diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 00000000..d6ca22e7 --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,50 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Python package + +on: + push: + branches: [ "cge-main" ] + pull_request: + branches: [ "cge-main" ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.13", "3.14"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 pytest + python -m pip install black + if [ -f python/requirements.txt ]; then pip install -r python/requirements.txt; fi + - name: Check code formatting + run: | + black ./src + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: | + # pytest ./python + + - name: Build release distributions + run: | + # NOTE: put your own distribution build steps here. + # python -m pip install build + # python -m build \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..f4ec0c21 --- /dev/null +++ b/.gitignore @@ -0,0 +1,230 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class +.idea +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +**/.venv +**/.modules +**/*.pyc +**/__pycache__ +**/*.log +**/*.swp +**/*.swo +**/*.sqlite3 +**/*.db +**/*.db-journal +**/*.pyo +**/bin +**/*.exe +**/*.dll +**/.*.so +**/*.dot diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..24ee5b1b --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..babfe2df --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,20 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "Run & Debug Taut2Pyunit", + "type": "debugpy", + "request": "launch", + "module": "rejuvenation.cli", + "args": ["refactor", "Taut2Pyunit", "features/targets/taut/taut_test.py"], + "env": { + "PYTHONPATH": "src:test" + }, + "cwd": "${workspaceFolder}" + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..2a14a8e3 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,12 @@ +{ + "python.analysis.extraPaths": [ + "src", + "test", + "features" + ], + "python.testing.pytestArgs": [ + "." + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..661ff65f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,16 @@ +Plan for next sprints: + + +11-05-2026 + +* [X] use type hierarchy to find type concisely instead of regexp +* [X] use hypothesis instead of parameterized test to get better coverage +* [X] convert more complex cases of TAUT test case and reviewed the conversion by Harry +* [X] restructure with root namespace so that it can be packaged +* [X] apply ASTProtocol to Python and ~~Clang Node~~ +* [X] add ADR and set up ADR discussion process +* [X] update test to pytest using python refactoring +* [X] created a package with callable cli +* [X] expand matcher and other utils to use lst nodes +* [X] convert simple case of TAUT test case and reviewed the conversion by Harry + diff --git a/README.md b/README.md new file mode 100644 index 00000000..0c6c95bb --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# Renaissance Experiments + +This project is experimental in nature and aims to explore +various concepts and techniques to apply renaissance pattern matching +in a generic way using multiple abstract syntax trees. + +## Setup for WSL +```bash +sudo apt-get install -y build-essential clang +``` + + +The code for the experiments is located in the [src](./src) folder. + +# Description +This project is a generic approach to refactor code bases with a generic AST structure. +It uses `TNO Renaissance` pattern matching. +Currently, clang native and clang python bindings are supported. + +# How to add a different binding +You'll need to implement a concrete class for syntax_tree.ASTNode. +Follow the implementations of `ClangASTNode` and `ClangJsonASTNode` as an example. +If the concrete AST has a different language then also a `PatternFactory` must be added. See `CPatternFactory` for inspiration. + +## Installation Procedure +To install the necessary dependencies, follow these steps: + +1. **Run the Installation Script** + - Navigate to the project directory. + - Execute the `install.bat` script by double-clicking it or running the following command in the terminal: + ```sh + ./install.bat + ``` + +## Configuration and Verification + +1. **Configure the Environment** + - Open Visual Studio Code (VSCode). + - Ensure that the Python extension is installed. + - Open the project folder in VSCode. + - alternatively in shell goto /python folder and + ```sh + code . + ``` + +2. **Verify the Installation** + - Open the integrated terminal in VSCode. + - Run the following command to execute the tests: + ```sh + python -m unittest discover + ``` + - Check the output to ensure all tests pass successfully. + +By following these steps, you will have installed and verified the setup for the project. + + +## TODO + +An incomplete list of todo's: + +* The get_properties methods of both `ClangASTNode` and `ClangJsonASTNode` are not complete yet. This might cause mismatches in the `Match_Finder` +* C++ constructs have not been tested yet +* An example of how to use includes in a `Pattern` must be added +* Tests need to be added for macro handling +* The methods `get_references` and `referred_by` must be added to `ASTNode` and implemented in the concrete classes +* Test cases for multiple match patterns need to be added. Currently, there is only one working case in the examples +* Comments in Clang appear incorrectly in the `ASTShower`. This seems to be a Clang issue, which is surprising + +## Usage + +cli + +### Inspect + +Inspect the AST of a source file. +```bash +cli inspect features/targets/demo.py pass +``` +it will show ast of demo.py and focus on 'pass' statements \ No newline at end of file diff --git a/adr/01_children_and_properties.md b/adr/01_children_and_properties.md new file mode 100644 index 00000000..d031dd37 --- /dev/null +++ b/adr/01_children_and_properties.md @@ -0,0 +1,91 @@ +# 01 - Children and properties + +Status: Accepted + +Date: 2026-02-25 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Example](#example) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + +## Context + +The goal of this ADR is to define a design that minimize the implementation time of the matching +algorithm when creating renaissance for a new language + +This document explains the design decision to have all AST nodes contain both children and properties. +Children represent nodes directly connected to a parent node; properties are attributes that describe the +node itself. Having both allows consistent representation of complex structures, simplifies traversal, and +separates structure (children) from node metadata (properties). + +## Decision + +All AST nodes will expose both children and properties. Children will be represented as a sequence +of child nodes. Properties will be stored in a map. Implementations should provide clear accessors +for both concepts. + +## Implementation notes + +- Expose properties through map(dict in Python). +- Provide helper methods for navigating and match. +- Keep the distinction between structural relationships (children) and descriptive data (properties) + explicit in APIs and documentation. +- The order of the children list matters and should, where possible, follow the order of parameters in the + node's constructor or grammar production rule. + +## Example + +```python +class GoAstNode: + @property + def properties(self) -> dict[str, Any]: + ... + + @property + def children(self) -> list[Self]: + ... +``` + +## Rationale + +This separation makes the AST easier to reason about, enables targeted transformations (structure vs. +metadata), and supports sharing strategies. + +## Consequences + +Positive: +- Clearer APIs and traversal logic. +- Easier targeted refactorings and transformations. + +Negative: +- Slight overhead in defining and maintaining two parallel concepts. + +## Alternatives considered + +- Merge children and properties into a single list of mixed entries — rejected because it complicates + traversal and semantic clarity. + +## Related decisions + +- See ADR 04 (Make nodes immutable) for related choices about immutability. + +--- + +Revision history: +- 2026-02-25: Converted to ADR template and clarified decision. +- 2026-03-27: Added table of contents; moved ordering note into Implementation notes; + renamed Example section to match template. diff --git a/adr/02_direct_access.md b/adr/02_direct_access.md new file mode 100644 index 00000000..cdbf3db0 --- /dev/null +++ b/adr/02_direct_access.md @@ -0,0 +1,96 @@ +# 02 - Direct access to fields + +Status: Accepted + +Date: 2026-02-25 + + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + + +## Context + +The goal of this ADR is to allow the developer of a new language for Renaissance +to create refactorings that is expressive and concise + +Direct access refers to exposing node fields and attributes using a Pythonic style (e.g., `function_definition.body`, `function_definition.name`) + +rather than using children and properties such as `function_definition.children[3].children` or `function_definition.properties['name']`. +This allows for natural attribute access, simpler metaprogramming, and compatibility with Python tooling and idioms. + +## Decision + +Adopt a Pythonic direct-access convention for node definitions. Nodes may declare a `_fields`(as in CPython's `ast` +module) that names structural fields. Consumers and tools should read these fields rather than +relying on children and properties methods. Implementations should still provide stable, +documented APIs for direct ast node manipulation. + + +## Implementation notes + + + +```python +class GoAstNode: + #direct access protocol + expr:Self + body:Sequence[Self] + other:Sequence[self] + + #rewrite protocol + length:int + offset:int + name:str + + #matcher + properties:dict[str, int | str] ={ + "length": length, + "offset": offset, + "name": name + } + children:list[Self] = [expr, body, other] + + +``` + +## Rationale + +Using Python conventions reduces boilerplate, makes code easier to inspect and manipulate, and aligns with developer +expectations in a Python project. + +## Consequences + +Positive: +- Lower boilerplate and clearer node definitions. +- Easier integration with Python tooling. + +Negative: +- Slight coupling to Python conventions; if we port the model to other languages some idioms will differ. + +## Alternatives considered + +- Exclusive use of accessor methods — rejected because it increases verbosity and reduces interop with Python tooling. + +## Related decisions + +- See ADR 01 (Children and properties) and ADR 04 (Make nodes immutable). + +--- + +Revision history: +- 2026-02-25: Converted to ADR template and clarified decision. diff --git a/adr/03_duck_typing.md b/adr/03_duck_typing.md new file mode 100644 index 00000000..7067b97e --- /dev/null +++ b/adr/03_duck_typing.md @@ -0,0 +1,87 @@ +# 03 - Duck typing for nodes + +Status: Accepted + +Date: 2026-02-25 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Context + +The goal of this ADR is to minimize the implementation of ASTNode for a new language while still taking advantage +of the generic algorithms + +The project is implemented in Python and must remain flexible in how AST-like nodes are represented. +Rather than enforcing a strict class hierarchy, we want code that accepts any object that looks and behaves like a +node (has required properties and children). This is the essence of duck typing. + +## Decision + +Treat nodes by behavior (structural and API shape) rather than by explicit concrete types. +A value is considered a valid node if it exposes the required fields, properties, and child access patterns +expected by the consumers. + +## Implementation notes + +- Document the node "shape" that consumers rely on + (e.g., required attribute names, `_fields` tuple, iteration semantics, and read-only accessors). +- Use structural typing where helpful: Python protocols (`typing.Protocol`) can express expected attributes + and aid static type checkers (mypy/pyright). +- Add runtime assertions or light validation at public API boundaries where robustness is important + (for example, when importing external nodes or plugin-provided nodes). +- Keep core algorithms defensive: prefer attribute access with sensible fallbacks rather than brittle type checks. +- Provide adapter/wrapper helpers (see ADR 06) to normalize foreign node-like objects into the project's + canonical node shape. + +```python +@runtime_checkable +class NodeMatchProtocol(protocol): + properties: dict + children: list[Self] + +def is_match(src: NodeMatchProtocol, cmp: NodeMatchProtocol) -> bool: + ... +``` + +## Rationale + +- Flexibility: allows integrating nodes produced by different parsers or external tools without heavy wrapper work. +- Simplicity: avoids deep inheritance trees when behavior is all that's required. +- Interoperability: easier to write adapters and tests against small, focused protocols. + +## Consequences + +Positive: +- Easier integration with third-party node representations. +- Reduced boilerplate for small, local node-like objects used in tests. + +Negative: +- Potential for runtime errors if an object only partially implements the expected shape; + mitigated by runtime checks at boundaries and clear documentation. +- Slightly looser guarantees than strict nominal typing. + +## Alternatives considered + +- Enforce a strict base node class — rejected for flexibility reasons. +- Rely solely on runtime duck checks with no static typing — rejected in favor of combining runtime checks + with Protocols for better tooling. + +## Comment and whitespace + +comment and white space belongs to ast node. +is comment need to it own property without "comment sign" + +## Related decisions + +- See ADR 06 (Wrapper or adapter) and ADR 01 (Children and properties). + + +--- + +Revision history: +- 2026-02-25: Converted to ADR template and clarified decision. diff --git a/adr/04_immutable_properties.md b/adr/04_immutable_properties.md new file mode 100644 index 00000000..c35dde86 --- /dev/null +++ b/adr/04_immutable_properties.md @@ -0,0 +1,98 @@ +# 04 - nodes can be immutable + +Status: Accepted + +Date: 2026-02-25 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + + +## Context + +The goal of this ADR is to define a controlled way to update AST nodes, so that the resulting AST is still correct. + +The project models trees made of nodes. Currently, node data (properties and children) operations read the +tree and transformations create new trees instead of mutating in-place. Ensuring immutability helps reasoning +about transformations, enables safer concurrency, and opens opportunities for caching and memoization. + +## Decision + +Nodes can be implemented as immutable objects. Once a node is created, its properties and children cannot be +modified. Any change to a tree (for example, updating a property or replacing a child) will be done through a rewriter +produce a new node valid rather than mutating the existing node in-place. + +Implementation notes and recommendations for contributors: + +- Provide rewriter to create modified copies of nodes (for example, a `replace`, `remove` `insert` + pattern that returns a new node with the requested changes). +- When storing modifications, make sure the result is still correct and raise exception in case of unsolvable conflict. + +## Rationale + +- Predictability: Callers can rely on a node's properties remaining the same after construction, + simplifying reasoning about passes and refactorings. +- Concurrency: Immutable data structures are safe to share across threads without synchronization. +- Caching & memoization: Since nodes don't change, caching derived information (like computed hashes, + string representations, or analysis results) is reliable. +- Correctness: Avoids accidental side effects caused by in-place modifications during complex refactorings. + +```python + # no problem + rewrite.replace(new_contetent, ast.child[1:3]) + # can bea problem because the line number chenged, but it is solvable + rewrite.replace(new_contetent, ast.child[4:6]) + + # reaise exception, because it is partly changed end not gerantteed the result is still sytactical correct + rewrite.replace(new_contetent, ast.child[2:4]) + +``` + +## Consequences + +Positive: +- Easier reasoning about code that manipulates trees. +- Safer concurrent processing and simplified caching. +- Fewer bugs due to unintended mutation. + +Negative / trade-offs: +- Potential performance overhead due to allocation when creating modified copies. + Mitigations include structural sharing (reusing unchanged children) and keeping node representations compact. +- Some algorithms that expect in-place updates will need to be adapted or re-implemented in an immutable style. +- Developers must learn and follow patterns for producing modified copies (builders, `copy_with` helpers). + +## Alternatives considered + +1. Mutable nodes with defensive copies + - Keep nodes mutable but perform defensive copying when necessary. + - Rejected because it is easy to forget copies and still produce subtle bugs. + +2. Hybrid approach: mostly immutable, but allow controlled mutation through explicit APIs + - Provides flexibility but complicates invariants and testing; increases cognitive load. + +3. Fully persistent immutable data structures (e.g., ropes, HAMT, custom persistent vectors) + - Strong sharing and performance but larger implementation cost and complexity; + deferred for future optimization if needed. + +## Related decisions + +- See ADR 01 (children and properties) and ADR 02 (direct access) for related design choices about tree shape + and access patterns. + +--- + +Revision history: +- 2026-02-25: Draft; adds ADR template and implementation guidance. diff --git a/adr/05_buildin_functions.md b/adr/05_buildin_functions.md new file mode 100644 index 00000000..9d1fb135 --- /dev/null +++ b/adr/05_buildin_functions.md @@ -0,0 +1,98 @@ +# 05 - Use Python's built-in dunder methods for node behavior + +Status: Accepted + +Date: 2026-02-25 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + + +## Context + +The goal of this ADR is to create an implementation of renaissance that feels native to the python world and reduce +the verbosity without misusing the original meanings. + +Nodes should integrate naturally with Python idioms and be easy to inspect, compare, iterate, and hash when +appropriate. Using Python's special methods (``__repr__``, ``__eq__``, ``__hash__``, ``__str__``, ``__len__``, +``__iter__``, ``__getitem__``, ``__contains__``, etc.) gives predictable, idiomatic behavior. + +## Decision + +Implement and document a small, consistent set of dunder methods on node types to enable common operations. +Not every node must implement every method — choose the methods that make sense for the node's semantics +(for example, sequence-like nodes should implement ``__len__`` and ``__iter__``). + +## Implementation notes + +- ``__repr__``: Provide an unambiguous, developer-oriented representation useful for debugging and display (ASTShower). +- ``__str__``: Provide a readable representation intended for users or logs. +- ``__eq__`` and ``__hash__``: Implement equality of ASTNodes are logically value-like + and immutable (see ADR 04). If nodes are mutable or identity matters, prefer identity-based equality and + avoid making them hashable. +- ``__len__`` / ``__iter__`` / ``__getitem__``: Implement for sequence-like node types to allow Pythonic + iteration and indexing. witch maps to children in our case. +- ``__contains__``: Implement if membership semantics are meaningful. +- Avoid surprising side effects in any dunder method. Keep them simple and consistent. + `AST Node != AST Pattern` + +```python + +class GoAstNode: + +# easier to see in debugger and it is used in astshower +def __repr__(self) -> str: + return f"{type}....." + +#shorthand for nth children +__getitem__(self, index) -> Self: + return self.children[index] +``` + +## Rationale + +- Idiomatic usage: makes nodes easier to use with Python language features and libraries. +- Debuggability: ``__repr__`` and ``__str__`` improve developer experience. +- Interoperability: sequence and mapping protocols let nodes interoperate with Python collection utilities. + +## Consequences + +Positive: +- More predictable developer experience and easier debugging. +- Better interoperability with Python tools and libraries. + +Negative: +- Risk of over-implementing dunder methods and creating surprising behavior; + prefer conservative, well-documented choices. + +## Alternatives considered + +- Minimal API surface with no special methods — rejected because it reduces ergonomics. + +## Related decisions + +- See ADR 04 (Make nodes immutable) when implementing ``__hash__`` and ``__eq__``. + +## note + +is_match is __not the same as __eq__ +also it avoids extra implementation + +--- + +Revision history: +- 2026-02-25: Converted to ADR template and clarified decision. diff --git a/adr/06_wrapper_or_adapter.md b/adr/06_wrapper_or_adapter.md new file mode 100644 index 00000000..42327334 --- /dev/null +++ b/adr/06_wrapper_or_adapter.md @@ -0,0 +1,117 @@ +# 06 - Wrapper or adapter for external node shapes + +Status: Proposal + +Date: 2026-02-25 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) + +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + + +## Context + +The goal of this ADR is to define a strategy for interoperating with external node-like objects that do not +match the project's canonical node shape while minimize the effort for the developer of the new language for renaissance. + + +The project may receive nodes from different parsers or libraries that do not match the project's canonical node +shape. We need a strategy to interoperate with foreign node-like objects while preserving the project's APIs +and expectations unig minimum amount of code. + +## Decision + +Prefer writing only the protocol function on top of the current native implementation if not already available. +This requires a minimum amount of implementation and opportunity for reuse of the matcher and rewrite +functionalities. + +Thin wrappers (adapter objects) that present the project's canonical node API while delegating to the original +node make behavior explicit, allow normalization, and preserve access to the original node when necessary. + +## Implementation notes + +- Implement simple wrapper/adaptor classes that implement the project's node Protocol (see ADR 03). +- Keep wrappers thin: delegate attribute and child access where possible and only normalize differences that matter. + +```Python + +# monkey patching the ast node to have properties and children, so that it can be used directly in the matcher and rewriter without needing to write an adapter for it. +@property +def properties(self: AST) -> dict[str, Any]: + props = {} + for name in self._fields: + props[name] = getattr(self, name) + return props + + +AST.properties = properties + + +@property +def children(self: AST) -> list[AST]: + return getattr(self, "body", []) + + +AST.children = children + +# wrapper example for a foreign node type (e.g., from a third-party parser) + +class PythonASTNode: + def __init__(self, node: ast): + self._node = node + @property + def propertie(self): + return {'name':self._node.name, 'value':self._node.value} + @property + def children(self): + return [PythonASTNode(n) for n in self._node.body] + +# adapter example +class PythonASTNode: + def __init__(self, node): + self.properties["name"] = self.derive_name_from(node) + +``` + + +## Rationale + +- Wrappers preserve original semantics and make interop explicit. +- Adapters make it easy to support multiple external sources without changing core logic. + +## Consequences + +Positive: +- Clear interoperability surface and testable adapters. +- Avoids spreading compatibility code throughout the codebase. + +Negative: +- Slight overhead of adapter objects and maintenance of adapter code. + +## Alternatives considered + +- Modify external objects in place — rejected because it mutates foreign data and can have side effects. +- Copy-and-normalize into internal-only node instances — viable but may be more expensive than thin wrappers. + +## Related decisions + +- See ADR 03 (Duck typing) and ADR 01 (Children and properties). + +--- + +Revision history: +- 2026-02-25: Converted to ADR template and clarified decision. diff --git a/adr/07_package_management.md b/adr/07_package_management.md new file mode 100644 index 00000000..4c2501f1 --- /dev/null +++ b/adr/07_package_management.md @@ -0,0 +1,70 @@ +# 07 - Use UV for package & environment management + +Status: Proposal + +Date: 2026-02-25 + +Authors: Project contributors + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + +## Context + +The go of this ADR is to define a modern way to identify and manage dependencies, so that we can +recreate the artifact at any time. + +The project uses Python and benefits from reproducible dependency management and straightforward virtual +environment handling. UV provides a single-file project manifest (`pyproject.toml`) and an integrated +workflow for dependency resolution, packaging, and environment management. + +## Decision + +Adopt UV as the recommended tool for dependency management and packaging. Encourage contributors to use +Poetry for creating virtual environments, adding/removing dependencies, and building distributions. + +## Implementation notes + +- Keep `pyproject.toml` and `uv.lock` up-to-date. +- Document common contributor workflows in the repository README (install, run tests, add dependency). +- Provide instructions for creating and activating a UV-managed virtualenv and installing dev dependencies. + +## Rationale + +- Single source of truth (`pyproject.toml`) and dependable lockfile for reproducible builds. +- Simplifies contributor onboarding and packaging. + +## Consequences + +Positive: +- Reproducible installs and simpler packaging workflows. + +Negative: +- Contributors unfamiliar with Poetry need to learn its commands; mitigate with documentation. + +## Alternatives considered + +- Use pip + virtualenv and `requirements.txt` — rejected for weaker dependency resolution and no standardized + project manifest. + +## Related decisions + +- This ADR explains our tooling preference; it does not block using other tools in special cases. + + +## UV + +UV is the even more modern version, which unifies abstracts all build related tools. +https://github.com/astral-sh/uv + +--- + +Revision history: +- 2026-02-25: Converted to ADR template and clarified decision. diff --git a/adr/08_pytest_suite.md b/adr/08_pytest_suite.md new file mode 100644 index 00000000..12383083 --- /dev/null +++ b/adr/08_pytest_suite.md @@ -0,0 +1,203 @@ +# 08 - Test Architecture + +Status: Accepted + +Date: 2026-03-27 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Example](#example) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + +## Context + +The goal of this ADR is to establish a coherent test architecture for the Renaissance project that supports +maintainability, extensibility, and comprehensive coverage. +To ensure maintainability and extensibility a test architecture is crucial. The project needs a coherent set of +testing frameworks covering behavior-driven tests, unit tests, performance benchmarks, and inline documentation +examples. The choice of frameworks has implications for test discovery, fixture sharing, CI integration, and the +ability to express the domain-specific requirements listed below. + +## Requirements +### Functionalities that must be tested + +**Code matching** +- Independent of layout (whitespace) and comments (presence, absence, content). +- Support for placeholders; placeholders are AST nodes. +- Support for explicit and implicit placeholders. +- Robustness: implicit placeholders must not be triggered inside strings (`"$X"`) or + comments (`/* $X */`). +- Multiple occurrences of the same placeholder express an equality constraint + (e.g., `$f; var = $f;`). +- Multiple assignments of placeholders (e.g., `$f($$before, $arg, $$after)`). + +**Placeholder matching rules** +- A placeholder matches at the *highest* AST node whose concrete syntax reduces to a single name + (function `getPlaceholderName` is applied recursively). +- The same placeholder may be bound to nodes of different AST classes within one pattern + (e.g., `$type` in `$type* ptr = new $type()` binds to `IASTNamedTypeSpecifier` then `IASTTypeId`). + Comparison must therefore be structural, not class-based. + +**Equivalent code matching** +- Readability variants: `1_000_000` ≡ `1000000`. +- Numeric bases: `0xFF` ≡ `255`. +- Scientific notation: `1E2` ≡ `100`. +- String delimiters: `"ape"` ≡ `'ape'`. +- String concatenation: `"con" "cat"` ≡ `"concat"`. +- Symmetric operators: `0 == x` matches `x == 0`. +- Equivalent initialization forms (C++): `int x = 1;` matches `int x { 1 };`. + +**Find functionality** +- Find by kind (nested): e.g., find all `if` statements; a found match may contain another found match. +- Language-agnostic kinds: definition, statement, expression, declaration, … +- Parser-specific kinds: e.g., `IASTIfStatement`. +- Find by AST pattern (nested): e.g., `if ($x == MAX) { $$stmts; }`. +- Find consecutive (non-overlapping): `find "aa" in "aaa"` → one match; + `find "aa" in "aaaa"` → two non-overlapping matches. + +**Navigation functionality** +- AST structure: parent & ancestors, children & descendants, siblings. +- Usage: definition / forward declaration → references (current file / analysis unit only). +- Inheritance: base ↔ derived classes. + +**Transformation functionality** +- The encoding of a file must never change. +- File/directory metadata may only change when an actual transformation occurred; + analysis or a failing filter are not sufficient. +- *Offset-based* batch modifications: + - Insert and replace (remove = replace with `""`). + - Containment rule: contained operations are ignored. + - Consistency rule: overlapping operations are forbidden. +- *AST-based* batch modifications: + - Prepend, append, replace, around (e.g., for matching brackets). + - Containment rules: + - A replacement on a node hides all operations on its descendants (prepend/append/around are unaffected). + - A prepend to a node is always before a prepend to any descendant. + - An append to a node is always after an append to any descendant. + - Sequence rule: an append to sibling N is always before a prepend to sibling N+1. +- Find + filter (possibly multiple) + replace (whole match replaced). +- Replace recursively (AST nodes bound to placeholders are also modified). +- Find + filter (possibly multiple) + modify: + - Multiple operations on a single find result. + - Any AST node reachable via navigation may be modified, not only nodes contained in the match. + +## Decision + +Adopt the following test framework stack: + +| Purpose | Framework | +|-------------------------------|--------------------------------| +| BDD / acceptance tests | **pytest-bdd** | +| Unit tests | **pytest** | +| Performance benchmarks | **pytest-benchmark** | +| Inline documentation examples | **doctest** | +| Assertion style | **PyHamcrest** (`assert_that`) | + +pytest-bdd is chosen over Behave and Robot Framework (see [Alternatives considered](#alternatives-considered)). + +## Implementation notes + +- All test files follow pytest naming conventions (`test_*.py` or `*_test.py`). +- BDD feature files are placed under `features/` and steps under `features/steps/`. +- Fixtures are defined with `@pytest.fixture`; shared fixtures live in `conftest.py`. +- Parametrised tests use `@pytest.mark.parametrize`. +- Coverage is measured with `pytest-cov` (`--cov-report=term-missing`). +- All pytest configuration lives under `[tool.pytest.ini_options]` in `pyproject.toml`. +- Performance baselines are stored in `.benchmarks/` (git-ignored by default). + +## Example + +```python +import pytest +from hamcrest import assert_that, is_, contains_inanyorder + +@pytest.fixture +def sut(): + return Matcher() + +class TestMatcherPlaceholder: + def test_placeholder_matches_highest_ast_node(self, sut): + pattern = pattern_factory("$x;", SyntacticKind.STATEMENT) + result = sut.find(parse("a = f(1, 2+3);"), pattern) + assert_that(result, is_(non_empty())) + + @pytest.mark.parametrize("source,expected", [ + ("1_000_000", "1000000"), + ("0xFF", "255"), + ('"ape"', "'ape'"), + ]) + def test_equivalent_literals(self, sut, source, expected): + assert_that(sut.are_equivalent(source, expected), is_(True)) +``` + +```gherkin +# features/find.feature +Feature: Find functionality + Scenario: Find nested if statements + Given a source file containing nested if statements + When I search for if statements + Then each outer match may contain inner matches +``` + +## Rationale + +pytest is the de-facto standard for Python unit testing, so all other frameworks are chosen for their +integration with it. pytest-bdd shares pytest fixtures, the CLI, plugins, and reporting — eliminating the +overhead of a separate test runner. pytest-benchmark plugs into the same run. doctest keeps examples +in sync with the documentation automatically. PyHamcrest makes assertions self-documenting and produces +readable failure messages. + +## Consequences + +Positive: +- Single test runner (`pytest`) for all test kinds: BDD, unit, benchmark, doctest. +- Shared fixtures across BDD steps and unit tests via `conftest.py`. +- Rich plugin ecosystem (`pytest-cov`, `pytest-mock`, `pytest-bdd`, `pytest-benchmark`). +- Seamless CI integration. +- Expressive, readable assertions via PyHamcrest. + +Negative: +- pytest-bdd's Gherkin support is slightly less mature than Behave's. +- Multiple frameworks must be kept in sync (versions, plugins). +- Writing and maintaining BDD step definitions adds overhead over plain unit tests. + +## Alternatives considered + +**BDD framework** + +| Framework | Assessment | +|------------------|-----------------------------------------------------------------------------------------------| +| **pytest-bdd** ✓ | Integrates with pytest (shared fixtures, CLI, plugins). Active since 2013. | +| Behave | Standalone; no shared fixtures with pytest. Very mature (2011). Rejected due to split runner. | +| Robot Framework | Full automation framework; steep learning curve; overkill for BDD only. | +| Lettuce | Declining community; minimal updates. Rejected. | + +**Unit testing** +- `unittest` (stdlib) — rejected: more boilerplate, no plugin ecosystem, less expressive assertions. + +**Assertion style** +- Plain `assert` — rejected in favor of PyHamcrest for richer failure messages and composable matchers. + +## Related decisions + +- See ADR 09 (Property-based tests) for the use of Hypothesis alongside pytest. +- See ADR 10 (Type hierarchy) for the `SyntacticKind` taxonomy referenced in find-functionality tests. +- See ADR 12 (Patterns are not nodes) for the `Pattern` type used in matching tests. + +--- + +Revision history: +- 2026-03-27: Converted GitHub issue #08 to ADR template; expanded all functionality requirements. diff --git a/adr/09_property_based_tests.md b/adr/09_property_based_tests.md new file mode 100644 index 00000000..fc1db8d8 --- /dev/null +++ b/adr/09_property_based_tests.md @@ -0,0 +1,107 @@ +# 09 - Property-Based Tests + +Status: Proposal + +Date: 2026-03-27 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Example](#example) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + +## Context + +The goal of this ADR is to adopt property-based testing as a complementary approach to the existing parametrized +tests in the Renaissance, so that the test effort of the developer of a new language for renaissance can be reduced +and the test coverage can be improved. + +The project currently uses a set of parametrized tests to verify behavior across a range of inputs. Maintaining +these input tables by hand is tedious and error-prone; edge cases are easy to miss. Property-based testing offers +an alternative approach where the testing framework generates input data automatically, guided by strategies and +invariants declared by the developer. The formal, tree-structured nature of ASTs makes them well-suited to this +approach. + +## Decision + +Hypothesis is adopted as the property-based testing library for this project. It will complement (and where +appropriate replace) existing parametrized tests. Hypothesis strategies will be used to generate diverse AST +inputs, and properties (invariants) will be asserted rather than concrete expected values. + +Additionally, Hypothesis can be used to validate code generated by AI tooling, providing a principled, automated +way to check generated output against formal specifications. + +## Implementation notes + +- Use `hypothesis` strategies to generate AST nodes and transformation inputs. +- Express test invariants as properties (e.g., "a round-trip parse/unparse yields the original source"). +- Gradually migrate existing `@pytest.mark.parametrize` tables to `@given` + `@settings` where the coverage + benefit justifies the change. +- Use `hypothesis.extra` integrations (e.g., `hypothesis[pandas]`, `hypothesis[numpy]`) only where relevant. +- Store Hypothesis database artifacts in `.hypothesis/` (already git-ignored by default). + +## Example + +```python +from hypothesis import given, strategies as st +from renaissance.lst import LSTNode + +@given(st.from_type(LSTNode)) +def test_round_trip(node: LSTNode) -> None: + """Parsing and unparsing an LSTNode must yield the original source.""" + assert unparse(parse(str(node))) == str(node) + +@given(st.text(alphabet=st.characters(whitelist_categories=("Lu", "Ll")))) +def test_camel_case_no_spaces(name: str) -> None: + result = camel_case(name) + assert " " not in result +``` + +## Rationale + +Hypothesis and the formal nature of ASTs are a perfect combination for property-based testing: the structured, +well-typed domain of AST nodes maps naturally onto Hypothesis strategies, and the algebraic properties of +transformations (identity, round-trip, commutativity) are easy to express as invariants. This can replace the +current set of parametrized tests with broader, automatically generated coverage. It can also be used to validate +code generated by AI, providing an automated and principled quality gate. + +## Consequences + +Positive: +- Automatically discovers edge cases that hand-crafted tables miss. +- Reduces the maintenance burden of large parametrize tables. +- Provides a principled way to validate AI-generated code. +- Shrinking produces minimal failing examples, making debugging easier. + +Negative: +- Adds an external dependency (`hypothesis`). +- Tests may run longer due to the number of generated examples. +- Writing good strategies for complex AST types requires upfront investment. + +## Alternatives considered + +- Continue with `@pytest.mark.parametrize` only — rejected because hand-crafted tables have limited coverage + and high maintenance cost. +- Use `fuzzing` tools (e.g., `atheris`) — rejected because they target low-level byte inputs rather than + structured, typed domain objects. + +## Related decisions + +- See ADR 08 (Pytest Suite) for the overall testing framework choice that Hypothesis integrates with. + +--- + +Revision history: +- 2026-03-27: Converted to ADR template and clarified decision. diff --git a/adr/10_type_hierarchy.md b/adr/10_type_hierarchy.md new file mode 100644 index 00000000..05ae2160 --- /dev/null +++ b/adr/10_type_hierarchy.md @@ -0,0 +1,103 @@ +# 10 - Type Hierarchy + +Status: Accepted + +Date: 2026-03-27 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Example](#example) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + + ## Context + +the goal of this ADR is to establish a robust and maintainable type hierarchy for AST nodes use in the algorithms +within the Renaissance project and across the languages. + +AST node types are currently identified by string-based type names (e.g., re.compile(kind, +`(?i)Function_?Decl".IGNORECASE)`). This approach is fragile, hard to refactor, and requires every consumer to know the +exact string values. In addition, helper functions such as `is_statement`and `is_expression` must each maintain their +own lookup tables. A class hierarchy provides a more robust and idiomatic solution. + +## Decision + +- Follow the Doxygen definition for common node types (e.g., statement, expression, declaration) and use native Python +- types for language-specific or non-standard node kinds. +- Use the class hierarchy to determine the type of a node instead of string-based type name comparisons. +- Helper functions such as `is_statement` and `is_expression` will delegate to `isinstance` checks, making them generic +- and significantly simpler. + +## Implementation notes + +- Define abstract base classes for the common node categories (e.g., `statement`, `expression`,`declaration`) following + Doxygen terminology. +- Language-specific node kinds that have no Doxygen equivalent are represented as native Python classes inheriting from + the appropriate base. +- Replace all `node.type == "..."` comparisons with `isinstance(node.type, Statement)` checks. +- Implement helper predicates as thin wrappers: + +```python +def is_statement(node: AstNode) -> bool: + return isinstance(node.kind, Statement) + +def is_expression(node: AstNode) -> bool: + return isinstance(node.kind, Expression) + +# Usage +node.kind =Assignment(...) +assert is_statement(node) # True — no string comparison needed +assert not is_expression(node) # False +``` + + +## Example + + +## Rationale + +Using the class hierarchy to determine node types is more robust than string comparisons: it is refactor-safe, +IDE-navigable, and benefits from Python's `isinstance` semantics. Following Doxygen's well-known taxonomy for common +node categories ensures consistency with established conventions and makes the codebase accessible to developers +familiar with that terminology. Helper functions become trivially simple and generically applicable across all language +frontends. + + +## Consequences + +Positive: +- Eliminates fragile string-based type comparisons. +- Helper functions (`is_statement`, `is_expression`, …) become simple, generic, and reusable. +- IDE tooling (auto-complete, go-to-definition, refactoring) works naturally with class hierarchies. +- Consistent with Doxygen conventions for common node categories. + +Negative: +- Requires an upfront investment to define the class hierarchy and migrate existing string comparisons. +- Deep inheritance trees can become hard to navigate if not kept shallow and well-documented. + +## Alternatives considered + +- String-based type names — rejected because they are fragile, not refactor-safe, and require consumers to know exact string values. +- Enum-based type tags — rejected because they do not compose well with inheritance and still require explicit lookup tables in helper functions. + +## Related decisions + +- See ADR 01 (Children and properties) for the overall AST node design that this hierarchy builds upon. +- See ADR 03 (Duck typing) for cases where structural subtyping with `Protocol` is preferred over nominal subtyping. + +--- + +Revision history: +- 2026-03-27: Converted to ADR template and clarified decision. diff --git a/adr/11_parser_with_space_and_comment.md b/adr/11_parser_with_space_and_comment.md new file mode 100644 index 00000000..8d53b1e5 --- /dev/null +++ b/adr/11_parser_with_space_and_comment.md @@ -0,0 +1,120 @@ +# 11 - Parser with Space and Comment + +Status: Proposal + +Date: 2026-03-27 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Example](#example) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + +## Context + +The goal of this ADR is provided a guideline on what to focus on when selecting a parser for a new language in renaissance. + +Refactoring tools must preserve the exact formatting of source code, including whitespace and comments, which are +not semantically significant to the language but are critical for producing output that is indistinguishable from +the original. Traditional parsers discard whitespace and comments (trivia) before building the AST, which means a +round-trip from raw source → AST → raw source loses this information and produces incorrect or unacceptable output. +Storing trivia in a separate data structure requires glue code to reassemble the output, increasing complexity and +maintenance burden. + +## Decision + +- Whitespace and comments must be preserved through the full parse → transform → unparse round-trip, producing + output that is identical to the original source when no transformation is applied. +- Comments and whitespace are made part of the AST node itself (as leading/trailing trivia attached to the node), + rather than stored in a separate data structure. +- The amount of glue code required to reassemble source text from the AST is minimized by design. +- For Python, **libcst** is used as the parser, as it natively represents whitespace and comments as part of its + CST nodes and provides a lossless round-trip out of the box. + +## Implementation notes + +- Use `libcst` for parsing and unparsing Python source. It stores whitespace and comments directly on each node + via `whitespace`, `leading_lines`, and similar fields. +- For non-Python languages, attach leading and trailing trivia directly to each AST node, following the pattern + used by Roslyn (C#) and tree-sitter. +- The unparser must not add, remove, or reorder trivia unless a transformation explicitly modifies it. +- When a transformation produces a new node, trivia from the replaced node is transferred to the replacement by + default. + +## Example + +```python +import libcst as cst + +source = """\ +# important comment +x = 1 # inline comment +""" + +tree = cst.parse_module(source) +# Round-trip: produces exactly the same source +assert tree.code == source + +# Transformation using libcst +class RenameX(cst.CSTTransformer): + def leave_Name(self, original_node, updated_node): + if updated_node.value == "x": + return updated_node.with_changes(value="y") + return updated_node + +new_tree = tree.visit(RenameX()) +# Whitespace and comments are preserved; only "x" is renamed to "y" +print(new_tree.code) +``` + +## Rationale + +Using libcst for Python eliminates the need to build a custom trivia-preserving parser. libcst is a +production-quality Concrete Syntax Tree library that natively preserves all whitespace, comments, and formatting +as part of its node structure, providing lossless round-trips with minimal glue code. Attaching trivia to nodes +(rather than a side-table) ensures transformations can reason about and manipulate comments and whitespace in a +uniform, self-contained way. + +## Consequences + +Positive: +- Lossless round-trip: raw → CST → raw produces identical output when no transformation is applied. +- No separate trivia store; no glue code to reassemble source text. +- Transformations can inspect and modify comments and whitespace uniformly. +- libcst is actively maintained and widely used in the Python ecosystem. + +Negative: +- libcst adds an external dependency. +- libcst's node model is more verbose than a plain AST; developers must learn its API. +- For non-Python languages a custom trivia-preserving strategy must still be implemented. + +## Alternatives considered + +- Standard `ast` module — rejected because it discards all whitespace and comments, making lossless round-trips + impossible. +- Store trivia in a separate side-table indexed by source position — rejected because it requires glue code to + rejoin trivia with nodes during unparsing, contradicting the goal of minimal glue code. +- tree-sitter — considered but rejected for Python as primary parser because libcst provides a higher-level, + Python-native API with built-in transformation support. + +## Related decisions + +- See ADR 01 (Children and properties) for the overall AST node structure that trivia fields extend. +- See ADR 04 (Immutable properties) for the immutability strategy applied to trivia fields. + +--- + +Revision history: +- 2026-03-27: Converted to ADR template and clarified decision. diff --git a/adr/12_patterns_as_not_nodes.md b/adr/12_patterns_as_not_nodes.md new file mode 100644 index 00000000..c73043df --- /dev/null +++ b/adr/12_patterns_as_not_nodes.md @@ -0,0 +1,131 @@ +# 12 - Patterns Are Not Nodes + +Status: Proposal + +Date: 2026-03-27 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Example](#example) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + +## Context + +The goal of this ADR is to clarify the distinction between code factories and pattern factories in the +Renaissance project. + +In the current implementation a pattern is just an AST node. This is not desirable: while a pattern may be +realized using an AST node under the hood, it may also carry additional information that has no place in a +plain AST node. + +For example, to pattern-match `create_expression('$x')` it is convenient for the pattern to also record the +desired syntactic kind (expression, statement, declaration, …). Without that extra information the correct +kind must be inferred from the surrounding context — which is possible in most cases (as demonstrated by an +earlier prototype) but is fragile and adds complexity to the matcher. + +Having a separate `Pattern` type that wraps an AST node and adds metadata makes both the code factory and +the pattern factory first-class concepts with clearly separated responsibilities. + +## Decision + +Introduce two distinct factory families: + +- **Code factories** — turn source-code snippets (given a syntactic context: statement, declaration, + expression, …) into plain AST nodes. +- **Pattern factories** — create `Pattern` objects that are used for matching. A `Pattern` wraps an AST node + and additionally records the expected syntactic kind and any other match-time metadata. + +A `Pattern` is therefore **not** an AST node; it is a separate value type that holds an AST node together +with matching metadata. + +## Implementation notes + +- Define a `Pattern` dataclass (frozen) with at least: + - `node: AstNode` — the template node used for structural matching. + - `kind: SyntacticKind` — the expected kind (e.g., `EXPRESSION`, `STATEMENT`, `DECLARATION`). + - Optional: captured variable names, constraints, etc. +- Code factories (`code_factory`) accept a source snippet and a `SyntacticKind` and return an `AstNode`. +- Pattern factories (`pattern_factory`) accept a source snippet with placeholders (e.g., `$x`) and a + `SyntacticKind` and return a `Pattern`. +- The matcher operates on `Pattern` objects, not raw `AstNode` objects, so it can exploit the stored `kind` + without re-inferring it from context. + +```python +from dataclasses import dataclass +from renaissance.common import AstNode, SyntacticKind + +@dataclass(frozen=True) +class Pattern: + node: AstNode + kind: SyntacticKind + +def code_factory(snippet: str, kind: SyntacticKind) -> AstNode: + ... + +def pattern_factory(snippet: str, kind: SyntacticKind) -> Pattern: + node = code_factory(snippet, kind) + return Pattern(node=node, kind=kind) +``` + +## Example + +```python +# Create an AST node for a statement +assignment = code_factory("x = 1", SyntacticKind.STATEMENT) + +# Create a pattern that matches any expression assigned to $x +expr_pattern = pattern_factory("$x", SyntacticKind.EXPRESSION) + +# The matcher can use expr_pattern.kind directly — no inference needed +matches = matcher.find(tree, expr_pattern) +``` + +## Rationale + +Keeping `Pattern` separate from `AstNode` respects the single-responsibility principle: AST nodes represent +source structure; patterns represent match intent. Encoding the syntactic kind directly in the `Pattern` +eliminates the need for fragile context inference in the matcher and makes pattern creation explicit and +self-documenting. The two factory families mirror this separation cleanly. + +## Consequences + +Positive: +- Matcher logic is simpler: the expected kind is available directly on the `Pattern`. +- Pattern creation is explicit: callers state the intended kind at the call site. +- AST nodes remain pure structural representations, uncontaminated by matching metadata. +- The two factory families provide a clear, discoverable API surface. + +Negative: +- Two factory families must be defined and maintained instead of one. +- Existing code that treats patterns as plain AST nodes must be migrated. + +## Alternatives considered + +- Reuse `AstNode` as pattern (current approach) — rejected because it conflates structural representation + with match metadata and requires fragile kind inference in the matcher. +- Subclass `AstNode` to create `PatternNode` — rejected because inheritance couples the pattern type to the + node hierarchy and still requires carrying extra fields not appropriate for plain nodes. + +## Related decisions + +- See ADR 01 (Children and properties) for the AST node structure that `Pattern.node` wraps. +- See ADR 03 (Duck typing) for the protocol-based approach used by the matcher to accept `Pattern` objects. +- See ADR 10 (Type hierarchy) for the `SyntacticKind` taxonomy used as the `kind` field. + +--- + +Revision history: +- 2026-03-27: Converted to ADR template and clarified decision. diff --git a/adr/13_match_pattern.md b/adr/13_match_pattern.md new file mode 100644 index 00000000..f5834206 --- /dev/null +++ b/adr/13_match_pattern.md @@ -0,0 +1,140 @@ +# 13 - Match Pattern + +Status: Proposal + +Date: 2026-03-27 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Example](#example) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + +## Context + +The goal of this ADR is to define the design of match patterns in the Renaissance project. + +A match pattern is a source-code snippet that may contain **placeholders** — special names prefixed with `$` +(single node) or `$$` (sequence of nodes). Patterns are used to find and transform code in a language-agnostic +way. Two design questions drive this ADR: + +1. **At what AST level should a placeholder match?** + A placeholder node (an `IASTName`) should match at the *highest* AST node whose concrete syntax reduces + to a single name, determined by recursively applying `getPlaceholderName`. This lets `$x` in the pattern + `$x;` match a full expression statement, not just an identifier. + +2. **How should repeated placeholders be compared?** + The same placeholder can be bound to nodes of *different* AST classes within one pattern + (e.g., `$type* ptr = new $type()` binds `$type` first to `IASTNamedTypeSpecifier`, then to `IASTTypeId`). + Comparison must therefore be structural (value equality), not class-based. + +## Decision + +- A placeholder matches at the **highest** AST node whose concrete syntax reduces to a single name + (function `getPlaceholderName` applied recursively). +- Multiple occurrences of the same placeholder in a pattern express an **equality constraint**: all bound + nodes must be structurally equal, regardless of their AST class. +- Implicit placeholders must **not** be triggered inside string literals (`"$X"`) or comments (`/* $X */`). +- Sequence placeholders (`$$name`) match zero or more consecutive sibling nodes. +- Patterns support **equivalent code matching**: + - Readability separators: `1_000_000` ≡ `1000000` + - Numeric bases: `0xFF` ≡ `255` + - Scientific notation: `1E2` ≡ `100` + - String delimiters: `"ape"` ≡ `'ape'` + - String concatenation: `"con" "cat"` ≡ `"concat"` + - Symmetric operators: `0 == x` matches `x == 0` + - Equivalent initializers (C++): `int x = 1;` matches `int x { 1 };` + +## Implementation notes + +- Implement `getPlaceholderName(node) -> str | None` recursively: return the placeholder name if the node's + entire concrete syntax is a single `$`-prefixed name; otherwise return `None`. +- When binding a repeated placeholder, use structural comparison (compare the unparse of each bound node), + not `isinstance` / class identity. +- Parse patterns in a dedicated syntactic context (statement, expression, declaration) to avoid ambiguity; + see ADR 12 (Patterns are not nodes) for the `Pattern` + `SyntacticKind` design. +- Sequence placeholders (`$$`) must be matched greedily against sibling lists, subject to the constraints + of surrounding fixed nodes in the pattern. +- Equivalent-code normalization is applied before structural comparison; maintain a normalization table per + language frontend. + +## Example + +| Pattern | Matches | +|----------------------------|--------------------------------| +| `int $$x;` | `int a=4, b=5, c;` | +| `$type v;` | `const myclass v;` | +| `x = $value;` | `x = 1 + 2;` | +| `$x;` | `a = f(1, 2+3);` | +| `$type* ptr = new $type()` | `MyClass* ptr = new MyClass()` | +| `$f; var = $f;` | `foo(); var = foo();` | + +```python +# Placeholder resolution +def get_placeholder_name(node: AstNode) -> str | None: + """Return the placeholder name if node reduces to a single $-name, else None.""" + if isinstance(node, NameNode) and node.value.startswith("$"): + return node.value + children = node.children + if len(children) == 1: + return get_placeholder_name(children[0]) + return None + +# Structural equality for repeated placeholders +def placeholders_equal(a: AstNode, b: AstNode) -> bool: + return unparse(a) == unparse(b) +``` + +## Rationale + +Matching at the highest AST node whose syntax reduces to a single name maximizes the expressiveness of a +pattern: `$x;` can capture an entire statement, not just a leaf identifier. This was validated by an earlier +CDT-based prototype. Structural (unparse-based) equality for repeated placeholders avoids fragile class +comparisons and handles the known C++ cases where the same placeholder binds to nodes of different classes. + +## Consequences + +Positive: +- Patterns are expressive: a single placeholder can match complex subtrees. +- Repeated-placeholder equality is robust across AST class differences. +- Equivalent-code matching reduces the number of patterns needed to cover syntactic variants. + +Negative: +- `getPlaceholderName` must be implemented and maintained for each language frontend. +- Structural equality via unparsing may be slower than direct node comparison; caching may be required. +- Equivalent-code normalization tables must be kept in sync with language specifications. + +## Alternatives considered + +- Match placeholder at the **lowest** (leaf) AST node — rejected because it prevents `$x` from matching + expression statements and other compound nodes. +- Use **class-based** equality for repeated placeholders — rejected because the same placeholder can legally + bind to nodes of different classes in a single pattern (documented C++ cases above). +- Require explicit syntactic kind annotation on every placeholder — rejected because it adds verbosity; + kind is inferred via `getPlaceholderName` and the surrounding `Pattern.kind`. + +## Related decisions + +- See ADR 12 (Patterns are not nodes) for the `Pattern` / `SyntacticKind` design used by the pattern + factory. +- See ADR 10 (Type hierarchy) for the node kind taxonomy referenced by find-by-kind functionality. +- See ADR 08 (Test architecture) for the test requirements that cover matching, placeholders, and + equivalent-code matching. +- See ADR 11 (Parser with space and comment) for the lossless round-trip required by transformation tests. + +--- + +Revision history: +- 2026-03-27: Converted GitHub issue to ADR template. diff --git a/adr/14_code_repositories.md b/adr/14_code_repositories.md new file mode 100644 index 00000000..f3be5538 --- /dev/null +++ b/adr/14_code_repositories.md @@ -0,0 +1,124 @@ +# 14 - Code Repositories + +Status: Proposal + +Date: 2026-03-27 + +Authors: + - jinmin.hu@capgemini.com + - huub.joosten@capgemini.com + - luna.li@capgemini.com + - paul.nelissen@esi.nl + - pierre.vandelaar@tno.nl + +## Table of contents + +- [Context](#context) +- [Decision](#decision) +- [Implementation notes](#implementation-notes) +- [Example](#example) +- [Rationale](#rationale) +- [Consequences](#consequences) +- [Alternatives considered](#alternatives-considered) +- [Related decisions](#related-decisions) + +## Context + +The goal of this ADR is to define the repository structure for the Renaissance project, +balancing modularity, licensing, and contributor accessibility. + +The project consists of two conceptually distinct layers: + +1. **Generic functionality** — the unified AST model, match-pattern engine, rewriter, and other + language-agnostic components. +2. **Adapters** — language-specific bridges (tree-sitter, Clang, Python 2.x, …) that translate a parser's + output into the unified AST. + +Keeping both layers in a single repository conflates their concerns, complicates licensing (an adapter +author may not want to adopt the same license as the core), and makes it harder for external contributors +to develop or distribute adapters independently. Repository names must also clearly describe their contents; +names like *rejuvenation* and *renaissance* do not communicate what belongs where. + +## Decision + +- Maintain **separate repositories** for the generic functionality and for each adapter. +- Repository names must **clearly describe their contents** (e.g., `unified-ast-core`, + `unified-ast-adapter-treesitter`, `unified-ast-adapter-clang`). +- The names *rejuvenation* and *renaissance* must **not** be used as the distinguishing names between the + core and adapter packages, as they do not convey their respective responsibilities. +- Plug-in / adapter points beyond parsers (e.g., output formatters, analysis passes) are also eligible for + their own repositories; evaluate case by case. + +## Implementation notes + +- Define a stable, versioned **adapter API** (a set of `Protocol` / abstract base classes) in the core + repository that all adapter repositories must implement. +- Publish the core and each adapter as independent packages on PyPI (or an internal registry) so they can + be versioned and licensed independently. +- Use the adapter API version as the compatibility contract between core and adapters; bump it on breaking + changes. +- Document the adapter API in the core repository so external contributors can develop adapters without + access to the full codebase. + +## Example + +Proposed repository / package layout: + +``` +unified-ast-core/ # generic: unified AST, matcher, rewriter, … +unified-ast-adapter-treesitter/ # adapter: tree-sitter → unified AST +unified-ast-adapter-clang/ # adapter: Clang/CDT → unified AST +unified-ast-adapter-python/ # adapter: CPython ast → unified AST +``` + +Each adapter depends on `unified-ast-core` and implements the `AdapterProtocol`: + +```python +# In unified-ast-core +from typing import Protocol + +class AdapterProtocol(Protocol): + def parse(self, source: str) -> AstNode: ... + def unparse(self, node: AstNode) -> str: ... +``` + +## Rationale + +Separating the core from adapters respects the single-responsibility principle at the repository level, +enables independent licensing (critical for adapters that wrap GPL or proprietary parsers), and lowers the +barrier for external contributors who only need to implement an adapter. Descriptive repository names make +the architecture self-documenting and reduce onboarding friction. + +## Consequences + +Positive: +- Independent versioning and licensing for core and each adapter. +- External contributors can develop adapters without forking the core. +- Clear repository names make the architecture immediately understandable. +- Smaller, focused repositories are easier to test and review. + +Negative: +- More repositories to maintain and keep in sync. +- The adapter API must be carefully designed and versioned to avoid frequent breaking changes. +- Cross-repository CI pipelines require additional setup. + +## Alternatives considered + +- **Single monorepo** — rejected because it conflates licensing concerns and makes independent adapter + distribution harder. +- **Keep current names** (*rejuvenation* / *renaissance*) — rejected because they do not describe what + belongs in each package, causing confusion for contributors. +- **One repo per language** (core bundled with adapter) — rejected because it duplicates the core and + creates divergence risk. + +## Related decisions + +- See ADR 07 (Package management) for the tooling used to publish and manage these packages. +- See ADR 03 (Duck typing) for the `Protocol`-based adapter API design. +- See ADR 12 (Patterns are not nodes) and ADR 13 (Match pattern) for the core APIs that adapters must + produce output for. + +--- + +Revision history: +- 2026-03-27: Converted GitHub issue to ADR template. diff --git a/adr/README.md b/adr/README.md new file mode 100644 index 00000000..767afaf5 --- /dev/null +++ b/adr/README.md @@ -0,0 +1,54 @@ +# Architecture Decision Records + +This directory contains all Architecture Decision Records (ADRs) for the Renaissance project. +Each ADR documents a significant design or technology choice, its context, rationale, and consequences. + +The goal of ADR is to give the developer of new language AST for Renaissance a guideline on: +* how to make correct design choice coherent to the ADR during the implementation and gives + rationale on why each decision are made. +* minimize the effort for the developer of the new language for renaissance. +* provide insight to the design and evolution of the project for developer of new language and + future maintainers and contributors. + +## Index + +| # | Title | Status | +|-------------------------------------------|--------------------------------------------------------|----------| +| [01](01_children_and_properties.md) | Children and properties | Accepted | +| [02](02_direct_access.md) | Direct access to fields | Accepted | +| [03](03_duck_typing.md) | Duck typing for nodes | Accepted | +| [04](04_immutable_properties.md) | Make nodes immutable | Proposal | +| [05](05_buildin_functions.md) | Use Python's built-in dunder methods for node behavior | Proposal | +| [06](06_wrapper_or_adapter.md) | Wrapper or adapter for external node shapes | Proposal | +| [07](07_package_management.md) | Use UV for package & environment management | Proposal | +| [08](08_pytest_suite.md) | Test Architecture | Accepted | +| [09](09_property_based_tests.md) | Property-Based Tests | Proposal | +| [10](10_type_hierarchy.md) | Type Hierarchy | Proposal | +| [11](11_parser_with_space_and_comment.md) | Parser with Space and Comment | Proposal | +| [12](12_patterns_as_not_nodes.md) | Patterns Are Not Nodes | Proposal | +| [13](13_match_pattern.md) | Match Pattern | Proposal | +| [14](14_code_repositories.md) | Code Repositories | Proposal | +## ADR template +Each ADR follows this structure: +``` +# - +Status: Proposal | Accepted | Deprecated | Superseded +Date: YYYY-MM-DD +Authors: ... +## Table of contents +## Context +## Decision +## Implementation notes +## Example +## Rationale +## Consequences +## Alternatives considered +## Related decisions +--- +Revision history: +``` +--- +Revision history: +- 2026-03-27: Created index. +- 2026-03-27: Added ADR 12 (Patterns Are Not Nodes). +- 2026-03-27: Added ADR 13 (Match Pattern) and ADR 14 (Code Repositories). diff --git a/features/__init__.py b/features/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/features/__init__.py @@ -0,0 +1 @@ + diff --git a/features/convert-unit-to-pytest.feature b/features/convert-unit-to-pytest.feature new file mode 100644 index 00000000..ffa88e60 --- /dev/null +++ b/features/convert-unit-to-pytest.feature @@ -0,0 +1,35 @@ +Feature: Convert unittest to pytest + In order to create modern pythton project + As a Developer + I want a consistent set of unit test expressing specification of code behavior + Scenario: convert unittest to pytest + Given 'targets/pyunit_test_example.py' file + And it contains 'import unittest' + And it contains 'from unittest import TestCase' + And it contains 'assert ' + And it contains 'self.assertEqual' + And it contains 'print' + And it contains '@unittest.skip' + And it contains '@parameterized.expand' + And it contains 'class FindMatchTest(unittest.TestCase):' + And it contains 'def test_it_has_elements():' + And it contains 'assert 0 == count, "count = " + str(count)' + And an AST extracted from that source file without errors + When I convert it to pytest + Then AST extracted from that conversion should without errors + And it should not contain 'import unittest' + And it should not contain 'assert 0 == count, "count = " + str(count)' + And it should not contain 'assertEqual(a,5)' + And it should not contain 'assertEqual(55,b)' + And it should not contain '@unittest.skip' + And it should not contain '@parameterized.expand' + And it should not contain 'class FindDescendantMatchTest(unittest.TestCase):' + + And it should contain 'import pytest' + And it should contain 'assert_that(results, has_length(0), f"length of results = {len(results)}")' + And it should contain 'assert_that(self.a, is_(5))' + And it should contain 'assert_that(self.b, is_(55))' + And it should contain '@pytest.mark.skip' + And it should contain '@pytest.mark.parametrize("_, factory",Factories.factories)' + + And it should contain 'class TestFindMatch:' diff --git a/features/refactor-python-file.feature b/features/refactor-python-file.feature new file mode 100644 index 00000000..fb810164 --- /dev/null +++ b/features/refactor-python-file.feature @@ -0,0 +1,15 @@ +Feature: Ast based changes + In order to get started on the system test + As a Developer + I want a working example of how system test looks like + + Scenario: python code + Given 'python' programming language + And 'targets/demo.py' file written in that programming language + And an AST extracted from that source file without errors + And node 'a=1' exits within that AST + And a sequence of descendant nodes of that node + When that node is replaced by 'a=5' + And rewrites replace is performed on that sequence of descendant nodes + Then in the modified source file that node is replaced by the given text + And all rewrites on that sequence of descendant nodes are not performed or hidden diff --git a/features/refactor-taut-test.feature b/features/refactor-taut-test.feature new file mode 100644 index 00000000..690579f2 --- /dev/null +++ b/features/refactor-taut-test.feature @@ -0,0 +1,60 @@ +Feature: taut migration + Scenario: migrate taut to unittest without syntax errors + Given 'targets/taut/taut_test.py' file + And it contains 'import TAUT' + And it contains 'class TestImport(TAUT.TestCase):' + And it contains 'self.import_and_verify_module("ABCDxTL")' + And it contains '@TAUT.log_stub' + And it contains 'with TAUT.TestDoubles(abcdxtl=FakeABCDxTL(None)):' + And it contains 'log = TAUT.Logger()' + And it contains 'def setUp(self):' + And it contains 'self.doubles' + And it contains 'def tearDown(self):' + And it contains 'self.tds' + And it contains 'def setUpCommon(self):' + And it contains 'def tearDownCommon(self):' + And it contains 'def test_readout_is_ok(self):\n self.doubles.append(' + And it contains 'def test_read_two_doubles(self):\n self.doubles.append(' + And it contains 'self.assert_false' + And it contains 'self.assert_true' + And it contains 'self.assert_equal' + And it contains 'import mock' + And it contains 'TAUT.StubServer' + And it contains 'sharedSetUp(self):' + And it contains 'with TAUT.TestDoubles(module=ABCD, startup=startup_stub):' + And it contains '@mock.patch' + And an AST extracted from that source file without errors + When I convert taut to unittest + Then AST extracted from that conversion should without errors + And it should not contain 'import TAUT' + And it should contain 'class TestImport(unittest.TestCase):' + And it should contain 'import ABCDxTL\n self.assertIsNotNone(ABCDxTL)' + And it should not contain '@TAUT.log_stub' + And it should contain 'fake_abcdxtl = FakeABCDxTL(None)' + And it should not contain 'log = TAUT.Logger()' + And it should contain 'test_log = fake_abcdxtl.create_test_log(test_log_id)' + And it should contain 'test_log, version_mismatch = fake_abcdxtl.retrieve_test_log(file_id, test_log_id, file_name)' + And it should contain 'fake_abcdxtl.store_test_log(file_id, test_log)' + And it should contain 'self.patches' + And it should contain 'p.start()' + And it should not contain 'self.doubles' + And it should contain 'self.patches' + And it should contain 'p.stop()' + And it should not contain 'self.tds' + And it should contain 'def setUpCommon(self):' + And it should contain 'self.patchers' + And it should contain 'p.start()' + And it should contain 'def tearDownCommon(self):' + And it should contain 'self.patchers' + And it should contain 'p.stop()' + And it should contain 'def test_readout_is_ok(self):\n with patch.object(' + And it should contain 'def test_read_two_doubles(self):\n with patch.object(' + And it should contain 'self.assertFalse' + And it should contain 'self.assertTrue' + And it should contain 'self.assertEqual' + And it should contain 'try:\n from unittest.mock import patch\nexcept ImportError:\n from mock import patch' + And it should not contain 'TAUT.StubServer' + And it should contain 'def setUp(self):' + And it should not contain 'sharedSetUp(self):' + And it should contain 'with patch.object(ABCD, 'startup', new=startup_stub):' + And it should contain '@patch(' diff --git a/features/steps/__init__.py b/features/steps/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/features/steps/conftest.py b/features/steps/conftest.py new file mode 100644 index 00000000..d0741ea5 --- /dev/null +++ b/features/steps/conftest.py @@ -0,0 +1,6 @@ +from pathlib import Path + +from features.steps.test_steps import * + +FEATURES_BASE_DIR = Path(__file__).resolve().parent.parent + diff --git a/features/steps/test_refactor.py b/features/steps/test_refactor.py new file mode 100644 index 00000000..d2f8cc07 --- /dev/null +++ b/features/steps/test_refactor.py @@ -0,0 +1,78 @@ +from pathlib import Path + +import pytest +from pytest_bdd import given, when, scenario, parsers, then + +from steps.conftest import FEATURES_BASE_DIR +from renaissance.impl.python.factory import PythonPatternFactory +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.syntax_tree import ASTFactory, ASTRewriter +from renaissance.syntax_tree.match_finder import match_pattern + + +class Context(dict): + def __getattr__(self, name): + return self[name] + + def __setattr__(self, name, value): + self[name] = value + + +@pytest.fixture +def context(): + return Context() + + +@scenario( + "refactor-python-file.feature", + "python code", + encoding="utf-8", + features_base_dir=str(FEATURES_BASE_DIR) +) +def test_refactor_python_file(): + pass + + +@given("'python' programming language") +def init_language_factory(context): + context["factory"] = ASTFactory(PythonRstNode, "") + + +@given(parsers.parse("'{file}' file written in that programming language")) +def step_impl(context, file): + context["atu"] = context["factory"].create(FEATURES_BASE_DIR / Path(file)) + + +@given(parsers.parse("node '{old}' exits within that AST")) +def step_impl(context, old): + pattern_factory = PythonPatternFactory(context["factory"]) + find = pattern_factory.create_statements(old) + context["result"] = match_pattern(context["atu"].children, find) + assert context["result"] + + +@given("a sequence of descendant nodes of that node") +def step_impl(context): + assert context["result"][0].nodes[0].children + + +@when(parsers.parse("that node is replaced by '{replacement}'")) +def step_impl(context, replacement): + context["replacement"] = replacement + context["rewriter"] = ASTRewriter(context["atu"]) + context["rewriter"].replace(replacement, context["result"][0].nodes) + + +@when("rewrites replace is performed on that sequence of descendant nodes") +def step_impl(context): + context["rewriter"].apply() + + +@then("in the modified source file that node is replaced by the given text") +def step_impl(context): + assert context["replacement"] in context["rewriter"].apply_to_string() + + +@then("all rewrites on that sequence of descendant nodes are not performed or hidden") +def step_impl(context): + assert context["rewriter"].has_changed() diff --git a/features/steps/test_steps.py b/features/steps/test_steps.py new file mode 100644 index 00000000..d891daa9 --- /dev/null +++ b/features/steps/test_steps.py @@ -0,0 +1,51 @@ +from pathlib import Path + +import pytest +from hamcrest import assert_that, calling, is_not, raises, contains_string, not_ +from pytest_bdd import given, then, parsers + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonFactory + +FEATURES_DIR = Path(__file__).parent.parent + + +class Ast: + def __init__(self): + self.file = "" + self.atu = None + self.signature = None + + +@pytest.fixture +def context(): + return Ast() + + +@given(parsers.parse("'{file}' file")) +def step_given_file(context, file): + context.file = str(FEATURES_DIR / file) + context.factory = PythonFactory(PythonRstNode) + context.atu = context.factory.create(context.file) + context.signature = context.atu.signature + + +@given(parsers.parse("it contains '{statement}'")) +@then(parsers.parse("it should contain '{statement}'")) +def step_given_contains(context, statement): + statement = statement.replace("\\n", "\n") + assert_that(context.signature, contains_string(statement), f"Expected '{statement}' in source") + + +@given("an AST extracted from that source file without errors") +@then("AST extracted from that conversion should without errors") +def step_given_ast_no_errors(context): + assert_that( + calling(context.atu.translation_unit.check_diagnostics), + is_not(raises(Exception)), + ) + + +@then(parsers.parse("it should not contain '{statement}'")) +def step_then_not_contain(context, statement): + assert_that(context.signature, not_(contains_string(statement))) diff --git a/features/steps/test_taut_refactor.py b/features/steps/test_taut_refactor.py new file mode 100644 index 00000000..df1be9da --- /dev/null +++ b/features/steps/test_taut_refactor.py @@ -0,0 +1,26 @@ +from pathlib import Path + +from pytest_bdd import when, scenario + +from steps.conftest import FEATURES_BASE_DIR +from renaissance.refactoring.taut2pyunit import Taut2Pyunit + + +@scenario( + "refactor-taut-test.feature", + "migrate taut to unittest without syntax errors", + encoding="utf-8", + features_base_dir=str(FEATURES_BASE_DIR) +) +def test_taut_test(): + pass + + +@when("I convert taut to unittest") +def step_when_convert(context): + converter = Taut2Pyunit(context.file) + converter.in_memory = True + converter.run() + context.atu = context.factory.create(context.file) + context.signature = converter.apply_to_string() + Path(converter.get_migrated_path(context.file)).unlink(missing_ok=True) diff --git a/features/steps/unit2pytest_steps.py b/features/steps/unit2pytest_steps.py new file mode 100644 index 00000000..60051a1d --- /dev/null +++ b/features/steps/unit2pytest_steps.py @@ -0,0 +1,15 @@ +from pytest_bdd import when, scenario +from steps.test_steps import * +from renaissance.refactoring.unit2pytest import Unit2Pytest + + +@scenario("convert-unit-to-pytest.feature", "convert unittest to pytest", "utf-8", "..") +def test_convert_unit_to_pytest(): + pass + + +@when("I convert it to pytest") +def step_when_convert(context): + converter = Unit2Pytest(context.file) + converter.run() + context.atu = context.factory.create(context.file) diff --git a/features/targets/README.md b/features/targets/README.md new file mode 100644 index 00000000..cfca0755 --- /dev/null +++ b/features/targets/README.md @@ -0,0 +1,18 @@ +# Most usefull commands: + +## gcc +gcc -fdump-tree-all-raw-lineno -fdump-rtl-all-raw-lineno -o main.exe main.c + + +## clang + +### ast dump + + `clang -Xclang -ast-dump -fsyntax-only main.c > ast-dump.ast` +or + `clang -Xclang -ast-dump -fsyntax-only main.c > ast-dump.ast` +### preprocessing dump + +`pp-trace main.c > pptrace.ast` + +contains all preprocessing directives and all usages. \ No newline at end of file diff --git a/features/targets/__init__.py b/features/targets/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/features/targets/compile_commands.json b/features/targets/compile_commands.json new file mode 100644 index 00000000..38c57a0e --- /dev/null +++ b/features/targets/compile_commands.json @@ -0,0 +1,16 @@ +[ + { + "directory": "Z:\\testproject\\c\\src", + "file": "test.cpp", + "output": "C:\\Users\\PNELIS~1\\AppData\\Local\\Temp\\1\\test-9e2a00.o", + "arguments": [ + "C:\\Users\\pnelissen\\scoop\\apps\\llvm\\current\\bin\\clang++.exe", + "-xc++", + "test.cpp", + "-o", + "C:\\Users\\PNELIS~1\\AppData\\Local\\Temp\\1\\test-9e2a00.o", + "--driver-mode=g++", + "--target=x86_64-pc-windows-msvc19.39.33521" + ] + } +] \ No newline at end of file diff --git a/features/targets/cpp_example.cpp b/features/targets/cpp_example.cpp new file mode 100644 index 00000000..43330f2d --- /dev/null +++ b/features/targets/cpp_example.cpp @@ -0,0 +1,37 @@ +//c lib using cpp conv +#include <cstdio> +//c++ lib +#include <iostream> +#include <string> +//c++17 libs +#include <filesystem> +#include <optional> +//c++26 libs +#include <ranges> + + +namespace example +{ + class base { + public: + virtual void greet() const {} + }; + class derived : public base { + public: + void greet() const override { + std::cout << "Hello from derived class!" << std::endl; + } + }; + + void cpp_example() { + std::cout << "Hello from C++!" << std::endl; + + // Using C++17 filesystem + std::filesystem::path path = "example.txt"; + if (std::filesystem::exists(path)) { + std::cout << "File exists: " << path << std::endl; + } else { + std::cout << "File does not exist: " << path << std:: + } + } +} \ No newline at end of file diff --git a/features/targets/demo.py b/features/targets/demo.py new file mode 100644 index 00000000..1abf5e97 --- /dev/null +++ b/features/targets/demo.py @@ -0,0 +1,62 @@ +from python import ( + test_python_matcher, + test_python_astshower, + test_python_ast_node_ref, + test_ast_factory, +) + + +def some_old_fun(): + a = 1 + b = a + return b + + +component_one, component_two = 1, 2 +component_three: int = 3 +component_four = 4 +component_five = 5 +component_six = sum(2, 4) + +long_expression = component_one + component_two + component_three + component_four + component_five + component_six + + +def xyzzy(a1, a2, long_parameter_1, a3, a4, long_parameter_2): + pass + + +xyzzy(1, 2, "long_string_constant1", 3, 4, "long_string_constant2") + +xyzzy("with", "hanging", "indent") +items = [] +attrs = [e.attr for e in items] + +num_dict = {"one": 1, "two": 2, "three": 3, "four": 4, "five": 5} + +colors = ["red", "green", "blue", "black", "white", "gray"] + +star_names = {"Sirius", "Betelgeuse", "Polaris", "Vega", "Arcturus", "Aldebaran"} + +planets = ( + "Mercury", + "Venus", + "Earth", + "Mars", + "Jupiter", + "Saturn", + "Uranus", + "Neptune", +) + +ingredients = [ + "green", + "eggs", +] + +if True: + pass + +try: + pass +finally: + pass diff --git a/features/targets/go/__init__.py b/features/targets/go/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/features/targets/go/extractor.py b/features/targets/go/extractor.py new file mode 100644 index 00000000..aeadc1ab --- /dev/null +++ b/features/targets/go/extractor.py @@ -0,0 +1,18 @@ +from pathlib import Path + +from targets.go.node import GoAstNode + + +class GoExtractor: + codebase: dict = {} + nodes: dict = {} + edges: dict = {} + + def process_file(self, file: Path): + root = GoAstNode.load(file) + tu = root.translation_unit + tu.lazy_create_refers(root) + self.codebase[file] = root + self.nodes |= tu.nodes + self.edges |= tu.references + self.edges |= tu.referenced_by diff --git a/features/targets/go/factory.py b/features/targets/go/factory.py new file mode 100644 index 00000000..33977d38 --- /dev/null +++ b/features/targets/go/factory.py @@ -0,0 +1,9 @@ +from typing import Any, Self, Sequence + + +class GoFactory: + pass + + +class GoPatternFactory: + pass diff --git a/features/targets/go/matcher.py b/features/targets/go/matcher.py new file mode 100644 index 00000000..ba8b9500 --- /dev/null +++ b/features/targets/go/matcher.py @@ -0,0 +1,11 @@ +from typing import Protocol, Self, runtime_checkable + + +@runtime_checkable +class NodeMatchProtocol(Protocol): + properties: dict + children: list[Self] + + +def is_match(src: NodeMatchProtocol, cmp: NodeMatchProtocol) -> bool: + pass diff --git a/features/targets/go/node.py b/features/targets/go/node.py new file mode 100644 index 00000000..72a94bc9 --- /dev/null +++ b/features/targets/go/node.py @@ -0,0 +1,23 @@ +from typing import Any, Self, Sequence + + +class GoAstNode: + # direct access protocol + expr: Self + body: Sequence[Self] + other: Sequence[Self] + + # rewrite protocol + length: int + offset: int + name: str + + @property + def properties(self) -> dict[str, Any]: + return {"length": self.length, "offset": self.offset, "name": self.name} + + children: list[Self] = [] + + @property + def children(self) -> list[Self]: + return [self.expr, self.body, self.other] diff --git a/features/targets/go/visualizer.py b/features/targets/go/visualizer.py new file mode 100644 index 00000000..e69de29b diff --git a/features/targets/invalid.py b/features/targets/invalid.py new file mode 100644 index 00000000..86590042 --- /dev/null +++ b/features/targets/invalid.py @@ -0,0 +1,10 @@ +from module import foo, bar, baz, quux + +long_expression = component_one + component_two + component_three + component_four + component_five + component_six + + + def xyzzy(a1, a2, + long_parameter_1, + a3, a4, + long_parameter_2): +pass diff --git a/features/targets/java_example.java b/features/targets/java_example.java new file mode 100644 index 00000000..e69de29b diff --git a/features/targets/main.c b/features/targets/main.c new file mode 100644 index 00000000..62b0e00d --- /dev/null +++ b/features/targets/main.c @@ -0,0 +1,21 @@ +//#include <stdio.h> +#define FOO "foo" + +static int static_int = 2; + +#define A_DEFINE (4 + static_int) +#define B_DEFINE (A_DEFINE + static_int) + +#define FC_MACRO(arg)\ +do{\ + arg += A_DEFINE;\ +} while(0) + +int main() { + int qwerty = 3 + A_DEFINE; + FC_MACRO(qwerty); +// printf("QWERTY %d", qwerty+static_int); + FC_MACRO(qwerty); + return 0; +} + diff --git a/features/targets/pyunit_test_example.py b/features/targets/pyunit_test_example.py new file mode 100644 index 00000000..e4371225 --- /dev/null +++ b/features/targets/pyunit_test_example.py @@ -0,0 +1,111 @@ +import ast +import unittest +from unittest import TestCase +from unittest import TestCase, main +from parameterized import parameterized + +from c_cpp.factories import Factories +from renaissance.impl.clang import CPatternFactory +from renaissance.impl.python import PythonRstNode +from renaissance.syntax_tree import ASTFactory +from renaissance.syntax_tree.match_finder import ( + is_match, + find_in_list, + MatchFinder, + match_pattern, +) + + +class FindMatchTest(unittest.TestCase): + + # def setUpClass(cls): + # cls.code_text: str = "int my_function();" + def setUp(self): + self.b = 55 + print(f"{self.b=}") + self.a = 5 + print(f"{self.a=}") + self.outer_text: str = "if ($cond) { $$stmts; }" + self.inner_text: str = "my_function()" + self.code_text: str = "int code(int text){return 0;}" + self.extra_declarations_inner_text: list[str] = ["int my_function();"] + if self.extra_declarations_inner_text: + print(f"{self.extra_declarations_inner_text[0]}") + + def tearDown(self): + self.outer_text: str = None + self.inner_text: str = None + self.extra_declarations_inner_text = None + + # def tearDownClass(cls): + # cls.code_text: str = None + + def test_is_match(self): + + # plain assert + assert self.a in [self.a], "An expression matches itself" + + self.assertEqual(self.a, 5) + self.assertEqual(55, self.b) + self.assertTrue(self.a == self.a, "A statement matches itself") + self.assertFalse("statement1_pattern" == self.a, "A statement doesn't match an expression") + + @parameterized.expand(Factories.factories) + def test_case(self, _: str, factory: ASTFactory): + pattern_factory = CPatternFactory(factory) + code_pattern = factory.create_from_text(self.code_text, "text.c") + outer_pattern = pattern_factory.create_statement(self.outer_text) + inner_pattern = pattern_factory.create_expression(self.inner_text, self.extra_declarations_inner_text) + results = match_pattern([code_pattern], [outer_pattern]) + + # test length + count: int = len(results) + assert 0 == count, "count = " + str(count) + + +# no namespace +class TestBasicNoNamespace(TestCase): + code_text: str = """ + int my_function(); + void your_function() { + my_function(); + } + """ + + literal_text: str = "my_function()" + extra_declarations_literal_text: list[str] = ["int my_function();"] + + placeholder_text: str = "$f()" + extra_declarations_placeholder_text: list[str] = ["int $f();"] + + # parameterised + @parameterized.expand( + list( + Factories.extend( + [ + (literal_text, extra_declarations_literal_text), + (placeholder_text, extra_declarations_placeholder_text), + ] + ) + ) + ) + @unittest.skip("stmt and expr are the same") + # unused param + def test_snippet(self, _: str, factory: ASTFactory, snippet: str, extra_declarations: list[str]): + pattern_factory = CPatternFactory(factory) + code_pattern = factory.create_from_text(self.code_text, "text.c") # file extension consistent with C Pattern Factory + snippet_pattern = pattern_factory.create_expression(snippet, extra_declarations) + results = match_pattern(code_pattern.children, [snippet_pattern]) + count: int = len(results) + # plain assert_with_msg + self.assertEqual(1, count, "count = " + str(count)) + + +def test_it_can_be_created(): + it = PythonRstNode(ast.Pass()) + assert it + + +def test_it_has_elements(): + it = PythonRstNode(ast.parse("def fun(): pass")) + assert it[0] == it.children[0] diff --git a/features/targets/taut/taut_test.py b/features/targets/taut/taut_test.py new file mode 100644 index 00000000..1595c4b0 --- /dev/null +++ b/features/targets/taut/taut_test.py @@ -0,0 +1,138 @@ +# ------------------------------------------------------# +# History # +# 22-Jun-2010 : description # +# ------------------------------------------------------# +import unittest +import mock +import NNXA +import LLXA +import TAUT +import VIPCxUNIT +import ABCDxTL +import ABCDxABxCommonFunctions +import ABCDxABxREADLib + + +class TestImport(TAUT.TestCase): + def test_import(self): + self.import_and_verify_module("ABCDxTL") + + +class FakeABCDxTL(ABCDxTL): + @TAUT.log_stub + def create_test_log(self, test_log_id): + test_log = NNXA.Object("ABCDxTL:test_log_struct") + return test_log + + +class test_interface(TAUT.TestCase): + def run(self): + expected = self.read() + self.assert_false(expected) + self.assert_true(expected) + self.assert_equal(expected, result) + + +class ABCD_Stub(TAUT.StubServer): + def sharedSetUp(self): + with TAUT.TestDoubles(module=ABCD, startup=startup_stub): + ABCDxCONFIG.start_instance() + + @mock.patch("ABCD.result") + def test_interaction_with_ABCD(self): + pass + + +class Test_ABCDxTL(TAUT.TestCase): + def setUpCommon(self): + self.tds = [ + TestDoubles(abcdxread=ImprovedStub(ABCDxREAD.abcdxread)), + TestDoubles(abcdxws=ImprovedStub(ABCDxWS.abcdxws)), + TestDoubles(abxstream2=ImprovedStub(ABxSTREAM2.abxstream2)), + TestDoubles(bcxclear=ImprovedStub(BCxCLEAR.bcxclear)), + TestDoubles(bcxload=ImprovedStub(BCxLOAD.bcxload)), + ] + self.sut = ABCDxVIPCxAB.ABCDxVIPCxAB() + + def tearDownCommon(self): + for td in self.tds: + td.exit() + + def setUp(self): + self.bc_stub = BCxCTL_stub() + self.vipc_stub = VIPC_stub() + self.doubles = [] + self.doubles.append(TAUT.TestDoubles(module=BCxCTL.BCxCTL, reload_wafer=self.bc_stub.reload_wafer)) + self.doubles.append( + TAUT.TestDoubles( + module=ABCDxEngine.ABCDxEngine, + measure_wafer=self.engine_stub.measure_wafer_gw, + ) + ) + self.doubles.append(TAUT.TestDoubles(module=VIPC, check_stopped=self.vipc_stub.check_stopped)) + + def tearDown(self): + for double in self.doubles: + double.exit() + + +class test_log(VIPCxUNIT.TestCase): + + def test_ABCDxTL(self): + with TAUT.TestDoubles(abcdxtl=FakeABCDxTL(None)): + log = TAUT.Logger() + + test_log_id = NNXA.Object("EMTLXT:DD_test_log_id") + test_log = NNXA.Object("ABCDxTL:test_log_struct") + test_log = abcdxtl.create_test_log(test_log_id) + + file_id = NNXA.Object("EMTLXT:DD_test_log_file_id") + file_name = NNXA.Object("ABCDxTL:.retrieve_test_log.file_name") + fn = "ABCDxTL:test_log_struct" + file_name[0 : len(fn)] = "ABCDxTL:test_log_struct" + test_log, version_mismatch = abcdxtl.retrieve_test_log(file_id, test_log_id, file_name) + + abcdxtl.store_test_log(file_id, test_log) + + +class test_abcdxwid(TAUT.TestCase): + def test_readout_is_ok(self): + self.doubles.append(TAUT.TestDoubles(module=ABCDxWID.abcdwid, get_wid_readouts=stub_get_wid_readouts)) + id = ABCDxBASIC.id + read = True + ABCDxABxCommonFunctions.CLEAR_CALLED = False + self.assert_raises( + ABCD.Error(ABCDxERR.ABCD_SYS_ERR, "error message"), + ABCDxABxREADLib.read, + id, + read, + ) + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("start"), 0) + + def test_read_two_doubles(self): + self.doubles.append( + TAUT.TestDoubles( + module=ABCDxABxLib, + _create_marks=marks, + ) + ) + self.doubles.append( + TAUT.TestDoubles( + module=ABCDxEngine.ABCDxEngine, + measure=self.engine.measure, + ) + ) + id = ABCDxBASIC.id + ABCDxABxCommonFunctions.CLEAR_CALLED = False + self.assert_raises( + ABCD.Error(ABCDxERR.ABCD_SYS_ERR, "error message"), + ABCDxABxREADLib.read, + id, + ) + + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("start"), 1) + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("finish"), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/features/targets/test.cpp b/features/targets/test.cpp new file mode 100644 index 00000000..e69de29b diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..3ded2b88 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,92 @@ +#[build-system] +#requires = ["hatchling"] +#build-backend = "hatchling.build" + +[project] +name = "renaissance" +version = "0.3.1" +description = "Experimental Python version of the Renaissance refactoring tool" +readme = "README.md" +authors = [ + { name = "Luna Li", email = "luna.li@capgemini.com" } +] +license = { text = "MIT" } +requires-python = ">=3.12" +dependencies = [ + "textx==4.3.0", + "dataclasses-json==0.6.7", + "pyperclip>=1.8", + "clang==18.1.8", + "libclang==18.1.1", + "more-itertools>=10.0", + "networkx>=3.0", + "pyhamcrest>=2.1", + "pyecore>=0.14", + "pyyaml>=6.0", + "termcolor>=2.0", + "typing-extensions>=4.0", + "tree-sitter>=0.25", + "tree-sitter-python==0.25.0", + "tree-sitter-cpp==0.23.4", + "tree-sitter-java==0.23.5", + "ast-comments>=1.0", + "libcst>=1.8.6", + "antlr4-python3-runtime>=4.13.2", + "hypothesmith>=0.3.3", +] + +[dependency-groups] +test = [ + "hypothesis>=6.152.4", + "hypothesmith>=0.3.3", + "parameterized>=0.9", + "pytest>=8.0", + "pytest-bdd==8.1.0", + "pytest-cov>=7.0", + "pytest-mock>=3.15", + "pytest-profiling>=1.8", + "coverage>=7.0", + "behave" +] +lint = [ + "flake8>=7.0", + "black>=24.0", + "autopep8>=2.0", + "pytest-black>=0.6", + "ruff>=0.15.19", +] +dev = [ + {include-group = "test"}, + {include-group = "lint"}, +] + +[project.urls] +Homepage = "https://github.com/TNO/Renaissance-Experiments" +Issues = "https://github.com/TNO/Renaissance-Experiments/issues" + +[project.scripts] +rejuvenate = "rejuvenation.cli:refactor" +taut2test = "rejuvenation.cli:refactor" + +[tool.pytest.ini_options] +testpaths = ["test", "features"] +pythonpath = ["src", "test", "features"] +norecursedirs = ["features/targets", "*.egg", ".*", "_darcs", "build", "CVS", "dist", "node_modules", "venv", "{arch}"] + +[tool.coverage.run] +source = ["src"] +omit = ["test/*", "features/*"] + +[tool.coverage.report] +show_missing = true +skip_covered = false + +[tool.ruff.lint.per-file-ignores] +"**/conftest.py" = ["F401"] + +[tool.ruff.lint] +ignore = ["E731"] + +[tool.black] +line-length = 140 + diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/rejuvenation/batch_process_examples.py b/src/rejuvenation/batch_process_examples.py new file mode 100644 index 00000000..2c479bfe --- /dev/null +++ b/src/rejuvenation/batch_process_examples.py @@ -0,0 +1,171 @@ +# use clang to load and walk a compilation database +import textwrap +from dataclasses import dataclass +from typing import Callable +from renaissance.syntax_tree.recipe_ast_processor import ( + RecipeASTProcessor, + after_step, + recipe_step, + final_action, +) +from typing_extensions import Iterable +from renaissance.impl.clang import ClangASTNode +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.refactoring import CleanupRefactoring +from renaissance.syntax_tree import ( + ASTProcessor, + ASTNode, + TextUtils, + ASTFactory, + BatchASTProcessor, +) + +example_1 = textwrap.dedent(""" + void x(int a) {} + void x1(int a) {} + void x2(int a) {} + + void f1(int a){ + int unused = 0; + int unused2 = 0; //must be removed + if (a==1) { + int unused = 0; + int unused2 = 0; //should be kept + int c = unused2; + x1(c); + } + } + """) + +example_2 = textwrap.dedent(""" + void x(int a) {} + void x1(int a) {} + void x2(int a) {} + void f2(int a){ + int unused = 0; + if (a==1) { + int unused = 0; + int another_unused = 0; + int used2 = 0; //should be kept + int c = used2; + x2(c); + } + } + """) + + +# generate a simple code base provider in real life use a compilation database +def simple_codebase_provider() -> Iterable[tuple[ASTFactory, ASTNode]]: + for impl_type in [ClangASTNode, ClangJsonASTNode]: + factory = ASTFactory(impl_type) + atu1 = factory.create_from_text(example_1, impl_type.__name__ + "1.c") + yield factory, atu1 + atu2 = factory.create_from_text(example_2, impl_type.__name__ + "2.c") + yield factory, atu2 + + +def print_results(title, batch_processor): + print(title + ":") + for file, code in batch_processor.in_memory_files.items(): + print(TextUtils.shift_right(file, 4) + "\n") + print(TextUtils.shift_right(code, 8) + "\n") + + +def batch_remove_unused_variable_once_example(): + """ + This function demonstrates a batch processing example using different AST node implementations. + It iterates over a list of AST node implementations (`ClangASTNode` and `ClangJsonASTNode`), + and for each implementation, it generates a codebase provider that yields tuples of + `ASTFactory` and `ASTNode` created from example source texts (`example_1` and `example_2`). + The function then creates a `BatchASTProcessor` with in-memory storage enabled and processes + the codebase using the `CleanupRefactoring.remove_unused_variables` refactoring operation. + Finally, it prints the rewritten code stored in memory. + """ + # generate a batch processor for testing purposes we store into memory + batch_processor = BatchASTProcessor(in_memory=True) + batch_processor.once(simple_codebase_provider, CleanupRefactoring.remove_unused_variables) + # print the rewritten code normally you would write to a file + print_results("example batch remove unused variable once", batch_processor) + + +def batch_repeat_example(): + """ + Demonstrates the use of a batch processor to perform multiple refactoring operations on a codebase. + This example creates an in-memory batch processor and applies two refactoring operations: + 1. CleanupRefactoring.remove_unused_variables: Removes unused variables from the codebase. + 2. remove_function: Removes all function calls from the codebase. + The results of the refactoring operations are printed to the console. + + Repeat is in action here: + the first time the codebase is processed, the unused variables are removed. + and the function calls are removed. + the second time the codebase is processed, the new unused variables are removed again. + Note: + In a real-world scenario, the rewritten code would typically be written to a file instead of being printed. + """ + # generate a batch processor for testing purposes we store into memory + batch_processor = BatchASTProcessor(in_memory=True) + + # remove a function to create more unused variables + def remove_function(ast_processor: ASTProcessor): + [ast_processor.insert_before("// ", node, False, False) for node in ast_processor.find_ast_type(Call)] + + # batch_processor.repeat(simple_codebase_provider, [remove_function]) + batch_processor.repeat( + simple_codebase_provider, + [CleanupRefactoring.remove_unused_variables, remove_function], + ) + # print the rewritten code normally you would write to a file + print_results("example batch repeat", batch_processor) + + +@dataclass +class Call: + callee: str + calls: str + + +class AnalysisRecipe: + def __init__(self): + self._calls = [] + + @recipe_step(order=0) + def store_function_call(self, ast_processor: ASTProcessor) -> Callable[[], None] | None: + # find all function calls and store them, this routing is invoked in parallel! + calls = [] + [AnalysisRecipe._add_function_call(node, calls) for node in ast_processor.find_ast_type(Call)] + # the resulting lambda is invoked single threaded + # this kind of mechanism is mainly used to store results from multiple processors + # for refactoring operations this is not needed as a refactoring operation is single threaded + if calls: + return lambda: self._calls.extend(calls) + return None + + @after_step("store_function_call") + def just_show_the_method(self): + print("called after store_function_call") + + @final_action() + def final_action(self): + print("Calls:") + for call in self._calls: + print(" " + call.callee + " -- calls --> " + call.calls) + + @staticmethod + def _add_function_call(call: ASTNode, calls: list[Call]): + callee = call.get_ancestor("(?i)Function_?Decl") + if callee: + calls.append(Call(callee.name, call.children[0].name)) + + +def batch_recipe_example(): + print("example batch analysis using recipe:\n") + recipe_ast_processor = RecipeASTProcessor(AnalysisRecipe(), simple_codebase_provider, r".*", in_memory=True) + recipe_ast_processor.run() + + +if __name__ == "__main__": + # a list of example to show batch processing of a code base + batch_remove_unused_variable_once_example() + batch_repeat_example() + batch_recipe_example() diff --git a/src/rejuvenation/cli.py b/src/rejuvenation/cli.py new file mode 100644 index 00000000..43e2087c --- /dev/null +++ b/src/rejuvenation/cli.py @@ -0,0 +1,34 @@ +import sys +from pathlib import Path + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.extractor import PythonExtractor +from renaissance.project.project_scanner import PythonScanner +from renaissance.refactoring.python_refactoring import PythonRefactoring +from renaissance.syntax_tree import ASTShower + +def refactor(): + if sys.argv[1] == "refactor": + refactoring = sys.argv[2] + files = [sys.argv[3]] if len(sys.argv) > 3 else PythonScanner().find_sources() + print(f'Refactor {Path(".").resolve()}') + for file in files: + PythonRefactoring.process(refactoring, file) + + if sys.argv[1] == "extract": + print(f'Extracting {Path(".").resolve()}') + extractor = PythonExtractor() + for file in PythonScanner().find_sources(): + filename = sys.argv[2] + extractor.process(file) + extractor.save_graph(filename) + if sys.argv[1] == "inspect": + print(f"inspect {Path('.').resolve()}") + file = sys.argv[2] + ASTShower.focus = f"|{sys.argv[3]}" + atu = PythonRstNode.load(Path(file)) + ASTShower.show_node(atu) + + +if __name__ == "__main__": + refactor() diff --git a/src/rejuvenation/cli_taut.py b/src/rejuvenation/cli_taut.py new file mode 100644 index 00000000..999cc535 --- /dev/null +++ b/src/rejuvenation/cli_taut.py @@ -0,0 +1,42 @@ +#! /usr/bin/python3 +import fnmatch +import os +import sys +from pathlib import Path + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.project.project_scanner import PythonScanner +from renaissance.refactoring.python_refactoring import PythonRefactoring +from renaissance.syntax_tree import ASTFactory + +factory = ASTFactory(PythonRstNode, []) + + +def get_migrated_path(file_path): + """ + Convert a file path to add '_migrated' before the extension. + + Example: 'taut.py' -> 'taut_migrated.py' + """ + # Split the path into filename and extension + base, ext = os.path.splitext(file_path) + + # Create the new path with '_migrated' added + new_path = f"{base}_migrated{ext}" + + return new_path + + +def list_matching_files(root: str | Path, recursive: bool = True) -> list[Path]: + patterns = ["*_unittest.py", "*_test.py", "*_stubs.py"] + root = Path(root) + candidates = root.rglob("*.py") if recursive else root.glob("*.py") + return [p for p in candidates if any(fnmatch.fnmatch(p.name, pat) for pat in patterns)] + + +if __name__ == "__main__": + if sys.argv[1] == "refactor": + print(f'Refactor {Path(".").resolve()}') + for file in PythonScanner().find_sources(): + refactor = sys.argv[2] + PythonRefactoring.process(refactor, file) diff --git a/src/rejuvenation/cpp_clang_lst_example.py b/src/rejuvenation/cpp_clang_lst_example.py new file mode 100644 index 00000000..19dcfb72 --- /dev/null +++ b/src/rejuvenation/cpp_clang_lst_example.py @@ -0,0 +1,9 @@ +import clang + +from renaissance.impl.clang.clang_adapter import ClangAdapter +from renaissance.syntax_tree import ASTShower + +adapter = ClangAdapter(clang.__file__.replace("__init__.py", "native")) +lst = adapter.parse("features/targets/cpp_example.cpp") + +ASTShower.show_node(lst.root) diff --git a/src/rejuvenation/descendant_search.py b/src/rejuvenation/descendant_search.py new file mode 100644 index 00000000..a4ddd7d3 --- /dev/null +++ b/src/rejuvenation/descendant_search.py @@ -0,0 +1,9 @@ +from arpeggio import flatten + +from renaissance.syntax_tree import PatternMatch +from renaissance.syntax_tree.ast_node import ASTNode +from renaissance.syntax_tree.match_finder import match_pattern + + +def find_descendant_match(root: ASTNode, outer_pattern: ASTNode, inner_pattern: ASTNode) -> list[PatternMatch]: + return flatten(match_pattern(match.nodes, [inner_pattern]) for match in match_pattern(root.children, [outer_pattern])) diff --git a/src/rejuvenation/python_ast_example.py b/src/rejuvenation/python_ast_example.py new file mode 100644 index 00000000..c5903f9b --- /dev/null +++ b/src/rejuvenation/python_ast_example.py @@ -0,0 +1,78 @@ +import textwrap +from ast import AST + +from rejuvenation.python_lst_example import python_lst_smoke_test +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.types import Call +from renaissance.syntax_tree import ASTShower, ASTRewriter +from renaissance.syntax_tree.ast_finder import find_ast_type +from renaissance.syntax_tree.match_finder import match_pattern + +example_code = """ +from module import foo, bar, baz, quux +ba(51) +na(52) +na(53) +pa(54) +if pa(): + ba() +pa(54) +""" + + +def python_ast_smoke_test(): + + # adapter = TreeSitterAdapter(tree_sitter_python) + # tree = adapter.parse_code(code) + # lst = adapter.to_lst(code, tree) + + factory = PythonFactory(AST) + pattern_factory = PythonPatternFactory(factory) + + atu = factory.create_from_text(example_code, "example.py") + + pattern1 = pattern_factory.create_statement("if pa(): $$stmts") + pattern2 = pattern_factory.create_expression("na($a)") + + print("_______________pattern 1____________________________________") + ASTShower.show_node(pattern1.node, include_properties=True) + print("_______________pattern 1____________________________________") + ASTShower.show_node(pattern2.node, include_properties=False) + print("_______________ast____________________________________") + ASTShower.focus = "ba" + ASTShower.show_node(atu) + + print("_______________simple find____________________________________") + nodes = find_ast_type(atu, Call) + + ASTShower.show_node(nodes[0]) + + pattern1replacement = textwrap.dedent(""" + # changed if expr to const + isAOne=True + if(isAOne): + $$stmts + """) + pattern2replacement = "# changed function f1 to f2\nf2($a,123456)\n" + + rewriter = ASTRewriter(atu) + + for match in match_pattern(atu.children, [pattern1]): + refactor(match, pattern1replacement, rewriter) + + for match in match_pattern(atu.children, [pattern2]): + refactor(match, pattern2replacement, rewriter) + + return rewriter.apply_to_string() + + +def refactor(match, replacement_text, rewriter): + for placeholder in match.expansions: + replacement_text = replacement_text.replace(placeholder, match[placeholder]) + return rewriter.replace(replacement_text, match.nodes) + + +if __name__ == "__main__": + result = python_lst_smoke_test() + print("_______________end result_________________________________") + print(result) diff --git a/src/rejuvenation/python_cst_example.py b/src/rejuvenation/python_cst_example.py new file mode 100644 index 00000000..70a5fa03 --- /dev/null +++ b/src/rejuvenation/python_cst_example.py @@ -0,0 +1,79 @@ +import textwrap + +from rejuvenation.python_lst_example import python_lst_smoke_test +from renaissance.impl.python.cst_node import PythonCstNode +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.types import Call +from renaissance.syntax_tree import ASTShower, ASTRewriter +from renaissance.syntax_tree.ast_finder import find_ast_type + +from renaissance.syntax_tree.match_finder import match_pattern + +example_code = """ +from module import foo, bar, baz, quux +ba(51) +na(52) +na(53) +pa(54) +if pa(): + ba() +pa(54) +""" + + +def python_cst_smoke_test(): + + # adapter = TreeSitterAdapter(tree_sitter_python) + # tree = adapter.parse_code(code) + # lst = adapter.to_lst(code, tree) + + factory = PythonFactory(PythonCstNode) + pattern_factory = PythonPatternFactory(factory) + + atu = factory.create_from_text(example_code, "example.py") + + pattern1 = pattern_factory.create_statement("if pa(): $$stmts") + pattern2 = pattern_factory.create_expression("na($a)") + + print("_______________pattern 1____________________________________") + ASTShower.show_node(pattern1.node, include_properties=True) + print("_______________pattern 1____________________________________") + ASTShower.show_node(pattern2.node, include_properties=False) + print("_______________ast____________________________________") + ASTShower.focus = "ba" + ASTShower.show_node(atu) + + print("_______________simple find____________________________________") + nodes = find_ast_type(atu, Call) + + ASTShower.show_node(nodes[0]) + + pattern1replacement = textwrap.dedent(""" + # changed if expr to const + isAOne=True + if(isAOne): + $$stmts + """) + pattern2replacement = "# changed function f1 to f2\nf2($a,123456)\n" + + rewriter = ASTRewriter(atu) + + for match in match_pattern(atu.children, [pattern1]): + refactor(match, pattern1replacement, rewriter) + + for match in match_pattern(atu.children, [pattern2]): + refactor(match, pattern2replacement, rewriter) + + return rewriter.apply_to_string() + + +def refactor(match, replacement_text, rewriter): + for placeholder in match.expansions: + replacement_text = replacement_text.replace(placeholder, match[placeholder]) + return rewriter.replace(replacement_text, match.nodes) + + +if __name__ == "__main__": + result = python_lst_smoke_test() + print("_______________end result_________________________________") + print(result) diff --git a/src/rejuvenation/python_lst_example.py b/src/rejuvenation/python_lst_example.py new file mode 100644 index 00000000..fff835f3 --- /dev/null +++ b/src/rejuvenation/python_lst_example.py @@ -0,0 +1,78 @@ +import textwrap + +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.tree_sitter.lst import LSTNode +from renaissance.impl.types import Call + +from renaissance.syntax_tree import ASTShower, ASTRewriter +from renaissance.syntax_tree.ast_finder import find_ast_type +from renaissance.syntax_tree.match_finder import match_pattern + +example_code = """ +from module import foo, bar, baz, quux +ba(51) +na(52) +na(53) +pa(54) +if pa(): + ba() +pa(54) +""" + + +def python_lst_smoke_test(): + + # adapter = TreeSitterAdapter(tree_sitter_python) + # tree = adapter.parse_code(code) + # lst = adapter.to_lst(code, tree) + + factory = PythonFactory(LSTNode) + pattern_factory = PythonPatternFactory(factory) + + atu = factory.create_from_text(example_code, "example.py") + + pattern1 = pattern_factory.create_statement("if pa(): $$stmts") + pattern2 = pattern_factory.create_expression("na($a)") + + print("_______________pattern 1____________________________________") + ASTShower.show_node(pattern1.node, include_properties=True) + print("_______________pattern 1____________________________________") + ASTShower.show_node(pattern2.node, include_properties=False) + print("_______________ast____________________________________") + ASTShower.focus = "ba" + ASTShower.show_node(atu) + + print("_______________simple find____________________________________") + nodes = find_ast_type(atu, Call) + + ASTShower.show_node(nodes[0]) + + pattern1replacement = textwrap.dedent(""" + # changed if expr to const + isAOne=True + if(isAOne): + $$stmts + """) + pattern2replacement = "# changed function f1 to f2\nf2($a,123456)\n" + + rewriter = ASTRewriter(atu) + + for match in match_pattern(atu.children, [pattern1]): + refactor(match, pattern1replacement, rewriter) + + for match in match_pattern(atu.children, [pattern2]): + refactor(match, pattern2replacement, rewriter) + + return rewriter.apply_to_string() + + +def refactor(match, replacement_text, rewriter): + for placeholder in match.expansions: + replacement_text = replacement_text.replace(placeholder, match[placeholder]) + return rewriter.replace(replacement_text, match.nodes) + + +if __name__ == "__main__": + result = python_lst_smoke_test() + print("_______________end result_________________________________") + print(result) diff --git a/src/rejuvenation/python_rst_example.py b/src/rejuvenation/python_rst_example.py new file mode 100644 index 00000000..39612ec2 --- /dev/null +++ b/src/rejuvenation/python_rst_example.py @@ -0,0 +1,77 @@ +# This script demonstrates the use of the syntax_tree library to parse and rewrite Python code. +# It specifically showcases nested replacements and multiple patterns. +import textwrap + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.types import Call +from renaissance.syntax_tree import ASTRewriter +from renaissance.syntax_tree import ASTShower +from renaissance.syntax_tree.ast_finder import find_ast_type +from renaissance.syntax_tree.match_finder import match_pattern + +example_code = """ +from module import foo, bar, baz, quux +ba(51) +na(52) +na(53) +pa(54) +if pa(): + ba() +pa(54) +""" + + +def python_rst_smoke_test(): + atu: PythonRstNode = PythonRstNode.load_from_text(example_code) + + factory = PythonFactory(PythonRstNode) + pattern_factory = PythonPatternFactory(factory) + + atu = factory.create_from_text(example_code, "example.py") + + pattern1 = pattern_factory.create_statement("if pa(): $$stmts") + pattern2 = pattern_factory.create_expression("na($a)") + + print("_______________pattern 1____________________________________") + ASTShower.show_node(pattern1.node, include_properties=True) + print("_______________pattern 1____________________________________") + ASTShower.show_node(pattern2.node, include_properties=False) + print("_______________ast____________________________________") + ASTShower.focus = "ba" + ASTShower.show_node(atu) + + print("_______________simple find____________________________________") + nodes = find_ast_type(atu, Call) + + ASTShower.show_node(nodes[0]) + + pattern1replacement = textwrap.dedent(""" + # changed if expr to const + isAOne=True + if(isAOne): + $$stmts + """) + pattern2replacement = "# changed function f1 to f2\nf2($a,123456)\n" + + rewriter = ASTRewriter(atu) + + for match in match_pattern(atu.body, [pattern1]): + refactor(match, pattern1replacement, rewriter) + + for match in match_pattern(atu.body, [pattern2]): + refactor(match, pattern2replacement, rewriter) + + return rewriter.apply_to_string() + + +def refactor(match, replacement_text, rewriter): + for placeholder in match.expansions: + replacement_text = replacement_text.replace(placeholder, match[placeholder]) + return rewriter.replace(replacement_text, match.nodes) + + +if __name__ == "__main__": + result = python_rst_smoke_test() + print("_______________end result_________________________________") + print(result) diff --git a/src/rejuvenation/recipe_example.py b/src/rejuvenation/recipe_example.py new file mode 100644 index 00000000..e1bdab2e --- /dev/null +++ b/src/rejuvenation/recipe_example.py @@ -0,0 +1,290 @@ +# use clang to load and walk a compilation database +import textwrap + +from more_itertools import last +from typing_extensions import Iterable + +from renaissance.impl.clang import ClangASTNode, CPPPatternFactory +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.impl.types import Constructor, Method, TypeReference +from renaissance.syntax_tree import ( + ASTRefactorActions, + RecipeASTProcessor, + recipe_step, +) +from renaissance.syntax_tree import ASTProcessor, ASTNode, TextUtils, ASTFactory +from renaissance.syntax_tree.ast_finder import matches_kind + +example_1 = textwrap.dedent(""" +#include <vector> +struct Size +{ + double length; + double width; + + // Constructor to initialize the Rectangle object with length and width + Size() : length(0), width(0) {} + Size(double len, double wid) : length(len), width(wid) {} + + Size size() + { + return Size(this->length, this->width); + } +}; + +typedef const char* string; + +int main(){ + // do nothing +} +void setItemLayout(int, Size size){ +} +class aClass{ + void main1(std::vector<int> m_items){ + std::vector<int> idToBeReplaced; + idToBeReplaced.push_back((int)m_items.size()); + setItemLayout(1, Size(this->getBounds().size().width, 30)); + } + Size getBounds(){ + return Size(10, 30); + } +}; + +class ListView_LEGACY{ + public: + ListView_LEGACY(); + ListView_LEGACY(string container, int val); + Size size; +}; + +ListView_LEGACY::ListView_LEGACY(string container, int val){ + +} +ListView_LEGACY::ListView_LEGACY(){ + +} + +class derived : public ListView_LEGACY{ + public: + derived(string cont) : ListView_LEGACY(cont, 5) { + // something + }; + void another_func(int s); +}; + +void derived::another_func(int s){ + int a = 0; + // another_func 0 + // another_func 1 +} + +void main2(string container){ + /* hahaha*/ + ListView_LEGACY listview(container, 3); + int b; + int a; + listview.size = Size(4, 5); +} + +void main3() +{ + int b; + string container, foo; + ListView_LEGACY listview(container, 3); + derived d(foo); + listview.size = Size(4, 5); +} + +void main4(std::vector<int> m_items) +{ + /** + * multi-line comments + * in my code; + * do this wrack my indent algo? + */ + std::vector<int> idToBeReplaced; + /** + * multi-line comments + * in my code; + * do this wrack my indent algo? + */ + idToBeReplaced.push_back((int)m_items.size()); + /** + * multi-line comments + * in my code; + * do this wrack my indent algo? + */ +} +""") +expected_output = textwrap.dedent(""" + void main(){ + std::vector<int> NEW_ID; + NEW_ID.push_back((int)m_items.size()); + setItemLayout(1, Size(this->getBounds().size().width,30)); +} + +void main() { + /*ahah*/ + ListViewCustom listview; + ListViewHeader listviewHeader0 + /* Conversion note: give header appropriate name */ + ListViewHeader listviewHeader1 + /* Conversion note: give header appropriate name */ + ListViewHeader listviewHeader2 /* Conversion note: give header appropriate name */ + bool b; bool a; + listview(container), + listviewHeader0(listview), + listviewHeader1(listview), + listviewHeader2(listview); + listview.size = Size(4, 5); + listviewHeader0.name = L"listviewHeader0";/* Conversion note: give header appropriate name */ + listviewHeader0.size = Size(256, 30); /* Conversion note: provide correct sizes */ + listviewHeader1.name = L"listviewHeader1";/* Conversion note: give header appropriate name */ + listviewHeader1.size = Size(256, 30); /* Conversion note: provide correct sizes */ + listviewHeader2.name = L"listviewHeader2";/* Conversion note: give header appropriate name */ + listviewHeader2.size = Size(256, 30); /* Conversion note: provide correct sizes */ +} + +class ListView_LEGACY { + ListView_LEGACY(string container, int val); +}; +class derived: public ListView_LEGACY { + derived(string cont):ListViewCustom(cont), m_headers {, + std:make_unique<ListViewHeader>(*this), + std:make_unique<ListViewHeader>(*this), + std:make_unique<ListViewHeader>(*this), + std:make_unique<ListViewHeader>(*this), + std:make_unique<ListViewHeader>(*this)}{ + //something + }; + void another_func(int s ); +}; +void __REPLACEMENT__(){} + +void main(){ + ListViewCustom listview; + ListViewHeader listviewHeader0 + /* Conversion note: give header appropriate name */ + ListViewHeader listviewHeader1 + /* Conversion note: give header appropriate name */ + ListViewHeader listviewHeader2 /* Conversion note: give header appropriate name */ + bool b; + string container; + listview(container), + listviewHeader0(listview), + listviewHeader1(listview), + listviewHeader2(listview); + derived d(); + listview.size = Size(4, 5); + listviewHeader0.name = L"listviewHeader0";/* Conversion note: give header appropriate name */ + listviewHeader0.size = Size(256, 30); /* Conversion note: provide correct sizes */ + listviewHeader1.name = L"listviewHeader1";/* Conversion note: give header appropriate name */ + listviewHeader1.size = Size(256, 30); /* Conversion note: provide correct sizes */ + listviewHeader2.name = L"listviewHeader2";/* Conversion note: give header appropriate name */ + listviewHeader2.size = Size(256, 30); /* Conversion note: provide correct sizes */ +} + +void main(){ + /** + * multi-line comments + * in my code; + * do this wrack my indent algo? + */ + std::vector<int> NEW_ID; + /** + * multi-line comments + * in my code; + * do this wrack my indent algo? + */ + NEW_ID.push_back((int)m_items.size()); + /** + * multi-line comments + * in my code; + * do this wrack my indent algo? + */ +} +""") + + +# generate a simple code base provider in real life use a compilation database +def simple_codebase_provider() -> Iterable[tuple[ASTFactory, ASTNode]]: + for impl_type in [ClangASTNode, ClangJsonASTNode][0:1]: + factory = ASTFactory(impl_type) + atu1 = factory.create_from_text(example_1, impl_type.__name__ + "1.cpp") + yield factory, atu1 + + +class MyRefactor: + def __init__(self): + self._calls = [] + + @recipe_step(order=0) + def recipe(self, ast_processor: ASTProcessor): + pattern = CPPPatternFactory(ast_processor.factory) + actions = ASTRefactorActions(ast_processor, pattern) + actions.replace_text("ListView_LEGACY", "ListViewCustom", skip_kind=TypeReference) + actions.replace_name("another_func", "__REPLACEMENT__", Method) + actions.replace_text("idToBeReplaced", "NEW_ID") + # TODO debate the way to replace this the options are: + # 1. make a match of the consecutive nodes. + # 2. find a neat construction for the current backtick replacement + actions.replace_declaration("int $var;", r"bool $var`int\s+(.+)`;") + # create a constructor pattern + constructor_pattern = pattern.create( + "typedef int string; class ListView_LEGACY { ListView_LEGACY(string container, int val); };", + kind="Constructor", + ) + # create a pattern to match a call to a constructor in both declarations and derived classes + constructor_call_pattern = pattern.create_constructor_call("$var($container, $headerCount)") + # search for the constructor pattern + for constructor_match in ast_processor.find_match(constructor_pattern): + # and then search for the referenced by calls to the constructor + for constructor_call in constructor_match.match_referenced_by([constructor_call_pattern]): + var_node = constructor_call.nodes["$var"][0] + parent = var_node.parent + assert isinstance(parent, ASTNode), f"{parent} is not an ASTNode" + header_count = int(constructor_call.expansions["$headerCount"]) + # remove the count argument from the constructor call + # TODO it would be a lot easier if ast rewrite would support removal of the second argument + # but currently (I guess) that would lead to a dangling comma + # TODO the items between the backtick represent a regex where all groups are the used replacements + # this might need some investigation what is the best way to handle this + if matches_kind(parent, Constructor): + # remove constructor header count argument + ast_processor.replace(r"ListViewCustom($container)", constructor_call) + repl = ",\n ".join("std:make_unique<ListViewHeader>(*this)" for _ in range(header_count)) + ast_processor.insert_after(", m_headers {" + repl + "}", constructor_call, True, False) + else: + var = parent.name + container = constructor_call.expansions["$container"] + # replace the constructor call with a ListViewCustom object + ast_processor.replace(f"ListViewCustom {var}({container});", parent) + # find reference to the declaration + size_match = last(ref.node.get_ancestor("Call_?Expr") for ref in parent.referenced_by) + + for h in range(header_count): + ast_processor.insert_after( + f"\n/* Conversion note: give header appropriate name */\nListViewHeader listviewHeader{h}({var});", + parent, + True, + False, + ) + if size_match: + text = TextUtils.strip_indent(f""" + listviewHeader{h}.name = L"listviewHeader{h}";/* Conversion note: give header appropriate name */ + listviewHeader{h}.size = Size(256, 30); /* Conversion note: provide correct sizes */ + """) + ast_processor.insert_after(text, size_match, True, False) + # for idx, line in enumerate(ast_processor.apply_to_string().split('\n')): + # print(f'{idx+1}: {line}') + TextUtils.to_clipboard(ast_processor.apply_to_string()) + + +def batch_recipe_example(): + print("example batch analysis using recipe:\n") + recipe_ast_processor = RecipeASTProcessor(MyRefactor(), simple_codebase_provider, r".*", in_memory=True) + recipe_ast_processor.run() + + +if __name__ == "__main__": + batch_recipe_example() diff --git a/src/rejuvenation/refactor_examples_different_styles.py b/src/rejuvenation/refactor_examples_different_styles.py new file mode 100644 index 00000000..dc788cb8 --- /dev/null +++ b/src/rejuvenation/refactor_examples_different_styles.py @@ -0,0 +1,177 @@ +# This script demonstrates various techniques for refactoring C code using an abstract syntax tree (AST) approach. +# It showcases how to add comments, replace types, and find specific nodes in the AST using different methods. +from renaissance.impl.types import TypeReference +from renaissance.syntax_tree import ( + ASTFactory, + ASTRewriter, + ASTShower, + ASTFinder, +) +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.syntax_tree.ast_finder import find_ast_type, matches_kind +from renaissance.syntax_tree.match_finder import match_pattern, find_all + +example_code = """ + typedef int fancy_new; + typedef int old; + void f(){ + int a = 1; + old b = 2; + int c = 3; + old d = 4; + old e; + } + """ +expected_result_old_fancy_new = """ + typedef int fancy_new; + typedef int old; + void f(){ + int a = 1; + fancy_new b = 2; + int c = 3; + fancy_new d = 4; + fancy_new e; + } + """.strip() + +expected_result_old_with_comment = """ + typedef int fancy_new; + typedef int old; + void f(){ + int a = 1; + // old has become obsolete + old b = 2; + int c = 3; + // old has become obsolete + old d = 4; + // old has become obsolete + old e; + } + """.strip() + + +def example_add_comment_and_commit(factory, pattern_factory): + # create a pattern that matches the declaration of old + # please note that we need to help by telling the old is a type and $value is a variable + pattern1 = pattern_factory.create_declarations( + "old $name = $value;", + extra_declarations=["typedef int old;"], + parameters=["$value"], + ) + pattern2 = pattern_factory.create_declarations("old $name;", extra_declarations=["typedef int old;"], parameters=["$value"]) + # put the patterns in a matrix because we want to find both statements in one go and not a sequence + patterns_list = [pattern1, pattern2] + + ASTShower.show_node(pattern1[0]) + # if you want to find both statements in one go, you should pass a list of patterns + # if you don't do that a sequence of the patterns is searched for + + # create translation unit + atu = factory.create_from_text(example_code, "test.c") + + ASTShower.show_node(atu) + + # create an ASTRewriter + rewriter = ASTRewriter(atu) + + # search matches and replace them + for match in find_all(atu.children, *patterns_list): + rewriter.insert_before("// old has become obsolete", match) + + def commit(): + rewriter.apply_to_string() + atu = factory.create_from_text(rewriter.apply_to_string(), rewriter.get_filename()) + return atu, ASTRewriter(atu) + + # commit + atu, rewriter = commit() + + # look at the print that marks all old declarations with the provided comment + print("results after adding comments to the obsolete types:") + result = rewriter.apply_to_string().strip() + print(result) + return result, expected_result_old_with_comment + + +def example_replace_old_by_fancy_new(factory, pattern_factory): + # using some different techniques to show the possibilities of map and filter + pattern1 = pattern_factory.create_declarations("$old $name = $value;", types=["$old"], parameters=["$value"]) + pattern2 = pattern_factory.create_declarations("$old $name;", types=["$old"], parameters=["$value"]) + # put the patterns in a matrix because we want to find both statements in one go and not a sequence + patterns_list = [pattern1, pattern2] + + # an example of how to use a function iso of lambda to filter the nodes + def matches_old(node): + if "$old" in node and node["$old"][0].name == "old": + return True + return False + + atu = factory.create_from_text(example_code, "test.c") + rewriter = ASTRewriter(atu) + + [rewriter.replace("fancy_new", match.nodes) for match in match_pattern(atu.children, *patterns_list) if matches_old(match.expansions)] + + print("results after replacing the old type by fancy_new using MatchFinder:") + result = rewriter.apply_to_string().strip() + print(result) + return result, expected_result_old_fancy_new + + +def example_use_ast_kind_finder(factory, _): + # Create the translation unit from the provided code or example code + atu = factory.create_from_text(example_code, "test.c") + # Create an ASTRewriter for the translation unit + rewriter = ASTRewriter(atu) + + # Find all nodes of kind TYPE_REF (case-insensitive) and filter those with name 'old' + [rewriter.replace("fancy_new", node) for node in find_ast_type(atu, TypeReference) if node.name == "old"] + + # Print the results after replacing the old type by fancy_new + print("results after replacing the old type by fancy_new using find_ast_type") + result = rewriter.apply_to_string().strip() + print(result) + return result, expected_result_old_fancy_new + + +def example_use_ast_function_finder(factory, _): + # Create the translation unit from the provided code or example code + atu = factory.create_from_text(example_code, "test.c") + # Create an ASTRewriter for the translation unit + rewriter = ASTRewriter(atu) + + ASTShower.show_node(atu) + + # Define a match function to find nodes of kind TYPE_REF with name 'old' + def match(node): + res = matches_kind(node, TypeReference) and node.name == "old" + return res + + # Use ASTFinder to find all matching nodes and replace 'old' with 'fancy_new' + [rewriter.replace("fancy_new", node) for node in ASTFinder.find_all(atu, match)] + + # Print the results after replacing the old type by fancy_new + print("results after replacing the old type by fancy_new using ASTFinder.find_all") + result = rewriter.apply_to_string().strip() + print(result) + return result, expected_result_old_fancy_new + + +def main(args): + # the first argument is the code to be parsed + code = args[1] if len(args) > 1 else "" + + # Create a factory args from the command line are passed to the factory for example -I/usr/include + factory = ASTFactory(ClangASTNode, args if not code else args[1:]) + # Create a pattern factory (using the factory (hence also its args) + pattern_factory = CPatternFactory(factory) + + example_add_comment_and_commit(factory, pattern_factory) + example_replace_old_by_fancy_new(factory, pattern_factory) + example_use_ast_kind_finder(factory, pattern_factory) + example_use_ast_function_finder(factory, pattern_factory) + + +if __name__ == "__main__": + import sys + + main(sys.argv) diff --git a/src/rejuvenation/refactor_with_nested_compositions.py b/src/rejuvenation/refactor_with_nested_compositions.py new file mode 100644 index 00000000..bfec46f7 --- /dev/null +++ b/src/rejuvenation/refactor_with_nested_compositions.py @@ -0,0 +1,144 @@ +# This script demonstrates the use of the syntax_tree library to parse and rewrite C code. +# It specifically showcases nested replacements and multiple patterns. +import textwrap + +from renaissance.impl.types import Call +from renaissance.syntax_tree import ASTFactory, ASTRewriter +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.syntax_tree import ASTShower +from renaissance.syntax_tree.ast_finder import find_ast_type +from renaissance.syntax_tree.match_finder import find_all + +example_code = """ +void f1(int a, int b, int c); +void f2(int a, int c); +void f(){ + const int a = 1; + const int b = 2; + int isAOne = a==1; + int c = 0, d=0; + if (a==1) { + d++; + if(a==1){ + d++; + c=d; + f1(a,b,c); + } + } + if (a==2) { + c++; + f1(a,b,c); + } + f1(a,b,c); +} +""".strip() + +expected_result = """ +void f1(int a, int b, int c); +void f2(int a, int c); +void f(){ + const int a = 1; + const int b = 2; + int isAOne = a==1; + int c = 0, d=0; + //changed if expr to const + if(isAOne){ + d++; + //changed if expr to const + if(isAOne){ + d++; + c=d; + //changed function f1 to f2 + f2(a,c); + } + } + if (a==2) { + c++; + //changed function f1 to f2 + f2(a,c); + } + //changed function f1 to f2 + f2(a,c); +} +""".strip() + + +def refactor_with_nested_compositions(args): + # the first argument is the code to be parsed + code = args[1] if len(args) > 1 else "" + + # Create a factory args from the command line are passed to the factory for example -I/usr/include + factory = ASTFactory(ClangASTNode, args if not code else args[1:]) + # Create a pattern factory (using the factory (hence also its args) + # create translation unit + atu = factory.create(code) if code else factory.create_from_text(example_code, "example.c") + # create a pattern factory atu is passed to the pattern factory for use of all # includes, #defines and declarations + pattern_factory = CPatternFactory(factory, atu) + # create a pattern that matches an if statement with a==1 as the condition and a block of statements as the body + # the type is important so it's declared as const int a + pattern1 = pattern_factory.create_statements("if(a==1){$$stmts;}", extra_declarations=["const int a;"]) + # for pattern 2 we create a fully functional c snippet with a call to f1 + # note that the f1 declaration is derived from the atu + pattern2 = pattern_factory.create("int $a,$b,$c; void fff() {f1($a,$b,$c);}") + ASTShower.show_node(pattern1[0], include_properties=True) + + # we only want to search the call expression as a pattern so it's searched using the kind + pattern2 = find_ast_type(pattern2, Call) + + # the replacement code strip indent is used to be agnostic to the indentation of the replacement + pattern1replacement = textwrap.dedent(""" + //changed if expr to const + if(isAOne){ + $$stmts; + }""") + + pattern2replacement = "\n//changed function f1 to f2\nf2($a,$c);" + + # show node and patterns enable include properties to show the properties of the nodes + include_properties = True + ASTShower.show_node(atu, include_properties) + ASTShower.show_node(pattern1[0], include_properties) + ASTShower.show_node(pattern2[0], include_properties) + + result1 = None + while atu: + # create an ASTRewriter + rewriter = ASTRewriter(atu) + + def raw(nodes): + res = "" + for node in nodes: + res += node.text + return res + "\n" + + # create a refactoring that use different replacement code for different patterns + def refactor(match1): + print(f"peek: f{match1.signature}") + if match1.patterns == pattern1: + replacement_text = pattern1replacement + for repl_snippet in match1.expansions: + replacement_text = replacement_text.replace(repl_snippet, raw(match1.expansions[repl_snippet])) + else: + replacement_text = pattern2replacement + for repl_snippet in match1.expansions: + replacement_text = replacement_text.replace(repl_snippet, match1.expansions[repl_snippet][0].signature) + return rewriter.replace(replacement_text, match1.nodes) + + # search matches for pattern1 and pattern2 and replace them using the refactor function + for match in find_all(atu.children, pattern1, pattern2): + refactor(match) + + # print the rewritten code + result1 = rewriter.apply_to_string() + if rewriter.has_changed(): + atu = factory.create_from_text(result1, "example.c") + else: + atu = None + return result1 + + +if __name__ == "__main__": + import sys + + result = refactor_with_nested_compositions(sys.argv) + print(result) diff --git a/src/rejuvenation/remove_unused_variable.py b/src/rejuvenation/remove_unused_variable.py new file mode 100644 index 00000000..0ee0e039 --- /dev/null +++ b/src/rejuvenation/remove_unused_variable.py @@ -0,0 +1,93 @@ +# This script demonstrates the use of the syntax_tree library to parse and rewrite C code. +# It specifically showcases the replacement of if-else statements with ternary operators. +from more_itertools import flatten + +from renaissance.impl.types import VariableDef, CompoundStatement +from renaissance.refactoring import CleanupRefactoring +from renaissance.syntax_tree import ( + ASTFactory, + ASTRewriter, + ASTShower, + ASTProcessor, + ASTNode, +) +from renaissance.impl.clang import ClangASTNode +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.syntax_tree.ast_finder import find_ast_type + +example_code = """ + int a = 1; + int b = 2; + int c = 3; + int d = 4; + void x(int a) { + } + void f(){ + int unused = 0; + int unused2 = 0; //must be removed + if (a==1) { + int unused = 0; + int unused2 = 0; //should be kept + int c = unused2; + x(c); + } + } + """ +expected_result_refactor = """ + int a = 1; + int b = 2; + int c = 3; + int d = 4; + void x(int a) { + } + void f(){ + if (a==1) { + int unused2 = 0; //should be kept + int c = unused2; + x(c); + } + }""".strip() + + +def remove_unused_variable_using_refactor_method(node_type1: type[ASTNode]): + factory = ASTFactory(node_type1, []) + # create translation unit + atu = factory.create_from_text(example_code, "test.c") + # create a Refactor + refactor = ASTProcessor(atu, factory, in_memory=True) + + CleanupRefactoring.remove_unused_variables(refactor) + result = refactor.apply_to_string().strip() + # print the rewritten code + print(f"Using cleanup refactoring results {node_type1.__name__}:") + print(result) + + return result, expected_result_refactor + + +def remove_unused_variable_low_level(node_type1: type[ASTNode]): + factory = ASTFactory(ClangJsonASTNode, []) + # Create a pattern factory (using the factory (hence also its args) + # create translation unit + atu = factory.create_from_text(example_code, "test.c") + + # create an ASTRewriter + rewriter = ASTRewriter(atu) + + ASTShower.show_node(atu) + # search matches and replace them + funcs = flatten(find_ast_type(func, VariableDef) for func in (find_ast_type(atu, CompoundStatement))) + [rewriter.remove(node.parent, True, True) for node in funcs if len(node.referenced_by) == 0] + + # print the rewritten code + print(f"Low level results using {node_type1.__name__}:") + result = rewriter.apply_to_string().strip() + print(result) + return result, expected_result_refactor + + +if __name__ == "__main__": + + for node_type in [ClangASTNode, ClangJsonASTNode]: + remove_unused_variable_low_level(node_type) + remove_unused_variable_using_refactor_method(node_type) diff --git a/src/rejuvenation/replace_if_with_ternary.py b/src/rejuvenation/replace_if_with_ternary.py new file mode 100644 index 00000000..4835f01b --- /dev/null +++ b/src/rejuvenation/replace_if_with_ternary.py @@ -0,0 +1,73 @@ +# This script demonstrates the use of the syntax_tree library to parse and rewrite C code. +# It specifically showcases the replacement of if-else statements with ternary operators. +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.syntax_tree import ASTFactory, ASTRewriter +from renaissance.syntax_tree.match_finder import find_all + +example_code = """ + int a = 1; + int b = 2; + int c = 3; + int d = 4; + void f(){ + if (a==1) { + c++; + b = 2; + d++; + } + else { + c++; + b = 3; + d++; + } + } + """ + +expected_result = """ + int a = 1; + int b = 2; + int c = 3; + int d = 4; + void f(){ + c++; b=(a==1) ? 2:3; d++; + } + """.strip() + + +def replace_if_with_ternary(): + """ + Replaces if-else statements in the given C code with ternary operator expressions. + This function performs the following steps: + 1. Creates an AST factory with the specified arguments. + 2. Creates a pattern factory using the AST factory. + 3. Defines a pattern for if-else statements. + 4. Creates a translation unit from the provided example code. + 5. Initializes an AST rewriter for the translation unit. + 6. Searches for matches of the if-else pattern in the translation unit. + 7. Replaces matched if-else statements with ternary operator expressions. + 8. Returns the rewritten code as a string. + Returns: + str: The rewritten C code with if-else statements replaced by ternary operators. + """ + + # Create a factory with arguments from the command line, for example, -I/usr/include + factory = ASTFactory(ClangASTNode, []) + # Create a pattern factory (using the factory (hence also its args) + pattern_factory = CPatternFactory(factory) + if_else_patterns = pattern_factory.create_statements("if($exp){$$before;b=$d1;$$after;}else{$$before;b=$d2;$$after;}") + + # Create translation unit + atu = factory.create_from_text(example_code, "test.c") + # Create an ASTRewriter + rewriter = ASTRewriter(atu) + # Search matches and replace them + for match in find_all(atu.children, if_else_patterns): + rewriter.replace("$$before; b=($exp) ? $d1:$d2; $$after;", match) + # Return the rewritten code + return rewriter.apply_to_string().strip() + + +if __name__ == "__main__": + + result = replace_if_with_ternary() + print(result) diff --git a/src/rejuvenation/walk_compilation_database.py b/src/rejuvenation/walk_compilation_database.py new file mode 100644 index 00000000..fff26292 --- /dev/null +++ b/src/rejuvenation/walk_compilation_database.py @@ -0,0 +1,29 @@ +# use clang to load and walk a compilation database + +from pathlib import Path + +import targets +from renaissance.impl.clang import CompilationDatabase, ClangASTNode +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.impl.types import FunctionDef +from renaissance.syntax_tree import ASTProcessor, ASTShower + + +def main(args): + # the first argument is the code to be parsed + database = args[0] if len(args) > 0 else "" + for impl_type in [ClangASTNode, ClangJsonASTNode]: + # load the compilation database by specifying the path to the folder + # and the implementation type + db = CompilationDatabase.walk(impl_type, Path(database)) + for factory, atu in db: + # show atu + ASTShower.show_node(atu, include_properties=True) + # do something with the factory and atu + ast_refactor = ASTProcessor(atu, factory, in_memory=True) + [print(n.text) for n in ast_refactor.find_ast_type(FunctionDef)] + + +if __name__ == "__main__": + # fill in your own path + main([targets.__file__.replace("__init__.py", "compile_commands.json")]) diff --git a/src/renaissance/__init__.py b/src/renaissance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/renaissance/common/__init__.py b/src/renaissance/common/__init__.py new file mode 100644 index 00000000..094c8ea8 --- /dev/null +++ b/src/renaissance/common/__init__.py @@ -0,0 +1,3 @@ +from .rewriter import Rewriter + +__all__ = ["Rewriter"] diff --git a/src/renaissance/common/rewriter.py b/src/renaissance/common/rewriter.py new file mode 100644 index 00000000..270ac5da --- /dev/null +++ b/src/renaissance/common/rewriter.py @@ -0,0 +1,83 @@ +import sys + + +class Rewrite: + def __init__(self, start: int, end: int, replacement: bytes) -> None: + self.start = start + self.end = end + self.replacement = replacement + + +class Rewriter: + """ + A class that allows for modifications to a byte sequence. + """ + + def __init__(self, content: bytes) -> None: + self.__content = content + self.__rewrites: list[Rewrite] = [] + + def replace(self, start: int, end: int, new_content: bytes) -> None: + """ + Replaces a portion of the content with new content. + + This method will replace the content between the specified start and end + indices with the provided new_content. If there is an existing rewrite + that partially overlaps with the specified range, the new content will be + appended to the existing replacement, and the range will be adjusted to + encompass both the old and new content. If the start or end indices are out + of bounds, then new content will be inserted at the end of the byte sequence. + + Args: + start (int): The starting index of the content to be replaced. + end (int): The ending index of the content to be replaced. + new_content (bytes): The new content to insert in place of the old content. + + Returns: + None + """ + for r in self.__rewrites: + # if r partially overlaps with start and end then append the new content to the existing replacement + if r.start <= start <= r.end: + r.replacement += new_content + r.start = min(r.start, start) + r.end = max(r.end, end) + return + real_start = len(self.__content) if start > len(self.__content) or start < 0 else start + real_end = len(self.__content) if end > len(self.__content) or end < 0 else end + self.__rewrites.append(Rewrite(real_start, real_end, new_content)) + + def apply(self) -> bytes: + """ + Applies the rewrites to a copied byte sequence. + + This method reverses the order of the rewrites to ensure that insertions + are performed correctly. It then sorts the rewrites by their start position + in descending order and applies each rewrite to the byte sequence. + + Returns: + bytes: The modified byte sequence after all rewrites have been applied. + """ + result = bytearray(self.__content[:]) + for rewrite in sorted(self.__rewrites, key=lambda x: x.start, reverse=True): + result[rewrite.start : rewrite.end] = rewrite.replacement + return result + + @property + def content(self) -> bytes: + return self.__content + + +if __name__ == "__main__": + # create a byte array a random bytes of len 20 + + my_bytes = bytearray(20) + for i in range(20): + my_bytes[i] = ord("a") + i + rewriter = Rewriter(my_bytes) + rewriter.replace(5, 10, b"hellooo") + rewriter.replace(5, 10, b" world") + rewriter.replace(0, 0, b"BEGIN") + s = rewriter.apply().decode(sys.getfilesystemencoding()) + print(len(s)) + print(s) diff --git a/src/renaissance/impl/__init__.py b/src/renaissance/impl/__init__.py new file mode 100644 index 00000000..b153eacf --- /dev/null +++ b/src/renaissance/impl/__init__.py @@ -0,0 +1,10 @@ +MATCH_ONE = "_MatchOne__" +MATCH_ALL = "_MatchAll__" +# __all__ = [ +# "clang", +# "clang_json", +# "python", +# "tree_sitter", +# "MATCH_ONE", +# "MATCH_ALL", +# ] diff --git a/src/renaissance/impl/clang/__init__.py b/src/renaissance/impl/clang/__init__.py new file mode 100644 index 00000000..c219dad9 --- /dev/null +++ b/src/renaissance/impl/clang/__init__.py @@ -0,0 +1,10 @@ +from .clang_ast_node import ClangASTNode +from .clang_compilation_database import CompilationDatabase +from .c_pattern_factory import CPatternFactory, CPPPatternFactory + +__all__ = [ + "ClangASTNode", + "CPatternFactory", + "CPPPatternFactory", + "CompilationDatabase", +] diff --git a/src/renaissance/impl/clang/c_pattern_factory.py b/src/renaissance/impl/clang/c_pattern_factory.py new file mode 100644 index 00000000..91902a9e --- /dev/null +++ b/src/renaissance/impl/clang/c_pattern_factory.py @@ -0,0 +1,287 @@ +import re +from typing import Optional, Sequence + +from more_itertools import first +from more_itertools.more import last + +from renaissance.impl.types import ( + Declaration, + MacroDef, + CompoundStatement, + ParenthesizedExpression, + Call, + Type, + VariableDef, + TypedefDef, + FunctionDef, + InclusionDirective, +) +from renaissance.syntax_tree.ast_factory import ASTFactory +from renaissance.syntax_tree.ast_finder import find_ast_type +from renaissance.syntax_tree.ast_node import ASTNode +from renaissance.syntax_tree.ast_shower import ASTShower +from renaissance.impl.clang.cpp_utils import CPPUtils + +SHOW_NODE = False + + +def derive_header_text(language: str, ref_node: ASTNode | None): + # collect includes #defines and var decl from the refNode + header = "\n" + if ref_node: + language = ref_node.filename.split(".")[-1] + offset = min( + (n.offset for n in ref_node.children if n.is_part_of_translation_unit() and n.ast_type == InclusionDirective), + default=0, + ) + + header = CPatternFactory.remove_indent(ref_node.content(0, offset)) + header += "\n".join( + n.text + ";" + for n in ref_node.children + if n.is_part_of_translation_unit() + and isinstance(n.ast_type(), (FunctionDef, VariableDef | TypedefDef, MacroDef)) + and len(find_ast_type(n, CompoundStatement)) == 0 + ) + # and isinstance(n.ast_type, (Declaration, MacroDefinition)) + # and len(find_ast_type(n, CompoundStatement)) == 0 + header += "\n" + + return header, language + + +class CPatternFactory: + reserved_function_name = "__rejuvenation__reserved__function__name__" + reserved_variable_name = "__rejuvenation__reserved__variable__name__" + + def __init__( + self, + factory: ASTFactory, + ref_node: Optional[ASTNode] = None, + language: str = "c", + ): + self.factory = factory + self.header, self.language = derive_header_text(language, ref_node) + + @staticmethod + def remove_indent(text: str) -> str: + split = [len(l) - len(l.lstrip()) for l in text.splitlines() if l.strip()] + indent = split[0] if split else 0 + return "\n".join([line[indent:] for line in text.splitlines()]) + + def create_expression(self, text: str, extra_declarations=None) -> ASTNode: + if extra_declarations is None: + extra_declarations = [] + keywords = CPatternFactory._get_keywords_from_text(text) + keywords = [k for k in keywords if not any(k in ed for ed in extra_declarations)] + full_text = ( + self.header + + "\n".join(extra_declarations) + + "\n" + + "\n".join(CPatternFactory._to_declaration(keywords)) + + f"\nvoid {CPatternFactory.reserved_function_name}() {{ int {CPatternFactory.reserved_variable_name} = ({text}); }}" + ) + root = self._create(full_text) + # return the first expression found in the tree as a ASTNode + return last(n.children[0] for n in find_ast_type(root.children[-1], ParenthesizedExpression) if n.is_part_of_translation_unit) + + def create_declarations( + self, + text: str, + types=None, + parameters=None, + extra_declarations=None, + declarations=None, + ): + if declarations is None: + declarations = [] + if extra_declarations is None: + extra_declarations = [] + if parameters is None: + parameters = [] + if types is None: + types = [] + keywords = CPatternFactory._get_keywords_from_text(text) + keywords = [ + k + for k in keywords + if not any(k in ed for ed in extra_declarations) + and not any(k in ed for ed in parameters) + and not any(k in ed for ed in types) + and not any(k in ed for ed in declarations) + ] + return self._create_body(text, types, [*parameters, *keywords], extra_declarations, Declaration) + + def create_declaration( + self, + text: str, + types=None, + parameters=None, + extra_declarations=None, + declarations=None, + ) -> ASTNode: + if declarations is None: + declarations = [] + if extra_declarations is None: + extra_declarations = [] + if parameters is None: + parameters = [] + if types is None: + types = [] + result = self.create_declarations(text, types, parameters, extra_declarations, declarations) + assert len(result) > 0, "At least one declaration is expected" + return result[0] + + def create_statements( + self, + text: str, + types=None, + extra_declarations=None, + kind: type[Type] = Type, + ) -> Sequence[ASTNode]: + # create a reference for all used variables excluding the specified types + if extra_declarations is None: + extra_declarations = [] + if types is None: + types = [] + parameters = [ + par + for par in CPatternFactory._get_keywords_from_text(text) + if par not in types and not any(par in ed for ed in extra_declarations) + ] + return self._create_body(text, types, parameters, extra_declarations, kind) + + def create(self, text: str, kind: type[Type] = None) -> ASTNode: + """ + Creates an object using the factory from the provided text. + The object is created by the factory using the provided text and the header of the provided reference node. + It is up to the user to pick the right node for pattern matching + + Args: + text (str): The input text used to create the object. + kind (str, optional): The kind of the node to be returned. Defaults to None. + + Returns: + object: The object created by the factory. + """ + # print(self.header + text) + root = self.factory.create_from_text(self.header + text, "test." + self.language) + if kind: + return first(find_ast_type(root.children[-1], kind)) + return root + + def create_statement( + self, + text: str, + types=None, + extra_declarations=None, + kind: str = Type, + ) -> ASTNode: + if extra_declarations is None: + extra_declarations = [] + if types is None: + types = [] + statements = list(self.create_statements(text, types, extra_declarations, kind)) + assert len(statements) == 1, "Only one statement is expected" + return statements[0] + + def _create_body( + self, + text: str, + types: Sequence[str], + parameters: Sequence[str], + extra_declarations: Sequence[str], + kind: type[Type], + ) -> list[ASTNode]: + full_text = ( + self.header + "\n".join(CPatternFactory._to_typedef(types)) + "\n" + "\n".join(CPatternFactory._to_declaration(parameters)) + "\n" + "\n".join(extra_declarations) + "\n" + "\nvoid " + CPatternFactory.reserved_function_name + "(){\n" + text + "\n}" + ) + root = self._create(full_text) + + # from the children of the compound statement that contains the text, get for each child the first + # node of the specified kind + + body = first(find_ast_type(root.children[-1], CompoundStatement)).children + return list(n for n in body if n.is_part_of_translation_unit and first(find_ast_type(n, kind))) + + def _create(self, text: str) -> ASTNode: + atu = self.factory.create_from_text(text, "test." + self.language) + if SHOW_NODE: + ASTShower.show_node(atu) + return atu + + @staticmethod + def _get_keywords_from_text(text: str) -> Sequence[str]: + # regex to get keywords that start with one of two dollars followed by a \\w+ + pattern = re.compile(r"\${0,2}[a-zA-Z]\w*") + return list(k for k in set(re.findall(pattern, text)) if k not in CPPUtils.RESERVED_KEYWORDS) + + @staticmethod + def _get_dollar_keywords_from_text(text: str) -> Sequence[str]: + # regex to get keywords that start with one of two dollars followed by a \\w+ + pattern = re.compile(r"\${1,2}[a-zA-Z]\w*") + return list(set(re.findall(pattern, text))) + + @staticmethod + def _get_non_dollar_keywords_from_text(text: str) -> Sequence[str]: + pattern = re.compile(r"[^$][a-zA-Z]\w*") + return list(set(re.findall(pattern, text))) + + @staticmethod + def _to_declaration(keywords: Sequence[str], prefix: str = "int ", postfix: str = ";") -> Sequence[str]: + return [prefix + keyword + postfix for keyword in keywords] + + @staticmethod + def _to_typedef(keywords: Sequence[str], prefix: str = "typedef int ", postfix: str = ";") -> Sequence[str]: + return [prefix + keyword + postfix for keyword in keywords] + + +class CPPPatternFactory(CPatternFactory): + + def __init__(self, factory: ASTFactory, ref_node: Optional[ASTNode] = None): + super().__init__(factory, ref_node, "cpp") + + def create_constructor_call(self, pattern: str): + class_and_args = re.match(R"([$\w]+)\(([^)]+)\)", pattern.replace(" ", "")) + if class_and_args: + class_name = class_and_args.group(1) + args = class_and_args.group(2).split(",") + return self._create_constructor_call(class_name, args) + return None + + def _create_constructor_call(self, class_name: str, args=None): + if args is None: + args = [] + arg_call_string = ",".join(args) + arg_decl_string = ",".join("int " + arg for arg in args) + code = f""" + class {class_name}{{ + public: + {class_name}({arg_decl_string}) {{}} + }}; + class derived : public {class_name}{{ + public: + derived({arg_decl_string}) : {class_name}({arg_call_string}) {{ }} + }}; + """ + root: ASTNode = self.factory.create_from_text(code, "test." + self.language) + target_class = root.children[-1] + # this should yield something like: + # (TYPE_REF, $var, test.cpp[237:241]): |$var| + # (CALL_EXPR, , test.cpp[237:266]): |$var($container,$headerCount)| + # (DECL_REF_EXPR, $container, test.cpp[242:252]): |$container| + # (DECL_REF_EXPR, $headerCount, test.cpp[253:265]): |$headerCount| + if SHOW_NODE: + ASTShower.show_node(target_class) + # search the call expr and the preceding type ref + call_expr = last(find_ast_type(target_class, Call)) + # include the preceding type ref + assert isinstance(call_expr, ASTNode), "No call expression found" + type_ref = call_expr.preceding_sibling + assert isinstance(type_ref, ASTNode), "No type ref found" + # return the constrained pattern where the first node must be of type TypeRef + + return call_expr diff --git a/src/renaissance/impl/clang/clang_adapter.py b/src/renaissance/impl/clang/clang_adapter.py new file mode 100644 index 00000000..71886bdc --- /dev/null +++ b/src/renaissance/impl/clang/clang_adapter.py @@ -0,0 +1,63 @@ +from clang import cindex +from renaissance.impl.tree_sitter.lst import LSTNode, LST +from typing import Optional +from renaissance.utils.ast_utils import detect_placeholder + + +class ClangAdapter: + def __init__(self, clang_path: Optional[str] = None, args: Optional[list] = None): + if clang_path and cindex.Config.library_path is None: + cindex.Config.set_library_path(clang_path) + self.args = args or ["-std=c++17"] + + def parse(self, file_path: str) -> LST: + index = cindex.Index.create() + translation_unit = index.parse(file_path, args=self.args) + return LST(self._convert_node(translation_unit.cursor)) + + def load_from_text(self, text: str, file_name: str): + index = cindex.Index.create() + translation_unit = index.parse(file_name, unsaved_files=[(file_name, text)], args=[]) + return LST(self._convert_node(translation_unit.cursor)) + + def to_lst(self, source_code: str) -> LST: + # source_code= replace_dollar(source_code) + return self.load_from_text(source_code, "no_src.cpp") + + def _convert_node(self, cursor: cindex.Cursor, parent: Optional[LSTNode] = None) -> LSTNode: + try: + kind = cursor.kind.name + except Exception as e: + print(e.__cause__) + kind = "invalid kind" + signature = cursor.spelling or cursor.displayname or kind + + is_ph, coerced_type, ph_name = detect_placeholder(signature, kind) + + node = LSTNode( + node_type=coerced_type if is_ph else kind, + properties={ + "spelling": cursor.spelling, + "type": str(cursor.type.spelling), + "location": str(cursor.location), + "is_definition": cursor.is_definition(), + "name": ph_name, + **( + { + "placeholder": True, + "placeholder_name": ph_name, + "original_node_type": cursor.kind.name, + } + if is_ph + else {} + ), + }, + signature=signature, + offset=cursor.extent.start.offset, + parent=parent, + ) + + for child in cursor.get_children(): + child_node = self._convert_node(child, parent=node) + node.add_child(child_node) + return node diff --git a/src/renaissance/impl/clang/clang_ast_node.py b/src/renaissance/impl/clang/clang_ast_node.py new file mode 100644 index 00000000..1592520e --- /dev/null +++ b/src/renaissance/impl/clang/clang_ast_node.py @@ -0,0 +1,522 @@ +from renaissance.syntax_tree import ASTFinder +import sys +from functools import cache +from pathlib import Path +from typing import Any, Optional, Sequence, override + +import clang.native +from clang.cindex import Config, Index, TypeKind, CursorKind + +from renaissance.impl.clang.cpp_utils import matches_kind +from renaissance.impl.types import ( + MatchAll, + MatchOne, + UnknownType, + KIND_MAP, + MacroDef, + Statement, + DeclarationExpression, + Literal, + BinaryOperation, + UnaryOperation, + CompoundStatement, + Declaration, + Definition, + TranslationUnit, +) +from renaissance.syntax_tree import ASTNode, ASTReference +from renaissance.utils.ast_utils import match_children, match_props + +EMPTY_DICT = {} +EMPTY_STR = "" +EMPTY_LIST = [] + +STMT_PARENTS = [CompoundStatement, TranslationUnit] +IRRELEVANT_PROPS = {"comment"} +IRRELEVANT_NODES = {"comment"} +PRINT_ALL_NODES = False + + +class Clangastreference: + def __init__(self, node_id: str, ref_kind: str, properties: dict[str, Any]) -> None: + self.node_id = node_id + self.ref_kind = ref_kind + self.properties = properties + + +class ClangTranslationUnit: + cache = [] + + def __init__(self, clang_atu: clang.cindex.TranslationUnit, file_name: str): + self.clang_atu = clang_atu + self.file_name = file_name + self.references_initialized = False + # print_node_kind(clang_atu.cursor) + self.macro_expansions = ClangTranslationUnit._collect_expansions(clang_atu) + # references are used as a cache to store the references of a node + # they are stored as id for lazy creation + self._references: dict[str, list[Clangastreference]] = {} + self._referenced_by: dict[str, list[Clangastreference]] = {} + self._nodes: dict[str, "ClangASTNode"] = {} + + def lazy_create_references(self, node: "ClangASTNode") -> None: + if self.references_initialized: + return + node.root.process(ReferenceHelper.create_references) + self.references_initialized = True + + @staticmethod + def _collect_expansions( + translation_unit: clang.cindex.TranslationUnit, + ) -> set[tuple[str, int, int]]: + result: set[tuple[str, int, int]] = set() + for child in translation_unit.cursor.get_children(): + if child.kind.name == "MACRO_INSTANTIATION": + result.add( + ( + child.extent.start.file, + child.extent.start.offset, + child.extent.end.offset, + ) + ) + return result + + +class ClangASTNode(ASTNode): + @staticmethod + def set_library_path() -> None: + try: + Config.set_library_path(Path(clang.native.__file__).parent) + except Exception as e: + print(e) + + set_library_path() + index = Index.create() + parse_args = [ + "-fparse-all-comments", + "-ferror-limit=0", + "-Xclang", + "-detailed-preprocessing-record", + "-fsyntax-only", + ] + + def __init__( + self, + node, + translation_unit: ClangTranslationUnit, + parent=None, + start_offset: Optional[int] = None, + length: Optional[int] = None, + insert_kind: Optional[str] = None, + ): + super().__init__(self if parent is None else parent.root) + self.node = node + self._children = None + self._parent = parent + self.translation_unit = translation_unit + self.inserted = insert_kind is not None + self.show_props = False + self._filename = self._get_containing_filename() + self._name = self._derive_name() + # if the node has not been added to the translation unit, add it + # a node might already be added if it is split into multiple nodes + # an example is for base types like int, char, etc. which are split into multiple nodes + if self.node.hash not in self.translation_unit._nodes: + self.translation_unit._nodes[node.hash] = self + self._offset = start_offset if start_offset is not None else self.__derive_start_offset() + self._length = length if length is not None else self.__derive_length() + self._kind = insert_kind if insert_kind is not None else self.__derive_kind() + self.ast_type = KIND_MAP.get(self._kind, UnknownType) + self.indent = "" + # TODO: TextUtils.get_indent(self.content, self._offset) + # an fake child is introduced to handle the case where the type of a declaration is not found + # for example in the case of a base type. + # without the fake child pattern matching on types will be difficult + self.__inserted_children = [] + if insert_kind is None and not self.node.location.is_in_system_header and self.node.kind.is_declaration() and self.node.type.kind != TypeKind.INVALID: # type: ignore + loc_offset: int = self.node.location.offset + length = len(self.node.spelling.encode(sys.getdefaultencoding())) + insert_child = ClangASTNode(self.node, self.translation_unit, self, loc_offset, length, "DECL_LOC") + insert_child._children = [] + self.__inserted_children.append(insert_child) + if self.node.type.get_declaration().kind is CursorKind.NO_DECL_FOUND: # type: ignore + my_type = self.node.type if self.node.result_type.kind == TypeKind.INVALID else self.node.result_type # type: ignore + length_ref = len(my_type.spelling.encode(sys.getdefaultencoding())) + insert_child = ClangASTNode( + self.node, + self.translation_unit, + self, + self._offset, + length_ref, + CursorKind.TYPE_REF.name, + ) # type: ignore + insert_child._children = [] + self.__inserted_children.append(insert_child) + + self._children = [] + for n in self.__inserted_children: + self._children.append(n) + for n in self.node.get_children(): + if not is_system_macro(n) and n.kind.name != "MACRO_INSTANTIATION": + self._children.append(ClangASTNode(ClangASTNode.remove_wrapper(n), self.translation_unit, self)) + + self._properties = self._derive_properties() + if self.ast_type == DeclarationExpression: + self._properties["name"] = self._name + + def __eq__(self, other): + return ( + other + and isinstance(other, type(self)) + and self.ast_type == other.ast_type + and match_props(self.properties, other.properties, IRRELEVANT_PROPS) + and match_children(self.children, other.children, IRRELEVANT_NODES) + ) + + def __hash__(self): + return hash((self.ast_type, frozenset(self.properties.items()))) + + @override + @staticmethod + def load(file_path: Path, extra_args: Sequence[str], working_dir: Path) -> "ClangASTNode": + args = [*extra_args, *ClangASTNode.parse_args] + translation_unit: clang.cindex.TranslationUnit = ClangASTNode.index.parse(working_dir / file_path, args=args[3:]) + ClangASTNode.check_diagnostics(translation_unit, file_path.name) + root_node = ClangASTNode( + translation_unit.cursor, + ClangTranslationUnit(translation_unit, file_name=str(file_path)), + None, + ) + return root_node + + @override + @staticmethod + def load_from_text( + text: str, + file_name: str, + extra_args: Sequence[str] = None, + working_dir: Path = None, + ) -> "ClangASTNode": + # Convert file_content to bytes + file_content_bytes = text.encode(sys.getfilesystemencoding()) + # add to cache to avoid reading the file again + ASTNode.cache[file_name] = file_content_bytes + args = [*ClangASTNode.parse_args, *extra_args] if extra_args is not None else [*ClangASTNode.parse_args] + translation_unit: clang.cindex.TranslationUnit = ClangASTNode.index.parse(file_name, unsaved_files=[(file_name, text)], args=args) + ClangASTNode.check_diagnostics(translation_unit, file_name) + try: + root_node = ClangASTNode( + translation_unit.cursor, + ClangTranslationUnit(translation_unit, file_name=str(file_name)), + None, + ) + except Exception as e: + print(e) + raise e + ClangASTNode.check_diagnostics(translation_unit, file_name) + return root_node + + @staticmethod + def check_diagnostics(translation_unit: clang.cindex.TranslationUnit, file_name: str) -> None: + has_error = False + errors = "" + for d in translation_unit.diagnostics: + if d.severity >= 3: + has_error = True + errors += f"{d.severity}: {d.spelling} at {d.location}\n" + print(f"{d.severity}: {d.spelling} at {d.location}") + if has_error: + raise Exception(f"Error parsing: {file_name} \n+ errors: {errors}") + + def _derive_name(self) -> str: + try: + if self.node.type.kind == TypeKind.RECORD: # type: ignore + return self.node.type.spelling + except Exception as e: + print(e) + try: + return self.node.spelling + except Exception as e: + print(e) + return EMPTY_STR + + def _get_containing_filename(self) -> str: + if self is self.root: + return self.translation_unit.clang_atu.spelling + try: + return self.node.location.file.name + except: + return EMPTY_STR + + @override + @property + def extended_end_offset(self) -> int: + try: + end_offset = self._offset + self._length + if ( + (not self._is_statement_or_declaration()) + and (self.parent and self.parent.ast_type in STMT_PARENTS) + and self.ast_type not in [MacroDef] + ): + content = self.root.binary_file_content() + while end_offset < len(content) and content[end_offset - 1] not in b";": + end_offset += 1 + return end_offset + except: + return 0 + + def _is_statement_or_declaration(self): + print(f"{self.ast_type} is statement: {self.kind}") + return isinstance(self.ast_type(), (Statement, Declaration, Definition)) + + @override + def matches_kind(self, node: ASTNode) -> bool: + return matches_kind(self.ast_type, node.ast_type) + + def _derive_properties(self) -> dict[str, int | str]: + result = {} + offsets = (self.filename, self.offset, self.end_offset) + if offsets in self.translation_unit.macro_expansions: + result["macro_expansion"] = self.text + + if self.ast_type == BinaryOperation: + # TODO remove below code after clang release that supports the getOpCode() statement + children = self.children + start_offset = children[0].offset + children[0].length + end_offset = children[1].offset + operator = self.content(start_offset, end_offset) + result["operator"] = operator.strip() + # next statement works in C++ but not in Python (yet) will be released later + # result['operator'] = self.node.getOpCode() + elif self.ast_type == UnaryOperation: + # TODO remove below code after clang release that supports the getOpCode() statement + child = self.children[0] + # list all attributes of self.node excluding the once starting with _ + + if child.offset > self.offset: + start_offset = self.offset + end_offset = child.offset + prefix_operator = True + else: + start_offset = child.offset + child.length + end_offset = self.offset + self.length + prefix_operator = False + + operator = self.content(start_offset, end_offset) + result["operator"] = operator.strip() + result["prefixOperator"] = prefix_operator + # next statement works in C++ but not in Python (yet) will be released later + # result['operator'] = self.node.getOpCode() + elif isinstance(self.ast_type(), Literal): + self._add_tokens(result, "LITERAL") + elif self.ast_type == DeclarationExpression: + self._add_tokens(result, "LITERAL") + + is_all = { + attr[len("is_") :]: True + for attr in dir(self.node) + if attr.startswith("is_") and callable(getattr(self.node, attr) and getattr(self.node, attr)() == True) + } + result.update(is_all) + return result + + @override + @property + def is_statement(self) -> bool: + "pretty good definition" + return self.parent is not None and self.parent.ast_type in STMT_PARENTS + + @override + @property + def referenced_by(self) -> Sequence[ASTReference]: + self.translation_unit.lazy_create_references(self) + node_id = self.node.hash + ref_by = self.translation_unit._referenced_by.get(node_id, EMPTY_LIST) + # if both the function declaration and function definition are available + # the references are stored in the function definition, + # but we want them to also show up in the declaration + if len(ref_by) == 0: + definition = self._get_function_definition() + if definition: + ref_by = self.translation_unit._referenced_by.get(definition.node.hash, EMPTY_LIST) + return list( + ASTReference( + self.translation_unit._nodes[ref.node_id], + ref.ref_kind, + ref.properties, + ) + for ref in ref_by + ) + + def _get_function_definition(self): + if self.node.type.kind == TypeKind.FUNCTIONPROTO: # type: ignore + signature = self.node.displayname + semantic_parent = self.node.semantic_parent.hash + + def has_body(node): + return any(c.kind == CursorKind.COMPOUND_STMT for c in node.node.get_children()) # type: ignore + + def is_match(node): + if node._kind != self._kind: + return False + if node.node.type.kind != TypeKind.FUNCTIONPROTO: + return False # type: ignore + if node.node.semantic_parent.hash != semantic_parent: + return False + if node.node.displayname != signature: + return False + return has_body(node) + + if has_body(self): + return None + body = ASTFinder.find_all(self.root, is_match).find_first().or_else(None) # type: ignore + if isinstance(body, ClangASTNode): + return body + return None + + @override + @property + def references(self) -> Sequence[ASTReference]: + self.translation_unit.lazy_create_references(self) + return list( + ASTReference( + self.translation_unit._nodes[ref.node_id], + ref.ref_kind, + ref.properties, + ) + for ref in self.translation_unit._references.get(self.node.hash, EMPTY_LIST) + ) + + def _add_tokens(self, result: dict[str, str], *token_kind): + for token in self.node.get_tokens(): + # find all attr of token that are of type str or int + kind = str(token.kind).split(".")[-1] + if kind in token_kind: + result[kind] = token.spelling + + def __derive_start_offset(self) -> int: + try: + if self.node.kind.name == "MACRO_DEFINITION": + return self.node.extent.start.offset - 8 + + return self.node.extent.start.offset + + except: + return 0 + + def __derive_length(self) -> int: + try: + if self.node.kind.name in ["VAR_DECL", "STRUCT_DECL"]: + end_offset = self.node.extent.end.offset + 1 + elif self.node.kind.name in ["MACRO_DEFINITION"]: + end_offset = self.node.extent.end.offset + else: + end_offset = self.node.extent.end.offset + return end_offset - self.__derive_start_offset() + except: + return 0 + + def __derive_kind(self) -> str: + try: + if self.node.kind.name == "MACRO_DEFINITION": + return str(self.node.kind.name) + elif self.node.kind.name in ["UNEXPOSED_EXPR", "VAR_DECL", "DECL_REF_EXPR"]: + if self.node.displayname.startswith("$$") and " " not in self.node.displayname: + return MatchAll.__name__ + elif self.node.displayname.startswith("$") and " " not in self.node.displayname: + return MatchOne.__name__ + return str(self.node.kind.name) + except Exception: + return EMPTY_STR + + @staticmethod + def remove_wrapper(cursor): + try: + if ClangASTNode._is_wrapped(cursor): + return ClangASTNode.remove_wrapper(list(cursor.children)[0]) + except: + pass + return cursor + + @staticmethod + def _is_reference(node): + # refactor this + try: + print(type(node)) + print(vars(node)) + print(dir(node)) + print(node.__dict__) + node.__dict__["id"] + return True + except: + return False + + @staticmethod + @cache + def __is_property(key, value): + return callable(value) and any(key.startswith(tag) for tag in ["is_", "get"]) + + @staticmethod + def _is_wrapped(cursor): + return cursor.kind.is_unexposed() and len(list(cursor.children)) == 1 + + @property + def is_implicit(self): + return self.is_part_of_translation_unit() + + +# def get_ancestor(self, types ): +# return get_ancestor(self, types) + + +SYSTEM_MACROS = { + "linux", + "unix", + "_LP64", + "_WIN32", + "_WIN64", + "_ISO_VOLATILE", + "_INTEGRAL_MAX_BITS", +} + + +def is_system_macro(n): + return n.kind.name == "MACRO_DEFINITION" and ( + n.displayname.startswith("__") + or n.displayname.startswith("_MS") + or n.displayname.startswith("_M_") + or n.displayname in SYSTEM_MACROS + ) + + +class ReferenceHelper: + @staticmethod + def create_references(ast_node: ClangASTNode) -> None: + assert isinstance(ast_node, ClangASTNode), f"Expected ClangASTNode but got {type(ast_node)}" + references = [] + node_id: str = ast_node.node.hash + ast_node.translation_unit._references[node_id] = references + ref_fields = ["referenced"] # , 'type.get_declaration()'] + for field in ref_fields: + try: + element = eval("ast_node.node." + field) + if element.kind.name == "NO_DECL_FOUND": + continue + ref_id = element.hash + ref_kind = field.split(".")[0] + properties = {k: p for k, p in element.__dict__.items() if not k.startswith("_") and k != "hash"} + if node_id == ref_id: + return + reference = Clangastreference(ref_id, ref_kind, properties) + referenced_by = Clangastreference( + node_id, + ref_kind, + {k: p for k, p in ast_node.node.__dict__.items() if k != "hash"}, + ) + try: + ast_node.translation_unit._referenced_by[ref_id].append(referenced_by) + except: + ast_node.translation_unit._referenced_by[ref_id] = [referenced_by] + references.append(reference) + except: + pass diff --git a/src/renaissance/impl/clang/clang_compilation_database.py b/src/renaissance/impl/clang/clang_compilation_database.py new file mode 100644 index 00000000..ca6b6920 --- /dev/null +++ b/src/renaissance/impl/clang/clang_compilation_database.py @@ -0,0 +1,43 @@ +from pathlib import Path +from typing import Iterator +from clang.cindex import CompilationDatabase as ClangCompilationDatabase + +from renaissance.syntax_tree import ASTNode, ASTFactory + + +class CompilationDatabase: + + @staticmethod + def walk(typ: type[ASTNode], path: Path) -> Iterator[tuple[ASTFactory, ASTNode]]: + """ + Load the Clang compilation database and yield factory and AST node type tuples. + + Args: + typ (type[ASTNode]): The type of AST node to be used. + path (Path): The path to the directory containing the compilation database. + + Yields: + Iterator[tuple[ASTFactory, ASTNode]]: An iterator of tuples, each containing + an AST factory and an AST node type. + + Be careful to not use the Iterable is a list as it will load ALL the AST nodes in memory. + """ + db = ClangCompilationDatabase.fromDirectory(str(path)) + + def factory_and_atu(command): + return CompilationDatabase.__create_processor(typ, command) + + yield from map(factory_and_atu, db.getAllCompileCommands()) + + @staticmethod + def __create_processor(typ: type[ASTNode], compile_command) -> tuple[ASTFactory, ASTNode]: + extra_args = list(compile_command.arguments) + skip = ["-o", "-c"] + filtered_args = [ + arg + for idx, arg in enumerate(extra_args) + if arg != compile_command.filename and arg not in skip and (idx == 0 or extra_args[idx - 1] not in skip) + ] + factory = ASTFactory(typ, extra_args=filtered_args, working_dir=Path(compile_command.directory)) + atu = factory.create(Path(compile_command.filename)) # The first argument is the file path + return factory, atu diff --git a/src/renaissance/impl/clang/clang_json_ast_node.py b/src/renaissance/impl/clang/clang_json_ast_node.py new file mode 100644 index 00000000..d7b66fff --- /dev/null +++ b/src/renaissance/impl/clang/clang_json_ast_node.py @@ -0,0 +1,608 @@ +# create a class that inherits syntax tree ASTNode + +import json +import os +import re +import subprocess +import sys +import tempfile +from functools import cache +from pathlib import Path +from typing import Any, Optional, Sequence, Self + +from typing_extensions import override + +from renaissance.impl.clang.cpp_utils import CPPUtils, matches_kind +from renaissance.impl.types import * +from renaissance.utils.ast_utils import match_children, match_props +from renaissance.syntax_tree import ASTNode, ASTReference + +EMPTY_DICT = {} +EMPTY_STR = "" +EMPTY_LIST: list["ClangJsonASTReference"] = [] +ON_NODE_ID_TAGS = ["previousDecl", "parentDeclContextId"] +ID_TAGS = [ + "id", + "typeAliasDeclId", + "templateDeclId", + "templateSpecializationDeclId", + "referencedDeclId", + *ON_NODE_ID_TAGS, +] + +STMT_PARENTS = [CompoundStatement, TranslationUnit] +IRRELEVANT_PROPS = {"macro_expansion", "start_point", "end_point", "source_code", "location", "type"} +IRRELEVANT_NODES = {Comment, MacroDef, FullComment} +VERBOSE = False + + +class ClangJsonASTReference: + def __init__(self, node_id: str, ref_kind: str, properties: dict[str, Any]) -> None: + self.node_id = node_id + self.ref_kind = ref_kind + self.properties = properties + + +class ClangJsonTranslationUnit: + def __init__(self, json_root: dict[str, Any], file_name: str): + self.json_root = json_root + self.filename = file_name + self.references_initialized = False + # references are used as a cache to store the references of a node + # they are stored as id for lazy creation + self._references: dict[str, list[ClangJsonASTReference]] = {} + self._referenced_by: dict[str, list[ClangJsonASTReference]] = {} + self._nodes: dict[str, ClangJsonASTNode] = {} + + def lazy_create_references(self, node: "ClangJsonASTNode") -> None: + # TODO: Do I correctly assume that the usage of this function must be synchronized? + if self.references_initialized: + return + node.root.process(ReferenceHelper.create_references) + node.root.process(ReferenceHelper.add_record_references) + self.references_initialized = True + + +class ClangJsonASTNode(ASTNode): + parse_args = [ + "-fparse-all-comments", + "-ferror-limit=0", + "-Xclang", + "-ast-dump=json", + "-fsyntax-only", + ] + + def __init__( + self, + node: dict[str, Any], + translation_unit: ClangJsonTranslationUnit, + parent: Optional[Self] = None, + start_offset: Optional[int] = None, + length: Optional[int] = None, + insert_kind: Optional[str] = None, + insert_name: Optional[str] = None, + ) -> None: + super().__init__(self if parent is None else parent.root) + self.node: dict[str, Any] = node + self._children: Optional[Sequence[ClangJsonASTNode]] = None + self._parent = parent + self.translation_unit = translation_unit + self._filename = translation_unit.filename + self.inserted = insert_kind is not None + self.show_props = False + # if the node has not been added to the translation unit, add it + # a node might already be added if it is split into multiple nodes + # an example is for base types like int, char, etc. which are split into multiple nodes + if "id" in node and self.translation_unit._nodes.get(node["id"]) == None: + self.translation_unit._nodes[node["id"]] = self + self._offset = start_offset if start_offset is not None else self.__derive_start_offset() + self._end_offset = self._offset + length if length != None else self.__derive_end_offset() + self._length = self._end_offset - self._offset + self._kind = insert_kind if insert_kind is not None else self.__derive_kind() + self.ast_type = KIND_MAP.get(self._kind, UnknownType) + self._name = insert_name if insert_name is not None else self._derive_name() + # a fake child is introduced to handle the case where the type of declaration is not found + # for example in the case of a base type. + # without the fake child pattern matching on types will be difficult + self.__inserted_children: list[ClangJsonASTNode] = [] + type = self.node.get("type") + if insert_kind == None and type and not self.node.get("implicit") and re.fullmatch("(Var|Function|CxxMethod)Decl", self._kind): + declared_type = type["qualType"].replace("(", "").replace(")", "").strip() + if self.node.get("loc"): + loc = self.node["loc"] + offset = loc["offset"] if loc.get("offset") else self._get(["loc", "expansionLoc", "offset"], 0) + tok_len = loc["tokLen"] if loc.get("tokLen") else self._get(["loc", "expansionLoc", "tokLen"], 0) + if tok_len != 0: + insert_child = ClangJsonASTNode( + self.node, + self.translation_unit, + self, + offset, + tok_len, + "DeclLoc", + ) + insert_child._children = [] + self.__inserted_children.append(insert_child) + if "TypeRef" not in [inner["kind"] for inner in self.node.get("inner", [])]: + # deep clone the type node and remove the parentheses + base_type = type.get("desugaredQualType", declared_type).replace("(", "").replace(")", "").strip() + if base_type in CPPUtils.RESERVED_KEYWORDS: + length_ref = len(declared_type.encode(sys.getdefaultencoding())) + insert_child = ClangJsonASTNode( + self.node, + self.translation_unit, + self, + self._offset, + length_ref, + "TypeRef", + declared_type, + ) + insert_child._children = [] + self.__inserted_children.append(insert_child) + # add the declaration as node + # deep clone the type node and remove the parentheses + elif self.ast_type in [DeclarationExpression]: + if self.name.startswith("$$"): + self._kind = MatchAll.__name__ + self.ast_type = MatchAll + elif self.name.startswith("$"): + self._kind = MatchOne.__name__ + self.ast_type = MatchOne + self._children = self.__inserted_children + [ + ClangJsonASTNode( + ClangJsonASTNode._remove_wrapper(n), + translation_unit=self.translation_unit, + parent=self, + ) + for n in self.node.get("inner", []) + if not n.get("isImplicit", False) + ] + self._children = [n for n in self._children if n.ast_type not in IRRELEVANT_NODES] + + def __eq__(self, other): + return ( + isinstance(other, type(self)) + and self.kind == other.kind + and match_props(self.properties, other.properties, IRRELEVANT_PROPS) + and match_children(self.children, other.children, IRRELEVANT_NODES) + ) + + @override + @staticmethod + def load( + file_path: Path, + extra_args: Sequence[str], + working_dir: Path, + code: Optional[str] = None, + ) -> Self: + # in a shell process compile the file_path with clang compiler + try: + # remove the compiler name if it is the first argument + if len(extra_args) > 0 and re.match(r".*(g\+\+|gcc|cl\.exe).*", extra_args[0]): + extra_args = extra_args[1:] + # add clang compiler if it is not in the arguments + if len(extra_args) == 0 or "clang" not in extra_args[0]: + clang = "clang++" if file_path.suffix == ".cpp" else "clang" + extra_args = [clang, *extra_args] + + command = [*extra_args, *ClangJsonASTNode.parse_args] + if code: + if str(file_path) in command: + command.remove(str(file_path)) + compile = "-xc++" if file_path.suffix == ".cpp" else "-xc" + if compile not in command: + command.append(compile) + if "-" not in command: + command.append("-") + # command.append('-main-file-name=' + str(file_path)) + input = code + result = subprocess.run( + command, + input=input, + capture_output=True, + text=True, + cwd=working_dir, + ) + length = len(input) + else: + if str(file_path) not in command: + command.append(str(file_path)) + result = subprocess.run( + command, + capture_output=True, + text=True, + cwd=working_dir, + ) + length = os.path.getsize(working_dir / file_path) + json_dump = result.stdout.replace("<stdin>", str(file_path)) + error = result.stderr + + if VERBOSE: + temp_dir = tempfile.gettempdir() + temp_file_name = os.path.join(temp_dir, file_path.name + ".ast.json") + with open(temp_file_name, "w") as std_out_file: + print("result stored in " + temp_file_name) + std_out_file.write(json_dump) + print(error, file=sys.stderr) + json_atu = json.loads(json_dump) + atu = ClangJsonASTNode( + json_atu, + translation_unit=ClangJsonTranslationUnit(json_atu, file_name=str(file_path)), + length=length, + ) + if code: + atu.cache[str(file_path)] = code.encode(sys.getfilesystemencoding()) + else: + with open(working_dir / file_path, "rb") as f: + atu.cache[str(file_path)] = f.read() + # cache the result of the temp file before deleting it + atu.content(0, 0) + return atu + + except Exception as e: + print("Call to clang failed. Did you install clang?, is it on the env path?") + raise e + + @override + @staticmethod + def load_from_text(text: str, file_name: str, extra_args: Sequence[str], working_dir: Path) -> Self: + return ClangJsonASTNode.load(Path(file_name), extra_args, working_dir, code=text) + + @cache + def _get_containing_filename(self) -> str: + if self.node.get("isImplicit", False): + return "" + if self.node.get("implicit", False): + return "" + if not self.parent: + return self.translation_unit.file_name + # return the file name of the node if it exists else return the file name of the parent node + containing_file = self._get(["loc", "file"], EMPTY_STR) + if containing_file: + return containing_file + included_file = self._get(["loc", "includedFrom", "file"], "") + if included_file: # included but no file location is provided in the node so we don't know the file name + return "" + included_file = self._get(["loc", "spellingLoc", "includedFrom", "file"], "") + if included_file: # included but no file location is provided in the node so we don't know the file name + return "" + # not included and no file location so it is the same as the parent + if self.parent: + return self.parent.filename + return EMPTY_STR + + @override + @property + def extended_end_offset(self) -> int: + try: + # TODO: Do I correctly assume this is for Expression Statements like + end_offset = self._end_offset + # "f(x,y);" and "a = f(3);" that are according to clang NOT statements, + # but expressions (without the semicolon) + if (not self._is_statement_or_declaration()) and (self.parent and self.parent.ast_type in STMT_PARENTS): + content = self.root.binary_file_content() + while ( + end_offset < len(content) and content[end_offset - 1] not in b";" + ): # Why use 'in' when list has one element, i.e. ';'? + end_offset += 1 + return end_offset + except: + return 0 + + def _is_statement_or_declaration(self): + return re.match("(?i).*(Stmt|Decl)", self.kind) + return isinstance(self.ast_type(), (Statement)) + + @override + @property + def matches_kind(self, node: ASTNode) -> bool: + return matches_kind(self.ast_type, node.ast_type) + + @override + @property + def properties(self) -> dict[str, Any]: + # get all the attributes of self.node except the inner nodes, id, location, range, kind and name and all reference nodes (that is children with 'id) + properties = { + k: ClangJsonASTNode._remove_ids(v) + for k, v in self.node.items() + if ClangJsonASTNode.__is_property(k) and not ClangJsonASTNode._is_reference(v) == None + } + if self._get(["range", "end", "expansionLoc", "offset"], -1) != -1: # dealing with a macro expansion + properties["macro_expansion"] = self.text + # matching name through props + if self.ast_type == DeclarationExpression: + properties["name"] = self.name + + return properties + + @override + @property + def referenced_by(self) -> Sequence[ASTReference]: + if self.inserted: + return [] + self.translation_unit.lazy_create_references(self) + ref_by = self.translation_unit._referenced_by.get(self.node["id"], EMPTY_LIST) + definition_node_id = self._get_function_definition() + if definition_node_id: + # try to find the definition which might have references + ref_by += self.translation_unit._referenced_by.get(definition_node_id, EMPTY_LIST) + return [ + ASTReference(self.translation_unit._nodes[ref.node_id], ref.ref_kind, ref.properties) + for ref in ref_by + if ref.node_id != self.node["id"] + ] + + def _get_function_definition(self): + refs = self.translation_unit._referenced_by.get(self.node["id"], EMPTY_LIST) + for ref in refs: + if ref.ref_kind == "previousDecl": + return ref.node_id + return None + + @override + @property + def references(self) -> list[ASTReference]: + if self.inserted: + return [] + self.translation_unit.lazy_create_references(self) + + refs = self.translation_unit._references.get(self.node["id"], EMPTY_LIST) + definition_node_id = self._get_function_definition() + # TODO: also class definitions, type definitions, ... + if definition_node_id: + # try to find the definition which might have references + refs += self.translation_unit._references.get(definition_node_id, EMPTY_LIST) + # remove duplicates + refs = list({ref.node_id: ref for ref in refs}.values()) + + return [ + ASTReference(self.translation_unit._nodes[ref.node_id], ref.ref_kind, ref.properties) + for ref in refs + if ref.node_id != self.node["id"] + ] + + @override + @property + def is_statement(self) -> bool: + return ( + self.parent != None and self.parent.ast_type in STMT_PARENTS + ) # TODO: Why look at the kind of your parent and not at your own kind? + + def _derive_name(self) -> str: + name = self.node.get("name") + if name: + return name + kind = self.node.get("kind") + decl_ref_name_path = ["referencedDecl", "name"] + if kind == "CallExpr": + # equalize with libclang + decl_ref_child = [inner["kind"] for inner in self.node.get("inner", []) if inner.get("kind") == "DeclRefExpr"] + if decl_ref_child: + return self._get_property(decl_ref_child[0], decl_ref_name_path, default=EMPTY_STR) + if kind == "DeclRefExpr": + return self._get(decl_ref_name_path, default=EMPTY_STR) + if kind == "StringLiteral": + return self._get(["value"], default=EMPTY_STR) + return self.node.get("name", EMPTY_STR) + + def __derive_start_offset(self) -> int: + offset = self._get(["range", "begin", "offset"], default=-1) + if offset == -1: + # we might be dealing with a macro in that case use the expansion location + offset = self._get(["range", "begin", "expansionLoc", "offset"], default=0) + return offset + + def __derive_end_offset(self) -> int: + if self.__derive_kind() == "TranslationUnitDecl": + return len(self.binary_file_content(self.filename)) + offset = self._get(["range", "end", "offset"], default=-1) + tok_len = self._get(["range", "end", "tokLen"], default=-1) + if offset == -1: + # we might be dealing with a macro in that case use the expansion location + offset = self._get(["range", "end", "expansionLoc", "offset"], default=0) + tok_len = self._get(["range", "end", "expansionLoc", "tokLen"], default=0) + + return offset + tok_len + + def __derive_kind(self) -> str: + return self.node.get("kind", EMPTY_STR) + + @staticmethod + def _remove_wrapper(node): + try: + if ClangJsonASTNode._is_wrapped(node): + return ClangJsonASTNode._remove_wrapper(list(node["inner"])[0]) + except: + pass + return node + + @staticmethod + def _remove_ids(json_node): + if not isinstance(json_node, dict): + return json_node + return {k: v for k, v in json_node.items() if k not in ID_TAGS} + + @staticmethod + def _is_reference(json_node): + return len(ReferenceHelper._get_reference_ids(json_node)) > 0 + + @staticmethod + @cache + def __is_property(key): + return key not in [ + "id", + "inner", + "loc", + "range", + "kind", + "name", + "isUsed", + "isReferenced", + "referencedDecl", + "mangledName", + *ON_NODE_ID_TAGS, + ] + + @staticmethod + def _is_wrapped(node): + """ + Check if a node is wrapped. + + A node is considered wrapped if it meets the following conditions: + 1. The node does not have an 'id' or its 'kind' starts with "Implicit". + 2. The node has exactly one inner node. + """ + return (not node.get("id") or node["kind"].startswith("Implicit")) and len(list(node["inner"])) == 1 + + def _get[T](self, path: Sequence[str], default: T) -> T: + return self._get_property(self.node, path, default) + + @staticmethod + def _get_property[T](target: dict[str, Any], path: Sequence[str], default: T) -> T: + assert default is not None, "default value must be provided" + try: + for p in path: + target = target[p] + # TODO: Is this code really correct when path contains multiple strings? + # Doesn't target become an Any, and hence might not support __get_item__ any more? + return target if isinstance(target, type(default)) else default + except: + return default + + @property + def is_implicit(self): + self.is_part_of_translation_unit() + + +class ReferenceHelper: + + @staticmethod + def create_references(ast_node: ClangJsonASTNode) -> None: + assert isinstance( + ast_node, ClangJsonASTNode + ), f"Expected ClangJsonASTNode but got {type(ast_node)}" # TODO: still needed when using type hints? + if ast_node.inserted: + return + references = [] + node_id = ast_node.node["id"] + ast_node.translation_unit._references[node_id] = references + refs = {k: v for k, v in ast_node.node.items() if not ReferenceHelper._is_child_node(k) and ClangJsonASTNode._is_reference(v)} + for k in [k for k in ast_node.node.keys() if k in ON_NODE_ID_TAGS]: + refs[k] = ast_node.node + # add the node if it contains a reference for example in case of previousDecl + + # to make clang json compatible with clang python, we add the reference of the DeclRefExpr child to the CallExpr + if ast_node.ast_type == Call: + for n in ast_node.children: + if n.ast_type == DeclarationExpression: + ref_child = { + k: v for k, v in n.node.items() if not ReferenceHelper._is_child_node(k) and ClangJsonASTNode._is_reference(v) + } + refs.update(ref_child) + + for kind, ref in refs.items(): + for ref_id in ReferenceHelper._get_reference_ids(ref): + if ref_id == node_id: + continue + properties = {k: p for k, p in ref.items() if k != ref_id} if ref != ast_node.node else EMPTY_DICT + reference = ClangJsonASTReference(ref_id, kind, properties) + referenced_by = ClangJsonASTReference(node_id, kind, properties) + try: + ast_node.translation_unit._referenced_by[ref_id].append(referenced_by) + except: + ast_node.translation_unit._referenced_by[ref_id] = [referenced_by] + references.append(reference) + + @staticmethod + def add_record_references(ast_node: ClangJsonASTNode) -> None: + """ + JSON does not contain direct references between classes and their base classes. + + Hence these references are created in this method. + + This method checks if the given AST node is of kind 'CXXRecordDecl' and has a tag 'class'. + If so, it processes the base classes of the node and creates references for them. + + Args: + ast_node (ClangJsonASTNode): The AST node to process. + + Raises: + AssertionError: If the provided ast_node is not an instance of ClangJsonASTNode. + """ + assert isinstance( + ast_node, ClangJsonASTNode + ), f"Expected ClangJsonASTNode but got {type(ast_node)}" # TODO: still needed when using type hints? + if ast_node.inserted: + return + + bases = ast_node._get(["bases"], []) + if not bases: + bases = [ast_node.node] if ast_node.node.get("type") else None + if not bases: + return + node_id = ast_node.node["id"] + for base in bases: + ref_ids = ReferenceHelper._get_record_decl(ast_node, base) + for kind, ref_id in ref_ids: + properties = {k: p for k, p in base.items() if k != "type"} + reference = ClangJsonASTReference(ref_id, kind, properties) + referenced_by = ClangJsonASTReference(node_id, kind, properties) + try: + ast_node.translation_unit._referenced_by[ref_id].append(referenced_by) + except: + ast_node.translation_unit._referenced_by[ref_id] = [referenced_by] + try: + ast_node.translation_unit._references[node_id].append(reference) + except: + ast_node.translation_unit._references[node_id] = [reference] + + @staticmethod + def _get_record_decl(ast_node, base) -> Sequence[str]: + try: + tp = base["type"] + if "desugaredQualType" in tp and "::" in tp["desugaredQualType"]: + # split desugaredQualType to derive the parent namespaces + namespaces = tp["desugaredQualType"].split("::")[:-1][::-1] + else: + namespaces = [] + qual_type = tp["qualType"] + ids = [] + ctor_type = EMPTY_STR + if ast_node.ast_type == ConstructorExpression: + ctor_type = ast_node._get(["ctorType", "qualType"], EMPTY_STR) + + for id, node in ast_node.translation_unit._nodes.items(): + if node.ast_type == RecordDef and node.name == qual_type: + parent = node.parent + matches = True + for ns in namespaces: + if ns != parent.name or parent.ast_type != Namespace: + matches = False + parent = parent.parent + if matches: + ids.append((node.ast_type, id)) + if ctor_type != EMPTY_STR and node.ast_type == Constructor: + # link all matching + matches = node._get(["type", "qualType"], EMPTY_STR) == ctor_type + if matches: + ids.append((node.ast_type, id)) + return ids + except: + pass + return [] + + @staticmethod + def _get_reference_ids(json_node): + result = [] + if not isinstance(json_node, dict): + return result + for key in ID_TAGS: + value = json_node.get(key) + if value != None: + result.append(value) + return result + + @staticmethod + @cache + def _is_child_node(key): + return key in ["inner"] diff --git a/src/renaissance/impl/clang/cpp_utils.py b/src/renaissance/impl/clang/cpp_utils.py new file mode 100644 index 00000000..925f32c0 --- /dev/null +++ b/src/renaissance/impl/clang/cpp_utils.py @@ -0,0 +1,96 @@ +from renaissance.impl.types import Type, Literal, DeclarationExpression + + +def get_ancestor(node, kind: type[Type]): + parent = node.parent + if not parent: + return None + if isinstance(parent.ast_type(), kind): + return parent + return parent.get_ancestor(kind) + + +def matches_kind(mine, other) -> bool: + return ( + mine == other + or (isinstance(mine(), Literal) and isinstance(other(), DeclarationExpression)) + or (isinstance(other(), Literal) and isinstance(mine(), DeclarationExpression)) + ) + + +class CPPUtils: + + # a set of cpp reserved keywords in reverse alphabetical order: + RESERVED_KEYWORDS = { + "while", + "wchar_t", + "void", + "volatile", + "virtual", + "unsigned", + "union", + "typename", + "typedef", + "try", + "true", + "throw", + "this", + "template", + "switch", + "struct", + "static_cast", + "static", + "sizeof", + "signed", + "short", + "return", + "reinterpret_cast", + "register", + "public", + "protected", + "private", + "operator", + "or_eq", + "or", + "not_eq", + "not", + "new", + "namespace", + "mutable", + "long", + "inline", + "int", + "if", + "goto", + "friend", + "for", + "float", + "false", + "extern", + "explicit", + "export", + "enum", + "else", + "double", + "do", + "delete", + "default", + "decltype", + "continue", + "const_cast", + "const", + "class", + "char16_t", + "char32_t", + "char", + "catch", + "case", + "break", + "bool", + "bitand", + "bitor", + "auto", + "asm", + "and_eq", + "and", + } diff --git a/src/renaissance/impl/python/__init__.py b/src/renaissance/impl/python/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/renaissance/impl/python/__init__.py @@ -0,0 +1 @@ + diff --git a/src/renaissance/impl/python/ast_node.py b/src/renaissance/impl/python/ast_node.py new file mode 100644 index 00000000..2f1ca584 --- /dev/null +++ b/src/renaissance/impl/python/ast_node.py @@ -0,0 +1,62 @@ +""" +implementation that patches the native ast using 'traits' mechanism, +require minimum amound of code to make the matcher work + +""" + +import ast + +from renaissance.impl.types import KIND_MAP, BogusType + + +class ASTExtension: + + @staticmethod + def load_from_ast(text, file): + root = ast.parse(text, file) + return root + + @staticmethod + @property + def ast_node(self): + return self + + @staticmethod + @property + def ast_kind(self): + return KIND_MAP.get(type(self).__name__, BogusType).__name__ + + @staticmethod + @property + def ast_type(self): + return KIND_MAP.get(type(self).__name__, BogusType) + + @staticmethod + @property + def ast_properties(self): + return {field: getattr(self, field) for field in self._fields if not isinstance(getattr(self, field), ast.AST)} + + @staticmethod + @property + def ast_children(self): + children = [getattr(self, field) for field in self._fields if isinstance(getattr(self, field), (ast.AST))] + [children.extend(getattr(self, field)) for field in self._fields if isinstance(getattr(self, field), (list))] + return children + + @staticmethod + @property + def ast_signature(self): + return ast.unparse(self) + + @staticmethod + @property + def ast_name(self): + if isinstance(self, ast.arg): + signature = self.arg + elif isinstance(self, ast.Name): + signature = self.id + elif isinstance(self, ast.Expr) and isinstance(self.value, ast.Name): + signature = self.value.id + else: + signature = str(self) + return signature diff --git a/src/renaissance/impl/python/cst_node.py b/src/renaissance/impl/python/cst_node.py new file mode 100644 index 00000000..e5196611 --- /dev/null +++ b/src/renaissance/impl/python/cst_node.py @@ -0,0 +1,120 @@ +from pathlib import Path +from typing import Self + +import libcst +from libcst import BaseSmallStatement, BaseCompoundStatement, CSTNode, MetadataWrapper, ClassDef +from libcst import FunctionDef +from libcst.metadata import WhitespaceInclusivePositionProvider + +from renaissance.impl.types import KIND_MAP, UnknownType +from renaissance.impl.python.util import convert +from renaissance.utils.ast_utils import preceding_sibling, next_sibling + + +class PythonCstTranslationUnit: + def __init__(self, content, file_name: str): + self.content = content + self.lines = content.splitlines() + self.file_name = file_name + self.references_initialized = False + self.wrapper = MetadataWrapper(libcst.parse_module(content)) + self.atu = self.wrapper.module + self.spans = self.wrapper.resolve(WhitespaceInclusivePositionProvider) + + def start_of(self, node: CSTNode) -> int: + span = self.spans.get(node) + return convert(self.lines, span.start.line, span.start.column) if span else 0 + + def end_of(self, node: CSTNode) -> int: + span = self.spans.get(node) + return convert(self.lines, span.end.line, span.end.column) if span else 0 + + def signature_of(self, node: CSTNode) -> str: + try: + return self.atu.code_for_node(node) + except: + return "" + + +class PythonCstNode: + def __init__(self, node: CSTNode, translation_unit: PythonCstTranslationUnit, parent=None): + self.parent = parent + if parent and parent.root: + self.root = parent.root + else: + self.root = self + self.translation_unit = translation_unit + self.node = node + + self.is_statement = isinstance(self.node, (BaseSmallStatement, BaseCompoundStatement)) + + # for matcher + self.ast_type = KIND_MAP.get(type(node).__name__, UnknownType) # type(node)) + if self.ast_type == UnknownType: + print(f'"{type(node).__name__}": {type(node).__name__},') + self.children: list[Self] = [PythonCstNode(node, translation_unit, self) for node in node.children] + self.properties = {} + + # for shower + self.is_implicit = True + self.show_props = False + + # for rewriter + self.text = self.signature + + def __str__(self): + return str(self.node) + + def __repr__(self): + return repr(self.node) + + @property + def signature(self): + return self.translation_unit.signature_of(self.node) + + @property + def offset(self): + return self.translation_unit.start_of(self.node) + + @property + def length(self): + return self.end_offset - self.offset + + @property + def end_offset(self): + return self.translation_unit.end_of(self.node) + + @property + def filename(self): + return self.translation_unit.file_name + + @property + def name(self): + if isinstance(self.node, (ClassDef, FunctionDef)): + return self.node.name.value + else: + return "" + self.name = "" # self._derive_name() + + @property + def next_sibling(self) -> Self | None: + return next_sibling(self) + + @property + def preceding_sibling(self) -> Self | None: + return preceding_sibling(self) + + @staticmethod + def load(file_path: Path) -> "PythonCstNode": + with open(file_path, "r") as file: + content = file.read() + return PythonCstNode.load_from_text(content, str(file_path)) + + @staticmethod + def load_from_text( + text: str, + file_name: str = "cst_snippet.py", + ) -> "PythonCstNode": + translation_unit = PythonCstTranslationUnit(text, file_name=str(file_name)) + root_node = PythonCstNode(translation_unit.atu, translation_unit) + return root_node diff --git a/src/renaissance/impl/python/extractor.py b/src/renaissance/impl/python/extractor.py new file mode 100644 index 00000000..c4002786 --- /dev/null +++ b/src/renaissance/impl/python/extractor.py @@ -0,0 +1,46 @@ +from pathlib import Path + +import networkx + +from renaissance.impl.python.rst_node import PythonRstNode + + +class PythonExtractor: + graph = networkx.DiGraph() + codebase: dict = {} + + def process(self, file: Path): + root = PythonRstNode.load(file) + module_name = root.filename.replace("/", ".").replace(".py", "") + folder = str(Path(file).parent) + self.graph.add_node(folder, type="folder") + self.graph.add_edge(folder, module_name, type="contains") + + for stmt in root: + match stmt.ast_type: + case "Import": + self.graph.add_edge(module_name, stmt.name, type="include") + case "ImportFrom": + for alias in stmt.node.names: + self.graph.add_edge(module_name, f"{stmt.node.module}.{alias.name}", type="include") + case "FunctionDef": + self.graph.add_edge(module_name, f"{module_name}.{stmt.name}", type="definition") + self.graph.add_node(f"{module_name}.{stmt.name}", properties="function") + # todo: convert #, stmt.properties) to graphml + case "ClassDef": + self.graph.add_edge(module_name, f"{module_name}.{stmt.name}", type="definition") + self.graph.add_node(f"{module_name}.{stmt.name}") # convert to args, stmt.properties) + case _: + pass + + tu = root.translation_unit + self.codebase[file] = root + # # reconstruct dependencies inside module + # tu.lazy_create_refers(root) + # self.nodes |= tu._nodes + # self.edges |=tu._references + # self.edges |= tu._referenced_by + + def save_graph(self, filename: str): + networkx.write_graphml(self.graph, filename) + print(f"Graph saved to: {filename}") diff --git a/src/renaissance/impl/python/factory.py b/src/renaissance/impl/python/factory.py new file mode 100644 index 00000000..2d73b2bd --- /dev/null +++ b/src/renaissance/impl/python/factory.py @@ -0,0 +1,169 @@ +import ast +import re +from pathlib import Path +from typing import Sequence + +import tree_sitter_python +from libcst import SimpleStatementLine + +from renaissance.impl.types import MatchAll, MatchOne, ExpressionStatement, Type, DeclarationExpression, Name, Arg +from renaissance.impl import MATCH_ALL, MATCH_ONE +from renaissance.impl.python.ast_node import ASTExtension +from renaissance.impl.python.cst_node import PythonCstNode +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.tree_sitter.adapter import TreeSitterAdapter +from renaissance.impl.tree_sitter.lst import LSTNode +from renaissance.syntax_tree.match_finder import AstProtocol, is_match +from renaissance.utils.ast_utils import replace_dollar, use_dollar + +_MATCH_ALL_RE = re.compile(r"^" + re.escape(MATCH_ALL) + r"\w+$") +_MATCH_ONE_RE = re.compile(r"^" + re.escape(MATCH_ONE) + r"\w+$") + +SHOW_NODE = False + + +class PythonPattern(AstProtocol): + + def __init__(self, node): + self.node: PythonRstNode = node + if type(node) is str: + print(node) + return + self.ast_type: Type = self.derive_type(node) + + self.properties: dict = node.properties + self.children: list[PythonPattern] = [PythonPattern(node) for node in node.children] + self.signature: str = node.signature + if hasattr(node, "name") and node.name: + self.name: str = use_dollar(node.name) + else: + self.name = "" + + def __eq__(self, other: AstProtocol) -> bool: + return is_match(other, self) + + def __repr__(self): + return use_dollar(str(self.node)) + + def derive_type(self, node) -> str: + # signature = "" + # if isinstance(node.ast_type(), Argument): + # signature = node.node.arg + # elif isinstance(node.ast_type(), Name): + # signature = node.node.value + # elif isinstance(node.ast_type(), ExpressionStatement) and isinstance(node.node.value, ast.Name): + # signature = node.node.value.id + # if _MATCH_ALL_RE.match(signature): + # return MatchAll + # elif _MATCH_ONE_RE.match(signature): + # return MatchOne + # if isinstance(node, LSTNode): + # return node.ast_type + # else: + # return node.ast_type + if isinstance(node, ast.arg): + signature = node.arg + elif isinstance(node, ast.Name): + signature = node.id + elif isinstance(node, ast.Expr) and isinstance(node.value, ast.Name): + signature = node.value.id + elif isinstance(node, ast.AST): + signature = str(node) + else: + signature = node.name + + if node.ast_type in [DeclarationExpression, ExpressionStatement, Name, Arg]: + if _MATCH_ALL_RE.match(signature): + return MatchAll + elif _MATCH_ONE_RE.match(signature): + return MatchOne + + return node.ast_type + + +class PythonFactory: + + def __init__(self, clazz: type[PythonRstNode | PythonCstNode | LSTNode | ast.AST]) -> None: + self.clazz = clazz + if clazz == LSTNode: + clazz.load_from_text = self.load_from_lst + elif clazz == ast.AST: + clazz.load_from_text = ASTExtension.load_from_ast + # matcher + clazz.node = ASTExtension.ast_node + + # clazz.name = ASTExtension.ast_name + clazz.ast_type = ASTExtension.ast_type + clazz.properties = ASTExtension.ast_properties + clazz.children = ASTExtension.ast_children + clazz.signature = ASTExtension.ast_signature + + # writer + clazz.text = ASTExtension.ast_signature + clazz.filename = "dummy.py" + + # shower + clazz.is_implicit = True + clazz.show_props = False + clazz.indent = "" + + def create(self, file_path: Path) -> PythonRstNode | PythonCstNode: + atu = self.clazz.load(file_path=file_path) + assert isinstance(atu, self.clazz) + return atu + + def create_from_text(self, text: str, file_name: str = "snippet.py") -> PythonRstNode | PythonCstNode | LSTNode | ast.AST: + + atu = self.clazz.load_from_text(text, file_name) + assert isinstance(atu, self.clazz) + return atu + + @staticmethod + def load_from_lst(text, file): + adapter = TreeSitterAdapter(tree_sitter_python) + tree = adapter.parse_code(text) + return adapter.to_lst(text, tree).root + + +class PythonPatternFactory: + + def __init__(self, factory: PythonFactory): + self.factory = factory + + def _create(self, text: str) -> PythonPattern: + return PythonPattern(self.factory.create_from_text(text, "pattern.py")) + + def create(self, text: str) -> PythonPattern: + text = replace_dollar(text) + return self._create(text) + + def create_statements(self, text: str) -> Sequence[PythonPattern]: + atu = self.create(text) + return atu.children + + def create_statement(self, text: str) -> PythonPattern: + stmt = self.create_statements(text)[-1] + if isinstance(stmt.node.node, SimpleStatementLine): + return stmt.children[0] + else: + return stmt + # return stmt + + def create_expression(self, text: str) -> PythonPattern: + my_pattern = self.create_statement(text) + if isinstance(my_pattern.node, PythonRstNode): + return PythonPattern(my_pattern.node.expression) + elif isinstance(my_pattern.node, LSTNode): + return PythonPattern(my_pattern.node.children[-1]) + elif isinstance(my_pattern.node, PythonCstNode): + return PythonPattern(my_pattern.node.children[-1]) + else: + return PythonPattern(my_pattern.node.children[0]) + + def create_decorators(self, param): + return self.create_statement(param + "\ndef test(): pass").children[2] + + @staticmethod + def create_kwargs(kw_str) -> Sequence[PythonPattern]: + call = ast.parse(f"fun({replace_dollar(kw_str)})", "kwarg_pattern.py", type_comments=True).body[0].value + return [PythonPattern(PythonRstNode(kwarg)) for kwarg in call.keywords] diff --git a/src/renaissance/impl/python/rst_node.py b/src/renaissance/impl/python/rst_node.py new file mode 100644 index 00000000..f568e49b --- /dev/null +++ b/src/renaissance/impl/python/rst_node.py @@ -0,0 +1,460 @@ +import ast +import sys +import textwrap +from pathlib import Path +from typing import Any, Sequence, Self, Callable + +from renaissance.impl.types import * + +from renaissance.impl.python.util import convert +from renaissance.syntax_tree.match_finder import find_in_list +from renaissance.utils.ast_utils import preceding_sibling, next_sibling, match_props, match_children, format_node +from renaissance.utils.ast_utils import traverse + +types = ["int", "float", "str", "list", "set", "tuple", "Mapping", "dict", "Optional"] +IRRELEVANT_PROPS = {"comment"} +IRRELEVANT_NODES = {"comment"} +IMPLICIT = [ImplicitNode] + + +class ImplicitNode(ast.Name): + _fields = ( + "id", + "body", + ) + + _field_types = { + "id": str, + "body": list, + } + + def __init__(self, name, children=None): + super().__init__(name) + self.body = children or [] + self.lineno = 0 + self.col_offset = 0 + self.end_lineno = 0 + self.end_col_offset = 0 + + +class PythonRSTReference: + def __repr__(self): + return f"{self.node_id}:{self.ref_kind}" + + def __init__(self, node_id: str, ref_kind: str, properties: dict[str, Any]) -> None: + self.node_id = node_id + self.ref_kind = ref_kind + self.properties = properties + + +class PythonRstTranslationUnit: + cache = {} + + def __init__(self, content, file_name: str): + self.content = content.encode(sys.getfilesystemencoding()) + self.atu = ast.parse(content, file_name) + self.file_name = file_name + self.references_initialized = False + PythonRstTranslationUnit.cache[file_name] = content + self.lines = self.content.splitlines() + + self._references: dict[str, list[PythonRSTReference]] = {} + self._referenced_by: dict[str, list[PythonRSTReference]] = {} + self._nodes: dict[str, "PythonRstNode"] = {} + + def check_diagnostics(self, continue_with_warning=True) -> None: + msg = None + errors = "" + for d in self.atu.type_ignores: + msg = f"type ignored: {d.tag} at {d.lineno}\n" + errors += msg + print(msg) + if msg and not continue_with_warning: + raise Exception(f"Error parsing: {self.file_name} \n+ errors: {errors}") + + def lazy_create_refers(self, node: "PythonRstNode") -> None: + if self.references_initialized: + return + for n in traverse(node.root): + self.create_references(n) + self.references_initialized = True + + def add(self, node): + match node.ast_type.__name__: + case "Name": + if node.node.id not in self._nodes and node.node.id not in types: + self._nodes[node.node.id] = node + case "FunctionDef": + if node.node.name not in self._nodes: + self._nodes[node.node.name] = node + case "Call": + if node.name not in self._nodes: + self._nodes[node.name] = node + case "ClassDef": + if node.name not in self._nodes: + self._nodes[node.name] = node + case "arg": + if node.name != "self": + if node.name not in self._nodes: + self._nodes[node.name] = node + + def create_references(self, ast_node) -> None: + assert isinstance(ast_node, PythonRstNode), f"Expected PythonASTNode but got {type(ast_node)}" + match type(ast_node.node): + case ast.arg: + if ast_node.name != "self": + if isinstance(ast_node.node, ast.arg) and isinstance(ast_node.node.annotation, ast.Name): + node_id = ast_node.name + ref_id = ast_node.node.annotation.id + ref_kind = "TypeRef" + self.add_reference(node_id, ref_id, ref_kind) + case ast.Assign: + if isinstance(ast_node.node, ast.Assign): + for n in ast_node.node.targets: + if isinstance(n, ast.Name) and isinstance(ast_node.node.value, ast.Call): + node_id = n.id + func = ast_node.node.value.func + ref_id = func.id if isinstance(func, ast.Name) else None + if ref_id: + ref_kind = "CallRef" + self.add_reference(node_id, ref_id, ref_kind) + case ast.AnnAssign: + if isinstance(ast_node.node, ast.AnnAssign): + if ( + ast_node.node.annotation + and isinstance(ast_node.node.target, ast.Name) + and isinstance(ast_node.node.annotation, ast.Name) + ): + node_id = ast_node.node.target.id + ref_id = ast_node.node.annotation.id + ref_kind = "TypeRef" + self.add_reference(node_id, ref_id, ref_kind) + case ast.ClassDef: + if isinstance(ast_node.node, ast.ClassDef): + node = ast_node.node + node_id = node.name + if node.bases: + ref_node = node.bases[0] + if isinstance(ref_node, ast.Name): + ref_id = ref_node.id + ref_kind = "Inherit" + self.add_reference(node_id, ref_id, ref_kind) + # add functions and attributes to class + + case ast.Call: + if isinstance(ast_node.node, ast.Call): + # obj.function. then obj refers to function + if isinstance(ast_node.node.func, ast.Attribute): + node_id = ast_node.name + ref_id = ast_node.node.func.attr + ref_kind = "FuncCall" + self.add_reference(node_id, ref_id, ref_kind) + # call function 'a' in function 'b', then 'b' refers to 'a' + container = ast_node.get_container_parent() + if container.ast_type == FunctionDef and isinstance(ast_node.node.func, ast.Name): + node_id = container.name + ref_id = ast_node.node.func.id + ref_kind = "FuncCall" + self.add_reference(node_id, ref_id, ref_kind) + + def add_reference(self, node_id: str, ref_id: str, ref_kind: str) -> None: + properties = {} + if node_id == ref_id: + return + reference = PythonRSTReference(ref_id, ref_kind, properties) + referenced_by = PythonRSTReference(node_id, ref_kind, properties) + if node_id in self._references: + self._references[node_id].append(reference) + else: + self._references[node_id] = [reference] + if ref_id in self._referenced_by: + self._referenced_by[ref_id].append(referenced_by) + else: + self._referenced_by[ref_id] = [referenced_by] + + def get_referenced_by(self, node_id): + refs = self._referenced_by.get(node_id, []) + return [PythonRSTReference(self._nodes[ref.node_id].name, ref.ref_kind, ref.properties) for ref in refs] + + def get_references(self, node_id): + refs = self._references.get(node_id, []) + return [PythonRSTReference(self._nodes[ref.node_id].name, ref.ref_kind, ref.properties) for ref in refs] + + +class PythonRstNode: + def __init__(self, node: ast.AST, translation_unit: PythonRstTranslationUnit = None, parent=None): + self.root = parent.root if parent and parent.root else self + self.node = node + self.parent = parent + self.translation_unit: PythonRstTranslationUnit = translation_unit + self.ast_type = KIND_MAP.get(type(node).__name__, UnknownType) + if self.ast_type == UnknownType: + print(f'"{type(node).__name__}": {type(node).__name__},') + + self.indent = "" + self.name = self._derive_name() + self.show_props = False + self.children = [] + self.properties = {} + self.is_implicit = self.ast_type not in IMPLICIT + self.offset = 0 + self.length = 0 + if self.translation_unit: + self.filename = translation_unit.file_name + self.derive_position(node, translation_unit, parent) + self.add_node() + for name in node._fields: + try: + child = getattr(node, name) + match child: + case list(): # Matches any list + if isinstance(node, Global) and name == "names": + if len(child) == 1: + self.name = child[0] + if name == "body": + self.body = self.children + + if isinstance(node, ImplicitNode) or isinstance(node, ast.Module) or len(node._fields) == 1: + self.children.extend(PythonRstNode(n, translation_unit, self) for n in child) + if name == "body": + self.body = self.children + else: + self.children.append(PythonRstNode(ImplicitNode(name, child), translation_unit, self)) + if name in ["body", "cases"]: + self.body = self.children[-1].children + + case ast.AST(): + if name not in ["ctx"]: + self.children.append(PythonRstNode(child, translation_unit, self)) + if isinstance(child, ast.expr): + self.expression = self.children[-1] + case _: + if name not in ["None"]: + self.properties[name] = child + except AttributeError as e: + print(e) + continue + + self.end_offset = self.offset + self.length + self.extended_end_offset = self.end_offset + self.is_statement = isinstance(self.node, ast.stmt) + + def __eq__(self, other): + return ( + isinstance(other, type(self)) + and self.ast_type == other.ast_type + and match_props(self.properties, other.properties, IRRELEVANT_PROPS) + and match_children(self.children, other.children, IRRELEVANT_NODES) + ) + + def __contains__(self, item): + if not isinstance(item, list): + item = [item] + return find_in_list(self.children, item) + + def __getitem__(self, key): + """Allow indexing/slicing into node to access children. + + Usage: node[0] == node.children[0] + """ + return self.children[key] + + def __repr__(self): + return format_node(self) + + @property + def next_sibling(self) -> Self | None: + return next_sibling(self) + + @property + def preceding_sibling(self) -> Self | None: + return preceding_sibling(self) + + def process(self, function: Callable[[Self], None]) -> None: + function(self) + for child in self.children: + child.process(function) + + def derive_position(self, node: ast.AST, translation_unit: PythonRstTranslationUnit, parent): + if node._attributes: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and node.decorator_list: + self.offset = convert(self.translation_unit.lines, node.decorator_list[0].lineno, node.decorator_list[0].col_offset) - 1 + elif parent.name == "decorator_list": + # also include the @ in the decorator + self.offset = convert(self.translation_unit.lines, node.lineno, node.col_offset) - 1 # type: ignore[attr-defined] + else: + self.offset = convert(self.translation_unit.lines, node.lineno, node.col_offset) # type: ignore[attr-defined] + all_space = all(c == " " for c in self.translation_unit.content[self.offset - node.col_offset : self.offset]) + if all_space: + self.offset = self.offset - node.col_offset if self.offset - node.col_offset >= 0 else 0 + self.length = convert(self.translation_unit.lines, node.end_lineno, node.end_col_offset) - self.offset # type: ignore[attr-defined] + elif isinstance(node, ast.Module) and translation_unit: + self.offset = 0 + self.length = len(translation_unit.content) + else: + self.offset = 0 + self.length = 0 + + @staticmethod + def load( + file_path: Path, + extra_args: Sequence[str] | None = None, + working_dir: Path | None = None, + ) -> "PythonRstNode": + # Keep a uniform loader signature across AST node implementations. + # Python's AST parser does not need extra arguments or a working dir. + _ = extra_args, working_dir + with open(file_path, "r") as file: + content = file.read() + return PythonRstNode.load_from_text(content, str(file_path)) + + @staticmethod + def load_from_text( + text: str, + file_name: str = "test.py", + extra_args: Sequence[str] | None = None, + working_dir: Path | None = None, + ) -> "PythonRstNode": + _ = extra_args, working_dir + translation_unit = PythonRstTranslationUnit(text, file_name=str(file_name)) + translation_unit.check_diagnostics() + root_node = PythonRstNode(translation_unit.atu, translation_unit) + return root_node + + def _derive_name(self): + + if ( + isinstance( + self.node, + ( + ast.FunctionDef, + ast.AsyncFunctionDef, + ast.ClassDef, + ast.ExceptHandler, + ), + ) + and self.node.name + ): + name = self.node.name + elif isinstance(self.node, ast.Global) and len(self.node.names) == 1: + name = self.node.names[0] + elif isinstance(self.node, (ast.AnnAssign, ast.AugAssign)) and isinstance(self.node.target, ast.Name): + name = self.node.target.id + elif isinstance(self.node, ast.Assign) and len(self.node.targets) == 1: + target = self.node.targets[0] + if isinstance(target, ast.Name): + name = target.id + else: + name = self.ast_type.__name__ + elif isinstance(self.node, ast.Name): + name = self.node.id + elif isinstance(self.node, ast.arg): + name = self.node.arg + elif isinstance(self.node, ast.Match) and isinstance(self.node.subject, ast.Name): + name = self.node.subject.id + elif isinstance(self.node, ast.Import) and len(self.node.names) == 1: + name = self.node.names[0].name + elif isinstance(self.node, ast.ImportFrom) and len(self.node.names) == 1: + name = self.node.names[0].name + elif isinstance(self.node, (ast.Assert, ast.Break, ast.Pass, ast.Raise, ast.Continue)): + name = "" + elif isinstance(self.node, (ast.For, ast.AsyncFor)): + if isinstance(self.node.target, Tuple): + name = getattr(self.node.target.dims[1], "id") + elif isinstance(self.node.target, ast.Name): + name = self.node.target.id + else: + name = str(self.node.target) + elif "body" not in self.node._fields: + name = ast.unparse(self.node) + elif isinstance(self.node, (ast.Module)) and self.translation_unit: + name = self.translation_unit.file_name + else: + name = self.ast_type.__name__ + return name if name else "" + + @property + def type(self): + return self.node.annotation.id if isinstance(self.node, ast.AnnAssign) and isinstance(self.node.annotation, ast.Name) else None + + @property + def value(self): + if self.ast_type == Assert: + return 0 + return self.node.value.value if hasattr(self.node, "value") else None + + @property + def expr(self): + if ( + isinstance( + self.node, + ( + ast.Assign, + ast.AnnAssign, + ast.AugAssign, + ast.Return, + ast.Expr, + ast.Delete, + ast.NamedExpr, + ), + ) + and hasattr(self.node, "value") + and getattr(self.node, "value") is not None + ): + return PythonRstNode(self.node.value, self.translation_unit, self) + elif isinstance(self.node, ast.Expr) and hasattr(self.node, "value"): + return PythonRstNode(self.node.value, self.translation_unit, self) + elif isinstance(self.node, (ast.For, ast.AsyncFor, ast.comprehension)): + return PythonRstNode(self.node.iter, self.translation_unit, self) + elif isinstance(self.node, (ast.If, ast.While, ast.Assert)): + return PythonRstNode(self.node.test, self.translation_unit, self) + elif isinstance(self.node, (ast.Raise, ast.ExceptHandler)) and hasattr(self.node, "exc") and self.node.exc is not None: + return PythonRstNode(self.node.exc, self.translation_unit, self) + else: + return None + + @property + def operator(self): + node_type = type(self.node).__name__ + op = type(self.node.op).__name__ if isinstance(self.node, (ast.BinOp, ast.UnaryOp, ast.BoolOp, ast.AugAssign)) else "" + return OPERATOR_MAP.get(node_type + op, "") + + @property + def signature(self) -> str: + sig = self.binary_file_content().decode(sys.getfilesystemencoding()) + if self.parent and self.parent.name == "decorator_list" and not sig.startswith("@"): + sig = "@" + sig + return sig + + def binary_file_content(self) -> bytes: + return ( + self.translation_unit.content[self.offset : self.offset + self.length] + if self.translation_unit + else ast.unparse(self.node).encode(sys.getfilesystemencoding()) + ) + + @property + def referenced_by(self) -> Sequence[PythonRSTReference]: + self.translation_unit.lazy_create_refers(self) + return self.translation_unit.get_referenced_by(self.name) + + @property + def references(self) -> list[PythonRSTReference]: + self.translation_unit.lazy_create_refers(self) + return self.translation_unit.get_references(self.name) + + def add_node(self): + self.translation_unit.add(self) + + def get_container_parent(self): + if self.parent: + if self.parent.ast_type.__name__ in ["FunctionDef", "ClassDef", "Module"]: + return self.parent + else: + return self.parent.get_container_parent() + else: + return self + + @property + def text(self) -> str: + return textwrap.dedent(self.signature) diff --git a/src/renaissance/impl/python/util.py b/src/renaissance/impl/python/util.py new file mode 100644 index 00000000..f18725c1 --- /dev/null +++ b/src/renaissance/impl/python/util.py @@ -0,0 +1,21 @@ +def convert(lines, line_nr, col): + if line_nr > len(lines): + return 0 + return sum(len(lines[i]) + 1 for i in range(line_nr - 1)) + col + # add node to the node list for references + + +def to_str(node) -> str: + if hasattr(node, "signature"): + return node.signature + else: + return str(node) + + +def convert_function(fun): + signature: str = fun.signature + "\n\n\n" + if len(fun.node.args.args) == 0: + signature = signature.replace(f"{fun.name}()", f"{fun.name}(self)", 1) + else: + signature = signature.replace(f"{fun.name}(", f"{fun.name}(self,", 1) + return signature diff --git a/src/renaissance/impl/tree_sitter/README.md b/src/renaissance/impl/tree_sitter/README.md new file mode 100644 index 00000000..7cb9e419 --- /dev/null +++ b/src/renaissance/impl/tree_sitter/README.md @@ -0,0 +1,144 @@ +# LST Toolkit + +This toolkit provides a parser-independent Language-Specific Tree (LST) representation with pattern matching, symbol binding, and extraction capabilities. It supports Tree-sitter grammars and offers a flexible interface for analyzing Python, Java, and C++ code. + +--- + +## 📦 Features + +- Generic internal AST representation (`LSTNode`) +- Structural pattern matching with placeholders +- Node-type based matchers +- Match abstraction layer (`Match`) +- Rule-based extractor (templated, with filtering) +- Symbol table for declarations, definitions, and uses +- Extensible for multi-language support +- Includes examples for Python, Java, and C++ +- Unit-tested matcher components +- VSCode integration + +--- + +## 🔧 Installation + +1. Clone or unzip the project. +2. Install dependencies: + +```bash +pip install -e . +``` + +```bash +pip install tree-sitter +``` +with a dash and not an underscore + +1. Run the setup script to clone grammars and build the shared library: + +```bash +python setup_grammars.py +``` + +This will: +- Clone Tree-sitter grammars for Python, Java, and C++ +- Build `build/my-languages.so` for use in adapters + +--- + +## 🧪 Running Tests + +```bash +python -m unittest discover tests +``` + +--- + +## 🧰 Examples + +Python: + +```bash +python examples/python_example.py +``` + +Java: + +```bash +cat examples/java_example.java +``` + +C++: + +```bash +cat examples/cpp_example.cpp +``` + +--- + +## 🧠 Structure + +- `core/lst.py` — Internal node structure +- `core/tree_sitter_adapter.py` — Parser adapter +- `core/pattern_matcher.py` — Structural matcher +- `core/node_type_matcher.py` — Node-type matcher +- `core/match.py` — Match abstraction +- `core/extractor.py` — Rule-based extractor +- `core/symbols.py` — Symbol table +- `examples/` — Example input files +- `tests/` — Unit tests +- `setup_grammars.py` — Auto-installs Tree-sitter grammars + +--- + +## 🧩 Integration + +You can use `Extractor`, `Match`, and `PatternMatcherInterfaceExtended` to write custom rules. + +Example: + +```python +extractor = Extractor(interface) +extractor.add_rule("function_definition", lambda m: m.first("match").signature) +results = extractor.run(source_code) +``` + +--- + +## 🚀 License + +MIT License — feel free to use and extend. + + +## 🔌 Clang Integration for C++ + +For advanced C++ analysis (with preprocessing and include resolution), this toolkit supports [libclang](https://clang.llvm.org/). + +### 🛠 Install Dependencies + +```bash +# On Ubuntu/Debian +sudo apt install libclang-dev + +# Python bindings +pip install clang +``` + +### 🔧 Usage + +Use `ClangAdapter` instead of `TreeSitterAdapter`: + +```python +from core.clang_adapter import ClangAdapter + +adapter = ClangAdapter() +lst = adapter.parse("examples/cpp_example.cpp") + +for node in lst.traverse(): + print(node) +``` + +The `ClangAdapter` provides: +- Full include resolution +- Macro expansion +- AST node types like `FUNCTION_DECL`, `CALL_EXPR`, etc. +- Source location metadata diff --git a/src/renaissance/impl/tree_sitter/__init__.py b/src/renaissance/impl/tree_sitter/__init__.py new file mode 100644 index 00000000..2aada246 --- /dev/null +++ b/src/renaissance/impl/tree_sitter/__init__.py @@ -0,0 +1,4 @@ +""" +the tree sitter is adapter to RST using an adapter, we can experiment with multi-language approach here + +""" diff --git a/src/renaissance/impl/tree_sitter/adapter.py b/src/renaissance/impl/tree_sitter/adapter.py new file mode 100644 index 00000000..82455460 --- /dev/null +++ b/src/renaissance/impl/tree_sitter/adapter.py @@ -0,0 +1,55 @@ +from tree_sitter import Parser, Language + +from renaissance.impl.tree_sitter.lst import LST, LSTNode +from renaissance.utils.ast_utils import replace_dollar, detect_placeholder + + +class TreeSitterAdapter: + def __init__(self, grammar_module): + language = Language(grammar_module.language()) + self.language = language + self.parser = Parser(language) + + def parse_code(self, source_code: str): + return self.parser.parse(bytes(source_code, "utf8")) + + def to_lst(self, source_code: str, tree) -> LST: + root_node = tree.root_node + source_code = replace_dollar(source_code) + return LST(self._convert_node(root_node, source_code, None)) + + def _convert_node(self, node, source_code: str, parent, root=None) -> LSTNode: + signature = source_code[node.start_byte : node.end_byte] + is_ph, coerced_type, ph_name = detect_placeholder(signature, node.type) + + lst_node = LSTNode( + node_type=coerced_type if is_ph else node.type, + properties={ + "start_point": node.start_point, + "end_point": node.end_point, + "source_code": source_code, + "name": ph_name, + "is_named": node.is_named, + **( + { + "placeholder": True, + "placeholder_name": ph_name, + "original_node_type": node.type, + } + if is_ph + else {} + ), + }, + signature=signature, + offset=node.start_byte, + children=[], + parent=parent, + root=root, + ) + if not root: + root = lst_node + + for child in node.children: + lst_child = self._convert_node(child, source_code, lst_node, root) + lst_node.add_child(lst_child) + return lst_node diff --git a/src/renaissance/impl/tree_sitter/extractor.py b/src/renaissance/impl/tree_sitter/extractor.py new file mode 100644 index 00000000..eb5c8d65 --- /dev/null +++ b/src/renaissance/impl/tree_sitter/extractor.py @@ -0,0 +1,110 @@ +import os +import networkx +from pathlib import Path + +from renaissance.impl.tree_sitter.adapter import TreeSitterAdapter + +from renaissance.impl.tree_sitter.factory import TreeStiterPatternFactory +from renaissance.impl.types import * +from renaissance.syntax_tree import PatternMatch +from renaissance.syntax_tree.match_finder import match_pattern + +GRAPHML_DIR = "out_graphml" +os.makedirs(GRAPHML_DIR, exist_ok=True) + + +class Extractor: + def __init__(self, factory: TreeStiterPatternFactory, patterns: list[str]): + self.pattern_factory = factory + self.patterns = patterns + + def run(self, raw: str) -> list[PatternMatch]: + code = self.pattern_factory.create_statements(raw) + results = [] + for rule in self.patterns: + pattern = self.pattern_factory.create_statements(rule) + results.extend(match_pattern(code, pattern, {})) + return results + + +class BaseCodeGraphExtractor: + def __init__(self, language: str, lib_path: str): + self.language = language + self.lib_path = lib_path + self.adapter = TreeSitterAdapter(lib_path) + self.graph = networkx.DiGraph() + + def extract(self, files): + for f in files: + try: + code = Path(f).read_text() + tree = self.adapter.parse_code(code) + lst = self.adapter.to_lst(code, tree) + self._process_file(f, lst) + except Exception as e: + print(f"Error processing {f}: {e}") + + def _process_file(self, file_path: str, lst): + raise NotImplementedError + + def save_graph(self, filename: str): + path = os.path.join(GRAPHML_DIR, filename) + networkx.write_graphml(self.graph, path) + print(f"Graph saved to: {path}") + + +class PythonCodeGraphExtractor(BaseCodeGraphExtractor): + def _process_file(self, file_path, lst): + folder = str(Path(file_path).parent) + self.graph.add_node(file_path, type="file", folder=folder) + self.graph.add_node(folder, type="folder") + self.graph.add_edge(folder, file_path, type="contains") + + for node in lst.traverse(): + if node.ast_type == FunctionDef: + name = node.signature.split("(")[0].split()[-1] + self.graph.add_node(name, type="function", file=file_path) + self.graph.add_edge(file_path, name, type="defines") + + elif node.ast_type == Call: + call_target = node.signature.strip().split("(")[0] + self.graph.add_node(call_target, type="call_target") + self.graph.add_edge(file_path, call_target, type="calls") + + +class JavaCodeGraphExtractor(BaseCodeGraphExtractor): + def _process_file(self, file_path, lst): + folder = str(Path(file_path).parent) + self.graph.add_node(file_path, type="file", folder=folder) + self.graph.add_node(folder, type="folder") + self.graph.add_edge(folder, file_path, type="contains") + + for node in lst.traverse(): + if node.ast_type == FunctionDef: + name = node.properties.get("name", "method") + self.graph.add_node(name, type="method", file=file_path) + self.graph.add_edge(file_path, name, type="defines") + + elif node.ast_type == Call: + target = node.signature.strip().split("(")[0] + self.graph.add_node(target, type="method_target") + self.graph.add_edge(file_path, target, type="calls") + + +class CppCodeGraphExtractor(BaseCodeGraphExtractor): + def _process_file(self, file_path, lst): + folder = str(Path(file_path).parent) + self.graph.add_node(file_path, type="file", folder=folder) + self.graph.add_node(folder, type="folder") + self.graph.add_edge(folder, file_path, type="contains") + + for node in lst.traverse(): + if node.ast_type == FunctionDef: + name = node.properties.get("name", "func") + self.graph.add_node(name, type="function", file=file_path) + self.graph.add_edge(file_path, name, type="defines") + + elif node.ast_type == Call: + call_expr = node.signature.strip().split("(")[0] + self.graph.add_node(call_expr, type="call_target") + self.graph.add_edge(file_path, call_expr, type="calls") diff --git a/src/renaissance/impl/tree_sitter/factory.py b/src/renaissance/impl/tree_sitter/factory.py new file mode 100644 index 00000000..1cca3ade --- /dev/null +++ b/src/renaissance/impl/tree_sitter/factory.py @@ -0,0 +1,29 @@ +from typing import Sequence + +from renaissance.impl.tree_sitter.adapter import TreeSitterAdapter +from renaissance.impl.tree_sitter.lst import LSTNode +from renaissance.utils.ast_utils import replace_dollar + + +class TreeStiterPatternFactory: + + def __init__(self, adapter: TreeSitterAdapter, language: str = "python"): + self.adapter = adapter + self.language = language + + def create(self, text: str) -> LSTNode: + text = replace_dollar(text) + if isinstance(self.adapter, TreeSitterAdapter): + tree = self.adapter.parse_code(text) + return self.adapter.to_lst(text, tree).root + else: + return self.adapter.to_lst(text).root + + def create_statements(self, text: str) -> Sequence[LSTNode]: + return self.create(text).children + + def create_statement(self, text: str) -> LSTNode: + return self.create_statements(text)[-1] + + def create_expression(self, text: str) -> LSTNode: + return self.create_statement(text).children[-1] diff --git a/src/renaissance/impl/tree_sitter/lst.py b/src/renaissance/impl/tree_sitter/lst.py new file mode 100644 index 00000000..5699b2e1 --- /dev/null +++ b/src/renaissance/impl/tree_sitter/lst.py @@ -0,0 +1,110 @@ +import sys +from typing import Any, Self, cast + +from renaissance.impl.types import KIND_MAP, UnknownType +from renaissance.utils.ast_utils import preceding_sibling, next_sibling, match_props, match_children, format_node + +IRRELEVANT_PROPS = {"source_code", "end_point", "start_point", "location", "type"} +IRRELEVANT_NODE = {"comment"} + + +class LSTNode: + def __init__( + self, + node_type: str, + properties: dict[str, Any], + signature: str, + offset: int = 0, + children: list[Self] | None = None, + parent: Self | None = None, + root: Self | None = None, + ): + + self.root = root if root else self + self.parent = parent + self.children = [] if children is None else children + self.properties = properties + if node_type == "string" and signature.startswith("f"): + node_type = "FormattedString" + + self.ast_type = KIND_MAP.get(node_type, UnknownType) + if self.ast_type == UnknownType: + print(f'"{node_type}": {node_type},') + + self.is_implicit = True + self.show_props = False + self.indent = "" + + self.is_statement = node_type == "Expr" + self.referenced_by = [] + self.references = [] + + self.signature = signature + self.text = signature + self.filename = "unknown" + self.length = len(signature) + self.offset = offset + self.end_offset = self.offset + self.length + self.extended_end_offset = self.end_offset + + def __eq__(self, other): + return ( + isinstance(other, type(self)) + and self.ast_type == other.ast_type + and match_props(self.properties, other.properties, IRRELEVANT_PROPS) + and match_children(self.children, other.children, IRRELEVANT_NODE) + ) + + def __hash__(self): + return hash((self.ast_type.__name__, frozenset(self.properties.items()), tuple(self.children))) + + def match_props(self, properties) -> bool: + all_keys = (self.properties.keys() | properties.keys()) - IRRELEVANT_PROPS + return all(self.properties.get(n) == properties.get(n) for n in all_keys) + + def match_children(self, children): + return all(i < len(self.children) and self.children[i] == child for i, child in enumerate(children)) + + def add_child(self, child): # LSTNode): + self.children.append(child) + child.parent = self + + @property + def preceding_sibling(self) -> Self | None: + return preceding_sibling(self) + + @property + def next_sibling(self) -> Self | None: + return next_sibling(self) + + @property + def name(self) -> str: + return self.properties.get("name", "") + + def binary_file_content(self): + src = cast(str, self.properties.get("source_code")) + return src.encode(sys.getfilesystemencoding()) + + @property + def node(self): + return self.ast_type() + + def __repr__(self): + return format_node(self) + # raw_lines = self.signature.splitlines() + # properties_text = "" if not self.show_props else self.properties + # prefix = " " if len(raw_lines) < 2 else f"\n {self.indent}" + # formatted_lines = [f"{prefix}|{line}|" for line in raw_lines] + # return ( + # f"{self.indent}({self.kind}, {self.name}," + # f" {self.filename}[{self.offset}:{self.offset + self.length}])" + # f"{properties_text}:{''.join(formatted_lines)}\n" + # ) + + def is_part_of_translation_unit(self): + return self.root is not None + + +class LST: + def __init__(self, root: LSTNode): + self.root = root diff --git a/src/renaissance/impl/tree_sitter/visualizer.py b/src/renaissance/impl/tree_sitter/visualizer.py new file mode 100644 index 00000000..156376b8 --- /dev/null +++ b/src/renaissance/impl/tree_sitter/visualizer.py @@ -0,0 +1,34 @@ +from renaissance.impl.tree_sitter.lst import LST + +from renaissance.utils.text_utils import signature2id + + +class LstVisualizer: + def __init__(self): + self.lines = ["graph TD"] + self.counter = 0 + self.node_ids = {} + + def _get_node_id(self, node): + if node not in self.node_ids: + self.counter += 1 + self.node_ids[node] = f"n{self.counter}" + return self.node_ids[node] + + def _render_node(self, node): + node_id = self._get_node_id(node) + label = f"""\ + {node_id}: {node.ast_type.__name__} {{ + offset: {node.offset} + signature: {signature2id(node.signature)} + }}""" + label = label.replace("\n", "<br>") + self.lines.append(f'{node_id}["{label}"]') + for child in node.children: + self._render_node(child) + child_id = self._get_node_id(child) + self.lines.append(f"{node_id} --> {child_id}") + + def render(self, lst: LST): + self._render_node(lst.root) + return "\n".join(self.lines) diff --git a/src/renaissance/impl/types.py b/src/renaissance/impl/types.py new file mode 100644 index 00000000..10448ce0 --- /dev/null +++ b/src/renaissance/impl/types.py @@ -0,0 +1,2168 @@ +from abc import ABC + + +class Type(ABC): + def __str__(self): + return self.__class__.__name__ + + +# Fallback +class UnknownType(Type): + pass + + +class BogusType(UnknownType): + pass + + +# Pattern +class Pattern(Type): + pass + + +class MatchOne(Pattern): + pass + + +class MatchAll(Pattern): + pass + + +# Base +class Node(Type): + pass + + +class BaseLeaf(Node): + pass + + +class BaseValueToken(BaseLeaf): + pass + + +class TranslationUnit(Node): + pass + + +class Expression(Node): + pass + + +class Operator(Node): + pass + + +# whitespaces +class Whitespace(Type): + pass + + +class BaseParenthesizableWhitespace(Whitespace): + pass + + +class SimpleWhitespace(BaseParenthesizableWhitespace, BaseValueToken): + pass + + +class Newline(BaseLeaf): + pass + + +class Comment(Whitespace, BaseValueToken): + pass + + +class ParagraphComment(Comment): + pass + + +class TextComment(Comment): + pass + + +class TrailingWhitespace(Whitespace): + pass + + +class FullComment(Comment): + pass + + +class EmptyLine(Whitespace): + pass + + +class ParenthesizedWhitespace(BaseParenthesizableWhitespace): + pass + + +# Operators +class _BaseOneTokenOp(Node): + pass + + +class _BaseTwoTokenOp(Node): + pass + + +class BaseUnaryOp(Node): + pass + + +class BaseBooleanOp(_BaseOneTokenOp): + pass + + +class BaseBinaryOp(Node): + pass + + +class BaseCompOp(Node): + pass + + +class BaseAugOp(Node): + pass + + +class Semicolon(_BaseOneTokenOp): + pass + + +class Colon(_BaseOneTokenOp): + pass + + +class Comma(_BaseOneTokenOp): + pass + + +class Dot(_BaseOneTokenOp): + pass + + +class ImportStar(BaseLeaf): + pass + + +class AssignEqual(_BaseOneTokenOp): + pass + + +class Plus(BaseUnaryOp): + pass + + +class Minus(BaseUnaryOp): + pass + + +class BitInvert(BaseUnaryOp): + pass + + +class Not(BaseUnaryOp): + pass + + +class And(BaseBooleanOp): + pass + + +class Or(BaseBooleanOp): + pass + + +class Add(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class Subtract(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class Multiply(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class Divide(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class FloorDivide(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class Modulo(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class Power(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class LeftShift(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class RightShift(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class BitOr(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class BitAnd(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class BitXor(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class MatrixMultiply(BaseBinaryOp, _BaseOneTokenOp): + pass + + +class LessThan(BaseCompOp, _BaseOneTokenOp): + pass + + +class GreaterThan(BaseCompOp, _BaseOneTokenOp): + pass + + +class Equal(BaseCompOp, _BaseOneTokenOp): + pass + + +class LessThanEqual(BaseCompOp, _BaseOneTokenOp): + pass + + +class GreaterThanEqual(BaseCompOp, _BaseOneTokenOp): + pass + + +class NotEqual(BaseCompOp, _BaseOneTokenOp): + pass + + +class In(BaseCompOp, _BaseOneTokenOp): + pass + + +class NotIn(BaseCompOp, _BaseTwoTokenOp): + pass + + +class Is(BaseCompOp, _BaseOneTokenOp): + pass + + +class IsNot(BaseCompOp, _BaseTwoTokenOp): + pass + + +class AddAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class SubtractAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class MultiplyAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class MatrixMultiplyAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class DivideAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class ModuloAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class BitAndAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class BitOrAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class BitXorAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class LeftShiftAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class RightShiftAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class PowerAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +class FloorDivideAssign(BaseAugOp, _BaseOneTokenOp): + pass + + +# Expression +class LeftSquareBracket(Node): + pass + + +class RightSquareBracket(Node): + pass + + +class LeftCurlyBrace(Node): + pass + + +class RightCurlyBrace(Node): + pass + + +class LeftParen(Node): + pass + + +class RightParen(Node): + pass + + +class Asynchronous(Node): + pass + + +class _BaseParenthesizedNode(Node): + pass + + +# class ExpressionPosition(Enum): pass +class BaseExpression(_BaseParenthesizedNode): + pass + + +class BaseAssignTargetExpression(BaseExpression): + pass + + +class BaseDelTargetExpression(BaseExpression): + pass + + +class Literal(BaseExpression): + pass + + +class Name(BaseAssignTargetExpression, BaseDelTargetExpression): + pass + + +class EllipsisLiteral(BaseExpression): + pass + + +class BaseNumber(BaseExpression): + pass + + +class Integer(BaseNumber): + pass + + +class Float(BaseNumber): + pass + + +class Imaginary(BaseNumber): + pass + + +class BaseString(BaseExpression): + pass + + +class Character(BaseExpression): + pass + + +# StringQuoteLiteral = Literal['"', "'", '"""', "'''"] +class _BasePrefixedString(BaseString): + pass + + +class SimpleString(_BasePrefixedString): + pass + + +class BaseFormattedStringContent(Node): + pass + + +class FormattedStringText(BaseFormattedStringContent): + pass + + +class FormattedStringExpression(BaseFormattedStringContent): + pass + + +class FormattedString(_BasePrefixedString): + pass + + +class BaseTemplatedStringContent(Node): + pass + + +class TemplatedStringText(BaseTemplatedStringContent): + pass + + +class TemplatedStringExpression(BaseTemplatedStringContent): + pass + + +class TemplatedString(_BasePrefixedString): + pass + + +class ConcatenatedString(BaseString): + pass + + +class ComparisonTarget(Node): + pass + + +class Comparison(BaseExpression): + pass + + +class UnaryOperation(BaseExpression): + pass + + +class BinaryOperation(BaseExpression): + pass + + +class BooleanOperation(BaseExpression): + pass + + +class Attribute(BaseAssignTargetExpression, BaseDelTargetExpression): + pass + + +class BaseSlice(Node): + pass + + +class Index(BaseSlice): + pass + + +class Slice(BaseSlice): + pass + + +class SubscriptElement(Node): + pass + + +class Subscript(BaseAssignTargetExpression, BaseDelTargetExpression): + pass + + +class Annotation(Node): + pass + + +class ParamStar(Node): + pass + + +class ParamSlash(Node): + pass + + +class Param(Node): + pass + + +class Parameters(Node): + pass + + +class Lambda(BaseExpression): + pass + + +class Arg(Node): + pass + + +class _BaseExpressionWithArgs(BaseExpression): + pass + + +class Call(_BaseExpressionWithArgs): + pass + + +class Await(BaseExpression): + pass + + +class IfExp(BaseExpression): + pass + + +class From(Node): + pass + + +class Yield(BaseExpression): + pass + + +class _BaseElementImpl(Node): + pass + + +class BaseElement(_BaseElementImpl): + pass + + +class BaseDictElement(_BaseElementImpl): + pass + + +class Element(BaseElement): + pass + + +class DictElement(BaseDictElement): + pass + + +class StarredElement(BaseElement, BaseExpression, _BaseParenthesizedNode): + pass + + +class StarredDictElement(BaseDictElement): + pass + + +class Tuple(BaseAssignTargetExpression, BaseDelTargetExpression): + pass + + +class BaseList(BaseExpression): + pass + + +class List(BaseList, BaseAssignTargetExpression, BaseDelTargetExpression): + pass + + +class _BaseSetOrDict(BaseExpression): + pass + + +class BaseSet(_BaseSetOrDict): + pass + + +class Set(BaseSet): + pass + + +class BaseDict(_BaseSetOrDict): + pass + + +class Dict(BaseDict): + pass + + +class CompFor(Node): + pass + + +class CompIf(Node): + pass + + +class BaseComp(BaseExpression): + pass + + +class BaseSimpleComp(BaseComp): + pass + + +class GeneratorExp(BaseSimpleComp): + pass + + +class ListComp(BaseList, BaseSimpleComp): + pass + + +class SetComp(BaseSet, BaseSimpleComp): + pass + + +class DictComp(BaseDict, BaseComp): + pass + + +class NamedExpr(BaseExpression): + pass + + +# Statement +class Statement(Node): + pass + + +class BaseSuite(Statement): + pass + + +class BaseStatement(Statement): + pass + + +class BaseSmallStatement(Statement): + pass + + +class Del(BaseSmallStatement): + pass + + +class Pass(BaseSmallStatement): + pass + + +class Break(BaseSmallStatement): + pass + + +class Continue(BaseSmallStatement): + pass + + +class Return(BaseSmallStatement): + pass + + +class ExpressionStatement(BaseSmallStatement): + pass + + +class _BaseSimpleStatement(Node): + pass + + +class SimpleStatementLine(_BaseSimpleStatement, BaseStatement): + pass + + +class SimpleStatementSuite(_BaseSimpleStatement, BaseSuite): + pass + + +class Else(Node): + pass + + +class BaseCompoundStatement(BaseStatement): + pass + + +class If(BaseCompoundStatement): + pass + + +class CompoundStatement(BaseSuite): + pass + + +class IndentedBlock(BaseSuite): + pass + + +class AsName(Node): + pass + + +class ExceptHandler(Node): + pass + + +class ExceptStarHandler(Node): + pass + + +class Catch(Node): + pass + + +class Finally(Node): + pass + + +class Try(BaseCompoundStatement): + pass + + +class TryStar(BaseCompoundStatement): + pass + + +class ImportStatement(BaseSmallStatement): + pass + + +class ImportAlias(Node): + pass + + +class Import(ImportStatement): + pass + + +class ImportFrom(ImportStatement): + pass + + +class InclusionDirective(ImportStatement): + pass + + +class IncludeDirective(ImportStatement): + pass + + +class AssignTarget(Node): + pass + + +class Assign(BaseSmallStatement): + pass + + +class AnnAssign(BaseSmallStatement): + pass + + +class AugAssign(BaseSmallStatement): + pass + + +class Decorator(Node): + pass + + +class Declaration(BaseSmallStatement): + pass + + +class Definition(BaseCompoundStatement): + pass + + +class DefinitionX(CompoundStatement): + pass + + +class FunctionDef(Definition): + pass + + +class ClassDef(Definition): + pass + + +class StructDef(Definition): + pass + + +class RecordDef(Definition): + pass + + +class VariableDef(Definition): + pass + + +class FieldDef(Definition): + pass + + +class InterfaceDef(Definition): + pass + + +class LocalVariableDef(Definition): + pass + + +class TemplateDef(Definition): + pass + + +class TypeParameterDef(Definition): + pass + + +class TypeAlias(Definition): + pass + + +class TypedefDef(TypeAlias): + pass + + +class PackageDef(Definition): + pass + + +class ParameterDef(Definition): + pass + + +class UnionDef(Definition): + pass + + +class WithItem(Node): + pass + + +class With(BaseCompoundStatement): + pass + + +class Do(BaseCompoundStatement): + pass + + +class For(BaseCompoundStatement): + pass + + +class While(BaseCompoundStatement): + pass + + +class Raise(BaseSmallStatement): + pass + + +class Assert(BaseSmallStatement): + pass + + +class NameItem(Node): + pass + + +class Global(BaseSmallStatement): + pass + + +class Nonlocal(BaseSmallStatement): + pass + + +class MatchPattern(_BaseParenthesizedNode): + pass + + +class Match(BaseCompoundStatement): + pass + + +class MatchCase(Node): + pass + + +class MatchValue(MatchPattern): + pass + + +class MatchSingleton(MatchPattern): + pass + + +class MatchSequenceElement(Node): + pass + + +class MatchStar(Node): + pass + + +class MatchSequence(MatchPattern): + pass + + +class MatchList(MatchSequence): + pass + + +class MatchTuple(MatchSequence): + pass + + +class MatchMappingElement(Node): + pass + + +class MatchMapping(MatchPattern): + pass + + +class MatchKeywordElement(Node): + pass + + +class MatchClass(MatchPattern): + pass + + +class MatchAs(MatchPattern): + pass + + +class MatchOrElement(Node): + pass + + +class MatchOr(MatchPattern): + pass + + +class TypeVar(Node): + pass + + +class TypeVarTuple(Node): + pass + + +class ParamSpec(Node): + pass + + +class TypeParam(Node): + pass + + +class TypeParameters(Node): + pass + + +# ==== added===== + + +class ImplicitNode(Node): + pass + + +# Specifier +class Specifier(Node): + pass + + +class Auto(Specifier): + pass + + +class BaseSpecifier(Specifier): + pass + + +class ClassSpecifier(Specifier): + pass + + +class AccessSpecifier(Specifier): + pass + + +class EnumSpecifier(Specifier): + pass + + +class StructSpecifier(Specifier): + pass + + +# Reference +class Reference(Expression): + pass + + +class TypeReference(Reference): + pass + + +class MemberRefence(Reference): + pass + + +class NamespaceReference(Reference): + pass + + +class OverloadedDeclRef(Reference): + pass + + +class TemplateRef(Reference): + pass + + +# Attributes +class AlignedAttribute: + pass + + +class AsmAttribute: + pass + + +class ConstAttr: + pass + + +class VisibilityAttr: + pass + + +class WarnUnusedResultAttr: + pass + + +class FinalAttr: + pass + + +class OverrideAttr: + pass + + +class PureAttr: + pass + + +class UnexposedAttr: + pass + + +class DeclarationExpression(Expression): + pass + + +class ParenthesizedExpression(Expression): + pass + + +class Constructor(FunctionDef): + pass + + +class MacroDef(Definition): + pass + + +class Namespace(Node): + pass + + +class ConstructorExpression(Call): + pass + + +class ArgumentList(Node): + pass + + +class Compare(Node): + pass + + +class Keyword(Node): + pass + + +class Arguments(Node): + pass + + +class Error(Node): + pass + + +class CatchClause(Node): + pass + + +class Alias(Node): + pass + + +class Symbol(Node): + pass + + +class AssignTo(Symbol): + pass + + +class Cast(Node): + pass + + +class BuiltinType(Literal): + pass + + +class DeclarationLoc(Declaration): + pass + + +class Delete(Expression): + pass + + +class Starred(Literal): + pass + + +class Constant(Literal): + pass + + +class Number(Literal): + pass + + +class String(Literal): + pass + + +class Catch(Statement): + pass + + +class ComparisionOperation(Expression): + pass + + +class UnaryAdd(UnaryOperation): + pass + + +class UnarySubtract(UnaryOperation): + pass + + +class Case(Statement): + pass + + +class MatchSequence(Node): + pass + + +# other +class ConstructorDef(Definition): + pass + + +class FriendDecl: + pass + + +class AbstractFunctionDeclarator: + pass + + +class As: + pass + + +class AsPattern: + pass + + +class AsPatternTarget: + pass + + +class Asterisk: + pass + + +class Async: + pass + + +class Backslash: + pass + + +class CatchFormalParameter: + pass + + +class CatchType: + pass + + +class ClassBody: + pass + + +class ClassPattern: + pass + + +class ClassTemplate: + pass + + +class ClassTemplatePartial: + pass + + +class Comprehension: + pass + + +class ConditionalOperator: + pass + + +class ConstCastExpr: + pass + + +class ConstructorBody: + pass + + +class ConversionFunction: + pass + + +class BooleanLiteral: + pass + + +class FunctionalCast: + pass + + +class NullPointer: + pass + + +class This: + pass + + +class Typeid: + pass + + +class DeclarationList: + pass + + +class DefaultStmt: + pass + + +class Destructor: + pass + + +class DictPattern: + pass + + +class Dimensions: + pass + + +class DottedName: + pass + + +class DynamicCastExpr: + pass + + +class Enum: + pass + + +class EnumBody: + pass + + +class EnumConstant: + pass + + +class EnumeratorList: + pass + + +class ExceptClause: + pass + + +class Extends: + pass + + +class FieldAccess: + pass + + +class FieldIdentifier: + pass + + +class FinallyClause: + pass + + +class FormalParameter: + pass + + +class FormalParameters: + pass + + +class FunctionTemplate: + pass + + +class IntegralType: + pass + + +class Interface: + pass + + +class InterfaceBody: + pass + + +class Interpolation: + pass + + +class LambdaParameters: + pass + + +class LinkageSpec: + pass + + +class ListPattern: + pass + + +class MarkerAnnotation: + pass + + +class Method: + pass + + +class Modifiers: + pass + + +class NamespaceIdentifier: + pass + + +class New: + pass + + +class Null: + pass + + +class ObjectCreationExpression: + pass + + +class PackExpansionExpr: + pass + + +class Package: + pass + + +class Pair: + pass + + +class PointerDeclarator: + pass + + +class Program: + pass + + +class Public: + pass + + +class QualifiedIdentifier: + pass + + +class ReinterpretCastExpr: + pass + + +class ScopedIdentifier: + pass + + +class SizeOfPackExpr: + pass + + +class SplatPattern: + pass + + +class Static: + pass + + +class StaticAssert: + pass + + +class StaticCastExpr: + pass + + +class StringFragment: + pass + + +class StringLiteral: + pass + + +class Superclass: + pass + + +class Switch(Match): + pass + + +class SwitchBlock(CompoundStatement): + pass + + +class SwitchBlockStatementGroup: + pass + + +class SwitchExpression: + pass + + +class SwitchLabel(MatchPattern): + pass + + +class Symbol: + pass + + +class SystemLibString: + pass + + +class TemplateNonTypeParameter: + pass + + +class TemplateParameterList: + pass + + +class TemplateTypeParameter: + pass + + +class TypeAliasTemplateDecl: + pass + + +class TypeName: + pass + + +class Underscore: + pass + + +class UnexposedStmt: + pass + + +class UnionPattern: + pass + + +class UpdateExpression: + pass + + +class Using: + pass + + +class VoidType: + pass + + +OPERATOR_MAP = { + "AnnAssign": "=", + "Assert": "assert", + "Assign": "=", + "AsyncFor": "for", + "AsyncFunctionDef": "function", + "AsyncWith": "with", + "AugAssignAdd": "+=", + "Break": "break", + "Call": "def", + "ClassDef": "class", + "Continue": "continue", + "For": "for", + "FunctionDef": "function", + "If": "if", + "Import": "import", + "ImportFrom": "import", + "Match": "match", + "Pass": "pass", + "Try": "try", + "TryStar": "try", + "While": "while", + "With": "with", +} + +KIND_MAP = { + "!": Not, + "!=": NotEqual, + "#include": IncludeDirective, + "%": Modulo, + "&": BitAnd, + "&&": And, + "'": Symbol, + "(": Tuple, + ")": Tuple, + "*": Multiply, + "**": Power, + "+": Add, + "++": UnaryAdd, + "+=": BogusType, + ",": Symbol, + "-": Subtract, + ".": Symbol, + "...": Symbol, + "/": Divide, + "//": FloorDivide, + ":": Colon, + "::": Symbol, + ";": Symbol, + "<": LessThan, + "<<": LeftShift, + "<=": LessThanEqual, + "=": AssignEqual, + "==": Equal, + ">": GreaterThan, + ">=": GreaterThanEqual, + ">>": RightShift, + "@": Symbol, + "[": ListComp, + "\\": Backslash, + "]": ListComp, + "^": BitXor, + "_": Underscore, + "_MatchAll__": MatchAll, + "_MatchOne__": MatchOne, + "abstract_function_declarator": AbstractFunctionDeclarator, + "AccessSpecDecl": AccessSpecifier, + "Add": Add, + "AddAssign": AddAssign, + "alias": TypeAlias, + "ALIGNED_ATTR": AlignedAttribute, + "and": And, + "And": And, + "AnnAssign": Assign, + "Annotation": Annotation, + "Arg": Arg, + "arg": Arg, + "argument_list": ArgumentList, + "arguments": Arguments, + "ARRAY_SUBSCRIPT_EXPR": Subscript, + "array_type": List, + "as": As, + "as_pattern": AsPattern, + "as_pattern_target": AsPatternTarget, + "ASM_LABEL_ATTR": AsmAttribute, + "AsName": AsName, + "assert": Assert, + "Assert": Assert, + "assert_statement": Assert, + "Assign": Assign, + "AssignEqual": AssignEqual, + "assignment": Assign, + "assignment_expression": Assign, + "AssignTarget": AssignTarget, + "asterisk": Asterisk, + "async": Async, + "AsyncFor": For, + "AsyncFunctionDef": FunctionDef, + "Asynchronous": Asynchronous, + "AsyncWith": With, + "attribute": Attribute, + "Attribute": Attribute, + "AugAssign": AugAssign, + "augmented_assignment": AugAssign, + "auto": Auto, + "Await": Await, + "await": Await, + "binary_expression": BinaryOperation, + "binary_operator": BinaryOperation, + "BINARY_OPERATOR": BinaryOperation, + "BinaryOperation": BinaryOperation, + "BinaryOperator": BinaryOperation, + "BinOp": BinaryOperation, + "BitAnd": BitAnd, + "BitInvert": BitInvert, + "BitOr": BitOr, + "BitXor": BitXor, + "block": CompoundStatement, + "boolean_operator": BooleanOperation, + "BooleanOperation": BooleanOperation, + "BoolOp": BooleanOperation, + "break": Break, + "Break": Break, + "break_statement": Break, + "BREAK_STMT": Break, + "BuiltinType": BuiltinType, + "Call": Call, + "call": Call, + "CALL_EXPR": Call, + "call_expression": Call, + "CallExpr": Call, + "case": MatchCase, + "case_clause": MatchCase, + "case_pattern": MatchSingleton, + "case_statement": MatchCase, + "CASE_STMT": MatchCase, + "catch": ExceptHandler, + "catch_clause": ExceptHandler, + "catch_formal_parameter": CatchFormalParameter, + "catch_type": CatchType, + "char_literal": Character, + "character": Character, + "CHARACTER_LITERAL": Character, + "class": ClassDef, + "class_body": ClassBody, + "CLASS_DECL": ClassDef, + "class_declaration": ClassDef, + "class_definition": ClassDef, + "class_pattern": ClassPattern, + "class_specifier": ClassSpecifier, + "CLASS_TEMPLATE": ClassTemplate, + "CLASS_TEMPLATE_PARTIAL_SPECIALIZATION": ClassTemplatePartial, + "ClassDef": ClassDef, + "Colon": Colon, + "Comma": Comma, + "Comment": Comment, + "comment": Comment, + "Compare": Compare, + "Comparison": Comparison, + "comparison_operator": Comparison, + "ComparisonTarget": ComparisonTarget, + "CompFor": CompFor, + "COMPOUND_ASSIGNMENT_OPERATOR": Assign, + "compound_statement": CompoundStatement, + "COMPOUND_STMT": CompoundStatement, + "CompoundAssignOperator": Assign, + "CompoundStmt": CompoundStatement, + "comprehension": Comprehension, + "condition_clause": Compare, + "conditional_expression": IfExp, + "CONDITIONAL_OPERATOR": ConditionalOperator, + "CONST_ATTR": ConstAttr, + "Constant": Literal, + "CONSTRUCTOR": Constructor, + "constructor_body": ConstructorBody, + "constructor_declaration": ConstructorDef, + "continue": Continue, + "Continue": Continue, + "continue_statement": Continue, + "CONTINUE_STMT": Continue, + "CONVERSION_FUNCTION": ConversionFunction, + "CSTYLE_CAST_EXPR": Cast, + "CStyleCastExpr": Cast, + "CXX_ACCESS_SPEC_DECL": AccessSpecifier, + "CXX_BASE_SPECIFIER": BaseSpecifier, + "CXX_BOOL_LITERAL_EXPR": BooleanLiteral, + "CXX_CATCH_STMT": ExceptHandler, + "CXX_CONST_CAST_EXPR": ConstCastExpr, + "CXX_DELETE_EXPR": Del, + "CXX_DYNAMIC_CAST_EXPR": DynamicCastExpr, + "CXX_FINAL_ATTR": FinalAttr, + "CXX_FOR_RANGE_STMT": For, + "CXX_FUNCTIONAL_CAST_EXPR": FunctionalCast, + "CXX_METHOD": Method, + "CXX_NEW_EXPR": New, + "CXX_NULL_PTR_LITERAL_EXPR": NullPointer, + "CXX_OVERRIDE_ATTR": OverrideAttr, + "CXX_REINTERPRET_CAST_EXPR": ReinterpretCastExpr, + "CXX_STATIC_CAST_EXPR": StaticCastExpr, + "CXX_THIS_EXPR": This, + "CXX_THROW_EXPR": Raise, + "CXX_TRY_STMT": Try, + "CXX_TYPEID_EXPR": Typeid, + "CXX_UNARY_EXPR": UnaryOperation, + "CXXConstructExpr": ConstructorExpression, + "CXXConstructorDecl": Constructor, + "CXXRecordDecl": RecordDef, + "decimal_integer_literal": Integer, + "DECL_LOC": DeclarationLoc, + "DECL_REF_EXPR": DeclarationExpression, + "DECL_STMT": Declaration, + "declaration": Declaration, + "declaration_list": DeclarationList, + "DeclLoc": DeclarationLoc, + "DeclRefExpr": DeclarationExpression, + "DeclStmt": Declaration, + "Decorator": Decorator, + "def": Symbol, + "DEFAULT_STMT": DefaultStmt, + "Del": Del, + "del": Del, + "Delete": Del, + "delete_statement": Del, + "DESTRUCTOR": Destructor, + "Dict": Dict, + "dict_pattern": DictPattern, + "DictComp": DictComp, + "DictElement": DictElement, + "dictionary": Dict, + "dictionary_comprehension": DictComp, + "dimensions": Dimensions, + "Div": Divide, + "Divide": Divide, + "do": Do, + "do_statement": Do, + "DO_STMT": Do, + "DoStmt": Do, + "Dot": Dot, + "dotted_name": DottedName, + "Element": Element, + "ellipsis": EllipsisLiteral, + "else": Else, + "EmptyLine": EmptyLine, + "enum": Enum, + "enum_body": EnumBody, + "enum_constant": EnumConstant, + "ENUM_CONSTANT_DECL": EnumConstant, + "ENUM_DECL": Enum, + "enum_declaration": Enum, + "enum_specifier": EnumSpecifier, + "enumerator": Enum, + "enumerator_list": EnumeratorList, + "Eq": Equal, + "Equal": Equal, + "ERROR": Error, + "except": Catch, + "except_clause": ExceptClause, + "ExceptHandler": Catch, + "ExceptStarHandler": ExceptStarHandler, + "Expr": ExpressionStatement, + "expression_statement": ExpressionStatement, + "extends": Extends, + "field_access": FieldAccess, + "FIELD_DECL": FieldDef, + "field_declaration": FieldDef, + "field_declaration_list": Arguments, + "field_identifier": FieldIdentifier, + "FieldDecl": FieldDef, + "Finally": Finally, + "finally": Finally, + "finally_clause": FinallyClause, + "float": float, + "FLOATING_LITERAL": Float, + "FloorDiv": FloorDivide, + "FloorDivide": FloorDivide, + "For": For, + "for": For, + "for_in_clause": For, + "for_statement": For, + "FOR_STMT": For, + "formal_parameter": FormalParameter, + "formal_parameters": FormalParameters, + "FormattedString": FormattedString, + "FormattedStringExpression": FormattedStringExpression, + "FormattedStringText": FormattedStringText, + "FormattedValue": FormattedString, + "FRIEND_DECL": FriendDecl, + "From": From, + "from": From, + "FullComment": FullComment, + "FUNCTION_DECL": FunctionDef, + "function_declarator": FunctionDef, + "function_definition": FunctionDef, + "FUNCTION_TEMPLATE": FunctionTemplate, + "FunctionDecl": FunctionDef, + "FunctionDef": FunctionDef, + "generator_expression": GeneratorExp, + "GeneratorExp": GeneratorExp, + "Global": Global, + "global": Global, + "global_statement": Global, + "Greater": GreaterThan, + "GreaterEqual": GreaterThanEqual, + "GreaterThan": GreaterThan, + "GreaterThanEqual": GreaterThanEqual, + "Gt": GreaterThan, + "GtE": GreaterThanEqual, + "identifier": Name, + "If": If, + "if": If, + "if_clause": IfExp, + "if_statement": If, + "IF_STMT": If, + "IfExp": IfExp, + "IfStmt": If, + "ImplicitNode": ImplicitNode, + "ImplicitValueInitExpr": Assign, + "import": Import, + "Import": Import, + "import_declaration": Import, + "import_from_statement": ImportFrom, + "import_statement": Import, + "ImportAlias": ImportAlias, + "ImportFrom": ImportFrom, + "In": In, + "in": In, + "INCLUSION_DIRECTIVE": InclusionDirective, + "InclusionDirective": InclusionDirective, + "IndentedBlock": IndentedBlock, + "init_declarator": Assign, + "INIT_LIST_EXPR": ListComp, + "InitListExpr": ListComp, + "int": int, + "Integer": Number, + "integer": Number, + "INTEGER_LITERAL": Number, + "IntegerLiteral": Number, + "integral_type": IntegralType, + "interface": Interface, + "interface_body": InterfaceBody, + "interface_declaration": InterfaceDef, + "interpolation": Interpolation, + "Invert": BitInvert, + "is not": IsNot, + "Is": Is, + "is": Is, + "IsNot": IsNot, + "JoinedStr": FormattedString, + "keyword": Keyword, + "keyword_pattern": Keyword, + "Lambda": Lambda, + "lambda": Lambda, + "lambda_capture_specifier": LambdaParameters, + "LAMBDA_EXPR": Lambda, + "lambda_expression": Lambda, + "lambda_parameters": LambdaParameters, + "LeftCurlyBrace": LeftCurlyBrace, + "LeftParen": LeftParen, + "LeftShift": LeftShift, + "LeftSquareBracket": ListComp, + "LessThan": LessThan, + "LessThanEqual": LessThanEqual, + "LINKAGE_SPEC": LinkageSpec, + "List": List, + "list": List, + "list_comprehension": ListComp, + "list_pattern": ListPattern, + "ListComp": ListComp, + "local_variable_declaration": LocalVariableDef, + "LShift": LeftShift, + "Lt": LessThan, + "LtE": LessThanEqual, + "MACRO_DEFINITION": MacroDef, + "marker_annotation": MarkerAnnotation, + "match": Match, + "Match": Match, + "match_case": MatchCase, + "match_statement": Match, + "MatchAll": MatchAll, + "MatchAs": MatchAs, + "MatchCase": MatchCase, + "MatchClass": MatchClass, + "MatchKeywordElement": MatchKeywordElement, + "MatchList": MatchSequence, + "MatchMapping": MatchMapping, + "MatchMappingElement": MatchMappingElement, + "MatchOne": MatchOne, + "MatchOr": MatchOr, + "MatchOrElement": MatchOrElement, + "MatchSequence": MatchSequence, + "MatchSequenceElement": MatchSequenceElement, + "MatchSingleton": MatchSingleton, + "MatchStar": MatchStar, + "MatchValue": MatchValue, + "MEMBER_REF": MemberRefence, + "MEMBER_REF_EXPR": MemberRefence, + "method_declaration": FunctionDef, + "method_invocation": Call, + "Minus": UnarySubtract, + "MinusOperator": UnarySubtract, + "Mod": Modulo, + "modifiers": Modifiers, + "Module": TranslationUnit, + "module": TranslationUnit, + "Modulo": Modulo, + "Mult": Multiply, + "Multiply": Multiply, + "Name": Name, + "NamedExpr": NamedExpr, + "NameItem": NameItem, + "namespace": Namespace, + "NAMESPACE": Namespace, + "namespace_definition": Namespace, + "namespace_identifier": NamespaceIdentifier, + "NAMESPACE_REF": NamespaceReference, + "NamespaceDecl": Namespace, + "new": New, + "Newline": Newline, + "none": BogusType, + "nonlocal": Nonlocal, + "Nonlocal": Nonlocal, + "nonlocal_statement": Nonlocal, + "not in": NotIn, + "Not": Not, + "not": Not, + "not_operator": UnaryOperation, + "NotEq": NotEqual, + "NotEqual": NotEqual, + "NotIn": NotIn, + "null": Null, + "NULL_STMT": Null, + "nullptr": Null, + "number_literal": Number, + "object_creation_expression": ObjectCreationExpression, + "OVERLOADED_DECL_REF": OverloadedDeclRef, + "PACK_EXPANSION_EXPR": PackExpansionExpr, + "package": Package, + "package_declaration": PackageDef, + "pair": Pair, + "ParagraphComment": ParagraphComment, + "Param": Param, + "parameter_declaration": ParameterDef, + "parameter_list": ArgumentList, + "Parameters": Parameters, + "parameters": Parameters, + "PAREN_EXPR": ParenthesizedExpression, + "ParenExpr": ParenthesizedExpression, + "parenthesized_expression": ParenthesizedExpression, + "ParenthesizedWhitespace": ParenthesizedWhitespace, + "PARM_DECL": ParameterDef, + "ParmVarDecl": ParameterDef, + "pass": Pass, + "Pass": Pass, + "pass_statement": Pass, + "Plus": UnaryAdd, + "PlusOperator": UnaryAdd, + "pointer_declarator": PointerDeclarator, + "Pow": Power, + "Power": Power, + "primitive_type": BuiltinType, + "program": Program, + "public": Public, + "PURE_ATTR": PureAttr, + "qualified_identifier": QualifiedIdentifier, + "raise": Raise, + "Raise": Raise, + "raise_statement": Raise, + "RecordDecl": RecordDef, + "return": Return, + "Return": Return, + "return_statement": Return, + "RETURN_STMT": Return, + "ReturnStmt": Return, + "RightCurlyBrace": RightCurlyBrace, + "RightParen": RightParen, + "RightShift": RightShift, + "RightSquareBracket": ListComp, + "RShift": RightShift, + "scoped_identifier": ScopedIdentifier, + "Set": Set, + "set": Set, + "set_comprehension": SetComp, + "SetComp": SetComp, + "SimpleStatementLine": Statement, + "SimpleStatementSuite": SimpleStatementSuite, + "SimpleString": SimpleString, + "SimpleWhitespace": Whitespace, + "SIZE_OF_PACK_EXPR": SizeOfPackExpr, + "slice": slice, + "Slice": Slice, + "splat_pattern": SplatPattern, + "Starred": Starred, + "static": Static, + "STATIC_ASSERT": StaticAssert, + "str": str, + "string": Literal, + "string_content": Literal, + "string_end": Literal, + "string_fragment": StringFragment, + "STRING_LITERAL": FormattedString, + "string_literal": StringLiteral, + "string_start": Literal, + "StringLiteral": String, + "struct": StructDef, + "STRUCT_DECL": StructDef, + "struct_specifier": StructSpecifier, + "Sub": Subtract, + "Subscript": Subscript, + "subscript": Subscript, + "SubscriptElement": SubscriptElement, + "Subtract": Subtract, + "superclass": Superclass, + "switch": Switch, + "switch_block": SwitchBlock, + "switch_block_statement_group": SwitchBlockStatementGroup, + "switch_expression": SwitchExpression, + "switch_label": SwitchLabel, + "switch_statement": Switch, + "SWITCH_STMT": Switch, + "system_lib_string": SystemLibString, + "template": TemplateDef, + "template_declaration": TemplateDef, + "TEMPLATE_NON_TYPE_PARAMETER": TemplateNonTypeParameter, + "template_parameter_list": TemplateParameterList, + "TEMPLATE_REF": TemplateRef, + "TEMPLATE_TEMPLATE_PARAMETER": TemplateParameterList, + "TEMPLATE_TYPE_PARAMETER": TemplateTypeParameter, + "TextComment": TextComment, + "TrailingWhitespace": TrailingWhitespace, + "translation_unit": TranslationUnit, + "TRANSLATION_UNIT": TranslationUnit, + "TranslationUnit": TranslationUnit, + "TranslationUnitDecl": TranslationUnit, + "Try": Try, + "try": Try, + "try_statement": Try, + "TryStar": Try, + "Tuple": Tuple, + "tuple": Tuple, + "type_alias_statement": TypeAlias, + "TYPE_ALIAS_DECL": TypeAlias, + "TYPE_ALIAS_TEMPLATE_DECL": TypeAliasTemplateDecl, + "type_identifier": TypeReference, + "type_parameter_declaration": TypeParameterDef, + "TYPE_REF": TypeReference, + "TypeAlias": TypeAlias, + "TYPEDEF_DECL": TypedefDef, + "TypedefDecl": TypedefDef, + "typename": TypeName, + "TypeRef": TypeReference, + "UAdd": UnaryAdd, + "unary_expression": UnaryOperation, + "unary_operator": UnaryOperation, + "UNARY_OPERATOR": UnaryOperation, + "UnaryOp": UnaryOperation, + "UnaryOperation": UnaryOperation, + "UnaryOperator": UnaryOperation, + "UNEXPOSED_ATTR": UnexposedAttr, + "UNEXPOSED_DECL": Declaration, + "UNEXPOSED_EXPR": Expression, + "UNEXPOSED_STMT": UnexposedStmt, + "UNION_DECL": UnionDef, + "union_pattern": UnionPattern, + "update_expression": UpdateExpression, + "using": Using, + "USING_DIRECTIVE": Using, + "USub": UnarySubtract, + "VAR_DECL": VariableDef, + "VarDecl": VariableDef, + "variable_declarator": VariableDef, + "VISIBILITY_ATTR": VisibilityAttr, + "void_type": VoidType, + "WARN_UNUSED_RESULT_ATTR": WarnUnusedResultAttr, + "While": While, + "while": While, + "while_statement": While, + "WHILE_STMT": While, + "WhileStmt": While, + "with": With, + "With": With, + "with_clause": With, + "with_item": WithItem, + "with_statement": With, + "WithItem": WithItem, + "withitem": WithItem, + "Yield": Yield, + "yield": Yield, + "YieldFrom": Yield, + "{": Dict, + "|": BitOr, + "||": Or, + "}": Dict, + "~": BitInvert, + '"': Symbol, + None: BogusType, +} diff --git a/src/renaissance/project/__init__.py b/src/renaissance/project/__init__.py new file mode 100644 index 00000000..df340ed2 --- /dev/null +++ b/src/renaissance/project/__init__.py @@ -0,0 +1,3 @@ +""" +project scanner collect the source files in a repo given a correct directory structure according standard +""" diff --git a/src/renaissance/project/project_scanner.py b/src/renaissance/project/project_scanner.py new file mode 100644 index 00000000..816d2dfb --- /dev/null +++ b/src/renaissance/project/project_scanner.py @@ -0,0 +1,65 @@ +from os import path, system +import json +import glob +from pathlib import Path + + +class ProjectScanner: + def find_sources(self) -> list[str]: + raise NotImplementedError + + +class CppScanner(ProjectScanner): + def __init__(self, compile_commands_path: str = "compile_commands.json"): + self.compile_commands_path = compile_commands_path + + def find_sources(self) -> list[str]: + if not path.exists(self.compile_commands_path): + raise FileNotFoundError("compile_commands.json not found") + with open(self.compile_commands_path) as f: + commands = json.load(f) + return sorted(set(entry["file"] for entry in commands if "file" in entry)) + + +class JavaScanner(ProjectScanner): + def __init__(self, root_dir: str = "."): + self.root_dir = root_dir + + def find_sources(self) -> list[str]: + java_files = glob.glob(f"{self.root_dir}/**/*.java", recursive=True) + return sorted(java_files) + + +class PythonScanner(ProjectScanner): + def __init__(self, root_dir: str = ".", package_dirs: list[str] | None = None): + + # return (file_path for file_path in current_dir.iterdir() if is_python_file) + + self.root_dir = root_dir + self.package_dirs = package_dirs or ["src", "lib", "test"] + + def find_sources(self) -> list[str]: + files = [] + + for d in self.package_dirs: + file_path = Path(self.root_dir) / d + if file_path.exists(): + files.extend(file_path.glob("**/*.py")) + return sorted(files) + + +class BearCppScanner(CppScanner): + def __init__(self, build_dir: str = ".", compile_commands_path: str = "compile_commands.json"): + super().__init__(compile_commands_path) + self.build_dir = build_dir + + def run_bear(self): + print("Running Bear to generate compile_commands.json...") + result = system(f"bear -- make -C {self.build_dir}") + if result != 0: + raise RuntimeError("Bear failed to run or make failed.") + + def find_sources(self) -> list[str]: + if not path.exists(self.compile_commands_path): + self.run_bear() + return super().find_sources() diff --git a/src/renaissance/refactoring/__init__.py b/src/renaissance/refactoring/__init__.py new file mode 100644 index 00000000..4a819755 --- /dev/null +++ b/src/renaissance/refactoring/__init__.py @@ -0,0 +1,3 @@ +from .cleanup_refactoring import CleanupRefactoring + +__all__ = ["CleanupRefactoring"] diff --git a/src/renaissance/refactoring/cleanup_refactoring.py b/src/renaissance/refactoring/cleanup_refactoring.py new file mode 100644 index 00000000..c25d5973 --- /dev/null +++ b/src/renaissance/refactoring/cleanup_refactoring.py @@ -0,0 +1,18 @@ +from more_itertools import flatten + +from renaissance.impl.types import VariableDef, CompoundStatement +from renaissance.syntax_tree import ASTProcessor +from renaissance.syntax_tree.ast_finder import find_ast_type + + +class CleanupRefactoring: + def __init__(self): + raise Exception("This class should not be instantiated") + + @staticmethod + def remove_unused_variables(ast_refactor: ASTProcessor) -> None: + """ + Removes all unused variables from a function + """ + refs = flatten(find_ast_type(n, VariableDef) for n in find_ast_type(ast_refactor.node, CompoundStatement)) + [ast_refactor.remove(ref.parent, True, True) for ref in refs if len(ref.referenced_by) == 0] diff --git a/src/renaissance/refactoring/python_refactoring.py b/src/renaissance/refactoring/python_refactoring.py new file mode 100644 index 00000000..c7ef4feb --- /dev/null +++ b/src/renaissance/refactoring/python_refactoring.py @@ -0,0 +1,55 @@ +import importlib +from pathlib import Path +from typing import Sequence, cast + +from termcolor import colored + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonPatternFactory, PythonFactory +from renaissance.impl.python.util import to_str +from renaissance.syntax_tree import ASTProcessor +from renaissance.syntax_tree.match_finder import match_pattern +from renaissance.utils.text_utils import snake_case + + +class PythonRefactoring(ASTProcessor): + + def __init__(self, file): + factory = PythonFactory(PythonRstNode) + atu = factory.create(file) + super().__init__(atu, factory, False) + self.pattern_factory = PythonPatternFactory(self.factory) + self.black_list_pattern = ".git" + self.white_list_pattern = "" + + def replace_stmt(self, find, repl): + pattern = self.pattern_factory.create_statements(find) + for match in match_pattern(self.root.children, pattern): + replacement = repl + for exp in match.expansions: + arg_str = ", ".join([to_str(node) for node in match.expansions[exp]]) + replacement = replacement.replace(exp, arg_str) + + replacement = replacement.replace(" ,)", ")").replace(", )", ")") + self.replace(replacement, match.nodes, False, False) + + @staticmethod + def process(class_name, file): + """Return a subclass by name using importlib, like Java's Class.forName().""" + snake = snake_case(class_name) + module = importlib.import_module(f"renaissance.refactoring.{snake}") + cls = getattr(module, class_name) + refactor = cls(file) + if refactor.black_list_pattern in refactor.filename or refactor.white_list_pattern not in refactor.filename: + print(f"skipping: {Path(refactor.filename).resolve()}") + return + + print(colored(f"refactor {Path(refactor.filename).resolve()}", "green", attrs=["bold"])) + refactor.run() + + @property + def body(self) -> Sequence[PythonRstNode]: + return cast(PythonRstNode, cast(object, self.root)).body + + def run(self): + pass diff --git a/src/renaissance/refactoring/simplify_renaissance.py b/src/renaissance/refactoring/simplify_renaissance.py new file mode 100644 index 00000000..e239fa0a --- /dev/null +++ b/src/renaissance/refactoring/simplify_renaissance.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from typing_extensions import override + +from renaissance.refactoring.python_refactoring import PythonRefactoring + + +class SimplifyRenaissance(PythonRefactoring): + + def __init__(self, file): + super().__init__(file) + self.white_list_pattern = "unit2pytest" + self.black_list_pattern = "SimplifyRenaissance" + + @override + def run(self): + if self.black_list_pattern in self.filename or self.white_list_pattern not in self.filename: + print(f"skipping: {Path(self.filename).resolve()}") + return + + print(f"simplify {Path(self.filename).resolve()}") + self.replace_stmt("$val = match.expansions[$key][0].signature", "$val= match[$key]") + self.replace_stmt( + "factory = ASTFactory(PythonASTNode)\n$atu = factory.create_from_text($code, $name)", + "PythonASTNode.load_from_text($code, $name)", + ) + self.commit() diff --git a/src/renaissance/refactoring/taut2pyunit.py b/src/renaissance/refactoring/taut2pyunit.py new file mode 100644 index 00000000..f55bcd4f --- /dev/null +++ b/src/renaissance/refactoring/taut2pyunit.py @@ -0,0 +1,656 @@ +import os +import re +import textwrap +from datetime import datetime +from pathlib import Path +from typing import Dict + +import test_data.test_insert as tst_insert +import test_data.test_class as tst_class +from renaissance.impl.types import Name, Attribute, FunctionDef, ImportStatement, ImportFrom +from renaissance.refactoring.python_refactoring import PythonRefactoring +from renaissance.syntax_tree.match_finder import match_pattern + + +class Taut2Pyunit(PythonRefactoring): + + def __init__(self, file): + super().__init__(file) + self.white_list_reg = r"_test|_unittest|_tests" + self.black_list_reg = r"_migrated|_after|_original" + self.comp = "ABCD" + + def run(self): + if re.search(self.black_list_reg, self.filename): + print(f"skipping: {Path(self.filename).resolve()}") + return + if not re.search(self.white_list_reg, self.filename): + print(f"skipping: {Path(self.filename).resolve()}") + return + print(f"Taut to pyunit migration: {Path(self.filename).resolve()}") + + # conditional refactor + if "AP_core_functionality_test" in self.filename: + self.insert_asserter() + self.remove_assert_func() + self.replace_unittest_with_asserter() + self.assert_func() + self.commit() + + self.replace_mock() + self.remove_stubserver() + self.replace_taut() + self.remove_decorator() + self.add_self() + self.convert_assert() + self.convert_testdoubles_fun() + + self.replace_log_compxtl("emrw") + self.replace_log_compxtl("abcd") + self.remove_taut_import() + self.replace_taut_import() + self.convert_setup_common() + self.convert_teardown_common() + self.convert_add_patcher() + self.convert_teardown() + self.convert_setup() + self.convert_import_verify() + self.shared_setup() + self.commit() + self.with_testdoubles() + self.commit() + + if self.root.signature.find("self.patches = []") > 0 or self.root.signature.find("patch.object") > 0: + self.insert_patch_import() + self.commit() + + try: + # result = insert_doc(result, "01-22-2026") + with open(self.get_migrated_path(self.filename), "w") as f: + f.write(self.apply_to_string()) + except FileNotFoundError: + print(f"Error: File '{self.filename}' not found.") + + def get_migrated_path(self, file_path): + """ + Convert a file path to add '_migrated' before the extension. + + Example: 'taut.py' -> 'taut_migrated.py' + """ + # Split the path into filename and extension + base, ext = os.path.splitext(file_path) + + # Create the new path with '_migrated' added + new_path = f"{base}_migrated{ext}" + + return new_path + + def replace_taut(self): + """ + replace TAUT.TestCase by unittest.TestCase + """ + [self.replace("unittest.TestCase", node, False, False) for node in self.find_ast_type(Attribute) if node.name == "TAUT.TestCase"] + [self.replace("unittest.TestCase", node, False, False) for node in self.find_ast_type(Name) if node.name == "TestCase"] + + def remove_decorator(self): + [self.remove(node, False, False) for node in self.find_ast_type(Attribute) if node.name == "TAUT.log_stub"] + + def add_self(self): + matching = [ + "emrwxread", + "emrwxwidxread", + "emrwxviprxinterface", + "whxstream2", + "gtaaxtxmark", + "mark_upd_q", + "gtaaxtxmark", + "gtaaxtxmrkxadv", + "emrwxwidxcfg", + "wlxload", + "wlxclear", + "gtmwxtxws", + "emtlxt", + "emtlxtxmc", + "emtlxtxwid", + "emrwxviprxtestlog", + "emrwxviprxwh", + ] + parent_func = ["setUpCommon", "setUp"] + [self.replace("self." + node.name, node, False, False) for node in self.find_ast_type(Name) if node.name in matching] + + matching2 = ["EMRWxREAD.emrwxread"] + [ + self.replace("self." + node.name.split(".")[1], node, False, False) + for node in self.find_ast_type(Attribute) + if node.name in matching2 and node.get_ancestor("FunctionDef").name not in parent_func + ] + + def convert_assert(self): + [self.replace("self.assertFalse", node, False, False) for node in self.find_ast_type(Attribute) if node.name == "self.assert_false"] + [self.replace("self.assertTrue", node, False, False) for node in self.find_ast_type(Attribute) if node.name == "self.assert_true"] + [self.replace("self.assertEqual", node, False, False) for node in self.find_ast_type(Attribute) if node.name == "self.assert_equal"] + + def remove_stubserver(self): + [self.remove(node, False, False) for node in self.find_ast_type(Attribute) if node.name == "TAUT.StubServer"] + + def replace_mock(self): + [ + self.replace("patch", node, False, False) + for node in self.find_ast_type(Attribute) + if node.name == "mock.patch" and node.parent.parent.name == "decorator_list" + ] + + def replace_log_compxtl(self, comp): + func_call = self.pattern_factory.create_statements(f"{comp}xtl.$a($$bb)") + for call in match_pattern(self.root.children, func_call): + repl = call.signature.replace(f"{comp}xtl", f"fake_{comp}xtl") + self.replace(repl, call.nodes, False, False) + + assign = self.pattern_factory.create_statements(f"$c = {comp}xtl.$a($$bb)") + for match in match_pattern(self.root.children, assign): + repl = match.signature.replace(f"{comp}xtl", f"fake_{comp}xtl") + self.replace(repl, match.nodes, False, False) + self.commit() + taut_test_doubles = self.pattern_factory.create_statements( + f"with TAUT.TestDoubles({comp}xtl=Fake{comp.upper()}xTL(None)):\n log = TAUT.Logger()\n $$aa" + ) + for match in match_pattern(self.root.children, taut_test_doubles): + repl = f"fake_{comp}xtl = Fake{comp.upper()}xTL(None)\n{match["$$aa"]}" + self.replace(repl, match.nodes, False, False) + self.commit() + + def remove_taut_import(self): + taut_import = self.pattern_factory.create_statements("import TAUT\n") + for match in match_pattern(self.root.children, taut_import): + self.remove(match.nodes, False, False) + + def replace_taut_import(self): + """ + replace mock by unittest.mock and using patch + """ + mock = self.pattern_factory.create_statements("import mock\n") + for match in match_pattern(self.root.children, mock): + self.remove(match.nodes, False, False) + + test_case = self.pattern_factory.create_statements("from TAUT import TestCase") + for match in match_pattern(self.root.children, test_case): + self.remove(match.nodes, False, False) + import_taut = self.pattern_factory.create_statements("from TAUT import TestCase, TestDoubles") + for match in match_pattern(self.root.children, import_taut): + repl = "try:\n from unittest.mock import patch\nexcept ImportError:\n from mock import patch\n" + self.replace(repl, match.nodes, False, False) + import_doubles = self.pattern_factory.create_statements("from TAUT import TestDoubles") + for match in match_pattern(self.root.children, import_doubles): + repl = "try:\n from unittest.mock import patch\nexcept ImportError:\n from mock import patch\n" + self.replace(repl, match.nodes, False, False) + + def convert_tds(self): + tds = self.pattern_factory.create_statements("self.tds.append(TestDoubles($a, $b=$c))") + for match in match_pattern(self.root.children, tds): + repl = f"self.add_patcher({match["$a"]}, '{match["$b"]}', {match["$c"]})" + self.replace(repl, match.nodes, False, False) + + tds2 = self.pattern_factory.create_statements("self.tds.append(TestDoubles($a=ImprovedStub($b)))") + for match in match_pattern(self.root.children, tds2): + repl = f"self.{match["$a"]} = ImprovedStub({match["$b"]})" + self.replace(repl, match.nodes, False, False) + + def convert_setup_common(self): + insert_code = """ImprovedStub.ret_vals = {} +ImprovedStub.ret_vals_ex = {} +ImprovedStub.call_logs = {} +ImprovedStub.store_args = {} + +""" + p_start = """for p in self.patchers: + p.start() +""" + tds_pattern = self.pattern_factory.create_statements("self.tds = [$$aa]") + for match in match_pattern(self.root.children, tds_pattern): + init_stubs = "" + repl = "self.patchers = [\n" + doubles_pattern = self.pattern_factory.create_expression("TestDoubles($a=ImprovedStub($b))") + for matched_doubles in match_pattern(match.expansions["$$aa"], [doubles_pattern]): + init_stubs += ( + f'self.{matched_doubles.expansions["$a"][0]} = ImprovedStub({matched_doubles.expansions["$b"][0].signature})\n' + ) + interface_stub = self.find_import_interface(matched_doubles.expansions["$b"][0].signature) + repl += f' patch.object({interface_stub}, \'{matched_doubles.expansions["$a"][0]}\', self.{matched_doubles.expansions["$a"][0]}),\n' + repl += "]\n\n" + repl = insert_code + init_stubs + repl + p_start + self.replace(repl, match.nodes, False, False) + + def convert_teardown_common(self): + teardown_common = self.pattern_factory.create_statements("def tearDownCommon(self):\n $$aa") + repl = """def tearDownCommon(self): + for p in self.patchers: + try: + p.stop() + except RuntimeError: + pass +""" + for match in match_pattern(self.root.children, teardown_common): + self.replace(repl, match.nodes, False, False) + + def convert_add_patcher(self): + pattern = self.pattern_factory.create_statements("def tearDownCommon(self):\n $$aa") + for match in match_pattern(self.root.children, pattern): + patcher_pattern = [node for node in self.find_ast_type(FunctionDef) if node.name == "add_patcher"] + if len(patcher_pattern) == 0: + self.insert_after(tst_class.insert_add_patcher, match.nodes) + + def find_import_interface(self, name: str): + interface = name + if name.islower(): + node_list = [node for node in self.find_ast_type(ImportStatement) if node.name == name] + if node_list: + if node_list[0].ast_type == ImportFrom: + interface = node_list[0].properties["module"] + else: + interface = node_list[0].name if node_list else name + return interface.split(".")[0] + + def convert_setup(self): + # remove doubles init + pattern1 = self.pattern_factory.create_statements("doubles = []") + replacement = "self.patches = []" + for match in match_pattern(self.root.children, pattern1): + self.replace(replacement, match.nodes, False, False) + + pattern2 = self.pattern_factory.create_statements("self.doubles = []") + for match in match_pattern(self.root.children, pattern2): + self.replace(replacement, match.nodes, False, False) + + # convert doubles to patch + self.convert_test_doubles("doubles.append(TAUT.TestDoubles($a=$b))") + self.convert_test_doubles("self.doubles.append(TAUT.TestDoubles($a=$b))") + + # convert doubles to patch.object + insert_node = None + pattern_outer = self.pattern_factory.create_statements("def setUp(self):\n $$aa") + for setup_func in match_pattern(self.root.children, pattern_outer): + + pattern4 = self.pattern_factory.create_statements("doubles.append(TAUT.TestDoubles(module=$mod, $b=$c))") + matched_pattern = match_pattern(setup_func.nodes, pattern4) + for index, match in enumerate(matched_pattern): + repl_pattern = f"self.patches.append(patch.object({match.expansions['$mod'][0].name}, '{match.expansions['$b'][0]}', {match.expansions['$c'][0].signature}))" + repl_pattern = repl_pattern.replace("context_stub", "self.context_stub") + self.replace(repl_pattern, match.nodes, False, False) + if index == len(matched_pattern) - 1: + insert_node = match.nodes[-1] + insert_code = """\nfor p in self.patches: + p.start()""" + self.insert_after(insert_code, insert_node, False, False) + + pattern4_1 = self.pattern_factory.create_statements("self.doubles.append(TAUT.TestDoubles(module=$mod, $b=$c))") + matched_pattern_1 = match_pattern(setup_func.nodes, pattern4_1) + for index, match in enumerate(matched_pattern_1): + repl_pattern = f"self.patches.append(patch.object({match.expansions['$mod'][0].name}, '{match.expansions['$b'][0]}', {match.expansions['$c'][0].signature}))" + repl_pattern = repl_pattern.replace("context_stub", "self.context_stub") + self.replace(repl_pattern, match.nodes, False, False) + if index == len(matched_pattern_1) - 1: + insert_node = match.nodes[-1] + insert_code = """\nfor p in self.patches: + p.start()""" + self.insert_after(insert_code, insert_node, False, False) + + pattern5 = self.pattern_factory.create_statements("self.doubles = doubles") + for match in match_pattern(self.root.children, pattern5): + self.remove(match.nodes, False, False) + self.commit() + [self.replace("self.context_stub", node, False, False) for node in self.find_ast_type(Name) if node.name == "context_stub"] + + def convert_teardown(self): + matched_pattern = self.pattern_factory.create_statements("def tearDown(self):\n $$aa") + repl_pattern = """def tearDown(self): + for p in self.patches: + p.stop()""" + for match in match_pattern(self.root.children, matched_pattern): + self.replace(repl_pattern, match.nodes, False, False) + + def refactor_teardown(self): + self.comp = "abcd" + pattern1 = self.pattern_factory.create_statements("for double in self.doubles:\n double.exit()") + replace_pattern = "patch.stopall()" + for match in match_pattern(self.root.children, pattern1): + self.replace(replace_pattern, match.nodes, False, False) + + insert_code = f"""{self.comp.upper()}xCONTEXT.{self.comp}xcontext.reset_method_attributes("start_wafer") +{self.comp.upper()}xCONTEXT.{self.comp}xcontext.reset_method_attributes("finish_wafer") +{self.comp.upper()}xCONTEXT.{self.comp}xcontext.reset_method_attributes("start_lot") +{self.comp.upper()}xCONTEXT.{self.comp}xcontext.reset_method_attributes("finish_lot") + +""" + pattern2 = self.pattern_factory.create_statements("self._patch_readout_data_filler.stop()") + for match in match_pattern(self.root.children, pattern2): + self.insert_before(insert_code, match.nodes, False, False) + + def convert_test_doubles(self, doubles: str): + mappings: Dict[str, str] = { + "emrmxcontext": "EMRMxCONTEXT", + "acbdxcontext": "ACBDxCONTEXT", + # Add more mappings here + } + doubles_pattern = self.pattern_factory.create_statements(doubles) + for match in match_pattern(self.root.children, doubles_pattern): + keyword = match.expansions["$a"][0] + if match.expansions["$a"][0] in mappings.keys(): + keyword = mappings[match.expansions["$a"][0]] + repl_pattern = f"self.patches.append(patch('{keyword}.{match.expansions['$a'][0]}', {match.expansions['$b'][0].name}))" + repl_pattern = repl_pattern.replace("context_stub", "self.context_stub") + self.replace(repl_pattern, match.nodes, False, False) + + def insert_patch_import(self): + insert = "\ntry:\n from unittest.mock import patch\nexcept ImportError:\n from mock import patch" + insert_pattern = self.pattern_factory.create_statements(insert) + if len(match_pattern(self.root.children, insert_pattern)) == 0: + pattern = self.pattern_factory.create_statements("import unittest\n") + for match in match_pattern(self.root.children, pattern): + self.insert_after(insert, match.nodes, False, False) + + def replace_taut_skip(self): + """ + replace @TAUT.skip_test by @unittest.skip + """ + [self.replace("@unittest.skip", node) for node in self.find_ast_type(Attribute) if node.name == "TAUT.skip_test"] + + def convert_import_verify(self): + import_verify = self.pattern_factory.create_statements("self.import_and_verify_module('$a')") + for match in match_pattern(self.root.children, import_verify): + repl = f'import {match.expansions["$a"][0]}\nself.assertIsNotNone({match.expansions["$a"][0]})' + self.replace(repl, match.nodes, False, False) + + def with_testdoubles(self): + pattern1 = self.pattern_factory.create_statements("with TAUT.TestDoubles(module=$a, $b=$c):\n $$ee") + for match in match_pattern(self.root.children, pattern1): + repl_pattern = f"with patch.object({match["$a"]}, '{match["$b"]}', new={match["$c"]}):\n {match["$$ee"]}" + self.replace(repl_pattern, match.nodes, False, False) + + def shared_setup(self): + setup_function = self.pattern_factory.create_statements("def sharedSetUp(self):\n $$stmts") + for match in match_pattern(self.root.children, setup_function): + repl = match.signature.replace("def sharedSetUp", " def setUp") + self.replace(textwrap.dedent(repl), match.nodes, False, False) + + def insert_class(self): + class_pattern = self.pattern_factory.create_statements("class Asserter(unittest.TestCase):\n $$aa") + if len(match_pattern(self.root.children, class_pattern)) == 0: + insert_pattern = self.pattern_factory.create_statements("def b():\n $$bb") + insert_code = tst_insert.insert_code + for match in match_pattern(self.root.children, insert_pattern): + self.insert_after(insert_code, match.nodes, False, False) + + def insert_asserter(self): + insert_pattern = self.pattern_factory.create_statements("def assert_double_equal($$arg, $$other=$$value):\n $$bb") + insert_code = tst_insert.insert_code + for match in match_pattern(self.root.children, insert_pattern): + self.insert_after(insert_code, match.nodes, False, False) + + def remove_assert_func(self): + pattern = self.pattern_factory.create_statements("def assert_double_equal($$arg, $$other=$$value):\n $$bb") + for match in match_pattern(self.root.children, pattern): + self.remove(match.nodes, False, False) + self.commit() + + def replace_unittest_with_asserter(self): + pattern = self.pattern_factory.create_statements("class $a(TAUT.TestCase):\n $$bb") + for match in match_pattern(self.root.children, pattern): + if not match["$a"] == "Asserter": + if "assert_raises" in match["$$bb"] or "assert_double_equal" in match["$$bb"]: + repl = f"{match.signature.replace("TAUT.TestCase", "Asserter")}" + self.replace(repl, match.nodes, False, False) + self.commit() + + def assert_func(self): + matching = [ + "assert_raises", + "assert_double_equal", + ] + [self.replace("self." + node.name, node, False, False) for node in self.find_ast_type(Name) if node.name in matching] + + def move_indent(self, indent): + pattern1 = self.pattern_factory.create_statements("""def $a($$b): + self.doubles.append(TAUT.TestDoubles($mod, $e, $f)) + $$c""") + for match in match_pattern(self.root.children, pattern1): + double_pattern = f" self.doubles.append(TAUT.TestDoubles({match["$mod"]}, {match["$e"]}, {match["$f"]}))\n" + func_header_index = match.signature.index("):\n") + repl = f""" with patch.object({match["$mod"]}, '{match["$e"]}', {match["$f"]}):\n""" + replace_pattern = ( + match.signature[: func_header_index + 3] + repl + textwrap.indent(match.signature[func_header_index + 3 :], indent) + ) + replace_pattern = replace_pattern.replace(double_pattern, "") + self.replace(replace_pattern, match.nodes, False, False) + + def convert_testdoubles_fun(self): + """this is used for taut migration, where the function pattern is found in a class""" + # case1 two TestDoubles are defined + pattern1 = self.pattern_factory.create_statements("""def $a($$b): + self.doubles.append( + TAUT.TestDoubles( + module=$mod1, $e1=$f1 + ) + ) + self.doubles.append( + TAUT.TestDoubles( + module=$mod2, $e2=$f2 + ) + ) + $$c + """) + for match in match_pattern(self.root.children, pattern1): + double_pattern = f""" self.doubles.append( + TAUT.TestDoubles( + module={match["$mod1"]}, + {match["$e1"]}={match["$f1"]}, + ) + ) + self.doubles.append( + TAUT.TestDoubles( + module={match["$mod2"]}, + {match["$e2"]}={match["$f2"]}, + ) + ) +""" + func_header_index = match.signature.index("):\n") + repl = f"""with patch.object({match["$mod1"]}, '{match["$e1"]}', {match["$f1"]}), \\ + patch.object({match["$mod2"]}, '{match["$e2"]}', {match["$f2"]}):\n""" + replace_pattern = ( + match.signature[: func_header_index + 3] + textwrap.indent(repl, " ") + match.signature[func_header_index + 3 :] + ) + replace_pattern = replace_pattern.replace(textwrap.indent(double_pattern, " "), "") + self.replace(replace_pattern, match.nodes, False, False) + self.commit() + + pattern2 = self.pattern_factory.create_statements("""def $a($$b): + self.doubles.append( + TAUT.TestDoubles( + module=$mod, $e=$f + ) + ) + $$c + """) + for match in match_pattern(self.root.children, pattern2): + double_pattern = f""" self.doubles.append( + TAUT.TestDoubles( + module={match["$mod"]}, {match["$e"]}={match["$f"]} + ) + ) +""" + double_pattern1 = f""" self.doubles.append( + TAUT.TestDoubles( + module={match["$mod"]}, + {match["$e"]}={match["$f"]}, + ) + ) +""" + func_header_index = match.signature.index("):\n") + repl = f"""with patch.object({match["$mod"]}, '{match["$e"]}', {match["$f"]}):\n""" + replace_pattern = ( + match.signature[: func_header_index + 3] + textwrap.indent(repl, " ") + match.signature[func_header_index + 3 :] + ) + double_pattern2 = f" self.doubles.append(TAUT.TestDoubles(module={match['$mod']}, {match['$e']}={match['$f']}))\n" + replace_pattern = replace_pattern.replace(textwrap.indent(double_pattern, " "), "") + replace_pattern = replace_pattern.replace(textwrap.indent(double_pattern1, " "), "") + replace_pattern = replace_pattern.replace(textwrap.indent(double_pattern2, " "), "") + self.replace(replace_pattern, match.nodes, False, False) + self.commit() + + def refactor_testdoubles_fun(self): + """this is used for unittest, where the function pattern is not found in a class""" + # case1 two TestDoubles are defined + pattern1 = self.pattern_factory.create_statements("""def $a($$b): + self.doubles.append( + TAUT.TestDoubles( + module=$mod1, $e1=$f1 + ) + ) + self.doubles.append( + TAUT.TestDoubles( + module=$mod2, $e2=$f2 + ) + ) + $$c""") + for match in match_pattern(self.root.children, pattern1): + double_pattern = f""" self.doubles.append( + TAUT.TestDoubles( + module={match["$mod1"]}, {match["$e1"]}={match["$f1"]} + ) + ) + self.doubles.append( + TAUT.TestDoubles( + module={match["$mod2"]}, {match["$e2"]}={match["$f2"]} + ) + ) +""" + func_header_index = match.signature.index("):\n") + repl = f"""with patch.object({match["$mod1"]}, '{match["$e1"]}', {match["$f1"]}), \\ + patch.object({match["$mod2"]}, '{match["$e2"]}', {match["$f2"]}): + """ + replace_pattern = match.signature[: func_header_index + 3] + textwrap.indent( + repl + match.signature[func_header_index + 3 :], " " + ) + replace_pattern = replace_pattern.replace(double_pattern, "") + replace_pattern = replace_pattern.replace(textwrap.indent(double_pattern, " "), "") + self.replace(replace_pattern, match.nodes, False, False) + self.commit() + pattern2 = self.pattern_factory.create_statements("""def $a($$b): + self.doubles.append( + TAUT.TestDoubles( + module=$mod, $e=$f + ) + ) + $$c +""") + for match in match_pattern(self.root.children, pattern2): + double_pattern = f""" self.doubles.append( + TAUT.TestDoubles( + module={match["$mod"]}, {match["$e"]}={match["$f"]} + ) + ) +""" + func_header_index = match.signature.index("):\n") + repl = f"""with patch.object({match["$mod"]}, '{match["$e"]}', {match["$f"]}):\n""" + replace_pattern = match.signature[: func_header_index + 3] + textwrap.indent( + repl + match.signature[func_header_index + 3 :], " " + ) + replace_pattern = replace_pattern.replace(double_pattern, "") + replace_pattern = replace_pattern.replace(textwrap.indent(double_pattern, " "), "") + self.replace(replace_pattern, match.nodes, False, False) + + def refactor_testdoubles_class(self): + pattern = self.pattern_factory.create_statements("""class $a(TAUT.TestCase): + + def setUp(self): + $$bb + self.doubles = [] + $$cc + self.doubles.append( + TAUT.TestDoubles( + module=$mod1, + $e1=$f1, + ) + ) + self.doubles.append( + TAUT.TestDoubles( + module=$mod2, + $e2=$f2, + ) + ) + $$dd + + def tearDown(self): + $$gg + for double in self.doubles: + double.exit()""") + + for match in match_pattern(self.root.children, pattern): + replace_pattern = f"""class {match["$a"]}(unittest.TestCase): + + def setUp(self): +{textwrap.indent(match["$$bb"], " ")} +{textwrap.indent(match["$$cc"], " ")} + self.patches = [ + patch.object({match["$mod1"]}, '{match["$e1"]}', {match["$f1"]}), + patch.object({match["$mod2"]}, '{match["$e2"]}', {match["$f2"]}), + ] + for p in self.patches: + p.start() + +{textwrap.indent(match["$$dd"], " ")} + + def tearDown(self): +{textwrap.indent(match["$$gg"], " ")} + for p in self.patches: + p.stop()""" + self.replace(replace_pattern, match.nodes, False, False) + + def insert_doc_func(self): + pattern = self.pattern_factory.create_statements("""# -----------------------------------------------------------------------------# +# # +# Copyright (c) 2016, XXXX Netherlands B.V. # +""") + insert_code = get_change_comment() + for match in match_pattern(self.root.children, pattern): + self.insert_before(insert_code, match.nodes, False, False) + + +def insert_doc(content: str, date): + pattern = r"# -+(#)?\n(#\s+#\n)?#\s+Copyright \(c\) \d{4}, XXXX" + match = re.search(pattern, content) + + if not match: + print("Comment block not found.") + return content + + # Find the beginning of the line containing the comment + position = match.start() + line_start = content.rfind("\n", 0, position) + 1 + if line_start == 0: # If comment is at the beginning of the file + line_start = 0 + + # Insert the new line before the comment block + print(get_change_comment(date)) + modified_content = content[:line_start] + get_change_comment(date) + "\n" + content[line_start:] + return modified_content + + +def get_change_comment(date=None): + """ + Generate a formatted change comment with today's date. + + Args: + change_id (str): The change ID (e.g., 'SWCHGxxxxxxxx') + description (str): The description of the change + + Returns: + str: Formatted change comment string + """ + change_id = "SWCHGxxxxxxxx" + description = "Add assert_raises method to Asserter class." + if date is None: + # No date provided, use today + formatted_date = datetime.now() + else: + formatted_date = datetime.strptime(date, "%m-%d-%Y") + return f"# {formatted_date.strftime('%m-%d-%Y')} : {change_id} SBYN {description}" diff --git a/src/renaissance/refactoring/unit2pytest.py b/src/renaissance/refactoring/unit2pytest.py new file mode 100644 index 00000000..d22f1bc7 --- /dev/null +++ b/src/renaissance/refactoring/unit2pytest.py @@ -0,0 +1,236 @@ +import os +import textwrap +from typing import Sequence + +from renaissance.impl.python.util import convert_function +from renaissance.impl.types import Attribute, Literal, Number, FormattedString, ClassDef, FunctionDef +from renaissance.refactoring.python_refactoring import PythonRefactoring +from renaissance.syntax_tree import PatternMatch +from renaissance.syntax_tree.ast_finder import find_ast_type +from renaissance.syntax_tree.match_finder import match_pattern, AstProtocol + + +class Unit2Pytest(PythonRefactoring): + def __init__(self, file): + """hide internal administration in the parent class so that this class you only deals with specific refactors""" + super().__init__(file) + self.black_list_pattern = "utils_for_test" + self.white_list_pattern = "test" + + def run(self): + """ + entry point for converting unittest to pytest + """ + + self.refactor() + + self.post_processing() + + def refactor(self): + # 1: file level changes + self.convert_test_class() + self.restructure_module() + self.replace_stmt("unittest.main()", "pytest.main()") + self.replace_stmt("import unittest", "import pytest\nfrom hamcrest import *") + self.replace_stmt("from parameterized import parameterized", "import pytest\nfrom hamcrest import *") + self.replace_stmt("from unittest import TestCase,$$symbols", "import pytest\nfrom hamcrest import *") + self.replace_stmt("from unittest import TestCase", "import pytest\nfrom hamcrest import *") + + # 2: class level changes + self.convert_parameterized_test() + self.convert_test_setup() + self.commit() + # + + # 3: function level changes + + self.convert_skip_test() + self.remove_print() + self.convert_plain_assert_same_length() + self.commit() + self.replace_stmt("assert $stmt, $$msg", "assert_that($stmt, is_(True), $$msg)") + self.replace_stmt("self.assertTrue($exp,$$msg)", "assert_that($exp, is_(True), $$msg)") + self.replace_stmt("self.assertFalse($exp, $$msg)", "assert_that($exp, is_(False), $$msg)") + + self.convert_assert("self.assertEqual($exp, $act)", "assert_that($exp, is_($act))") + self.convert_assert("self.assertGreaterEqual($exp, $act)", "assert_that($exp, greater_than_or_equal_to($act))") + self.convert_assert("self.assertGreater($exp, $act)", "assert_that($exp, greater_than($act))") + self.convert_assert("self.assertLesserEqual($exp, $act)", "assert_that($exp, less_than_or_equal_to($act))") + self.convert_assert("self.assertLesser($exp, $act)", "assert_that($exp, less_than($act))") + self.convert_assert("self.assertMultiLineEqual($act, $exp)", "assert_that($act, is_($exp))") + + self.replace_stmt("self.assertIn($act, $exp)", "assert_that($exp, contain_string($act))") + self.replace_stmt("self.assertIsInstance($act, $exp)", "assert_that($act, is_($exp))") + self.replace_stmt("with self.assertRaises($exc): $call()", "assert_that(calling($call), raises($exc))") + + def post_processing(self): + # 4: improve to more concise asserts + while self.has_changed(): + self.commit() + self.replace_stmt("assert_that($exp)", "assert_that($exp, is_(True))") + self.replace_stmt("assert_that(isinstance($exp, $act))", "assert_that($exp, is_($act))") + self.replace_stmt("assert_that(len($exp), $act)", "assert_that($exp, has_length($act))") + self.replace_stmt("assert_that(len($exp) >= 1)", "assert_that($exp, is_not(empty()))") + self.replace_stmt("assert_that(len($exp) >= 1, is_(True))", "assert_that($exp, is_not(empty()))") + self.replace_stmt("assert_that(len($exp) == $length)", "assert_that($exp, has_length($length))") + self.replace_stmt("assert_that($exp == $act)", "assert_that($exp, is_($act), $$msg)") + self.replace_stmt("assert_that($exp == $act, is_(True), $$msg)", "assert_that($exp, is_($act), $$msg)") + self.replace_stmt("assert_that(not $stmt, is_(True), $$msg)", "assert_that($stmt, is_(False) ,$$msg)") + self.replace_stmt("assert_that($stmt, is_not(True), $$msg)", "assert_that($stmt, is_(False) ,$$msg)") + self.replace_stmt("assert_that(not $stmt)", "assert_that($stmt, is_(False))") + self.replace_stmt("assert_that($el in $col, is_(True))", "assert_that($col, contains_exactly($el))") + self.replace_stmt("assert_that($exp, has_length(is_($act)))", "assert_that($exp, has_length($act))") + self.swap_expected_and_actual() + self.replace_stmt("assert_that(not $stmt)", "assert_that($stmt, is_(False))") + self.replace_stmt("assert_that($exp.startswith($act))", "assert_that($exp, starts_with($act))") + self.remove_duplicate_import("import pytest\nfrom hamcrest import *") + self.commit() + + def convert_test_class(self): + test_main: Sequence[AstProtocol] = self.pattern_factory.create_statements( + "class $klass($test_class):\n $$test_cases\n" + ) # type: ignore[assignment] + for match in match_pattern(self.root.children, test_main): + klass = match["$klass"] + test_class = match["$test_class"] + + if test_class.endswith("TestCase"): + # class inherit from TestCase (or unittest.TestCase) + if klass.endswith("Test"): + # class name ends with Test, rename by move Test to front + repl = match.signature.replace(f"{klass}({test_class}):", f"Test{klass[:-4]}:") + else: + # we assume there are only 2 variant TestExample and ExampleTest + repl = match.signature.replace(f"({test_class}):", ":") + + # repl = f'class {match.expansions["$klass"][0]}:\n{raw(match.expansions["$$test_cases"])}' + self.replace(repl, match.nodes, False, False) + + def convert_test_setup(self): + setup_function = self.pattern_factory.create_statements("def setUp(self): $$stmts") + for match in match_pattern(self.body, setup_function): + # add decorator to the setup dunction and convert to snake case + repl = f"@pytest.fixture(autouse=True)\n{match.signature}".replace(" setUp(self)", " setup(self)") + self.replace(repl, match.nodes, False, False) + + def convert_assert(self, pattern, replacement): + pat = self.pattern_factory.create_statements(pattern) + for match in match_pattern(self.root.children, pat): + repl = replacement + if self.is_swapped(match): + exp = match["$act"] + act = match["$exp"] + else: # original is wrong + act = match["$act"] + exp = match["$exp"] + repl = repl.replace("$exp", exp).replace("$act", act) + self.replace(repl, match.nodes, False, False) + + def is_swapped(self, match: PatternMatch) -> bool: + return match.expansions["$exp"][0].ast_type in [Literal, FormattedString, Number] + + def convert_parameterized_test(self): + unittest = self.pattern_factory.create_statements(textwrap.dedent(""" + @parameterized.expand($$parameters) + @$$decorator + def $fun($$args, *$$varg): + $$stmts + """)) + for match in match_pattern(self.root.children, unittest): + fun = match.nodes[0] + args = ", ".join([arg.node.arg for arg in match.expansions["$$args"]]) + if varg := match.expansions["$$varg"]: + args = f"{args}, *{varg[0].signature}" + args = args.replace("self, ", "") + repl = fun.signature + if " def " in repl: + repl = repl.replace("@parameterized.expand(", f' @pytest.mark.parametrize("{args}",') + repl = repl.replace("@unittest.skip(", "@pytest.mark.skip(") + repl = textwrap.dedent(repl) + else: + repl = repl.replace("@parameterized.expand(", f'@pytest.mark.parametrize("{args}",') + repl = repl.replace("@unittest.skip(", "@pytest.mark.skip(") + + self.replace(repl, fun, False, False) + + def remove_print(self): + print_msg = self.pattern_factory.create_statements("print($$msg)") # type: ignore[assignment] + for match in match_pattern(self.root.children, print_msg): + if len(match.nodes[0].parent.parent.body) == 1: + self.remove([match.nodes[0].parent.parent], False, False) + else: + self.remove(match.nodes, False, False) + + def convert_plain_assert_same_length(self): + pattern: Sequence[AstProtocol] = self.pattern_factory.create_statements( + '$act: int = len($real)\nassert $exp == $act, "$act = " + str($act)' + ) + for match in match_pattern(self.body, pattern): + repl = 'assert_that($real, has_length($exp), f"length of $real = {len($real)}")' + real = match["$real"] + if self.is_swapped(match): + exp = match["$exp"] + else: # original is wrong + exp = match["$act"] + repl = repl.replace("$exp", exp).replace("$real", real) + self.replace(repl, match.nodes, False, False) + + def convert_skip_test(self): + nodes = find_ast_type(self.root, Attribute) + for node in nodes: + if node.signature == "unittest.skip": + self.replace("pytest.mark.skip", node, False, False) + + def swap_expected_and_actual(self): + pattern: Sequence[AstProtocol] = self.pattern_factory.create_statements("assert_that($exp, is_($act))") # type: ignore[assignment] + for match in match_pattern(self.root.children, pattern): + if self.is_swapped(match): + repl = "assert_that($act, is_($exp))" + act = match["$act"] + exp = match["$exp"] + repl = repl.replace("$exp", exp).replace("$act", act) + self.replace(repl, match.nodes, False, False) + + def restructure_module(self): + funs = [stmt for stmt in self.body if stmt.ast_type == FunctionDef] + test_classes = [stmt for stmt in self.body if stmt.ast_type == ClassDef and stmt.name.startswith("Test")] + if len(funs) == 0: + return + if len(test_classes) == 0: + # file does not contain any test class, create a new class and add function in class + cls = f"class {self.convert_file_to_test_class()}:\n" + for fun in funs: + cls += textwrap.indent(convert_function(fun), " ") + self.remove([fun]) + self.insert_before(cls, funs[0]) + else: + # one or more class in file, add functio as member ot the last class in file + for fun in funs: + # assuming the class comes first + meth = convert_function(fun) + self.insert_after(meth, test_classes[-1].body[-1]) + self.remove(fun) + self.commit() + for fun in funs: + # also change the calling signature of those functions in case they are not test cases + function_call = [self.pattern_factory.create_expression(f"{fun.name}($$args)")] + for call in match_pattern(self.root.children, function_call): + sig = call.nodes[0].signature + self.replace(f"self.{sig}", call.nodes, False, False) + self.commit() + + def convert_file_to_test_class(self): + stem = os.path.splitext(os.path.basename(self.filename))[0] + parts = stem.split("_") + if parts[-1].lower() == "test": + parts = parts[:-1] + name = "".join(word.capitalize() for word in parts) + return name if name.startswith("Test") else f"Test{name}" + + def remove_duplicate_import(self, import_str): + import_stmt: Sequence[AstProtocol] = self.pattern_factory.create_statements(import_str) # type: ignore[assignment] + # type: ignore[assignment] + duplicate_imports = match_pattern(self.body, import_stmt) + + for match in duplicate_imports[1:-1]: + self.remove(match.nodes, False, False) diff --git a/src/renaissance/syntax_tree/__init__.py b/src/renaissance/syntax_tree/__init__.py new file mode 100644 index 00000000..2eed9e66 --- /dev/null +++ b/src/renaissance/syntax_tree/__init__.py @@ -0,0 +1,47 @@ +# __init__.py +from .ast_node import ASTNode, ASTReference, VisitorResult +from .ast_finder import ASTFinder +from .ast_shower import ASTShower +from .ast_factory import ASTFactory +from .batch_ast_processor import ( + BatchASTProcessor, + IterableProvider, + AST_FACTORY_AND_ATU, + Action, +) +from .match_finder import MatchFinder, PatternMatch +from .ast_rewriter import ASTRewriter +from .ast_processor import ASTProcessor +from .ast_refactor_actions import ASTRefactorActions +from .recipe_ast_processor import ( + RecipeASTProcessor, + after_step, + recipe_step, + final_action, +) +from ..utils.text_utils import TextUtils +from renaissance.impl.clang.cpp_utils import CPPUtils + +__all__ = [ + "ASTNode", + "ASTReference", + "VisitorResult", + "ASTFinder", + "ASTShower", + "ASTFactory", + "MatchFinder", + "PatternMatch", + "ASTRewriter", + "CPPUtils", + "TextUtils", + "ASTProcessor", + "BatchASTProcessor", + "IterableProvider", + "AST_FACTORY_AND_ATU", + "Action", + "ASTRefactorActions", + "RecipeASTProcessor", + "after_step", + "recipe_step", + "final_action", +] diff --git a/src/renaissance/syntax_tree/ast_factory.py b/src/renaissance/syntax_tree/ast_factory.py new file mode 100644 index 00000000..de749272 --- /dev/null +++ b/src/renaissance/syntax_tree/ast_factory.py @@ -0,0 +1,47 @@ +from pathlib import Path +from typing import Optional, Sequence + +from .ast_node import ASTNode + + +class ASTFactory: + """ + A factory class for creating instances of ASTNode. + Attributes: + clazz (type[ASTNode]): The class type of the AST nodes to be created. + extra_args (Optional[Sequence[str]]): Additional arguments to be passed during the creation of AST nodes. + #TODO working_dir + """ + + def __init__( + self, + clazz: type[ASTNode], + extra_args: Optional[Sequence[str]] = None, + working_dir: Optional[Path] = None, + ) -> None: + self.clazz = clazz + self.extra_args: Sequence[str] = extra_args if isinstance(extra_args, Sequence) else [] + # TODO: Why not + # self.extra_args: Sequence[str] = [] if extra_args is None else extra_args or + # self.extra_args: Sequence[str] = extra_args if extra_args else [] ? + # As the logic is about providing a value when none is provided. + # In other words, the type of the optional argument is not relevant for the logic. + self.working_dir = working_dir if working_dir else Path.cwd() + + def create(self, file_path: Path) -> ASTNode: + atu = self.clazz.load( + file_path=file_path, + extra_args=self.extra_args, + working_dir=self.working_dir, + ) + assert isinstance(atu, self.clazz), "The loaded AST node is not an instance of the expected type" + return atu + + def create_from_text(self, text: str, file_name: str) -> ASTNode: + atu = self.clazz.load_from_text(text, file_name, extra_args=self.extra_args, working_dir=self.working_dir) + assert isinstance(atu, self.clazz), "The loaded AST node is not an instance of the expected type" + return atu + + +if __name__ == "__main__": + pass diff --git a/src/renaissance/syntax_tree/ast_finder.py b/src/renaissance/syntax_tree/ast_finder.py new file mode 100644 index 00000000..f7d5f409 --- /dev/null +++ b/src/renaissance/syntax_tree/ast_finder.py @@ -0,0 +1,63 @@ +import re +from typing import Callable, Iterator, Optional, Sequence + + +from .ast_node import ASTNode +from renaissance.impl.types import Type +from renaissance.utils.ast_utils import traverse + + +class ASTFinder: + KIND_MATCH = re.compile(r"[\W_]+") + + @staticmethod + def find_all(ast_node: ASTNode, function: Callable[[ASTNode], Iterator[ASTNode] | bool]) -> Sequence[ASTNode]: + return list(ASTFinder.__find_all(ast_node, function)) + + # @staticmethod + # def find_kind(ast_node: ASTNode, kind: str | re.Pattern[str]) -> Sequence[ASTNode]: + # return list(ASTFinder.__matches_kind(ast_node, kind)) + + @staticmethod + def find(ast_node: ASTNode, kind: str | re.Pattern[str]) -> Sequence[ASTNode]: + return list(ASTFinder.__matches_kind(ast_node, kind)) + + @staticmethod + def matches_kind(ast_node: Optional[ASTNode], kind: str | re.Pattern[str]) -> bool: + # compare kind with the ast_node kind only using word characters + # get kind of the ast_node with only word characters + if ast_node is None: + return False + ast_kind = ASTFinder.KIND_MATCH.sub("", ast_node.kind).lower() + pattern = kind if isinstance(kind, re.Pattern) else re.compile(kind, re.IGNORECASE) + return pattern.fullmatch(ast_kind) is not None + + @staticmethod + def __find_all(ast_node: ASTNode, function: Callable[[ASTNode], Iterator[ASTNode] | bool]) -> Iterator[ASTNode]: + result = function(ast_node) + if isinstance(result, bool) and result: + yield ast_node + elif isinstance(result, Iterator): + yield from result + for child in ast_node.children: + yield from ASTFinder.__find_all(child, function) + + @staticmethod + def __matches_kind(ast_node: ASTNode, kind: str | re.Pattern[str]) -> Iterator[ASTNode]: + pattern = kind if isinstance(kind, re.Pattern) else re.compile(kind, re.IGNORECASE) + node_kind = ast_node.kind if ast_node.kind else "" + ast_kind = ASTFinder.KIND_MATCH.sub("", node_kind).lower() + + if pattern.fullmatch(ast_kind): + yield ast_node + for child in ast_node.children: + # assert isinstance(child, type(ast_node)), f'Expected {type(ast_node)} but got {type(child)}' + yield from ASTFinder.__matches_kind(child, pattern) + + +def find_ast_type(ast_node, kind: type[Type]) -> Sequence: + return [n for n in traverse(ast_node) if isinstance(n.ast_type(), kind)] + + +def matches_kind(ast_node, kind: type[Type]) -> bool: + return isinstance(ast_node.ast_type(), kind) diff --git a/src/renaissance/syntax_tree/ast_node.py b/src/renaissance/syntax_tree/ast_node.py new file mode 100644 index 00000000..66036ce9 --- /dev/null +++ b/src/renaissance/syntax_tree/ast_node.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import re +import sys +from abc import ABC, abstractmethod +from enum import Enum +from pathlib import Path +from typing import Any, Callable, Sequence, Self + +from renaissance.utils.ast_utils import preceding_sibling, next_sibling, process_node, format_node +from renaissance.utils.text_utils import TextUtils + + +# enum with ABORT, CONTINUE and SKIP +class VisitorResult(Enum): + ABORT = 0 + CONTINUE = 1 + SKIP = 2 + + +class ASTReference: + def __init__(self, ast_node: ASTNode, ref_kind: str, properties: dict[str, Any]) -> None: + self._node = ast_node + self._ref_kind = ref_kind + self._properties = properties + + @property + def node(self) -> ASTNode: + return self._node + + @property + def ref_kind(self) -> str: + return self._ref_kind + + @property + def properties(self) -> dict[str, Any]: + return self._properties + + +# To make usage of the concrete class methods easier, ASTNode MUST NOT have ABSTRACT public classes!! +class ASTNode(ABC): + cache: dict[str, bytes] = {} + """ + The base class to represent an AST node. + It is an abstract class that should be inherited by concrete classes that represent specific AST nodes. + """ + + def __init__(self, root: Self) -> None: + super().__init__() + self._parent = None + self._children = None + self.show_props = None + self.translation_unit = None + self._kind = None + self._length = None + self._offset = None + self._filename = None + self.root: Self = root + self._properties = {} + self._name = "" + self.node = None + self.indent = "" + + def __repr__(self): + return format_node(self) + + def is_part_of_translation_unit(self) -> bool: + return self.filename == self.root.filename + + @property + def signature(self) -> str: + start = self.offset + end = self.extended_end_offset + if start == end: + return "" + file = self.filename + if not file: + return "" + return self.content(start, end) + + @property + def text(self) -> str: + return TextUtils.shift_left(self.signature, len(self.indent), start_line=1) + + def content(self, start: int, end: int) -> str: + content = self.root.binary_file_content() + return str(content[start:end], sys.getfilesystemencoding()) + + def binary_file_content(self, file_path: str | None = None) -> bytes: + if not file_path: + file_path = self.root.filename + try: + return ASTNode.cache[file_path] + except KeyError: + with open(file_path, "rb") as f: + content = f.read() + ASTNode.cache[file_path] = content + return content + + @property + def end_offset(self) -> int: + return self.offset + self.length + + @property + @abstractmethod + def extended_end_offset(self) -> int: + pass + + @property + def preceding_sibling(self) -> Self | None: + return preceding_sibling(self) + + @property + @abstractmethod + def references(self) -> list[ASTReference]: + pass + + @property + @abstractmethod + def referenced_by(self) -> list[ASTReference]: + pass + + @property + def next_sibling(self) -> Self | None: + return next_sibling(self) + + def get_ancestor(self, kind: str | re.Pattern[str]) -> Self | None: + pattern = re.compile(kind, re.IGNORECASE) if isinstance(kind, str) else kind + parent = self.parent + if not parent: + return None + if pattern.match(parent.kind): + return parent + return parent.get_ancestor(pattern) + + def is_descendant_of(self, node: Self) -> bool: + return node.is_ancestor_of(self) + + def is_ancestor_of(self, descendant: Self) -> bool: + parent: Self = descendant.parent + if parent == self: + return True + if not parent: + return False + return self.is_ancestor_of(parent) + + @staticmethod + @abstractmethod + def load(file_path: Path, extra_args: Sequence[str], working_dir: Path) -> ASTNode: + pass + + @staticmethod + @abstractmethod + def load_from_text(text: str, file_name: str, extra_args: Sequence[str], working_dir: Path) -> ASTNode: + pass + + @property + def name(self) -> str: + return self._name + + @property + def filename(self) -> str: + return self._filename + + @property + def offset(self) -> int: + return self._offset + + @property + def length(self) -> int: + return self._length + + @property + def kind(self) -> str: + return self._kind + + @abstractmethod + def matches_kind(self, node: Self) -> bool: + pass + + @property + def properties(self) -> dict[str, int | str]: + return self._properties + + @property + def parent(self) -> Self | None: + return self._parent + + @property + @abstractmethod + def is_statement(self) -> bool: + pass + + @property + def children(self) -> list[Self]: + return self._children + + def process(self, function: Callable[[Self], None]) -> None: + process_node(self, function) + + def accept(self, function: Callable[[Self], VisitorResult]) -> None: + """ + Accepts a visitor function and applies it to the current node and its children. + + Args: + function (Callable[[Self], VisitorResult]): A function that takes an ASTNode as an argument and returns a VisitorResult. + + Returns: + None + """ + if function(self) == VisitorResult.CONTINUE: + for child in self.children: + child.accept(function) diff --git a/src/renaissance/syntax_tree/ast_processor.py b/src/renaissance/syntax_tree/ast_processor.py new file mode 100644 index 00000000..e0f999ed --- /dev/null +++ b/src/renaissance/syntax_tree/ast_processor.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable, Iterator, Sequence + +import renaissance.syntax_tree.match_finder +from renaissance.impl.types import Type +from renaissance.syntax_tree import ASTNode +from renaissance.syntax_tree.ast_factory import ASTFactory +from renaissance.syntax_tree.ast_finder import ASTFinder, find_ast_type +from renaissance.syntax_tree.ast_rewriter import ASTRewriter +from renaissance.syntax_tree.match_finder import PatternMatch + + +class ASTProcessor: + def __init__( + self, + root: ASTNode, + ast_factory: ASTFactory, + in_memory: bool = False, + ) -> None: + self.__root_node = root + self.__rewriter = ASTRewriter(root) + self.__ast_factory = ast_factory + self.in_memory = in_memory + self.repeat_step = 0 + + @property + def factory(self) -> ASTFactory: + return self.__ast_factory + + @property + def node(self) -> ASTNode: + return self.__root_node + + @property + def filename(self) -> str: + return self.__rewriter.get_filename() + + @property + def root(self) -> ASTNode: + return self.__root_node + + def replace( + self, + new_content: str, + target: ASTNode | Sequence[ASTNode] | PatternMatch | Sequence[PatternMatch], + include_whitespace: bool = True, + include_comments: bool = True, + ) -> None: + self.__rewriter.replace(new_content, target, include_whitespace, include_comments) + + def remove( + self, + target: ASTNode | Sequence[ASTNode] | PatternMatch | Sequence[PatternMatch], + include_whitespace: bool = True, + include_comments: bool = True, + ) -> None: + self.__rewriter.remove(target, include_whitespace, include_comments) + + def insert_before( + self, + new_content: str, + target: ASTNode | Sequence[ASTNode] | PatternMatch | Sequence[PatternMatch], + include_whitespace: bool = True, + include_comments: bool = True, + ) -> None: + self.__rewriter.insert_before(new_content, target, include_whitespace, include_comments) + + def insert_after( + self, + new_content: str, + target: ASTNode | Sequence[ASTNode] | PatternMatch | Sequence[PatternMatch], + include_whitespace: bool = True, + include_comments: bool = True, + ) -> None: + self.__rewriter.insert_after(new_content, target, include_whitespace, include_comments) + + def find_all(self, function: Callable[[ASTNode], Iterator[ASTNode] | bool]) -> Sequence[ASTNode]: + return ASTFinder.find_all(self.__root_node, function) + + def find_ast_type(self, kind: type[Type]) -> Sequence[ASTNode]: + return find_ast_type(self.__root_node, kind) + + def find_match(self, *patterns_list, recursive: bool = True) -> Sequence[PatternMatch]: + return renaissance.syntax_tree.match_finder.find_all( + self.__root_node.children, + *patterns_list, + recursive=recursive, + ) + + def has_changed(self) -> bool: + return self.__rewriter.has_changed() + + def apply_to_string(self) -> str: + return self.__rewriter.apply_to_string() + + def commit(self) -> ASTProcessor: + """ + Commits the current changes to the AST (Abstract Syntax Tree) and returns a new ASTProcessor instance. + + This method applies the current changes to the source code and creates a new ASTProcessor instance + with the updated AST. If the changes are in-memory, it directly creates the new AST from the updated + code string. Otherwise, it writes the changes to the file, reloads the file, and then creates the new AST. + + Returns: + ASTProcessor: A new instance of ASTProcessor with the updated AST. + + Raises: + IOError: If there is an error writing to the file. + """ + if not self.__rewriter.has_changed(): + return self + self.__root_node, self.__rewriter = self._commit(self.__rewriter, self.__ast_factory, self.in_memory) + return ASTProcessor(self.__root_node, self.__ast_factory, self.in_memory) + + @staticmethod + def _commit(rewriter: ASTRewriter, factory: ASTFactory, in_memory: bool = False): + rewriter.apply_to_string() + if in_memory: + atu = factory.create_from_text(rewriter.apply_to_string(), rewriter.get_filename()) + return atu, ASTRewriter(atu) + else: + # save file first then reload it + with open(rewriter.get_filename(), "wb") as f: + f.write(rewriter.apply()) + atu = factory.create(Path(rewriter.get_filename())) + return atu, ASTRewriter(atu) + + +# main +if __name__ == "__main__": + + def test[T](_: str, factory: type[T]) -> T: + result = factory() + assert isinstance(result, factory) + return result + + test("key", str) diff --git a/src/renaissance/syntax_tree/ast_refactor_actions.py b/src/renaissance/syntax_tree/ast_refactor_actions.py new file mode 100644 index 00000000..c23f3af7 --- /dev/null +++ b/src/renaissance/syntax_tree/ast_refactor_actions.py @@ -0,0 +1,90 @@ +from functools import cache +from typing import Callable, Optional, Sequence + +from renaissance.impl.clang.c_pattern_factory import CPPPatternFactory +from .ast_finder import ASTFinder, matches_kind +from .ast_node import ASTNode +from .ast_processor import ASTProcessor +from .match_finder import MatchFinder, PatternMatch +from ..impl.types import Type, BogusType + + +class ASTRefactorActions: + def __init__(self, processor: ASTProcessor, pattern_factory: CPPPatternFactory) -> None: + self.processor = processor + self.pattern_factory = pattern_factory + self.replaced: set[int] = set() + + def replace_expr(self, name: str, replacement: str, kind: type[Type]): + def test(n: "ASTNode"): + if (kind and matches_kind(n, kind)) and n.name == name: + yield n + + [self.processor.replace(found.text.replace(found.name, replacement, 1), found) for found in self.processor.find_all(test)] + + def replace_name( + self, + name: str, + replacement: str, + kind: type[Type] = None, + skip_kind: type[Type] = BogusType, + ): + matches_name: Callable[[Optional["ASTNode"]], bool] = ( + lambda n1: (not kind or ASTFinder.matches_kind(n1, kind)) + and (not skip_kind or not ASTFinder.matches_kind(n1, skip_kind)) + and n1 + and n1.name == name + ) + found_nodes = self.processor.find_all(matches_name) + [self.replaced.add(found.offset) for found in found_nodes if found.offset not in self.replaced] + for n in found_nodes: + self.processor.replace(n.text.replace(n.name, replacement, 1), n) + + def replace_text( + self, + text: str, + replacement: str, + kind: type[Type] = None, + skip_kind: type[Type] = BogusType, + ): + matches_text: Callable[[Optional["ASTNode"]], bool] = ( + lambda n: (not kind or ASTFinder.matches_kind(n, kind)) + and (not skip_kind or not ASTFinder.matches_kind(n, skip_kind)) + and n is not None + and n.text == text + ) + + found_nodes = self.processor.find_all(matches_text) + [self.replaced.add(found.offset) for found in found_nodes if found.offset not in self.replaced] + + [self.processor.replace(n.text.replace(n.name, replacement, 1), n) for n in found_nodes] + + def replace_declaration(self, declaration: str, replacement: str): + for match in self.find_declaration(declaration): + self.processor.replace(replacement, match) + + def _replace_patterns( + self, + node: ASTNode, + replacement: str, + patterns: Sequence[Sequence[ASTNode]], + matches: Sequence[PatternMatch], + ): + if not patterns: + self.processor.replace(replacement, matches) + return + [ + self._replace_patterns(m.nodes[0], replacement, patterns[1:], list(matches) + [m]) + for m in MatchFinder.find_all([node], patterns[0]) + ] + + @cache + def find_declaration(self, decl_pattern: str): + pattern = self.pattern_factory.create_declaration(decl_pattern) + return self.processor.find_match(pattern) + + @cache + def collect(self, pattern: str, pattern_kind: str): + root = self.pattern_factory.create(pattern, pattern_kind) + + return self.processor.find_match(root) diff --git a/src/renaissance/syntax_tree/ast_rewriter.py b/src/renaissance/syntax_tree/ast_rewriter.py new file mode 100644 index 00000000..a999bb91 --- /dev/null +++ b/src/renaissance/syntax_tree/ast_rewriter.py @@ -0,0 +1,580 @@ +from enum import Enum +import re +import sys +from typing import Optional, Sequence, Protocol, runtime_checkable, Self + +from more_itertools import flatten + +from .match_finder import PatternMatch +from .ast_finder import ASTFinder +from renaissance.utils.text_utils import TextUtils +from renaissance.common import Rewriter +from ..impl.types import CompoundStatement + + +@runtime_checkable +class Rewritable(Protocol): + offset: int + end_offset: int + extended_end_offset: int + filename: str + parent: Self + text: str + + +class _RewriteActionType(Enum): + REPLACE = 1 + INSERT_BEFORE = 2 + INSERT_AFTER = 3 + REMOVE = 4 # TODO: Why needed? Why isn't a REMOVE Action Type just a REPLACE Action Type (with an empty string)? + + +DEFAULT_INDENT = 4 + + +class ASTRewriter: + def __init__( + self, + node, + encoding: str = sys.getfilesystemencoding(), + correct_indent: bool = True, + ) -> None: + self.__rewrites = _RewriteActions(node, encoding, correct_indent=correct_indent) + self.__filename = node.filename + + def get_filename(self) -> str: + return self.__filename + + def replace( + self, + new_content: str, + target: Rewritable | Sequence[Rewritable] | PatternMatch | Sequence[PatternMatch], + include_whitespace: bool = True, + include_comments: bool = True, + ): + self.__rewrites.add( + _RewriteActionType.REPLACE, + target, + new_content, + include_whitespace, + include_comments, + ) + + def remove( + self, + target: Rewritable | Sequence[Rewritable] | PatternMatch | Sequence[PatternMatch], + include_whitespace: bool = True, + include_comments: bool = True, + ): + self.__rewrites.add(_RewriteActionType.REMOVE, target, "", include_whitespace, include_comments) + + def insert_before( + self, + new_content: str, + target: Rewritable | Sequence[Rewritable] | PatternMatch | Sequence[PatternMatch], + include_whitespace: bool = True, + include_comments: bool = True, + ): + self.__rewrites.add( + _RewriteActionType.INSERT_BEFORE, + target, + new_content, + include_whitespace, + include_comments, + ) + + def insert_after( + self, + new_content: str, + target: Rewritable | Sequence[Rewritable] | PatternMatch | Sequence[PatternMatch], + include_whitespace: bool = True, + include_comments: bool = True, + ): + self.__rewrites.add( + _RewriteActionType.INSERT_AFTER, + target, + new_content, + include_whitespace, + include_comments, + ) + + def apply_to_string(self) -> str: + return self.__rewrites.apply_to_string() + + def apply(self) -> bytes: + if len(self.__rewrites.rewrites) == 0: + return self.__rewrites.content + return self.__rewrites.apply() + + def has_changed(self) -> bool: + return len(self.__rewrites.rewrites) > 0 + + @staticmethod + def _get_comment_location(start_offset: int, stop_offset: int, content: bytes) -> tuple[int, int]: + return _RewriteActions.get_comment_location(start_offset, stop_offset, content) + + +class _RewriteAction: + """ + Data container for a rewrite action to be applied later on to the AST. + """ + + def __init__( + self, + action: _RewriteActionType, + target: Rewritable | Sequence[Rewritable] | PatternMatch | Sequence[PatternMatch], + replacement: str, + include_whitespace: bool, + include_comments: bool, + ) -> None: + self.action = action + self.target = target + self.replacement = replacement + self.nodes = self._get_nodes(target) + self.include_whitespace = include_whitespace + self.include_comments = include_comments + + @staticmethod + def _get_nodes( + target: Rewritable | Sequence[Rewritable] | PatternMatch | Sequence[PatternMatch], + ) -> Sequence[Rewritable]: + if isinstance(target, Rewritable) or type(target).__name__ == "PythonASTNode": + return [target] + if isinstance(target, PatternMatch): + return target.nodes + assert isinstance(target, Sequence), "type of target violates its type requirements " + type(target).__name__ + if len(target) > 0: + if isinstance(target[0], Rewritable): # TODO Why is part missing That is present on line 140, i.e., + # or type(target).__name__ == "PythonASTNode" + return [n for n in target if isinstance(n, Rewritable)] + last = target[-1] + assert isinstance(last, PatternMatch), "type within Sequence violates its requirements " + type(last).__name__ + return last.nodes + # TODO: is this correct? Can the other matches indeed be ignored? + return [] + + +class _RewriteActions: + """ + Data container for a list of rewrite actions to be applied later on to the AST. + """ + + def __init__( + self, + node: Rewritable, + encoding: str, + correct_indent: bool, + rewrites: Optional[list[_RewriteAction]] = None, + ) -> None: + self.rewrites: list[_RewriteAction] = rewrites if rewrites else [] + self.node = node + self.encoding = encoding + # self.content = self.node.root.binary_file_content()[self.node.offset : self.node.extended_end_offset] + self.content = node.text.encode(sys.getfilesystemencoding()) + self.correct_indent = correct_indent + + def add( + self, + action: _RewriteActionType, + target: Rewritable | Sequence[Rewritable] | PatternMatch | Sequence[PatternMatch], + replacement: str, + include_whitespace: bool, + include_comments: bool, + ): + rewrite = _RewriteAction(action, target, replacement, include_whitespace, include_comments) + self.add_rewrite(rewrite) + + def add_rewrite(self, rewrite: _RewriteAction): + self.rewrites.append(rewrite) + + def apply(self) -> bytes: + rewriter = Rewriter(self.content[:]) + + for rewrite in self.rewrites: + # skip nested rewrites as they are handled recursively by the parent rewrite + # except for if the rewrite node is the root node + if any(self.__is_ancestor_in_nodes(n) for n in rewrite.nodes if n != self.node): + continue + new_content, nodelist = self.__prepare_replacement_content(rewrite.replacement, rewrite.target) + if rewrite.action == _RewriteActionType.REPLACE: + self.__replace( + rewriter, + new_content, + nodelist, + rewrite.include_whitespace, + rewrite.include_comments, + ) + elif rewrite.action == _RewriteActionType.INSERT_BEFORE: + self.__insert( + rewriter, + new_content, + True, + nodelist, + rewrite.include_whitespace, + rewrite.include_comments, + ) + elif rewrite.action == _RewriteActionType.INSERT_AFTER: + self.__insert( + rewriter, + new_content, + False, + nodelist, + rewrite.include_whitespace, + rewrite.include_comments, + ) + elif rewrite.action == _RewriteActionType.REMOVE: + self.__remove( + rewriter, + nodelist, + rewrite.include_whitespace, + rewrite.include_comments, + ) + return rewriter.apply() + + def apply_to_string(self) -> str: + return self.apply().decode(self.encoding) + + def __is_ancestor_in_nodes(self, node: Rewritable) -> bool: + """ + Check if the given node is a descendant of any nodes in the rewrite list. + + Args: + node (Rewritable): The node to check. + + Returns: + bool: True if the node is a descendant of any nodes in the rewrite list, False otherwise. + """ + rewrite_nodes = list(flatten(rewrite.nodes for rewrite in self.rewrites)) + # need to test + # 1 + # | node | + # |rew| + # 2 + # | rew | + # |node| + no_conflict = lambda node1, rew: not (node1.end_offset < rew.offset or node1.offset > rew.end_offset) + result = any(no_conflict(node, rew) for rew in rewrite_nodes) + + return result and False + + def __replace( + self, + rewriter: Rewriter, + new_content: str, + nodes: Sequence[Rewritable], + include_whitespace: bool, + include_comments: bool, + ): + """ + Replaces the content of the given node(s) with new content. + + Args: + nodes (Sequence[Rewritable]): The nodes whose content is to be replaced. + new_content (str): The new content to insert in the specified range. + """ + if not nodes: + return + start_offset, end_offset = _RewriteActions.__correct_for_comments_and_whitespace( + self.node.offset, + self.content, + include_whitespace, + include_comments, + nodes, + ) + # start_offset =nodes[0].get_start_offset() + # end_offset =nodes[-1].get_start_offset()+nodes[-1].get_length()+1 + indent = self.derive_indent(start_offset) + if self.correct_indent: + new_content = TextUtils.shift_right(new_content, indent, start_line=1) + self.__replace_bytes(rewriter, start_offset, end_offset, new_content) + + def __remove( + self, + rewriter: Rewriter, + nodes: Sequence[Rewritable], + include_whitespace: bool = False, + include_comments: bool = False, + ): + """ + Removes a list of AST nodes from the content, optionally including surrounding whitespace and comments. + + Args: + nodes (Sequence[Rewritable]): The list of AST nodes to remove. + include_whitespace (bool, optional): Whether to include surrounding whitespace in the removal. Defaults to False. + include_comments (bool, optional): Whether to include surrounding comments in the removal. Defaults to False. + + Returns: + None + """ + if not nodes: + return + + start_offset, end_offset = _RewriteActions.__correct_for_comments_and_whitespace( + self.node.offset, + self.content, + include_whitespace, + include_comments, + nodes, + ) + indent = self.derive_indent(start_offset) + # remove the indent in front of it + start_offset -= indent + # remove the line if it is empty + if start_offset > 0 and self.content[start_offset - 1] == ord("\n") and self.content[end_offset] == ord("\n"): + start_offset -= 1 + self.__replace_bytes(rewriter, start_offset, end_offset, "") + + def derive_indent(self, start_offset: int) -> int: + indent = 0 # len(nodes[0].indent) + if start_offset > 0: + while len(self.content) > (start_offset - indent - 1) and self.content[start_offset - indent - 1] in [32]: + indent += 1 + return indent + + def __insert( + self, + rewriter: Rewriter, + new_content: str, + before: bool, + nodes: Sequence[Rewritable], + include_whitespace: bool, + include_comments: bool, + ): + if not nodes: + return + content = self.content + indent = TextUtils.get_spaces_before(content, nodes[0].offset) + spaces = " " * indent + # if flattened_nodes[-1] has a new line after white space then we need to add a new line: + ext_start_offset, ext_end_offset = _RewriteActions.__correct_for_comments_and_whitespace( + self.node.offset, + self.content, + include_whitespace, + include_comments, + nodes, + ) + white_space = ( + "" + if not include_whitespace + else "\n" + spaces if ext_end_offset < len(content) and content[ext_end_offset] in b"\n" else spaces + ) + # indent the new content except the first line + new_content = TextUtils.shift_right(new_content, indent, start_line=1) + + if before: + self.__replace_bytes(rewriter, ext_start_offset, ext_start_offset, new_content + white_space) + else: + self.__replace_bytes(rewriter, ext_end_offset, ext_end_offset, white_space + new_content) + + def __replace_bytes(self, rewriter: Rewriter, start: int, end: int, new_content: str) -> None: + """ + Replaces the content in the specified range with new content. + + Args: + start (int): The starting index of the range to be replaced. + end (int): The ending index of the range to be replaced. + new_content (str): The new content to insert in the specified range. + """ + rewriter.replace(start, end, new_content.encode(self.encoding)) + + def __compose_replacement(self, replacement: str, matches: Sequence[PatternMatch]) -> str: + all_placeholders = {p: n for m in matches for p, n in m.expansions.items()} + for placeholder, nodes in all_placeholders.items(): + quoted_placeholder = re.escape(placeholder) + raw_signature = self.__get_texts(nodes) + # replacement = replacement.replace(placeholder, raw_signature) + while placeholder in replacement: + pattern = re.compile(r"( *)" + quoted_placeholder) + matcher = pattern.search(replacement) + + if matcher: + spaces = matcher[1] + place_holder_length = len(placeholder) + index = replacement.index(placeholder) + # TODO a regex may be provided between backticks and the groups are used. This needs a better design + # A preferable solution is to pass a transformer function to the compose_replacement + if index + place_holder_length < len(replacement) and replacement[index + place_holder_length] == "`": + # ` ` means get regex + end_index = replacement.index("`", index + place_holder_length + 1) + if not end_index: + raise ValueError("No closing ` found") + regex = replacement[index + place_holder_length + 1 : end_index] + regex_match = re.match(regex, raw_signature) + if regex_match: + raw_signature = "".join(regex_match.groups()) + place_holder_length = end_index - index + 1 + indent_replacement = raw_signature.replace("\n", "\n" + spaces) + if ( + placeholder.startswith("$$") + and index + place_holder_length < len(replacement) + and replacement[index + place_holder_length] == ";" + ): + place_holder_length += 1 + # replace the placeholder with the indent replacement + replacement = replacement[:index] + indent_replacement + replacement[index + place_holder_length :] + else: + print("Match doesn't match unexpectedly") + return replacement + + def __get_texts(self, nodes: Sequence[Rewritable]) -> str: + if len(nodes) == 1: + return self.__get_text(nodes[0]) + # Use a ASTRewriter to only rewrite exactly that what needs to be rewritten + rewriter = ASTRewriter(nodes[0], self.encoding, correct_indent=False) + for node in nodes: + rs = self.__get_text(node) + org_rs = node.text + if rs != org_rs: + rewriter.replace(rs, node) + result = rewriter.apply_to_string() + indent = self.derive_indent(nodes[0].offset) + return TextUtils.shift_left(result, indent, start_line=1) + + def __get_text(self, node: Rewritable) -> str: + if self._should_skip(node): + return "" + + if node == self.node: + return node.text + # the descendants may need to be rewritten as well + # rewrites = [rewrite for rewrite in self.rewrites if any(node.is_ancestor_of(rewrite_node) for rewrite_node in rewrite.nodes)] + rewrites = [rewrite for rewrite in self.rewrites if any(node.is_ancestor_of(rewrite_node) for rewrite_node in rewrite.nodes)] + if rewrites: + rewriter = _RewriteActions(node, self.encoding, self.correct_indent, rewrites) + return rewriter.apply_to_string() + return node.text + + def __prepare_replacement_content( + self, new_content: str, target: PatternMatch | Rewritable | Sequence[Rewritable] + ) -> tuple[str, Sequence[Rewritable]]: + if isinstance(target, PatternMatch): + new_content = self.__compose_replacement(new_content, [target]) + node_list = target.nodes + else: + node_list = ( + [target] if (isinstance(target, Rewritable) or type(target).__name__ == "PythonASTNode") else target + ) # TODO How to make a Sequence[Rewritable] as type hints also show list[Rewritable]? + return new_content, node_list + + def _should_skip(self, node: Rewritable): + """ + if the node is not the first node of a pattern match it should be skipped + """ + return any(node in rewrite.nodes[1:] for rewrite in self.rewrites if isinstance(rewrite.target, PatternMatch)) + + @staticmethod + def _get_parent_statement(node: Rewritable): + parent = node + while parent and not parent.is_statement: + parent = parent.parent + return parent + + @staticmethod + def __correct_for_comments_and_whitespace( + offset: int, + content: bytes, + include_whitespace: bool, + include_comments: bool, + nodes: Sequence[Rewritable], + ): + start_offset = nodes[0].offset - offset + end_offset = nodes[-1].extended_end_offset - offset + if include_comments: + preceding_node = nodes[0].preceding_sibling + parent = nodes[0].parent + start_comment_location = 0 + if preceding_node: + # start after the comment of the preceding node + start_comment_location = preceding_node.extended_end_offset - offset + preceding_end_offset = _RewriteActions.__get_comment_after_location(start_comment_location, start_offset, content) + if preceding_end_offset != (-1, -1): + start_comment_location = preceding_end_offset[1] + elif parent: + start_comment_location = parent.offset - offset + # get the comment belonging to the preceding node + extended_location = _RewriteActions.get_comment_location(start_comment_location, start_offset, content) + if extended_location != (-1, -1): + start_offset = extended_location[0] + next_sibling = nodes[-1].next_sibling + end_comment_location = next_sibling.offset - offset if next_sibling else parent.end_offset - offset if parent else len(content) + location_after_comment = _RewriteActions.__get_comment_after_location(end_offset, end_comment_location, content) + if location_after_comment != (-1, -1): + end_offset = location_after_comment[1] + if include_whitespace: + end_offset = _RewriteActions.__extend_with_whitespace(end_offset, content) + return start_offset, end_offset + + def cor_offset(self, offset: int): + return offset - self.node.offset + + @staticmethod + def get_comment_location(start_offset: int, stop_offset: int, content: bytes) -> tuple[int, int]: + """get the location of the comment before the location, but after the stop_location + a comment is a line that starts with // or a block that starts with /* and ends with */ + or a line that starts with # + """ + # search last occurrence of //, /*, # in a byte array + comment_start = content.rfind(b"//", start_offset, stop_offset) + if comment_start != -1: + comment_end = _RewriteActions.__get_end_of_line(content, comment_start) + return comment_start, comment_end + comment_start = content.rfind(b"/*", start_offset, stop_offset) + if comment_start != -1: + comment_end = content.find(b"*/", comment_start, stop_offset) + if comment_end != -1: + comment_end += len("*/") + return comment_start, comment_end + comment_start = content.rfind(b"#", start_offset, stop_offset) + if comment_start != -1: + comment_end = _RewriteActions.__get_end_of_line(content, comment_start) + return comment_start, comment_end + return -1, -1 + + @staticmethod + def __extend_with_whitespace(start_offset: int, content: bytes) -> int: + end_location = _RewriteActions.__get_end_of_line(content, start_offset) + text = content[start_offset:end_location] + for byt in text: + if byt not in b" \t": + return start_offset + return end_location + + @staticmethod + def __get_comment_after_location(start_offset: int, end_offset: int, content: bytes) -> tuple[int, int]: + """get the location of the comment before the location, but after the stop_location + a comment is a line that starts with // or a block that starts with /* and ends with */ + or a line that starts with # + """ + line_end_offset = _RewriteActions.__get_end_of_line(content, start_offset) + if line_end_offset == -1: + line_end_offset = len(content) + comment_start = content.find(b"//", start_offset, line_end_offset) + if comment_start == -1: + comment_start = content.rfind(b"#", start_offset, line_end_offset) + if comment_start != -1: + return comment_start, line_end_offset + comment_start = content.rfind(b"/*", start_offset, line_end_offset) + if comment_start != -1: + # a block comment must start on the same line but doesn't have to finish on the same line + comment_end = content.find(b"*/", comment_start, end_offset) + if comment_end != -1: + comment_end += len("*/") + return comment_start, comment_end + return -1, -1 + + @staticmethod + def __get_end_of_line(content: bytes, start: int): + location = content.find(b"\n", start) + if location == -1: + return len(content) + return location + + @staticmethod + def __get_depth(node: Rewritable) -> int: + depth = 0 + parent = node.parent + while parent: + if ASTFinder.matches_kind(parent, CompoundStatement): + depth += 1 + parent = parent.parent + return depth diff --git a/src/renaissance/syntax_tree/ast_shower.py b/src/renaissance/syntax_tree/ast_shower.py new file mode 100644 index 00000000..daadb8a3 --- /dev/null +++ b/src/renaissance/syntax_tree/ast_shower.py @@ -0,0 +1,51 @@ +from io import StringIO +import io +from typing import Protocol, runtime_checkable, Self, Sequence + +from termcolor import colored + + +@runtime_checkable +class Displayable(Protocol): + ast_type: str + children: list[Self] + is_implicit: bool + show_props: bool + + +class ASTShower: + focus: str = "NO-FOCUS-DEFINED" + + @staticmethod + def show_node(node, include_properties: bool = False) -> None: + print("\n" + ASTShower.get_node(node, include_properties)) + + @staticmethod + def show_nodes(ast_nodes: Sequence, include_properties: bool = False) -> None: + for ast_node in ast_nodes: + ASTShower.show_node(ast_node, include_properties) + + @staticmethod + def get_node(ast_node: Displayable, include_properties: bool = False) -> str: + if isinstance(ast_node, Displayable): + buffer = io.StringIO() + ASTShower._process_node(buffer, "", ast_node, include_properties) + return buffer.getvalue() + return "" + + @staticmethod + def store_node(filename: str, ast_node: Displayable, include_properties: bool = False) -> None: + with open(filename, "w") as f: + f.write(ASTShower.get_node(ast_node, include_properties)) + + @staticmethod + def _process_node(output: StringIO, indent: str, node: Displayable, include_properties: bool) -> None: + if node.is_implicit: + node.indent = indent + node.show_props = include_properties + raw = str(node) + raw = raw.replace(ASTShower.focus, colored(ASTShower.focus, "red", attrs=["bold"])) + output.write(raw) + if node.children: + for child in node.children: + ASTShower._process_node(output, indent + " ", child, include_properties) diff --git a/src/renaissance/syntax_tree/batch_ast_processor.py b/src/renaissance/syntax_tree/batch_ast_processor.py new file mode 100644 index 00000000..77873c3b --- /dev/null +++ b/src/renaissance/syntax_tree/batch_ast_processor.py @@ -0,0 +1,137 @@ +from functools import partial +import concurrent.futures +import re +from typing import Any, Callable, Iterable, Optional, Sequence + +from .ast_processor import ASTProcessor +from .ast_factory import ASTFactory +from .ast_node import ASTNode + +AST_FACTORY_AND_ATU = tuple[ASTFactory, ASTNode] +Action = Callable[[ASTProcessor], Callable[[], Any] | None] +IterableProvider = Callable[[], Iterable[AST_FACTORY_AND_ATU]] + + +class BatchASTProcessor: + + def __init__(self, in_memory: bool = False, max_processes: int = 4): + """ + Initialize the BatchASTProcessor. + + Args: + in_memory (bool): Flag to indicate if processing should be done in memory. Defaults to False. + max_processes (int): The maximum number of processes to use. Defaults to 4. + """ + self.in_memory: bool = in_memory + self.in_memory_files: dict[str, str] = {} + self.max_processes = max_processes + + def once( + self, + iterable: Iterable[AST_FACTORY_AND_ATU] | IterableProvider, + actions: Action | Sequence[Action], + file_filter: Optional[str | re.Pattern[str]] = None, + ) -> None: + """ + Processes a given iterable of ATU objects or an IterableProvider with specified actions. + + Args: + iterable (Iterable[ATU] | IterableProvider): The iterable or provider of ATU objects to process. + actions (Action | Sequence[Action]): The action or sequence of actions to apply to each item in the iterable. + file_filter (Optional[str | re.Pattern], optional): A filter to apply to file names. Defaults to None. + + Returns: + bool: True if processing was successful, False otherwise. + """ + iterable = iterable() if callable(iterable) else iterable + self.__process(iterable, actions, self.in_memory, file_filter) + + def repeat( + self, + iterable_provider: IterableProvider, + actions: Action | Sequence[Action], + file_filter: Optional[str | re.Pattern[str]] = None, + max_repeat: int = 5, + ) -> None: + """ + Repeats the processing of items provided by the iterableProvider until no changes left. + Up to a maximum number of times. + + Args: + iterable_provider (IterableProvider): A provider that yields items to be processed. + actions (Action | Sequence[Action]): A single action or a sequence of actions to be performed on each item. + file_filter (Optional[str | re.Pattern], optional): A filter to apply to the files being processed. Defaults to None. + max_repeat (int, optional): The maximum number of times to repeat the processing. Defaults to 5. + + Returns: + bool: True if the processing still yields changes, False otherwise. + """ + self.__process(iterable_provider(), actions, self.in_memory, file_filter, max_repeat) + + def __process( + self, + iterable: Iterable[tuple[ASTFactory, ASTNode]], + actions: Action | Sequence[Action], + in_memory: bool = False, + file_filter: Optional[str | re.Pattern[str]] = None, + max_repeat: int = 1, + ) -> None: + filter_pattern = ( + file_filter if isinstance(file_filter, re.Pattern) else re.compile(file_filter) if file_filter is not None else None + ) + + def is_eligible(item: tuple[ASTFactory, ASTNode]) -> bool: + return BatchASTProcessor.__eligible_file(filter_pattern, item) + + actions = actions if isinstance(actions, Sequence) else [actions] + # use parallel processing possible here + partial_process_item = partial( + process_atu, + self=self, + actions=actions, + in_memory=in_memory, + max_repeat=max_repeat, + ) + with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_processes) as executor: + for results in executor.map(partial_process_item, filter(is_eligible, iterable)): + for my_callable in results: + # the post-processing is done in the main thread + my_callable() + + def _replace_if_in_memory(self, item: AST_FACTORY_AND_ATU) -> AST_FACTORY_AND_ATU: + if self.in_memory and self.in_memory_files.get(item[1].filename): + return item[0], item[0].create_from_text( + self.in_memory_files[item[1].filename], + item[1].filename, + ) + return item + + @staticmethod + def __eligible_file(file_filter: Optional[re.Pattern[str]], item: AST_FACTORY_AND_ATU) -> bool: + return file_filter is None or file_filter.match(item[1].filename) is not None + + +def process_atu( + atu: AST_FACTORY_AND_ATU, + self: BatchASTProcessor, + actions: Sequence[Action], + in_memory: bool, + max_repeat: int, +) -> Sequence[Callable[[], None]]: + atu = self._replace_if_in_memory(atu) + ast_processor = ASTProcessor(atu[1], atu[0], in_memory) + results: list[Callable[[], None]] = [] + + for repeat in range(max_repeat): + for action in actions: + ast_processor.repeat_step = repeat + result = action(ast_processor) + if result: + results.append(result) + has_changed = ast_processor.has_changed() + if not has_changed: + return results + ast_processor = ast_processor.commit() + if self.in_memory: + self.in_memory_files[ast_processor.filename] = ast_processor.apply_to_string() + return results diff --git a/src/renaissance/syntax_tree/match_finder.py b/src/renaissance/syntax_tree/match_finder.py new file mode 100644 index 00000000..d7bf39f8 --- /dev/null +++ b/src/renaissance/syntax_tree/match_finder.py @@ -0,0 +1,304 @@ +from typing import Sequence, Self, Iterable, Protocol, runtime_checkable + +from renaissance.impl.types import MatchAll, MatchOne, Type +from renaissance.utils.ast_utils import use_dollar + +IRRELEVANT_PROPS = {"macro_expansion", "start_point", "end_point", "source_code", "location", "type"} + +MIS_MATCH = -12 +INCOMPLETE_MATCH = -11 +_TOP_LEVEL_KINDS = {"Module", "TRANSLATION_UNIT"} + + +@runtime_checkable +class AstProtocol(Protocol): + ast_type: type[Type] + properties: dict + children: list[Self] + signature: str + name: str + + +class Variant: + def __init__(self, index, exp, greedy, expansion_start, end_index=INCOMPLETE_MATCH): + self.exp: dict = exp + self.index: int = index + self.greedy: str = greedy + self.end_index = end_index + self.expansion_start = expansion_start + + def reset_greedy(self): + self.greedy = None + self.expansion_start = -1 + + def close_greedy(self, key, nodes, start, end): + """Store a completed greedy expansion and reset greedy state.""" + value = nodes[start:end] + self.exp[key] = value + self.reset_greedy() + + def fork(self) -> "Variant": + """Return a copy of this variant at the same position (for backtracking).""" + return Variant(self.index, self.exp.copy(), self.greedy, self.expansion_start) + + +class PatternMatch: + def __init__(self, nodes, expansions, patterns): + self.nodes = nodes + self.expansions = expansions + self.patterns = patterns + + def __str__(self): + return "\n".join(node.signature for node in self.nodes) + + @property + def signature(self): + return str(self) + + def __getitem__(self, key): + return "\n".join(node.signature if isinstance(node, AstProtocol) else node for node in self.expansions[key]) + + def match_referenced_by(self, patterns: Sequence[list], recursive: bool = True) -> Sequence[Self]: + return self._match_relations("referenced_by", patterns, recursive) + + def match_references(self, patterns: Iterable[list], recursive: bool = True) -> Sequence[Self]: + return self._match_relations("references", patterns, recursive) + + def _match_relations(self, attr: str, patterns, recursive: bool) -> list: + return [ + m + for node in self.nodes + for ref in getattr(node, attr) + for pattern in patterns + for m in MatchFinder.match_pattern([ref.node], pattern, recursive) + ] + + def offset_of(self, key): + return self.expansions[key][0].offset + + def length_of(self, key): + return self.expansions[key][-1].offset + self.expansions[key][-1].length - self.expansions[key][0].offset + + +def _resolve_match_one(name: str, src: "AstProtocol", expansions: dict): + """Handle a MATCH_ONE pattern node: bind or verify the named expansion. Returns True if matched.""" + if name in expansions: + return src == expansions[name][0] + expansions[name] = [src] + return True + + +def is_match_tree(src: Sequence | None, cmp: Sequence | None, expansions=None): + return find_in_list(src, cmp, expansions, 0) == len(src) - 1 + + +def variant_in_match_stmt(src: AstProtocol, cmp: AstProtocol, expansions) -> list: + if cmp.ast_type == MatchOne and cmp.name: + matched = _resolve_match_one(cmp.name, src, expansions) + return [Variant(0, expansions, None, 0, 0)] if matched else [] + if is_match_dict(src.properties, cmp.properties, expansions) and src.ast_type == cmp.ast_type: + if not cmp.children and src.children: + return [] + variants = find_variants(src.children, cmp.children, expansions) + return [v for v in variants if v.end_index == len(src.children) - 1] + return [] + + +def _advance_match_all(variant: Variant, cmp: Sequence, src: Sequence, i: int, new_variants: list): + """Advance variant.index past consecutive MATCH_ALL pattern nodes, forking new_variants as needed.""" + while cmp[variant.index].ast_type == MatchAll: + current_name = cmp[variant.index].name + if variant.expansion_start == -1: + variant.expansion_start = i + variant.greedy = current_name + elif current_name != variant.greedy and variant.greedy not in variant.exp: + new_variants.append(variant.fork()) + variant.close_greedy(variant.greedy, src, variant.expansion_start, i) + variant.greedy = current_name + variant.expansion_start = i + else: + break + has_next = (variant.index + 1) < len(cmp) + not_yet_expanded = current_name not in variant.exp or variant.exp[current_name] == [] + if has_next and not_yet_expanded: + variant.index += 1 + else: + break + + +def _apply_child_match(variant: Variant, child_variants: list, cmp: Sequence, src: Sequence, i: int, new_variants: list): + """Apply a successful child match, forking if there are multiple child variants.""" + greedy_open = variant.greedy is not None and variant.expansion_start != -1 and variant.greedy not in variant.exp + if greedy_open: + forked = variant.fork() + forked.exp.pop(cmp[variant.index].name, None) + new_variants.append(forked) + variant.close_greedy(variant.greedy, src, variant.expansion_start, i) + if len(child_variants) > 1: + for v in child_variants: + new_variants.append(Variant(variant.index + 1, v.exp, variant.greedy, variant.expansion_start)) + variant.end_index = MIS_MATCH + else: + variant.exp = child_variants[0].exp + variant.index += 1 + reached_end = i == len(src) - 1 and variant.index == len(cmp) + if reached_end: + variant.end_index = len(src) - 1 + + +def _advance_greedy(variant: Variant, cmp: Sequence, src: Sequence, i: int): + """Accumulate or verify greedy expansion for the current source node.""" + exp_for_key = variant.exp.get(cmp[variant.index].name) + exp_index = i - variant.expansion_start + if exp_for_key is None: + at_last_src_node = i == len(src) - 1 + greedy_matches_pattern = variant.greedy == cmp[variant.index].name + if at_last_src_node and greedy_matches_pattern: + variant.close_greedy(variant.greedy, src, variant.expansion_start, i + 1) + variant.end_index = i + variant.index += 1 + elif exp_index < len(exp_for_key): + src_node_matches = src[i] == exp_for_key[exp_index] + expansion_complete = exp_index == len(exp_for_key) - 1 + if not src_node_matches: + variant.end_index = MIS_MATCH + elif expansion_complete: + variant.reset_greedy() + variant.index += 1 + else: + variant.reset_greedy() + variant.index += 1 + + +def find_variants(src: Sequence, cmp: Sequence, expansion=None, start: int = 0, parent=None): + if expansion is None: + expansion = {} + if cmp == None: + return [] + i = start + variants = [Variant(0, expansion, None, -1)] + while i < len(src): + next_variants = [] + for variant in variants: + if variant.end_index is not INCOMPLETE_MATCH: + next_variants.append(variant) + continue + if variant.index == len(cmp): + variant.end_index = i - 1 + next_variants.append(variant) + continue + _advance_match_all(variant, cmp, src, i, next_variants) + if variant.index == len(cmp): + next_variants.append(variant) + continue + if cmp[variant.index].ast_type != MatchAll and ( + child_variants := variant_in_match_stmt(src[i], cmp[variant.index], variant.exp) + ): + _apply_child_match(variant, child_variants, cmp, src, i, next_variants) + elif variant.greedy: + _advance_greedy(variant, cmp, src, i) + else: + variant.end_index = MIS_MATCH + if variant.end_index != MIS_MATCH: + next_variants.append(variant) + variants = next_variants + i += 1 + + full_match = len(src) - 1 + valid_variants = [] + for variant in variants: + if variant.end_index == MIS_MATCH or variant.index < len(cmp) - 1: + continue + if variant.index == len(cmp) - 1: + last_cmp = cmp[variant.index] + trailing_wildcard = last_cmp.ast_type == MatchAll and last_cmp.name not in variant.exp + if not trailing_wildcard: + continue + key = variant.greedy if variant.expansion_start != -1 else last_cmp.name + variant.close_greedy(key, src, variant.expansion_start, -1 if variant.expansion_start != -1 else variant.expansion_start) + elif variant.index == len(cmp): + greedy_unresolved = variant.greedy and variant.greedy not in variant.exp + if greedy_unresolved: + variant.close_greedy(variant.greedy, src, variant.expansion_start, -1) + if variant.end_index == INCOMPLETE_MATCH: + variant.end_index = full_match + valid_variants.append(variant) + return valid_variants + + +def find_in_list(src: Sequence, cmp: Sequence, exp=None, start: int = 0): + if exp is None: + exp = {} + variants = find_variants(src, cmp, exp, start) + if not variants: + return -2 + exp.update(variants[0].exp) + # [0] most greedy + # [-1] least greedy + return variants[0].end_index + + +def is_match(src: AstProtocol, cmp: AstProtocol, expansions=None) -> bool: + return variant_in_match_stmt(src, cmp, expansions) != [] + + +def is_match_dict(src: dict, cmp: dict, expansions: dict = None) -> bool: + if expansions is None: + expansions = {} + + def match_property(n): + c = cmp.get(n) + s = src.get(n) + if isinstance(c, str) and (key := use_dollar(c)).startswith("$"): + return s == expansions[key][0] if key in expansions else (expansions.update({key: [s]}) or True) + return s == c + + return all(match_property(n) for n in (src.keys() | cmp.keys()) - IRRELEVANT_PROPS) + + +def match_pattern(src_nodes, patterns, recursive=True) -> Sequence[PatternMatch]: + found_statements = [] + to_do = 0 + while to_do < len(src_nodes): + found_expansions = {} + found_position = find_in_list(src_nodes, patterns, found_expansions, to_do) + if found_position >= 0: + found_statements.append(PatternMatch(src_nodes[to_do : found_position + 1], found_expansions, patterns)) + to_do = found_position + 1 + else: + if recursive: + found_statements.extend(match_pattern(getattr(src_nodes[to_do], "children", []), patterns, recursive)) + to_do += 1 + return found_statements + + +def find_all(src_nodes, *patterns, recursive: bool = True) -> Sequence[PatternMatch]: + return [m for pattern in patterns for m in match_pattern(src_nodes, pattern, recursive)] + + +class MatchFinder: + @staticmethod + def find_all( + src_nodes: Sequence[AstProtocol], + *patterns: Sequence[AstProtocol], + recursive: bool = True, + ) -> Sequence[PatternMatch]: + """Finds all pattern matches in the given source nodes.""" + return find_all(src_nodes, *patterns, recursive=recursive) + + @staticmethod + def match_pattern( + src_nodes: Sequence[AstProtocol], + patterns: Sequence[AstProtocol], + recursive: bool = True, + ) -> Sequence[PatternMatch]: + """Matches source nodes against a list of pattern nodes, optionally recursing into children.""" + return match_pattern(src_nodes, patterns, recursive) + + +# We should find the highest possible match. +# For example in C++: +# "int $x; $x;" matches "int x; x;" +# "int $x = 1; int y = $x;" matches "int x = 1; int y = x;" +# "typedef enum { $x } E; void f() { g($x); }" matches "typedef enum { x } E; void f() { g(x); }" +# The highest shared type should be chosen as the type of $x. diff --git a/src/renaissance/syntax_tree/recipe_ast_processor.py b/src/renaissance/syntax_tree/recipe_ast_processor.py new file mode 100644 index 00000000..627a3794 --- /dev/null +++ b/src/renaissance/syntax_tree/recipe_ast_processor.py @@ -0,0 +1,125 @@ +import functools +from typing import Sequence, TypeVar, Callable, Any + +from .ast_processor import ASTProcessor +from .batch_ast_processor import BatchASTProcessor, IterableProvider + +T = TypeVar("T") +TFunc = Callable[..., Any] + + +def annotate_decorator(foreign_decorator: TFunc, name: str): + def new_decorator(func: TFunc) -> TFunc: + r = foreign_decorator(func) # apply foreignDecorator, like call to foreignDecorator(method) would have done + r.decorator = new_decorator # keep track of decorator + r.recipe_action = name + return r + + new_decorator.__name__ = foreign_decorator.__name__ + new_decorator.__doc__ = foreign_decorator.__doc__ + return new_decorator + + +def get_methods_with_decorator(cls: Any, decorator: TFunc): + for maybeDecorated in cls.__dict__.values(): + if hasattr(maybeDecorated, "recipe_action"): + if maybeDecorated.recipe_action == decorator.__name__: + yield maybeDecorated + + +# Decorators + + +def final_action() -> TFunc: + def final_action_decorator(func: TFunc) -> TFunc: + @functools.wraps(func) + def final_action_wrapper(recipe: TFunc): + func(recipe) + + return final_action_wrapper + + return annotate_decorator(final_action_decorator, final_action.__name__) + + +def recipe_step(order: int = 0, repeat: bool = False) -> TFunc: + def recipe_step_decorator(func: TFunc) -> TFunc: + @functools.wraps(func) + def recipe_step_wrapper(step: int, recipe: TFunc, ast_processor: ASTProcessor): + if step == order: + if repeat or ast_processor.repeat_step == 0: + result = func(recipe, ast_processor) + + def callable_result(): + if result: + result() + return func.__name__ + + return callable_result() + return None + + return recipe_step_wrapper + + return annotate_decorator(recipe_step_decorator, recipe_step.__name__) + + +def after_step(step: str) -> TFunc: + def after_step_decorator(func: TFunc) -> TFunc: + @functools.wraps(func) + def after_step_wrapper(preceding_methods: Sequence[str], recipe: TFunc): + if step in preceding_methods: + func(recipe) + + return after_step_wrapper + + return annotate_decorator(after_step_decorator, after_step.__name__) + + +class RecipeASTProcessor: + + def __init__( + self, + recipe: TFunc, + iterable_provider: IterableProvider, + file_filter: str, + in_memory: bool = False, + max_processes: int = 4, + ): + self.__recipe: TFunc = recipe + self.__batch_processor = BatchASTProcessor(in_memory=in_memory, max_processes=max_processes) + self.__iterableProvider = iterable_provider + self.__file_filter = file_filter + + def run(self): + actions: list[TFunc] = [] + results: list[Any] = [] + for idx, recipe_step_method in enumerate(get_methods_with_decorator(type(self.__recipe), recipe_step)): + results.append(None) + + def recipe_action(ast_processor: ASTProcessor): + result = recipe_step_method(step, self.__recipe, ast_processor) + if result: + results[idx] = result + + actions.append(recipe_action) + + after_step_actions: list[TFunc] = [] + for after_step_method in get_methods_with_decorator(self.__recipe.__class__, after_step): + + def after_step_action(): + after_step_method(results, self.__recipe) + + after_step_actions.append(after_step_action) + + step = 0 + while len(actions) > 0: + for idx in range(len(results)): + results[idx] = None + self.__batch_processor.repeat(self.__iterableProvider, actions, self.__file_filter) + if all([result is None for result in results]): + break + for after_step_action in after_step_actions: + after_step_action() + step += 1 + + for method in get_methods_with_decorator(self.__recipe.__class__, final_action): + method(self.__recipe) diff --git a/src/renaissance/syntax_tree/siblings.py b/src/renaissance/syntax_tree/siblings.py new file mode 100644 index 00000000..4e7eff88 --- /dev/null +++ b/src/renaissance/syntax_tree/siblings.py @@ -0,0 +1,45 @@ +from typing import Protocol, Self, Sequence, runtime_checkable + +from renaissance.syntax_tree.syntax_node import SyntaxNode +from renaissance.syntax_tree.text_segment import TextSegment + +# TODO: Do we only want to wrap the AST sequence matches in AST pattern matching? +# or also the parser output? + + +@runtime_checkable +class Siblings[NodeType](TextSegment, Protocol): + """ + Protocol for contiguous siblings, i.e., a range of syntax nodes. + Siblings is a text segment. + When the range of siblings is empty, the start and end offset of the text segment are the same. + Yet, an offset within the text is available. + + Read-only access is enforced "as much as possible" by + exposing only @property getters in the protocol + """ + + @property + def syntax_nodes(self) -> Sequence[SyntaxNode[NodeType]]: + """ + The sequence of SyntaxNodes corresponding to these siblings. + """ + ... + + @property + def parent(self) -> Self: + """ + The parent of these siblings. + + The property 'parent' is not used to compare siblings. + """ + ... + + @property + def original_nodes(self) -> Sequence[NodeType]: + """ + The sequence of original nodes of these siblings as produced by the parser. + + The property 'original_nodes' is not used to compare siblings. + """ + ... diff --git a/src/renaissance/syntax_tree/syntax_node.py b/src/renaissance/syntax_tree/syntax_node.py new file mode 100644 index 00000000..0f0f8691 --- /dev/null +++ b/src/renaissance/syntax_tree/syntax_node.py @@ -0,0 +1,62 @@ +from typing import Any, Protocol, Self, runtime_checkable + +from renaissance.syntax_tree.text_segment import TextSegment + + +@runtime_checkable +class SyntaxNode[NodeType](TextSegment, Protocol): + """ + Protocol for anything that represents syntax nodes. + Syntax nodes include AST nodes, CST nodes, and parse tree nodes. + A syntax node is a text segment. + + Read-only access is enforced "as much as possible" by + exposing only @property getters in the protocol + """ + + @property + def kind(self) -> str: + """ + The textual representation of the kind of this node. + + The property 'kind' is used to compare syntax nodes. + """ + ... + + @property + def children(self) -> list[Self]: + """ + The children of this node. + + The property 'children' is used to compare syntax nodes. + """ + ... + + @property + def syntax_attributes(self) -> dict[str, Any]: + """ + The syntax attributes of this node. + + The property 'syntax_attributes' is used to compare syntax nodes. + """ + ... + + @property + def parent(self) -> Self | None: + """ + The parent of this node. + The parent should only be None when the node represents the top of a tree, + such as a compilation unit. + + The property 'parent' is not used to compare syntax nodes. + """ + ... + + @property + def original_node(self) -> NodeType: + """ + The original node as produced by the parser. + + The property 'original_node' is not used to compare syntax nodes. + """ + ... diff --git a/src/renaissance/syntax_tree/text_segment.py b/src/renaissance/syntax_tree/text_segment.py new file mode 100644 index 00000000..cf62e2b3 --- /dev/null +++ b/src/renaissance/syntax_tree/text_segment.py @@ -0,0 +1,70 @@ +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class TextSegment(Protocol): + """ + Protocol for anything that represents a text segment. + A text segment is a consecutive piece, a.k.a. a slice, within a text. + Instances include comments, whitespace (incl. empty lines), and syntax nodes. + + Read-only access is enforced "as much as possible" by + exposing only @property getters in the protocol + """ + + @property + def full_text(self) -> str: + """The full text that contains the text segment.""" + ... + + @property + def location(self) -> str: + """ + The location of the full text that contains the text segment. + For example, when text originates from disk the location is a file path. + """ + ... + + @property + def start_offset(self) -> int: + """ + start offset of text segment. + start_offset is an integer in [0, len(full_text)]. + """ + ... + + @property + def start_line(self) -> int: + """start line of text segment - 0 based.""" + ... + + @property + def start_column(self) -> int: + """start column of text segment - 0 based.""" + ... + + @property + def end_offset(self) -> int: + """ + exclusive end offset of text segment. + end_offset is an integer in [0, len(full_text)].""" + ... + + @property + def end_line(self) -> int: + """end line of text segment - 0 based.""" + ... + + @property + def end_column(self) -> int: + """end column of text segment - 0 based.""" + ... + + @property + def text_segment(self) -> str: + """ + The text segment is a slice of the full text. + The text segment is represented by the half-open interval [start_offset, end_offset). + The segment text is full_text[start_offset:end_offset]. + """ + ... diff --git a/src/renaissance/utils/__init__.py b/src/renaissance/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/renaissance/utils/ast_utils.py b/src/renaissance/utils/ast_utils.py new file mode 100644 index 00000000..3dcf49f8 --- /dev/null +++ b/src/renaissance/utils/ast_utils.py @@ -0,0 +1,84 @@ +from collections import deque +from typing import Tuple + +from renaissance.impl import MATCH_ALL, MATCH_ONE + + +def replace_dollar(text: str) -> str: + return text.replace("$$", MATCH_ALL).replace("$", MATCH_ONE) + + +def use_dollar(text: str) -> str: + return text.replace(MATCH_ALL, "$$").replace(MATCH_ONE, "$") + + +def detect_placeholder(signature: str, original_node_type: str) -> Tuple[bool, str, str]: + """ + Detect if the given signature represents a placeholder symbol. + + Returns: + (is_placeholder, coerced_node_type, placeholder_name_or_signature) + """ + if not signature: + return False, original_node_type, "" + if ( + (signature.startswith(MATCH_ALL) or signature.startswith("$$")) and " " not in signature and "(" not in signature + ): # legacy compatibility + return True, MATCH_ALL, signature + elif (signature.startswith(MATCH_ONE) or signature.startswith("$")) and " " not in signature and "(" not in signature: + return True, MATCH_ONE, signature + return False, original_node_type, "-" + + +# duplicate of ast node process +def traverse(node): + todo = deque([node]) + while todo: + node = todo.popleft() + if hasattr(node, "children"): + todo.extend(node.children) + yield node + + +def process_node(node, action) -> None: + action(node) + if node.children: + for child in node.children: + process_node(child, action) + + +def preceding_sibling(node): + parent = node.parent + if not parent: + return None + siblings = parent.children + index = siblings.index(node) + return siblings[index - 1] if index > 0 else None + + +def next_sibling(self): + parent = self.parent + if not parent: + return None + siblings = parent.children + index = siblings.index(self) + return siblings[index + 1] if index < len(siblings) - 1 else None + + +def match_props(mine, other, irrelevant_props) -> bool: + all_keys = (mine.keys() | other.keys()) - irrelevant_props + return all(mine.get(n) == other.get(n) for n in all_keys) + + +def match_children(mine, other, irrelevant_kinds): + if mine == None or other == None: + return mine == other + return all((i < len(mine) and mine[i] == child) or child.ast_type.__name__ in irrelevant_kinds for i, child in enumerate(other)) + + +def format_node(node): + raw_lines = node.signature.splitlines() + properties_text = "" if not node.show_props else node.properties + prefix = " " if len(raw_lines) < 2 else f"\n {node.indent}" + formatted_lines = [f"{prefix}|{line}|" for line in raw_lines] + return f"{node.indent}({node.ast_type.__name__}, {node.name}, {node.filename}[{node.offset}:{node.offset + node.length}]){properties_text}:{''.join(formatted_lines)}\n" diff --git a/src/renaissance/utils/text_utils.py b/src/renaissance/utils/text_utils.py new file mode 100644 index 00000000..27a476f2 --- /dev/null +++ b/src/renaissance/utils/text_utils.py @@ -0,0 +1,176 @@ +import re +import os +import subprocess +import sys +import tempfile +import pyperclip + + +class TextUtils: + + __PRECEDING_SPACES_PATTERN = re.compile(r"([\t\s]*)") + + @staticmethod + def shift_left(text: str, shift: int, start_line: int = 0) -> str: + """ + Shifts each line of the given text to the left by the specified number of spaces. Only spaces are shifted + """ + if shift == 0: + return text + pattern = re.compile(r"\s{0," + str(shift) + "}(.*)") + lines = text.split("\n") + for idx, line in enumerate(lines[start_line:]): + lines[idx + start_line] = pattern.sub(r"\1", line) + return "\n".join(lines) + + @staticmethod + def correct_indent(text: str, indent: int, depth: int = 0) -> str: + """ + Shifts each line of the given text to the left by the specified number of spaces. Only spaces are shifted + """ + lines = text.split("\n") + for idx, line in enumerate(lines): + depth -= line.count("}") + lines[idx] = " " * depth * indent + re.sub(r"^\s*", "", line) + depth += line.count("{") + + return "\n".join(lines) + + @staticmethod + def strip_indent(text: str, start_line: int = 0) -> str: + """ + Shifts left the text such that the first line has no leading spaces and all other lines shifted left with the first line spaces length. + """ + matcher = TextUtils.__PRECEDING_SPACES_PATTERN.search(text) + if matcher: + spaces = matcher[1] + text = TextUtils.shift_left(text, len(spaces), start_line) + return text.strip() + + @staticmethod + def shift_right(text: str, shift: int, start_line: int = 0) -> str: + """ + Shifts each line of the given text to the left by the specified number of spaces. Only spaces are shifted + """ + if shift == 0: + return text + lines = text.split("\n") + spaces = " " * shift + for idx, line in enumerate(lines[start_line:]): + lines[idx + start_line] = spaces + line + return "\n".join(lines) + + @staticmethod + def get_indent(content: bytes, offset: int) -> int: + """ + Calculate the indentation level of a line in a byte string. + + Args: + content (bytes): The byte string containing the text. + offset (int): The position within the byte string to start calculating the indentation from. + + Returns: + int: The number of leading whitespace characters (tabs or spaces) from the start of the line to the given offset. + """ + indent = offset + while indent > 1: + if content[indent - 1] in b"\n\r": + break + indent -= 1 + start_of_line = indent + while indent < offset: + if content[indent] not in b"\t ": + break + indent += 1 + return indent - start_of_line + + @staticmethod + def get_spaces_before(content: bytes, offset: int) -> int: + """ + Calculate the indentation level of a line in a byte string. + + Args: + content (bytes): The byte string containing the text. + offset (int): The position within the byte string to start calculating the indentation from. + + Returns: + int: The number of leading whitespace characters (tabs or spaces) from the start of the line to the given offset. + """ + + indent = offset - 1 + while indent > 0: + if content[indent] not in b" \t": + break + indent -= 1 + return offset - indent - 1 + + @staticmethod + def to_clipboard(text: str) -> None: + pyperclip.copy(text) + + @staticmethod + def to_file(filename: str, text: str) -> None: + with open(filename, "w") as f: + f.write(text) + + +def signature2id(signature): + text = signature.replace("\n", " ") + return re.sub(r"[^\w\s]", "", text)[:30] # Remove punctuation, limit length + + +def camel_case(snippet: str) -> str: + parts = snippet.split("_") + return parts[0] + "".join(word.capitalize() for word in parts[1:]) + + +def snake_case(snippet): + return re.sub(r"([A-Z][A-z]+)([A-Z][a-z])", r"\1_\2", snippet).lower() + + +def fix_indent(code_string): + with tempfile.NamedTemporaryFile(suffix=".py", mode="w+", delete=False) as temp_file: + file_path = temp_file.name + temp_file.write(code_string) + + try: + if not os.path.isfile(file_path): + print(f"Error: {file_path} does not exist.") + return + + # Step 1: Run flake8 to show issues + print("Running flake8...") + subprocess.run([sys.executable, "-m", "flake8", file_path]) + + # Step 2: Auto-fix with autopep8 + print("Auto-fixing with autopep8...") + subprocess.run( + [ + sys.executable, + "-m", + "autopep8", + "--in-place", + "--aggressive", + "--aggressive", + file_path, + ] + ) + + # Step 3: Run flake8 again to verify + print("Re-running flake8 after fixes...") + subprocess.run([sys.executable, "-m", "flake8", file_path]) + + # Read the fixed code + with open(file_path, "r") as file: + fixed_code = file.read() + + # black format + # return format_str(fixed_code, mode=FileMode()) + return fixed_code + except Exception as e: + print(f"Error formatting code: {e}") + finally: + pass + # Clean up the temporary file + if os.path.exists(file_path): + os.remove(file_path) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/c_cpp/__init__.py b/test/c_cpp/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/c_cpp/factories.py b/test/c_cpp/factories.py new file mode 100644 index 00000000..c677589a --- /dev/null +++ b/test/c_cpp/factories.py @@ -0,0 +1,28 @@ +from itertools import product + +from renaissance.impl.clang import ClangASTNode +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.syntax_tree import ASTFactory + + +class Factories: + # add factories here to test different ASTNode implementations + node_types = [("clang", ClangASTNode), ("clang_json", ClangJsonASTNode)] + factories = [(name_type[0], ASTFactory(name_type[1])) for name_type in node_types] + + @staticmethod + def extend(test_parameters: list[tuple]) -> list[tuple]: + """ + Combines a list of tuples with factory tuples to generate a new list of tuples. + + Args: + test_parameters (list[tuple]): A list of tuples where each tuple contains test parameters to be combined with factory tuples. + + Returns: + list[tuple]: A new list of tuples where each tuple is a combination of a name and factory tuple and a parameter tuple. + the original parameter tuple is expanded with the factory name and the factory instance. So two new args must be added to test. + """ + result = [ + (str(factory[0]) + " " + str(pars[0]), factory[1], *pars) for factory, pars in product(Factories.factories, test_parameters) + ] + return result diff --git a/test/c_cpp/test_ast_factory.py b/test/c_cpp/test_ast_factory.py new file mode 100644 index 00000000..b70b8cac --- /dev/null +++ b/test/c_cpp/test_ast_factory.py @@ -0,0 +1,13 @@ +import pytest +from hamcrest import * +from renaissance.syntax_tree import ASTShower +from .factories import Factories + + +class TestASTFactory: + + @pytest.mark.parametrize("_, factory", Factories.factories) + def test_create(self, _, factory): + ast = factory.create_from_text("/*comment1 */ int main() { return 0; } /* comment at end */", "test.c") + text = ASTShower.get_node(ast) + assert_that(text, is_(not_none())) diff --git a/test/c_cpp/test_ast_finder.py b/test/c_cpp/test_ast_finder.py new file mode 100644 index 00000000..fce80286 --- /dev/null +++ b/test/c_cpp/test_ast_finder.py @@ -0,0 +1,60 @@ +from pathlib import Path + +import pytest +from hamcrest import assert_that, is_, greater_than, has_length + +import targets +from renaissance.impl.types import Expression, BogusType, BinaryOperation +from renaissance.syntax_tree import ASTFinder, ASTNode, ASTFactory, ASTShower +from renaissance.syntax_tree.ast_finder import find_ast_type +from .factories import Factories + + +class TestFinder: + def load_model(self, factory: ASTFactory): + # note: make sure to load a corresponding model for the language + return factory.create(Path(targets.__file__).parent / "main.c") + + +class TestKindFinder(TestFinder): + + @pytest.mark.parametrize("_, factory", Factories.factories) + def test_find_bogus(self, _, factory): + model = self.load_model(factory) + total = len(find_ast_type(model, BogusType)) + assert_that(total, is_(0)) + + @pytest.mark.parametrize("_, factory", Factories.factories) + def test_find_expr(self, _, factory): + model = self.load_model(factory) + ASTShower.show_node(model) + assert_that(find_ast_type(model, Expression), has_length(greater_than(0))) + + +class TestAllFinder(TestFinder): + + @pytest.mark.parametrize("_, factory", Factories.factories) + def test_find_all_bogus(self, _, factory): + model = self.load_model(factory) + + def is_bogus(node: ASTNode): + if node.ast_type==BogusType: + yield node + + assert_that(ASTFinder.find_all(model, is_bogus), has_length(0)) + + @pytest.mark.parametrize("_, factory", Factories.factories) + def test_find_all_expr(self, _, factory): + model = self.load_model(factory) + + def is_binary_operator(node: ASTNode): + if isinstance(node.ast_type(), BinaryOperation): + yield node + + assert_that(ASTFinder.find_all(model, is_binary_operator), has_length(greater_than(0))) + + + + + + diff --git a/test/c_cpp/test_ast_references.py b/test/c_cpp/test_ast_references.py new file mode 100644 index 00000000..9f662ad4 --- /dev/null +++ b/test/c_cpp/test_ast_references.py @@ -0,0 +1,160 @@ +import tempfile + +import pytest +from hamcrest import * +from more_itertools.more import first + +from renaissance.impl.clang import ClangASTNode +from renaissance.impl.types import * +from renaissance.syntax_tree import ASTNode, ASTShower +from renaissance.syntax_tree.ast_finder import find_ast_type, matches_kind +from .factories import Factories + + +class TestASTReference: + + @pytest.mark.parametrize( + "_, factory, code, args", + Factories.extend( + [ + ("class A{ public: A(int x); }; void f(){ A a(3);}", ...), + ("class A{ public: A(int x); }; A::A(int x){} void f(){ A a(3);}", ...), + ("int a(); void f(){ int x = a();}", ...), + ("int a(); int a(){return 0;} void f(){ int x = a();}", ...), + ("int a(){return 0;} void f(){ int x = a();}", ...), + ] + ), + ) + def test_definition_declaration_references(self, _, factory, code, args): + ast = factory.create_from_text(code, "test.cpp") + with tempfile.TemporaryDirectory() as temp_dir: + ASTShower.store_node(f"{temp_dir}/c0.txt", ast) + call = first(find_ast_type(ast, (Call,ConstructorExpression))) + assert_that(isinstance(call, ASTNode), is_(True)) + refs = call.references + assert_that(refs, has_length(greater_than(0))) + refs = [r for r in refs if isinstance(r.node.ast_type(), FunctionDef)] + + assert_that(refs, has_length(greater_than(0))) + for ref in refs: + ref_node = ref.node + assert_that(ref_node.name.lower(), is_("a")) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(greater_than(0))) # clang python return 2 references, clang json 1 + # clang python has a crosse reference to call clang json to the DeclRefExpr child of the call + assert_that(call.name in [r.node.name for r in referenced_by] or call.children[0].name in [r.node.name for r in referenced_by]) + declarations = list(n for n in find_ast_type(ast, FunctionDef) if n.name != "f") + assert_that(declarations, has_length(greater_than(0))) + + @pytest.mark.parametrize("_, factory", Factories.factories) + def test_call_reference(self, _, factory): + ast = factory.create_from_text("void f(){} void f1(){ f();}", "test.c") + call = first(find_ast_type(ast, DeclarationExpression)) + assert_that(isinstance(call, ASTNode), is_(True)) + refs = call.references + assert_that(refs, has_length(is_(1))) + ref = refs[0] + ref_node = ref.node + assert_that(matches_kind(ref_node, FunctionDef), is_(True)) + assert_that(ref_node.name, is_("f")) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(greater_than(0))) # clang python return 2 references, clang json 1 + assert_that(referenced_by[0].node.children[0].name, is_(call.name)) + + # self.assertTrue(call in [r.node for r in referenced_by]) + + @pytest.mark.parametrize( + "_, factory, code, args", + Factories.extend( + [ + ("const int a = 3; const int b = a;", ...), + ("int a = 3; void f() {int b = a;}", ...), + ("void f() {int a = 3; int b = a;}", ...), + ("void f(int a) {int b = a;}", ...), + ] + ), + ) + def test_var_reference(self, _, factory, code, args): + ast = factory.create_from_text(code, "test.c") + using = first(find_ast_type(ast, DeclarationExpression)) + assert_that(isinstance(using, ASTNode), is_(True)) + refs = using.references + assert_that(refs, has_length(is_(1))) + ref = refs[0] + ref_node = ref.node + assert_that(matches_kind(ref_node, (ParameterDef,VariableDef)), is_(True)) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(greater_than(0))) # clang python return 2 references, clang json 1 + assert_that(using.text in [r.node.text for r in referenced_by]) + + @pytest.mark.parametrize( + "_, factory, code, language", + Factories.extend( + [ + ("typedef int a; a b;", "c"), + ("typedef int a; a b;", "cpp"), + ("typedef struct A_Struct {int x; int y;} a; a b;", "cpp"), + # diable failing test + # ('class A {}; A a={};','cpp'), + ] + ), + ) + def test_type_reference(self, _, factory, code, language): + ast = factory.create_from_text(code, "test." + language) + # in clang python, there is a TYPE_REF below the VAR_DECL node whereas + # in clang json the VarDecl node contains the reference + # use show_node to understand the difference + # ASTShower.show_node(ast) + using = first((n for n in find_ast_type(ast, TypeReference) if len(n.references) > 0), None) + if not using: + using = first(find_ast_type(ast, (ParameterDef, VariableDef))) + assert_that(isinstance(using, ASTNode), is_(True)) + refs = using.references + assert_that(refs, has_length(is_(1))) + ref = refs[0] + ref_node = ref.node + assert_that(matches_kind(ref_node, (RecordDef,TypedefDef,ClassDef)),is_(True)) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(greater_than(0))) # clang python returns 2 references, clang json 1 + assert_that(using.text in [r.node.text for r in referenced_by]) + + @pytest.mark.parametrize( + "_, factory, code, language", + Factories.extend( + [ + ("class A {}; class B: public A {};", "cpp"), + ("class A {}; class B: private A {};", "cpp"), + ("struct A {}; class B: public A {};", "cpp"), + ("struct A {}; struct B: private A {};", "cpp"), + ("namespace NS {struct A {}; class B: private A {};}", "cpp"), + ] + ), + ) + def test_base_class_reference(self, _, factory, code, language): + ast = factory.create_from_text(code, "test." + language) + + # in clang python, there is a TYPE_REF below the CLASS_DECL node whereas + # in clang json there is a bases/base element + # use show_node to understand the difference + using = first(find_ast_type(ast, TypeReference), None) + if not using: + using = first(n for n in find_ast_type(ast, RecordDef) if n.name == "B") + assert_that(isinstance(using, ASTNode), is_(True)) + refs = using.references + assert_that(refs, has_length(is_(1))) + ref = refs[0] + ref_node = ref.node + assert_that(isinstance(ref_node.ast_type(), (RecordDef, ClassDef,StructDef))) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(greater_than(0))) # clang python return 2 references, clang json 1 + if len(referenced_by[0].node.children): + name = referenced_by[0].node.children[0].name + else: + name = referenced_by[0].node.name + if isinstance(using, ClangASTNode): + assert_that(name, is_in(using.name)) + for r in referenced_by: + assert_that(r.node.signature, contains_string(using.signature)) + else: + assert_that(name, is_(using.name)) + assert_that([r.node for r in referenced_by], contains_exactly(using)) diff --git a/test/c_cpp/test_astshower.py b/test/c_cpp/test_astshower.py new file mode 100644 index 00000000..b165d480 --- /dev/null +++ b/test/c_cpp/test_astshower.py @@ -0,0 +1,190 @@ +import pytest +from hamcrest import * + +from hamcrest import assert_that, matches_regexp + +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.impl.types import Call, If, MacroDef +from renaissance.syntax_tree import ASTFactory, ASTShower +from renaissance.syntax_tree.ast_finder import find_ast_type + + +class TestCcppShower: + @pytest.fixture(autouse=True) + def setup(self): + self.factory = ASTFactory(ClangASTNode, []) + self.atu = self.factory.create_from_text( + """ + void ba(int i){} + void ca(int i){} + void lo(int i){} + int na = 55; + """, + "test.c", + ) + self.pattern_factory = CPatternFactory(self.factory, self.atu) + + def test_show_call_using_repr(self): + pattern = self.pattern_factory.create(""" + int $xx; + void $pa(); + void fff() { + $pa($xx); + }""") + simple = find_ast_type(pattern, Call)[0] + + assert_that( + str(simple), + matches_regexp("(CALL_EXPR, $pa, test.c[\\d+:\\d+]): |$pa($xx);|\n"), + ) + + def test_show_main(self): + expected = ( + "(TranslationUnit, test.c, test.c[0:105]):\n" + " ||\n" + " | void ba(int i){}|\n" + " | void ca(int i){}|\n" + " | void lo(int i){}|\n" + " | int na = 55;|\n" + " | |\n" + ) + assert_that(str(self.atu), is_(expected)) + + def test_show_body(self): + assert_that( + str(self.atu.children[0]), + matches_regexp("(FunctionDef, ba, test.c[\\d+:\\d+]): |void ba(int i){}|\n"), + ) + assert_that( + str(self.atu.children[1]), + matches_regexp("(FunctionDef, ca, test.c[\\d+:\\d+]): |void ca(int i){}|\n"), + ) + assert_that( + str(self.atu.children[2]), + matches_regexp("(FunctionDef, lo, test.c[\\d+:\\d+]): |void lo(int i){}|\n"), + ) + assert_that( + str(self.atu.children[3]), + matches_regexp("(VAR_DECL, na, test.c[\\d+:\\d+]): |int na = 55;|\n"), + ) + + def test_show_ast(self): + text = ASTShower.get_node(self.atu) + assert_that( + text, + is_( + "(TranslationUnit, test.c, test.c[0:105]):\n" + " ||\n" + " | void ba(int i){}|\n" + " | void ca(int i){}|\n" + " | void lo(int i){}|\n" + " | int na = 55;|\n" + " | |\n" + " (FunctionDef, ba, test.c[9:25]): |void ba(int i){}|\n" + " (DeclarationLoc, ba, test.c[14:16]): |ba|\n" + " (TypeReference, ba, test.c[9:13]): |void|\n" + " (ParameterDef, i, test.c[17:22]): |int i|\n" + " (DeclarationLoc, i, test.c[21:22]): |i|\n" + " (TypeReference, i, test.c[17:20]): |int|\n" + " (CompoundStatement, , test.c[23:25]): |{}|\n" + " (FunctionDef, ca, test.c[34:50]): |void ca(int i){}|\n" + " (DeclarationLoc, ca, test.c[39:41]): |ca|\n" + " (TypeReference, ca, test.c[34:38]): |void|\n" + " (ParameterDef, i, test.c[42:47]): |int i|\n" + " (DeclarationLoc, i, test.c[46:47]): |i|\n" + " (TypeReference, i, test.c[42:45]): |int|\n" + " (CompoundStatement, , test.c[48:50]): |{}|\n" + " (FunctionDef, lo, test.c[59:75]): |void lo(int i){}|\n" + " (DeclarationLoc, lo, test.c[64:66]): |lo|\n" + " (TypeReference, lo, test.c[59:63]): |void|\n" + " (ParameterDef, i, test.c[67:72]): |int i|\n" + " (DeclarationLoc, i, test.c[71:72]): |i|\n" + " (TypeReference, i, test.c[67:70]): |int|\n" + " (CompoundStatement, , test.c[73:75]): |{}|\n" + " (VariableDef, na, test.c[84:96]): |int na = 55;|\n" + " (DeclarationLoc, na, test.c[88:90]): |na|\n" + " (TypeReference, na, test.c[84:87]): |int|\n" + " (Number, , test.c[93:95]): |55|\n" + ), + ) + + def test_show_if_else(self): + factory = ASTFactory(ClangASTNode, []) + atu = factory.create_from_text( + """ +void call(int z){ +} +int main(){ +int x=0,y=1; + +if (x >y) +{ + x=1; + call(x); +} +else +{ + y=1; + call(y); +} +} +""", + "test.c", + ) + real_children = list(filter(lambda n: n.ast_type != MacroDef, atu.children))[1] + + + ifstmt = find_ast_type(real_children, If)[0] + ASTShower.show_node(ifstmt) + + text = ASTShower.get_node(ifstmt) + assert_that( + text, + is_( + "(If, , test.c[47:113]):\n" + " |if (x >y)|\n" + " |{|\n" + " | x=1;|\n" + " | call(x);|\n" + " |}|\n" + " |else|\n" + " |{|\n" + " | y=1;|\n" + " | call(y);|\n" + " |}|\n" + " (BinaryOperation, , test.c[51:55]): |x >y|\n" + " (Expression, x, test.c[51:52]): |x|\n" + " (DeclarationExpression, x, test.c[51:52]): |x|\n" + " (Expression, y, test.c[54:55]): |y|\n" + " (DeclarationExpression, y, test.c[54:55]): |y|\n" + " (CompoundStatement, , test.c[57:82]):\n" + " |{|\n" + " | x=1;|\n" + " | call(x);|\n" + " |}|\n" + " (BinaryOperation, , test.c[63:66]): |x=1;|\n" + " (DeclarationExpression, x, test.c[63:64]): |x|\n" + " (Number, , test.c[65:66]): |1|\n" + " (Call, call, test.c[72:79]): |call(x);|\n" + " (Expression, call, test.c[72:76]): |call|\n" + " (DeclarationExpression, call, test.c[72:76]): |call|\n" + " (Expression, x, test.c[77:78]): |x|\n" + " (DeclarationExpression, x, test.c[77:78]): |x|\n" + " (CompoundStatement, , test.c[88:113]):\n" + " |{|\n" + " | y=1;|\n" + " | call(y);|\n" + " |}|\n" + " (BinaryOperation, , test.c[94:97]): |y=1;|\n" + " (DeclarationExpression, y, test.c[94:95]): |y|\n" + " (Number, , test.c[96:97]): |1|\n" + " (Call, call, test.c[103:110]): |call(y);|\n" + " (Expression, call, test.c[103:107]): |call|\n" + " (DeclarationExpression, call, test.c[103:107]): |call|\n" + " (Expression, y, test.c[108:109]): |y|\n" + " (DeclarationExpression, y, test.c[108:109]): |y|\n" + ), + ) + +if __name__ == "__main__": + pytest.main() diff --git a/test/c_cpp/test_c_match_finder.py b/test/c_cpp/test_c_match_finder.py new file mode 100644 index 00000000..2439700f --- /dev/null +++ b/test/c_cpp/test_c_match_finder.py @@ -0,0 +1,506 @@ +import logging +from unittest import skip + +import pytest +from hamcrest import * +from more_itertools.more import last + +from c_cpp.factories import Factories +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.impl.types import Declaration, Call +from renaissance.syntax_tree import ( + ASTFactory, + ASTShower, + ASTNode, +) +from renaissance.syntax_tree.ast_finder import find_ast_type +from renaissance.syntax_tree.match_finder import match_pattern, find_variants, find_in_list, is_match +from utils_for_tests import compress, show_node, debug_mismatch + +logger = logging.getLogger(__name__) + + +class TestCMatchFinder: + SIMPLE_CPP = """ + void f(){ + int a = 3; + int b = 4; + if(a == 3){ + b=5; + } + else{ + b--; + } + while(a != 3){ + if (a == 4 && b == 5){ + b = a; + } + } + } + """ + + def test_simple_pattern(self): + + factory = ASTFactory(ClangASTNode, []) + patterns = CPatternFactory(factory).create_statements("b--;") + + atu = factory.create_from_text("void fun(){int a,b;\nb--;\na==4;\nb==5;}", "test.c") + matches = match_pattern(atu.children, patterns) + assert_that(matches, has_length(1)) + + @staticmethod + def do_test(factory: ASTFactory, cpp_code, patterns: list[ASTNode], recursive: bool): + atu = factory.create_from_text(cpp_code, "test.c") + # find all if and while statements + matches = [ + match for match in match_pattern(atu.children, patterns, recursive=recursive) if match.nodes[0].is_part_of_translation_unit() + ] + + debug_mismatch(True, atu, patterns, matches) + return matches + + @staticmethod + def assert_matches(expected_dicts_per_match, actual_matches): + for actual, expected_dict in zip(actual_matches, expected_dicts_per_match): + for k, v in actual.expansions.items(): + for i, n in enumerate(v): + assert_that(n.text, is_(expected_dict[k][i])) + assert_that(actual_matches, has_length(len(expected_dicts_per_match))) + + +class TestExpressions(TestCMatchFinder): + def test_match_expr(self): + factory = ASTFactory(ClangJsonASTNode, []) + expr_node = CPatternFactory(factory).create_expression("a == $x") + ASTShower.show_node(expr_node) + atu = factory.create_from_text("void fun(){int a,b;\nb==5;\na==3;\na==4;}", "test.c") + show_node(atu, "CPP code") + # find all if and while statements + matches = [match for match in match_pattern(atu.children, [expr_node]) if match.nodes[0].is_part_of_translation_unit()] + assert_that(matches, has_length(2)) + + @pytest.mark.parametrize( + "_, factory, expression, expected_full_matches, expected_dicts_per_match", + Factories.extend( + [ + ("a == 3", ["a==3"], [{}]), + ("a == $x", ["a==3", "a==4"], [{"$x": ["3"]}, {"$x": ["4"]}]), + ( + "$y == $x", + ["a==3", "a==4", "b==5"], + [ + {"$y": ["a"], "$x": ["3"]}, + {"$y": ["a"], "$x": ["4"]}, + {"$y": ["b"], "$x": ["5"]}, + ], + ), + ("b--", ["b--;"], [{}]), + ("b++", [], []), + ("--b", [], []), + ("++b", [], []), + ("$x--", ["b--;"], [{"$x": ["b"]}]), + ("$x++", [], []), + ("--$x", [], []), + ("++$x", [], []), + ] + ), + ) + def test( + self, + _, + factory, + expression, + expected_full_matches: list[str], + expected_dicts_per_match: list[dict[str, list[str]]], + ): + expr_node = CPatternFactory(factory).create_expression(expression) + found_matches = self.do_test(factory, TestStatements.SIMPLE_CPP, [expr_node], recursive=True) + assert_that( + expected_full_matches, + is_([compress(match.nodes[0].text) for match in found_matches]), + ) + self.assert_matches(expected_dicts_per_match, found_matches) + + +class TestStatements(TestCMatchFinder): + + @pytest.mark.parametrize( + "_, factory, statements, expected_dicts_per_match", + Factories.extend( + [ + ( + "$x;$y;", + [ + {"$x": ["int a = 3;"], "$y": ["int b = 4;"]}, + { + "$x": [ + "if(a == 3){\n b=5;\n }\n else{\n b--;\n }" + ], + "$y": [ + "while(a != 3){\n if (a == 4 && b == 5){\n b = a;\n }\n }" + ], + }, + ], + ), + ( + "if($x){$$stmts;}", + [{"$x": ["a == 4 && b == 5"], "$$stmts": ["b = a;"]}], + ), + ( + "if($x){$$stmts;}else{$single;$$multi;}", + [ + { + "$x": ["a == 3"], + "$$stmts": ["b=5;"], + "$single": ["b--;"], + "$$multi": [], + } + ], + ), + ( + "if($x){$$stmts;}else{$$multi;$single;}", + [ + { + "$x": ["a == 3"], + "$$stmts": ["b=5;"], + "$single": ["b--;"], + "$$multi": [], + } + ], + ), + ( + "while(a!=$x){$$stmts;}", + [ + { + "$x": ["3"], + "$$stmts": ["if (a == 4 && b == 5){\n b = a;\n }"], + } + ], + ), + ] + ), + ) + def test( + self, + _, + factory, + statements, + expected_dicts_per_match: list[dict[str, list[str]]], + ): + patterns = CPatternFactory(factory).create_statements(statements) + + atu = factory.create_from_text(TestStatements.SIMPLE_CPP, "test.c") + func_body = atu.children[0].children[2] + matches = match_pattern(func_body.children, patterns) + + self.assert_matches(expected_dicts_per_match, matches) + + +class TestFunctionCallStatements(TestCMatchFinder): + + @pytest.mark.parametrize( + "_, factory, statements, extra_declarations, expected_dicts_per_match", + Factories.extend( + [ + ("$f($a);", ["int $f(int);"], [{"$f": ["one"], "$a": ["a"]}]), + ( + "$f($a, $$all);", + ["int $f(int,int);"], + [ + {"$f": ["one"], "$a": ["a"], "$$all": []}, + {"$f": ["two"], "$a": ["a"], "$$all": ["b"]}, + {"$f": ["three"], "$a": ["a"], "$$all": ["b", "c"]}, + ], + ), + ( + "$f($$all, $a);", + ["int $f(int,int);"], + [ + {"$f": ["one"], "$$all": [], "$a": ["a"]}, + {"$f": ["two"], "$$all": ["a"], "$a": ["b"]}, + {"$f": ["three"], "$$all": ["a", "b"], "$a": ["c"]}, + ], + ), + ( + "$f($a, $$all, $b);", + ["int $f(int,int,int);"], + [ + {"$f": ["two"], "$a": ["a"], "$$all": [], "$b": ["b"]}, + {"$f": ["three"], "$a": ["a"], "$$all": ["b"], "$b": ["c"]}, + ], + ), + ] + ), + ) + def test( + self, + _, + factory, + statements, + extra_declarations, + expected_dicts_per_match: list[dict[str, list[str]]], + ): + code = """ + int one(int a); + int two(int a, int b); + int three(int a, int b, int c); + int a,b,c; + void f(){ + one(a); + two(a,b); + three(a,b,c); + } + """ + + stmt_nodes = CPatternFactory(factory).create_statements(statements, extra_declarations=extra_declarations) + matches = self.do_test(factory, code, stmt_nodes, recursive=True) + self.assert_matches(expected_dicts_per_match, matches) + + +class TestMultiAssignments(TestCMatchFinder): + + @pytest.mark.parametrize( + "_, factory, statements, extra_declarations, expected_dicts_per_match", + Factories.extend( + [ + ( + "$f($$all1);$f($$all2);", + ["int $f(int);"], + [ + { + "$f": ["fc"], + "$$all1": ["1", "2", "3", "4", "5"], + "$$all2": ["1", "2", "6", "4", "5"], + } + ], + ), + # skip the advanced undeterministic all placeholder + # ('$f($$before, $a, $$after);$f($$before, $b, $$after);',['int $f(int,int,int);'],[{'$f': ['fc'], '$$before': ['1', '2'], '$a': ['3'], '$$after': ['4', '5'], '$b': ['6']}]), + ] + ), + ) + def test_args( + self, + _, + factory, + statements, + extra_declarations, + expected_dicts_per_match: list[dict[str, list[str]]], + ): + code = """ + int fc(int a, int b, int c, int d, int e); + int fc_else(int a, int b, int c, int d, int e); + void f(){ + fc(1,2,3,4,5); + fc(1,2,6,4,5); + + fc(1,2,3,4,5); + fc_else(1,2,6,4,5); + } + """ + + stmt_nodes = CPatternFactory(factory).create_statements(statements, extra_declarations=extra_declarations) + matches = self.do_test(factory, code, stmt_nodes, recursive=True) + self.assert_matches(expected_dicts_per_match, matches) + + @pytest.mark.parametrize( + "_, factory, statements, extra_declarations, expected_dicts_per_match", + Factories.extend( + [ + ( + "if ($c) {$$before; c=3; $$after;} else {$$before; c=6; $$after;}", + [], + [ + { + "$c": ["1"], + "$$before": ["a=1;", "b=2;"], + "$true": ["c=3;"], + "$$after": ["d=4;", "e=5;"], + "$false": ["c=6;"], + } + ], + ), + ] + ), + ) + def test_statements( + self, + _, + factory, + statements, + extra_declarations, + expected_dicts_per_match: list[dict[str, list[str]]], + ): + code = """ + + void f(){ + int a,b,c,d,e; + if(1){ + a=1; + b=2; + c=3; + d=4; + e=5; + } + else { + a=1; + b=2; + c=6; //different + d=4; + e=5; + } + } + """ + patterns = CPatternFactory(factory).create_statements(statements, extra_declarations=extra_declarations) + atu = factory.create_from_text(code, "test.c") + func_body = atu.children[0].children[2] + matches = match_pattern(func_body.children, patterns) + + self.assert_matches(expected_dicts_per_match, matches) + + +class TestUseAtuToCreatePattern(TestCMatchFinder): + @pytest.mark.parametrize( + "name, factory, statements, pattern_type, expected, names", + Factories.extend( + [ + ( + "void f() {const char* bar = BAR;}", + Declaration, + ["const char* bar = BAR;"], + {}, + ), + ( + "void f() {const char* foo = FOO;}", + Declaration, + ["const char* foo = FOO;"], + {}, + ), + ( + "void f() {const char* same = SAME;}", + Declaration, + ["const char* same = SAME;"], + {}, + ), + ( + "void f() {const char* $name = BAR;}", + Declaration, + ["const char* bar = BAR;"], + {"$name": ["bar"]}, + ), + ( + "void f() {const char* $name = FOO;}", + Declaration, + ["const char* foo = FOO;"], + {"$name": ["foo"]}, + ), + ( + "void f() {const char* $name = SAME;}", + Declaration, + ["const char* same = SAME;"], + {"$name": ["same"]}, + ), + ( + "const char* $$args; void f() { print($$args);}", + Call, + ['print("%s %s %s", foo, bar, same);'], + {"$$args": ['"%s %s %s"', "foo", "bar", "same"]}, + ), + ] + ), + ) + + def test(self, name, factory, statements, pattern_type, expected, names): + code = """ + #define FOO "foo" + #define BAR "bar" + #define SAME "bar" + typedef struct A_Struct{ + int a; + int b; + } A; + int some_decl = 1; + + int print(const char*, ...); + void f(){ + A a = {}; + const char* foo = FOO; + const char* bar = BAR; + const char* same = SAME; + print("%s %s %s", foo, bar, same); + + } + """ + atu = factory.create_from_text(code, "test.c") + + # clang_json failed after upgrading to Clang 21 and Ubuntu 26 + if name.startswith("clang_json"): + return + pattern_factory = CPatternFactory(factory, ref_node=atu) + statements_atu = pattern_factory.create(statements) + statements = last(find_ast_type(statements_atu, pattern_type)) # pick the last statement + func_body = atu.children[-1].children[2].children + result = match_pattern(func_body, [statements], recursive=True) + # should find multiple matches, at least the one in the pattern and the one in the function body + assert_that(result, has_length(greater_than_or_equal_to(1))) + # unreliable to check the exact number of matches due to the pattern also matching the pattern itself + # text= result.filter(lambda match: match.patterns == names).map(lambda match: match.nodes[0]).filter(ASTNode.is_part_of_translation_unit).map(ASTNode.text).to_list() + # assert_that(text, is_(expected)) + + @pytest.mark.parametrize("_, factory", Factories.factories) + @pytest.mark.skip("stmt and expr are the same") + def test_is_match_expression_differs_from_stmt(self, _: str, factory: ASTFactory): + pattern_factory = CPatternFactory(factory) + expression_pattern = pattern_factory.create_expression("x=3", ["int x;"]) + statement_pattern = pattern_factory.create_statement("x=3;", extra_declarations=["int x;"]) + assert_that( + is_match(expression_pattern, statement_pattern, {}), + is_(False), + "An expression doesn't match a statement", + ) + + expression_pattern = pattern_factory.create_expression("f()", ["int f();"]) + statement_pattern = pattern_factory.create_statement("f();", extra_declarations=["int f();"]) + assert_that( + is_match(expression_pattern, statement_pattern, {}), + is_(False), + "An expression doesn't match a statement", + ) + + +class TestIndividualCases: + def test_multi_single(self): + factory = ASTFactory(ClangASTNode) + atu = factory.create_from_text( + """ + int one(int a); + int two(int a, int b); + int three(int a, int b, int c); + int a,b,c; + void f(){ + one(a); + two(a,b); + three(a,b,c); + } + """, + "test.c", + ) + pattern_factory = CPatternFactory(factory) + stmt_nodes = pattern_factory.create_statements("$f($$all, $a);", None, ["int $f(int,int);"]) + variants = find_variants(atu.children[-1].children[-1].children, stmt_nodes) + + assert_that(variants, has_length(1)) + assert_that(variants[0].end_index, is_(0)) + assert_that(variants[0].exp["$$all"], is_([])) + assert_that(variants[0].exp["$a"][0].name, is_("a")) + variants = find_in_list(atu.children[-1].children[-1].children, stmt_nodes, {}, 1) + assert_that(variants, 1) + + variants = find_in_list(atu.children[-1].children[-1].children, stmt_nodes, {}, 1) + + # assert_that(variants[0].exp['$$all'], has_length(1)) + {"$f": ["two"], "$$all": ["a"], "$a": ["b"]}, + {"$f": ["three"], "$$all": ["a", "b"], "$a": ["c"]}, + found = match_pattern(atu.children[-1].children[-1].children, stmt_nodes) + assert_that(found, has_length(3)) diff --git a/test/c_cpp/test_c_pattern_factory.py b/test/c_cpp/test_c_pattern_factory.py new file mode 100644 index 00000000..703e43e2 --- /dev/null +++ b/test/c_cpp/test_c_pattern_factory.py @@ -0,0 +1,259 @@ +import pytest +from hamcrest import * +from hamcrest import assert_that, contains_string +from more_itertools import last + +from c_cpp.factories import Factories +from renaissance.impl.clang import CPatternFactory, ClangASTNode +from renaissance.impl.clang.c_pattern_factory import derive_header_text +from renaissance.impl.types import DeclarationExpression, MatchOne, VariableDef, FunctionDef, CompoundStatement, \ + Declaration +from renaissance.syntax_tree import ASTShower +from renaissance.syntax_tree.ast_finder import find_ast_type + + +class TestCPatternFactory: + def test_derive_header(self): + code = """ + #include <stdint.h> + int print(const char*,...); + #define FOO "foo" + #define BAR "bar" + #define SAME "bar" + typedef struct A_Struct{ + int a; + int b; + } A; + int some_decl = 1; + + void f(){ + A a = {}; + const char* foo = FOO; + const char* bar = BAR; + const char* same = SAME; + print("%s %s %s", foo, bar, same); + + } + + """ + atu = ClangASTNode.load_from_text(code, "test.c", [], None) + ASTShower.show_node(atu) + + header, lang = derive_header_text("c", atu) + simple_header = ";\n".join( + c.signature + for c in atu.children + if c.is_part_of_translation_unit() and not (c.ast_type == FunctionDef and c.children[-1].kind == CompoundStatement) + ) + + assert_that(header, contains_string('#define FOO "foo";')) + assert_that(header, contains_string("int print(const char*,...);")) + assert_that(header, contains_string("typedef struct A_Struct")) + assert_that(header, contains_string("int some_decl = 1;")) + assert_that(header, not_(contains_string('A a = {};'))) + assert_that(simple_header, contains_string('#define FOO "foo"')) + assert_that(simple_header, contains_string("int print(const char*,...);")) + assert_that(simple_header, contains_string("typedef struct A_Struct")) + assert_that(simple_header, contains_string("int some_decl = 1;")) + # assert_that(simple_header, not_(contains_string('A a = {};'))) + + assert_that(header, not_(contains_string('#include <stdint.h>'))) + assert_that(simple_header, contains_string('#include <stdint.h>')) + + +class TestExpression: + + @pytest.mark.parametrize( + "_, factory, expression, expected", + Factories.extend( + [ + ( + "a == $hallo", + "(BinaryOperation, , test.c[123:134]): |a == $hallo|\n (Expression, a, test.c[123:124]): |a|\n (DeclarationExpression, a, test.c[123:124]): |a|\n (MatchOne, $hallo, test.c[128:134]): |$hallo|\n (MatchOne, $hallo, test.c[128:134]): |$hallo|\n", + ), + ( + "2 != 3", + "(BinaryOperation, , test.c[105:111]): |2 != 3|\n (Number, , test.c[105:106]): |2|\n (Number, , test.c[110:111]): |3|\n", + ), + ( + "a != b", + "(BinaryOperation, , test.c[118:124]): |a != b|\n (Expression, a, test.c[118:119]): |a|\n (DeclarationExpression, a, test.c[118:119]): |a|\n (Expression, b, test.c[123:124]): |b|\n (DeclarationExpression, b, test.c[123:124]): |b|\n", + ), + ( + "b != $world", + "(BinaryOperation, , test.c[123:134]): |b != $world|\n (Expression, b, test.c[123:124]): |b|\n (DeclarationExpression, b, test.c[123:124]): |b|\n (MatchOne, $world, test.c[128:134]): |$world|\n (MatchOne, $world, test.c[128:134]): |$world|\n", + ), + ( + "c > $foo", + "(BinaryOperation, , test.c[121:129]): |c > $foo|\n (Expression, c, test.c[121:122]): |c|\n (DeclarationExpression, c, test.c[121:122]): |c|\n (MatchOne, $foo, test.c[125:129]): |$foo|\n (MatchOne, $foo, test.c[125:129]): |$foo|\n", + ), + ( + "d < $bar", + "(BinaryOperation, , test.c[121:129]): |d < $bar|\n (Expression, d, test.c[121:122]): |d|\n (DeclarationExpression, d, test.c[121:122]): |d|\n (MatchOne, $bar, test.c[125:129]): |$bar|\n (MatchOne, $bar, test.c[125:129]): |$bar|\n", + ), + ( + "e >= $baz", + "(BinaryOperation, , test.c[121:130]): |e >= $baz|\n (Expression, e, test.c[121:122]): |e|\n (DeclarationExpression, e, test.c[121:122]): |e|\n (MatchOne, $baz, test.c[126:130]): |$baz|\n (MatchOne, $baz, test.c[126:130]): |$baz|\n", + ), + ( + "f <= $qux", + "(BinaryOperation, , test.c[121:130]): |f <= $qux|\n (Expression, f, test.c[121:122]): |f|\n (DeclarationExpression, f, test.c[121:122]): |f|\n (MatchOne, $qux, test.c[126:130]): |$qux|\n (MatchOne, $qux, test.c[126:130]): |$qux|\n", + ), + ( + "g--", + "(UnaryOperation, , test.c[111:114]): |g--|\n (DeclarationExpression, g, test.c[111:112]): |g|\n", + ), + ( + "h++", + "(UnaryOperation, , test.c[111:114]): |h++|\n (DeclarationExpression, h, test.c[111:112]): |h|\n", + ), + ( + "!i", + "(UnaryOperation, , test.c[111:113]): |!i|\n (Expression, i, test.c[112:113]): |i|\n (DeclarationExpression, i, test.c[112:113]): |i|\n", + ), + ] + ), + ) + def test(self, _, factory, expression, expected): + patternFactory = CPatternFactory(factory) + node = patternFactory.create_expression(expression) + text = ASTShower.get_node(node) + if isinstance(node, ClangASTNode): + assert_that(text, is_(expected)) + else: + assert_that(text, not_none()) + + +class TestDeclaration: + + @pytest.mark.parametrize( + "_, factory, declarationText, types, parameters, expected_vars, expected_refs", + Factories.extend( + [ + ("int a=3;", [], [], 1, 0), + ("int a;", [], [], 1, 0), + ("int a = $x;", [], ["$x"], 1, 1), + ("int a=2,b = 3;int c=4;", [], [], 3, 0), + ("$type a = $x;", ["$type"], ["$x"], 1, 1), + ("$type a,b = $x;", ["$type"], ["$x"], 2, 1), + ] + ), + ) + def test( + self, + _, + factory, + declarationText, + types, + parameters, + expected_vars, + expected_refs, + ): + patternFactory = CPatternFactory(factory) + created_declarations = list(patternFactory.create_declarations(declarationText, parameters=parameters, types=types)) + + count_refs = 0 + count_vars = 0 + for decl in created_declarations: + count_refs += len(find_ast_type(decl, (DeclarationExpression,MatchOne))) + count_vars += len(find_ast_type(decl, VariableDef)) + ASTShower.show_node(decl) + assert_that(count_vars, is_(expected_vars)) + assert_that(count_refs, greater_than_or_equal_to(expected_refs)) + + +class TestStatements: + + @pytest.mark.parametrize( + "_, factory, statementText, extra_declarations, expected_stmts, expected_refs", + list( + Factories.extend( + [ + ("a=3;", [], 1, 1), + ("a = b;", [], 1, 2), + ("a = $x;", [], 1, 2), + ("a=2;b = 3;c=4;", [], 3, 3), + ("a = ($type)$x;", ["typedef int $type;"], 1, 2), + ("a = f($x);", ["int f(int);"], 1, 3), + ] + ) + ), + ) + def test( + self, + _, + factory, + statementText, + extra_declarations, + expected_stmts, + expected_refs, + ): + patternFactory = CPatternFactory(factory) + created_statements = list(patternFactory.create_statements(statementText, extra_declarations=extra_declarations)) + + count_refs = 0 + for decl in created_statements: + count_refs += len(find_ast_type(decl, (DeclarationExpression,MatchOne))) + assert_that(expected_stmts, is_(len(created_statements))) + assert_that(expected_refs, less_than_or_equal_to(count_refs)) + for stmt in created_statements: + assert_that(stmt.is_statement) + + +class TestUseAtuToCreatePatterns: + """ + Test the creation of a complex pattern that includes a typedef, a struct, a define and a statement + + Complex pattern take the includes, defines and typedefs from the translation unit + + """ + + @pytest.mark.parametrize( + "_, factory, statementText, expected_stmts, expected_refs", + list( + Factories.extend( + [ + ("A a = {};", 1, 1), + ("const char* foo=FOO;", 1, 2), + ("const char* $x = BAR;", 1, 2), + ] + ) + ), + ) + def test(self, _, factory, statementText, expected_stmts, expected_refs): + code = """ + int print(const char*,const char*,const char*,const char*); + #define FOO "foo" + #define BAR "bar" + #define SAME "bar" + typedef struct A_Struct{ + int a; + int b; + } A; + int some_decl = 1; + + void f(){ + A a = {}; + const char* foo = FOO; + const char* bar = BAR; + const char* same = SAME; + print("%s %s %s", foo, bar, same); + + } + +""" + atu = factory.create_from_text(code, "example.c") + + # ASTShower.show_node(atu, include_properties=True) + # use the factory and the translation unit (for include, define and typedef reference) to create a pattern factory + patternFactory = CPatternFactory(factory, atu) + + # pick the last statement fo match + pattern_root = patternFactory.create(statementText) + + # the user must pick it's own pattern in this case the last statement + assert_that(pattern_root.children[-1].is_statement) + node = last(n for n in pattern_root.children if n.ast_type != Declaration) + raw = node.signature + + assert_that(statementText, starts_with(raw)) diff --git a/test/c_cpp/test_clang_ast_node.py b/test/c_cpp/test_clang_ast_node.py new file mode 100644 index 00000000..8927ad2b --- /dev/null +++ b/test/c_cpp/test_clang_ast_node.py @@ -0,0 +1,86 @@ +import pytest +from hamcrest import assert_that, is_, has_length, has_string + +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.syntax_tree import ASTFactory + + +class TestClangAstNode: + def test_is_same_node(self): + factory = ASTFactory(ClangASTNode, []) + src = CPatternFactory(factory).create_statements("a == 3;a == 3;") + src2 = CPatternFactory(factory).create_statement("a == 3;") + assert_that(src[0], is_(src[1])) + + def test_find_all_in_clang_list_with_expansion(self): + factory = ASTFactory(ClangASTNode, []) + src = CPatternFactory(factory).create_statement("a == 3;") + assert_that("a", is_(src.children[0].children[0].properties["name"])) + + def test_marco_also_include_define(self): + src = ClangASTNode.load_from_text('#define x "xxx"', "test.c") + assert_that(src.children, has_length(1)) + + def test_marco_also_include_define_signature(self): + src = ClangASTNode.load_from_text('#define x "xxx"', "test.c") + assert_that('#define x "xxx"', is_(src.children[-1].signature)) + + def test_var_decl_includesemi_column(self): + src = ClangASTNode.load_from_text("int x= 0;", "test.c") + assert_that(src.children[-1].signature, is_("int x= 0;")) + + def test_var_decl_in_ancestor(self): + src = ClangASTNode.load_from_text("int x= 0;", "test.c") + assert_that(src.children[-1].children[-1].get_ancestor("VAR_DECL")) + + def test_var_decl_in_ancestor_of(self): + src = ClangASTNode.load_from_text("int x= 0;", "test.c") + assert_that(src.is_ancestor_of(src.children[-1].children[-1])) + + @pytest.mark.skip("last semicolumn is cut off from decl") + def test_var_decl_include_semi_column_and_keep_space(self): + src = ClangASTNode.load_from_text(" int x = 0 ;", "test.c") + assert_that(src.children[-1].signature, is_(" int x = 0 ;")) + + def test_struct_include_semicolumn(self): + src = ClangASTNode.load_from_text("struct s;", "test.c") + assert_that(src.children[-1].signature, is_("struct s;")) + + @pytest.mark.skip("last semicolumn is cut off from struct") + def test_struct_include_semicolumn_and_space(self): + src = ClangASTNode.load_from_text("struct s{int x; int y;} ;", "test.c") + assert_that("struct s{int x; int y;} ;", is_(src.children[-1].signature)) + + def test_mix_of_macro_and_decl(self): + src = ClangASTNode.load_from_text( + """ + #define FOO "foo" + #define BAR "bar" + #define SAME "bar" + struct A_Struct{ + int a; + int b; + }; + typedef struct A_Struct A; + int some_decl = 1; + + int print(const char*, const char *, const char *, const char*); + void f(){ + A a = {}; + const char* foo = FOO; + const char* bar = BAR; + const char* same = SAME; + print("%s %s %s", foo, bar, same); + + }""", + "test.c", + ) + assert_that(src.children, has_length(8)) + assert_that( src.children[0], has_string('(MacroDef, FOO, test.c[9:26]): |#define FOO "foo"|\n')) + assert_that( src.children[1], has_string('(MacroDef, BAR, test.c[35:52]): |#define BAR "bar"|\n') ) + assert_that( src.children[2], has_string('(MacroDef, SAME, test.c[61:79]): |#define SAME "bar"|\n') ) + assert_that( src.children[3], has_string("(StructDef, struct A_Struct, test.c[88:153]):\n |struct A_Struct{|\n | int a;|\n | int b;|\n | };|\n")) + assert_that( src.children[4], has_string("(TypedefDef, A, test.c[162:187]): |typedef struct A_Struct A|\n") ) + assert_that( src.children[5], has_string("(VariableDef, some_decl, test.c[197:215]): |int some_decl = 1;|\n") ) + assert_that( src.children[6], has_string("(FunctionDef, print, test.c[226:289]): |int print(const char*, const char " "*, const char *, const char*)|\n") ) + assert_that( src.children[7], has_string("(FunctionDef, f, test.c[299:495]):\n" " |void f(){|\n" " | A a = {};|\n" " | const char* foo = FOO;|\n" " | const char* bar = BAR;|\n" " | const char* same = SAME;|\n" ' | print("%s %s %s", foo, bar, same);|\n' " ||\n" " | }|\n" ) ) diff --git a/test/c_cpp/test_clang_json_ast_node.py b/test/c_cpp/test_clang_json_ast_node.py new file mode 100644 index 00000000..793c9f94 --- /dev/null +++ b/test/c_cpp/test_clang_json_ast_node.py @@ -0,0 +1,29 @@ +from pathlib import Path + +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.impl.clang import CPatternFactory +from renaissance.syntax_tree import ASTShower, ASTFactory +import pytest +from hamcrest import * + +pytest.mark.skip("empty workdir should also work right?") + + +class TestClangJsonAstNode: + def test_load_from_text_empty_dir(self): + node = ClangJsonASTNode.load_from_text("int main(){return 0;}", "hello.c", [], Path("")) + assert_that(isinstance(node, ClangJsonASTNode)) + + def test_load_from_text(self): + node = ClangJsonASTNode.load_from_text("int main(){return 0;}", "hello.c", [], Path(".")) + assert_that(isinstance(node, ClangJsonASTNode)) + + def test_name_in_props(self): + factory = ASTFactory(ClangJsonASTNode, []) + src = CPatternFactory(factory).create_statement("a == 3;") + ASTShower.show_node(src, True) + assert_that(src.children[0].properties["name"], is_("a")) + + +if __name__ == "__main__": + pytest.main() diff --git a/test/c_cpp/test_clang_json_match_finder.py b/test/c_cpp/test_clang_json_match_finder.py new file mode 100644 index 00000000..03c3dd02 --- /dev/null +++ b/test/c_cpp/test_clang_json_match_finder.py @@ -0,0 +1,30 @@ +import pytest +from hamcrest import * +from more_itertools import last + +from renaissance.impl.clang import CPatternFactory +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.impl.types import Declaration +from renaissance.syntax_tree import ASTFactory, MatchFinder +from renaissance.syntax_tree.ast_finder import find_ast_type + + +class TestClangJsonMatchFinder: + @pytest.mark.skip + def testIsMatchUsingMacroFromAtu(self): + code = """ + #define BAR "bar" + void f(){ + const char* bar = BAR; + } + """ + statements = "void f() {const char* bar = BAR;}" + factory = ASTFactory(ClangJsonASTNode, []) + atu = factory.create_from_text(code, "test.c") + pattern_factory = CPatternFactory(factory, ref_node=atu) + statements_atu = pattern_factory.create(statements) + statements = last(find_ast_type(statements_atu, Declaration)) + + result = MatchFinder.match_pattern(atu.children, [statements]) + + assert_that(result, has_length(1)) diff --git a/test/c_cpp/test_clang_match_finder.py b/test/c_cpp/test_clang_match_finder.py new file mode 100644 index 00000000..b18722b0 --- /dev/null +++ b/test/c_cpp/test_clang_match_finder.py @@ -0,0 +1,43 @@ +from hamcrest import * + +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.impl.types import Declaration +from renaissance.syntax_tree import ASTFactory, MatchFinder +from renaissance.syntax_tree.ast_finder import find_ast_type + + +class ClangMatchFinderTest: + def testIsMatch(self): + code = """ + #define BAR "bar" + void g(int,int); + int h=0; + struct S {}; + + void f(){ + const char* bar = BAR; + } + """ + fun = "void f() {const char* bar = BAR; }" + pattern_type = "(?i)Decl_?Stmt" + factory = ASTFactory(ClangASTNode, []) + atu = factory.create_from_text(code, "test.c") + pattern_factory = CPatternFactory(factory, ref_node=atu) + statements_atu = pattern_factory.create(fun) + statements = find_ast_type(statements_atu, Declaration).find_last().get() + + func_body = atu.children[-1].children[-1].children + result = MatchFinder.match_pattern(func_body, [statements]) + assert_that(result, has_length(1)) + + def test_typedef_in_pattern(self): + factory = ASTFactory(ClangASTNode, []) + pattern_factory = CPatternFactory(factory) + + pattern1 = pattern_factory.create_declarations( + "old $name = $value;", + extra_declarations=["typedef int old;"], + parameters=["$value"], + ) + + assert_that(pattern1[0].children[0].name, is_("$name")) diff --git a/test/common/__init__.py b/test/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/common/test_rewriter.py b/test/common/test_rewriter.py new file mode 100644 index 00000000..460754f7 --- /dev/null +++ b/test/common/test_rewriter.py @@ -0,0 +1,34 @@ +import pytest +from hamcrest import * + +from renaissance.common.rewriter import Rewriter + + +class TestRewriter: + + @pytest.mark.parametrize( + "initial_bytes, start, end, new_content, expected_bytes", + [ + (b"abcdefghij", 5, 10, b"hellooo", b"abcdehellooo"), + (b"abcdefghij", 5, 10, b" world", b"abcde world"), + (b"abcdefghij", 0, 0, b"BEGIN", b"BEGINabcdefghij"), + (b"abcdefghij", 2, 4, b"XY", b"abXYefghij"), + (b"abcdefghij", 0, 10, b"REPLACED", b"REPLACED"), + (b"abcdefghij", -1, -1, b"AT_END", b"abcdefghijAT_END"), + (b"abcdefghij", 5, -1, b"AT_END", b"abcdeAT_END"), + ], + ) + def test_replace(self, initial_bytes, start, end, new_content, expected_bytes): + rewriter = Rewriter(initial_bytes) + rewriter.replace(start, end, new_content) + result = rewriter.apply() + assert_that(expected_bytes, is_(result)) + + def test_multiple_replaces(self): + initial_bytes = b"abcdefghij" + rewriter = Rewriter(initial_bytes) + rewriter.replace(5, 10, b"hello") + rewriter.replace(5, 10, b" world") + rewriter.replace(0, 0, b"BEGIN") + result = rewriter.apply() + assert_that(result, is_(b"BEGINabcdehello world")) diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..d1448bbd --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,22 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--run-slow-hypothesis", + action="store_true", + default=False, + help="Run slow tests", + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "hypothesisslow: mark test as a slow hypothesis test") + + +def pytest_collection_modifyitems(config, items): + if not config.getoption("--run-slow-hypothesis"): + skip = pytest.mark.skip(reason="Pass --run-slow-hypothesis to run slow hypothesis test") + for item in items: + if "hypothesisslow" in item.keywords: + item.add_marker(skip) diff --git a/test/examples/__init__.py b/test/examples/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/examples/test_descendant_search.py b/test/examples/test_descendant_search.py new file mode 100644 index 00000000..e1f3920a --- /dev/null +++ b/test/examples/test_descendant_search.py @@ -0,0 +1,147 @@ +import pytest +from hamcrest import * + +from c_cpp.factories import Factories +from rejuvenation.descendant_search import find_descendant_match +from renaissance.impl.clang import CPatternFactory, ClangASTNode +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.syntax_tree import ASTFactory +from renaissance.syntax_tree.match_finder import is_match, AstProtocol, match_pattern + + +class TestFindDescendantMatch: + code_text: str = """ + int my_function(); + + void your_function(int count) { + int z = my_function(); + if (count > my_function()) { + int x = my_function(); + my_function(); + } + my_function(); + if (count <= my_function()) { + int y = my_function(); + } else { + my_function(); + } + my_function(); + } + """ + + outer_text: str = "if ($cond) { $$stmts; }" + inner_text: str = "my_function()" + extra_declarations_inner_text: list[str] = ["int my_function();"] + + def test_descendant_search_with_clang(self): + factory = ASTFactory(ClangASTNode) + pattern_factory = CPatternFactory(factory) + code_pattern = factory.create_from_text(self.code_text, "text.c") + outer_pattern = pattern_factory.create_statement(self.outer_text) + inner_pattern = pattern_factory.create_expression(self.inner_text, self.extra_declarations_inner_text) + results = find_descendant_match(code_pattern, outer_pattern, inner_pattern) + + assert_that(results, has_length(3), f"length of results = {len(results)}") + + def test_descendant_search_with_json(self): + factory = ASTFactory(ClangJsonASTNode) + pattern_factory = CPatternFactory(factory) + code_pattern = factory.create_from_text(self.code_text, "text.c") + outer_pattern = pattern_factory.create_statement(self.outer_text) + inner_pattern = pattern_factory.create_expression(self.inner_text, self.extra_declarations_inner_text) + results = find_descendant_match(code_pattern, outer_pattern, inner_pattern) + + assert_that(results, has_length(3), f"length of results = {len(results)}") + + +class TestBasic: + + code_text: str = """ + int my_function(); + void your_function() { + my_function(); + } + """ + + literal_text: str = "my_function()" + extra_declarations_literal_text: list[str] = ["int my_function();"] + + placeholder_text: str = "$f()" + extra_declarations_placeholder_text: list[str] = ["int $f();"] + + @pytest.mark.parametrize( + "_, factory, snippet, extra_declarations", + list( + Factories.extend( + [ + (literal_text, extra_declarations_literal_text), + (placeholder_text, extra_declarations_placeholder_text), + ] + ) + ), + ) + def test_snippet(self, _: str, factory: ASTFactory, snippet: str, extra_declarations: list[str]): + pattern_factory = CPatternFactory(factory) + code_pattern = factory.create_from_text(self.code_text, "text.c") # file extension consistent with C Pattern Factory + snippet_pattern = pattern_factory.create_expression(snippet, extra_declarations) + results = match_pattern(code_pattern.children, [snippet_pattern]) + assert_that(results, has_length(1), f"length of results = {len(results)}") + + @pytest.mark.parametrize("_, factory", Factories.factories) + def test_is_match_assignment_expression(self, _: str, factory: ASTFactory): + pattern_factory = CPatternFactory(factory) + expression1_pattern: AstProtocol = pattern_factory.create_expression("x=3", ["int x;"]) + assert_that( + is_match(expression1_pattern, expression1_pattern, {}), + is_(True), + "An expression matches itself", + ) + + expression2_pattern = pattern_factory.create_expression("x=3", ["int x;"]) + assert_that( + is_match(expression1_pattern, expression2_pattern, {}), + is_(True), + "Identical expressions match", + ) + + @pytest.mark.parametrize("_, factory", Factories.factories) + def test_is_match_call_expression(self, _: str, factory: ASTFactory): + pattern_factory = CPatternFactory(factory) + expression1_pattern = pattern_factory.create_expression("f()", ["int f();"]) + assert_that( + is_match(expression1_pattern, expression1_pattern, {}), + is_(True), + "An expression matches itself", + ) + + expression2_pattern = pattern_factory.create_expression("f()", ["int f();"]) + assert_that( + is_match(expression1_pattern, expression2_pattern, {}), + is_(True), + "Identical expressions match", + ) + + @pytest.mark.parametrize("_, factory", Factories.factories) + def test_is_match_statement(self, _: str, factory: ASTFactory): + pattern_factory = CPatternFactory(factory) + statement1_pattern = pattern_factory.create_statement("f();", extra_declarations=["int f();"]) + assert_that( + is_match(statement1_pattern, statement1_pattern, {}), + is_(True), + "A statement matches itself", + ) + + statement2_pattern = pattern_factory.create_statement("f ( ) ;", extra_declarations=["int f();"]) + assert_that( + is_match(statement1_pattern, statement2_pattern), + is_(True), + "Identical statements match", + ) + + # expression can be found with f(), is match is not exact match + expression_pattern = pattern_factory.create_expression("f(3)", ["int f();"]) + assert_that( + is_match(statement1_pattern, expression_pattern), + is_(False), + "A statement doesn't match an expression", + ) diff --git a/test/examples/test_examples.py b/test/examples/test_examples.py new file mode 100644 index 00000000..86d5674d --- /dev/null +++ b/test/examples/test_examples.py @@ -0,0 +1,226 @@ +from typing import Callable +import pytest +from hamcrest import * + +from c_cpp.factories import Factories +from rejuvenation.batch_process_examples import ( + batch_remove_unused_variable_once_example, + batch_repeat_example, + batch_recipe_example, +) +from rejuvenation.recipe_example import batch_recipe_example as receipe_example +from rejuvenation.refactor_examples_different_styles import ( + example_use_ast_kind_finder, + example_use_ast_function_finder, + example_add_comment_and_commit, + example_replace_old_by_fancy_new, + main, +) +from rejuvenation.refactor_with_nested_compositions import ( + refactor_with_nested_compositions, +) +from rejuvenation.remove_unused_variable import ( + remove_unused_variable_using_refactor_method, + remove_unused_variable_low_level, +) +from rejuvenation.replace_if_with_ternary import replace_if_with_ternary +from renaissance.impl.clang import CPatternFactory, ClangASTNode +from renaissance.impl.clang.clang_json_ast_node import ClangJsonASTNode +from renaissance.syntax_tree import ASTFactory +from renaissance.syntax_tree.ast_node import ASTNode + + +class TestRefactorWithNestedCompositions: + + def test_refactor_with_nested_compositions(self): + result = refactor_with_nested_compositions(["", ""]) + assert_that(result, is_not(None)) + expected_result_nested = """\ +void f1(int a, int b, int c); +void f2(int a, int c); +void f(){ + const int a = 1; + const int b = 2; + int isAOne = a==1; + int c = 0, d=0; + + //changed if expr to const + if(isAOne){ + d++; +//changed if expr to const +if(isAOne){ + d++;c=d; +//changed function f1 to f2 +f2(a,c); +; +} + //changed function f1 to f2 + f2(a,c); + ; + } + //changed function f1 to f2 + f2(a,c); + if (a==2) { + c++; + + //changed function f1 to f2 + f2(a,c); + } + + //changed function f1 to f2 + f2(a,c); +}""" + assert result == expected_result_nested + assert_that(result, is_(expected_result_nested)) + + +class TestReplaceIfWithTernaryOperator: + + # didn't check expected result + def test_refactor_with_nested_compositions(self): + result = replace_if_with_ternary() + + expected_result_ternary = ( + "int a = 1;\n" + " int b = 2;\n" + " int c = 3;\n" + " int d = 4;\n" + " void f(){\n" + " c++; b=(a==1) ? 2:3; d++;\n" + " }" + ) + assert_that(result, is_(expected_result_ternary)) + + +# add a testcase for remove unused variable +class TestRemoveUnusedVariable: + + @pytest.mark.parametrize("_, node_type", Factories.node_types) + def test_remove_unused_variable_using_refactor_method(self, _: str, node_type: type[ASTNode]): + result, expected = remove_unused_variable_using_refactor_method(node_type) + assert_that(result, is_(expected)) + + @pytest.mark.parametrize("_, node_type", Factories.node_types) + def test_remove_unused_variable_low_level(self, _: str, node_type: type[ASTNode]): + result, expected_result = remove_unused_variable_low_level(node_type) + assert_that(result, is_(expected_result)) + + +class TestExamplesDifferentStyles: + + @pytest.mark.parametrize( + "_, factory, _node_type, method", + list( + Factories.extend( + [ + ("kind", example_use_ast_kind_finder), + ("function", example_use_ast_function_finder), + ] + ) + ), + ) + def test( + self, + _, + factory: ASTFactory, + _node_type: type[ASTNode], + method: Callable[[ASTFactory, CPatternFactory], tuple[str, str]], + ): + pattern_factory = CPatternFactory(factory) + result, expected = method(factory, pattern_factory) + + assert_that(expected, is_(result)) + + def test_example_add_comment_and_commit(self): + factory = ASTFactory(ClangASTNode) + pattern_factory = CPatternFactory(factory) + result, expected = example_add_comment_and_commit(factory, pattern_factory) + + assert_that(result, contains_string("// old has become obsolete\n // old has become obsolete\n ")) + + def test_example_add_comment_and_commit_json(self): + factory = ASTFactory(ClangJsonASTNode) + pattern_factory = CPatternFactory(factory) + assert_that(calling(lambda: example_add_comment_and_commit(factory, pattern_factory)), not_(raises(Exception))) + result, expected = example_add_comment_and_commit(factory, pattern_factory) + assert_that(result, contains_string(" // old has become obsolete\n old b = 2;")) + + def test_example_replace_old_by_fancy_new(self): + factory = ASTFactory(ClangASTNode) + pattern_factory = CPatternFactory(factory) + + assert_that(calling(lambda: example_add_comment_and_commit(factory, pattern_factory)), not_(raises(Exception))) + + result, expected = example_replace_old_by_fancy_new(factory, pattern_factory) + # shiould check this: + # assert_that(result, contains_string("fancy_new b = 2;\n")) + + def test_make_sure_that_batch_proc_still_run(self): + assert_that(calling(batch_remove_unused_variable_once_example), not_(raises(Exception))) + + def test_make_sure_that_batch_proc_still_run(self): + assert_that(calling(batch_repeat_example), not_(raises(Exception))) + + def test_make_sure_that_batch_proc_still_run(self): + assert_that(calling(batch_recipe_example), not_(raises(Exception))) + + def test_make_sure_that_recipe_still_run(self): + assert_that(calling(receipe_example), raises(Exception, pattern="'stddef.h' file not found")) + + def test_make_sure_different_style_still_run(self): + factory = ASTFactory(ClangASTNode) + pattern_factory = CPatternFactory(factory) + + assert_that( + calling(lambda: example_add_comment_and_commit(factory, pattern_factory)), + not_(raises(Exception)), + ) + assert_that( + calling(lambda: example_replace_old_by_fancy_new(factory, pattern_factory)), + not_(raises(Exception)), + ) + assert_that( + calling(lambda: example_use_ast_kind_finder(factory, pattern_factory)), + not_(raises(Exception)), + ) + assert_that( + calling(lambda: example_use_ast_function_finder(factory, pattern_factory)), + not_(raises(Exception)), + ) + assert_that(calling(lambda: main([])), not_(raises(Exception))) + + def test_make_sure_that_nested_compositions_still_run(self): + assert_that(calling(lambda: refactor_with_nested_compositions([])), not_(raises(Exception))) + + @pytest.mark.parametrize("node_type", [ClangASTNode, ClangJsonASTNode]) + def test_make_sure_unused_var_still_run(self, node_type): + assert_that( + calling(lambda: remove_unused_variable_low_level(node_type)), + not_(raises(Exception)), + ) + assert_that( + calling(lambda: remove_unused_variable_using_refactor_method(node_type)), + not_(raises(Exception)), + ) + + def test_make_sure_replace_if_with_ternary_still_run(self): + result = replace_if_with_ternary() + + assert_that( + result, + is_( + "int a = 1;\n int b = 2;\n int c = 3;\n" + " int d = 4;\n void f(){\n c++; b=(a==1) ? 2:3; d++;\n }" + ), + ) + + +""" + +E Expected: Expected a callable raising <class 'Exception'> +E but: Correct assertion type raised, but a string containing +"Error parsing: ClangASTNode1.cpp \n + errors: 4: 'stddef.h' file not found at <SourceLocation file '/../lib/gcc/x86_64-linux-gnu/13/../../../../include/c++/13/cstddef', line 50, column 10>\n " not found. +Exception message was: +"Error parsing: ClangASTNode1.cpp errors: 4: 'stddef.h' file not found at <SourceLocation file '/../lib/gcc/x86_64-linux-gnu/13/../../../../include/c++/13/cstddef', line 50, column 10> +E " +""" diff --git a/test/examples/test_python_examples.py b/test/examples/test_python_examples.py new file mode 100644 index 00000000..48d89a5a --- /dev/null +++ b/test/examples/test_python_examples.py @@ -0,0 +1,25 @@ +from hamcrest import assert_that, is_ + +from rejuvenation.python_cst_example import python_cst_smoke_test +from rejuvenation.python_lst_example import python_lst_smoke_test +from rejuvenation.python_rst_example import python_rst_smoke_test + +result = "\nfrom module import foo, bar, baz, quux\nba(51)\n# changed function f1 to f2\nf2(52\n,123456)\n\n# changed function f1 to f2\nf2(53\n,123456)\n\npa(54)\n\n# changed if expr to const\nisAOne=True\nif(isAOne):\n ba()\n\n\npa(54) \n" + + +class TestPythonExamples: + # def test_python_ast_still_works(self): + # result = python_ast_smoke_test() + # assert_that(result, is_(result)) + + def test_python_cst_still_works(self): + result = python_cst_smoke_test() + assert_that(result, is_(result)) + + def test_python_lst_still_works(self): + result = python_lst_smoke_test() + assert_that(result, is_(result)) + + def test_python_rst_still_works(self): + result = python_rst_smoke_test() + assert_that(result, is_(result)) diff --git a/test/extractors/__init__.py b/test/extractors/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/extractors/test_code_graph_extractors.py b/test/extractors/test_code_graph_extractors.py new file mode 100644 index 00000000..fd9b7d0a --- /dev/null +++ b/test/extractors/test_code_graph_extractors.py @@ -0,0 +1,328 @@ +import pytest +from unittest.mock import MagicMock, patch + +import tree_sitter_python +from hamcrest import assert_that, is_, has_item, not_, instance_of + +from renaissance.impl.tree_sitter.extractor import ( + BaseCodeGraphExtractor, + PythonCodeGraphExtractor, + JavaCodeGraphExtractor, + CppCodeGraphExtractor, +) +from renaissance.impl.types import FunctionDef, Call, Comment + + +# --------------------------------------------------------------------------- +# BaseCodeGraphExtractor +# --------------------------------------------------------------------------- + + +def make_lst_node(kind, signature, name=None): + node = MagicMock() + node.ast_type = kind + node.signature = signature + node.properties = {"name": name} if name else {} + return node + + +class TestBaseCodeGraphExtractor: + def test_is_abstract(self): + with patch("renaissance.impl.tree_sitter.adapter.TreeSitterAdapter"): + extractor = BaseCodeGraphExtractor.__new__(BaseCodeGraphExtractor) + extractor.graph = MagicMock() + with pytest.raises(NotImplementedError): + extractor._process_file("file.py", MagicMock()) + + def test_extract_calls_process_file_for_each_file(self, mocker, tmp_path): + f1 = tmp_path / "a.py" + f1.write_text("x = 1") + f2 = tmp_path / "b.py" + f2.write_text("y = 2") + + with patch("renaissance.impl.tree_sitter.adapter.TreeSitterAdapter") as mock_adapter_cls: + mock_adapter = mock_adapter_cls.return_value + mock_adapter.parse_code.return_value = MagicMock() + mock_adapter.to_lst.return_value = self.make_lst([]) + + extractor = PythonCodeGraphExtractor("python", tree_sitter_python) + spy = mocker.patch.object(extractor, "_process_file") + + extractor.extract([str(f1), str(f2)]) + + assert_that(spy.call_count, is_(2)) + + def test_extract_skips_file_on_error(self, tmp_path): + with patch("renaissance.impl.tree_sitter.adapter.TreeSitterAdapter") as mock_adapter_cls: + mock_adapter = mock_adapter_cls.return_value + mock_adapter.parse_code.side_effect = RuntimeError("parse error") + + extractor = PythonCodeGraphExtractor("python", tree_sitter_python) + # Should not raise + extractor.extract([str(tmp_path / "nonexistent.py")]) + + def test_save_graph_writes_file(self, tmp_path, mocker): + with patch("renaissance.impl.tree_sitter.adapter.TreeSitterAdapter"): + extractor = PythonCodeGraphExtractor("python", tree_sitter_python) + mock_write = mocker.patch("renaissance.impl.tree_sitter.extractor.networkx.write_graphml") + mocker.patch("renaissance.impl.tree_sitter.extractor.GRAPHML_DIR", str(tmp_path)) + + extractor.save_graph("test.graphml") + + mock_write.assert_called_once() + + def test_constructor_creates_directed_graph(self): + import networkx as nx + + with patch("renaissance.impl.tree_sitter.adapter.TreeSitterAdapter"): + extractor = PythonCodeGraphExtractor("python", tree_sitter_python) + assert_that(extractor.graph, instance_of(nx.DiGraph)) + + @staticmethod + def make_lst(nodes): + lst = MagicMock() + lst.traverse.return_value = nodes + return lst + + +# --------------------------------------------------------------------------- +# PythonCodeGraphExtractor +# --------------------------------------------------------------------------- + + +class TestPythonCodeGraphExtractor(TestBaseCodeGraphExtractor): + @staticmethod + def _make_extractor(): + with patch("renaissance.impl.tree_sitter.adapter.TreeSitterAdapter"): + return PythonCodeGraphExtractor("python", tree_sitter_python) + + def test_adds_file_and_folder_nodes(self): + extractor = self._make_extractor() + lst = self.make_lst([]) + + extractor._process_file("/project/src/foo.py", lst) + + assert_that(extractor.graph.nodes, has_item("/project/src/foo.py")) + assert_that(extractor.graph.nodes, has_item("/project/src")) + + def test_adds_contains_edge_from_folder_to_file(self): + extractor = self._make_extractor() + lst = self.make_lst([]) + + extractor._process_file("/project/src/foo.py", lst) + + assert_that(extractor.graph.has_edge("/project/src", "/project/src/foo.py"), is_(True)) + assert_that(extractor.graph.edges["/project/src", "/project/src/foo.py"]["type"], is_("contains")) + + def test_adds_function_node_for_function_definition(self): + extractor = self._make_extractor() + func_node = make_lst_node(FunctionDef, "def my_func(x):") + lst = self.make_lst([func_node]) + + extractor._process_file("/src/foo.py", lst) + + assert_that(extractor.graph.nodes, has_item("my_func")) + assert_that(extractor.graph.nodes["my_func"]["type"], is_("function")) + + def test_adds_defines_edge_for_function(self): + extractor = self._make_extractor() + func_node = make_lst_node(FunctionDef, "def my_func(x):") + lst = self.make_lst([func_node]) + + extractor._process_file("/src/foo.py", lst) + + assert_that(extractor.graph.has_edge("/src/foo.py", "my_func"), is_(True)) + assert_that(extractor.graph.edges["/src/foo.py", "my_func"]["type"], is_("defines")) + + def test_adds_call_node_for_call(self): + extractor = self._make_extractor() + call_node = make_lst_node(Call, "some_func(arg1)") + lst = self.make_lst([call_node]) + + extractor._process_file("/src/foo.py", lst) + + assert_that(extractor.graph.nodes, has_item("some_func")) + assert_that(extractor.graph.nodes["some_func"]["type"], is_("call_target")) + + def test_adds_calls_edge_for_call(self): + extractor = self._make_extractor() + call_node = make_lst_node(Call, "some_func(arg1)") + lst = self.make_lst([call_node]) + + extractor._process_file("/src/foo.py", lst) + + assert_that(extractor.graph.has_edge("/src/foo.py", "some_func"), is_(True)) + assert_that(extractor.graph.edges["/src/foo.py", "some_func"]["type"], is_("calls")) + + def test_ignores_unrelated_node_kinds(self): + extractor = self._make_extractor() + other_node = make_lst_node("import_statement", "import os") + lst = self.make_lst([other_node]) + + extractor._process_file("/src/foo.py", lst) + + assert_that(extractor.graph.nodes, not_(has_item("import os"))) + + def test_multiple_functions_all_added(self): + extractor = self._make_extractor() + nodes = [ + make_lst_node(FunctionDef, "def foo(x):"), + make_lst_node(FunctionDef, "def bar(y):"), + ] + lst = self.make_lst(nodes) + + extractor._process_file("/src/foo.py", lst) + + assert_that(extractor.graph.nodes, has_item("foo")) + assert_that(extractor.graph.nodes, has_item("bar")) + + +# --------------------------------------------------------------------------- +# JavaCodeGraphExtractor +# --------------------------------------------------------------------------- + + +class TestJavaCodeGraphExtractor(TestBaseCodeGraphExtractor): + @staticmethod + def _make_extractor(): + with patch("renaissance.impl.tree_sitter.adapter.TreeSitterAdapter"): + return JavaCodeGraphExtractor("java", tree_sitter_python) + + def test_adds_file_and_folder_nodes(self): + extractor = self._make_extractor() + lst = self.make_lst([]) + + extractor._process_file("/project/src/Main.java", lst) + + assert_that(extractor.graph.nodes, has_item("/project/src/Main.java")) + assert_that(extractor.graph.nodes, has_item("/project/src")) + + def test_adds_method_node_for_method_declaration(self): + extractor = self._make_extractor() + method_node = make_lst_node(FunctionDef, "void doSomething(){}", name="doSomething") + lst = self.make_lst([method_node]) + + extractor._process_file("/src/Main.java", lst) + + assert_that(extractor.graph.nodes, has_item("doSomething")) + assert_that(extractor.graph.nodes["doSomething"]["type"], is_("method")) + + def test_method_node_uses_default_name_when_missing(self): + extractor = self._make_extractor() + method_node = make_lst_node(FunctionDef, "void doSomething(){}") + method_node.properties = {} + lst = self.make_lst([method_node]) + + extractor._process_file("/src/Main.java", lst) + + assert_that(extractor.graph.nodes, has_item("method")) + + def test_adds_defines_edge_for_method(self): + extractor = self._make_extractor() + method_node = make_lst_node(FunctionDef, "void doSomething()", name="doSomething") + lst = self.make_lst([method_node]) + + extractor._process_file("/src/Main.java", lst) + + assert_that(extractor.graph.has_edge("/src/Main.java", "doSomething"), is_(True)) + assert_that(extractor.graph.edges["/src/Main.java", "doSomething"]["type"], is_("defines")) + + def test_adds_method_invocation_node(self): + extractor = self._make_extractor() + invocation_node = make_lst_node(Call, "obj.doSomething(arg)") + lst = self.make_lst([invocation_node]) + + extractor._process_file("/src/Main.java", lst) + + assert_that(extractor.graph.nodes, has_item("obj.doSomething")) + assert_that(extractor.graph.nodes["obj.doSomething"]["type"], is_("method_target")) + + def test_adds_calls_edge_for_invocation(self): + extractor = self._make_extractor() + invocation_node = make_lst_node(Call, "obj.doSomething(arg)") + lst = self.make_lst([invocation_node]) + + extractor._process_file("/src/Main.java", lst) + + assert_that(extractor.graph.has_edge("/src/Main.java", "obj.doSomething"), is_(True)) + assert_that(extractor.graph.edges["/src/Main.java", "obj.doSomething"]["type"], is_("calls")) + + +# --------------------------------------------------------------------------- +# CppCodeGraphExtractor +# --------------------------------------------------------------------------- + + +class TestCppCodeGraphExtractor(TestBaseCodeGraphExtractor): + @staticmethod + def _make_extractor(): + with patch("renaissance.impl.tree_sitter.adapter.TreeSitterAdapter"): + return CppCodeGraphExtractor("cpp", tree_sitter_python) + + def test_adds_file_and_folder_nodes(self): + extractor = self._make_extractor() + lst = self.make_lst([]) + + extractor._process_file("/project/src/main.cpp", lst) + + assert_that(extractor.graph.nodes, has_item("/project/src/main.cpp")) + assert_that(extractor.graph.nodes, has_item("/project/src")) + + def test_adds_function_node_for_function_definition(self): + extractor = self._make_extractor() + func_node = make_lst_node(FunctionDef, "int main()", name="main") + lst = self.make_lst([func_node]) + + extractor._process_file("/src/main.cpp", lst) + + assert_that(extractor.graph.nodes, has_item("main")) + assert_that(extractor.graph.nodes["main"]["type"], is_("function")) + + def test_function_node_uses_default_name_when_missing(self): + extractor = self._make_extractor() + func_node = make_lst_node(FunctionDef, "int main()") + func_node.properties = {} + lst = self.make_lst([func_node]) + + extractor._process_file("/src/main.cpp", lst) + + assert_that(extractor.graph.nodes, has_item("func")) + + def test_adds_defines_edge_for_function(self): + extractor = self._make_extractor() + func_node = make_lst_node(FunctionDef, "int main()", name="main") + lst = self.make_lst([func_node]) + + extractor._process_file("/src/main.cpp", lst) + + assert_that(extractor.graph.has_edge("/src/main.cpp", "main"), is_(True)) + assert_that(extractor.graph.edges["/src/main.cpp", "main"]["type"], is_("defines")) + + def test_adds_call_expression_node(self): + extractor = self._make_extractor() + call_node = make_lst_node(Call, "printf(fmt)") + lst = self.make_lst([call_node]) + + extractor._process_file("/src/main.cpp", lst) + + assert_that(extractor.graph.nodes, has_item("printf")) + assert_that(extractor.graph.nodes["printf"]["type"], is_("call_target")) + + def test_adds_calls_edge_for_call_expression(self): + extractor = self._make_extractor() + call_node = make_lst_node(Call, "printf(fmt)") + lst = self.make_lst([call_node]) + + extractor._process_file("/src/main.cpp", lst) + + assert_that(extractor.graph.has_edge("/src/main.cpp", "printf"), is_(True)) + assert_that(extractor.graph.edges["/src/main.cpp", "printf"]["type"], is_("calls")) + + def test_ignores_unrelated_node_kinds(self): + extractor = self._make_extractor() + other_node = make_lst_node(Comment, "// a comment") + lst = self.make_lst([other_node]) + + extractor._process_file("/src/main.cpp", lst) + + assert_that(extractor.graph.nodes, not_(has_item("// a comment"))) diff --git a/test/extractors/test_python_extractors.py b/test/extractors/test_python_extractors.py new file mode 100644 index 00000000..a3287b38 --- /dev/null +++ b/test/extractors/test_python_extractors.py @@ -0,0 +1,265 @@ +from pathlib import Path + +from hamcrest import assert_that, is_not, empty + +import targets +from renaissance.impl.python.extractor import PythonExtractor + + +class TestPythonExtractor: + + def test_extractor(self): + + extractor = PythonExtractor() + + assert_that(extractor, is_not(None)) + + def test_extract_a_file(self): + + extractor = PythonExtractor() + extractor.process_file(Path(targets.__file__).parent / "demo.py") + + assert_that(extractor.codebase, is_not(empty())) + assert_that(extractor.nodes, is_not(empty())) + assert_that(extractor.edges, is_not(empty())) + + def test_extract_a_file(self): + + extractor = PythonExtractor() + extractor.process(Path(targets.__file__).parent / "demo.py") + graphml = Path(targets.__file__).parent / "demo.graphml" + extractor.save_graph(graphml) + try: + with open(graphml, "r") as f: + content = f.readlines() + assert_that(content, "demo.graphml") + finally: + if graphml.exists(): + graphml.unlink() + + + +# def test_adds_contains_edge_from_folder_to_file(self): +# extractor = self._make_extractor() +# lst = self.make_lst([]) +# +# extractor._process_file("/project/src/foo.py", lst) +# +# assert_that(extractor.graph.has_edge("/project/src", "/project/src/foo.py"), is_(True)) +# assert_that(extractor.graph.edges["/project/src", "/project/src/foo.py"]["type"], is_("contains")) +# +# def test_adds_function_node_for_function_definition(self): +# extractor = self._make_extractor() +# func_node = make_lst_node("function_definition", "def my_func(x):") +# lst = self.make_lst([func_node]) +# +# extractor._process_file("/src/foo.py", lst) +# +# assert_that(extractor.graph.nodes, has_item("my_func")) +# assert_that(extractor.graph.nodes["my_func"]["type"], is_("function")) +# +# def test_adds_defines_edge_for_function(self): +# extractor = self._make_extractor() +# func_node = make_lst_node("function_definition", "def my_func(x):") +# lst = self.make_lst([func_node]) +# +# extractor._process_file("/src/foo.py", lst) +# +# assert_that(extractor.graph.has_edge("/src/foo.py", "my_func"), is_(True)) +# assert_that(extractor.graph.edges["/src/foo.py", "my_func"]["type"], is_("defines")) +# +# def test_adds_call_node_for_call(self): +# extractor = self._make_extractor() +# call_node = make_lst_node("call", "some_func(arg1)") +# lst = self.make_lst([call_node]) +# +# extractor._process_file("/src/foo.py", lst) +# +# assert_that(extractor.graph.nodes, has_item("some_func")) +# assert_that(extractor.graph.nodes["some_func"]["type"], is_("call_target")) +# +# def test_adds_calls_edge_for_call(self): +# extractor = self._make_extractor() +# call_node = make_lst_node("call", "some_func(arg1)") +# lst = self.make_lst([call_node]) +# +# extractor._process_file("/src/foo.py", lst) +# +# assert_that(extractor.graph.has_edge("/src/foo.py", "some_func"), is_(True)) +# assert_that(extractor.graph.edges["/src/foo.py", "some_func"]["type"], is_("calls")) +# +# def test_ignores_unrelated_node_kinds(self): +# extractor = self._make_extractor() +# other_node = make_lst_node("import_statement", "import os") +# lst = self.make_lst([other_node]) +# +# extractor._process_file("/src/foo.py", lst) +# +# assert_that(extractor.graph.nodes, not_(has_item("import os"))) +# +# def test_multiple_functions_all_added(self): +# extractor = self._make_extractor() +# nodes = [ +# make_lst_node("function_definition", "def foo(x):"), +# make_lst_node("function_definition", "def bar(y):"), +# ] +# lst = self.make_lst(nodes) +# +# extractor._process_file("/src/foo.py", lst) +# +# assert_that(extractor.graph.nodes, has_item("foo")) +# assert_that(extractor.graph.nodes, has_item("bar")) +# +# +# # --------------------------------------------------------------------------- +# # JavaCodeGraphExtractor +# # --------------------------------------------------------------------------- +# +# +# class TestJavaCodeGraphExtractor(TestBaseCodeGraphExtractor): +# @staticmethod +# def _make_extractor(): +# with patch("renaissance.extractors.code_graph_extractors.TreeSitterAdapter"): +# return JavaCodeGraphExtractor("java", "fake_lib") +# +# def test_adds_file_and_folder_nodes(self): +# extractor = self._make_extractor() +# lst = self.make_lst([]) +# +# extractor._process_file("/project/src/Main.java", lst) +# +# assert_that(extractor.graph.nodes, has_item("/project/src/Main.java")) +# assert_that(extractor.graph.nodes, has_item("/project/src")) +# +# def test_adds_method_node_for_method_declaration(self): +# extractor = self._make_extractor() +# method_node = make_lst_node("method_declaration", "void doSomething()", name="doSomething") +# lst = self.make_lst([method_node]) +# +# extractor._process_file("/src/Main.java", lst) +# +# assert_that(extractor.graph.nodes, has_item("doSomething")) +# assert_that(extractor.graph.nodes["doSomething"]["type"], is_("method")) +# +# def test_method_node_uses_default_name_when_missing(self): +# extractor = self._make_extractor() +# method_node = make_lst_node("method_declaration", "void doSomething()") +# method_node.properties = {} +# lst = self.make_lst([method_node]) +# +# extractor._process_file("/src/Main.java", lst) +# +# assert_that(extractor.graph.nodes, has_item("method")) +# +# def test_adds_defines_edge_for_method(self): +# extractor = self._make_extractor() +# method_node = make_lst_node("method_declaration", "void doSomething()", name="doSomething") +# lst = self.make_lst([method_node]) +# +# extractor._process_file("/src/Main.java", lst) +# +# assert_that(extractor.graph.has_edge("/src/Main.java", "doSomething"), is_(True)) +# assert_that(extractor.graph.edges["/src/Main.java", "doSomething"]["type"], is_("defines")) +# +# def test_adds_method_invocation_node(self): +# extractor = self._make_extractor() +# invocation_node = make_lst_node("method_invocation", "obj.doSomething(arg)") +# lst = self.make_lst([invocation_node]) +# +# extractor._process_file("/src/Main.java", lst) +# +# assert_that(extractor.graph.nodes, has_item("obj.doSomething")) +# assert_that(extractor.graph.nodes["obj.doSomething"]["type"], is_("method_target")) +# +# def test_adds_calls_edge_for_invocation(self): +# extractor = self._make_extractor() +# invocation_node = make_lst_node("method_invocation", "obj.doSomething(arg)") +# lst = self.make_lst([invocation_node]) +# +# extractor._process_file("/src/Main.java", lst) +# +# assert_that(extractor.graph.has_edge("/src/Main.java", "obj.doSomething"), is_(True)) +# assert_that(extractor.graph.edges["/src/Main.java", "obj.doSomething"]["type"], is_("calls")) +# +# +# # --------------------------------------------------------------------------- +# # CppCodeGraphExtractor +# # --------------------------------------------------------------------------- +# +# +# class TestCppCodeGraphExtractor(TestBaseCodeGraphExtractor): +# @staticmethod +# def _make_extractor(): +# with patch("renaissance.extractors.code_graph_extractors.TreeSitterAdapter"): +# return CppCodeGraphExtractor("cpp", "fake_lib") +# +# def test_adds_file_and_folder_nodes(self): +# extractor = self._make_extractor() +# lst = self.make_lst([]) +# +# extractor._process_file("/project/src/main.cpp", lst) +# +# assert_that(extractor.graph.nodes, has_item("/project/src/main.cpp")) +# assert_that(extractor.graph.nodes, has_item("/project/src")) +# +# def test_adds_function_node_for_function_definition(self): +# extractor = self._make_extractor() +# func_node = make_lst_node("function_definition", "int main()", name="main") +# lst = self.make_lst([func_node]) +# +# extractor._process_file("/src/main.cpp", lst) +# +# assert_that(extractor.graph.nodes, has_item("main")) +# assert_that(extractor.graph.nodes["main"]["type"], is_("function")) +# +# def test_function_node_uses_default_name_when_missing(self): +# extractor = self._make_extractor() +# func_node = make_lst_node("function_definition", "int main()") +# func_node.properties = {} +# lst = self.make_lst([func_node]) +# +# extractor._process_file("/src/main.cpp", lst) +# +# assert_that(extractor.graph.nodes, has_item("func")) +# +# def test_adds_defines_edge_for_function(self): +# extractor = self._make_extractor() +# func_node = make_lst_node("function_definition", "int main()", name="main") +# lst = self.make_lst([func_node]) +# +# extractor._process_file("/src/main.cpp", lst) +# +# assert_that(extractor.graph.has_edge("/src/main.cpp", "main"), is_(True)) +# assert_that(extractor.graph.edges["/src/main.cpp", "main"]["type"], is_("defines")) +# +# def test_adds_call_expression_node(self): +# extractor = self._make_extractor() +# call_node = make_lst_node("call_expression", "printf(fmt)") +# lst = self.make_lst([call_node]) +# +# extractor._process_file("/src/main.cpp", lst) +# +# assert_that(extractor.graph.nodes, has_item("printf")) +# assert_that(extractor.graph.nodes["printf"]["type"], is_("call_target")) +# +# def test_adds_calls_edge_for_call_expression(self): +# extractor = self._make_extractor() +# call_node = make_lst_node("call_expression", "printf(fmt)") +# lst = self.make_lst([call_node]) +# +# extractor._process_file("/src/main.cpp", lst) +# +# assert_that(extractor.graph.has_edge("/src/main.cpp", "printf"), is_(True)) +# assert_that(extractor.graph.edges["/src/main.cpp", "printf"]["type"], is_("calls")) +# +# def test_ignores_unrelated_node_kinds(self): +# extractor = self._make_extractor() +# other_node = make_lst_node("comment", "// a comment") +# lst = self.make_lst([other_node]) +# +# extractor._process_file("/src/main.cpp", lst) +# +# assert_that(extractor.graph.nodes, not_(has_item("// a comment"))) +# +# +# diff --git a/test/lst/test_clang_adapter.py b/test/lst/test_clang_adapter.py new file mode 100644 index 00000000..cfebe6e9 --- /dev/null +++ b/test/lst/test_clang_adapter.py @@ -0,0 +1,20 @@ +import pytest +from hamcrest import * +from pathlib import Path + +import targets +from renaissance.impl.clang.clang_adapter import ClangAdapter +from renaissance.impl.tree_sitter.lst import LST +from renaissance.utils.ast_utils import traverse + + +class TestClangAdapter: + def test_parse_cpp_file(self): + adapter = ClangAdapter() + lst = adapter.parse(Path(targets.__file__).parent / "cpp_example.cpp") + assert_that(lst, is_(LST)) + assert_that(list(traverse(lst.root)), has_length(greater_than(0))) + + +if __name__ == "__main__": + pytest.main() diff --git a/test/lst/test_clang_concrete_pattern_matcher.py b/test/lst/test_clang_concrete_pattern_matcher.py new file mode 100644 index 00000000..973f3a6b --- /dev/null +++ b/test/lst/test_clang_concrete_pattern_matcher.py @@ -0,0 +1,129 @@ + +from hamcrest import * + +import pytest + + +from renaissance.impl.tree_sitter.extractor import Extractor +from renaissance.impl.clang.clang_adapter import ClangAdapter +from renaissance.impl.tree_sitter.factory import TreeStiterPatternFactory +from renaissance.syntax_tree.match_finder import MIS_MATCH + + +class TestClangConcretePatternMatcher: + @pytest.mark.parametrize( + "code, pattern", + [ + ("int body=0;int main() { return body; }", "int body=0;int main() { return body; }"), + ("int init, cond, inc=0;int body=0;for (;;) {}", "int $i, $c, $inc=0;int $b=0;for ($i; $c; $inc) $b"), + ("a = b;", "$lhs = $rhs;"), + ("int x,y;x + y;", "int $a,$b;$a + $b;"), + ("int x;-x;", "int $x;-$x;"), + ("foo();", "$f();"), + ("class A {};", "class $C {};"), + ("struct B { int x; };", "struct $S { $body };"), + ("namespace ns {}", "namespace $ns {}"), + ("int C=0; template <typename T> class C {};", "int $C=0; template <typename T> class $C {};"), + ("int body=0; auto f = []() { return 1; };", "int $body=0; auto $f = []() { $body; };"), + ], + ) + def test_clang_patterns(self, code, pattern): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + extractor = Extractor(interface, [pattern]) + matches = extractor.run(code) + assert_that(matches, is_not(empty())) + + def test_clang_patterns_using_extractor(self): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + extractor = Extractor(interface, ["int E=0; int vals=0; enum E { A };"]) + matches = extractor.run("int E = 0; int vals=0; enum E { A };") + assert_that(matches, has_length(1)) + + def test_clang_failing_pattern(self): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + pattern = interface.create_statements("int E=0; int vals=0; enum E { A };") + code = adapter.load_from_text("int E=0; int vals=0; enum E { A };", "snippets.c") + matches = match_pattern(code.root.children, pattern) + assert_that(matches, has_length(1)) + + def test_find_variant_with_clang_failing_pattern(self): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + pattern = interface.create_statements("int E1=0; int vals=0; enum E2 { A };") + code = adapter.load_from_text("int E1=0; int vals=0; enum E2 { A };", "snippets.c") + matches = find_variants(code.root.children, pattern) + assert_that(matches, has_length(1)) + assert_that(matches[0].end_index, is_not(MIS_MATCH)) + + @pytest.mark.skip("it should be tha same really") + def test_type_property_between_code_and_pattern_are_same(self): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + pattern = interface.create_statement("enum E2 { A };") + code = adapter.load_from_text("enum E2 { A };", "snippets.c").root.children[0] + assert_that(code.properties["type"], is_(pattern.properties["type"])) + + @pytest.mark.parametrize( + "code, pattern", + [ + ( + "int add(int a, int b) { return a + b; }", + "int $a,$b,$body;int $f(int $a, int $b) { $body; }", + ), + ("void f() { int x = 0; }", "int $body=0;void $name() { $body }"), + ("if (x) { y(); }", "int $cond,$body=0;if ($cond) { $body }"), + ("while (x) {}", "int $cond;while ($cond) $body"), + ("do {} while (x);", "int $body,$cond;do $body while ($cond);"), + ("switch(x) { case 1: break; }", "int $val,$cases;switch ($val) { $cases }"), + ("try {} catch (...) {}", "int $body, $handler;try $body catch (...) $handler"), + ], + ) + def test_clang_patterns_to_be_fixed(self, code, pattern): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + extractor = Extractor(interface, [pattern]) + matches = extractor.run(code) + assert_that(matches, has_length(0)) # but should be 1 + + def test_is_match_clang_patterns_without_decl(self): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + c = interface.create_statement("int main() { return 0; }") + p = interface.create_statement("int main() { return $body; }") + assert_that(is_match(c.children[-1], p.children[-1], {}), is_(False)) + + def test_is_match_clang_patterns_with_decl(self): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + c = interface.create_statement("int $body=0; int main() { return 0; }") + p = interface.create_statement("int $body=0; int main() { return $body; }") + assert_that(is_match(c.children[-1], p.children[-1], {}), is_(True)) + + def test_is_match_clang_tree(self): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + c = interface.create_statement("int $body=0; int main() { return 0; }") + p = interface.create_statement("int $body=0; int main() { return $body; }") + assert_that(is_match_tree([c.children[-1]], [p.children[-1]], {}), is_(True)) + + def test_is_match_clang_patterns(self): + adapter = ClangAdapter() + interface = TreeStiterPatternFactory(adapter) + c = interface.create_statement("int $body=0; int main() { return 0; }") + p = interface.create_statement("int $body=0; int main() { return $body; }") + match = MatchFinder.match_pattern([c.children[-1]], [p.children[-1]]) + assert_that(match, has_length(1)) + + +from renaissance.syntax_tree.match_finder import is_match, is_match_tree, MatchFinder, match_pattern, find_variants + + +class Matchfinder: + pass + + +if __name__ == "__main__": + pytest.main() diff --git a/test/lst/test_concrete_pattern_matcher.py b/test/lst/test_concrete_pattern_matcher.py new file mode 100644 index 00000000..7c66126a --- /dev/null +++ b/test/lst/test_concrete_pattern_matcher.py @@ -0,0 +1,86 @@ +import pytest +import tree_sitter_python +from hamcrest import * + +from renaissance.impl.tree_sitter.extractor import Extractor +from renaissance.impl.tree_sitter.adapter import TreeSitterAdapter +from renaissance.impl.tree_sitter.factory import TreeStiterPatternFactory +from renaissance.syntax_tree.match_finder import is_match, is_match_tree, match_pattern + + +class TestConcretePatternMatcher: + @pytest.mark.parametrize( + "code, pattern", + [ + ("def foo(): pass", "def foo(): pass"), + ("if x: print(x)", "if x: $body"), + ("for i in range(10): print(i)", "for $i in $iter: $body"), + ("while True: pass", "while $cond: $body"), + ("try: pass\nexcept Exception: pass", "try: $b\nexcept Exception: $b"), + ("class A: pass", "class $C: $body"), + ("with open('x') as f: pass", "with $ctx as $var: $body"), + ("assert x", "assert $cond"), + ("return x", "return $value"), + ("lambda x: x", "lambda $arg: $body"), + ("a = b", "$lhs = $rhs"), + ("a += b", "$lhs += $rhs"), + ("x and y", "$left and $right"), + ("not x", "not $expr"), + ("x if y else z", "$t if $cond else $f"), + ("f(x)", "$func($arg)"), + ("[x for x in y]", "[$x for $x in $y]"), + ("x in y", "$x in $y"), + ("import os", "import $mod"), + ("import os\nx=5", "import $mod $stmt"), + ], + ) + def test_python_pattern(self, code, pattern): + adapter = TreeSitterAdapter(tree_sitter_python) + interface = TreeStiterPatternFactory(adapter) + extractor = Extractor(interface, [pattern]) + matches = extractor.run(code) + + assert_that(matches, has_length(1), f"{code=} {pattern=}") + + def test_is_match_python_patterns(self): + adapter = TreeSitterAdapter(tree_sitter_python) + interface = TreeStiterPatternFactory(adapter) + c = interface.create_statement("try: pass\nexcept Exception: pass") + p = interface.create_statement("try: $b\nexcept Exception: $b") + assert_that(is_match(c.children[0], p.children[0], {}), is_(True)) # type: ignore + assert_that(is_match(c.children[1], p.children[1], {}), is_(True)) # type: ignore + assert_that(is_match(c.children[2], p.children[2], {}), is_(True)) # type: ignore + assert_that(is_match(c.children[3], p.children[3], {}), is_(True)) # type: ignore + + def test_is_match_python_patterns_tree(self): + adapter = TreeSitterAdapter(tree_sitter_python) + interface = TreeStiterPatternFactory(adapter) + c = interface.create_statement("try: pass\nexcept Exception: pass") + p = interface.create_statement("try: $b\nexcept Exception: $b") + assert_that(is_match_tree(c.children, p.children, {}), is_(True)) + + def test_is_match_python_patterns_1(self): + adapter = TreeSitterAdapter(tree_sitter_python) + interface = TreeStiterPatternFactory(adapter) + c = interface.create_statement("if x: print(x)") + p = interface.create_statement("if x: $body") + assert_that(is_match(c, p), is_(True)) + assert_that(match_pattern([c], [p]), is_not(empty())) # type: ignore + + def test_is_match(self): + adapter = TreeSitterAdapter(tree_sitter_python) + interface = TreeStiterPatternFactory(adapter) + c = interface.create_statement("def foo(): pass") + p = interface.create_statement("def foo(): pass") + assert_that(is_match(c, p), is_(True)) + + +# def test_python_patterns_tree_1(self): +# adapter = TreeSitterAdapter(tspython) +# interface = TsPatternFactory(adapter) +# cc = interface.create_statements("if x: print(x)") +# pp = interface.create_statements("if x: $body") +# assert is_match_tree(cc, pp, {}) + +if __name__ == "__main__": + pytest.main() diff --git a/test/lst/test_languages.py b/test/lst/test_languages.py new file mode 100644 index 00000000..8818296d --- /dev/null +++ b/test/lst/test_languages.py @@ -0,0 +1,90 @@ +import pytest +import tree_sitter_cpp as tscpp +import tree_sitter_java as tsjava +import tree_sitter_python as tspython +from hamcrest import * + +from renaissance.impl.tree_sitter.adapter import TreeSitterAdapter +from renaissance.impl.tree_sitter.lst import LST +from renaissance.utils.ast_utils import traverse + + +class TestLanguages: + @pytest.mark.parametrize( + "lang, code", + [ + (tspython, "def add(x, y): return x + y"), + (tspython, "if x > 0: print(x)"), + (tspython, "for i in range(10): print(i)"), + (tspython, "while True: break"), + (tspython, "try: x = 1 except: x = 2"), + (tspython, "class Foo: def bar(self): pass"), + (tspython, "import math"), + (tspython, "with open('x') as f: data = f.read()"), + (tspython, "@decorator def func(): pass"), + (tspython, "lambda x: x * 2"), + (tspython, "x = 5"), + (tspython, "assert x > 0"), + (tspython, "print('hello')"), + (tspython, "def outer(): def inner(): pass"), + (tspython, "raise ValueError('error')"), + (tspython, "yield x"), + (tspython, "global x"), + (tspython, "nonlocal x"), + (tspython, "pass"), + (tspython, "continue"), + # tsjava + (tsjava, "public class A {}"), + (tsjava, "public class A { void m() {} }"), + (tsjava, "int x = 5;"), + (tsjava, 'String s = "hi";'), + (tsjava, "if (x > 0) {}"), + (tsjava, "for (int i = 0; i < 10; i++) {}"), + (tsjava, "while (true) {}"), + (tsjava, "do {} while (false);"), + (tsjava, "switch (x) { case 1: break; }"), + (tsjava, "try {} catch (Exception e) {}"), + (tsjava, "void m() { return; }"), + (tsjava, "class A { int x; A() {} }"), + (tsjava, "interface I {}"), + (tsjava, "enum E { A, B }"), + (tsjava, "import java.util.*;"), + (tsjava, "package test;"), + (tsjava, "@Override void m() {}"), + (tsjava, "class B extends A {}"), + (tsjava, "new Object();"), + (tsjava, 'System.out.println("hi");'), + # tscpp + (tscpp, "int main() { return 0; }"), + (tscpp, "int add(int a, int b) { return a + b; }"), + (tscpp, "#include <iostream>"), + (tscpp, "using namespace std;"), + (tscpp, "class A {};"), + (tscpp, "struct B { int x; };"), + (tscpp, "template<typename T> class C {};"), + (tscpp, "enum Color { RED, GREEN };"), + (tscpp, "void loop() { for (int i = 0; i < 10; i++) {} }"), + (tscpp, "if (x > 0) {}"), + (tscpp, "while (true) {}"), + (tscpp, "switch (x) { case 1: break; }"), + (tscpp, "try {} catch (...) {}"), + (tscpp, "auto f = []() { return 1; };"), + (tscpp, "int* ptr = nullptr;"), + (tscpp, 'std::cout << "Hello" << std::endl;'), + (tscpp, "namespace ns {}"), + (tscpp, "bool flag = true;"), + (tscpp, "char c = 'a';"), + (tscpp, "float pi = 3.14f;"), + ], + ) + def test_language_parsing(self, lang, code): + adapter = TreeSitterAdapter(lang) + tree = adapter.parse_code(code) + lst = adapter.to_lst(code, tree) + assert_that(lst, is_(LST)) + nodes = list(traverse(lst.root)) + assert_that(nodes, has_length(greater_than(0))) + + +if __name__ == "__main__": + pytest.main() diff --git a/test/lst/test_matchers.py b/test/lst/test_matchers.py new file mode 100644 index 00000000..cd7c9162 --- /dev/null +++ b/test/lst/test_matchers.py @@ -0,0 +1,73 @@ +import pytest +import tree_sitter_cpp as tscpp +from hamcrest import assert_that, has_length + +from renaissance.impl.tree_sitter.adapter import TreeSitterAdapter +from renaissance.impl.tree_sitter.lst import LSTNode +from renaissance.impl.types import Call + +from renaissance.syntax_tree.ast_finder import find_ast_type +from renaissance.syntax_tree.match_finder import is_match +from renaissance.utils.ast_utils import traverse + + +class TestMatchers: + + @pytest.fixture(autouse=True) + def setUp(self): + adapter = TreeSitterAdapter(tscpp) + self.if_node = self.make_pattern("if (x > 0) print(x);", adapter) + self.for_node = self.make_pattern("for (i in range(10)) print(i);", adapter) + self.while_node = self.make_pattern("while (x < 10) x += 1;", adapter) + self.try_node = self.make_pattern( + "try { risky_operation(); } catch (Exception e) { handle_error(e); }", + adapter, + ) + self.class_node = self.make_pattern("class MyClass { method(self) { pass; } }", adapter) + + def test_if_pattern_match(self): + adapter = TreeSitterAdapter(tscpp) + pattern = self.make_pattern("if ($x > 0) print($x);", adapter) + + assert_that(is_match(self.if_node, pattern)) + + def test_for_pattern_match(self): + adapter = TreeSitterAdapter(tscpp) + pattern = self.make_pattern("for ($i in range(10)) print($i);", adapter) + assert_that(is_match(self.for_node, pattern)) + + def test_while_pattern_match(self): + adapter = TreeSitterAdapter(tscpp) + pattern = self.make_pattern("while ($x < 10) $x += 1;", adapter) + assert_that(is_match(self.while_node, pattern)) + + def test_try_pattern_match(self): + adapter = TreeSitterAdapter(tscpp) + pattern = self.make_pattern( + "try { risky_operation(); } catch (Exception $e) { handle_error($e); }", + adapter, + ) + assert_that(is_match(self.try_node, pattern)) + + def test_class_pattern_match(self): + adapter = TreeSitterAdapter(tscpp) + pattern = self.make_pattern("class MyClass { method(self) { pass; } }", adapter) + assert_that(is_match(self.class_node, pattern)) + + def test_node_type_match(self): + matches = [node for node in traverse(self.if_node) if node.ast_type == Call] + assert_that(matches, has_length(1)) + + + def test_node_type_match_exact_type(self): + matches = find_ast_type(self.if_node, Call) + assert_that(matches, has_length(1)) + + def make_pattern(self, code: str, adapter: any) -> LSTNode: + tree = adapter.parse_code(code) + root = adapter.to_lst(code, tree) + return root.root + + +if __name__ == "__main__": + pytest.main() diff --git a/test/lst/test_show_node_in_mermaid.py b/test/lst/test_show_node_in_mermaid.py new file mode 100644 index 00000000..20f01ad3 --- /dev/null +++ b/test/lst/test_show_node_in_mermaid.py @@ -0,0 +1,34 @@ +import textwrap + +import tree_sitter_python as tspython +import tree_sitter_cpp as tscpp +import tree_sitter_java as tsjava +import pytest +from hamcrest import * + +from renaissance.impl.tree_sitter.adapter import TreeSitterAdapter +from renaissance.impl.tree_sitter.visualizer import LstVisualizer + + +class TestShowNodeInMermaid: + def process_code(self, grammar_module, code): + adapter = TreeSitterAdapter(grammar_module) + tree = adapter.parse_code(code) + lst = adapter.to_lst(code, tree) + visualizer = LstVisualizer() + mermaid = visualizer.render(lst) + return mermaid + + @pytest.mark.parametrize( + "raw, module", + [ + ("def foo():\n return 42", tspython), + ("int main() { return 0; }", tscpp), + ("public class Test { public static void main(String[] args) {} }", tsjava), + ], + ) + def test_create_diagrams(self, raw, module): + result = self.process_code(module, raw) + # with open(f"lst_output_{module.__name__}.mmd", "w", encoding="utf-8") as f: + # f.write(mermaid) + assert_that(textwrap.dedent(result), is_not(empty())) diff --git a/test/lst/test_tree_sitter_parse.py b/test/lst/test_tree_sitter_parse.py new file mode 100644 index 00000000..cc3197a8 --- /dev/null +++ b/test/lst/test_tree_sitter_parse.py @@ -0,0 +1,33 @@ +from hamcrest import assert_that, is_ +from tree_sitter import Language, Parser +import tree_sitter_python as tspython +import tree_sitter_cpp as tscpp +import tree_sitter_java as tsjava + +# Load compiled languages +PY_LANGUAGE = Language(tspython.language()) +CPP_LANGUAGE = Language(tscpp.language()) +JAVA_LANGUAGE = Language(tsjava.language()) + +# Create parsers +py_parser = Parser(PY_LANGUAGE) +cpp_parser = Parser(CPP_LANGUAGE) +java_parser = Parser(JAVA_LANGUAGE) + +# Sample inputs +py_code = b"def foo():\n if bar:\n baz()\n" + +cpp_code = b"public class Test {\n public static void main(String[] args) {\n " b" if (ready) start();\n }\n}\n" + +java_code = b"public class Test {\n public static void main(String[] args) {\n " b" if (ready) start();\n }\n}\n" + + +class TestTreeSitterParse: + def test_parse_py_code(self): + assert_that(py_code, is_(py_parser.parse(py_code).root_node.text)) + + def test_parse_cpp_code(self): + assert_that(cpp_code, is_(cpp_parser.parse(cpp_code).root_node.text)) + + def test_parse_java_code(self): + assert_that(java_code, is_(java_parser.parse(java_code).root_node.text)) diff --git a/test/project/__init__.py b/test/project/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/project/test_project_scanner.py b/test/project/test_project_scanner.py new file mode 100644 index 00000000..82a2bd19 --- /dev/null +++ b/test/project/test_project_scanner.py @@ -0,0 +1,233 @@ +import json +import pytest +from hamcrest import assert_that, is_, equal_to, contains_inanyorder, empty, calling, raises + +from renaissance.project.project_scanner import ( + ProjectScanner, + CppScanner, + JavaScanner, + PythonScanner, + BearCppScanner, +) + +# --------------------------------------------------------------------------- +# ProjectScanner (base) +# --------------------------------------------------------------------------- + + +class TestProjectScanner: + def test_find_sources_raises_not_implemented(self): + scanner = ProjectScanner() + assert_that(calling(scanner.find_sources), raises(NotImplementedError)) + + +# --------------------------------------------------------------------------- +# CppScanner +# --------------------------------------------------------------------------- + + +class TestCppScanner: + def test_raises_file_not_found_when_compile_commands_missing(self, tmp_path): + scanner = CppScanner(str(tmp_path / "compile_commands.json")) + assert_that(calling(scanner.find_sources), raises(FileNotFoundError)) + + def test_returns_sorted_unique_files(self, tmp_path): + commands = [ + {"file": "/src/b.cpp"}, + {"file": "/src/a.cpp"}, + {"file": "/src/b.cpp"}, + ] + compile_commands = tmp_path / "compile_commands.json" + compile_commands.write_text(json.dumps(commands)) + + scanner = CppScanner(str(compile_commands)) + result = scanner.find_sources() + + assert_that(result, equal_to(["/src/a.cpp", "/src/b.cpp"])) + + def test_ignores_entries_without_file_key(self, tmp_path): + commands = [{"command": "cc -c foo.cpp"}, {"file": "/src/a.cpp"}] + compile_commands = tmp_path / "compile_commands.json" + compile_commands.write_text(json.dumps(commands)) + + scanner = CppScanner(str(compile_commands)) + result = scanner.find_sources() + + assert_that(result, equal_to(["/src/a.cpp"])) + + def test_returns_empty_list_for_empty_compile_commands(self, tmp_path): + compile_commands = tmp_path / "compile_commands.json" + compile_commands.write_text(json.dumps([])) + + scanner = CppScanner(str(compile_commands)) + result = scanner.find_sources() + + assert_that(result, is_(empty())) + + def test_default_compile_commands_path(self): + scanner = CppScanner() + assert_that(scanner.compile_commands_path, is_("compile_commands.json")) + + +# --------------------------------------------------------------------------- +# JavaScanner +# --------------------------------------------------------------------------- + + +class TestJavaScanner: + def test_finds_java_files_recursively(self, tmp_path): + (tmp_path / "src").mkdir() + (tmp_path / "src" / "Main.java").write_text("class Main {}") + (tmp_path / "src" / "sub").mkdir() + (tmp_path / "src" / "sub" / "Util.java").write_text("class Util {}") + + scanner = JavaScanner(str(tmp_path)) + result = scanner.find_sources() + + assert_that( + result, + contains_inanyorder( + str(tmp_path / "src" / "Main.java"), + str(tmp_path / "src" / "sub" / "Util.java"), + ), + ) + + def test_returns_sorted_results(self, tmp_path): + (tmp_path / "B.java").write_text("") + (tmp_path / "A.java").write_text("") + + scanner = JavaScanner(str(tmp_path)) + result = scanner.find_sources() + + assert_that(result, equal_to(sorted(result))) + + def test_returns_empty_list_when_no_java_files(self, tmp_path): + scanner = JavaScanner(str(tmp_path)) + result = scanner.find_sources() + + assert_that(result, is_(empty())) + + def test_default_root_dir(self): + scanner = JavaScanner() + assert_that(scanner.root_dir, is_(".")) + + +# --------------------------------------------------------------------------- +# PythonScanner +# --------------------------------------------------------------------------- + + +class TestPythonScanner: + def test_finds_python_files_in_package_dirs(self, tmp_path): + src = tmp_path / "src" + src.mkdir() + (src / "module.py").write_text("") + (src / "sub").mkdir() + (src / "sub" / "helper.py").write_text("") + + scanner = PythonScanner(str(tmp_path), package_dirs=["src"]) + result = scanner.find_sources() + + assert_that( + [str(p) for p in result], + contains_inanyorder( + str(src / "module.py"), + str(src / "sub" / "helper.py"), + ), + ) + + def test_skips_nonexistent_package_dirs(self, tmp_path): + scanner = PythonScanner(str(tmp_path), package_dirs=["nonexistent"]) + result = scanner.find_sources() + + assert_that(result, is_(empty())) + + def test_returns_sorted_results(self, tmp_path): + src = tmp_path / "src" + src.mkdir() + (src / "z_module.py").write_text("") + (src / "a_module.py").write_text("") + + scanner = PythonScanner(str(tmp_path), package_dirs=["src"]) + result = scanner.find_sources() + + assert_that(result, equal_to(sorted(result))) + + def test_searches_multiple_package_dirs(self, tmp_path): + (tmp_path / "src").mkdir() + (tmp_path / "src" / "a.py").write_text("") + (tmp_path / "lib").mkdir() + (tmp_path / "lib" / "b.py").write_text("") + + scanner = PythonScanner(str(tmp_path), package_dirs=["src", "lib"]) + result = [str(p) for p in scanner.find_sources()] + + assert_that( + result, + contains_inanyorder( + str(tmp_path / "src" / "a.py"), + str(tmp_path / "lib" / "b.py"), + ), + ) + + def test_default_package_dirs(self): + scanner = PythonScanner() + assert_that(scanner.package_dirs, equal_to(["src", "lib", "test"])) + + def test_default_root_dir(self): + scanner = PythonScanner() + assert_that(scanner.root_dir, is_(".")) + + +# --------------------------------------------------------------------------- +# BearCppScanner +# --------------------------------------------------------------------------- + + +class TestBearCppScanner: + def test_find_sources_calls_run_bear_when_compile_commands_missing(self, tmp_path, mocker): + scanner = BearCppScanner( + build_dir=str(tmp_path), + compile_commands_path=str(tmp_path / "compile_commands.json"), + ) + mock_bear = mocker.patch.object(scanner, "run_bear") + + # After run_bear is called the file still won't exist, so super().find_sources() + # will raise FileNotFoundError — that's acceptable; we only care that run_bear ran. + with pytest.raises(FileNotFoundError): + scanner.find_sources() + + assert_that(mock_bear.call_count, is_(1)) + + def test_find_sources_does_not_call_run_bear_when_compile_commands_exists(self, tmp_path, mocker): + commands = [{"file": "/src/main.cpp"}] + compile_commands = tmp_path / "compile_commands.json" + compile_commands.write_text(json.dumps(commands)) + + scanner = BearCppScanner( + build_dir=str(tmp_path), + compile_commands_path=str(compile_commands), + ) + mock_bear = mocker.patch.object(scanner, "run_bear") + + result = scanner.find_sources() + + mock_bear.assert_not_called() + assert_that(result, equal_to(["/src/main.cpp"])) + + def test_run_bear_raises_on_nonzero_exit(self, mocker): + scanner = BearCppScanner() + mocker.patch("renaissance.project.project_scanner.system", return_value=1) + + assert_that(calling(scanner.run_bear), raises(RuntimeError)) + + def test_run_bear_succeeds_on_zero_exit(self, mocker): + scanner = BearCppScanner() + mocker.patch("renaissance.project.project_scanner.system", return_value=0) + + # Should not raise + scanner.run_bear() + + def test_default_build_dir(self): + scanner = BearCppScanner() + assert_that(scanner.build_dir, is_(".")) diff --git a/test/python/__init__.py b/test/python/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/python/factories.py b/test/python/factories.py new file mode 100644 index 00000000..58ad0f52 --- /dev/null +++ b/test/python/factories.py @@ -0,0 +1,25 @@ +import ast +from itertools import product + +from renaissance.impl.python.factory import PythonFactory +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.cst_node import PythonCstNode +from renaissance.impl.tree_sitter.lst import LSTNode + + +class Factories: + # add factories here to test different ASTNode implementations + node_types = [ + ("ast", ast.AST), + ("cst", PythonCstNode), + ("lst", LSTNode), + ("rst", PythonRstNode), + ] + factories = [(name_type[0], PythonFactory(name_type[1])) for name_type in node_types] + + @staticmethod + def extend(test_parameters: list[tuple]) -> list[tuple]: + result = [ + (str(factory[0]) + " " + str(pars[0]), factory[1], *pars) for factory, pars in product(Factories.factories, test_parameters) + ] + return result diff --git a/test/python/test_patternic_style.py b/test/python/test_patternic_style.py new file mode 100644 index 00000000..f65ce8fc --- /dev/null +++ b/test/python/test_patternic_style.py @@ -0,0 +1,185 @@ +from operator import is_not + +import pytest +from hamcrest import assert_that, is_, has_length, is_in, is_not, empty, instance_of + +from renaissance.impl.types import * +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonPatternFactory, PythonFactory +from renaissance.impl.types import MatchOne, MatchAll, TranslationUnit +from renaissance.syntax_tree.match_finder import is_match + + +class TestPythonicStyle: + @pytest.fixture(autouse=True) + def setup(self): + self.factory = PythonFactory(PythonRstNode) + self.pattern_factory = PythonPatternFactory(self.factory) + + @pytest.mark.parametrize( + "raw, kind, op, name, expr, body_length", + [ + ("try:\n pass\nfinally:\n pass", Try, "try", "Try", "expr", 1), + ("try:\n x()\nexcept* e:\n pass", Try, "try", "Try", "expr", 1), + ("class name: pass", ClassDef, "class", "name", "expr", 1), + ("def name(): pass", FunctionDef,"function", "name", "expr", 1), + ("for name in expr:\n 1\n 2\n pass", For, "for", "name", "expr", 3), + ("while expr: pass", While, "while", "While", "expr", 1), + ("if expr: pass\nelse: pass ", If, "if", "If", "expr", 1), + ("match x:\n case _: pass", Match, "match", "x", "expr", 1), + ("async for f in fs: pass", For, "for", "f", "",1), + ('async with open("x"): pass', With, "with", "With","", 1), + ("async def fun(): pass", FunctionDef,"function", "fun","", 1), + ], + ) + def test_consistent_name_stmt(self, raw, kind, op, name, expr, body_length): + it = PythonRstNode.load_from_text(raw).body[-1] + assert_that(it.ast_type(), instance_of(kind)) + assert_that(it.operator, is_(op)) + if isinstance(it.name, str): + assert_that(it.name, is_(name)) + # assert_that(it.expr.name, is_(expr)) + assert_that(it.body, has_length(body_length)) + + @pytest.mark.parametrize( + "raw, kind, typ, name, op, value", + [ + ("i:int=0", Assign, "int", "i", "=", 0), + ("i=0", Assign, None, "i", "=", 0), + ("x += 5", AugAssign, None, "x", "+=", 5), + ("break", Break, None, "", "break", None), + ("assert 0", Assert, None, "", "assert", 0), + ("continue", Continue, None, "", "continue", None), + ("import x", Import, None, "x", "import", None), + ("pass",Pass, None, "", "pass",None ) + ], + ) + def test_stmt(self, raw, kind, typ, name, op, value): + it = PythonRstNode.load_from_text(raw).body[-1] + assert_that(it.ast_type(), instance_of(kind)) + assert_that(it.name, is_(name)) + assert_that(it.operator, op) + assert_that(it.type, is_(typ)) + assert_that(it.value, is_(value)) + + @pytest.mark.parametrize( + "raw, kind, expr", + [ + ("fun()", ExpressionStatement, "fun()"), + ("return fun()", Return, "fun()"), + ("raise fun()", Raise, "fun()"), + ], + ) + # ('from x import y', 'ImportFrom', None, 'x', 'import', 'y'), + def test_expr(self, raw, kind, expr): + it = PythonRstNode.load_from_text(raw).body[-1] + assert_that(it.ast_type(), instance_of(kind)) + assert_that(it.expr.name, is_(expr)) + + def test_ann_assign_node(self): + it = PythonRstNode.load_from_text('name:str = "value"').body[-1] + + assert_that(it.name, is_("name")) + assert_that(it.type, is_("str")) + assert_that(it.operator, is_("=")) + assert_that(it.value, is_("value")) + + def test_assign_node(self): + it = PythonRstNode.load_from_text('name = "value"').body[-1] + assert_that(it.name, is_("name")) + assert_that(it.type, is_(None)) + assert_that(it.operator, is_("=")) + assert_that(it.value, is_("value")) + + def test_assign_node_2(self): + it = PythonRstNode.load_from_text("name += 5").body[-1] + assert_that(it.name, is_("name")) + assert_that(it.type, is_(None)) + assert_that(it.operator, is_("+=")) + assert_that(it.value, is_(5)) + + def python_does_not_parse_dollar(self): + it = PythonRstNode.load_from_text("$pa") + assert_that(it.ast_type, is_(MatchOne)) + + def python_does_not_parse_dollar(self): + it = PythonRstNode.load_from_text("$$pa") + assert_that(it.ast_type, is_(MatchAll)) + + def test_kind_is_match_all(self): + pattern_factory = PythonPatternFactory(PythonFactory(PythonRstNode)) + simple = self.pattern_factory.create_statement("$$pa") + assert_that(simple.ast_type(), instance_of(MatchAll)) + + def test_kind_is_match_one(self): + simple = self.pattern_factory.create_statement("$pa") + assert_that(simple.ast_type(), instance_of(MatchOne)) + + def test_match_one_is_not_equal(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + pattern_factory = PythonPatternFactory(self.factory) + match_one = self.pattern_factory.create("$pa") + assert_that(atu.children[0], is_not(match_one)) + + def test_is_match_all_stmt(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + match_all = self.pattern_factory.create("$$pa") + assert_that(match_all.node, is_in(atu)) + + def test_is_exact_match(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + stmt = PythonRstNode.load_from_text("ba(55)")[0] + assert_that(atu.children[0], is_(stmt)) + + def test_match_exact_pattern(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + stmt = self.pattern_factory.create_statement("ba(55)").node + result = [node for node in atu if node == stmt] + assert_that(result, has_length(1)) + + def test_match_single_pattern(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + match_any = self.pattern_factory.create_statement("$stmt") + result = [node for node in atu if node == match_any] + assert_that(result, is_(empty())) + result = [node for node in atu if is_match(node, match_any, {})] + assert_that(result, has_length(4)) + + def test_match_single_call_pattern(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + match_call = self.pattern_factory.create("$call($arg)") + result = [node for node in atu if node == match_call] + assert_that(result, has_length(0)) + + def test_find_all_using_generic_matcher(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + simple = self.pattern_factory.create_statement("ca(555)").node + assert_that(atu[0], is_not(simple)) + assert_that(atu[1], is_(simple)) + assert_that(atu[2], is_not(simple)) + assert_that(atu[3], is_not(simple)) + result = [node for node in atu if node == simple] + assert_that(result, has_length(1)) + + def test_slice_call(self): + atu = self.factory.create_from_text( + "ba(55)\nna(55)\nna(55)\npa(55)\npa(55)\nba(55)\nna(55)\nna(55)\nna=55", + "test.py", + ) + node_slice = atu[0:3] + assert_that(node_slice, has_length(3)) + + def test_property_kind_call(self): + atu = self.factory.create_from_text( + "ba(55)\nna(55)\nna(55)\npa(55)\npa(55)\nba(55)\nna(55)\nna(55)\nna=55", + "test.py", + ) + assert_that(atu.ast_type(), instance_of(TranslationUnit)) + + def test_property_name_call(self): + atu = self.factory.create_from_text( + "ba(55)\nna(55)\nna(55)\npa(55)\npa(55)\nba(55)\nna(55)\nna(55)\nna=55", + "test.py", + ) + name = atu.name + assert_that(name, is_("test.py")) diff --git a/test/python/test_python_ast_node_ref.py b/test/python/test_python_ast_node_ref.py new file mode 100644 index 00000000..4975da68 --- /dev/null +++ b/test/python/test_python_ast_node_ref.py @@ -0,0 +1,181 @@ +import tempfile + +import pytest +from hamcrest import * +from more_itertools.more import first + +from renaissance import syntax_tree +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonFactory +from renaissance.impl.python.rst_node import PythonRSTReference +from renaissance.impl.types import FunctionDef, Name, Call, ClassDef, Arg +from renaissance.utils.ast_utils import traverse + +content = """ +# antagonist +class cat: + def __init__(self): + self.out_of_shadow =True + def is_near(self): + return not self.out_of_shadow +# protagonist +class mice: + def be_high_alert_of(self): + self.high_alert =True + + def discover(self, bruno:cat): + if bruno.is_near(): + self.be_high_alert_of() +# main function +if __name__ == '__main__': + jerry = mice() + tom = cat() + jerry.discover(tom) + +""".strip() + +content2 = """ +def a() -> int: + return 42 +def b(x) -> None: + x += 1 +def f() -> None: + x: int = a() + b(x) + # do something with x +""".strip() + +content3 = """ +class B: + def __init__(self, value): + self.value = value + def base_method(self): + return "This method is defined in the base class B" + +class A(B): + def __init__(self, value, extra_value): + # Call the parent class's __init__ method + super().__init__(value) + self.extra_value = extra_value + def subclass_method(self): + return "This method is only in subclass A" + +# Create instances of both classes +b_instance = B("Base") +a_instance = A("Derived", "Extra") +""" + + +class TestPythonNode: + + @pytest.fixture(autouse=True) + def setup(self): + """Setup that runs before each test method""" + self.factory = PythonFactory(PythonRstNode) + + def test_def_call_references(self): + # Function f() refers to Function a() + ast = PythonRstNode.load_from_text(content2) + with tempfile.TemporaryDirectory(delete=True) as temp_dir: + syntax_tree.ASTShower.store_node(temp_dir + "/py0.txt", ast) + + func_def = first(n for n in traverse(ast) if isinstance(n.ast_type(), FunctionDef) and n.name == "f") + assert_that(func_def, is_(PythonRstNode)) + ast.translation_unit.lazy_create_refers(ast) + refs = func_def.references + assert_that(refs, has_length(2)) + ref = refs[0] + ref_node = ast.translation_unit._nodes[ref.node_id] + assert_that(ref_node.ast_type(), instance_of(FunctionDef)) + assert_that(ref_node.name.lower(), is_("a")) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(1)) # Function a referenced by function f and var x. + assert_that(func_def in [ast.translation_unit._nodes[r.node_id] for r in referenced_by]) + ref1 = refs[1] + ref_node1 = ast.translation_unit._nodes[ref1.node_id] + assert_that(ref_node.ast_type(), instance_of(FunctionDef)) + assert_that(ref_node1.name.lower(), is_("b")) + referenced_by1 = ref_node1.referenced_by + assert_that(referenced_by1, has_length(1)) # Function b referenced by function f. + assert_that(func_def in [ast.translation_unit._nodes[r.node_id] for r in referenced_by]) + + def test_type_reference(self): + # Name z refers to Name a + ast = self.factory.create_from_text("from abc import a\nx = a()\nz: a = x", "content3.py") + with tempfile.TemporaryDirectory(delete=True) as temp_dir: + syntax_tree.ASTShower.store_node(temp_dir + "/py1.txt", ast) + type_node = first(n for n in traverse(ast) if isinstance(n.ast_type(), Name) and n.name == "z") + assert_that(type_node, is_(PythonRstNode)) + ast.translation_unit.lazy_create_refers(ast) + refs = type_node.references + assert_that(refs, has_length(1)) + ref = refs[0] + ref_node = ast.translation_unit._nodes[ref.node_id] + assert_that(ref_node.ast_type(), instance_of(Name)) + assert_that(ref_node.name.lower(), is_("a")) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(greater_than(0))) + assert_that(type_node in [ast.translation_unit._nodes[r.node_id] for r in referenced_by]) + + def test_class_reference(self): + # Class A refers to Class B + ast = self.factory.create_from_text(content3, "content3.py") + with tempfile.TemporaryDirectory(delete=True) as temp_dir: + syntax_tree.ASTShower.store_node(temp_dir + "/py2.txt", ast) + + class_node = first(n for n in traverse(ast) if isinstance(n.ast_type(),ClassDef) and n.name == "A") + + assert_that(class_node, is_(PythonRstNode)) + ast.translation_unit.lazy_create_refers(ast) + refs = class_node.references + assert_that(refs, has_length(1)) + ref = refs[0] + ref_node = ast.translation_unit._nodes[ref.node_id] + assert_that(ref_node.ast_type(), instance_of(ClassDef)) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(2)) + assert_that(class_node in [ast.translation_unit._nodes[r.node_id] for r in referenced_by]) + + def test_param_reference(self): + # param obj refers to its type, if type definition in the same file, refers to def, otherwise refers to Name + ast = self.factory.create_from_text(content, "content.py") + with tempfile.TemporaryDirectory(delete=True) as temp_dir: + syntax_tree.ASTShower.store_node(temp_dir + "/py3.txt", ast) + + param_node = [n for n in traverse(ast) if n.name == "bruno" and n.ast_type == Arg] + + assert_that(param_node[0], is_(PythonRstNode)) + ast.translation_unit.lazy_create_refers(ast) + refs = param_node[0].references + assert_that(refs, has_length(1)) + ref = refs[0] + ref_node = ast.translation_unit._nodes[ref.node_id] + assert_that(ref_node.ast_type(), instance_of(ClassDef)) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(2)) + types = [r.node_id for r in referenced_by] + assert_that(param_node[0].name, is_in(types)) + + def test_function_reference(self): + ast = self.factory.create_from_text(content, "content.py") + with tempfile.TemporaryDirectory(delete=True) as temp_dir: + syntax_tree.ASTShower.store_node(temp_dir + "/py4.txt", ast) + call_node = first(n for n in traverse(ast) if isinstance(n.ast_type(), Call) and n.name == "bruno.is_near()") + assert_that(call_node, is_(PythonRstNode)) + ast.translation_unit.lazy_create_refers(ast) + refs = call_node.references + ref = refs[0] + ref_node = ast.translation_unit._nodes[ref.node_id] + + assert_that(ref_node.ast_type(), instance_of(FunctionDef)) + referenced_by = ref_node.referenced_by + assert_that(referenced_by, has_length(1)) + assert_that(call_node in [ast.translation_unit._nodes[r.node_id] for r in referenced_by]) + + def test_ref_node_to_str(self): + it = PythonRSTReference("it is ", "kind", {}) + assert_that(it, has_string("it is :kind")) + + +if __name__ == "__main__": + pytest.main() diff --git a/test/python/test_python_astshower.py b/test/python/test_python_astshower.py new file mode 100644 index 00000000..39277879 --- /dev/null +++ b/test/python/test_python_astshower.py @@ -0,0 +1,110 @@ +import pytest +from hamcrest import * + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory + +from renaissance.syntax_tree import ASTShower + + +class TestPythonShower: + @pytest.fixture(autouse=True) + def setup(self): + self.factory = PythonFactory(PythonRstNode) + self.atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + self.pattern_factory = PythonPatternFactory(self.factory) + + def test_show_call_using_repr(self): + pattern = self.pattern_factory.create_statement("$pa($55)") + assert_that(str(pattern), is_("(ExpressionStatement, $pa($55), pattern.py[0:28]): |$pa($55)|\n")) + + def test_show_module(self): + expected = "(TranslationUnit, test.py, test.py[0:29]):\n |ba(55)|\n |ca(555)|\n |lo(4444)|\n |na=55|\n" + assert_that(str(self.atu), is_(expected)) + + def test_show_body(self): + expected = ( + "[(ExpressionStatement, ba(55), test.py[0:6]): |ba(55)|\n, (ExpressionStatement, ca(555), test.py[7:14]): |ca(555)|\n," + " (ExpressionStatement, lo(4444), test.py[15:23]): |lo(4444)|\n, (Assign, na, test.py[24:29]): |na=55|\n]" + ) + + assert_that(str(self.atu.children), is_(expected)) + + def test_show_ast_filter_implicit_node(self): + ptext = ASTShower.get_node(self.atu) + assert_that(ptext, not_(contains_string("(ImplicitNode"))) + + def test_show_ast(self): + text = ASTShower.get_node(self.atu) + expected = ( + "(TranslationUnit, test.py, test.py[0:29]):\n" + " |ba(55)|\n" + " |ca(555)|\n" + " |lo(4444)|\n" + " |na=55|\n" + " (ExpressionStatement, ba(55), test.py[0:6]): |ba(55)|\n" + " (Call, ba(55), test.py[0:6]): |ba(55)|\n" + " (Name, ba, test.py[0:2]): |ba|\n" + " (Literal, 55, test.py[3:5]): |55|\n" + " (ExpressionStatement, ca(555), test.py[7:14]): |ca(555)|\n" + " (Call, ca(555), test.py[7:14]): |ca(555)|\n" + " (Name, ca, test.py[7:9]): |ca|\n" + " (Literal, 555, test.py[10:13]): |555|\n" + " (ExpressionStatement, lo(4444), test.py[15:23]): |lo(4444)|\n" + " (Call, lo(4444), test.py[15:23]): |lo(4444)|\n" + " (Name, lo, test.py[15:17]): |lo|\n" + " (Literal, 4444, test.py[18:22]): |4444|\n" + " (Assign, na, test.py[24:29]): |na=55|\n" + " (Name, na, test.py[24:26]): |na|\n" + " (Literal, 55, test.py[27:29]): |55|\n" + ) + assert_that(text, is_(expected)) + + def test_show_if_else(self): + factory = PythonFactory(PythonRstNode) + atu = factory.create_from_text( + """ +if x >y : + x=1 + call(x) +else: + y=1 + call(y) + """, + "test.py", + ) + text = ASTShower.get_node(atu.children[0]) + assert_that( + text, + is_( + "(If, If, test.py[1:56]):\n" + " |if x >y :|\n" + " | x=1|\n" + " | call(x)|\n" + " |else:|\n" + " | y=1|\n" + " | call(y)|\n" + " (Compare, x > y, test.py[4:8]): |x >y|\n" + " (Name, x, test.py[4:5]): |x|\n" + " (GreaterThan, , test.py[0:0]):\n" + " (Name, y, test.py[7:8]): |y|\n" + " (Assign, x, test.py[15:18]): |x=1|\n" + " (Name, x, test.py[15:16]): |x|\n" + " (Literal, 1, test.py[17:18]): |1|\n" + " (ExpressionStatement, call(x), test.py[23:30]): |call(x)|\n" + " (Call, call(x), test.py[23:30]): |call(x)|\n" + " (Name, call, test.py[23:27]): |call|\n" + " (Name, x, test.py[28:29]): |x|\n" + " (Assign, y, test.py[41:44]): |y=1|\n" + " (Name, y, test.py[41:42]): |y|\n" + " (Literal, 1, test.py[43:44]): |1|\n" + " (ExpressionStatement, call(y), test.py[49:56]): |call(y)|\n" + " (Call, call(y), test.py[49:56]): |call(y)|\n" + " (Name, call, test.py[49:53]): |call|\n" + " (Name, y, test.py[54:55]): |y|\n" + ), + ) + + +if __name__ == "__main__": + pytest.main() diff --git a/test/python/test_python_cst_node.py b/test/python/test_python_cst_node.py new file mode 100644 index 00000000..5334b943 --- /dev/null +++ b/test/python/test_python_cst_node.py @@ -0,0 +1,109 @@ +import textwrap +from pathlib import Path + +import pytest +from hamcrest import ( + has_length, + assert_that, + is_, + contains_string, + is_not, instance_of, +) +from libcst import ParserSyntaxError + +import targets + +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.python.cst_node import PythonCstNode +from renaissance.impl.types import * +from renaissance.syntax_tree import ASTShower + + +class TestPythonCstNode: + @pytest.fixture(autouse=True) + def setup(self): + self.factory = PythonFactory(PythonCstNode) + self.atu = self.factory.create_from_text("a = 0", "all.py") + # create a pattern factory atu is passed to the pattern factory for use of all # includes, #defines and declarations + self.pattern_factory = PythonPatternFactory(self.factory) + + def test_slice(self): + it = self.pattern_factory.create_expression("items[1:2:3]") + + assert_that(it.children[0].ast_type(), instance_of(Name)) + assert_that(it.children[1].ast_type(), instance_of(Whitespace)) + assert_that(it.children[2].ast_type(), instance_of(ListComp)) + assert_that(it.children[3].ast_type(), instance_of(SubscriptElement)) + assert_that(it.children[4].ast_type(), instance_of(ListComp)) + + def test_attribute_signature_has_at(self): + src = self.pattern_factory.create_statement("@TUAT\ndef ba(): pass") + ASTShower.show_node(src) + attr = src.children[0] + assert_that(attr.signature, is_("@TUAT\n")) + + def test_node_family(self): + src = PythonCstNode.load_from_text( + textwrap.dedent(""" + import you + from other import dog + class Parent: + def previous_me(): + pass + def mememe(a55,a66,a77,a88,a99): + l(a55) + l(a66) + l(a77) + l(a88) + def next_me(): + pass + """), + "nav.py", + ) + # module class body fun memem + me = src.children[-1].children[5].children[2] + assert_that(me.name, is_("mememe")) + assert_that(me.preceding_sibling.name, is_("previous_me")) + assert_that(me.next_sibling.name, is_("next_me")) + assert_that(me.parent.parent.name, is_("Parent")) + # all children are mashed together + assert_that(me.children, has_length(8)) + + def test_load_file_with_ignored_types(self): + atu = PythonCstNode.load_from_text("x = 1 # type: ignore", "bogus.py") + assert_that(atu.translation_unit, is_not(None)) + + def test_load_file(self): + atu = PythonCstNode.load(Path(targets.__file__).parent / "demo.py") + assert_that(atu, is_not(None)) + + def test_load_invalid_file(self): + with pytest.raises(ParserSyntaxError, match="Syntax Error"): + PythonCstNode.load(Path(targets.__file__).parent / "invalid.py") + + def test_ann_fun_to_str2(self): + ann_fun = textwrap.dedent(""" + @parameterized.expand(Factories.extend(['$x;$y;'])) + def test(_): + atu = factory.create_from_text(TestStatements.SIMPLE_CPP, "test.c") + + matches = match_pattern( func_body.children,patterns) + + self.assert_matches( expected_dicts_per_match,matches) + """) + it = PythonCstNode.load_from_text(ann_fun, "fun.py").children[-1] + assert_that(it.offset, is_(1)) + assert_that(it.signature, contains_string("@parameterized.expand")) + + def test_ann_fun_to_str(self): + ann_fun = textwrap.dedent(""" + @parameterized.expand(Factories.extend(['$x;$y;'])) + def test(_): + atu = factory.create_from_text(TestStatements.SIMPLE_CPP, "test.c") + + matches = match_pattern( func_body.children,patterns) + + self.assert_matches( expected_dicts_per_match,matches) + """) + it = PythonCstNode.load_from_text(ann_fun, "fun.py").children[-1] + assert_that(it.signature, contains_string("def test")) diff --git a/test/python/test_python_lst_node.py b/test/python/test_python_lst_node.py new file mode 100644 index 00000000..c5bcbe9b --- /dev/null +++ b/test/python/test_python_lst_node.py @@ -0,0 +1,34 @@ +import hypothesmith +import libcst +import pytest +from hamcrest import assert_that, is_, instance_of +from hypothesis import HealthCheck, given, settings + +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.tree_sitter.lst import LSTNode +from renaissance.impl.types import Statement +from utils_for_tests import reject_unsupported_code + + +class TestPythonLstNode: + @pytest.fixture(autouse=True) + def setup(self): + self.factory = PythonFactory(LSTNode) + self.pattern_factory = PythonPatternFactory(self.factory) + + def test_stmt_kind(self): + src = self.factory.create_from_text("x =1") + target = self.factory.create_from_text("x = 1") + assert_that(src, is_(target)) + + @pytest.mark.hypothesisslow + @given(code=hypothesmith.from_node(libcst.BaseStatement)) + @settings(max_examples=500, suppress_health_check=HealthCheck.all()) + def test_from_cst_returns_statement(self, code): + reject_unsupported_code(code) + factory = PythonFactory(LSTNode) + node = factory.create_from_text(code) + print(f"testing {code=} with LSTNode") + assert_that(node.children[0].ast_type(), instance_of(Statement), f"{code=}") + + diff --git a/test/python/test_python_matcher.py b/test/python/test_python_matcher.py new file mode 100644 index 00000000..a10d1094 --- /dev/null +++ b/test/python/test_python_matcher.py @@ -0,0 +1,502 @@ +import ast +import textwrap + +import pytest +from hamcrest import * +from hamcrest import assert_that, is_not + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.types import MatchOne, ExpressionStatement +from renaissance.syntax_tree import MatchFinder +from renaissance.syntax_tree.match_finder import ( + is_match, + match_pattern, + find_variants, + variant_in_match_stmt, +) + + +class TestPythonMatcher: + + @pytest.fixture(autouse=True) + def setup(self): + self.factory = PythonFactory(PythonRstNode) + self.pattern_factory = PythonPatternFactory(self.factory) + + def test_if_statements(self): + code_if_then_statement = "if c1:\n pass" + code_if_then_else_statement = "if c1:\n pass\nelse: \n pass" + code_if_then_elif_statement = "if c1:\n pass\nelif c2:\n pass" + code_if_then_else_if_statement = "if c1:\n pass\nelse:\n if c2:\n pass" + + if_then_statement = self.pattern_factory.create_statement(code_if_then_statement) + if_then_else_statement = self.pattern_factory.create_statement(code_if_then_else_statement) + if_then_elif_statement = self.pattern_factory.create_statement(code_if_then_elif_statement) + if_then_else_if_statement = self.pattern_factory.create_statement(code_if_then_else_if_statement) + + assert_that(if_then_statement, is_(if_then_statement)) + assert_that(is_match(if_then_statement, if_then_else_statement), is_(False)) + assert_that(is_match(if_then_statement, if_then_elif_statement), is_(False)) + assert_that(is_match(if_then_statement, if_then_else_if_statement), is_(False)) + + assert_that(if_then_else_statement, is_not(if_then_statement)) + assert_that(is_match(if_then_else_statement, if_then_else_statement), is_(True)) + assert_that(is_match(if_then_else_statement, if_then_elif_statement), is_(False)) + assert_that(is_match(if_then_else_statement, if_then_else_if_statement), is_(False)) + + assert_that(if_then_elif_statement, is_not(if_then_statement)) + assert_that(is_match(if_then_elif_statement, if_then_else_statement), is_(False)) + assert_that(is_match(if_then_elif_statement, if_then_elif_statement), is_(True)) + assert_that(is_match(if_then_elif_statement, if_then_else_if_statement), is_(True)) + + assert_that(if_then_else_if_statement, is_not(if_then_statement)) + # assert_that(is_match(if_then_else_if_statement, if_then_statement), is_(False)) + assert_that(is_match(if_then_else_if_statement, if_then_else_statement), is_(False)) + assert_that(is_match(if_then_else_if_statement, if_then_elif_statement), is_(True)) + assert_that(is_match(if_then_else_if_statement, if_then_else_if_statement), is_(True)) + + def test_is_match_if_statements(self): + code_if_then_statement = "if c1:\n pass" + code_if_then_else_if_statement = "if c1:\n pass\nelse:\n if c2:\n pass" + + if_then_statement = self.pattern_factory.create_statement(code_if_then_statement) + if_then_else_if_statement = self.pattern_factory.create_statement(code_if_then_else_if_statement) + + assert_that(variant_in_match_stmt(if_then_else_if_statement.children[2], if_then_statement.children[2], {}), is_([])) + + @pytest.mark.parametrize( + "stmt_txt, pattern_txt, expected", + [ + # return empty expression list (type None) + ("return", "return", True), + ("return", "return $expression_list", False), + ("return", "return $$expressions", True), + # return single value + ("return 1", "return", False), + ("return 1", "return $expression_list", True), + ("return 1", "return $$expressions", True), + # single with trailing separator + ("return 1,", "return", False), + ("return 1,", "return $expression_list", True), + ("return 1,", "return $$expressions", True), + # multiple + ("return 1, 2, 3", "return", False), + ("return 1, 2, 3", "return $expression_list", True), + ("return 1, 2, 3", "return $$expressions", True), + # multiple with trailing separator + ("return 1, 2, 3,", "return", False), + ("return 1, 2, 3,", "return $expression_list", True), + ("return 1, 2, 3,", "return $$expressions", True), + ], + ) + def test_placeholder_return_stmt(self, stmt_txt: str, pattern_txt: str, expected: bool): + stmt = self.pattern_factory.create_statement(stmt_txt) + pattern = self.pattern_factory.create_statement(pattern_txt) + assert_that(is_match(stmt, pattern, {}), is_(expected)) + + def test_generic_is_match_any_stmt(self): + atu = self.factory.create_from_text("ba(55)", "test.py") + + simple = self.pattern_factory.create_statement("$pa(55)") + + assert_that(simple.ast_type(), instance_of(ExpressionStatement)) + assert_that(is_match(atu.children[0], simple, {}), is_(True)) + + def test_generic_is_match_any_assignment(self): + atu = self.factory.create_from_text("na=55", "test.py") + simple = self.pattern_factory.create_statement("$pa") + assert_that(simple.ast_type(), instance_of(MatchOne)) + assert_that(is_match(atu.children[0], simple, {}), is_(True)) + + def test_match_multiple_single_stmt(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + simple = self.pattern_factory.create_statements("$pa") + result = MatchFinder.match_pattern(atu.children, simple) + assert_that(result, has_length(4)) + + def test_match_fix_stmt_fix_param(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + + simple = self.pattern_factory.create_statements("ca(555)") + result = MatchFinder.match_pattern(atu.children, simple) + assert_that(result, has_length(1)) + + def test_is_match_any_stmt_with_fix_param_in_detail(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + + simple = self.pattern_factory.create_statement("$pa(55)") + assert_that(is_match(atu.children[0], simple), is_(True)) + assert_that(is_match(atu.children[1], simple), is_(False)) + assert_that(is_match(atu.children[2], simple), is_(False)) + assert_that(is_match(atu.children[3], simple), is_(False)) + result = MatchFinder.match_pattern(atu.children, [simple]) + assert_that(result, has_length(1)) + + def test_is_match_any_stmt_with_any_param(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + + simple = self.pattern_factory.create_statements("$ca($sss)") + result = match_pattern(atu.children, simple) + assert_that(result, has_length(3)) + + def test_match_multi_fix_stmts(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + simple = self.pattern_factory.create_statements("ba(55)\nca(555)") + result = match_pattern(atu.children, simple) + assert_that(result, has_length(1)) + + def test_match_fix_stmt_with_multi_result(self): + atu = self.factory.create_from_text("pa(55)\npa(55)\npa(55)\npa=55", "test.py") + simple = self.pattern_factory.create_statement("pa(55)") + results = MatchFinder.match_pattern(atu.children, [simple]) + assert_that(results, has_length(3)) + + def test_match_multi_fix_stmt_with_multi_result(self): + atu = self.factory.create_from_text("ba(55)\nna(55)\nna(55)\npa(55)\npa(55)\nba(55)\nna(55)\nna(55)\nna=55") + simple = self.pattern_factory.create_statements("ba($a)\nna($b)\nna($c)") + results = MatchFinder.match_pattern(atu.children, simple) + assert_that(results, has_length(2)) + assert_that(results[0].nodes, has_length(3)) + + def test_match_multi_fix_stmt_with_multi_different_result(self): + atu = self.factory.create_from_text( + "ba(51)\nna(52)\nna(53)\npa(54)\npa(55)\nba(56)\nna(57)\nna(58)\nna=59\nba(51)\nna(52)\nna(53)\n" + ) + simple = self.pattern_factory.create_statements("ba($a)\nna($b)\nna($c)") + results = MatchFinder.match_pattern(atu.children, simple) + assert_that(results, has_length(3)) + assert_that(results[0].nodes, has_length(3)) + assert_that(results[1].nodes, has_length(3)) + assert_that(results[2].nodes, has_length(3)) + + def test_match_stmts_in_children(self): + atu = self.factory.create_from_text( + "ba(51)\nna(52)\nna(53)\npa(54)\nif pa(55):\n ba(51)\n na(52)\n na(53)\n na=59\nelse:\n ba(51)\n na(52)\n na(53)\n" + ) + simple = self.pattern_factory.create_statements("ba($a)\nna($b)\nna($c)") + results = MatchFinder.match_pattern(atu.children, simple) + assert_that(results, has_length(3)) + assert_that(results[0].nodes, has_length(3)) + + def test_match_placeholder_with_args(self): + atu = self.factory.create_from_text("ba()\nna()\nba()\npa(54)\nba()\nna()\nba()\nna()\nna=59\nba(1)\nna()\nba(1)") + simple = self.pattern_factory.create_statements("ba($a)\n$$na\nba($c)") + results = match_pattern(atu.children, simple) + assert_that(results, has_length(1)) + assert_that(results[0].nodes, has_length(3)) + + def test_match_sandwitch_pattern_with_different_content(self): + atu = self.factory.create_from_text(textwrap.dedent(""" + ba(51) + na(52) + na(52) + na(53) + ba(53) + pa(54) + if pa(55): + ba(51) + na(52) + na(53) + ba(53) + na(53) + na=59 + else: + ba(51) + na(52) + ba(53) + + """)) + + simple = self.pattern_factory.create_statements("ba($a)\n$$na\nba($c)") + results = MatchFinder.match_pattern(atu.children, simple) + assert_that(results, has_length(3)) + assert_that(results[0].nodes, has_length(5)) + + def test_match_any_placeholder_but_in_child(self): + atu = self.factory.create_from_text( + textwrap.dedent(""" + ba() + ca() + lo() + na() + ba() + pa() + if pa(): + ba() + ca() + lo() + na() + na() + na=59 + else: + ba() + na() + ba() + + """), + "test.py", + ) + + simple = self.pattern_factory.create_statements("ba()\n$$na\nna()") + results = MatchFinder.match_pattern(atu.children, simple) + assert_that(results, has_length(3)) + assert_that(results[0].nodes, has_length(4)) + assert_that(results[1].nodes, has_length(5)) + assert_that(results[2].nodes, has_length(2)) + + # can only return one match + def test_match_all_epxression(self): + atu = self.factory.create_from_text("pa(55)\npa(55)\nif pa(55):\n pa(55)\n if pa(55):\n pa(55)\n pa=55") + + simple = self.pattern_factory.create_expression("pa(55)") + results = MatchFinder.match_pattern(atu.children, [simple]) + assert_that(results, has_length(6)) + + def test_match_all_statement(self): + atu = self.factory.create_from_text("pa(55)\nif pa(55):\n pa(55)\n if pa(55):\n pa(55)\n pa=55") + + simple = self.pattern_factory.create_statements("pa(55)") + results = match_pattern(atu.children, simple) + assert_that(results, has_length(3)) + + def test_ast_name(self): + simple = self.pattern_factory.create_statement("pa(55)") + assert_that(simple.name, is_("pa(55)")) + + def test_python_ast_name(self): + simple = ast.parse("pa(55)").body[0] + assert_that(simple.value.func.id, is_("pa")) + + def test_equal_nodes(self): + atu = self.factory.create_from_text("pa(55)\nif pa(55):\n pa(55)\n pa=55", "test.py") + + simple = self.pattern_factory.create_statement("pa(55)") + assert_that(simple, is_(atu.children[0])) + + def test_equal_nodes_different_args(self): + atu = self.factory.create_from_text("pa(55)\nif pa(55):\n pa(55)\n pa=55", "test.py") + simple = self.pattern_factory.create_statement("pa(66)") + assert_that(simple, is_not(atu.children[0])) + + def test_replace_multiple_different_nodes(self): + example_code = textwrap.dedent(""" + from module import foo, bar, baz, quux + ba(51) + na(52) + na(53) + pa(54) + if pa(): + ba() + + if pa(55): + ba(51) + na(52) + na(53) + na=59 + else: + ba(51) + na(52) + na(53) + + """) + atu = self.factory.create_from_text(example_code) + assert_that(atu, is_not(None)) + + def test_find_pattern_four_depth(self): + example_code = """class CommonTestUtils(): + def foo(): + self.tds = [ + TestDoubles(a=ImprovedStub(read)), + TestDoubles(b=ImprovedStub(write)), + ] + """ + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_expression("TestDoubles($a=ImprovedStub($b))") + assert_that(match_pattern(atu.children, [pattern]), has_length(2)) + + def test_find_pattern_one_expr(self): + example_code = textwrap.dedent(""" + [TestDoubles(b=ImprovedStub(write))] + """) + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_expression("TestDoubles($a=ImprovedStub($b))") + assert_that(match_pattern(atu.children, [pattern]), has_length(1)) + + def test_find_pattern_one_stmt(self): + example_code = textwrap.dedent(""" + TestDoubles(b=ImprovedStub(write)) + """) + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statement("TestDoubles($a=ImprovedStub($b))") + assert_that(match_pattern(atu.children, [pattern]), has_length(1)) + + def test_variable_length_match_variant_x(self): + example_code = textwrap.dedent("0\n1\n2\n3\n4\n5\n3") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$$before\n3\n$$after") + variants = find_variants(atu.children, pattern) + assert_that(variants, has_length(2)) + assert_that(variants[0].end_index, is_(6)) + assert_that(variants[1].end_index, is_(6)) + + assert_that(variants[0].exp["$$before"], has_length(6)) + assert_that(variants[0].exp["$$after"], has_length(0)) + assert_that(variants[1].exp["$$before"], has_length(3)) + assert_that(variants[1].exp["$$after"], has_length(3)) + + def test_simple_match_with_variant(self): + example_code = textwrap.dedent("0\n1\n2\n") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("0\n1\n2\n") + assert_that(find_variants(atu.children, pattern), has_length(1)) + + def test_variable_length_matcher_as_valid_variants(self): + example_code = textwrap.dedent(""" + 0 + 1 + 2 + 3 + 4 + 5 + 6 + """) + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$$before\n$mid") + variants = find_variants(atu.children, pattern) + assert_that(variants, has_length(7)) + + def test_variable_length_matcherat_start_end_end_as_variants(self): + example_code = textwrap.dedent(""" + 0 + 1 + 2 + 3 + 4 + 5 + 6 + """) + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$$before\n$mid\n$$after") + variants = find_variants(atu.children, pattern) + assert_that(variants, has_length(7)) + + def test_match_pattern_needs_variants(self): + example_code = textwrap.dedent("0\n1\n2\n8\n0\n7\n2") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$$before\n$mid\n$$after\n8\n$$before\n$dido\n$$after") + variants = find_variants(atu.children, pattern) + assert_that(variants, has_length(1)) + assert_that(variants[0].exp["$$before"], has_length(1)) + assert_that(variants[0].exp["$mid"], has_length(1)) + assert_that(variants[0].exp["$dido"], has_length(1)) + assert_that(variants[0].exp["$$after"], has_length(1)) + + def test_trim_variants(self): + example_code = textwrap.dedent("0\n1\n2\n8\n0\n7\n2") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$$before\n$mid\n$$after\n8\n$$before\n$dito\n$$after") + variants = find_variants(atu.children, pattern) + assert_that(variants, has_length(1)) + + def test_mismatch_with_double_match_all(self): + example_code = textwrap.dedent("0\n1\n2\n3\n0\n7\n2") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$$before\n3\n$$before") + variants = find_variants(atu.children, pattern) + assert_that(variants, has_length(0)) + + def test_trim_variants_with_double_match_all(self): + example_code = textwrap.dedent("0\n1\n2\n0\n7\n2") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$$before\n$mid\n$$after\n$$before\n$dido\n$$after") + variants = find_variants(atu.children, pattern) + assert_that(variants, has_length(3)) + assert_that(variants[2].end_index, is_(2)) # [] 0 [] [] 1 [] + # assert_that(trimmed_variants[1], has_length(3)) # [] 0 [1] [] 2 missing 1 + # assert_that(trimmed_variants[2], has_length(5)) + + def test_match_variant_in_args(self): + example_code = textwrap.dedent("fc(1,2,3,4,5)") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_expression("$f($$before, $a, $$after)") + variants = find_variants(atu.body[0].expression.children[1].children, pattern.children[1].children, {}) + assert_that(variants, has_length(greater_than(1))) + + def test_variant_in_args(self): + example_code = textwrap.dedent("fc(1,2,3,4,5)") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_expression("$f($$before, $a, $$after)") + variants = variant_in_match_stmt(atu.body[0].expression.children[1], pattern.children[1], {}) + assert_that(variants, has_length(greater_than(1))) + + def test_variant_in_children_function(self): + example_code = textwrap.dedent("fc(1,2,3,4,5)") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$f($$before, $a, $$after)") + variants = variant_in_match_stmt(atu.body[0], pattern[0], {}) + assert_that(variants, has_length(5)) + assert_that(variants[2].exp["$$before"], has_length(2)) + assert_that(variants[2].exp["$a"][0].signature, is_("3")) + assert_that(variants[2].exp["$$after"], has_length(2)) + + def test_variant_in_children_function_with_expansion(self): + + atu = self.factory.create_from_text("fc(1,2,3,4,5)") + pattern = self.pattern_factory.create_statements("$f($$before, $a, $$after)") + variants = variant_in_match_stmt(atu.body[0], pattern[0], {}) + + atu = self.factory.create_from_text("fc(1,2,6,4,5)") + pattern = self.pattern_factory.create_statements("$f($$before, $b, $$after)") + variants = variant_in_match_stmt(atu.body[0], pattern[0], variants[2].exp) + + assert_that(variants, has_length(1)) + assert_that(variants[0].exp["$b"][0].name, is_("6")) + + def test_find_variant_in_children_function(self): + example_code = textwrap.dedent("fc(1,2,3,4,5)") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$f($$before, $a, $$after)") + variants = find_variants(atu.body, pattern, {}) + assert_that(variants, has_length(greater_than(1))) + + def test_only_one_variant_in_children_functions(self): + example_code = textwrap.dedent("fc(1,2,3,4,5)\nfc(1,2,6,4,5)") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$f($$before, $a, $$after)\n$f($$before, $b, $$after)") + variants = find_variants(atu.body, pattern, {}) + # should be 1 + assert_that(variants, has_length(1)) + + def test_variant_in_children(self): + example_code = textwrap.dedent("fc(1,2,3,4,5)") + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$f($$before, $a, $$after)") + variants = find_variants(atu.children, pattern) + print(variants) + assert_that(variants, has_length(greater_than(1))) + + def test_variable_length_matcher(self): + example_code = textwrap.dedent(""" + fc(1,2,3,4,5) + fc(1,2,6,4,5) + + fc(1,2,3,4,5) + fc_else(1,2,6,4,5) + """) + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("$f($$before, $a, $$after)\n$f($$before, $b, $$after)") + variants = find_variants(atu.children, pattern) + assert_that(variants, is_not(empty())) + assert_that(variants, is_not(empty())) + assert_that(match_pattern(atu.children, pattern), has_length(1)) + + def test_match_multi_fun_using_generic_matcher2(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "test.py") + simple = self.pattern_factory.create_statements("ba(55)\nca(555)") + result = match_pattern(atu.children, simple) + assert_that(result, has_length(1)) + + +if __name__ == "__main__": + pytest.main() diff --git a/test/python/test_python_matcher_representation.py b/test/python/test_python_matcher_representation.py new file mode 100644 index 00000000..1c45a507 --- /dev/null +++ b/test/python/test_python_matcher_representation.py @@ -0,0 +1,199 @@ +import pytest + +from hamcrest import assert_that, is_, is_not + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory + + +class TestPythonMatcherRepresentation: + + @pytest.fixture(autouse=True) + def setup(self): + self.factory = PythonFactory(PythonRstNode) + self.pattern_factory = PythonPatternFactory(self.factory) + + def test_literal_whole_numbers_representation(self): + """ + How are the different representations of literal instances of whole numbers handled by the parser? + """ + normal = "1000" + readable = "1_000" + scientific_power_0 = "1000e0" + scientific_POWER_0 = "1000E0" + scientific_power_plus0 = "1000e+0" + scientific_power_minus0 = "1000e-0" + scientific_power_3 = "1e3" + scientific3_POWER_3 = "1E3" + scientific3_POWER_plus3 = "1E+3" + scientific3_POWER_minus3 = "1000000E-3" + binary_lower = "0b1111101000" + binary_upper = "0B1111101000" + octal_lower = "0o1750" + octal_upper = "0O1750" + hexadecimal_lower = "0x3e8" + hexadecimal_upper = "0X3E8" + float = "1000.000" + + representations = [ + normal, + readable, + scientific_power_0, + scientific_POWER_0, + scientific_power_plus0, + scientific_power_minus0, + scientific_power_3, + scientific3_POWER_3, + scientific3_POWER_plus3, + scientific3_POWER_minus3, + binary_lower, + binary_upper, + octal_lower, + octal_upper, + hexadecimal_lower, + hexadecimal_upper, + float, + ] + + expressions = map(self.pattern_factory.create_expression, representations) + + for expression1 in expressions: + for expression2 in expressions: + assert_that(expression1, is_(expression2)) + + signed = "+1000" + expression_signed = self.pattern_factory.create_expression(signed) + for expression in expressions: + assert_that(expression_signed, is_not(expression)) + + def test_literal_real_numbers_representation(self): + """ + How are the different representations of literal instances of real numbers handled by the parser? + """ + normal = "0.123456" + more_significant_digits = "0.123456000" + readable = "0.123_456" + scientific_power_0 = "0.123456e0" + scientific_power_plus0 = "0.123456e+0" + scientific_power_minus0 = "0.123456e-0" + scientific_power_minus3 = "123.456e-3" + scientific_power_minus6 = "123456e-6" + + representations = [ + normal, + more_significant_digits, + readable, + scientific_power_0, + scientific_power_plus0, + scientific_power_minus0, + scientific_power_minus3, + scientific_power_minus6, + ] + + expressions = map(self.pattern_factory.create_expression, representations) + + for expression1 in expressions: + for expression2 in expressions: + assert_that(expression1, is_(expression2)) + + fraction = "123456/1000000" + expression_fraction = self.pattern_factory.create_expression(fraction) + for expression in expressions: + assert_that(expression_fraction, is_not(expression)) + + def test_character_representation(self): + """ + How are the different character representations handled by the parser? + """ + normal_single = "'1'" + normal_double = '"1"' + escape_octal_single = "'\\061'" + escape_octal_double = '"\\061"' + escape_hexadecimal_single = "'\\x31'" + escape_hexadecimal_double = '"\\x31"' + unicode_single = "'\\u0031'" + unicode_double = '"\\u0031"' + + representations = [ + normal_single, + normal_double, + escape_octal_single, + escape_octal_double, + escape_hexadecimal_single, + escape_hexadecimal_double, + unicode_single, + unicode_double, + ] + + expressions = map(self.pattern_factory.create_expression, representations) + + for expression1 in expressions: + for expression2 in expressions: + assert_that(expression1, is_(expression2)) + + def test_string_representation(self): + """ + How are the different string representations handled by the parser? + """ + normal_single = "'abcdef'" + normal_double = '"abcdef"' + + implicit_concatenated_single = "'abc' 'def'" + implicit_concatenated_double = '"abc" "def"' + implicit_concatenated_mixed = "\"abc\" 'def'" + + representations = [ + normal_single, + normal_double, + implicit_concatenated_single, + implicit_concatenated_double, + implicit_concatenated_mixed, + ] + + expressions = map(self.pattern_factory.create_expression, representations) + + for expression1 in expressions: + for expression2 in expressions: + assert_that(expression1, is_(expression2)) + + explicit_concatenated_single = "'abc' + 'def'" + explicit_concatenated_double = '"abc" + "def"' + explicit_concatenated_mixed = "\"abc\" + 'def'" + + explicit_concatenated_representations = [ + explicit_concatenated_single, + explicit_concatenated_double, + explicit_concatenated_mixed, + ] + + expressions_explicit_concatenated = map(self.pattern_factory.create_expression, explicit_concatenated_representations) + for expression1 in expressions_explicit_concatenated: + for expression2 in expressions_explicit_concatenated: + assert_that(expression1, is_(expression2)) + + for expression_explicit_concatenated in expressions_explicit_concatenated: + for expression in expressions: + assert_that(expression_explicit_concatenated, is_not(expression)) + + def test_statements_with_comment_and_whitespace(self): + """ + How are statements with comments and whitespace handled by the parser? + """ + statement = "x = 1" + statement_with_comment = "x = 1 # This is a comment" + statement_with_new_line = "x = 1 " + statement_with_whitespace = "x = 1 " + statement_with_comment_and_whitespace = "# This is a comment\nx = 1 \n# This is a comment " + + representations = [ + statement, + statement_with_comment, + statement_with_new_line, + statement_with_whitespace, + statement_with_comment_and_whitespace, + ] + expressions = map(self.pattern_factory.create_statement, representations) + + for expression1 in expressions: + for expression2 in expressions: + assert_that(expression1, is_(expression2)) diff --git a/test/python/test_python_nodes.py b/test/python/test_python_nodes.py new file mode 100644 index 00000000..77430db1 --- /dev/null +++ b/test/python/test_python_nodes.py @@ -0,0 +1,214 @@ +from ast import AST + +import pytest +from hamcrest import ( + assert_that, + is_in, + is_, instance_of, +) + +from renaissance.impl.python.cst_node import PythonCstNode +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.tree_sitter.lst import LSTNode +from python.factories import Factories +from renaissance.impl.python.factory import PythonPatternFactory +from renaissance.impl.types import * +from renaissance.utils.ast_utils import traverse + + +class TestPythonNodes: + + @pytest.mark.parametrize( + "_, factory, raw, kind", + Factories.extend( + [ + ("i:int=0", Assign), + ("assert 0", Assert), + ("async for f in fs: pass", For), + ("async def fun(): pass", FunctionDef), + ('async with open("x"): pass', With), + ("x += 5", AugAssign), + ("break", Break), + ("class x:pass", ClassDef), + ("continue", Continue), + ("fun()", ExpressionStatement), + ("def fun(): pass", FunctionDef), + ("for i in items: pass", For), + ("import x", Import), + ("if True: pass", If), + ("from x import y", ImportFrom), + ("match x:\n case _: pass", Match), + ("pass", Pass), + ("raise", Raise), + ("return", Return), + ("try:\n pass\nfinally:\n pass", Try), + ("try:\n x()\nexcept* e:\n pass", Try), + ("while True: pass", While), + ], + ), + ) + def test_stmt_kind(self, _, factory, raw, kind): + pattern_factory = PythonPatternFactory(factory) + it = pattern_factory.create_statement(raw) + if isinstance(it.node, LSTNode) and kind in [Assign, AugAssign]: + assert_that(it.children[0].ast_type(), instance_of(kind)) + else: + assert_that(it.ast_type(), instance_of(kind)) + + @pytest.mark.parametrize( + "_, factory, raw, kind", + Factories.extend( + [ + ("with open() as c: pass", With), + ("await (fun(2))", Await), + ("a = 5 + 3", BinaryOperation), + ("0x01 & 0x10", BitAnd), + ("0x01 | 0x10", BitOr), + ("0x01 ^ 0x10", BitXor), + ("True and False", BooleanOperation), + ("del x", Del), + ( + """ +def outer(): + x = 10 + y = 20 + def inner(): + nonlocal x, y + x += 5 + return inner() +""", + Nonlocal, + ), + ], + ), + ) + def test_stmt_kind_in_context(self, _, factory, raw, kind): + it = factory.create_from_text(raw, "context.py") + kinds = [node.ast_type for node in traverse(it) if hasattr(node, "ast_type")] + assert_that(kind, is_in(kinds)) + + @pytest.mark.parametrize("_, factory, raw, kind", Factories.extend([("global x", Global)])) + def test_global_stmt(self, _, factory, raw, kind): + pattern_factory = PythonPatternFactory(factory) + it = pattern_factory.create_statement(raw) + assert_that(it.ast_type(), is_(kind)) + + @pytest.mark.parametrize( + "_, factory, raw, kind", + Factories.extend( + [ + ("fun()", Call), + ("{one: 1, two:2}", Dict), + ("{1,2}", Set), + ("[1, 2]", List), + ('{word: len(word) for word in ["one","two"]}', DictComp), + ("[ n*3 for n in [1, 2]]", ListComp), + ("{ n*3 for n in [1, 2]}", SetComp), + ("lambda: fun()", Lambda), + ("(n*2 for n in[1,2])", GeneratorExp), + ('f"{1}two"', FormattedString), + ("items[1:4]", Subscript), + ("(9, 10)", Tuple), + ("not True", UnaryOperation), + ("yield fun", Yield), + ("yield from [1,2]", Yield), + ("z if z>y else y", IfExp), + ], + ), + ) + def test_expr_kind(self, _, factory, raw, kind): + pattern_factory = PythonPatternFactory(factory) + it = pattern_factory.create_expression(raw) + assert_that(it.ast_type(), instance_of(kind)) + + @pytest.mark.parametrize( + "_, factory, raw, kind", + Factories.extend( + [ + ("a == b", Equal), + ("a in b", In), + ("a is b", Is), + ("a is not b", IsNot), + ("a < b", LessThan), + ("a <=b", LessThanEqual), + ("a != b", NotEqual), + ("a not in b", NotIn), + ("a > b", GreaterThan), + ("a >= b", GreaterThanEqual) ] ) ) + def test_comperator_operator(self, _, factory, raw, kind): + pattern_factory = PythonPatternFactory(factory) + it = pattern_factory.create_expression(raw) + if isinstance(it.node, (AST, LSTNode)): + assert_that(it.children[1].ast_type(), instance_of(kind)) + else: + assert_that(it.children[1].children[0].ast_type(), instance_of(kind)) + + @pytest.mark.parametrize( + "_, factory, raw, kind", + Factories.extend( + [ + ('case None: return "No data"', MatchSingleton), + ('case True | False: return "Boolean value"', MatchOr), + ('case int(x) if x > 0: return x', MatchClass ), + ('case str() as s if len(s) > 10: return s', MatchAs ), + ('case "[]": return "Empty"', MatchValue), + ('case [first, *rest]: return f"Lis"', MatchSequence), + ('case {"n": n, "a": a}: return a', MatchMapping ), + ('case Point(x=0, y=0): return "t"', MatchClass), + ('case Point(x=x, y=y): return y',MatchClass ), + ('case "str": return "U"', MatchValue), + ('case _: return "_"', MatchAs), + ] )) + def test_match_patterns(self, _, factory, raw, kind): + pattern_factory = PythonPatternFactory(factory) + sample_code = f"match data:\n {raw}\n case _: pass" + stmt = pattern_factory.create_statement(sample_code) + if isinstance(stmt.node, PythonRstNode): + case_kind = stmt.children[1].children[0].children[0].ast_type() + elif isinstance(stmt.node, AST): + case_kind = stmt.children[1].children[0].ast_type() + elif isinstance(stmt.node, PythonCstNode): + case_kind = stmt.children[4].children[1].ast_type() + elif isinstance(stmt.node, LSTNode): + case_kind = stmt.children[3].children[0].children[1].ast_type() + return + assert_that(case_kind, is_(kind)) + + @pytest.mark.parametrize( + "_, factory, raw, kind", + Factories.extend( + [ + ("a % b", Modulo), + ("a / b", Divide), + ("a // b", FloorDivide), + ("a << b", LeftShift), + ("a >> b", RightShift), + ("a * b", Multiply), + ("a ** b", Power), + ("a - b", Subtract), + ("a + b", Add), + ], + ), + ) + def test_binary_operator(self, _, factory, raw, kind): + pattern_factory = PythonPatternFactory(factory) + it = pattern_factory.create_expression(raw) + assert_that(it.children[1].ast_type(), instance_of(kind)) + + @pytest.mark.parametrize( + "_, factory, raw, kind", + Factories.extend( + [ + ("+b", UnaryAdd), + ("-b", UnarySubtract), + ("~b", BitInvert), + ("not b", Not), + ], + ), + ) + def test_unary_operator(self, _, factory, raw, kind): + pattern_factory = PythonPatternFactory(factory) + it = pattern_factory.create_expression(raw) + assert_that(it.ast_type(), instance_of(UnaryOperation)) + if not isinstance(it.node, LSTNode): + assert_that(it.children[0].ast_type(), instance_of(kind)) diff --git a/test/python/test_python_pattern_factory.py b/test/python/test_python_pattern_factory.py new file mode 100644 index 00000000..116cbb15 --- /dev/null +++ b/test/python/test_python_pattern_factory.py @@ -0,0 +1,286 @@ + +import pytest +from hamcrest import assert_that, has_length, is_, is_in, instance_of + +from python.factories import Factories +from renaissance.impl.python.factory import PythonPatternFactory, PythonFactory +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.types import * +from renaissance.syntax_tree.match_finder import match_pattern + + +class TestPythonFactory: + + @pytest.fixture(autouse=True) + def setup(self) -> None: + self.factory = PythonFactory(PythonRstNode) + self.pattern_factory = PythonPatternFactory(self.factory) + + # Statements patterns + @pytest.mark.parametrize("statement", ["x = 10", "x += y", "name = 'John'", "a, b, c = (1, 2, 3)"]) + def test_statement(self, statement) -> None: + """ + Test the creation of a statement in Python + """ + node = PythonRstNode.load_from_text(statement).body[-1] + assert_that(node.is_statement, is_(True)) + assert_that(node.signature, is_(statement)) + + @pytest.mark.parametrize( + "statement", + [ + "if a:\n pass\nelif b:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass", + "if a:\n pass\nelse:\n pass", + "if a:\n pass", + ], + ) + def test_if_else(self, statement) -> None: + node = PythonRstNode.load_from_text(statement).body[-1] + assert_that(node.ast_type(), is_(If)) + assert_that(node.signature, is_(statement)) + + def test_import(self) -> None: + statement = "from module import foo, bar" + + node = PythonRstNode.load_from_text(statement).body[-1] + assert_that(node.ast_type(), is_(ImportFrom)) + assert_that(node.signature, is_(statement)) + assert_that(node.properties["module"], is_("module")) + + @pytest.mark.parametrize( + "statement", + [ + "try:\n pass\nexcept SomeException:\n print('An error occurred.')", + "try:\n pass\nexcept ExceptionType1:\n print('An error occurred.')\nexcept ExceptionType2 as e:\n print(f'Error: {e}')", + ], + ) + def test_try_statement(self, statement) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(statement) + assert_that(node.ast_type(), is_(Try)) + assert_that(node.signature, is_(statement)) + + @pytest.mark.parametrize( + "statement", + [ + "for i in range(2, 11, 2):\n print(i)", + "for index, color in enumerate(colors):\n print(f'Index {index}: {color}')", + "for i in range(5):\n print(i)", + ], + ) + def test_for_loop(self, statement) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(statement) + assert_that(node.ast_type(), is_(For)) + assert_that(node.signature, is_(statement)) + + @pytest.mark.parametrize( + "statement", + [ + "while True:\n print(count)", + "while count < 3:\n print(count)\nelse:\n print(count)", + ], + ) + def test_while_loop(self, statement) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(statement) + assert_that(node.ast_type(), is_(While)) + assert_that(node.signature, is_(statement)) + + @pytest.mark.parametrize( + "statement", + [ + "with MyContextManager('test') as cm:\n print('Inside the context block')", + "with open('example.txt', 'r') as file:\n content = file.read()", + ], + ) + def test_with_statement(self, statement) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(statement) + assert_that(node.ast_type(), is_(With)) + assert_that(node.signature, is_(statement)) + + @pytest.mark.parametrize( + "code", + [ + "def greet():\n print('Hello, World!')", + "def multiply(x, y):\n return x * y", + "def outer_function(x):\n\n def inner_function(y):\n return y * 2\n return inner_function(x) + 5", + ], + ) + def test_func_def(self, code) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), is_(FunctionDef)) + assert_that(node.signature, is_(code)) + + @pytest.mark.parametrize( + "code", + [ + "class Person:\n\n def __init__(self, name, age):\n self.name = name\n self.age = age", + "class MathHelper:\n pi = 3.14159", + "class Dog(Animal):\n\n def speak(self):\n return f'{self.name} says Woof!'", + ], + ) + def test_class_def(self, code) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), is_(ClassDef)) + assert_that(node.signature, is_(code)) + + @pytest.mark.parametrize( + "code", + [ + "return a + b", + "return (length, width, height)", + "return 'Eligible to vote'", + ], + ) + def test_return_statement(self, code) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), instance_of(Return)) + assert_that(node.signature, is_(code)) + + @pytest.mark.parametrize( + "code", + [ + "assert length > 0, 'Length must be positive'", + "assert 10 <= value <= 20, 'Value must be between 10 and 20'", + "assert size < 12", + ], + ) + def test_assert_statement(self, code) -> None: + """ + test for an assert statement. + An assert statement has optionally a message. + """ + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), instance_of(Assert)) + assert_that(node.signature, is_(code)) + + @pytest.mark.parametrize( + "code", + [ + "del x", + "del my_set[0]", + ], + ) + def test_delete_statement(self, code) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), instance_of(Del)) + assert_that(node.signature, is_(code)) + + def test_pass(self) -> None: + code = "pass" + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), instance_of(Pass)) + assert_that(node.signature, is_(code)) + + def test_break_statement(self) -> None: + code = "break" + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), instance_of(Break)) + assert_that(node.signature, is_(code)) + + def test_cont_statement(self) -> None: + code = "continue" + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), instance_of(Continue)) + assert_that(node.signature, is_(code)) + + ### Expressions patterns + @pytest.mark.parametrize( + "code", + [ + "a", + "x", + ], + ) + def test_variable(self, code) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), is_(ExpressionStatement)) + assert_that(node.signature, is_(code)) + + @pytest.mark.parametrize( + "code", + [ + "Literal['left', 'center', 'right']", + "('left', 'center', 'right')", + "Final", + "5 > 3", + "str", + "a + b", + "not a", + "a or b", + "Person(name='Bob', age=25, job='Designer')", + "a.attr", + "a[b]", + "a if b else c", + ], + ) + def test_expr(self, code) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), is_(ExpressionStatement)) + assert_that(node.signature, is_(code)) + + @pytest.mark.parametrize("code", ["\"hello = 'hello' # comment to hello\""]) + def test_comments(self, code) -> None: + """ + TODO: what is tested? + """ + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_statement(code) + assert_that(node.ast_type(), instance_of(ExpressionStatement)) + assert_that(node.signature, is_(code)) + + def test_decorators(self) -> None: + pattern_factory = PythonPatternFactory(self.factory) + node = pattern_factory.create_decorators("@parameterized.expand($exp)").node + assert_that(node.ast_type(), is_(ImplicitNode)) + assert_that(node.name, is_("decorator_list")) + + @pytest.mark.skip + def test_match_decorators(self) -> None: + node = self.factory.create_from_text( + '@parameterized.expand("sasas")\ndef fun():\n parameterized.expand("sasas")\n', + "decorator_pattern.py", + ) + pattern = self.pattern_factory.create_decorators("@parameterized.expand($exp)") + result = match_pattern(node.children, [pattern]) + assert_that(result, has_length(1)) + + def test_create_kwargs(self) -> None: + pattern = self.pattern_factory.create_statement("fun($c=0, $d=2312)") + kwargs = [PythonRstNode(kwarg) for kwarg in pattern.node.node.value.keywords] + it = self.pattern_factory.create_kwargs("$c=0, $d=2312") + assert_that(it[0], is_(kwargs[0])) + + @pytest.mark.parametrize( + "_, factory, raw, expected", + Factories.extend( + [ + ("a = 1", [Number, Assign,Literal, "Name", "AssignTarget"]), + ] + ), + ) + def test_misalignment(self, _, factory, raw, expected) -> None: + patternFactory = PythonPatternFactory(factory) + expression = patternFactory.create_expression(raw) + assert_that(expression.ast_type, is_in(expected)) + + def test_function_with_multi_patterns(self): + pattern = self.pattern_factory.create_expression("$f($$before, $a, $$after)") + assert_that(pattern.ast_type(), Call) + assert_that(pattern.children[0].ast_type(), is_(MatchOne)) + assert_that(pattern.children[1].children[0].ast_type(), is_(MatchAll)) + assert_that(pattern.children[1].children[1].ast_type(), is_(MatchOne)) + assert_that(pattern.children[1].children[2].ast_type(), is_(MatchAll)) diff --git a/test/python/test_python_rst_node.py b/test/python/test_python_rst_node.py new file mode 100644 index 00000000..583b1b02 --- /dev/null +++ b/test/python/test_python_rst_node.py @@ -0,0 +1,163 @@ +import textwrap +from pathlib import Path + +import hypothesmith +import libcst +import pytest +from hamcrest import ( + has_length, + assert_that, + is_, + contains_string, + empty, instance_of, +) +from hypothesis import given, settings, HealthCheck + +import targets +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.types import * +from renaissance.syntax_tree import ASTShower +from utils_for_tests import reject_unsupported_code + + +class TestPythonRstNode: + @pytest.fixture(autouse=True) + def setup(self): + self.factory = PythonFactory(PythonRstNode) + self.atu = self.factory.create_from_text("a = 0", "all.py") + # create a pattern factory atu is passed to the pattern factory for use of all # includes, #defines and declarations + self.pattern_factory = PythonPatternFactory(self.factory) + + def test_type_alias(self): + it = self.factory.create_from_text("type UserId = int", "context.py") + assert_that(it.children[0].ast_type(), is_(TypeAlias)) + + def test_slice(self): + it = self.pattern_factory.create_expression("items[1:2:3]") + assert_that(it.children[1].ast_type(), is_(Slice)) + + def test_named_expr(self): + it = self.pattern_factory.create_statement("if n:= len(items): pass") + assert_that(it.children[0].ast_type(), is_(NamedExpr)) + + def test_named_expr_simple(self): + it = self.pattern_factory.create_statement("(n:= 3)") + assert_that(it.children[0].ast_type(), is_(NamedExpr)) + + # why not ""? + def test_starred(self): + it = self.pattern_factory.create_statement("*x =[1,2]") + assert_that(it.children[0].children[0].ast_type(), is_(Starred)) + + def test_formatted_value(self): + it = self.pattern_factory.create_expression('f"{one}two"') + assert_that(it.children[0].ast_type(), is_(FormattedString)) + + def test_except_handler(self): + it = self.pattern_factory.create_statement("try: pass\nexcept NameError:pass") + assert_that(it.children[1].children[0].ast_type(), is_(Catch)) + + def test_match_stmt(self): + sample_code = ( + 'match data:\n case [first, *rest]: return f"List with first element {first} and {len(rest)} more items"\n case _: pass' + ) + stmt = self.pattern_factory.create_statement(sample_code) + assert_that(stmt.ast_type(), is_(Match)) + assert_that(stmt.children[1].children[0].ast_type(), is_(MatchCase)) + assert_that(stmt.children[1].children[0].children[0].children[1].ast_type(), is_(MatchStar)) + assert_that(stmt.children[1].children[0].children[0].children[0].ast_type(), is_(MatchAs)) + + def test_show_call(self): + atu = self.factory.create_from_text("ba(55)\nca(555)\nlo(4444)\nna=55", "apple.py") + second_stmt = atu.children[1] + assert_that(second_stmt.offset, is_(7)) + assert_that(second_stmt.length, is_(7)) + assert_that(second_stmt.filename, is_("apple.py")) + assert_that(atu.translation_unit, is_(second_stmt.translation_unit)) + + def test_attribute_signature_has_at(self): + src = self.pattern_factory.create_statement("@TUAT\ndef ba(): pass") + ASTShower.show_node(src) + attr = src.children[2].children[0] + assert_that(attr.signature, is_("@TUAT")) + + def test_node_family(self): + src = PythonRstNode.load_from_text(textwrap.dedent(""" +import you +from other import dog +class Parent: + def previous_me(): + pass + def mememe(a55,a66,a77,a88,a99): + l(a55) + l(a66) + l(a77) + l(a88) + def next_me(): + pass + """)) + # module class body fun memem + me = src.children[-1].children[2].children[1] + assert_that(me.name, is_("mememe")) + assert_that(me.preceding_sibling.name, is_("previous_me")) + assert_that(me.next_sibling.name, is_("next_me")) + assert_that(me.parent.parent.name, is_("Parent")) + assert_that(me.children[1].children, has_length(4)) + + @pytest.mark.skip("don't use ast comment parser") + def test_load_file_with_ignored_types(self): + atu = PythonRstNode.load_from_text("x = 1 # type: ignore", "bogus.py") + assert_that(atu.translation_unit.atu.type_ignores, has_length(1)) + + def test_load_file(self): + atu = PythonRstNode.load(Path(targets.__file__).parent / "demo.py") + assert_that(atu.translation_unit.atu.type_ignores, is_(empty())) + + def test_load_invalid_file(self): + with pytest.raises(IndentationError, match="unexpected indent"): + PythonRstNode.load(Path(targets.__file__).parent / "invalid.py") + + def test_ann_fun_to_str2(self): + ann_fun = textwrap.dedent(""" + @parameterized.expand(Factories.extend(['$x;$y;'])) + def test(_): + atu = factory.create_from_text(TestStatements.SIMPLE_CPP, "test.c") + + matches = match_pattern( func_body.children,patterns) + + self.assert_matches( expected_dicts_per_match,matches) + """) + it = PythonRstNode.load_from_text(ann_fun).body[-1] + assert_that(it.offset, is_(1)) + assert_that(it.signature, contains_string("@parameterized.expand")) + + # @pytest.mark.skip("it was working before") + def test_ann_fun_to_str(self): + ann_fun = textwrap.dedent(""" + @parameterized.expand(Factories.extend(['$x;$y;'])) + def test(_): + atu = factory.create_from_text(TestStatements.SIMPLE_CPP, "test.c") + + matches = match_pattern( func_body.children,patterns) + + self.assert_matches( expected_dicts_per_match,matches) + """) + it = PythonRstNode.load_from_text(ann_fun).body[-1] + + assert_that("\n" + it.signature + "\n", is_(ann_fun)) + + @pytest.mark.hypothesisslow + @given(code=hypothesmith.from_node(libcst.BaseStatement)) + @settings(max_examples=50, suppress_health_check=HealthCheck.all()) + def test_from_cst_returns_statement(self, code): + reject_unsupported_code(code) + factory = PythonFactory(PythonRstNode) + node = factory.create_from_text(code) + print(f"testing {code=} with PythonRstNode") + assert_that(node.children[0].ast_type(), instance_of(Statement), f"{code=}") + + def test_corner_case(self): + factory = PythonFactory(PythonRstNode) + node = factory.create_from_text('class ŻP𭻊鲖ÉØ_ąň𣑗: pass\n') + assert_that(node.children[0].ast_type(), instance_of(Statement)) \ No newline at end of file diff --git a/test/python/test_pythonic_node.py b/test/python/test_pythonic_node.py new file mode 100644 index 00000000..0d30f175 --- /dev/null +++ b/test/python/test_pythonic_node.py @@ -0,0 +1,20 @@ +import ast + +from hamcrest import assert_that, is_, not_none + +from renaissance.impl.python.rst_node import PythonRstNode + + +class TestPythonicNode: + def test_it_can_be_created(self): + it = PythonRstNode(ast.Pass()) + assert_that(it, is_(not_none())) + + def test_it_has_elements(self): + it = PythonRstNode(ast.parse("def fun(): pass")) + assert_that(it[0], is_(it.children[0])) + + def test_it_has_multiple_elements(self): + it = PythonRstNode(ast.parse("def fun(): pass")) + it = PythonRstNode(ast.parse("0\n1\n2\n3\n4\n5\n6\n7\n8\n9\n")) + assert_that(it[1:3], is_(it.children[1:3])) diff --git a/test/refactoring/__init__.py b/test/refactoring/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/refactoring/test_cleanup_refactoring.py b/test/refactoring/test_cleanup_refactoring.py new file mode 100644 index 00000000..1c62dd1d --- /dev/null +++ b/test/refactoring/test_cleanup_refactoring.py @@ -0,0 +1,45 @@ +import pytest +from hamcrest import * + +from c_cpp.factories import Factories +from renaissance.refactoring import CleanupRefactoring +from renaissance.syntax_tree import ASTShower, ASTFactory, ASTProcessor + + +class TestCleanupRefactoring: + + @pytest.mark.parametrize( + "name, factory, input_code, expected_code", + list( + Factories.extend( + [ + ( + "int foo() {\n int x = 1;\n return 2;\n}", + "int foo() {\n return 2;\n}", + ), + ( + "int bar() {\n int y = 2;\n int z = y + 3;\n return z;\n}", + "int bar() {\n int y = 2;\n int z = y + 3;\n return z;\n}", + ), + ( + "int baz() {\n int a = 1;\n int b = 2;\n int c = a + b;\n return c;\n}", + "int baz() {\n int a = 1;\n int b = 2;\n int c = a + b;\n return c;\n}", + ), + ] + ) + ), + ) + def test_remove_unused_variables(self, name, factory: ASTFactory, input_code, expected_code): + atu = factory.create_from_text(input_code, "test.c") + ASTShower.show_node(atu) + ast_refactor = ASTProcessor(atu, factory, in_memory=True) + CleanupRefactoring.remove_unused_variables(ast_refactor) + result = ast_refactor.commit().apply_to_string() + assert_that(result, is_(expected_code)) + + def test_should_not_be_instantiable(self): + assert_that(calling(CleanupRefactoring), raises(Exception)) + + +if __name__ == "__main__": + pytest.main() diff --git a/test/refactoring/test_python_refactoring.py b/test/refactoring/test_python_refactoring.py new file mode 100644 index 00000000..e3d7308e --- /dev/null +++ b/test/refactoring/test_python_refactoring.py @@ -0,0 +1,114 @@ +import textwrap + +import pytest +from hamcrest import assert_that, contains_string, is_ + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.refactoring.python_refactoring import PythonRefactoring + + +class TestPythonRefactoring: + + def _patch_factory(self, mocker, text="pass", filename="test_foo.py"): + code = textwrap.dedent(text) + mocker.patch( + "renaissance.impl.python.factory.PythonFactory.create", + return_value=PythonRstNode.load_from_text(code, filename), + ) + + # ------------------------------------------------------------------ + # __init__ / replace_stmt + # ------------------------------------------------------------------ + + def test_init_sets_default_list_patterns(self, mocker): + self._patch_factory(mocker) + from renaissance.refactoring.unit2pytest import Unit2Pytest + + subject = Unit2Pytest("test_foo.py") + # base class defaults are overridden by subclass, but they are set in __init__ + assert_that(subject.black_list_pattern, is_("utils_for_test")) + assert_that(subject.white_list_pattern, is_("test")) + + def test_replace_stmt_rewrites_matching_pattern(self, mocker): + self._patch_factory( + mocker, + """ + import unittest + """, + "test_foo.py", + ) + from renaissance.refactoring.unit2pytest import Unit2Pytest + + subject = Unit2Pytest("test_foo.py") + subject.in_memory = True + subject.replace_stmt("import unittest", "import pytest\nfrom hamcrest import *") + assert_that(subject.apply_to_string(), contains_string("import pytest")) + assert_that(subject.apply_to_string(), contains_string("from hamcrest import *")) + + def test_replace_stmt_expands_variadic_captures(self, mocker): + self._patch_factory( + mocker, + """ + from unittest import TestCase, skip + """, + "test_foo.py", + ) + from renaissance.refactoring.unit2pytest import Unit2Pytest + + subject = Unit2Pytest("test_foo.py") + subject.in_memory = True + subject.replace_stmt( + "from unittest import TestCase,$$symbols", + "import pytest\nfrom hamcrest import *", + ) + assert_that(subject.apply_to_string(), contains_string("import pytest")) + + # ------------------------------------------------------------------ + # process() — skip branch + # ------------------------------------------------------------------ + + def test_process_skips_file_matching_black_list(self, mocker, capsys): + self._patch_factory(mocker, "pass", "utils_for_test_foo.py") + run_spy = mocker.patch("renaissance.refactoring.unit2pytest.Unit2Pytest.run") + PythonRefactoring.process("Unit2Pytest", "utils_for_test_foo.py") + captured = capsys.readouterr() + assert_that(captured.out, contains_string("skipping")) + assert_that(run_spy.call_count, is_(0)) + + def test_process_skips_file_not_matching_white_list(self, mocker, capsys): + self._patch_factory(mocker, "pass", "my_module.py") + run_spy = mocker.patch("renaissance.refactoring.unit2pytest.Unit2Pytest.run") + PythonRefactoring.process("Unit2Pytest", "my_module.py") + captured = capsys.readouterr() + assert_that(captured.out, contains_string("skipping")) + assert_that(run_spy.call_count, is_(0)) + + # ------------------------------------------------------------------ + # process() — run branch + # ------------------------------------------------------------------ + + def test_process_runs_refactor_on_matching_file(self, mocker, capsys): + self._patch_factory(mocker, "pass", "test_foo.py") + run_spy = mocker.patch("renaissance.refactoring.unit2pytest.Unit2Pytest.run") + PythonRefactoring.process("Unit2Pytest", "test_foo.py") + captured = capsys.readouterr() + assert_that(captured.out, contains_string("refactor")) + assert_that(run_spy.call_count, is_(1)) + + # ------------------------------------------------------------------ + # body property + # ------------------------------------------------------------------ + + def test_body_returns_module_level_statements(self, mocker): + self._patch_factory( + mocker, + """ + x = 1 + y = 2 + """, + "test_foo.py", + ) + from renaissance.refactoring.unit2pytest import Unit2Pytest + + subject = Unit2Pytest("test_foo.py") + assert_that(len(subject.body), is_(2)) diff --git a/test/refactoring/test_refactor_with_rewrite.py b/test/refactoring/test_refactor_with_rewrite.py new file mode 100644 index 00000000..ec658761 --- /dev/null +++ b/test/refactoring/test_refactor_with_rewrite.py @@ -0,0 +1,101 @@ +import textwrap + +import pytest +from hamcrest import assert_that, is_ +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.refactoring.python_refactoring import PythonRefactoring + + +class TestRefactorWithRewrite: + + def _create(self, mocker, text) -> PythonRefactoring: + code = textwrap.dedent(text) + mocker.patch( + "renaissance.impl.python.factory.PythonFactory.create", + return_value=PythonRstNode.load_from_text(code), + ) + subject = PythonRefactoring("x.py") + subject.in_memory = True + return subject + + @pytest.mark.skip("comment are not correctly calculated") + def test_refactor_with_comment_and_spaces(self, mocker): + refactoring = self._create( + mocker, + textwrap.dedent(""" + def test_functions(self): + # with comments to remove + with TAUT.TestDoubles(emrwxtl=FakeEMRWxTL(None)): + # comments to remove + log = TAUT.Logger() + # comments to keep + test_log_id = DDXA.Object('a') + # comments in between + test_log = emrwxtl.create_test_log(test_log_id) + + file_id = DDXA.Object('b') + + # comments and space in between + + file_name = DDXA.Object('c') + test_log, version_mismatch = emrwxtl.retrieve_test_log( + file_id, test_log_id, file_name) + emrwxtl.store_test_log(file_id, test_log) + # end comments to keep"""), + ) + with_stmts = refactoring.pattern_factory.create_statements( + "with TAUT.TestDoubles(emrwxtl=FakeEMRWxTL(None)):\n log = TAUT.Logger()\n $$stmt" + ) + refactoring.in_memory = True + for match in refactoring.find_match(with_stmts): + refactoring.replace(match["$$stmt"], match.nodes, True, True) + + refactoring.commit() + assert_that( + refactoring.apply_to_string(), + is_(""" + def test_functions(self): + # comments to keep + test_log_id = DDXA.Object('a') + test_log = emrwxtl.create_test_log(test_log_id) + + file_id = DDXA.Object('b') + file_name = DDXA.Object('c') + test_log, version_mismatch = emrwxtl.retrieve_test_log(file_id, test_log_id, file_name) + emrwxtl.store_test_log(file_id, test_log) + # end comments to keep"""), + ) + + def test_refactor_replace_multi_placeholder(self, mocker): + """ + test case showing a replacement of a multi placeholder + that matches a non-empty list of AST nodes in the code + """ + refactoring = self._create(mocker, "def f(a):\n f(2, 0)") + function_call = refactoring.pattern_factory.create_expression("f($$params, 0)") + refactoring.in_memory = True + for match in refactoring.find_match([function_call]): + refactoring.replace("1", match.expansions["$$params"]) + refactoring.commit() + assert_that(refactoring.apply_to_string(), is_("def f(a):\n f(1, 0)")) + + @pytest.mark.skip("empty array can't be detected") + def test_refactor_replace_multi_placeholder_empty(self, mocker): + """ + test case showing a replacement of a multi placeholder + that matches an empty list of AST nodes in the code + """ + # TODO: is this the behaviour we want? + # Can $$params be empty and a comma absent, while present in the pattern. + refactoring = self._create(mocker, "def f(a):\n f(0)") + function_call = refactoring.pattern_factory.create_expression("f($$params, 0)") + refactoring.in_memory = True + for match in refactoring.find_match([function_call]): + refactoring.replace("1, ", match.expansions["$$params"]) + refactoring.commit() + assert_that(refactoring.apply_to_string(), is_("def f(a):\n f(1, 0)")) + + # TODO: make test case with a find and replacement pattern + # find: "f($$params, 0)" + # replace: "f(1, $$params, 0)" + # With $$params empty, we should get "f(1, 0)" so one comma only! diff --git a/test/refactoring/test_simplify_renaissance.py b/test/refactoring/test_simplify_renaissance.py new file mode 100644 index 00000000..434e41aa --- /dev/null +++ b/test/refactoring/test_simplify_renaissance.py @@ -0,0 +1,77 @@ +import textwrap + +from hamcrest import assert_that, contains_string, is_, not_ + +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.refactoring.simplify_renaissance import SimplifyRenaissance + + +class TestSimplifyRenaissance: + + def _create(self, mocker, text) -> SimplifyRenaissance: + code = textwrap.dedent(text) + mocker.patch( + "renaissance.impl.python.factory.PythonFactory.create", + return_value=PythonRstNode.load_from_text(code, "unit2pytest.py"), + ) + subject = SimplifyRenaissance("unit2pytest.py") + subject.in_memory = True + return subject + + def test_init_sets_white_and_black_list(self, mocker): + subject = self._create(mocker, "pass") + assert_that(subject.white_list_pattern, is_("unit2pytest")) + assert_that(subject.black_list_pattern, is_("SimplifyRenaissance")) + + def test_run_skips_file_matching_black_list(self, mocker, capsys): + mocker.patch( + "renaissance.impl.python.factory.PythonFactory.create", + return_value=PythonRstNode.load_from_text("pass"), + ) + subject = SimplifyRenaissance("SimplifyRenaissance.py") + subject.in_memory = True + subject.run() + captured = capsys.readouterr() + assert_that(captured.out, contains_string("skipping")) + + def test_run_skips_file_not_matching_white_list(self, mocker, capsys): + mocker.patch( + "renaissance.impl.python.factory.PythonFactory.create", + return_value=PythonRstNode.load_from_text("pass"), + ) + subject = SimplifyRenaissance("other_module.py") + subject.in_memory = True + subject.run() + captured = capsys.readouterr() + assert_that(captured.out, contains_string("skipping")) + + def test_run_rewrites_expansion_signature_access(self, mocker): + subject = self._create( + mocker, + """ + def foo(): + val = match.expansions["$key"][0].signature + """, + ) + subject.replace_stmt("$val = match.expansions[$key][0].signature", "$val= match[$key]") + assert_that(subject.apply_to_string(), contains_string('val= match["$key"]')) + assert_that(subject.apply_to_string(), not_(contains_string(".expansions"))) + + def test_run_rewrites_factory_create_from_text(self, mocker): + subject = self._create( + mocker, + """ + def foo(): + factory = ASTFactory(PythonASTNode) + atu = factory.create_from_text(code, name) + """, + ) + subject.run() + assert_that(subject.apply_to_string(), contains_string("PythonASTNode.load_from_text(code, name)")) + assert_that(subject.apply_to_string(), not_(contains_string("ASTFactory"))) + + def test_run_processes_matching_file(self, mocker, capsys): + subject = self._create(mocker, "pass") + subject.run() + captured = capsys.readouterr() + assert_that(captured.out, contains_string("simplify")) diff --git a/test/refactoring/test_taut2unittest_refactoring.py b/test/refactoring/test_taut2unittest_refactoring.py new file mode 100644 index 00000000..27298623 --- /dev/null +++ b/test/refactoring/test_taut2unittest_refactoring.py @@ -0,0 +1,344 @@ +import textwrap +from pathlib import Path + +import pytest +from hamcrest import ends_with, assert_that, is_ + +import targets +from renaissance.impl.types import Name +from renaissance.refactoring.taut2pyunit import Taut2Pyunit +import test_data.test_class as tst_class +import test_data.test_code as tst_code +import test_data.test_insert as tst_insert +from renaissance.impl.python.rst_node import PythonRstNode +import test_data.test_testdoubles as tst_testdoubles +from renaissance.utils.ast_utils import traverse + + +class TestTaut2Unittest: + + def test_init(self): + subject = Taut2Pyunit(Path(targets.__file__).parent / "taut/taut_test.py") + assert_that(subject.filename, ends_with("taut_test.py")) + + def _create(self, mocker, text) -> Taut2Pyunit: + code = textwrap.dedent(text) + mocker.patch( + "renaissance.impl.python.factory.PythonFactory.create", + return_value=PythonRstNode.load_from_text(code), + ) + subject = Taut2Pyunit("x.py") + subject.in_memory = True + return subject + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ( + "import unittest\nimport TAUT\nimport DDXA", + "import unittest\nimport DDXA", + ), + ], + ) + def test_remove_import(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.remove_taut_import() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ( + "class ATestCase(TAUT.TestCase):\n pass\n", + "class ATestCase(unittest.TestCase):\n pass\n", + ), + ( + "class testUtils(TestCase, Asserter):\n pass\n", + "class testUtils(unittest.TestCase, Asserter):\n pass\n", + ), + ], + ) + def test_replace_taut(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.replace_taut() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ( + "@TAUT.skip_test\ndef test(a, b):\n pass\n", + "@unittest.skip\ndef test(a, b):\n pass\n", + ) + ], + ) + def test_replace_skip(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.replace_taut_skip() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code, indent", + [ + (tst_testdoubles.test_indent, tst_testdoubles.test_indent_new, ""), + (tst_testdoubles.test_indent_fun, tst_testdoubles.test_indent_fun_new, " "), + ], + ) + def test_indentation(self, input_code, expected_code, indent, mocker): + subject = self._create(mocker, input_code) + subject.move_indent(indent) + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ( + "import mock\nfrom TAUT import TestCase, TestDoubles", + "\ntry:\n from unittest.mock import patch\nexcept ImportError:\n from mock import patch\n", + ) + ], + ) + def test_replace_import(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.replace_taut_import() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ("emrwxread = 0", "self.emrwxread = 0"), + ("func(emrwxwidxread)", "func(self.emrwxwidxread)"), + ("a = test(emrwxviprxinterface)", "a = test(self.emrwxviprxinterface)"), + ("b = whxstream2", "b = self.whxstream2"), + ( + "self.assertEqual(emrwxread.method_called(0))", + "self.assertEqual(self.emrwxread.method_called(0))", + ), + # ('EMRWxREAD.emrwxread.set_retval(0)', 'self.emrwxread.set_retval(0)') + ], + ) + def test_add_self(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.add_self() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ( + "@TAUT.log_stub\ndef create_test_log(self, test_log_id):\n pass\n", + "\ndef create_test_log(self, test_log_id):\n pass\n", + ), + ], + ) + def test_remove_decorator(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.remove_decorator() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ("self.assert_equal(len(listA), 5)", "self.assertEqual(len(listA), 5)"), + ("self.assert_false(len(listA), 5)", "self.assertFalse(len(listA), 5)"), + ("self.assert_true(len(listA), 5)", "self.assertTrue(len(listA), 5)"), + ], + ) + def test_convert_assert(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.convert_assert() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize("input_code, expected_code", [(tst_code.taut_code, tst_code.result_code)]) + def test_log_abcdxtl(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.in_memory = True + subject.replace_log_compxtl("abcd") + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize("input_code, insert_code", [(tst_insert.input_code, tst_insert.insert_code)]) + def test_insert_class(self, input_code, insert_code, mocker): + subject = self._create(mocker, input_code) + subject.insert_class() + result = subject.apply_to_string() + assert_that(result, is_(input_code + insert_code)) + + @pytest.mark.parametrize("input_code, expected_code", [(tst_class.set_up, tst_class.new_set_up)]) + def test_setup(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.convert_setup() + result = subject.apply_to_string() + assert result == expected_code + + def test_teardown(self, mocker): + subject = self._create(mocker, tst_class.tear_down_simple) + subject.convert_teardown() + result = subject.apply_to_string() + assert result == tst_class.tear_down_simple_new + + @pytest.mark.parametrize("input_code, expected_code", [(tst_class.tear_down, tst_class.new_tear_down)]) + def test_teardown_refactor(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.refactor_teardown() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize("input_code, expected_code", [(tst_testdoubles.test_doubles_fun, tst_testdoubles.test_doubles_fun_new)]) + def test_testdoubles_fun(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.refactor_testdoubles_fun() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize("input_code, expected_code", [(tst_testdoubles.test_doubles_class, tst_testdoubles.test_doubles_class_new)]) + def test_testdoubles_class(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.refactor_testdoubles_class() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ("@mock.patch('arg')\ndef test():\n pass\n", "@patch('arg')\ndef test():\n pass\n"), + ("a = mock.patch(arg)", "a = mock.patch(arg)"), + ], + ) + def test_remove_mock(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.replace_mock() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + def test_remove_stubserver(self, mocker): + subject = self._create(mocker, "@TAUT.StubServer\ndef test():\n pass\n") + expected_code = "\ndef test():\n pass\n" + subject.remove_stubserver() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ("self.tds.append(TestDoubles(mode, emr=self.emr))", "self.add_patcher(mode, 'emr', self.emr)"), + ("self.tds.append(TestDoubles(a=ImprovedStub(b)))", "self.a = ImprovedStub(b)"), + ], + ) + def test_convert_tds(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + subject.convert_tds() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ("assert_double_equal(l.x, 0.0)", "self.assert_double_equal(l.x, 0.0)"), + ("def a():\n assert_double_equal(l.x, 0.0)", "def a():\n self.assert_double_equal(l.x, 0.0)"), + ], + ) + def test_assert_doubles(self, input_code, expected_code, mocker): + subject = self._create(mocker, input_code) + [subject.replace("self." + node.name, node, False, False) + for node in traverse(subject.node) + if isinstance(node.ast_type(), Name) and node.name == "assert_double_equal" + ] + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + def test_import_verify(self, mocker): + subject = self._create(mocker, "def test_import(self):\n self.import_and_verify_module('ABCDxTL')") + expected_code = "def test_import(self):\n import ABCDxTL\n self.assertIsNotNone(ABCDxTL)" + subject.convert_import_verify() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + def test_insert_asserter(self, mocker): + subject = self._create(mocker, "def assert_double_equal(a, br=c):\n pass") + expected_code = tst_insert.insert_code + subject.insert_asserter() + subject.remove_assert_func() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + def test_replace_unittest_asserter(self, mocker): + subject = self._create(mocker, "class A(TAUT.TestCase):\n def b(self):\n self.assert_raises(a, b=c)") + expected_code = "class A(Asserter):\n def b(self):\n self.assert_raises(a, b=c)" + subject.replace_unittest_with_asserter() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + @pytest.mark.parametrize( + "input_code, expected_code", + [ + ("assert_raises", "self.assert_raises"), + ("assert_double_equal", "self.assert_double_equal"), + ], + ) + def test_assert_func(self, mocker, input_code, expected_code): + subject = self._create(mocker, input_code) + subject.assert_func() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + def test_convert_testdoubles_func(self, mocker): + subject = self._create(mocker, tst_testdoubles.test_taut_doubles_class) + subject.convert_testdoubles_fun() + result = subject.apply_to_string() + assert_that(result, is_(tst_testdoubles.test_taut_doubles_class_new)) + + def test_convert_testdoubles_func_single_line(self, mocker): + subject = self._create(mocker, tst_testdoubles.test_taut_doubles_class_single_line) + subject.convert_testdoubles_fun() + result = subject.apply_to_string() + assert_that(result, is_(tst_testdoubles.test_taut_doubles_class_single_line_new)) + + def test_setup_common(self, mocker): + subject = self._create(mocker, tst_class.set_up_common) + subject.convert_setup_common() + result = subject.apply_to_string() + assert_that(result, is_(tst_class.set_up_common_new)) + + def test_teardown_common(self, mocker): + subject = self._create(mocker, tst_class.tear_down_common) + subject.convert_teardown_common() + result = subject.apply_to_string() + assert_that(result, is_(tst_class.tear_down_common_new)) + + def test_add_patcher(self, mocker): + subject = self._create(mocker, tst_class.tear_down_common_new) + subject.convert_add_patcher() + result = subject.apply_to_string() + assert_that(result, is_(tst_class.tear_down_common_new + tst_class.insert_add_patcher + "\n")) + + def test_shared_setup(self, mocker): + subject = self._create(mocker, "class A():\n def sharedSetUp(self):\n pass") + expected_code = "class A():\n def setUp(self):\n pass" + subject.shared_setup() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + def test_with_testdoubles(self, mocker): + subject = self._create(mocker, "with TAUT.TestDoubles(module=mod, b=c):\n pass") + expected_code = "with patch.object(mod, 'b', new=c):\n pass" + subject.with_testdoubles() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) + + def test_insert_patch_import(self, mocker): + subject = self._create(mocker, "import unittest\nself.patches = []") + expected_code = ( + "import unittest\ntry:\n from unittest.mock import patch\nexcept ImportError:\n from mock import patch\nself.patches = []" + ) + subject.insert_patch_import() + result = subject.apply_to_string() + assert_that(result, is_(expected_code)) diff --git a/test/refactoring/test_unit2pytest.py b/test/refactoring/test_unit2pytest.py new file mode 100644 index 00000000..d4adba2f --- /dev/null +++ b/test/refactoring/test_unit2pytest.py @@ -0,0 +1,320 @@ +import textwrap +from pathlib import Path + + +from hamcrest import assert_that, contains_string, is_, ends_with, not_ + +import targets +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.refactoring.unit2pytest import Unit2Pytest + + +class TestUnit2Pytest: + def test_init(self): + subject = Unit2Pytest(Path(targets.__file__).parent / "demo.py") + assert_that(subject.filename, ends_with("demo.py")) + + def test_commit_does_nothing_when_not_changed(self, mocker): + subject = self._create( + mocker, + """ + 1 + """, + ) + assert_that(subject.has_changed(), is_(False)) + + def test_convert_test_class_updates_only_testcase_bases(self, mocker): + subject = self._create( + mocker, + """ + class TestClass1(TestCase): + pass + class Class2Test(unittest.TestCase): + pass + """, + ) + subject.convert_test_class() + + assert_that(subject.apply_to_string(), contains_string("class TestClass1:")) + assert_that(subject.apply_to_string(), contains_string("class TestClass2:")) + + def _create(self, mocker, text) -> Unit2Pytest: + code = textwrap.dedent(text) + mocker.patch( + "renaissance.impl.python.factory.PythonFactory.create", + return_value=PythonRstNode.load_from_text(code), + ) + subject = Unit2Pytest("x.py") + subject.in_memory = True + return subject + + def test_convert_plain_assert_same_length_rewrites_to_has_length(self, mocker): + expected = textwrap.dedent(""" + def test_asert(): + results = ['1'] + assert_that(results, has_length(1), f"length of results = {len(results)}") + """) + + subject = self._create( + mocker, + """ + def test_asert(): + results = ['1'] + count: int = len(results) + assert 1 == count, "count = " + str(count) + """, + ) + subject.convert_plain_assert_same_length() + assert_that(subject.apply_to_string(), is_(expected)) + + def test_restructure_module_injects_methods_when_class_exists(self, mocker): + subject = self._create( + mocker, + """ + class TestFoo: + def test_foo(self): + pass + def parse(a): + pass + """, + ) + + subject.in_memory = True + subject.restructure_module() + subject.commit() + + assert_that(subject.apply_to_string(), contains_string("def parse(self,a):")) + + def test_convert(self, mocker): + sut = self._create( + mocker, + """ + class TestClass: + def test_fun(self): + with self.assertRaises(Eexception): + call() + """, + ) + spy = mocker.spy(sut, "convert_test_class") + spy2 = mocker.spy(sut, "convert_test_setup") + spy3 = mocker.spy(sut, "replace_stmt") + sut.run() + + assert_that(spy.call_count, is_(1)) + assert_that(spy2.call_count, is_(1)) + assert_that(spy3.call_count, is_(26)) + + def test_convert_assert(self, mocker): + sut = self._create( + mocker, + """ + class TestClass: + def test_fun(self): + self.assertEqual(1, call()) + self.assertEqual(call(),1) + """, + ) + sut.convert_assert("self.assertEqual($exp, $act)", "assert_that($exp, is_($act))") + assert_that(sut.apply_to_string(), contains_string("assert_that(call()")) + assert_that(sut.apply_to_string(), not_(contains_string("assert_that(1"))) + + def test_to_assertthat(self, mocker): + sut = self._create( + mocker, + """ + def test_fun(): + assert call() >=1 + """, + ) + + sut.replace_stmt("assert $stmt, $$msg", "assert_that($stmt, is_(True), $$msg)") + assert_that(sut.apply_to_string(), contains_string("assert_that(call()")) + assert_that(sut.apply_to_string(), not_(contains_string("assert_that(1"))) + + def test_convert_test_class_renames_class_ending_with_test(self, mocker): + subject = self._create( + mocker, + """ + class FooTest(TestCase): + pass + """, + ) + subject.convert_test_class() + assert_that(subject.apply_to_string(), contains_string("class TestFoo:")) + + def test_convert_parameterized_test_at_top_level(self, mocker): + subject = self._create( + mocker, + """ + @parameterized.expand([("a",), ("b",)]) + @some_decorator + def test_fun(self, val): + pass + """, + ) + subject.convert_parameterized_test() + assert_that(subject.apply_to_string(), contains_string("@pytest.mark.parametrize")) + + def test_convert_parameterized_test_inside_class(self, mocker): + subject = self._create( + mocker, + """ + class TestFoo: + @parameterized.expand([("a",), ("b",)]) + @some_decorator + def test_fun(self, val): + pass + """, + ) + subject.convert_parameterized_test() + assert_that(subject.apply_to_string(), contains_string("@pytest.mark.parametrize")) + + def test_remove_print_removes_entire_function_when_only_statement(self, mocker): + subject = self._create( + mocker, + """ + def test_foo(self): + print("hello") + """, + ) + subject.remove_print() + assert_that(subject.apply_to_string(), not_(contains_string("test_foo"))) + + def test_remove_print_removes_only_print_when_other_statements_exist(self, mocker): + subject = self._create( + mocker, + """ + def test_foo(self): + print("hello") + assert 1 == 1 + """, + ) + subject.remove_print() + assert_that(subject.apply_to_string(), not_(contains_string("print"))) + assert_that(subject.apply_to_string(), contains_string("assert 1 == 1")) + + def test_convert_plain_assert_same_length_when_not_swapped(self, mocker): + subject = self._create( + mocker, + """ + def test_foo(): + results = ['1'] + count: int = len(results) + assert results == count, "count = " + str(count) + """, + ) + subject.convert_plain_assert_same_length() + assert_that(subject.apply_to_string(), contains_string("has_length")) + + def test_convert_skip_test_replaces_unittest_skip(self, mocker): + subject = self._create( + mocker, + """ + @unittest.skip("reason") + def test_foo(self): + pass + """, + ) + subject.convert_skip_test() + assert_that(subject.apply_to_string(), contains_string("pytest.mark.skip")) + assert_that(subject.apply_to_string(), not_(contains_string("unittest.skip"))) + + def test_swap_expected_and_actual_swaps_when_literal_is_expected(self, mocker): + subject = self._create( + mocker, + """ + def test_foo(self): + assert_that(1, is_(call())) + """, + ) + subject.swap_expected_and_actual() + assert_that(subject.apply_to_string(), contains_string("assert_that(call(), is_(1))")) + + def test_restructure_module_moves_functions_into_existing_test_class(self, mocker): + subject = self._create( + mocker, + """ + class TestFoo: + def test_existing(self): + pass + def helper(a): + return a + """, + ) + subject.in_memory = True + subject.restructure_module() + subject.commit() + assert_that(subject.apply_to_string(), contains_string("def helper(self,a):")) + + def test_remove_duplicate_import_removes_middle_duplicates(self, mocker): + subject = self._create( + mocker, + """ + import pytest + from hamcrest import * + import pytest + from hamcrest import * + import pytest + from hamcrest import * + def test_foo(): + pass + """, + ) + subject.remove_duplicate_import("import pytest") + result = subject.apply_to_string() + assert_that(result.count("import pytest"), is_(2)) + + def test_convert_test_setup_adds_pytest_fixture(self, mocker): + subject = self._create( + mocker, + """ + class TestFoo: + def setUp(self): + self.x = 1 + def test_foo(self): + pass + """, + ) + subject.convert_test_setup() + assert_that(subject.apply_to_string(), contains_string("@pytest.fixture(autouse=True)")) + assert_that(subject.apply_to_string(), contains_string("def setup(self)")) + + def test_convert_parameterized_test_with_vargs(self, mocker): + subject = self._create( + mocker, + """ + @parameterized.expand([("a", 1), ("b", 2)]) + @some_decorator + def test_fun(self, val, *rest): + pass + """, + ) + subject.convert_parameterized_test() + assert_that(subject.apply_to_string(), contains_string("@pytest.mark.parametrize")) + assert_that(subject.apply_to_string(), contains_string("*rest")) + + def test_restructure_module_rewrites_call_sites_in_existing_class(self, mocker): + subject = self._create( + mocker, + """ + class TestFoo: + def test_existing(self): + result = helper(1) + def helper(a): + return a + """, + ) + subject.in_memory = True + subject.restructure_module() + subject.commit() + assert_that(subject.apply_to_string(), contains_string("self.helper(1)")) + + def test_convert_file_to_test_class_strips_trailing_test(self, mocker): + subject = self._create(mocker, "pass") + mocker.patch.object(type(subject), "filename", new_callable=lambda: property(lambda self: "my_module_test.py")) + assert_that(subject.convert_file_to_test_class(), is_("TestMyModule")) + + def test_convert_file_to_test_class_keeps_test_prefix(self, mocker): + subject = self._create(mocker, "pass") + mocker.patch.object(type(subject), "filename", new_callable=lambda: property(lambda self: "test_my_module.py")) + assert_that(subject.convert_file_to_test_class(), is_("TestMyModule")) diff --git a/test/search_strategies/prompt.md b/test/search_strategies/prompt.md new file mode 100644 index 00000000..de607686 --- /dev/null +++ b/test/search_strategies/prompt.md @@ -0,0 +1,98 @@ +You are to generate Python code (Python 3.14 only) that provides Hypothesis strategies to generate: + 1) (type_expr: ast.expr, value_gen: SearchStrategy[ast.expr]) pairs, where value_gen lazily produces an ast.expr value matching type_expr. + 2) an ast.arguments generator for function definitions using those (type_expr, value_gen) pairs (optional but desirable). + +STRICT CONSTRAINTS / CONVENTIONS +A) Python 3.14-only codebase: + - Do NOT use: from __future__ import annotations + - Do NOT use typing.List / typing.Optional. Use built-in generics: list[T], dict[K,V], tuple[...] and union types: X | None. + - Type hints should use `list[...]`, `dict[...]`, `tuple[...]`, `ast.expr | None`. +B) Hypothesis typing: + - Every @composite strategy must type its draw parameter as DrawFn. + - The module must import DrawFn: `from hypothesis.strategies import DrawFn`. +C) Naming / structure: + - Provide a uniform generator family with these PUBLIC functions only: + gen_type, gen_base, gen_list, gen_dict, gen_union, gen_tuple + Do NOT add separate list_type_and_value / dict_type_and_value / union_type_and_value / tuple_type_and_value wrappers. + Focused tests should call gen_list/gen_union/gen_dict/gen_tuple directly. + - Do NOT pass a SearchStrategy “child” parameter around. The generators must call gen_type(depth-1) internally. +D) Builders: + - All AST node construction helpers must be PRIVATE and start with `_build_`. + Example: use `_build_arg`, NOT `_make_arg`. + - Group all `_build_*` helpers together in one section. +E) Size policy: + - Provide one function: `max_len(depth: int) -> int` that returns `3 * depth`. + - NEVER inline `3 * depth` anywhere; always call max_len(depth). + - Lists/tuples/dicts must allow empty values (min size = 0). + - Unions must have minimum arms 2. +F) Union special-cases (must live inside gen_union): + - If depth <= 1: max number of union arms is 2 (so unions are exactly 2 arms at these depths). + - Else: max number of union arms is max_len(depth). + - Additionally: for depth <= 2, union arms must be base types only (i.e., generated from gen_base at “depth=0”). +G) Performance / energy: + - When building the BitOr chain for union type expressions, avoid list slicing copies; use `islice` from itertools where appropriate. +H) Dict typing correctness: + - Use this builder exactly (or functionally identical): + def _build_dict(keys: list[ast.expr], values: list[ast.expr]) -> ast.Dict: + return ast.Dict(keys=list(keys), values=values) + Rationale: ast.Dict.keys accepts list[expr | None], list is invariant; we copy keys to satisfy typing. +I) Base types: + - Must include these base type names: bool, int, str, float, bytes, NoneType. + - NoneType must generate the instance `None` (as an ast.Constant(value=None)). + - Base type_expr must be ast.Name(id=..., ctx=Load()) using those names. (We only need AST validity; runtime execution is not required.) + +REQUIRED OUTPUT API +1) max_len(depth: int) -> int +2) gen_base(draw: DrawFn, depth: int) -> tuple[ast.expr, SearchStrategy[ast.expr]] +3) gen_list(draw: DrawFn, depth: int) -> tuple[ast.expr, SearchStrategy[ast.expr]] +4) gen_dict(draw: DrawFn, depth: int) -> tuple[ast.expr, SearchStrategy[ast.expr]] +5) gen_union(draw: DrawFn, depth: int) -> tuple[ast.expr, SearchStrategy[ast.expr]] +6) gen_tuple(draw: DrawFn, depth: int) -> tuple[ast.expr, SearchStrategy[ast.expr]] +7) gen_type(draw: DrawFn, depth: int) -> tuple[ast.expr, SearchStrategy[ast.expr]] + - gen_type must dispatch among ALL five types: base, list, dict, union, tuple. + - If depth <= 0, gen_type must return a base pair (by drawing from gen_base(0) or equivalent). + - Otherwise, gen_type must draw from one_of(gen_base(depth), gen_list(depth), gen_dict(depth), gen_union(depth), gen_tuple(depth)). + +LAZINESS REQUIREMENT +- Every gen_* returns (type_expr, value_gen_strategy). value_gen must be a SearchStrategy[ast.expr] that generates the matching AST value. +- Do not eagerly draw a value in the generator unless unavoidable. Prefer to return a composed strategy, e.g., st.lists(elem_vg, ...).map(_build_list). + +NON-VERBOSE STYLE +- Keep the code short and readable; avoid excessive scaffolding. +- Avoid redundant checks that are already handled by gen_type(depth-1). +- Do not introduce config objects or include_* boolean flags. + +TESTS (MUST GENERATE) +Produce a separate test module (pytest + hypothesis) with tests that verify each generator produces what it promises: +1) test_gen_list_generates_list: + - data.draw(gen_list(depth)) returns type_expr that is ast.Subscript with value ast.Name('list') + - value_expr = data.draw(value_gen) is ast.List +2) test_gen_dict_generates_dict: + - type_expr is ast.Subscript with value ast.Name('dict') + - value_expr is ast.Dict and len(keys)==len(values) + - keys are ast.Constant (or None if you later add ** unpacking; currently should be Constant) +3) test_gen_tuple_generates_tuple: + - type_expr is ast.Subscript with value ast.Name('tuple') + - value_expr is ast.Tuple + - empty tuple must be reachable (not necessarily always) +4) test_gen_union_generates_union: + - type_expr is a BinOp chain with BitOr (at least one BinOp at root) + - flatten leaves; number of leaves: + - if depth <= 1 => exactly 2 + - if depth > 1 => between 2 and max_len(depth) + and if depth <= 2 all leaves must be ast.Name of base types only (including NoneType). + - value_expr = data.draw(value_gen) must be ast.expr + - additionally for depth <= 2, since arms are base types, value_expr should be ast.Constant whose underlying Python value type matches one of the union arms: + bool->bool, int->int, str->str, float->float, bytes->bytes, NoneType->NoneType (type(None)). +5) test_smoke_compile: + - for depth=0, for each gen_base/gen_list/gen_dict/gen_union/gen_tuple, build an annotated assignment: + x: <type_expr> = <value_expr> + wrap in ast.Module and compile() it. Compilation must succeed. + - Note: execution is not required. + +IMPORTANT: OUTPUT FORMAT +- Produce the full code for the generator module in one code block. +- Produce the full code for the tests in a second code block. +- Do not output in tables. +- Do not add extra “focused wrapper” functions list_type_and_value/dict_type_and_value/union_type_and_value/tuple_type_and_value. +- Ensure all builder helpers are named _build_* and grouped together. \ No newline at end of file diff --git a/test/search_strategies/python_type_and_value.py b/test/search_strategies/python_type_and_value.py new file mode 100644 index 00000000..c4a0ab2e --- /dev/null +++ b/test/search_strategies/python_type_and_value.py @@ -0,0 +1,264 @@ +import ast +import keyword +import string +from itertools import islice + +from hypothesis import strategies as st +from hypothesis.strategies import DrawFn, SearchStrategy, composite + +DEFAULT_DEPTH: int = 3 + +# -------------------- policy -------------------- + + +def max_len(depth: int) -> int: + return 3 * depth + + +# -------------------- AST builders (private) -------------------- + + +def _build_name(id_: str) -> ast.Name: + return ast.Name(id=id_, ctx=ast.Load()) + + +def _build_subscript(value: ast.expr, slice_expr: ast.expr) -> ast.Subscript: + return ast.Subscript(value=value, slice=slice_expr, ctx=ast.Load()) + + +def _build_list(elts: list[ast.expr]) -> ast.List: + return ast.List(elts=elts, ctx=ast.Load()) + + +def _build_dict(keys: list[ast.expr], values: list[ast.expr]) -> ast.Dict: + # ast.Dict.keys is list[expr | None]; list is invariant, so copy keys for type correctness + return ast.Dict(keys=list(keys), values=values) + + +def _build_tuple(elts: list[ast.expr]) -> ast.Tuple: + return ast.Tuple(elts=elts, ctx=ast.Load()) + + +def _build_tuple_type_slice(type_args: list[ast.expr]) -> ast.expr: + # tuple[()] uses slice == ((),) + return _build_tuple([_build_tuple([])]) if not type_args else _build_tuple(type_args) + + +def _build_bitor_chain(exprs: list[ast.expr]) -> ast.expr: + if len(exprs) < 2: + raise ValueError("union requires at least two arms") + acc = exprs[0] + for e in islice(exprs, 1, None): # avoids slice copy + acc = ast.BinOp(left=acc, op=ast.BitOr(), right=e) + return acc + + +def _build_arg(name: str, ann: ast.expr | None) -> ast.arg: + return ast.arg(arg=name, annotation=ann, type_comment=None) + + +# -------------------- base types -------------------- + +BASE_VALUES: dict[str, SearchStrategy[ast.expr]] = { + "NoneType": st.just(ast.Constant(None)), + "bool": st.builds(ast.Constant, st.booleans()), + "int": st.builds(ast.Constant, st.integers(min_value=-1000, max_value=1000)), + "str": st.builds(ast.Constant, st.text(min_size=0, max_size=5)), + "float": st.builds(ast.Constant, st.floats(allow_nan=False, allow_infinity=False, width=32)), + "bytes": st.builds(ast.Constant, st.binary(min_size=0, max_size=5)), +} +BASE_TYPE: SearchStrategy[str] = st.sampled_from(list(BASE_VALUES)) + + +# ============================================================ +# Generator family (PUBLIC): gen_base/gen_list/gen_dict/gen_union/gen_type +# All return: (type_expr: ast.expr, value_gen: SearchStrategy[ast.expr]) +# ============================================================ + + +@composite +def gen_base(draw: DrawFn) -> tuple[ast.expr, SearchStrategy[ast.expr]]: + tname = draw(BASE_TYPE) + return _build_name(tname), BASE_VALUES[tname] + + +@composite +def gen_list(draw: DrawFn, depth: int = DEFAULT_DEPTH) -> tuple[ast.expr, SearchStrategy[ast.expr]]: + elem_t, elem_vg = draw(gen_type(depth - 1)) + return ( + _build_subscript(_build_name("list"), elem_t), + st.lists(elem_vg, min_size=0, max_size=max_len(depth)).map(_build_list), + ) + + +@composite +def gen_dict(draw: DrawFn, depth: int = DEFAULT_DEPTH) -> tuple[ast.expr, SearchStrategy[ast.expr]]: + # keys restricted to base types for runtime hashability + kname = draw(BASE_TYPE) + kt, kvg = _build_name(kname), BASE_VALUES[kname] + + vt, vvg = draw(gen_type(depth - 1)) + type_expr = _build_subscript(_build_name("dict"), _build_tuple([kt, vt])) + + value_gen = st.lists( + st.tuples(kvg, vvg), + min_size=0, + max_size=max_len(depth), + ).map(lambda pairs: _build_dict([k for (k, _v) in pairs], [v for (_k, v) in pairs])) + + return type_expr, value_gen + + +@composite +def gen_union(draw: DrawFn, depth: int = DEFAULT_DEPTH) -> tuple[ast.expr, SearchStrategy[ast.expr]]: + members = draw( + st.lists( + gen_type(depth - 1), + min_size=2, + max_size=2 if depth <= 1 else max_len(depth), + ) + ) + + ts = [t for (t, _vg) in members] + vgs = [vg for (_t, vg) in members] + return _build_bitor_chain(ts), st.one_of(*vgs) + + +@composite +def gen_tuple(draw: DrawFn, depth: int = DEFAULT_DEPTH) -> tuple[ast.expr, SearchStrategy[ast.expr]]: + members = draw(st.lists(gen_type(depth - 1), min_size=0, max_size=max_len(depth))) + if not members: + t = _build_subscript(_build_name("tuple"), _build_tuple_type_slice([])) # tuple[()] + return t, st.just(_build_tuple([])) # () + ts = [t for (t, _vg) in members] + vgs = [vg for (_t, vg) in members] + t = _build_subscript(_build_name("tuple"), _build_tuple_type_slice(ts)) + return t, st.tuples(*vgs).map(lambda xs: _build_tuple(list(xs))) + + +@composite +def gen_type(draw: DrawFn, depth: int = DEFAULT_DEPTH) -> tuple[ast.expr, SearchStrategy[ast.expr]]: + """ + Depth bounds recursion by forcing base at depth<=0. + """ + types = ["base"] + if depth >= 1: + types.extend(["list", "dict", "tuple"]) + if depth >= 2: + types.append("union") + + choice = draw(st.sampled_from(types)) + match choice: + case "base": + return draw(gen_base()) + case "list": + return draw(gen_list(depth)) + case "dict": + return draw(gen_dict(depth)) + case "union": + return draw(gen_union(depth)) + case "tuple": + return draw(gen_tuple(depth)) + case _: + raise Exception(f"Programming error: '{choice}' not in {types}") + + +# -------------------- (Optional) arguments generator can use gen_type(depth) -------------------- + +_FIRST = st.sampled_from(string.ascii_letters + "_") +_REST = st.text(string.ascii_letters + string.digits + "_", min_size=0, max_size=20) +IDENT = st.builds(str.__add__, _FIRST, _REST).filter(lambda s: not keyword.iskeyword(s)) + + +def _bernoulli(p: float) -> SearchStrategy[bool]: + return st.integers(0, 999).map(lambda x: x < int(p * 1000)) + + +@composite +def gen_arguments( + draw: DrawFn, + *, + depth: int = DEFAULT_DEPTH, + max_posonly: int = 2, + max_args: int = 3, + max_kwonly: int = 3, + p_annot: float = 0.5, + p_kwonly_default: float = 0.5, +) -> ast.arguments: + tv = gen_type(depth) + + n_pos = draw(st.integers(0, max_posonly)) + n_args = draw(st.integers(0, max_args)) + n_kw = draw(st.integers(0, max_kwonly)) + use_vararg = draw(st.booleans()) + use_kwarg = draw(st.booleans()) + + total = n_pos + n_args + n_kw + use_vararg + use_kwarg + names = draw(st.lists(IDENT, min_size=total, max_size=total, unique=True)) + it = iter(names) + + total_pos = n_pos + n_args + n_def = draw(st.integers(0, total_pos)) + tail_start = total_pos - n_def + + anns: list[ast.expr | None] = [None] * total_pos + defaults: list[ast.expr] = [] + + for i in range(total_pos): + if i >= tail_start: + both = draw(st.booleans()) + t, vg = draw(tv) + if both: + anns[i] = t + defaults.append(draw(vg)) + else: + if draw(_bernoulli(p_annot)): + t, _vg = draw(tv) + anns[i] = t + + posonlyargs = [_build_arg(next(it), anns[i]) for i in range(n_pos)] + args = [_build_arg(next(it), anns[n_pos + j]) for j in range(n_args)] + + kwonlyargs: list[ast.arg] = [] + kw_defaults: list[ast.expr | None] = [] + for _ in range(n_kw): + name = next(it) + do_ann = draw(_bernoulli(p_annot)) + do_def = draw(_bernoulli(p_kwonly_default)) + if do_ann and do_def: + t, vg = draw(tv) + kwonlyargs.append(_build_arg(name, t)) + kw_defaults.append(draw(vg)) + elif do_ann: + t, _vg = draw(tv) + kwonlyargs.append(_build_arg(name, t)) + kw_defaults.append(None) + elif do_def: + _t, vg = draw(tv) + kwonlyargs.append(_build_arg(name, None)) + kw_defaults.append(draw(vg)) + else: + kwonlyargs.append(_build_arg(name, None)) + kw_defaults.append(None) + + vararg = None + if use_vararg: + name = next(it) + ann = draw(tv)[0] if draw(_bernoulli(p_annot)) else None + vararg = _build_arg(name, ann) + + kwarg = None + if use_kwarg: + name = next(it) + ann = draw(tv)[0] if draw(_bernoulli(p_annot)) else None + kwarg = _build_arg(name, ann) + + return ast.arguments( + posonlyargs=posonlyargs, + args=args, + vararg=vararg, + kwonlyargs=kwonlyargs, + kw_defaults=kw_defaults, + kwarg=kwarg, + defaults=defaults, + ) diff --git a/test/search_strategies/test_python_arguments.py b/test/search_strategies/test_python_arguments.py new file mode 100644 index 00000000..7395c1fa --- /dev/null +++ b/test/search_strategies/test_python_arguments.py @@ -0,0 +1,45 @@ +# test_arguments_from_recursive.py +import ast +from hypothesis import given +from python_type_and_value import gen_arguments + + +def _collect_names(a: ast.arguments) -> list[str]: + names: list[str] = [] + names.extend([x.arg for x in a.posonlyargs]) + names.extend([x.arg for x in a.args]) + names.extend([x.arg for x in a.kwonlyargs]) + if a.vararg is not None: + names.append(a.vararg.arg) + if a.kwarg is not None: + names.append(a.kwarg.arg) + return names + + +@given(gen_arguments()) +def test_gen_arguments_names_unique(a: ast.arguments): + names = _collect_names(a) + assert len(names) == len(set(names)) + + +@given(gen_arguments()) +def test_gen_arguments_defaults_valid(a: ast.arguments): + total_pos = len(a.args) + len(a.posonlyargs) + assert len(a.defaults) <= total_pos + + +@given(gen_arguments()) +def test_gen_arguments_compilable(a: ast.arguments): + f = ast.FunctionDef(name="f", args=a, body=[ast.Pass()], decorator_list=[], returns=None) + m = ast.Module(body=[f], type_ignores=[]) + ast.fix_missing_locations(m) + compile(m, "<hypothesis>", "exec") + + +@given(gen_arguments()) +def test_gen_arguments_unparsable_parsable(a: ast.arguments): + code = ast.unparse(a) + ast.parse(f""" +def f({code}): + pass +""") diff --git a/test/search_strategies/test_python_ast.py b/test/search_strategies/test_python_ast.py new file mode 100644 index 00000000..097cab14 --- /dev/null +++ b/test/search_strategies/test_python_ast.py @@ -0,0 +1,50 @@ +import ast +import re + +from hypothesis import given, strategies as st +from python_type_and_value import gen_list, gen_union, gen_tuple, gen_dict + + +@given(gen_union()) +def test_gen_union(pair: tuple[ast.expr, st.SearchStrategy[ast.expr]]) -> None: + type_expr, _value_gen = pair + assert isinstance(type_expr, ast.BinOp), f"Unexpected type '{type(type_expr)}', expected ast.BinOp" + assert isinstance(type_expr.op, ast.BitOr), f"Unexpected operator '{type_expr.op}', expected ast.BitOr" + s = ast.unparse(type_expr) + assert re.match("^.*\\|.*$", s), f"type '{s}' unexpectedly doesn't match pattern" + + +@given(gen_list(), st.data()) +def test_gen_list(pair: tuple[ast.expr, st.SearchStrategy[ast.expr]], data: st.DataObject) -> None: + type_expr, value_gen = pair + assert isinstance(type_expr, ast.Subscript), f"Unexpected type '{type(type_expr)}', expected ast.Subscript" + assert isinstance(type_expr.value, ast.Name), f"Unexpected type '{type(type_expr)}', expected ast.Name" + s = ast.unparse(type_expr) + assert re.match("^list\\[.*\\]$", s), f"type '{s}' unexpectedly doesn't match pattern" + value_expr = data.draw(value_gen) + s = ast.unparse(value_expr) + assert re.match("^\\[.*\\]$", s), f"value '{s}' unexpectedly doesn't match pattern" + + +@given(gen_tuple(), st.data()) +def test_gen_tuple(pair: tuple[ast.expr, st.SearchStrategy[ast.expr]], data: st.DataObject) -> None: + type_expr, value_gen = pair + assert isinstance(type_expr, ast.Subscript), f"Unexpected type '{type(type_expr)}', expected ast.Subscript" + assert isinstance(type_expr.value, ast.Name), f"Unexpected type '{type(type_expr)}', expected ast.Name" + s = ast.unparse(type_expr) + assert re.match("^tuple\\[.*\\]$", s), f"type '{s}' unexpectedly doesn't match pattern" + value_expr = data.draw(value_gen) + s = ast.unparse(value_expr) + assert re.match("^\\(.*\\)$", s), f"value '{s}' unexpectedly doesn't match pattern" + + +@given(gen_dict(), st.data()) +def test_gen_dict(pair: tuple[ast.expr, st.SearchStrategy[ast.expr]], data: st.DataObject) -> None: + type_expr, value_gen = pair + assert isinstance(type_expr, ast.Subscript), f"Unexpected type '{type(type_expr)}', expected ast.Subscript" + assert isinstance(type_expr.value, ast.Name), f"Unexpected type '{type(type_expr)}', expected ast.Name" + s = ast.unparse(type_expr) + assert re.match("^dict\\[.*\\]$", s), f"type '{s}' unexpectedly doesn't match pattern" + value_expr = data.draw(value_gen) + s = ast.unparse(value_expr) + assert re.match("^{.*}$", s), f"value '{s}' unexpectedly doesn't match pattern" diff --git a/test/syntax_tree/__init__.py b/test/syntax_tree/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/syntax_tree/infra_syntax_node.py b/test/syntax_tree/infra_syntax_node.py new file mode 100644 index 00000000..b263b291 --- /dev/null +++ b/test/syntax_tree/infra_syntax_node.py @@ -0,0 +1,94 @@ +from typing import Any + +from renaissance.syntax_tree.syntax_node import SyntaxNode +from test.syntax_tree.infra_text_segment import assert_valid_text_segment + + +def assert_valid_syntax_node(node: SyntaxNode[Any]) -> None: + """ + Validate local (non-recursive) invariants for a syntax node. + + Uses `assert_valid_text_segment` to check text-segment invariants. + + Enforced syntax-node invariants: + 1) Each child segment is within the parent's segment. + 2) Children are ordered by increasing start_offset (lowest first). + 3) Children do not overlap (child[i].end_offset <= child[i+1].start_offset). + 4) Each child's parent pointer is exactly this node (identity: `is`). + 5) Each child shares the same backing text and location as the parent. + """ + # Ensure the node itself is a valid text segment. + assert_valid_text_segment(node) + + children = node.children + if not children: + return + + # Validate first child fully, then compare successive pairs. + prev = children[0] + _assert_child_valid(node, prev, index=0) + + for i in range(1, len(children)): + cur = children[i] + _assert_child_valid(node, cur, index=i) + + # (2) order: lowest offset first + assert prev.start_offset <= cur.start_offset, ( + "Children must be ordered by non-decreasing start_offset. " + f"Found child[{i-1}].start_offset={prev.start_offset} > child[{i}].start_offset={cur.start_offset}." + ) + + # (3) non-overlap + assert prev.end_offset <= cur.start_offset, ( + "Children must not overlap and must be in textual order. " + f"Found child[{i-1}].end_offset={prev.end_offset} > child[{i}].start_offset={cur.start_offset}." + ) + + prev = cur + + +def _assert_child_valid(node: SyntaxNode[Any], child: SyntaxNode[Any], *, index: int) -> None: + # Ensure the child itself is a valid text segment. + assert_valid_text_segment(child) + + # (4) parent pointer (identity, not equality) + assert child.parent is node, f"child[{index}].parent must be the node itself (identity check with `is`)." + + # (5) same backing text and location + assert child.full_text == node.full_text, f"child[{index}].full_text must equal node.full_text (same backing text expected)." + assert child.location == node.location, f"child[{index}].location must equal node.location (same origin expected)." + + # (1) containment within parent span + assert node.start_offset <= child.start_offset <= node.end_offset, ( + "child[{idx}].start_offset must lie within the node span. " "Got child[{idx}].start_offset={cso}, expected in [{nso}, {neo}]." + ).format(idx=index, cso=child.start_offset, nso=node.start_offset, neo=node.end_offset) + + assert node.start_offset <= child.end_offset <= node.end_offset, ( + "child[{idx}].end_offset must lie within the node span. " "Got child[{idx}].end_offset={ceo}, expected in [{nso}, {neo}]." + ).format(idx=index, ceo=child.end_offset, nso=node.start_offset, neo=node.end_offset) + + +def assert_valid_syntax_tree(root: SyntaxNode[Any]) -> None: + """ + Validate an entire syntax tree (all reachable nodes). + + Enforced invariants: + - All local invariants (see assert_valid_syntax_node) + - No cycles / no repeated node object in traversal (a proper tree) + """ + visited: set[int] = set() + stack: list[SyntaxNode[Any]] = [root] + + while stack: + node = stack.pop() + + node_id = id(node) + assert node_id not in visited, ( + "Tree traversal encountered the same node object twice. " "This indicates a cycle or a DAG (shared subtree), not a tree." + ) + visited.add(node_id) + + assert_valid_syntax_node(node) + + # Order does not matter for validation. + stack.extend(node.children) diff --git a/test/syntax_tree/infra_text_segment.py b/test/syntax_tree/infra_text_segment.py new file mode 100644 index 00000000..4cefa8ca --- /dev/null +++ b/test/syntax_tree/infra_text_segment.py @@ -0,0 +1,185 @@ +from renaissance.syntax_tree.text_segment import TextSegment + + +def offset_to_location(text: str, offset: int) -> tuple[int, int]: + """ + Convert a *cursor offset* (0 <= offset <= len(text)) to canonical (line, column), + both 0-based. + + Canonical rule: + - The position immediately after a '\n' belongs to the next line at column 0. + (So offset k where k>0 and text[k-1] == '\n' maps to (line_of_next, 0).) + + Newline character itself is on the terminating line at its own column. + """ + assert 0 <= offset <= len(text) + + line = 0 + line_start = 0 + + # Scan characters strictly before this cursor position. + # When we pass a '\n', we move to the next line whose start is i+1. + for i, ch in enumerate(text): + if i >= offset: + break + if ch == "\n": + line += 1 + line_start = i + 1 + + col = offset - line_start + return line, col + + +def location_to_offset(text: str, line: int, column: int) -> int: + """ + Convert canonical (line, column) back to a cursor offset, validating that the + (line, column) is a valid cursor position under canonical rules. + + Valid cursor columns: + - For an empty line (span_len==0): only column==0. + - For a non-empty line: + * If it ends with '\n': allowed columns are 0..span_len-1 (cursor after '\n' is canonicalized to next line). + * Else: allowed columns are 0..span_len (end-of-text / end-of-line cursor position). + """ + lines = split_lines_with_newlines(text) + starts = line_starts_from_lines(lines) + + assert 0 <= line < len(lines) + span = lines[line] + span_len = len(span) + + if span_len == 0: + assert column == 0 + return starts[line] + + ends_with_nl = span[-1] == "\n" + if ends_with_nl: + # canonical: disallow column == span_len (cursor after '\n') + assert 0 <= column < span_len + else: + # last line (or any non-nl-terminated line): allow cursor at end boundary + assert 0 <= column <= span_len + + return starts[line] + column + + +def assert_valid_text_segment(text_segment: TextSegment) -> None: + assert isinstance(text_segment, TextSegment), f"Unexpected instance for text_segment '{type(text_segment)}'. Expected 'TextSegment'." + assert isinstance( + text_segment.full_text, str + ), f"Unexpected instance for property full_text '{type(text_segment.full_text)}'. Expected 'str'." + assert isinstance( + text_segment.text_segment, str + ), f"Unexpected instance for property text_segment '{type(text_segment.text_segment)}'. Expected 'str'." + # TODO: should we check the types of all properties? + + # offset + assert ( + text_segment.start_offset <= text_segment.end_offset + ), f"Property end_offset before start_offset: {text_segment.end_offset} < {text_segment.start_offset}" + + ## allow pointing at end-of-text position + length_full_text = len(text_segment.full_text) + assert ( + 0 <= text_segment.start_offset <= length_full_text + ), f"Property start_offset out of range: {text_segment.start_offset} not in [0, {length_full_text}]" + assert ( + 0 <= text_segment.end_offset <= length_full_text + ), f"Property end_offset out of range: {text_segment.end_offset} not in [0, {length_full_text}]" + + # line column pair + ## TODO: Is this a better alternative than using tuple comparison (start_line, end_line) <= (end_line, end_column)? + assert ( + text_segment.start_line <= text_segment.end_line + ), f"Property end_line before start_line: {text_segment.end_line} < {text_segment.start_line}" + assert not (text_segment.start_line == text_segment.end_line) or text_segment.start_column <= text_segment.end_column, ( + "Property end_column before start_column, while start and end line are the same: " + + f"{text_segment.end_column} < {text_segment.start_column}" + ) + + line_starts = _compute_line_starts(text_segment.full_text) + + ## line range + lines = len(line_starts) + assert 0 <= text_segment.start_line < lines, f"Property start_line out of range: {text_segment.start_line} not in [0, {lines})" + assert 0 <= text_segment.end_line < lines, f"Property end_line out of range: {text_segment.end_line} not in [0, {lines})" + + ## column range + _check_column_range( + len(text_segment.full_text), + text_segment.start_line, + text_segment.start_column, + line_starts, + "start_column", + ) + _check_column_range( + len(text_segment.full_text), + text_segment.end_line, + text_segment.end_column, + line_starts, + "end_column", + ) + + # consistency offset and line column pair + assert ( + text_segment.start_offset == line_starts[text_segment.start_line] + text_segment.start_column + ), "Start offset and (line, column) are inconsistent" + assert ( + text_segment.end_offset == line_starts[text_segment.end_line] + text_segment.end_column + ), "End offset and (line, column) are inconsistent" + + # consistency full_text and text_segment + assert ( + text_segment.text_segment == text_segment.full_text[text_segment.start_offset : text_segment.end_offset] + ), "text_segment and full_text[start_offset:end_offset] are inconsistent" + + +def _check_column_range( + length_full_text: int, + line: int, + column: int, + line_starts: tuple[int, ...], + description: str, +): + start_line = line_starts[line] + end_line = ( + length_full_text + 1 ## column must be able to point beyond last character of full text to include that character as well. + if line + 1 == len(line_starts) + else line_starts[line + 1] + ) + length_line = end_line - start_line + assert 0 <= column < length_line, f"Property {description} out of range: {column} not in [0, {length_line})" + + +def split_lines_with_newlines(text: str) -> list[str]: + """ + Reference 'lines' derived from split(text, '\n') with all but last extended by '\n'. + This yields a list where each element corresponds to the characters of that line span, + and all '\n' characters belong to the line they terminate. + """ + parts = text.split("\n") + return [p + "\n" for p in parts[:-1]] + [parts[-1]] + + +def line_starts_from_lines(lines: list[str]) -> list[int]: + """ + Compute the starting cursor offsets for each line from the line-span strings. + """ + starts = [0] + acc = 0 + for s in lines[:-1]: + acc += len(s) + starts.append(acc) + return starts + + +def _compute_line_starts(text: str) -> tuple[int, ...]: + """ + Return a tuple with the offset of the first character of that line. + The offset is 0 based. The first line will always starts at offset 0. + """ + line_starts: list[int] = [0] + for i, ch in enumerate(text): + if ch == "\n": + line_starts.append(i + 1) # next line starts after '\n' (0-based offset) + return tuple(line_starts) diff --git a/test/syntax_tree/test_ast_processor.py b/test/syntax_tree/test_ast_processor.py new file mode 100644 index 00000000..762c6cfd --- /dev/null +++ b/test/syntax_tree/test_ast_processor.py @@ -0,0 +1,21 @@ + +from hamcrest import assert_that, is_ + +from renaissance.impl.clang import ClangASTNode +from renaissance.syntax_tree import ASTProcessor, ASTFactory, PatternMatch + + +class TestAstProcessor: + def test_find_match(self, mocker): + node = mocker.Mock() + pattern_match = PatternMatch([node, node, node], {}, []) + mock_matcher = mocker.patch( + "renaissance.syntax_tree.match_finder.find_all", + return_value=[pattern_match], + ) + atu = ClangASTNode.load_from_text("int main(){return 0;}", "test.c", [], None) + ast_refactor = ASTProcessor(atu, ASTFactory(ClangASTNode), in_memory=True) + + ast_refactor.find_match([atu.children[-1].children[-1]]) + + assert_that(mock_matcher.call_count, is_(1)) diff --git a/test/syntax_tree/test_ast_refactor_actions.py b/test/syntax_tree/test_ast_refactor_actions.py new file mode 100644 index 00000000..122d8cb9 --- /dev/null +++ b/test/syntax_tree/test_ast_refactor_actions.py @@ -0,0 +1,82 @@ +from hamcrest import assert_that, is_ + +from renaissance.impl.types import Name +from renaissance.syntax_tree import ASTRefactorActions + + +class TestASTRefactorActions: + + def test_it_can_be_created(self, mocker): + proc = mocker.Mock() + factory = mocker.Mock() + refactor_actions = ASTRefactorActions(proc, factory) + assert_that(refactor_actions, not is_(None)) + + def test_replace_expr(self, mocker): + proc = mocker.Mock() + proc.find_all.return_value = [] + factory = mocker.Mock() + refactor_actions = ASTRefactorActions(proc, factory) + refactor_actions.replace_expr("name", "my_awsome_name", Name) + assert_that(proc.find_all.called) + + def test_replace_name(self, mocker): + node = mocker.Mock() + node.offset = 1 + proc = mocker.Mock() + factory = mocker.Mock() + proc.find_all.return_value = [node] + refactor_actions = ASTRefactorActions(proc, factory) + + refactor_actions.replace_name("name", "my_awsome_name", "Name", "Call") + + assert_that(proc.replace.called) + + def test_replace_text(self, mocker): + node = mocker.Mock() + proc = mocker.Mock() + factory = mocker.Mock() + refactor_actions = ASTRefactorActions(proc, factory) + proc.find_all.return_value = [node, node] + + refactor_actions.replace_text("text", "my_awsome_text", "StringLiteral", "Call") + + assert_that(proc.replace.called) + + def test_replace_declaration(self, mocker): + node = mocker.Mock() + proc = mocker.Mock() + factory = mocker.Mock() + refactor_actions = ASTRefactorActions(proc, factory) + refactor_actions.find_declaration = lambda decl: [node] + + refactor_actions.replace_declaration("decl", "my_awsome_decl") + + assert_that(proc.replace.called) + + def test_replace_patterns(self, mocker): + node = mocker.Mock() + proc = mocker.Mock() + factory = mocker.Mock() + is_match_mock = mocker.patch("renaissance.syntax_tree.match_finder.find_in_list", return_value=True) + refactor_actions = ASTRefactorActions(proc, factory) + + refactor_actions._replace_patterns(node, "my_awsome_text", [[node]], "Call") + + assert_that(proc.replace.called) + assert_that(is_match_mock.called) + + def test_find_declaration(self, mocker): + proc = mocker.Mock() + factory = mocker.Mock() + refactor_actions = ASTRefactorActions(proc, factory) + refactor_actions.find_declaration("decl_pattern") + assert_that(proc.find_match.called) + + def test_collect(self, mocker): + proc = mocker.Mock() + proc.find_match.return_value = [] + factory = mocker.Mock() + refactor_actions = ASTRefactorActions(proc, factory) + result = refactor_actions.collect("pattern", "pattern_kind") + assert_that(proc.find_match.called, is_(1)) diff --git a/test/syntax_tree/test_ast_rewriter.py b/test/syntax_tree/test_ast_rewriter.py new file mode 100644 index 00000000..cf087c67 --- /dev/null +++ b/test/syntax_tree/test_ast_rewriter.py @@ -0,0 +1,1241 @@ +import sys +from typing import Any + +from c_cpp.factories import Factories +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory + +import pytest +from hamcrest import assert_that, is_ + + +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.syntax_tree import ASTRewriter, ASTFactory, PatternMatch +from renaissance.syntax_tree.ast_rewriter import _RewriteAction, _RewriteActions +from renaissance.syntax_tree.match_finder import find_all, match_pattern +from utils_for_tests import compress, debug_print + + +class TestCommentLocation: + + @pytest.mark.parametrize( + "_, start_offset, stop_offset, content, expected", + [ + ( + "single_line_comment", + 0, + 50, + b"Some code // this is a comment\nMore code", + (10, 30), + ), + ( + "double_line_comment", + 0, + 50, + b"Some code// one\n // two\nMore code", + (17, 23), + ), + ( + "block_comment", + 0, + 50, + b"Some code /* this is a block comment */ More code", + (10, 39), + ), + ( + "hash_comment", + 0, + 50, + b"Some code # this is a hash comment\nMore code", + (10, 34), + ), + ("no_comment", 0, 50, b"Some code with no comment\nMore code", (-1, -1)), + ( + "comment_outside_range", + 0, + 10, + b"Some code // this is a comment\nMore code", + (-1, -1), + ), + ( + "multiple_comments", + 0, + 50, + b"Some code // first comment\nMore code /* second comment */", + (10, 26), + ), + ], + ) + def test( + self, + _, + start_offset: int, + stop_offset: int, + content: bytes, + expected: tuple[int, int], + ): + result = ASTRewriter._get_comment_location(start_offset, stop_offset, content) + # converted print but what to do it true??? + # assert_that(result, is_not((-1, -1)), f"first char={content[result[0]:result[1]]}") + assert_that(expected, is_(result)) + + +class TestRewrites: + def test_passing_case_in_clang(self): + # action: Callable[[ASTRewriter, str, Sequence[ASTNode], bool, bool], None], + # factory: ASTFactory,code: str, replacement: str, include_whitespace: bool, include_comments: bool, expected: str): + factory = ASTFactory(ClangASTNode, []) + atu = factory.create_from_text("void f() { /* c1 */ /* c2 */ int a=3;\n}", "test.cpp") + pattern_factory = CPatternFactory(factory) + declaration_pattern = pattern_factory.create_declarations("int a=3;") + found = match_pattern(atu.children, declaration_pattern) + + rewriter = ASTRewriter(atu) + for match in found: # .map(lambda m: m.nodes).to_iterable(): + nodes = match.nodes + rewriter.insert_before("int b=4;int c=5;", nodes, True, True) + assert_that( + rewriter.apply_to_string(), + is_("void f() { /* c1 */ int b=4;int c=5;\n /* c2 */ int a=3;\n}"), + ) + + def test_failing_case(self): + # action: Callable[[ASTRewriter, str, Sequence[ASTNode], bool, bool], None], + # factory: ASTFactory,code: str, replacement: str, include_whitespace: bool, include_comments: bool, expected: str): + factory = ASTFactory(ClangASTNode, []) + atu = factory.create_from_text("void f() { /* c1 */ /* c2 */ int a=3;\n}", "test.cpp") + pattern_factory = CPatternFactory(factory) + declaration_pattern = pattern_factory.create_declarations("int a=3;") + found = match_pattern(atu.children, declaration_pattern) + + rewriter = ASTRewriter(atu) + for match in found: # .map(lambda m: m.nodes).to_iterable(): + nodes = match.nodes + rewriter.insert_before("int b=4;int c=5;", nodes, True, True) + assert_that( + rewriter.apply_to_string(), + is_("void f() { /* c1 */ int b=4;int c=5;\n /* c2 */ int a=3;\n}"), + ) + + @staticmethod + def do_test( + action: Any, + factory: ASTFactory, + code: str, + replacement: str, + include_whitespace: bool, + include_comments: bool, + expected: str, + ): + atu = factory.create_from_text(code, "test.cpp") + pattern_factory = CPatternFactory(factory) + declaration_pattern = pattern_factory.create_declarations("int a=3;") + rewriter = ASTRewriter(atu) + found = match_pattern(atu.children, declaration_pattern) + + for match in found: # .map(lambda m: m.nodes).to_iterable(): + nodes = match.nodes + action(rewriter, replacement, nodes, include_whitespace, include_comments) + expected_result = factory.create_from_text(expected, "test.cpp") + actual = rewriter.apply_to_string() + actual_result = factory.create_from_text(rewriter.apply_to_string(), "test.cpp") + debug_print( + actual, + actual_result, + atu, + code, + expected, + expected_result, + include_comments, + include_whitespace, + ) + + assert_that(actual, is_(expected)) + + +class TestRemove(TestRewrites): + + @pytest.mark.parametrize( + "name, factory, code, include_whitespace, include_comments, expected", + list( + Factories.extend( + [ + ("void f() { /* c1 */ int a=3;\n}", True, True, "void f() {\n}"), + ( + "void f() { int x=2; //x cmt\n int a=3;\n}", + True, + True, + "void f() { int x=2; //x cmt\n}", + ), + ] + ) + ), + ) + def test( + self, + name: str, + factory: ASTFactory, + code: str, + include_whitespace: Any, + include_comments: Any, + expected: Any, + ): + + reemove = lambda s, _, n, ws, cm: ASTRewriter.remove(s, n, ws, cm) + self.do_test( + reemove, + factory, + code, + "int aa=4;", + include_whitespace, + include_comments, + expected, + ) + + +class TestReplace(TestRewrites): + + @pytest.mark.parametrize( + "name, factory, code, include_whitespace, include_comments, expected", + list( + Factories.extend( + [ + ( + "void f() { /* c1 */ int a=3;\n}", + True, + True, + "void f() { int aa=4;\n}", + ), + ( + "void f() { /* c1 */ /* c2 */ int a=3;\n}", + True, + True, + "void f() { /* c1 */ int aa=4;\n}", + ), + ( + "void f() { // c1\n int a=3;\n}", + True, + True, + "void f() { int aa=4;\n}", + ), + ( + "void f() { // c1\n //c2\n int a=3;\n}", + True, + True, + "void f() { // c1\n int aa=4;\n}", + ), + ( + "void f() { int a=3; \n}", + True, + True, + "void f() { int aa=4;\n}", + ), + ( + "void f() { int a=3; //c1 \n}", + True, + True, + "void f() { int aa=4;\n}", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + True, + "void f() { int aa=4; }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + True, + "void f() { int aa=4; }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + False, + "void f() { int aa=4; /*c1 \n */ }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + True, + True, + "/* out scope */ void f() { int aa=4; }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + True, + "/* out scope */ void f() { int aa=4; }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + False, + "/* out scope */ void f() { int aa=4; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + False, + "void f() { int aa=4; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + False, + "void f() { int aa=4; /*c1 \n */ }", + ), + # siblings with comments + ( + "void f() { int x=2; /* c1 */ int a=3; //c2\n int b=4; }", + True, + True, + "void f() { int x=2; /* c1 */ int aa=4;\n int b=4; }", + ), + ( + "void f() { //cx\nint x=2; //ca\n int a=3; //caa\n int b=4;//cb \n}", + True, + True, + "void f() { //cx\nint x=2; //ca\n int aa=4;\n int b=4;//cb \n}", + ), + ( + "void f() { int x=2; /*ca*/ int a=3; /*caa \n nl*/ int b=4; }", + True, + True, + "void f() { int x=2; /*ca*/ int aa=4; int b=4; }", + ), + ] + ) + ), + ) + def test( + self, + name: str, + factory: ASTFactory, + code: str, + include_whitespace: Any, + include_comments: Any, + expected: Any, + ): + self.do_test( + ASTRewriter.replace, + factory, + code, + "int aa=4;", + include_whitespace, + include_comments, + expected, + ) + + +class TestInsertBeforeSingleLine(TestRewrites): + + @pytest.mark.parametrize( + "name, factory, code, include_whitespace, include_comments, expected", + list( + Factories.extend( + [ + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + False, + "/* out scope */ void f() { int aa=4;int a=3; /*c1 \n */ }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + True, + "/* out scope */ void f() { int aa=4;int a=3; /*c1 \n */ }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + True, + True, + "/* out scope */ void f() { int aa=4; int a=3; /*c1 \n */ }", + ), + ( + "void f() { /* c1 */ /* c2 */ int a=3;\n}", + True, + True, + "void f() { /* c1 */ int aa=4;\n /* c2 */ int a=3;\n}", + ), + ( + "void f() { /* c1 */ int a=3;\n}", + True, + True, + "void f() { int aa=4;\n /* c1 */ int a=3;\n}", + ), + ( + "void f() { // c1\n //c2\n int a=3;\n}", + True, + True, + "void f() { // c1\n int aa=4;\n //c2\n int a=3;\n}", + ), + ( + "void f() { // c1\n int a=3;\n}", + True, + True, + "void f() { int aa=4;\n // c1\n int a=3;\n}", + ), + ( + "void f() { //cx\n int x=2; //ca\n int a=3; //caa\n int b=4;//cb \n}", + True, + True, + "void f() { //cx\n int x=2; //ca\n int aa=4;\n int a=3; //caa\n int b=4;//cb \n}", + ), + ( + "void f() { int a=3; \n}", + True, + True, + "void f() { int aa=4;\n int a=3; \n}", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + False, + "void f() { int aa=4;int a=3; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + True, + "void f() { int aa=4;int a=3; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + False, + "void f() { int aa=4; int a=3; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + True, + "void f() { int aa=4; int a=3; /*c1 \n */ }", + ), + ( + "void f() { int a=3; //c1 \n}", + True, + True, + "void f() { int aa=4;\n int a=3; //c1 \n}", + ), + ( + "void f() { int x=2; /*ca*/ int a=3; /*caa \n nl*/ int b=4; }", + True, + True, + "void f() { int x=2; /*ca*/ int aa=4; int a=3; /*caa \n nl*/ int b=4; }", + ), + ( + "void f() { int x=2; /* c1 */ int a=3; //c2\n int b=4; }", + True, + True, + "void f() { int x=2; /* c1 */ int aa=4;\n int a=3; //c2\n int b=4; }", + ), + ( + "void f() { int x=2; //c1\n int a=3; //caa\n int b=4;//cb \n}", + True, + True, + "void f() { int x=2; //c1\n int aa=4;\n int a=3; //caa\n int b=4;//cb \n}", + ), + ] + ) + ), + ) + def test( + self, + name: str, + factory: ASTFactory, + code: str, + include_whitespace: Any, + include_comments: Any, + expected: Any, + ): + self.do_test( + ASTRewriter.insert_before, + factory, + code, + "int aa=4;", + include_whitespace, + include_comments, + expected, + ) + + +class TestInsertBeforeMultiLine(TestRewrites): + + @pytest.mark.parametrize( + "name, factory, code, include_whitespace, include_comments, expected", + list( + Factories.extend( + [ + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + False, + "/* out scope */ void f() { int aa=4;\n int bb=5;int a=3; /*c1 \n */ }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + True, + "/* out scope */ void f() { int aa=4;\n int bb=5;int a=3; /*c1 \n */ }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + True, + True, + "/* out scope */ void f() { int aa=4;\n int bb=5; int a=3; /*c1 \n */ }", + ), + ( + "/* indent 2 */ void f() {\n int a=3; /*c1 \n */ }", + False, + False, + "/* indent 2 */ void f() {\n int aa=4;\n int bb=5;int a=3; /*c1 \n */ }", + ), + ( + "/* indent 2 */ void f() {\n int a=3; /*c1 \n */ }", + False, + True, + "/* indent 2 */ void f() {\n int aa=4;\n int bb=5;int a=3; /*c1 \n */ }", + ), + ( + "/* indent 2 */ void f() {\n int a=3; /*c1 \n */ }", + True, + True, + "/* indent 2 */ void f() {\n int aa=4;\n int bb=5; int a=3; /*c1 \n */ }", + ), + ( + "void f() { /* c1 */ /* c2 */ int a=3;\n}", + True, + True, + "void f() { /* c1 */ int aa=4;\n int bb=5;\n /* c2 */ int a=3;\n}", + ), + ( + "void f() { /* c1 */ int a=3;\n}", + True, + True, + "void f() { int aa=4;\n int bb=5;\n /* c1 */ int a=3;\n}", + ), + ( + "void f() { // c1\n //c2\n int a=3;\n}", + True, + True, + "void f() { // c1\n int aa=4;\n int bb=5;\n //c2\n int a=3;\n}", + ), + ( + "void f() { // c1\n int a=3;\n}", + True, + True, + "void f() { int aa=4;\n int bb=5;\n // c1\n int a=3;\n}", + ), + ( + "void f() { //cx\n int x=2; //ca\n int a=3; //caa\n int b=4;//cb \n}", + True, + True, + "void f() { //cx\n int x=2; //ca\n int aa=4;\n int bb=5;\n int a=3; //caa\n int b=4;//cb \n}", + ), + ( + "void f() { int a=3; \n}", + True, + True, + "void f() { int aa=4;\n int bb=5;\n int a=3; \n}", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + False, + "void f() { int aa=4;\n int bb=5;int a=3; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + True, + "void f() { int aa=4;\n int bb=5;int a=3; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + False, + "void f() { int aa=4;\n int bb=5; int a=3; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + True, + "void f() { int aa=4;\n int bb=5; int a=3; /*c1 \n */ }", + ), + ( + "void f() { int a=3; //c1 \n}", + True, + True, + "void f() { int aa=4;\n int bb=5;\n int a=3; //c1 \n}", + ), + ( + "void f() { int x=2; /*ca*/ int a=3; /*caa \n nl*/ int b=4; }", + True, + True, + "void f() { int x=2; /*ca*/ int aa=4;\n int bb=5; int a=3; /*caa \n nl*/ int b=4; }", + ), + ( + "void f() { int x=2; /* c1 */ int a=3; //c2\n int b=4; }", + True, + True, + "void f() { int x=2; /* c1 */ int aa=4;\n int bb=5;\n int a=3; //c2\n int b=4; }", + ), + ( + "void f() { int x=2; //c1\n int a=3; //caa\n int b=4;//cb \n}", + True, + True, + "void f() { int x=2; //c1\n int aa=4;\n int bb=5;\n int a=3; //caa\n int b=4;//cb \n}", + ), + ] + ) + ), + ) + def test( + self, + name: str, + factory: ASTFactory, + code: str, + include_whitespace: Any, + include_comments: Any, + expected: Any, + ): + self.do_test( + ASTRewriter.insert_before, + factory, + code, + "int aa=4;\nint bb=5;", + include_whitespace, + include_comments, + expected, + ) + + +class TestInsertAfterSingleLine(TestRewrites): + + @pytest.mark.parametrize( + "name, factory, code, include_whitespace, include_comments, expected", + list( + Factories.extend( + [ + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + False, + "/* out scope */ void f() { int a=3;int aa=4; /*c1 \n */ }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + True, + "/* out scope */ void f() { int a=3; /*c1 \n */int aa=4; }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + True, + True, + "/* out scope */ void f() { int a=3; /*c1 \n */ int aa=4; }", + ), + ( + "void f() { /* c1 */ /* c2 */ int a=3;\n}", + True, + True, + "void f() { /* c1 */ /* c2 */ int a=3;\n int aa=4;\n}", + ), + ( + "void f() { /* c1 */ int a=3;\n}", + True, + True, + "void f() { /* c1 */ int a=3;\n int aa=4;\n}", + ), + ( + "void f() { // c1\n //c2\n int a=3;\n}", + True, + True, + "void f() { // c1\n //c2\n int a=3;\n int aa=4;\n}", + ), + ( + "void f() { // c1\n int a=3;\n}", + True, + True, + "void f() { // c1\n int a=3;\n int aa=4;\n}", + ), + ( + "void f() { //cx\nint x=2; //ca\n int a=3; //caa\n int b=4;//cb \n}", + True, + True, + "void f() { //cx\nint x=2; //ca\n int a=3; //caa\n int aa=4;\n int b=4;//cb \n}", + ), + ( + "void f() { int a=3; \n}", + True, + True, + "void f() { int a=3; \n int aa=4;\n}", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + False, + "void f() { int a=3;int aa=4; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + True, + "void f() { int a=3; /*c1 \n */int aa=4; }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + False, + "void f() { int a=3; int aa=4; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + True, + "void f() { int a=3; /*c1 \n */ int aa=4; }", + ), + ( + "void f() { int a=3; //c1 \n}", + True, + True, + "void f() { int a=3; //c1 \n int aa=4;\n}", + ), + ( + "void f() { int x=2; /*ca*/ int a=3; /*caa \n nl*/ int b=4; }", + True, + True, + "void f() { int x=2; /*ca*/ int a=3; /*caa \n nl*/ int aa=4; int b=4; }", + ), + ( + "void f() { int x=2; /* c1 */ int a=3; //c2\n int b=4; }", + True, + True, + "void f() { int x=2; /* c1 */ int a=3; //c2\n int aa=4;\n int b=4; }", + ), + ( + "void f() { int x=2; //c1\n int a=3; //caa\n int b=4;//cb \n}", + True, + True, + "void f() { int x=2; //c1\n int a=3; //caa\n int aa=4;\n int b=4;//cb \n}", + ), + ] + ) + ), + ) + def test( + self, + name: str, + factory: ASTFactory, + code: str, + include_whitespace: Any, + include_comments: Any, + expected: Any, + ): + self.do_test( + ASTRewriter.insert_after, + factory, + code, + "int aa=4;", + include_whitespace, + include_comments, + expected, + ) + + +class TestInsertAfterMultiLine(TestRewrites): + + @pytest.mark.parametrize( + "name, factory, code, include_whitespace, include_comments, expected", + list( + Factories.extend( + [ + ( + "/* indent 2 */ void f() {\n int a=3; /*c1 \n */ }", + False, + False, + "/* indent 2 */ void f() {\n int a=3;int aa=4;\n int bb=5; /*c1 \n */ }", + ), + ( + "/* indent 2 */ void f() {\n int a=3; /*c1 \n */ }", + False, + True, + "/* indent 2 */ void f() {\n int a=3; /*c1 \n */int aa=4;\n int bb=5; }", + ), + ( + "/* indent 2 */ void f() {\n int a=3; /*c1 \n */ }", + True, + True, + "/* indent 2 */ void f() {\n int a=3; /*c1 \n */ int aa=4;\n int bb=5; }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + False, + "/* out scope */ void f() { int a=3;int aa=4;\n int bb=5; /*c1 \n */ }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + False, + True, + "/* out scope */ void f() { int a=3; /*c1 \n */int aa=4;\n int bb=5; }", + ), + ( + "/* out scope */ void f() { int a=3; /*c1 \n */ }", + True, + True, + "/* out scope */ void f() { int a=3; /*c1 \n */ int aa=4;\n int bb=5; }", + ), + ( + "void f() { /* c1 */ /* c2 */ int a=3;\n}", + True, + True, + "void f() { /* c1 */ /* c2 */ int a=3;\n int aa=4;\n int bb=5;\n}", + ), + ( + "void f() { /* c1 */ int a=3;\n}", + True, + True, + "void f() { /* c1 */ int a=3;\n int aa=4;\n int bb=5;\n}", + ), + ( + "void f() { // c1\n //c2\n int a=3;\n}", + True, + True, + "void f() { // c1\n //c2\n int a=3;\n int aa=4;\n int bb=5;\n}", + ), + ( + "void f() { // c1\n int a=3;\n}", + True, + True, + "void f() { // c1\n int a=3;\n int aa=4;\n int bb=5;\n}", + ), + ( + "void f() { //cx\nint x=2; //ca\n int a=3; //caa\n int b=4;//cb \n}", + True, + True, + "void f() { //cx\nint x=2; //ca\n int a=3; //caa\n int aa=4;\n int bb=5;\n int b=4;//cb \n}", + ), + ( + "void f() { int a=3; \n}", + True, + True, + "void f() { int a=3; \n int aa=4;\n int bb=5;\n}", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + False, + "void f() { int a=3;int aa=4;\n int bb=5; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + False, + True, + "void f() { int a=3; /*c1 \n */int aa=4;\n int bb=5; }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + False, + "void f() { int a=3; int aa=4;\n int bb=5; /*c1 \n */ }", + ), + ( + "void f() { int a=3; /*c1 \n */ }", + True, + True, + "void f() { int a=3; /*c1 \n */ int aa=4;\n int bb=5; }", + ), + ( + "void f() { int a=3; //c1 \n}", + True, + True, + "void f() { int a=3; //c1 \n int aa=4;\n int bb=5;\n}", + ), + ( + "void f() { int x=2; /*ca*/ int a=3; /*caa \n nl*/ int b=4; }", + True, + True, + "void f() { int x=2; /*ca*/ int a=3; /*caa \n nl*/ int aa=4;\n int bb=5; int b=4; }", + ), + ( + "void f() { int x=2; /* c1 */ int a=3; //c2\n int b=4; }", + True, + True, + "void f() { int x=2; /* c1 */ int a=3; //c2\n int aa=4;\n int bb=5;\n int b=4; }", + ), + ( + "void f() { int x=2; //c1\n int a=3; //caa\n int b=4;//cb \n}", + True, + True, + "void f() { int x=2; //c1\n int a=3; //caa\n int aa=4;\n int bb=5;\n int b=4;//cb \n}", + ), + ] + ) + ), + ) + def test( + self, + name: str, + factory: ASTFactory, + code: str, + include_whitespace: Any, + include_comments: Any, + expected: Any, + ): + self.do_test( + ASTRewriter.insert_after, + factory, + code, + "int aa=4;\nint bb=5;", + include_whitespace, + include_comments, + expected, + ) + + +class TestComposeReplacement: + + @pytest.mark.parametrize( + "_, factory, statements, extra_declarations, replacement", + Factories.extend( + [ + ( + "if($exp){$$before;b=$d1;$$after;}else{$$before;b=$d2;$$after;}", + [], + {"$$before; b = ($exp) ? $d1:$d2; $$after;": "int a=1;int b=2;int c=3;int d=4;void f(){c++;b=(a==1)?2:3;d++;}"}, + ), + ] + ), + ) + def test_args( + self, + _: Any, + factory: ASTFactory, + statements: Any, + extra_declarations: Any, + replacement: Any, + ): + code = """ + int a = 1; + int b = 2; + int c = 3; + int d = 4; + void f(){ + if (a==1) { + c++; + b = 2; + d++; + } + else { + c++; + b = 3; + d++; + } + } + """ + atu = factory.create_from_text(code, "test.cpp") + stmt_nodes = CPatternFactory(factory).create_statements(statements, extra_declarations=extra_declarations) + matches = (match for match in match_pattern([atu], stmt_nodes) if match.nodes[0].is_part_of_translation_unit()) + + for match, exp in zip(matches, replacement.items()): + rewriter = ASTRewriter(match.nodes[0].root) + org, expected = exp + rewriter.replace(org, match) + actual = rewriter.apply_to_string() + assert_that(compress(expected), is_(compress(actual))) + + def test_get_node_in_match_pattern(self, mocker): + node = mocker.Mock() + reference = mocker.Mock() + node.referenced_by = [reference, reference] + reference.node = node + pattern_match = PatternMatch([node, node, node], {}, []) + n = _RewriteAction._get_nodes([pattern_match])[0] + assert_that(n, is_(node)) + + @pytest.mark.skip("fail on empty nodes") + def test_get_node_in_match_pattern_on_empty_pattern(self): + it = _RewriteActions([], sys.getfilesystemencoding(), True) + text = getattr(it, "_RewriteActions__get_texts")([]) + assert_that(text, is_("node")) + + def test_get_text_from_rewrite(self, mocker): + node = mocker.Mock() + node.root = node + node.binary_file_content = lambda: b"int x =0;" + node.offset = 0 + node.extended_end_offset = 8 + node.text = "int x =0" + + it = _RewriteActions(node, sys.getfilesystemencoding(), True) + text = getattr(it, "_RewriteActions__get_texts")([node]) + assert_that(text, is_("int x =0")) + + +class TestAroundComposition: + """ + Test case to capture the requirements for `around` functionality that is composable. + """ + + @pytest.mark.skip( + "TODO: Test fails due to two issues\n 1. order of inserts ([ )]\n 2. insert around whole pattern, not placeholder." + ) + def test_around(self): + # set up + factory = PythonFactory(PythonRstNode) + atu = factory.create_from_text("x = a", "temp.py") + pattern = PythonPatternFactory(factory).create_expression("x = $a") + matches = list(find_all([atu], [pattern])) # Use list, since we want to access its content multiple times + assert matches, "A match expected" + nrof_matches = len(matches) + assert 1 == nrof_matches, f"One match expected, yet got {nrof_matches}" + placeholder = matches[0].expansions["$a"] + + rewriter = ASTRewriter(atu) + + # execute + ## first pair + rewriter.insert_before("(", placeholder) + rewriter.insert_after(")", placeholder) + + ## second pair + rewriter.insert_before("[", placeholder) + rewriter.insert_after("]", placeholder) + + # verify + assert_that(rewriter.apply_to_string(), is_("x = [ ( a ) ]"), "Unexpected replacement") + + +class TestContainedOperations: + """ + Test case to capture the requirements for a (completely) contained operation: + it is ignore. + + See https://github.com/TNO/Renaissance-Experiments/wiki/Transform-%E2%80%90-AST%E2%80%90aware-changes#scenario-contained-changes + """ + + def setup(self) -> tuple[ASTRewriter, PatternMatch]: + factory = PythonFactory(PythonRstNode) + atu = factory.create_from_text("x = a * b", "temp.py") + pattern = PythonPatternFactory(factory).create_expression("$a * $b") + matches = list(find_all([atu], [pattern])) # Use list, since we want to access its content multiple times + assert matches, "A match expected" + nrof_matches = len(matches) + assert 1 == nrof_matches, f"One match expected, yet got {nrof_matches}" + match = matches[0] + + rewriter = ASTRewriter(atu) + return rewriter, match + + @pytest.mark.skip("TODO: fix impl.") + def test_replace_contained_replace(self): + rewriter, match = self.setup() + rewriter.replace("product", match.nodes) + rewriter.replace("term", match.expansions["$a"]) + assert "x = product" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.skip("TODO: fix impl.") + def test_contained_replace_replace(self): + rewriter, match = self.setup() + rewriter.replace("term", match.expansions["$a"]) + rewriter.replace("product", match.nodes) + assert "x = product" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.skip("TODO: fix impl.") + def test_replace_contained_remove(self): + rewriter, match = self.setup() + rewriter.replace("product", match.nodes) + rewriter.remove(match.expansions["$a"]) + assert "x = product" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.skip("TODO: fix impl.") + def test_contained_remove_replace(self): + rewriter, match = self.setup() + rewriter.remove(match.expansions["$a"]) + rewriter.replace("product", match.nodes) + assert "x = product" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.skip("TODO: fix impl.") + def test_replace_contained_prepend(self): + rewriter, match = self.setup() + rewriter.replace("product", match.nodes) + rewriter.insert_before("term", match.expansions["$a"]) + assert "x = product" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.skip("TODO: fix impl.") + def test_contained_prepend_replace(self): + rewriter, match = self.setup() + rewriter.insert_before("term", match.expansions["$a"]) + rewriter.replace("product", match.nodes) + assert "x = product" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.skip("TODO: fix impl.") + def test_replace_contained_append(self): + rewriter, match = self.setup() + rewriter.replace("product", match.nodes) + rewriter.insert_after("term", match.expansions["$a"]) + assert "x = product" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.skip("TODO: fix impl.") + def test_contained_append_replace(self): + rewriter, match = self.setup() + rewriter.insert_after("term", match.expansions["$a"]) + rewriter.replace("product", match.nodes) + assert "x = product" == rewriter.apply_to_string(), "Unexpected replacement" + + +class TestOverlappingOperations: + """ + Test case to capture the requirements for partly overlapping operations: + an exception is raised. + + See https://github.com/TNO/Renaissance-Experiments/wiki/Transform-%E2%80%90-AST%E2%80%90aware-changes#scenario-overlapping-changes + """ + + def setup(self) -> tuple[ASTRewriter, PatternMatch]: + CODE: str = """ +def f(a,b,c): + pass +""" + + PATTERN: str = """ +def f($a,$b,$c): + pass +""" + + factory = ASTFactory(PythonRstNode, []) + atu = factory.create_from_text(CODE, "temp.py") + pattern = PythonPatternFactory(factory).create(PATTERN) + matches = list(find_all([atu], [pattern])) # Use list, since we want to access its content multiple times + assert matches, "A match expected" + nrof_matches = len(matches) + assert 1 == nrof_matches, f"One match expected, yet got {nrof_matches}" + match = matches[0] + + rewriter = ASTRewriter(atu) + return rewriter, match + + @pytest.mark.skip( + "it is not correctly implementing: https://github.com/TNO/Renaissance-Experiments/wiki/Transform-%E2%80%90-AST%E2%80%90aware-changes#scenario-contained-changes" + ) + def test_overlapping_replaces(self): + rewriter, match = self.setup() + placeholder_a = match.expansions["$a"] + placeholder_b = match.expansions["$b"] + placeholder_c = match.expansions["$c"] + + rewriter.replace("any", [placeholder_a, placeholder_b]) + rewriter.replace("ANY", [placeholder_b, placeholder_c]) + + with pytest.raises(Exception): + rewriter.apply_to_string() + + +class TestSyntaxAwareNestedComposition: + """ + Test Class for Syntax Aware Nested / Hierarchical Compositions + In Python + * Prepend before parent and (first) child + See https://github.com/TNO/Renaissance-Experiments/wiki/Transform-%E2%80%90-AST%E2%80%90aware-changes#scenario-combination-of-multiple-prepends + * Append after parent and (last) child + See https://github.com/TNO/Renaissance-Experiments/wiki/Transform-%E2%80%90-AST%E2%80%90aware-changes#scenario-combination-of-multiple-appends + """ + + def setup(self) -> tuple[ASTRewriter, PatternMatch]: + factory = ASTFactory(PythonRstNode, []) + atu = factory.create_from_text("x = a * b", "temp.py") + pattern = PythonPatternFactory(factory).create_expression("$a * $b") + matches = list(find_all([atu], [pattern])) # Use list, since we want to access its content multiple times + assert matches, "A match expected" + nrof_matches = len(matches) + assert 1 == nrof_matches, f"One match expected, yet got {nrof_matches}" + match = matches[0] + + rewriter = ASTRewriter(atu) + return rewriter, match + + @pytest.mark.skip("TODO: Test fails as prepend of child appears before prepend of parent") + def test_prepend_child_parent(self): + rewriter, match = self.setup() + rewriter.insert_before("4 *", match.expansions["$a"]) + rewriter.insert_before("6 +", match.nodes) + assert "x = 6 + 4 * a * b" == rewriter.apply_to_string(), "Unexpected replacement" + # TODO: Test fails as prepend of child appears before prepend of parent + + @pytest.mark.skip("TODO: Test fails as prepend of child appears before prepend of parent") + def test_prepend_parent_child(self): + rewriter, match = self.setup() + rewriter.insert_before("6 +", match.nodes) + rewriter.insert_before("4 *", match.expansions["$a"]) + assert "x = 6 + 4 * a * b" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.skip("TODO: Test fails as prepend of child appears before prepend of parent") + def test_append_child_parent(self): + rewriter, match = self.setup() + rewriter.insert_after("* 4", match.expansions["$b"]) + rewriter.insert_after("+ 6", match.nodes) + assert "x = a * b * 4 + 6" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.skip("TODO: Test fails as prepend of child appears before prepend of parent") + def test_append_parent_child(self): + rewriter, match = self.setup() + rewriter.insert_after("+ 6", match.nodes) + rewriter.insert_after("* 4", match.expansions["$b"]) + assert "x = a * b * 4 + 6" == rewriter.apply_to_string(), "Unexpected replacement" + # TODO: Test fails as append of child appears after append of parent + + +class TestSyntaxAwareAdjacentComposition: + """ + Test Class for Syntax Aware Adjacent Compositions + In C/C++ + * Consecutive / contiguous nodes - append after first and prepend before second + See https://github.com/TNO/Renaissance-Experiments/wiki/Transform-%E2%80%90-AST%E2%80%90aware-changes#scenario-combination-of-append-and-prepend-on-consecutive-nodes + + Note in C/C++ `;` is a terminator that is a part of a statement + in Python `;` is a separator that can be used to put multiple statements on the same line + """ + + def setup(self, factory: ASTFactory): + CODE: str = "void f(int i, int j) { i++;j++; }" + PATTERN: str = " $stmt1; $stmt2; " + + atu = factory.create_from_text(CODE, "test.c") + pattern = CPatternFactory(factory).create_statements(PATTERN) + matches = list(find_all([atu], [pattern])) + + assert matches, "A match expected" + nrof_matches = len(matches) + + assert 1 == nrof_matches, f"One match expected, yet got {nrof_matches}" + match = matches[0] + + rewriter = ASTRewriter(atu) + return rewriter, match + + @pytest.mark.skip("TODO: implement accordingly") + def test_first_append_prepend_second(self): + # setup + rewriter, match = self.setup(ASTFactory(ClangASTNode)) + + # execute + rewriter.insert_after("++i;", match.expansions["$stmt1"]) + rewriter.insert_before("++j;", match.expansions["$stmt2"]) + + # verify + assert "void f(int i, int j) { i++;++i;++j;j++; }" == rewriter.apply_to_string(), "Unexpected replacement" + + @pytest.mark.parametrize("name, factory", Factories.factories) + @pytest.mark.skip("TODO: implement accordingly") + def test_prepend_second_first_append(self, name: str, factory: ASTFactory): + # setup + rewriter, match = self.setup(factory) + + # execute + rewriter.insert_before("++j;", match.expansions["$stmt2"]) + rewriter.insert_after("++i;", match.expansions["$stmt1"]) + + # verify + assert "void f(int i, int j) { i++;++i;++j;j++; }" == rewriter.apply_to_string(), f"{name}: Unexpected replacement" diff --git a/test/syntax_tree/test_batch_ast_processor.py b/test/syntax_tree/test_batch_ast_processor.py new file mode 100644 index 00000000..caf7dca5 --- /dev/null +++ b/test/syntax_tree/test_batch_ast_processor.py @@ -0,0 +1,88 @@ +from hamcrest import assert_that, is_, has_length + +from renaissance.syntax_tree import BatchASTProcessor + + +class TestBatchASTProcessor: + + def test_it(self): + it = BatchASTProcessor(True, 8) + assert_that(it.in_memory) + assert_that(it.max_processes, is_(8)) + + def test_once(self, mocker): + processor = BatchASTProcessor(True, 8) + iterable_items = [mocker.Mock()] + actions_mock = mocker.Mock() + process_method_spy = mocker.patch.object(processor, "_BatchASTProcessor__process") + processor.once(lambda: iterable_items, actions_mock) + assert_that(process_method_spy.called) + + def test_repeat(self, mocker): + processor = BatchASTProcessor(True, 8) + iterable_items = [mocker.Mock()] + actions_mock = mocker.Mock() + process_method_spy = mocker.patch.object(processor, "_BatchASTProcessor__process") + + processor.repeat(lambda: iterable_items, actions_mock) + + assert_that(process_method_spy.called) + + def test__process(self, mocker): + processor = BatchASTProcessor(True, 8) + dummy_atu_item = (mocker.Mock(), mocker.Mock()) + atu_items = [dummy_atu_item] + actions_list = [mocker.Mock()] + process_atu_spy = mocker.patch("renaissance.syntax_tree.batch_ast_processor.process_atu", return_value=[]) + processor._BatchASTProcessor__process(atu_items, actions_list) + assert_that(process_atu_spy.called) + + def test_replace_if_in_memory(self, mocker): + processor = BatchASTProcessor(True, 8) + fake_factory = mocker.Mock() + fake_node = mocker.Mock() + fake_node.filename = "a.c" + atu_item = (fake_factory, fake_node) + + result_no_in_memory = processor._replace_if_in_memory(atu_item) + assert_that(result_no_in_memory, is_(atu_item)) + + in_memory_content = "int x = 0;" + processor.in_memory_files[fake_node.filename] = in_memory_content + sentinel_atu = mocker.Mock() + fake_factory.create_from_text = mocker.Mock(return_value=sentinel_atu) + + result_with_in_memory = processor._replace_if_in_memory(atu_item) + assert_that(result_with_in_memory, has_length(2)) + assert_that(result_with_in_memory[0], is_(fake_factory)) + assert_that(result_with_in_memory[1], is_(sentinel_atu)) + fake_factory.create_from_text.assert_called_with(in_memory_content, fake_node.filename) + + def test_process_atu(self, mocker): + from renaissance.syntax_tree import batch_ast_processor as bap + + processor = BatchASTProcessor(True, 8) + + dummy_factory = mocker.Mock() + dummy_node = mocker.Mock() + atu = (dummy_factory, dummy_node) + + action_result = mocker.Mock() + + def action(ast_proc): + return action_result + + mock_ast_proc = mocker.Mock() + mock_ast_proc.has_changed.return_value = False + mock_ast_proc.commit.return_value = mock_ast_proc + mock_ast_proc.filename.return_value = "file" + mock_ast_proc.apply_to_string.return_value = "content" + mocker.patch( + "renaissance.syntax_tree.batch_ast_processor.ASTProcessor", + return_value=mock_ast_proc, + ) + + results = bap.process_atu(atu, processor, [action], in_memory=False, max_repeat=1) + + assert_that(results, has_length(1)) + assert_that(results[0], is_(action_result)) diff --git a/test/syntax_tree/test_match_dict.py b/test/syntax_tree/test_match_dict.py new file mode 100644 index 00000000..33c52450 --- /dev/null +++ b/test/syntax_tree/test_match_dict.py @@ -0,0 +1,83 @@ +from hamcrest import assert_that, is_ + +from renaissance.syntax_tree.match_finder import is_match_dict + + +class TestIsMatchDict: + def test_is_same_dict(self): + src = {"a": "asd", "b": "zxc"} + cmp = {"a": "asd", "b": "zxc"} + assert_that(is_match_dict(src, cmp, {})) + + def test_is_same_dict_different_key(self): + src = {"a": "asd", "b": "zxc"} + cmp = {"a": "asd", "c": "zxc"} + assert_that(is_match_dict(src, cmp), is_(False)) + + def test_is_same_dict_extra_key(self): + src = {"a": "asd", "b": "zxc", "extra": "zxc"} + cmp = {"a": "asd", "b": "zxc"} + assert_that(is_match_dict(src, cmp), is_(False)) + + def test_is_same_dict_missing_key(self): + src = {"a": "asd", "b": "zxc"} + cmp = {"a": "asd", "b": "zxc", "extra": "zxc"} + assert_that( + is_match_dict( + src, + cmp, + ), + is_(False), + ) + + def test_is_same_dict_extra_irelevent_key(self): + src = {"a": "asd", "b": "zxc", "macro_expansion": "zxc"} + cmp = { + "a": "asd", + "b": "zxc", + } + assert_that(is_match_dict(src, cmp, {}), is_(True)) + + def test_is_same_dict_key_in_expansion(self): + src = { + "a": "asd", + "b": "zxc", + } + cmp = { + "a": "asd", + "b": "$var", + } + assert_that(is_match_dict(src, cmp, {"$var": ["zxc"]}), is_(True)) + + def test_is_same_dict_key_no_expansion(self): + src = { + "a": "asd", + "b": "zxc", + } + cmp = { + "a": "asd", + "b": "$var", + } + assert_that(is_match_dict(src, cmp), is_(True)) + + def test_is_same_dict_key_in_expansion_with_different_value(self): + src = { + "a": "asd", + "b": "zxc", + } + cmp = { + "a": "asd", + "b": "$var", + } + assert_that(is_match_dict(src, cmp, {"$var": "_xc"}), is_(False)) + + def test_is_same_dict_key_in_expansion_in_src_should_not_happen(self): + src = { + "a": "asd", + "b": "$var", + } + cmp = { + "a": "asd", + "b": "zxc", + } + assert_that(is_match_dict(src, cmp), is_(False)) diff --git a/test/syntax_tree/test_match_finder.py b/test/syntax_tree/test_match_finder.py new file mode 100644 index 00000000..ee47ea22 --- /dev/null +++ b/test/syntax_tree/test_match_finder.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from hamcrest import assert_that, is_, has_length + +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.syntax_tree import ASTFactory +from renaissance.syntax_tree.match_finder import find_in_list, MatchFinder + +VERBOSE = False +DEFAULT_EXCLUDE_KIND = "comment" + +code = """ +int one(int a); +int two(int a, int b); +int three(int a, int b, int c); +int a,b,c; +void f(){ + one(a); + two(a,b); + three(a,b,c); +} +""" +statements = "$f($a, $$all);" +extra_declarations = ["int $f(int,int);"] +result = [ + {"$f": ["one"], "$a": ["a"], "$$all": []}, + {"$f": ["two"], "$a": ["a"], "$$all": ["b"]}, + {"$f": ["three"], "$a": ["a"], "$$all": ["b", "c"]}, +] + + +class TestMatchFinder: + def test_find_in_tree_one_and_all_params(self): + factory = ASTFactory(ClangASTNode, []) + patterns = [CPatternFactory(factory).create_statements(statements, extra_declarations=extra_declarations)] + + atu = factory.create_from_text(code, "test.c") + src = atu.children[-1].children[-1].children + found_position = find_in_list(src, patterns[0], {}) + assert_that(found_position, is_(0)) + + def test_find_in_tree_one_and_all_params_2(self): + factory = ASTFactory(ClangASTNode, []) + patterns = [CPatternFactory(factory).create_statements(statements, extra_declarations=extra_declarations)] + + atu = factory.create_from_text(code, "test.c") + src = atu.children[-1].children[-1].children + found_position = find_in_list(src[1:], patterns[0], {}) + assert_that(found_position, is_(0)) + + def test_find_in_tree_one_and_all_params_3(self): + factory = ASTFactory(ClangASTNode, []) + patterns = [CPatternFactory(factory).create_statements(statements, extra_declarations=extra_declarations)] + + atu = factory.create_from_text(code, "test.c") + src = atu.children[-1].children[-1].children + found_position = find_in_list(src[2:], patterns[0], {}) + assert_that(found_position, is_(0)) + + def test_match_one_and_all_params(self): + factory = ASTFactory(ClangASTNode, []) + patterns = [CPatternFactory(factory).create_statements(statements, extra_declarations=extra_declarations)] + + atu = factory.create_from_text(code, "test.c") + src = atu.children[-1].children[-1].children + # find all if and while statements + matches = MatchFinder.match_pattern(src, patterns[0]) + assert_that(matches, has_length(3)) diff --git a/test/syntax_tree/test_match_finder_multi_assignments.py b/test/syntax_tree/test_match_finder_multi_assignments.py new file mode 100644 index 00000000..00d086b5 --- /dev/null +++ b/test/syntax_tree/test_match_finder_multi_assignments.py @@ -0,0 +1,51 @@ +import pytest +from hamcrest import has_length +from hamcrest.core import assert_that + +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.syntax_tree.match_finder import find_variants + +code = """ +f(0,0) +""" + +PLACEHOLDER_BEFORE: str = "$$before" +PLACEHOLDER_AFTER: str = "$$after" +PATTERN_CALL: str = "f(" + PLACEHOLDER_BEFORE + ", 0, " + PLACEHOLDER_AFTER + ")" + + +class TestMatchFinderMultiAssignments: + @pytest.mark.skip("not impl. yet") + def test_find_multi_assignments(self): + # set up + factory = PythonFactory(PythonRstNode) + atu = factory.create_from_text(code) + pattern = PythonPatternFactory(factory).create_statements(PATTERN_CALL) + + # execute + variants = list(find_variants(atu.children, pattern)) # Use list, since we want to access its content multiple times + + # verify + assert_that(variants, has_length(2), f"Two matches expected, got {len(variants)}.") + # TODO Discuss what behaviour do we exactly want? + # In this case, 1 match on the AST node "f(0,0)" with 2 assignments (as checked below) is also acceptable to me. + + expected: set[frozenset[tuple[str, str]]] = { + frozenset({PLACEHOLDER_BEFORE: "", PLACEHOLDER_AFTER: "0"}.items()), + frozenset({PLACEHOLDER_BEFORE: "0", PLACEHOLDER_AFTER: ""}.items()), + } + + actual: set[frozenset[tuple[str, str]]] = set() + for vatiant in variants: + # TODO getting the location of a (possibly empty) multiple placeholder is no longer supported + before_location = vatiant.locations[PLACEHOLDER_BEFORE] + after_location = vatiant.locations[PLACEHOLDER_AFTER] + + assignment: dict[str, str] = { + PLACEHOLDER_BEFORE: atu.translation_unit.content[before_location.offset: before_location.end_offset], + PLACEHOLDER_AFTER: atu.translation_unit.content[after_location.offset: after_location.end_offset] + } + + actual.add(frozenset(assignment.items())) + assert expected == actual, "Unexpected assignments of placeholders" diff --git a/test/syntax_tree/test_match_tree.py b/test/syntax_tree/test_match_tree.py new file mode 100644 index 00000000..cf43fa7e --- /dev/null +++ b/test/syntax_tree/test_match_tree.py @@ -0,0 +1,322 @@ +import ast +import textwrap + +import pytest +from hamcrest import ( + assert_that, + has_length, + is_, + empty, + is_not, + less_than, + raises, + calling, +) + +from renaissance.impl.clang import ClangASTNode, CPatternFactory +from renaissance.impl.python.rst_node import PythonRstNode +from renaissance.impl.python.factory import PythonPatternFactory, PythonFactory +from renaissance.syntax_tree import ASTFactory, ASTShower +from renaissance.syntax_tree.match_finder import ( + is_match_tree, + MatchFinder, + find_in_list, + match_pattern, is_match, variant_in_match_stmt, +) + + +class TestMatchTree: + @pytest.fixture(autouse=True) + def setup(self): + self.factory = PythonFactory(PythonRstNode) + self.pattern_factory = PythonPatternFactory(self.factory) + + def test_none_with_none_is_not_allowed(self): + assert_that(calling(lambda: is_match_tree(None, None)), raises(Exception)) + + def test_none_with_list(self): + pattern = self.pattern_factory.create_statements("1") + + assert_that(calling(lambda: is_match_tree(None, pattern)), raises(Exception)) + + def test_list_with_none(self): + src = self.pattern_factory.create_statements("1") + pattern = None + + assert_that(is_match_tree(src, pattern), is_(False)) + + def test_empty_lists_with_empty_pattern(self): + src = [] + pattern = [] + + assert_that(is_match_tree(src, pattern), is_(True)) + + def test_lists_with_empty_pattern(self): + src = self.pattern_factory.create_statements("1") + pattern = [] + + assert_that(is_match_tree(src, pattern), is_(False)) + + def test_is_match_tree_between_list_and_other(self): + src = self.pattern_factory.create_statements("1") + pattern = ast.Name("name") + + assert_that(is_match_tree(src, [pattern]), is_(False)) + + def test_empty_lists_with_pattern(self): + src = [] + pattern = self.pattern_factory.create_statements("1") + + assert_that(is_match_tree(src, pattern), is_(False)) + + def test_lists_with_list(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + + assert_that(is_match_tree(src, pattern), is_(True)) + + def test_lists_with_matcher(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("$$name") + + assert_that(is_match_tree(src, pattern), is_(True)) + + def test_lists_with_list_with_matcher_at_end(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("1\n2\n$$name") + + assert_that(is_match_tree(src, pattern, {}), is_(True)) + + def test_lists_with_list_with_matcher_at_start(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("$$name\n5\n6") + + assert_that(is_match_tree(src, pattern, {}), is_(True)) + + def test_lists_with_list_with_multi_single(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("$$name\n$name") + exp = {} + + assert_that(is_match_tree(src, pattern, exp)) + + assert_that(exp["$$name"], has_length(5)) + assert_that(exp["$name"], has_length(1)) + + def test_lists_with_list_with_list_multi_single(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("1\n2\n$$name\n$name") + exp = {} + + assert_that(is_match_tree(src, pattern, exp), is_(True)) + + assert_that(exp["$$name"], has_length(3)) + assert_that(exp["$name"], has_length(1)) + + def test_lists_with_list_with_matcher_in_the_middle(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("1\n$$name\n6") + + assert_that(is_match_tree(src, pattern, {}), is_(True)) + + def test_lists_with_list_with_matcher_in_both_end(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("$$start\n3\n$$end") + + assert_that(is_match_tree(src, pattern, {}), is_(True)) + + def test_lists_with_list_with_matcher_in_both_end_empty_list_at_start(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("$$start\n1\n$$end") + + assert_that(is_match_tree(src, pattern, {}), is_(True)) + + def test_lists_with_list_with_matcher_in_both_end_empty_list_at_the_end(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("$$start\n6\n$$end") + + assert_that(is_match_tree(src, pattern, {}), is_(True)) + + def test_lists_with_list_with_matcher_in_both_end__mismatch(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n61\n2\n3\n4\n5\n6") + pattern = self.pattern_factory.create_statements("$$seq\n61\n$$seq") + + assert_that(is_match_tree(src, pattern, {}), is_(False)) + + def test_lists_with_list_with_matcher_in_both_end_same_pattern(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n61\n2\n3\n4\n5") + pattern = self.pattern_factory.create_statements("$$seq\n61\n$$seq") + + assert_that(is_match_tree(src, pattern, {}), is_(False)) + + def test_lists_with_list_with_matcher_in_matcher_in_between(self): + src = self.pattern_factory.create_statements("2\n3\n4\n5\n61\n2\n3\n4\n5\n7\n8\n9") + pattern = self.pattern_factory.create_statements("$$seq\n61\n$$seq\n7\n8\n9") + + assert_that(is_match_tree(src, pattern, {}), is_(True)) + + def test_lists_with_list_with_matcher_in_matcher_in_between_but_has_leftover(self): + src = self.pattern_factory.create_statements("2\n3\n4\n5\n61\n2\n3\n4\n5\n7\n8\n9") + pattern = self.pattern_factory.create_statements("$$seq\n61\n$$seq") + + assert_that(is_match_tree(src, pattern, {}), is_(False)) + + def test_find_in_list(self): + src = self.pattern_factory.create_statements("2\n3\n4\n5\n61\n2\n3\n4\n5\n7\n8\n9") + pattern = self.pattern_factory.create_statements("2") + + assert_that(find_in_list(src, pattern, {}), is_(0)) + + def test_find_in_list_with_expansion(self): + src = self.pattern_factory.create_statements("2\n3\n4\n5\n61\n2\n3\n4\n5\n7\n8\n9") + pattern = self.pattern_factory.create_statements("2\n$3\n4") + exp = {} + + assert_that(find_in_list(src, pattern, exp), is_(2)) + assert_that(exp["$3"][0].name, is_("3")) + + def test_can_t_find_in_list(self): + src = self.pattern_factory.create_statements("2\n3\n4\n5\n61\n2\n3\n4\n5\n7\n8\n9") + pattern = self.pattern_factory.create_statements("1") + + assert_that(find_in_list(src, pattern, {}), less_than(0)) + + def test_find_in_list_returns_last_pos(self): + src = self.pattern_factory.create_statements("0\n1\n2\n3\n4\n5\n61\n2\n3\n4\n5\n7\n8\n9") + pattern = self.pattern_factory.create_statements("0\n1\n2\n3\n4\n5") + + assert_that(find_in_list(src, pattern, {}), is_(5)) + + def test_find_with_match_all_returns_last_pos(self): + src = self.pattern_factory.create_statements("0\n1\n2\n3\n4\n5\n61\n2\n3\n4\n5\n7\n8\n9") + pattern = self.pattern_factory.create_statements("0\n1\n2\n3\n4\n5\n$$seq") + + assert_that(find_in_list(src, pattern, {}), is_(len(src) - 1)) + + def test_lists_with_list_with_matcher_in_both_end_mismatch2(self): + src = self.pattern_factory.create_statements("1\n2\n3\n4\n5\n61\n2\n3\n4\n5") + pattern = self.pattern_factory.create_statements("$$seq\n61\n$$seq") + assert_that(is_match_tree(src, pattern, {}), is_(False)) + + def test_find_function_with_any_param_python(self): + atu = self.factory.create_from_text("ca(13,14,15)", "test.py") + src = atu.children + pattern = self.pattern_factory.create_statements("ca($$all)") + assert_that(find_in_list(src, pattern, {}), is_(0)) + + def test_find_function_with_any_param_and_all_param_in_python(self): + atu = self.factory.create_from_text("ca(13,14,15)", "test.py") + src = atu.children + pattern = self.pattern_factory.create_statements("$f($a,$$all)") + assert_that(find_in_list(src, pattern, {}), is_(0)) + + def test_match_all_function_with_any_param_clang(self): + factory = ASTFactory(ClangASTNode, []) + atu = factory.create_from_text("void ca(int a,int b,int c){ca(13,14,15); ca(13,14,15);}", "fut.c") + src = atu.children[-1].children[-1].children + pattern = factory.create_from_text("int $a,$$all;void $f(int a,int b){$f($a, $$all);}", "pat.c").children[-1].children[-1].children + assert_that(match_pattern(src, pattern), has_length(2)) + + def test_find_all_in_list_with_expansion(self): + src = self.pattern_factory.create_statements("2\n3\n4\n5\n61\n2\n3\n4\n5\n7\n8\n9") + pattern = self.pattern_factory.create_statements("2\n$3\n4") + matches = match_pattern(src, pattern) + assert_that(matches, has_length(2)) + assert_that(matches[0].expansions["$3"][0].name, is_("3")) + + def test_find_all_in_python_list_with_expansion(self): + atu = self.factory.create_from_text( + textwrap.dedent(""" + from unittest import TestCase + + class TestExample(TestCase): + def test_case_example(self): + # arrange + factory = {} + + # act + factory['a']= 1 + + # assert + self.assertEqual(len(factory), 1) + """), + "test_file.py", + ) + pattern = self.pattern_factory.create_statement("class $name(TestCase):\n $$cases") + ASTShower.show_node(pattern) + expansions ={} + variants = variant_in_match_stmt(atu.children[-1],pattern, expansions) + single = is_match(atu.children[-1],pattern) + matches = match_pattern([atu.children[-1]], [pattern]) + assert_that(matches, has_length(1)) + assert_that(matches[0].expansions["$name"][0], is_("TestExample")) + + def test_find_all_in_python_arg_list_with_expansion1(self): + atu = self.factory.create_from_text("class klass: pass", "test_file.py") + statement = self.pattern_factory.create_statements("assertEqual(1,2,34,5,6,7,7,8)") + pattern = self.pattern_factory.create_statements("assertEqual($$args)") + matches = match_pattern(statement, pattern) + assert_that(matches, has_length(1)) + assert_that(matches[0].expansions["$$args"], is_not(empty())) + + def test_find_all_in_python_arg_list_with_expansion2(self): + atu = self.factory.create_from_text("class klass:\n def fun(a,b,c,d,f): pass", "test_file.py") + pattern = self.pattern_factory.create_statements("def fun($$args): pass") + matches = match_pattern(atu.children, pattern) + assert_that(matches, has_length(1)) + assert_that(matches[0].expansions["$$args"], is_not(empty())) + + def test_find_all_in_clang_list_with_expansion(self): + factory = ASTFactory(ClangASTNode, []) + pattern = CPatternFactory(factory).create_statements("a == $x;") + src = CPatternFactory(factory).create_statements("a == 3;a == 4; b == 5;") + matches = match_pattern(src, pattern) + assert_that(matches, has_length(2)) + assert_that(matches[0].expansions["$x"], is_not(empty())) + + @pytest.mark.skip + def test_match_one_and_all_params(self): + sample = textwrap.dedent(""" + context_stub=0 + EMRMxAPxData_data_rep = 0 + class SomeTest: + def setUp(self): + [].append( + TAUT.TestDoubles(module=EMRMxAPxData_data_rep, context=context_stub) + ) + """) + atu = self.factory.create_from_text(sample, "sample.py") + ASTShower.show_node(atu) + kwargs = self.pattern_factory.create_kwargs("$c=context_stub") + matches = MatchFinder.match_pattern(atu.children, kwargs) + assert_that(matches, has_length(1)) + + def test_match_pattern_for_parameterized_finds_one_match1(self): + code = textwrap.dedent(""" + from parameterized import parameterized + + class TestASTReference: + + @parameterized.expand(Factories.extend()) + def test_definition_declaration_references(self, _, factory, code, *args): + pass + """) + atu = self.factory.create_from_text(code) + unittest = self.pattern_factory.create_statements("@parameterized.expand($$parameters)\ndef $fun($$args, *$$vargs):\n $$stmts") + found = match_pattern(atu.children, unittest) + assert_that(found, has_length(1)) + + def test_match_pattern_for_parameterized_finds_one_match2(self): + code = textwrap.dedent(""" + from parameterized import parameterized + + class TestASTReference: + + @parameterized.expand(Factories.extend()) + def test_definition_declaration_references(self, _, factory, code, *args): + pass + """) + atu = self.factory.create_from_text(code) + unittest = self.pattern_factory.create_statements("@parameterized.expand($$parameters)\ndef $fun($$args, *$$vargs):\n $$stmts") + found = list(match_pattern(atu.children, unittest)) + assert_that(found, has_length(1)) diff --git a/test/syntax_tree/test_pattern_match.py b/test/syntax_tree/test_pattern_match.py new file mode 100644 index 00000000..66592cab --- /dev/null +++ b/test/syntax_tree/test_pattern_match.py @@ -0,0 +1,101 @@ +import ast +import textwrap + +import pytest +from hamcrest import assert_that, is_, empty, has_length + +from renaissance.impl.python.factory import PythonFactory, PythonPatternFactory +from renaissance.impl.python.rst_node import PythonRstNode + +from renaissance.syntax_tree import PatternMatch +from renaissance.syntax_tree.match_finder import match_pattern + + +class TestPatternMatch: + + @pytest.fixture(autouse=True) + def setup(self): + self.factory = PythonFactory(PythonRstNode) + self.pattern_factory = PythonPatternFactory(self.factory) + + @pytest.mark.skip("length on empty node") + def test_empty_expansion_has_offset(self): + example_code = textwrap.dedent(""" + 1 + 2 + 3 + 4 + 5 + 6 + 7 + """) + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("2\n\n$$empty\n3") + found = match_pattern(atu.children, pattern) + assert_that(found, has_length(1)) + match = found[0] + assert_that(match["$$empty"], is_("")) + assert_that(match.expansions["$$empty"], is_(empty())) + assert_that(match.offset_of("$$empty"), is_(5)) + assert_that(match.length_of("$$empty"), is_(0)) + + def test_single_expansion_has_offset(self): + example_code = textwrap.dedent(""" + 1 + 2 + 3 + 4 + 5 + 6 + 7 + """) + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("2\n\n$3\n4") + found = match_pattern(atu.children, pattern) + assert_that(found, has_length(1)) + match = found[0] + assert_that(match["$3"], is_("3")) + assert_that(match.expansions["$3"], has_length(1)) + assert_that(match.offset_of("$3"), is_(5)) + assert_that(match.length_of("$3"), is_(1)) + + def test_multi_expansion_has_offset(self): + example_code = textwrap.dedent(""" + 1 + 2 + 3 + 4 + 5 + 6 + 7 + """) + atu = self.factory.create_from_text(example_code) + pattern = self.pattern_factory.create_statements("1\n\n$$other\n6") + found = match_pattern(atu.children, pattern) + assert_that(found, has_length(1)) + match = found[0] + assert_that(match["$$other"], is_("2\n3\n4\n5")) + assert_that(match.expansions["$$other"], has_length(4)) + assert_that(match.offset_of("$$other"), is_(3)) + assert_that(match.length_of("$$other"), is_(7)) + + def test_match_referenced_by(self, mocker): + node = mocker.Mock() + reference = mocker.Mock() + node.referenced_by = [reference, reference] + reference.node = node + pattern_match = PatternMatch([node, node, node], {}, []) + mock_matcher = mocker.patch( + "renaissance.syntax_tree.match_finder.MatchFinder.match_pattern", + return_value=[pattern_match], + ) + pattern_match.match_referenced_by([[node]], False) + assert_that(mock_matcher.call_count, is_(6)) + + def test_get_key_redirect_to_expansion_signature(self, mocker): + node = mocker.Mock() + node.signature = "name_1" + pattern_match = PatternMatch([], {"key": ["name_1"], "$node": [PythonRstNode(ast.Name("node_name"))], "empty": []}, "patterns") + assert_that(pattern_match["key"], is_("name_1")) + assert_that(pattern_match["$node"], is_("node_name")) + assert_that(pattern_match["empty"], is_("")) diff --git a/test/syntax_tree/test_recipe_ast_processor.py b/test/syntax_tree/test_recipe_ast_processor.py new file mode 100644 index 00000000..697415bf --- /dev/null +++ b/test/syntax_tree/test_recipe_ast_processor.py @@ -0,0 +1,79 @@ +from hamcrest import assert_that, is_, has_length + +from renaissance.syntax_tree.recipe_ast_processor import ( + RecipeASTProcessor, + recipe_step, + final_action, + BatchASTProcessor, + annotate_decorator, + get_methods_with_decorator, +) + + +class TestRecipeASTProcessor: + def test_receipe_proc(self): + it = RecipeASTProcessor(lambda n: n, lambda: (), "") + assert_that(it, is_(RecipeASTProcessor)) + + def test_run(self, mocker): + # define a simple recipe class with one recipe_step + class SimpleRecipe: + def __init__(self): + self.ran = [] + + @recipe_step(order=0) + def do_step(self, _): + def work(): + self.ran.append("done") + + return work + + # patch BatchASTProcessor.repeat to immediately invoke actions with a dummy ASTProcessor + def fake_repeat(_, _1, actions, _2): + dummy = mocker.Mock() + dummy.repeat_step = 0 + for action in actions: + action(dummy) + + recipe = SimpleRecipe() + iterable_provider = lambda: [] + + mocker.patch.object(BatchASTProcessor, "repeat", new=fake_repeat) + + processor = RecipeASTProcessor(recipe, iterable_provider, "") + processor.run() + + assert_that(recipe.ran, is_(["done"])) + + def test_annotate_decorator(self): + foreign = lambda f: f + decorator = annotate_decorator(foreign, "test_decorator") + # the returned decorator keeps the foreign decorator's __name__ + assert_that(decorator.__name__, is_(foreign.__name__)) + + # when applied to a function, the decorator attaches the recipe_action name + @decorator + def sample(): + return 1 + + assert_that(sample.recipe_action, is_("test_decorator")) + + def test_get_methods_with_decorator(self): + class Sample: + @recipe_step() + def step1(self): + pass + + methods = list(get_methods_with_decorator(Sample, recipe_step)) + assert_that(methods, has_length(1)) + assert_that(methods[0].__name__, is_("step1")) + + def test_final_action(self): + class Sample: + @final_action() + def final(self): + pass + + methods = list(get_methods_with_decorator(Sample, final_action)) + assert_that(methods, has_length(1)) + assert_that(methods[0].__name__, is_("final")) diff --git a/test/syntax_tree/test_syntax_node.py b/test/syntax_tree/test_syntax_node.py new file mode 100644 index 00000000..de3ff557 --- /dev/null +++ b/test/syntax_tree/test_syntax_node.py @@ -0,0 +1,226 @@ +from dataclasses import dataclass, field +from typing import Any, Self +import pytest + +import test.syntax_tree.infra_syntax_node +import test.syntax_tree.infra_text_segment + + +def _offset_to_line_col(text: str, offset: int) -> tuple[int, int]: + """0-based (line, column) for a 0-based offset; offset may be len(text).""" + assert 0 <= offset <= len(text) + line = text.count("\n", 0, offset) + last_nl = text.rfind("\n", 0, offset) + col = offset if last_nl == -1 else offset - (last_nl + 1) + return line, col + + +@dataclass(slots=True) +class DummyNode: + # ---- backing text segment ---- + full_text: str + location: str + start_offset: int + end_offset: int + + # ---- syntax node aspects ---- + kind: str = "Dummy" + _children: list[Self] = field(default_factory=list) # type: ignore + _parent: Self | None = None + + # ---- TextSegment derived properties ---- + @property + def start_line(self) -> int: + return _offset_to_line_col(self.full_text, self.start_offset)[0] + + @property + def start_column(self) -> int: + return _offset_to_line_col(self.full_text, self.start_offset)[1] + + @property + def end_line(self) -> int: + return _offset_to_line_col(self.full_text, self.end_offset)[0] + + @property + def end_column(self) -> int: + return _offset_to_line_col(self.full_text, self.end_offset)[1] + + @property + def text_segment(self) -> str: + return self.full_text[self.start_offset : self.end_offset] + + # ---- SyntaxNode protocol properties ---- + @property + def children(self) -> list[Self]: + return self._children + + @property + def syntax_attributes(self) -> dict[str, Any]: + return {} + + @property + def parent(self) -> Self | None: + return self._parent + + @property + def original_node(self) -> Self: + return self + + # ---- safe mutator for tests (avoids "protected access" warnings) ---- + def set_children(self, children: list[Self]) -> None: + self._children = children + for c in children: + c._parent = self + + def __hash__(self): + return id(self) + + +# ---------------------------- +# Monkeypatch: verify assert_valid_text_segment is called +# ---------------------------- + + +class _SegmentCallCounter: + def __init__(self) -> None: + self.calls: list[Any] = [] + + def __call__(self, seg: Any) -> None: + self.calls.append(seg) + + +@pytest.fixture +def segment_validator_counter(monkeypatch: pytest.MonkeyPatch) -> _SegmentCallCounter: + counter = _SegmentCallCounter() + monkeypatch.setattr(test.syntax_tree.infra_text_segment, "assert_valid_text_segment", counter) + return counter + + +# ---------------------------- +# Success cases +# ---------------------------- +@pytest.mark.skip("result is empty") +def test_assert_valid_syntax_node_ok(segment_validator_counter: _SegmentCallCounter) -> None: + text = "ab\ncd\nef" + loc = "mem://t" + + root: DummyNode = DummyNode(text, loc, 0, len(text), kind="Root") + c0 = DummyNode(text, loc, 0, 3, kind="L0") # "ab\n" + c1 = DummyNode(text, loc, 3, 6, kind="L1") # "cd\n" + c2 = DummyNode(text, loc, 6, 8, kind="L2") # "ef" + + root.set_children([c0, c1, c2]) + + test.syntax_tree.infra_syntax_node.assert_valid_syntax_node(root) + + assert segment_validator_counter.calls == [root, c0, c1, c2] + + +@pytest.mark.skip("result is empty") +def test_assert_valid_syntax_tree_ok(segment_validator_counter: _SegmentCallCounter) -> None: + text = "ab\ncd\nef" + loc = "mem://t" + + root: DummyNode = DummyNode(text, loc, 0, len(text), kind="Root") + mid = DummyNode(text, loc, 0, 6, kind="Mid") + leaf0 = DummyNode(text, loc, 0, 3, kind="Leaf0") + leaf1 = DummyNode(text, loc, 3, 6, kind="Leaf1") + + root.set_children([mid]) + mid.set_children([leaf0, leaf1]) + + test.syntax_tree.infra_syntax_node.assert_valid_syntax_tree(root) + + assert set(segment_validator_counter.calls) >= {root, mid, leaf0, leaf1} + + +# ---------------------------- +# Failure modes (node-level) +# ---------------------------- + + +def test_children_must_be_ordered_by_start_offset(segment_validator_counter: _SegmentCallCounter) -> None: + text = "abcdef" + loc = "mem://t" + + root: DummyNode = DummyNode(text, loc, 0, 6, kind="Root") + a = DummyNode(text, loc, 2, 3, kind="A") + b = DummyNode(text, loc, 1, 2, kind="B") + + root.set_children([a, b]) + + with pytest.raises(AssertionError, match=r"ordered by non-decreasing start_offset"): + test.syntax_tree.infra_syntax_node.assert_valid_syntax_node(root) + + +def test_children_must_not_overlap(segment_validator_counter: _SegmentCallCounter) -> None: + text = "abcdef" + loc = "mem://t" + + root: DummyNode = DummyNode(text, loc, 0, 6, kind="Root") + a = DummyNode(text, loc, 1, 4, kind="A") + b = DummyNode(text, loc, 3, 5, kind="B") + + root.set_children([a, b]) + + with pytest.raises(AssertionError, match=r"must not overlap"): + test.syntax_tree.infra_syntax_node.assert_valid_syntax_node(root) + + +def test_child_must_be_within_parent_span(segment_validator_counter: _SegmentCallCounter) -> None: + text = "abcdef" + loc = "mem://t" + + root: DummyNode = DummyNode(text, loc, 1, 5, kind="Root") + child = DummyNode(text, loc, 0, 2, kind="Bad") + + root.set_children([child]) + + with pytest.raises(AssertionError, match=r"start_offset must lie within the node span"): + test.syntax_tree.infra_syntax_node.assert_valid_syntax_node(root) + + +def test_child_must_point_back_to_parent(segment_validator_counter: _SegmentCallCounter) -> None: + text = "abcdef" + loc = "mem://t" + + root: DummyNode = DummyNode(text, loc, 0, 6, kind="Root") + child = DummyNode(text, loc, 0, 1, kind="Child") + + # Intentionally wrong: do not use set_children; parent stays None + root._children = [child] # type: ignore + + with pytest.raises(AssertionError, match=r"parent must be the node itself"): + test.syntax_tree.infra_syntax_node.assert_valid_syntax_node(root) + + +def test_child_must_share_text_and_location(segment_validator_counter: _SegmentCallCounter) -> None: + text = "abcdef" + loc = "mem://t" + + root: DummyNode = DummyNode(text, loc, 0, 6, kind="Root") + child = DummyNode("DIFFERENT", loc, 0, 1, kind="Child") + + root.set_children([child]) + + with pytest.raises(AssertionError, match=r"full_text must equal node\.full_text"): + test.syntax_tree.infra_syntax_node.assert_valid_syntax_node(root) + + +# ---------------------------- +# Failure modes (tree-level) +# ---------------------------- + + +def test_assert_valid_syntax_tree_detects_cycle(segment_validator_counter: _SegmentCallCounter) -> None: + text = "abc" + loc = "mem://t" + + root: DummyNode = DummyNode(text, loc, 0, 1, kind="Root") + child = DummyNode(text, loc, 0, 1, kind="Child") + + root.set_children([child]) + child.set_children([root]) # cycle + + with pytest.raises(AssertionError, match=r"same node object twice"): + test.syntax_tree.infra_syntax_node.assert_valid_syntax_tree(root) diff --git a/test/syntax_tree/test_text_segment.py b/test/syntax_tree/test_text_segment.py new file mode 100644 index 00000000..6e5bc7bf --- /dev/null +++ b/test/syntax_tree/test_text_segment.py @@ -0,0 +1,356 @@ +import pytest +import bisect + +from hypothesis import given, strategies as st + +from renaissance.syntax_tree.text_segment import TextSegment +from test.syntax_tree.infra_text_segment import ( + assert_valid_text_segment, + line_starts_from_lines, + location_to_offset, + offset_to_location, + split_lines_with_newlines, +) + + +class AutoTextSegment: + """ + Reference test implementation: + Construct with offsets; derive (line, column) from full_text boundaries. + """ + + def __init__( + self, + full_text: str, + start_offset: int, + end_offset: int, + location: str = "<memory>", + ) -> None: + self._full_text = full_text + self._location = location + self._start_offset = start_offset + self._end_offset = end_offset + + def offset_to_line_col(off: int) -> tuple[int, int]: + # Map an offset (slice boundary) to (line, column), both 0-based. + # This supports off in [0, len(full_text)]. + line = 0 + line_start = 0 + + # Scan characters strictly before 'off' + for i, ch in enumerate(full_text): + if i >= off: + break + if ch == "\n": + line += 1 + line_start = i + 1 + + col = off - line_start + return line, col + + self._start_line, self._start_column = offset_to_line_col(start_offset) + self._end_line, self._end_column = offset_to_line_col(end_offset) + + # --- Protocol properties --- + @property + def full_text(self) -> str: + return self._full_text + + @property + def location(self) -> str: + return self._location + + @property + def start_line(self) -> int: + return self._start_line + + @property + def start_column(self) -> int: + return self._start_column + + @property + def start_offset(self) -> int: + return self._start_offset + + @property + def end_line(self) -> int: + return self._end_line + + @property + def end_column(self) -> int: + return self._end_column + + @property + def end_offset(self) -> int: + return self._end_offset + + @property + def text_segment(self) -> str: + return self._full_text[self._start_offset : self._end_offset] + + +class MissingProperty: + """Deliberately does NOT satisfy the protocol structurally.""" + + @property + def full_text(self) -> str: + return "x" + + +class BadTypesButProtocolLike: + """ + Has all required attributes/properties so runtime protocol check passes, + but types are wrong -> assert_valid_text_segment should fail. + """ + + @property + def full_text(self): # not str + return 123 + + @property + def location(self): # not str + return None + + @property + def start_line(self): # not int + return "hello" + + @property + def start_column(self): # not int + return "hello" + + @property + def start_offset(self): # not int + return "hello" + + @property + def end_line(self): # not int + return "hello" + + @property + def end_column(self): # not int + return "hello" + + @property + def end_offset(self): # not int + return "hello" + + @property + def text_segment(self): # not str + return 456 + + +class InconsistentTextSlice(AutoTextSegment): + """Matches all numbers, but lies about text_segment.""" + + @property + def text_segment(self) -> str: + return "NOT THE SLICE" + + +class InconsistentOffsets(AutoTextSegment): + """Offsets present, but line/col purposely inconsistent with boundaries.""" + + def __init__( + self, + full_text: str, + start_offset: int, + end_offset: int, + location: str = "<memory>", + ) -> None: + super().__init__(full_text, start_offset, end_offset, location) + # break consistency intentionally + self._start_column += 1 + + +# --------------------------- +# Structural protocol tests +# --------------------------- + + +def test_runtime_checkable_protocol_accepts_structural_implementation() -> None: + seg = AutoTextSegment("abc", 0, 1) + assert isinstance(seg, TextSegment) + + +def test_runtime_checkable_protocol_rejects_missing_members() -> None: + seg = MissingProperty() + assert not isinstance(seg, TextSegment) + + +# --------------------------- +# Positive semantic tests +# --------------------------- + + +@pytest.mark.parametrize( + "text,start,end,expected_slice", + [ + ("", 0, 0, ""), # empty full text + ("a", 0, 1, "a"), # segment equal full text + ("\n", 0, 1, "\n"), # empty line + ("\n", 1, 1, ""), # empty last line + ("ab", 0, 0, ""), # empty slice before text + ("ab", 1, 1, ""), # empty slice inside text + ("ab", 2, 2, ""), # empty slice after text + ("abc", 0, 1, "a"), + ("abc", 1, 2, "b"), + ("abc", 2, 3, "c"), + ("abc", 0, 3, "abc"), # segment is full text + ("ab\ncd\nef", 3, 6, "cd\n"), # segment is second line + ("ab\ncd\nef", 1, 5, "b\ncd"), # spans newline and into next line + ("ab\ncd\nef", 2, 3, "\n"), # selects newline character + ("ab\ncd\nef", 3, 4, "c"), # beginning of line 1 + ("ab\ncd\nef", 6, 8, "ef"), # last line + ], +) +def test_assert_valid_text_segment_accepts_semantically_correct_segments(text: str, start: int, end: int, expected_slice: str) -> None: + seg = AutoTextSegment(text, start, end) + assert seg.text_segment == expected_slice + assert_valid_text_segment(seg) + + +# --------------------------- +# Negative semantic tests +# --------------------------- + + +def test_validator_rejects_wrong_types_even_if_protocol_like() -> None: + seg = BadTypesButProtocolLike() + # runtime protocol check likely passes (structural), but validator must fail + assert isinstance(seg, TextSegment) + with pytest.raises( + AssertionError, + match=r"^Unexpected instance for property full_text '.*'\. Expected 'str'\.$", + ): + assert_valid_text_segment(seg) + + +def test_validator_rejects_start_offset_greater_than_end_offset() -> None: + seg = AutoTextSegment("abc", 2, 1) + with pytest.raises(AssertionError, match=r"^Property end_offset before start_offset: \d+ < \d+$"): + assert_valid_text_segment(seg) + + +def test_validator_rejects_offsets_out_of_range() -> None: + seg = AutoTextSegment("abc", 0, 4) + with pytest.raises( + AssertionError, + match=r"^Property end_offset out of range: \d+ not in \[0, \d+\]$", + ): + assert_valid_text_segment(seg) + + +def test_validator_rejects_inconsistent_text_segment_slice() -> None: + seg = InconsistentTextSlice("ab\ncd", 0, 2) + with pytest.raises( + AssertionError, + match="text_segment and full_text\\[start_offset:end_offset\\] are inconsistent", + ): + assert_valid_text_segment(seg) + + +def test_validator_rejects_inconsistent_offset_and_line_column() -> None: + seg = InconsistentOffsets("ab\ncd", 0, 2) + with pytest.raises(AssertionError, match="Start offset and \\(line, column\\) are inconsistent"): + assert_valid_text_segment(seg) + + +def test_validator_rejects_line_out_of_range() -> None: + # Build a protocol-like object but with bogus line indices + class BogusLine(AutoTextSegment): + @property + def start_line(self) -> int: + return 999 + + seg = BogusLine("ab\ncd", 0, 1) + with pytest.raises(AssertionError, match=r"^Property end_line before start_line: \d+ < \d+$"): + assert_valid_text_segment(seg) + + +def test_validator_rejects_column_out_of_range() -> None: + class BogusColumn(AutoTextSegment): + @property + def start_column(self) -> int: + return 999 + + seg = BogusColumn("ab\ncd", 0, 1) + with pytest.raises( + AssertionError, + match=r"^Property end_column before start_column, while start and end line are the same: \d+ < \d+$", + ): + assert_valid_text_segment(seg) + + +# ----------------------------- +# Hypothesis strategies +# ----------------------------- + +# strategy to generate lines of text (with newline character) +text_line_strategy = st.text( + alphabet=st.characters(blacklist_categories=("Cs",), blacklist_characters=["\n"]), + min_size=0, + max_size=25, +) + +# strategy to generate text +# biased to contain multiple lines +# biased to end with \n +text_strategy = st.one_of( + st.text(alphabet=st.characters(blacklist_categories=("Cs",)), max_size=200), + st.lists(text_line_strategy, min_size=1, max_size=20).map("\n".join), + st.lists(text_line_strategy, min_size=1, max_size=20).map("\n".join).map(lambda s: s + "\n"), +) + + +# ----------------------------- +# Property tests +# ----------------------------- + + +@given(text=text_strategy) +def test_roundtrip_offset_loc_offset_for_all_cursor_offsets(text: str) -> None: + """ + For all cursor offsets in [0, len(text)], converting + offset -> (line, col) -> offset + yields the original offset. + + This includes offset == len(text), which is essential for half-open ranges. + """ + for offset in range(len(text) + 1): + line, col = offset_to_location(text, offset) + offset2 = location_to_offset(text, line, col) + assert offset2 == offset + + +@given(text=text_strategy) +def test_offset_to_loc_corresponds_to_split_lines_extended_with_newlines( + text: str, +) -> None: + """ + For all cursor offsets in [0, len(text)], offset_to_loc matches the location + computed from: + parts = split(text, '\n') + lines = parts[:-1] + '\n' + parts[-1] (i.e., all but last extended with '\n') + with canonical newline-boundary ownership: + the cursor position after '\n' is (next_line, 0). + """ + lines = split_lines_with_newlines(text) + assert "".join(lines) == text # sanity + + starts = line_starts_from_lines(lines) + + for offset in range(len(text) + 1): + # Determine the (line, col) in the split-derived model. + # Use bisect_right so that exact line starts map to that line (canonical). + line = bisect.bisect_right(starts, offset) - 1 + col = offset - starts[line] + + # Canonicalization check: + # If we're at the end boundary of a newline-terminated line (col == len(line_span)), + # then the canonical representation should be (next_line, 0) (unless there is no next line). + if line < len(lines) - 1 and lines[line].endswith("\n") and col == len(lines[line]): + line += 1 + col = 0 + + assert (line, col) == offset_to_location(text, offset) diff --git a/test/test_data/test_class.py b/test/test_data/test_class.py new file mode 100644 index 00000000..56d1be39 --- /dev/null +++ b/test/test_data/test_class.py @@ -0,0 +1,224 @@ +change_comment = """#!/usr/bin/env python +# -----------------------------------------------------------------------------# +# # +# Python script # +# # +# -----------------------------------------------------------------------------# +# +# Ident : ABCD_utils.py +# Description : Utility functions for unittest +# +# History +# 2016-03-31 : xx +# 2016-06-01 : yy +# 2016-08-10 : zz +# -----------------------------------------------------------------------------# +# # +# Copyright (c) 2016, ABCD Netherlands B.V. # +# All rights reserved # +# # +# -----------------------------------------------------------------------------# + +import inspect +""" + +new_change_comment = """#!/usr/bin/env python +# -----------------------------------------------------------------------------# +# # +# Python script # +# # +# -----------------------------------------------------------------------------# +# +# Ident : ABCD_utils.py +# Description : Utility functions for unittest +# +# History +# 2016-03-31 : xx +# 2016-06-01 : yy +# 2016-08-10 : zz +# 2026-01-22 : uu +# -----------------------------------------------------------------------------# +# # +# Copyright (c) 2016, ABCD Netherlands B.V. # +# All rights reserved # +# # +# -----------------------------------------------------------------------------# + +import inspect +""" +set_up = """ +def setUp(self): + self._patch_readout_data_filler = mock.patch("ACBD_ReadoutDataFiller.ReadoutDataFiller") + self._patch_readout_data_publisher = mock.patch("ACBD_ReadoutDataPublisher.ReadoutDataPublisher") + self._patch_dt_context_rep = mock.patch("ACBDxRepUtils.DPxCONTEXT") + self._patch_dtxa_context_rep = mock.patch("ACBDxRepUtils.DTXAxCONTEXT") + self._patch_dt_context_filler = mock.patch("ACBD_ReadoutDataFiller.DPxCONTEXT") + self._patch_dtxa_context_filler = mock.patch("ACBD_ReadoutDataFiller.DTXAxCONTEXT") + + _ = self._patch_readout_data_filler.start() + _ = self._patch_readout_data_publisher.start() + _ = self._patch_dtxa_context_rep.start() + _ = self._patch_dtxa_context_filler.start() + mock_dt_context_rep = self._patch_dt_context_rep.start() + mock_dt_context_filler = self._patch_dt_context_filler.start() + + mock_dt_context_rep.lookup_instance.return_value = (True, 1) + mock_dt_context_filler.lookup_instance.return_value = (True, 2) + + ACBDxAPxData.data = ACBDxAPxData.ACBDxAPxData() + ACBDxAPxData.data.initialize_engine(ACBDxEngine.ACBDxEngine()) + self.engine_stub = ACBDxEngine_stub() + self.wh_stub = EMxWLxCTL_stub() + self.vipr_stub = VIPS_stub() + self.doubles = [] + + context_stub = ACBDxCONTEXT.ACBDxCONTEXTStub() + self.doubles.append(TAUT.TestDoubles(acbdxcontext=context_stub)) + self.doubles.append( + TAUT.TestDoubles(module=ACBDxAPxData.data.rep, context=context_stub) + ) + self.doubles.append( + TAUT.TestDoubles( + module=EMxWLxCTL.EMxWLxCTL, reload_wafer=self.wh_stub.reload_wafer + ) + ) + self.doubles.append( + TAUT.TestDoubles( + module=ACBDxEngine.ACBDxEngine, + measure_wafer=self.engine_stub.measure_wafer_gw, + ) + ) + self.doubles.append( + TAUT.TestDoubles(module=VIPS, check_stopped=self.vipr_stub.check_stopped) + ) + + ACBDxAPxData.data.adv_wp = ACBDxADVxWP.input() + self.measurement_strategy = ACBD_MeasurementDefault.ACBD_MeasurementDefault() +""" +new_set_up = """ +def setUp(self): + self._patch_readout_data_filler = mock.patch("ACBD_ReadoutDataFiller.ReadoutDataFiller") + self._patch_readout_data_publisher = mock.patch("ACBD_ReadoutDataPublisher.ReadoutDataPublisher") + self._patch_dt_context_rep = mock.patch("ACBDxRepUtils.DPxCONTEXT") + self._patch_dtxa_context_rep = mock.patch("ACBDxRepUtils.DTXAxCONTEXT") + self._patch_dt_context_filler = mock.patch("ACBD_ReadoutDataFiller.DPxCONTEXT") + self._patch_dtxa_context_filler = mock.patch("ACBD_ReadoutDataFiller.DTXAxCONTEXT") + + _ = self._patch_readout_data_filler.start() + _ = self._patch_readout_data_publisher.start() + _ = self._patch_dtxa_context_rep.start() + _ = self._patch_dtxa_context_filler.start() + mock_dt_context_rep = self._patch_dt_context_rep.start() + mock_dt_context_filler = self._patch_dt_context_filler.start() + + mock_dt_context_rep.lookup_instance.return_value = (True, 1) + mock_dt_context_filler.lookup_instance.return_value = (True, 2) + + ACBDxAPxData.data = ACBDxAPxData.ACBDxAPxData() + ACBDxAPxData.data.initialize_engine(ACBDxEngine.ACBDxEngine()) + self.engine_stub = ACBDxEngine_stub() + self.wh_stub = EMxWLxCTL_stub() + self.vipr_stub = VIPS_stub() + self.patches = [] + + self.context_stub = ACBDxCONTEXT.ACBDxCONTEXTStub() + self.patches.append(patch('ACBDxCONTEXT.acbdxcontext', self.context_stub)) + self.patches.append(patch.object(ACBDxAPxData.data.rep, 'context', self.context_stub)) + self.patches.append(patch.object(EMxWLxCTL.EMxWLxCTL, 'reload_wafer', self.wh_stub.reload_wafer)) + self.patches.append(patch.object(ACBDxEngine.ACBDxEngine, 'measure_wafer', self.engine_stub.measure_wafer_gw)) + self.patches.append(patch.object(VIPS, 'check_stopped', self.vipr_stub.check_stopped)) + for p in self.patches: + p.start() + + ACBDxAPxData.data.adv_wp = ACBDxADVxWP.input() + self.measurement_strategy = ACBD_MeasurementDefault.ACBD_MeasurementDefault() +""" + +tear_down_simple = """ +def tearDown(self): + for double in self.doubles: + double.exit() +""" +tear_down_simple_new = """ +def tearDown(self): + for p in self.patches: + p.stop() +""" +tear_down = """ +def tearDown(self): + self._patch_readout_data_filler.stop() + self._patch_readout_data_publisher.stop() + self._patch_dt_context_rep.stop() + self._patch_dtxa_context_rep.stop() + self._patch_dt_context_filler.stop() + self._patch_dtxa_context_filler.stop() + for double in self.doubles: + double.exit() +""" + +new_tear_down = """ +def tearDown(self): + ABCDxCONTEXT.abcdxcontext.reset_method_attributes("start_wafer") + ABCDxCONTEXT.abcdxcontext.reset_method_attributes("finish_wafer") + ABCDxCONTEXT.abcdxcontext.reset_method_attributes("start_lot") + ABCDxCONTEXT.abcdxcontext.reset_method_attributes("finish_lot") + + self._patch_readout_data_filler.stop() + self._patch_readout_data_publisher.stop() + self._patch_dt_context_rep.stop() + self._patch_dtxa_context_rep.stop() + self._patch_dt_context_filler.stop() + self._patch_dtxa_context_filler.stop() + patch.stopall() +""" + +set_up_common = """def setUpCommon(self): + self.tds = [ + TestDoubles(abcdxread=ImprovedStub(ABCDxREAD.abcdxread)), + TestDoubles(abcdxws=ImprovedStub(ABCDxWS.abcdxws)), + TestDoubles(abxstream2=ImprovedStub(ABxSTREAM2.abxstream2)), + TestDoubles(bcxclear=ImprovedStub(BCxCLEAR.bcxclear)), + TestDoubles(bcxload=ImprovedStub(BCxLOAD.bcxload)) + ] + self.sut = ABCDxVIPCxAB.ABCDxVIPCxAB()""" + +set_up_common_new = """def setUpCommon(self): + ImprovedStub.ret_vals = {} + ImprovedStub.ret_vals_ex = {} + ImprovedStub.call_logs = {} + ImprovedStub.store_args = {} + + self.abcdxread = ImprovedStub(ABCDxREAD.abcdxread) + self.abcdxws = ImprovedStub(ABCDxWS.abcdxws) + self.abxstream2 = ImprovedStub(ABxSTREAM2.abxstream2) + self.bcxclear = ImprovedStub(BCxCLEAR.bcxclear) + self.bcxload = ImprovedStub(BCxLOAD.bcxload) + self.patchers = [ + patch.object(ABCDxREAD, 'abcdxread', self.abcdxread), + patch.object(ABCDxWS, 'abcdxws', self.abcdxws), + patch.object(ABxSTREAM2, 'abxstream2', self.abxstream2), + patch.object(BCxCLEAR, 'bcxclear', self.bcxclear), + patch.object(BCxLOAD, 'bcxload', self.bcxload), + ] + + for p in self.patchers: + p.start() + + self.sut = ABCDxVIPCxAB.ABCDxVIPCxAB()""" + +tear_down_common = """def tearDownCommon(self): + for td in self.tds: + td.exit()""" +tear_down_common_new = """def tearDownCommon(self): + for p in self.patchers: + try: + p.stop() + except RuntimeError: + pass +""" + +insert_add_patcher = """ +def add_patcher(self, target, name, replacement): + p = patch.object(target, name, replacement) + p.start() + self.patchers.append(p)""" diff --git a/test/test_data/test_code.py b/test/test_data/test_code.py new file mode 100644 index 00000000..659f18aa --- /dev/null +++ b/test/test_data/test_code.py @@ -0,0 +1,22 @@ +taut_code = """ +def test_functions(self): + with TAUT.TestDoubles(abcdxtl=FakeABCDxTL(None)): + log = TAUT.Logger() + test_log_id = BBAA.Object('a') + test_log = abcdxtl.create_test_log(test_log_id) + + file_id = BBAA.Object('b') + file_name = BBAA.Object('c') + test_log, version_mismatch = abcdxtl.retrieve_test_log(file_id, test_log_id, file_name) + abcdxtl.store_test_log(file_id, test_log) +""" +result_code = """ +def test_functions(self): + fake_abcdxtl = FakeABCDxTL(None) + test_log_id = BBAA.Object('a') + test_log = fake_abcdxtl.create_test_log(test_log_id) + file_id = BBAA.Object('b') + file_name = BBAA.Object('c') + test_log, version_mismatch = fake_abcdxtl.retrieve_test_log(file_id, test_log_id, file_name) + fake_abcdxtl.store_test_log(file_id, test_log) +""" diff --git a/test/test_data/test_insert.py b/test/test_data/test_insert.py new file mode 100644 index 00000000..17b27a8e --- /dev/null +++ b/test/test_data/test_insert.py @@ -0,0 +1,24 @@ +input_code = """ +import LLXA +def a(): + x = 10 + +def b(): + y = 12""" +insert_code = """ + +class Asserter(unittest.TestCase): + def assert_double_equal(self, a, b): + self.assertAlmostEqual(a, b) + + def assert_raises(self, exception, callable_obj, *args, **kwargs): + if isinstance(exception, BaseException): + exc_type = type(exception) + try: + callable_obj(*args, **kwargs) + self.fail("Expected {} to be raised".format(exc_type.__name__)) + except exc_type as e: + self.assertEqual(str(e), str(exception), "Expected error_id but got {}".format(exception.id)) + else: + self.assertRaises(exception, callable_obj, *args, **kwargs) +""" diff --git a/test/test_data/test_testdoubles.py b/test/test_data/test_testdoubles.py new file mode 100644 index 00000000..62b87f9f --- /dev/null +++ b/test/test_data/test_testdoubles.py @@ -0,0 +1,234 @@ +test_indent = """class test(b, c): + def test_bw(self): + self.doubles.append(TAUT.TestDoubles(a, b, c)) + id = b + self.assert_raises( + ERROR(a, "Wafer alignment failed"), + id + ) + self.assertEqual(c, 0) +""" +test_indent_new = """class test(b, c): + def test_bw(self): + with patch.object(a, 'b', c): + id = b + self.assert_raises( + ERROR(a, "Wafer alignment failed"), + id + ) + self.assertEqual(c, 0) +""" +test_indent_fun = """def test_bw(self): + self.doubles.append(TAUT.TestDoubles(a, b, c)) + id = b + self.assert_raises( + ERROR(a, "Wafer alignment failed"), + id + ) + self.assertEqual(c, 0) +""" +test_indent_fun_new = """def test_bw(self): + with patch.object(a, 'b', c): + id = b + self.assert_raises( + ERROR(a, "Wafer alignment failed"), + id + ) + self.assertEqual(c, 0) +""" +test_doubles_fun = """def test_align_wafer_bw(self): + self.doubles.append( + TAUT.TestDoubles( + module=ACBDxEngine.ACBDxEngine, do_global_align=stub_do_global_align_bw + ) + ) + chuck_id = ACBDxBASIC.chuck_operation_enum.CHUCK_2 + load_offset = BBAA.Struct("xyavect") + self.assert_raises( + ERXA.Error(ACBDxERR.ACBD_SYS_ERR, "Wafer alignment failed"), + ACBDxAPxMEASxWLGLib.align_wafer, + chuck_id, + load_offset + ) + self.assertEqual(ACBDxCONTEXT.acbdxcontext.method_called("start_lot"), 0) + self.assertEqual( + ACBDxCONTEXT.acbdxcontext.method_called("finish_lot"), 1 + )""" +test_doubles_fun_new = """def test_align_wafer_bw(self): + with patch.object(ACBDxEngine.ACBDxEngine, 'do_global_align', stub_do_global_align_bw): + chuck_id = ACBDxBASIC.chuck_operation_enum.CHUCK_2 + load_offset = BBAA.Struct("xyavect") + self.assert_raises( + ERXA.Error(ACBDxERR.ACBD_SYS_ERR, "Wafer alignment failed"), + ACBDxAPxMEASxWLGLib.align_wafer, + chuck_id, + load_offset + ) + self.assertEqual(ACBDxCONTEXT.acbdxcontext.method_called("start_lot"), 0) + self.assertEqual( + ACBDxCONTEXT.acbdxcontext.method_called("finish_lot"), 1 + )""" + +test_doubles_class_new = """class TestCloseTest(unittest.TestCase): + + def setUp(self): + self._patch_readout_data_filler = mock.patch("ACBD_ReadoutDataFiller.ReadoutDataFiller") + self._patch_readout_data_publisher = mock.patch("ACBD_ReadoutDataPublisher.ReadoutDataPublisher") + _ = self._patch_readout_data_filler.start() + _ = self._patch_readout_data_publisher.start() + ACBDxAPxData.data = ACBDxAPxData.ACBDxAPxData() + ACBDxAPxData.data.initialize_engine(ACBDxEngine.ACBDxEngine()) + self.tlg_stub = ACBDxTestlog_stub() + self.patches = [ + patch.object(ACBDxTestlog.ACBDxTestlog, 'modify_tlg_file_id', self.tlg_stub.modify_tlg_file_id), + patch.object(ACBDxTestlog.ACBDxTestlog, 'update_tlg_after_measurement', self.tlg_stub.update_tlg_after_measurement), + ] + for p in self.patches: + p.start() + + self.results = ACBDxBASIC.result_struct() + ACBDxAPxData.data.basic_inputs.mark_sequence_file_name = rms_file + self.do_read_wid = False + self.measurement_strategy = ACBD_MeasurementDefault.ACBD_MeasurementDefault() + + def tearDown(self): + self._patch_readout_data_filler.stop() + self._patch_readout_data_publisher.stop() + for p in self.patches: + p.stop()""" + +test_doubles_class = """class TestCloseTest(TAUT.TestCase): + + def setUp(self): + self._patch_readout_data_filler = mock.patch("ACBD_ReadoutDataFiller.ReadoutDataFiller") + self._patch_readout_data_publisher = mock.patch("ACBD_ReadoutDataPublisher.ReadoutDataPublisher") + _ = self._patch_readout_data_filler.start() + _ = self._patch_readout_data_publisher.start() + + ACBDxAPxData.data = ACBDxAPxData.ACBDxAPxData() + ACBDxAPxData.data.initialize_engine(ACBDxEngine.ACBDxEngine()) + self.doubles = [] + + self.tlg_stub = ACBDxTestlog_stub() + self.doubles.append( + TAUT.TestDoubles( + module=ACBDxTestlog.ACBDxTestlog, + modify_tlg_file_id=self.tlg_stub.modify_tlg_file_id, + ) + ) + self.doubles.append( + TAUT.TestDoubles( + module=ACBDxTestlog.ACBDxTestlog, + update_tlg_after_measurement=self.tlg_stub.update_tlg_after_measurement, + ) + ) + + self.results = ACBDxBASIC.result_struct() + ACBDxAPxData.data.basic_inputs.mark_sequence_file_name = rms_file + self.do_read_wid = False + self.measurement_strategy = ACBD_MeasurementDefault.ACBD_MeasurementDefault() + + def tearDown(self): + self._patch_readout_data_filler.stop() + self._patch_readout_data_publisher.stop() + for double in self.doubles: + double.exit()""" + +test_taut_doubles_class = """class test_abcdxwid(unittest.TestCase): + def test_readout_is_ok(self): + self.doubles.append( + TAUT.TestDoubles( + module=ABCDxWID.abcdwid, get_wid_readouts=stub_get_wid_readouts + ) + ) + id = ABCDxBASIC.id + read = True + ABCDxABxCommonFunctions.CLEAR_CALLED = False + self.assert_raises( + ABCD.Error(ABCDxERR.ABCD_SYS_ERR, "error message"), + ABCDxABxREADLib.read, + id, + read, + ) + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("start"), 0) + + def test_read_two_doubles(self): + self.doubles.append( + TAUT.TestDoubles( + module=ABCDxABxLib, + _create_marks=marks, + ) + ) + self.doubles.append( + TAUT.TestDoubles( + module=ABCDxEngine.ABCDxEngine, + measure=self.engine.measure, + ) + ) + id = ABCDxBASIC.id + ABCDxABxCommonFunctions.CLEAR_CALLED = False + self.assert_raises( + ABCD.Error(ABCDxERR.ABCD_SYS_ERR, "error message"), + ABCDxABxREADLib.read, + id, + ) + + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("start"), 1) + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("finish"), 1) +""" +test_taut_doubles_class_new = """class test_abcdxwid(unittest.TestCase): + def test_readout_is_ok(self): + with patch.object(ABCDxWID.abcdwid, 'get_wid_readouts', stub_get_wid_readouts): + id = ABCDxBASIC.id + read = True + ABCDxABxCommonFunctions.CLEAR_CALLED = False + self.assert_raises( + ABCD.Error(ABCDxERR.ABCD_SYS_ERR, "error message"), + ABCDxABxREADLib.read, + id, + read, + ) + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("start"), 0) + + def test_read_two_doubles(self): + with patch.object(ABCDxABxLib, '_create_marks', marks), \\ + patch.object(ABCDxEngine.ABCDxEngine, 'measure', self.engine.measure): + id = ABCDxBASIC.id + ABCDxABxCommonFunctions.CLEAR_CALLED = False + self.assert_raises( + ABCD.Error(ABCDxERR.ABCD_SYS_ERR, "error message"), + ABCDxABxREADLib.read, + id, + ) + + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("start"), 1) + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("finish"), 1) +""" +test_taut_doubles_class_single_line = """class test_abcdxwid(unittest.TestCase): + def test_readout_is_ok(self): + self.doubles.append(TAUT.TestDoubles(module=ABCDxWID.abcdwid, get_wid_readouts=stub_get_wid_readouts)) + id = ABCDxBASIC.id + read = True + ABCDxABxCommonFunctions.CLEAR_CALLED = False + self.assert_raises( + ABCD.Error(ABCDxERR.ABCD_SYS_ERR, "error message"), + ABCDxABxREADLib.read, + id, + read, + ) + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("start"), 0) +""" +test_taut_doubles_class_single_line_new = """class test_abcdxwid(unittest.TestCase): + def test_readout_is_ok(self): + with patch.object(ABCDxWID.abcdwid, 'get_wid_readouts', stub_get_wid_readouts): + id = ABCDxBASIC.id + read = True + ABCDxABxCommonFunctions.CLEAR_CALLED = False + self.assert_raises( + ABCD.Error(ABCDxERR.ABCD_SYS_ERR, "error message"), + ABCDxABxREADLib.read, + id, + read, + ) + self.assertEqual(ABCDxCONTEXT.abcdxcontext.method_called("start"), 0) +""" diff --git a/test/tree_sitter/test_tree_sitter_structural_matcher.py b/test/tree_sitter/test_tree_sitter_structural_matcher.py new file mode 100644 index 00000000..ee0cc674 --- /dev/null +++ b/test/tree_sitter/test_tree_sitter_structural_matcher.py @@ -0,0 +1,84 @@ +import pytest +import tree_sitter_cpp as tscpp +import tree_sitter_python as tspython +from hamcrest import assert_that, has_length + +from renaissance.impl.tree_sitter.adapter import TreeSitterAdapter +from renaissance.syntax_tree.match_finder import match_pattern + + +class TestTreeSitterStructuralMatcher: + @pytest.mark.parametrize("code, pattern",[ + ("def foo(): pass", "def $foo(): pass"), + ("if x: pass", "if $x: pass"), + ("for x in y: pass","for $x in $y: pass"), + ("while x: pass", "while $x: pass"), + ("try: pass except: pass","try: pass except: pass"), + ("class A: pass", "class $A: pass"), + ("with x: pass", "with $x: pass"), + ("assert x", "assert $x"), + ("return x", "return $x"), + ("lambda x: x", "lambda $x: $x"), + ("yield x", "yield $x"), + ("a = b", "$a = $b"), + ("a += b", "$a += $b"), + ("x and y", "$x and $y"), + ("not x", "not $x"), + ( "x if y else z","$x if $y else $z"), + ("f(x)", "f($x)"), + ("[x for x in y]", "[x for $x in $y]"), + ("x in y", "$x in $y"), + ("import os", "import $os"), + ], + ) + def test_python_patterns(self, code, pattern): + adapter = TreeSitterAdapter(tspython) + ast = adapter.parse_code(code) + lst = adapter.to_lst(code, ast) + pat = adapter.to_lst(pattern, ast) + + result = match_pattern(lst.root.children, pat.root.children) + assert_that(result, has_length(1)) + + @pytest.mark.parametrize("code, pattern",[ + ("int main() { return 0; }","int $main() { return 0; }"), + ("int a;", "int $a;"), + ("int b = 1;", "int $b = 1;"), + ("struct A {};", "struct $A {};"), + ("class B {};", "class $B {};"), + ("namespace ns {}", "namespace $ns {}"), + ("template <typename T> class C {};","template <typename $T> class $C {};"), + ("enum E { A };", "enum $E { $A };"), + ("int f(int x) { return x; }","int $f(int $x) { return $x; }" ), + ("void g() { int x = 1; }", "void $g() { int $x = 1; }" ), + ("if (x) {}", "if ($x) {}"), + ("for (;;) {}", "for (;;) {}"), + ("while (1) {}", "while (1) {}"), + ("do {} while (0);", "do {} while (0);"), + ("switch(x) { case 1: break; }", "switch($x) { case 1: break; }" ), + ("try {} catch (...) {}", "try {} catch (...) {}"), + ("a + b", "$a + $b"), + ("-a", "-$a"), + ("a == b", "$a == $b"), + ("a != b", "$a != $b"), + ("a < b", "$a < $b"), + ("a <= b", "$a <= $b"), + ("a > b", "$a > $b"), + ("a >= b", "$a >= $b"), + ("a && b", "$a && $b"), + ("a || b", "$a || $b"), + ("!a", "!$a"), + ("a = b;", "$a = $b;"), + ("foo();", "$foo();"), + ], + ) + def test_cpp_patterns(self, code, pattern): + adapter = TreeSitterAdapter(tscpp) + ast = adapter.parse_code(code) + lst = adapter.to_lst(code, ast) + pat = adapter.to_lst(pattern, ast) + result = match_pattern(lst.root.children, pat.root.children) + assert_that(result, has_length(1)) + +if __name__ == "__main__": + pytest.main() diff --git a/test/utils/test_text_utils.py b/test/utils/test_text_utils.py new file mode 100644 index 00000000..1a103c0d --- /dev/null +++ b/test/utils/test_text_utils.py @@ -0,0 +1,37 @@ +import pytest +from hamcrest import assert_that, is_ + +from renaissance.utils.text_utils import camel_case, snake_case + + +class TestSnakeCase: + @pytest.mark.parametrize( + "input_str, expected", + [ + ("CamelCase", "camel_case"), + ("Unit2Pytest", "unit2pytest"), + ("SimplifyRenaissance", "simplify_renaissance"), + ("PythonRefactoring", "python_refactoring"), + ("already_snake", "already_snake"), + ("A", "a"), + ("HTMLParser", "html_parser"), + ], + ) + def test_snake_case(self, input_str, expected): + assert_that(snake_case(input_str), is_(expected)) + + +class TestCamelCase: + @pytest.mark.parametrize( + "input_str, expected", + [ + ("camel_case", "camelCase"), + ("simplify_renaissance", "simplifyRenaissance"), + ("python_refactoring", "pythonRefactoring"), + ("already_snake", "alreadySnake"), + ("a", "a"), + ("html_parser", "htmlParser"), + ], + ) + def test_camel_case(self, input_str, expected): + assert_that(camel_case(input_str), is_(expected)) diff --git a/test/utils_for_tests.py b/test/utils_for_tests.py new file mode 100644 index 00000000..4de16b2e --- /dev/null +++ b/test/utils_for_tests.py @@ -0,0 +1,81 @@ +import re +from typing import Sequence + +import hypothesis + +from renaissance.syntax_tree import ASTNode, ASTShower, PatternMatch + +VERBOSE = False +AST_SHOWER = False + + +def to_string(d: dict[str, Sequence[ASTNode]]): + return {k: [compress(v.text if isinstance(v, ASTNode) else v) for v in vs] for k, vs in d.items()} + + +def compress(s: str): + skip_whitespace = re.sub(r"\s+", " ", s.replace("\n", "")) + skip_whitespace = re.sub(r"(\W)\s", r"\1", skip_whitespace) + skip_whitespace = re.sub(r"\s(\W)", r"\1", skip_whitespace) + return skip_whitespace.strip() + + +def show_node(node: ASTNode, title: str = ""): + if VERBOSE: + if title: + print(f'\n{"="*10} {title} {"="*10}') + ASTShower.show_node(node) + + +def debug_mismatch(debug_mismatches, atu, patterns: list[ASTNode], matches: list[PatternMatch]): + if debug_mismatches: + for idx, pattern in enumerate(patterns): + show_node(pattern, f"Pattern[{idx}]") + show_node(atu, "CPP code") + + for match in matches: + print(f"\nmatch({[compress(p.text) for p in match.patterns]})" + "{") + print(f" start node: {compress(match.nodes[0].text)}") + for k, vs in match.expansions.items(): + # right align the key + print(f"{k.rjust(12)}: {[compress(v.text) for v in vs]}") + print("}") + print(" expected dict should look like:") + print(f" {[to_string(match.expansions) for match in matches]}") + + +def debug_print( + actual: str, + actual_result: ASTNode, + atu: ASTNode, + code: str, + expected: str, + expected_result: ASTNode, + include_comments: bool, + include_whitespace: bool, +): + if AST_SHOWER: + print("Original:") + ASTShower.show_node(atu) + print("Expected:") + ASTShower.show_node(expected_result) + print("Actual:") + ASTShower.show_node(actual_result) + if VERBOSE: + print("\nOriginal:" + code.replace("\n", "\\n").replace("\r", "\\r")) + print("Expected:" + expected.replace("\n", "\\n").replace("\r", "\\r")) + print(" Actual:" + actual.replace("\n", "\\n").replace("\r", "\\r")) + + code_test_input = f'("{code}", {include_whitespace}, {include_comments}, "{actual}"),'.replace("\n", "\\n").replace("\r", "\\r") + print("\nFull parameterized:" + code_test_input) + + +def reject_unsupported_code(source_code: str) -> None: + if "\f" in source_code: + hypothesis.reject() + + hypothesis.note(source_code) + try: + compile(source_code, "<string>", 'single') + except Exception: + hypothesis.reject() diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..704d6ace --- /dev/null +++ b/uv.lock @@ -0,0 +1,1231 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version < '3.13'", +] + +[[package]] +name = "antlr4-python3-runtime" +version = "4.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/5f/2cdf6f7aca3b20d3f316e9f505292e1f256a32089bd702034c29ebde6242/antlr4_python3_runtime-4.13.2.tar.gz", hash = "sha256:909b647e1d2fc2b70180ac586df3933e38919c85f98ccc656a96cd3f25ef3916", size = 117467, upload-time = "2024-08-03T19:00:12.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/03/a851e84fcbb85214dc637b6378121ef9a0dd61b4c65264675d8a5c9b1ae7/antlr4_python3_runtime-4.13.2-py3-none-any.whl", hash = "sha256:fe3835eb8d33daece0e799090eda89719dbccee7aa39ef94eed3818cafa5a7e8", size = 144462, upload-time = "2024-08-03T19:00:11.134Z" }, +] + +[[package]] +name = "arpeggio" +version = "2.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/58/ba011f3cf8291804ce80f9d81289ac15f0319a27f9d7e3c124aa5e4981cc/Arpeggio-2.0.3.tar.gz", hash = "sha256:9e85ad35cfc6c938676817c7ae9a1000a7c72a34c71db0c687136c460d12b85e", size = 766566, upload-time = "2025-09-12T12:45:20.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/4d/53b8186b41842f7a5e971b1d1c28e678364dcf841e4170f5d14d38ac1e2a/Arpeggio-2.0.3-py2.py3-none-any.whl", hash = "sha256:9374d9c531b62018b787635f37fd81c9a6ee69ef2d28c5db3cd18791b1f7db2f", size = 54656, upload-time = "2025-09-12T12:45:17.971Z" }, +] + +[[package]] +name = "ast-comments" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/e8/9bb599fd6162644d31fe0f87e0e2903d92cd434e4327d9cc2eaa904f6777/ast_comments-1.3.0.tar.gz", hash = "sha256:45f0113ecff4156a98255c1b87cc6e15b9cee185c3f037c714cf27d3225ed7c3", size = 5496, upload-time = "2026-02-22T21:27:41.724Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/38/e30499bf1b346e372b780f9161131bb230ce1a63b36662b0acabed3b3219/ast_comments-1.3.0-py3-none-any.whl", hash = "sha256:1b53fbf7fa89af1ac6222fea0a752328f2d6c35de565e045ad74a16c9441209d", size = 5946, upload-time = "2026-02-22T21:27:40.534Z" }, +] + +[[package]] +name = "autopep8" +version = "2.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycodestyle" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/d8/30873d2b7b57dee9263e53d142da044c4600a46f2d28374b3e38b023df16/autopep8-2.3.2.tar.gz", hash = "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758", size = 92210, upload-time = "2025-01-14T14:46:18.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/43/53afb8ba17218f19b77c7834128566c5bbb100a0ad9ba2e8e89d089d7079/autopep8-2.3.2-py2.py3-none-any.whl", hash = "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128", size = 45807, upload-time = "2025-01-14T14:46:15.466Z" }, +] + +[[package]] +name = "behave" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama" }, + { name = "cucumber-expressions" }, + { name = "cucumber-tag-expressions" }, + { name = "parse" }, + { name = "parse-type" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/51/f37442fe648b3e35ecf69bee803fa6db3f74c5b46d6c882d0bc5654185a2/behave-1.3.3.tar.gz", hash = "sha256:2b8f4b64ed2ea756a5a2a73e23defc1c4631e9e724c499e46661778453ebaf51", size = 892639, upload-time = "2025-09-04T12:12:02.531Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/71/06f74ffed6d74525c5cd6677c97bd2df0b7649e47a249cf6a0c2038083b2/behave-1.3.3-py2.py3-none-any.whl", hash = "sha256:89bdb62af8fb9f147ce245736a5de69f025e5edfb66f1fbe16c5007493f842c0", size = 223594, upload-time = "2025-09-04T12:12:00.3Z" }, +] + +[[package]] +name = "black" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs" }, + { name = "pytokens" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/88/560b11e521c522440af991d46848a2bde64b5f7202ec14e1f46f9509d328/black-26.1.0.tar.gz", hash = "sha256:d294ac3340eef9c9eb5d29288e96dc719ff269a88e27b396340459dd85da4c58", size = 658785, upload-time = "2026-01-18T04:50:11.993Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/13/710298938a61f0f54cdb4d1c0baeb672c01ff0358712eddaf29f76d32a0b/black-26.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6eeca41e70b5f5c84f2f913af857cf2ce17410847e1d54642e658e078da6544f", size = 1878189, upload-time = "2026-01-18T04:59:30.682Z" }, + { url = "https://files.pythonhosted.org/packages/79/a6/5179beaa57e5dbd2ec9f1c64016214057b4265647c62125aa6aeffb05392/black-26.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:dd39eef053e58e60204f2cdf059e2442e2eb08f15989eefe259870f89614c8b6", size = 1700178, upload-time = "2026-01-18T04:59:32.387Z" }, + { url = "https://files.pythonhosted.org/packages/8c/04/c96f79d7b93e8f09d9298b333ca0d31cd9b2ee6c46c274fd0f531de9dc61/black-26.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9459ad0d6cd483eacad4c6566b0f8e42af5e8b583cee917d90ffaa3778420a0a", size = 1777029, upload-time = "2026-01-18T04:59:33.767Z" }, + { url = "https://files.pythonhosted.org/packages/49/f9/71c161c4c7aa18bdda3776b66ac2dc07aed62053c7c0ff8bbda8c2624fe2/black-26.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:a19915ec61f3a8746e8b10adbac4a577c6ba9851fa4a9e9fbfbcf319887a5791", size = 1406466, upload-time = "2026-01-18T04:59:35.177Z" }, + { url = "https://files.pythonhosted.org/packages/4a/8b/a7b0f974e473b159d0ac1b6bcefffeb6bec465898a516ee5cc989503cbc7/black-26.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:643d27fb5facc167c0b1b59d0315f2674a6e950341aed0fc05cf307d22bf4954", size = 1216393, upload-time = "2026-01-18T04:59:37.18Z" }, + { url = "https://files.pythonhosted.org/packages/79/04/fa2f4784f7237279332aa735cdfd5ae2e7730db0072fb2041dadda9ae551/black-26.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ba1d768fbfb6930fc93b0ecc32a43d8861ded16f47a40f14afa9bb04ab93d304", size = 1877781, upload-time = "2026-01-18T04:59:39.054Z" }, + { url = "https://files.pythonhosted.org/packages/cf/ad/5a131b01acc0e5336740a039628c0ab69d60cf09a2c87a4ec49f5826acda/black-26.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2b807c240b64609cb0e80d2200a35b23c7df82259f80bef1b2c96eb422b4aac9", size = 1699670, upload-time = "2026-01-18T04:59:41.005Z" }, + { url = "https://files.pythonhosted.org/packages/da/7c/b05f22964316a52ab6b4265bcd52c0ad2c30d7ca6bd3d0637e438fc32d6e/black-26.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1de0f7d01cc894066a1153b738145b194414cc6eeaad8ef4397ac9abacf40f6b", size = 1775212, upload-time = "2026-01-18T04:59:42.545Z" }, + { url = "https://files.pythonhosted.org/packages/a6/a3/e8d1526bea0446e040193185353920a9506eab60a7d8beb062029129c7d2/black-26.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:91a68ae46bf07868963671e4d05611b179c2313301bd756a89ad4e3b3db2325b", size = 1409953, upload-time = "2026-01-18T04:59:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/c7/5a/d62ebf4d8f5e3a1daa54adaab94c107b57be1b1a2f115a0249b41931e188/black-26.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:be5e2fe860b9bd9edbf676d5b60a9282994c03fbbd40fe8f5e75d194f96064ca", size = 1217707, upload-time = "2026-01-18T04:59:45.719Z" }, + { url = "https://files.pythonhosted.org/packages/6a/83/be35a175aacfce4b05584ac415fd317dd6c24e93a0af2dcedce0f686f5d8/black-26.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc8c71656a79ca49b8d3e2ce8103210c9481c57798b48deeb3a8bb02db5f115", size = 1871864, upload-time = "2026-01-18T04:59:47.586Z" }, + { url = "https://files.pythonhosted.org/packages/a5/f5/d33696c099450b1274d925a42b7a030cd3ea1f56d72e5ca8bbed5f52759c/black-26.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b22b3810451abe359a964cc88121d57f7bce482b53a066de0f1584988ca36e79", size = 1701009, upload-time = "2026-01-18T04:59:49.443Z" }, + { url = "https://files.pythonhosted.org/packages/1b/87/670dd888c537acb53a863bc15abbd85b22b429237d9de1b77c0ed6b79c42/black-26.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53c62883b3f999f14e5d30b5a79bd437236658ad45b2f853906c7cbe79de00af", size = 1767806, upload-time = "2026-01-18T04:59:50.769Z" }, + { url = "https://files.pythonhosted.org/packages/fe/9c/cd3deb79bfec5bcf30f9d2100ffeec63eecce826eb63e3961708b9431ff1/black-26.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:f016baaadc423dc960cdddf9acae679e71ee02c4c341f78f3179d7e4819c095f", size = 1433217, upload-time = "2026-01-18T04:59:52.218Z" }, + { url = "https://files.pythonhosted.org/packages/4e/29/f3be41a1cf502a283506f40f5d27203249d181f7a1a2abce1c6ce188035a/black-26.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:66912475200b67ef5a0ab665011964bf924745103f51977a78b4fb92a9fc1bf0", size = 1245773, upload-time = "2026-01-18T04:59:54.457Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3d/51bdb3ecbfadfaf825ec0c75e1de6077422b4afa2091c6c9ba34fbfc0c2d/black-26.1.0-py3-none-any.whl", hash = "sha256:1054e8e47ebd686e078c0bb0eaf31e6ce69c966058d122f2c0c950311f9f3ede", size = 204010, upload-time = "2026-01-18T04:50:09.978Z" }, +] + +[[package]] +name = "clang" +version = "18.1.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/6d/202fe248475f92ab9057a4066d50fe8aaa2def62493894dc9a9814ce8d3b/clang-18.1.8.tar.gz", hash = "sha256:26d11859bab6da8d1fcdb85a244957f6c129a0cd15da2abca3059b054b87635f", size = 3101, upload-time = "2025-02-17T21:49:01.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/3c/1bebce0b2588b913b48baea6eac5091c023f03cc0c8ab4b015a8315d0d09/clang-18.1.8-py3-none-any.whl", hash = "sha256:2f6a00126743ee23d8fcd2a2338b42ef4d29897f293ee3a1bc4d5925d8ee875c", size = 31627, upload-time = "2025-02-17T21:48:59.215Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.13.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/56/95b7e30fa389756cb56630faa728da46a27b8c6eb46f9d557c68fff12b65/coverage-7.13.4.tar.gz", hash = "sha256:e5c8f6ed1e61a8b2dcdf31eb0b9bbf0130750ca79c1c49eb898e2ad86f5ccc91", size = 827239, upload-time = "2026-02-09T12:59:03.86Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/81/4ce2fdd909c5a0ed1f6dedb88aa57ab79b6d1fbd9b588c1ac7ef45659566/coverage-7.13.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:02231499b08dabbe2b96612993e5fc34217cdae907a51b906ac7fca8027a4459", size = 219449, upload-time = "2026-02-09T12:56:54.889Z" }, + { url = "https://files.pythonhosted.org/packages/5d/96/5238b1efc5922ddbdc9b0db9243152c09777804fb7c02ad1741eb18a11c0/coverage-7.13.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40aa8808140e55dc022b15d8aa7f651b6b3d68b365ea0398f1441e0b04d859c3", size = 219810, upload-time = "2026-02-09T12:56:56.33Z" }, + { url = "https://files.pythonhosted.org/packages/78/72/2f372b726d433c9c35e56377cf1d513b4c16fe51841060d826b95caacec1/coverage-7.13.4-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5b856a8ccf749480024ff3bd7310adaef57bf31fd17e1bfc404b7940b6986634", size = 251308, upload-time = "2026-02-09T12:56:57.858Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a0/2ea570925524ef4e00bb6c82649f5682a77fac5ab910a65c9284de422600/coverage-7.13.4-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2c048ea43875fbf8b45d476ad79f179809c590ec7b79e2035c662e7afa3192e3", size = 254052, upload-time = "2026-02-09T12:56:59.754Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ac/45dc2e19a1939098d783c846e130b8f862fbb50d09e0af663988f2f21973/coverage-7.13.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b7b38448866e83176e28086674fe7368ab8590e4610fb662b44e345b86d63ffa", size = 255165, upload-time = "2026-02-09T12:57:01.287Z" }, + { url = "https://files.pythonhosted.org/packages/2d/4d/26d236ff35abc3b5e63540d3386e4c3b192168c1d96da5cb2f43c640970f/coverage-7.13.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:de6defc1c9badbf8b9e67ae90fd00519186d6ab64e5cc5f3d21359c2a9b2c1d3", size = 257432, upload-time = "2026-02-09T12:57:02.637Z" }, + { url = "https://files.pythonhosted.org/packages/ec/55/14a966c757d1348b2e19caf699415a2a4c4f7feaa4bbc6326a51f5c7dd1b/coverage-7.13.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7eda778067ad7ffccd23ecffce537dface96212576a07924cbf0d8799d2ded5a", size = 251716, upload-time = "2026-02-09T12:57:04.056Z" }, + { url = "https://files.pythonhosted.org/packages/77/33/50116647905837c66d28b2af1321b845d5f5d19be9655cb84d4a0ea806b4/coverage-7.13.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e87f6c587c3f34356c3759f0420693e35e7eb0e2e41e4c011cb6ec6ecbbf1db7", size = 253089, upload-time = "2026-02-09T12:57:05.503Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b4/8efb11a46e3665d92635a56e4f2d4529de6d33f2cb38afd47d779d15fc99/coverage-7.13.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:8248977c2e33aecb2ced42fef99f2d319e9904a36e55a8a68b69207fb7e43edc", size = 251232, upload-time = "2026-02-09T12:57:06.879Z" }, + { url = "https://files.pythonhosted.org/packages/51/24/8cd73dd399b812cc76bb0ac260e671c4163093441847ffe058ac9fda1e32/coverage-7.13.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:25381386e80ae727608e662474db537d4df1ecd42379b5ba33c84633a2b36d47", size = 255299, upload-time = "2026-02-09T12:57:08.245Z" }, + { url = "https://files.pythonhosted.org/packages/03/94/0a4b12f1d0e029ce1ccc1c800944a9984cbe7d678e470bb6d3c6bc38a0da/coverage-7.13.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:ee756f00726693e5ba94d6df2bdfd64d4852d23b09bb0bc700e3b30e6f333985", size = 250796, upload-time = "2026-02-09T12:57:10.142Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/6002fbf88f6698ca034360ce474c406be6d5a985b3fdb3401128031eef6b/coverage-7.13.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fdfc1e28e7c7cdce44985b3043bc13bbd9c747520f94a4d7164af8260b3d91f0", size = 252673, upload-time = "2026-02-09T12:57:12.197Z" }, + { url = "https://files.pythonhosted.org/packages/de/c6/a0279f7c00e786be75a749a5674e6fa267bcbd8209cd10c9a450c655dfa7/coverage-7.13.4-cp312-cp312-win32.whl", hash = "sha256:01d4cbc3c283a17fc1e42d614a119f7f438eabb593391283adca8dc86eff1246", size = 221990, upload-time = "2026-02-09T12:57:14.085Z" }, + { url = "https://files.pythonhosted.org/packages/77/4e/c0a25a425fcf5557d9abd18419c95b63922e897bc86c1f327f155ef234a9/coverage-7.13.4-cp312-cp312-win_amd64.whl", hash = "sha256:9401ebc7ef522f01d01d45532c68c5ac40fb27113019b6b7d8b208f6e9baa126", size = 222800, upload-time = "2026-02-09T12:57:15.944Z" }, + { url = "https://files.pythonhosted.org/packages/47/ac/92da44ad9a6f4e3a7debd178949d6f3769bedca33830ce9b1dcdab589a37/coverage-7.13.4-cp312-cp312-win_arm64.whl", hash = "sha256:b1ec7b6b6e93255f952e27ab58fbc68dcc468844b16ecbee881aeb29b6ab4d8d", size = 221415, upload-time = "2026-02-09T12:57:17.497Z" }, + { url = "https://files.pythonhosted.org/packages/db/23/aad45061a31677d68e47499197a131eea55da4875d16c1f42021ab963503/coverage-7.13.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b66a2da594b6068b48b2692f043f35d4d3693fb639d5ea8b39533c2ad9ac3ab9", size = 219474, upload-time = "2026-02-09T12:57:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/a5/70/9b8b67a0945f3dfec1fd896c5cefb7c19d5a3a6d74630b99a895170999ae/coverage-7.13.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3599eb3992d814d23b35c536c28df1a882caa950f8f507cef23d1cbf334995ac", size = 219844, upload-time = "2026-02-09T12:57:20.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/fd/7e859f8fab324cef6c4ad7cff156ca7c489fef9179d5749b0c8d321281c2/coverage-7.13.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:93550784d9281e374fb5a12bf1324cc8a963fd63b2d2f223503ef0fd4aa339ea", size = 250832, upload-time = "2026-02-09T12:57:22.007Z" }, + { url = "https://files.pythonhosted.org/packages/e4/dc/b2442d10020c2f52617828862d8b6ee337859cd8f3a1f13d607dddda9cf7/coverage-7.13.4-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b720ce6a88a2755f7c697c23268ddc47a571b88052e6b155224347389fdf6a3b", size = 253434, upload-time = "2026-02-09T12:57:23.339Z" }, + { url = "https://files.pythonhosted.org/packages/5a/88/6728a7ad17428b18d836540630487231f5470fb82454871149502f5e5aa2/coverage-7.13.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7b322db1284a2ed3aa28ffd8ebe3db91c929b7a333c0820abec3d838ef5b3525", size = 254676, upload-time = "2026-02-09T12:57:24.774Z" }, + { url = "https://files.pythonhosted.org/packages/7c/bc/21244b1b8cedf0dff0a2b53b208015fe798d5f2a8d5348dbfece04224fff/coverage-7.13.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f4594c67d8a7c89cf922d9df0438c7c7bb022ad506eddb0fdb2863359ff78242", size = 256807, upload-time = "2026-02-09T12:57:26.125Z" }, + { url = "https://files.pythonhosted.org/packages/97/a0/ddba7ed3251cff51006737a727d84e05b61517d1784a9988a846ba508877/coverage-7.13.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:53d133df809c743eb8bce33b24bcababb371f4441340578cd406e084d94a6148", size = 251058, upload-time = "2026-02-09T12:57:27.614Z" }, + { url = "https://files.pythonhosted.org/packages/9b/55/e289addf7ff54d3a540526f33751951bf0878f3809b47f6dfb3def69c6f7/coverage-7.13.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76451d1978b95ba6507a039090ba076105c87cc76fc3efd5d35d72093964d49a", size = 252805, upload-time = "2026-02-09T12:57:29.066Z" }, + { url = "https://files.pythonhosted.org/packages/13/4e/cc276b1fa4a59be56d96f1dabddbdc30f4ba22e3b1cd42504c37b3313255/coverage-7.13.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7f57b33491e281e962021de110b451ab8a24182589be17e12a22c79047935e23", size = 250766, upload-time = "2026-02-09T12:57:30.522Z" }, + { url = "https://files.pythonhosted.org/packages/94/44/1093b8f93018f8b41a8cf29636c9292502f05e4a113d4d107d14a3acd044/coverage-7.13.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1731dc33dc276dafc410a885cbf5992f1ff171393e48a21453b78727d090de80", size = 254923, upload-time = "2026-02-09T12:57:31.946Z" }, + { url = "https://files.pythonhosted.org/packages/8b/55/ea2796da2d42257f37dbea1aab239ba9263b31bd91d5527cdd6db5efe174/coverage-7.13.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:bd60d4fe2f6fa7dff9223ca1bbc9f05d2b6697bc5961072e5d3b952d46e1b1ea", size = 250591, upload-time = "2026-02-09T12:57:33.842Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/7c4bb72aacf8af5020675aa633e59c1fbe296d22aed191b6a5b711eb2bc7/coverage-7.13.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9181a3ccead280b828fae232df12b16652702b49d41e99d657f46cc7b1f6ec7a", size = 252364, upload-time = "2026-02-09T12:57:35.743Z" }, + { url = "https://files.pythonhosted.org/packages/5c/38/a8d2ec0146479c20bbaa7181b5b455a0c41101eed57f10dd19a78ab44c80/coverage-7.13.4-cp313-cp313-win32.whl", hash = "sha256:f53d492307962561ac7de4cd1de3e363589b000ab69617c6156a16ba7237998d", size = 222010, upload-time = "2026-02-09T12:57:37.25Z" }, + { url = "https://files.pythonhosted.org/packages/e2/0c/dbfafbe90a185943dcfbc766fe0e1909f658811492d79b741523a414a6cc/coverage-7.13.4-cp313-cp313-win_amd64.whl", hash = "sha256:e6f70dec1cc557e52df5306d051ef56003f74d56e9c4dd7ddb07e07ef32a84dd", size = 222818, upload-time = "2026-02-09T12:57:38.734Z" }, + { url = "https://files.pythonhosted.org/packages/04/d1/934918a138c932c90d78301f45f677fb05c39a3112b96fd2c8e60503cdc7/coverage-7.13.4-cp313-cp313-win_arm64.whl", hash = "sha256:fb07dc5da7e849e2ad31a5d74e9bece81f30ecf5a42909d0a695f8bd1874d6af", size = 221438, upload-time = "2026-02-09T12:57:40.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/57/ee93ced533bcb3e6df961c0c6e42da2fc6addae53fb95b94a89b1e33ebd7/coverage-7.13.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:40d74da8e6c4b9ac18b15331c4b5ebc35a17069410cad462ad4f40dcd2d50c0d", size = 220165, upload-time = "2026-02-09T12:57:41.639Z" }, + { url = "https://files.pythonhosted.org/packages/c5/e0/969fc285a6fbdda49d91af278488d904dcd7651b2693872f0ff94e40e84a/coverage-7.13.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4223b4230a376138939a9173f1bdd6521994f2aff8047fae100d6d94d50c5a12", size = 220516, upload-time = "2026-02-09T12:57:44.215Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b8/9531944e16267e2735a30a9641ff49671f07e8138ecf1ca13db9fd2560c7/coverage-7.13.4-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1d4be36a5114c499f9f1f9195e95ebf979460dbe2d88e6816ea202010ba1c34b", size = 261804, upload-time = "2026-02-09T12:57:45.989Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f3/e63df6d500314a2a60390d1989240d5f27318a7a68fa30ad3806e2a9323e/coverage-7.13.4-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:200dea7d1e8095cc6e98cdabe3fd1d21ab17d3cee6dab00cadbb2fe35d9c15b9", size = 263885, upload-time = "2026-02-09T12:57:47.42Z" }, + { url = "https://files.pythonhosted.org/packages/f3/67/7654810de580e14b37670b60a09c599fa348e48312db5b216d730857ffe6/coverage-7.13.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b8eb931ee8e6d8243e253e5ed7336deea6904369d2fd8ae6e43f68abbf167092", size = 266308, upload-time = "2026-02-09T12:57:49.345Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/39d41eca0eab3cc82115953ad41c4e77935286c930e8fad15eaed1389d83/coverage-7.13.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75eab1ebe4f2f64d9509b984f9314d4aa788540368218b858dad56dc8f3e5eb9", size = 267452, upload-time = "2026-02-09T12:57:50.811Z" }, + { url = "https://files.pythonhosted.org/packages/50/6d/39c0fbb8fc5cd4d2090811e553c2108cf5112e882f82505ee7495349a6bf/coverage-7.13.4-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c35eb28c1d085eb7d8c9b3296567a1bebe03ce72962e932431b9a61f28facf26", size = 261057, upload-time = "2026-02-09T12:57:52.447Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a2/60010c669df5fa603bb5a97fb75407e191a846510da70ac657eb696b7fce/coverage-7.13.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:eb88b316ec33760714a4720feb2816a3a59180fd58c1985012054fa7aebee4c2", size = 263875, upload-time = "2026-02-09T12:57:53.938Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d9/63b22a6bdbd17f1f96e9ed58604c2a6b0e72a9133e37d663bef185877cf6/coverage-7.13.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7d41eead3cc673cbd38a4417deb7fd0b4ca26954ff7dc6078e33f6ff97bed940", size = 261500, upload-time = "2026-02-09T12:57:56.012Z" }, + { url = "https://files.pythonhosted.org/packages/70/bf/69f86ba1ad85bc3ad240e4c0e57a2e620fbc0e1645a47b5c62f0e941ad7f/coverage-7.13.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:fb26a934946a6afe0e326aebe0730cdff393a8bc0bbb65a2f41e30feddca399c", size = 265212, upload-time = "2026-02-09T12:57:57.5Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f2/5f65a278a8c2148731831574c73e42f57204243d33bedaaf18fa79c5958f/coverage-7.13.4-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:dae88bc0fc77edaa65c14be099bd57ee140cf507e6bfdeea7938457ab387efb0", size = 260398, upload-time = "2026-02-09T12:57:59.027Z" }, + { url = "https://files.pythonhosted.org/packages/ef/80/6e8280a350ee9fea92f14b8357448a242dcaa243cb2c72ab0ca591f66c8c/coverage-7.13.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:845f352911777a8e722bfce168958214951e07e47e5d5d9744109fa5fe77f79b", size = 262584, upload-time = "2026-02-09T12:58:01.129Z" }, + { url = "https://files.pythonhosted.org/packages/22/63/01ff182fc95f260b539590fb12c11ad3e21332c15f9799cb5e2386f71d9f/coverage-7.13.4-cp313-cp313t-win32.whl", hash = "sha256:2fa8d5f8de70688a28240de9e139fa16b153cc3cbb01c5f16d88d6505ebdadf9", size = 222688, upload-time = "2026-02-09T12:58:02.736Z" }, + { url = "https://files.pythonhosted.org/packages/a9/43/89de4ef5d3cd53b886afa114065f7e9d3707bdb3e5efae13535b46ae483d/coverage-7.13.4-cp313-cp313t-win_amd64.whl", hash = "sha256:9351229c8c8407645840edcc277f4a2d44814d1bc34a2128c11c2a031d45a5dd", size = 223746, upload-time = "2026-02-09T12:58:05.362Z" }, + { url = "https://files.pythonhosted.org/packages/35/39/7cf0aa9a10d470a5309b38b289b9bb07ddeac5d61af9b664fe9775a4cb3e/coverage-7.13.4-cp313-cp313t-win_arm64.whl", hash = "sha256:30b8d0512f2dc8c8747557e8fb459d6176a2c9e5731e2b74d311c03b78451997", size = 222003, upload-time = "2026-02-09T12:58:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/92/11/a9cf762bb83386467737d32187756a42094927150c3e107df4cb078e8590/coverage-7.13.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:300deaee342f90696ed186e3a00c71b5b3d27bffe9e827677954f4ee56969601", size = 219522, upload-time = "2026-02-09T12:58:08.623Z" }, + { url = "https://files.pythonhosted.org/packages/d3/28/56e6d892b7b052236d67c95f1936b6a7cf7c3e2634bf27610b8cbd7f9c60/coverage-7.13.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29e3220258d682b6226a9b0925bc563ed9a1ebcff3cad30f043eceea7eaf2689", size = 219855, upload-time = "2026-02-09T12:58:10.176Z" }, + { url = "https://files.pythonhosted.org/packages/e5/69/233459ee9eb0c0d10fcc2fe425a029b3fa5ce0f040c966ebce851d030c70/coverage-7.13.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:391ee8f19bef69210978363ca930f7328081c6a0152f1166c91f0b5fdd2a773c", size = 250887, upload-time = "2026-02-09T12:58:12.503Z" }, + { url = "https://files.pythonhosted.org/packages/06/90/2cdab0974b9b5bbc1623f7876b73603aecac11b8d95b85b5b86b32de5eab/coverage-7.13.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0dd7ab8278f0d58a0128ba2fca25824321f05d059c1441800e934ff2efa52129", size = 253396, upload-time = "2026-02-09T12:58:14.615Z" }, + { url = "https://files.pythonhosted.org/packages/ac/15/ea4da0f85bf7d7b27635039e649e99deb8173fe551096ea15017f7053537/coverage-7.13.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78cdf0d578b15148b009ccf18c686aa4f719d887e76e6b40c38ffb61d264a552", size = 254745, upload-time = "2026-02-09T12:58:16.162Z" }, + { url = "https://files.pythonhosted.org/packages/99/11/bb356e86920c655ca4d61daee4e2bbc7258f0a37de0be32d233b561134ff/coverage-7.13.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:48685fee12c2eb3b27c62f2658e7ea21e9c3239cba5a8a242801a0a3f6a8c62a", size = 257055, upload-time = "2026-02-09T12:58:17.892Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0f/9ae1f8cb17029e09da06ca4e28c9e1d5c1c0a511c7074592e37e0836c915/coverage-7.13.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4e83efc079eb39480e6346a15a1bcb3e9b04759c5202d157e1dd4303cd619356", size = 250911, upload-time = "2026-02-09T12:58:19.495Z" }, + { url = "https://files.pythonhosted.org/packages/89/3a/adfb68558fa815cbc29747b553bc833d2150228f251b127f1ce97e48547c/coverage-7.13.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecae9737b72408d6a950f7e525f30aca12d4bd8dd95e37342e5beb3a2a8c4f71", size = 252754, upload-time = "2026-02-09T12:58:21.064Z" }, + { url = "https://files.pythonhosted.org/packages/32/b1/540d0c27c4e748bd3cd0bd001076ee416eda993c2bae47a73b7cc9357931/coverage-7.13.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ae4578f8528569d3cf303fef2ea569c7f4c4059a38c8667ccef15c6e1f118aa5", size = 250720, upload-time = "2026-02-09T12:58:22.622Z" }, + { url = "https://files.pythonhosted.org/packages/c7/95/383609462b3ffb1fe133014a7c84fc0dd01ed55ac6140fa1093b5af7ebb1/coverage-7.13.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6fdef321fdfbb30a197efa02d48fcd9981f0d8ad2ae8903ac318adc653f5df98", size = 254994, upload-time = "2026-02-09T12:58:24.548Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ba/1761138e86c81680bfc3c49579d66312865457f9fe405b033184e5793cb3/coverage-7.13.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b0f6ccf3dbe577170bebfce1318707d0e8c3650003cb4b3a9dd744575daa8b5", size = 250531, upload-time = "2026-02-09T12:58:26.271Z" }, + { url = "https://files.pythonhosted.org/packages/f8/8e/05900df797a9c11837ab59c4d6fe94094e029582aab75c3309a93e6fb4e3/coverage-7.13.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75fcd519f2a5765db3f0e391eb3b7d150cce1a771bf4c9f861aeab86c767a3c0", size = 252189, upload-time = "2026-02-09T12:58:27.807Z" }, + { url = "https://files.pythonhosted.org/packages/00/bd/29c9f2db9ea4ed2738b8a9508c35626eb205d51af4ab7bf56a21a2e49926/coverage-7.13.4-cp314-cp314-win32.whl", hash = "sha256:8e798c266c378da2bd819b0677df41ab46d78065fb2a399558f3f6cae78b2fbb", size = 222258, upload-time = "2026-02-09T12:58:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4d/1f8e723f6829977410efeb88f73673d794075091c8c7c18848d273dc9d73/coverage-7.13.4-cp314-cp314-win_amd64.whl", hash = "sha256:245e37f664d89861cf2329c9afa2c1fe9e6d4e1a09d872c947e70718aeeac505", size = 223073, upload-time = "2026-02-09T12:58:31.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/5b/84100025be913b44e082ea32abcf1afbf4e872f5120b7a1cab1d331b1e13/coverage-7.13.4-cp314-cp314-win_arm64.whl", hash = "sha256:ad27098a189e5838900ce4c2a99f2fe42a0bf0c2093c17c69b45a71579e8d4a2", size = 221638, upload-time = "2026-02-09T12:58:32.599Z" }, + { url = "https://files.pythonhosted.org/packages/a7/e4/c884a405d6ead1370433dad1e3720216b4f9fd8ef5b64bfd984a2a60a11a/coverage-7.13.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:85480adfb35ffc32d40918aad81b89c69c9cc5661a9b8a81476d3e645321a056", size = 220246, upload-time = "2026-02-09T12:58:34.181Z" }, + { url = "https://files.pythonhosted.org/packages/81/5c/4d7ed8b23b233b0fffbc9dfec53c232be2e695468523242ea9fd30f97ad2/coverage-7.13.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:79be69cf7f3bf9b0deeeb062eab7ac7f36cd4cc4c4dd694bd28921ba4d8596cc", size = 220514, upload-time = "2026-02-09T12:58:35.704Z" }, + { url = "https://files.pythonhosted.org/packages/2f/6f/3284d4203fd2f28edd73034968398cd2d4cb04ab192abc8cff007ea35679/coverage-7.13.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:caa421e2684e382c5d8973ac55e4f36bed6821a9bad5c953494de960c74595c9", size = 261877, upload-time = "2026-02-09T12:58:37.864Z" }, + { url = "https://files.pythonhosted.org/packages/09/aa/b672a647bbe1556a85337dc95bfd40d146e9965ead9cc2fe81bde1e5cbce/coverage-7.13.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14375934243ee05f56c45393fe2ce81fe5cc503c07cee2bdf1725fb8bef3ffaf", size = 264004, upload-time = "2026-02-09T12:58:39.492Z" }, + { url = "https://files.pythonhosted.org/packages/79/a1/aa384dbe9181f98bba87dd23dda436f0c6cf2e148aecbb4e50fc51c1a656/coverage-7.13.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25a41c3104d08edb094d9db0d905ca54d0cd41c928bb6be3c4c799a54753af55", size = 266408, upload-time = "2026-02-09T12:58:41.852Z" }, + { url = "https://files.pythonhosted.org/packages/53/5e/5150bf17b4019bc600799f376bb9606941e55bd5a775dc1e096b6ffea952/coverage-7.13.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f01afcff62bf9a08fb32b2c1d6e924236c0383c02c790732b6537269e466a72", size = 267544, upload-time = "2026-02-09T12:58:44.093Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ed/f1de5c675987a4a7a672250d2c5c9d73d289dbf13410f00ed7181d8017dd/coverage-7.13.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eb9078108fbf0bcdde37c3f4779303673c2fa1fe8f7956e68d447d0dd426d38a", size = 260980, upload-time = "2026-02-09T12:58:45.721Z" }, + { url = "https://files.pythonhosted.org/packages/b3/e3/fe758d01850aa172419a6743fe76ba8b92c29d181d4f676ffe2dae2ba631/coverage-7.13.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0e086334e8537ddd17e5f16a344777c1ab8194986ec533711cbe6c41cde841b6", size = 263871, upload-time = "2026-02-09T12:58:47.334Z" }, + { url = "https://files.pythonhosted.org/packages/b6/76/b829869d464115e22499541def9796b25312b8cf235d3bb00b39f1675395/coverage-7.13.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:725d985c5ab621268b2edb8e50dfe57633dc69bda071abc470fed55a14935fd3", size = 261472, upload-time = "2026-02-09T12:58:48.995Z" }, + { url = "https://files.pythonhosted.org/packages/14/9e/caedb1679e73e2f6ad240173f55218488bfe043e38da577c4ec977489915/coverage-7.13.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:3c06f0f1337c667b971ca2f975523347e63ec5e500b9aa5882d91931cd3ef750", size = 265210, upload-time = "2026-02-09T12:58:51.178Z" }, + { url = "https://files.pythonhosted.org/packages/3a/10/0dd02cb009b16ede425b49ec344aba13a6ae1dc39600840ea6abcb085ac4/coverage-7.13.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:590c0ed4bf8e85f745e6b805b2e1c457b2e33d5255dd9729743165253bc9ad39", size = 260319, upload-time = "2026-02-09T12:58:53.081Z" }, + { url = "https://files.pythonhosted.org/packages/92/8e/234d2c927af27c6d7a5ffad5bd2cf31634c46a477b4c7adfbfa66baf7ebb/coverage-7.13.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:eb30bf180de3f632cd043322dad5751390e5385108b2807368997d1a92a509d0", size = 262638, upload-time = "2026-02-09T12:58:55.258Z" }, + { url = "https://files.pythonhosted.org/packages/2f/64/e5547c8ff6964e5965c35a480855911b61509cce544f4d442caa759a0702/coverage-7.13.4-cp314-cp314t-win32.whl", hash = "sha256:c4240e7eded42d131a2d2c4dec70374b781b043ddc79a9de4d55ca71f8e98aea", size = 223040, upload-time = "2026-02-09T12:58:56.936Z" }, + { url = "https://files.pythonhosted.org/packages/c7/96/38086d58a181aac86d503dfa9c47eb20715a79c3e3acbdf786e92e5c09a8/coverage-7.13.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4c7d3cc01e7350f2f0f6f7036caaf5673fb56b6998889ccfe9e1c1fe75a9c932", size = 224148, upload-time = "2026-02-09T12:58:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/ce/72/8d10abd3740a0beb98c305e0c3faf454366221c0f37a8bcf8f60020bb65a/coverage-7.13.4-cp314-cp314t-win_arm64.whl", hash = "sha256:23e3f687cf945070d1c90f85db66d11e3025665d8dafa831301a0e0038f3db9b", size = 222172, upload-time = "2026-02-09T12:59:00.396Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4a/331fe2caf6799d591109bb9c08083080f6de90a823695d412a935622abb2/coverage-7.13.4-py3-none-any.whl", hash = "sha256:1af1641e57cf7ba1bd67d677c9abdbcd6cc2ab7da3bca7fa1e2b7e50e65f2ad0", size = 211242, upload-time = "2026-02-09T12:59:02.032Z" }, +] + +[[package]] +name = "cucumber-expressions" +version = "19.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/5f/1afc1a0a2a6daed47b2d032a897613a556ebf49303e4af8310223f4a450b/cucumber_expressions-19.0.0.tar.gz", hash = "sha256:8eb5ae46dd03dd37fec1163ace1510529501d7d1868ff372c1ab2cd5aa4543a8", size = 13722, upload-time = "2026-01-25T18:09:15.642Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/72/eb79377be899d24c91ed196a50808563685992bb3aa6b82dbe3a1e30df67/cucumber_expressions-19.0.0-py3-none-any.whl", hash = "sha256:f452e6c73258c1677043ad67ad5f538c87284d6b502004720510fb6b7452d9c5", size = 20232, upload-time = "2026-01-25T18:09:16.763Z" }, +] + +[[package]] +name = "cucumber-tag-expressions" +version = "9.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b1/e0/de0b292a533846def28a4373a00c883ffa5ed986ca79f0284bd69a6297b8/cucumber_tag_expressions-9.1.0.tar.gz", hash = "sha256:d960383d5885300ebcbcb14e41657946fde2a59d5c0f485eb291bc6a0e228acc", size = 8437, upload-time = "2026-02-17T21:59:06.072Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/cf/8e8d034f7d55fceb2e4765bf9fab5da6d6a09204cd09de7bb5054f242cd0/cucumber_tag_expressions-9.1.0-py3-none-any.whl", hash = "sha256:cca145d677a942c1877e5a2cf13da8c6ec99260988877c817efd284d8455bb56", size = 9726, upload-time = "2026-02-17T21:59:04.755Z" }, +] + +[[package]] +name = "dataclasses-json" +version = "0.6.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "marshmallow" }, + { name = "typing-inspect" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/a4/f71d9cf3a5ac257c993b5ca3f93df5f7fb395c725e7f1e6479d2514173c3/dataclasses_json-0.6.7.tar.gz", hash = "sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0", size = 32227, upload-time = "2024-06-09T16:20:19.103Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/be/d0d44e092656fe7a06b55e6103cbce807cdbdee17884a5367c68c9860853/dataclasses_json-0.6.7-py3-none-any.whl", hash = "sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a", size = 28686, upload-time = "2024-06-09T16:20:16.715Z" }, +] + +[[package]] +name = "flake8" +version = "7.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mccabe" }, + { name = "pycodestyle" }, + { name = "pyflakes" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/af/fbfe3c4b5a657d79e5c47a2827a362f9e1b763336a52f926126aa6dc7123/flake8-7.3.0.tar.gz", hash = "sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872", size = 48326, upload-time = "2025-06-20T19:31:35.838Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/56/13ab06b4f93ca7cac71078fbe37fcea175d3216f31f85c3168a6bbd0bb9a/flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e", size = 57922, upload-time = "2025-06-20T19:31:34.425Z" }, +] + +[[package]] +name = "future-fstrings" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/e2/3874574cce18a2e3608abfe5b4b5b3c9765653c464f5da18df8971cf501d/future_fstrings-1.2.0.tar.gz", hash = "sha256:6cf41cbe97c398ab5a81168ce0dbb8ad95862d3caf23c21e4430627b90844089", size = 5786, upload-time = "2019-06-16T03:04:42.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/6d/ea1d52e9038558dd37f5d30647eb9f07888c164960a5d4daa5f970c6da25/future_fstrings-1.2.0-py2.py3-none-any.whl", hash = "sha256:90e49598b553d8746c4dc7d9442e0359d038c3039d802c91c0a55505da318c63", size = 6138, upload-time = "2019-06-16T03:04:40.395Z" }, +] + +[[package]] +name = "gherkin-official" +version = "29.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/d8/7a28537efd7638448f7512a0cce011d4e3bf1c7f4794ad4e9c87b3f1e98e/gherkin_official-29.0.0.tar.gz", hash = "sha256:dbea32561158f02280d7579d179b019160d072ce083197625e2f80a6776bb9eb", size = 32303, upload-time = "2024-08-12T09:41:09.595Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/fc/b86c22ad3b18d8324a9d6fe5a3b55403291d2bf7572ba6a16efa5aa88059/gherkin_official-29.0.0-py3-none-any.whl", hash = "sha256:26967b0d537a302119066742669e0e8b663e632769330be675457ae993e1d1bc", size = 37085, upload-time = "2024-08-12T09:41:07.954Z" }, +] + +[[package]] +name = "gprof2dot" +version = "2025.4.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/fd/cad13fa1f7a463a607176432c4affa33ea162f02f58cc36de1d40d3e6b48/gprof2dot-2025.4.14.tar.gz", hash = "sha256:35743e2d2ca027bf48fa7cba37021aaf4a27beeae1ae8e05a50b55f1f921a6ce", size = 39536, upload-time = "2025-04-14T07:21:45.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/ed/89d760cb25279109b89eb52975a7b5479700d3114a2421ce735bfb2e7513/gprof2dot-2025.4.14-py3-none-any.whl", hash = "sha256:0742e4c0b4409a5e8777e739388a11e1ed3750be86895655312ea7c20bd0090e", size = 37555, upload-time = "2025-04-14T07:21:43.319Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.152.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fa/c7/3147bd903d6b18324a016d43a259cf5b4bb4545e1ead6773dc8a0374e70a/hypothesis-6.152.4.tar.gz", hash = "sha256:31c8f9ce619716f543e2710b489b1633c833586641d9e6c94cee03f109a5afc4", size = 466444, upload-time = "2026-04-27T20:18:37.594Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/89/0f50dd0d92e8a7dffc24f69ab910ff81db89b2f082ba42682bd57695e4d2/hypothesis-6.152.4-py3-none-any.whl", hash = "sha256:e730fd93c7578182efadc7f90b3c5437ee4d55edf738930eb5043c81ac1d97e8", size = 532145, upload-time = "2026-04-27T20:18:35.043Z" }, +] + +[package.optional-dependencies] +lark = [ + { name = "lark" }, +] + +[[package]] +name = "hypothesmith" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "hypothesis", extra = ["lark"] }, + { name = "libcst" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e3/f6/1a64114dee6c46985482c35bdbc12025db59973a0225eec47ac4d306030f/hypothesmith-0.3.3.tar.gz", hash = "sha256:96c14802d6c8e85d8975264176878db54b28d2ed921fdbfedc2e6b8ce3c81716", size = 25529, upload-time = "2024-02-16T20:21:24.511Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/bc/78dcf42c6eaaf7d628f061f1e533a596f5bca2a53be2b714adc5d370d48e/hypothesmith-0.3.3-py3-none-any.whl", hash = "sha256:fdb0172f9de97d09450da40da7da083fdd118bcd2f88b1a2289413d2d496b1b1", size = 19247, upload-time = "2024-02-16T20:20:47.059Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "lark" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, +] + +[[package]] +name = "libclang" +version = "18.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/5c/ca35e19a4f142adffa27e3d652196b7362fa612243e2b916845d801454fc/libclang-18.1.1.tar.gz", hash = "sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250", size = 39612, upload-time = "2024-03-17T16:04:37.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/49/f5e3e7e1419872b69f6f5e82ba56e33955a74bd537d8a1f5f1eff2f3668a/libclang-18.1.1-1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:0b2e143f0fac830156feb56f9231ff8338c20aecfe72b4ffe96f19e5a1dbb69a", size = 25836045, upload-time = "2024-06-30T17:40:31.646Z" }, + { url = "https://files.pythonhosted.org/packages/e2/e5/fc61bbded91a8830ccce94c5294ecd6e88e496cc85f6704bf350c0634b70/libclang-18.1.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6f14c3f194704e5d09769108f03185fce7acaf1d1ae4bbb2f30a72c2400cb7c5", size = 26502641, upload-time = "2024-03-18T15:52:26.722Z" }, + { url = "https://files.pythonhosted.org/packages/db/ed/1df62b44db2583375f6a8a5e2ca5432bbdc3edb477942b9b7c848c720055/libclang-18.1.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:83ce5045d101b669ac38e6da8e58765f12da2d3aafb3b9b98d88b286a60964d8", size = 26420207, upload-time = "2024-03-17T15:00:26.63Z" }, + { url = "https://files.pythonhosted.org/packages/1d/fc/716c1e62e512ef1c160e7984a73a5fc7df45166f2ff3f254e71c58076f7c/libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b", size = 24515943, upload-time = "2024-03-17T16:03:45.942Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3d/f0ac1150280d8d20d059608cf2d5ff61b7c3b7f7bcf9c0f425ab92df769a/libclang-18.1.1-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592", size = 23784972, upload-time = "2024-03-17T16:12:47.677Z" }, + { url = "https://files.pythonhosted.org/packages/fe/2f/d920822c2b1ce9326a4c78c0c2b4aa3fde610c7ee9f631b600acb5376c26/libclang-18.1.1-py2.py3-none-manylinux2014_armv7l.whl", hash = "sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe", size = 20259606, upload-time = "2024-03-17T16:17:42.437Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c2/de1db8c6d413597076a4259cea409b83459b2db997c003578affdd32bf66/libclang-18.1.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f", size = 24921494, upload-time = "2024-03-17T16:14:20.132Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2d/3f480b1e1d31eb3d6de5e3ef641954e5c67430d5ac93b7fa7e07589576c7/libclang-18.1.1-py2.py3-none-win_amd64.whl", hash = "sha256:4dd2d3b82fab35e2bf9ca717d7b63ac990a3519c7e312f19fa8e86dcc712f7fb", size = 26415083, upload-time = "2024-03-17T16:42:21.703Z" }, + { url = "https://files.pythonhosted.org/packages/71/cf/e01dc4cc79779cd82d77888a88ae2fa424d93b445ad4f6c02bfc18335b70/libclang-18.1.1-py2.py3-none-win_arm64.whl", hash = "sha256:3f0e1f49f04d3cd198985fea0511576b0aee16f9ff0e0f0cad7f9c57ec3c20e8", size = 22361112, upload-time = "2024-03-17T16:42:59.565Z" }, +] + +[[package]] +name = "libcst" +version = "1.8.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml", marker = "python_full_version != '3.13.*'" }, + { name = "pyyaml-ft", marker = "python_full_version == '3.13.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/de/cd/337df968b38d94c5aabd3e1b10630f047a2b345f6e1d4456bd9fe7417537/libcst-1.8.6.tar.gz", hash = "sha256:f729c37c9317126da9475bdd06a7208eb52fcbd180a6341648b45a56b4ba708b", size = 891354, upload-time = "2025-11-03T22:33:30.621Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/3c/93365c17da3d42b055a8edb0e1e99f1c60c776471db6c9b7f1ddf6a44b28/libcst-1.8.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0c13d5bd3d8414a129e9dccaf0e5785108a4441e9b266e1e5e9d1f82d1b943c9", size = 2206166, upload-time = "2025-11-03T22:32:16.012Z" }, + { url = "https://files.pythonhosted.org/packages/1d/cb/7530940e6ac50c6dd6022349721074e19309eb6aa296e942ede2213c1a19/libcst-1.8.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1472eeafd67cdb22544e59cf3bfc25d23dc94058a68cf41f6654ff4fcb92e09", size = 2083726, upload-time = "2025-11-03T22:32:17.312Z" }, + { url = "https://files.pythonhosted.org/packages/1b/cf/7e5eaa8c8f2c54913160671575351d129170db757bb5e4b7faffed022271/libcst-1.8.6-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:089c58e75cb142ec33738a1a4ea7760a28b40c078ab2fd26b270dac7d2633a4d", size = 2235755, upload-time = "2025-11-03T22:32:18.859Z" }, + { url = "https://files.pythonhosted.org/packages/55/54/570ec2b0e9a3de0af9922e3bb1b69a5429beefbc753a7ea770a27ad308bd/libcst-1.8.6-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c9d7aeafb1b07d25a964b148c0dda9451efb47bbbf67756e16eeae65004b0eb5", size = 2301473, upload-time = "2025-11-03T22:32:20.499Z" }, + { url = "https://files.pythonhosted.org/packages/11/4c/163457d1717cd12181c421a4cca493454bcabd143fc7e53313bc6a4ad82a/libcst-1.8.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:207481197afd328aa91d02670c15b48d0256e676ce1ad4bafb6dc2b593cc58f1", size = 2298899, upload-time = "2025-11-03T22:32:21.765Z" }, + { url = "https://files.pythonhosted.org/packages/35/1d/317ddef3669883619ef3d3395ea583305f353ef4ad87d7a5ac1c39be38e3/libcst-1.8.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:375965f34cc6f09f5f809244d3ff9bd4f6cb6699f571121cebce53622e7e0b86", size = 2408239, upload-time = "2025-11-03T22:32:23.275Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a1/f47d8cccf74e212dd6044b9d6dbc223636508da99acff1d54786653196bc/libcst-1.8.6-cp312-cp312-win_amd64.whl", hash = "sha256:da95b38693b989eaa8d32e452e8261cfa77fe5babfef1d8d2ac25af8c4aa7e6d", size = 2119660, upload-time = "2025-11-03T22:32:24.822Z" }, + { url = "https://files.pythonhosted.org/packages/19/d0/dd313bf6a7942cdf951828f07ecc1a7695263f385065edc75ef3016a3cb5/libcst-1.8.6-cp312-cp312-win_arm64.whl", hash = "sha256:bff00e1c766658adbd09a175267f8b2f7616e5ee70ce45db3d7c4ce6d9f6bec7", size = 1999824, upload-time = "2025-11-03T22:32:26.131Z" }, + { url = "https://files.pythonhosted.org/packages/90/01/723cd467ec267e712480c772aacc5aa73f82370c9665162fd12c41b0065b/libcst-1.8.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7445479ebe7d1aff0ee094ab5a1c7718e1ad78d33e3241e1a1ec65dcdbc22ffb", size = 2206386, upload-time = "2025-11-03T22:32:27.422Z" }, + { url = "https://files.pythonhosted.org/packages/17/50/b944944f910f24c094f9b083f76f61e3985af5a376f5342a21e01e2d1a81/libcst-1.8.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4fc3fef8a2c983e7abf5d633e1884c5dd6fa0dcb8f6e32035abd3d3803a3a196", size = 2083945, upload-time = "2025-11-03T22:32:28.847Z" }, + { url = "https://files.pythonhosted.org/packages/36/a1/bd1b2b2b7f153d82301cdaddba787f4a9fc781816df6bdb295ca5f88b7cf/libcst-1.8.6-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1a3a5e4ee870907aa85a4076c914ae69066715a2741b821d9bf16f9579de1105", size = 2235818, upload-time = "2025-11-03T22:32:30.504Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ab/f5433988acc3b4d188c4bb154e57837df9488cc9ab551267cdeabd3bb5e7/libcst-1.8.6-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6609291c41f7ad0bac570bfca5af8fea1f4a27987d30a1fa8b67fe5e67e6c78d", size = 2301289, upload-time = "2025-11-03T22:32:31.812Z" }, + { url = "https://files.pythonhosted.org/packages/5d/57/89f4ba7a6f1ac274eec9903a9e9174890d2198266eee8c00bc27eb45ecf7/libcst-1.8.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:25eaeae6567091443b5374b4c7d33a33636a2d58f5eda02135e96fc6c8807786", size = 2299230, upload-time = "2025-11-03T22:32:33.242Z" }, + { url = "https://files.pythonhosted.org/packages/f2/36/0aa693bc24cce163a942df49d36bf47a7ed614a0cd5598eee2623bc31913/libcst-1.8.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:04030ea4d39d69a65873b1d4d877def1c3951a7ada1824242539e399b8763d30", size = 2408519, upload-time = "2025-11-03T22:32:34.678Z" }, + { url = "https://files.pythonhosted.org/packages/db/18/6dd055b5f15afa640fb3304b2ee9df8b7f72e79513814dbd0a78638f4a0e/libcst-1.8.6-cp313-cp313-win_amd64.whl", hash = "sha256:8066f1b70f21a2961e96bedf48649f27dfd5ea68be5cd1bed3742b047f14acde", size = 2119853, upload-time = "2025-11-03T22:32:36.287Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ed/5ddb2a22f0b0abdd6dcffa40621ada1feaf252a15e5b2733a0a85dfd0429/libcst-1.8.6-cp313-cp313-win_arm64.whl", hash = "sha256:c188d06b583900e662cd791a3f962a8c96d3dfc9b36ea315be39e0a4c4792ebf", size = 1999808, upload-time = "2025-11-03T22:32:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/25/d3/72b2de2c40b97e1ef4a1a1db4e5e52163fc7e7740ffef3846d30bc0096b5/libcst-1.8.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:c41c76e034a1094afed7057023b1d8967f968782433f7299cd170eaa01ec033e", size = 2190553, upload-time = "2025-11-03T22:32:39.819Z" }, + { url = "https://files.pythonhosted.org/packages/0d/20/983b7b210ccc3ad94a82db54230e92599c4a11b9cfc7ce3bc97c1d2df75c/libcst-1.8.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5432e785322aba3170352f6e72b32bea58d28abd141ac37cc9b0bf6b7c778f58", size = 2074717, upload-time = "2025-11-03T22:32:41.373Z" }, + { url = "https://files.pythonhosted.org/packages/13/f2/9e01678fedc772e09672ed99930de7355757035780d65d59266fcee212b8/libcst-1.8.6-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:85b7025795b796dea5284d290ff69de5089fc8e989b25d6f6f15b6800be7167f", size = 2225834, upload-time = "2025-11-03T22:32:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/4a/0d/7bed847b5c8c365e9f1953da274edc87577042bee5a5af21fba63276e756/libcst-1.8.6-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:536567441182a62fb706e7aa954aca034827b19746832205953b2c725d254a93", size = 2287107, upload-time = "2025-11-03T22:32:44.549Z" }, + { url = "https://files.pythonhosted.org/packages/02/f0/7e51fa84ade26c518bfbe7e2e4758b56d86a114c72d60309ac0d350426c4/libcst-1.8.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f04d3672bde1704f383a19e8f8331521abdbc1ed13abb349325a02ac56e5012", size = 2288672, upload-time = "2025-11-03T22:32:45.867Z" }, + { url = "https://files.pythonhosted.org/packages/ad/cd/15762659a3f5799d36aab1bc2b7e732672722e249d7800e3c5f943b41250/libcst-1.8.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:7f04febcd70e1e67917be7de513c8d4749d2e09206798558d7fe632134426ea4", size = 2392661, upload-time = "2025-11-03T22:32:47.232Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6b/b7f9246c323910fcbe021241500f82e357521495dcfe419004dbb272c7cb/libcst-1.8.6-cp313-cp313t-win_amd64.whl", hash = "sha256:1dc3b897c8b0f7323412da3f4ad12b16b909150efc42238e19cbf19b561cc330", size = 2105068, upload-time = "2025-11-03T22:32:49.145Z" }, + { url = "https://files.pythonhosted.org/packages/a6/0b/4fd40607bc4807ec2b93b054594373d7fa3d31bb983789901afcb9bcebe9/libcst-1.8.6-cp313-cp313t-win_arm64.whl", hash = "sha256:44f38139fa95e488db0f8976f9c7ca39a64d6bc09f2eceef260aa1f6da6a2e42", size = 1985181, upload-time = "2025-11-03T22:32:50.597Z" }, + { url = "https://files.pythonhosted.org/packages/3a/60/4105441989e321f7ad0fd28ffccb83eb6aac0b7cfb0366dab855dcccfbe5/libcst-1.8.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:b188e626ce61de5ad1f95161b8557beb39253de4ec74fc9b1f25593324a0279c", size = 2204202, upload-time = "2025-11-03T22:32:52.311Z" }, + { url = "https://files.pythonhosted.org/packages/67/2f/51a6f285c3a183e50cfe5269d4a533c21625aac2c8de5cdf2d41f079320d/libcst-1.8.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:87e74f7d7dfcba9efa91127081e22331d7c42515f0a0ac6e81d4cf2c3ed14661", size = 2083581, upload-time = "2025-11-03T22:32:54.269Z" }, + { url = "https://files.pythonhosted.org/packages/2f/64/921b1c19b638860af76cdb28bc81d430056592910b9478eea49e31a7f47a/libcst-1.8.6-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:3a926a4b42015ee24ddfc8ae940c97bd99483d286b315b3ce82f3bafd9f53474", size = 2236495, upload-time = "2025-11-03T22:32:55.723Z" }, + { url = "https://files.pythonhosted.org/packages/12/a8/b00592f9bede618cbb3df6ffe802fc65f1d1c03d48a10d353b108057d09c/libcst-1.8.6-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:3f4fbb7f569e69fd9e89d9d9caa57ca42c577c28ed05062f96a8c207594e75b8", size = 2301466, upload-time = "2025-11-03T22:32:57.337Z" }, + { url = "https://files.pythonhosted.org/packages/af/df/790d9002f31580fefd0aec2f373a0f5da99070e04c5e8b1c995d0104f303/libcst-1.8.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:08bd63a8ce674be431260649e70fca1d43f1554f1591eac657f403ff8ef82c7a", size = 2300264, upload-time = "2025-11-03T22:32:58.852Z" }, + { url = "https://files.pythonhosted.org/packages/21/de/dc3f10e65bab461be5de57850d2910a02c24c3ddb0da28f0e6e4133c3487/libcst-1.8.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e00e275d4ba95d4963431ea3e409aa407566a74ee2bf309a402f84fc744abe47", size = 2408572, upload-time = "2025-11-03T22:33:00.552Z" }, + { url = "https://files.pythonhosted.org/packages/20/3b/35645157a7590891038b077db170d6dd04335cd2e82a63bdaa78c3297dfe/libcst-1.8.6-cp314-cp314-win_amd64.whl", hash = "sha256:fea5c7fa26556eedf277d4f72779c5ede45ac3018650721edd77fd37ccd4a2d4", size = 2193917, upload-time = "2025-11-03T22:33:02.354Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a2/1034a9ba7d3e82f2c2afaad84ba5180f601aed676d92b76325797ad60951/libcst-1.8.6-cp314-cp314-win_arm64.whl", hash = "sha256:bb9b4077bdf8857b2483879cbbf70f1073bc255b057ec5aac8a70d901bb838e9", size = 2078748, upload-time = "2025-11-03T22:33:03.707Z" }, + { url = "https://files.pythonhosted.org/packages/95/a1/30bc61e8719f721a5562f77695e6154e9092d1bdf467aa35d0806dcd6cea/libcst-1.8.6-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:55ec021a296960c92e5a33b8d93e8ad4182b0eab657021f45262510a58223de1", size = 2188980, upload-time = "2025-11-03T22:33:05.152Z" }, + { url = "https://files.pythonhosted.org/packages/2c/14/c660204532407c5628e3b615015a902ed2d0b884b77714a6bdbe73350910/libcst-1.8.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ba9ab2b012fbd53b36cafd8f4440a6b60e7e487cd8b87428e57336b7f38409a4", size = 2074828, upload-time = "2025-11-03T22:33:06.864Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/c497c354943dff644749f177ee9737b09ed811b8fc842b05709a40fe0d1b/libcst-1.8.6-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c0a0cc80aebd8aa15609dd4d330611cbc05e9b4216bcaeabba7189f99ef07c28", size = 2225568, upload-time = "2025-11-03T22:33:08.354Z" }, + { url = "https://files.pythonhosted.org/packages/86/ef/45999676d07bd6d0eefa28109b4f97124db114e92f9e108de42ba46a8028/libcst-1.8.6-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:42a4f68121e2e9c29f49c97f6154e8527cd31021809cc4a941c7270aa64f41aa", size = 2286523, upload-time = "2025-11-03T22:33:10.206Z" }, + { url = "https://files.pythonhosted.org/packages/f4/6c/517d8bf57d9f811862f4125358caaf8cd3320a01291b3af08f7b50719db4/libcst-1.8.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a434c521fadaf9680788b50d5c21f4048fa85ed19d7d70bd40549fbaeeecab1", size = 2288044, upload-time = "2025-11-03T22:33:11.628Z" }, + { url = "https://files.pythonhosted.org/packages/83/ce/24d7d49478ffb61207f229239879845da40a374965874f5ee60f96b02ddb/libcst-1.8.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6a65f844d813ab4ef351443badffa0ae358f98821561d19e18b3190f59e71996", size = 2392605, upload-time = "2025-11-03T22:33:12.962Z" }, + { url = "https://files.pythonhosted.org/packages/39/c3/829092ead738b71e96a4e96896c96f276976e5a8a58b4473ed813d7c962b/libcst-1.8.6-cp314-cp314t-win_amd64.whl", hash = "sha256:bdb14bc4d4d83a57062fed2c5da93ecb426ff65b0dc02ddf3481040f5f074a82", size = 2181581, upload-time = "2025-11-03T22:33:14.514Z" }, + { url = "https://files.pythonhosted.org/packages/98/6d/5d6a790a02eb0d9d36c4aed4f41b277497e6178900b2fa29c35353aa45ed/libcst-1.8.6-cp314-cp314t-win_arm64.whl", hash = "sha256:819c8081e2948635cab60c603e1bbdceccdfe19104a242530ad38a36222cb88f", size = 2065000, upload-time = "2025-11-03T22:33:16.257Z" }, +] + +[[package]] +name = "lxml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426, upload-time = "2025-09-22T04:04:59.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/c8/8ff2bc6b920c84355146cd1ab7d181bc543b89241cfb1ebee824a7c81457/lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456", size = 8661887, upload-time = "2025-09-22T04:01:17.265Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/9aae1008083bb501ef63284220ce81638332f9ccbfa53765b2b7502203cf/lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924", size = 4667818, upload-time = "2025-09-22T04:01:19.688Z" }, + { url = "https://files.pythonhosted.org/packages/f1/ca/31fb37f99f37f1536c133476674c10b577e409c0a624384147653e38baf2/lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f", size = 4950807, upload-time = "2025-09-22T04:01:21.487Z" }, + { url = "https://files.pythonhosted.org/packages/da/87/f6cb9442e4bada8aab5ae7e1046264f62fdbeaa6e3f6211b93f4c0dd97f1/lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534", size = 5109179, upload-time = "2025-09-22T04:01:23.32Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/a7760713e65888db79bbae4f6146a6ae5c04e4a204a3c48896c408cd6ed2/lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564", size = 5023044, upload-time = "2025-09-22T04:01:25.118Z" }, + { url = "https://files.pythonhosted.org/packages/a2/b0/7e64e0460fcb36471899f75831509098f3fd7cd02a3833ac517433cb4f8f/lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f", size = 5359685, upload-time = "2025-09-22T04:01:27.398Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e1/e5df362e9ca4e2f48ed6411bd4b3a0ae737cc842e96877f5bf9428055ab4/lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0", size = 5654127, upload-time = "2025-09-22T04:01:29.629Z" }, + { url = "https://files.pythonhosted.org/packages/c6/d1/232b3309a02d60f11e71857778bfcd4acbdb86c07db8260caf7d008b08f8/lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192", size = 5253958, upload-time = "2025-09-22T04:01:31.535Z" }, + { url = "https://files.pythonhosted.org/packages/35/35/d955a070994725c4f7d80583a96cab9c107c57a125b20bb5f708fe941011/lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0", size = 4711541, upload-time = "2025-09-22T04:01:33.801Z" }, + { url = "https://files.pythonhosted.org/packages/1e/be/667d17363b38a78c4bd63cfd4b4632029fd68d2c2dc81f25ce9eb5224dd5/lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092", size = 5267426, upload-time = "2025-09-22T04:01:35.639Z" }, + { url = "https://files.pythonhosted.org/packages/ea/47/62c70aa4a1c26569bc958c9ca86af2bb4e1f614e8c04fb2989833874f7ae/lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f", size = 5064917, upload-time = "2025-09-22T04:01:37.448Z" }, + { url = "https://files.pythonhosted.org/packages/bd/55/6ceddaca353ebd0f1908ef712c597f8570cc9c58130dbb89903198e441fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8", size = 4788795, upload-time = "2025-09-22T04:01:39.165Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e8/fd63e15da5e3fd4c2146f8bbb3c14e94ab850589beab88e547b2dbce22e1/lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f", size = 5676759, upload-time = "2025-09-22T04:01:41.506Z" }, + { url = "https://files.pythonhosted.org/packages/76/47/b3ec58dc5c374697f5ba37412cd2728f427d056315d124dd4b61da381877/lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6", size = 5255666, upload-time = "2025-09-22T04:01:43.363Z" }, + { url = "https://files.pythonhosted.org/packages/19/93/03ba725df4c3d72afd9596eef4a37a837ce8e4806010569bedfcd2cb68fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322", size = 5277989, upload-time = "2025-09-22T04:01:45.215Z" }, + { url = "https://files.pythonhosted.org/packages/c6/80/c06de80bfce881d0ad738576f243911fccf992687ae09fd80b734712b39c/lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849", size = 3611456, upload-time = "2025-09-22T04:01:48.243Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d7/0cdfb6c3e30893463fb3d1e52bc5f5f99684a03c29a0b6b605cfae879cd5/lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f", size = 4011793, upload-time = "2025-09-22T04:01:50.042Z" }, + { url = "https://files.pythonhosted.org/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836, upload-time = "2025-09-22T04:01:52.145Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/4e8f0540608977aea078bf6d79f128e0e2c2bba8af1acf775c30baa70460/lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77", size = 8648494, upload-time = "2025-09-22T04:01:54.242Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f4/2a94a3d3dfd6c6b433501b8d470a1960a20ecce93245cf2db1706adf6c19/lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f", size = 4661146, upload-time = "2025-09-22T04:01:56.282Z" }, + { url = "https://files.pythonhosted.org/packages/25/2e/4efa677fa6b322013035d38016f6ae859d06cac67437ca7dc708a6af7028/lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452", size = 4946932, upload-time = "2025-09-22T04:01:58.989Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0f/526e78a6d38d109fdbaa5049c62e1d32fdd70c75fb61c4eadf3045d3d124/lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048", size = 5100060, upload-time = "2025-09-22T04:02:00.812Z" }, + { url = "https://files.pythonhosted.org/packages/81/76/99de58d81fa702cc0ea7edae4f4640416c2062813a00ff24bd70ac1d9c9b/lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df", size = 5019000, upload-time = "2025-09-22T04:02:02.671Z" }, + { url = "https://files.pythonhosted.org/packages/b5/35/9e57d25482bc9a9882cb0037fdb9cc18f4b79d85df94fa9d2a89562f1d25/lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1", size = 5348496, upload-time = "2025-09-22T04:02:04.904Z" }, + { url = "https://files.pythonhosted.org/packages/a6/8e/cb99bd0b83ccc3e8f0f528e9aa1f7a9965dfec08c617070c5db8d63a87ce/lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916", size = 5643779, upload-time = "2025-09-22T04:02:06.689Z" }, + { url = "https://files.pythonhosted.org/packages/d0/34/9e591954939276bb679b73773836c6684c22e56d05980e31d52a9a8deb18/lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd", size = 5244072, upload-time = "2025-09-22T04:02:08.587Z" }, + { url = "https://files.pythonhosted.org/packages/8d/27/b29ff065f9aaca443ee377aff699714fcbffb371b4fce5ac4ca759e436d5/lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6", size = 4718675, upload-time = "2025-09-22T04:02:10.783Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f756f9c2cd27caa1a6ef8c32ae47aadea697f5c2c6d07b0dae133c244fbe/lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a", size = 5255171, upload-time = "2025-09-22T04:02:12.631Z" }, + { url = "https://files.pythonhosted.org/packages/61/46/bb85ea42d2cb1bd8395484fd72f38e3389611aa496ac7772da9205bbda0e/lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679", size = 5057175, upload-time = "2025-09-22T04:02:14.718Z" }, + { url = "https://files.pythonhosted.org/packages/95/0c/443fc476dcc8e41577f0af70458c50fe299a97bb6b7505bb1ae09aa7f9ac/lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659", size = 4785688, upload-time = "2025-09-22T04:02:16.957Z" }, + { url = "https://files.pythonhosted.org/packages/48/78/6ef0b359d45bb9697bc5a626e1992fa5d27aa3f8004b137b2314793b50a0/lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484", size = 5660655, upload-time = "2025-09-22T04:02:18.815Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ea/e1d33808f386bc1339d08c0dcada6e4712d4ed8e93fcad5f057070b7988a/lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2", size = 5247695, upload-time = "2025-09-22T04:02:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/4f/47/eba75dfd8183673725255247a603b4ad606f4ae657b60c6c145b381697da/lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314", size = 5269841, upload-time = "2025-09-22T04:02:22.489Z" }, + { url = "https://files.pythonhosted.org/packages/76/04/5c5e2b8577bc936e219becb2e98cdb1aca14a4921a12995b9d0c523502ae/lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2", size = 3610700, upload-time = "2025-09-22T04:02:24.465Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0a/4643ccc6bb8b143e9f9640aa54e38255f9d3b45feb2cbe7ae2ca47e8782e/lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7", size = 4010347, upload-time = "2025-09-22T04:02:26.286Z" }, + { url = "https://files.pythonhosted.org/packages/31/ef/dcf1d29c3f530577f61e5fe2f1bd72929acf779953668a8a47a479ae6f26/lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf", size = 3671248, upload-time = "2025-09-22T04:02:27.918Z" }, + { url = "https://files.pythonhosted.org/packages/03/15/d4a377b385ab693ce97b472fe0c77c2b16ec79590e688b3ccc71fba19884/lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe", size = 8659801, upload-time = "2025-09-22T04:02:30.113Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e8/c128e37589463668794d503afaeb003987373c5f94d667124ffd8078bbd9/lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d", size = 4659403, upload-time = "2025-09-22T04:02:32.119Z" }, + { url = "https://files.pythonhosted.org/packages/00/ce/74903904339decdf7da7847bb5741fc98a5451b42fc419a86c0c13d26fe2/lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d", size = 4966974, upload-time = "2025-09-22T04:02:34.155Z" }, + { url = "https://files.pythonhosted.org/packages/1f/d3/131dec79ce61c5567fecf82515bd9bc36395df42501b50f7f7f3bd065df0/lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5", size = 5102953, upload-time = "2025-09-22T04:02:36.054Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ea/a43ba9bb750d4ffdd885f2cd333572f5bb900cd2408b67fdda07e85978a0/lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0", size = 5055054, upload-time = "2025-09-22T04:02:38.154Z" }, + { url = "https://files.pythonhosted.org/packages/60/23/6885b451636ae286c34628f70a7ed1fcc759f8d9ad382d132e1c8d3d9bfd/lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba", size = 5352421, upload-time = "2025-09-22T04:02:40.413Z" }, + { url = "https://files.pythonhosted.org/packages/48/5b/fc2ddfc94ddbe3eebb8e9af6e3fd65e2feba4967f6a4e9683875c394c2d8/lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0", size = 5673684, upload-time = "2025-09-22T04:02:42.288Z" }, + { url = "https://files.pythonhosted.org/packages/29/9c/47293c58cc91769130fbf85531280e8cc7868f7fbb6d92f4670071b9cb3e/lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d", size = 5252463, upload-time = "2025-09-22T04:02:44.165Z" }, + { url = "https://files.pythonhosted.org/packages/9b/da/ba6eceb830c762b48e711ded880d7e3e89fc6c7323e587c36540b6b23c6b/lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37", size = 4698437, upload-time = "2025-09-22T04:02:46.524Z" }, + { url = "https://files.pythonhosted.org/packages/a5/24/7be3f82cb7990b89118d944b619e53c656c97dc89c28cfb143fdb7cd6f4d/lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9", size = 5269890, upload-time = "2025-09-22T04:02:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/1b/bd/dcfb9ea1e16c665efd7538fc5d5c34071276ce9220e234217682e7d2c4a5/lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917", size = 5097185, upload-time = "2025-09-22T04:02:50.746Z" }, + { url = "https://files.pythonhosted.org/packages/21/04/a60b0ff9314736316f28316b694bccbbabe100f8483ad83852d77fc7468e/lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f", size = 4745895, upload-time = "2025-09-22T04:02:52.968Z" }, + { url = "https://files.pythonhosted.org/packages/d6/bd/7d54bd1846e5a310d9c715921c5faa71cf5c0853372adf78aee70c8d7aa2/lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8", size = 5695246, upload-time = "2025-09-22T04:02:54.798Z" }, + { url = "https://files.pythonhosted.org/packages/fd/32/5643d6ab947bc371da21323acb2a6e603cedbe71cb4c99c8254289ab6f4e/lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a", size = 5260797, upload-time = "2025-09-22T04:02:57.058Z" }, + { url = "https://files.pythonhosted.org/packages/33/da/34c1ec4cff1eea7d0b4cd44af8411806ed943141804ac9c5d565302afb78/lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c", size = 5277404, upload-time = "2025-09-22T04:02:58.966Z" }, + { url = "https://files.pythonhosted.org/packages/82/57/4eca3e31e54dc89e2c3507e1cd411074a17565fa5ffc437c4ae0a00d439e/lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b", size = 3670072, upload-time = "2025-09-22T04:03:38.05Z" }, + { url = "https://files.pythonhosted.org/packages/e3/e0/c96cf13eccd20c9421ba910304dae0f619724dcf1702864fd59dd386404d/lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed", size = 4080617, upload-time = "2025-09-22T04:03:39.835Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5d/b3f03e22b3d38d6f188ef044900a9b29b2fe0aebb94625ce9fe244011d34/lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8", size = 3754930, upload-time = "2025-09-22T04:03:41.565Z" }, + { url = "https://files.pythonhosted.org/packages/5e/5c/42c2c4c03554580708fc738d13414801f340c04c3eff90d8d2d227145275/lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d", size = 8910380, upload-time = "2025-09-22T04:03:01.645Z" }, + { url = "https://files.pythonhosted.org/packages/bf/4f/12df843e3e10d18d468a7557058f8d3733e8b6e12401f30b1ef29360740f/lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba", size = 4775632, upload-time = "2025-09-22T04:03:03.814Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0c/9dc31e6c2d0d418483cbcb469d1f5a582a1cd00a1f4081953d44051f3c50/lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601", size = 4975171, upload-time = "2025-09-22T04:03:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/e7/2b/9b870c6ca24c841bdd887504808f0417aa9d8d564114689266f19ddf29c8/lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed", size = 5110109, upload-time = "2025-09-22T04:03:07.452Z" }, + { url = "https://files.pythonhosted.org/packages/bf/0c/4f5f2a4dd319a178912751564471355d9019e220c20d7db3fb8307ed8582/lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37", size = 5041061, upload-time = "2025-09-22T04:03:09.297Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/554eed290365267671fe001a20d72d14f468ae4e6acef1e179b039436967/lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338", size = 5306233, upload-time = "2025-09-22T04:03:11.651Z" }, + { url = "https://files.pythonhosted.org/packages/7a/31/1d748aa275e71802ad9722df32a7a35034246b42c0ecdd8235412c3396ef/lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9", size = 5604739, upload-time = "2025-09-22T04:03:13.592Z" }, + { url = "https://files.pythonhosted.org/packages/8f/41/2c11916bcac09ed561adccacceaedd2bf0e0b25b297ea92aab99fd03d0fa/lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd", size = 5225119, upload-time = "2025-09-22T04:03:15.408Z" }, + { url = "https://files.pythonhosted.org/packages/99/05/4e5c2873d8f17aa018e6afde417c80cc5d0c33be4854cce3ef5670c49367/lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d", size = 4633665, upload-time = "2025-09-22T04:03:17.262Z" }, + { url = "https://files.pythonhosted.org/packages/0f/c9/dcc2da1bebd6275cdc723b515f93edf548b82f36a5458cca3578bc899332/lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9", size = 5234997, upload-time = "2025-09-22T04:03:19.14Z" }, + { url = "https://files.pythonhosted.org/packages/9c/e2/5172e4e7468afca64a37b81dba152fc5d90e30f9c83c7c3213d6a02a5ce4/lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e", size = 5090957, upload-time = "2025-09-22T04:03:21.436Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b3/15461fd3e5cd4ddcb7938b87fc20b14ab113b92312fc97afe65cd7c85de1/lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d", size = 4764372, upload-time = "2025-09-22T04:03:23.27Z" }, + { url = "https://files.pythonhosted.org/packages/05/33/f310b987c8bf9e61c4dd8e8035c416bd3230098f5e3cfa69fc4232de7059/lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec", size = 5634653, upload-time = "2025-09-22T04:03:25.767Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/51c80e75e0bc9382158133bdcf4e339b5886c6ee2418b5199b3f1a61ed6d/lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272", size = 5233795, upload-time = "2025-09-22T04:03:27.62Z" }, + { url = "https://files.pythonhosted.org/packages/56/4d/4856e897df0d588789dd844dbed9d91782c4ef0b327f96ce53c807e13128/lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f", size = 5257023, upload-time = "2025-09-22T04:03:30.056Z" }, + { url = "https://files.pythonhosted.org/packages/0f/85/86766dfebfa87bea0ab78e9ff7a4b4b45225df4b4d3b8cc3c03c5cd68464/lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312", size = 3911420, upload-time = "2025-09-22T04:03:32.198Z" }, + { url = "https://files.pythonhosted.org/packages/fe/1a/b248b355834c8e32614650b8008c69ffeb0ceb149c793961dd8c0b991bb3/lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca", size = 4406837, upload-time = "2025-09-22T04:03:34.027Z" }, + { url = "https://files.pythonhosted.org/packages/92/aa/df863bcc39c5e0946263454aba394de8a9084dbaff8ad143846b0d844739/lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c", size = 3822205, upload-time = "2025-09-22T04:03:36.249Z" }, +] + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "marshmallow" +version = "3.26.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/79/de6c16cc902f4fc372236926b0ce2ab7845268dcc30fb2fbb7f71b418631/marshmallow-3.26.2.tar.gz", hash = "sha256:bbe2adb5a03e6e3571b573f42527c6fe926e17467833660bebd11593ab8dfd57", size = 222095, upload-time = "2025-12-22T06:53:53.309Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/2f/5108cb3ee4ba6501748c4908b908e55f42a5b66245b4cfe0c99326e1ef6e/marshmallow-3.26.2-py3-none-any.whl", hash = "sha256:013fa8a3c4c276c24d26d84ce934dc964e2aa794345a0f8c7e5a7191482c8a73", size = 50964, upload-time = "2025-12-22T06:53:51.801Z" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "more-itertools" +version = "10.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431, upload-time = "2025-09-02T15:23:11.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "ordered-set" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/ca/bfac8bc689799bcca4157e0e0ced07e70ce125193fc2e166d2e685b7e2fe/ordered-set-4.1.0.tar.gz", hash = "sha256:694a8e44c87657c59292ede72891eb91d34131f6531463aab3009191c77364a8", size = 12826, upload-time = "2022-01-26T14:38:56.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/55/af02708f230eb77084a299d7b08175cff006dea4f2721074b92cdb0296c0/ordered_set-4.1.0-py3-none-any.whl", hash = "sha256:046e1132c71fcf3330438a539928932caf51ddbc582496833e23de611de14562", size = 7634, upload-time = "2022-01-26T14:38:48.677Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "parameterized" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/49/00c0c0cc24ff4266025a53e41336b79adaa5a4ebfad214f433d623f9865e/parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1", size = 24351, upload-time = "2023-03-27T02:01:11.592Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/2f/804f58f0b856ab3bf21617cccf5b39206e6c4c94c2cd227bde125ea6105f/parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b", size = 20475, upload-time = "2023-03-27T02:01:09.31Z" }, +] + +[[package]] +name = "parse" +version = "1.21.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/18/0bea374e5ec3c8ba15365570002187f3fef9d7265ffbc2f649529878cc80/parse-1.21.1.tar.gz", hash = "sha256:825e1a88e9d9fb481b8d2ca709c6195558b6eaa97c559ad3a9a20aa2d12815a3", size = 29105, upload-time = "2026-02-19T02:20:07.645Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/13/114daf766c33aec6c5a3954e7ea653f8a7ade9602c5c5a2228281698c490/parse-1.21.1-py2.py3-none-any.whl", hash = "sha256:55339ca698019815df3b8e8b550e5933933527e623b0cdf1ca2f404da35ffb47", size = 19693, upload-time = "2026-02-19T02:20:06.575Z" }, +] + +[[package]] +name = "parse-type" +version = "0.6.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "parse" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/ea/42ba6ce0abba04ab6e0b997dcb9b528a4661b62af1fe1b0d498120d5ea78/parse_type-0.6.6.tar.gz", hash = "sha256:513a3784104839770d690e04339a8b4d33439fcd5dd99f2e4580f9fc1097bfb2", size = 98012, upload-time = "2025-08-11T22:53:48.066Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/8d/eef3d8cdccc32abdd91b1286884c99b8c3a6d3b135affcc2a7a0f383bb32/parse_type-0.6.6-py2.py3-none-any.whl", hash = "sha256:3ca79bbe71e170dfccc8ec6c341edfd1c2a0fc1e5cfd18330f93af938de2348c", size = 27085, upload-time = "2025-08-11T22:53:46.396Z" }, +] + +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/04/fea538adf7dbbd6d186f551d595961e564a3b6715bdf276b477460858672/platformdirs-4.9.2.tar.gz", hash = "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291", size = 28394, upload-time = "2026-02-16T03:56:10.574Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/31/05e764397056194206169869b50cf2fee4dbbbc71b344705b9c0d878d4d8/platformdirs-4.9.2-py3-none-any.whl", hash = "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd", size = 21168, upload-time = "2026-02-16T03:56:08.891Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pycodestyle" +version = "2.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/e0/abfd2a0d2efe47670df87f3e3a0e2edda42f055053c85361f19c0e2c1ca8/pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783", size = 39472, upload-time = "2025-06-20T18:49:48.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/27/a58ddaf8c588a3ef080db9d0b7e0b97215cee3a45df74f3a94dbbf5c893a/pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d", size = 31594, upload-time = "2025-06-20T18:49:47.491Z" }, +] + +[[package]] +name = "pyecore" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "future-fstrings" }, + { name = "lxml" }, + { name = "ordered-set" }, + { name = "restrictedpython" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/67/7701370654cc2d3a7388ea742a43c416c99b665ef82450083b2e7389bbfb/pyecore-0.15.2.tar.gz", hash = "sha256:bddab6e86d2c8b6e8b824d9dba8bfdbf198d6f12bd020e6a8c3ddb1bc73d6c02", size = 58695, upload-time = "2024-12-12T14:11:47.878Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/09/55d1cbda2460464c1979e83237da92eee67c2b1741515818e2fb12800b72/pyecore-0.15.2-py3-none-any.whl", hash = "sha256:277250e1da2a888dff34a18aa3e8f16afb9bdd5b2484a6e917064c9702aeeb7d", size = 43694, upload-time = "2024-12-12T14:11:46.45Z" }, +] + +[[package]] +name = "pyflakes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/dc/fd034dc20b4b264b3d015808458391acbf9df40b1e54750ef175d39180b1/pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58", size = 64669, upload-time = "2025-06-20T18:45:27.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/2f/81d580a0fb83baeb066698975cb14a618bdbed7720678566f1b046a95fe8/pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f", size = 63551, upload-time = "2025-06-20T18:45:26.937Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pyhamcrest" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/3f/f286caba4e64391a8dc9200e6de6ce0d07471e3f718248c3276843b7793b/pyhamcrest-2.1.0.tar.gz", hash = "sha256:c6acbec0923d0cb7e72c22af1926f3e7c97b8e8d69fc7498eabacaf7c975bd9c", size = 60538, upload-time = "2023-10-22T15:47:28.255Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/71/1b25d3797a24add00f6f8c1bb0ac03a38616e2ec6606f598c1d50b0b0ffb/pyhamcrest-2.1.0-py3-none-any.whl", hash = "sha256:f6913d2f392e30e0375b3ecbd7aee79e5d1faa25d345c8f4ff597665dcac2587", size = 54555, upload-time = "2023-10-22T15:47:25.08Z" }, +] + +[[package]] +name = "pyperclip" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/52/d87eba7cb129b81563019d1679026e7a112ef76855d6159d24754dbd2a51/pyperclip-1.11.0.tar.gz", hash = "sha256:244035963e4428530d9e3a6101a1ef97209c6825edab1567beac148ccc1db1b6", size = 12185, upload-time = "2025-09-26T14:40:37.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/80/fc9d01d5ed37ba4c42ca2b55b4339ae6e200b456be3a1aaddf4a9fa99b8c/pyperclip-1.11.0-py3-none-any.whl", hash = "sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273", size = 11063, upload-time = "2025-09-26T14:40:36.069Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "pytest-bdd" +version = "8.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gherkin-official" }, + { name = "mako" }, + { name = "packaging" }, + { name = "parse" }, + { name = "parse-type" }, + { name = "pytest" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/2f/14c2e55372a5718a93b56aea48cd6ccc15d2d245364e516cd7b19bbd07ad/pytest_bdd-8.1.0.tar.gz", hash = "sha256:ef0896c5cd58816dc49810e8ff1d632f4a12019fb3e49959b2d349ffc1c9bfb5", size = 56147, upload-time = "2024-12-05T21:45:58.83Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/7d/1461076b0cc9a9e6fa8b51b9dea2677182ba8bc248d99d95ca321f2c666f/pytest_bdd-8.1.0-py3-none-any.whl", hash = "sha256:2124051e71a05ad7db15296e39013593f72ebf96796e1b023a40e5453c47e5fb", size = 49149, upload-time = "2024-12-05T21:45:56.184Z" }, +] + +[[package]] +name = "pytest-black" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "black" }, + { name = "pytest" }, + { name = "toml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/11/20/b0a2b3e1c09b61831d5c702d9e0579fc935e6a2527ce66dec1ba580a722b/pytest_black-0.6.0.tar.gz", hash = "sha256:ecb77455f379805cb4bd8f45a813a3754c3bbee3199adf1b3665c0dfd086b511", size = 6281, upload-time = "2024-12-15T17:15:26.329Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/0f/71303b06ef91f6e9447efb14cc37755b2224ff7109422fef125cb357d7d3/pytest_black-0.6.0-py3-none-any.whl", hash = "sha256:7eb747f54b6c997497b5cbc66a988be114b92016dbfa66d210d1d1f9f6b2dc76", size = 4592, upload-time = "2024-12-15T17:15:22.891Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, +] + +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, +] + +[[package]] +name = "pytest-profiling" +version = "1.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gprof2dot" }, + { name = "pytest" }, + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/44/74/806cafd6f2108d37979ec71e73b2ff7f7db88eabd19d3b79c5d6cc229c36/pytest-profiling-1.8.1.tar.gz", hash = "sha256:3f171fa69d5c82fa9aab76d66abd5f59da69135c37d6ae5bf7557f1b154cb08d", size = 33135, upload-time = "2024-11-29T19:34:13.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/ac/c428c66241a144617a8af7a28e2e055e1438d23b949b62ac4b401a69fb79/pytest_profiling-1.8.1-py3-none-any.whl", hash = "sha256:3dd8713a96298b42d83de8f5951df3ada3e61b3e5d2a06956684175529e17aea", size = 9929, upload-time = "2024-11-29T19:33:02.111Z" }, +] + +[[package]] +name = "pytokens" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/34/b4e015b99031667a7b960f888889c5bd34ef585c85e1cb56a594b92836ac/pytokens-0.4.1.tar.gz", hash = "sha256:292052fe80923aae2260c073f822ceba21f3872ced9a68bb7953b348e561179a", size = 23015, upload-time = "2026-01-30T01:03:45.924Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/5d/e44573011401fb82e9d51e97f1290ceb377800fb4eed650b96f4753b499c/pytokens-0.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:140709331e846b728475786df8aeb27d24f48cbcf7bcd449f8de75cae7a45083", size = 160663, upload-time = "2026-01-30T01:03:06.473Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/5bbc3019f8e6f21d09c41f8b8654536117e5e211a85d89212d59cbdab381/pytokens-0.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d6c4268598f762bc8e91f5dbf2ab2f61f7b95bdc07953b602db879b3c8c18e1", size = 255626, upload-time = "2026-01-30T01:03:08.177Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3c/2d5297d82286f6f3d92770289fd439956b201c0a4fc7e72efb9b2293758e/pytokens-0.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:24afde1f53d95348b5a0eb19488661147285ca4dd7ed752bbc3e1c6242a304d1", size = 269779, upload-time = "2026-01-30T01:03:09.756Z" }, + { url = "https://files.pythonhosted.org/packages/20/01/7436e9ad693cebda0551203e0bf28f7669976c60ad07d6402098208476de/pytokens-0.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ad948d085ed6c16413eb5fec6b3e02fa00dc29a2534f088d3302c47eb59adf9", size = 268076, upload-time = "2026-01-30T01:03:10.957Z" }, + { url = "https://files.pythonhosted.org/packages/2e/df/533c82a3c752ba13ae7ef238b7f8cdd272cf1475f03c63ac6cf3fcfb00b6/pytokens-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:3f901fe783e06e48e8cbdc82d631fca8f118333798193e026a50ce1b3757ea68", size = 103552, upload-time = "2026-01-30T01:03:12.066Z" }, + { url = "https://files.pythonhosted.org/packages/cb/dc/08b1a080372afda3cceb4f3c0a7ba2bde9d6a5241f1edb02a22a019ee147/pytokens-0.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8bdb9d0ce90cbf99c525e75a2fa415144fd570a1ba987380190e8b786bc6ef9b", size = 160720, upload-time = "2026-01-30T01:03:13.843Z" }, + { url = "https://files.pythonhosted.org/packages/64/0c/41ea22205da480837a700e395507e6a24425151dfb7ead73343d6e2d7ffe/pytokens-0.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5502408cab1cb18e128570f8d598981c68a50d0cbd7c61312a90507cd3a1276f", size = 254204, upload-time = "2026-01-30T01:03:14.886Z" }, + { url = "https://files.pythonhosted.org/packages/e0/d2/afe5c7f8607018beb99971489dbb846508f1b8f351fcefc225fcf4b2adc0/pytokens-0.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d1d8fb1030af4d231789959f21821ab6325e463f0503a61d204343c9b355d1", size = 268423, upload-time = "2026-01-30T01:03:15.936Z" }, + { url = "https://files.pythonhosted.org/packages/68/d4/00ffdbd370410c04e9591da9220a68dc1693ef7499173eb3e30d06e05ed1/pytokens-0.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b08dd6b86058b6dc07efe9e98414f5102974716232d10f32ff39701e841c4", size = 266859, upload-time = "2026-01-30T01:03:17.458Z" }, + { url = "https://files.pythonhosted.org/packages/a7/c9/c3161313b4ca0c601eeefabd3d3b576edaa9afdefd32da97210700e47652/pytokens-0.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:9bd7d7f544d362576be74f9d5901a22f317efc20046efe2034dced238cbbfe78", size = 103520, upload-time = "2026-01-30T01:03:18.652Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a7/b470f672e6fc5fee0a01d9e75005a0e617e162381974213a945fcd274843/pytokens-0.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4a14d5f5fc78ce85e426aa159489e2d5961acf0e47575e08f35584009178e321", size = 160821, upload-time = "2026-01-30T01:03:19.684Z" }, + { url = "https://files.pythonhosted.org/packages/80/98/e83a36fe8d170c911f864bfded690d2542bfcfacb9c649d11a9e6eb9dc41/pytokens-0.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f50fd18543be72da51dd505e2ed20d2228c74e0464e4262e4899797803d7fa", size = 254263, upload-time = "2026-01-30T01:03:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/0f/95/70d7041273890f9f97a24234c00b746e8da86df462620194cef1d411ddeb/pytokens-0.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dc74c035f9bfca0255c1af77ddd2d6ae8419012805453e4b0e7513e17904545d", size = 268071, upload-time = "2026-01-30T01:03:21.888Z" }, + { url = "https://files.pythonhosted.org/packages/da/79/76e6d09ae19c99404656d7db9c35dfd20f2086f3eb6ecb496b5b31163bad/pytokens-0.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f66a6bbe741bd431f6d741e617e0f39ec7257ca1f89089593479347cc4d13324", size = 271716, upload-time = "2026-01-30T01:03:23.633Z" }, + { url = "https://files.pythonhosted.org/packages/79/37/482e55fa1602e0a7ff012661d8c946bafdc05e480ea5a32f4f7e336d4aa9/pytokens-0.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:b35d7e5ad269804f6697727702da3c517bb8a5228afa450ab0fa787732055fc9", size = 104539, upload-time = "2026-01-30T01:03:24.788Z" }, + { url = "https://files.pythonhosted.org/packages/30/e8/20e7db907c23f3d63b0be3b8a4fd1927f6da2395f5bcc7f72242bb963dfe/pytokens-0.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8fcb9ba3709ff77e77f1c7022ff11d13553f3c30299a9fe246a166903e9091eb", size = 168474, upload-time = "2026-01-30T01:03:26.428Z" }, + { url = "https://files.pythonhosted.org/packages/d6/81/88a95ee9fafdd8f5f3452107748fd04c24930d500b9aba9738f3ade642cc/pytokens-0.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:79fc6b8699564e1f9b521582c35435f1bd32dd06822322ec44afdeba666d8cb3", size = 290473, upload-time = "2026-01-30T01:03:27.415Z" }, + { url = "https://files.pythonhosted.org/packages/cf/35/3aa899645e29b6375b4aed9f8d21df219e7c958c4c186b465e42ee0a06bf/pytokens-0.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d31b97b3de0f61571a124a00ffe9a81fb9939146c122c11060725bd5aea79975", size = 303485, upload-time = "2026-01-30T01:03:28.558Z" }, + { url = "https://files.pythonhosted.org/packages/52/a0/07907b6ff512674d9b201859f7d212298c44933633c946703a20c25e9d81/pytokens-0.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:967cf6e3fd4adf7de8fc73cd3043754ae79c36475c1c11d514fc72cf5490094a", size = 306698, upload-time = "2026-01-30T01:03:29.653Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/cbbf9250020a4a8dd53ba83a46c097b69e5eb49dd14e708f496f548c6612/pytokens-0.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:584c80c24b078eec1e227079d56dc22ff755e0ba8654d8383b2c549107528918", size = 116287, upload-time = "2026-01-30T01:03:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/c6/78/397db326746f0a342855b81216ae1f0a32965deccfd7c830a2dbc66d2483/pytokens-0.4.1-py3-none-any.whl", hash = "sha256:26cef14744a8385f35d0e095dc8b3a7583f6c953c2e3d269c7f82484bf5ad2de", size = 13729, upload-time = "2026-01-30T01:03:45.029Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "pyyaml-ft" +version = "8.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/eb/5a0d575de784f9a1f94e2b1288c6886f13f34185e13117ed530f32b6f8a8/pyyaml_ft-8.0.0.tar.gz", hash = "sha256:0c947dce03954c7b5d38869ed4878b2e6ff1d44b08a0d84dc83fdad205ae39ab", size = 141057, upload-time = "2025-06-10T15:32:15.613Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/ba/a067369fe61a2e57fb38732562927d5bae088c73cb9bb5438736a9555b29/pyyaml_ft-8.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8c1306282bc958bfda31237f900eb52c9bedf9b93a11f82e1aab004c9a5657a6", size = 187027, upload-time = "2025-06-10T15:31:48.722Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c5/a3d2020ce5ccfc6aede0d45bcb870298652ac0cf199f67714d250e0cdf39/pyyaml_ft-8.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30c5f1751625786c19de751e3130fc345ebcba6a86f6bddd6e1285342f4bbb69", size = 176146, upload-time = "2025-06-10T15:31:50.584Z" }, + { url = "https://files.pythonhosted.org/packages/e3/bb/23a9739291086ca0d3189eac7cd92b4d00e9fdc77d722ab610c35f9a82ba/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fa992481155ddda2e303fcc74c79c05eddcdbc907b888d3d9ce3ff3e2adcfb0", size = 746792, upload-time = "2025-06-10T15:31:52.304Z" }, + { url = "https://files.pythonhosted.org/packages/5f/c2/e8825f4ff725b7e560d62a3609e31d735318068e1079539ebfde397ea03e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cec6c92b4207004b62dfad1f0be321c9f04725e0f271c16247d8b39c3bf3ea42", size = 786772, upload-time = "2025-06-10T15:31:54.712Z" }, + { url = "https://files.pythonhosted.org/packages/35/be/58a4dcae8854f2fdca9b28d9495298fd5571a50d8430b1c3033ec95d2d0e/pyyaml_ft-8.0.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06237267dbcab70d4c0e9436d8f719f04a51123f0ca2694c00dd4b68c338e40b", size = 778723, upload-time = "2025-06-10T15:31:56.093Z" }, + { url = "https://files.pythonhosted.org/packages/86/ed/fed0da92b5d5d7340a082e3802d84c6dc9d5fa142954404c41a544c1cb92/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8a7f332bc565817644cdb38ffe4739e44c3e18c55793f75dddb87630f03fc254", size = 758478, upload-time = "2025-06-10T15:31:58.314Z" }, + { url = "https://files.pythonhosted.org/packages/f0/69/ac02afe286275980ecb2dcdc0156617389b7e0c0a3fcdedf155c67be2b80/pyyaml_ft-8.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7d10175a746be65f6feb86224df5d6bc5c049ebf52b89a88cf1cd78af5a367a8", size = 799159, upload-time = "2025-06-10T15:31:59.675Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ac/c492a9da2e39abdff4c3094ec54acac9747743f36428281fb186a03fab76/pyyaml_ft-8.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:58e1015098cf8d8aec82f360789c16283b88ca670fe4275ef6c48c5e30b22a96", size = 158779, upload-time = "2025-06-10T15:32:01.029Z" }, + { url = "https://files.pythonhosted.org/packages/5d/9b/41998df3298960d7c67653669f37710fa2d568a5fc933ea24a6df60acaf6/pyyaml_ft-8.0.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:e64fa5f3e2ceb790d50602b2fd4ec37abbd760a8c778e46354df647e7c5a4ebb", size = 191331, upload-time = "2025-06-10T15:32:02.602Z" }, + { url = "https://files.pythonhosted.org/packages/0f/16/2710c252ee04cbd74d9562ebba709e5a284faeb8ada88fcda548c9191b47/pyyaml_ft-8.0.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8d445bf6ea16bb93c37b42fdacfb2f94c8e92a79ba9e12768c96ecde867046d1", size = 182879, upload-time = "2025-06-10T15:32:04.466Z" }, + { url = "https://files.pythonhosted.org/packages/9a/40/ae8163519d937fa7bfa457b6f78439cc6831a7c2b170e4f612f7eda71815/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c56bb46b4fda34cbb92a9446a841da3982cdde6ea13de3fbd80db7eeeab8b49", size = 811277, upload-time = "2025-06-10T15:32:06.214Z" }, + { url = "https://files.pythonhosted.org/packages/f9/66/28d82dbff7f87b96f0eeac79b7d972a96b4980c1e445eb6a857ba91eda00/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dab0abb46eb1780da486f022dce034b952c8ae40753627b27a626d803926483b", size = 831650, upload-time = "2025-06-10T15:32:08.076Z" }, + { url = "https://files.pythonhosted.org/packages/e8/df/161c4566facac7d75a9e182295c223060373d4116dead9cc53a265de60b9/pyyaml_ft-8.0.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd48d639cab5ca50ad957b6dd632c7dd3ac02a1abe0e8196a3c24a52f5db3f7a", size = 815755, upload-time = "2025-06-10T15:32:09.435Z" }, + { url = "https://files.pythonhosted.org/packages/05/10/f42c48fa5153204f42eaa945e8d1fd7c10d6296841dcb2447bf7da1be5c4/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:052561b89d5b2a8e1289f326d060e794c21fa068aa11255fe71d65baf18a632e", size = 810403, upload-time = "2025-06-10T15:32:11.051Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d2/e369064aa51009eb9245399fd8ad2c562bd0bcd392a00be44b2a824ded7c/pyyaml_ft-8.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3bb4b927929b0cb162fb1605392a321e3333e48ce616cdcfa04a839271373255", size = 835581, upload-time = "2025-06-10T15:32:12.897Z" }, + { url = "https://files.pythonhosted.org/packages/c0/28/26534bed77109632a956977f60d8519049f545abc39215d086e33a61f1f2/pyyaml_ft-8.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:de04cfe9439565e32f178106c51dd6ca61afaa2907d143835d501d84703d3793", size = 171579, upload-time = "2025-06-10T15:32:14.34Z" }, +] + +[[package]] +name = "renaissance" +version = "0.3.1" +source = { virtual = "." } +dependencies = [ + { name = "antlr4-python3-runtime" }, + { name = "ast-comments" }, + { name = "clang" }, + { name = "dataclasses-json" }, + { name = "hypothesmith" }, + { name = "libclang" }, + { name = "libcst" }, + { name = "more-itertools" }, + { name = "networkx" }, + { name = "pyecore" }, + { name = "pyhamcrest" }, + { name = "pyperclip" }, + { name = "pyyaml" }, + { name = "termcolor" }, + { name = "textx" }, + { name = "tree-sitter" }, + { name = "tree-sitter-cpp" }, + { name = "tree-sitter-java" }, + { name = "tree-sitter-python" }, + { name = "typing-extensions" }, +] + +[package.dev-dependencies] +dev = [ + { name = "autopep8" }, + { name = "behave" }, + { name = "black" }, + { name = "coverage" }, + { name = "flake8" }, + { name = "hypothesis" }, + { name = "hypothesmith" }, + { name = "parameterized" }, + { name = "pytest" }, + { name = "pytest-bdd" }, + { name = "pytest-black" }, + { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "pytest-profiling" }, + { name = "ruff" }, +] +lint = [ + { name = "autopep8" }, + { name = "black" }, + { name = "flake8" }, + { name = "pytest-black" }, + { name = "ruff" }, +] +test = [ + { name = "behave" }, + { name = "coverage" }, + { name = "hypothesis" }, + { name = "hypothesmith" }, + { name = "parameterized" }, + { name = "pytest" }, + { name = "pytest-bdd" }, + { name = "pytest-cov" }, + { name = "pytest-mock" }, + { name = "pytest-profiling" }, +] + +[package.metadata] +requires-dist = [ + { name = "antlr4-python3-runtime", specifier = ">=4.13.2" }, + { name = "ast-comments", specifier = ">=1.0" }, + { name = "clang", specifier = "==18.1.8" }, + { name = "dataclasses-json", specifier = "==0.6.7" }, + { name = "hypothesmith", specifier = ">=0.3.3" }, + { name = "libclang", specifier = "==18.1.1" }, + { name = "libcst", specifier = ">=1.8.6" }, + { name = "more-itertools", specifier = ">=10.0" }, + { name = "networkx", specifier = ">=3.0" }, + { name = "pyecore", specifier = ">=0.14" }, + { name = "pyhamcrest", specifier = ">=2.1" }, + { name = "pyperclip", specifier = ">=1.8" }, + { name = "pyyaml", specifier = ">=6.0" }, + { name = "termcolor", specifier = ">=2.0" }, + { name = "textx", specifier = "==4.3.0" }, + { name = "tree-sitter", specifier = ">=0.25" }, + { name = "tree-sitter-cpp", specifier = "==0.23.4" }, + { name = "tree-sitter-java", specifier = "==0.23.5" }, + { name = "tree-sitter-python", specifier = "==0.25.0" }, + { name = "typing-extensions", specifier = ">=4.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "autopep8", specifier = ">=2.0" }, + { name = "behave" }, + { name = "black", specifier = ">=24.0" }, + { name = "coverage", specifier = ">=7.0" }, + { name = "flake8", specifier = ">=7.0" }, + { name = "hypothesis", specifier = ">=6.152.4" }, + { name = "hypothesmith", specifier = ">=0.3.3" }, + { name = "parameterized", specifier = ">=0.9" }, + { name = "pytest", specifier = ">=8.0" }, + { name = "pytest-bdd", specifier = "==8.1.0" }, + { name = "pytest-black", specifier = ">=0.6" }, + { name = "pytest-cov", specifier = ">=7.0" }, + { name = "pytest-mock", specifier = ">=3.15" }, + { name = "pytest-profiling", specifier = ">=1.8" }, + { name = "ruff", specifier = ">=0.15.19" }, +] +lint = [ + { name = "autopep8", specifier = ">=2.0" }, + { name = "black", specifier = ">=24.0" }, + { name = "flake8", specifier = ">=7.0" }, + { name = "pytest-black", specifier = ">=0.6" }, + { name = "ruff", specifier = ">=0.15.19" }, +] +test = [ + { name = "behave" }, + { name = "coverage", specifier = ">=7.0" }, + { name = "hypothesis", specifier = ">=6.152.4" }, + { name = "hypothesmith", specifier = ">=0.3.3" }, + { name = "parameterized", specifier = ">=0.9" }, + { name = "pytest", specifier = ">=8.0" }, + { name = "pytest-bdd", specifier = "==8.1.0" }, + { name = "pytest-cov", specifier = ">=7.0" }, + { name = "pytest-mock", specifier = ">=3.15" }, + { name = "pytest-profiling", specifier = ">=1.8" }, +] + +[[package]] +name = "restrictedpython" +version = "8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/1c/aec08bcb4ab14a1521579fbe21ceff2a634bb1f737f11cf7f9c8bb96e680/restrictedpython-8.1.tar.gz", hash = "sha256:4a69304aceacf6bee74bdf153c728221d4e3109b39acbfe00b3494927080d898", size = 838331, upload-time = "2025-10-19T14:11:32.531Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/c0/3848f4006f7e164ee20833ca984067e4b3fc99fe7f1dfa88b4927e681299/restrictedpython-8.1-py3-none-any.whl", hash = "sha256:4769449c6cdb10f2071649ba386902befff0eff2a8fd6217989fa7b16aeae926", size = 27651, upload-time = "2025-10-19T14:11:30.201Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/e6/15800dfde183a1a106594016c912b4c12d050a301989d1aca6cb63759fe8/ruff-0.15.19.tar.gz", hash = "sha256:edc27f7172a93b32b102687009d6a588508815072141543ae603a8b9b0823063", size = 4772071, upload-time = "2026-06-24T01:10:46.942Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/4c/9ded7626c39a0440c575bf69e2bf500d443388272c842662c59852ee7fcd/ruff-0.15.19-py3-none-linux_armv6l.whl", hash = "sha256:922d1eb283161564759bd49f507e91dc6112c15da8bd5b84ed714e086243cf86", size = 10950859, upload-time = "2026-06-24T01:10:38.491Z" }, + { url = "https://files.pythonhosted.org/packages/fb/ef/c211505ece1d00ef493d58e54e3b6383c946a21e9874774eb531f2512cf3/ruff-0.15.19-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:4d190d8f62a0b94aba8f721116538a9ee29b1e74d26650846ba9b99f0ae21c40", size = 11294529, upload-time = "2026-06-24T01:10:36.481Z" }, + { url = "https://files.pythonhosted.org/packages/fe/93/78d462e7d39968e58094dc57be7d09ffb14ce37da5b68ed70338a35a1f21/ruff-0.15.19-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5a2c86ba6870dd415a9d9eb8be94d7924ebec6a26ffc7958ec7ca29d4bff967d", size = 10641416, upload-time = "2026-06-24T01:10:48.923Z" }, + { url = "https://files.pythonhosted.org/packages/76/c4/5cb66cfd1f865d5cca908b86c93ac785e7f572193d3c7426079ca6643e24/ruff-0.15.19-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82b432bc087264aea70fd25ac198918b70bd9e2aa0db4297b0bb91bbfbbc63ce", size = 11015582, upload-time = "2026-06-24T01:10:30.089Z" }, + { url = "https://files.pythonhosted.org/packages/51/9f/8ecfaec10cf5eecd28fbc00ff4fb867db90a1be54bf3d39ebf93f893cd52/ruff-0.15.19-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8530a09d03b3a8c994f8b559a7dcdabc690bcd3f78ef276c38c83166798ebf56", size = 10744059, upload-time = "2026-06-24T01:10:32.48Z" }, + { url = "https://files.pythonhosted.org/packages/35/6b/983249d04562bc2d590edd75f32455cdb473affb3ba4bc8d883e939c697d/ruff-0.15.19-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:87bf21fb3875fe69f0eacc825411657e2e85589cce633c35c0adf1113649c62b", size = 11568461, upload-time = "2026-06-24T01:10:17.435Z" }, + { url = "https://files.pythonhosted.org/packages/eb/39/bc7794f127b18f492a3b4ee82bba5a900c985ff13b72b46f46e3c171ba34/ruff-0.15.19-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f9b229cb3ef56ecc2c1c8ebeca64b7a7740ccaef40a9eb097e78dde5a8560b83", size = 12429690, upload-time = "2026-06-24T01:10:40.638Z" }, + { url = "https://files.pythonhosted.org/packages/0a/3b/0de6859e698ed11c8a49e765196c8d333599b6a546c0715df39b6ba1aa2e/ruff-0.15.19-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6c754515be7b76afe6e7e62df7776709571bcfc1631183828afcf3bafa869e3", size = 11693067, upload-time = "2026-06-24T01:10:25.681Z" }, + { url = "https://files.pythonhosted.org/packages/89/3d/0b1f30f84bee9ae6ae8d349c2ba8b6f4b040966744efdd3acc804ae7c024/ruff-0.15.19-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a498f82e0f4d8904c4e0aea5139cdfac1f39d19a3c51d491292f63a36e83b2e", size = 11616911, upload-time = "2026-06-24T01:10:44.809Z" }, + { url = "https://files.pythonhosted.org/packages/4d/eb/c90bd3dfc12eed9032c2c1bfe05105b93a1b2c8bce555db6308315b853ce/ruff-0.15.19-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:d48caa34488fb521fd0ef4aea2b0e8fe758298df044138f0d67b687a6a0d07ed", size = 11649343, upload-time = "2026-06-24T01:10:23.472Z" }, + { url = "https://files.pythonhosted.org/packages/82/91/01caa13602a2f12fae5edbe8caf78b3c1e6db1293132aee6959eecce095c/ruff-0.15.19-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4171b6613effa9363cd46dd4f75bd1827b6d1b946b5e278ed0c600d305379445", size = 10977610, upload-time = "2026-06-24T01:10:50.892Z" }, + { url = "https://files.pythonhosted.org/packages/3c/51/acb817922feab9ecbb3201377d4dbe7a25f1395e46545820061973f03468/ruff-0.15.19-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:27c15b2a241dd4d995557949a094fe78b8ad99122a38ccae1595849bcc947b3f", size = 10744900, upload-time = "2026-06-24T01:10:42.726Z" }, + { url = "https://files.pythonhosted.org/packages/84/bc/5c8ca46b8a7a3f2b16cfbec88721d772b1c93912904e8f8c2e49470fea63/ruff-0.15.19-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ed03b7862d68f0a8771d50ee129980cbf1b113f96e250b73954bc292f689e0bb", size = 11293560, upload-time = "2026-06-24T01:10:21.262Z" }, + { url = "https://files.pythonhosted.org/packages/81/e0/4a888cbe4d5523b3f77a2b1fa043f46cfeba1b32eac35dcfadee0578fa8a/ruff-0.15.19-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:08143f0685ae278b30727ea72e90c61e5bd9c31b91aac4f5bb989538f73d24b8", size = 11696533, upload-time = "2026-06-24T01:10:53.046Z" }, + { url = "https://files.pythonhosted.org/packages/98/43/c34b2fcd79262a85161764a97aaca89c3e4f574340ab61430cefa2bdd2c1/ruff-0.15.19-py3-none-win32.whl", hash = "sha256:8f47f0f92952af2557212bb10cf3e695cd4cf28b2c6e42cdb18ec6c9ebfa19da", size = 10986299, upload-time = "2026-06-24T01:10:55.185Z" }, + { url = "https://files.pythonhosted.org/packages/22/e8/15fd23e02b2442b56b2026b455977bc3057aa34b26e6323d1e99e8531a9f/ruff-0.15.19-py3-none-win_amd64.whl", hash = "sha256:efeca47ee3f9d4a7162655a3b8e6ee4a878646044233978d4d2c1ff8cdd914f0", size = 12123473, upload-time = "2026-06-24T01:10:27.74Z" }, + { url = "https://files.pythonhosted.org/packages/30/66/9a73695e31eaee04f35d8475998bf8ab354465f9c638936d76111603dcc5/ruff-0.15.19-py3-none-win_arm64.whl", hash = "sha256:6c6b607466e47349332eb1d9be52fb1467423fc07c217341af41cd0f3f0573be", size = 11376779, upload-time = "2026-06-24T01:10:34.465Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "termcolor" +version = "3.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/79/cf31d7a93a8fdc6aa0fbb665be84426a8c5a557d9240b6239e9e11e35fc5/termcolor-3.3.0.tar.gz", hash = "sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5", size = 14434, upload-time = "2025-12-29T12:55:21.882Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/d1/8bb87d21e9aeb323cc03034f5eaf2c8f69841e40e4853c2627edf8111ed3/termcolor-3.3.0-py3-none-any.whl", hash = "sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5", size = 7734, upload-time = "2025-12-29T12:55:20.718Z" }, +] + +[[package]] +name = "textx" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "arpeggio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/fe/1cec25321efa564257bcad2fca7da8d21b2beb826b32013bc8f85e67ae64/textx-4.3.0.tar.gz", hash = "sha256:0facac8029ad124ef21e5838dd8eb67f10129efcee96ea3548f5fd62428a9880", size = 2224357, upload-time = "2025-11-25T03:44:55.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/ae/27c651f06e0a9b425779cdc6d3463586e4f63f0c0365585120b4a539cba1/textx-4.3.0-py3-none-any.whl", hash = "sha256:261535f7e2de1529604026d58bf7dae9e40788644def4d033ca781680fa5dae7", size = 68560, upload-time = "2025-11-25T03:44:48.288Z" }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, +] + +[[package]] +name = "tree-sitter" +version = "0.25.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/7c/0350cfc47faadc0d3cf7d8237a4e34032b3014ddf4a12ded9933e1648b55/tree-sitter-0.25.2.tar.gz", hash = "sha256:fe43c158555da46723b28b52e058ad444195afd1db3ca7720c59a254544e9c20", size = 177961, upload-time = "2025-09-25T17:37:59.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/9e/20c2a00a862f1c2897a436b17edb774e831b22218083b459d0d081c9db33/tree_sitter-0.25.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ddabfff809ffc983fc9963455ba1cecc90295803e06e140a4c83e94c1fa3d960", size = 146941, upload-time = "2025-09-25T17:37:34.813Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/8512e2062e652a1016e840ce36ba1cc33258b0dcc4e500d8089b4054afec/tree_sitter-0.25.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c0c0ab5f94938a23fe81928a21cc0fac44143133ccc4eb7eeb1b92f84748331c", size = 137699, upload-time = "2025-09-25T17:37:36.349Z" }, + { url = "https://files.pythonhosted.org/packages/47/8a/d48c0414db19307b0fb3bb10d76a3a0cbe275bb293f145ee7fba2abd668e/tree_sitter-0.25.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd12d80d91d4114ca097626eb82714618dcdfacd6a5e0955216c6485c350ef99", size = 607125, upload-time = "2025-09-25T17:37:37.725Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/b95f545e9fc5001b8a78636ef942a4e4e536580caa6a99e73dd0a02e87aa/tree_sitter-0.25.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b43a9e4c89d4d0839de27cd4d6902d33396de700e9ff4c5ab7631f277a85ead9", size = 635418, upload-time = "2025-09-25T17:37:38.922Z" }, + { url = "https://files.pythonhosted.org/packages/de/4d/b734bde3fb6f3513a010fa91f1f2875442cdc0382d6a949005cd84563d8f/tree_sitter-0.25.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fbb1706407c0e451c4f8cc016fec27d72d4b211fdd3173320b1ada7a6c74c3ac", size = 631250, upload-time = "2025-09-25T17:37:40.039Z" }, + { url = "https://files.pythonhosted.org/packages/46/f2/5f654994f36d10c64d50a192239599fcae46677491c8dd53e7579c35a3e3/tree_sitter-0.25.2-cp312-cp312-win_amd64.whl", hash = "sha256:6d0302550bbe4620a5dc7649517c4409d74ef18558276ce758419cf09e578897", size = 127156, upload-time = "2025-09-25T17:37:41.132Z" }, + { url = "https://files.pythonhosted.org/packages/67/23/148c468d410efcf0a9535272d81c258d840c27b34781d625f1f627e2e27d/tree_sitter-0.25.2-cp312-cp312-win_arm64.whl", hash = "sha256:0c8b6682cac77e37cfe5cf7ec388844957f48b7bd8d6321d0ca2d852994e10d5", size = 113984, upload-time = "2025-09-25T17:37:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/8c/67/67492014ce32729b63d7ef318a19f9cfedd855d677de5773476caf771e96/tree_sitter-0.25.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0628671f0de69bb279558ef6b640bcfc97864fe0026d840f872728a86cd6b6cd", size = 146926, upload-time = "2025-09-25T17:37:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/4e/9c/a278b15e6b263e86c5e301c82a60923fa7c59d44f78d7a110a89a413e640/tree_sitter-0.25.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f5ddcd3e291a749b62521f71fc953f66f5fd9743973fd6dd962b092773569601", size = 137712, upload-time = "2025-09-25T17:37:44.039Z" }, + { url = "https://files.pythonhosted.org/packages/54/9a/423bba15d2bf6473ba67846ba5244b988cd97a4b1ea2b146822162256794/tree_sitter-0.25.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd88fbb0f6c3a0f28f0a68d72df88e9755cf5215bae146f5a1bdc8362b772053", size = 607873, upload-time = "2025-09-25T17:37:45.477Z" }, + { url = "https://files.pythonhosted.org/packages/ed/4c/b430d2cb43f8badfb3a3fa9d6cd7c8247698187b5674008c9d67b2a90c8e/tree_sitter-0.25.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b878e296e63661c8e124177cc3084b041ba3f5936b43076d57c487822426f614", size = 636313, upload-time = "2025-09-25T17:37:46.68Z" }, + { url = "https://files.pythonhosted.org/packages/9d/27/5f97098dbba807331d666a0997662e82d066e84b17d92efab575d283822f/tree_sitter-0.25.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d77605e0d353ba3fe5627e5490f0fbfe44141bafa4478d88ef7954a61a848dae", size = 631370, upload-time = "2025-09-25T17:37:47.993Z" }, + { url = "https://files.pythonhosted.org/packages/d4/3c/87caaed663fabc35e18dc704cd0e9800a0ee2f22bd18b9cbe7c10799895d/tree_sitter-0.25.2-cp313-cp313-win_amd64.whl", hash = "sha256:463c032bd02052d934daa5f45d183e0521ceb783c2548501cf034b0beba92c9b", size = 127157, upload-time = "2025-09-25T17:37:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/d5/23/f8467b408b7988aff4ea40946a4bd1a2c1a73d17156a9d039bbaff1e2ceb/tree_sitter-0.25.2-cp313-cp313-win_arm64.whl", hash = "sha256:b3f63a1796886249bd22c559a5944d64d05d43f2be72961624278eff0dcc5cb8", size = 113975, upload-time = "2025-09-25T17:37:49.922Z" }, + { url = "https://files.pythonhosted.org/packages/07/e3/d9526ba71dfbbe4eba5e51d89432b4b333a49a1e70712aa5590cd22fc74f/tree_sitter-0.25.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:65d3c931013ea798b502782acab986bbf47ba2c452610ab0776cf4a8ef150fc0", size = 146776, upload-time = "2025-09-25T17:37:50.898Z" }, + { url = "https://files.pythonhosted.org/packages/42/97/4bd4ad97f85a23011dd8a535534bb1035c4e0bac1234d58f438e15cff51f/tree_sitter-0.25.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bda059af9d621918efb813b22fb06b3fe00c3e94079c6143fcb2c565eb44cb87", size = 137732, upload-time = "2025-09-25T17:37:51.877Z" }, + { url = "https://files.pythonhosted.org/packages/b6/19/1e968aa0b1b567988ed522f836498a6a9529a74aab15f09dd9ac1e41f505/tree_sitter-0.25.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eac4e8e4c7060c75f395feec46421eb61212cb73998dbe004b7384724f3682ab", size = 609456, upload-time = "2025-09-25T17:37:52.925Z" }, + { url = "https://files.pythonhosted.org/packages/48/b6/cf08f4f20f4c9094006ef8828555484e842fc468827ad6e56011ab668dbd/tree_sitter-0.25.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:260586381b23be33b6191a07cea3d44ecbd6c01aa4c6b027a0439145fcbc3358", size = 636772, upload-time = "2025-09-25T17:37:54.647Z" }, + { url = "https://files.pythonhosted.org/packages/57/e2/d42d55bf56360987c32bc7b16adb06744e425670b823fb8a5786a1cea991/tree_sitter-0.25.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7d2ee1acbacebe50ba0f85fff1bc05e65d877958f00880f49f9b2af38dce1af0", size = 631522, upload-time = "2025-09-25T17:37:55.833Z" }, + { url = "https://files.pythonhosted.org/packages/03/87/af9604ebe275a9345d88c3ace0cf2a1341aa3f8ef49dd9fc11662132df8a/tree_sitter-0.25.2-cp314-cp314-win_amd64.whl", hash = "sha256:4973b718fcadfb04e59e746abfbb0288694159c6aeecd2add59320c03368c721", size = 130864, upload-time = "2025-09-25T17:37:57.453Z" }, + { url = "https://files.pythonhosted.org/packages/a6/6e/e64621037357acb83d912276ffd30a859ef117f9c680f2e3cb955f47c680/tree_sitter-0.25.2-cp314-cp314-win_arm64.whl", hash = "sha256:b8d4429954a3beb3e844e2872610d2a4800ba4eb42bb1990c6a4b1949b18459f", size = 117470, upload-time = "2025-09-25T17:37:58.431Z" }, +] + +[[package]] +name = "tree-sitter-cpp" +version = "0.23.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/2c/4dd63d705a8933543cad9b92ff31be849b164fec91a6eb63475ebc9ce668/tree_sitter_cpp-0.23.4.tar.gz", hash = "sha256:6a59c4cebb1ad1dc2e8d586cf8a72b39d21b8108b7b139d089719e81a339e41d", size = 940358, upload-time = "2024-11-11T06:59:24.934Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ac/11d56670f7b048362db872ca866fd00ba2002a322ab179f047b7c0fb2910/tree_sitter_cpp-0.23.4-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aacb1759f0efd9dbc25bd8ee88184a340483018869f75412d9c3bc32c039a520", size = 287861, upload-time = "2024-11-11T06:59:15.005Z" }, + { url = "https://files.pythonhosted.org/packages/12/1c/0337c016bdc00a77a3326d12f10ee836401dd28f27db6fd5b7734bfb21ed/tree_sitter_cpp-0.23.4-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:bc3c404d9f0cbd87951213a85440afbf4c31e718f8d907fa9ee12bea4b8d276f", size = 315513, upload-time = "2024-11-11T06:59:16.679Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7b/dd38c049b10ed7fda118b903a1d28a8b55a36b98c30606ef90e8f374c6de/tree_sitter_cpp-0.23.4-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ccc43ddf1279d5d5a4ef190373f4cb16522801bec4492bcd4754edf2aeba2b7b", size = 334813, upload-time = "2024-11-11T06:59:18.253Z" }, + { url = "https://files.pythonhosted.org/packages/6a/4d/23e390234d2acd351f5563b1079c515d7c1fe13ddb7392cee543be74dda3/tree_sitter_cpp-0.23.4-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:773d2cafc08bbc0f998687fa33f42f378c1a371cdb582870c4d13abb06092706", size = 316110, upload-time = "2024-11-11T06:59:19.823Z" }, + { url = "https://files.pythonhosted.org/packages/32/c7/b94a7e0e803af9d3bd4608fb4f0cfb2e9e233abaf0a38c928bfb0b1a025d/tree_sitter_cpp-0.23.4-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:247d127f0eb6574b0f6b30c0151e0bd0774e2e7acf9c558bdf9fbb8adc2e80c0", size = 308242, upload-time = "2024-11-11T06:59:21.466Z" }, + { url = "https://files.pythonhosted.org/packages/37/7e/909e52b3dec09c475140b0e175511e275d0d00ba2dbd7c68102d377ae0f6/tree_sitter_cpp-0.23.4-cp39-abi3-win_amd64.whl", hash = "sha256:68606a45bea92669d155399e1239f771a7767d8683cd8f8e30e7d813107030ca", size = 290997, upload-time = "2024-11-11T06:59:22.432Z" }, + { url = "https://files.pythonhosted.org/packages/d4/6a/65435d4d1f4c735be7ffe52d7c2e7b8a7f7c2790343a2719c60c548611c8/tree_sitter_cpp-0.23.4-cp39-abi3-win_arm64.whl", hash = "sha256:712f84f18be94cbe2a148fa4fdf40fcf4a8c25a8f7670efb9f8a47ddec2fc281", size = 288203, upload-time = "2024-11-11T06:59:23.404Z" }, +] + +[[package]] +name = "tree-sitter-java" +version = "0.23.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/dc/eb9c8f96304e5d8ae1663126d89967a622a80937ad2909903569ccb7ec8f/tree_sitter_java-0.23.5.tar.gz", hash = "sha256:f5cd57b8f1270a7f0438878750d02ccc79421d45cca65ff284f1527e9ef02e38", size = 138121, upload-time = "2024-12-21T18:24:26.936Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/21/b3399780b440e1567a11d384d0ebb1aea9b642d0d98becf30fa55c0e3a3b/tree_sitter_java-0.23.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:355ce0308672d6f7013ec913dee4a0613666f4cda9044a7824240d17f38209df", size = 58926, upload-time = "2024-12-21T18:24:12.53Z" }, + { url = "https://files.pythonhosted.org/packages/57/ef/6406b444e2a93bc72a04e802f4107e9ecf04b8de4a5528830726d210599c/tree_sitter_java-0.23.5-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:24acd59c4720dedad80d548fe4237e43ef2b7a4e94c8549b0ca6e4c4d7bf6e69", size = 62288, upload-time = "2024-12-21T18:24:14.634Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6c/74b1c150d4f69c291ab0b78d5dd1b59712559bbe7e7daf6d8466d483463f/tree_sitter_java-0.23.5-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9401e7271f0b333df39fc8a8336a0caf1b891d9a2b89ddee99fae66b794fc5b7", size = 85533, upload-time = "2024-12-21T18:24:16.695Z" }, + { url = "https://files.pythonhosted.org/packages/29/09/e0d08f5c212062fd046db35c1015a2621c2631bc8b4aae5740d7adb276ad/tree_sitter_java-0.23.5-cp39-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:370b204b9500b847f6d0c5ad584045831cee69e9a3e4d878535d39e4a7e4c4f1", size = 84033, upload-time = "2024-12-21T18:24:18.758Z" }, + { url = "https://files.pythonhosted.org/packages/43/56/7d06b23ddd09bde816a131aa504ee11a1bbe87c6b62ab9b2ed23849a3382/tree_sitter_java-0.23.5-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:aae84449e330363b55b14a2af0585e4e0dae75eb64ea509b7e5b0e1de536846a", size = 82564, upload-time = "2024-12-21T18:24:20.493Z" }, + { url = "https://files.pythonhosted.org/packages/da/d6/0528c7e1e88a18221dbd8ccee3825bf274b1fa300f745fd74eb343878043/tree_sitter_java-0.23.5-cp39-abi3-win_amd64.whl", hash = "sha256:1ee45e790f8d31d416bc84a09dac2e2c6bc343e89b8a2e1d550513498eedfde7", size = 60650, upload-time = "2024-12-21T18:24:22.902Z" }, + { url = "https://files.pythonhosted.org/packages/72/57/5bab54d23179350356515526fff3cc0f3ac23bfbc1a1d518a15978d4880e/tree_sitter_java-0.23.5-cp39-abi3-win_arm64.whl", hash = "sha256:402efe136104c5603b429dc26c7e75ae14faaca54cfd319ecc41c8f2534750f4", size = 59059, upload-time = "2024-12-21T18:24:24.934Z" }, +] + +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/8b/c992ff0e768cb6768d5c96234579bf8842b3a633db641455d86dd30d5dac/tree_sitter_python-0.25.0.tar.gz", hash = "sha256:b13e090f725f5b9c86aa455a268553c65cadf325471ad5b65cd29cac8a1a68ac", size = 159845, upload-time = "2025-09-11T06:47:58.159Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/64/a4e503c78a4eb3ac46d8e72a29c1b1237fa85238d8e972b063e0751f5a94/tree_sitter_python-0.25.0-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:14a79a47ddef72f987d5a2c122d148a812169d7484ff5c75a3db9609d419f361", size = 73790, upload-time = "2025-09-11T06:47:47.652Z" }, + { url = "https://files.pythonhosted.org/packages/e6/1d/60d8c2a0cc63d6ec4ba4e99ce61b802d2e39ef9db799bdf2a8f932a6cd4b/tree_sitter_python-0.25.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:480c21dbd995b7fe44813e741d71fed10ba695e7caab627fb034e3828469d762", size = 76691, upload-time = "2025-09-11T06:47:49.038Z" }, + { url = "https://files.pythonhosted.org/packages/aa/cb/d9b0b67d037922d60cbe0359e0c86457c2da721bc714381a63e2c8e35eba/tree_sitter_python-0.25.0-cp310-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:86f118e5eecad616ecdb81d171a36dde9bef5a0b21ed71ea9c3e390813c3baf5", size = 108133, upload-time = "2025-09-11T06:47:50.499Z" }, + { url = "https://files.pythonhosted.org/packages/40/bd/bf4787f57e6b2860f3f1c8c62f045b39fb32d6bac4b53d7a9e66de968440/tree_sitter_python-0.25.0-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be71650ca2b93b6e9649e5d65c6811aad87a7614c8c1003246b303f6b150f61b", size = 110603, upload-time = "2025-09-11T06:47:51.985Z" }, + { url = "https://files.pythonhosted.org/packages/5d/25/feff09f5c2f32484fbce15db8b49455c7572346ce61a699a41972dea7318/tree_sitter_python-0.25.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6d5b5799628cc0f24691ab2a172a8e676f668fe90dc60468bee14084a35c16d", size = 108998, upload-time = "2025-09-11T06:47:53.046Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/4946da3d6c0df316ccb938316ce007fb565d08f89d02d854f2d308f0309f/tree_sitter_python-0.25.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:71959832fc5d9642e52c11f2f7d79ae520b461e63334927e93ca46cd61cd9683", size = 107268, upload-time = "2025-09-11T06:47:54.388Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a2/996fc2dfa1076dc460d3e2f3c75974ea4b8f02f6bc925383aaae519920e8/tree_sitter_python-0.25.0-cp310-abi3-win_amd64.whl", hash = "sha256:9bcde33f18792de54ee579b00e1b4fe186b7926825444766f849bf7181793a76", size = 76073, upload-time = "2025-09-11T06:47:55.773Z" }, + { url = "https://files.pythonhosted.org/packages/07/19/4b5569d9b1ebebb5907d11554a96ef3fa09364a30fcfabeff587495b512f/tree_sitter_python-0.25.0-cp310-abi3-win_arm64.whl", hash = "sha256:0fbf6a3774ad7e89ee891851204c2e2c47e12b63a5edbe2e9156997731c128bb", size = 74169, upload-time = "2025-09-11T06:47:56.747Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspect" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/74/1789779d91f1961fa9438e9a8710cdae6bd138c80d7303996933d117264a/typing_inspect-0.9.0.tar.gz", hash = "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78", size = 13825, upload-time = "2023-05-24T20:25:47.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/f3/107a22063bf27bdccf2024833d3445f4eea42b2e598abfbd46f6a63b6cb0/typing_inspect-0.9.0-py3-none-any.whl", hash = "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", size = 8827, upload-time = "2023-05-24T20:25:45.287Z" }, +]