From 8ff82814cacd80e2833b672a0d0d32435c5d7c1a Mon Sep 17 00:00:00 2001 From: MWR27 <64335495+MWR27@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:34:59 -0400 Subject: [PATCH] Add foundation for automatic refactoring tool --- pygrate2/__init__.py | 0 pygrate2/__main__.py | 3 ++ pygrate2/fixes/__init__.py | 0 pygrate2/fixes/fix_literal_div.py | 23 +++++++++ pygrate2/main.py | 77 +++++++++++++++++++++++++++++++ pygrate2/refactor_util.py | 10 ++++ pygrate2/semfixer_base.py | 6 +++ pygrate2/semrefactor.py | 59 +++++++++++++++++++++++ 8 files changed, 178 insertions(+) create mode 100644 pygrate2/__init__.py create mode 100644 pygrate2/__main__.py create mode 100644 pygrate2/fixes/__init__.py create mode 100644 pygrate2/fixes/fix_literal_div.py create mode 100644 pygrate2/main.py create mode 100644 pygrate2/refactor_util.py create mode 100644 pygrate2/semfixer_base.py create mode 100644 pygrate2/semrefactor.py diff --git a/pygrate2/__init__.py b/pygrate2/__init__.py new file mode 100644 index 000000000000000..e69de29bb2d1d64 diff --git a/pygrate2/__main__.py b/pygrate2/__main__.py new file mode 100644 index 000000000000000..7186d05800419da --- /dev/null +++ b/pygrate2/__main__.py @@ -0,0 +1,3 @@ +from .main import main + +main() \ No newline at end of file diff --git a/pygrate2/fixes/__init__.py b/pygrate2/fixes/__init__.py new file mode 100644 index 000000000000000..e69de29bb2d1d64 diff --git a/pygrate2/fixes/fix_literal_div.py b/pygrate2/fixes/fix_literal_div.py new file mode 100644 index 000000000000000..ea3deb7cf446d41 --- /dev/null +++ b/pygrate2/fixes/fix_literal_div.py @@ -0,0 +1,23 @@ +from lib2to3 import fixer_base, pytree +from lib2to3.pgen2 import token + +from ..refactor_util import is_integer_literal + +class FixLiteralDiv(fixer_base.BaseFix): + # assumes FixNumliterals is run first + + PATTERN = "term< dividend=NUMBER operator='/' divisor=NUMBER >" + + def match(self, node): + results = super(FixLiteralDiv, self).match(node) + if not results: + return False + + if is_integer_literal(results['dividend']) and is_integer_literal(results['divisor']): + return results + + return False + + def transform(self, node, results): + operator = results["operator"] + operator.replace(pytree.Leaf(token.DOUBLESLASH, u"//", prefix=operator.prefix)) diff --git a/pygrate2/main.py b/pygrate2/main.py new file mode 100644 index 000000000000000..8011ec5c7adb02c --- /dev/null +++ b/pygrate2/main.py @@ -0,0 +1,77 @@ +import sys +import os +import subprocess +import re +import optparse + +from lib2to3 import refactor +from .semrefactor import ProgramContext, SemanticRefactoringTool + +def main(): + parser = optparse.OptionParser(usage="pygrate2 [options] source dest") + parser.add_option("-r", "--run", action="store_true", + help="Run program with warnings. If source is a directory, it runs as a module.") + + options, args = parser.parse_args() + + if len(args) == 0: + parser.print_help() + return + if len(args) == 1: + print 'Missing dest argument' + return + if len(args) > 2: + print 'Too many arguments' + return + if not os.path.exists(args[0]): + print 'Source path does not exist' + return + + input_base_dir = args[0] + if (not input_base_dir.endswith(os.sep) and not os.path.isdir(input_base_dir)): + input_base_dir = os.path.dirname(input_base_dir) + input_base_dir = input_base_dir.rstrip(os.sep) + + lib2to3_fixer_names = ['lib2to3.fixes.fix_numliterals'] + fixer_names = lib2to3_fixer_names + refactor.get_fixers_from_package('pygrate2.fixes') + + program_context = ProgramContext() + + tool = SemanticRefactoringTool( + fixers=fixer_names, + program_context=program_context, + options=None, + explicit=None, + nobackups=True, + show_diffs=True, + input_base_dir=input_base_dir, + output_dir=args[1]) + + if os.path.isdir(args[0]): + tool.refactor_dir(args[0], write=True) + else: + tool.refactor_file(args[0], write=True) + + if not options.run: + return + + proc_args = [sys.executable, '-3'] + if os.path.isdir(args[0]): + proc_args.append('-m') + proc_args.append(args[0]) + + proc = subprocess.Popen(proc_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + stdoutdata, stderrdata = proc.communicate() + + print '' + print stdoutdata + + warning_pattern = re.compile(r"(.+):(\d+): (.*)Warning: (.*)") + + for line in stderrdata.splitlines(): + m = re.match(warning_pattern, line) + + if m is not None: + warning = m.group(1, 2, 3, 4) + print warning diff --git a/pygrate2/refactor_util.py b/pygrate2/refactor_util.py new file mode 100644 index 000000000000000..13c96672077bdc0 --- /dev/null +++ b/pygrate2/refactor_util.py @@ -0,0 +1,10 @@ +import re +import ast + +from lib2to3.pytree import Leaf + +# assumes FixNumliterals was run +def is_integer_literal(node): + if isinstance(node, Leaf): + return re.match('(?:[1-9][0-9]*|0(?:[oO][0-7]+|[xX][0-9a-fA-F]+|[bB][01]+)?)$', node.value) is not None + return False diff --git a/pygrate2/semfixer_base.py b/pygrate2/semfixer_base.py new file mode 100644 index 000000000000000..f7deabb24bdb3bb --- /dev/null +++ b/pygrate2/semfixer_base.py @@ -0,0 +1,6 @@ +from lib2to3.fixer_base import BaseFix + +class SemanticFix(BaseFix): + def __init__(self, options, log, program_context): + super(SemanticFix, self).__init__(options, log) + self._program_context = program_context diff --git a/pygrate2/semrefactor.py b/pygrate2/semrefactor.py new file mode 100644 index 000000000000000..c555d93e467e5ba --- /dev/null +++ b/pygrate2/semrefactor.py @@ -0,0 +1,59 @@ +import operator + +from lib2to3.main import StdoutRefactoringTool +from .semfixer_base import SemanticFix + +class ProgramContext(object): + # example of a method that would read the source to find the possible types of a variable + def get_possible_types(self, variable): + pass + +class SemanticRefactoringTool(StdoutRefactoringTool): + def __init__(self, fixers, program_context, options, explicit, nobackups, show_diffs, + input_base_dir='', output_dir='', append_suffix=''): + self._program_context = program_context + super(SemanticRefactoringTool, self).__init__(fixers, options, explicit, nobackups, show_diffs, + input_base_dir, output_dir, append_suffix) + + def get_fixers(self): + """Inspects the options to load the requested patterns and handlers. + + Returns: + (pre_order, post_order), where pre_order is the list of fixers that + want a pre-order AST traversal, and post_order is the list that want + post-order traversal. + """ + pre_order_fixers = [] + post_order_fixers = [] + for fix_mod_path in self.fixers: + mod = __import__(fix_mod_path, {}, {}, ["*"]) + fix_name = fix_mod_path.rsplit(".", 1)[-1] + if fix_name.startswith(self.FILE_PREFIX): + fix_name = fix_name[len(self.FILE_PREFIX):] + parts = fix_name.split("_") + class_name = self.CLASS_PREFIX + "".join([p.title() for p in parts]) + try: + fix_class = getattr(mod, class_name) + except AttributeError: + raise FixerError("Can't find %s.%s" % (fix_name, class_name)) + if issubclass(fix_class, SemanticFix): + fixer = fix_class(self.options, self.fixer_log, self._program_context) + else: + fixer = fix_class(self.options, self.fixer_log) + if fixer.explicit and self.explicit is not True and \ + fix_mod_path not in self.explicit: + self.log_message("Skipping optional fixer: %s", fix_name) + continue + + self.log_debug("Adding transformation: %s", fix_name) + if fixer.order == "pre": + pre_order_fixers.append(fixer) + elif fixer.order == "post": + post_order_fixers.append(fixer) + else: + raise FixerError("Illegal fixer order: %r" % fixer.order) + + key_func = operator.attrgetter("run_order") + pre_order_fixers.sort(key=key_func) + post_order_fixers.sort(key=key_func) + return (pre_order_fixers, post_order_fixers)