From 517cc4e6ba4ddc5b14b7974b1971a262e3c3fbf5 Mon Sep 17 00:00:00 2001 From: Tim Hedger-Gourlay Date: Thu, 23 Jul 2026 18:35:36 +0100 Subject: [PATCH 1/3] multi-token extension capability added --- src/unasync/__init__.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/unasync/__init__.py b/src/unasync/__init__.py index 333d748..c5b2037 100644 --- a/src/unasync/__init__.py +++ b/src/unasync/__init__.py @@ -69,7 +69,7 @@ def _unasync_file(self, filepath): with open(filepath, encoding=encoding) as f: tokens = tokenize_rt.src_to_tokens(f.read()) - tokens = self._unasync_tokens(tokens) + tokens = self._transform_tokens(tokens) result = tokenize_rt.tokens_to_src(tokens) outfilepath = self.map_in_to_out_file_path(filepath) os.makedirs(os.path.dirname(outfilepath), exist_ok=True) @@ -105,6 +105,25 @@ def _unasync_tokens(self, tokens): yield token + def _transform_tokens(self, tokens): + """ + Perform all token transformations. + + The default implementation performs the standard async→sync + conversion. Subclasses may override this method to perform + additional token-level transformations. + """ + tokens = self._unasync_tokens(tokens) + return self._postprocess_tokens(tokens) + + def _postprocess_tokens(self, tokens): + """ + Hook for subclasses. + + Called after the standard async→sync conversion. + """ + return token + def unasync_name(self, name): if name in self.token_replacements: return self.token_replacements[name] From e93392857bed6c0358319d3d376262caf4204009 Mon Sep 17 00:00:00 2001 From: Tim Hedger-Gourlay Date: Thu, 30 Jul 2026 23:28:47 +0100 Subject: [PATCH 2/3] Update src/unasync/__init__.py Bug fix (after my testing focused on using the hook to override the post processor method instead of testing the default post processor Co-authored-by: Quentin Pradet --- src/unasync/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/unasync/__init__.py b/src/unasync/__init__.py index c5b2037..31c680c 100644 --- a/src/unasync/__init__.py +++ b/src/unasync/__init__.py @@ -122,7 +122,7 @@ def _postprocess_tokens(self, tokens): Called after the standard async→sync conversion. """ - return token + return tokens def unasync_name(self, name): if name in self.token_replacements: From 51ae2f0d945ea0172bf807298690b3b7f6d3e238 Mon Sep 17 00:00:00 2001 From: Tim Hedger-Gourlay Date: Fri, 31 Jul 2026 13:47:49 +0100 Subject: [PATCH 3/3] Add regression test for postprocess hook Test demonstrates a simple real world multitoken transformation from: asyncio.current_task() to: current_thread() it also has data to confirm that those tokens on their own are not touched (so asyncio remains asyncio and curent_task() remains current_task()) --- tests/data/postprocess/async/hello.py | 3 ++ tests/data/postprocess/sync/hello.py | 3 ++ tests/test_post_process.py | 63 +++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 tests/data/postprocess/async/hello.py create mode 100644 tests/data/postprocess/sync/hello.py create mode 100644 tests/test_post_process.py diff --git a/tests/data/postprocess/async/hello.py b/tests/data/postprocess/async/hello.py new file mode 100644 index 0000000..9e9e0df --- /dev/null +++ b/tests/data/postprocess/async/hello.py @@ -0,0 +1,3 @@ +self._update_task = asyncio.current_task() +asyncio +current_task() diff --git a/tests/data/postprocess/sync/hello.py b/tests/data/postprocess/sync/hello.py new file mode 100644 index 0000000..57d4fbf --- /dev/null +++ b/tests/data/postprocess/sync/hello.py @@ -0,0 +1,3 @@ +self._update_task = current_thread() +asyncio +current_task() diff --git a/tests/test_post_process.py b/tests/test_post_process.py new file mode 100644 index 0000000..6722e9b --- /dev/null +++ b/tests/test_post_process.py @@ -0,0 +1,63 @@ +import os + +from unasync import Rule + +TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") +TEST_DIR = os.path.join(TEST_DIR, "postprocess") +ASYNC_DIR = os.path.join(TEST_DIR, "async") +SYNC_DIR = os.path.join(TEST_DIR, "sync") +TEST_FILES = sorted(f for f in os.listdir(ASYNC_DIR) if f.endswith(".py")) + +class PostProcessRule(Rule): + + def _postprocess_tokens(self, tokens): + # Replace: + # asyncio.current_task() + # with: + # current_thread() + + prev2 = None + prev1 = None + + for token in tokens: + + if ( + prev2 is not None + and prev2.src == "asyncio" + and prev1.src == "." + and token.src == "current_task" + ): + yield token._replace(src="current_thread") + + prev2 = None + prev1 = None + + elif prev2 is not None: + yield prev2 + prev2 = prev1 + prev1 = token + + else: + prev2 = prev1 + prev1 = token + + if prev2 is not None: + yield prev2 + if prev1 is not None: + yield prev1 + + +def test_postprocess(tmpdir): + rule = PostProcessRule(fromdir=ASYNC_DIR, todir=str(tmpdir)) + + for source_file in TEST_FILES: + rule._unasync_file(os.path.join(ASYNC_DIR, source_file)) + + for source_file in TEST_FILES: + with open(os.path.join(SYNC_DIR, source_file)) as f: + truth = f.read() + + with open(os.path.join(str(tmpdir), source_file)) as f: + unasynced = f.read() + + assert unasynced == truth