gh-145234: keep tokenizer source pointer after UTF-8 translation

pablogsal · pablogsal · commit cb60a1642dce · 2026-02-26T22:18:08.000Z
diff --git a/Parser/tokenizer/string_tokenizer.c b/Parser/tokenizer/string_tokenizer.c
@@ -120,6 +120,7 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr
         str = translated;
         Py_CLEAR(utf8);
     }
+    tok->str = str;
     assert(tok->decoding_buffer == NULL);
     tok->decoding_buffer = utf8; /* CAUTION */
     return str;

Original file line number	Diff line number	Diff line change
`@@ -120,6 +120,7 @@ decode_str(const char input, int single, struct tok_state tok, int preserve_cr`
`120`	`120`	`str = translated;`
`121`	`121`	`Py_CLEAR(utf8);`
`122`	`122`	`}`
	`123`	`+ tok->str = str;`
`123`	`124`	`assert(tok->decoding_buffer == NULL);`
`124`	`125`	`tok->decoding_buffer = utf8; /* CAUTION */`
`125`	`126`	`return str;`