From b413caab2c7b08fd08e9fe8f3d5f1407d0370942 Mon Sep 17 00:00:00 2001 From: boris Date: Mon, 13 Jul 2026 15:50:12 -0700 Subject: [PATCH] fix: cpython ast fixes --- .../pegjava/java_generator.py | 2 +- .../graal/python/pegparser/AbstractParser.java | 16 ++++++++++++++++ .../oracle/graal/python/pegparser/Parser.java | 2 +- .../com/oracle/graal/python/PythonLanguage.java | 1 + .../builtins/modules/BuiltinFunctions.java | 1 + .../python/builtins/modules/ast/AstBuiltins.java | 12 ++++++++++-- .../objects/type/PythonBuiltinClass.java | 6 +++--- .../builtins/objects/type/TypeBuiltins.java | 9 +++++++-- .../oracle/graal/python/nodes/ErrorMessages.java | 1 + .../attributes/WriteAttributeToObjectNode.java | 2 +- 10 files changed, 42 insertions(+), 10 deletions(-) diff --git a/graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py b/graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py index f936050b3b..8b0254e53d 100644 --- a/graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py +++ b/graalpython/com.oracle.graal.python.pegparser.generator/pegjava/java_generator.py @@ -258,7 +258,7 @@ def __str__(self) -> str: '_PyAST_Starred ( a , Load , EXTRA )': (3, 'factory.createStarred(a, ExprContextTy.Load, $RANGE)'), '_PyAST_Try ( b , NULL , NULL , f , EXTRA )': (1, 'factory.createTry(b, null, null, f, $RANGE)'), '_PyAST_Try ( b , ex , el , f , EXTRA )': (1, 'factory.createTry(b, ex, el, f, $RANGE)'), - 'CHECK_VERSION ( stmt_ty , 11 , "Exception groups are" , _PyAST_TryStar ( b , ex , el , f , EXTRA ) )': (1, 'checkVersion(10, "Pattern matching is", factory.createTryStar(b, ex, el, f, $RANGE))'), + 'CHECK_VERSION ( stmt_ty , 11 , "Exception groups are" , _PyAST_TryStar ( b , ex , el , f , EXTRA ) )': (1, 'checkVersion(11, "Exception groups are", factory.createTryStar(b, ex, el, f, $RANGE))'), '_PyAST_Tuple ( CHECK ( asdl_expr_seq* , _PyPegen_seq_insert_in_front ( p , a , b ) ) , Load , EXTRA )': (2, 'factory.createTuple(this.insertInFront(a, b), ExprContextTy.Load, $RANGE)'), '_PyAST_Tuple ( CHECK ( asdl_expr_seq* , _PyPegen_seq_insert_in_front ( p , a , b ) ) , Store , EXTRA )': (1, 'factory.createTuple(this.insertInFront(a,b), ExprContextTy.Store, $RANGE)'), '_PyAST_Tuple ( CHECK ( asdl_expr_seq* , _PyPegen_singleton_seq ( p , a ) ) , Load , EXTRA )': (2, 'factory.createTuple(new ExprTy[] {a}, ExprContextTy.Load, $RANGE)'), diff --git a/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java b/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java index 75798b79e4..f24e8e51f7 100644 --- a/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java +++ b/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/AbstractParser.java @@ -117,6 +117,10 @@ public enum Flags { private static final String BARRY_AS_BDFL = "with Barry as BDFL, use '<>' instead of '!='"; + private static final int INT_MAX_STR_DIGITS_THRESHOLD = 640; + private static final String INT_MAX_STR_DIGITS_ERROR_FMT = "Exceeds the limit (%d digits) for integer string conversion: value has %d digits; " + + "use sys.set_int_max_str_digits() to increase the limit - Consider hexadecimal for huge integer literals to avoid decimal conversion limits."; + private int currentPos; // position of the mark private final ArrayList tokens; private final Tokenizer tokenizer; @@ -126,6 +130,7 @@ public enum Flags { private final EnumSet flags; final int featureVersion; + private int intMaxStrDigits = 4300; protected int level = 0; boolean callInvalidRules = false; @@ -423,6 +428,10 @@ public Token string_token() { return expect(Token.Kind.STRING); } + public void setIntMaxStrDigits(int intMaxStrDigits) { + this.intMaxStrDigits = intMaxStrDigits; + } + /** * _PyPegen_number_token */ @@ -492,6 +501,13 @@ public ExprTy number_token() { } if (overunder) { // overflow + if (base == 10) { + int numDigits = number.length() - start; + if (numDigits > INT_MAX_STR_DIGITS_THRESHOLD && intMaxStrDigits > 0 && numDigits > intMaxStrDigits) { + raiseSyntaxError(INT_MAX_STR_DIGITS_ERROR_FMT, intMaxStrDigits, numDigits); + return null; // to skip BigInteger constructor + } + } BigInteger bigResult = BigInteger.valueOf(result); BigInteger bigBase = BigInteger.valueOf(base); while (i < number.length()) { diff --git a/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/Parser.java b/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/Parser.java index d7c0846fd0..4161cac78d 100644 --- a/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/Parser.java +++ b/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/Parser.java @@ -3950,7 +3950,7 @@ public StmtTy try_stmt_rule() if (endToken == null) { return null; } - _res = checkVersion(10, "Pattern matching is", factory.createTryStar(b, ex, el, f, startToken.sourceRange.withEnd(endToken.sourceRange))); + _res = checkVersion(11, "Exception groups are", factory.createTryStar(b, ex, el, f, startToken.sourceRange.withEnd(endToken.sourceRange))); return (StmtTy)_res; } reset(_mark); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java index c7a68d788b..a1f855cc4b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java @@ -595,6 +595,7 @@ public RootCallTarget parse(PythonContext context, Source source, InputType type LOGGER.log(Level.FINE, () -> "parse '" + source.getName() + "'"); } Parser parser = Compiler.createParser(source.getCharacters().toString(), errorCb, type, interactiveTerminal, allowIncompleteInput); + parser.setIntMaxStrDigits(context.getIntMaxStrDigits()); ModTy mod = (ModTy) parser.parse(); assert mod != null; return compileModule(context, mod, source, topLevel, optimize, argumentNames, errorCb, futureFeatures); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java index 6c287f845f..b8e8d175da 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java @@ -1042,6 +1042,7 @@ Object compile(TruffleString expression, TruffleString filename, TruffleString m PythonLanguage.LOGGER.log(Level.FINE, () -> "parse '" + source.getName() + "'"); } Parser parser = Compiler.createParser(code.toJavaStringUncached(), parserCb, type, compilerFlags, featureVersion); + parser.setIntMaxStrDigits(context.getIntMaxStrDigits()); ModTy mod = (ModTy) parser.parse(); parserCb.triggerDeprecationWarnings(); return AstModuleBuiltins.sst2Obj(getContext(), mod); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ast/AstBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ast/AstBuiltins.java index a522a56722..3f26200499 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ast/AstBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ast/AstBuiltins.java @@ -47,6 +47,7 @@ import static com.oracle.graal.python.nodes.SpecialAttributeNames.J___DICT__; import static com.oracle.graal.python.nodes.SpecialAttributeNames.T___DICT__; import static com.oracle.graal.python.nodes.SpecialMethodNames.J___REDUCE__; +import static com.oracle.graal.python.runtime.exception.PythonErrorType.AttributeError; import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError; import static com.oracle.graal.python.util.PythonUtils.EMPTY_OBJECT_ARRAY; import static com.oracle.graal.python.util.PythonUtils.TS_ENCODING; @@ -66,6 +67,7 @@ import com.oracle.graal.python.builtins.objects.dict.PDict; import com.oracle.graal.python.builtins.objects.function.PKeyword; import com.oracle.graal.python.builtins.objects.object.PythonObject; +import com.oracle.graal.python.builtins.objects.type.PythonBuiltinClass; import com.oracle.graal.python.builtins.objects.type.TpSlots; import com.oracle.graal.python.builtins.objects.type.TypeNodes; import com.oracle.graal.python.lib.PyObjectLookupAttr; @@ -125,15 +127,21 @@ abstract static class InitNode extends PythonVarargsBuiltinNode { @Specialization protected Object doIt(VirtualFrame frame, Object self, Object[] args, PKeyword[] kwArgs, @Bind Node inliningTarget, + @Cached GetClassNode getClassNode, @Cached PyObjectLookupAttr lookupAttrNode, @Cached SequenceNodes.GetObjectArrayNode getObjectArrayNode, @Cached PyObjectSetAttrO setAttrNode, @Cached TruffleString.EqualNode equalNode, + @Cached TypeNodes.GetNameNode getTypeNameNode, @Cached PRaiseNode raiseNode) { - Object fieldsObj = lookupAttrNode.execute(frame, inliningTarget, self, T__FIELDS); + Object selfType = getClassNode.execute(inliningTarget, self); + Object fieldsObj = lookupAttrNode.execute(frame, inliningTarget, selfType, T__FIELDS); Object[] fields; if (fieldsObj == PNone.NO_VALUE) { - fields = EMPTY_OBJECT_ARRAY; + PythonBuiltinClassType builtinSelfType = selfType instanceof PythonBuiltinClassType pbct ? pbct + : selfType instanceof PythonBuiltinClass pbc ? pbc.getType() : null; + TruffleString typeName = builtinSelfType != null ? builtinSelfType.getPrintName() : getTypeNameNode.execute(inliningTarget, selfType); + throw raiseNode.raise(inliningTarget, AttributeError, ErrorMessages.TYPE_S_HAS_NO_ATTR, typeName, T__FIELDS); } else { if (!(fieldsObj instanceof PSequence)) { throw raiseNode.raise(inliningTarget, TypeError, IS_NOT_A_SEQUENCE, fieldsObj); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/PythonBuiltinClass.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/PythonBuiltinClass.java index f603f0ce30..39bcd09e34 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/PythonBuiltinClass.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/PythonBuiltinClass.java @@ -76,7 +76,7 @@ public PythonBuiltinClass(PythonLanguage lang, PythonBuiltinClassType builtinCla @Override public void setAttribute(TruffleString name, Object value) { CompilerAsserts.neverPartOfCompilation(); - if (!PythonContext.get(null).isCoreInitialized()) { + if (!PythonContext.get(null).isCoreInitialized() || type == PythonBuiltinClassType.AST) { setAttributeUnsafe(name, value); } else { throw PRaiseNode.raiseStatic(null, TypeError, ErrorMessages.CANT_SET_ATTRIBUTE_R_OF_IMMUTABLE_TYPE_N, PyObjectReprAsTruffleStringNode.executeUncached(name), this); @@ -97,7 +97,7 @@ public PythonBuiltinClassType getType() { @TruffleBoundary @Override public void onAttributeUpdate(TruffleString key, Object newValue) { - assert !PythonContext.get(null).isCoreInitialized(); + assert !PythonContext.get(null).isCoreInitialized() || type == PythonBuiltinClassType.AST; // Ideally, startup code should not create ASTs that rely on assumptions of props of // builtins assert getMethodResolutionOrder().getFinalAttributeAssumption(key) == null; @@ -105,7 +105,7 @@ public void onAttributeUpdate(TruffleString key, Object newValue) { // NO_VALUE changes MRO lookup results without actually changing any Shapes in the MRO, this // can prevent some optimizations, so it is best to avoid any code that triggers such code // paths during initialization - assert newValue != PNone.NO_VALUE; + assert newValue != PNone.NO_VALUE || type == PythonBuiltinClassType.AST; PythonClass.updateMroShapeSubTypes(this); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java index a7ffa2ac9f..116fd8a3e5 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeBuiltins.java @@ -657,8 +657,13 @@ void setBuiltin(Object object, Object key, Object value) { } protected static boolean isImmutable(Object type) { - // TODO should also check Py_TPFLAGS_IMMUTABLETYPE - return type instanceof PythonBuiltinClass || type instanceof PythonBuiltinClassType; + PythonBuiltinClassType builtinType = null; + if (type instanceof PythonBuiltinClass pbc) { + builtinType = pbc.getType(); + } else if (type instanceof PythonBuiltinClassType pbct) { + builtinType = pbct; + } + return builtinType != null && builtinType != PythonBuiltinClassType.AST; } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java index 25a3cb9424..27a01efcb1 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java @@ -358,6 +358,7 @@ public abstract class ErrorMessages { public static final TruffleString HANDLER_MUST_BE_CALLABLE = tsLiteral("handler must be callable"); public static final TruffleString HAS_NO_ATTR = tsLiteral("%p has no attribute '%s'"); public static final TruffleString TYPE_N_HAS_NO_ATTR = tsLiteral("type object '%N' has no attribute '%s'"); + public static final TruffleString TYPE_S_HAS_NO_ATTR = tsLiteral("type object '%s' has no attribute '%s'"); public static final TruffleString P_HAS_NO_ATTRS_S_TO_ASSIGN = tsLiteral("'%p' object has no attributes (assign to .%s)"); public static final TruffleString P_HAS_NO_ATTRS_S_TO_DELETE = tsLiteral("'%p' object has no attributes (del .%s)"); public static final TruffleString P_HAS_RO_ATTRS_S_TO_ASSIGN = tsLiteral("'%p' object has only read-only attributes (assign to .%s)"); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/attributes/WriteAttributeToObjectNode.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/attributes/WriteAttributeToObjectNode.java index 05053c6696..064f7b724a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/attributes/WriteAttributeToObjectNode.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/attributes/WriteAttributeToObjectNode.java @@ -120,7 +120,7 @@ boolean writeToDynamicStorageBuiltinType(Object klassOrType, TruffleString key, } else { klass = context.lookupType((PythonBuiltinClassType) klassOrType); } - if (context.isInitialized() || value == PNone.NO_VALUE) { + if ((context.isInitialized() || value == PNone.NO_VALUE) && klass.getType() != PythonBuiltinClassType.AST) { throw PRaiseNode.raiseStatic(this, TypeError, ErrorMessages.CANT_SET_ATTRIBUTE_R_OF_IMMUTABLE_TYPE_N, key, klass); } else { PDict dict = GetDictIfExistsNode.getUncached().execute(klass);