Fix stack overflow with mutually recursive type aliases

Summary:
Fixes #2982

Cyclic type aliases like type T = U; type U = T were correctly detected and reported as errors, but the cyclic UntypedAlias(Ref(...)) was still emitted as the alias's type. Downstream operations (attribute lookup, subset checks) would then infinitely bounce between the two refs, overflowing the stack. Fix: when check_type_alias_for_cyclic_reference finds a cycle, short-circuit wrap_type_alias to return an error type instead of the cyclic body. {F1987625179}

Reviewed By: rchen152

Differential Revision: D99314495

fbshipit-source-id: 87937d7fca8c556eaadb641160a23cfc63b0a5a8

Author: tejasreddyvepala

pyrefly/lib/alt/solve.rs

@@ -1252,13 +1252,14 @@ impl<'a, Ans: LookupAnswer> AnswersSolver<'a, Ans> {
     /// user-defined `class C[T]: x: T | None` makes `type A = C[A]`
     /// inhabitable (e.g. `C(x=C(x=None))`), so we can't assume all generic
     /// containers require their type parameter.
+    /// Returns `true` if a cyclic self-reference was found.
     fn check_type_alias_for_cyclic_reference(
         &self,
         name: &Name,
         ta: &TypeAlias,
         range: TextRange,
         errors: &ErrorCollector,
-    ) {
+    ) -> bool {
         // Unwrap the type[body] wrapper. We operate on the inner body because
         // map_over_union wraps inner union members in type[...] when traversing
         // inside Type::Type, which would prevent matching UntypedAlias nodes.
@@ -1267,7 +1268,7 @@ impl<'a, Ans: LookupAnswer> AnswersSolver<'a, Ans> {
         let ty = ta.as_type();
         let body = match &ty {
             Type::Type(inner) => inner.as_ref(),
-            _ => return,
+            _ => return false,
         };
         let is_self_ref = |ty: &Type| matches!(ty, Type::UntypedAlias(ta) if ta.name() == name);
 
@@ -1338,7 +1339,9 @@ impl<'a, Ans: LookupAnswer> AnswersSolver<'a, Ans> {
                 ErrorInfo::Kind(ErrorKind::InvalidTypeAlias),
                 format!("Found cyclic self-reference in `{name}`"),
             );
+            return true;
         }
+        false
     }
 
     /// `typealiastype_tparams` refers specifically to the elements of the tuple literal passed to the `TypeAliasType` constructor
@@ -1371,7 +1374,12 @@ impl<'a, Ans: LookupAnswer> AnswersSolver<'a, Ans> {
         }
 
         // Step 2: Check for cyclic self-references after expansion.
-        self.check_type_alias_for_cyclic_reference(name, &ta, range, errors);
+        // If a cycle is found, replace the body with an error type to prevent
+        // infinite recursion when downstream operations (e.g. attribute lookup,
+        // subset checks) try to resolve the alias.
+        if self.check_type_alias_for_cyclic_reference(name, &ta, range, errors) {
+            return self.heap.mk_any_error();
+        }
 
         // Step 3: Extract type parameters from the (now expanded) body.
         let mut seen_type_vars = SmallMap::new();

pyrefly/lib/test/recursive_alias.rs

@@ -258,6 +258,19 @@ type Z = int | Y  # E: cyclic self-reference in `Z`
     "#,
 );
 
+testcase!(
+    test_cyclic_mutual_usage,
+    r#"
+# Mutually recursive type aliases with no base case should not stack overflow
+# when the aliases are used in expressions. 
+type T = U  # E: cyclic self-reference in `T`
+type U = T  # E: cyclic self-reference in `U`
+
+x: T = 1
+not x
+    "#,
+);
+
 testcase!(
     test_cyclic_no_base_case,
     r#"