Add proactive stack overflow detection via MaxCallDepth + tests

src/Lua/Exceptions.cs

@@ -99,7 +99,7 @@ static string GetMessageWithNearToken(string message, string? nearToken)
 
 public class LuaUndumpException(string message) : Exception(message);
 
-class LuaStackOverflowException() : Exception("stack overflow")
+public class LuaStackOverflowException() : Exception("stack overflow")
 {
     public override string ToString()
     {

src/Lua/LuaState.cs

@@ -197,6 +197,8 @@ public void Release()
     public LuaTable PreloadModules => GlobalState.PreloadModules;
     public LuaState MainThread => GlobalState.MainThread;
 
+    public int MaxCallDepth { get; set; } = 100_000;
+
     public ILuaModuleLoader? ModuleLoader
     {
         get => GlobalState.ModuleLoader;
@@ -285,6 +287,11 @@ int callerInstructionIndex
     [MethodImpl(MethodImplOptions.AggressiveInlining)]
     internal void PushCallStackFrame(in CallStackFrame frame)
     {
+        if (CallStackFrameCount >= MaxCallDepth)
+        {
+            throw new LuaStackOverflowException();
+        }
+
         CurrentException?.BuildOrGet();
         CurrentException = null;
         ref var callStack = ref CoreData!.CallStack;

src/Lua/Runtime/LuaVirtualMachine.cs

@@ -1419,6 +1419,11 @@ static bool Call(VirtualMachineExecutionContext context, out bool doRestart)
 
         var newFrame = func.CreateNewFrame(context, newBase, RA, variableArgumentCount);
 
+        if (!RuntimeHelpers.TryEnsureSufficientExecutionStack())
+        {
+            throw new LuaStackOverflowException();
+        }
+
         state.PushCallStackFrame(newFrame);
         if (state.CallOrReturnHookMask.Value != 0 && !context.State.IsInHook)
         {
@@ -1619,6 +1624,12 @@ static bool TailCall(VirtualMachineExecutionContext context, out bool doRestart)
         );
         newBase = context.FrameBase + variableArgumentCount;
         stack.PopUntil(newBase + argumentCount);
+
+        if (!RuntimeHelpers.TryEnsureSufficientExecutionStack())
+        {
+            throw new LuaStackOverflowException();
+        }
+
         var lastFrame = state.GetCurrentFrame();
         state.LastPc = context.Pc;
         state.LastCallerFunction = lastFrame.Function;

src/Lua/Standard/BasicLibrary.cs

@@ -367,6 +367,8 @@ CancellationToken cancellationToken
                     throw;
                 case OperationCanceledException:
                     throw new LuaCanceledException(context.State, cancellationToken, ex);
+                case LuaStackOverflowException:
+                    return context.Return(false, ex.Message);
                 case LuaRuntimeException luaEx:
                 {
                     if (

tests/Lua.Tests/StackOverflowTests.cs

@@ -0,0 +1,87 @@
+using Lua.Standard;
+
+namespace Lua.Tests;
+
+public class StackOverflowTests
+{
+    [Test]
+    public void ExceedingMaxCallDepth_ThrowsLuaRuntimeException()
+    {
+        var state = LuaState.Create();
+        state.MaxCallDepth = 5;
+
+        var ex = Assert.ThrowsAsync<LuaRuntimeException>(async () =>
+            await state.DoStringAsync(
+                """
+                local function f()
+                    f()
+                end
+                f()
+                """
+            ).AsTask()
+        );
+
+        Assert.That(ex!.InnerException, Is.TypeOf<LuaStackOverflowException>());
+    }
+
+    [Test]
+    public async Task ExceedingMaxCallDepth_WithPCall_ReturnsErrorMessage()
+    {
+        var state = LuaState.Create();
+        state.OpenStandardLibraries();
+        state.MaxCallDepth = 5;
+
+        var result = await state.DoStringAsync(
+            """
+            local function f()
+                f()
+            end
+            local ok, msg = pcall(f)
+            return ok, msg
+            """
+        );
+
+        Assert.That(result, Has.Length.EqualTo(2));
+        Assert.That(result[0], Is.EqualTo(new LuaValue(false)));
+        Assert.That(result[1].Read<string>(), Does.Contain("stack overflow"));
+    }
+
+    [Test]
+    public async Task UnderMaxCallLimit_Succeeds()
+    {
+        var state = LuaState.Create();
+        state.MaxCallDepth = 100;
+
+        var result = await state.DoStringAsync(
+            """
+            local function f(n)
+                if n <= 0 then return 'done' end
+                return f(n - 1)
+            end
+            return f(10)
+            """
+        );
+
+        Assert.That(result, Has.Length.EqualTo(1));
+        Assert.That(result[0], Is.EqualTo(new LuaValue("done")));
+    }
+
+    [Test]
+    public async Task DefaultMaxCallDepth_AllowsDeepRecursion()
+    {
+        var state = LuaState.Create();
+
+        var result = await state.DoStringAsync(
+            """
+            local function f(n)
+                if n <= 0 then return n end
+                return f(n - 1)
+            end
+            return f(1000)
+            """
+        );
+
+        Assert.That(result, Has.Length.EqualTo(1));
+        Assert.That(result[0], Is.EqualTo(new LuaValue(0)));
+    }
+}