Don't call error handler for memory errors in coroutines

src/thread.rs

@@ -136,6 +136,10 @@ impl<'lua> Thread<'lua> {
 
             let ret = ffi::lua_resume(thread_state, state, nargs, &mut nresults as *mut c_int);
             if ret != ffi::LUA_OK && ret != ffi::LUA_YIELD {
+                if ret == ffi::LUA_ERRMEM {
+                    // Don't call error handler for memory errors
+                    return Err(pop_error(thread_state, ret));
+                }
                 check_stack(state, 3)?;
                 protect_lua!(state, 0, 1, |state| error_traceback_thread(
                     state,

src/util.rs

@@ -667,7 +667,7 @@ where
 }
 
 pub unsafe extern "C" fn error_traceback(state: *mut ffi::lua_State) -> c_int {
-    // This is a workaround for bug in Luau, when it calls error handler for memory allocation error
+    // Luau calls error handler for memory allocation errors, skip it
     // See https://github.com/Roblox/luau/issues/880
     #[cfg(feature = "luau")]
     if MemoryState::limit_reached(state) {

tests/memory.rs

@@ -38,6 +38,29 @@ fn test_memory_limit() -> Result<()> {
     Ok(())
 }
 
+#[test]
+fn test_memory_limit_thread() -> Result<()> {
+    let lua = Lua::new();
+
+    let f = lua
+        .load("local t = {}; for i = 1,10000 do t[i] = i end")
+        .into_function()?;
+
+    if cfg!(feature = "luajit") && cfg!(not(feature = "vendored")) {
+        // we don't support setting memory limit for non-vendored luajit
+        return Ok(());
+    }
+
+    lua.set_memory_limit(lua.used_memory() + 10000)?;
+    let thread = lua.create_thread(f)?;
+    match thread.resume::<_, ()>(()) {
+        Err(Error::MemoryError(_)) => {}
+        something_else => panic!("did not trigger memory error: {:?}", something_else),
+    };
+
+    Ok(())
+}
+
 #[test]
 fn test_gc_control() -> Result<()> {
     let lua = Lua::new();