From 4ded555cc5f6fa9ab7cddb12c0b9e295058da68c Mon Sep 17 00:00:00 2001
From: XmiliaH <45106915+XmiliaH@users.noreply.github.com>
Date: Wed, 17 Aug 2022 00:32:48 +0200
Subject: [PATCH] Prevent overflow in lua_newuserdatadtor (#639)

In case a large userdata size is passed to lua_newuserdatadtor it might overflow the size resulting in luaU_newudata actually allocating the object without a memory error. This will then result in overwriting part of the metatable pointer of the userdata.
This PR fixes this issue by checking for the overflow and in such cases pass a size value which will cause a memory error in luaU_newudata.
---
 VM/src/lapi.cpp            |  5 ++++-
 tests/Conformance.test.cpp | 17 +++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/VM/src/lapi.cpp b/VM/src/lapi.cpp
index bb994fb..77788e4 100644
--- a/VM/src/lapi.cpp
+++ b/VM/src/lapi.cpp
@@ -1209,7 +1209,10 @@ void* lua_newuserdatadtor(lua_State* L, size_t sz, void (*dtor)(void*))
 {
     luaC_checkGC(L);
     luaC_checkthreadsleep(L);
-    Udata* u = luaU_newudata(L, sz + sizeof(dtor), UTAG_IDTOR);
+    size_t as = sz + sizeof(dtor);
+    if (as < sizeof(dtor))
+        as = SIZE_MAX; // Will cause a memory error in luaU_newudata.
+    Udata* u = luaU_newudata(L, as, UTAG_IDTOR);
     memcpy(&u->data + sz, &dtor, sizeof(dtor));
     setuvalue(L, L->top, u);
     api_incr_top(L);
diff --git a/tests/Conformance.test.cpp b/tests/Conformance.test.cpp
index e07ba12..17c7ac5 100644
--- a/tests/Conformance.test.cpp
+++ b/tests/Conformance.test.cpp
@@ -716,6 +716,23 @@ TEST_CASE("Reference")
     CHECK(dtorhits == 2);
 }
 
+TEST_CASE("NewUserdataOverflow")
+{
+    StateRef globalState(luaL_newstate(), lua_close);
+    lua_State* L = globalState.get();
+
+    lua_pushcfunction(L, [](lua_State* L1) {
+        // The following userdata request might cause an overflow.
+        lua_newuserdatadtor(L1, SIZE_MAX, [](void* d){});
+        // The overflow might segfault in the following call.
+        lua_getmetatable(L1, -1);
+        return 0;
+    }, "PCall");
+
+    CHECK(lua_pcall(L, 0, 0, 0) == LUA_ERRRUN);
+    CHECK(strcmp(lua_tostring(L, -1), "memory allocation error: block too big") == 0);
+}
+
 TEST_CASE("ApiTables")
 {
     StateRef globalState(luaL_newstate(), lua_close);