Prevent overflow in lua_newuserdatadtor (#639)

In case a large userdata size is passed to lua_newuserdatadtor it might overflow the size resulting in luaU_newudata actually allocating the object without a memory error. This will then result in overwriting part of the metatable pointer of the userdata.
This PR fixes this issue by checking for the overflow and in such cases pass a size value which will cause a memory error in luaU_newudata.
This commit is contained in:
XmiliaH 2022-08-17 00:32:48 +02:00 committed by GitHub
parent cd26f88d56
commit 4ded555cc5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 1 deletions

View File

@ -1209,7 +1209,10 @@ void* lua_newuserdatadtor(lua_State* L, size_t sz, void (*dtor)(void*))
{
luaC_checkGC(L);
luaC_checkthreadsleep(L);
Udata* u = luaU_newudata(L, sz + sizeof(dtor), UTAG_IDTOR);
size_t as = sz + sizeof(dtor);
if (as < sizeof(dtor))
as = SIZE_MAX; // Will cause a memory error in luaU_newudata.
Udata* u = luaU_newudata(L, as, UTAG_IDTOR);
memcpy(&u->data + sz, &dtor, sizeof(dtor));
setuvalue(L, L->top, u);
api_incr_top(L);

View File

@ -716,6 +716,23 @@ TEST_CASE("Reference")
CHECK(dtorhits == 2);
}
TEST_CASE("NewUserdataOverflow")
{
StateRef globalState(luaL_newstate(), lua_close);
lua_State* L = globalState.get();
lua_pushcfunction(L, [](lua_State* L1) {
// The following userdata request might cause an overflow.
lua_newuserdatadtor(L1, SIZE_MAX, [](void* d){});
// The overflow might segfault in the following call.
lua_getmetatable(L1, -1);
return 0;
}, "PCall");
CHECK(lua_pcall(L, 0, 0, 0) == LUA_ERRRUN);
CHECK(strcmp(lua_tostring(L, -1), "memory allocation error: block too big") == 0);
}
TEST_CASE("ApiTables")
{
StateRef globalState(luaL_newstate(), lua_close);