michael/luau
michael
/
luau
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Sync to upstream/release/511 (#324) 

- TableOperations lint now includes a warning for table.create(N, {}) (which is likely a mistake since the table is shared by all entries)
- Type checker now type checks #v when v is a union
- Parser now rejects sources that consists of a single unfinished long comment
- Work around significant MSVC 2022 performance regression, bringing it more or less in line with MSVC 2019
- Compiler now predicts array size for newly allocated tables when the table is filled in a short loop
- Small improvements in compilation throughput (~2% faster)
- Implement paged sweeper for GC which improves sweep throughput 2-3x and reduces memory consumption by 8 bytes per object (once it is stabilized we will see additional 8 bytes per object of savings)
- Improve Repl Tab completion
- Repl now supports -i (interactive mode to run code in context of a script's environment) and -On (to control optimization flags)
This commit is contained in:
Arseny Kapoulkine 2022-01-21 09:00:19 -08:00 committed by GitHub
parent 478a3da634
commit 8fe95c9963
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
36 changed files with 1275 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Analysis/include/Luau/TypeVar.h
View File

@ -1,6 +1,7 @@
// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details
#pragma once
#include "Luau/DenseHash.h"
#include "Luau/Predicate.h"
#include "Luau/Unifiable.h"
#include "Luau/Variant.h"
@ -499,6 +500,9 @@ bool maybeGeneric(const TypeId ty);
// Checks if a type is of the form T1|...|Tn where one of the Ti is a singleton
bool maybeSingleton(TypeId ty);
// Checks if the length operator can be applied on the value of type
bool hasLength(TypeId ty, DenseHashSet<TypeId>& seen, int* recursionCount);
struct SingletonTypes
{
const TypeId nilType;

15
Analysis/src/Linter.cpp
View File

@ -12,6 +12,8 @@
#include <math.h>
#include <limits.h>
LUAU_FASTFLAGVARIABLE(LuauLintTableCreateTable, false)
namespace Luau
{
@ -2153,6 +2155,19 @@ private:
"table.move uses index 0 but arrays are 1-based; did you mean 1 instead?");
}
if (FFlag::LuauLintTableCreateTable && func->index == "create" && node->args.size == 2)
{
// table.create(n, {...})
if (args[1]->is<AstExprTable>())
emitWarning(*context, LintWarning::Code_TableOperations, args[1]->location,
"table.create with a table literal will reuse the same object for all elements; consider using a for loop instead");
// table.create(n, {...} :: ?)
if (AstExprTypeAssertion* as = args[1]->as<AstExprTypeAssertion>(); as && as->expr->is<AstExprTable>())
emitWarning(*context, LintWarning::Code_TableOperations, as->expr->location,
"table.create with a table literal will reuse the same object for all elements; consider using a for loop instead");
}
return true;
}

29
Analysis/src/TypeInfer.cpp
View File

@ -33,6 +33,7 @@ LUAU_FASTFLAGVARIABLE(DebugLuauFreezeDuringUnification, false)
LUAU_FASTFLAGVARIABLE(LuauRecursiveTypeParameterRestriction, false)
LUAU_FASTFLAGVARIABLE(LuauIfElseBranchTypeUnion, false)
LUAU_FASTFLAGVARIABLE(LuauIfElseExpectedType2, false)
LUAU_FASTFLAGVARIABLE(LuauLengthOnCompositeType, false)
LUAU_FASTFLAGVARIABLE(LuauQuantifyInPlace2, false)
LUAU_FASTFLAGVARIABLE(LuauSealExports, false)
LUAU_FASTFLAGVARIABLE(LuauSingletonTypes, false)
@ -2066,17 +2067,27 @@ ExprResult<TypeId> TypeChecker::checkExpr(const ScopePtr& scope, const AstExprUn
if (get<ErrorTypeVar>(operandType))
return {errorRecoveryType(scope)};
if (get<AnyTypeVar>(operandType))
return {numberType}; // Not strictly correct: metatables permit overriding this
if (auto p = get<PrimitiveTypeVar>(operandType))
if (FFlag::LuauLengthOnCompositeType)
{
if (p->type == PrimitiveTypeVar::String)
return {numberType};
}
DenseHashSet<TypeId> seen{nullptr};
if (!getTableType(operandType))
reportError(TypeError{expr.location, NotATable{operandType}});
if (!hasLength(operandType, seen, &recursionCount))
reportError(TypeError{expr.location, NotATable{operandType}});
}
else
{
if (get<AnyTypeVar>(operandType))
return {numberType}; // Not strictly correct: metatables permit overriding this
if (auto p = get<PrimitiveTypeVar>(operandType))
{
if (p->type == PrimitiveTypeVar::String)
return {numberType};
}
if (!getTableType(operandType))
reportError(TypeError{expr.location, NotATable{operandType}});
}
return {numberType};

46
Analysis/src/TypeVar.cpp
View File

@ -5,6 +5,7 @@
#include "Luau/Common.h"
#include "Luau/DenseHash.h"
#include "Luau/Error.h"
#include "Luau/RecursionCounter.h"
#include "Luau/StringUtils.h"
#include "Luau/ToString.h"
#include "Luau/TypeInfer.h"
@ -19,6 +20,8 @@
LUAU_FASTINTVARIABLE(LuauTypeMaximumStringifierLength, 500)
LUAU_FASTINTVARIABLE(LuauTableTypeMaximumStringifierLength, 0)
LUAU_FASTINT(LuauTypeInferRecursionLimit)
LUAU_FASTFLAG(LuauLengthOnCompositeType)
LUAU_FASTFLAGVARIABLE(LuauMetatableAreEqualRecursion, false)
LUAU_FASTFLAGVARIABLE(LuauRefactorTagging, false)
LUAU_FASTFLAG(LuauErrorRecoveryType)
@ -326,6 +329,49 @@ bool maybeSingleton(TypeId ty)
return false;
}
bool hasLength(TypeId ty, DenseHashSet<TypeId>& seen, int* recursionCount)
{
LUAU_ASSERT(FFlag::LuauLengthOnCompositeType);
RecursionLimiter _rl(recursionCount, FInt::LuauTypeInferRecursionLimit);
ty = follow(ty);
if (seen.contains(ty))
return true;
if (isPrim(ty, PrimitiveTypeVar::String) || get<AnyTypeVar>(ty) || get<TableTypeVar>(ty) || get<MetatableTypeVar>(ty))
return true;
if (auto uty = get<UnionTypeVar>(ty))
{
seen.insert(ty);
for (TypeId part : uty->options)
{
if (!hasLength(part, seen, recursionCount))
return false;
}
return true;
}
if (auto ity = get<IntersectionTypeVar>(ty))
{
seen.insert(ty);
for (TypeId part : ity->parts)
{
if (hasLength(part, seen, recursionCount))
return true;
}
return false;
}
return false;
}
FunctionTypeVar::FunctionTypeVar(TypePackId argTypes, TypePackId retType, std::optional<FunctionDefinition> defn, bool hasSelf)
: argTypes(argTypes)
, retType(retType)

6
Analysis/src/Unifier.cpp
View File

@ -13,6 +13,7 @@
LUAU_FASTINT(LuauTypeInferRecursionLimit);
LUAU_FASTINT(LuauTypeInferTypePackLoopLimit);
LUAU_FASTFLAGVARIABLE(LuauCommittingTxnLogFreeTpPromote, false)
LUAU_FASTFLAG(LuauUseCommittingTxnLog)
LUAU_FASTINTVARIABLE(LuauTypeInferIterationLimit, 2000);
LUAU_FASTFLAGVARIABLE(LuauTableSubtypingVariance2, false);
@ -99,6 +100,11 @@ struct PromoteTypeLevels
bool operator()(TypePackId tp, const FreeTypePack&)
{
// Surprise, it's actually a BoundTypePack that hasn't been committed yet.
// Calling getMutable on this will trigger an assertion.
if (FFlag::LuauCommittingTxnLogFreeTpPromote && FFlag::LuauUseCommittingTxnLog && !log.is<FreeTypePack>(tp))
return true;
promote(tp, FFlag::LuauUseCommittingTxnLog ? log.getMutable<FreeTypePack>(tp) : getMutable<FreeTypePack>(tp));
return true;
}

3
Ast/include/Luau/Ast.h
View File

@ -1265,7 +1265,8 @@ struct hash<Luau::AstName>
size_t operator()(const Luau::AstName& value) const
{
// note: since operator== uses pointer identity, hashing function uses it as well
return value.value ? std::hash<const void*>()(value.value) : 0;
// the hasher is the same as DenseHashPointer (DenseHash.h)
return (uintptr_t(value.value) >> 4) ^ (uintptr_t(value.value) >> 9);
}
};

8
Ast/include/Luau/DenseHash.h
View File

@ -12,10 +12,6 @@
namespace Luau
{
// Internal implementation of DenseHashSet and DenseHashMap
namespace detail
{
struct DenseHashPointer
{
size_t operator()(const void* key) const
@ -24,6 +20,10 @@ struct DenseHashPointer
}
};
// Internal implementation of DenseHashSet and DenseHashMap
namespace detail
{
template<typename T>
using DenseHashDefault = std::conditional_t<std::is_pointer_v<T>, DenseHashPointer, std::hash<T>>;

20
Ast/src/Parser.cpp
View File

@ -14,6 +14,7 @@ LUAU_FASTFLAGVARIABLE(LuauFixAmbiguousErrorRecoveryInAssign, false)
LUAU_FASTFLAGVARIABLE(LuauParseSingletonTypes, false)
LUAU_FASTFLAGVARIABLE(LuauParseTypeAliasDefaults, false)
LUAU_FASTFLAGVARIABLE(LuauParseRecoverTypePackEllipsis, false)
LUAU_FASTFLAGVARIABLE(LuauStartingBrokenComment, false)
namespace Luau
{
@ -174,10 +175,23 @@ ParseResult Parser::parse(const char* buffer, size_t bufferSize, AstNameTable& n
const Lexeme::Type type = p.lexer.current().type;
const Location loc = p.lexer.current().location;
p.lexer.next();
if (FFlag::LuauStartingBrokenComment)
{
if (options.captureComments)
p.commentLocations.push_back(Comment{type, loc});
if (options.captureComments)
p.commentLocations.push_back(Comment{type, loc});
if (type == Lexeme::BrokenComment)
break;
p.lexer.next();
}
else
{
p.lexer.next();
if (options.captureComments)
p.commentLocations.push_back(Comment{type, loc});
}
}
p.lexer.setSkipComments(true);

110
CLI/Repl.cpp
View File

@ -35,10 +35,15 @@ enum class CompileFormat
Binary
};
struct GlobalOptions
{
int optimizationLevel = 1;
} globalOptions;
static Luau::CompileOptions copts()
{
Luau::CompileOptions result = {};
result.optimizationLevel = 1;
result.optimizationLevel = globalOptions.optimizationLevel;
result.debugLevel = 1;
result.coverageLevel = coverageActive() ? 2 : 0;
@ -232,13 +237,14 @@ static std::string runCode(lua_State* L, const std::string& source)
static void completeIndexer(lua_State* L, const char* editBuffer, size_t start, std::vector<std::string>& completions)
{
std::string_view lookup = editBuffer + start;
char lastSep = 0;
for (;;)
{
size_t dot = lookup.find('.');
std::string_view prefix = lookup.substr(0, dot);
size_t sep = lookup.find_first_of(".:");
std::string_view prefix = lookup.substr(0, sep);
if (dot == std::string_view::npos)
if (sep == std::string_view::npos)
{
// table, key
lua_pushnil(L);
@ -249,11 +255,22 @@ static void completeIndexer(lua_State* L, const char* editBuffer, size_t start,
{
// table, key, value
std::string_view key = lua_tostring(L, -2);
int valueType = lua_type(L, -1);
if (!key.empty() && Luau::startsWith(key, prefix))
completions.push_back(editBuffer + std::string(key.substr(prefix.size())));
// If the last separator was a ':' (i.e. a method call) then only functions should be completed.
bool requiredValueType = (lastSep != ':' || valueType == LUA_TFUNCTION);
if (!key.empty() && requiredValueType && Luau::startsWith(key, prefix))
{
std::string completion(editBuffer + std::string(key.substr(prefix.size())));
if (valueType == LUA_TFUNCTION)
{
// Add an opening paren for function calls by default.
completion += "(";
}
completions.push_back(completion);
}
}
lua_pop(L, 1);
}
@ -266,10 +283,21 @@ static void completeIndexer(lua_State* L, const char* editBuffer, size_t start,
lua_rawget(L, -2);
lua_remove(L, -2);
if (!lua_istable(L, -1))
if (lua_type(L, -1) == LUA_TSTRING)
{
// Replace the string object with the string class to perform further lookups of string functions
// Note: We retrieve the string class from _G to prevent issues if the user assigns to `string`.
lua_getglobal(L, "_G");
lua_pushlstring(L, "string", 6);
lua_rawget(L, -2);
lua_remove(L, -2);
LUAU_ASSERT(lua_istable(L, -1));
}
else if (!lua_istable(L, -1))
break;
lookup.remove_prefix(dot + 1);
lastSep = lookup[sep];
lookup.remove_prefix(sep + 1);
}
}
@ -279,7 +307,7 @@ static void completeIndexer(lua_State* L, const char* editBuffer, size_t start,
static void completeRepl(lua_State* L, const char* editBuffer, std::vector<std::string>& completions)
{
size_t start = strlen(editBuffer);
while (start > 0 && (isalnum(editBuffer[start - 1]) || editBuffer[start - 1] == '.' || editBuffer[start - 1] == '_'))
while (start > 0 && (isalnum(editBuffer[start - 1]) || editBuffer[start - 1] == '.' || editBuffer[start - 1] == ':' || editBuffer[start - 1] == '_'))
start--;
// look the value up in current global table first
@ -319,15 +347,8 @@ struct LinenoiseScopedHistory
std::string historyFilepath;
};
static void runRepl()
static void runReplImpl(lua_State* L)
{
std::unique_ptr<lua_State, void (*)(lua_State*)> globalState(luaL_newstate(), lua_close);
lua_State* L = globalState.get();
setupState(L);
luaL_sandboxthread(L);
linenoise::SetCompletionCallback([L](const char* editBuffer, std::vector<std::string>& completions) {
completeRepl(L, editBuffer, completions);
});
@ -368,7 +389,18 @@ static void runRepl()
}
}
static bool runFile(const char* name, lua_State* GL)
static void runRepl()
{
std::unique_ptr<lua_State, void (*)(lua_State*)> globalState(luaL_newstate(), lua_close);
lua_State* L = globalState.get();
setupState(L);
luaL_sandboxthread(L);
runReplImpl(L);
}
// `repl` is used it indicate if a repl should be started after executing the file.
static bool runFile(const char* name, lua_State* GL, bool repl)
{
std::optional<std::string> source = readFile(name);
if (!source)
@ -419,6 +451,10 @@ static bool runFile(const char* name, lua_State* GL)
fprintf(stderr, "%s", error.c_str());
}
if (repl)
{
runReplImpl(L);
}
lua_pop(GL, 1);
return status == 0;
}
@ -457,7 +493,7 @@ static bool compileFile(const char* name, CompileFormat format)
bcb.setDumpSource(*source);
}
Luau::compileOrThrow(bcb, *source);
Luau::compileOrThrow(bcb, *source, copts());
switch (format)
{
@ -495,9 +531,11 @@ static void displayHelp(const char* argv0)
printf(" --compile[=format]: compile input files and output resulting formatted bytecode (binary or text)\n");
printf("\n");
printf("Available options:\n");
printf(" -h, --help: Display this usage message.\n");
printf(" --profile[=N]: profile the code using N Hz sampling (default 10000) and output results to profile.out\n");
printf(" --coverage: collect code coverage while running the code and output results to coverage.out\n");
printf(" -h, --help: Display this usage message.\n");
printf(" -i, --interactive: Run an interactive REPL after executing the last script specified.\n");
printf(" -O<n>: use compiler optimization level (n=0-2).\n");
printf(" --profile[=N]: profile the code using N Hz sampling (default 10000) and output results to profile.out\n");
printf(" --timetrace: record compiler time tracing information into trace.json\n");
}
@ -519,6 +557,7 @@ int main(int argc, char** argv)
CompileFormat compileFormat{};
int profile = 0;
bool coverage = false;
bool interactive = false;
// Set the mode if the user has explicitly specified one.
int argStart = 1;
@ -540,8 +579,8 @@ int main(int argc, char** argv)
}
else
{
fprintf(stdout, "Error: Unrecognized value for '--compile' specified.\n");
return -1;
fprintf(stderr, "Error: Unrecognized value for '--compile' specified.\n");
return 1;
}
}
@ -552,6 +591,20 @@ int main(int argc, char** argv)
displayHelp(argv[0]);
return 0;
}
else if (strcmp(argv[i], "-i") == 0 || strcmp(argv[i], "--interactive") == 0)
{
interactive = true;
}
else if (strncmp(argv[i], "-O", 2) == 0)
{
int level = atoi(argv[i] + 2);
if (level < 0 || level > 2)
{
fprintf(stderr, "Error: Optimization level must be between 0 and 2 inclusive.\n");
return 1;
}
globalOptions.optimizationLevel = level;
}
else if (strcmp(argv[i], "--profile") == 0)
{
profile = 10000; // default to 10 KHz
@ -575,7 +628,7 @@ int main(int argc, char** argv)
}
else if (argv[i][0] == '-')
{
fprintf(stdout, "Error: Unrecognized option '%s'.\n\n", argv[i]);
fprintf(stderr, "Error: Unrecognized option '%s'.\n\n", argv[i]);
displayHelp(argv[0]);
return 1;
}
@ -623,8 +676,11 @@ int main(int argc, char** argv)
int failed = 0;
for (const std::string& path : files)
failed += !runFile(path.c_str(), L);
for (size_t i = 0; i < files.size(); ++i)
{
bool isLastFile = i == files.size() - 1;
failed += !runFile(files[i].c_str(), L, interactive && isLastFile);
}
if (profile)
{

6
CMakeLists.txt
View File

@ -78,6 +78,12 @@ target_compile_options(Luau.Ast PRIVATE ${LUAU_OPTIONS})
target_compile_options(Luau.Analysis PRIVATE ${LUAU_OPTIONS})
target_compile_options(Luau.VM PRIVATE ${LUAU_OPTIONS})
if (MSVC AND MSVC_VERSION GREATER_EQUAL 1924)
# disable partial redundancy elimination which regresses interpreter codegen substantially in VS2022:
# https://developercommunity.visualstudio.com/t/performance-regression-on-a-complex-interpreter-lo/1631863
set_source_files_properties(VM/src/lvmexecute.cpp PROPERTIES COMPILE_FLAGS /d2ssa-pre-)
endif()
if(LUAU_BUILD_CLI)
target_compile_options(Luau.Repl.CLI PRIVATE ${LUAU_OPTIONS})
target_compile_options(Luau.Analyze.CLI PRIVATE ${LUAU_OPTIONS})

49
Compiler/src/TableShape.cpp
View File

@ -1,11 +1,16 @@
// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details
#include "TableShape.h"
LUAU_FASTFLAGVARIABLE(LuauPredictTableSizeLoop, false)
namespace Luau
{
namespace Compile
{
// conservative limit for the loop bound that establishes table array size
static const int kMaxLoopBound = 16;
static AstExprTable* getTableHint(AstExpr* expr)
{
// unadorned table literal
@ -27,7 +32,7 @@ struct ShapeVisitor : AstVisitor
{
size_t operator()(const std::pair<AstExprTable*, AstName>& p) const
{
return std::hash<AstExprTable*>()(p.first) ^ std::hash<AstName>()(p.second);
return DenseHashPointer()(p.first) ^ std::hash<AstName>()(p.second);
}
};
@ -36,10 +41,13 @@ struct ShapeVisitor : AstVisitor
DenseHashMap<AstLocal*, AstExprTable*> tables;
DenseHashSet<std::pair<AstExprTable*, AstName>, Hasher> fields;
DenseHashMap<AstLocal*, unsigned int> loops; // iterator => upper bound for 1..k
ShapeVisitor(DenseHashMap<AstExprTable*, TableShape>& shapes)
: shapes(shapes)
, tables(nullptr)
, fields(std::pair<AstExprTable*, AstName>())
, loops(nullptr)
{
}
@ -63,16 +71,31 @@ struct ShapeVisitor : AstVisitor
void assignField(AstExpr* expr, AstExpr* index)
{
AstExprLocal* lv = expr->as<AstExprLocal>();
AstExprConstantNumber* number = index->as<AstExprConstantNumber>();
if (!lv)
return;
if (lv && number)
AstExprTable** table = tables.find(lv->local);
if (!table)
return;
if (AstExprConstantNumber* number = index->as<AstExprConstantNumber>())
{
if (AstExprTable** table = tables.find(lv->local))
TableShape& shape = shapes[*table];
if (number->value == double(shape.arraySize + 1))
shape.arraySize += 1;
}
else if (AstExprLocal* iter = index->as<AstExprLocal>())
{
if (!FFlag::LuauPredictTableSizeLoop)
return;
if (const unsigned int* bound = loops.find(iter->local))
{
TableShape& shape = shapes[*table];
if (number->value == double(shape.arraySize + 1))
shape.arraySize += 1;
if (shape.arraySize == 0)
shape.arraySize = *bound;
}
}
}
@ -117,6 +140,20 @@ struct ShapeVisitor : AstVisitor
return false;
}
bool visit(AstStatFor* node) override
{
if (!FFlag::LuauPredictTableSizeLoop)
return true;
AstExprConstantNumber* from = node->from->as<AstExprConstantNumber>();
AstExprConstantNumber* to = node->to->as<AstExprConstantNumber>();
if (from && to && from->value == 1.0 && to->value >= 1.0 && to->value <= double(kMaxLoopBound) && !node->step)
loops[node->var] = unsigned(to->value);
return true;
}
};
void predictTableShapes(DenseHashMap<AstExprTable*, TableShape>& shapes, AstNode* root)

9
VM/src/ldo.cpp
View File

@ -150,12 +150,11 @@ l_noret luaD_throw(lua_State* L, int errcode)
static void correctstack(lua_State* L, TValue* oldstack)
{
CallInfo* ci;
GCObject* up;
L->top = (L->top - oldstack) + L->stack;
for (up = L->openupval; up != NULL; up = up->gch.next)
gco2uv(up)->v = (gco2uv(up)->v - oldstack) + L->stack;
for (ci = L->base_ci; ci <= L->ci; ci++)
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
for (UpVal* up = L->openupval; up != NULL; up = (UpVal*)up->next)
up->v = (up->v - oldstack) + L->stack;
for (CallInfo* ci = L->base_ci; ci <= L->ci; ci++)
{
ci->top = (ci->top - oldstack) + L->stack;
ci->base = (ci->base - oldstack) + L->stack;

96
VM/src/lfunc.cpp
View File

@ -7,10 +7,11 @@
#include "lgc.h"
LUAU_FASTFLAGVARIABLE(LuauNoDirectUpvalRemoval, false)
LUAU_FASTFLAG(LuauGcPagedSweep)
Proto* luaF_newproto(lua_State* L)
{
Proto* f = luaM_new(L, Proto, sizeof(Proto), L->activememcat);
Proto* f = luaM_newgco(L, Proto, sizeof(Proto), L->activememcat);
luaC_link(L, f, LUA_TPROTO);
f->k = NULL;
f->sizek = 0;
@ -38,7 +39,7 @@ Proto* luaF_newproto(lua_State* L)
Closure* luaF_newLclosure(lua_State* L, int nelems, Table* e, Proto* p)
{
Closure* c = luaM_new(L, Closure, sizeLclosure(nelems), L->activememcat);
Closure* c = luaM_newgco(L, Closure, sizeLclosure(nelems), L->activememcat);
luaC_link(L, c, LUA_TFUNCTION);
c->isC = 0;
c->env = e;
@ -53,7 +54,7 @@ Closure* luaF_newLclosure(lua_State* L, int nelems, Table* e, Proto* p)
Closure* luaF_newCclosure(lua_State* L, int nelems, Table* e)
{
Closure* c = luaM_new(L, Closure, sizeCclosure(nelems), L->activememcat);
Closure* c = luaM_newgco(L, Closure, sizeCclosure(nelems), L->activememcat);
luaC_link(L, c, LUA_TFUNCTION);
c->isC = 1;
c->env = e;
@ -69,10 +70,9 @@ Closure* luaF_newCclosure(lua_State* L, int nelems, Table* e)
UpVal* luaF_findupval(lua_State* L, StkId level)
{
global_State* g = L->global;
GCObject** pp = &L->openupval;
UpVal** pp = &L->openupval;
UpVal* p;
UpVal* uv;
while (*pp != NULL && (p = gco2uv(*pp))->v >= level)
while (*pp != NULL && (p = *pp)->v >= level)
{
LUAU_ASSERT(p->v != &p->u.value);
if (p->v == level)
@ -81,53 +81,95 @@ UpVal* luaF_findupval(lua_State* L, StkId level)
changewhite(obj2gco(p)); /* resurrect it */
return p;
}
pp = &p->next;
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
pp = (UpVal**)&p->next;
}
uv = luaM_new(L, UpVal, sizeof(UpVal), L->activememcat); /* not found: create a new one */
UpVal* uv = luaM_newgco(L, UpVal, sizeof(UpVal), L->activememcat); /* not found: create a new one */
uv->tt = LUA_TUPVAL;
uv->marked = luaC_white(g);
uv->memcat = L->activememcat;
uv->v = level; /* current value lives in the stack */
uv->next = *pp; /* chain it in the proper position */
*pp = obj2gco(uv);
uv->u.l.prev = &g->uvhead; /* double link it in `uvhead' list */
// chain the upvalue in the threads open upvalue list at the proper position
UpVal* next = *pp;
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
uv->next = (GCObject*)next;
if (FFlag::LuauGcPagedSweep)
{
uv->u.l.threadprev = pp;
if (next)
{
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
next->u.l.threadprev = (UpVal**)&uv->next;
}
}
*pp = uv;
// double link the upvalue in the global open upvalue list
uv->u.l.prev = &g->uvhead;
uv->u.l.next = g->uvhead.u.l.next;
uv->u.l.next->u.l.prev = uv;
g->uvhead.u.l.next = uv;
LUAU_ASSERT(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv);
return uv;
}
static void unlinkupval(UpVal* uv)
void luaF_unlinkupval(UpVal* uv)
{
// unlink upvalue from the global open upvalue list
LUAU_ASSERT(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv);
uv->u.l.next->u.l.prev = uv->u.l.prev; /* remove from `uvhead' list */
uv->u.l.next->u.l.prev = uv->u.l.prev;
uv->u.l.prev->u.l.next = uv->u.l.next;
if (FFlag::LuauGcPagedSweep)
{
// unlink upvalue from the thread open upvalue list
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and this and the following cast will not be required
*uv->u.l.threadprev = (UpVal*)uv->next;
if (UpVal* next = (UpVal*)uv->next)
next->u.l.threadprev = uv->u.l.threadprev;
}
}
void luaF_freeupval(lua_State* L, UpVal* uv)
void luaF_freeupval(lua_State* L, UpVal* uv, lua_Page* page)
{
if (uv->v != &uv->u.value) /* is it open? */
unlinkupval(uv); /* remove from open list */
luaM_free(L, uv, sizeof(UpVal), uv->memcat); /* free upvalue */
luaF_unlinkupval(uv); /* remove from open list */
luaM_freegco(L, uv, sizeof(UpVal), uv->memcat, page); /* free upvalue */
}
void luaF_close(lua_State* L, StkId level)
{
UpVal* uv;
global_State* g = L->global; // TODO: remove with FFlagLuauNoDirectUpvalRemoval
while (L->openupval != NULL && (uv = gco2uv(L->openupval))->v >= level)
UpVal* uv;
while (L->openupval != NULL && (uv = L->openupval)->v >= level)
{
GCObject* o = obj2gco(uv);
LUAU_ASSERT(!isblack(o) && uv->v != &uv->u.value);
L->openupval = uv->next; /* remove from `open' list */
if (!FFlag::LuauNoDirectUpvalRemoval && isdead(g, o))
if (!FFlag::LuauGcPagedSweep)
L->openupval = (UpVal*)uv->next; /* remove from `open' list */
if (FFlag::LuauGcPagedSweep && isdead(g, o))
{
luaF_freeupval(L, uv); /* free upvalue */
// by removing the upvalue from global/thread open upvalue lists, L->openupval will be pointing to the next upvalue
luaF_unlinkupval(uv);
// close the upvalue without copying the dead data so that luaF_freeupval will not unlink again
uv->v = &uv->u.value;
}
else if (!FFlag::LuauNoDirectUpvalRemoval && isdead(g, o))
{
luaF_freeupval(L, uv, NULL); /* free upvalue */
}
else
{
unlinkupval(uv);
// by removing the upvalue from global/thread open upvalue lists, L->openupval will be pointing to the next upvalue
luaF_unlinkupval(uv);
setobj(L, &uv->u.value, uv->v);
uv->v = &uv->u.value; /* now current value lives here */
luaC_linkupval(L, uv); /* link upvalue into `gcroot' list */
@ -135,7 +177,7 @@ void luaF_close(lua_State* L, StkId level)
}
}
void luaF_freeproto(lua_State* L, Proto* f)
void luaF_freeproto(lua_State* L, Proto* f, lua_Page* page)
{
luaM_freearray(L, f->code, f->sizecode, Instruction, f->memcat);
luaM_freearray(L, f->p, f->sizep, Proto*, f->memcat);
@ -146,13 +188,13 @@ void luaF_freeproto(lua_State* L, Proto* f)
luaM_freearray(L, f->upvalues, f->sizeupvalues, TString*, f->memcat);
if (f->debuginsn)
luaM_freearray(L, f->debuginsn, f->sizecode, uint8_t, f->memcat);
luaM_free(L, f, sizeof(Proto), f->memcat);
luaM_freegco(L, f, sizeof(Proto), f->memcat, page);
}
void luaF_freeclosure(lua_State* L, Closure* c)
void luaF_freeclosure(lua_State* L, Closure* c, lua_Page* page)
{
int size = c->isC ? sizeCclosure(c->nupvalues) : sizeLclosure(c->nupvalues);
luaM_free(L, c, size, c->memcat);
luaM_freegco(L, c, size, c->memcat, page);
}
const LocVar* luaF_getlocal(const Proto* f, int local_number, int pc)

7
VM/src/lfunc.h
View File

@ -12,7 +12,8 @@ LUAI_FUNC Closure* luaF_newLclosure(lua_State* L, int nelems, Table* e, Proto* p
LUAI_FUNC Closure* luaF_newCclosure(lua_State* L, int nelems, Table* e);
LUAI_FUNC UpVal* luaF_findupval(lua_State* L, StkId level);
LUAI_FUNC void luaF_close(lua_State* L, StkId level);
LUAI_FUNC void luaF_freeproto(lua_State* L, Proto* f);
LUAI_FUNC void luaF_freeclosure(lua_State* L, Closure* c);
LUAI_FUNC void luaF_freeupval(lua_State* L, UpVal* uv);
LUAI_FUNC void luaF_freeproto(lua_State* L, Proto* f, struct lua_Page* page);
LUAI_FUNC void luaF_freeclosure(lua_State* L, Closure* c, struct lua_Page* page);
void luaF_unlinkupval(UpVal* uv);
LUAI_FUNC void luaF_freeupval(lua_State* L, UpVal* uv, struct lua_Page* page);
LUAI_FUNC const LocVar* luaF_getlocal(const Proto* func, int local_number, int pc);

251
VM/src/lgc.cpp
View File

@ -8,12 +8,16 @@
#include "lfunc.h"
#include "lstring.h"
#include "ldo.h"
#include "lmem.h"
#include "ludata.h"
#include <string.h>
LUAU_FASTFLAGVARIABLE(LuauGcPagedSweep, false)
#define GC_SWEEPMAX 40
#define GC_SWEEPCOST 10
#define GC_SWEEPPAGESTEPCOST 4
#define GC_INTERRUPT(state) \
{ \
@ -457,31 +461,31 @@ static void shrinkstack(lua_State* L)
condhardstacktests(luaD_reallocstack(L, s_used));
}
static void freeobj(lua_State* L, GCObject* o)
static void freeobj(lua_State* L, GCObject* o, lua_Page* page)
{
switch (o->gch.tt)
{
case LUA_TPROTO:
luaF_freeproto(L, gco2p(o));
luaF_freeproto(L, gco2p(o), page);
break;
case LUA_TFUNCTION:
luaF_freeclosure(L, gco2cl(o));
luaF_freeclosure(L, gco2cl(o), page);
break;
case LUA_TUPVAL:
luaF_freeupval(L, gco2uv(o));
luaF_freeupval(L, gco2uv(o), page);
break;
case LUA_TTABLE:
luaH_free(L, gco2h(o));
luaH_free(L, gco2h(o), page);
break;
case LUA_TTHREAD:
LUAU_ASSERT(gco2th(o) != L && gco2th(o) != L->global->mainthread);
luaE_freethread(L, gco2th(o));
luaE_freethread(L, gco2th(o), page);
break;
case LUA_TSTRING:
luaS_free(L, gco2ts(o));
luaS_free(L, gco2ts(o), page);
break;
case LUA_TUSERDATA:
luaU_freeudata(L, gco2u(o));
luaU_freeudata(L, gco2u(o), page);
break;
default:
LUAU_ASSERT(0);
@ -492,6 +496,8 @@ static void freeobj(lua_State* L, GCObject* o)
static GCObject** sweeplist(lua_State* L, GCObject** p, size_t count, size_t* traversedcount)
{
LUAU_ASSERT(!FFlag::LuauGcPagedSweep);
GCObject* curr;
global_State* g = L->global;
int deadmask = otherwhite(g);
@ -502,7 +508,7 @@ static GCObject** sweeplist(lua_State* L, GCObject** p, size_t count, size_t* tr
int alive = (curr->gch.marked ^ WHITEBITS) & deadmask;
if (curr->gch.tt == LUA_TTHREAD)
{
sweepwholelist(L, &gco2th(curr)->openupval, traversedcount); /* sweep open upvalues */
sweepwholelist(L, (GCObject**)&gco2th(curr)->openupval, traversedcount); /* sweep open upvalues */
lua_State* th = gco2th(curr);
@ -524,7 +530,7 @@ static GCObject** sweeplist(lua_State* L, GCObject** p, size_t count, size_t* tr
*p = curr->gch.next;
if (curr == g->rootgc) /* is the first element of the list? */
g->rootgc = curr->gch.next; /* adjust first */
freeobj(L, curr);
freeobj(L, curr, NULL);
}
}
@ -537,14 +543,16 @@ static GCObject** sweeplist(lua_State* L, GCObject** p, size_t count, size_t* tr
static void deletelist(lua_State* L, GCObject** p, GCObject* limit)
{
LUAU_ASSERT(!FFlag::LuauGcPagedSweep);
GCObject* curr;
while ((curr = *p) != limit)
{
if (curr->gch.tt == LUA_TTHREAD) /* delete open upvalues of each thread */
deletelist(L, &gco2th(curr)->openupval, NULL);
deletelist(L, (GCObject**)&gco2th(curr)->openupval, NULL);
*p = curr->gch.next;
freeobj(L, curr);
freeobj(L, curr, NULL);
}
}
@ -567,23 +575,62 @@ static void shrinkbuffersfull(lua_State* L)
luaS_resize(L, hashsize); /* table is too big */
}
static bool deletegco(void* context, lua_Page* page, GCObject* gco)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
// we are in the process of deleting everything
// threads with open upvalues will attempt to close them all on removal
// but those upvalues might point to stack values that were already deleted
if (gco->gch.tt == LUA_TTHREAD)
{
lua_State* th = gco2th(gco);
while (UpVal* uv = th->openupval)
{
luaF_unlinkupval(uv);
// close the upvalue without copying the dead data so that luaF_freeupval will not unlink again
uv->v = &uv->u.value;
}
}
lua_State* L = (lua_State*)context;
freeobj(L, gco, page);
return true;
}
void luaC_freeall(lua_State* L)
{
global_State* g = L->global;
LUAU_ASSERT(L == g->mainthread);
LUAU_ASSERT(L->next == NULL); /* mainthread is at the end of rootgc list */
deletelist(L, &g->rootgc, obj2gco(L));
if (FFlag::LuauGcPagedSweep)
{
luaM_visitgco(L, L, deletegco);
for (int i = 0; i < g->strt.size; i++) /* free all string lists */
deletelist(L, &g->strt.hash[i], NULL);
for (int i = 0; i < g->strt.size; i++) /* free all string lists */
LUAU_ASSERT(g->strt.hash[i] == NULL);
LUAU_ASSERT(L->global->strt.nuse == 0);
deletelist(L, &g->strbufgc, NULL);
// unfortunately, when string objects are freed, the string table use count is decremented
// even when the string is a buffer that wasn't placed into the table
L->global->strt.nuse = 0;
LUAU_ASSERT(L->global->strt.nuse == 0);
LUAU_ASSERT(g->strbufgc == NULL);
}
else
{
LUAU_ASSERT(L->next == NULL); /* mainthread is at the end of rootgc list */
deletelist(L, &g->rootgc, obj2gco(L));
for (int i = 0; i < g->strt.size; i++) /* free all string lists */
deletelist(L, (GCObject**)&g->strt.hash[i], NULL);
LUAU_ASSERT(L->global->strt.nuse == 0);
deletelist(L, (GCObject**)&g->strbufgc, NULL);
// unfortunately, when string objects are freed, the string table use count is decremented
// even when the string is a buffer that wasn't placed into the table
L->global->strt.nuse = 0;
}
}
static void markmt(global_State* g)
@ -648,12 +695,88 @@ static size_t atomic(lua_State* L)
/* flip current white */
g->currentwhite = cast_byte(otherwhite(g));
g->sweepstrgc = 0;
g->sweepgc = &g->rootgc;
g->gcstate = GCSsweepstring;
if (FFlag::LuauGcPagedSweep)
{
g->sweepgcopage = g->allgcopages;
g->gcstate = GCSsweep;
}
else
{
g->sweepgc = &g->rootgc;
g->gcstate = GCSsweepstring;
}
return work;
}
static bool sweepgco(lua_State* L, lua_Page* page, GCObject* gco)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
global_State* g = L->global;
int deadmask = otherwhite(g);
LUAU_ASSERT(testbit(deadmask, FIXEDBIT)); // make sure we never sweep fixed objects
int alive = (gco->gch.marked ^ WHITEBITS) & deadmask;
g->gcstats.currcycle.sweepitems++;
if (gco->gch.tt == LUA_TTHREAD)
{
lua_State* th = gco2th(gco);
if (alive)
{
resetbit(th->stackstate, THREAD_SLEEPINGBIT);
shrinkstack(th);
}
}
if (alive)
{
LUAU_ASSERT(!isdead(g, gco));
makewhite(g, gco); // make it white (for next cycle)
return false;
}
LUAU_ASSERT(isdead(g, gco));
freeobj(L, gco, page);
return true;
}
// a version of generic luaM_visitpage specialized for the main sweep stage
static int sweepgcopage(lua_State* L, lua_Page* page)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
char* start;
char* end;
int busyBlocks;
int blockSize;
luaM_getpagewalkinfo(page, &start, &end, &busyBlocks, &blockSize);
for (char* pos = start; pos != end; pos += blockSize)
{
GCObject* gco = (GCObject*)pos;
// skip memory blocks that are already freed
if (gco->gch.tt == LUA_TNIL)
continue;
// when true is returned it means that the element was deleted
if (sweepgco(L, page, gco))
{
// if the last block was removed, page would be removed as well
if (--busyBlocks == 0)
return int(pos - start) / blockSize + 1;
}
}
return int(end - start) / blockSize;
}
static size_t gcstep(lua_State* L, size_t limit)
{
size_t cost = 0;
@ -706,15 +829,21 @@ static size_t gcstep(lua_State* L, size_t limit)
g->gcstats.currcycle.atomicstarttotalsizebytes = g->totalbytes;
cost = atomic(L); /* finish mark phase */
LUAU_ASSERT(g->gcstate == GCSsweepstring);
if (FFlag::LuauGcPagedSweep)
LUAU_ASSERT(g->gcstate == GCSsweep);
else
LUAU_ASSERT(g->gcstate == GCSsweepstring);
break;
}
case GCSsweepstring:
{
LUAU_ASSERT(!FFlag::LuauGcPagedSweep);
while (g->sweepstrgc < g->strt.size && cost < limit)
{
size_t traversedcount = 0;
sweepwholelist(L, &g->strt.hash[g->sweepstrgc++], &traversedcount);
sweepwholelist(L, (GCObject**)&g->strt.hash[g->sweepstrgc++], &traversedcount);
g->gcstats.currcycle.sweepitems += traversedcount;
cost += GC_SWEEPCOST;
@ -727,7 +856,7 @@ static size_t gcstep(lua_State* L, size_t limit)
uint32_t nuse = L->global->strt.nuse;
size_t traversedcount = 0;
sweepwholelist(L, &g->strbufgc, &traversedcount);
sweepwholelist(L, (GCObject**)&g->strbufgc, &traversedcount);
L->global->strt.nuse = nuse;
@ -738,19 +867,44 @@ static size_t gcstep(lua_State* L, size_t limit)
}
case GCSsweep:
{
while (*g->sweepgc && cost < limit)
if (FFlag::LuauGcPagedSweep)
{
size_t traversedcount = 0;
g->sweepgc = sweeplist(L, g->sweepgc, GC_SWEEPMAX, &traversedcount);
while (g->sweepgcopage && cost < limit)
{
lua_Page* next = luaM_getnextgcopage(g->sweepgcopage); // page sweep might destroy the page
g->gcstats.currcycle.sweepitems += traversedcount;
cost += GC_SWEEPMAX * GC_SWEEPCOST;
int steps = sweepgcopage(L, g->sweepgcopage);
g->sweepgcopage = next;
cost += steps * GC_SWEEPPAGESTEPCOST;
}
// nothing more to sweep?
if (g->sweepgcopage == NULL)
{
// don't forget to visit main thread
sweepgco(L, NULL, obj2gco(g->mainthread));
shrinkbuffers(L);
g->gcstate = GCSpause; /* end collection */
}
}
else
{
while (*g->sweepgc && cost < limit)
{
size_t traversedcount = 0;
g->sweepgc = sweeplist(L, g->sweepgc, GC_SWEEPMAX, &traversedcount);
if (*g->sweepgc == NULL)
{ /* nothing more to sweep? */
shrinkbuffers(L);
g->gcstate = GCSpause; /* end collection */
g->gcstats.currcycle.sweepitems += traversedcount;
cost += GC_SWEEPMAX * GC_SWEEPCOST;
}
if (*g->sweepgc == NULL)
{ /* nothing more to sweep? */
shrinkbuffers(L);
g->gcstate = GCSpause; /* end collection */
}
}
break;
}
@ -877,12 +1031,19 @@ void luaC_fullgc(lua_State* L)
{
/* reset sweep marks to sweep all elements (returning them to white) */
g->sweepstrgc = 0;
g->sweepgc = &g->rootgc;
if (FFlag::LuauGcPagedSweep)
g->sweepgcopage = g->allgcopages;
else
g->sweepgc = &g->rootgc;
/* reset other collector lists */
g->gray = NULL;
g->grayagain = NULL;
g->weak = NULL;
g->gcstate = GCSsweepstring;
if (FFlag::LuauGcPagedSweep)
g->gcstate = GCSsweep;
else
g->gcstate = GCSsweepstring;
}
LUAU_ASSERT(g->gcstate == GCSsweepstring || g->gcstate == GCSsweep);
/* finish any pending sweep phase */
@ -979,8 +1140,11 @@ void luaC_barrierback(lua_State* L, Table* t)
void luaC_linkobj(lua_State* L, GCObject* o, uint8_t tt)
{
global_State* g = L->global;
o->gch.next = g->rootgc;
g->rootgc = o;
if (!FFlag::LuauGcPagedSweep)
{
o->gch.next = g->rootgc;
g->rootgc = o;
}
o->gch.marked = luaC_white(g);
o->gch.tt = tt;
o->gch.memcat = L->activememcat;
@ -990,8 +1154,13 @@ void luaC_linkupval(lua_State* L, UpVal* uv)
{
global_State* g = L->global;
GCObject* o = obj2gco(uv);
o->gch.next = g->rootgc; /* link upvalue into `rootgc' list */
g->rootgc = o;
if (!FFlag::LuauGcPagedSweep)
{
o->gch.next = g->rootgc; /* link upvalue into `rootgc' list */
g->rootgc = o;
}
if (isgray(o))
{
if (keepinvariant(g))

1
VM/src/lgc.h
View File

@ -13,6 +13,7 @@
#define GCSpropagate 1
#define GCSpropagateagain 2
#define GCSatomic 3
// TODO: remove with FFlagLuauGcPagedSweep
#define GCSsweepstring 4
#define GCSsweep 5

77
VM/src/lgcdebug.cpp
View File

@ -2,16 +2,19 @@
// This code is based on Lua 5.x implementation licensed under MIT License; see lua_LICENSE.txt for details
#include "lgc.h"
#include "lfunc.h"
#include "lmem.h"
#include "lobject.h"
#include "lstate.h"
#include "ltable.h"
#include "lfunc.h"
#include "lstring.h"
#include "ltable.h"
#include "ludata.h"
#include <string.h>
#include <stdio.h>
LUAU_FASTFLAG(LuauGcPagedSweep)
static void validateobjref(global_State* g, GCObject* f, GCObject* t)
{
LUAU_ASSERT(!isdead(g, t));
@ -101,10 +104,11 @@ static void validatestack(global_State* g, lua_State* l)
if (l->namecall)
validateobjref(g, obj2gco(l), obj2gco(l->namecall));
for (GCObject* uv = l->openupval; uv; uv = uv->gch.next)
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
for (UpVal* uv = l->openupval; uv; uv = (UpVal*)uv->next)
{
LUAU_ASSERT(uv->gch.tt == LUA_TUPVAL);
LUAU_ASSERT(gco2uv(uv)->v != &gco2uv(uv)->u.value);
LUAU_ASSERT(uv->tt == LUA_TUPVAL);
LUAU_ASSERT(uv->v != &uv->u.value);
}
}
@ -178,6 +182,8 @@ static void validateobj(global_State* g, GCObject* o)
static void validatelist(global_State* g, GCObject* o)
{
LUAU_ASSERT(!FFlag::LuauGcPagedSweep);
while (o)
{
validateobj(g, o);
@ -216,6 +222,17 @@ static void validategraylist(global_State* g, GCObject* o)
}
}
static bool validategco(void* context, lua_Page* page, GCObject* gco)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
lua_State* L = (lua_State*)context;
global_State* g = L->global;
validateobj(g, gco);
return false;
}
void luaC_validate(lua_State* L)
{
global_State* g = L->global;
@ -231,11 +248,18 @@ void luaC_validate(lua_State* L)
validategraylist(g, g->gray);
validategraylist(g, g->grayagain);
for (int i = 0; i < g->strt.size; ++i)
validatelist(g, g->strt.hash[i]);
if (FFlag::LuauGcPagedSweep)
{
luaM_visitgco(L, L, validategco);
}
else
{
for (int i = 0; i < g->strt.size; ++i)
validatelist(g, (GCObject*)(g->strt.hash[i]));
validatelist(g, g->rootgc);
validatelist(g, g->strbufgc);
validatelist(g, g->rootgc);
validatelist(g, (GCObject*)(g->strbufgc));
}
for (UpVal* uv = g->uvhead.u.l.next; uv != &g->uvhead; uv = uv->u.l.next)
{
@ -499,6 +523,8 @@ static void dumpobj(FILE* f, GCObject* o)
static void dumplist(FILE* f, GCObject* o)
{
LUAU_ASSERT(!FFlag::LuauGcPagedSweep);
while (o)
{
dumpref(f, o);
@ -509,22 +535,45 @@ static void dumplist(FILE* f, GCObject* o)
// thread has additional list containing collectable objects that are not present in rootgc
if (o->gch.tt == LUA_TTHREAD)
dumplist(f, gco2th(o)->openupval);
dumplist(f, (GCObject*)gco2th(o)->openupval);
o = o->gch.next;
}
}
static bool dumpgco(void* context, lua_Page* page, GCObject* gco)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
FILE* f = (FILE*)context;
dumpref(f, gco);
fputc(':', f);
dumpobj(f, gco);
fputc(',', f);
fputc('\n', f);
return false;
}
void luaC_dump(lua_State* L, void* file, const char* (*categoryName)(lua_State* L, uint8_t memcat))
{
global_State* g = L->global;
FILE* f = static_cast<FILE*>(file);
fprintf(f, "{\"objects\":{\n");
dumplist(f, g->rootgc);
dumplist(f, g->strbufgc);
for (int i = 0; i < g->strt.size; ++i)
dumplist(f, g->strt.hash[i]);
if (FFlag::LuauGcPagedSweep)
{
luaM_visitgco(L, f, dumpgco);
}
else
{
dumplist(f, g->rootgc);
dumplist(f, (GCObject*)(g->strbufgc));
for (int i = 0; i < g->strt.size; ++i)
dumplist(f, (GCObject*)(g->strt.hash[i]));
}
fprintf(f, "\"0\":{\"type\":\"userdata\",\"cat\":0,\"size\":0}\n"); // to avoid issues with trailing ,
fprintf(f, "},\"roots\":{\n");

347
VM/src/lmem.cpp
View File

@ -8,6 +8,8 @@
#include <string.h>
LUAU_FASTFLAG(LuauGcPagedSweep)
#ifndef __has_feature
#define __has_feature(x) 0
#endif
@ -42,13 +44,21 @@ static_assert(sizeof(LuaNode) == ABISWITCH(32, 32, 32), "size mismatch for table
#endif
static_assert(offsetof(TString, data) == ABISWITCH(24, 20, 20), "size mismatch for string header");
// TODO (FFlagLuauGcPagedSweep): this will become ABISWITCH(16, 16, 16)
static_assert(offsetof(Udata, data) == ABISWITCH(24, 16, 16), "size mismatch for userdata header");
// TODO (FFlagLuauGcPagedSweep): this will become ABISWITCH(48, 32, 32)
static_assert(sizeof(Table) == ABISWITCH(56, 36, 36), "size mismatch for table header");
// TODO (FFlagLuauGcPagedSweep): new code with old 'next' pointer requires that GCObject start at the same point as TString/UpVal
static_assert(offsetof(GCObject, uv) == 0, "UpVal data must be located at the start of the GCObject");
static_assert(offsetof(GCObject, ts) == 0, "TString data must be located at the start of the GCObject");
const size_t kSizeClasses = LUA_SIZECLASSES;
const size_t kMaxSmallSize = 512;
const size_t kPageSize = 16 * 1024 - 24; // slightly under 16KB since that results in less fragmentation due to heap metadata
const size_t kBlockHeader = sizeof(double) > sizeof(void*) ? sizeof(double) : sizeof(void*); // suitable for aligning double & void* on all platforms
// TODO (FFlagLuauGcPagedSweep): when 'next' is removed, 'kBlockHeader' can be used unconditionally
const size_t kGCOHeader = sizeof(GCheader) > kBlockHeader ? sizeof(GCheader) : kBlockHeader;
struct SizeClassConfig
{
@ -96,6 +106,7 @@ const SizeClassConfig kSizeClassConfig;
// metadata for a block is stored in the first pointer of the block
#define metadata(block) (*(void**)(block))
#define freegcolink(block) (*(void**)((char*)block + kGCOHeader))
/*
** About the realloc function:
@ -117,15 +128,22 @@ const SizeClassConfig kSizeClassConfig;
struct lua_Page
{
// list of pages with free blocks
lua_Page* prev;
lua_Page* next;
// list of all gco pages
lua_Page* gcolistprev;
lua_Page* gcolistnext;
int busyBlocks;
int blockSize;
void* freeList;
int freeNext;
int pageSize;
union
{
char data[1];
@ -141,6 +159,8 @@ l_noret luaM_toobig(lua_State* L)
static lua_Page* luaM_newpage(lua_State* L, uint8_t sizeClass)
{
LUAU_ASSERT(!FFlag::LuauGcPagedSweep);
global_State* g = L->global;
lua_Page* page = (lua_Page*)(*g->frealloc)(L, g->ud, NULL, 0, kPageSize);
if (!page)
@ -155,6 +175,9 @@ static lua_Page* luaM_newpage(lua_State* L, uint8_t sizeClass)
page->prev = NULL;
page->next = NULL;
page->gcolistprev = NULL;
page->gcolistnext = NULL;
page->busyBlocks = 0;
page->blockSize = blockSize;
@ -171,8 +194,69 @@ static lua_Page* luaM_newpage(lua_State* L, uint8_t sizeClass)
return page;
}
static lua_Page* newpage(lua_State* L, lua_Page** gcopageset, int pageSize, int blockSize, int blockCount)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
global_State* g = L->global;
LUAU_ASSERT(pageSize - int(offsetof(lua_Page, data)) >= blockSize * blockCount);
lua_Page* page = (lua_Page*)(*g->frealloc)(L, g->ud, NULL, 0, pageSize);
if (!page)
luaD_throw(L, LUA_ERRMEM);
ASAN_POISON_MEMORY_REGION(page->data, blockSize * blockCount);
// setup page header
page->prev = NULL;
page->next = NULL;
page->gcolistprev = NULL;
page->gcolistnext = NULL;
page->busyBlocks = 0;
page->blockSize = blockSize;
// note: we start with the last block in the page and move downward
// either order would work, but that way we don't need to store the block count in the page
// additionally, GC stores objects in singly linked lists, and this way the GC lists end up in increasing pointer order
page->freeList = NULL;
page->freeNext = (blockCount - 1) * blockSize;
page->pageSize = pageSize;
if (gcopageset)
{
page->gcolistnext = *gcopageset;
if (page->gcolistnext)
page->gcolistnext->gcolistprev = page;
*gcopageset = page;
}
return page;
}
static lua_Page* newclasspage(lua_State* L, lua_Page** freepageset, lua_Page** gcopageset, uint8_t sizeClass, bool storeMetadata)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
int blockSize = kSizeClassConfig.sizeOfClass[sizeClass] + (storeMetadata ? kBlockHeader : 0);
int blockCount = (kPageSize - offsetof(lua_Page, data)) / blockSize;
lua_Page* page = newpage(L, gcopageset, kPageSize, blockSize, blockCount);
// prepend a page to page freelist (which is empty because we only ever allocate a new page when it is!)
LUAU_ASSERT(!freepageset[sizeClass]);
freepageset[sizeClass] = page;
return page;
}
static void luaM_freepage(lua_State* L, lua_Page* page, uint8_t sizeClass)
{
LUAU_ASSERT(!FFlag::LuauGcPagedSweep);
global_State* g = L->global;
// remove page from freelist
@ -188,6 +272,44 @@ static void luaM_freepage(lua_State* L, lua_Page* page, uint8_t sizeClass)
(*g->frealloc)(L, g->ud, page, kPageSize, 0);
}
static void freepage(lua_State* L, lua_Page** gcopageset, lua_Page* page)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
global_State* g = L->global;
if (gcopageset)
{
// remove page from alllist
if (page->gcolistnext)
page->gcolistnext->gcolistprev = page->gcolistprev;
if (page->gcolistprev)
page->gcolistprev->gcolistnext = page->gcolistnext;
else if (*gcopageset == page)
*gcopageset = page->gcolistnext;
}
// so long
(*g->frealloc)(L, g->ud, page, page->pageSize, 0);
}
static void freeclasspage(lua_State* L, lua_Page** freepageset, lua_Page** gcopageset, lua_Page* page, uint8_t sizeClass)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
// remove page from freelist
if (page->next)
page->next->prev = page->prev;
if (page->prev)
page->prev->next = page->next;
else if (freepageset[sizeClass] == page)
freepageset[sizeClass] = page->next;
freepage(L, gcopageset, page);
}
static void* luaM_newblock(lua_State* L, int sizeClass)
{
global_State* g = L->global;
@ -195,7 +317,12 @@ static void* luaM_newblock(lua_State* L, int sizeClass)
// slow path: no page in the freelist, allocate a new one
if (!page)
page = luaM_newpage(L, sizeClass);
{
if (FFlag::LuauGcPagedSweep)
page = newclasspage(L, g->freepages, NULL, sizeClass, true);
else
page = luaM_newpage(L, sizeClass);
}
LUAU_ASSERT(!page->prev);
LUAU_ASSERT(page->freeList || page->freeNext >= 0);
@ -236,6 +363,55 @@ static void* luaM_newblock(lua_State* L, int sizeClass)
return (char*)block + kBlockHeader;
}
static void* luaM_newgcoblock(lua_State* L, int sizeClass)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
global_State* g = L->global;
lua_Page* page = g->freegcopages[sizeClass];
// slow path: no page in the freelist, allocate a new one
if (!page)
page = newclasspage(L, g->freegcopages, &g->allgcopages, sizeClass, false);
LUAU_ASSERT(!page->prev);
LUAU_ASSERT(page->freeList || page->freeNext >= 0);
LUAU_ASSERT(page->blockSize == kSizeClassConfig.sizeOfClass[sizeClass]);
void* block;
if (page->freeNext >= 0)
{
block = &page->data + page->freeNext;
ASAN_UNPOISON_MEMORY_REGION(block, page->blockSize);
page->freeNext -= page->blockSize;
page->busyBlocks++;
}
else
{
// when separate block metadata is not used, free list link is stored inside the block data itself
block = (char*)page->freeList - kGCOHeader;
ASAN_UNPOISON_MEMORY_REGION((char*)block + kGCOHeader, page->blockSize - kGCOHeader);
page->freeList = freegcolink(block);
page->busyBlocks++;
}
// if we allocate the last block out of a page, we need to remove it from free list
if (!page->freeList && page->freeNext < 0)
{
g->freegcopages[sizeClass] = page->next;
if (page->next)
page->next->prev = NULL;
page->next = NULL;
}
// the user data is right after the metadata
return (char*)block;
}
static void luaM_freeblock(lua_State* L, int sizeClass, void* block)
{
global_State* g = L->global;
@ -270,12 +446,45 @@ static void luaM_freeblock(lua_State* L, int sizeClass, void* block)
// if it's the last block in the page, we don't need the page
if (page->busyBlocks == 0)
luaM_freepage(L, page, sizeClass);
{
if (FFlag::LuauGcPagedSweep)
freeclasspage(L, g->freepages, NULL, page, sizeClass);
else
luaM_freepage(L, page, sizeClass);
}
}
static void luaM_freegcoblock(lua_State* L, int sizeClass, void* block, lua_Page* page)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
global_State* g = L->global;
// if the page wasn't in the page free list, it should be now since it got a block!
if (!page->freeList && page->freeNext < 0)
{
LUAU_ASSERT(!page->prev);
LUAU_ASSERT(!page->next);
page->next = g->freegcopages[sizeClass];
if (page->next)
page->next->prev = page;
g->freegcopages[sizeClass] = page;
}
// when separate block metadata is not used, free list link is stored inside the block data itself
freegcolink(block) = page->freeList;
page->freeList = (char*)block + kGCOHeader;
ASAN_POISON_MEMORY_REGION((char*)block + kGCOHeader, page->blockSize - kGCOHeader);
page->busyBlocks--;
// if it's the last block in the page, we don't need the page
if (page->busyBlocks == 0)
freeclasspage(L, g->freegcopages, &g->allgcopages, page, sizeClass);
}
/*
** generic allocation routines.
*/
void* luaM_new_(lua_State* L, size_t nsize, uint8_t memcat)
{
global_State* g = L->global;
@ -292,6 +501,43 @@ void* luaM_new_(lua_State* L, size_t nsize, uint8_t memcat)
return block;
}
GCObject* luaM_newgco_(lua_State* L, size_t nsize, uint8_t memcat)
{
if (!FFlag::LuauGcPagedSweep)
return (GCObject*)luaM_new_(L, nsize, memcat);
global_State* g = L->global;
int nclass = sizeclass(nsize);
void* block = NULL;
if (nclass >= 0)
{
LUAU_ASSERT(nsize > 8);
block = luaM_newgcoblock(L, nclass);
}
else
{
lua_Page* page = newpage(L, &g->allgcopages, offsetof(lua_Page, data) + int(nsize), int(nsize), 1);
block = &page->data;
ASAN_UNPOISON_MEMORY_REGION(block, page->blockSize);
page->freeNext -= page->blockSize;
page->busyBlocks++;
}
if (block == NULL && nsize > 0)
luaD_throw(L, LUA_ERRMEM);
g->totalbytes += nsize;
g->memcatbytes[memcat] += nsize;
return (GCObject*)block;
}
void luaM_free_(lua_State* L, void* block, size_t osize, uint8_t memcat)
{
global_State* g = L->global;
@ -308,6 +554,36 @@ void luaM_free_(lua_State* L, void* block, size_t osize, uint8_t memcat)
g->memcatbytes[memcat] -= osize;
}
void luaM_freegco_(lua_State* L, GCObject* block, size_t osize, uint8_t memcat, lua_Page* page)
{
if (!FFlag::LuauGcPagedSweep)
{
luaM_free_(L, block, osize, memcat);
return;
}
global_State* g = L->global;
LUAU_ASSERT((osize == 0) == (block == NULL));
int oclass = sizeclass(osize);
if (oclass >= 0)
{
block->gch.tt = LUA_TNIL;
luaM_freegcoblock(L, oclass, block, page);
}
else
{
LUAU_ASSERT(page->busyBlocks == 1);
freepage(L, &g->allgcopages, page);
}
g->totalbytes -= osize;
g->memcatbytes[memcat] -= osize;
}
void* luaM_realloc_(lua_State* L, void* block, size_t osize, size_t nsize, uint8_t memcat)
{
global_State* g = L->global;
@ -344,3 +620,64 @@ void* luaM_realloc_(lua_State* L, void* block, size_t osize, size_t nsize, uint8
g->memcatbytes[memcat] += nsize - osize;
return result;
}
void luaM_getpagewalkinfo(lua_Page* page, char** start, char** end, int* busyBlocks, int* blockSize)
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
int blockCount = (page->pageSize - offsetof(lua_Page, data)) / page->blockSize;
*start = page->data + page->freeNext + page->blockSize;
*end = page->data + blockCount * page->blockSize;
*busyBlocks = page->busyBlocks;
*blockSize = page->blockSize;
}
lua_Page* luaM_getnextgcopage(lua_Page* page)
{
return page->gcolistnext;
}
void luaM_visitpage(lua_Page* page, void* context, bool (*visitor)(void* context, lua_Page* page, GCObject* gco))
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
char* start;
char* end;
int busyBlocks;
int blockSize;
luaM_getpagewalkinfo(page, &start, &end, &busyBlocks, &blockSize);
for (char* pos = start; pos != end; pos += blockSize)
{
GCObject* gco = (GCObject*)pos;
// skip memory blocks that are already freed
if (gco->gch.tt == LUA_TNIL)
continue;
// when true is returned it means that the element was deleted
if (visitor(context, page, gco))
{
// if the last block was removed, page would be removed as well
if (--busyBlocks == 0)
break;
}
}
}
void luaM_visitgco(lua_State* L, void* context, bool (*visitor)(void* context, lua_Page* page, GCObject* gco))
{
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
global_State* g = L->global;
for (lua_Page* curr = g->allgcopages; curr;)
{
lua_Page* next = curr->gcolistnext; // page blockvisit might destroy the page
luaM_visitpage(curr, context, visitor);
curr = next;
}
}

15
VM/src/lmem.h
View File

@ -4,8 +4,15 @@
#include "lua.h"
struct lua_Page;
union GCObject;
// TODO: remove with FFlagLuauGcPagedSweep and rename luaM_newgco to luaM_new
#define luaM_new(L, t, size, memcat) cast_to(t*, luaM_new_(L, size, memcat))
#define luaM_newgco(L, t, size, memcat) cast_to(t*, luaM_newgco_(L, size, memcat))
// TODO: remove with FFlagLuauGcPagedSweep and rename luaM_freegco to luaM_free
#define luaM_free(L, p, size, memcat) luaM_free_(L, (p), size, memcat)
#define luaM_freegco(L, p, size, memcat, page) luaM_freegco_(L, obj2gco(p), size, memcat, page)
#define luaM_arraysize_(n, e) ((cast_to(size_t, (n)) <= SIZE_MAX / (e)) ? (n) * (e) : (luaM_toobig(L), SIZE_MAX))
@ -15,7 +22,15 @@
((v) = cast_to(t*, luaM_realloc_(L, v, (oldn) * sizeof(t), luaM_arraysize_(n, sizeof(t)), memcat)))
LUAI_FUNC void* luaM_new_(lua_State* L, size_t nsize, uint8_t memcat);
LUAI_FUNC GCObject* luaM_newgco_(lua_State* L, size_t nsize, uint8_t memcat);
LUAI_FUNC void luaM_free_(lua_State* L, void* block, size_t osize, uint8_t memcat);
LUAI_FUNC void luaM_freegco_(lua_State* L, GCObject* block, size_t osize, uint8_t memcat, lua_Page* page);
LUAI_FUNC void* luaM_realloc_(lua_State* L, void* block, size_t osize, size_t nsize, uint8_t memcat);
LUAI_FUNC l_noret luaM_toobig(lua_State* L);
LUAI_FUNC void luaM_getpagewalkinfo(lua_Page* page, char** start, char** end, int* busyBlocks, int* blockSize);
LUAI_FUNC lua_Page* luaM_getnextgcopage(lua_Page* page);
LUAI_FUNC void luaM_visitpage(lua_Page* page, void* context, bool (*visitor)(void* context, lua_Page* page, GCObject* gco));
LUAI_FUNC void luaM_visitgco(lua_State* L, void* context, bool (*visitor)(void* context, lua_Page* page, GCObject* gco));

16
VM/src/lobject.h
View File

@ -11,12 +11,11 @@
typedef union GCObject GCObject;
/*
** Common Header for all collectible objects (in macro form, to be
** included in other objects)
** Common Header for all collectible objects (in macro form, to be included in other objects)
*/
// clang-format off
#define CommonHeader \
GCObject* next; \
GCObject* next; /* TODO: remove with FFlagLuauGcPagedSweep */ \
uint8_t tt; uint8_t marked; uint8_t memcat
// clang-format on
@ -229,8 +228,10 @@ typedef TValue* StkId; /* index to stack elements */
typedef struct TString
{
CommonHeader;
// 1 byte padding
int16_t atom;
// 2 byte padding
unsigned int hash;
unsigned int len;
@ -314,14 +315,21 @@ typedef struct LocVar
typedef struct UpVal
{
CommonHeader;
// 1 (x86) or 5 (x64) byte padding
TValue* v; /* points to stack or to its own value */
union
{
TValue value; /* the value (when closed) */
struct
{ /* double linked list (when open) */
{
/* global double linked list (when open) */
struct UpVal* prev;
struct UpVal* next;
/* thread double linked list (when open) */
// TODO: when FFlagLuauGcPagedSweep is removed, old outer 'next' value will be placed here
/* note: this is the location of a pointer to this upvalue in the previous element that can be either an UpVal or a lua_State */
struct UpVal** threadprev;
} l;
} u;
} UpVal;

37
VM/src/lstate.cpp
View File

@ -10,6 +10,8 @@
#include "ldo.h"
#include "ldebug.h"
LUAU_FASTFLAG(LuauGcPagedSweep)
/*
** Main thread combines a thread state and the global state
*/
@ -86,14 +88,21 @@ static void close_state(lua_State* L)
global_State* g = L->global;
luaF_close(L, L->stack); /* close all upvalues for this thread */
luaC_freeall(L); /* collect all objects */
LUAU_ASSERT(g->rootgc == obj2gco(L));
if (!FFlag::LuauGcPagedSweep)
LUAU_ASSERT(g->rootgc == obj2gco(L));
LUAU_ASSERT(g->strbufgc == NULL);
LUAU_ASSERT(g->strt.nuse == 0);
luaM_freearray(L, L->global->strt.hash, L->global->strt.size, TString*, 0);
freestack(L, L);
LUAU_ASSERT(g->totalbytes == sizeof(LG));
for (int i = 0; i < LUA_SIZECLASSES; i++)
{
LUAU_ASSERT(g->freepages[i] == NULL);
if (FFlag::LuauGcPagedSweep)
LUAU_ASSERT(g->freegcopages[i] == NULL);
}
if (FFlag::LuauGcPagedSweep)
LUAU_ASSERT(g->allgcopages == NULL);
LUAU_ASSERT(g->totalbytes == sizeof(LG));
LUAU_ASSERT(g->memcatbytes[0] == sizeof(LG));
for (int i = 1; i < LUA_MEMORY_CATEGORIES; i++)
LUAU_ASSERT(g->memcatbytes[i] == 0);
@ -102,7 +111,7 @@ static void close_state(lua_State* L)
lua_State* luaE_newthread(lua_State* L)
{
lua_State* L1 = luaM_new(L, lua_State, sizeof(lua_State), L->activememcat);
lua_State* L1 = luaM_newgco(L, lua_State, sizeof(lua_State), L->activememcat);
luaC_link(L, L1, LUA_TTHREAD);
preinit_state(L1, L->global);
L1->activememcat = L->activememcat; // inherit the active memory category
@ -113,7 +122,7 @@ lua_State* luaE_newthread(lua_State* L)
return L1;
}
void luaE_freethread(lua_State* L, lua_State* L1)
void luaE_freethread(lua_State* L, lua_State* L1, lua_Page* page)
{
luaF_close(L1, L1->stack); /* close all upvalues for this thread */
LUAU_ASSERT(L1->openupval == NULL);
@ -121,7 +130,7 @@ void luaE_freethread(lua_State* L, lua_State* L1)
if (g->cb.userthread)
g->cb.userthread(NULL, L1);
freestack(L, L1);
luaM_free(L, L1, sizeof(lua_State), L1->memcat);
luaM_freegco(L, L1, sizeof(lua_State), L1->memcat, page);
}
void lua_resetthread(lua_State* L)
@ -162,7 +171,8 @@ lua_State* lua_newstate(lua_Alloc f, void* ud)
return NULL;
L = (lua_State*)l;
g = &((LG*)L)->g;
L->next = NULL;
if (!FFlag::LuauGcPagedSweep)
L->next = NULL;
L->tt = LUA_TTHREAD;
L->marked = g->currentwhite = bit2mask(WHITE0BIT, FIXEDBIT);
L->memcat = 0;
@ -185,9 +195,11 @@ lua_State* lua_newstate(lua_Alloc f, void* ud)
g->strt.hash = NULL;
setnilvalue(registry(L));
g->gcstate = GCSpause;
g->rootgc = obj2gco(L);
if (!FFlag::LuauGcPagedSweep)
g->rootgc = obj2gco(L);
g->sweepstrgc = 0;
g->sweepgc = &g->rootgc;
if (!FFlag::LuauGcPagedSweep)
g->sweepgc = &g->rootgc;
g->gray = NULL;
g->grayagain = NULL;
g->weak = NULL;
@ -197,7 +209,16 @@ lua_State* lua_newstate(lua_Alloc f, void* ud)
g->gcstepmul = LUAI_GCSTEPMUL;
g->gcstepsize = LUAI_GCSTEPSIZE << 10;
for (i = 0; i < LUA_SIZECLASSES; i++)
{
g->freepages[i] = NULL;
if (FFlag::LuauGcPagedSweep)
g->freegcopages[i] = NULL;
}
if (FFlag::LuauGcPagedSweep)
{
g->allgcopages = NULL;
g->sweepgcopage = NULL;
}
for (i = 0; i < LUA_T_COUNT; i++)
g->mt[i] = NULL;
for (i = 0; i < LUA_UTAG_LIMIT; i++)

15
VM/src/lstate.h
View File

@ -22,7 +22,7 @@
typedef struct stringtable
{
GCObject** hash;
TString** hash;
uint32_t nuse; /* number of elements */
int size;
} stringtable;
@ -149,13 +149,15 @@ typedef struct global_State
int sweepstrgc; /* position of sweep in `strt' */
// TODO: remove with FFlagLuauGcPagedSweep
GCObject* rootgc; /* list of all collectable objects */
// TODO: remove with FFlagLuauGcPagedSweep
GCObject** sweepgc; /* position of sweep in `rootgc' */
GCObject* gray; /* list of gray objects */
GCObject* grayagain; /* list of objects to be traversed atomically */
GCObject* weak; /* list of weak tables (to be cleared) */
GCObject* strbufgc; // list of all string buffer objects
TString* strbufgc; // list of all string buffer objects
size_t GCthreshold; // when totalbytes > GCthreshold; run GC step
@ -164,7 +166,10 @@ typedef struct global_State
int gcstepmul; // see LUAI_GCSTEPMUL
int gcstepsize; // see LUAI_GCSTEPSIZE
struct lua_Page* freepages[LUA_SIZECLASSES]; /* free page linked list for each size class */
struct lua_Page* freepages[LUA_SIZECLASSES]; // free page linked list for each size class for non-collectable objects
struct lua_Page* freegcopages[LUA_SIZECLASSES]; // free page linked list for each size class for collectable objects
struct lua_Page* allgcopages; // page linked list with all pages for all classes
struct lua_Page* sweepgcopage; // position of the sweep in `allgcopages'
size_t memcatbytes[LUA_MEMORY_CATEGORIES]; /* total amount of memory used by each memory category */
@ -231,7 +236,7 @@ struct lua_State
TValue l_gt; /* table of globals */
TValue env; /* temporary place for environments */
GCObject* openupval; /* list of open upvalues in this stack */
UpVal* openupval; /* list of open upvalues in this stack */
GCObject* gclist;
TString* namecall; /* when invoked from Luau using NAMECALL, what method do we need to invoke? */
@ -268,4 +273,4 @@ union GCObject
#define obj2gco(v) check_exp(iscollectable(v), cast_to(GCObject*, (v) + 0))
LUAI_FUNC lua_State* luaE_newthread(lua_State* L);
LUAI_FUNC void luaE_freethread(lua_State* L, lua_State* L1);
LUAI_FUNC void luaE_freethread(lua_State* L, lua_State* L1, struct lua_Page* page);

136
VM/src/lstring.cpp
View File

@ -7,6 +7,8 @@
#include <string.h>
LUAU_FASTFLAG(LuauGcPagedSweep)
unsigned int luaS_hash(const char* str, size_t len)
{
// Note that this hashing algorithm is replicated in BytecodeBuilder.cpp, BytecodeBuilder::getStringHash
@ -44,26 +46,25 @@ unsigned int luaS_hash(const char* str, size_t len)
void luaS_resize(lua_State* L, int newsize)
{
GCObject** newhash;
stringtable* tb;
int i;
if (L->global->gcstate == GCSsweepstring)
return; /* cannot resize during GC traverse */
newhash = luaM_newarray(L, newsize, GCObject*, 0);
tb = &L->global->strt;
for (i = 0; i < newsize; i++)
TString** newhash = luaM_newarray(L, newsize, TString*, 0);
stringtable* tb = &L->global->strt;
for (int i = 0; i < newsize; i++)
newhash[i] = NULL;
/* rehash */
for (i = 0; i < tb->size; i++)
for (int i = 0; i < tb->size; i++)
{
GCObject* p = tb->hash[i];
TString* p = tb->hash[i];
while (p)
{ /* for each node in the list */
GCObject* next = p->gch.next; /* save next */
unsigned int h = gco2ts(p)->hash;
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
TString* next = (TString*)p->next; /* save next */
unsigned int h = p->hash;
int h1 = lmod(h, newsize); /* new position */
LUAU_ASSERT(cast_int(h % newsize) == lmod(h, newsize));
p->gch.next = newhash[h1]; /* chain it */
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
p->next = (GCObject*)newhash[h1]; /* chain it */
newhash[h1] = p;
p = next;
}
@ -79,7 +80,7 @@ static TString* newlstr(lua_State* L, const char* str, size_t l, unsigned int h)
stringtable* tb;
if (l > MAXSSIZE)
luaM_toobig(L);
ts = luaM_new(L, TString, sizestring(l), L->activememcat);
ts = luaM_newgco(L, TString, sizestring(l), L->activememcat);
ts->len = unsigned(l);
ts->hash = h;
ts->marked = luaC_white(L->global);
@ -90,8 +91,9 @@ static TString* newlstr(lua_State* L, const char* str, size_t l, unsigned int h)
ts->atom = L->global->cb.useratom ? L->global->cb.useratom(ts->data, l) : -1;
tb = &L->global->strt;
h = lmod(h, tb->size);
ts->next = tb->hash[h]; /* chain new entry */
tb->hash[h] = obj2gco(ts);
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the case will not be required
ts->next = (GCObject*)tb->hash[h]; /* chain new entry */
tb->hash[h] = ts;
tb->nuse++;
if (tb->nuse > cast_to(uint32_t, tb->size) && tb->size <= INT_MAX / 2)
luaS_resize(L, tb->size * 2); /* too crowded */
@ -101,28 +103,41 @@ static TString* newlstr(lua_State* L, const char* str, size_t l, unsigned int h)
static void linkstrbuf(lua_State* L, TString* ts)
{
global_State* g = L->global;
GCObject* o = obj2gco(ts);
o->gch.next = g->strbufgc;
g->strbufgc = o;
o->gch.marked = luaC_white(g);
if (FFlag::LuauGcPagedSweep)
{
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
ts->next = (GCObject*)g->strbufgc;
g->strbufgc = ts;
ts->marked = luaC_white(g);
}
else
{
GCObject* o = obj2gco(ts);
o->gch.next = (GCObject*)g->strbufgc;
g->strbufgc = gco2ts(o);
o->gch.marked = luaC_white(g);
}
}
static void unlinkstrbuf(lua_State* L, TString* ts)
{
global_State* g = L->global;
GCObject** p = &g->strbufgc;
TString** p = &g->strbufgc;
while (GCObject* curr = *p)
while (TString* curr = *p)
{
if (curr == obj2gco(ts))
if (curr == ts)
{
*p = curr->gch.next;
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
*p = (TString*)curr->next;
return;
}
else
{
p = &curr->gch.next;
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
p = (TString**)&curr->next;
}
}
@ -134,7 +149,7 @@ TString* luaS_bufstart(lua_State* L, size_t size)
if (size > MAXSSIZE)
luaM_toobig(L);
TString* ts = luaM_new(L, TString, sizestring(size), L->activememcat);
TString* ts = luaM_newgco(L, TString, sizestring(size), L->activememcat);
ts->tt = LUA_TSTRING;
ts->memcat = L->activememcat;
@ -152,15 +167,14 @@ TString* luaS_buffinish(lua_State* L, TString* ts)
int bucket = lmod(h, tb->size);
// search if we already have this string in the hash table
for (GCObject* o = tb->hash[bucket]; o != NULL; o = o->gch.next)
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
for (TString* el = tb->hash[bucket]; el != NULL; el = (TString*)el->next)
{
TString* el = gco2ts(o);
if (el->len == ts->len && memcmp(el->data, ts->data, ts->len) == 0)
{
// string may be dead
if (isdead(L->global, o))
changewhite(o);
if (isdead(L->global, obj2gco(el)))
changewhite(obj2gco(el));
return el;
}
@ -173,8 +187,9 @@ TString* luaS_buffinish(lua_State* L, TString* ts)
// Complete string object
ts->atom = L->global->cb.useratom ? L->global->cb.useratom(ts->data, ts->len) : -1;
ts->next = tb->hash[bucket]; // chain new entry
tb->hash[bucket] = obj2gco(ts);
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
ts->next = (GCObject*)tb->hash[bucket]; // chain new entry
tb->hash[bucket] = ts;
tb->nuse++;
if (tb->nuse > cast_to(uint32_t, tb->size) && tb->size <= INT_MAX / 2)
@ -185,24 +200,63 @@ TString* luaS_buffinish(lua_State* L, TString* ts)
TString* luaS_newlstr(lua_State* L, const char* str, size_t l)
{
GCObject* o;
unsigned int h = luaS_hash(str, l);
for (o = L->global->strt.hash[lmod(h, L->global->strt.size)]; o != NULL; o = o->gch.next)
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
for (TString* el = L->global->strt.hash[lmod(h, L->global->strt.size)]; el != NULL; el = (TString*)el->next)
{
TString* ts = gco2ts(o);
if (ts->len == l && (memcmp(str, getstr(ts), l) == 0))
if (el->len == l && (memcmp(str, getstr(el), l) == 0))
{
/* string may be dead */
if (isdead(L->global, o))
changewhite(o);
return ts;
if (isdead(L->global, obj2gco(el)))
changewhite(obj2gco(el));
return el;
}
}
return newlstr(L, str, l, h); /* not found */
}
void luaS_free(lua_State* L, TString* ts)
static bool unlinkstr(lua_State* L, TString* ts)
{
L->global->strt.nuse--;
luaM_free(L, ts, sizestring(ts->len), ts->memcat);
LUAU_ASSERT(FFlag::LuauGcPagedSweep);
global_State* g = L->global;
TString** p = &g->strt.hash[lmod(ts->hash, g->strt.size)];
while (TString* curr = *p)
{
if (curr == ts)
{
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
*p = (TString*)curr->next;
return true;
}
else
{
// TODO (FFlagLuauGcPagedSweep): 'next' type will change after removal of the flag and the cast will not be required
p = (TString**)&curr->next;
}
}
return false;
}
void luaS_free(lua_State* L, TString* ts, lua_Page* page)
{
if (FFlag::LuauGcPagedSweep)
{
// Unchain from the string table
if (!unlinkstr(L, ts))
unlinkstrbuf(L, ts); // An unlikely scenario when we have a string buffer on our hands
else
L->global->strt.nuse--;
luaM_freegco(L, ts, sizestring(ts->len), ts->memcat, page);
}
else
{
L->global->strt.nuse--;
luaM_free(L, ts, sizestring(ts->len), ts->memcat);
}
}

2
VM/src/lstring.h
View File

@ -20,7 +20,7 @@ LUAI_FUNC unsigned int luaS_hash(const char* str, size_t len);
LUAI_FUNC void luaS_resize(lua_State* L, int newsize);
LUAI_FUNC TString* luaS_newlstr(lua_State* L, const char* str, size_t l);
LUAI_FUNC void luaS_free(lua_State* L, TString* ts);
LUAI_FUNC void luaS_free(lua_State* L, TString* ts, struct lua_Page* page);
LUAI_FUNC TString* luaS_bufstart(lua_State* L, size_t size);
LUAI_FUNC TString* luaS_buffinish(lua_State* L, TString* ts);

8
VM/src/ltable.cpp
View File

@ -424,7 +424,7 @@ static void rehash(lua_State* L, Table* t, const TValue* ek)
Table* luaH_new(lua_State* L, int narray, int nhash)
{
Table* t = luaM_new(L, Table, sizeof(Table), L->activememcat);
Table* t = luaM_newgco(L, Table, sizeof(Table), L->activememcat);
luaC_link(L, t, LUA_TTABLE);
t->metatable = NULL;
t->flags = cast_byte(~0);
@ -443,12 +443,12 @@ Table* luaH_new(lua_State* L, int narray, int nhash)
return t;
}
void luaH_free(lua_State* L, Table* t)
void luaH_free(lua_State* L, Table* t, lua_Page* page)
{
if (t->node != dummynode)
luaM_freearray(L, t->node, sizenode(t), LuaNode, t->memcat);
luaM_freearray(L, t->array, t->sizearray, TValue, t->memcat);
luaM_free(L, t, sizeof(Table), t->memcat);
luaM_freegco(L, t, sizeof(Table), t->memcat, page);
}
static LuaNode* getfreepos(Table* t)
@ -741,7 +741,7 @@ int luaH_getn(Table* t)
Table* luaH_clone(lua_State* L, Table* tt)
{
Table* t = luaM_new(L, Table, sizeof(Table), L->activememcat);
Table* t = luaM_newgco(L, Table, sizeof(Table), L->activememcat);
luaC_link(L, t, LUA_TTABLE);
t->metatable = tt->metatable;
t->flags = tt->flags;

2
VM/src/ltable.h
View File

@ -20,7 +20,7 @@ LUAI_FUNC TValue* luaH_set(lua_State* L, Table* t, const TValue* key);
LUAI_FUNC Table* luaH_new(lua_State* L, int narray, int lnhash);
LUAI_FUNC void luaH_resizearray(lua_State* L, Table* t, int nasize);
LUAI_FUNC void luaH_resizehash(lua_State* L, Table* t, int nhsize);
LUAI_FUNC void luaH_free(lua_State* L, Table* t);
LUAI_FUNC void luaH_free(lua_State* L, Table* t, struct lua_Page* page);
LUAI_FUNC int luaH_next(lua_State* L, Table* t, StkId key);
LUAI_FUNC int luaH_getn(Table* t);
LUAI_FUNC Table* luaH_clone(lua_State* L, Table* tt);

6
VM/src/ludata.cpp
View File

@ -11,7 +11,7 @@ Udata* luaU_newudata(lua_State* L, size_t s, int tag)
{
if (s > INT_MAX - sizeof(Udata))
luaM_toobig(L);
Udata* u = luaM_new(L, Udata, sizeudata(s), L->activememcat);
Udata* u = luaM_newgco(L, Udata, sizeudata(s), L->activememcat);
luaC_link(L, u, LUA_TUSERDATA);
u->len = int(s);
u->metatable = NULL;
@ -20,7 +20,7 @@ Udata* luaU_newudata(lua_State* L, size_t s, int tag)
return u;
}
void luaU_freeudata(lua_State* L, Udata* u)
void luaU_freeudata(lua_State* L, Udata* u, lua_Page* page)
{
LUAU_ASSERT(u->tag < LUA_UTAG_LIMIT || u->tag == UTAG_IDTOR);
@ -33,5 +33,5 @@ void luaU_freeudata(lua_State* L, Udata* u)
if (dtor)
dtor(u->data);
luaM_free(L, u, sizeudata(u->len), u->memcat);
luaM_freegco(L, u, sizeudata(u->len), u->memcat, page);
}

2
VM/src/ludata.h
View File

@ -10,4 +10,4 @@
#define sizeudata(len) (offsetof(Udata, data) + len)
LUAI_FUNC Udata* luaU_newudata(lua_State* L, size_t s, int tag);
LUAI_FUNC void luaU_freeudata(lua_State* L, Udata* u);
LUAI_FUNC void luaU_freeudata(lua_State* L, Udata* u, struct lua_Page* page);

2
VM/src/lvmexecute.cpp
View File

@ -496,7 +496,7 @@ static void luau_execute(lua_State* L)
Instruction insn = *pc++;
StkId ra = VM_REG(LUAU_INSN_A(insn));
if (L->openupval && gco2uv(L->openupval)->v >= ra)
if (L->openupval && L->openupval->v >= ra)
luaF_close(L, ra);
VM_NEXT();
}

24
tests/Compiler.test.cpp
View File

@ -756,6 +756,30 @@ RETURN R0 1
)");
}
TEST_CASE("TableSizePredictionLoop")
{
ScopedFastFlag sff("LuauPredictTableSizeLoop", true);
CHECK_EQ("\n" + compileFunction0(R"(
local t = {}
for i=1,4 do
t[i] = 0
end
return t
)"),
R"(
NEWTABLE R0 0 4
LOADN R3 1
LOADN R1 4
LOADN R2 1
FORNPREP R1 +3
LOADN R4 0
SETTABLE R4 R0 R3
FORNLOOP R1 -3
RETURN R0 1
)");
}
TEST_CASE("ReflectionEnums")
{
CHECK_EQ("\n" + compileFunction0("return Enum.EasingStyle.Linear"), R"(

9
tests/Linter.test.cpp
View File

@ -1396,6 +1396,8 @@ end
TEST_CASE_FIXTURE(Fixture, "TableOperations")
{
ScopedFastFlag sff("LuauLintTableCreateTable", true);
LintResult result = lintTyped(R"(
local t = {}
local tt = {}
@ -1416,9 +1418,12 @@ table.insert(t, string.find("hello", "h"))
table.move(t, 0, #t, 1, tt)
table.move(t, 1, #t, 0, tt)
table.create(42, {})
table.create(42, {} :: {})
)");
REQUIRE_EQ(result.warnings.size(), 8);
REQUIRE_EQ(result.warnings.size(), 10);
CHECK_EQ(result.warnings[0].text, "table.insert will insert the value before the last element, which is likely a bug; consider removing the "
"second argument or wrap it in parentheses to silence");
CHECK_EQ(result.warnings[1].text, "table.insert will append the value to the table; consider removing the second argument for efficiency");
@ -1430,6 +1435,8 @@ table.move(t, 1, #t, 0, tt)
"table.insert may change behavior if the call returns more than one result; consider adding parentheses around second argument");
CHECK_EQ(result.warnings[6].text, "table.move uses index 0 but arrays are 1-based; did you mean 1 instead?");
CHECK_EQ(result.warnings[7].text, "table.move uses index 0 but arrays are 1-based; did you mean 1 instead?");
CHECK_EQ(result.warnings[8].text, "table.create with a table literal will reuse the same object for all elements; consider using a for loop instead");
CHECK_EQ(result.warnings[9].text, "table.create with a table literal will reuse the same object for all elements; consider using a for loop instead");
}
TEST_CASE_FIXTURE(Fixture, "DuplicateConditions")

13
tests/Parser.test.cpp
View File

@ -1498,6 +1498,17 @@ return
CHECK_EQ(std::string(str->value.data, str->value.size), "\n");
}
TEST_CASE_FIXTURE(Fixture, "parse_error_broken_comment")
{
ScopedFastFlag luauStartingBrokenComment{"LuauStartingBrokenComment", true};
const char* expected = "Expected identifier when parsing expression, got unfinished comment";
matchParseError("--[[unfinished work", expected);
matchParseError("--!strict\n--[[unfinished work", expected);
matchParseError("local x = 1 --[[unfinished work", expected);
}
TEST_CASE_FIXTURE(Fixture, "string_literals_escapes_broken")
{
const char* expected = "String literal contains malformed escape sequence";
@ -2333,7 +2344,7 @@ TEST_CASE_FIXTURE(Fixture, "capture_broken_comment_at_the_start_of_the_file")
ParseOptions options;
options.captureComments = true;
ParseResult result = parseEx(R"(
ParseResult result = tryParse(R"(
--[[
)",
options);

48
tests/TypeInfer.tables.test.cpp
View File

@ -2180,4 +2180,52 @@ b()
CHECK_EQ(toString(result.errors[0]), R"(Cannot call non-function t1 where t1 = { @metatable { __call: t1 }, { } })");
}
TEST_CASE_FIXTURE(Fixture, "length_operator_union")
{
ScopedFastFlag luauLengthOnCompositeType{"LuauLengthOnCompositeType", true};
CheckResult result = check(R"(
local x: {number} | {string}
local y = #x
)");
LUAU_REQUIRE_NO_ERRORS(result);
}
TEST_CASE_FIXTURE(Fixture, "length_operator_intersection")
{
ScopedFastFlag luauLengthOnCompositeType{"LuauLengthOnCompositeType", true};
CheckResult result = check(R"(
local x: {number} & {z:string} -- mixed tables are evil
local y = #x
)");
LUAU_REQUIRE_NO_ERRORS(result);
}
TEST_CASE_FIXTURE(Fixture, "length_operator_non_table_union")
{
ScopedFastFlag luauLengthOnCompositeType{"LuauLengthOnCompositeType", true};
CheckResult result = check(R"(
local x: {number} | any | string
local y = #x
)");
LUAU_REQUIRE_NO_ERRORS(result);
}
TEST_CASE_FIXTURE(Fixture, "length_operator_union_errors")
{
ScopedFastFlag luauLengthOnCompositeType{"LuauLengthOnCompositeType", true};
CheckResult result = check(R"(
local x: {number} | number | string
local y = #x
)");
LUAU_REQUIRE_ERROR_COUNT(1, result);
}
TEST_SUITE_END();

29
tests/TypeInfer.test.cpp
View File

@ -5129,4 +5129,33 @@ local c = a or b
LUAU_REQUIRE_NO_ERRORS(result);
}
TEST_CASE_FIXTURE(Fixture, "bound_typepack_promote")
{
ScopedFastFlag luauCommittingTxnLogFreeTpPromote{"LuauCommittingTxnLogFreeTpPromote", true};
// No assertions should trigger
check(R"(
local function p()
local this = {}
this.pf = foo()
function this:IsActive() end
function this:Start(o) end
return this
end
local function h(tp, o)
ep = tp
tp:Start(o)
tp.pf.Connect(function()
ep:IsActive()
end)
end
function on()
local t = p()
h(t)
end
)");
}
TEST_SUITE_END();

15
tests/conformance/closure.lua
View File

@ -419,5 +419,20 @@ co = coroutine.create(function ()
return loadstring("return a")()
end)
-- large closure size
do
local a1, a2, a3, a4, a5, a6, a7, a8, a9, a0
local b1, b2, b3, b4, b5, b6, b7, b8, b9, b0
local c1, c2, c3, c4, c5, c6, c7, c8, c9, c0
local d1, d2, d3, d4, d5, d6, d7, d8, d9, d0
local f = function()
return
a1 + a2 + a3 + a4 + a5 + a6 + a7 + a8 + a9 + a0 +
b1 + b2 + b3 + b4 + b5 + b6 + b7 + b8 + b9 + b0 +
c1 + c2 + c3 + c4 + c5 + c6 + c7 + c8 + c9 + c0 +
d1 + d2 + d3 + d4 + d5 + d6 + d7 + d8 + d9 + d0
end
end
return 'OK'

28
tests/conformance/gc.lua
View File

@ -291,4 +291,32 @@ do
for i = 1,10 do table.insert(___Glob, newproxy(true)) end
end
-- create threads that die together with their unmarked upvalues
do
local t = {}
for i = 1,100 do
local c = coroutine.wrap(function()
local uv = {i + 1}
local function f()
return uv[1] * 10
end
coroutine.yield(uv[1])
uv = {i + 2}
coroutine.yield(f())
end)
assert(c() == i + 1)
table.insert(t, c)
end
for i = 1,100 do
t[i] = nil
end
collectgarbage()
end
return('OK')