String interpolation (#614)

Implements the string interpolation RFC (#165).

Adds the string interpolation as per the RFC.

```lua
local name = "world"
print(`Hello {name}!`) -- Hello world!
```

Co-authored-by: Arseny Kapoulkine <arseny.kapoulkine@gmail.com>
Co-authored-by: Alexander McCord <11488393+alexmccord@users.noreply.github.com>
This commit is contained in:
boyned//Kampfkarren 2022-08-24 12:01:00 -07:00 committed by GitHub
parent 0ce4c45436
commit da9d8e8c60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 915 additions and 36 deletions

View File

@ -107,6 +107,7 @@ struct TypeChecker
WithPredicate<TypeId> checkExpr(const ScopePtr& scope, const AstExprTypeAssertion& expr);
WithPredicate<TypeId> checkExpr(const ScopePtr& scope, const AstExprError& expr);
WithPredicate<TypeId> checkExpr(const ScopePtr& scope, const AstExprIfElse& expr, std::optional<TypeId> expectedType = std::nullopt);
WithPredicate<TypeId> checkExpr(const ScopePtr& scope, const AstExprInterpString& expr);
TypeId checkExprTable(const ScopePtr& scope, const AstExprTable& expr, const std::vector<std::pair<TypeId, TypeId>>& fieldTypes,
std::optional<TypeId> expectedType);

View File

@ -445,6 +445,14 @@ struct AstJsonEncoder : public AstVisitor
});
}
void write(class AstExprInterpString* node)
{
writeNode(node, "AstExprInterpString", [&]() {
PROP(strings);
PROP(expressions);
});
}
void write(class AstExprTable* node)
{
writeNode(node, "AstExprTable", [&]() {
@ -888,6 +896,12 @@ struct AstJsonEncoder : public AstVisitor
return false;
}
bool visit(class AstExprInterpString* node) override
{
write(node);
return false;
}
bool visit(class AstExprLocal* node) override
{
write(node);

View File

@ -206,6 +206,24 @@ static bool similar(AstExpr* lhs, AstExpr* rhs)
return true;
}
CASE(AstExprIfElse) return similar(le->condition, re->condition) && similar(le->trueExpr, re->trueExpr) && similar(le->falseExpr, re->falseExpr);
CASE(AstExprInterpString)
{
if (le->strings.size != re->strings.size)
return false;
if (le->expressions.size != re->expressions.size)
return false;
for (size_t i = 0; i < le->strings.size; ++i)
if (le->strings.data[i].size != re->strings.data[i].size || memcmp(le->strings.data[i].data, re->strings.data[i].data, le->strings.data[i].size) != 0)
return false;
for (size_t i = 0; i < le->expressions.size; ++i)
if (!similar(le->expressions.data[i], re->expressions.data[i]))
return false;
return true;
}
else
{
LUAU_ASSERT(!"Unknown expression type");

View File

@ -511,6 +511,28 @@ struct Printer
writer.keyword("else");
visualize(*a->falseExpr);
}
else if (const auto& a = expr.as<AstExprInterpString>())
{
writer.symbol("`");
size_t index = 0;
for (const auto& string : a->strings)
{
writer.write(escape(std::string_view(string.data, string.size), /* escapeForInterpString = */ true));
if (index < a->expressions.size)
{
writer.symbol("{");
visualize(*a->expressions.data[index]);
writer.symbol("}");
}
index++;
}
writer.symbol("`");
}
else if (const auto& a = expr.as<AstExprError>())
{
writer.symbol("(error-expr");

View File

@ -1805,6 +1805,8 @@ WithPredicate<TypeId> TypeChecker::checkExpr(const ScopePtr& scope, const AstExp
result = checkExpr(scope, *a);
else if (auto a = expr.as<AstExprIfElse>())
result = checkExpr(scope, *a, expectedType);
else if (auto a = expr.as<AstExprInterpString>())
result = checkExpr(scope, *a);
else
ice("Unhandled AstExpr?");
@ -2999,6 +3001,14 @@ WithPredicate<TypeId> TypeChecker::checkExpr(const ScopePtr& scope, const AstExp
return {types.size() == 1 ? types[0] : addType(UnionTypeVar{std::move(types)})};
}
WithPredicate<TypeId> TypeChecker::checkExpr(const ScopePtr& scope, const AstExprInterpString& expr)
{
for (AstExpr* expr : expr.expressions)
checkExpr(scope, *expr);
return {stringType};
}
TypeId TypeChecker::checkLValue(const ScopePtr& scope, const AstExpr& expr)
{
return checkLValueBinding(scope, expr);

View File

@ -134,6 +134,10 @@ public:
{
return visit((class AstExpr*)node);
}
virtual bool visit(class AstExprInterpString* node)
{
return visit((class AstExpr*)node);
}
virtual bool visit(class AstExprError* node)
{
return visit((class AstExpr*)node);
@ -732,6 +736,22 @@ public:
AstExpr* falseExpr;
};
class AstExprInterpString : public AstExpr
{
public:
LUAU_RTTI(AstExprInterpString)
AstExprInterpString(const Location& location, const AstArray<AstArray<char>>& strings, const AstArray<AstExpr*>& expressions);
void visit(AstVisitor* visitor) override;
/// An interpolated string such as `foo{bar}baz` is represented as
/// an array of strings for "foo" and "bar", and an array of expressions for "baz".
/// `strings` will always have one more element than `expressions`.
AstArray<AstArray<char>> strings;
AstArray<AstExpr*> expressions;
};
class AstStatBlock : public AstStat
{
public:

View File

@ -61,6 +61,12 @@ struct Lexeme
SkinnyArrow,
DoubleColon,
InterpStringBegin,
InterpStringMid,
InterpStringEnd,
// An interpolated string with no expressions (like `x`)
InterpStringSimple,
AddAssign,
SubAssign,
MulAssign,
@ -80,6 +86,8 @@ struct Lexeme
BrokenString,
BrokenComment,
BrokenUnicode,
BrokenInterpDoubleBrace,
Error,
Reserved_BEGIN,
@ -208,6 +216,11 @@ private:
Lexeme readLongString(const Position& start, int sep, Lexeme::Type ok, Lexeme::Type broken);
Lexeme readQuotedString();
Lexeme readInterpolatedStringBegin();
Lexeme readInterpolatedStringSection(Position start, Lexeme::Type formatType, Lexeme::Type endType);
void readBackslashInString();
std::pair<AstName, Lexeme::Type> readName();
Lexeme readNumber(const Position& start, unsigned int startOffset);
@ -231,6 +244,14 @@ private:
bool skipComments;
bool readNames;
enum class BraceType
{
InterpolatedString,
Normal
};
std::vector<BraceType> braceStack;
};
inline bool isSpace(char ch)

View File

@ -228,6 +228,9 @@ private:
// TODO: Add grammar rules here?
AstExpr* parseIfElseExpr();
// stringinterp ::= <INTERP_BEGIN> exp {<INTERP_MID> exp} <INTERP_END>
AstExpr* parseInterpString();
// Name
std::optional<Name> parseNameOpt(const char* context = nullptr);
Name parseName(const char* context = nullptr);
@ -379,6 +382,7 @@ private:
std::vector<unsigned int> matchRecoveryStopOnToken;
std::vector<AstStat*> scratchStat;
std::vector<AstArray<char>> scratchString;
std::vector<AstExpr*> scratchExpr;
std::vector<AstExpr*> scratchExprAux;
std::vector<AstName> scratchName;

View File

@ -35,6 +35,6 @@ bool equalsLower(std::string_view lhs, std::string_view rhs);
size_t hashRange(const char* data, size_t size);
std::string escape(std::string_view s);
std::string escape(std::string_view s, bool escapeForInterpString = false);
bool isIdentifier(std::string_view s);
} // namespace Luau

View File

@ -349,6 +349,22 @@ AstExprError::AstExprError(const Location& location, const AstArray<AstExpr*>& e
{
}
AstExprInterpString::AstExprInterpString(const Location& location, const AstArray<AstArray<char>>& strings, const AstArray<AstExpr*>& expressions)
: AstExpr(ClassIndex(), location)
, strings(strings)
, expressions(expressions)
{
}
void AstExprInterpString::visit(AstVisitor* visitor)
{
if (visitor->visit(this))
{
for (AstExpr* expr : expressions)
expr->visit(visitor);
}
}
void AstExprError::visit(AstVisitor* visitor)
{
if (visitor->visit(this))

View File

@ -6,6 +6,8 @@
#include <limits.h>
LUAU_FASTFLAG(LuauInterpolatedStringBaseSupport)
namespace Luau
{
@ -89,7 +91,18 @@ Lexeme::Lexeme(const Location& location, Type type, const char* data, size_t siz
, length(unsigned(size))
, data(data)
{
LUAU_ASSERT(type == RawString || type == QuotedString || type == Number || type == Comment || type == BlockComment);
LUAU_ASSERT(
type == RawString
|| type == QuotedString
|| type == InterpStringBegin
|| type == InterpStringMid
|| type == InterpStringEnd
|| type == InterpStringSimple
|| type == BrokenInterpDoubleBrace
|| type == Number
|| type == Comment
|| type == BlockComment
);
}
Lexeme::Lexeme(const Location& location, Type type, const char* name)
@ -160,6 +173,18 @@ std::string Lexeme::toString() const
case QuotedString:
return data ? format("\"%.*s\"", length, data) : "string";
case InterpStringBegin:
return data ? format("`%.*s{", length, data) : "the beginning of an interpolated string";
case InterpStringMid:
return data ? format("}%.*s{", length, data) : "the middle of an interpolated string";
case InterpStringEnd:
return data ? format("}%.*s`", length, data) : "the end of an interpolated string";
case InterpStringSimple:
return data ? format("`%.*s`", length, data) : "interpolated string";
case Number:
return data ? format("'%.*s'", length, data) : "number";
@ -175,6 +200,9 @@ std::string Lexeme::toString() const
case BrokenComment:
return "unfinished comment";
case BrokenInterpDoubleBrace:
return "'{{', which is invalid (did you mean '\\{'?)";
case BrokenUnicode:
if (codepoint)
{
@ -515,6 +543,32 @@ Lexeme Lexer::readLongString(const Position& start, int sep, Lexeme::Type ok, Le
return Lexeme(Location(start, position()), broken);
}
void Lexer::readBackslashInString()
{
LUAU_ASSERT(peekch() == '\\');
consume();
switch (peekch())
{
case '\r':
consume();
if (peekch() == '\n')
consume();
break;
case 0:
break;
case 'z':
consume();
while (isSpace(peekch()))
consume();
break;
default:
consume();
}
}
Lexeme Lexer::readQuotedString()
{
Position start = position();
@ -535,27 +589,7 @@ Lexeme Lexer::readQuotedString()
return Lexeme(Location(start, position()), Lexeme::BrokenString);
case '\\':
consume();
switch (peekch())
{
case '\r':
consume();
if (peekch() == '\n')
consume();
break;
case 0:
break;
case 'z':
consume();
while (isSpace(peekch()))
consume();
break;
default:
consume();
}
readBackslashInString();
break;
default:
@ -568,6 +602,69 @@ Lexeme Lexer::readQuotedString()
return Lexeme(Location(start, position()), Lexeme::QuotedString, &buffer[startOffset], offset - startOffset - 1);
}
Lexeme Lexer::readInterpolatedStringBegin()
{
LUAU_ASSERT(peekch() == '`');
Position start = position();
consume();
return readInterpolatedStringSection(start, Lexeme::InterpStringBegin, Lexeme::InterpStringSimple);
}
Lexeme Lexer::readInterpolatedStringSection(Position start, Lexeme::Type formatType, Lexeme::Type endType)
{
unsigned int startOffset = offset;
while (peekch() != '`')
{
switch (peekch())
{
case 0:
case '\r':
case '\n':
return Lexeme(Location(start, position()), Lexeme::BrokenString);
case '\\':
// Allow for \u{}, which would otherwise be consumed by looking for {
if (peekch(1) == 'u' && peekch(2) == '{')
{
consume(); // backslash
consume(); // u
consume(); // {
break;
}
readBackslashInString();
break;
case '{':
{
braceStack.push_back(BraceType::InterpolatedString);
if (peekch(1) == '{')
{
Lexeme brokenDoubleBrace = Lexeme(Location(start, position()), Lexeme::BrokenInterpDoubleBrace, &buffer[startOffset], offset - startOffset);
consume();
consume();
return brokenDoubleBrace;
}
Lexeme lexemeOutput(Location(start, position()), Lexeme::InterpStringBegin, &buffer[startOffset], offset - startOffset);
consume();
return lexemeOutput;
}
default:
consume();
}
}
consume();
return Lexeme(Location(start, position()), endType, &buffer[startOffset], offset - startOffset - 1);
}
Lexeme Lexer::readNumber(const Position& start, unsigned int startOffset)
{
LUAU_ASSERT(isDigit(peekch()));
@ -660,6 +757,36 @@ Lexeme Lexer::readNext()
}
}
case '{':
{
consume();
if (!braceStack.empty())
braceStack.push_back(BraceType::Normal);
return Lexeme(Location(start, 1), '{');
}
case '}':
{
consume();
if (braceStack.empty())
{
return Lexeme(Location(start, 1), '}');
}
const BraceType braceStackTop = braceStack.back();
braceStack.pop_back();
if (braceStackTop != BraceType::InterpolatedString)
{
return Lexeme(Location(start, 1), '}');
}
return readInterpolatedStringSection(position(), Lexeme::InterpStringMid, Lexeme::InterpStringEnd);
}
case '=':
{
consume();
@ -716,6 +843,15 @@ Lexeme Lexer::readNext()
case '\'':
return readQuotedString();
case '`':
if (FFlag::LuauInterpolatedStringBaseSupport)
return readInterpolatedStringBegin();
else
{
consume();
return Lexeme(Location(start, 1), '`');
}
case '.':
consume();
@ -817,8 +953,6 @@ Lexeme Lexer::readNext()
case '(':
case ')':
case '{':
case '}':
case ']':
case ';':
case ',':

View File

@ -23,10 +23,14 @@ LUAU_FASTFLAGVARIABLE(LuauErrorDoubleHexPrefix, false)
LUAU_FASTFLAGVARIABLE(LuauLintParseIntegerIssues, false)
LUAU_DYNAMIC_FASTFLAGVARIABLE(LuaReportParseIntegerIssues, false)
LUAU_FASTFLAGVARIABLE(LuauInterpolatedStringBaseSupport, false)
bool lua_telemetry_parsed_out_of_range_bin_integer = false;
bool lua_telemetry_parsed_out_of_range_hex_integer = false;
bool lua_telemetry_parsed_double_prefix_hex_integer = false;
#define ERROR_INVALID_INTERP_DOUBLE_BRACE "Double braces are not permitted within interpolated strings. Did you mean '\\{'?"
namespace Luau
{
@ -1567,6 +1571,12 @@ AstTypeOrPack Parser::parseSimpleTypeAnnotation(bool allowPack)
else
return {reportTypeAnnotationError(begin, {}, /*isMissing*/ false, "String literal contains malformed escape sequence")};
}
else if (lexer.current().type == Lexeme::InterpStringBegin || lexer.current().type == Lexeme::InterpStringSimple)
{
parseInterpString();
return {reportTypeAnnotationError(begin, {}, /*isMissing*/ false, "Interpolated string literals cannot be used as types")};
}
else if (lexer.current().type == Lexeme::BrokenString)
{
Location location = lexer.current().location;
@ -2215,15 +2225,24 @@ AstExpr* Parser::parseSimpleExpr()
{
return parseNumber();
}
else if (lexer.current().type == Lexeme::RawString || lexer.current().type == Lexeme::QuotedString)
else if (lexer.current().type == Lexeme::RawString || lexer.current().type == Lexeme::QuotedString || (FFlag::LuauInterpolatedStringBaseSupport && lexer.current().type == Lexeme::InterpStringSimple))
{
return parseString();
}
else if (FFlag::LuauInterpolatedStringBaseSupport && lexer.current().type == Lexeme::InterpStringBegin)
{
return parseInterpString();
}
else if (lexer.current().type == Lexeme::BrokenString)
{
nextLexeme();
return reportExprError(start, {}, "Malformed string");
}
else if (lexer.current().type == Lexeme::BrokenInterpDoubleBrace)
{
nextLexeme();
return reportExprError(start, {}, ERROR_INVALID_INTERP_DOUBLE_BRACE);
}
else if (lexer.current().type == Lexeme::Dot3)
{
if (functionStack.back().vararg)
@ -2614,11 +2633,11 @@ AstArray<AstTypeOrPack> Parser::parseTypeParams()
std::optional<AstArray<char>> Parser::parseCharArray()
{
LUAU_ASSERT(lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::RawString);
LUAU_ASSERT(lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::RawString || lexer.current().type == Lexeme::InterpStringSimple);
scratchData.assign(lexer.current().data, lexer.current().length);
if (lexer.current().type == Lexeme::QuotedString)
if (lexer.current().type == Lexeme::QuotedString || lexer.current().type == Lexeme::InterpStringSimple)
{
if (!Lexer::fixupQuotedString(scratchData))
{
@ -2645,6 +2664,70 @@ AstExpr* Parser::parseString()
return reportExprError(location, {}, "String literal contains malformed escape sequence");
}
AstExpr* Parser::parseInterpString()
{
TempVector<AstArray<char>> strings(scratchString);
TempVector<AstExpr*> expressions(scratchExpr);
Location startLocation = lexer.current().location;
do {
Lexeme currentLexeme = lexer.current();
LUAU_ASSERT(
currentLexeme.type == Lexeme::InterpStringBegin
|| currentLexeme.type == Lexeme::InterpStringMid
|| currentLexeme.type == Lexeme::InterpStringEnd
|| currentLexeme.type == Lexeme::InterpStringSimple
);
Location location = currentLexeme.location;
Location startOfBrace = Location(location.end, 1);
scratchData.assign(currentLexeme.data, currentLexeme.length);
if (!Lexer::fixupQuotedString(scratchData))
{
nextLexeme();
return reportExprError(startLocation, {}, "Interpolated string literal contains malformed escape sequence");
}
AstArray<char> chars = copy(scratchData);
nextLexeme();
strings.push_back(chars);
if (currentLexeme.type == Lexeme::InterpStringEnd || currentLexeme.type == Lexeme::InterpStringSimple)
{
AstArray<AstArray<char>> stringsArray = copy(strings);
AstArray<AstExpr*> expressionsArray = copy(expressions);
return allocator.alloc<AstExprInterpString>(startLocation, stringsArray, expressionsArray);
}
AstExpr* expression = parseExpr();
expressions.push_back(expression);
switch (lexer.current().type)
{
case Lexeme::InterpStringBegin:
case Lexeme::InterpStringMid:
case Lexeme::InterpStringEnd:
break;
case Lexeme::BrokenInterpDoubleBrace:
nextLexeme();
return reportExprError(location, {}, ERROR_INVALID_INTERP_DOUBLE_BRACE);
case Lexeme::BrokenString:
nextLexeme();
return reportExprError(location, {}, "Malformed interpolated string, did you forget to add a '}'?");
default:
return reportExprError(location, {}, "Malformed interpolated string, got %s", lexer.current().toString().c_str());
}
} while (true);
}
AstExpr* Parser::parseNumber()
{
Location start = lexer.current().location;

View File

@ -230,19 +230,25 @@ bool isIdentifier(std::string_view s)
return (s.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890_") == std::string::npos);
}
std::string escape(std::string_view s)
std::string escape(std::string_view s, bool escapeForInterpString)
{
std::string r;
r.reserve(s.size() + 50); // arbitrary number to guess how many characters we'll be inserting
for (uint8_t c : s)
{
if (c >= ' ' && c != '\\' && c != '\'' && c != '\"')
if (c >= ' ' && c != '\\' && c != '\'' && c != '\"' && c != '`' && c != '{')
r += c;
else
{
r += '\\';
if (escapeForInterpString && (c == '`' || c == '{'))
{
r += c;
continue;
}
switch (c)
{
case '\a':

View File

@ -14,6 +14,8 @@
#include <algorithm>
#include <bitset>
#include <memory>
#include <math.h>
LUAU_FASTINTVARIABLE(LuauCompileLoopUnrollThreshold, 25)
@ -25,6 +27,8 @@ LUAU_FASTINTVARIABLE(LuauCompileInlineDepth, 5)
LUAU_FASTFLAGVARIABLE(LuauCompileXEQ, false)
LUAU_FASTFLAG(LuauInterpolatedStringBaseSupport)
LUAU_FASTFLAGVARIABLE(LuauCompileOptimalAssignment, false)
LUAU_FASTFLAGVARIABLE(LuauCompileExtractK, false)
@ -1585,6 +1589,76 @@ struct Compiler
}
}
void compileExprInterpString(AstExprInterpString* expr, uint8_t target, bool targetTemp)
{
size_t formatCapacity = 0;
for (AstArray<char> string : expr->strings)
{
formatCapacity += string.size + std::count(string.data, string.data + string.size, '%');
}
std::string formatString;
formatString.reserve(formatCapacity);
size_t stringsLeft = expr->strings.size;
for (AstArray<char> string : expr->strings)
{
if (memchr(string.data, '%', string.size))
{
for (size_t characterIndex = 0; characterIndex < string.size; ++characterIndex)
{
char character = string.data[characterIndex];
formatString.push_back(character);
if (character == '%')
formatString.push_back('%');
}
}
else
formatString.append(string.data, string.size);
stringsLeft--;
if (stringsLeft > 0)
formatString += "%*";
}
size_t formatStringSize = formatString.size();
// We can't use formatStringRef.data() directly, because short strings don't have their data
// pinned in memory, so when interpFormatStrings grows, these pointers will move and become invalid.
std::unique_ptr<char[]> formatStringPtr(new char[formatStringSize]);
memcpy(formatStringPtr.get(), formatString.data(), formatStringSize);
AstArray<char> formatStringArray{formatStringPtr.get(), formatStringSize};
interpStrings.emplace_back(std::move(formatStringPtr)); // invalidates formatStringPtr, but keeps formatStringArray intact
int32_t formatStringIndex = bytecode.addConstantString(sref(formatStringArray));
if (formatStringIndex < 0)
CompileError::raise(expr->location, "Exceeded constant limit; simplify the code to compile");
RegScope rs(this);
uint8_t baseReg = allocReg(expr, uint8_t(2 + expr->expressions.size));
emitLoadK(baseReg, formatStringIndex);
for (size_t index = 0; index < expr->expressions.size; ++index)
compileExprTempTop(expr->expressions.data[index], uint8_t(baseReg + 2 + index));
BytecodeBuilder::StringRef formatMethod = sref(AstName("format"));
int32_t formatMethodIndex = bytecode.addConstantString(formatMethod);
if (formatMethodIndex < 0)
CompileError::raise(expr->location, "Exceeded constant limit; simplify the code to compile");
bytecode.emitABC(LOP_NAMECALL, baseReg, baseReg, uint8_t(BytecodeBuilder::getStringHash(formatMethod)));
bytecode.emitAux(formatMethodIndex);
bytecode.emitABC(LOP_CALL, baseReg, uint8_t(expr->expressions.size + 2), 2);
bytecode.emitABC(LOP_MOVE, target, baseReg, 0);
}
static uint8_t encodeHashSize(unsigned int hashSize)
{
size_t hashSizeLog2 = 0;
@ -2059,6 +2133,10 @@ struct Compiler
{
compileExprIfElse(expr, target, targetTemp);
}
else if (AstExprInterpString* interpString = node->as<AstExprInterpString>(); FFlag::LuauInterpolatedStringBaseSupport && interpString)
{
compileExprInterpString(interpString, target, targetTemp);
}
else
{
LUAU_ASSERT(!"Unknown expression type");
@ -3808,6 +3886,7 @@ struct Compiler
std::vector<Loop> loops;
std::vector<InlineFrame> inlineFrames;
std::vector<Capture> captures;
std::vector<std::unique_ptr<char[]>> interpStrings;
};
void compileOrThrow(BytecodeBuilder& bytecode, const ParseResult& parseResult, const AstNameTable& names, const CompileOptions& inputOptions)

View File

@ -349,6 +349,11 @@ struct ConstantVisitor : AstVisitor
if (cond.type != Constant::Type_Unknown)
result = cond.isTruthful() ? trueExpr : falseExpr;
}
else if (AstExprInterpString* expr = node->as<AstExprInterpString>())
{
for (AstExpr* expression : expr->expressions)
analyze(expression);
}
else
{
LUAU_ASSERT(!"Unknown expression type");

View File

@ -215,6 +215,16 @@ struct CostVisitor : AstVisitor
{
return model(expr->condition) + model(expr->trueExpr) + model(expr->falseExpr) + 2;
}
else if (AstExprInterpString* expr = node->as<AstExprInterpString>())
{
// Baseline cost of string.format
Cost cost = 3;
for (AstExpr* innerExpression : expr->expressions)
cost += model(innerExpression);
return cost;
}
else
{
LUAU_ASSERT(!"Unknown expression type");

View File

@ -44,7 +44,8 @@ functioncall = prefixexp funcargs | prefixexp ':' NAME funcargs
exp = (asexp | unop exp) { binop exp }
ifelseexp = 'if' exp 'then' exp {'elseif' exp 'then' exp} 'else' exp
asexp = simpleexp ['::' Type]
simpleexp = NUMBER | STRING | 'nil' | 'true' | 'false' | '...' | tableconstructor | 'function' body | prefixexp | ifelseexp
stringinterp = INTERP_BEGIN exp { INTERP_MID exp } INTERP_END
simpleexp = NUMBER | STRING | 'nil' | 'true' | 'false' | '...' | tableconstructor | 'function' body | prefixexp | ifelseexp | stringinterp
funcargs = '(' [explist] ')' | tableconstructor | STRING
tableconstructor = '{' [fieldlist] '}'

View File

@ -2,6 +2,7 @@
#include "Luau/Ast.h"
#include "Luau/AstJsonEncoder.h"
#include "Luau/Parser.h"
#include "ScopedFlags.h"
#include "doctest.h"
@ -175,6 +176,17 @@ TEST_CASE_FIXTURE(JsonEncoderFixture, "encode_AstExprIfThen")
CHECK(toJson(statement) == expected);
}
TEST_CASE_FIXTURE(JsonEncoderFixture, "encode_AstExprInterpString")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
AstStat* statement = expectParseStatement("local a = `var = {x}`");
std::string_view expected =
R"({"type":"AstStatLocal","location":"0,0 - 0,17","vars":[{"luauType":null,"name":"a","type":"AstLocal","location":"0,6 - 0,7"}],"values":[{"type":"AstExprInterpString","location":"0,10 - 0,17","strings":["var = ",""],"expressions":[{"type":"AstExprGlobal","location":"0,18 - 0,19","global":"x"}]}]})";
CHECK(toJson(statement) == expected);
}
TEST_CASE("encode_AstExprLocal")
{

View File

@ -2708,6 +2708,15 @@ a = if temp then even else abc@3
CHECK(ac.entryMap.count("abcdef"));
}
TEST_CASE_FIXTURE(ACFixture, "autocomplete_interpolated_string")
{
check(R"(f(`expression = {@1}`))");
auto ac = autocomplete('1');
CHECK(ac.entryMap.count("table"));
CHECK_EQ(ac.context, AutocompleteContext::Expression);
}
TEST_CASE_FIXTURE(ACFixture, "autocomplete_explicit_type_pack")
{
check(R"(

View File

@ -1230,6 +1230,58 @@ RETURN R0 0
)");
}
TEST_CASE("InterpStringWithNoExpressions")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
CHECK_EQ(compileFunction0(R"(return "hello")"), compileFunction0("return `hello`"));
}
TEST_CASE("InterpStringZeroCost")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
CHECK_EQ(
"\n" + compileFunction0(R"(local _ = `hello, {"world"}!`)"),
R"(
LOADK R1 K0
LOADK R3 K1
NAMECALL R1 R1 K2
CALL R1 2 1
MOVE R0 R1
RETURN R0 0
)"
);
}
TEST_CASE("InterpStringRegisterCleanup")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
CHECK_EQ(
"\n" + compileFunction0(R"(
local a, b, c = nil, "um", "uh oh"
a = `foo{"bar"}`
print(a)
)"),
R"(
LOADNIL R0
LOADK R1 K0
LOADK R2 K1
LOADK R3 K2
LOADK R5 K3
NAMECALL R3 R3 K4
CALL R3 2 1
MOVE R0 R3
GETIMPORT R3 6
MOVE R4 R0
CALL R3 1 0
RETURN R0 0
)"
);
}
TEST_CASE("ConstantFoldArith")
{
CHECK_EQ("\n" + compileFunction0("return 10 + 2"), R"(

View File

@ -294,6 +294,14 @@ TEST_CASE("Strings")
runConformance("strings.lua");
}
TEST_CASE("StringInterp")
{
ScopedFastFlag sffInterpStrings{"LuauInterpolatedStringBaseSupport", true};
ScopedFastFlag sffTostringFormat{"LuauTostringFormatSpecifier", true};
runConformance("stringinterp.lua");
}
TEST_CASE("VarArg")
{
runConformance("vararg.lua");

View File

@ -138,4 +138,90 @@ TEST_CASE("lookahead")
CHECK_EQ(lexer.lookahead().type, Lexeme::Eof);
}
TEST_CASE("string_interpolation_basic")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
const std::string testInput = R"(`foo {"bar"}`)";
Luau::Allocator alloc;
AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table);
Lexeme interpBegin = lexer.next();
CHECK_EQ(interpBegin.type, Lexeme::InterpStringBegin);
Lexeme quote = lexer.next();
CHECK_EQ(quote.type, Lexeme::QuotedString);
Lexeme interpEnd = lexer.next();
CHECK_EQ(interpEnd.type, Lexeme::InterpStringEnd);
}
TEST_CASE("string_interpolation_double_brace")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
const std::string testInput = R"(`foo{{bad}}bar`)";
Luau::Allocator alloc;
AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table);
auto brokenInterpBegin = lexer.next();
CHECK_EQ(brokenInterpBegin.type, Lexeme::BrokenInterpDoubleBrace);
CHECK_EQ(std::string(brokenInterpBegin.data, brokenInterpBegin.length), std::string("foo"));
CHECK_EQ(lexer.next().type, Lexeme::Name);
auto interpEnd = lexer.next();
CHECK_EQ(interpEnd.type, Lexeme::InterpStringEnd);
CHECK_EQ(std::string(interpEnd.data, interpEnd.length), std::string("}bar"));
}
TEST_CASE("string_interpolation_double_but_unmatched_brace")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
const std::string testInput = R"(`{{oops}`, 1)";
Luau::Allocator alloc;
AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table);
CHECK_EQ(lexer.next().type, Lexeme::BrokenInterpDoubleBrace);
CHECK_EQ(lexer.next().type, Lexeme::Name);
CHECK_EQ(lexer.next().type, Lexeme::InterpStringEnd);
CHECK_EQ(lexer.next().type, ',');
CHECK_EQ(lexer.next().type, Lexeme::Number);
}
TEST_CASE("string_interpolation_unmatched_brace")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
const std::string testInput = R"({
`hello {"world"}
} -- this might be incorrectly parsed as a string)";
Luau::Allocator alloc;
AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table);
CHECK_EQ(lexer.next().type, '{');
CHECK_EQ(lexer.next().type, Lexeme::InterpStringBegin);
CHECK_EQ(lexer.next().type, Lexeme::QuotedString);
CHECK_EQ(lexer.next().type, Lexeme::BrokenString);
CHECK_EQ(lexer.next().type, '}');
}
TEST_CASE("string_interpolation_with_unicode_escape")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
const std::string testInput = R"(`\u{1F41B}`)";
Luau::Allocator alloc;
AstNameTable table(alloc);
Lexer lexer(testInput.c_str(), testInput.size(), table);
CHECK_EQ(lexer.next().type, Lexeme::InterpStringSimple);
CHECK_EQ(lexer.next().type, Lexeme::Eof);
}
TEST_SUITE_END();

View File

@ -1662,17 +1662,31 @@ TEST_CASE_FIXTURE(Fixture, "WrongCommentOptimize")
{
LintResult result = lint(R"(
--!optimize
--!optimize
--!optimize me
--!optimize 100500
--!optimize 2
)");
REQUIRE_EQ(result.warnings.size(), 4);
REQUIRE_EQ(result.warnings.size(), 3);
CHECK_EQ(result.warnings[0].text, "optimize directive requires an optimization level");
CHECK_EQ(result.warnings[1].text, "optimize directive requires an optimization level");
CHECK_EQ(result.warnings[2].text, "optimize directive uses unknown optimization level 'me', 0..2 expected");
CHECK_EQ(result.warnings[3].text, "optimize directive uses unknown optimization level '100500', 0..2 expected");
CHECK_EQ(result.warnings[1].text, "optimize directive uses unknown optimization level 'me', 0..2 expected");
CHECK_EQ(result.warnings[2].text, "optimize directive uses unknown optimization level '100500', 0..2 expected");
result = lint("--!optimize ");
REQUIRE_EQ(result.warnings.size(), 1);
CHECK_EQ(result.warnings[0].text, "optimize directive requires an optimization level");
}
TEST_CASE_FIXTURE(Fixture, "TestStringInterpolation")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
LintResult result = lint(R"(
--!nocheck
local _ = `unknown {foo}`
)");
REQUIRE_EQ(result.warnings.size(), 1);
}
TEST_CASE_FIXTURE(Fixture, "IntegerParsing")

View File

@ -905,6 +905,146 @@ TEST_CASE_FIXTURE(Fixture, "parse_compound_assignment_error_multiple")
}
}
TEST_CASE_FIXTURE(Fixture, "parse_interpolated_string_double_brace_begin")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
try
{
parse(R"(
_ = `{{oops}}`
)");
FAIL("Expected ParseErrors to be thrown");
}
catch (const ParseErrors& e)
{
CHECK_EQ("Double braces are not permitted within interpolated strings. Did you mean '\\{'?", e.getErrors().front().getMessage());
}
}
TEST_CASE_FIXTURE(Fixture, "parse_interpolated_string_double_brace_mid")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
try
{
parse(R"(
_ = `{nice} {{oops}}`
)");
FAIL("Expected ParseErrors to be thrown");
}
catch (const ParseErrors& e)
{
CHECK_EQ("Double braces are not permitted within interpolated strings. Did you mean '\\{'?", e.getErrors().front().getMessage());
}
}
TEST_CASE_FIXTURE(Fixture, "parse_interpolated_string_without_end_brace")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
auto columnOfEndBraceError = [this](const char* code)
{
try
{
parse(code);
FAIL("Expected ParseErrors to be thrown");
return UINT_MAX;
}
catch (const ParseErrors& e)
{
CHECK_EQ(e.getErrors().size(), 1);
auto error = e.getErrors().front();
CHECK_EQ("Malformed interpolated string, did you forget to add a '}'?", error.getMessage());
return error.getLocation().begin.column;
}
};
// This makes sure that the error is coming from the brace itself
CHECK_EQ(columnOfEndBraceError("_ = `{a`"), columnOfEndBraceError("_ = `{abcdefg`"));
CHECK_NE(columnOfEndBraceError("_ = `{a`"), columnOfEndBraceError("_ = `{a`"));
}
TEST_CASE_FIXTURE(Fixture, "parse_interpolated_string_without_end_brace_in_table")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
try
{
parse(R"(
_ = { `{a` }
)");
FAIL("Expected ParseErrors to be thrown");
}
catch (const ParseErrors& e)
{
CHECK_EQ(e.getErrors().size(), 2);
CHECK_EQ("Malformed interpolated string, did you forget to add a '}'?", e.getErrors().front().getMessage());
CHECK_EQ("Expected '}' (to close '{' at line 2), got <eof>", e.getErrors().back().getMessage());
}
}
TEST_CASE_FIXTURE(Fixture, "parse_interpolated_string_mid_without_end_brace_in_table")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
try
{
parse(R"(
_ = { `x {"y"} {z` }
)");
FAIL("Expected ParseErrors to be thrown");
}
catch (const ParseErrors& e)
{
CHECK_EQ(e.getErrors().size(), 2);
CHECK_EQ("Malformed interpolated string, did you forget to add a '}'?", e.getErrors().front().getMessage());
CHECK_EQ("Expected '}' (to close '{' at line 2), got <eof>", e.getErrors().back().getMessage());
}
}
TEST_CASE_FIXTURE(Fixture, "parse_interpolated_string_as_type_fail")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
try
{
parse(R"(
local a: `what` = `???`
local b: `what {"the"}` = `???`
local c: `what {"the"} heck` = `???`
)");
FAIL("Expected ParseErrors to be thrown");
}
catch (const ParseErrors& parseErrors)
{
CHECK_EQ(parseErrors.getErrors().size(), 3);
for (ParseError error : parseErrors.getErrors())
CHECK_EQ(error.getMessage(), "Interpolated string literals cannot be used as types");
}
}
TEST_CASE_FIXTURE(Fixture, "parse_interpolated_string_call_without_parens")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
try
{
parse(R"(
_ = print `{42}`
)");
FAIL("Expected ParseErrors to be thrown");
}
catch (const ParseErrors& e)
{
CHECK_EQ("Expected identifier when parsing expression, got `{", e.getErrors().front().getMessage());
}
}
TEST_CASE_FIXTURE(Fixture, "parse_nesting_based_end_detection")
{
try

View File

@ -6,6 +6,7 @@
#include "Luau/Transpiler.h"
#include "Fixture.h"
#include "ScopedFlags.h"
#include "doctest.h"
@ -678,4 +679,22 @@ TEST_CASE_FIXTURE(Fixture, "transpile_for_in_multiple_types")
CHECK_EQ(code, transpile(code, {}, true).code);
}
TEST_CASE_FIXTURE(Fixture, "transpile_string_interp")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
std::string code = R"( local _ = `hello {name}` )";
CHECK_EQ(code, transpile(code, {}, true).code);
}
TEST_CASE_FIXTURE(Fixture, "transpile_string_literal_escape")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
std::string code = R"( local _ = ` bracket = \{, backtick = \` = {'ok'} ` )";
CHECK_EQ(code, transpile(code, {}, true).code);
}
TEST_SUITE_END();

View File

@ -8,6 +8,7 @@
#include "Luau/VisitTypeVar.h"
#include "Fixture.h"
#include "ScopedFlags.h"
#include "doctest.h"
@ -828,6 +829,41 @@ end
LUAU_REQUIRE_NO_ERRORS(result);
}
TEST_CASE_FIXTURE(Fixture, "tc_interpolated_string_basic")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
CheckResult result = check(R"(
local foo: string = `hello {"world"}`
)");
LUAU_REQUIRE_NO_ERRORS(result);
}
TEST_CASE_FIXTURE(Fixture, "tc_interpolated_string_with_invalid_expression")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
CheckResult result = check(R"(
local function f(x: number) end
local foo: string = `hello {f("uh oh")}`
)");
LUAU_REQUIRE_ERROR_COUNT(1, result);
}
TEST_CASE_FIXTURE(Fixture, "tc_interpolated_string_constant_type")
{
ScopedFastFlag sff{"LuauInterpolatedStringBaseSupport", true};
CheckResult result = check(R"(
local foo: "hello" = `hello`
)");
LUAU_REQUIRE_NO_ERRORS(result);
}
/*
* If it wasn't instantly obvious, we have the fuzzer to thank for this gem of a test.
*

View File

@ -0,0 +1,59 @@
local function assertEq(left, right)
assert(typeof(left) == "string", "left is a " .. typeof(left))
assert(typeof(right) == "string", "right is a " .. typeof(right))
if left ~= right then
error(string.format("%q ~= %q", left, right))
end
end
assertEq(`hello {"world"}`, "hello world")
assertEq(`Welcome {"to"} {"Luau"}!`, "Welcome to Luau!")
assertEq(`2 + 2 = {2 + 2}`, "2 + 2 = 4")
assertEq(`{1} {2} {3} {4} {5} {6} {7}`, "1 2 3 4 5 6 7")
local combo = {5, 2, 8, 9}
assertEq(`The lock combinations are: {table.concat(combo, ", ")}`, "The lock combinations are: 5, 2, 8, 9")
assertEq(`true = {true}`, "true = true")
local name = "Luau"
assertEq(`Welcome to {
name
}!`, "Welcome to Luau!")
local nameNotConstantEvaluated = (function() return "Luau" end)()
assertEq(`Welcome to {nameNotConstantEvaluated}!`, "Welcome to Luau!")
assertEq(`This {localName} does not exist`, "This nil does not exist")
assertEq(`Welcome to \
{name}!`, "Welcome to \nLuau!")
assertEq(`empty`, "empty")
assertEq(`Escaped brace: \{}`, "Escaped brace: {}")
assertEq(`Escaped brace \{} with {"expression"}`, "Escaped brace {} with expression")
assertEq(`Backslash \ that escapes the space is not a part of the string...`, "Backslash that escapes the space is not a part of the string...")
assertEq(`Escaped backslash \\`, "Escaped backslash \\")
assertEq(`Escaped backtick: \``, "Escaped backtick: `")
assertEq(`Hello {`from inside {"a nested string"}`}`, "Hello from inside a nested string")
assertEq(`1 {`2 {`3 {4}`}`}`, "1 2 3 4")
local health = 50
assert(`You have {health}% health` == "You have 50% health")
local function shadowsString(string)
return `Value is {string}`
end
assertEq(shadowsString("hello"), "Value is hello")
assertEq(shadowsString(1), "Value is 1")
assertEq(`\u{0041}\t`, "A\t")
return "OK"