// This file is part of the Luau programming language and is licensed under MIT License; see LICENSE.txt for details #include "Luau/CodeGen.h" #include "Luau/AssemblyBuilderX64.h" #include "Luau/Common.h" #include "Luau/CodeAllocator.h" #include "Luau/CodeBlockUnwind.h" #include "Luau/IrAnalysis.h" #include "Luau/IrBuilder.h" #include "Luau/UnwindBuilder.h" #include "Luau/UnwindBuilderDwarf2.h" #include "Luau/UnwindBuilderWin.h" #include "CustomExecUtils.h" #include "CodeGenX64.h" #include "EmitCommonX64.h" #include "EmitInstructionX64.h" #include "IrLoweringX64.h" #include "NativeState.h" #include "lapi.h" #include #if defined(__x86_64__) || defined(_M_X64) #ifdef _MSC_VER #include // __cpuid #else #include // __cpuid #endif #endif LUAU_FASTFLAGVARIABLE(DebugUseOldCodegen, false) namespace Luau { namespace CodeGen { constexpr uint32_t kFunctionAlignment = 32; struct InstructionOutline { int pcpos; int length; }; static void assembleHelpers(AssemblyBuilderX64& build, ModuleHelpers& helpers) { if (build.logText) build.logAppend("; exitContinueVm\n"); helpers.exitContinueVm = build.setLabel(); emitExit(build, /* continueInVm */ true); if (build.logText) build.logAppend("; exitNoContinueVm\n"); helpers.exitNoContinueVm = build.setLabel(); emitExit(build, /* continueInVm */ false); if (build.logText) build.logAppend("; continueCallInVm\n"); helpers.continueCallInVm = build.setLabel(); emitContinueCallInVm(build); } static int emitInst(AssemblyBuilderX64& build, NativeState& data, ModuleHelpers& helpers, Proto* proto, LuauOpcode op, const Instruction* pc, int i, Label* labelarr, Label& next, Label& fallback) { int skip = 0; switch (op) { case LOP_NOP: break; case LOP_LOADNIL: emitInstLoadNil(build, pc); break; case LOP_LOADB: emitInstLoadB(build, pc, i, labelarr); break; case LOP_LOADN: emitInstLoadN(build, pc); break; case LOP_LOADK: emitInstLoadK(build, pc); break; case LOP_LOADKX: emitInstLoadKX(build, pc); break; case LOP_MOVE: emitInstMove(build, pc); break; case LOP_GETGLOBAL: emitInstGetGlobal(build, pc, i, fallback); break; case LOP_SETGLOBAL: emitInstSetGlobal(build, pc, i, next, fallback); break; case LOP_NAMECALL: emitInstNameCall(build, pc, i, proto->k, next, fallback); break; case LOP_CALL: emitInstCall(build, helpers, pc, i); break; case LOP_RETURN: emitInstReturn(build, helpers, pc, i); break; case LOP_GETTABLE: emitInstGetTable(build, pc, fallback); break; case LOP_SETTABLE: emitInstSetTable(build, pc, next, fallback); break; case LOP_GETTABLEKS: emitInstGetTableKS(build, pc, i, fallback); break; case LOP_SETTABLEKS: emitInstSetTableKS(build, pc, i, next, fallback); break; case LOP_GETTABLEN: emitInstGetTableN(build, pc, fallback); break; case LOP_SETTABLEN: emitInstSetTableN(build, pc, next, fallback); break; case LOP_JUMP: emitInstJump(build, pc, i, labelarr); break; case LOP_JUMPBACK: emitInstJumpBack(build, pc, i, labelarr); break; case LOP_JUMPIF: emitInstJumpIf(build, pc, i, labelarr, /* not_ */ false); break; case LOP_JUMPIFNOT: emitInstJumpIf(build, pc, i, labelarr, /* not_ */ true); break; case LOP_JUMPIFEQ: emitInstJumpIfEq(build, pc, i, labelarr, /* not_ */ false, fallback); break; case LOP_JUMPIFLE: emitInstJumpIfCond(build, pc, i, labelarr, ConditionX64::LessEqual, fallback); break; case LOP_JUMPIFLT: emitInstJumpIfCond(build, pc, i, labelarr, ConditionX64::Less, fallback); break; case LOP_JUMPIFNOTEQ: emitInstJumpIfEq(build, pc, i, labelarr, /* not_ */ true, fallback); break; case LOP_JUMPIFNOTLE: emitInstJumpIfCond(build, pc, i, labelarr, ConditionX64::NotLessEqual, fallback); break; case LOP_JUMPIFNOTLT: emitInstJumpIfCond(build, pc, i, labelarr, ConditionX64::NotLess, fallback); break; case LOP_JUMPX: emitInstJumpX(build, pc, i, labelarr); break; case LOP_JUMPXEQKNIL: emitInstJumpxEqNil(build, pc, i, labelarr); break; case LOP_JUMPXEQKB: emitInstJumpxEqB(build, pc, i, labelarr); break; case LOP_JUMPXEQKN: emitInstJumpxEqN(build, pc, proto->k, i, labelarr); break; case LOP_JUMPXEQKS: emitInstJumpxEqS(build, pc, i, labelarr); break; case LOP_ADD: emitInstBinary(build, pc, TM_ADD, fallback); break; case LOP_SUB: emitInstBinary(build, pc, TM_SUB, fallback); break; case LOP_MUL: emitInstBinary(build, pc, TM_MUL, fallback); break; case LOP_DIV: emitInstBinary(build, pc, TM_DIV, fallback); break; case LOP_MOD: emitInstBinary(build, pc, TM_MOD, fallback); break; case LOP_POW: emitInstBinary(build, pc, TM_POW, fallback); break; case LOP_ADDK: emitInstBinaryK(build, pc, TM_ADD, fallback); break; case LOP_SUBK: emitInstBinaryK(build, pc, TM_SUB, fallback); break; case LOP_MULK: emitInstBinaryK(build, pc, TM_MUL, fallback); break; case LOP_DIVK: emitInstBinaryK(build, pc, TM_DIV, fallback); break; case LOP_MODK: emitInstBinaryK(build, pc, TM_MOD, fallback); break; case LOP_POWK: emitInstPowK(build, pc, proto->k, fallback); break; case LOP_NOT: emitInstNot(build, pc); break; case LOP_MINUS: emitInstMinus(build, pc, fallback); break; case LOP_LENGTH: emitInstLength(build, pc, fallback); break; case LOP_NEWTABLE: emitInstNewTable(build, pc, i, next); break; case LOP_DUPTABLE: emitInstDupTable(build, pc, i, next); break; case LOP_SETLIST: emitInstSetList(build, pc, next); break; case LOP_GETUPVAL: emitInstGetUpval(build, pc); break; case LOP_SETUPVAL: emitInstSetUpval(build, pc, next); break; case LOP_CLOSEUPVALS: emitInstCloseUpvals(build, pc, next); break; case LOP_FASTCALL: // We want to lower next instruction at skip+2, but this instruction is only 1 long, so we need to add 1 skip = emitInstFastCall(build, pc, i, next) + 1; break; case LOP_FASTCALL1: // We want to lower next instruction at skip+2, but this instruction is only 1 long, so we need to add 1 skip = emitInstFastCall1(build, pc, i, next) + 1; break; case LOP_FASTCALL2: skip = emitInstFastCall2(build, pc, i, next); break; case LOP_FASTCALL2K: skip = emitInstFastCall2K(build, pc, i, next); break; case LOP_FORNPREP: emitInstForNPrep(build, pc, i, next, labelarr[i + 1 + LUAU_INSN_D(*pc)]); break; case LOP_FORNLOOP: emitInstForNLoop(build, pc, i, labelarr[i + 1 + LUAU_INSN_D(*pc)], next); break; case LOP_FORGLOOP: emitinstForGLoop(build, pc, i, labelarr[i + 1 + LUAU_INSN_D(*pc)], next, fallback); break; case LOP_FORGPREP_NEXT: emitInstForGPrepNext(build, pc, labelarr[i + 1 + LUAU_INSN_D(*pc)], fallback); break; case LOP_FORGPREP_INEXT: emitInstForGPrepInext(build, pc, labelarr[i + 1 + LUAU_INSN_D(*pc)], fallback); break; case LOP_AND: emitInstAnd(build, pc); break; case LOP_ANDK: emitInstAndK(build, pc); break; case LOP_OR: emitInstOr(build, pc); break; case LOP_ORK: emitInstOrK(build, pc); break; case LOP_GETIMPORT: emitInstGetImport(build, pc, fallback); break; case LOP_CONCAT: emitInstConcat(build, pc, i, next); break; case LOP_COVERAGE: emitInstCoverage(build, i); break; default: emitFallback(build, data, op, i); break; } return skip; } static void emitInstFallback(AssemblyBuilderX64& build, NativeState& data, LuauOpcode op, const Instruction* pc, int i, Label* labelarr) { switch (op) { case LOP_GETIMPORT: emitSetSavedPc(build, i + 1); emitInstGetImportFallback(build, LUAU_INSN_A(*pc), pc[1]); break; case LOP_GETTABLE: emitInstGetTableFallback(build, pc, i); break; case LOP_SETTABLE: emitInstSetTableFallback(build, pc, i); break; case LOP_GETTABLEN: emitInstGetTableNFallback(build, pc, i); break; case LOP_SETTABLEN: emitInstSetTableNFallback(build, pc, i); break; case LOP_NAMECALL: // TODO: fast-paths that we've handled can be removed from the fallback emitFallback(build, data, op, i); break; case LOP_JUMPIFEQ: emitInstJumpIfEqFallback(build, pc, i, labelarr, /* not_ */ false); break; case LOP_JUMPIFLE: emitInstJumpIfCondFallback(build, pc, i, labelarr, ConditionX64::LessEqual); break; case LOP_JUMPIFLT: emitInstJumpIfCondFallback(build, pc, i, labelarr, ConditionX64::Less); break; case LOP_JUMPIFNOTEQ: emitInstJumpIfEqFallback(build, pc, i, labelarr, /* not_ */ true); break; case LOP_JUMPIFNOTLE: emitInstJumpIfCondFallback(build, pc, i, labelarr, ConditionX64::NotLessEqual); break; case LOP_JUMPIFNOTLT: emitInstJumpIfCondFallback(build, pc, i, labelarr, ConditionX64::NotLess); break; case LOP_ADD: emitInstBinaryFallback(build, pc, i, TM_ADD); break; case LOP_SUB: emitInstBinaryFallback(build, pc, i, TM_SUB); break; case LOP_MUL: emitInstBinaryFallback(build, pc, i, TM_MUL); break; case LOP_DIV: emitInstBinaryFallback(build, pc, i, TM_DIV); break; case LOP_MOD: emitInstBinaryFallback(build, pc, i, TM_MOD); break; case LOP_POW: emitInstBinaryFallback(build, pc, i, TM_POW); break; case LOP_ADDK: emitInstBinaryKFallback(build, pc, i, TM_ADD); break; case LOP_SUBK: emitInstBinaryKFallback(build, pc, i, TM_SUB); break; case LOP_MULK: emitInstBinaryKFallback(build, pc, i, TM_MUL); break; case LOP_DIVK: emitInstBinaryKFallback(build, pc, i, TM_DIV); break; case LOP_MODK: emitInstBinaryKFallback(build, pc, i, TM_MOD); break; case LOP_POWK: emitInstBinaryKFallback(build, pc, i, TM_POW); break; case LOP_MINUS: emitInstMinusFallback(build, pc, i); break; case LOP_LENGTH: emitInstLengthFallback(build, pc, i); break; case LOP_FORGLOOP: emitinstForGLoopFallback(build, pc, i, labelarr[i + 1 + LUAU_INSN_D(*pc)]); break; case LOP_FORGPREP_NEXT: case LOP_FORGPREP_INEXT: emitInstForGPrepXnextFallback(build, pc, i, labelarr[i + 1 + LUAU_INSN_D(*pc)]); break; case LOP_GETGLOBAL: // TODO: luaV_gettable + cachedslot update instead of full fallback emitFallback(build, data, op, i); break; case LOP_SETGLOBAL: // TODO: luaV_settable + cachedslot update instead of full fallback emitFallback(build, data, op, i); break; case LOP_GETTABLEKS: // Full fallback required for LOP_GETTABLEKS because 'luaV_gettable' doesn't handle builtin vector field access // It is also required to perform cached slot update // TODO: extra fast-paths could be lowered before the full fallback emitFallback(build, data, op, i); break; case LOP_SETTABLEKS: // TODO: luaV_settable + cachedslot update instead of full fallback emitFallback(build, data, op, i); break; default: LUAU_ASSERT(!"Expected fallback for instruction"); } } static NativeProto* assembleFunction(AssemblyBuilderX64& build, NativeState& data, ModuleHelpers& helpers, Proto* proto, AssemblyOptions options) { NativeProto* result = new NativeProto(); result->proto = proto; if (options.includeAssembly || options.includeIr) { if (proto->debugname) build.logAppend("; function %s()", getstr(proto->debugname)); else build.logAppend("; function()"); if (proto->linedefined >= 0) build.logAppend(" line %d\n", proto->linedefined); else build.logAppend("\n"); } if (!FFlag::DebugUseOldCodegen) { build.align(kFunctionAlignment, AlignmentDataX64::Ud2); Label start = build.setLabel(); IrBuilder builder; builder.buildFunctionIr(proto); updateUseInfo(builder.function); IrLoweringX64 lowering(build, helpers, data, proto, builder.function); lowering.lower(options); result->instTargets = new uintptr_t[proto->sizecode]; for (int i = 0; i < proto->sizecode; i++) { auto [irLocation, asmLocation] = builder.function.bcMapping[i]; result->instTargets[i] = irLocation == ~0u ? 0 : asmLocation - start.location; } result->location = start.location; if (build.logText) build.logAppend("\n"); return result; } std::vector