diff --git a/src/jit/Backend.cpp b/src/jit/Backend.cpp index 2553bd5d9..00d542e03 100644 --- a/src/jit/Backend.cpp +++ b/src/jit/Backend.cpp @@ -44,6 +44,9 @@ extern "C" { #define OffsetOfContextField(field) \ (static_cast(offsetof(ExecutionContext, field))) +#define OffsetOfStackTmp(type, field) \ + (stackTmpStart + static_cast(offsetof(type, field))) + #if !(defined SLJIT_INDIRECT_CALL && SLJIT_INDIRECT_CALL) #define GET_FUNC_ADDR(type, func) (reinterpret_cast(func)) #else @@ -219,6 +222,7 @@ CompileContext::CompileContext(Module* module, JITCompiler* compiler) #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) , shuffleOffset(0) #endif /* SLJIT_CONFIG_X86 */ + , stackTmpStart(0) , nextTryBlock(0) , currentTryBlock(InstanceConstData::globalTryBlock) , trapBlocksStart(0) @@ -1016,6 +1020,7 @@ JITCompiler::JITCompiler(Module* module, uint32_t JITFlags) , m_options(0) , m_savedIntegerRegCount(0) , m_savedFloatRegCount(0) + , m_stackTmpSize(0) { if (module->m_jitModule != nullptr) { ASSERT(module->m_jitModule->m_instanceConstData != nullptr); @@ -1471,6 +1476,7 @@ void JITCompiler::clear() m_first = nullptr; m_last = nullptr; m_branchTableSize = 0; + m_stackTmpSize = 0; #if (defined SLJIT_CONFIG_X86 && SLJIT_CONFIG_X86) m_context.shuffleOffset = 0; #endif /* SLJIT_CONFIG_X86 */ @@ -1507,17 +1513,15 @@ void JITCompiler::emitProlog() options |= SLJIT_ENTER_USE_VEX; #endif /* !SLJIT_CONFIG_X86 */ +#if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) + ASSERT(m_stackTmpSize <= 32); +#else /* !SLJIT_CONFIG_ARM_32 */ + ASSERT(m_stackTmpSize <= 16); +#endif /* SLJIT_CONFIG_ARM_32 */ + sljit_emit_enter(m_compiler, options, SLJIT_ARGS0(P), SLJIT_NUMBER_OF_SCRATCH_REGISTERS | SLJIT_ENTER_FLOAT(SLJIT_NUMBER_OF_SCRATCH_FLOAT_REGISTERS), - (m_savedIntegerRegCount + 2) | SLJIT_ENTER_FLOAT(m_savedFloatRegCount), sizeof(ExecutionContext::CallFrame)); - - // Setup new frame. - sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_R0, 0, SLJIT_MEM1(kContextReg), OffsetOfContextField(lastFrame)); - - sljit_get_local_base(m_compiler, SLJIT_R1, 0, 0); - sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_MEM1(kContextReg), OffsetOfContextField(lastFrame), SLJIT_R1, 0); - sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_MEM1(SLJIT_SP), offsetof(ExecutionContext::CallFrame, frameStart), kFrameReg, 0); - sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_MEM1(SLJIT_SP), offsetof(ExecutionContext::CallFrame, prevFrame), SLJIT_R0, 0); + (m_savedIntegerRegCount + 2) | SLJIT_ENTER_FLOAT(m_savedFloatRegCount), m_stackTmpSize); m_context.branchTableOffset = 0; size_t size = func.branchTableSize * sizeof(sljit_up); @@ -1555,10 +1559,6 @@ void JITCompiler::emitEpilog() m_context.earlyReturns.clear(); } - // Restore previous frame. - sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_R1, 0, SLJIT_MEM1(SLJIT_SP), offsetof(ExecutionContext::CallFrame, prevFrame)); - sljit_emit_op1(m_compiler, SLJIT_MOV_P, SLJIT_MEM1(kContextReg), OffsetOfContextField(lastFrame), SLJIT_R1, 0); - sljit_emit_return(m_compiler, SLJIT_MOV_P, SLJIT_R0, 0); m_context.emitSlowCases(m_compiler); diff --git a/src/jit/ByteCodeParser.cpp b/src/jit/ByteCodeParser.cpp index 56328b247..f34da7ae2 100644 --- a/src/jit/ByteCodeParser.cpp +++ b/src/jit/ByteCodeParser.cpp @@ -549,6 +549,7 @@ static void compileFunction(JITCompiler* compiler) group = Instruction::Binary; paramType = ParamTypes::ParamSrc2Dst; #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) + compiler->increaseStackTmpSize(16); info = Instruction::kIsCallback; requiredInit = OTDivRemI64; #else /* !SLJIT_32BIT_ARCHITECTURE */ @@ -790,6 +791,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; requiredInit = OTConvertInt32FromFloat32Callback; + compiler->increaseStackTmpSize(4); #else /* !SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertInt32FromFloat32; #endif /* SLJIT_32BIT_ARCHITECTURE */ @@ -808,6 +810,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; requiredInit = OTConvertInt32FromFloat64Callback; + compiler->increaseStackTmpSize(8); #else /* !SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertInt32FromFloat64; #endif /* SLJIT_32BIT_ARCHITECTURE */ @@ -819,6 +822,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; requiredInit = OTConvertInt64FromFloat32Callback; + compiler->increaseStackTmpSize(8); #else /* !SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertInt64FromFloat32; #endif /* SLJIT_32BIT_ARCHITECTURE */ @@ -830,6 +834,7 @@ static void compileFunction(JITCompiler* compiler) paramType = ParamTypes::ParamSrcDst; info = Instruction::kIsCallback; requiredInit = OTConvertInt64FromFloat32Callback; + compiler->increaseStackTmpSize(8); break; } case ByteCode::I64TruncF64SOpcode: { @@ -838,6 +843,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; requiredInit = OTConvertInt64FromFloat64Callback; + compiler->increaseStackTmpSize(8); #else /* !SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertInt64FromFloat64; #endif /* SLJIT_32BIT_ARCHITECTURE */ @@ -849,6 +855,7 @@ static void compileFunction(JITCompiler* compiler) paramType = ParamTypes::ParamSrcDst; info = Instruction::kIsCallback; requiredInit = OTConvertInt64FromFloat64Callback; + compiler->increaseStackTmpSize(8); break; } case ByteCode::I32TruncSatF32UOpcode: { @@ -857,6 +864,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; requiredInit = OTConvertInt32FromFloat32Callback; + compiler->increaseStackTmpSize(4); #else /* !SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertInt32FromFloat32; #endif /* SLJIT_32BIT_ARCHITECTURE */ @@ -868,6 +876,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; requiredInit = OTConvertInt32FromFloat64Callback; + compiler->increaseStackTmpSize(8); #else /* !SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertInt32FromFloat64; #endif /* SLJIT_32BIT_ARCHITECTURE */ @@ -879,6 +888,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; requiredInit = OTConvertInt64FromFloat32Callback; + compiler->increaseStackTmpSize(8); #else /* !SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertInt64FromFloat32; #endif /* SLJIT_32BIT_ARCHITECTURE */ @@ -890,6 +900,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; requiredInit = OTConvertInt64FromFloat64Callback; + compiler->increaseStackTmpSize(8); #else /* !SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertInt64FromFloat64; #endif /* SLJIT_32BIT_ARCHITECTURE */ @@ -908,6 +919,7 @@ static void compileFunction(JITCompiler* compiler) paramType = ParamTypes::ParamSrcDst; #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; + compiler->increaseStackTmpSize(8); #endif /* SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertFloat32FromInt64; break; @@ -925,6 +937,7 @@ static void compileFunction(JITCompiler* compiler) paramType = ParamTypes::ParamSrcDst; #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) info = Instruction::kIsCallback; + compiler->increaseStackTmpSize(8); #endif /* SLJIT_32BIT_ARCHITECTURE */ requiredInit = OTConvertFloat64FromInt64; break; @@ -1235,6 +1248,7 @@ static void compileFunction(JITCompiler* compiler) Instruction* instr = compiler->append(byteCode, Instruction::Table, opcode, 3, 0); instr->addInfo(Instruction::kIsCallback); instr->setRequiredRegsDescriptor(OTCallback3Arg); + compiler->increaseStackTmpSize(sizeof(InitTableArguments)); Operand* operands = instr->operands(); operands[0] = STACK_OFFSET(tableInit->srcOffsets()[0]); @@ -1254,6 +1268,7 @@ static void compileFunction(JITCompiler* compiler) Instruction* instr = compiler->append(byteCode, Instruction::Table, opcode, 3, 0); instr->addInfo(Instruction::kIsCallback); instr->setRequiredRegsDescriptor(OTCallback3Arg); + compiler->increaseStackTmpSize(sizeof(TableCopyArguments)); Operand* operands = instr->operands(); operands[0] = STACK_OFFSET(tableCopy->srcOffsets()[0]); @@ -1267,6 +1282,7 @@ static void compileFunction(JITCompiler* compiler) Instruction* instr = compiler->append(byteCode, Instruction::Table, opcode, 3, 0); instr->addInfo(Instruction::kIsCallback); instr->setRequiredRegsDescriptor(OTCallback3Arg); + compiler->increaseStackTmpSize(sizeof(TableFillArguments)); Operand* operands = instr->operands(); operands[0] = STACK_OFFSET(tableFill->srcOffsets()[0]); @@ -1314,6 +1330,7 @@ static void compileFunction(JITCompiler* compiler) Instruction* instr = compiler->append(byteCode, Instruction::Memory, opcode, 3, 0); instr->addInfo(Instruction::kIsCallback); instr->setRequiredRegsDescriptor(OTCallback3Arg); + compiler->increaseStackTmpSize(sizeof(MemoryInitArguments)); Operand* operands = instr->operands(); operands[0] = STACK_OFFSET(memoryInit->srcOffsets()[0]); @@ -1663,6 +1680,7 @@ static void compileFunction(JITCompiler* compiler) paramType = ParamTypes::ParamSrc2Dst; #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) info = Instruction::kIsCallback; + compiler->increaseStackTmpSize(32); #endif /* SLJIT_CONFIG_ARM_32 */ requiredInit = OTMinMaxV128; break; @@ -1741,6 +1759,7 @@ static void compileFunction(JITCompiler* compiler) paramType = ParamTypes::ParamSrcDst; #if (defined SLJIT_CONFIG_ARM_32 && SLJIT_CONFIG_ARM_32) info = Instruction::kIsCallback; + compiler->increaseStackTmpSize(16); #endif /* SLJIT_CONFIG_ARM_32 */ requiredInit = OTOp1V128CB; break; @@ -1841,6 +1860,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) if (opcode == ByteCode::I64AtomicLoadOpcode) { info = Instruction::kIsCallback; + compiler->increaseStackTmpSize(8); } #endif /* SLJIT_32BIT_ARCHITECTURE */ if (requiredInit == OTNone) { @@ -1864,6 +1884,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) if (opcode == ByteCode::I64AtomicStoreOpcode) { info = Instruction::kIsCallback; + compiler->increaseStackTmpSize(8); } #endif /* SLJIT_32BIT_ARCHITECTURE */ if (requiredInit == OTNone) { @@ -1902,6 +1923,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) if (info == 0) { info = Instruction::kIsCallback; + compiler->increaseStackTmpSize(16); } #endif /* SLJIT_32BIT_ARCHITECTURE */ FALLTHROUGH; @@ -1947,6 +1969,7 @@ static void compileFunction(JITCompiler* compiler) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) if (info == 0) { info = Instruction::kIsCallback; + compiler->increaseStackTmpSize(16); } #endif /* SLJIT_32BIT_ARCHITECTURE */ FALLTHROUGH; @@ -1978,6 +2001,7 @@ static void compileFunction(JITCompiler* compiler) ByteCodeOffset4Value* memoryAtomicWait = reinterpret_cast(byteCode); Operand* operands = instr->operands(); instr->setRequiredRegsDescriptor(requiredInit != OTNone ? requiredInit : OTAtomicWaitI32); + compiler->increaseStackTmpSize(16); operands[0] = STACK_OFFSET(memoryAtomicWait->src0Offset()); operands[1] = STACK_OFFSET(memoryAtomicWait->src1Offset()); diff --git a/src/jit/Compiler.h b/src/jit/Compiler.h index 3ec74b74c..2ae8492c0 100644 --- a/src/jit/Compiler.h +++ b/src/jit/Compiler.h @@ -37,6 +37,7 @@ class BrTableInstruction; class Label; class JITModule; struct CompileContext; +struct ExecutionContext; // Defined in ObjectType.h. class FunctionType; @@ -548,6 +549,28 @@ struct TrapJump { sljit_jump* jump; }; +struct MemoryInitArguments { + ExecutionContext* context; + uint32_t segmentIndex; +}; + +struct InitTableArguments { + ExecutionContext* context; + uint32_t tableIndex; + uint32_t segmentIndex; +}; + +struct TableCopyArguments { + ExecutionContext* context; + uint32_t srcIndex; + uint32_t dstIndex; +}; + +struct TableFillArguments { + ExecutionContext* context; + uint32_t tableIndex; +}; + struct CompileContext { CompileContext(Module* module, JITCompiler* compiler); @@ -565,6 +588,7 @@ struct CompileContext { size_t globalsStart; size_t tableStart; size_t functionsStart; + sljit_sw stackTmpStart; size_t nextTryBlock; size_t currentTryBlock; size_t trapBlocksStart; @@ -729,6 +753,13 @@ class JITCompiler { m_branchTableSize += value; } + void increaseStackTmpSize(uint8_t value) + { + if (m_stackTmpSize < value) { + m_stackTmpSize = value; + } + } + void setModuleFunction(ModuleFunction* moduleFunction) { m_moduleFunction = moduleFunction; @@ -797,6 +828,7 @@ class JITCompiler { uint32_t m_options; uint8_t m_savedIntegerRegCount; uint8_t m_savedFloatRegCount; + uint8_t m_stackTmpSize; std::vector m_tryBlocks; std::vector m_functionList; diff --git a/src/jit/FloatConvInl.h b/src/jit/FloatConvInl.h index a54321084..8f4773d76 100644 --- a/src/jit/FloatConvInl.h +++ b/src/jit/FloatConvInl.h @@ -708,10 +708,12 @@ static void emitConvertFloat(sljit_compiler* compiler, Instruction* instr) } #endif /* SLJIT_64BIT_ARCHITECTURE */ + sljit_sw stackTmpStart = CompileContext::get(compiler)->stackTmpStart; + if (arg.arg == SLJIT_MEM1(kFrameReg)) { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, kFrameReg, 0, SLJIT_IMM, arg.argw); } else { - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_get_local_base(compiler, SLJIT_R0, 0, stackTmpStart); } #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE) @@ -737,14 +739,14 @@ static void emitConvertFloat(sljit_compiler* compiler, Instruction* instr) #if (defined SLJIT_64BIT_ARCHITECTURE && SLJIT_64BIT_ARCHITECTURE) sljit_s32 movOp = (flags & DestinationIs64Bit) ? SLJIT_MOV : SLJIT_MOV32; - sljit_emit_op1(compiler, movOp, arg.arg, arg.argw, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1)); + sljit_emit_op1(compiler, movOp, arg.arg, arg.argw, SLJIT_MEM1(SLJIT_SP), stackTmpStart); #else /* !SLJIT_64BIT_ARCHITECTURE */ if (!(flags & DestinationIs64Bit)) { - sljit_emit_op1(compiler, SLJIT_MOV, arg.arg, arg.argw, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1)); + sljit_emit_op1(compiler, SLJIT_MOV, arg.arg, arg.argw, SLJIT_MEM1(SLJIT_SP), stackTmpStart); return; } - sljit_emit_op1(compiler, SLJIT_MOV, argPair.arg1, argPair.arg1w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET); - sljit_emit_op1(compiler, SLJIT_MOV, argPair.arg2, argPair.arg2w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, argPair.arg1, argPair.arg1w, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_LOW_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, argPair.arg2, argPair.arg2w, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_HIGH_OFFSET); #endif /* SLJIT_32BIT_ARCHITECTURE */ } diff --git a/src/jit/IntMath32Inl.h b/src/jit/IntMath32Inl.h index 6b7082604..f35d37397 100644 --- a/src/jit/IntMath32Inl.h +++ b/src/jit/IntMath32Inl.h @@ -502,6 +502,7 @@ static void emitDivRem64(sljit_compiler* compiler, sljit_s32 opcode, JITArgPair* sljit_sw addr; bool isImm = SLJIT_IS_IMM(args[1].arg1); CompileContext* context = CompileContext::get(compiler); + sljit_sw stackTmpStart = context->stackTmpStart; if (isImm) { if ((args[1].arg1w | args[1].arg2w) == 0) { @@ -516,34 +517,34 @@ static void emitDivRem64(sljit_compiler* compiler, sljit_s32 opcode, JITArgPair* } if (args[0].arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET, args[0].arg1, args[0].arg1w); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET, args[0].arg2, args[0].arg2w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_LOW_OFFSET, args[0].arg1, args[0].arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_HIGH_OFFSET, args[0].arg2, args[0].arg2w); } if (args[1].arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_LOW_OFFSET, args[1].arg1, args[1].arg1w); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_HIGH_OFFSET, args[1].arg2, args[1].arg2w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 8 + WORD_LOW_OFFSET, args[1].arg1, args[1].arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 8 + WORD_HIGH_OFFSET, args[1].arg2, args[1].arg2w); } if (args[0].arg1 == SLJIT_MEM1(kFrameReg)) { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, kFrameReg, 0, SLJIT_IMM, args[0].arg1w - WORD_LOW_OFFSET); } else { ASSERT(SLJIT_IS_REG(args[0].arg1) || SLJIT_IS_IMM(args[0].arg1)); - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_get_local_base(compiler, SLJIT_R0, 0, stackTmpStart); } if (args[1].arg1 == SLJIT_MEM1(kFrameReg)) { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kFrameReg, 0, SLJIT_IMM, args[1].arg1w - WORD_LOW_OFFSET); } else { ASSERT(SLJIT_IS_REG(args[1].arg1) || SLJIT_IS_IMM(args[1].arg1)); - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp2)); + sljit_get_local_base(compiler, SLJIT_R1, 0, stackTmpStart + 8); } if (args[2].arg1 == SLJIT_MEM1(kFrameReg)) { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R2, 0, kFrameReg, 0, SLJIT_IMM, args[2].arg1w - WORD_LOW_OFFSET); } else { ASSERT(SLJIT_IS_REG(args[2].arg1)); - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R2, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_get_local_base(compiler, SLJIT_R2, 0, stackTmpStart); } switch (opcode) { @@ -575,8 +576,8 @@ static void emitDivRem64(sljit_compiler* compiler, sljit_s32 opcode, JITArgPair* } if (args[2].arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op1(compiler, SLJIT_MOV, args[2].arg1, args[2].arg1w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET); - sljit_emit_op1(compiler, SLJIT_MOV, args[2].arg2, args[2].arg2w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, args[2].arg1, args[2].arg1w, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_LOW_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, args[2].arg2, args[2].arg2w, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_HIGH_OFFSET); } } diff --git a/src/jit/IntMath64Inl.h b/src/jit/IntMath64Inl.h index 56c1f32c4..f10cae07d 100644 --- a/src/jit/IntMath64Inl.h +++ b/src/jit/IntMath64Inl.h @@ -16,6 +16,8 @@ /* Only included by jit-backend.cc */ +#define WORD_LOW_OFFSET 0 + enum DivRemOptions : sljit_s32 { DivRem32 = 1 << 1, DivRemSigned = 1 << 0, diff --git a/src/jit/MemoryInl.h b/src/jit/MemoryInl.h index 528f23d77..b515067ca 100644 --- a/src/jit/MemoryInl.h +++ b/src/jit/MemoryInl.h @@ -312,8 +312,10 @@ static void emitAtomicLoadStore64(sljit_compiler* compiler, Instruction* instr) } JITArgPair valueArgPair(operands + 1); + sljit_sw stackTmpStart = CompileContext::get(compiler)->stackTmpStart; sljit_s32 type = SLJIT_ARGS2V(P, P); sljit_s32 faddr; + if (instr->opcode() == ByteCode::I64AtomicLoadOpcode) { faddr = GET_FUNC_ADDR(sljit_sw, atomicRmwLoad64); } else { @@ -321,8 +323,8 @@ static void emitAtomicLoadStore64(sljit_compiler* compiler, Instruction* instr) faddr = GET_FUNC_ADDR(sljit_sw, atomicRmwStore64); if (valueArgPair.arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET, valueArgPair.arg1, valueArgPair.arg1w); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET, valueArgPair.arg2, valueArgPair.arg2w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_LOW_OFFSET, valueArgPair.arg1, valueArgPair.arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_HIGH_OFFSET, valueArgPair.arg2, valueArgPair.arg2w); } } @@ -330,15 +332,15 @@ static void emitAtomicLoadStore64(sljit_compiler* compiler, Instruction* instr) sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kFrameReg, 0, SLJIT_IMM, valueArgPair.arg1w - WORD_LOW_OFFSET); } else { ASSERT(SLJIT_IS_REG(valueArgPair.arg1) || SLJIT_IS_IMM(valueArgPair.arg1)); - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_get_local_base(compiler, SLJIT_R1, 0, stackTmpStart); } sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, GET_SOURCE_REG(addr.memArg.arg, instr->requiredReg(0)), 0); sljit_emit_icall(compiler, SLJIT_CALL, type, SLJIT_IMM, faddr); if ((instr->opcode() == ByteCode::I64AtomicLoadOpcode) && (valueArgPair.arg1 != SLJIT_MEM1(kFrameReg))) { - sljit_emit_op1(compiler, SLJIT_MOV, valueArgPair.arg1, valueArgPair.arg1w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET); - sljit_emit_op1(compiler, SLJIT_MOV, valueArgPair.arg2, valueArgPair.arg2w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, valueArgPair.arg1, valueArgPair.arg1w, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_LOW_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, valueArgPair.arg2, valueArgPair.arg2w, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_HIGH_OFFSET); } } #endif /* ENABLE_EXTENDED_FEATURES && SLJIT_32BIT_ARCHITECTURE */ @@ -1076,6 +1078,7 @@ static void emitAtomicRmw64(sljit_compiler* compiler, Instruction* instr) JITArgPair srcArgPair(operands + 1); JITArgPair dstArgPair(operands + 2); + sljit_sw stackTmpStart = CompileContext::get(compiler)->stackTmpStart; sljit_s32 functionAddr; switch (instr->opcode()) { @@ -1110,8 +1113,8 @@ static void emitAtomicRmw64(sljit_compiler* compiler, Instruction* instr) } if (srcArgPair.arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET, srcArgPair.arg1, srcArgPair.arg1w); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET, srcArgPair.arg2, srcArgPair.arg2w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_LOW_OFFSET, srcArgPair.arg1, srcArgPair.arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_HIGH_OFFSET, srcArgPair.arg2, srcArgPair.arg2w); } if (addr.memArg.arg != SLJIT_R0) { @@ -1119,13 +1122,13 @@ static void emitAtomicRmw64(sljit_compiler* compiler, Instruction* instr) } if (srcArgPair.arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_get_local_base(compiler, SLJIT_R1, 0, stackTmpStart); } else { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kFrameReg, 0, SLJIT_IMM, srcArgPair.arg1w - WORD_LOW_OFFSET); } if (dstArgPair.arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R2, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp2)); + sljit_get_local_base(compiler, SLJIT_R2, 0, stackTmpStart + 8); } else { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R2, 0, kFrameReg, 0, SLJIT_IMM, dstArgPair.arg1w - WORD_LOW_OFFSET); } @@ -1133,8 +1136,8 @@ static void emitAtomicRmw64(sljit_compiler* compiler, Instruction* instr) sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS3V(P, P, P), SLJIT_IMM, functionAddr); if (dstArgPair.arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op1(compiler, SLJIT_MOV, dstArgPair.arg1, dstArgPair.arg1w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_LOW_OFFSET); - sljit_emit_op1(compiler, SLJIT_MOV, dstArgPair.arg2, dstArgPair.arg2w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_HIGH_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, dstArgPair.arg1, dstArgPair.arg1w, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 8 + WORD_LOW_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, dstArgPair.arg2, dstArgPair.arg2w, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 8 + WORD_HIGH_OFFSET); } } @@ -1158,6 +1161,7 @@ static void emitAtomicRmwCmpxchg64(sljit_compiler* compiler, Instruction* instr) JITArgPair srcExpectedArgPair(operands + 1); JITArgPair srcValueArgPair(operands + 2); JITArgPair dstArgPair(operands + 3); + sljit_sw stackTmpStart = CompileContext::get(compiler)->stackTmpStart; sljit_s32 type = SLJIT_ARGS3V(P, P, P); sljit_s32 functionAddr = GET_FUNC_ADDR(sljit_sw, atomicRmwCmpxchg64); @@ -1167,13 +1171,13 @@ static void emitAtomicRmwCmpxchg64(sljit_compiler* compiler, Instruction* instr) sljit_emit_op1(compiler, SLJIT_MOV, dstArgPair.arg2, dstArgPair.arg2w, srcExpectedArgPair.arg2, srcExpectedArgPair.arg2w); } } else { - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET, srcExpectedArgPair.arg1, srcExpectedArgPair.arg1w); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET, srcExpectedArgPair.arg2, srcExpectedArgPair.arg2w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_LOW_OFFSET, srcExpectedArgPair.arg1, srcExpectedArgPair.arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_HIGH_OFFSET, srcExpectedArgPair.arg2, srcExpectedArgPair.arg2w); } if (srcValueArgPair.arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_LOW_OFFSET, srcValueArgPair.arg1, srcValueArgPair.arg1w); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_HIGH_OFFSET, srcValueArgPair.arg2, srcValueArgPair.arg2w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 8 + WORD_LOW_OFFSET, srcValueArgPair.arg1, srcValueArgPair.arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 8 + WORD_HIGH_OFFSET, srcValueArgPair.arg2, srcValueArgPair.arg2w); } if (addr.memArg.arg != SLJIT_R0) { @@ -1183,20 +1187,20 @@ static void emitAtomicRmwCmpxchg64(sljit_compiler* compiler, Instruction* instr) if (srcExpectedArgPair.arg1 == SLJIT_MEM1(kFrameReg)) { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, SLJIT_EXTRACT_REG(dstArgPair.arg1), 0, SLJIT_IMM, dstArgPair.arg1w); } else { - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_get_local_base(compiler, SLJIT_R1, 0, stackTmpStart); } - if (srcValueArgPair.arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R2, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp2)); - } else { + if (srcValueArgPair.arg1 == SLJIT_MEM1(kFrameReg)) { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R2, 0, kFrameReg, 0, SLJIT_IMM, srcValueArgPair.arg1w - WORD_LOW_OFFSET); + } else { + sljit_get_local_base(compiler, SLJIT_R2, 0, stackTmpStart + 8); } sljit_emit_icall(compiler, SLJIT_CALL, type, SLJIT_IMM, functionAddr); if (srcExpectedArgPair.arg1 != SLJIT_MEM1(kFrameReg)) { - sljit_emit_op1(compiler, SLJIT_MOV, dstArgPair.arg1, dstArgPair.arg1w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET); - sljit_emit_op1(compiler, SLJIT_MOV, dstArgPair.arg2, dstArgPair.arg2w, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, dstArgPair.arg1, dstArgPair.arg1w, SLJIT_MEM1(kContextReg), stackTmpStart + WORD_LOW_OFFSET); + sljit_emit_op1(compiler, SLJIT_MOV, dstArgPair.arg2, dstArgPair.arg2w, SLJIT_MEM1(kContextReg), stackTmpStart + WORD_HIGH_OFFSET); } } @@ -1543,7 +1547,7 @@ static void emitAtomic(sljit_compiler* compiler, Instruction* instr) #undef OP_XCHG #undef OP_CMPXCHG -static sljit_s32 atomicWaitCallback(ExecutionContext* context, uint8_t* address, sljit_s32 size) +static sljit_s32 atomicWaitCallback(ExecutionContext* context, uint8_t* address, int64_t* args, sljit_s32 size) { Instance* instance = context->instance; @@ -1552,8 +1556,8 @@ static sljit_s32 atomicWaitCallback(ExecutionContext* context, uint8_t* address, } uint32_t result = 0; - int64_t timeout = context->tmp2[0]; - int64_t expect = context->tmp1[0]; + int64_t expect = args[0]; + int64_t timeout = args[1]; if (size == 8) { instance->memory(0)->atomicWait(context->state, instance->module()->store(), address, expect, timeout, &result); @@ -1561,13 +1565,14 @@ static sljit_s32 atomicWaitCallback(ExecutionContext* context, uint8_t* address, instance->memory(0)->atomicWait(context->state, instance->module()->store(), address, (int32_t)expect, timeout, &result); } - context->tmp2[0] = result; + args[0] = result; return ExecutionContext::NoError; } static void emitAtomicWait(sljit_compiler* compiler, Instruction* instr) { CompileContext* context = CompileContext::get(compiler); + sljit_sw stackTmpStart = context->stackTmpStart; sljit_s32 size = (instr->opcode() == ByteCode::MemoryAtomicWait64Opcode ? 8 : 4); ByteCodeOffset4Value* atomicWaitOperation = reinterpret_cast(instr->byteCode()); @@ -1599,40 +1604,40 @@ static void emitAtomicWait(sljit_compiler* compiler, Instruction* instr) #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) if (instr->opcode() == ByteCode::MemoryAtomicWait64Opcode) { - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_LOW_OFFSET, expectedPair.arg1, expectedPair.arg1w); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1) + WORD_HIGH_OFFSET, expectedPair.arg2, expectedPair.arg2w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_LOW_OFFSET, expectedPair.arg1, expectedPair.arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_HIGH_OFFSET, expectedPair.arg2, expectedPair.arg2w); } else { - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), expected.arg, expected.argw); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart, expected.arg, expected.argw); } #else /* !SLJIT_32BIT_ARCHITECTURE */ - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), expected.arg, expected.argw); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart, expected.arg, expected.argw); #endif /* SLJIT_32BIT_ARCHITECTURE */ #if (defined SLJIT_32BIT_ARCHITECTURE && SLJIT_32BIT_ARCHITECTURE) - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_LOW_OFFSET, timeout.arg1, timeout.arg1w); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2) + WORD_HIGH_OFFSET, timeout.arg2, timeout.arg2w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 8 + WORD_LOW_OFFSET, timeout.arg1, timeout.arg1w); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 8 + WORD_HIGH_OFFSET, timeout.arg2, timeout.arg2w); #else /* !SLJIT_32BIT_ARCHITECTURE */ - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2), timeout.arg, timeout.argw); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 8, timeout.arg, timeout.argw); #endif /* SLJIT_32BIT_ARCHITECTURE */ sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, SLJIT_EXTRACT_REG(addr.memArg.arg), 0); sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, kContextReg, 0); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R2, 0, SLJIT_IMM, size); + sljit_get_local_base(compiler, SLJIT_R2, 0, stackTmpStart); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R3, 0, SLJIT_IMM, size); - sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS3(W, P, W, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, atomicWaitCallback)); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, P, W, W, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, atomicWaitCallback)); memoryShared = sljit_emit_cmp(compiler, SLJIT_EQUAL, SLJIT_IMM, ExecutionContext::NoError, SLJIT_R0, 0); context->appendTrapJump(ExecutionContext::ExpectedSharedMemError, sljit_emit_jump(compiler, SLJIT_JUMP)); sljit_set_label(memoryShared, sljit_emit_label(compiler)); - sljit_emit_op1(compiler, SLJIT_MOV, dst.arg, dst.argw, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2)); + sljit_emit_op1(compiler, SLJIT_MOV, dst.arg, dst.argw, SLJIT_MEM1(SLJIT_SP), stackTmpStart + WORD_LOW_OFFSET); } -static sljit_s32 atomicNotifyCallback(ExecutionContext* context, uint8_t* address) +static sljit_s32 atomicNotifyCallback(ExecutionContext* context, uint8_t* address, int32_t count) { Instance* instance = context->instance; uint32_t result = 0; - int32_t count = context->tmp1[0]; instance->memory(0)->atomicNotify(instance->module()->store(), address, count, &result); return result; } @@ -1649,14 +1654,24 @@ static void emitAtomicNotify(sljit_compiler* compiler, Instruction* instr) JITArg count(operands + 1); JITArg dst(operands + 2); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), count.arg, count.argw); + if (count.arg == SLJIT_R1) { + if (SLJIT_EXTRACT_REG(addr.memArg.arg) != SLJIT_R2) { + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R2, 0, count.arg, count.argw); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, SLJIT_EXTRACT_REG(addr.memArg.arg), 0); + } else { + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_TMP_DEST_REG, 0, SLJIT_R1, 0); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, SLJIT_R2, 0); + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R2, 0, SLJIT_TMP_DEST_REG, 0); + } + } else { + MOVE_TO_REG(compiler, SLJIT_MOV, SLJIT_R1, SLJIT_EXTRACT_REG(addr.memArg.arg), 0); + MOVE_TO_REG(compiler, SLJIT_MOV, SLJIT_R2, count.arg, count.argw); + } - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, SLJIT_EXTRACT_REG(addr.memArg.arg), 0); sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, kContextReg, 0); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS3(W, P, W, 32), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, atomicNotifyCallback)); - sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS2(W, P, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, atomicNotifyCallback)); - - sljit_emit_op1(compiler, SLJIT_MOV, dst.arg, dst.argw, SLJIT_R0, 0); + MOVE_FROM_REG(compiler, SLJIT_MOV, dst.arg, dst.argw, SLJIT_R0); } #endif /* ENABLE_EXTENDED_FEATURES */ diff --git a/src/jit/MemoryUtilInl.h b/src/jit/MemoryUtilInl.h index 2dbc587fc..96dde337b 100644 --- a/src/jit/MemoryUtilInl.h +++ b/src/jit/MemoryUtilInl.h @@ -16,10 +16,11 @@ /* Only included by jit-backend.cc */ -static sljit_sw initMemory(uint32_t dstStart, uint32_t srcStart, uint32_t srcSize, ExecutionContext* context) +static sljit_sw initMemory(uint32_t dstStart, uint32_t srcStart, uint32_t srcSize, MemoryInitArguments* args) { + ExecutionContext* context = args->context; Memory* memory = context->instance->memory(0); - DataSegment& sg = context->instance->dataSegment(*(sljit_u32*)&context->tmp1); + DataSegment& sg = context->instance->dataSegment(args->segmentIndex); if (!memory->checkAccess(dstStart, srcSize)) { return ExecutionContext::OutOfBoundsMemAccessError; @@ -91,22 +92,26 @@ static void emitMemory(sljit_compiler* compiler, Instruction* instr) ASSERT(instr->info() & Instruction::kIsCallback); emitInitR0R1R2(compiler, SLJIT_MOV32, params); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R3, 0, kContextReg, 0); sljit_sw addr; if (opcode == ByteCode::MemoryInitOpcode) { MemoryInit* memoryInit = reinterpret_cast(instr->byteCode()); - sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), SLJIT_IMM, memoryInit->segmentIndex()); + sljit_sw stackTmpStart = CompileContext::get(compiler)->stackTmpStart; + sljit_get_local_base(compiler, SLJIT_R3, 0, stackTmpStart); + sljit_emit_op1(compiler, SLJIT_MOV_P, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(MemoryInitArguments, context), kContextReg, 0); + sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(MemoryInitArguments, segmentIndex), SLJIT_IMM, memoryInit->segmentIndex()); addr = GET_FUNC_ADDR(sljit_sw, initMemory); } else if (opcode == ByteCode::MemoryCopyOpcode) { + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R3, 0, kContextReg, 0); addr = GET_FUNC_ADDR(sljit_sw, copyMemory); } else { + sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R3, 0, kContextReg, 0); addr = GET_FUNC_ADDR(sljit_sw, fillMemory); } - sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, 32, 32, 32, W), SLJIT_IMM, addr); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, 32, 32, 32, P), SLJIT_IMM, addr); sljit_jump* cmp = sljit_emit_cmp(compiler, SLJIT_NOT_EQUAL, SLJIT_R0, 0, SLJIT_IMM, ExecutionContext::NoError); context->appendTrapJump(ExecutionContext::GenericTrap, cmp); diff --git a/src/jit/SimdArm32Inl.h b/src/jit/SimdArm32Inl.h index 1feed4a16..7a621b7cd 100644 --- a/src/jit/SimdArm32Inl.h +++ b/src/jit/SimdArm32Inl.h @@ -442,19 +442,21 @@ static void simdEmitF64x2Logical(sljit_compiler* compiler, uint32_t op, sljit_s3 static void simdEmitFloatUnaryOpWithCB(sljit_compiler* compiler, JITArg src, JITArg dst, sljit_sw funcAddr) { + sljit_sw stackTmpStart = CompileContext::get(compiler)->stackTmpStart; + if (src.arg == SLJIT_MEM1(kFrameReg)) { ASSERT((src.argw & (sizeof(void*) - 1)) == 0); sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, kFrameReg, 0, SLJIT_IMM, src.argw); } else if (SLJIT_IS_REG(src.arg)) { - sljit_emit_simd_mov(compiler, SLJIT_SIMD_STORE | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, src.arg, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1)); - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_emit_simd_mov(compiler, SLJIT_SIMD_STORE | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, src.arg, SLJIT_MEM1(SLJIT_SP), stackTmpStart); + sljit_get_local_base(compiler, SLJIT_R0, 0, stackTmpStart); } else { ASSERT(src.arg == SLJIT_MEM0() && (src.argw & (sizeof(void*) - 1)) == 0); sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, SLJIT_IMM, src.argw); } if (SLJIT_IS_REG(dst.arg)) { - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_get_local_base(compiler, SLJIT_R1, 0, stackTmpStart); } else { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kFrameReg, 0, SLJIT_IMM, dst.argw); } @@ -462,7 +464,7 @@ static void simdEmitFloatUnaryOpWithCB(sljit_compiler* compiler, JITArg src, JIT sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS2V(P, P), SLJIT_IMM, funcAddr); if (SLJIT_IS_REG(dst.arg)) { - sljit_emit_simd_mov(compiler, SLJIT_SIMD_LOAD | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, dst.arg, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1)); + sljit_emit_simd_mov(compiler, SLJIT_SIMD_LOAD | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, dst.arg, SLJIT_MEM1(SLJIT_SP), stackTmpStart); } } @@ -995,12 +997,14 @@ static void F32x4Max(float* src0, float* src1, float* dst) static void simdEmitFloatBinaryOpWithCB(sljit_compiler* compiler, JITArg src[2], JITArg dst, sljit_sw funcAddr) { + sljit_sw stackTmpStart = CompileContext::get(compiler)->stackTmpStart; + if (src[0].arg == SLJIT_MEM1(kFrameReg)) { ASSERT((src[0].argw & (sizeof(void*) - 1)) == 0); sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, kFrameReg, 0, SLJIT_IMM, src[0].argw); } else if (SLJIT_IS_REG(src[0].arg)) { - sljit_emit_simd_mov(compiler, SLJIT_SIMD_STORE | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, src[0].arg, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1)); - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R0, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_emit_simd_mov(compiler, SLJIT_SIMD_STORE | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, src[0].arg, SLJIT_MEM1(SLJIT_SP), stackTmpStart); + sljit_get_local_base(compiler, SLJIT_R0, 0, stackTmpStart); } else { ASSERT(src[0].arg == SLJIT_MEM0() && (src[0].argw & (sizeof(void*) - 1)) == 0); sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R0, 0, SLJIT_IMM, src[0].argw); @@ -1010,15 +1014,15 @@ static void simdEmitFloatBinaryOpWithCB(sljit_compiler* compiler, JITArg src[2], ASSERT((src[1].argw & (sizeof(void*) - 1)) == 0); sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kFrameReg, 0, SLJIT_IMM, src[1].argw); } else if (SLJIT_IS_REG(src[1].arg)) { - sljit_emit_simd_mov(compiler, SLJIT_SIMD_STORE | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, src[1].arg, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2)); - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R1, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp2)); + sljit_emit_simd_mov(compiler, SLJIT_SIMD_STORE | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, src[1].arg, SLJIT_MEM1(SLJIT_SP), stackTmpStart + 16); + sljit_get_local_base(compiler, SLJIT_R1, 0, stackTmpStart + 16); } else { ASSERT(src[1].arg == SLJIT_MEM0() && (src[1].argw & (sizeof(void*) - 1)) == 0); sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R1, 0, SLJIT_IMM, src[1].argw); } if (SLJIT_IS_REG(dst.arg)) { - sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R2, 0, kContextReg, 0, SLJIT_IMM, OffsetOfContextField(tmp1)); + sljit_get_local_base(compiler, SLJIT_R2, 0, stackTmpStart); } else { sljit_emit_op2(compiler, SLJIT_ADD, SLJIT_R2, 0, kFrameReg, 0, SLJIT_IMM, dst.argw); } @@ -1026,7 +1030,7 @@ static void simdEmitFloatBinaryOpWithCB(sljit_compiler* compiler, JITArg src[2], sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS3V(P, P, P), SLJIT_IMM, funcAddr); if (SLJIT_IS_REG(dst.arg)) { - sljit_emit_simd_mov(compiler, SLJIT_SIMD_LOAD | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, dst.arg, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1)); + sljit_emit_simd_mov(compiler, SLJIT_SIMD_LOAD | SLJIT_SIMD_REG_128 | SLJIT_SIMD_ELEM_128, dst.arg, SLJIT_MEM1(SLJIT_SP), stackTmpStart); } } diff --git a/src/jit/TableInl.h b/src/jit/TableInl.h index 88d955c82..7a2c009ee 100644 --- a/src/jit/TableInl.h +++ b/src/jit/TableInl.h @@ -16,16 +16,11 @@ /* Only included by Backend.cpp */ -static void emitLoad3Arguments(sljit_compiler* compiler, Operand* params) +static sljit_sw initTable(uint32_t dstStart, uint32_t srcStart, uint32_t srcSize, InitTableArguments* args) { - emitInitR0R1R2(compiler, SLJIT_MOV32, params); - sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R3, 0, kContextReg, 0); -} - -static sljit_sw initTable(uint32_t dstStart, uint32_t srcStart, uint32_t srcSize, ExecutionContext* context) -{ - auto table = context->instance->table(*(sljit_u32*)&context->tmp1); - auto source = (context->instance->elementSegment(*(sljit_u32*)&context->tmp2)); + ExecutionContext* context = args->context; + auto table = context->instance->table(args->tableIndex); + auto source = (context->instance->elementSegment(args->segmentIndex)); if (UNLIKELY((uint64_t)dstStart + (uint64_t)srcSize > (uint64_t)table->size()) || UNLIKELY(!source.element() || (srcStart + srcSize) > source.element()->exprFunctions().size())) { @@ -40,10 +35,11 @@ static sljit_sw initTable(uint32_t dstStart, uint32_t srcStart, uint32_t srcSize return ExecutionContext::NoError; } -static sljit_sw copyTable(uint32_t dstIndex, uint32_t srcIndex, uint32_t n, ExecutionContext* context) +static sljit_sw copyTable(uint32_t dstIndex, uint32_t srcIndex, uint32_t n, TableCopyArguments* args) { - auto srcTable = context->instance->table(context->tmp1[0]); - auto dstTable = context->instance->table(context->tmp2[0]); + ExecutionContext* context = args->context; + auto srcTable = context->instance->table(args->srcIndex); + auto dstTable = context->instance->table(args->dstIndex); if (UNLIKELY(((uint64_t)srcIndex + (uint64_t)n > (uint64_t)srcTable->size()) || ((uint64_t)dstIndex + (uint64_t)n > (uint64_t)dstTable->size()))) { return ExecutionContext::OutOfBoundsTableAccessError; @@ -53,9 +49,10 @@ static sljit_sw copyTable(uint32_t dstIndex, uint32_t srcIndex, uint32_t n, Exec return ExecutionContext::NoError; } -static sljit_sw fillTable(uint32_t index, void* ptr, uint32_t n, ExecutionContext* context) +static sljit_sw fillTable(uint32_t index, void* ptr, uint32_t n, TableFillArguments* args) { - auto srcTable = context->instance->table(context->tmp1[0]); + ExecutionContext* context = args->context; + auto srcTable = context->instance->table(args->tableIndex); if ((uint64_t)index + (uint64_t)n > (uint64_t)srcTable->size()) { return ExecutionContext::OutOfBoundsTableAccessError; @@ -99,15 +96,19 @@ static void emitTable(sljit_compiler* compiler, Instruction* instr) { sljit_sw addr; CompileContext* context = CompileContext::get(compiler); + sljit_sw stackTmpStart = context->stackTmpStart; switch (instr->opcode()) { case ByteCode::TableInitOpcode: { ASSERT(instr->info() & Instruction::kIsCallback); - emitLoad3Arguments(compiler, instr->operands()); + emitInitR0R1R2(compiler, SLJIT_MOV32, instr->operands()); - sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), SLJIT_IMM, (reinterpret_cast(instr->byteCode()))->tableIndex()); - sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2), SLJIT_IMM, (reinterpret_cast(instr->byteCode()))->segmentIndex()); - sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4V(32, 32, 32, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, initTable)); + TableInit* tableInit = reinterpret_cast(instr->byteCode()); + sljit_get_local_base(compiler, SLJIT_R3, 0, stackTmpStart); + sljit_emit_op1(compiler, SLJIT_MOV_P, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(InitTableArguments, context), kContextReg, 0); + sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(InitTableArguments, tableIndex), SLJIT_IMM, tableInit->tableIndex()); + sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(InitTableArguments, segmentIndex), SLJIT_IMM, tableInit->segmentIndex()); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4V(32, 32, 32, P), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, initTable)); sljit_jump* cmp = sljit_emit_cmp(compiler, SLJIT_NOT_EQUAL, SLJIT_R0, 0, SLJIT_IMM, ExecutionContext::NoError); context->appendTrapJump(ExecutionContext::GenericTrap, cmp); @@ -123,11 +124,14 @@ static void emitTable(sljit_compiler* compiler, Instruction* instr) } case ByteCode::TableCopyOpcode: { ASSERT(instr->info() & Instruction::kIsCallback); - emitLoad3Arguments(compiler, instr->operands()); + emitInitR0R1R2(compiler, SLJIT_MOV32, instr->operands()); - sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), SLJIT_IMM, ((reinterpret_cast(instr->byteCode()))->srcIndex())); - sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp2), SLJIT_IMM, ((reinterpret_cast(instr->byteCode()))->dstIndex())); - sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4V(32, 32, 32, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, copyTable)); + TableCopy* tableCopy = reinterpret_cast(instr->byteCode()); + sljit_get_local_base(compiler, SLJIT_R3, 0, stackTmpStart); + sljit_emit_op1(compiler, SLJIT_MOV_P, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(TableCopyArguments, context), kContextReg, 0); + sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(TableCopyArguments, srcIndex), SLJIT_IMM, tableCopy->srcIndex()); + sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(TableCopyArguments, dstIndex), SLJIT_IMM, tableCopy->dstIndex()); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4V(32, 32, 32, P), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, copyTable)); sljit_jump* cmp = sljit_emit_cmp(compiler, SLJIT_NOT_EQUAL, SLJIT_R0, 0, SLJIT_IMM, ExecutionContext::NoError); context->appendTrapJump(ExecutionContext::GenericTrap, cmp); @@ -135,10 +139,12 @@ static void emitTable(sljit_compiler* compiler, Instruction* instr) } case ByteCode::TableFillOpcode: { ASSERT(instr->info() & Instruction::kIsCallback); - emitLoad3Arguments(compiler, instr->operands()); + emitInitR0R1R2(compiler, SLJIT_MOV32, instr->operands()); - sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(kContextReg), OffsetOfContextField(tmp1), SLJIT_IMM, ((reinterpret_cast(instr->byteCode()))->tableIndex())); - sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4V(32, P, 32, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, fillTable)); + sljit_get_local_base(compiler, SLJIT_R3, 0, stackTmpStart); + sljit_emit_op1(compiler, SLJIT_MOV_P, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(TableFillArguments, context), kContextReg, 0); + sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_MEM1(SLJIT_SP), OffsetOfStackTmp(TableFillArguments, tableIndex), SLJIT_IMM, (reinterpret_cast(instr->byteCode()))->tableIndex()); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4V(32, P, 32, P), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, fillTable)); sljit_jump* cmp = sljit_emit_cmp(compiler, SLJIT_NOT_EQUAL, SLJIT_R0, 0, SLJIT_IMM, ExecutionContext::NoError); context->appendTrapJump(ExecutionContext::GenericTrap, cmp); @@ -154,7 +160,7 @@ static void emitTable(sljit_compiler* compiler, Instruction* instr) sljit_emit_op1(compiler, SLJIT_MOV32, SLJIT_R2, 0, SLJIT_IMM, ((reinterpret_cast(instr->byteCode()))->tableIndex())); sljit_emit_op1(compiler, SLJIT_MOV, SLJIT_R3, 0, kContextReg, 0); - sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, P, 32, 32, W), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, growTable)); + sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS4(W, P, 32, 32, P), SLJIT_IMM, GET_FUNC_ADDR(sljit_sw, growTable)); MOVE_FROM_REG(compiler, SLJIT_MOV32, args[2].arg, args[2].argw, SLJIT_R0); break; } diff --git a/src/runtime/JITExec.h b/src/runtime/JITExec.h index 06b112fbd..032440257 100644 --- a/src/runtime/JITExec.h +++ b/src/runtime/JITExec.h @@ -28,16 +28,6 @@ class Memory; class InstanceConstData; struct ExecutionContext { - // Describes a function in the call frame - // chain (stored on the machine stack). - struct CallFrame { - // Previous call frame. - CallFrame* prevFrame; - // Start address of the data used by - // the function (also stored in kFrameReg). - void* frameStart; - }; - enum ErrorCodes : uint32_t { NoError, CapturedException, @@ -64,8 +54,7 @@ struct ExecutionContext { }; ExecutionContext(InstanceConstData* currentInstanceConstData, ExecutionState& state, Instance* instance) - : lastFrame(nullptr) - , currentInstanceConstData(currentInstanceConstData) + : currentInstanceConstData(currentInstanceConstData) , state(state) , instance(instance) , capturedException(nullptr) @@ -73,15 +62,12 @@ struct ExecutionContext { { } - CallFrame* lastFrame; InstanceConstData* currentInstanceConstData; ExecutionState& state; Instance* instance; Exception* capturedException; Memory::TargetBuffer memory0; ErrorCodes error; - uint64_t tmp1[2]; - uint64_t tmp2[2]; }; class JITModule {