diff --git a/src/action/evm_bytecode_visitor.h b/src/action/evm_bytecode_visitor.h index ccc7377e..01324394 100644 --- a/src/action/evm_bytecode_visitor.h +++ b/src/action/evm_bytecode_visitor.h @@ -601,6 +601,7 @@ template class EVMByteCodeVisitor { case OP_MSTORE8: { Builder.noteMemoryOpcodeInBlock(Opcode, PC); + maybePrepareLinearBlockMemoryPrecheck(Opcode); Operand Addr = pop(); Operand Value = pop(); Builder.handleMStore8(Addr, Value); @@ -1337,6 +1338,47 @@ template class EVMByteCodeVisitor { return Plan; } + BlockLinearPrecheckPlan analyzeLinearMstore8DirectMemoryBlockPrecheck( + const Byte *Bytecode, size_t BytecodeSize, uint64_t EntryPC) { + uint64_t ScanPC = 0; + if (!consumeLinearRecurrencePrefix(Bytecode, BytecodeSize, EntryPC, + ScanPC)) { + return {}; + } + + uint64_t CoveredDirectOps = 0; + while (ScanPC < BytecodeSize) { + evmc_opcode Opcode = static_cast(Bytecode[ScanPC]); + if (Opcode == OP_JUMPDEST || isBlockTerminatorOpcode(Opcode)) { + break; + } + + uint64_t MotifPC = ScanPC; + if (!consumeExpectedOpcode(Bytecode, BytecodeSize, MotifPC, OP_DUP1) || + !consumeExpectedOpcode(Bytecode, BytecodeSize, MotifPC, OP_DUP1) || + !consumeExpectedOpcode(Bytecode, BytecodeSize, MotifPC, OP_MSTORE8) || + !consumeExpectedOpcode(Bytecode, BytecodeSize, MotifPC, OP_DUP2) || + !consumeExpectedOpcode(Bytecode, BytecodeSize, MotifPC, OP_ADD)) { + return {}; + } + + ++CoveredDirectOps; + ScanPC = MotifPC; + } + + if (CoveredDirectOps < 2) { + return {}; + } + + BlockLinearPrecheckPlan Plan; + Plan.Eligible = true; + Plan.CoveredOpcode = OP_MSTORE8; + Plan.AccessWidth = 1; + Plan.CoveredDirectOps = CoveredDirectOps; + Plan.StrideStackIndex = 3; + return Plan; + } + BlockLinearPrecheckPlan analyzeLinearDirectMemoryBlockPrecheck( const Byte *Bytecode, size_t BytecodeSize, uint64_t EntryPC) { BlockLinearPrecheckPlan Plan = analyzeLinearMloadDirectMemoryBlockPrecheck( @@ -1344,8 +1386,13 @@ template class EVMByteCodeVisitor { if (Plan.Eligible) { return Plan; } - return analyzeLinearMstoreDirectMemoryBlockPrecheck(Bytecode, BytecodeSize, + Plan = analyzeLinearMstoreDirectMemoryBlockPrecheck(Bytecode, BytecodeSize, EntryPC); + if (Plan.Eligible) { + return Plan; + } + return analyzeLinearMstore8DirectMemoryBlockPrecheck(Bytecode, BytecodeSize, + EntryPC); } void maybePrepareLinearBlockMemoryPrecheck(evmc_opcode Opcode) { diff --git a/src/compiler/evm_frontend/evm_mir_compiler.cpp b/src/compiler/evm_frontend/evm_mir_compiler.cpp index 660b8b87..e80020c5 100644 --- a/src/compiler/evm_frontend/evm_mir_compiler.cpp +++ b/src/compiler/evm_frontend/evm_mir_compiler.cpp @@ -4967,18 +4967,29 @@ void EVMMirBuilder::handleMStore8(Operand AddrComponents, const bool OffsetWasConst = AddrComponents.isConstU64(); const uint64_t OriginalConstOffset = OffsetWasConst ? AddrComponents.getConstValue()[0] : 0; - const bool OffsetKnownU64 = OffsetWasConst; + bool OffsetKnownU64 = OffsetWasConst; const bool CanUseConstBaseDispPath = OffsetWasConst && (ConstAddr = OriginalConstOffset) <= static_cast(INT32_MAX); - normalizeOperandU64(AddrComponents); #ifdef ZEN_ENABLE_EVM_GAS_REGISTER syncGasToMemory(); #endif MType *I64Type = &Ctx.I64Type; - U256Inst AddrParts = extractU256Operand(AddrComponents); - MInstruction *Offset = AddrParts[0]; + const bool CanUseLinearU64AddrFastPath = + CurBlockLinearPrecheckPlan.Active && + CurBlockLinearPrecheckPlan.CoveredDirectOpsRemaining != 0; + MInstruction *Offset = nullptr; + bool UsedLinearPrecheck = false; + if (CanUseLinearU64AddrFastPath) { + Offset = extractKnownU64LowOperand(AddrComponents); + UsedLinearPrecheck = tryConsumeLinearBlockMemoryPrecheck(Offset, nullptr); + OffsetKnownU64 = OffsetKnownU64 || UsedLinearPrecheck; + } + if (!UsedLinearPrecheck) { + normalizeOperandU64(AddrComponents); + Offset = extractKnownU64LowOperand(AddrComponents); + } U256Inst ValueParts = extractU256Operand(ValueComponents); MInstruction *SizeConst = createIntConstInstruction(I64Type, 1); @@ -4993,7 +5004,8 @@ void EVMMirBuilder::handleMStore8(Operand AddrComponents, MInstruction *Overflow = createInstruction( false, CmpInstruction::Predicate::ICMP_ULT, I64Type, RequiredSize, Offset); - bool UsedSharedPrecheck = tryConsumeConstBlockMemoryPrecheck(); + bool UsedSharedPrecheck = + UsedLinearPrecheck || tryConsumeConstBlockMemoryPrecheck(); noteSmallFrameMemoryOp(SmallFrameMemoryOp::MStore8, OffsetWasConst, OriginalConstOffset, OffsetKnownU64, 1, UsedSharedPrecheck); diff --git a/src/tests/evm_jit_frontend_tests.cpp b/src/tests/evm_jit_frontend_tests.cpp index 8e0ecdeb..8d34a55d 100644 --- a/src/tests/evm_jit_frontend_tests.cpp +++ b/src/tests/evm_jit_frontend_tests.cpp @@ -386,8 +386,17 @@ class MockEVMBuilder { void beginMemoryCompileBlock(uint64_t) {} void setMemoryCompileBlockConstPrecheckPlan(uint64_t, uint64_t) {} - void setMemoryCompileBlockLinearPrecheckPlan(uint64_t, uint64_t, bool) {} - void prepareLinearBlockMemoryPrecheck(Operand) {} + void setMemoryCompileBlockLinearPrecheckPlan(uint64_t AccessWidth, + uint64_t CoveredDirectOps, + bool ValueEqualsFirstAddr) { + LinearPrecheckPlanCount++; + LastLinearPrecheckAccessWidth = AccessWidth; + LastLinearPrecheckCoveredDirectOps = CoveredDirectOps; + LastLinearPrecheckValueEqualsFirstAddr = ValueEqualsFirstAddr; + } + void prepareLinearBlockMemoryPrecheck(Operand) { + LinearPrecheckPrepareCount++; + } void noteMemoryOpcodeInBlock(evmc_opcode, uint64_t) {} void noteHelperOpcodeInBlock(evmc_opcode, uint64_t) {} void endMemoryCompileBlock() {} @@ -455,6 +464,24 @@ class MockEVMBuilder { return RuntimeStack.back().resolvedValue(); } + uint32_t linearPrecheckPlanCount() const { return LinearPrecheckPlanCount; } + + uint64_t lastLinearPrecheckAccessWidth() const { + return LastLinearPrecheckAccessWidth; + } + + uint64_t lastLinearPrecheckCoveredDirectOps() const { + return LastLinearPrecheckCoveredDirectOps; + } + + bool lastLinearPrecheckValueEqualsFirstAddr() const { + return LastLinearPrecheckValueEqualsFirstAddr; + } + + uint32_t linearPrecheckPrepareCount() const { + return LinearPrecheckPrepareCount; + } + bool Trapped = false; bool Undefined = false; @@ -478,6 +505,11 @@ class MockEVMBuilder { std::vector RuntimeStack; MockOperand::U256Value LastPushValue = {0, 0, 0, 0}; bool HasLastPushValue = false; + uint32_t LinearPrecheckPlanCount = 0; + uint64_t LastLinearPrecheckAccessWidth = 0; + uint64_t LastLinearPrecheckCoveredDirectOps = 0; + bool LastLinearPrecheckValueEqualsFirstAddr = false; + uint32_t LinearPrecheckPrepareCount = 0; #undef MOCK_OPERAND_STUB #undef MOCK_VOID_STUB @@ -1140,6 +1172,44 @@ TEST(EVMJITFrontendVisitorTest, FusesLinearMStoreNextMotifIntoMeteredRange) { EXPECT_EQ(Builder.topStackValue()[0], 0x60U); } +TEST(EVMJITFrontendVisitorTest, PlansLinearMStore8NextMotifMemoryPrecheck) { + const std::vector Bytecode = { + 0x5f, // PUSH0 calldata offset for stride + 0x35, // CALLDATALOAD + 0x5f, // PUSH0 current + 0x80, // DUP1 + 0x80, // DUP1 + 0x53, // MSTORE8 + 0x81, // DUP2 + 0x01, // ADD + 0x80, // DUP1 + 0x80, // DUP1 + 0x53, // MSTORE8 + 0x81, // DUP2 + 0x01, // ADD + 0x00 // STOP + }; + + COMPILER::EVMFrontendContext Ctx; + Ctx.setRevision(EVMC_CANCUN); + Ctx.setBytecode(reinterpret_cast(Bytecode.data()), + Bytecode.size()); + + MockEVMBuilder Builder; + COMPILER::EVMByteCodeVisitor Visitor(Builder, &Ctx); + EXPECT_TRUE(Visitor.compile()); + EXPECT_FALSE(Builder.Trapped); + EXPECT_FALSE(Builder.Undefined); + + EXPECT_EQ(Builder.linearPrecheckPlanCount(), 1U); + EXPECT_EQ(Builder.lastLinearPrecheckAccessWidth(), 1U); + EXPECT_EQ(Builder.lastLinearPrecheckCoveredDirectOps(), 2U); + EXPECT_FALSE(Builder.lastLinearPrecheckValueEqualsFirstAddr()); + EXPECT_EQ(Builder.linearPrecheckPrepareCount(), 2U); + EXPECT_EQ(Builder.meteredOpcodeCount(OP_MSTORE8), 2U); + EXPECT_EQ(Builder.runtimeStackDepth(), 2U); +} + TEST(EVMJITFrontendVisitorTest, FusesCallerSlotKeccakIntoMeteredRange) { const std::vector Bytecode = { 0x33, // CALLER