worker.mlog.jinja

#%# linked buildings

#% set UART_START_LINK = 16

#% set REGISTERS = 'cell1'
#% set LABELS = 'cell2'

#% set CSRS = 'processor17'
#% set INCR = 'processor18'
#% set CONFIG = 'processor19'
#% set CSR_LABELS = 'processor20'
#% set CONTROLLER = 'processor21'

#% set ERROR_OUTPUT = 'message1'

#%# constants

#% set LN_2 = '0.6931471805599453'

#%# Sv32
#% set PAGESIZE = 2**12
#% set LEVELS = 2
#% set PTESIZE = 4

#%# macros

#% macro start_assert_length(n)
#{- n
#directive start_assert_length {{ n }}
#%- endmacro

reset:
    setrate 1000

    # load config
    read MEMORY_X {{CONFIG}} "MEMORY_X"
    read MEMORY_Y {{CONFIG}} "MEMORY_Y"
    read MEMORY_WIDTH {{CONFIG}} "MEMORY_WIDTH"
    read PROGRAM_ROM_ICACHE_SIZE {{CONFIG}} "PROGRAM_ROM_ICACHE_SIZE"
    read ROM_SIZE {{CONFIG}} "ROM_SIZE"
    read RAM_SIZE {{CONFIG}} "RAM_SIZE"
    read RAM_END {{CONFIG}} "RAM_END"
    read ICACHE_END {{CONFIG}} "ICACHE_END"
    read RAM_ICACHE_OFFSET {{CONFIG}} "RAM_ICACHE_OFFSET"
    read UART_FIFO_MODULO {{CONFIG}} "UART_FIFO_MODULO"

    op idiv RAM_START_PROC ROM_SIZE {{ROM_PROC_BYTES}}

    wait 1e-5

    # @counter is moved here when the next worker starts executing
next_tick:
    # fetch saved variables from the previous worker
    read prev_proc {{CONTROLLER}} "prev_proc"

#directive start_fetch

#% macro fetch_variables()
#% do reset_locals()
    read {{local_variable()}} prev_proc "{{ local_variable(1) }}"
    read {{local_variable()}} prev_proc "{{ local_variable(2) }}"
    read {{local_variable()}} prev_proc "{{ local_variable(3) }}"
    read {{local_variable()}} prev_proc "{{ local_variable(4) }}"
    read {{local_variable()}} prev_proc "{{ local_variable(5) }}"
    read {{local_variable()}} prev_proc "{{ local_variable(6) }}"
    read {{local_variable()}} prev_proc "{{ local_variable(7) }}"
    read {{local_variable()}} prev_proc "{{ local_variable(8) }}"

    read nonlocal1 prev_proc "nonlocal1"
    read nonlocal2 prev_proc "nonlocal2"

    read __etext prev_proc "__etext"
    read _address prev_proc "_address"
    read address prev_proc "address"
    read breakpoint_address prev_proc "breakpoint_address"
    read csr_mcycle prev_proc "csr_mcycle"
    read csr_mie prev_proc "csr_mie"
    read csr_minstret prev_proc "csr_minstret"
    read csr_mip prev_proc "csr_mip"
    read csr_mstatus prev_proc "csr_mstatus"
    read csr_mtime prev_proc "csr_mtime"
    read csr_mtimecmp prev_proc "csr_mtimecmp"
    read csr_mtimecmph prev_proc "csr_mtimecmph"
    read csr_mtimeh prev_proc "csr_mtimeh"
    read csr_stimecmp prev_proc "csr_stimecmp"
    read csr_stimecmph prev_proc "csr_stimecmph"
    read effective_privilege_mode prev_proc "effective_privilege_mode"
    read icache_ram prev_proc "icache_ram"
    read icache_var prev_proc "icache_var"
    read imm prev_proc "imm"
    read length prev_proc "length"
    read mcause prev_proc "mcause"
    read mtval prev_proc "mtval"
    read next_pc prev_proc "next_pc"
    read op_id prev_proc "op_id"
    read pc prev_proc "pc"
    read privilege_mode prev_proc "privilege_mode"
    read ram prev_proc "ram"
    read rd prev_proc "rd"
    read rd_id prev_proc "rd_id"
    read result prev_proc "result"
    read rs1 prev_proc "rs1"
    read rs1_id prev_proc "rs1_id"
    read rs2 prev_proc "rs2"
    read rs2_id prev_proc "rs2_id"
    read state prev_proc "state"
    read uart_flags prev_proc "uart_flags"
    read value prev_proc "value"
    read variable prev_proc "variable"

    read ret prev_proc "ret"
    read ret2 prev_proc "ret2"
    read ret3 prev_proc "ret3"
    read ret4 prev_proc "ret4"
#% endmacro

#{ fetch_variables()

#directive end_fetch

    jump reset notEqual state "running"

    # tell the next worker to fetch the hart state from us
    write @this {{CONTROLLER}} "prev_proc"

    # these variables don't need to be saved because this should only ever run at the start of a tick

    # increment mcycle
    op add csr_mcycle csr_mcycle 1
    jump reset__no_overflow lessThan csr_mcycle 0x100000000

    # overflow mcycle into mcycleh
    # TODO: handle mcycleh overflow
    set csr_mcycle 0
#directive push_saved __mcycleh
    read __mcycleh {{CSRS}} "{{ 'mcycleh'|csr }}"
    op add __mcycleh __mcycleh 1
    write __mcycleh {{CSRS}} "{{ 'mcycleh'|csr }}"
#directive pop_saved __mcycleh
reset__no_overflow:

    # if we just started executing, don't restore @counter
    jump poll_interrupts equal prev_proc {{CONTROLLER}}

#directive push_saved __counter
    # otherwise, jump to whatever instruction the previous worker would have executed next
    read __counter prev_proc "@counter"
    write %next_tick% prev_proc "@counter"
    set @counter __counter
#directive pop_saved __counter

# check if software-manipulable interrupts should become pending
# this runs at the start of the first worker each tick, and at the start of the next worker if certain CSRs/MMRs are modified
poll_interrupts:
#% do reset_locals()
    # poll                   -
    # MEIP                                       -
    # MTIP                                           -
    # STIP                                             -
    op and csr_mip csr_mip 0b01111111111111111111011101011111

    # machine external interrupt (UARTs)
    op shr $uart_interrupts uart_flags 4
    op and $uart_interrupts $uart_interrupts uart_flags
    select $mip.meip notEqual $uart_interrupts 0 0b100000000000 0
    op or csr_mip csr_mip $mip.meip

    # machine timer interrupt
    # fire if mtimeh > mtimecmph || (mtimeh == mtimecmph && mtime >= mtimecmp)
    jump poll_interrupts__no_mti lessThan csr_mtimeh csr_mtimecmph
    select $mip.mtip greaterThan csr_mtimeh csr_mtimecmph 0b10000000 0
    select $mip.mtip greaterThanEq csr_mtime csr_mtimecmp 0b10000000 $mip.mtip
    op or csr_mip csr_mip $mip.mtip
poll_interrupts__no_mti:

    # supervisor timer interrupt
    # fire if mtimeh > stimecmph || (mtimeh == stimecmph && mtime >= stimecmp)
    jump poll_interrupts__no_sti lessThan csr_mtimeh csr_stimecmph
    select $mip.stip greaterThan csr_mtimeh csr_stimecmph 0b100000 0
    select $mip.stip greaterThanEq csr_mtime csr_stimecmp 0b100000 $mip.stip
    op or csr_mip csr_mip $mip.stip
poll_interrupts__no_sti:

# check if interrupts should fire
# this runs at the start of the first worker each tick, if certain CSRs are modified, and after an xRET instruction
fire_interrupts:
#% set trap_locals = ["$$mxdeleg"]
#% do declare_locals(trap_locals)
    op and csr_mip csr_mip 0b10111111111111111111111111111111 # clear interrupt_flags.fire

    # fire interrupt i if bit i is set in both mip and mie
    op and $interrupts csr_mip csr_mie

    # if no interrupts would be fired, short-circuit
    jump main equal $interrupts 0

    # read mideleg for later
    read $$mxdeleg {{CSRS}} "{{ 'mideleg'|csr }}"

    # in U-mode, interrupts are always enabled
    jump fire_interrupts__ok lessThan privilege_mode 0b01

    # in S-mode or M-mode, interrupts are enabled if mstatus.xIE is 1
    # conveniently, SIE is bit 1 and MIE is bit 3
    # FIXME: this is incorrect, we should still take interrupts to M-mode if in S-mode with SIE=0
    op shl $mstatus.xie 1 privilege_mode
    op and $mstatus.xie $mstatus.xie csr_mstatus
    jump main equal $mstatus.xie 0

    # in M-mode, don't take delegated interrupts
    jump fire_interrupts__ok lessThan privilege_mode 0b11

    op not $mask $$mxdeleg
    op and $interrupts $interrupts $mask

fire_interrupts__ok:
    # interrupt priority order: MEI, MSI*, MTI, SEI, SSI, STI, LCOFI*
    # * not implemented

    # we can check consecutive interrupts i and j at the same time if i > j
    # for example, if MEI (11) and MTI (7) are both set, then floor(log_2(0b100010000000)) = 11

    # MEI                       -
    # MTI                           -
    op and mcause $interrupts 0b100010000000
    jump interrupt notEqual mcause 0

    # SEI                         -
    # SSI                                 -
    op and mcause $interrupts 0b001000000010
    jump interrupt notEqual mcause 0

    # STI                             -
    op and mcause $interrupts 0b000000100000
    jump interrupt notEqual mcause 0

main:
    # store the pc for the following instruction in a separate variable, so jumps and traps don't need to account for the pc being incremented at the end of an instruction
    op add next_pc pc 4

    # get the current instruction cache processor and variable
    jump main__read_icache notEqual icache_var null

    # convert virtual pc to physical address
    # we check privilege_mode here because mstatus.MPRV does not affect instruction fetch
    set address pc
    set mcause 1 # instruction access fault
    set mtval pc
    op add ret3 @counter 1
    jump translate_virtual_address lessThan privilege_mode 0b11

    # fall back to slow instruction fetch/decode if not in valid icache range
    # note: MLOGSYS enforces __etext <= ICACHE_SIZE and __etext <= PROGRAM_ROM_SIZE
    jump main__access_rom_icache      lessThan address __etext
    jump main__slow_instruction_fetch lessThan address {{RAM_START}}
    op add _address RAM_ICACHE_OFFSET address
    jump main__access_ram_icache      lessThan address ICACHE_END

main__slow_instruction_fetch:
    # prevent executing from MMIO
    op add ret @counter 1
    select @counter greaterThanEq address RAM_END %trap% %load_word%

    set address null

    # IMPORTANT: remember to update this value if anything in access_icache or read_icache is changed
    op add ret3 @counter {{# start_assert_length(10) }}
    jump decode_value_in_result always

main__access_rom_icache:
    # the icache is stored in RAM cells immediately after the end of RAM with the same density as regular memory
    # if address in ROM: _address = RAM_SIZE + address
    # else:              _address = RAM_SIZE + PROGRAM_ROM_SIZE + address - RAM_START
    #                             = RAM_ICACHE_OFFSET + address
    op add _address RAM_SIZE address
main__access_ram_icache:
    op add ret2 @counter 1
    jump access_ram_raw always
    set icache_ram ram
    set icache_var variable

main__read_icache:
    # load instruction and unpack arguments
    read result icache_ram icache_var
    read icache_var {{INCR}} icache_var
    op and imm result 0xffffffff
    op shr op_id result 47
#directive end_assert_length
    # ret3 jumps here for the slow path
    # we can't jump any later because decode doesn't fully isolate the register fields
    # NOTE: FENCE assumes the icache payload is in result

    op shr rd_id result 32
    op and rd_id rd_id 0b11111

    op shr rs1_id result 37
    op and rs1_id rs1_id 0b11111

    op shr rs2_id result 42
    op and rs2_id rs2_id 0b11111

    # read rs1 and rs2
    # this wastes 1 instruction for I-type and 2 for U-type and J-type, but it saves a huge amount of space
    read rs1 {{REGISTERS}} rs1_id
    read rs2 {{REGISTERS}} rs2_id

    jump main__breakpoint strictEqual pc breakpoint_address
    jump main__no_breakpoint notEqual breakpoint_address "*"
main__breakpoint:
    # the controller will increment @counter and set state to "running" when it's time to unpause
    set state "breakpoint"
    stop
main__no_breakpoint:

#% do reset_locals()
    # jump to instruction handler
    # op_id is in the range [-64, 63]
    # and the instruction handlers in {{LABELS}} start at index 128
    # so we need to add 64 + 128 = 192
    op add $index op_id 192
    read @counter {{LABELS}} $index

# this is a normal length for a label
end_instruction_with_rd_and_poll_interrupts:
    op or csr_mip csr_mip 0b10000000000000000000000000000000 # set interrupt_flags.poll
    # continue into end_instruction_with_rd_and_fire_interrupts

# this is too
end_instruction_with_rd_and_fire_interrupts:
    op or csr_mip csr_mip 0b01000000000000000000000000000000 # set interrupt_flags.fire
    # continue into end_instruction_with_rd

# most instructions with an output register jump here after completing successfully
# writes value rd to register rd_id if rd_id is not x0
end_instruction_with_rd:
    jump end_instruction equal rd_id 0
    write rd {{REGISTERS}} rd_id
    # continue into end_instruction

# all instructions jump here after completing successfully
end_instruction:
#% do reset_locals()
    # increment instret
    op add csr_minstret csr_minstret 1
    jump end_instruction_trap lessThan csr_minstret 0x100000000

    # overflow instret
    set csr_minstret 0
    read $minstreth {{CSRS}} "{{ 'minstreth'|csr }}"
    op add $minstreth $minstreth 1
    write $minstreth {{CSRS}} "{{ 'minstreth'|csr }}"

end_instruction_trap:
    # next_pc is set to pc + 4 at the start of main
    # if we're in the icache, icache_var was already incremented
    # if next_pc was modified, icache_var should have also been set to null
    set pc next_pc

    jump main            lessThan csr_mip 0b01000000000000000000000000000000
    jump fire_interrupts lessThan csr_mip 0b10000000000000000000000000000000
    jump poll_interrupts always

# exceptions

default_xtvec_handler:
    # if we set state before printing, atomicity should be guaranteed
    set state "halt"
    print "Trap taken to {{SYSCON}} (default trap vector), halting."
    printflush {{ERROR_OUTPUT}}
    jump reset always

interrupt:
#% do declare_locals(trap_locals)
    # the highest bit set in mcause is the bit in (mip & mie) that triggered the interrupt
    # so convert mcause to the actual trap code via floor(log_2(mcause)) = ln(mcause) // ln(2)
    op log mcause mcause
    op idiv mcause mcause {{LN_2}}

    # set interrupt bit
    op or mcause mcause 0x80000000

    # continue into trap_without_mtval

trap_without_mtval:
    set mtval 0
    jump trap__check_mxdeleg greaterThanEq mcause 0x80000000
    # continue into trap

trap:
#% do declare_locals(trap_locals)
    # check medeleg
    read $$mxdeleg {{CSRS}} "{{ 'medeleg'|csr }}"

trap__check_mxdeleg:
    # don't trap from M-mode to S-mode
    jump trap__machine greaterThan privilege_mode 0b01

    # if not in M-mode, trap to S-mode if the corresponding mxdeleg bit is set, else trap to M-mode
    # this is safe for interrupt mcause values: https://docs.oracle.com/javase/specs/jls/se17/html/jls-15.html#jls-15.19
    op shr $$mxdeleg $$mxdeleg mcause
    op and $$mxdeleg $$mxdeleg 0b1
    jump trap__machine equal $$mxdeleg 0

#% do reset_locals()
    # set sstatus.SPIE to sstatus.SIE, set sstatus.SIE to 0, and set sstatus.SPP to privilege_mode
    op and $mstatus.xpie csr_mstatus 0b10

    op shl $mstatus.xpp privilege_mode 8

    op and csr_mstatus csr_mstatus 0b11111111111111111111111011011101

    # set scause
    write mcause {{CSRS}} "{{ 'scause'|csr }}"

    # set stval
    write mtval {{CSRS}} "{{ 'stval'|csr }}"

    # set sepc to current pc
    write pc {{CSRS}} "{{ 'sepc'|csr }}"

    # we only support direct mode, so set next_pc to stvec
    read next_pc {{CSRS}} "{{ 'stvec'|csr }}"

    # trap into S-mode
    set privilege_mode 0b01

    jump trap__common always

trap__machine:
#% do reset_locals()
    # set mstatus.MPIE to mstatus.MIE, set mstatus.MIE to 0, and set mstatus.MPP to privilege_mode
    op and $mstatus.xpie csr_mstatus 0b1000

    op shl $mstatus.xpp privilege_mode 11

    op and csr_mstatus csr_mstatus 0b11111111111111111110011101110111

    # set mcause
    write mcause {{CSRS}} "{{ 'mcause'|csr }}"

    # set mtval
    write mtval {{CSRS}} "{{ 'mtval'|csr }}"

    # set mepc to current pc
    write pc {{CSRS}} "{{ 'mepc'|csr }}"

    # we only support direct mode, so set next_pc to mtvec
    read next_pc {{CSRS}} "{{ 'mtvec'|csr }}"

    # trap into M-mode
    set privilege_mode 0b11

trap__common:
    op shl $mstatus.xpie $mstatus.xpie 4
    op or csr_mstatus csr_mstatus $mstatus.xpie
    op or csr_mstatus csr_mstatus $mstatus.xpp

    # check for breakpoints
    jump trap__breakpoint strictEqual pc breakpoint_address
    jump trap__no_breakpoint notEqual breakpoint_address "*"
trap__breakpoint:
    set state "trap_breakpoint"
    stop
trap__no_breakpoint:

    # update effective_privilege_mode and check for unhandled traps
    # tail call into end_instruction_trap so we don't update instret
    select ret equal next_pc {{SYSCON}} %default_xtvec_handler% %end_instruction_trap%
    # continue into get_effective_privilege_mode_and_invalidate_icache

# update effective_privilege_mode
# we don't need to check the current privilege mode here because MRET/SRET clear MPRV if leaving M-mode
# this must stay directly after trap
get_effective_privilege_mode_and_invalidate_icache:
    set icache_var null
get_effective_privilege_mode:
#% do reset_locals()
    op shr $mstatus.mpp csr_mstatus 11
    op and $mstatus.mpp $mstatus.mpp 0b11

    op and $mstatus.mprv csr_mstatus 0b100000000000000000

    # if MPRV=0, return privilege_mode
    # if MPRV=1, return mstatus.MPP
    select effective_privilege_mode equal $mstatus.mprv 0 privilege_mode $mstatus.mpp
    set @counter ret

# helper functions

# apply Sv32 virtual address translation
# jumps to this function must be conditional on privilege_mode or effective_privilege_mode
# address is assumed to already be truncated to 32 bits
# mcause must contain the access fault code for the original access type (ie. 1, 5, or 7)
# mtval must contain the same value as address
# address, mcause, mtval -> address
# ret: ret3
# clobbers: locals, nonlocal2
translate_virtual_address:
#% do declare_locals("$$a")
    # if this function is called, we know that the current effective privilege mode permits address translation
    # so the only thing we need to check here is satp.MODE
    read $satp {{CSRS}} "{{ 'satp'|csr }}"
    jump translate_virtual_address__unchanged lessThan $satp 0x80000000

    # begin virtual address translation algorithm
    # PAGESIZE=2^12, LEVELS=2, PTESIZE=4

    # step 1

    # a = satp.ppn * PAGESIZE
    op and $$a $satp 0x3fffff # satp.ppn
    op mul $$a $$a {{PAGESIZE}}
    # i = LEVELS - 1 = 1
    # nonlocal2 = (i + 1) * 10
    # i is stored in this form because it simplifies step 2 and doesn't really matter for other steps
    # we use nonlocal2 because amo uses nonlocal1
    set nonlocal2 20 # i=1

    # step 2
translate_virtual_address__step_2:

    # pte = load(a + va.vpn[i] * PTESIZE)
    #     = load(a + (va.vpn[i] << 2))
    #     = load(a + (va[y:x] << 2))
    #     = load(a + (va >> (x - 2))[11:2])
    op shr address mtval nonlocal2
    op and address address 0b111111111100
    op add address address $$a

    op add ret @counter 1
    jump load_word always
    # result = pte

    # step 3

    # if pte is not a pointer, we check pte.a here instead of step 7
    # so pte is invalid if v=0, or if r=0 and w=1, or if (r=1 or x=1) and a=0

    # a00xwrv

    # a | x | w | r | v | index | valid?
    # - | - | - | - | - | ----- | --------
    # 0 | 0 | 0 | 0 | 0 | 0     | no (v=0)
    # 0 | 0 | 0 | 0 | 1 | 1     | yes
    # 0 | 0 | 0 | 1 | 0 | 2     | no (v=0)
    # 0 | 0 | 0 | 1 | 1 | 3     | no (a=0)
    # 0 | 0 | 1 | 0 | 0 | 4     | no (v=0)
    # 0 | 0 | 1 | 0 | 1 | 5     | no (a=0)
    # 0 | 0 | 1 | 1 | 0 | 6     | no (v=0)
    # 0 | 0 | 1 | 1 | 1 | 7     | no (a=0)
    # 0 | 1 | 0 | 0 | 0 | 8     | no (v=0)
    # 0 | 1 | 0 | 0 | 1 | 9     | no (a=0)
    # 0 | 1 | 0 | 1 | 0 | 10    | no (v=0)
    # 0 | 1 | 0 | 1 | 1 | 11    | no (a=0)
    # 0 | 1 | 1 | 0 | 0 | 12    | no (v=0)
    # 0 | 1 | 1 | 0 | 1 | 13    | no (a=0)
    # 0 | 1 | 1 | 1 | 0 | 14    | no (v=0)
    # 0 | 1 | 1 | 1 | 1 | 15    | no (a=0)
    # 1 | 0 | 0 | 0 | 0 | 64    | no (v=0)
    # 1 | 0 | 0 | 0 | 1 | 65    | yes
    # 1 | 0 | 0 | 1 | 0 | 66    | no (v=0)
    # 1 | 0 | 0 | 1 | 1 | 67    | yes
    # 1 | 0 | 1 | 0 | 0 | 68    | no (v=0)
    # 1 | 0 | 1 | 0 | 1 | 69    | no (r=0)
    # 1 | 0 | 1 | 1 | 0 | 70    | no (v=0)
    # 1 | 0 | 1 | 1 | 1 | 71    | yes
    # 1 | 1 | 0 | 0 | 0 | 72    | no (v=0)
    # 1 | 1 | 0 | 0 | 1 | 73    | yes
    # 1 | 1 | 0 | 1 | 0 | 74    | no (v=0)
    # 1 | 1 | 0 | 1 | 1 | 75    | yes
    # 1 | 1 | 1 | 0 | 0 | 76    | no (v=0)
    # 1 | 1 | 1 | 0 | 1 | 77    | no (r=0)
    # 1 | 1 | 1 | 1 | 0 | 78    | no (v=0)
    # 1 | 1 | 1 | 1 | 1 | 79    | yes

#% set valid_indices = [1, 65, 67, 71, 73, 75, 79]

#% do reset_locals()
    op and $a_x_w_r_v result 0b1001111
    read $valid "{% for i in range(80) %}{{ (i in valid_indices)|int }}{% endfor %}" $a_x_w_r_v
    jump translate_virtual_address__page_fault notEqual $valid 49 # ascii 1

    # step 4

    # if pte.r=1 or pte.x=1, this is a leaf PTE
    op and $x_r result 0b1010
    jump translate_virtual_address__step_5 notEqual $x_r 0

    # otherwise, if nonlocal2 has been updated, then i-1 < 0, so raise a page fault exception
    jump translate_virtual_address__page_fault notEqual nonlocal2 20

    # otherwise, let a=pte.ppn * PAGESIZE, let i=i-1, and go to step 2
#% do declare_locals("$$a")
    op and $$a result 0xfffffc00
    op shl $$a $$a 2
    set nonlocal2 10 # i=0
    jump translate_virtual_address__step_2 always

    # step 5
translate_virtual_address__step_5:

    # effective_privilege_mode is either S (1) or U (0)

    # SUCCEED IF

    # execute:
    #   x == 1
    #   && (effective_privilege_mode == U ? u == 1 : u == 0)

    # load:
    #   (r == 1 || MXR == 1 && x == 1)
    #   && (effective_privilege_mode == U ? u == 1 : (u == 0 || SUM == 1))

    # store:
    #   w == 1
    #   && (effective_privilege_mode == U ? u == 1 : (u == 0 || SUM == 1))

    # FAIL IF

    # execute:
    #   x == 0
    #   || u == 0 && effective_privilege_mode == 0
    #   || u == 1 && effective_privilege_mode == 1
    # load:
    #   r == 0 && (MXR == 0 || x == 0)
    #   || u == 0 && effective_privilege_mode == 0
    #   || u == 1 && effective_privilege_mode == 1 && SUM == 0
    # store:
    #   w == 0
    #   || u == 0 && effective_privilege_mode == 0
    #   || u == 1 && effective_privilege_mode == 1 && SUM == 0

    # prevent user mode from accessing non-user pages
#% do reset_locals()
    op and $u_priv result 0b10000
    op or $u_priv $u_priv effective_privilege_mode
    jump translate_virtual_address__page_fault equal $u_priv 0

    # prevent supervisor mode from executing user pages
    jump translate_virtual_address__step_5__execute equal mcause 1 # execute

    # prevent supervisor mode from reading/writing user pages if mstatus.SUM is not set
    op and $mstatus.sum csr_mstatus 0b1000000000000000000
    jump translate_virtual_address__step_5__sum notEqual $mstatus.sum 0

translate_virtual_address__step_5__execute:
    jump translate_virtual_address__page_fault equal $u_priv 0b10001
translate_virtual_address__step_5__sum:

    # check if the required rwx bits are set

    # PTE: ...daguxwrv
    # if we mask off g and u, we can use ASCII characters 0b0011___0

    # type    | mcause | MXR | PTE mask || index | char
    # ------- | ------ | --- | -------- || ----- | ----
    # execute | 1      | 0   | 0b1000   || 1     | '8' (0011 1000)
    # execute | 1      | 1   | 0b1000   || 2     | '8' (0011 1000)
    # load    | 5      | 0   | 0b0010   || 5     | '2' (0011 0010)
    # load    | 5      | 1   | 0b1010   || 6     | ':' (0011 1010)
    # store   | 7      | 0   | 0b0100   || 7     | '4' (0011 0100)
    # store   | 7      | 1   | 0b0100   || 8     | '4' (0011 0100)

    # extract fields from PTE
    # this value is also used in step 7
#% do reset_locals()
    # ------- DO NOT RESET LOCALS AFTER THIS LINE -------
    op and $d_x_w_r result 0b10001110

    # extract MXR bit
    op shr $mstatus.mxr csr_mstatus 19
    op and $mstatus.mxr $mstatus.mxr 0b1

    # fetch the bitmask from the string as per the above table
    op add $mask mcause $mstatus.mxr
    read $mask "_88__2:44" $mask

    # fail if NONE of the required bits are set in pte
    # this is fine because none of the things we're checking here require more than one bit to be set
    # in particular, if MXR is 1, loads succeed if r is 1 OR x is 1
    op and $mask $d_x_w_r $mask
    jump translate_virtual_address__page_fault equal $mask 0

    # step 6

    # skip this step if i == 0
    # nonlocal2 == (i + 1) * 10
    jump translate_virtual_address__step_7 equal nonlocal2 10

    # if i > 0 and pte.ppn[i-1:0] != 0, this is a misaligned superpage
    # in Sv32, i is either 1 or 0
    # so we need to check pte.ppn[0]

    op and $pte.ppn[0] result 0xffc00
    jump translate_virtual_address__page_fault notEqual $pte.ppn[0] 0

    # step 7
translate_virtual_address__step_7:

    # we implement Svade for this step because it's simpler
    # if pte.a=0, or if the original memory access is a store and pte.d=0, raise a page fault exception

    # pte.a was checked in step 3

    # fault if storing and pte.d=0
    jump translate_virtual_address__step_8 notEqual mcause 7 # store/AMO
    jump translate_virtual_address__page_fault lessThan $d_x_w_r 0b10000000
    # ------- end critical locals section -------

    # step 8
translate_virtual_address__step_8:

    # move the pte.ppn fields into position
    op shl result result 2

    # raise an access fault exception if the resulting physical address will be wider than 32 bits
    # TODO: is there any case where we *don't* want to do this?
    jump trap greaterThan result 0xffffffff

    # if i>0, this is a superpage
    jump translate_virtual_address__step_8__superpage greaterThan nonlocal2 10

    # pa.pgoff = va.pgoff
    op and address mtval 0xfff

    # pa.ppn = pte.ppn
    op and result result 0xfffff000
    op or address address result

    set @counter ret3

translate_virtual_address__step_8__superpage:
    # pa.pgoff = va.pgoff
    # pa.ppn[0] = va.ppn[0]
    op and address mtval 0x3fffff

    # pa.ppn[1] = pte.ppn[1]
    op and result result 0xffc00000
    op or address address result

translate_virtual_address__unchanged:
    set @counter ret3

translate_virtual_address__page_fault:
    # we need to convert mcause from access fault to page fault
    # instruction: 1 -> 12
    # load:        5 -> 13
    # store/AMO:   7 -> 15
    # for load/store/AMO, page fault = access fault + 8
    # for instruction, 1 + 8 = 9 < 12 < 13 < 15
    # instruction: max(1 + 8, 12) = max(9, 12)  = 12
    # load:        max(5 + 8, 12) = max(13, 12) = 13
    # store/AMO:   max(7 + 8, 12) = max(15, 12) = 15
    op add mcause mcause 8
    op max mcause mcause 12
    jump trap always

# caller must ensure offset is 4-byte aligned and in uart range
# ret: ret2
access_uart:
#% set access_uart_locals = ["$$offset", "$$uart_shift", "$$uart", "$$rx_read", "$$rx_write", "$$tx_read", "$$tx_write", "$$next_tx_write"]
#% do declare_locals(access_uart_locals)
    op sub $$offset $$offset {{UART_RX_OFFSET}}

    op idiv $$uart_shift $$offset {{UART_MMIO_SIZE}}
    op mod $$offset $$offset {{UART_MMIO_SIZE}}

    op add $$offset $$offset {{UART_RX_OFFSET}}

    op add $$uart $$uart_shift {{UART_START_LINK}}
    getlink $$uart $$uart

    read $$rx_read $$uart {{UART_RX_READ}}

    read $$rx_write $$uart {{UART_RX_WRITE}}

    read $$tx_read $$uart {{UART_TX_READ}}

    read $$tx_write $$uart {{UART_TX_WRITE}}

    op add $$next_tx_write $$tx_write 1
    op mod $$next_tx_write $$next_tx_write UART_FIFO_MODULO

    set @counter ret2

# loads the word from memory that contains the specified address
# mcause is required in order to raise the correct exception in case of access fault
# address, mcause, mtval -> result
# ret: ret
# clobbers: locals, ret2, value
load_word:
    jump load_rom_word_unchecked lessThan address ROM_SIZE

load_ram_or_mmio_word:
    jump load_mmio_word greaterThanEq address {{MMIO_START}}

    jump trap lessThan address {{RAM_START}}
    jump trap greaterThanEq address RAM_END
    # continue into load_ram_word_unchecked

load_ram_word_unchecked:
    # locate and read value from ram
    op add ret2 @counter {{# start_assert_length(17) }}
    # continue into access_ram

# helper function to find the ram proc and variable for a given address
# address -> ram, variable
# ret: ret2
access_ram:
#% do reset_locals()
    # _address is named because it's also an input for lookup_variable
    op sub _address address {{RAM_START}}

# _address -> ram, variable
access_ram_raw:
    # we store 4 bytes in each value
    op idiv _address _address 4

    # get the ram proc containing this address
    op idiv $ram_index _address {{RAM_PROC_VARS}}
    op add $ram_index $ram_index RAM_START_PROC

    op mod $ram_x $ram_index MEMORY_WIDTH
    op add $ram_x $ram_x MEMORY_X

    op idiv $ram_y $ram_index MEMORY_WIDTH
    op add $ram_y $ram_y MEMORY_Y

    getblock building ram $ram_x $ram_y

    # get the variable within the ram proc containing this address
    op mod _address _address {{RAM_PROC_VARS}}

# given a value 0 <= _address < RAM_PROC_VARS, resolve that variable in the lookup table
# this must stay directly after access_ram
# _address -> variable
# ret: ret2
# NOTE: modify_csr assumes this function uses one local
lookup_variable:
#% do reset_locals()
    # the lookup is assumed to start at link 0, so we don't need to add an offset here
    op idiv $lookup _address {{LOOKUP_PROC_SIZE}}
    getlink $lookup $lookup

    op mod variable _address {{LOOKUP_PROC_SIZE}}
    lookup block variable variable
    sensor variable variable @name
    read variable $lookup variable

    set @counter ret2

#directive end_assert_length
    read value ram variable
    # null is coerced to 0 by swap_endianness

    # tail call, swap_endianness will jump to the ret value of load_word's caller

# converts a little endian 32-bit number to big endian, or vice versa
# https://stackoverflow.com/a/2182184
# this MUST stay directly after load_word
# value -> result
swap_endianness:
#% do reset_locals()
    # byte 3 -> byte 0
    op shr result value 24
    op and result result 0xff

    # byte 1 -> byte 2
    op shl $tmp value 8
    op and $tmp $tmp 0xff0000
    op or result result $tmp

    # byte 2 -> byte 1
    op shr $tmp value 8
    op and $tmp $tmp 0xff00
    op or result result $tmp

    # byte 0 -> byte 3
    op shl $tmp value 24
    op and $tmp $tmp 0xff000000
    op or result result $tmp

    set @counter ret

# loads a word from ROM
# address -> result
load_rom_word_unchecked:
#% do declare_locals("$$rom", "$$str_index")
    # align to 4 bytes
    op and $address address 0xfffffffc

    # get rom proc data
    op idiv $rom_index $address {{ROM_PROC_BYTES}}
    op mod $$str_index $address {{ROM_PROC_BYTES}}

    op mod $rom_x $rom_index MEMORY_WIDTH
    op add $rom_x $rom_x MEMORY_X

    op idiv $rom_y $rom_index MEMORY_WIDTH
    op add $rom_y $rom_y MEMORY_Y

    getblock building $$rom $rom_x $rom_y
    read $$rom $$rom "v"

#% do declare_locals("$$rom", "$$str_index")
    # read bytes of word
    read $byte0 $$rom $$str_index

    op add $$str_index $$str_index 1
    read $byte1 $$rom $$str_index

    op add $$str_index $$str_index 1
    read $byte2 $$rom $$str_index

    op add $$str_index $$str_index 1
    read $$rom $$rom $$str_index # overwrite rom variable to avoid issues with the vars menu and block data size

    # merge into big endian
    op shl $byte1 $byte1 8
    op shl $byte2 $byte2 16
    op shl $$rom $$rom 24

    op add result $byte0 $byte1
    op add result result $byte2
    op add result result $$rom

    op sub result result {{# ROM_BYTE_OFFSET * (1 + 2**8 + 2**16 + 2**24) }}
    op max result result 0 # if we're reading out of bounds, default to 0

    set @counter ret

# address, mcause, mtval -> result
load_mmio_word:
#% do declare_locals(access_uart_locals)
    # linux syscon driver does a read-modify-write, so the syscon needs to be readable
    set result 0
    jump load_mmio_word__ret equal address {{SYSCON}}

    # the extra 3 at the start is in case we jump here from address translation
    # in that case, address may be up to 34 bits
    op sub $$offset address {{MMIO_START}}
    op and $$offset $$offset 0x3fffffffc
    jump trap greaterThanEq $$offset {{# (UART_RX_OFFSET|int(base=16) + 4 * UART_MMIO_SIZE|int(base=16))|hex }}

    # if we're in UART range, look it up
    # skip privilege check if accessing uart
    op add ret2 @counter {{# start_assert_length(2) }}
    jump access_uart greaterThanEq $$offset {{UART_RX_OFFSET}}
    # otherwise, we're in machine timer range, so trap for PMA if not in M-mode
    # MMIO is not executable, so always use the effective privilege mode here
    jump trap lessThan effective_privilege_mode 0b11
#directive end_assert_length

    # return 0 for write-only registers
    jump load_mmio_word__uart_status equal $$offset {{UART_STATUS_OFFSET}}
    jump load_mmio_word__not_zero lessThanEq $$offset {{UART_RX_OFFSET}}
load_mmio_word__ret:
    set @counter ret
load_mmio_word__not_zero:

    op idiv $$offset $$offset 2
    op add @counter @counter $$offset

    # mtime
    set result csr_mtime
    set @counter ret
    # mtimeh
    set result csr_mtimeh
    set @counter ret
    # mtimecmp
    set result csr_mtimecmp
    set @counter ret
    # mtimecmph
    set result csr_mtimecmph
    set @counter ret
    # UART RX FIFO
#% do declare_locals(access_uart_locals)
#% do free_locals("$$offset")
    # return 0 if the FIFO is empty
    jump load_mmio_word__ret equal $$rx_read $$rx_write

    # HACK: if mcause is 7, we're loading this word as part of a store instruction, so return 0 and don't modify UART state
    jump load_mmio_word__ret equal mcause 7

    # read one value from the FIFO
    read result $$uart $$rx_read
    op and result result 0xffffffff

    # advance read
    op add $$rx_read $$rx_read 1
    op mod $$rx_read $$rx_read UART_FIFO_MODULO
    write $$rx_read $$uart {{UART_RX_READ}}

    set @counter ret

load_mmio_word__uart_status:
#% do declare_locals(access_uart_locals)
#% do free_locals("$$offset", "$$uart")
    # interrupt enabled
    op shr result uart_flags $$uart_shift
    op and result result 0b10000

    # TX FIFO empty
    jump load_mmio_word__uart_status__tx_not_empty notEqual $$tx_read $$tx_write
    op or result result 0b00100
load_mmio_word__uart_status__tx_not_empty:

    # TX FIFO full
    jump load_mmio_word__uart_status__tx_not_full notEqual $$tx_read $$next_tx_write
    op or result result 0b01000
load_mmio_word__uart_status__tx_not_full:

    # RX FIFO valid data
    jump load_mmio_word__uart_status__rx_empty equal $$rx_read $$rx_write
    op or result result 0b00001
load_mmio_word__uart_status__rx_empty:

    # RX FIFO full
    op add $next_rx_write $$rx_write 1
    op mod $next_rx_write $next_rx_write UART_FIFO_MODULO
    jump load_mmio_word__uart_status__rx_not_full notEqual $$rx_read $next_rx_write
    op or result result 0b00010
load_mmio_word__uart_status__rx_not_full:

    # check if interrupt is pending for this port
    op shl $mask 1 $$uart_shift
    op and $mask uart_flags $mask
    jump load_mmio_word__uart_status__no_clear_interrupt equal $mask 0

    # clear pending interrupt
    op not $mask $mask
    op and uart_flags uart_flags $mask

    # set interrupt_flags.poll
    op or csr_mip csr_mip 0b10000000000000000000000000000000
load_mmio_word__uart_status__no_clear_interrupt:

    set @counter ret

# instruction decoder

# same as decode, but the value argument is passed in the result variable as a code size optimization
decode_value_in_result:
    set value result
    # continue into decode

# decoder entry point: decodes a single instruction and writes it to the icache
# the caller MUST set address (skip write) or variable (prevent fast write) to null before the first consecutive call to this function
# value (instruction), address -> op_id, imm, result (icache_payload)
# ret: ret3
decode:
#% set decode_locals = ["$$imm_4_0", "$$funct12", "$$funct7", "$$funct3", "$$opcode_6_2"]
#% do declare_locals(decode_locals)
    # decode and pack registers into the payload
    op and result value 0b1111111111000000000000000 # rs1/rs2

    # the lowest 2 bits are always "11" for non-compressed instructions
    # check this here to increase the odds of an early exit for non-code addresses
    # this needs to be after putting a value in result so we don't potentially corrupt op_id
    op and $tmp value 0b11
    jump decode_illegal_instruction notEqual $tmp 0b11

    # continue packing registers
    op and $instruction_11_7 value 0b0000000000000111110000000

    op shl result result 22 # instruction[24:15] -> icache_payload[46:37]
    op shl $icache_payload_rd $instruction_11_7 25 # instruction[11:7] -> icache_payload[36:32]

    # we put the registers into the result variable here so that we don't have to fully pack the payload if decoding during instruction fetch
    op add result result $icache_payload_rd

    # this is used in both S-type and B-type
    op shr $$imm_4_0 $instruction_11_7 7 # instruction[11:7] -> imm[4:0]

    # decode common fields
    # funct12
    # NOTE: this is unsafe if instruction is longer than 32 bits, but that should never happen. hopefully.
    op shr $$funct12 value 20

    # funct7
    op shr $$funct7 $$funct12 5

    # funct3
    op shr $$funct3 value 12
    op and $$funct3 $$funct3 0b111

    # jump table based on opcode[6:2] since the low 2 bits have already been checked
    # the jump table is stored in LABELS by the controller, starting at index 0 at multiples of 4
    op and $$opcode_6_2 value 0b1111100
    read @counter {{LABELS}} $$opcode_6_2

# U-type: imm, rd_id
decode_LUI:
decode_AUIPC:
#% do declare_locals(decode_locals)
    # -53: LUI
    # -8: AUIPC
    select op_id equal $$opcode_6_2 0b0110100 -53 -8

    # imm
    op and imm value 0b11111111111111111111000000000000
    jump end_decode always

# J-type: imm, rd_id
decode_JAL:
#% do declare_locals(decode_locals)
#% do free_locals("$$opcode_6_2")
    # imm
    op shr $imm_20 value 11 # instruction[31] -> imm[20]
    # instruction: instruction[19:12] -> imm[19:12]
    op shr $imm_11 value 9 # instruction[20] -> imm[11]
    # funct12: instruction[30:21] -> imm[10:1]

    op and $imm_20 $imm_20     0b100000000000000000000
    op and $imm_19_12 value    0b011111111000000000000
    op and $imm_11 $imm_11     0b000000000100000000000
    op and $imm_10_1 $$funct12 0b000000000011111111110

    op add imm $imm_20 $imm_19_12
    op add imm imm $imm_11
    op add imm imm $imm_10_1

    # extend sign
    op xor imm imm 0b100000000000000000000
    op sub imm imm 0b100000000000000000000

    set op_id -62 # JAL
    jump end_decode always

decode_JALR:
    set op_id -61 # JALR
    jump decode_I-type always

# B-type: rs1, rs2, imm
decode_BRANCH:
#% do declare_locals(decode_locals)
#% do free_locals("$$opcode_6_2")
    jump decode_illegal_instruction equal $$funct3 2
    jump decode_illegal_instruction equal $$funct3 3

    # imm
    op shr $imm_12 value 19 # instruction[31] -> imm[12]
    op shl $imm_11 value 4 # instruction[7] -> imm[11]
    # funct12: instruction[30:25] -> imm[10:5]
    # imm_4_0/rd: instruction[11:8] -> imm[4:1]

    op and $imm_12 $imm_12     0b1000000000000
    op and $imm_11 $imm_11     0b0100000000000
    op and $imm_10_5 $$funct12 0b0011111100000
    op and $imm_4_1 $$imm_4_0  0b0000000011110

    op add imm $imm_12 $imm_11
    op add imm imm $imm_10_5
    op add imm imm $imm_4_1

    # extend sign
    op xor imm imm 0b1000000000000
    op sub imm imm 0b1000000000000

    # -64: BEQ
    # -63: BNE
    # -62: (JAL)
    # -61: (JALR)
    # -60: BLT
    # -59: BGE
    # -58: BLTU
    # -57: BGEU
    op add op_id -64 $$funct3
    jump end_decode always

decode_LOAD:
#% do declare_locals(decode_locals)
    jump decode_illegal_instruction equal $$funct3 3

    # -56: LB
    # -55: LH
    # -54: LW
    # -53: (LUI)
    # -52: LBU
    # -51: LHU
    op add op_id -56 $$funct3
    select @counter lessThan $$funct3 6 %decode_I-type% %decode_illegal_instruction%

# S-type: rs1, rs2, imm
decode_STORE:
#% do declare_locals(decode_locals)
    # imm
    op shl $imm_11_5 $$funct7 5 # instruction[31:25] -> imm[11:5]
    # imm_4_0: instruction[11:7] -> imm[4:0]

    op add imm $imm_11_5 $$imm_4_0

    # extend sign
    op xor imm imm 0b100000000000
    op sub imm imm 0b100000000000

    # -50: SB
    # -49: SH
    # -48: SW
    op add op_id -50 $$funct3
    select @counter lessThan $$funct3 3 %end_decode% %decode_illegal_instruction%

# R-type: rs1, rs2, rd_id
decode_AMO:
#% do declare_locals(decode_locals)
    jump decode_illegal_instruction notEqual $$funct3 0b010

    # funct5
    # mlogv32 is single-hart, so just ignore aq and rl
    op shr $funct5 $$funct7 2

    # low bits
    # -47: AMOADD.W
    # -46: AMOSWAP.W
    # -45: LR.W
    # -44: SC.W
    op add op_id -47 $funct5

    # funct5 either has bits in [4:2] or [1:0] set, never both
    # (ab)use that so we don't need 32 lookup table entries
    jump end_decode lessThanEq $funct5 0b11

    # if funct5 > 0b11, the low two bits must be clear
    op and $low $funct5 0b11

    # high bits
    # -43: AMOXOR.W
    # -42: AMOOR.W
    # -41: AMOAND.W
    # -40: AMOMIN.W
    # -39: AMOMAX.W
    # -38: AMOMINU.W
    # -37: AMOMAXU.W
    op shr $funct5 $funct5 2
    op add op_id -44 $funct5 # funct5 >> 2 starts at 1, not 0

    select @counter equal $low 0b00 %end_decode% %decode_illegal_instruction%

decode_OP-IMM:
#% do declare_locals(decode_locals)
    jump decode_SRLI_SRAI equal $$funct3 5
    jump decode_SLLI equal $$funct3 1

    # -36: ADDI
    # -35: (SUB)
    # -34: SLTI
    # -33: SLTIU
    # -32: XORI
    # -31: (SRA)
    # -30: ORI
    # -29: ANDI
    op add op_id -36 $$funct3
    # continue into decode_I-type

# common decoder for I-type instructions
# this should be called AFTER setting op_id
# I-type: rs1, imm, rd_id
decode_I-type:
#% do declare_locals(decode_locals)
    # imm is just funct12 but sign-extended
    op xor $$funct12 $$funct12 0b100000000000
    op sub $$funct12 $$funct12 0b100000000000
    # continue into decode_I-type_no_extend_sign

# common decoder for I-type instructions without a sign-extended immediate
# I-type: rs1, imm, rd_id
decode_I-type_no_extend_sign:
#% do declare_locals(decode_locals)
    # imm
    # NOTE: SRAI and decode_I-type both mutate funct12
    set imm $$funct12
    jump end_decode always

decode_SLLI:
#% do declare_locals(decode_locals)
    # shamt is encoded in the low 5 bits of imm, so sign extension is redundant
    set op_id -3 # SLLI
    select @counter equal $$funct7 0b0000000 %decode_I-type_no_extend_sign% %decode_illegal_instruction%

decode_SRLI_SRAI:
#% do declare_locals(decode_locals)
    # SRLI
    set op_id -2 # SRLI
    jump decode_I-type_no_extend_sign equal $$funct7 0b0000000

    # SRAI
    op and $$funct12 $$funct12 0b11111 # HACK: zero out the high bits of funct12 so funct7 isn't included in imm
    set op_id -1 # SRAI
    select @counter equal $$funct7 0b0100000 %decode_I-type_no_extend_sign% %decode_illegal_instruction%

# R-type: rs1, rs2, rd_id
decode_OP:
#% do declare_locals(decode_locals)
    # -28: ADD
    # -27: SLL
    # -26: SLT
    # -25: SLTU
    # -24: XOR
    # -23: SRL
    # -22: OR
    # -21: AND
    op add op_id -28 $$funct3
    jump end_decode equal $$funct7 0b0000000

    # -20: MUL
    # -19: MULH
    # -18: MULHSU
    # -17: MULHU
    # -16: DIV
    # -15: DIVU
    # -14: REM
    # -13: REMU
    op add op_id -20 $$funct3
    jump end_decode equal $$funct7 0b0000001

    jump decode_illegal_instruction notEqual $$funct7 0b0100000

    set op_id -35 # SUB
    jump end_decode equal $$funct3 0b000

    set op_id -31 # SRA
    select @counter equal $$funct3 0b101 %end_decode% %decode_illegal_instruction%

decode_MISC-MEM:
#% do declare_locals(decode_locals)
    # NOTE: we put FENCE at -4 because -3 to -1 are placed manually, not by a funct3 addition
    # so we can move them to different op_ids if we want to add a separate handler for FENCE.I
    set op_id -4 # FENCE/FENCE.I

    # FENCE=0, FENCE.I=1
    select @counter lessThan $$funct3 2 %decode_I-type_no_extend_sign% %decode_illegal_instruction%

decode_SYSTEM:
#% do declare_locals(decode_locals)
    # -12: PRIV
    # -11: CSRRW
    # -10: CSRRS
    # -9: CSRRC
    # -8: (AUIPC)
    # -7: CSRRWI
    # -6: CSRRSI
    # -5: CSRRCI
    op add op_id -12 $$funct3
    select @counter notEqual $$funct3 4 %decode_I-type_no_extend_sign% %decode_illegal_instruction%

# Xmlogsys
decode_custom-0:
#% do declare_locals(decode_locals)
    # 1: MLOGSYS
    op add op_id 1 $$funct3
    jump decode_I-type_no_extend_sign lessThan $$funct3 1
    # continue into decode_illegal_instruction if funct3 >= 1

# must stay directly before end_decode
decode_illegal_instruction:
    # use 0 as the illegal instruction id because null coerces to 0
    set op_id 0
    # continue into end_decode

# all decoders jump here after completing successfully
end_decode:
#% do declare_locals(decode_locals)
    # do this here because several decoders need to do it for sign extension
    # and to make sure the immediate doesn't corrupt the registers
    op and imm imm 0xffffffff

    # this branch is taken if decoding during instruction fetch
    # using select here doesn't save on code size, but it speeds up the slow fetch a bit
    select @counter strictEqual address null ret3 @counter

    # get the current instruction cache processor and variable
    jump end_decode__decode_var_valid notEqual variable null

    select $offset lessThan address {{RAM_START}} RAM_SIZE RAM_ICACHE_OFFSET
    op add _address $offset address

    op add ret2 @counter 1
    jump access_ram_raw always

end_decode__decode_var_valid:
    # pack op_id, registers (already in result), and imm into a single double
    # this is skipped if decoding for instruction fetch because op_id and imm are already in variables
    # IMPORTANT: use or instead of add because op_id may be negative
    op shl $icache_payload_op_id op_id 47
    op or result result $icache_payload_op_id
    op or result result imm

    # write to the icache
    write result ram variable

    set @counter ret3

# instruction handlers

LUI:
    # U-type: imm, rd_id
    set rd imm
    jump end_instruction_with_rd always

AUIPC:
    # U-type: imm, rd_id
    op add rd imm pc
    jump end_instruction_with_truncated_rd always

JALR:
    # I-type: rs1, imm, rd_id
    op add next_pc rs1 imm
    op and next_pc next_pc 0xfffffffe # clear LSB
    jump unconditional_jump always

BEQ:
    # B-type: rs1, rs2, imm
    select @counter equal rs1 rs2 %conditional_branch% %end_instruction%

BNE:
    # B-type: rs1, rs2, imm
    select @counter notEqual rs1 rs2 %conditional_branch% %end_instruction%

BLT:
    # B-type: rs1, rs2, imm
    op add ret @counter {{# start_assert_length(5) }}
    # implicit jump to unsigned_to_signed_rs1_rs2

# converts a pair of two's complement unsigned values to signed in-place
# this converts two values at once because most usages of it need to convert two values
# also, the values are assumed to be in rs1 and rs2
# rs1, rs2 -> rs1, rs2
unsigned_to_signed_rs1_rs2:
    # value < 2^31 ? value : value - 2^32
    jump unsigned_to_signed__done_1 lessThan rs1 0x80000000 # 2^31
    op sub rs1 rs1 0x100000000 # 2^32
unsigned_to_signed__done_1:

    jump unsigned_to_signed__done_2 lessThan rs2 0x80000000
    op sub rs2 rs2 0x100000000
unsigned_to_signed__done_2:

    set @counter ret

#directive end_assert_length
    # continue into BLTU

BLTU:
    # B-type: rs1, rs2, imm
    select @counter lessThan rs1 rs2 %conditional_branch% %end_instruction%

BGE:
    # B-type: rs1, rs2, imm
    op add ret @counter 1
    jump unsigned_to_signed_rs1_rs2 always
    # continue into BGEU

BGEU:
    # B-type: rs1, rs2, imm
    select @counter greaterThanEq rs1 rs2 %conditional_branch% %end_instruction%

conditional_branch:
    set rd_id 0 # prevent end_instruction_with_rd from writing to a register
    # continue into JAL

JAL:
    # J-type: imm, rd_id
    op add next_pc pc imm
    # continue into unconditional_jump

unconditional_jump:
#% do reset_locals()
    op and next_pc next_pc 0xffffffff

    op mod $remainder next_pc 4
    set mcause 0 # instruction address misaligned
    set mtval next_pc
    jump trap notEqual $remainder 0

    # write pc+4 to rd
    op add rd pc 4

    # if we're in an icache-covered page, and the target address is within the same page
    # then just do a variable lookup instead of forcing a full cache miss

    # we assume __etext, PROGRAM_ROM_SIZE, and ICACHE_END are all page-aligned
    # so this should be adequate to ensure we're in the same page and both addresses are cached
    jump end_instruction_with_rd equal icache_var null

    op and $pc_page pc 0xfffff000
    op and $next_pc_page next_pc 0xfffff000
    jump unconditional_jump__no_icache_lookup notEqual $pc_page $next_pc_page

    # now we know we're in the same page, but we need to figure out which of the four pages within the RAM proc it is

    read _address icache_var 0
    read _address "{{ VARIABLE_0_TO_PAGE_OFFSET }}" _address
    op and _address _address 0b11
    op mul _address _address 4096

    # and finally we can add the page offset and look up the variable

    op and $pgoff next_pc 0xfff
    op add _address _address $pgoff
    op idiv _address _address 4

    op add ret2 @counter 1
    jump lookup_variable always

unconditional_jump__no_icache_lookup:
    select icache_var notEqual $pc_page $next_pc_page null variable
    jump end_instruction_with_rd always

LB:
    # I-type: rs1, imm, rd_id
    set length 8
    set nonlocal1 true
    jump load_value always

LH:
    # I-type: rs1, imm, rd_id
    set length 16
    set nonlocal1 true
    jump load_value always

LW:
    # I-type: rs1, imm, rd_id
    set length 32
    jump load_value_unsigned always

LBU:
    # I-type: rs1, imm, rd_id
    set length 8
    jump load_value_unsigned always

LHU:
    # I-type: rs1, imm, rd_id
    set length 16
    # continue into load_value_unsigned

load_value_unsigned:
    set nonlocal1 false
    # continue into load_value

# helper for LOAD instructions to load a value with a specified length in bits (8, 16, or 32) from memory
# raises a misaligned-load exception if the address is misaligned for the specified length
# rs1 (address), imm (offset), rd_id, length, nonlocal1 (is_signed) ->
load_value:
#% do reset_locals()
    set mcause 4 # load address misaligned
    op add ret3 @counter {{# start_assert_length(8) }}

setup_load_and_store_value:
    op add address rs1 imm
    op mod address address 0x100000000 # 2^32 (trunc 32)

    # check alignment
    op idiv $bytes length 8
    op mod $alignment address $bytes
    set mtval address
    jump trap notEqual $alignment 0

    # translate to physical address (tail call)
    op add mcause mcause 1 # access fault
    select @counter lessThan effective_privilege_mode 0b11 %translate_virtual_address% ret3

#directive end_assert_length
    # load the word containing the address
    op add ret @counter 1
    jump load_word always

#% do reset_locals()

    # right shift by (address % 4) bits
    op mod $shift address 4
    op mul $shift $shift 8
    op shr rd result $shift

    # truncate to the specified number of bits
    op pow $divisor 2 length
    op mod rd rd $divisor

    # extend the sign of the value if it's signed
    select @counter equal nonlocal1 true %extend_sign_rd% %end_instruction_with_rd%

SB:
    # S-type: rs1, rs2, imm
    set length 8
    jump store_value always

SH:
    # S-type: rs1, rs2, imm
    set length 16
    jump store_value always

SW:
    # S-type: rs1, rs2, imm
    set length 32
    # continue into store_value

# helper for STORE instructions to store a value with a specified length in bits (8, 16, or 32) to memory
# raises a misaligned-store exception if the address is misaligned for the specified length
# TODO: the 32-bit case could be optimized a lot
# rs1 (address), rs2 (value), imm (offset), length ->
store_value:
#% do reset_locals()
    # we also use parts of this procedure for atomic operations, so it jumps to end_instruction_with_rd at the end
    # so if we jump to store_value, set rd_id to 0 so we don't write garbage to a random register
    set rd_id 0

    set mcause 6 # store/AMO address misaligned
    op add ret3 @counter 1
    jump setup_load_and_store_value always

    # handle write-only MMIO
    jump store_value__syscon equal address {{SYSCON}}

    # fast path for 32-bit writes
    set value rs2
    op add ret2 @counter {{# start_assert_length(13) }}
    jump access_ram equal length 32

    # create a bitmask with 0 in the section we want to replace and 1 everywhere else
    op mod $shift address 4
    op mul $shift $shift 8

    op pow $length_squared 2 length
    op sub nonlocal1 $length_squared 1 # nonlocal1 = bitmask
    op shl nonlocal1 nonlocal1 $shift
    op sub nonlocal1 0xffffffff nonlocal1

    # take the low n bits of the value to store, and shift it to line up with the zeros in the mask
    op mod nonlocal2 rs2 $length_squared
    op shl nonlocal2 nonlocal2 $shift

    # load the word containing the address
    op add ret @counter 1
    jump load_ram_or_mmio_word always

#% do reset_locals()

    # insert the value into the word
    op and result result nonlocal1 # bitmask
    op add value result nonlocal2 # inserted value

#directive end_assert_length
    # align address to word
    # NOTE: we jump to this line from access_ram if length == 32
    op and address address 0xfffffffc

    # if it's in the MMIO range, handle it separately
    jump store_mmio_word_unchecked greaterThanEq address {{MMIO_START}}

store_ram_word_unchecked:
    # otherwise, store the new value
    op add ret @counter 1
    jump swap_endianness always

    write result ram variable

    # if in icache range, decode the new value and update the icache to allow executing from RAM and self-modifying code
    # TODO: add a way for the CPU to temporarily disable this for performance?
    # or make use of FENCE.I
    set variable null
    set ret3 %end_instruction_with_rd%
    select @counter lessThan address ICACHE_END %decode% ret3

    # atomics are not supported for MMIO, so we don't need to write to rd here
    # however, rd_id is set regardless, so we can safely jump to end_instruction_with_rd_and_time_update
store_mmio_word_unchecked:
#% do declare_locals(access_uart_locals)
    op sub $$offset address {{MMIO_START}}

    # if we're in UART range, look it up
    op add ret2 @counter 1
    jump access_uart greaterThanEq $$offset {{UART_RX_OFFSET}}

    # ignore writes to read-only registers
    jump store_mmio_word_unchecked__uart_tx equal $$offset {{UART_TX_OFFSET}}
    jump store_mmio_word_unchecked__uart_control equal $$offset {{UART_CONTROL_OFFSET}}
    jump end_instruction greaterThanEq $$offset {{UART_RX_OFFSET}}

    # NOTE: if we get to this point, the MMIO address must be valid, so we don't need bounds or privilege checks
    op idiv $$offset $$offset 2
    op add @counter @counter $$offset

    # mtime
    set csr_mtime value
    jump end_instruction_with_rd_and_poll_interrupts always
    # mtimeh
    set csr_mtimeh value
    jump end_instruction_with_rd_and_poll_interrupts always
    # mtimecmp
    set csr_mtimecmp value
    jump end_instruction_with_rd_and_poll_interrupts always
    # mtimecmph
    set csr_mtimecmph value
    jump end_instruction_with_rd_and_poll_interrupts always

store_mmio_word_unchecked__uart_tx:
#% do declare_locals(access_uart_locals)
#% do free_locals("$$offset")
    # append value to queue
    # NOTE: we don't check for overflow here, see readme
    op add $tmp $$tx_write {{UART_TX_START}}
    write value $$uart $tmp

    # advance write
    write $$next_tx_write $$uart {{UART_TX_WRITE}}

    jump end_instruction always

store_mmio_word_unchecked__uart_control:
#% do declare_locals(access_uart_locals)
#% do free_locals("$$offset")
    # enable/disable interrupts
    op shl $tmp 0b10000 $$uart_shift
    op not $tmp $tmp
    op and uart_flags uart_flags $tmp

    op and $tmp value 0b10000
    op shl $tmp $tmp $$uart_shift
    op or uart_flags uart_flags $tmp

    # reset RX FIFO
    op and $tmp value 0b10
    jump store_mmio_word_unchecked__uart_control__no_reset_rx equal $tmp 0
    write $$rx_write $$uart {{UART_RX_READ}}
store_mmio_word_unchecked__uart_control__no_reset_rx:

    # reset TX FIFO
    op and $tmp value 0b01
    jump store_mmio_word_unchecked__uart_control__no_reset_tx equal $tmp 0
    write $$tx_read $$uart {{UART_TX_WRITE}}
store_mmio_word_unchecked__uart_control__no_reset_tx:

    jump end_instruction always

store_value__syscon:
    jump end_instruction greaterThan rs2 2
    # halt: 0 or 2 (for linux syscon)
    # reset: 1
    select state equal rs2 1 "reset" "halt"
    jump reset always

LR.W:
    # R-type: rs1, rs2=0, rd_id

    # check address and alignment
    set mcause 4 # load address misaligned
    op add ret4 @counter {{# start_assert_length(9) }}
    # implicit jump to atomic_validate_address

    # ret: ret4
atomic_validate_address:
#% do reset_locals()
    set address rs1
    set mtval address

    op mod $alignment address 4
    jump trap notEqual $alignment 0

    # translate to physical address
    op add mcause mcause 1 # load/store/AMO access fault
    op add ret3 @counter 1
    jump translate_virtual_address lessThan effective_privilege_mode 0b11

    jump trap lessThan address {{RAM_START}}
    select @counter greaterThanEq address RAM_END %trap% ret4
#directive end_assert_length

    # load value
    op add ret @counter 1
    jump load_ram_word_unchecked always
    set rd result

    # register reservation set for physical address
    # TODO: implement Za64rs?
    write address {{CONTROLLER}} "reservation_set"

    jump end_instruction_with_rd always

SC.W:
    # R-type: rs1, rs2, rd_id
#% do reset_locals()

    # check address and alignment
    set mcause 6 # store/AMO address misaligned
    op add ret4 @counter 1
    jump atomic_validate_address always

    # check if reservation is valid and contains the physical address corresponding to rs1
    read $reservation_set {{CONTROLLER}} "reservation_set"
    op strictEqual $success address $reservation_set

    # invalidate reservation
    write null {{CONTROLLER}} "reservation_set"

    # write 0 to rd if successful, else write 1
    op notEqual rd $success true

    # conditionally write value
    # if success is true, we jump to access_ram and then tail call into store_ram_word_unchecked
    # otherwise, we jump directly to end_instruction_with_rd
    set value rs2
    set ret2 %store_ram_word_unchecked%
    select @counter equal $success true %access_ram% %end_instruction_with_rd%

AMOADD.W:
AMOSWAP.W:
AMOXOR.W:
AMOOR.W:
AMOAND.W:
AMOMIN.W:
AMOMINU.W:
AMOMAX.W:
AMOMAXU.W:
    # R-type: rs1, rs2, rd_id

    # instruction | op_id | amo_op
    # AMOADD.W    | -47   | 0
    # AMOSWAP.W   | -46   | 2
    # AMOXOR.W    | -43   | 4
    # AMOOR.W     | -42   | 6
    # AMOAND.W    | -41   | 8
    # AMOMIN.W    | -40   | 10
    # AMOMAX.W    | -39   | 14
    # AMOMINU.W   | -38   | 12
    # AMOMAXU.W   | -37   | 16

    # nonlocal1 = amo_op
    op add nonlocal1 op_id 47
    read nonlocal1 "@B__DFHJNLP" nonlocal1
    op sub nonlocal1 nonlocal1 64

    # check address and alignment
    set mcause 6 # store/AMO address misaligned
    op add ret4 @counter 1
    jump atomic_validate_address always

    # read
    op add ret @counter 1
    jump load_ram_word_unchecked always

    # put original value in rd
    set rd result

    # modify
    set rs1 result # hack
    op add @counter @counter nonlocal1

    # add
    op add result result rs2
    jump amo__done_modify always
    # swap
    set result rs2
    jump amo__done_modify always
    # xor
    op xor result result rs2
    jump amo__done_modify always
    # or
    op or result result rs2
    jump amo__done_modify always
    # and
    op and result result rs2
    jump amo__done_modify always
    # min
    op add ret @counter 1
    jump unsigned_to_signed_rs1_rs2 always
    # continue into minu
    # minu
    op min result rs1 rs2
    jump amo__done_modify always
    # max
    op add ret @counter 1
    jump unsigned_to_signed_rs1_rs2 always
    # continue into maxu
    # maxu
    op max result rs1 rs2
amo__done_modify:

    # write
    op and value result 0xffffffff
    jump store_ram_word_unchecked always

# most of the register-register instructions are identical to their register-immediate counterparts
# so just set imm to rs2 and reuse the existing logic, placing them directly before to save space

SUB:
    # R-type: rs1, rs2, rd_id
    # apply two's complement to negate the value in rs2
    op sub rs2 0x100000000 rs2
    op mod rs2 rs2 0x100000000
    # continue into ADD

ADD:
    # R-type: rs1, rs2, rd_id
    set imm rs2
    # continue into ADDI

ADDI:
    # I-type: rs1, imm, rd_id
    op add rd rs1 imm
    jump end_instruction_with_truncated_rd always

SLTI:
    # I-type: rs1, imm, rd_id
    set rs2 imm
    # continue into SLT

SLT:
    # R-type: rs1, rs2, rd_id
    op add ret @counter 1
    jump unsigned_to_signed_rs1_rs2 always
    # continue into SLTU

SLTU:
    # R-type: rs1, rs2, rd_id
    set imm rs2
    # continue into SLTIU

SLTIU:
    # I-type: rs1, imm, rd_id
    op lessThan rd rs1 imm
    jump end_instruction_with_rd always

XOR:
    # R-type: rs1, rs2, rd_id
    set imm rs2
    # continue into XORI

XORI:
    # I-type: rs1, imm, rd_id
    op xor rd rs1 imm
    jump end_instruction_with_rd always

OR:
    # R-type: rs1, rs2, rd_id
    set imm rs2
    # continue into ORI

ORI:
    # I-type: rs1, imm, rd_id
    op or rd rs1 imm
    jump end_instruction_with_rd always

AND:
    # R-type: rs1, rs2, rd_id
    set imm rs2
    # continue into ANDI

ANDI:
    # I-type: rs1, imm, rd_id
    op and rd rs1 imm
    jump end_instruction_with_rd always

SLL:
    # R-type: rs1, rs2, rd_id
    op mod imm rs2 32 # 2^5 (trunc 5)
    # continue into SLLI

SLLI:
    # I-type: rs1, imm, rd_id
#% do reset_locals()
    # truncate before shifting to avoid overflowing 52 integer bits in double
    op sub $length 32 imm
    op pow $divisor 2 $length

    op mod rd rs1 $divisor
    op shl rd rd imm

    jump end_instruction_with_rd always

SRL:
    # R-type: rs1, rs2, rd_id
    op mod imm rs2 32 # 2^5 (trunc 5)
    # continue into SRLI

SRLI:
    # I-type: rs1, imm, rd_id
    op shr rd rs1 imm
    jump end_instruction_with_rd always

SRA:
    # R-type: rs1, rs2, rd_id
    op mod imm rs2 32 # 2^5 (trunc 5)
    # continue into SRAI

SRAI:
    # I-type: rs1, imm, rd_id
    op shr rd rs1 imm

    op sub length 32 imm
    # continue into extend_sign_rd

# in-place variable length sign extension function for rd
# if length is known statically, a sequence of xor-sub-and can be used instead
# ends by jumping to end_instruction_with_rd
# rd, length ->
extend_sign_rd:
#% do reset_locals()
    # https://graphics.stanford.edu/~seander/bithacks.html#VariableSignExtend
    op sub $mask length 1
    op shl $mask 1 $mask

    op xor rd rd $mask
    op sub rd rd $mask
    # continue into end_instruction_with_truncated_rd to convert signed to unsigned

end_instruction_with_truncated_rd:
    op and rd rd 0xffffffff
    jump end_instruction_with_rd always

#% set mul_locals = ["$$high_high", "$$high_low", "$$low_high", "$$low_low"]

MUL:
    # R-type: rs1, rs2, rd_id
    # "return" to the rest of MUL, and implicitly jump to mul_decomp
    # this is cursed. but it saves an instruction
    op add ret @counter {{# start_assert_length(9) }}

mul_decomp:
#% do declare_locals(mul_locals)

    # https://github.com/es-shims/Math.imul/blob/dab2a3bfc9ce5f4af97057c64ae1880fdbac57a5/implementation.js

    op shr $rs1_high rs1 16 # no truncate necessary, rs1 is already 32 bits
    op and $rs1_low rs1 0xffff
    op shr $rs2_high rs2 16
    op and $rs2_low rs2 0xffff

    op mul $$high_high $rs1_high $rs2_high
    op mul $$high_low $rs1_high $rs2_low
    op mul $$low_high $rs1_low $rs2_high
    op mul $$low_low $rs1_low $rs2_low

    set @counter ret

    # MUL ret jumps here
#directive end_assert_length
    op add rd $$high_low $$low_high
    op shl rd rd 16
    op add rd rd $$low_low

    jump end_instruction_with_truncated_rd always

# nonlocal1 = rs1_negative
# nonlocal2 = rs2_negative

MULH:
    # R-type: rs1, rs2, rd_id
    op greaterThanEq nonlocal2 rs2 0x80000000 # check if the sign bit is set
MULH_mulhsu:
    op greaterThanEq nonlocal1 rs1 0x80000000
    # continue into mulh_s_u

mulh_s_u:
#% do declare_locals(mul_locals)
    op add ret @counter 1
    jump mul_decomp always

    # https://github.com/scala-js/scala-js/blob/7e9dbfcd5c9a9eb3fe065db568c4c96ef4e19a81/linker-private-library/src/main/scala/org/scalajs/linker/runtime/RuntimeLong.scala#L422
    op shr rd $$low_low 16
    op add rd rd $$low_high
    op add rd rd $$high_low
    op shr rd rd 16
    op add rd rd $$high_high

    # https://stackoverflow.com/a/22847373
    jump mulh_s_u__rs1_positive notEqual nonlocal1 true
    op sub rd rd rs2
mulh_s_u__rs1_positive:

    jump mulh_s_u__rs2_positive notEqual nonlocal2 true
    op sub rd rd rs1
mulh_s_u__rs2_positive:

    jump end_instruction_with_truncated_rd always

MULHSU:
    # R-type: rs1, rs2, rd_id
#% do declare_locals(mul_locals)
    set nonlocal2 false
    jump MULH_mulhsu always

MULHU:
    # R-type: rs1, rs2, rd_id
#% do declare_locals(mul_locals)
    set nonlocal1 false
    set nonlocal2 false
    jump mulh_s_u always

DIV:
    # R-type: rs1, rs2, rd_id
    op add ret @counter 1
    jump unsigned_to_signed_rs1_rs2 always
    # continue into DIVU

DIVU:
    # R-type: rs1, rs2, rd_id

    # use div instead of idiv and then cast to long so that it rounds towards zero
    # https://stackoverflow.com/a/8387377
    op div rd rs1 rs2

    # division by zero returns all bits set
    select rd equal rs2 0 0xffffffff rd

    jump end_instruction_with_truncated_rd always

REM:
    # R-type: rs1, rs2, rd_id
    op add ret @counter 1
    jump unsigned_to_signed_rs1_rs2 always
    # continue into REMU

# reimu?
REMU:
    # R-type: rs1, rs2, rd_id

    # for REM, the sign of a nonzero result equals the sign of the dividend
    # this matches java's % operator, so use mod
    op mod rd rs1 rs2

    # modulo by zero returns dividend
    select rd equal rs2 0 rs1 rd

    # truncation is only necessary for signed
    jump end_instruction_with_truncated_rd always

FENCE:
    # I-type: rs1, imm, rd_id
    # note: this currently also handles FENCE.I, but that will need to change if we want either of them to not be a no-op
#% do reset_locals()

    # no-op if not PAUSE
    # PAUSE is encoded as a FENCE instruction with imm=0b000000010000, rd=x0, and rs1=x0
    # we can differentiate FENCE.I from PAUSE because FENCE.I is supposed to have imm=0
    # NOTE: result always contains the icache payload at this point
    op and $rs1id_rdid_imm result 0x3ffffffffff
    select @counter notEqual $rs1id_rdid_imm 0b000000010000 %end_instruction% %pause_one_tick%

WFI:
#% do reset_locals()
    # the simplest possible implementation of WFI would be a plain no-op
    # however, it seems like another valid implementation is to just stall for a bounded amount of time
    # since the wait is bounded, we don't need to check mstatus.TW
    # we check interrupt_flags first to avoid unnecessarily stalling if there may be an interrupt ready
    jump end_instruction greaterThanEq csr_mip 0b01000000000000000000000000000000

pause_one_tick:
    set state "pause"
    wait 1e-5
    set state "running"
    jump end_instruction always

PRIV:
    # I-type: rs1, rs1_id=uimm, imm=funct12/csr, rd_id
    # we partially decode this at runtime to simplify the decoding logic
    # privilege mode changes should be relatively rare, so it should be fine for them to be a bit slower
#% do reset_locals()

    # unprivileged
    jump ECALL  equal imm 0b000000000000
    jump EBREAK equal imm 0b000000000001
    jump WFI    equal imm 0b000100000101

    # S-mode
    jump ILLEGAL_OP lessThan privilege_mode 0b01
    jump SRET   equal imm 0b000100000010
    op shr $funct7 imm 5
    jump SFENCE.VMA equal $funct7 0b0001001

    # M-mode
    jump ILLEGAL_OP lessThan privilege_mode 0b11
    jump MRET   equal imm 0b001100000010

ILLEGAL_OP:
    # TODO: return faulting instruction bits?
    set mcause 2
    jump trap_without_mtval always

ECALL:
    # 0b1000 (8) = ecall from U-mode
    # 0b1001 (9) = ecall from S-mode
    # 0b1011 (11) = ecall from M-mode
    op or mcause 0b1000 privilege_mode
    jump trap_without_mtval always

EBREAK:
    set mcause 3 # breakpoint
    set mtval pc
    jump trap always

SRET:
#% do reset_locals()
    # if executing in S-mode, raise an illegal instruction exception if TSR=1
    jump SRET__no_tsr notEqual privilege_mode 0b01
    op and $mstatus.tsr csr_mstatus 0b10000000000000000000000
    jump ILLEGAL_OP notEqual $mstatus.tsr 0
SRET__no_tsr:

    # set privilege_mode to mstatus.SPP
    op and privilege_mode csr_mstatus 0b100000000
    op shr privilege_mode privilege_mode 8

    # set mstatus.SPP to U (and clear mstatus.SIE)
    op and csr_mstatus csr_mstatus 0b11111111111111111111111011111101

    # set next_pc to SEPC
    read next_pc {{CSRS}} "{{ 'sepc'|csr }}"

    jump xRET always

MRET:
#% do reset_locals()
    # set privilege_mode to mstatus.MPP
    op and privilege_mode csr_mstatus 0b1100000000000
    op shr privilege_mode privilege_mode 11

    # set mstatus.MPP to U (and clear mstatus.MIE)
    op and csr_mstatus csr_mstatus 0b11111111111111111110011111110111

    # set next_pc to MEPC
    read next_pc {{CSRS}} "{{ 'mepc'|csr }}"

xRET:
    # 0b000100000010 = SRET
    select $xpie_mask equal imm 0b000100000010 0b100000 0b10000000

    # set mstatus.xIE to mstatus.xPIE
    op and $mstatus.xie csr_mstatus $xpie_mask
    op shr $mstatus.xie $mstatus.xie 4
    op or csr_mstatus csr_mstatus $mstatus.xie

    # set mstatus.xPIE to 1
    op or csr_mstatus csr_mstatus $xpie_mask

    # if changing the privilege mode to a mode less privileged than M, set mstatus.MPRV to 0
    jump xRET__mprv_m_mode equal privilege_mode 0b11
    op and csr_mstatus csr_mstatus 0b11111111111111011111111111111111
xRET__mprv_m_mode:

    op or csr_mip csr_mip 0b01000000000000000000000000000000 # set interrupt_flags.fire

    # update effective_privilege_mode, and tail call
    set ret %end_instruction%
    jump get_effective_privilege_mode_and_invalidate_icache always

SFENCE.VMA:
#% do reset_locals()
    # if executing in S-mode, raise an illegal instruction exception if TVM=1
    jump SFENCE.VMA__no_tvm notEqual privilege_mode 0b01
    op and $mstatus.tvm csr_mstatus 0b100000000000000000000
    jump ILLEGAL_OP notEqual $mstatus.tvm 0
SFENCE.VMA__no_tvm:

    # invalidate icache_var to force a page table walk
    set icache_var null

    jump end_instruction always

#%# the first local is clobbered by lookup_variable
#% set modify_csr_locals = ['$$_', '$$modify_csr_op', '$$csr_labels', '$$tmp']

#% do declare_locals(modify_csr_locals)

CSRRWI:
    # CSRI-type: rs1_id=uimm, imm=csr, rd_id
    set rs1 rs1_id
    # continue into CSRRW

CSRRW:
    # CSR-type: rs1, imm=csr, rd_id
    set $$modify_csr_op %modify_csr__rw%
    jump modify_csr always

CSRRSI:
    # CSRI-type: rs1_id=uimm, imm=csr, rd_id
    set rs1 rs1_id
    # continue into CSRRS

CSRRS:
    # CSR-type: rs1, imm=csr, rd_id
    set $$modify_csr_op %modify_csr__rs%
    jump modify_csr always

CSRRCI:
    # CSRI-type: rs1_id=uimm, imm=csr, rd_id
    set rs1 rs1_id
    # continue into CSRRC

CSRRC:
    # CSR-type: rs1, imm=csr, rd_id
    set $$modify_csr_op %modify_csr__rc%
    # continue into modify_csr

modify_csr:
    # csr is stored in imm
    # csr[9:8] encodes the lowest privilege level that can access the CSR
    op shr $required_privilege imm 8
    op and $required_privilege $required_privilege 0b11
    jump ILLEGAL_OP lessThan privilege_mode $required_privilege

    # look up the CSR labels
    # NOTE: this value in _address is later used by check_xtimecmp
    # csr_labels = csr_read * 1000 + csr_write
    set _address imm
    op add ret2 @counter 1
    jump lookup_variable always
    read $$csr_labels {{CSR_LABELS}} variable

    # read

    # a few handlers assume that we set rd to 0
    set rd 0

    # jump to read handler
    op mod @counter $$csr_labels 1000

    # modify
    # the old value is stored in the variable rd by this point

modify_csr__rw:
    # read/write (always write)
    set result rs1
    jump modify_csr__always_write always
modify_csr__rs:
    # read and set bits (conditional write)
    op or result rs1 rd
    jump modify_csr__conditional_write always
modify_csr__rc:
    # read and clear bits (conditional write)
    op not result rs1
    op and result result rd

    # write

modify_csr__conditional_write:
    # we don't throw an exception if uimm is 0 or rs1 is x0
    # but we do if rs1 is some other register that happens to contain 0
    jump end_instruction_with_rd equal rs1_id 0

modify_csr__always_write:
    # apply static mask to result
    # 0xffffffff * 1000000 takes up 52 bits, so this SHOULD be safe
    op idiv $$tmp $$csr_labels 1000000
    op and result result $$tmp

    # jump to write handler
    # if the CSR is read-only, this jumps to ILLEGAL_OP
    op idiv $$tmp $$csr_labels 1000
    op mod @counter $$tmp 1000

csr_read_csrs:
#% do declare_locals(modify_csr_locals)
    read rd {{CSRS}} variable
    # continue into csr_read_zero

csr_read_zero:
    # rd is set to 0 by modify_csr
    set @counter $$modify_csr_op

# this is up here instead of after csr_read_satp to save a jump
csr_write_satp:
#% do declare_locals(modify_csr_locals)
    # we don't use ASID for anything, but riscv-sail doesn't allow us to configure ASIDLEN, so the tests will fail if we zero ASID
    # https://github.com/riscv/sail-riscv/blob/d8c3006f0ff4dbd33be876d9d62abb085c7f5298/model/riscv_xlen.sail#L25

    # invalidate icache_var when writing satp because next_pc may now point to a different page
    # TODO: this may not be strictly required by the spec
    set icache_var null
    # continue into csr_write_csrs

csr_write_csrs:
    # write the modified value back to {{CSRS}}
    write result {{CSRS}} variable
    # continue into csr_write_readonly

csr_write_readonly:
    jump end_instruction_with_rd always

csr_read_mstatus:
csr_read_sstatus:
#% do declare_locals(modify_csr_locals)
    # sstatus is a subset of mstatus
    # MXR                                                           -
    # SUM                                                            -
    # SPP                                                                      -
    # SPIE                                                                        -
    # SIE                                                                             -
    select $mask equal variable "{{ 'sstatus'|csr }}" 0b00000000000011000000000100100010 0xffffffff
    op and rd csr_mstatus $mask
    set @counter $$modify_csr_op

csr_write_mstatus:
    # no read-only non-zero fields
    set csr_mstatus 0

    # if MPP was set to 10 (reserved), set it to 0
    op and $mstatus.mpp result                   0b1100000000000
    jump csr_write_sstatus notEqual $mstatus.mpp 0b1000000000000
    op and result result      0b11111111111111111110011111111111
    # continue into csr_write_sstatus

csr_write_sstatus:
    # insert new writable fields
    # MXR                                        -
    # SUM                                         -
    # SPP                                                   -
    # SPIE                                                     -
    # SIE                                                          -
    op and csr_mstatus csr_mstatus 0b11111111111100111111111011011101

    op or csr_mstatus csr_mstatus result

    # update effective_privilege_mode
    set ret %end_instruction_with_rd_and_fire_interrupts%
    jump get_effective_privilege_mode always

csr_read_sip:
#% do declare_locals(modify_csr_locals)
    # subset of mip according to mideleg
    read $mideleg {{CSRS}} "{{ 'mideleg'|csr }}"
    op and rd csr_mip $mideleg
    set @counter $$modify_csr_op

csr_write_sip:
    # clear delegated fields
    op not $mask $mideleg
    op and csr_mip csr_mip $mask

    # write new delegated fields
    op and result result $mideleg
    jump csr_write_xip always

csr_read_mip:
#% do declare_locals(modify_csr_locals)
    # hide internal interrupt flags
    op and rd csr_mip 0b00111111111111111111111111111111
    set @counter $$modify_csr_op

csr_write_mip:
    # insert new writable fields
    # SEIP                                         -
    # SSIP                                                 -
    op and csr_mip csr_mip 0b11111111111111111111110111111101

csr_write_xip:
    op or csr_mip csr_mip result
    jump end_instruction_with_rd_and_fire_interrupts always

csr_read_sie:
#% do declare_locals(modify_csr_locals)
    # subset of mie according to mideleg
    read $mideleg {{CSRS}} "{{ 'mideleg'|csr }}"
    op and rd csr_mie $mideleg
    set @counter $$modify_csr_op

csr_write_sie:
    # clear delegated fields
    op not $mask $mideleg
    op and csr_mie csr_mie $mask

    # set new delegated fields
    op and result result $mideleg
    op or csr_mie csr_mie result

    jump end_instruction_with_rd_and_fire_interrupts always

csr_read_mie:
#% do declare_locals(modify_csr_locals)
    set rd csr_mie
    set @counter $$modify_csr_op

csr_write_mie:
    # get new writable fields
    # MEIE                                      -
    # SEIE                                        -
    # MTIE                                          -
    # STIE                                            -
    # SSIE                                                -
    op and csr_mie result 0b00000000000000000000101010100010
    jump end_instruction_with_rd_and_fire_interrupts always

csr_read_satp:
#% do declare_locals(modify_csr_locals)
    # if executing in S-mode, raise an illegal instruction exception if TVM=1
    jump csr_read_csrs notEqual privilege_mode 0b01
    op and $mstatus.tvm csr_mstatus 0b100000000000000000000
    select @counter equal $mstatus.tvm 0 %csr_read_csrs% %ILLEGAL_OP%

csr_read_stimecmp:
csr_read_stimecmph:
#% do declare_locals(modify_csr_locals)
    select rd equal variable "{{ 'stimecmp'|csr }}" csr_stimecmp csr_stimecmph

    # stimecmp is controlled by the TM bit in mcounteren
    # so we set _address to 1 and use check_xcounteren to check the TM bit
    set _address 1
    jump check_xcounteren always

csr_write_stimecmp:
    set csr_stimecmp result
    jump end_instruction_with_rd_and_poll_interrupts always

csr_write_stimecmph:
    set csr_stimecmph result
    jump end_instruction_with_rd_and_poll_interrupts always

csr_read_mcycle:
#% do declare_locals(modify_csr_locals)
    set rd csr_mcycle
    jump check_xcounteren always

csr_write_mcycle:
    set csr_mcycle result
    jump end_instruction_with_rd always

csr_read_cycleh:
#% do declare_locals(modify_csr_locals)
    read rd {{CSRS}} "{{ 'mcycleh'|csr }}"
    jump check_xcounteren always

csr_read_minstret:
#% do declare_locals(modify_csr_locals)
    set rd csr_minstret
    jump check_xcounteren always

csr_write_minstret:
    # a value written to instret by one instruction should be the value read by the following instruction
    op sub csr_minstret result 1
    jump end_instruction_with_rd always

csr_read_minstreth:
#% do declare_locals(modify_csr_locals)
    read rd {{CSRS}} "{{ 'minstreth'|csr }}"
    jump check_xcounteren always

csr_write_minstreth:
    # a value written to instret by one instruction should be the value read by the following instruction
    # TODO: is this necessary for minstreth?
    op sub csr_minstret csr_minstret 1
    jump csr_write_csrs always

csr_read_time:
csr_read_timeh:
#% do declare_locals(modify_csr_locals)
    select rd equal variable "{{ 'time'|csr }}" csr_mtime csr_mtimeh
    # continue into check_xcounteren via csr_read_hpmcounter

csr_read_hpmcounter:
#% do declare_locals(modify_csr_locals)
    # rd is set to 0 by modify_csr
    # continue into check_xcounteren

check_xcounteren:
    # if in M-mode, always allow counter accesses
    select @counter equal privilege_mode 0b11 $$modify_csr_op @counter

    # if in S-mode, only require the bit to be set in mcounteren
    read $xcounteren {{CSRS}} "{{ 'mcounteren'|csr }}"
    jump check_xcounteren__s_mode equal privilege_mode 0b01

    # if in U-mode, also require the bit to be set in scounteren
    read $scounteren {{CSRS}} "{{ 'scounteren'|csr }}"
    op and $xcounteren $xcounteren $scounteren
check_xcounteren__s_mode:

    # trap if the bit in xcounteren corresponding to the CSR being read is clear
    # we use _address instead of imm here to allow check_stimecmp to change it without messing up debug info
    op and $bit _address 31
    op shr $bit $xcounteren $bit
    op and $bit $bit 0b1

    select @counter equal $bit 0 %ILLEGAL_OP% $$modify_csr_op

MLOGSYS:
    # I-type: rs1, imm=funct12, rd_id

    # for now we can just ignore imm, since MLOGSYS is only used for decoding
    # jump ILLEGAL_OP greaterThan imm 0

    # init icache
    # require M-mode so that we don't need to do address translation
    jump ILLEGAL_OP lessThan privilege_mode 0b11

    # round up to the nearest page boundary
    # if we're exactly on a page boundary, this results in decoding one extra page
    op add rs1 rs1 4096
    op and rs1 rs1 0xfffff000

    op min __etext rs1 PROGRAM_ROM_ICACHE_SIZE

    # start at -4 so the first address read is 0
    set address -4
    set variable null

    set ret3 @counter
    # loop begins

    op add address address 4
    read variable {{INCR}} variable

    # load_rom_word_unchecked does not clobber ram or variable
    # decode returns to the top of this loop
    set ret %decode_value_in_result%
    select @counter lessThan address __etext %load_rom_word_unchecked% %end_instruction%

#% if false
# disable mlogls warnings
# put this at the end of the file so if we go over the 1000 instruction limit, we can ignore the errors below this line
# {% raw %}
# common preprocessor constants
set {{ null
set {{LOOKUP_PROC_SIZE}} null
set {{ROM_PROC_BYTES}} null
set {{RAM_PROC_VARS}} null
set {{RAM_START}} null
set {{MMIO_START}} null
set {{UART_RX_OFFSET}} null
set {{UART_TX_OFFSET}} null
set {{UART_STATUS_OFFSET}} null
set {{UART_CONTROL_OFFSET}} null
set {{UART_MMIO_SIZE}} null
set {{SYSCON}} null
set {{UART_RX_READ}} null
set {{UART_RX_WRITE}} null
set {{UART_TX_START}} null
set {{UART_TX_READ}} null
set {{UART_TX_WRITE}} null
# local preprocessor constants
set {{UART_START_LINK}} null
set {{LABELS}} null
set {{REGISTERS}} null
set {{CSRS}} null
set {{INCR}} null
set {{CONFIG}} null
set {{CSR_LABELS}} null
set {{CONTROLLER}} null
set {{ERROR_OUTPUT}} null
set {{LN_2}} null
set {{PAGESIZE}} null
# unused variables
set _ {{local_variable()}}
# symbolic labels
set %ILLEGAL_OP% null
set %next_tick% null
set %end_instruction_trap% null
set %end_instruction_with_rd% null
set %end_instruction% null
set %end_instruction_with_rd_and_fire_interrupts% null
set %default_xtvec_handler% null
set %trap% null
set %translate_virtual_address% null
set %access_ram% null
set %load_word% null
set %load_rom_word_unchecked% null
set %decode_value_in_result% null
set %decode% null
set %decode_I-type% null
set %decode_I-type_no_extend_sign% null
set %decode_illegal_instruction% null
set %end_decode% null
set %conditional_branch% null
set %store_ram_word_unchecked% null
set %extend_sign_rd% null
set %modify_csr__rw% null
set %modify_csr__rs% null
set %modify_csr__rc% null
set %csr_read_csrs% null
set %pause_one_tick% null
# unused labels
jump next_tick always
jump default_xtvec_handler always
jump decode always
jump conditional_branch always
jump extend_sign_rd always
jump BEQ always
jump BNE always
jump JAL always
jump JALR always
jump BLT always
jump BGE always
jump BLTU always
jump BGEU always
jump LB always
jump LH always
jump LW always
jump LUI always
jump LBU always
jump LHU always
jump SB always
jump SH always
jump SW always
jump AMOADD.W always
jump AMOSWAP.W always
jump LR.W always
jump SC.W always
jump AMOXOR.W always
jump AMOOR.W always
jump AMOAND.W always
jump AMOMIN.W always
jump AMOMAX.W always
jump AMOMINU.W always
jump AMOMAXU.W always
jump ADDI always
jump SUB always
jump SLTI always
jump SLTIU always
jump XORI always
jump SRA always
jump ORI always
jump ANDI always
jump ADD always
jump SLL always
jump SLT always
jump SLTU always
jump XOR always
jump SRL always
jump OR always
jump AND always
jump MUL always
jump MULH always
jump MULHSU always
jump MULHU always
jump DIV always
jump DIVU always
jump REM always
jump REMU always
jump PRIV always
jump CSRRW always
jump CSRRS always
jump CSRRC always
jump AUIPC always
jump CSRRWI always
jump CSRRSI always
jump CSRRCI always
jump FENCE always
jump SLLI always
jump SRLI always
jump SRAI always
jump ILLEGAL_OP always
jump MLOGSYS always
jump decode_LOAD always
jump decode_custom-0 always
jump decode_MISC-MEM always
jump decode_OP-IMM always
jump decode_AUIPC always
jump decode_STORE always
jump decode_AMO always
jump decode_OP always
jump decode_LUI always
jump decode_BRANCH always
jump decode_JALR always
jump decode_JAL always
jump decode_SYSTEM always
jump modify_csr__rw always
jump modify_csr__rs always
jump modify_csr__rc always
jump csr_read_csrs always
jump csr_read_mcycle always
jump csr_read_cycleh always
jump csr_read_hpmcounter always
jump csr_read_mie always
jump csr_read_minstret always
jump csr_read_minstreth always
jump csr_read_mip always
jump csr_read_mstatus always
jump csr_read_satp always
jump csr_read_sie always
jump csr_read_sip always
jump csr_read_sstatus always
jump csr_read_stimecmp always
jump csr_read_stimecmph always
jump csr_read_time always
jump csr_read_timeh always
jump csr_read_zero always
jump csr_write_csrs always
jump csr_write_mcycle always
jump csr_write_mie always
jump csr_write_minstret always
jump csr_write_minstreth always
jump csr_write_mip always
jump csr_write_mstatus always
jump csr_write_readonly always
jump csr_write_satp always
jump csr_write_sie always
jump csr_write_sip always
jump csr_write_sstatus always
jump csr_write_stimecmp always
jump csr_write_stimecmph always
jump pause_one_tick always
# {% endraw %}
#% endif