diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e12bf4d5..3da6de63a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,6 +44,9 @@ jobs: sudo apt-get update -q -y sudo apt-get install -q -y libsdl2-dev libsdl2-mixer-dev .ci/riscv-toolchain-install.sh + wget https://apt.llvm.org/llvm.sh + sudo chmod +x ./llvm.sh + sudo ./llvm.sh 17 shell: bash - name: default build run: make @@ -65,14 +68,14 @@ jobs: make distclean ENABLE_GDBSTUB=1 gdbstub-test - name: JIT test run: | - make clean && make ENABLE_JIT=1 check -j$(nproc) - make clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check -j$(nproc) - make clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check -j$(nproc) - make clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check -j$(nproc) + make ENABLE_JIT=1 clean && make ENABLE_JIT=1 check -j$(nproc) + make ENABLE_JIT=1 clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check -j$(nproc) + make ENABLE_JIT=1 clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check -j$(nproc) + make ENABLE_JIT=1 clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check -j$(nproc) - name: undefined behavior test run: | make clean && make ENABLE_UBSAN=1 check -j$(nproc) - make clean && make ENABLE_JIT=1 ENABLE_UBSAN=1 check -j$(nproc) + make ENABLE_JIT=1 clean clean && make ENABLE_JIT=1 ENABLE_UBSAN=1 check -j$(nproc) host-arm64: needs: [detect-code-related-file-changes] @@ -91,18 +94,21 @@ jobs: # No 'sudo' is available install: | apt-get update -q -y - apt-get install -q -y git build-essential libsdl2-dev libsdl2-mixer-dev + apt-get install -q -y git build-essential libsdl2-dev libsdl2-mixer-dev lsb-release wget software-properties-common gnupg git config --global --add safe.directory ${{ github.workspace }} git config --global --add safe.directory ${{ github.workspace }}/src/softfloat git config --global --add safe.directory ${{ github.workspace }}/src/mini-gdbstub + wget https://apt.llvm.org/llvm.sh + chmod +x ./llvm.sh + ./llvm.sh 17 # Append custom commands here run: | make -j$(nproc) make check -j$(nproc) - make clean && make ENABLE_JIT=1 check -j$(nproc) - make clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check -j$(nproc) - make clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check -j$(nproc) - make clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check -j$(nproc) + make ENABLE_JIT=1 clean && make ENABLE_JIT=1 check -j$(nproc) + make ENABLE_JIT=1 clean && make ENABLE_EXT_A=0 ENABLE_JIT=1 check -j$(nproc) + make ENABLE_JIT=1 clean && make ENABLE_EXT_F=0 ENABLE_JIT=1 check -j$(nproc) + make ENABLE_JIT=1 clean && make ENABLE_EXT_C=0 ENABLE_JIT=1 check -j$(nproc) coding-style: needs: [detect-code-related-file-changes] @@ -132,7 +138,8 @@ jobs: - name: run scan-build without JIT run: make distclean && scan-build -v -o ~/scan-build --status-bugs --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability make ENABLE_EXT_F=0 ENABLE_SDL=0 ENABLE_JIT=0 - name: run scan-build with JIT - run: make distclean && scan-build -v -o ~/scan-build --status-bugs --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability make ENABLE_EXT_F=0 ENABLE_SDL=0 ENABLE_JIT=1 + run: | + make ENABLE_JIT=1 distclean && scan-build -v -o ~/scan-build --status-bugs --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability make ENABLE_EXT_F=0 ENABLE_SDL=0 ENABLE_JIT=1 compliance-test: needs: [detect-code-related-file-changes] diff --git a/Makefile b/Makefile index e6c94b95e..d83591109 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,32 @@ endif ENABLE_JIT ?= 0 $(call set-feature, JIT) ifeq ($(call has, JIT), 1) -OBJS_EXT += jit.o +OBJS_EXT += jit.o +# tier-2 JIT compiler powered LLVM +LLVM_CONFIG = llvm-config-17 +LLVM_CONFIG := $(shell which $(LLVM_CONFIG)) +ifndef LLVM_CONFIG +# Try Homebrew on macOS +LLVM_CONFIG = /opt/homebrew/opt/llvm@17/bin/llvm-config +LLVM_CONFIG := $(shell which $(LLVM_CONFIG)) +ifdef LLVM_CONFIG +LDFLAGS += -L/opt/homebrew/opt/llvm@17/lib +endif +endif +ifneq ("$(LLVM_CONFIG)", "") +ifneq ("$(findstring -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS, "$(shell $(LLVM_CONFIG) --cflags)")", "") +ENABLE_T2C := 1 +$(call set-feature, T2C) +OBJS_EXT += t2c.o +CFLAGS += -g $(shell $(LLVM_CONFIG) --cflags) +LDFLAGS += $(shell $(LLVM_CONFIG) --libs) +else +ENABLE_T2C := 0 +$(call set-feature, T2C) +$(warning No llvm-config-17 installed. Check llvm-config-17 installation in advance) +endif +endif + ifneq ($(processor),$(filter $(processor),x86_64 aarch64 arm64)) $(error JIT mode only supports for x64 and arm64 target currently.) endif @@ -136,6 +161,10 @@ src/rv32_jit.c: $(OUT)/jit.o: src/jit.c src/rv32_jit.c $(VECHO) " CC\t$@\n" $(Q)$(CC) -o $@ $(CFLAGS) -c -MMD -MF $@.d $< + +$(OUT)/t2c.o: src/t2c.c src/t2c_template.c + $(VECHO) " CC\t$@\n" + $(Q)$(CC) -o $@ $(CFLAGS) -c -MMD -MF $@.d $< endif # For tail-call elimination, we need a specific set of build flags applied. # FIXME: On macOS + Apple Silicon, -fno-stack-protector might have a negative impact. diff --git a/README.md b/README.md index f36dd2569..d5a926c29 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Features: * Implementation of commonly used newlib system calls * Experimental SDL-based display/event/audio system calls for running video games * Support for remote GDB debugging -* Experimental JIT compiler for performance boost while maintaining a small footprint +* Tiered JIT compilation for performance boost while maintaining a small footprint ## Build and Verify @@ -40,6 +40,12 @@ and [SDL2_Mixer library](https://wiki.libsdl.org/SDL2_mixer) installed. * macOS: `brew install sdl2 sdl2_mixer` * Ubuntu Linux / Debian: `sudo apt install libsdl2-dev libsdl2-mixer-dev` +### JIT compiler +The tier-2 JIT compiler in `rv32emu` leverages LLVM for powerful optimization. Therefore, the target system must have [`LLVM`](https://llvm.org/) installed, with version 17 recommended. If `LLVM` is not installed, only the tier-1 JIT compiler will be used for performance enhancement. + +* macOS: `brew install llvm@17` +* Ubuntu Linux / Debian: `sudo apt-get install llvm-17` + Build the emulator. ```shell $ make diff --git a/src/emulate.c b/src/emulate.c index eb7646510..0152896c4 100644 --- a/src/emulate.c +++ b/src/emulate.c @@ -304,7 +304,9 @@ static block_t *block_alloc(riscv_t *rv) #if RV32_HAS(JIT) block->translatable = true; block->hot = false; + block->hot2 = false; block->has_loops = false; + block->n_invoke = 0; INIT_LIST_HEAD(&block->list); #endif return block; @@ -911,8 +913,6 @@ static bool runtime_profiler(riscv_t *rv, block_t *block) return true; return false; } - -typedef void (*exec_block_func_t)(riscv_t *rv, uintptr_t); #endif void rv_step(void *arg) @@ -985,15 +985,31 @@ void rv_step(void *arg) } last_pc = rv->PC; #if RV32_HAS(JIT) - /* execute by tier-1 JIT compiler */ +#if RV32_HAS(T2C) + /* executed through the tier-2 JIT compiler */ + if (block->hot2) { + ((exec_t2c_func_t) block->func)(rv); + prev = NULL; + continue; + } /* check if the execution path is strong hotspot */ + if (block->n_invoke >= THRESHOLD) { + t2c_compile(block, + (uint64_t) ((memory_t *) PRIV(rv)->mem)->mem_base); + ((exec_t2c_func_t) block->func)(rv); + prev = NULL; + continue; + } +#endif + /* executed through the tier-1 JIT compiler */ struct jit_state *state = rv->jit_state; if (block->hot) { + block->n_invoke++; ((exec_block_func_t) state->buf)( rv, (uintptr_t) (state->buf + block->offset)); prev = NULL; continue; - } /* check if using frequency of block exceed threshold */ - else if (block->translatable && runtime_profiler(rv, block)) { + } /* check if the execution path is potential hotspot */ + if (block->translatable && runtime_profiler(rv, block)) { jit_translate(rv, block); ((exec_block_func_t) state->buf)( rv, (uintptr_t) (state->buf + block->offset)); diff --git a/src/feature.h b/src/feature.h index 3371a798d..706c4603a 100644 --- a/src/feature.h +++ b/src/feature.h @@ -52,5 +52,10 @@ #define RV32_FEATURE_JIT 0 #endif +/* Experimental tier-2 just-in-time compiler */ +#ifndef RV32_FEATURE_T2C +#define RV32_FEATURE_T2C 0 +#endif + /* Feature test macro */ #define RV32_HAS(x) RV32_FEATURE_##x diff --git a/src/jit.h b/src/jit.h index ca01106cd..8859822bc 100644 --- a/src/jit.h +++ b/src/jit.h @@ -48,3 +48,9 @@ struct host_reg { struct jit_state *jit_state_init(size_t size); void jit_state_exit(struct jit_state *state); void jit_translate(riscv_t *rv, block_t *block); +typedef void (*exec_block_func_t)(riscv_t *rv, uintptr_t); + +#if RV32_HAS(T2C) +void t2c_compile(block_t *block, uint64_t mem_base); +typedef void (*exec_t2c_func_t)(riscv_t *); +#endif diff --git a/src/riscv_private.h b/src/riscv_private.h index 25e9aea7b..95efe76da 100644 --- a/src/riscv_private.h +++ b/src/riscv_private.h @@ -65,11 +65,14 @@ typedef struct block { rv_insn_t *ir_head, *ir_tail; /**< the first and last ir for this block */ #if RV32_HAS(JIT) - bool hot; /**< Determine the block is hotspot or not */ - uint32_t offset; + bool hot; /**< Determine the block is potential hotspot or not */ + bool hot2; /**< Determine the block is strong hotspot or not */ bool translatable; /**< Determine the block has RV32AF insturctions or not */ bool has_loops; /**< Determine the block has loop or not */ + uint32_t offset; /**< The machine code offset in T1 code cache */ + uint32_t n_invoke; /**< The invoking times of T1 machine code */ + void *func; /**< The function pointer of T2 machine code */ struct list_head list; #endif } block_t; diff --git a/src/t2c.c b/src/t2c.c new file mode 100644 index 000000000..363853564 --- /dev/null +++ b/src/t2c.c @@ -0,0 +1,302 @@ +/* + * rv32emu is freely redistributable under the MIT License. See the file + * "LICENSE" for information on usage and redistribution of this file. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "jit.h" +#include "riscv_private.h" + +#define MAX_BLOCKS 8152 + +struct LLVM_block_map_entry { + uint32_t pc; + LLVMBasicBlockRef block; +}; + +struct LLVM_block_map { + uint32_t count; + struct LLVM_block_map_entry map[MAX_BLOCKS]; +}; + +FORCE_INLINE void t2c_block_map_insert(struct LLVM_block_map *map, + LLVMBasicBlockRef *entry, + uint32_t pc) +{ + struct LLVM_block_map_entry map_entry = { + .block = *entry, + .pc = pc, + }; + map->map[map->count++] = map_entry; + return; +} + +FORCE_INLINE LLVMBasicBlockRef t2c_block_map_search(struct LLVM_block_map *map, + uint32_t pc) +{ + for (uint32_t i = 0; i < map->count; i++) { + if (map->map[i].pc == pc) { + return map->map[i].block; + } + } + return NULL; +} + +#define T2C_OP(inst, code) \ + static void t2c_##inst( \ + LLVMBuilderRef *builder UNUSED, LLVMTypeRef *param_types UNUSED, \ + LLVMValueRef start UNUSED, LLVMBasicBlockRef *entry UNUSED, \ + LLVMBuilderRef *taken_builder UNUSED, \ + LLVMBuilderRef *untaken_builder UNUSED, uint64_t mem_base UNUSED, \ + rv_insn_t *ir UNUSED) \ + { \ + code; \ + } + +#define T2C_LLVM_GEN_ADDR(reg, rv_member, ir_member) \ + FORCE_INLINE LLVMValueRef t2c_gen_##reg##_addr( \ + LLVMValueRef start, LLVMBuilderRef *builder, UNUSED rv_insn_t *ir) \ + { \ + LLVMValueRef offset = LLVMConstInt( \ + LLVMInt32Type(), \ + offsetof(riscv_t, rv_member) / sizeof(int) + ir_member, true); \ + return LLVMBuildInBoundsGEP2(*builder, LLVMInt32Type(), \ + LLVMGetParam(start, 0), &offset, 1, ""); \ + } + +T2C_LLVM_GEN_ADDR(rs1, X, ir->rs1); +T2C_LLVM_GEN_ADDR(rs2, X, ir->rs2); +T2C_LLVM_GEN_ADDR(rd, X, ir->rd); +T2C_LLVM_GEN_ADDR(ra, X, rv_reg_ra); +T2C_LLVM_GEN_ADDR(sp, X, rv_reg_sp); +T2C_LLVM_GEN_ADDR(PC, PC, 0); + +#define T2C_LLVM_GEN_STORE_IMM32(builder, val, addr) \ + LLVMBuildStore(builder, LLVMConstInt(LLVMInt32Type(), val, true), addr) + +#define T2C_LLVM_GEN_LOAD_VMREG(reg, size, addr) \ + LLVMValueRef val_##reg = \ + LLVMBuildLoad2(*builder, LLVMInt##size##Type(), addr, ""); + +#define T2C_LLVM_GEN_ALU32_IMM(op, dst, imm) \ + LLVMBuild##op(*builder, dst, LLVMConstInt(LLVMInt32Type(), imm, true), "") + +#define T2C_LLVM_GEN_ALU64_IMM(op, dst, imm) \ + LLVMBuild##op(*builder, dst, LLVMConstInt(LLVMInt64Type(), imm, true), "") + +#define T2C_LLVM_GEN_CMP(cond, rs1, rs2) \ + LLVMValueRef cmp = LLVMBuildICmp(*builder, LLVMInt##cond, rs1, rs2, "") + +#define T2C_LLVM_GEN_CMP_IMM32(cond, rs1, imm) \ + LLVMValueRef cmp = \ + LLVMBuildICmp(*builder, LLVMInt##cond, rs1, \ + LLVMConstInt(LLVMInt32Type(), imm, false), "") + +FORCE_INLINE LLVMValueRef t2c_gen_mem_loc(LLVMValueRef start, + LLVMBuilderRef *builder, + UNUSED rv_insn_t *ir, + uint64_t mem_base) +{ + LLVMValueRef val_rs1 = + LLVMBuildZExt(*builder, + LLVMBuildLoad2(*builder, LLVMInt32Type(), + t2c_gen_rs1_addr(start, builder, ir), ""), + LLVMInt64Type(), ""); + LLVMValueRef addr = + T2C_LLVM_GEN_ALU64_IMM(Add, val_rs1, ir->imm + mem_base); + addr = LLVMBuildIntToPtr(*builder, addr, + LLVMPointerType(LLVMInt32Type(), 0), ""); + return addr; +} + +FORCE_INLINE void t2c_gen_call_io_func(LLVMValueRef start, + LLVMBuilderRef *builder, + LLVMTypeRef *param_types, + int offset) +{ + LLVMValueRef func_offset = LLVMConstInt(LLVMInt32Type(), offset, true); + LLVMValueRef addr_io_func = LLVMBuildInBoundsGEP2( + *builder, LLVMPointerType(LLVMVoidType(), 0), LLVMGetParam(start, 0), + &func_offset, 1, "addr_io_func"); + LLVMValueRef io_func = LLVMBuildLoad2( + *builder, + LLVMPointerType(LLVMFunctionType(LLVMVoidType(), param_types, 1, 0), 0), + addr_io_func, "io_func"); + LLVMValueRef io_param = LLVMGetParam(start, 0); + LLVMBuildCall2(*builder, + LLVMFunctionType(LLVMVoidType(), param_types, 1, 0), io_func, + &io_param, 1, ""); +} + +#include "t2c_template.c" +#undef T2C_OP + +static const void *dispatch_table[] = { +/* RV32 instructions */ +#define _(inst, can_branch, insn_len, translatable, reg_mask) \ + [rv_insn_##inst] = t2c_##inst, + RV_INSN_LIST +#undef _ +/* Macro operation fusion instructions */ +#define _(inst) [rv_insn_##inst] = t2c_##inst, + FUSE_INSN_LIST +#undef _ +}; + +FORCE_INLINE bool t2c_insn_is_terminal(uint8_t opcode) +{ + switch (opcode) { + case rv_insn_ecall: + case rv_insn_ebreak: + case rv_insn_jalr: + case rv_insn_sret: + case rv_insn_mret: +#if RV32_HAS(EXT_C) + case rv_insn_cjalr: + case rv_insn_cjr: + case rv_insn_cebreak: +#endif + return true; + } + return false; +} + +typedef void (*t2c_codegen_block_func_t)(LLVMBuilderRef *builder UNUSED, + LLVMTypeRef *param_types UNUSED, + LLVMValueRef start UNUSED, + LLVMBasicBlockRef *entry UNUSED, + LLVMBuilderRef *taken_builder UNUSED, + LLVMBuilderRef *untaken_builder UNUSED, + uint64_t mem_base UNUSED, + rv_insn_t *ir UNUSED); + +static void t2c_trace_ebb(LLVMBuilderRef *builder, + LLVMTypeRef *param_types UNUSED, + LLVMValueRef start, + LLVMBasicBlockRef *entry, + uint64_t mem_base, + rv_insn_t *ir, + set_t *set, + struct LLVM_block_map *map) +{ + if (set_has(set, ir->pc)) + return; + set_add(set, ir->pc); + t2c_block_map_insert(map, entry, ir->pc); + LLVMBuilderRef tk, utk; + + while (1) { + ((t2c_codegen_block_func_t) dispatch_table[ir->opcode])( + builder, param_types, start, entry, &tk, &utk, mem_base, ir); + if (!ir->next) + break; + ir = ir->next; + } + + if (!t2c_insn_is_terminal(ir->opcode)) { + if (ir->branch_untaken) { + if (set_has(set, ir->branch_untaken->pc)) + LLVMBuildBr(utk, + t2c_block_map_search(map, ir->branch_untaken->pc)); + else { + LLVMBasicBlockRef untaken_entry = + LLVMAppendBasicBlock(start, + "untaken_" + "entry"); + LLVMBuilderRef untaken_builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(untaken_builder, untaken_entry); + LLVMBuildBr(utk, untaken_entry); + t2c_trace_ebb(&untaken_builder, param_types, start, + &untaken_entry, mem_base, ir->branch_untaken, set, + map); + } + } + if (ir->branch_taken) { + if (set_has(set, ir->branch_taken->pc)) + LLVMBuildBr(tk, + t2c_block_map_search(map, ir->branch_taken->pc)); + else { + LLVMBasicBlockRef taken_entry = LLVMAppendBasicBlock(start, + "taken_" + "entry"); + LLVMBuilderRef taken_builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(taken_builder, taken_entry); + LLVMBuildBr(tk, taken_entry); + t2c_trace_ebb(&taken_builder, param_types, start, &taken_entry, + mem_base, ir->branch_taken, set, map); + } + } + } +} + +void t2c_compile(block_t *block, uint64_t mem_base) +{ + LLVMModuleRef module = LLVMModuleCreateWithName("my_module"); + LLVMTypeRef io_members[] = { + LLVMPointerType(LLVMVoidType(), 0), LLVMPointerType(LLVMVoidType(), 0), + LLVMPointerType(LLVMVoidType(), 0), LLVMPointerType(LLVMVoidType(), 0), + LLVMPointerType(LLVMVoidType(), 0), LLVMPointerType(LLVMVoidType(), 0), + LLVMPointerType(LLVMVoidType(), 0), LLVMPointerType(LLVMVoidType(), 0), + LLVMPointerType(LLVMVoidType(), 0), LLVMPointerType(LLVMVoidType(), 0), + LLVMPointerType(LLVMVoidType(), 0), LLVMInt8Type()}; + LLVMTypeRef struct_io = LLVMStructType(io_members, 12, false); + LLVMTypeRef arr_X = LLVMArrayType(LLVMInt32Type(), 32); + LLVMTypeRef rv_members[] = {LLVMInt8Type(), struct_io, arr_X, + LLVMInt32Type()}; + LLVMTypeRef struct_rv = LLVMStructType(rv_members, 4, false); + LLVMTypeRef param_types[] = {LLVMPointerType(struct_rv, 0)}; + LLVMValueRef start = LLVMAddFunction( + module, "start", LLVMFunctionType(LLVMVoidType(), param_types, 1, 0)); + LLVMBasicBlockRef first_block = LLVMAppendBasicBlock(start, "first_block"); + LLVMBuilderRef first_builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(first_builder, first_block); + LLVMBasicBlockRef entry = LLVMAppendBasicBlock(start, "entry"); + LLVMBuilderRef builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder, entry); + LLVMBuildBr(first_builder, entry); + set_t set; + set_reset(&set); + struct LLVM_block_map map; + map.count = 0; + /* Translate custon IR into LLVM IR */ + t2c_trace_ebb(&builder, param_types, start, &entry, mem_base, + block->ir_head, &set, &map); + /* Offload LLVM IR to LLVM backend */ + char *error = NULL, *triple = LLVMGetDefaultTargetTriple(); + LLVMExecutionEngineRef engine; + LLVMTargetRef target; + LLVMLinkInMCJIT(); + LLVMInitializeNativeTarget(); + LLVMInitializeNativeAsmPrinter(); + if (LLVMGetTargetFromTriple(triple, &target, &error) != 0) { + fprintf(stderr, + "failed to create " + "Target\n"); + abort(); + } + LLVMTargetMachineRef tm = LLVMCreateTargetMachine( + target, triple, LLVMGetHostCPUName(), LLVMGetHostCPUFeatures(), + LLVMCodeGenLevelNone, LLVMRelocDefault, LLVMCodeModelJITDefault); + LLVMPassBuilderOptionsRef pb_option = LLVMCreatePassBuilderOptions(); + /* Run aggressive optimization level and some selected Passes */ + LLVMRunPasses(module, "default,early-cse,instcombine", tm, + pb_option); + + if (LLVMCreateExecutionEngineForModule(&engine, module, &error) != 0) { + fprintf(stderr, + "failed to create " + "execution engine\n"); + abort(); + } + + /* Return the function pointer of T2C generated machine code */ + block->func = (exec_t2c_func_t) LLVMGetPointerToGlobal(engine, start); + block->hot2 = true; +} diff --git a/src/t2c_template.c b/src/t2c_template.c new file mode 100644 index 000000000..041bcb8d7 --- /dev/null +++ b/src/t2c_template.c @@ -0,0 +1,847 @@ +/* This file maps each custom IR to the corresponding LLVM IRs and builds LLVM + * IR through LLVM-C API. The built LLVM IR is offloaded to the LLVM backend, + * where it undergoes optimization through several selected LLVM passes. + * Subsequently, the optimized LLVM IR is passed to the LLVM execution engine, + * which compiles the optimized LLVM IR and returns a function pointer to the + * generated machine code. + */ + +T2C_OP(nop, { return; }) + +T2C_OP(lui, { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->imm, + t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(auipc, { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + ir->imm, + t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(jal, { + if (ir->rd) + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + 4, + t2c_gen_rd_addr(start, builder, ir)); + + if (ir->branch_taken) + *taken_builder = *builder; + else { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + ir->imm, + t2c_gen_PC_addr(start, builder, ir)); + LLVMBuildRetVoid(*builder); + } +}) + +T2C_OP(jalr, { + if (ir->rd) + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + 4, + t2c_gen_rd_addr(start, builder, ir)); + + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + val_rs1 = T2C_LLVM_GEN_ALU32_IMM(Add, val_rs1, ir->imm); + val_rs1 = T2C_LLVM_GEN_ALU32_IMM(And, val_rs1, ~1U); + LLVMBuildStore(*builder, val_rs1, t2c_gen_PC_addr(start, builder, ir)); + LLVMBuildRetVoid(*builder); +}) + +#define BRANCH_FUNC(type, cond) \ + T2C_OP(type, { \ + LLVMValueRef addr_PC = t2c_gen_PC_addr(start, builder, ir); \ + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, \ + t2c_gen_rs1_addr(start, builder, ir)); \ + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, \ + t2c_gen_rs2_addr(start, builder, ir)); \ + T2C_LLVM_GEN_CMP(cond, val_rs1, val_rs2); \ + LLVMBasicBlockRef taken = LLVMAppendBasicBlock(start, "taken"); \ + LLVMBuilderRef builder2 = LLVMCreateBuilder(); \ + LLVMPositionBuilderAtEnd(builder2, taken); \ + if (ir->branch_taken) \ + *taken_builder = builder2; \ + else { \ + T2C_LLVM_GEN_STORE_IMM32(builder2, ir->pc + ir->imm, addr_PC); \ + LLVMBuildRetVoid(builder2); \ + } \ + LLVMBasicBlockRef untaken = LLVMAppendBasicBlock(start, "untaken"); \ + LLVMBuilderRef builder3 = LLVMCreateBuilder(); \ + LLVMPositionBuilderAtEnd(builder3, untaken); \ + if (ir->branch_untaken) \ + *untaken_builder = builder3; \ + else { \ + T2C_LLVM_GEN_STORE_IMM32(builder3, ir->pc + 4, addr_PC); \ + LLVMBuildRetVoid(builder3); \ + } \ + LLVMBuildCondBr(*builder, cmp, taken, untaken); \ + }) + +BRANCH_FUNC(beq, EQ) +BRANCH_FUNC(bne, NE) +BRANCH_FUNC(blt, SLT) +BRANCH_FUNC(bge, SGE) +BRANCH_FUNC(bltu, ULT) +BRANCH_FUNC(bgeu, UGE) + +T2C_OP(lb, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = LLVMBuildSExt( + *builder, LLVMBuildLoad2(*builder, LLVMInt8Type(), mem_loc, "res"), + LLVMInt32Type(), "sext8to32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(lh, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = LLVMBuildSExt( + *builder, LLVMBuildLoad2(*builder, LLVMInt16Type(), mem_loc, "res"), + LLVMInt32Type(), "sext16to32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + + +T2C_OP(lw, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = + LLVMBuildLoad2(*builder, LLVMInt32Type(), mem_loc, "res"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(lbu, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = LLVMBuildZExt( + *builder, LLVMBuildLoad2(*builder, LLVMInt8Type(), mem_loc, "res"), + LLVMInt32Type(), "zext8to32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(lhu, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = LLVMBuildZExt( + *builder, LLVMBuildLoad2(*builder, LLVMInt16Type(), mem_loc, "res"), + LLVMInt32Type(), "zext16to32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(sb, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 8, t2c_gen_rs2_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs2, mem_loc); +}) + +T2C_OP(sh, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 16, t2c_gen_rs2_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs2, mem_loc); +}) + +T2C_OP(sw, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs2, mem_loc); +}) + +T2C_OP(addi, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(Add, val_rs1, ir->imm); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(slti, { + LLVMValueRef addr_rd = t2c_gen_rd_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_CMP_IMM32(SLT, val_rs1, ir->imm); + LLVMBasicBlockRef new_entry = LLVMAppendBasicBlock(start, "new_entry"); + LLVMBuilderRef new_builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(new_builder, new_entry); + LLVMBasicBlockRef taken = LLVMAppendBasicBlock(start, "taken"); + LLVMBuilderRef builder2 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder2, taken); + T2C_LLVM_GEN_STORE_IMM32(builder2, 1, addr_rd); + LLVMBuildBr(builder2, new_entry); + LLVMBasicBlockRef untaken = LLVMAppendBasicBlock(start, "untaken"); + LLVMBuilderRef builder3 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder3, untaken); + T2C_LLVM_GEN_STORE_IMM32(builder3, 0, addr_rd); + LLVMBuildBr(builder3, new_entry); + LLVMBuildCondBr(*builder, cmp, taken, untaken); + *entry = new_entry; + *builder = new_builder; +}) + +T2C_OP(sltiu, { + LLVMValueRef addr_rd = t2c_gen_rd_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_CMP_IMM32(ULT, val_rs1, ir->imm); + LLVMBasicBlockRef new_entry = LLVMAppendBasicBlock(start, "new_entry"); + LLVMBuilderRef new_builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(new_builder, new_entry); + LLVMBasicBlockRef taken = LLVMAppendBasicBlock(start, "taken"); + LLVMBuilderRef builder2 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder2, taken); + T2C_LLVM_GEN_STORE_IMM32(builder2, 1, addr_rd); + LLVMBuildBr(builder2, new_entry); + LLVMBasicBlockRef untaken = LLVMAppendBasicBlock(start, "untaken"); + LLVMBuilderRef builder3 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder3, untaken); + T2C_LLVM_GEN_STORE_IMM32(builder3, 0, addr_rd); + LLVMBuildBr(builder3, new_entry); + LLVMBuildCondBr(*builder, cmp, taken, untaken); + *entry = new_entry; + *builder = new_builder; +}) + +T2C_OP(xori, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(Xor, val_rs1, ir->imm); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(ori, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(Or, val_rs1, ir->imm); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(andi, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(And, val_rs1, ir->imm); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(slli, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(Shl, val_rs1, ir->imm & 0x1f); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(srli, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(LShr, val_rs1, ir->imm & 0x1f); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(srai, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(AShr, val_rs1, ir->imm & 0x1f); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(add, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildAdd(*builder, val_rs1, val_rs2, "add"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(sub, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildSub(*builder, val_rs1, val_rs2, "sub"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(sll, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + val_rs2 = T2C_LLVM_GEN_ALU32_IMM(And, val_rs2, 0x1f); + LLVMValueRef res = LLVMBuildShl(*builder, val_rs1, val_rs2, "sll"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(slt, { + LLVMValueRef addr_rd = t2c_gen_rd_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + T2C_LLVM_GEN_CMP(SLT, val_rs1, val_rs2); + LLVMBasicBlockRef new_entry = LLVMAppendBasicBlock(start, "new_entry"); + LLVMBuilderRef new_builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(new_builder, new_entry); + LLVMBasicBlockRef taken = LLVMAppendBasicBlock(start, "taken"); + LLVMBuilderRef builder2 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder2, taken); + T2C_LLVM_GEN_STORE_IMM32(builder2, 1, addr_rd); + LLVMBuildBr(builder2, new_entry); + LLVMBasicBlockRef untaken = LLVMAppendBasicBlock(start, "untaken"); + LLVMBuilderRef builder3 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder3, untaken); + T2C_LLVM_GEN_STORE_IMM32(builder3, 0, addr_rd); + LLVMBuildBr(builder3, new_entry); + LLVMBuildCondBr(*builder, cmp, taken, untaken); + *entry = new_entry; + *builder = new_builder; +}) + +T2C_OP(sltu, { + LLVMValueRef addr_rd = t2c_gen_rd_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + T2C_LLVM_GEN_CMP(ULT, val_rs1, val_rs2); + LLVMBasicBlockRef new_entry = LLVMAppendBasicBlock(start, "new_entry"); + LLVMBuilderRef new_builder = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(new_builder, new_entry); + LLVMBasicBlockRef taken = LLVMAppendBasicBlock(start, "taken"); + LLVMBuilderRef builder2 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder2, taken); + T2C_LLVM_GEN_STORE_IMM32(builder2, 1, addr_rd); + LLVMBuildBr(builder2, new_entry); + LLVMBasicBlockRef untaken = LLVMAppendBasicBlock(start, "untaken"); + LLVMBuilderRef builder3 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder3, untaken); + T2C_LLVM_GEN_STORE_IMM32(builder3, 0, addr_rd); + LLVMBuildBr(builder3, new_entry); + LLVMBuildCondBr(*builder, cmp, taken, untaken); + *entry = new_entry; + *builder = new_builder; +}) + +T2C_OP(xor, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildXor(*builder, val_rs1, val_rs2, "xor"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(srl, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + val_rs2 = T2C_LLVM_GEN_ALU32_IMM(And, val_rs2, 0x1f); + LLVMValueRef res = LLVMBuildLShr(*builder, val_rs1, val_rs2, "sll"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(sra, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + val_rs2 = T2C_LLVM_GEN_ALU32_IMM(And, val_rs2, 0x1f); + LLVMValueRef res = LLVMBuildAShr(*builder, val_rs1, val_rs2, "sll"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(or, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildOr(*builder, val_rs1, val_rs2, "xor"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(and, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildAnd(*builder, val_rs1, val_rs2, "xor"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(ecall, { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc, + t2c_gen_PC_addr(start, builder, ir)); + t2c_gen_call_io_func(start, builder, param_types, 8); + LLVMBuildRetVoid(*builder); +}) + +T2C_OP(ebreak, { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc, + t2c_gen_PC_addr(start, builder, ir)); + t2c_gen_call_io_func(start, builder, param_types, 9); + LLVMBuildRetVoid(*builder); +}) + +T2C_OP(wfi, { __UNREACHABLE; }) + +T2C_OP(uret, { __UNREACHABLE; }) + +T2C_OP(sret, { __UNREACHABLE; }) + +T2C_OP(hret, { __UNREACHABLE; }) + +T2C_OP(mret, { __UNREACHABLE; }) + +#if RV32_HAS(Zifencei) +T2C_OP(fencei, { __UNREACHABLE; }) +#endif + +#if RV32_HAS(Zicsr) +T2C_OP(csrrw, { __UNREACHABLE; }) + +T2C_OP(csrrs, { __UNREACHABLE; }) + +T2C_OP(csrrc, { __UNREACHABLE; }) + +T2C_OP(csrrwi, { __UNREACHABLE; }) + +T2C_OP(csrrsi, { __UNREACHABLE; }) + +T2C_OP(csrrci, { __UNREACHABLE; }) +#endif + +#if RV32_HAS(EXT_M) +T2C_OP(mul, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + val_rs1 = LLVMBuildSExt(*builder, val_rs1, LLVMInt64Type(), "sextrs1to64"); + val_rs2 = LLVMBuildSExt(*builder, val_rs2, LLVMInt64Type(), "sextrs2to64"); + LLVMValueRef res = LLVMBuildMul(*builder, val_rs1, val_rs2, "mul"); + res = T2C_LLVM_GEN_ALU64_IMM(And, res, 0xFFFFFFFF); + res = LLVMBuildTrunc(*builder, res, LLVMInt32Type(), "sextresto32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(mulh, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + val_rs1 = LLVMBuildSExt(*builder, val_rs1, LLVMInt64Type(), "sextrs1to64"); + val_rs2 = LLVMBuildSExt(*builder, val_rs2, LLVMInt64Type(), "sextrs2to64"); + LLVMValueRef res = LLVMBuildMul(*builder, val_rs1, val_rs2, "mul"); + res = T2C_LLVM_GEN_ALU64_IMM(LShr, res, 32); + res = LLVMBuildTrunc(*builder, res, LLVMInt32Type(), "sextresto32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(mulhsu, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + val_rs1 = LLVMBuildSExt(*builder, val_rs1, LLVMInt64Type(), "sextrs1to64"); + val_rs2 = LLVMBuildZExt(*builder, val_rs2, LLVMInt64Type(), "zextrs2to64"); + LLVMValueRef res = LLVMBuildMul(*builder, val_rs1, val_rs2, "mul"); + res = T2C_LLVM_GEN_ALU64_IMM(LShr, res, 32); + res = LLVMBuildTrunc(*builder, res, LLVMInt32Type(), "sextresto32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(mulhu, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + val_rs1 = LLVMBuildZExt(*builder, val_rs1, LLVMInt64Type(), "sextrs1to64"); + val_rs2 = LLVMBuildZExt(*builder, val_rs2, LLVMInt64Type(), "zextrs2to64"); + LLVMValueRef res = LLVMBuildMul(*builder, val_rs1, val_rs2, "mul"); + res = T2C_LLVM_GEN_ALU64_IMM(LShr, res, 32); + res = LLVMBuildTrunc(*builder, res, LLVMInt32Type(), "sextresto32"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(div, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildSDiv(*builder, val_rs1, val_rs2, "sdiv"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(divu, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildUDiv(*builder, val_rs1, val_rs2, "udiv"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(rem, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildSRem(*builder, val_rs1, val_rs2, "srem"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(remu, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildURem(*builder, val_rs1, val_rs2, "urem"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) +#endif + +#if RV32_HAS(EXT_A) +T2C_OP(lrw, { __UNREACHABLE; }) + +T2C_OP(scw, { __UNREACHABLE; }) + +T2C_OP(amoswapw, { __UNREACHABLE; }) + +T2C_OP(amoaddw, { __UNREACHABLE; }) + +T2C_OP(amoxorw, { __UNREACHABLE; }) + +T2C_OP(amoandw, { __UNREACHABLE; }) + +T2C_OP(amoorw, { __UNREACHABLE; }) + +T2C_OP(amominw, { __UNREACHABLE; }) + +T2C_OP(amomaxw, { __UNREACHABLE; }) + +T2C_OP(amominuw, { __UNREACHABLE; }) + +T2C_OP(amomaxuw, { __UNREACHABLE; }) +#endif + +#if RV32_HAS(EXT_F) +T2C_OP(flw, { __UNREACHABLE; }) + +T2C_OP(fsw, { __UNREACHABLE; }) + +T2C_OP(fmadds, { __UNREACHABLE; }) + +T2C_OP(fmsubs, { __UNREACHABLE; }) + +T2C_OP(fnmsubs, { __UNREACHABLE; }) + +T2C_OP(fnmadds, { __UNREACHABLE; }) + +T2C_OP(fadds, { __UNREACHABLE; }) + +T2C_OP(fsubs, { __UNREACHABLE; }) + +T2C_OP(fmuls, { __UNREACHABLE; }) + +T2C_OP(fdivs, { __UNREACHABLE; }) + +T2C_OP(fsqrts, { __UNREACHABLE; }) + +T2C_OP(fsgnjs, { __UNREACHABLE; }) + +T2C_OP(fsgnjns, { __UNREACHABLE; }) + +T2C_OP(fsgnjxs, { __UNREACHABLE; }) + +T2C_OP(fmins, { __UNREACHABLE; }) + +T2C_OP(fmaxs, { __UNREACHABLE; }) + +T2C_OP(fcvtws, { __UNREACHABLE; }) + +T2C_OP(fcvtwus, { __UNREACHABLE; }) + +T2C_OP(fmvxw, { __UNREACHABLE; }) + +T2C_OP(feqs, { __UNREACHABLE; }) + +T2C_OP(flts, { __UNREACHABLE; }) + +T2C_OP(fles, { __UNREACHABLE; }) + +T2C_OP(fclasss, { __UNREACHABLE; }) + +T2C_OP(fcvtsw, { __UNREACHABLE; }) + +T2C_OP(fcvtswu, { __UNREACHABLE; }) + +T2C_OP(fmvwx, { __UNREACHABLE; }) +#endif + +#if RV32_HAS(EXT_C) +T2C_OP(caddi4spn, { + T2C_LLVM_GEN_LOAD_VMREG(sp, 32, t2c_gen_sp_addr(start, builder, ir)); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(Add, val_sp, (int16_t) ir->imm); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) +T2C_OP(clw, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + LLVMValueRef res = + LLVMBuildLoad2(*builder, LLVMInt32Type(), mem_loc, "res"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(csw, { + LLVMValueRef mem_loc = t2c_gen_mem_loc(start, builder, ir, mem_base); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs2, mem_loc); +}) + +T2C_OP(cnop, { return; }) + +T2C_OP(caddi, { + LLVMValueRef addr_rd = t2c_gen_rd_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rd, 32, addr_rd); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(Add, val_rd, (int16_t) ir->imm); + LLVMBuildStore(*builder, res, addr_rd); +}) + +T2C_OP(cjal, { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + 2, + t2c_gen_ra_addr(start, builder, ir)); + if (ir->branch_taken) + *taken_builder = *builder; + else { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + ir->imm, + t2c_gen_PC_addr(start, builder, ir)); + LLVMBuildRetVoid(*builder); + } +}) + +T2C_OP(cli, { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->imm, + t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(caddi16sp, { + LLVMValueRef addr_rd = t2c_gen_rd_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rd, 32, addr_rd); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(Add, val_rd, ir->imm); + LLVMBuildStore(*builder, res, addr_rd); +}) + +T2C_OP(clui, { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->imm, + t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(csrli, { + LLVMValueRef addr_rs1 = t2c_gen_rs1_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, addr_rs1); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(LShr, val_rs1, ir->shamt); + LLVMBuildStore(*builder, res, addr_rs1); +}) + +T2C_OP(csrai, { + LLVMValueRef addr_rs1 = t2c_gen_rs1_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, addr_rs1); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(AShr, val_rs1, ir->shamt); + LLVMBuildStore(*builder, res, addr_rs1); +}) + +T2C_OP(candi, { + LLVMValueRef addr_rs1 = t2c_gen_rs1_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, addr_rs1); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(And, val_rs1, ir->imm); + LLVMBuildStore(*builder, res, addr_rs1); +}) + +T2C_OP(csub, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildSub(*builder, val_rs1, val_rs2, "sub"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(cxor, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildXor(*builder, val_rs1, val_rs2, "xor"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(cor, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildOr(*builder, val_rs1, val_rs2, "xor"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(cand, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildAnd(*builder, val_rs1, val_rs2, "xor"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(cj, { + if (ir->branch_taken) + *taken_builder = *builder; + else { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + ir->imm, + t2c_gen_PC_addr(start, builder, ir)); + LLVMBuildRetVoid(*builder); + } +}) + +T2C_OP(cbeqz, { + LLVMValueRef addr_PC = t2c_gen_PC_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_CMP_IMM32(EQ, val_rs1, 0); + LLVMBasicBlockRef taken = LLVMAppendBasicBlock(start, "taken"); + LLVMBuilderRef builder2 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder2, taken); + if (ir->branch_taken) + *taken_builder = builder2; + else { + T2C_LLVM_GEN_STORE_IMM32(builder2, ir->pc + ir->imm, addr_PC); + LLVMBuildRetVoid(builder2); + } + + LLVMBasicBlockRef untaken = LLVMAppendBasicBlock(start, "untaken"); + LLVMBuilderRef builder3 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder3, untaken); + if (ir->branch_untaken) + *untaken_builder = builder3; + else { + T2C_LLVM_GEN_STORE_IMM32(builder3, ir->pc + 2, addr_PC); + LLVMBuildRetVoid(builder3); + } + LLVMBuildCondBr(*builder, cmp, taken, untaken); +}) + +T2C_OP(cbnez, { + LLVMValueRef addr_PC = t2c_gen_PC_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_CMP_IMM32(NE, val_rs1, 0); + LLVMBasicBlockRef taken = LLVMAppendBasicBlock(start, "taken"); + LLVMBuilderRef builder2 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder2, taken); + if (ir->branch_taken) + *taken_builder = builder2; + else { + T2C_LLVM_GEN_STORE_IMM32(builder2, ir->pc + ir->imm, addr_PC); + LLVMBuildRetVoid(builder2); + } + + LLVMBasicBlockRef untaken = LLVMAppendBasicBlock(start, "untaken"); + LLVMBuilderRef builder3 = LLVMCreateBuilder(); + LLVMPositionBuilderAtEnd(builder3, untaken); + if (ir->branch_untaken) + *untaken_builder = builder3; + else { + T2C_LLVM_GEN_STORE_IMM32(builder3, ir->pc + 2, addr_PC); + LLVMBuildRetVoid(builder3); + } + LLVMBuildCondBr(*builder, cmp, taken, untaken); +}) + +T2C_OP(cslli, { + LLVMValueRef addr_rd = t2c_gen_rd_addr(start, builder, ir); + T2C_LLVM_GEN_LOAD_VMREG(rd, 32, addr_rd); + LLVMValueRef res = T2C_LLVM_GEN_ALU32_IMM(Shl, val_rd, (uint8_t) ir->imm); + LLVMBuildStore(*builder, res, addr_rd); +}) + +T2C_OP(clwsp, { + LLVMValueRef val_sp = LLVMBuildZExt( + *builder, + LLVMBuildLoad2(*builder, LLVMInt32Type(), + t2c_gen_sp_addr(start, builder, ir), "val_sp"), + LLVMInt64Type(), "zext32to64"); + LLVMValueRef addr = LLVMBuildAdd( + *builder, val_sp, + LLVMConstInt(LLVMInt64Type(), ir->imm + mem_base, true), "addr"); + LLVMValueRef cast_addr = LLVMBuildIntToPtr( + *builder, addr, LLVMPointerType(LLVMInt32Type(), 0), "cast"); + LLVMValueRef res = + LLVMBuildLoad2(*builder, LLVMInt32Type(), cast_addr, "res"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(cjr, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs1, t2c_gen_PC_addr(start, builder, ir)); + LLVMBuildRetVoid(*builder); +}) + +T2C_OP(cmv, { + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs2, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(cebreak, { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc, + t2c_gen_PC_addr(start, builder, ir)); + t2c_gen_call_io_func(start, builder, param_types, 9); + LLVMBuildRetVoid(*builder); +}) + +T2C_OP(cjalr, { + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->pc + 2, + t2c_gen_ra_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + LLVMBuildStore(*builder, val_rs1, t2c_gen_PC_addr(start, builder, ir)); + LLVMBuildRetVoid(*builder); +}) + +T2C_OP(cadd, { + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, t2c_gen_rs2_addr(start, builder, ir)); + LLVMValueRef res = LLVMBuildAdd(*builder, val_rs1, val_rs2, "add"); + LLVMBuildStore(*builder, res, t2c_gen_rd_addr(start, builder, ir)); +}) + +T2C_OP(cswsp, { + LLVMValueRef addr_rs2 = t2c_gen_rs2_addr(start, builder, ir); + LLVMValueRef val_sp = LLVMBuildZExt( + *builder, + LLVMBuildLoad2(*builder, LLVMInt32Type(), + t2c_gen_sp_addr(start, builder, ir), "val_sp"), + LLVMInt64Type(), "zext32to64"); + T2C_LLVM_GEN_LOAD_VMREG(rs2, 32, addr_rs2); + LLVMValueRef addr = LLVMBuildAdd( + *builder, val_sp, + LLVMConstInt(LLVMInt64Type(), ir->imm + mem_base, true), "addr"); + LLVMValueRef cast_addr = LLVMBuildIntToPtr( + *builder, addr, LLVMPointerType(LLVMInt32Type(), 0), "cast"); + LLVMBuildStore(*builder, val_rs2, cast_addr); +}) +#endif + +#if RV32_HAS(EXT_C) && RV32_HAS(EXT_F) +T2C_OP(cflwsp, { __UNREACHABLE; }) + +T2C_OP(cfswsp, { __UNREACHABLE; }) + +T2C_OP(cflw, { __UNREACHABLE; }) + +T2C_OP(cfsw, { __UNREACHABLE; }) +#endif + +T2C_OP(fuse1, { + opcode_fuse_t *fuse = ir->fuse; + for (int i = 0; i < ir->imm2; i++) { + LLVMValueRef rd_offset = + LLVMConstInt(LLVMInt32Type(), + offsetof(riscv_t, X) / sizeof(int) + fuse[i].rd, true); + LLVMValueRef addr_rd = LLVMBuildInBoundsGEP2(*builder, LLVMInt32Type(), + LLVMGetParam(start, 0), + &rd_offset, 1, "addr_rd"); + LLVMBuildStore(*builder, + LLVMConstInt(LLVMInt32Type(), fuse[i].imm, true), + addr_rd); + } +}) + +T2C_OP(fuse2, { + LLVMValueRef addr_rd = t2c_gen_rd_addr(start, builder, ir); + T2C_LLVM_GEN_STORE_IMM32(*builder, ir->imm, addr_rd); + T2C_LLVM_GEN_LOAD_VMREG(rs1, 32, t2c_gen_rs1_addr(start, builder, ir)); + T2C_LLVM_GEN_LOAD_VMREG(rd, 32, addr_rd); + LLVMValueRef res = LLVMBuildAdd(*builder, val_rs1, val_rd, "add"); + LLVMBuildStore(*builder, res, t2c_gen_rs2_addr(start, builder, ir)); +}) + +T2C_OP(fuse3, { + opcode_fuse_t *fuse = ir->fuse; + for (int i = 0; i < ir->imm2; i++) { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, (rv_insn_t *) (&fuse[i]), mem_base); + T2C_LLVM_GEN_LOAD_VMREG( + rs2, 32, + t2c_gen_rs2_addr(start, builder, (rv_insn_t *) (&fuse[i]))); + LLVMBuildStore(*builder, val_rs2, mem_loc); + } +}) + +T2C_OP(fuse4, { + opcode_fuse_t *fuse = ir->fuse; + for (int i = 0; i < ir->imm2; i++) { + LLVMValueRef mem_loc = + t2c_gen_mem_loc(start, builder, (rv_insn_t *) (&fuse[i]), mem_base); + LLVMValueRef res = + LLVMBuildLoad2(*builder, LLVMInt32Type(), mem_loc, "res"); + LLVMBuildStore( + *builder, res, + t2c_gen_rd_addr(start, builder, (rv_insn_t *) (&fuse[i]))); + } +}) + +T2C_OP(fuse5, { + opcode_fuse_t *fuse = ir->fuse; + for (int i = 0; i < ir->imm2; i++) { + switch (fuse[i].opcode) { + case rv_insn_slli: + t2c_slli(builder, param_types, start, entry, taken_builder, + untaken_builder, mem_base, (rv_insn_t *) (&fuse[i])); + break; + case rv_insn_srli: + t2c_srli(builder, param_types, start, entry, taken_builder, + untaken_builder, mem_base, (rv_insn_t *) (&fuse[i])); + break; + case rv_insn_srai: + t2c_srai(builder, param_types, start, entry, taken_builder, + untaken_builder, mem_base, (rv_insn_t *) (&fuse[i])); + break; + default: + __UNREACHABLE; + break; + } + } +})