From 72c94d987e823f06ca134ff6d71f4589d9f800ee Mon Sep 17 00:00:00 2001 From: Alex Mikhalevich <2990126+alexmikhalevich@users.noreply.github.com> Date: Mon, 26 Sep 2022 19:39:17 -0300 Subject: [PATCH] feat: implement hypervisor extension --- hypervisor.md | 129 ++ lib/grpc-interfaces | 2 +- set_lua_env_hp.sh | 4 + src/clua-i-virtual-machine.cpp | 57 + src/clua-machine-util.cpp | 54 + src/grpc-virtual-machine.cpp | 152 +++ src/grpc-virtual-machine.h | 38 + src/i-state-access.h | 247 ++++ src/i-virtual-machine.h | 228 ++++ src/interpret.cpp | 2000 +++++++++++++++++++++++----- src/jsonrpc-virtual-machine.cpp | 144 ++ src/jsonrpc-virtual-machine.h | 38 + src/machine-c-api.cpp | 23 +- src/machine-c-api.h | 381 +++++- src/machine-config.h | 19 + src/machine-state.h | 43 +- src/machine.cpp | 359 ++++- src/machine.h | 183 ++- src/protobuf-util.cpp | 76 ++ src/riscv-constants.h | 345 ++++- src/shadow-state-factory.cpp | 19 + src/shadow-state.h | 38 + src/state-access.h | 168 ++- src/test-machine-c-api.cpp | 68 +- src/tests/machine-bind.lua | 23 +- src/translate-virtual-address.h | 213 ++- src/uarch-bridge.h | 135 +- src/virtual-machine.cpp | 152 +++ src/virtual-machine.h | 38 + uarch/uarch-machine-state-access.h | 174 ++- 30 files changed, 5082 insertions(+), 468 deletions(-) create mode 100644 hypervisor.md create mode 100755 set_lua_env_hp.sh diff --git a/hypervisor.md b/hypervisor.md new file mode 100644 index 000000000..710c154c7 --- /dev/null +++ b/hypervisor.md @@ -0,0 +1,129 @@ +# Hypervisor usage instructions + +The hypervisor feature is still at a highly experimental stage. + +In this document, we will use the term _host_ (or _host machine_) to address a cartesi machine instance that hosts a hypervisor, and the term _guest_ (or _guest machine_, or _virtual machine_) to address a machine running inside a hypervisor. + +## Quick start + +### Prepare the artifacts + +To test the hypervisor without building everything from scratch, you could download the rootfs and the kernel from google drive: +- [rootfs](https://drive.google.com/file/d/1Hy9VibEf6SZU4qtqqb8n5CzHq-eN7uO7/view?usp=share_link) +- [kernel](https://drive.google.com/file/d/1Wc9yAJLpxxg14aFDPvYcQQmrA7JPt1ZC/view?usp=share_link) + +### Build the emulator with a hypervisor support + +To build the emulator with hypervisor support you have to use the `hypervisor` [branch](https://github.com/cartesi-corp/machine-emulator/tree/feature/hypervisor) and follow the [regular build instructions](https://github.com/cartesi-corp/machine-emulator/blob/develop/README.md). After you execute `make install`, the emulator with hypervisor support will be installed in the `/opt/cartesi-hp` folder. + +### Run the virtual machine + +First of all, you have to make sure that you are using the correct emulator version. There is a special `set_lua_env_hp.sh` script for this in the repo root you may want to use. The command `source set_lua_env_hp.sh` will do the job. + +#### Booting a host + +To launch a new cartesi machine instance, use the following command: + +```bash +/opt/cartesi-hp/bin/cartesi-machine \ + --ram-image=/path/to/downloaded/opensbi.bin \ + --rom-image=/opt/cartesi/share/images/rom-v0.13.0.bin \ + --ram-length=1024Mi \ + --flash-drive=label:root,filename:/path/to/downloaded/rootfs-v0.15.0-dirty.ext2 \ + -i -- "/bin/sh" +``` + +This will give you a command prompt inside a host. + +#### Booting a guest + +All the hypervisor stuff you need is located inside the `/hp` folder on the host. To execute a virtual machine follow these steps: +1. load the `kvm` kernel module: `cd /hp && insmod kvm.ko`; +1. start the virtual machine with the provided script: `./start_machine.sh`. + +At this point, you should get a command prompt inside a virtual machine. + +### Looking around + +#### Testing network + +Inside both host and guest file systems you will find a `sender.py` and a `receiver.py` Python scripts. For the host, the scripts are located inside the `/hp` folder; for the guest look for them inside the `/opt` folder. These scripts could be used to test a network connection between the host and the guest. + - `receiver.py
`: listens for the incoming data on the given address, and prints the data as soon as it is received; + - `sender.py
`: sends the `data` to the given address. + +#### Guest startup script + +The `init` process inside the guest executes an `/opt/start.sh` script. You may want to modify this script to customize the guest startup behavior. + +#### Modifying a guest file system + +At some point, you may want to persist changes in the guest file system. The corresponding `.ext2` file is located inside the host file system: `/hp/rootfs-virt.ext2`. You can mount this file and make any changes to the corresponding file system: + +```bash +$ sudo mount /path/to/downloaded/rootfs-v0.15.0-dirty.ext2 /mnt +$ sudo mount /mnt/hp/rootfs-virt.ext2 /mnt-virt +... +$ sudo umount /mnt-virt +$ sudo umount /mnt +``` + +## Bootstrapping from scratch + +### Build the emulator with a hypervisor support + +Please, refer to [this section](https://github.com/cartesi-corp/machine-emulator/blob/hypervisor/hypervisor.md#build-the-emulator-with-a-hypervisor-support). + +### Build a kernel + +The hypervisor extension support for the RISC-V architecture is available only in the Linux kernel v6.0.9+. So, the first step you have to do is to clone the `cartesi-corp/linux` repo and checkout the corresponding branch: + +```bash +$ git clone git@github.com:cartesi-corp/linux.git +$ git checkout update/linux-6.0.9-ctsi-y +``` + +To build the kernel you have to use the [correct config](https://github.com/cartesi-corp/image-kernel/blob/hypervisor-config/configs/kvm-linux-config). Do not forget to copy it to your Linux kernel repo root directory. + +You also have to use `opensbi` to boot the kernel to have the compatible SBI interface version: + +```bash +$ git clone git@github.com:cartesi-corp/opensbi.git +$ git checkout feature/cartesi-legacy +``` + +Now you are ready to build a kernel using the Cartesi toolchain: + +```bash +$ docker run -v /path/to/linux:/linux -v /path/to/opensbi:/opensbi -it cartesicorp/toolchain:0.14.0 +$ export ARCH=riscv; export CROSS_COMPILE=/opt/riscv/riscv64-cartesi-linux-gnu/bin/riscv64-cartesi-linux-gnu-; make Image +$ export FW_PAYLOAD_PATH=/linux/arch/riscv/boot/Image; export PLATFORM=cartesi; make +``` + +You should have three files as output: +1. host kernel: `/path/to/opensbi/platform/cartesi/firmware/fw_payload.bin`; +1. guest kernel: `/path/to/linux/arch/riscv/boot/Image`; +1. kvm kernel module: `/path/to/linux/arch/riscv/kvm/kvm.ko`. + +### Build root file systems + +To be able to work with a hypervisor, you need root user permissions. The current root file system build does not provide this capability, so you have to use the version from the `hypervisor` [branch](https://github.com/cartesi-corp/image-rootfs/tree/hypervisor) to fix this. Other aspects are not different from the [regular build process](https://github.com/cartesi-corp/image-rootfs/blob/develop/README.md). Just keep in mind that after the host rootfs is compiled, you will have to copy the hypervisor-related files to it (KVM kernel module, lkvm tool, guest kernel, guest root file system, and any supporting scripts), so there should be enough free space. + +The same rootfs build may be used both for the host and the guest. + +### Build the LKVM tool + +To run a virtual machine [we need](https://github.com/kvm-riscv/howto/wiki/KVM-RISCV64-on-Spike#4-add-libfdt-library-to-cross_compile-sysroot-directory) lkvm tool. Here are the steps to build it (should be executed inside the toolchain container): + +```bash +$ git clone git://git.kernel.org/pub/scm/utils/dtc/dtc.git +$ cd dtc +$ export ARCH=riscv; export CROSS_COMPILE=/opt/riscv/riscv64-cartesi-linux-gnu/bin/riscv64-cartesi-linux-gnu-; export CC="${CROSS_COMPILE}gcc -mabi=lp64d -march=rv64gc" +$ SYSROOT=$($CC -print-sysroot) +$ make libfdt +$ make NO_PYTHON=1 NO_YAML=1 DESTDIR=$SYSROOT PREFIX=/usr LIBDIR=/usr/lib64/lp64d install-lib install-includes +$ cd .. +$ git clone https://git.kernel.org/pub/scm/linux/kernel/git/will/kvmtool.git; cd kvmtool; make lkvm-static +$ ${CROSS_COMPILE}strip lkvm-static +``` + +The above commands will create `kvmtool/lkvm-static` that you need to copy to your host root file system. diff --git a/lib/grpc-interfaces b/lib/grpc-interfaces index 4ec6e6620..193192249 160000 --- a/lib/grpc-interfaces +++ b/lib/grpc-interfaces @@ -1 +1 @@ -Subproject commit 4ec6e6620abce44048b9a8493124683ef15e0731 +Subproject commit 1931922492dbdaf64a0cf17d91e7649c03edcebc diff --git a/set_lua_env_hp.sh b/set_lua_env_hp.sh new file mode 100755 index 000000000..f4682393d --- /dev/null +++ b/set_lua_env_hp.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +export LUA_CPATH="/opt/cartesi-hp/lib/?.so;/opt/cartesi-hp/lib/lua/5.3/?.so;/opt/cartesi-hp/lib/lua/5.3/cartesi-hp/?.so;/usr/lib/lua/5.3/?.so" +export LUA_PATH="/opt/cartesi-hp/bin/?.lua;/opt/cartesi-hp/share/lua/5.3/?.lua;/usr/share/lua/5.3/?.lua" diff --git a/src/clua-i-virtual-machine.cpp b/src/clua-i-virtual-machine.cpp index bcbca0026..5033d1744 100644 --- a/src/clua-i-virtual-machine.cpp +++ b/src/clua-i-virtual-machine.cpp @@ -138,6 +138,25 @@ IMPL_MACHINE_OBJ_READ_WRITE(stval) IMPL_MACHINE_OBJ_READ_WRITE(satp) IMPL_MACHINE_OBJ_READ_WRITE(scounteren) IMPL_MACHINE_OBJ_READ_WRITE(senvcfg) +IMPL_MACHINE_OBJ_READ_WRITE(hstatus) +IMPL_MACHINE_OBJ_READ_WRITE(hideleg) +IMPL_MACHINE_OBJ_READ_WRITE(hedeleg) +IMPL_MACHINE_OBJ_READ_WRITE(hie) +IMPL_MACHINE_OBJ_READ_WRITE(hip) +IMPL_MACHINE_OBJ_READ_WRITE(hvip) +IMPL_MACHINE_OBJ_READ_WRITE(hgatp) +IMPL_MACHINE_OBJ_READ_WRITE(henvcfg) +IMPL_MACHINE_OBJ_READ_WRITE(htimedelta) +IMPL_MACHINE_OBJ_READ_WRITE(htval) +IMPL_MACHINE_OBJ_READ_WRITE(vsepc) +IMPL_MACHINE_OBJ_READ_WRITE(vsstatus) +IMPL_MACHINE_OBJ_READ_WRITE(vscause) +IMPL_MACHINE_OBJ_READ_WRITE(vstval) +IMPL_MACHINE_OBJ_READ_WRITE(vstvec) +IMPL_MACHINE_OBJ_READ_WRITE(vsscratch) +IMPL_MACHINE_OBJ_READ_WRITE(vsatp) +IMPL_MACHINE_OBJ_READ_WRITE(vsie) +IMPL_MACHINE_OBJ_READ_WRITE(vsip) IMPL_MACHINE_OBJ_READ_WRITE(ilrsc) IMPL_MACHINE_OBJ_READ_WRITE(iflags) IMPL_MACHINE_OBJ_READ_WRITE(htif_tohost) @@ -588,6 +607,25 @@ static const auto machine_obj_index = cartesi::clua_make_luaL_Reg_array({ {"read_sscratch", machine_obj_index_read_sscratch}, {"read_stval", machine_obj_index_read_stval}, {"read_stvec", machine_obj_index_read_stvec}, + {"read_hstatus", machine_obj_index_read_hstatus}, + {"read_hideleg", machine_obj_index_read_hideleg}, + {"read_hedeleg", machine_obj_index_read_hedeleg}, + {"read_hie", machine_obj_index_read_hie}, + {"read_hip", machine_obj_index_read_hip}, + {"read_hvip", machine_obj_index_read_hvip}, + {"read_hgatp", machine_obj_index_read_hgatp}, + {"read_henvcfg", machine_obj_index_read_henvcfg}, + {"read_htimedelta", machine_obj_index_read_htimedelta}, + {"read_htval", machine_obj_index_read_htval}, + {"read_vsepc", machine_obj_index_read_vsepc}, + {"read_vsstatus", machine_obj_index_read_vsstatus}, + {"read_vscause", machine_obj_index_read_vscause}, + {"read_vstval", machine_obj_index_read_vstval}, + {"read_vstvec", machine_obj_index_read_vstvec}, + {"read_vsscratch", machine_obj_index_read_vsscratch}, + {"read_vsatp", machine_obj_index_read_vsatp}, + {"read_vsie", machine_obj_index_read_vsie}, + {"read_vsip", machine_obj_index_read_vsip}, {"read_word", machine_obj_index_read_word}, {"read_x", machine_obj_index_read_x}, {"read_f", machine_obj_index_read_f}, @@ -637,6 +675,25 @@ static const auto machine_obj_index = cartesi::clua_make_luaL_Reg_array({ {"write_sscratch", machine_obj_index_write_sscratch}, {"write_stval", machine_obj_index_write_stval}, {"write_stvec", machine_obj_index_write_stvec}, + {"write_hstatus", machine_obj_index_write_hstatus}, + {"write_hideleg", machine_obj_index_write_hideleg}, + {"write_hedeleg", machine_obj_index_write_hedeleg}, + {"write_hie", machine_obj_index_write_hie}, + {"write_hip", machine_obj_index_write_hip}, + {"write_hvip", machine_obj_index_write_hvip}, + {"write_hgatp", machine_obj_index_write_hgatp}, + {"write_henvcfg", machine_obj_index_write_henvcfg}, + {"write_htimedelta", machine_obj_index_write_htimedelta}, + {"write_htval", machine_obj_index_write_htval}, + {"write_vsepc", machine_obj_index_write_vsepc}, + {"write_vsstatus", machine_obj_index_write_vsstatus}, + {"write_vscause", machine_obj_index_write_vscause}, + {"write_vstval", machine_obj_index_write_vstval}, + {"write_vstvec", machine_obj_index_write_vstvec}, + {"write_vsscratch", machine_obj_index_write_vsscratch}, + {"write_vsatp", machine_obj_index_write_vsatp}, + {"write_vsie", machine_obj_index_write_vsie}, + {"write_vsip", machine_obj_index_write_vsip}, {"write_x", machine_obj_index_write_x}, {"write_f", machine_obj_index_write_f}, {"replace_memory_range", machine_obj_index_replace_memory_range}, diff --git a/src/clua-machine-util.cpp b/src/clua-machine-util.cpp index 76c3ff26e..4f777d471 100644 --- a/src/clua-machine-util.cpp +++ b/src/clua-machine-util.cpp @@ -540,6 +540,24 @@ CM_PROC_CSR clua_check_cm_proc_csr(lua_State *L, int idx) try { {"satp", CM_PROC_SATP}, {"scounteren", CM_PROC_SCOUNTEREN}, {"senvcfg", CM_PROC_SENVCFG}, + {"hstatus", CM_PROC_HSTATUS}, + {"hedeleg", CM_PROC_HEDELEG}, + {"hideleg", CM_PROC_HIDELEG}, + {"hip", CM_PROC_HIP}, + {"hie", CM_PROC_HIE}, + {"hvip", CM_PROC_HVIP}, + {"hgatp", CM_PROC_HGATP}, + {"htimedelta", CM_PROC_HTIMEDELTA}, + {"htval", CM_PROC_HTVAL}, + {"vsepc", CM_PROC_VSEPC}, + {"vsstatus", CM_PROC_VSSTATUS}, + {"vscause", CM_PROC_VSCAUSE}, + {"vstval", CM_PROC_VSTVAL}, + {"vstvec", CM_PROC_VSTVEC}, + {"vsscratch", CM_PROC_VSSCRATCH}, + {"vsatp", CM_PROC_VSATP}, + {"vsip", CM_PROC_VSIP}, + {"vsie", CM_PROC_VSIE}, {"ilrsc", CM_PROC_ILRSC}, {"iflags", CM_PROC_IFLAGS}, {"clint_mtimecmp", CM_PROC_CLINT_MTIMECMP}, @@ -755,6 +773,24 @@ static void push_cm_processor_config(lua_State *L, const cm_processor_config *p) PUSH_CM_PROCESSOR_CONFIG_CSR(satp); PUSH_CM_PROCESSOR_CONFIG_CSR(scounteren); PUSH_CM_PROCESSOR_CONFIG_CSR(senvcfg); + PUSH_CM_PROCESSOR_CONFIG_CSR(hstatus); + PUSH_CM_PROCESSOR_CONFIG_CSR(hideleg); + PUSH_CM_PROCESSOR_CONFIG_CSR(hedeleg); + PUSH_CM_PROCESSOR_CONFIG_CSR(hip); + PUSH_CM_PROCESSOR_CONFIG_CSR(hvip); + PUSH_CM_PROCESSOR_CONFIG_CSR(hie); + PUSH_CM_PROCESSOR_CONFIG_CSR(hgatp); + PUSH_CM_PROCESSOR_CONFIG_CSR(htimedelta); + PUSH_CM_PROCESSOR_CONFIG_CSR(htval); + PUSH_CM_PROCESSOR_CONFIG_CSR(vsepc); + PUSH_CM_PROCESSOR_CONFIG_CSR(vsstatus); + PUSH_CM_PROCESSOR_CONFIG_CSR(vscause); + PUSH_CM_PROCESSOR_CONFIG_CSR(vstval); + PUSH_CM_PROCESSOR_CONFIG_CSR(vstvec); + PUSH_CM_PROCESSOR_CONFIG_CSR(vsscratch); + PUSH_CM_PROCESSOR_CONFIG_CSR(vsatp); + PUSH_CM_PROCESSOR_CONFIG_CSR(vsie); + PUSH_CM_PROCESSOR_CONFIG_CSR(vsip); PUSH_CM_PROCESSOR_CONFIG_CSR(ilrsc); PUSH_CM_PROCESSOR_CONFIG_CSR(iflags); } @@ -1079,6 +1115,24 @@ static void check_cm_processor_config(lua_State *L, int tabidx, cm_processor_con p->satp = opt_uint_field(L, -1, "satp", def->satp); p->scounteren = opt_uint_field(L, -1, "scounteren", def->scounteren); p->senvcfg = opt_uint_field(L, -1, "senvcfg", def->senvcfg); + p->hstatus = opt_uint_field(L, -1, "hstatus", def->hstatus); + p->hideleg = opt_uint_field(L, -1, "hideleg", def->hideleg); + p->hedeleg = opt_uint_field(L, -1, "hedeleg", def->hedeleg); + p->hip = opt_uint_field(L, -1, "hip", def->hip); + p->hvip = opt_uint_field(L, -1, "hvip", def->hvip); + p->hie = opt_uint_field(L, -1, "hie", def->hie); + p->hgatp = opt_uint_field(L, -1, "hgatp", def->hgatp); + p->htimedelta = opt_uint_field(L, -1, "htimedelta", def->htimedelta); + p->htval = opt_uint_field(L, -1, "htval", def->htval); + p->vsepc = opt_uint_field(L, -1, "vsepc", def->vsepc); + p->vsstatus = opt_uint_field(L, -1, "vsstatus", def->vsstatus); + p->vscause = opt_uint_field(L, -1, "vscause", def->vscause); + p->vstval = opt_uint_field(L, -1, "vstval", def->vstval); + p->vstvec = opt_uint_field(L, -1, "vstvec", def->vstvec); + p->vsscratch = opt_uint_field(L, -1, "vsscratch", def->vsscratch); + p->vsatp = opt_uint_field(L, -1, "vsatp", def->vsatp); + p->vsie = opt_uint_field(L, -1, "vsie", def->vsie); + p->vsip = opt_uint_field(L, -1, "vsip", def->vsip); p->ilrsc = opt_uint_field(L, -1, "ilrsc", def->ilrsc); p->iflags = opt_uint_field(L, -1, "iflags", def->iflags); lua_pop(L, 1); diff --git a/src/grpc-virtual-machine.cpp b/src/grpc-virtual-machine.cpp index 3832e25ac..82570bfd4 100644 --- a/src/grpc-virtual-machine.cpp +++ b/src/grpc-virtual-machine.cpp @@ -665,6 +665,158 @@ void grpc_virtual_machine::do_write_senvcfg(uint64_t val) { write_csr(csr::senvcfg, val); } +uint64_t grpc_virtual_machine::do_read_hstatus(void) const { + return read_csr(csr::hstatus); +} + +void grpc_virtual_machine::do_write_hstatus(uint64_t val) { + write_csr(csr::hstatus, val); +} + +uint64_t grpc_virtual_machine::do_read_hideleg(void) const { + return read_csr(csr::hideleg); +} + +void grpc_virtual_machine::do_write_hideleg(uint64_t val) { + write_csr(csr::hideleg, val); +} + +uint64_t grpc_virtual_machine::do_read_hedeleg(void) const { + return read_csr(csr::hedeleg); +} + +void grpc_virtual_machine::do_write_hedeleg(uint64_t val) { + write_csr(csr::hedeleg, val); +} + +uint64_t grpc_virtual_machine::do_read_hip(void) const { + return read_csr(csr::hip); +} + +void grpc_virtual_machine::do_write_hip(uint64_t val) { + write_csr(csr::hip, val); +} + +uint64_t grpc_virtual_machine::do_read_hvip(void) const { + return read_csr(csr::hvip); +} + +void grpc_virtual_machine::do_write_hvip(uint64_t val) { + write_csr(csr::hvip, val); +} + +uint64_t grpc_virtual_machine::do_read_hie(void) const { + return read_csr(csr::hie); +} + +void grpc_virtual_machine::do_write_hie(uint64_t val) { + write_csr(csr::hie, val); +} + +uint64_t grpc_virtual_machine::do_read_hgatp(void) const { + return read_csr(csr::hgatp); +} + +void grpc_virtual_machine::do_write_hgatp(uint64_t val) { + write_csr(csr::hgatp, val); +} + +uint64_t grpc_virtual_machine::do_read_henvcfg(void) const { + return read_csr(csr::henvcfg); +} + +void grpc_virtual_machine::do_write_henvcfg(uint64_t val) { + write_csr(csr::henvcfg, val); +} + +uint64_t grpc_virtual_machine::do_read_htimedelta(void) const { + return read_csr(csr::htimedelta); +} + +void grpc_virtual_machine::do_write_htimedelta(uint64_t val) { + write_csr(csr::htimedelta, val); +} + +uint64_t grpc_virtual_machine::do_read_htval(void) const { + return read_csr(csr::htval); +} + +void grpc_virtual_machine::do_write_htval(uint64_t val) { + write_csr(csr::htval, val); +} + +uint64_t grpc_virtual_machine::do_read_vsepc(void) const { + return read_csr(csr::vsepc); +} + +void grpc_virtual_machine::do_write_vsepc(uint64_t val) { + write_csr(csr::vsepc, val); +} + +uint64_t grpc_virtual_machine::do_read_vsstatus(void) const { + return read_csr(csr::vsstatus); +} + +void grpc_virtual_machine::do_write_vsstatus(uint64_t val) { + write_csr(csr::vsstatus, val); +} + +uint64_t grpc_virtual_machine::do_read_vscause(void) const { + return read_csr(csr::vscause); +} + +void grpc_virtual_machine::do_write_vscause(uint64_t val) { + write_csr(csr::vscause, val); +} + +uint64_t grpc_virtual_machine::do_read_vstval(void) const { + return read_csr(csr::vstval); +} + +void grpc_virtual_machine::do_write_vstval(uint64_t val) { + write_csr(csr::vstval, val); +} + +uint64_t grpc_virtual_machine::do_read_vstvec(void) const { + return read_csr(csr::vstvec); +} + +void grpc_virtual_machine::do_write_vstvec(uint64_t val) { + write_csr(csr::vstvec, val); +} + +uint64_t grpc_virtual_machine::do_read_vsscratch(void) const { + return read_csr(csr::vsscratch); +} + +void grpc_virtual_machine::do_write_vsscratch(uint64_t val) { + write_csr(csr::vsscratch, val); +} + +uint64_t grpc_virtual_machine::do_read_vsatp(void) const { + return read_csr(csr::vsatp); +} + +void grpc_virtual_machine::do_write_vsatp(uint64_t val) { + write_csr(csr::vsatp, val); +} + +uint64_t grpc_virtual_machine::do_read_vsip(void) const { + return read_csr(csr::vsip); +} + +void grpc_virtual_machine::do_write_vsip(uint64_t val) { + write_csr(csr::vsip, val); +} + +uint64_t grpc_virtual_machine::do_read_vsie(void) const { + return read_csr(csr::vsie); +} + +void grpc_virtual_machine::do_write_vsie(uint64_t val) { + write_csr(csr::vsie, val); +} + uint64_t grpc_virtual_machine::do_read_ilrsc(void) const { return read_csr(csr::ilrsc); } diff --git a/src/grpc-virtual-machine.h b/src/grpc-virtual-machine.h index d8a53359b..33adad4d4 100644 --- a/src/grpc-virtual-machine.h +++ b/src/grpc-virtual-machine.h @@ -175,6 +175,44 @@ class grpc_virtual_machine : public i_virtual_machine { void do_write_scounteren(uint64_t val) override; uint64_t do_read_senvcfg(void) const override; void do_write_senvcfg(uint64_t val) override; + uint64_t do_read_hstatus(void) const override; + void do_write_hstatus(uint64_t val) override; + uint64_t do_read_hideleg(void) const override; + void do_write_hideleg(uint64_t val) override; + uint64_t do_read_hedeleg(void) const override; + void do_write_hedeleg(uint64_t val) override; + uint64_t do_read_hip(void) const override; + void do_write_hip(uint64_t val) override; + uint64_t do_read_hvip(void) const override; + void do_write_hvip(uint64_t val) override; + uint64_t do_read_hie(void) const override; + void do_write_hie(uint64_t val) override; + uint64_t do_read_hgatp(void) const override; + void do_write_hgatp(uint64_t val) override; + uint64_t do_read_henvcfg(void) const override; + void do_write_henvcfg(uint64_t val) override; + uint64_t do_read_htimedelta(void) const override; + void do_write_htimedelta(uint64_t val) override; + uint64_t do_read_htval(void) const override; + void do_write_htval(uint64_t val) override; + uint64_t do_read_vsepc(void) const override; + void do_write_vsepc(uint64_t val) override; + uint64_t do_read_vsstatus(void) const override; + void do_write_vsstatus(uint64_t val) override; + uint64_t do_read_vscause(void) const override; + void do_write_vscause(uint64_t val) override; + uint64_t do_read_vstval(void) const override; + void do_write_vstval(uint64_t val) override; + uint64_t do_read_vstvec(void) const override; + void do_write_vstvec(uint64_t val) override; + uint64_t do_read_vsscratch(void) const override; + void do_write_vsscratch(uint64_t val) override; + uint64_t do_read_vsatp(void) const override; + void do_write_vsatp(uint64_t val) override; + uint64_t do_read_vsip(void) const override; + void do_write_vsip(uint64_t val) override; + uint64_t do_read_vsie(void) const override; + void do_write_vsie(uint64_t val) override; uint64_t do_read_ilrsc(void) const override; void do_write_ilrsc(uint64_t val) override; uint64_t do_read_iflags(void) const override; diff --git a/src/i-state-access.h b/src/i-state-access.h index e6f5a9fa3..162b8b0a4 100644 --- a/src/i-state-access.h +++ b/src/i-state-access.h @@ -362,6 +362,234 @@ class i_state_access { // CRTP return derived().do_write_senvcfg(val); } + /// \brief Reads CSR hstatus. + /// \returns Register value. + uint64_t read_hstatus(void) { + return derived().do_read_hstatus(); + } + + /// \brief Writes CSR hstatus. + /// \param val New register value. + void write_hstatus(uint64_t val) { + return derived().do_write_hstatus(val); + } + + /// \brief Reads CSR hideleg. + /// \returns Register value. + uint64_t read_hideleg(void) { + return derived().do_read_hideleg(); + } + + /// \brief Writes CSR hideleg. + /// \param val New register value. + void write_hideleg(uint64_t val) { + return derived().do_write_hideleg(val); + } + + /// \brief Reads CSR hedeleg. + /// \returns Register value. + uint64_t read_hedeleg(void) { + return derived().do_read_hedeleg(); + } + + /// \brief Writes CSR hedeleg. + /// \param val New register value. + void write_hedeleg(uint64_t val) { + return derived().do_write_hedeleg(val); + } + + /// \brief Reads CSR hip. + /// \returns Register value. + uint64_t read_hip(void) { + return derived().do_read_hip(); + } + + /// \brief Writes CSR hip. + /// \param val New register value. + void write_hip(uint64_t val) { + return derived().do_write_hip(val); + } + + /// \brief Reads CSR hvip. + /// \returns Register value. + uint64_t read_hvip(void) { + return derived().do_read_hvip(); + } + + /// \brief Writes CSR hvip. + /// \param val New register value. + void write_hvip(uint64_t val) { + return derived().do_write_hvip(val); + } + + /// \brief Reads CSR hie. + /// \returns Register value. + uint64_t read_hie(void) { + return derived().do_read_hie(); + } + + /// \brief Writes CSR hie. + /// \param val New register value. + void write_hie(uint64_t val) { + return derived().do_write_hie(val); + } + + /// \brief Reads CSR hgatp. + /// \returns Register value. + uint64_t read_hgatp(void) { + return derived().do_read_hgatp(); + } + + /// \brief Writes CSR hgatp. + /// \param val New register value. + void write_hgatp(uint64_t val) { + return derived().do_write_hgatp(val); + } + + /// \brief Reads CSR henvcfg. + /// \returns Register value. + uint64_t read_henvcfg(void) { + return derived().do_read_henvcfg(); + } + + /// \brief Writes CSR henvcfg. + /// \param val New register value. + void write_henvcfg(uint64_t val) { + return derived().do_write_henvcfg(val); + } + + /// \brief Reads CSR htimedelta. + /// \returns Register value. + uint64_t read_htimedelta(void) { + return derived().do_read_htimedelta(); + } + + /// \brief Writes CSR htimedelta. + /// \param val New register value. + void write_htimedelta(uint64_t val) { + return derived().do_write_htimedelta(val); + } + + /// \brief Reads CSR htval. + /// \returns Register value. + uint64_t read_htval(void) { + return derived().do_read_htval(); + } + + /// \brief Writes CSR htval. + /// \param val New register value. + void write_htval(uint64_t val) { + return derived().do_write_htval(val); + } + + /// \brief Reads CSR vsepc. + /// \returns Register value. + uint64_t read_vsepc(void) { + return derived().do_read_vsepc(); + } + + /// \brief Writes CSR vsepc. + /// \param val New register value. + void write_vsepc(uint64_t val) { + return derived().do_write_vsepc(val); + } + + /// \brief Reads CSR vsstatus. + /// \returns Register value. + uint64_t read_vsstatus(void) { + return derived().do_read_vsstatus(); + } + + /// \brief Writes CSR vsstatus. + /// \param val New register value. + void write_vsstatus(uint64_t val) { + return derived().do_write_vsstatus(val); + } + + /// \brief Reads CSR vscause. + /// \returns Register value. + uint64_t read_vscause(void) { + return derived().do_read_vscause(); + } + + /// \brief Writes CSR vscause. + /// \param val New register value. + void write_vscause(uint64_t val) { + return derived().do_write_vscause(val); + } + + /// \brief Reads CSR vstval. + /// \returns Register value. + uint64_t read_vstval(void) { + return derived().do_read_vstval(); + } + + /// \brief Writes CSR vstval. + /// \param val New register value. + void write_vstval(uint64_t val) { + return derived().do_write_vstval(val); + } + + /// \brief Reads CSR vstvec. + /// \returns Register value. + uint64_t read_vstvec(void) { + return derived().do_read_vstvec(); + } + + /// \brief Writes CSR vstvec. + /// \param val New register value. + void write_vstvec(uint64_t val) { + return derived().do_write_vstvec(val); + } + + /// \brief Reads CSR vsscratch. + /// \returns Register value. + uint64_t read_vsscratch(void) { + return derived().do_read_vsscratch(); + } + + /// \brief Writes CSR vsscratch. + /// \param val New register value. + void write_vsscratch(uint64_t val) { + return derived().do_write_vsscratch(val); + } + + /// \brief Reads CSR vsatp. + /// \returns Register value. + uint64_t read_vsatp(void) { + return derived().do_read_vsatp(); + } + + /// \brief Writes CSR vsatp. + /// \param val New register value. + void write_vsatp(uint64_t val) { + return derived().do_write_vsatp(val); + } + + /// \brief Reads CSR vsip. + /// \returns Register value. + uint64_t read_vsip(void) { + return derived().do_read_vsip(); + } + + /// \brief Writes CSR vsip. + /// \param val New register value. + void write_vsip(uint64_t val) { + return derived().do_write_vsip(val); + } + + /// \brief Reads CSR vsie. + /// \returns Register value. + uint64_t read_vsie(void) { + return derived().do_read_vsie(); + } + + /// \brief Writes CSR vsie. + /// \param val New register value. + void write_vsie(uint64_t val) { + return derived().do_write_vsie(val); + } + /// \brief Reads CSR stvec. /// \returns Register value. uint64_t read_stvec(void) { @@ -511,6 +739,25 @@ class i_state_access { // CRTP return derived().do_read_iflags_X(); } + /// \brief Reads the iflags_VRT flag. + /// \returns The flag value. + /// \details This is Cartesi-specific. + bool read_iflags_VRT(void) { + return derived().do_read_iflags_VRT(); + } + + /// \brief Resets the iflags_VRT flag. + /// \details This is Cartesi-specific. + void reset_iflags_VRT(void) { + return derived().do_reset_iflags_VRT(); + } + + /// \brief Sets the iflags_VRT flag. + /// \details This is Cartesi-specific. + void set_iflags_VRT(void) { + return derived().do_set_iflags_VRT(); + } + /// \brief Reads the current privilege mode from iflags_PRV. /// \details This is Cartesi-specific. /// \returns Current privilege mode. diff --git a/src/i-virtual-machine.h b/src/i-virtual-machine.h index 5cf5c18d3..0f6626c6d 100644 --- a/src/i-virtual-machine.h +++ b/src/i-virtual-machine.h @@ -437,6 +437,196 @@ class i_virtual_machine { do_write_senvcfg(val); } + /// \brief Reads the hstatus register + uint64_t read_hstatus(void) const { + return do_read_hstatus(); + } + + /// \brief Writes the hstatus register + void write_hstatus(uint64_t val) { + do_write_hstatus(val); + } + + /// \brief Reads the hedeleg register + uint64_t read_hedeleg(void) const { + return do_read_hedeleg(); + } + + /// \brief Writes the hedeleg register + void write_hedeleg(uint64_t val) { + do_write_hedeleg(val); + } + + /// \brief Reads the hideleg register + uint64_t read_hideleg(void) const { + return do_read_hideleg(); + } + + /// \brief Writes the hideleg register + void write_hideleg(uint64_t val) { + do_write_hideleg(val); + } + + /// \brief Reads the hip register + uint64_t read_hip(void) const { + return do_read_hip(); + } + + /// \brief Writes the hip register + void write_hip(uint64_t val) { + do_write_hip(val); + } + + /// \brief Reads the hvip register + uint64_t read_hvip(void) const { + return do_read_hvip(); + } + + /// \brief Writes the hvip register + void write_hvip(uint64_t val) { + do_write_hvip(val); + } + + /// \brief Reads the hie register + uint64_t read_hie(void) const { + return do_read_hie(); + } + + /// \brief Writes the hie register + void write_hie(uint64_t val) { + do_write_hie(val); + } + + /// \brief Reads the hgatp register + uint64_t read_hgatp(void) const { + return do_read_hgatp(); + } + + /// \brief Writes the hgatp register + void write_hgatp(uint64_t val) { + do_write_hgatp(val); + } + + /// \brief Reads the henvcfg register + uint64_t read_henvcfg(void) const { + return do_read_henvcfg(); + } + + /// \brief Writes the henvcfg register + void write_henvcfg(uint64_t val) { + do_write_henvcfg(val); + } + + /// \brief Reads the htimedelta register + uint64_t read_htimedelta(void) const { + return do_read_htimedelta(); + } + + /// \brief Writes the htimedelta register + void write_htimedelta(uint64_t val) { + do_write_htimedelta(val); + } + + /// \brief Reads the htval register + uint64_t read_htval(void) const { + return do_read_htval(); + } + + /// \brief Writes the htval register + void write_htval(uint64_t val) { + do_write_htval(val); + } + + /// \brief Reads the vsepc register + uint64_t read_vsepc(void) const { + return do_read_vsepc(); + } + + /// \brief Writes the vsepc register + void write_vsepc(uint64_t val) { + do_write_vsepc(val); + } + + /// \brief Reads the vsstatus register + uint64_t read_vsstatus(void) const { + return do_read_vsstatus(); + } + + /// \brief Writes the vsstatus register + void write_vsstatus(uint64_t val) { + do_write_vsstatus(val); + } + + /// \brief Reads the vscause register + uint64_t read_vscause(void) const { + return do_read_vscause(); + } + + /// \brief Writes the vscause register + void write_vscause(uint64_t val) { + do_write_vscause(val); + } + + /// \brief Reads the vstval register + uint64_t read_vstval(void) const { + return do_read_vstval(); + } + + /// \brief Writes the vstval register + void write_vstval(uint64_t val) { + do_write_vstval(val); + } + + /// \brief Reads the vstvec register + uint64_t read_vstvec(void) const { + return do_read_vstvec(); + } + + /// \brief Writes the vstvec register + void write_vstvec(uint64_t val) { + do_write_vstvec(val); + } + + /// \brief Reads the vsscratch register + uint64_t read_vsscratch(void) const { + return do_read_vsscratch(); + } + + /// \brief Writes the vsscratch register + void write_vsscratch(uint64_t val) { + do_write_vsscratch(val); + } + + /// \brief Reads the vsatp register + uint64_t read_vsatp(void) const { + return do_read_vsatp(); + } + + /// \brief Writes the vsatp register + void write_vsatp(uint64_t val) { + do_write_vsatp(val); + } + + /// \brief Reads the vsie register + uint64_t read_vsie(void) const { + return do_read_vsie(); + } + + /// \brief Writes the vsie register + void write_vsie(uint64_t val) { + do_write_vsie(val); + } + + /// \brief Reads the vsip register + uint64_t read_vsip(void) const { + return do_read_vsip(); + } + + /// \brief Writes the vsip register + void write_vsip(uint64_t val) { + do_write_vsip(val); + } + /// \brief Reads the ilrsc register uint64_t read_ilrsc(void) const { return do_read_ilrsc(); @@ -713,6 +903,44 @@ class i_virtual_machine { virtual void do_write_scounteren(uint64_t val) = 0; virtual uint64_t do_read_senvcfg(void) const = 0; virtual void do_write_senvcfg(uint64_t val) = 0; + virtual uint64_t do_read_hstatus(void) const = 0; + virtual void do_write_hstatus(uint64_t val) = 0; + virtual uint64_t do_read_hedeleg(void) const = 0; + virtual void do_write_hedeleg(uint64_t val) = 0; + virtual uint64_t do_read_hideleg(void) const = 0; + virtual void do_write_hideleg(uint64_t val) = 0; + virtual uint64_t do_read_hip(void) const = 0; + virtual void do_write_hip(uint64_t val) = 0; + virtual uint64_t do_read_hvip(void) const = 0; + virtual void do_write_hvip(uint64_t val) = 0; + virtual uint64_t do_read_hie(void) const = 0; + virtual void do_write_hie(uint64_t val) = 0; + virtual uint64_t do_read_hgatp(void) const = 0; + virtual void do_write_hgatp(uint64_t val) = 0; + virtual uint64_t do_read_henvcfg(void) const = 0; + virtual void do_write_henvcfg(uint64_t val) = 0; + virtual uint64_t do_read_htimedelta(void) const = 0; + virtual void do_write_htimedelta(uint64_t val) = 0; + virtual uint64_t do_read_htval(void) const = 0; + virtual void do_write_htval(uint64_t val) = 0; + virtual uint64_t do_read_vsepc(void) const = 0; + virtual void do_write_vsepc(uint64_t val) = 0; + virtual uint64_t do_read_vsstatus(void) const = 0; + virtual void do_write_vsstatus(uint64_t val) = 0; + virtual uint64_t do_read_vscause(void) const = 0; + virtual void do_write_vscause(uint64_t val) = 0; + virtual uint64_t do_read_vstval(void) const = 0; + virtual void do_write_vstval(uint64_t val) = 0; + virtual uint64_t do_read_vstvec(void) const = 0; + virtual void do_write_vstvec(uint64_t val) = 0; + virtual uint64_t do_read_vsscratch(void) const = 0; + virtual void do_write_vsscratch(uint64_t val) = 0; + virtual uint64_t do_read_vsatp(void) const = 0; + virtual void do_write_vsatp(uint64_t val) = 0; + virtual uint64_t do_read_vsie(void) const = 0; + virtual void do_write_vsie(uint64_t val) = 0; + virtual uint64_t do_read_vsip(void) const = 0; + virtual void do_write_vsip(uint64_t val) = 0; virtual uint64_t do_read_ilrsc(void) const = 0; virtual void do_write_ilrsc(uint64_t val) = 0; virtual uint64_t do_read_iflags(void) const = 0; diff --git a/src/interpret.cpp b/src/interpret.cpp index 048347e2a..5c723c7df 100644 --- a/src/interpret.cpp +++ b/src/interpret.cpp @@ -155,7 +155,7 @@ static void dump_exception_or_interrupt(uint64_t cause, STATE &s) { (void) fprintf(stderr, "supervisor software interrupt"); break; case 2: - (void) fprintf(stderr, "reserved software interrupt"); + (void) fprintf(stderr, "virtual software interrupt"); break; case 3: (void) fprintf(stderr, "machine software interrupt"); @@ -167,7 +167,7 @@ static void dump_exception_or_interrupt(uint64_t cause, STATE &s) { (void) fprintf(stderr, "supervisor timer interrupt"); break; case 6: - (void) fprintf(stderr, "reserved timer interrupt"); + (void) fprintf(stderr, "virtual timer interrupt"); break; case 7: (void) fprintf(stderr, "machine timer interrupt"); @@ -179,7 +179,7 @@ static void dump_exception_or_interrupt(uint64_t cause, STATE &s) { (void) fprintf(stderr, "supervisor external interrupt"); break; case 10: - (void) fprintf(stderr, "reserved external interrupt"); + (void) fprintf(stderr, "virtual external interrupt"); break; case 11: (void) fprintf(stderr, "machine external interrupt"); @@ -235,6 +235,18 @@ static void dump_exception_or_interrupt(uint64_t cause, STATE &s) { case 15: (void) fprintf(stderr, "store/amo page fault"); break; + case 20: + (void) fprintf(stderr, "instruction guest page fault"); + break; + case 21: + (void) fprintf(stderr, "load guest page fault"); + break; + case 22: + (void) fprintf(stderr, "virtual instruction page fault"); + break; + case 23: + (void) fprintf(stderr, "store/amo guest page fault"); + break; default: (void) fprintf(stderr, "reserved"); break; @@ -294,11 +306,56 @@ static inline bool csr_is_read_only(CSR_address csraddr) { return ((to_underlying(csraddr) & 0xc00) == 0xc00); } -/// \brief Extract privilege level from CSR address. +/// \brief Checks if CSR access is allowed from the current mode. +/// \param a Machine state accessor object. /// \param csr Address of CSR in file. -/// \returns Privilege level. -static inline uint32_t csr_priv(CSR_address csr) { - return (to_underlying(csr) >> 8) & 3; +/// \param status cause code in case of failure. +/// \returns true if CSR access is allowed from the current mode, false otherwise. +template +static inline bool csr_is_allowed_access(STATE_ACCESS &a, CSR_address csr, MCAUSE_constants &status) { + const uint8_t priv = a.read_iflags_PRV(); + const uint8_t access_priv = priv == NOM_S ? 2 : priv; + const uint8_t register_priv = (to_underlying(csr) >> 8) & 3; + + if (access_priv < register_priv) { + // we should not access Hypervisor and Virtual registers directly from the virtual mode + // for Hypervisor and Virtual registers register_priv = 2 + if (a.read_iflags_VRT() && register_priv <= (NOM_S + 1)) { + status = MCAUSE_VIRTUAL_INSTRUCTION; + } else { + status = MCAUSE_ILLEGAL_INSN; + } + return false; + } + + return true; +} + +static inline uint8_t encode_access_mode(uint8_t priv, bool virt) { + uint8_t access_mode = priv; + if (virt) { + access_mode |= ACCESS_MODE_V_MASK; + } + return access_mode; +} + +/// \brief Gets currently machine memory access mode. +/// \param a Machine state accessor object. +/// \returns machine memory access mode. +template +static inline uint8_t get_current_memory_access_mode(STATE_ACCESS &a) { + uint8_t priv = a.read_iflags_PRV(); + bool virt = a.read_iflags_VRT(); + + // When MPRV is set, data loads and stores use privilege in MPP + // and virtual state in MPV instead of the current privilege level + // (code access, virtual-machine load/store instructions, HLV, HLVX, HSV are unaffected) + if ((a.read_mstatus() & MSTATUS_MPRV_MASK) && !virt && !FETCH) { + priv = (a.read_mstatus() & MSTATUS_MPP_MASK) >> MSTATUS_MPP_SHIFT; + virt = (a.read_mstatus() & MSTATUS_MPV_MASK) >> MSTATUS_MPV_SHIFT; + } + + return encode_access_mode(priv, virt); } /// \brief Changes privilege level. @@ -310,6 +367,7 @@ template static NO_INLINE void set_priv(STATE_ACCESS &a, int new_prv) { INC_COUNTER(a.get_statistics(), priv_level[new_prv]); a.write_iflags_PRV(new_prv); + // Invalidate all TLB entries a.flush_all_tlb(); INC_COUNTER(a.get_statistics(), tlb_flush_all); @@ -320,6 +378,137 @@ static NO_INLINE void set_priv(STATE_ACCESS &a, int new_prv) { a.write_ilrsc(-1); // invalidate reserved address } +template +static inline uint64_t raise_exception_M(STATE_ACCESS &a, uint64_t pc, uint64_t cause, uint64_t tval) { + auto priv = a.read_iflags_PRV(); + set_priv(a, NOM_M); + + uint64_t mstatus = a.read_mstatus(); + const uint64_t mie = (mstatus & MSTATUS_MIE_MASK) >> MSTATUS_MIE_SHIFT; + mstatus = (mstatus & ~MSTATUS_MPIE_MASK) | (mie << MSTATUS_MPIE_SHIFT); + mstatus &= ~MSTATUS_MPV_MASK; + mstatus &= ~MSTATUS_GVA_MASK; + mstatus &= ~MSTATUS_MIE_MASK; + if (a.read_iflags_VRT()) { + mstatus |= MSTATUS_MPV_MASK; + // For any trap (breakpoint, address misaligned, access fault, page fault, or guest-page fault) + // that writes a guest virtual address to stval, GVA is set to 1. + switch (cause) { + case MCAUSE_BREAKPOINT: + case MCAUSE_INSN_ADDRESS_MISALIGNED: + case MCAUSE_INSN_ACCESS_FAULT: + case MCAUSE_LOAD_ADDRESS_MISALIGNED: + case MCAUSE_LOAD_ACCESS_FAULT: + case MCAUSE_STORE_AMO_ADDRESS_MISALIGNED: + case MCAUSE_STORE_AMO_ACCESS_FAULT: + case MCAUSE_FETCH_PAGE_FAULT: + case MCAUSE_LOAD_PAGE_FAULT: + case MCAUSE_STORE_AMO_PAGE_FAULT: + case MCAUSE_INSTRUCTION_GUEST_PAGE_FAULT: + case MCAUSE_LOAD_GUEST_PAGE_FAULT: + case MCAUSE_STORE_AMO_GUEST_PAGE_FAULT: + mstatus |= MSTATUS_GVA_MASK; + default: + break; + } + } + + mstatus = (mstatus & ~MSTATUS_MPP_MASK) | (priv << MSTATUS_MPP_SHIFT); + a.write_mstatus(mstatus); + a.write_mcause(cause); + a.write_mepc(pc); + a.write_mtval(tval); + a.reset_iflags_VRT(); + +#ifdef DUMP_COUNTERS + if (cause & MCAUSE_INTERRUPT_FLAG) + INC_COUNTER(a.get_naked_state(), m_int); + else if (cause >= MCAUSE_ECALL_BASE && cause <= MCAUSE_ECALL_BASE + NOM_M) // Do not count environment calls + INC_COUNTER(a.get_naked_state(), m_ex); +#endif + return a.read_mtvec(); +} + +template +static inline uint64_t raise_exception_HS(STATE_ACCESS &a, uint64_t pc, uint64_t cause, uint64_t tval, uint64_t tval2) { + auto priv = a.read_iflags_PRV(); + set_priv(a, NOM_S); + uint64_t mstatus = a.read_mstatus(); + uint64_t hstatus = a.read_hstatus(); + + const uint64_t sie = (mstatus & MSTATUS_SIE_MASK) >> MSTATUS_SIE_SHIFT; + hstatus &= ~HSTATUS_SPV_MASK; + hstatus &= ~HSTATUS_GVA_MASK; + mstatus = (mstatus & ~MSTATUS_SPIE_MASK) | (sie << MSTATUS_SPIE_SHIFT); + mstatus = (mstatus & ~MSTATUS_SPP_MASK) | (priv << MSTATUS_SPP_SHIFT); + mstatus &= ~MSTATUS_SIE_MASK; + // if V was 1 before the trap, field hstatus.SPVP is set the same as sstatus.SPP + if (a.read_iflags_VRT()) { + hstatus = (hstatus & ~HSTATUS_SPVP_MASK) | (priv << HSTATUS_SPVP_SHIFT); + hstatus |= HSTATUS_SPV_MASK; + // For any trap (breakpoint, address misaligned, access fault, page fault, or guest-page fault) + // that writes a guest virtual address to stval, GVA is set to 1. + switch (cause) { + case MCAUSE_BREAKPOINT: + case MCAUSE_INSN_ADDRESS_MISALIGNED: + case MCAUSE_INSN_ACCESS_FAULT: + case MCAUSE_LOAD_ADDRESS_MISALIGNED: + case MCAUSE_LOAD_ACCESS_FAULT: + case MCAUSE_STORE_AMO_ADDRESS_MISALIGNED: + case MCAUSE_STORE_AMO_ACCESS_FAULT: + case MCAUSE_FETCH_PAGE_FAULT: + case MCAUSE_LOAD_PAGE_FAULT: + case MCAUSE_STORE_AMO_PAGE_FAULT: + case MCAUSE_INSTRUCTION_GUEST_PAGE_FAULT: + case MCAUSE_LOAD_GUEST_PAGE_FAULT: + case MCAUSE_STORE_AMO_GUEST_PAGE_FAULT: + hstatus |= HSTATUS_GVA_MASK; + default: + break; + } + } + + a.write_hstatus(hstatus); + a.write_mstatus(mstatus); + a.write_scause(cause); + a.write_sepc(pc); + a.write_stval(tval); + a.write_htval(tval2); + a.reset_iflags_VRT(); + +#ifdef DUMP_COUNTERS + if (cause & MCAUSE_INTERRUPT_FLAG) + INC_COUNTER(a.get_naked_state(), sv_int); + else if (cause >= MCAUSE_ECALL_BASE && cause <= MCAUSE_ECALL_BASE + NOM_M) // Do not count environment calls + INC_COUNTER(a.get_naked_state(), sv_ex); +#endif + return a.read_stvec(); +} + +template +static inline uint64_t raise_exception_VS(STATE_ACCESS &a, uint64_t pc, uint64_t cause, uint64_t tval) { + auto priv = a.read_iflags_PRV(); + set_priv(a, NOM_S); + + uint64_t vsstatus = a.read_vsstatus(); + const uint64_t sie = (vsstatus & MSTATUS_SIE_MASK) >> MSTATUS_SIE_SHIFT; + vsstatus = (vsstatus & ~MSTATUS_SPIE_MASK) | (sie << MSTATUS_SPIE_SHIFT); + vsstatus = (vsstatus & ~MSTATUS_SPP_MASK) | (priv << MSTATUS_SPP_SHIFT); + vsstatus &= ~MSTATUS_SIE_MASK; + + a.write_vsstatus(vsstatus); + a.write_vscause(cause); + a.write_vsepc(pc); + a.write_vstval(tval); +#ifdef DUMP_COUNTERS + if (cause & MCAUSE_INTERRUPT_FLAG) + INC_COUNTER(a.get_naked_state(), sv_int); + else if (cause >= MCAUSE_ECALL_BASE && cause <= MCAUSE_ECALL_BASE + NOM_M) // Do not count environment calls + INC_COUNTER(a.get_naked_state(), sv_ex); +#endif + return a.read_vstvec(); +} + /// \brief Raise an exception (or interrupt). /// \param a Machine state accessor object. /// \param pc Machine current program counter. @@ -328,7 +517,7 @@ static NO_INLINE void set_priv(STATE_ACCESS &a, int new_prv) { /// \returns The new program counter, pointing to the raised exception trap handler. /// \details This function is outlined to minimize host CPU code cache pressure. template -static NO_INLINE uint64_t raise_exception(STATE_ACCESS &a, uint64_t pc, uint64_t cause, uint64_t tval) { +static NO_INLINE uint64_t raise_exception(STATE_ACCESS &a, uint64_t pc, uint64_t cause, uint64_t tval, uint64_t tval2) { #if defined(DUMP_EXCEPTIONS) || defined(DUMP_MMU_EXCEPTIONS) || defined(DUMP_INTERRUPTS) || \ defined(DUMP_ILLEGAL_INSN_EXCEPTIONS) { @@ -365,68 +554,76 @@ static NO_INLINE uint64_t raise_exception(STATE_ACCESS &a, uint64_t pc, uint64_t } #endif - // Check if exception should be delegated to supervisor privilege - // For each interrupt or exception number, there is a bit at mideleg - // or medeleg saying if it should be delegated - bool deleg = false; - auto priv = a.read_iflags_PRV(); - if (priv <= PRV_S) { - if (cause & MCAUSE_INTERRUPT_FLAG) { - // Clear the MCAUSE_INTERRUPT_FLAG bit before shifting - deleg = (a.read_mideleg() >> (cause & (XLEN - 1))) & 1; - } else { - deleg = (a.read_medeleg() >> cause) & 1; - } - } - // Every raised exception increases the exception counter, so we can compute minstret later a.write_icycleinstret(a.read_icycleinstret() + 1); - uint64_t new_pc = 0; - if (deleg) { - a.write_scause(cause); - a.write_sepc(pc); - a.write_stval(tval); - uint64_t mstatus = a.read_mstatus(); - mstatus = (mstatus & ~MSTATUS_SPIE_MASK) | (((mstatus >> MSTATUS_SIE_SHIFT) & 1) << MSTATUS_SPIE_SHIFT); - mstatus = (mstatus & ~MSTATUS_SPP_MASK) | (priv << MSTATUS_SPP_SHIFT); - mstatus &= ~MSTATUS_SIE_MASK; - a.write_mstatus(mstatus); - if (priv != PRV_S) { - set_priv(a, PRV_S); + // check if the exception should be delegated + const uint8_t priv = a.read_iflags_PRV(); + const bool virt = a.read_iflags_VRT(); + uint64_t vsdeleg = 0; + uint64_t hsdeleg = 0; + const bool interrupt = cause & MCAUSE_INTERRUPT_FLAG; + if (interrupt) { + if (virt && (priv == NOM_S || priv == NOM_U)) { + vsdeleg = a.read_hideleg(); } - new_pc = a.read_stvec(); -#ifdef DUMP_COUNTERS - if (cause & MCAUSE_INTERRUPT_FLAG) { - INC_COUNTER(a.get_statistics(), sv_int); - } else if (cause >= MCAUSE_ECALL_BASE && cause <= MCAUSE_ECALL_BASE + PRV_M) { // Do not count environment calls - INC_COUNTER(a.get_statistics(), sv_ex); + if (priv != NOM_M) { + hsdeleg = a.read_mideleg(); } -#endif } else { - a.write_mcause(cause); - a.write_mepc(pc); - a.write_mtval(tval); - uint64_t mstatus = a.read_mstatus(); - mstatus = (mstatus & ~MSTATUS_MPIE_MASK) | (((mstatus >> MSTATUS_MIE_SHIFT) & 1) << MSTATUS_MPIE_SHIFT); - mstatus = (mstatus & ~MSTATUS_MPP_MASK) | (priv << MSTATUS_MPP_SHIFT); - mstatus &= ~MSTATUS_MIE_MASK; - a.write_mstatus(mstatus); - if (priv != PRV_M) { - set_priv(a, PRV_M); + if (virt && (priv == NOM_S || priv == NOM_U)) { + vsdeleg = a.read_medeleg() & a.read_hedeleg(); } - new_pc = a.read_mtvec(); -#ifdef DUMP_COUNTERS - if (cause & MCAUSE_INTERRUPT_FLAG) { - INC_COUNTER(a.get_statistics(), m_int); - } else if (cause >= MCAUSE_ECALL_BASE && cause <= MCAUSE_ECALL_BASE + PRV_M) { // Do not count environment calls - INC_COUNTER(a.get_statistics(), m_ex); + if (priv != NOM_M) { + hsdeleg = a.read_medeleg(); } -#endif + } + + // delegate exception to the corresponding mode + uint64_t new_pc = 0; + if (priv <= NOM_S && ((vsdeleg >> cause) & 1)) { + if (interrupt) { + --cause; + } + new_pc = raise_exception_VS(a, pc, cause, tval); + } else if (priv <= NOM_S && ((hsdeleg >> cause) & 1)) { + new_pc = raise_exception_HS(a, pc, cause, tval, tval2); + } else { + new_pc = raise_exception_M(a, pc, cause, tval); } return new_pc; } +template +static inline uint32_t get_enabled_irq_mask_M(STATE_ACCESS &a, uint64_t mie, uint8_t priv) { + const uint32_t deleg = ~a.read_mideleg(); + if (priv < NOM_M || (priv == NOM_M && mie)) { // enabled M + return deleg; + } else { + return 0; + } +} + +template +static inline uint32_t get_enabled_irq_mask_HS(STATE_ACCESS &a, uint64_t sie, uint8_t priv) { + const uint32_t deleg = a.read_mideleg() & ~a.read_hideleg(); + if (a.read_iflags_VRT() || priv < NOM_S || (priv == NOM_S && sie)) { // enabled S + return deleg; + } else { + return 0; + } +} + +template +static inline uint32_t get_enabled_irq_mask_VS(STATE_ACCESS &a, uint64_t sie, uint8_t priv) { + const uint32_t deleg = a.read_hideleg(); + if (priv < NOM_S || (priv == NOM_S && sie)) { // enabled VS + return deleg; + } else { + return 0; + } +} + /// \brief Obtains a mask of pending and enabled interrupts. /// \param a Machine state accessor object. /// \returns The mask. @@ -435,43 +632,26 @@ static inline uint32_t get_pending_irq_mask(STATE_ACCESS &a) { const uint64_t mip = a.read_mip(); const uint64_t mie = a.read_mie(); - // interrupt trap condition 2: bit i is set in both mip and mie const uint32_t pending_ints = mip & mie; + if (pending_ints == 0) { return 0; } - uint32_t enabled_ints = 0; auto priv = a.read_iflags_PRV(); - switch (priv) { - // interrupt trap condition 1a: the current privilege mode is M - case PRV_M: { - const uint64_t mstatus = a.read_mstatus(); - // interrupt trap condition 1a: ... and the MIE bit in the mstatus - // register is set - if (mstatus & MSTATUS_MIE_MASK) { - // interrupt trap condition 3: bit i is not set in mideleg - enabled_ints = ~a.read_mideleg(); - } - break; - } - // interrupt trap condition 1b: the current privilege mode has less - // privilege than M-mode - case PRV_S: { - const uint64_t mstatus = a.read_mstatus(); - // Interrupts not set in mideleg are machine-mode - // and cannot be masked by supervisor mode - if (mstatus & MSTATUS_SIE_MASK) { - enabled_ints = -1; - } else { - // interrupt trap condition 3: bit i is not set in mideleg - enabled_ints = ~a.read_mideleg(); - } - break; + auto mstatus = a.read_mstatus(); + const uint64_t status_mie = (mstatus & MSTATUS_MIE_MASK) >> MSTATUS_MIE_SHIFT; + uint64_t status_sie = (mstatus & MSTATUS_SIE_MASK) >> MSTATUS_SIE_SHIFT; + if (a.read_iflags_VRT()) { + status_sie = (a.read_vsstatus() & MSTATUS_SIE_MASK) >> MSTATUS_SIE_SHIFT; + } + + uint32_t enabled_ints = get_enabled_irq_mask_M(a, status_mie, priv); + if ((enabled_ints & pending_ints) == 0) { + enabled_ints = get_enabled_irq_mask_HS(a, status_sie, priv); + if (a.read_iflags_VRT() && ((enabled_ints & pending_ints) == 0)) { + enabled_ints = get_enabled_irq_mask_VS(a, status_sie, priv); } - default: - enabled_ints = -1; - break; } return pending_ints & enabled_ints; @@ -493,8 +673,9 @@ static inline uint32_t get_highest_priority_irq_num(uint32_t v) { // Multiple simultaneous interrupts destined for HS-mode are handled in the following decreasing // priority order: SEI, SSI, STI, SGEI, VSEI, VSSI, VSTI. const std::array interrupts_priority{ - MIP_MEIP_MASK, MIP_MSIP_MASK, MIP_MTIP_MASK, // Machine interrupts has higher priority - MIP_SEIP_MASK, MIP_SSIP_MASK, MIP_STIP_MASK // Supervisor interrupts + MIP_MEIP_MASK, MIP_MSIP_MASK, MIP_MTIP_MASK, // Machine interrupts has higher priority + MIP_SEIP_MASK, MIP_SSIP_MASK, MIP_STIP_MASK, // Supervisor interrupts + MIP_VSEIP_MASK, MIP_VSSIP_MASK, MIP_VSTIP_MASK, // Virtual interrupts }; for (const uint32_t mask : interrupts_priority) { if (v & mask) { @@ -515,7 +696,7 @@ static inline uint64_t raise_interrupt_if_any(STATE_ACCESS &a, uint64_t pc) { const uint32_t mask = get_pending_irq_mask(a); if (unlikely(mask != 0)) { const uint64_t irq_num = get_highest_priority_irq_num(mask); - return raise_exception(a, pc, irq_num | MCAUSE_INTERRUPT_FLAG, 0); + return raise_exception(a, pc, irq_num | MCAUSE_INTERRUPT_FLAG, 0, 0); } return pc; } @@ -813,19 +994,32 @@ static FORCE_INLINE int32_t insn_get_C_SWSP_imm(uint32_t insn) { /// instead of always storing it in register (this is an optimization). template static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS &a, uint64_t pc, uint64_t mcycle, - uint64_t vaddr, T *pval) { + uint64_t vaddr, uint8_t access_mode, uint8_t xwr_shift, T *pval) { using U = std::make_unsigned_t; // No support for misaligned accesses: They are handled by a trap in BBL if (unlikely(vaddr & (sizeof(T) - 1))) { - pc = raise_exception(a, pc, - RAISE_STORE_EXCEPTIONS ? MCAUSE_STORE_AMO_ADDRESS_MISALIGNED : MCAUSE_LOAD_ADDRESS_MISALIGNED, vaddr); + MCAUSE_constants c = MCAUSE_LOAD_ADDRESS_MISALIGNED; // NOLINT(misc-const-correctness) + if constexpr (RAISE_STORE_EXCEPTIONS) { + c = MCAUSE_STORE_AMO_ADDRESS_MISALIGNED; + } + pc = raise_exception(a, pc, c, vaddr, 0); return {false, pc}; } // Deal with aligned accesses uint64_t paddr{}; - if (unlikely(!translate_virtual_address(a, &paddr, vaddr, PTE_XWR_R_SHIFT))) { - pc = raise_exception(a, pc, RAISE_STORE_EXCEPTIONS ? MCAUSE_STORE_AMO_PAGE_FAULT : MCAUSE_LOAD_PAGE_FAULT, - vaddr); + uint8_t cause = // NOLINT(misc-const-correctness) + translate_virtual_address(a, &paddr, vaddr, access_mode, xwr_shift); + if (unlikely(cause)) { + if constexpr (RAISE_STORE_EXCEPTIONS) { + if (cause == MCAUSE_LOAD_PAGE_FAULT) { + cause = MCAUSE_STORE_AMO_PAGE_FAULT; + } else { + cause = MCAUSE_STORE_AMO_GUEST_PAGE_FAULT; + } + } + // When a guest-page-fault trap is taken into HS-mode, htval is written with the guest + // physical address that faulted, shifted right by 2 bits. + pc = raise_exception(a, pc, cause, vaddr, paddr >> 2); return {false, pc}; } auto &pma = a.template find_pma_entry(paddr); @@ -846,8 +1040,11 @@ static NO_INLINE std::pair read_virtual_memory_slow(STATE_ACCESS } } } - pc = raise_exception(a, pc, RAISE_STORE_EXCEPTIONS ? MCAUSE_STORE_AMO_ACCESS_FAULT : MCAUSE_LOAD_ACCESS_FAULT, - vaddr); + MCAUSE_constants c = MCAUSE_LOAD_ACCESS_FAULT; // NOLINT(misc-const-correctness) + if constexpr (RAISE_STORE_EXCEPTIONS) { + c = MCAUSE_STORE_AMO_ACCESS_FAULT; + } + pc = raise_exception(a, pc, c, vaddr, 0); return {false, pc}; } @@ -863,10 +1060,11 @@ template (vaddr, pval)))) { + auto access_mode = get_current_memory_access_mode(a); // Outline the slow path into a function call to minimize host CPU code cache pressure INC_COUNTER(a.get_statistics(), tlb_rmiss); - auto [status, new_pc] = - read_virtual_memory_slow(a, pc, mcycle, vaddr, pval); + auto [status, new_pc] = read_virtual_memory_slow(a, pc, mcycle, vaddr, + access_mode, PTE_XWR_R_SHIFT, pval); pc = new_pc; return status; } @@ -890,17 +1088,21 @@ static FORCE_INLINE bool read_virtual_memory(STATE_ACCESS &a, uint64_t &pc, uint /// instead of always storing it in register (this is an optimization). template static NO_INLINE std::pair write_virtual_memory_slow(STATE_ACCESS &a, uint64_t pc, - uint64_t mcycle, uint64_t vaddr, uint64_t val64) { + uint64_t mcycle, uint64_t vaddr, uint8_t access_mode, uint64_t val64) { using U = std::make_unsigned_t; // No support for misaligned accesses: They are handled by a trap in BBL if (unlikely(vaddr & (sizeof(T) - 1))) { - pc = raise_exception(a, pc, MCAUSE_STORE_AMO_ADDRESS_MISALIGNED, vaddr); + pc = raise_exception(a, pc, MCAUSE_STORE_AMO_ADDRESS_MISALIGNED, vaddr, 0); return {execute_status::failure, pc}; } // Deal with aligned accesses uint64_t paddr{}; - if (unlikely(!translate_virtual_address(a, &paddr, vaddr, PTE_XWR_W_SHIFT))) { - pc = raise_exception(a, pc, MCAUSE_STORE_AMO_PAGE_FAULT, vaddr); + const uint8_t cause = + translate_virtual_address(a, &paddr, vaddr, access_mode, PTE_XWR_W_SHIFT); + if (unlikely(cause)) { + // When a guest-page-fault trap is taken into HS-mode, htval is written with the guest + // physical address that faulted, shifted right by 2 bits. + pc = raise_exception(a, pc, cause, vaddr, paddr >> 2); return {execute_status::failure, pc}; } auto &pma = a.template find_pma_entry(paddr); @@ -919,7 +1121,7 @@ static NO_INLINE std::pair write_virtual_memory_slow(S } } } - pc = raise_exception(a, pc, MCAUSE_STORE_AMO_ACCESS_FAULT, vaddr); + pc = raise_exception(a, pc, MCAUSE_STORE_AMO_ACCESS_FAULT, vaddr, 0); return {execute_status::failure, pc}; } @@ -937,8 +1139,9 @@ static FORCE_INLINE execute_status write_virtual_memory(STATE_ACCESS &a, uint64_ // Try hitting the TLB if (unlikely((!a.template write_memory_word_via_tlb(vaddr, static_cast(val64))))) { INC_COUNTER(a.get_statistics(), tlb_wmiss); + auto access_mode = get_current_memory_access_mode(a); // Outline the slow path into a function call to minimize host CPU code cache pressure - auto [status, new_pc] = write_virtual_memory_slow(a, pc, mcycle, vaddr, val64); + auto [status, new_pc] = write_virtual_memory_slow(a, pc, mcycle, vaddr, access_mode, val64); pc = new_pc; return status; } @@ -956,11 +1159,12 @@ static void dump_insn(STATE_ACCESS &a, uint64_t pc, uint32_t insn, const char *n #endif #ifdef DUMP_INSN uint64_t ppc = 0; + auto access_mode = get_current_memory_access_mode(a); // If we are running in the microinterpreter, we may or may not be collecting a step access log. // To prevent additional address translation end up in the log, // the following check will always be false when MICROARCHITECTURE is defined. if (std::is_same::value && - !translate_virtual_address(a, &ppc, pc, PTE_XWR_X_SHIFT)) { + translate_virtual_address(a, &ppc, pc, access_mode, PTE_XWR_X_SHIFT)) { ppc = pc; fprintf(stderr, "v %08" PRIx64, ppc); } else { @@ -986,7 +1190,13 @@ static void dump_insn(STATE_ACCESS &a, uint64_t pc, uint32_t insn, const char *n /// illegal. template static FORCE_INLINE execute_status raise_illegal_insn_exception(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { - pc = raise_exception(a, pc, MCAUSE_ILLEGAL_INSN, insn); + pc = raise_exception(a, pc, MCAUSE_ILLEGAL_INSN, insn, 0); + return execute_status::failure; +} + +template +static FORCE_INLINE execute_status raise_virtual_insn_exception(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + pc = raise_exception(a, pc, MCAUSE_VIRTUAL_INSTRUCTION, insn, 0); return execute_status::failure; } @@ -998,7 +1208,7 @@ static FORCE_INLINE execute_status raise_illegal_insn_exception(STATE_ACCESS &a, /// \details This function is tail-called whenever the caller identified that the next value of pc is misaligned. template static FORCE_INLINE execute_status raise_misaligned_fetch_exception(STATE_ACCESS &a, uint64_t &pc, uint64_t new_pc) { - pc = raise_exception(a, pc, MCAUSE_INSN_ADDRESS_MISALIGNED, new_pc); + pc = raise_exception(a, pc, MCAUSE_INSN_ADDRESS_MISALIGNED, new_pc, 0); return execute_status::failure; } @@ -1503,9 +1713,9 @@ template static inline bool rdcounteren(STATE_ACCESS &a, uint64_t mask) { uint64_t counteren = MCOUNTEREN_R_MASK; auto priv = a.read_iflags_PRV(); - if (priv <= PRV_S) { + if (priv <= NOM_S) { counteren &= a.read_mcounteren(); - if (priv < PRV_S) { + if (priv < NOM_S) { counteren &= a.read_scounteren(); } } @@ -1536,13 +1746,124 @@ static inline uint64_t read_csr_time(STATE_ACCESS &a, uint64_t mcycle, bool *sta if (unlikely(!rdcounteren(a, MCOUNTEREN_TM_MASK))) { return read_csr_fail(status); } - const uint64_t mtime = rtc_cycle_to_time(mcycle); + uint64_t mtime = rtc_cycle_to_time(mcycle); + if (a.read_iflags_VRT()) { + mtime += a.read_htimedelta(); + } return read_csr_success(mtime, status); } +template +static inline uint64_t read_csr_hstatus(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_hstatus() & HSTATUS_R_MASK, status); +} + +template +static inline uint64_t read_csr_hedeleg(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_hedeleg(), status); +} + +template +static inline uint64_t read_csr_hideleg(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_hideleg(), status); +} + +template +static inline uint64_t read_csr_hie(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_hie(), status); +} + +template +static inline uint64_t read_csr_hip(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_hip(), status); +} + +template +static inline uint64_t read_csr_hvip(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_hvip(), status); +} + +template +static inline uint64_t read_csr_henvcfg(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_henvcfg() & MENVCFG_R_MASK, status); +} + +template +static inline uint64_t read_csr_hgatp(STATE_ACCESS &a, bool *status) { + const uint64_t mstatus = a.read_mstatus(); + const uint8_t priv = a.read_iflags_PRV(); + + // When mstatus.TVM=1, attempts to read or write hgatp while executing in HS-mode will + // raise an illegal instruction exception. + if (unlikely(priv == NOM_S && (mstatus & MSTATUS_TVM_MASK))) { + return execute_status::failure; + } + + return read_csr_success(a.read_hgatp(), status); +} + +template +static inline uint64_t read_csr_htimedelta(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_htimedelta(), status); +} + +template +static inline uint64_t read_csr_htval(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_htval(), status); +} + +template +static inline uint64_t read_csr_vsstatus(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_vsstatus(), status); +} + +template +static inline uint64_t read_csr_vsie(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_vsie(), status); +} + +template +static inline uint64_t read_csr_vstvec(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_vstvec(), status); +} + +template +static inline uint64_t read_csr_vsscratch(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_vsscratch(), status); +} + +template +static inline uint64_t read_csr_vsepc(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_vsepc(), status); +} + +template +static inline uint64_t read_csr_vscause(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_vscause(), status); +} + +template +static inline uint64_t read_csr_vstval(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_vstval(), status); +} + +template +static inline uint64_t read_csr_vsip(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_vsip(), status); +} + +template +static inline uint64_t read_csr_vsatp(STATE_ACCESS &a, bool *status) { + return read_csr_success(a.read_vsatp(), status); +} + template static inline uint64_t read_csr_sstatus(STATE_ACCESS &a, bool *status) { - return read_csr_success(a.read_mstatus() & SSTATUS_R_MASK, status); + if (a.read_iflags_VRT()) { + return read_csr_success(a.read_vsstatus() & VSSTATUS_R_MASK, status); + } else { + return read_csr_success(a.read_mstatus() & SSTATUS_R_MASK, status); + } } template @@ -1552,14 +1873,23 @@ static inline uint64_t read_csr_senvcfg(STATE_ACCESS &a, bool *status) { template static inline uint64_t read_csr_sie(STATE_ACCESS &a, bool *status) { - const uint64_t mie = a.read_mie(); - const uint64_t mideleg = a.read_mideleg(); - return read_csr_success(mie & mideleg, status); + if (a.read_iflags_VRT()) { + const uint64_t sie = a.read_vsie(); + return read_csr_success(sie, status); + } else { + const uint64_t mie = a.read_mie() & (MIP_SSIP_MASK | MIP_SEIP_MASK | MIP_STIP_MASK); + const uint64_t mideleg = a.read_mideleg(); + return read_csr_success(mie & mideleg, status); + } } template static inline uint64_t read_csr_stvec(STATE_ACCESS &a, bool *status) { - return read_csr_success(a.read_stvec(), status); + if (a.read_iflags_VRT()) { + return read_csr_success(a.read_vstvec(), status); + } else { + return read_csr_success(a.read_stvec(), status); + } } template @@ -1569,42 +1899,75 @@ static inline uint64_t read_csr_scounteren(STATE_ACCESS &a, bool *status) { template static inline uint64_t read_csr_sscratch(STATE_ACCESS &a, bool *status) { - return read_csr_success(a.read_sscratch(), status); + if (a.read_iflags_VRT()) { + return read_csr_success(a.read_vsscratch(), status); + } else { + return read_csr_success(a.read_sscratch(), status); + } } template static inline uint64_t read_csr_sepc(STATE_ACCESS &a, bool *status) { - return read_csr_success(a.read_sepc(), status); + if (a.read_iflags_VRT()) { + return read_csr_success(a.read_vsepc(), status); + } else { + return read_csr_success(a.read_sepc(), status); + } } template static inline uint64_t read_csr_scause(STATE_ACCESS &a, bool *status) { - return read_csr_success(a.read_scause(), status); + if (a.read_iflags_VRT()) { + return read_csr_success(a.read_vscause(), status); + } else { + return read_csr_success(a.read_scause(), status); + } } template static inline uint64_t read_csr_stval(STATE_ACCESS &a, bool *status) { - return read_csr_success(a.read_stval(), status); + if (a.read_iflags_VRT()) { + return read_csr_success(a.read_vstval(), status); + } else { + return read_csr_success(a.read_stval(), status); + } } template static inline uint64_t read_csr_sip(STATE_ACCESS &a, bool *status) { - // Ensure values are are loaded in order: do not nest with operator - const uint64_t mip = a.read_mip(); - const uint64_t mideleg = a.read_mideleg(); - return read_csr_success(mip & mideleg, status); + // Ensure values are are loaded in order: do not nest with operator + if (a.read_iflags_VRT()) { + const uint64_t sip = a.read_vsip(); + return read_csr_success(sip, status); + } else { + const uint64_t mip = a.read_mip() & (MIP_SSIP_MASK | MIP_SEIP_MASK | MIP_STIP_MASK); + const uint64_t mideleg = a.read_mideleg(); + return read_csr_success(mip & mideleg, status); + } } template static inline uint64_t read_csr_satp(STATE_ACCESS &a, bool *status) { - const uint64_t mstatus = a.read_mstatus(); auto priv = a.read_iflags_PRV(); - // When TVM=1, attempts to read or write the satp CSR - // while executing in S-mode will raise an illegal instruction exception - if (unlikely(priv == PRV_S && (mstatus & MSTATUS_TVM_MASK))) { - return read_csr_fail(status); + if (a.read_iflags_VRT()) { + const uint64_t hstatus = a.read_hstatus(); + // When VTVM=1, an attempt in VS-mode to access CSR satp raises a virtual instruction + // exception + if (unlikely(priv == NOM_S && (hstatus & HSTATUS_VTVM_MASK))) { + return read_csr_fail(status); + } + const uint64_t atp = a.read_vsatp(); + return read_csr_success(atp, status); + } else { + const uint64_t mstatus = a.read_mstatus(); + // When TVM=1, attempts to read or write the satp CSR + // while executing in S-mode will raise an illegal instruction exception + if (unlikely(priv == NOM_S && (mstatus & MSTATUS_TVM_MASK))) { + return read_csr_fail(status); + } + const uint64_t atp = a.read_satp(); + return read_csr_success(atp, status); } - return read_csr_success(a.read_satp(), status); } template @@ -1704,6 +2067,12 @@ static inline uint64_t read_csr_fflags(STATE_ACCESS &a, bool *status) { if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return read_csr_fail(status); } + if (a.read_iflags_VRT()) { + const uint64_t vsstatus = a.read_vsstatus(); + if (unlikely((vsstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { + return read_csr_fail(status); + } + } const uint64_t fflags = (a.read_fcsr() & FCSR_FFLAGS_RW_MASK) >> FCSR_FFLAGS_SHIFT; return read_csr_success(fflags, status); } @@ -1714,6 +2083,12 @@ static inline uint64_t read_csr_frm(STATE_ACCESS &a, bool *status) { if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return read_csr_fail(status); } + if (a.read_iflags_VRT()) { + const uint64_t vsstatus = a.read_vsstatus(); + if (unlikely((vsstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { + return read_csr_fail(status); + } + } const uint64_t frm = (a.read_fcsr() & FCSR_FRM_RW_MASK) >> FCSR_FRM_SHIFT; return read_csr_success(frm, status); } @@ -1724,103 +2099,206 @@ static inline uint64_t read_csr_fcsr(STATE_ACCESS &a, bool *status) { if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return read_csr_fail(status); } + if (a.read_iflags_VRT()) { + auto vsstatus = a.read_vsstatus(); + if (unlikely((vsstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { + return read_csr_fail(status); + } + } return read_csr_success(a.read_fcsr(), status); } /// \brief Reads the value of a CSR given its address /// \param a Machine state accessor object. /// \param csraddr Address of CSR in file. -/// \param status Returns the status of the operation (true for success, false otherwise). -/// \returns Register value. -/// \details This function is outlined to minimize host CPU code cache pressure. +/// \param cause cause code in case of operation failure +/// \param val register value in case of operation success +/// \returns true if operation succeeded, false otherwise. template -static NO_INLINE uint64_t read_csr(STATE_ACCESS &a, uint64_t mcycle, CSR_address csraddr, bool *status) { - if (unlikely(csr_priv(csraddr) > a.read_iflags_PRV())) { - return read_csr_fail(status); +static NO_INLINE bool read_csr(STATE_ACCESS &a, uint64_t mcycle, CSR_address csraddr, uint64_t &val, + MCAUSE_constants &cause) { + if (unlikely(!csr_is_allowed_access(a, csraddr, cause))) { + return false; } + bool status = false; + MCAUSE_constants _cause = MCAUSE_ILLEGAL_INSN; switch (csraddr) { case CSR_address::fflags: - return read_csr_fflags(a, status); + val = read_csr_fflags(a, &status); + break; case CSR_address::frm: - return read_csr_frm(a, status); + val = read_csr_frm(a, &status); + break; case CSR_address::fcsr: - return read_csr_fcsr(a, status); - + val = read_csr_fcsr(a, &status); + break; case CSR_address::ucycle: - return read_csr_cycle(a, mcycle, status); + val = read_csr_cycle(a, mcycle, &status); + break; case CSR_address::uinstret: - return read_csr_instret(a, mcycle, status); + val = read_csr_instret(a, mcycle, &status); + break; case CSR_address::utime: - return read_csr_time(a, mcycle, status); - + val = read_csr_time(a, mcycle, &status); + break; case CSR_address::sstatus: - return read_csr_sstatus(a, status); + val = read_csr_sstatus(a, &status); + break; case CSR_address::senvcfg: - return read_csr_senvcfg(a, status); + val = read_csr_senvcfg(a, &status); + break; case CSR_address::sie: - return read_csr_sie(a, status); + val = read_csr_sie(a, &status); + break; case CSR_address::stvec: - return read_csr_stvec(a, status); + val = read_csr_stvec(a, &status); + break; case CSR_address::scounteren: - return read_csr_scounteren(a, status); + val = read_csr_scounteren(a, &status); + break; case CSR_address::sscratch: - return read_csr_sscratch(a, status); + val = read_csr_sscratch(a, &status); + break; case CSR_address::sepc: - return read_csr_sepc(a, status); + val = read_csr_sepc(a, &status); + break; case CSR_address::scause: - return read_csr_scause(a, status); + val = read_csr_scause(a, &status); + break; case CSR_address::stval: - return read_csr_stval(a, status); + val = read_csr_stval(a, &status); + break; case CSR_address::sip: - return read_csr_sip(a, status); + val = read_csr_sip(a, &status); + break; case CSR_address::satp: - return read_csr_satp(a, status); + val = read_csr_satp(a, &status); + if (!status) { + _cause = MCAUSE_VIRTUAL_INSTRUCTION; + } + break; case CSR_address::mstatus: - return read_csr_mstatus(a, status); + val = read_csr_mstatus(a, &status); + break; case CSR_address::menvcfg: - return read_csr_menvcfg(a, status); + val = read_csr_menvcfg(a, &status); + break; case CSR_address::misa: - return read_csr_misa(a, status); + val = read_csr_misa(a, &status); + break; case CSR_address::medeleg: - return read_csr_medeleg(a, status); + val = read_csr_medeleg(a, &status); + break; case CSR_address::mideleg: - return read_csr_mideleg(a, status); + val = read_csr_mideleg(a, &status); + break; case CSR_address::mie: - return read_csr_mie(a, status); + val = read_csr_mie(a, &status); + break; case CSR_address::mtvec: - return read_csr_mtvec(a, status); + val = read_csr_mtvec(a, &status); + break; case CSR_address::mcounteren: - return read_csr_mcounteren(a, status); + val = read_csr_mcounteren(a, &status); + break; case CSR_address::mscratch: - return read_csr_mscratch(a, status); + val = read_csr_mscratch(a, &status); + break; case CSR_address::mepc: - return read_csr_mepc(a, status); + val = read_csr_mepc(a, &status); + break; case CSR_address::mcause: - return read_csr_mcause(a, status); + val = read_csr_mcause(a, &status); + break; case CSR_address::mtval: - return read_csr_mtval(a, status); + val = read_csr_mtval(a, &status); + break; case CSR_address::mip: - return read_csr_mip(a, status); + val = read_csr_mip(a, &status); + break; case CSR_address::mcycle: - return read_csr_mcycle(mcycle, status); + val = read_csr_mcycle(mcycle, &status); + break; case CSR_address::minstret: - return read_csr_minstret(a, mcycle, status); - + val = read_csr_minstret(a, mcycle, &status); + break; case CSR_address::mvendorid: - return read_csr_mvendorid(a, status); + val = read_csr_mvendorid(a, &status); + break; case CSR_address::marchid: - return read_csr_marchid(a, status); + val = read_csr_marchid(a, &status); + break; case CSR_address::mimpid: - return read_csr_mimpid(a, status); + val = read_csr_mimpid(a, &status); + break; - // All hardwired to zero - case CSR_address::mhartid: - case CSR_address::mcountinhibit: - case CSR_address::mconfigptr: + case CSR_address::hstatus: + val = read_csr_hstatus(a, &status); + break; + case CSR_address::hedeleg: + val = read_csr_hedeleg(a, &status); + break; + case CSR_address::hideleg: + val = read_csr_hideleg(a, &status); + break; + case CSR_address::hie: + val = read_csr_hie(a, &status); + break; + case CSR_address::hip: + val = read_csr_hip(a, &status); + break; + case CSR_address::hvip: + val = read_csr_hvip(a, &status); + break; + case CSR_address::henvcfg: + val = read_csr_henvcfg(a, &status); + break; + case CSR_address::hgatp: + val = read_csr_hgatp(a, &status); + break; + case CSR_address::htimedelta: + val = read_csr_htimedelta(a, &status); + break; + case CSR_address::htval: + val = read_csr_htval(a, &status); + break; + + case CSR_address::vsstatus: + val = read_csr_vsstatus(a, &status); + break; + case CSR_address::vsie: + val = read_csr_vsie(a, &status); + break; + case CSR_address::vstvec: + val = read_csr_vstvec(a, &status); + break; + case CSR_address::vsscratch: + val = read_csr_vsscratch(a, &status); + break; + case CSR_address::vsepc: + val = read_csr_vsepc(a, &status); + break; + case CSR_address::vscause: + val = read_csr_vscause(a, &status); + break; + case CSR_address::vstval: + val = read_csr_vstval(a, &status); + break; + case CSR_address::vsip: + val = read_csr_vsip(a, &status); + break; + case CSR_address::vsatp: + val = read_csr_vsatp(a, &status); + break; + + // All hardwired to zero + case CSR_address::mhartid: + case CSR_address::mcountinhibit: + case CSR_address::mconfigptr: case CSR_address::mhpmcounter3: case CSR_address::mhpmcounter4: case CSR_address::mhpmcounter5: @@ -1879,25 +2357,321 @@ static NO_INLINE uint64_t read_csr(STATE_ACCESS &a, uint64_t mcycle, CSR_address case CSR_address::mhpmevent29: case CSR_address::mhpmevent30: case CSR_address::mhpmevent31: + case CSR_address::mtval2: + case CSR_address::mtinst: case CSR_address::tselect: case CSR_address::tdata1: case CSR_address::tdata2: case CSR_address::tdata3: - return read_csr_success(0, status); + case CSR_address::htinst: + case CSR_address::hgeip: + case CSR_address::hgeie: + case CSR_address::hcounteren: + val = read_csr_success(0, &status); + break; default: // Invalid CSRs #ifdef DUMP_INVALID_CSR fprintf(stderr, "csr_read: invalid CSR=0x%x\n", static_cast(csraddr)); #endif - return read_csr_fail(status); + val = read_csr_fail(&status); + } + + if (!status) { + cause = _cause; } + return status; } template -static execute_status write_csr_sstatus(STATE_ACCESS &a, uint64_t val) { +static void propagate_virtual_ip(STATE_ACCESS &a, uint64_t val) { + // this method syncronizes virtual 'interrupt pending' bits accross all + // the registers that keep track of them: mip, hip, hvip, vsip. + // the final registers' values depend on mideleg and hideleg. + + // 1. take VSEI, VSTI, VSSI, SGEI bits values + // for bits of mideleg that are zero, the corresponding bits in hip are read-only zeros + auto virtual_bits = val & HIX_MASK & a.read_mideleg(); + + // 2. write virtual bits to hvip + // hvip does not have SGEIP bit, thus applying the mask here + const uint64_t hvip = (a.read_hvip() & ~HVIP_RW_MASK) | (virtual_bits & HVIP_RW_MASK); + a.write_hvip(hvip); + + // 3. write virtual bits to hip + // hip.SGEIP is read-only 0 as we do not support guest external interrupts + // hip.VSEIP is the logical-OR of hvip.VSEIP and the bit of hgeip selected by hstatus.VGEIN (not supported) + // hip.VSTIP is the logical-OR of hvip.VSTIP and any other platform-specific timer interrupt signal + // directed to VS-level (not supported) + // hip.VSSIP is an alias of hvip.VSSIP + // thus, hip = hvip + const uint64_t hip = (a.read_hip() & ~HVIP_RW_MASK) | (virtual_bits & HVIP_RW_MASK); + a.write_hip(hip); + + // 4. write virtual bits to mip + // SGEIP, VSEIP, VSTIP, and VSSIP in mip are aliases for the same bits in hip + auto mip = (a.read_mip() & ~HIP_R_MASK) | virtual_bits; + a.write_mip(mip); + + // 5. write VSEI, VSTI, VSSI bits to vsip (SGEI is absent in vsip) + // when bits 2, 6, 10 in hideleg are zero, bits 1, 5, 9 in vsip are read-only zero + // else, bits 1, 5, 9 in vsip are aliases for the bits 2, 6, 10 in hip + auto alias_bits = (a.read_hip() >> 1) & VSIP_RW_MASK & (a.read_hideleg() >> 1); + auto vsip = (a.read_vsip() & ~VSIP_RW_MASK) | alias_bits; + a.write_vsip(vsip); +} + +template +static void propagate_virtual_ie(STATE_ACCESS &a, uint64_t val) { + // this method syncronizes virtual 'interrupt enabled' bits accross all + // the registers that keep track of them: mie, hie, vsie. + // the final registers' values depend on mideleg and hideleg. + + // 1. take VSEI, VSTI, VSSI, SGEI bits values + // for bits of mideleg that are zero, the corresponding bits in hie are read-only zeros + auto virtual_bits = val & HIE_W_MASK & a.read_mideleg(); + + // 2. write virtual bits to hie + auto hie = (a.read_hie() & ~HIE_W_MASK) | virtual_bits; + a.write_hie(hie); + + // 3. write virtual bits to mie + // SGEIE, VSEIE, VSTIE, and VSSIE in mie are aliases for the same bits in hie + auto mie = (a.read_mie() & ~HIE_W_MASK) | virtual_bits; + a.write_mie(mie); + + // 4. write VSEI, VSTI, VSSI bits to vsie (SGEI is absent in vsie) + // when bits 2, 6, 10 in hideleg are zero, bits 1, 5, 9 in vsie are read-only zero + // else, bits 1, 5, 9 in vsie are aliases for the bits 2, 6, 10 in hie + auto alias_bits = (a.read_hie() >> 1) & VSIE_RW_MASK & (a.read_hideleg() >> 1); + auto vsie = (a.read_vsie() & ~VSIE_RW_MASK) | alias_bits; + a.write_vsie(vsie); +} + +template +static execute_status write_csr_hstatus(STATE_ACCESS &a, uint64_t val) { + a.write_hstatus(val & HSTATUS_W_MASK); + return execute_status::success; +} + +template +static execute_status write_csr_hedeleg(STATE_ACCESS &a, uint64_t val) { + a.write_hedeleg(val & HEDELEG_W_MASK); + return execute_status::success; +} + +template +static execute_status write_csr_hideleg(STATE_ACCESS &a, uint64_t val) { + a.write_hideleg(val & HIDELEG_W_MASK); + // propagating the current hie/hip values will enable no new interrupts but + // will take into account hideleg changes for the hie/hip and vsie/vsip registers + propagate_virtual_ie(a, a.read_hie() & STANDARD_BITS_MASK); + propagate_virtual_ip(a, a.read_hip() & STANDARD_BITS_MASK); + + return execute_status::success_and_serve_interrupts; +} + +template +static execute_status write_csr_hie(STATE_ACCESS &a, uint64_t val) { + // write non-standard bits + a.write_hie(val & ~STANDARD_BITS_MASK); + // hie has only virtual bits, so we have to only propagate them here + propagate_virtual_ie(a, val & HIE_W_MASK); + + return execute_status::success_and_serve_interrupts; +} + +template +static execute_status write_csr_hip(STATE_ACCESS &a, uint64_t val) { + // write non-standard portion + a.write_hip(val & ~STANDARD_BITS_MASK); + // we only have to propagate VSSIP here as it is the only + // writable bit in hip + propagate_virtual_ip(a, val & HIP_W_MASK); + return execute_status::success_and_serve_interrupts; +} + +template +static execute_status write_csr_hgatp(STATE_ACCESS &a, uint64_t val) { + // the TLB shutdown is not happening here as in the write_csr_(v)satp methods: + // hgatp can only be changed when virtual mode is off, and when virtual mode is off + // the address translation function will not use it. Enabling virtual mode will trigger + // a TLB shootdown. const uint64_t mstatus = a.read_mstatus(); - return write_csr_mstatus(a, (mstatus & ~SSTATUS_W_MASK) | (val & SSTATUS_W_MASK)); + const uint8_t priv = a.read_iflags_PRV(); + + // When mstatus.TVM=1, attempts to read or write hgatp while executing in HS-mode will + // raise an illegal instruction exception. + if (unlikely(priv == NOM_S && (mstatus & MSTATUS_TVM_MASK))) { + return execute_status::failure; + } + + auto atp_mode = val >> SATP_MODE_SHIFT; + switch (atp_mode) { + case SATP_MODE_BARE: + case SATP_MODE_SV39: + case SATP_MODE_SV48: + case SATP_MODE_SV57: + // lowest two bits of the PPN in hgatp always read as zeroes + a.write_hgatp((val >> 2) << 2); + break; + } + return execute_status::success; +} + +template +static execute_status write_csr_hvip(STATE_ACCESS &a, uint64_t val) { + // write non-standard portion + a.write_hvip(val & ~STANDARD_BITS_MASK); + // hvip has only virtual bits, so we have to only propagate them here + propagate_virtual_ip(a, val & HVIP_RW_MASK); + return execute_status::success_and_serve_interrupts; +} + +template +static execute_status write_csr_henvcfg(STATE_ACCESS &a, uint64_t val) { + uint64_t henvcfg = a.read_henvcfg() & MENVCFG_R_MASK; + // Modify only bits that can be written to + henvcfg = (henvcfg & ~MENVCFG_W_MASK) | (val & MENVCFG_W_MASK); + // Store results + a.write_henvcfg(henvcfg); + return execute_status::success; +} + +template +static execute_status write_csr_htimedelta(STATE_ACCESS &a, uint64_t val) { + a.write_htimedelta(val); + return execute_status::success; +} + +template +static execute_status write_csr_htval(STATE_ACCESS &a, uint64_t val) { + a.write_htval(val); + return execute_status::success; +} + +template +static execute_status write_csr_vsstatus(STATE_ACCESS &a, uint64_t val) { + const uint64_t old_vsstatus = a.read_vsstatus() & VSSTATUS_R_MASK; + uint64_t vsstatus = (old_vsstatus & ~VSSTATUS_W_MASK) | (val & VSSTATUS_W_MASK); + auto mstatus = a.read_mstatus(); + if ((vsstatus & MSTATUS_FS_MASK) != MSTATUS_FS_OFF) { + mstatus |= MSTATUS_FS_DIRTY; + mstatus |= MSTATUS_SD_MASK; + vsstatus |= MSTATUS_FS_DIRTY; + vsstatus |= MSTATUS_SD_MASK; + } else { + vsstatus &= ~MSTATUS_SD_MASK; + } + a.write_mstatus(mstatus); + a.write_vsstatus(vsstatus); + const uint64_t mod = flush_tlb(a, old_vsstatus, vsstatus); + + // When changing an interrupt enabled bit, we may have to service any pending interrupt + if (mod & MSTATUS_SIE_MASK) { + return execute_status::success_and_serve_interrupts; + } + return execute_status::success; +} + +template +static execute_status write_csr_vsie(STATE_ACCESS &a, uint64_t val) { + // store non-standard bits + a.write_vsie(val & ~STANDARD_BITS_MASK); + + // vsie has only virtual bits, so we have to only propagate them here + auto vsie_bits = (val & VSIE_RW_MASK) << 1; + propagate_virtual_ie(a, vsie_bits); + + return execute_status::success_and_serve_interrupts; +} + +template +static execute_status write_csr_vstvec(STATE_ACCESS &a, uint64_t val) { + a.write_vstvec(val); + return execute_status::success; +} + +template +static execute_status write_csr_vsscratch(STATE_ACCESS &a, uint64_t val) { + a.write_vsscratch(val); + return execute_status::success; +} + +template +static execute_status write_csr_vsepc(STATE_ACCESS &a, uint64_t val) { + a.write_vsepc(val); + return execute_status::success; +} + +template +static execute_status write_csr_vscause(STATE_ACCESS &a, uint64_t val) { + a.write_vscause(val); + return execute_status::success; +} + +template +static execute_status write_csr_vstval(STATE_ACCESS &a, uint64_t val) { + a.write_vstval(val); + return execute_status::success; +} + +template +static execute_status write_csr_vsip(STATE_ACCESS &a, uint64_t val) { + // store non-standard bits + a.write_vsip(val & ~STANDARD_BITS_MASK); + + // vsip has only virtual bits, so we have to only propagate them here + propagate_virtual_ip(a, val & VSIP_RW_MASK); + + return execute_status::success_and_serve_interrupts; +} + +template +static execute_status write_csr_vsatp(STATE_ACCESS &a, uint64_t val) { + const uint64_t old_satp = a.read_vsatp(); + const uint64_t atp = old_satp; + auto mode = val >> SATP_MODE_SHIFT; + switch (mode) { + case SATP_MODE_BARE: + case SATP_MODE_SV39: + case SATP_MODE_SV48: + case SATP_MODE_SV57: + a.write_vsatp(val); + break; + default: + return execute_status::success; + } + +#ifdef DUMP_COUNTERS + uint64_t asid = (atp & SATP_ASID_MASK) >> SATP_ASID_SHIFT; + if (asid != ASID_MAX_MASK) { // Software is not testing ASID bits + a.get_statistics().max_asid = std::max(a.get_statistics().max_asid, asid); + } +#endif + + // Changes to MODE and ASID, flushes the TLBs. + // Note that there is no need to flush the TLB when PPN has changed, + // because software is required to execute SFENCE.VMA when recycling an ASID. + const uint64_t mod = old_satp ^ atp; + if (mod & (SATP_ASID_MASK | SATP_MODE_MASK)) { + a.flush_all_tlb(); + INC_COUNTER(a.get_statistics(), tlb_flush_all); + INC_COUNTER(a.get_statistics(), tlb_flush_satp); + return execute_status::success_and_flush_fetch; + } + return execute_status::success; +} + +template +static execute_status write_csr_sstatus(STATE_ACCESS &a, uint64_t val) { + if (a.read_iflags_VRT()) { + return write_csr_vsstatus(a, val); + } else { + const uint64_t mstatus = a.read_mstatus(); + return write_csr_mstatus(a, (mstatus & ~SSTATUS_W_MASK) | (val & SSTATUS_W_MASK)); + } } template @@ -1909,16 +2683,22 @@ static execute_status write_csr_senvcfg(STATE_ACCESS &a, uint64_t val) { template static execute_status write_csr_sie(STATE_ACCESS &a, uint64_t val) { - uint64_t mie = a.read_mie(); - const uint64_t mask = a.read_mideleg(); - mie = (mie & ~mask) | (val & mask); - a.write_mie(mie); - return execute_status::success_and_serve_interrupts; + if (a.read_iflags_VRT()) { + return write_csr_vsie(a, val); + } else { + const uint64_t mie = (a.read_mie() & ~SIE_RW_MASK) | (val & SIE_RW_MASK); + a.write_mie(mie); + return execute_status::success_and_serve_interrupts; + } } template static execute_status write_csr_stvec(STATE_ACCESS &a, uint64_t val) { - a.write_stvec(val & ~1); + if (a.read_iflags_VRT()) { + a.write_vstvec(val & ~1); + } else { + a.write_stvec(val & ~1); + } return execute_status::success; } @@ -1930,50 +2710,83 @@ static execute_status write_csr_scounteren(STATE_ACCESS &a, uint64_t val) { template static execute_status write_csr_sscratch(STATE_ACCESS &a, uint64_t val) { - a.write_sscratch(val); + if (a.read_iflags_VRT()) { + a.write_vsscratch(val); + } else { + a.write_sscratch(val); + } return execute_status::success; } template static execute_status write_csr_sepc(STATE_ACCESS &a, uint64_t val) { - a.write_sepc(val & ~1); + if (a.read_iflags_VRT()) { + a.write_vsepc(val & ~1); + } else { + a.write_sepc(val & ~1); + } return execute_status::success; } template static execute_status write_csr_scause(STATE_ACCESS &a, uint64_t val) { - a.write_scause(val); + if (a.read_iflags_VRT()) { + a.write_vscause(val); + } else { + a.write_scause(val); + } return execute_status::success; } template static execute_status write_csr_stval(STATE_ACCESS &a, uint64_t val) { - a.write_stval(val); + if (a.read_iflags_VRT()) { + a.write_vstval(val); + } else { + a.write_stval(val); + } return execute_status::success; } template static execute_status write_csr_sip(STATE_ACCESS &a, uint64_t val) { - const uint64_t mask = a.read_mideleg(); - uint64_t mip = a.read_mip(); - mip = (mip & ~mask) | (val & mask); - a.write_mip(mip); + if (a.read_iflags_VRT()) { + return write_csr_vsip(a, val); + } else { + const uint64_t mip = (a.read_mip() & ~SIP_RW_MASK) | (val & SIP_RW_MASK); + a.write_mip(mip); + return execute_status::success_and_serve_interrupts; + } + return execute_status::success_and_serve_interrupts; } template static NO_INLINE execute_status write_csr_satp(STATE_ACCESS &a, uint64_t val) { - const uint64_t mstatus = a.read_mstatus(); auto priv = a.read_iflags_PRV(); + uint64_t old_satp = 0; - // When TVM=1, attempts to read or write the satp CSR - // while executing in S-mode will raise an illegal instruction exception - if (unlikely(priv == PRV_S && (mstatus & MSTATUS_TVM_MASK))) { - return execute_status::failure; + if (a.read_iflags_VRT()) { + const uint64_t hstatus = a.read_hstatus(); + // When VTVM=1, an attempt in VS-mode to access CSR satp raises a virtual instruction + // exception + if (unlikely(priv == NOM_S && (hstatus & HSTATUS_VTVM_MASK))) { + return execute_status::failure; + } + + old_satp = a.read_vsatp(); + } else { + const uint64_t mstatus = a.read_mstatus(); + // When TVM=1, attempts to read or write the satp CSR while executing in S-mode will + // raise an illegal instruction exception + if (unlikely(priv == NOM_S && (mstatus & MSTATUS_TVM_MASK))) { + return execute_status::failure; + } + + old_satp = a.read_satp(); } - const uint64_t old_satp = a.read_satp(); - uint64_t stap = old_satp; + uint64_t atp = old_satp; const uint64_t mode = val >> SATP_MODE_SHIFT; // Checks for supported MODE @@ -1982,7 +2795,7 @@ static NO_INLINE execute_status write_csr_satp(STATE_ACCESS &a, uint64_t val) { case SATP_MODE_SV39: case SATP_MODE_SV48: case SATP_MODE_SV57: - stap = (val & SATP_PPN_MASK) | (val & SATP_ASID_MASK) | (val & SATP_MODE_MASK); + atp = (val & SATP_PPN_MASK) | (val & SATP_ASID_MASK) | (val & SATP_MODE_MASK); break; default: // Implementations are not required to support all MODE settings, @@ -1990,10 +2803,14 @@ static NO_INLINE execute_status write_csr_satp(STATE_ACCESS &a, uint64_t val) { // the entire write has no effect; no fields in satp are modified. return execute_status::success; } - a.write_satp(stap); + if (a.read_iflags_VRT()) { + a.write_vsatp(atp); + } else { + a.write_satp(atp); + } #ifdef DUMP_COUNTERS - uint64_t asid = (stap & SATP_ASID_MASK) >> SATP_ASID_SHIFT; + uint64_t asid = (atp & SATP_ASID_MASK) >> SATP_ASID_SHIFT; if (asid != ASID_MAX_MASK) { // Software is not testing ASID bits a.get_statistics().max_asid = std::max(a.get_statistics().max_asid, asid); } @@ -2002,7 +2819,7 @@ static NO_INLINE execute_status write_csr_satp(STATE_ACCESS &a, uint64_t val) { // Changes to MODE and ASID, flushes the TLBs. // Note that there is no need to flush the TLB when PPN has changed, // because software is required to execute SFENCE.VMA when recycling an ASID. - const uint64_t mod = old_satp ^ stap; + const uint64_t mod = old_satp ^ atp; if (mod & (SATP_ASID_MASK | SATP_MODE_MASK)) { a.flush_all_tlb(); INC_COUNTER(a.get_statistics(), tlb_flush_all); @@ -2012,39 +2829,12 @@ static NO_INLINE execute_status write_csr_satp(STATE_ACCESS &a, uint64_t val) { return execute_status::success; } -template -static execute_status write_csr_mstatus(STATE_ACCESS &a, uint64_t val) { - const uint64_t old_mstatus = a.read_mstatus() & MSTATUS_R_MASK; - - // M-mode software can determine whether a privilege mode is implemented - // by writing that mode to MPP then reading it back. - if (PRV_HS == ((val & MSTATUS_MPP_MASK) >> MSTATUS_MPP_SHIFT)) { - // HS-mode is not supported yet, set val MPP to U-mode - val = val & ~MSTATUS_MPP_MASK; - } - - // Modify only bits that can be written to - uint64_t mstatus = (old_mstatus & ~MSTATUS_W_MASK) | (val & MSTATUS_W_MASK); - // Is FS enabled? - if ((mstatus & MSTATUS_FS_MASK) != MSTATUS_FS_OFF) { - // Implementations may choose to track the dirtiness of the floating-point register state - // imprecisely by reporting the state to be dirty even when it has not been modified. - // In our implementation an attempt to set FS to Initial or Clean causes FS to be set to Dirty, - // therefore FS is always Dirty when enabled. - mstatus |= MSTATUS_FS_DIRTY; - // The SD bit is read-only and is set when either the FS, VS, or XS bits encode a Dirty state - mstatus |= MSTATUS_SD_MASK; - } else { - // No FS, VS or XS dirty state, SD bit can be cleared - mstatus &= ~MSTATUS_SD_MASK; - } - // Store results - a.write_mstatus(mstatus); - +template +static uint64_t flush_tlb(STATE_ACCESS &a, uint64_t old_status, uint64_t new_status) { + const uint64_t mod = old_status ^ new_status; // If MMU configuration was changed, we may have to flush the TLBs bool flush_tlb_read = false; bool flush_tlb_write = false; - const uint64_t mod = old_mstatus ^ mstatus; if ((mod & MSTATUS_MXR_MASK) != 0) { // MXR allows read access to execute-only pages, // therefore it only affects read translations @@ -2056,12 +2846,15 @@ static execute_status write_csr_mstatus(STATE_ACCESS &a, uint64_t val) { flush_tlb_read = true; flush_tlb_write = true; } - if ((mod & MSTATUS_MPRV_MASK) != 0 || ((mstatus & MSTATUS_MPRV_MASK) && (mod & MSTATUS_MPP_MASK) != 0)) { - // When MPRV is set, data loads and stores use privilege in MPP - // instead of the current privilege level, but code access is unaffected, - // therefore it only affects read/write translations - flush_tlb_read = true; - flush_tlb_write = true; + + if constexpr (control_mprv) { + if ((mod & MSTATUS_MPRV_MASK) != 0 || ((new_status & MSTATUS_MPRV_MASK) && (mod & MSTATUS_MPP_MASK) != 0)) { + // When MPRV is set, data loads and stores use privilege in MPP + // instead of the current privilege level, but code access is unaffected, + // therefore it only affects read/write translations + flush_tlb_read = true; + flush_tlb_write = true; + } } // Flush TLBs when needed @@ -2077,6 +2870,38 @@ static execute_status write_csr_mstatus(STATE_ACCESS &a, uint64_t val) { INC_COUNTER(a.get_statistics(), tlb_flush_mstatus); } + return mod; +} + +template +static execute_status write_csr_mstatus(STATE_ACCESS &a, uint64_t val) { + const uint64_t old_mstatus = a.read_mstatus() & MSTATUS_R_MASK; + + // M-mode software can determine whether a privilege mode is implemented + // by writing that mode to MPP then reading it back. + if (NOM_RSVD == ((val & MSTATUS_MPP_MASK) >> MSTATUS_MPP_SHIFT)) { + val = val & ~MSTATUS_MPP_MASK; + } + + // Modify only bits that can be written to + uint64_t mstatus = (old_mstatus & ~MSTATUS_W_MASK) | (val & MSTATUS_W_MASK); + // Is FS enabled? + if ((mstatus & MSTATUS_FS_MASK) != MSTATUS_FS_OFF) { + // Implementations may choose to track the dirtiness of the floating-point register state + // imprecisely by reporting the state to be dirty even when it has not been modified. + // In our implementation an attempt to set FS to Initial or Clean causes FS to be set to Dirty, + // therefore FS is always Dirty when enabled. + mstatus |= MSTATUS_FS_DIRTY; + // The SD bit is read-only and is set when either the FS, VS, or XS bits encode a Dirty state + mstatus |= MSTATUS_SD_MASK; + } else { + // No FS, VS or XS dirty state, SD bit can be cleared + mstatus &= ~MSTATUS_SD_MASK; + } + // Store results + a.write_mstatus(mstatus); + const uint64_t mod = flush_tlb(a, old_mstatus, mstatus); + // When changing an interrupt enabled bit, we may have to service any pending interrupt if ((mod & (MSTATUS_SIE_MASK | MSTATUS_MIE_MASK)) != 0) { return execute_status::success_and_serve_interrupts; @@ -2088,7 +2913,6 @@ static execute_status write_csr_mstatus(STATE_ACCESS &a, uint64_t val) { template static execute_status write_csr_menvcfg(STATE_ACCESS &a, uint64_t val) { uint64_t menvcfg = a.read_menvcfg() & MENVCFG_R_MASK; - // Modify only bits that can be written to menvcfg = (menvcfg & ~MENVCFG_W_MASK) | (val & MENVCFG_W_MASK); // Store results @@ -2098,27 +2922,34 @@ static execute_status write_csr_menvcfg(STATE_ACCESS &a, uint64_t val) { template static execute_status write_csr_medeleg(STATE_ACCESS &a, uint64_t val) { - // For exceptions that cannot occur in less privileged modes, - // the corresponding medeleg bits should be read-only zero - a.write_medeleg((a.read_medeleg() & ~MEDELEG_W_MASK) | (val & MEDELEG_W_MASK)); + a.write_medeleg(val & MEDELEG_W_MASK); return execute_status::success; } template static execute_status write_csr_mideleg(STATE_ACCESS &a, uint64_t val) { - const uint64_t mask = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK; uint64_t mideleg = a.read_mideleg(); - mideleg = (mideleg & ~mask) | (val & mask); + mideleg = (mideleg & ~MIDELEG_W_MASK) | (val & MIDELEG_W_MASK); a.write_mideleg(mideleg); + + // for bits of mideleg that are zero, the corresponding bits in hideleg are read-only zeros + a.write_hideleg(a.read_hideleg() & mideleg); + + // propagating the current hie/hip values will enable no new interrupts but will take into + // account mideleg and hideleg changes for the hie/hip and vsie/vsip registers + propagate_virtual_ie(a, a.read_hie() & STANDARD_BITS_MASK); + propagate_virtual_ip(a, a.read_hip() & STANDARD_BITS_MASK); + return execute_status::success_and_serve_interrupts; } template static execute_status write_csr_mie(STATE_ACCESS &a, uint64_t val) { - const uint64_t mask = MIP_MSIP_MASK | MIP_MTIP_MASK | MIP_MEIP_MASK | MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK; - uint64_t mie = a.read_mie(); - mie = (mie & ~mask) | (val & mask); - a.write_mie(mie); + a.write_mie((val & ~STANDARD_BITS_MASK) | (val & MIE_W_MASK)); + + // virtual exceptions bits are not writable in mie, + // so we are not calling `propagate_virtual_ie` here + return execute_status::success_and_serve_interrupts; } @@ -2182,10 +3013,11 @@ static execute_status write_csr_mtval(STATE_ACCESS &a, uint64_t val) { template static execute_status write_csr_mip(STATE_ACCESS &a, uint64_t val) { - const uint64_t mask = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK; - auto mip = a.read_mip(); - mip = (mip & ~mask) | (val & mask); - a.write_mip(mip); + a.write_mip((val & ~STANDARD_BITS_MASK) | (val & MIP_W_MASK)); + + // virtual exceptions bits are not writable in mip, + // so we are not calling `propagate_virtual_ip` here + return execute_status::success_and_serve_interrupts; } @@ -2196,6 +3028,12 @@ static inline execute_status write_csr_fflags(STATE_ACCESS &a, uint64_t val) { if (unlikely((mstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return execute_status::failure; } + if (a.read_iflags_VRT()) { + const uint64_t vsstatus = a.read_vsstatus(); + if (unlikely((vsstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { + return execute_status::failure; + } + } const uint64_t fcsr = (a.read_fcsr() & ~FCSR_FFLAGS_RW_MASK) | ((val << FCSR_FFLAGS_SHIFT) & FCSR_FFLAGS_RW_MASK); a.write_fcsr(fcsr); return execute_status::success; @@ -2208,6 +3046,12 @@ static inline execute_status write_csr_frm(STATE_ACCESS &a, uint64_t val) { if (unlikely((mstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return execute_status::failure; } + if (a.read_iflags_VRT()) { + const uint64_t vsstatus = a.read_vsstatus(); + if (unlikely((vsstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { + return execute_status::failure; + } + } const uint64_t fcsr = (a.read_fcsr() & ~FCSR_FRM_RW_MASK) | ((val << FCSR_FRM_SHIFT) & FCSR_FRM_RW_MASK); a.write_fcsr(fcsr); return execute_status::success; @@ -2220,6 +3064,12 @@ static inline execute_status write_csr_fcsr(STATE_ACCESS &a, uint64_t val) { if (unlikely((mstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return execute_status::failure; } + if (a.read_iflags_VRT()) { + const uint64_t vsstatus = a.read_vsstatus(); + if (unlikely((vsstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { + return execute_status::failure; + } + } const uint64_t fcsr = val & FCSR_RW_MASK; a.write_fcsr(fcsr); return execute_status::success; @@ -2229,19 +3079,22 @@ static inline execute_status write_csr_fcsr(STATE_ACCESS &a, uint64_t val) { /// \param a Machine state accessor object. /// \param csraddr Address of CSR in file. /// \param val New register value. +/// \param status cause code in case of error. /// \returns The status of the operation (true for success, false otherwise). /// \details This function is outlined to minimize host CPU code cache pressure. template -static NO_INLINE execute_status write_csr(STATE_ACCESS &a, uint64_t mcycle, CSR_address csraddr, uint64_t val) { +static NO_INLINE execute_status write_csr(STATE_ACCESS &a, uint64_t mcycle, CSR_address csraddr, uint64_t val, + MCAUSE_constants &status) { #if defined(DUMP_CSR) fprintf(stderr, "csr_write: csr=0x%03x val=0x", static_cast(csraddr)); print_uint64_t(val); fprintf(stderr, "\n"); #endif if (unlikely(csr_is_read_only(csraddr))) { + status = MCAUSE_ILLEGAL_INSN; return execute_status::failure; } - if (unlikely(csr_priv(csraddr) > a.read_iflags_PRV())) { + if (unlikely(!csr_is_allowed_access(a, csraddr, status))) { return execute_status::failure; } @@ -2309,6 +3162,46 @@ static NO_INLINE execute_status write_csr(STATE_ACCESS &a, uint64_t mcycle, CSR_ case CSR_address::minstret: return write_csr_minstret(a, mcycle, val); + case CSR_address::hstatus: + return write_csr_hstatus(a, val); + case CSR_address::hedeleg: + return write_csr_hedeleg(a, val); + case CSR_address::hideleg: + return write_csr_hideleg(a, val); + case CSR_address::hie: + return write_csr_hie(a, val); + case CSR_address::hip: + return write_csr_hip(a, val); + case CSR_address::hvip: + return write_csr_hvip(a, val); + case CSR_address::henvcfg: + return write_csr_henvcfg(a, val); + case CSR_address::hgatp: + return write_csr_hgatp(a, val); + case CSR_address::htimedelta: + return write_csr_htimedelta(a, val); + case CSR_address::htval: + return write_csr_htval(a, val); + + case CSR_address::vsstatus: + return write_csr_vsstatus(a, val); + case CSR_address::vsie: + return write_csr_vsie(a, val); + case CSR_address::vstvec: + return write_csr_vstvec(a, val); + case CSR_address::vsscratch: + return write_csr_vsscratch(a, val); + case CSR_address::vsepc: + return write_csr_vsepc(a, val); + case CSR_address::vscause: + return write_csr_vscause(a, val); + case CSR_address::vstval: + return write_csr_vstval(a, val); + case CSR_address::vsip: + return write_csr_vsip(a, val); + case CSR_address::vsatp: + return write_csr_vsatp(a, val); + // Ignore writes case CSR_address::misa: case CSR_address::mhpmcounter3: @@ -2370,10 +3263,15 @@ static NO_INLINE execute_status write_csr(STATE_ACCESS &a, uint64_t mcycle, CSR_ case CSR_address::mhpmevent29: case CSR_address::mhpmevent30: case CSR_address::mhpmevent31: + case CSR_address::mtval2: + case CSR_address::mtinst: case CSR_address::tselect: case CSR_address::tdata1: case CSR_address::tdata2: case CSR_address::tdata3: + case CSR_address::htinst: + case CSR_address::hgeie: + case CSR_address::hcounteren: return execute_status::success; default: @@ -2381,6 +3279,7 @@ static NO_INLINE execute_status write_csr(STATE_ACCESS &a, uint64_t mcycle, CSR_ #ifdef DUMP_INVALID_CSR fprintf(stderr, "csr_write: invalid CSR=0x%x\n", static_cast(csraddr)); #endif + status = MCAUSE_ILLEGAL_INSN; return execute_status::failure; } } @@ -2390,23 +3289,28 @@ static FORCE_INLINE execute_status execute_csr_RW(STATE_ACCESS &a, uint64_t &pc, const RS1VAL &rs1val) { auto csraddr = static_cast(insn_I_get_uimm(insn)); // Try to read old CSR value - bool status = true; + MCAUSE_constants status = MCAUSE_INSN_ADDRESS_MISALIGNED; uint64_t csrval = 0; // If rd=r0, we do not read from the CSR to avoid side-effects const uint32_t rd = insn_get_rd(insn); if (rd != 0) { - csrval = read_csr(a, mcycle, csraddr, &status); - } - if (unlikely(!status)) { - return raise_illegal_insn_exception(a, pc, insn); + if (unlikely(!read_csr(a, mcycle, csraddr, csrval, status))) { + if (status == MCAUSE_VIRTUAL_INSTRUCTION) { + return raise_virtual_insn_exception(a, pc, insn); + } + return raise_illegal_insn_exception(a, pc, insn); + } } // Try to write new CSR value //??D When we optimize the inner interpreter loop, we // will have to check if there was a change to the // memory manager and report back from here so we // break out of the inner loop - const execute_status wstatus = write_csr(a, mcycle, csraddr, rs1val(a, insn)); + const execute_status wstatus = write_csr(a, mcycle, csraddr, rs1val(a, insn), status); if (unlikely(wstatus == execute_status::failure)) { + if (status == MCAUSE_VIRTUAL_INSTRUCTION) { + return raise_virtual_insn_exception(a, pc, insn); + } return raise_illegal_insn_exception(a, pc, insn); } // Write to rd only after potential read/write exceptions @@ -2438,11 +3342,15 @@ static FORCE_INLINE execute_status execute_csr_SC(STATE_ACCESS &a, uint64_t &pc, const F &f) { auto csraddr = static_cast(insn_I_get_uimm(insn)); // Try to read old CSR value - bool status = false; - const uint64_t csrval = read_csr(a, mcycle, csraddr, &status); - if (unlikely(!status)) { + MCAUSE_constants status = MCAUSE_INSN_ADDRESS_MISALIGNED; + uint64_t csrval = 0; + if (unlikely(!read_csr(a, mcycle, csraddr, csrval, status))) { + if (status == MCAUSE_VIRTUAL_INSTRUCTION) { + return raise_virtual_insn_exception(a, pc, insn); + } return raise_illegal_insn_exception(a, pc, insn); } + // Load value of rs1 before potentially overwriting it // with the value of the csr when rd=rs1 const uint32_t rs1 = insn_get_rs1(insn); @@ -2453,8 +3361,11 @@ static FORCE_INLINE execute_status execute_csr_SC(STATE_ACCESS &a, uint64_t &pc, // will have to check if there was a change to the // memory manager and report back from here so we // break out of the inner loop - wstatus = write_csr(a, mcycle, csraddr, f(csrval, rs1val)); + wstatus = write_csr(a, mcycle, csraddr, f(csrval, rs1val), status); if (unlikely(wstatus == execute_status::failure)) { + if (status == MCAUSE_VIRTUAL_INSTRUCTION) { + return raise_virtual_insn_exception(a, pc, insn); + } return raise_illegal_insn_exception(a, pc, insn); } } @@ -2486,9 +3397,12 @@ static FORCE_INLINE execute_status execute_csr_SCI(STATE_ACCESS &a, uint64_t &pc const F &f) { auto csraddr = static_cast(insn_I_get_uimm(insn)); // Try to read old CSR value - bool status = false; - const uint64_t csrval = read_csr(a, mcycle, csraddr, &status); - if (unlikely(!status)) { + MCAUSE_constants status = MCAUSE_INSN_ADDRESS_MISALIGNED; + uint64_t csrval = 0; + if (unlikely(!read_csr(a, mcycle, csraddr, csrval, status))) { + if (status == MCAUSE_VIRTUAL_INSTRUCTION) { + return raise_virtual_insn_exception(a, pc, insn); + } return raise_illegal_insn_exception(a, pc, insn); } const uint32_t rs1 = insn_get_rs1(insn); @@ -2498,8 +3412,11 @@ static FORCE_INLINE execute_status execute_csr_SCI(STATE_ACCESS &a, uint64_t &pc // will have to check if there was a change to the // memory manager and report back from here so we // break out of the inner loop - wstatus = write_csr(a, mcycle, csraddr, f(csrval, rs1)); + wstatus = write_csr(a, mcycle, csraddr, f(csrval, rs1), status); if (unlikely(wstatus == execute_status::failure)) { + if (status == MCAUSE_VIRTUAL_INSTRUCTION) { + return raise_virtual_insn_exception(a, pc, insn); + } return raise_illegal_insn_exception(a, pc, insn); } } @@ -2530,8 +3447,11 @@ static FORCE_INLINE execute_status execute_CSRRCI(STATE_ACCESS &a, uint64_t &pc, template static FORCE_INLINE execute_status execute_ECALL(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { dump_insn(a, pc, insn, "ecall"); - auto priv = a.read_iflags_PRV(); - pc = raise_exception(a, pc, MCAUSE_ECALL_BASE + priv, 0); + auto ecall_base_shift = a.read_iflags_PRV(); + if (a.read_iflags_VRT() && ecall_base_shift == NOM_S) { + ++ecall_base_shift; // differentiate between VS and HS ecalls + } + pc = raise_exception(a, pc, MCAUSE_ECALL_BASE + ecall_base_shift, 0, 0); return execute_status::failure; } @@ -2540,7 +3460,7 @@ template static FORCE_INLINE execute_status execute_EBREAK(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { (void) a; dump_insn(a, pc, insn, "ebreak"); - pc = raise_exception(a, pc, MCAUSE_BREAKPOINT, pc); + pc = raise_exception(a, pc, MCAUSE_BREAKPOINT, pc, 0); return execute_status::failure; } @@ -2548,29 +3468,52 @@ static FORCE_INLINE execute_status execute_EBREAK(STATE_ACCESS &a, uint64_t &pc, template static FORCE_INLINE execute_status execute_SRET(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { dump_insn(a, pc, insn, "sret"); - auto priv = a.read_iflags_PRV(); uint64_t mstatus = a.read_mstatus(); - if (unlikely(priv < PRV_S || (priv == PRV_S && (mstatus & MSTATUS_TSR_MASK)))) { + uint64_t hstatus = a.read_hstatus(); + + const uint8_t priv = a.read_iflags_PRV(); + const bool virt = a.read_iflags_VRT(); + if (unlikely(priv < NOM_S || (priv == NOM_S && (mstatus & MSTATUS_TSR_MASK)))) { return raise_illegal_insn_exception(a, pc, insn); } - auto spp = (mstatus & MSTATUS_SPP_MASK) >> MSTATUS_SPP_SHIFT; - /* set the IE state to previous IE state */ - auto spie = (mstatus & MSTATUS_SPIE_MASK) >> MSTATUS_SPIE_SHIFT; - mstatus = (mstatus & ~MSTATUS_SIE_MASK) | (spie << MSTATUS_SIE_SHIFT); - /* set SPIE to 1 */ - mstatus |= MSTATUS_SPIE_MASK; - /* set SPP to U */ - mstatus &= ~MSTATUS_SPP_MASK; - /* An SRET instruction that changes the privilege mode to a mode - * less privileged than M also sets MPRV = 0 */ - if (spp < PRV_M) { - mstatus &= ~MSTATUS_MPRV_MASK; - } - a.write_mstatus(mstatus); - if (priv != spp) { + + if (!virt && (priv == NOM_S || priv == NOM_M)) { + auto spp = (mstatus & MSTATUS_SPP_MASK) >> MSTATUS_SPP_SHIFT; + auto spv = (hstatus & HSTATUS_SPV_MASK) >> HSTATUS_SPV_SHIFT; + auto spie = (mstatus & MSTATUS_SPIE_MASK) >> MSTATUS_SPIE_SHIFT; + mstatus = (mstatus & ~MSTATUS_SIE_MASK) | (spie << MSTATUS_SIE_SHIFT); + mstatus |= MSTATUS_SPIE_MASK; + mstatus &= ~MSTATUS_SPP_MASK; + set_priv(a, spp); + if (spv == 1) { + a.set_iflags_VRT(); + } + hstatus &= ~HSTATUS_SPV_MASK; + /* An SRET instruction that changes the privilege mode to a mode + * less privileged than M also sets MPRV = 0 */ + if (spp < NOM_M) { + mstatus &= ~MSTATUS_MPRV_MASK; + } + a.write_hstatus(hstatus); + a.write_mstatus(mstatus); + pc = a.read_sepc(); + } else if (virt && priv == NOM_S) { // VS mode + if (hstatus & HSTATUS_VTSR_MASK) { + pc = raise_exception(a, pc, MCAUSE_VIRTUAL_INSTRUCTION, insn, 0); + return advance_to_raised_exception(a, pc); + } + uint64_t vsstatus = a.read_vsstatus(); + auto spp = (vsstatus & MSTATUS_SPP_MASK) >> MSTATUS_SPP_SHIFT; + auto spie = (vsstatus & MSTATUS_SPIE_MASK) >> MSTATUS_SPIE_SHIFT; + vsstatus = (vsstatus & ~MSTATUS_SIE_MASK) | (spie << MSTATUS_SIE_SHIFT); + vsstatus |= MSTATUS_SPIE_MASK; + vsstatus &= ~MSTATUS_SPP_MASK; set_priv(a, spp); + a.write_vsstatus(vsstatus); + pc = a.read_vsepc(); + } else { + return raise_illegal_insn_exception(a, pc, insn); } - pc = a.read_sepc(); return execute_status::success_and_serve_interrupts; } @@ -2579,28 +3522,39 @@ template static FORCE_INLINE execute_status execute_MRET(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { dump_insn(a, pc, insn, "mret"); auto priv = a.read_iflags_PRV(); - if (unlikely(priv < PRV_M)) { + if (unlikely(priv < NOM_M)) { return raise_illegal_insn_exception(a, pc, insn); } uint64_t mstatus = a.read_mstatus(); auto mpp = (mstatus & MSTATUS_MPP_MASK) >> MSTATUS_MPP_SHIFT; + auto mpv = (mstatus & MSTATUS_MPV_MASK) >> MSTATUS_MPV_SHIFT; + if (mpp == NOM_M) { + a.reset_iflags_VRT(); + } else { + if (mpv) { + a.set_iflags_VRT(); + } else { + a.reset_iflags_VRT(); + } + } + //??D we can save one shift here, but maybe the compiler already does /* set the IE state to previous IE state */ auto mpie = (mstatus & MSTATUS_MPIE_MASK) >> MSTATUS_MPIE_SHIFT; mstatus = (mstatus & ~MSTATUS_MIE_MASK) | (mpie << MSTATUS_MIE_SHIFT); /* set MPIE to 1 */ mstatus |= MSTATUS_MPIE_MASK; - /* set MPP to U */ + /* set MPP to 0 */ mstatus &= ~MSTATUS_MPP_MASK; + /* set MPV to 0 */ + mstatus &= ~MSTATUS_MPV_MASK; /* An MRET instruction that changes the privilege mode to a mode * less privileged than M also sets MPRV = 0 */ - if (mpp < PRV_M) { + if (mpp < NOM_M) { mstatus &= ~MSTATUS_MPRV_MASK; } a.write_mstatus(mstatus); - if (priv != mpp) { - set_priv(a, mpp); - } + set_priv(a, mpp); pc = a.read_mepc(); return execute_status::success_and_serve_interrupts; } @@ -2611,12 +3565,17 @@ template static FORCE_INLINE execute_status execute_WFI(STATE_ACCESS &a, uint64_t &pc, uint64_t &mcycle, uint32_t insn) { dump_insn(a, pc, insn, "wfi"); // Check privileges and do nothing else - auto priv = a.read_iflags_PRV(); + const uint8_t priv = a.read_iflags_PRV(); const uint64_t mstatus = a.read_mstatus(); // WFI can always causes an illegal instruction exception in less-privileged modes when TW=1 - if (unlikely(priv == PRV_U || (priv < PRV_M && (mstatus & MSTATUS_TW_MASK)))) { + if (unlikely(priv == NOM_U || (priv < NOM_M && (mstatus & MSTATUS_TW_MASK)))) { return raise_illegal_insn_exception(a, pc, insn); } + const uint64_t hstatus = a.read_hstatus(); + if (unlikely(priv == NOM_S && a.read_iflags_VRT() && (hstatus & HSTATUS_VTW_MASK))) { + return raise_virtual_insn_exception(a, pc, insn); + } + // Poll console, this may advance mcycle when in interactive mode mcycle = a.poll_console(mcycle); return advance_to_next_insn(a, pc); @@ -3203,24 +4162,8 @@ static FORCE_INLINE execute_status execute_JALR(STATE_ACCESS &a, uint64_t &pc, u return execute_jump(a, pc, new_pc); } -/// \brief Implementation of the SFENCE.VMA instruction. -/// \details This function is outlined to minimize host CPU code cache pressure. template -static FORCE_INLINE execute_status execute_SFENCE_VMA(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { - // rs1 and rs2 are arbitrary, rest is set - if (unlikely((insn & 0b11111110000000000111111111111111) != 0b00010010000000000000000001110011)) { - return raise_illegal_insn_exception(a, pc, insn); - } - INC_COUNTER(a.get_statistics(), fence_vma); - dump_insn(a, pc, insn, "sfence.vma"); - auto priv = a.read_iflags_PRV(); - const uint64_t mstatus = a.read_mstatus(); - - // When TVM=1, attempts to execute an SFENCE.VMA while executing in S-mode - // will raise an illegal instruction exception. - if (unlikely(priv == PRV_U || (priv == PRV_S && (mstatus & MSTATUS_TVM_MASK)))) { - return raise_illegal_insn_exception(a, pc, insn); - } +static execute_status execute_XFENCE(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { const uint32_t rs1 = insn_get_rs1(insn); const uint32_t rs2 = insn_get_rs2(insn); if (rs1 == 0) { @@ -3254,6 +4197,265 @@ static FORCE_INLINE execute_status execute_SFENCE_VMA(STATE_ACCESS &a, uint64_t return advance_to_next_insn(a, pc, execute_status::success_and_flush_fetch); } +/// \brief Implementation of the SFENCE.VMA instruction. +/// \details This function is outlined to minimize host CPU code cache pressure. +template +static execute_status execute_SFENCE_VMA(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + INC_COUNTER(a.get_naked_state(), fence_vma); + dump_insn(a, pc, insn, "sfence.vma"); + auto priv = a.read_iflags_PRV(); + + if (a.read_iflags_VRT()) { + const uint64_t hstatus = a.read_hstatus(); + if (unlikely(priv == NOM_S && (hstatus & HSTATUS_VTVM_MASK))) { + return raise_virtual_insn_exception(a, pc, insn); + } + } else { + const uint64_t mstatus = a.read_mstatus(); + if (unlikely(priv == NOM_U || (priv == NOM_S && (mstatus & MSTATUS_TVM_MASK)))) { + return raise_illegal_insn_exception(a, pc, insn); + } + } + + return execute_XFENCE(a, pc, insn); +} + +/// \brief Implementation of the HFENCE.VVMA instruction. +template +static execute_status execute_HFENCE_VVMA(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + const uint8_t priv = a.read_iflags_PRV(); + // HFENCE.VVMA is valid only in M-mode or HS-mode + if (unlikely(priv != NOM_M && priv != NOM_S)) { + return raise_illegal_insn_exception(a, pc, insn); + } + return execute_XFENCE(a, pc, insn); +} + +/// \brief Implementation of the HFENCE.GVMA instruction. +template +static execute_status execute_HFENCE_GVMA(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + const uint8_t priv = a.read_iflags_PRV(); + // HFENCE.GVMA is valid only in HS-mode when mstatus.TVM=0, or in M-mode (irrespective of mstatus.TVM) + if (unlikely(priv != NOM_M && priv != NOM_S)) { + return raise_illegal_insn_exception(a, pc, insn); + } + const uint64_t mstatus = a.read_mstatus(); + if (unlikely(priv == NOM_S && (mstatus & MSTATUS_TVM_MASK))) { + return raise_illegal_insn_exception(a, pc, insn); + } + return execute_XFENCE(a, pc, insn); +} + +/// \brief Implementation of the SINVAL.VMA instruction. +template +static execute_status execute_SINVAL_VMA(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + // this instruction is a part of the Svinval extension + return raise_illegal_insn_exception(a, pc, insn); +} + +/// \brief Implementation of the SFENCE.W.INVAL instruction. +template +static execute_status execute_SFENCE_W_INVAL(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + // this instruction is a part of the Svinval extension + return raise_illegal_insn_exception(a, pc, insn); +} + +/// \brief Implementation of the SFENCE.INVAL.IR instruction. +template +static execute_status execute_SFENCE_INVAL_IR(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + // this instruction is a part of the Svinval extension + return raise_illegal_insn_exception(a, pc, insn); +} + +/// \brief Implementation of the HINVAL.VVMA instruction. +template +static execute_status execute_HINVAL_VVMA(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + // this instruction is a part of the Svinval extension + return raise_illegal_insn_exception(a, pc, insn); +} + +/// \brief Implementation of the HINVAL.GVMA instruction. +template +static execute_status execute_HINVAL_GVMA(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + // this instruction is a part of the Svinval extension + return raise_illegal_insn_exception(a, pc, insn); +} + +template +static inline execute_status check_HV_insn_allowed(STATE_ACCESS &a, uint8_t &access_mode, uint64_t &pc, uint32_t insn) { + uint8_t priv = a.read_iflags_PRV(); + const bool virt = a.read_iflags_VRT(); + // HV instructions are not allowed in virtual mode + if (unlikely(virt)) { + return raise_virtual_insn_exception(a, pc, insn); + } + const uint64_t hstatus = a.read_hstatus(); + // HV instructions are not allowed in user mode unless hstatus.HU is set + if (unlikely(priv == NOM_U && !(hstatus & HSTATUS_HU_MASK))) { + return raise_illegal_insn_exception(a, pc, insn); + } + + // hstatus.SPVP controls the privilege level of access + priv = NOM_U; + if (hstatus & HSTATUS_SPVP_MASK) { + priv = NOM_S; + } + // HV instructions perform a memory access as though we are in a virtual mode + access_mode = encode_access_mode(priv, true); + return execute_status::success; +} + +template +static inline execute_status execute_HLV(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn, + uint8_t xwr_shift) { + uint8_t access_mode = NOM_U; + auto hv_allowed = check_HV_insn_allowed(a, access_mode, pc, insn); + if (unlikely(hv_allowed != execute_status::success)) { + return hv_allowed; + } + + const uint64_t vaddr = a.read_x(insn_get_rs1(insn)); + T val; + auto [status, new_pc] = + read_virtual_memory_slow(a, pc, mcycle, vaddr, access_mode, xwr_shift, &val); + pc = new_pc; + if (status) { + const uint32_t rd = insn_get_rd(insn); + // don't write x0 + if (rd != 0) { + // This static branch is eliminated by the compiler + if (std::is_signed::value) { + a.write_x(rd, static_cast(val)); + } else { + a.write_x(rd, static_cast(val)); + } + } + return advance_to_next_insn(a, pc); + } else { + return advance_to_raised_exception(a, pc); + } +} + +template +static inline execute_status execute_HLV(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + return execute_HLV(a, pc, mcycle, insn, PTE_XWR_R_SHIFT); +} + +/// \brief Implementation of the HLV.B instruction. +template +static execute_status execute_HLV_B(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hlv.b"); + return execute_HLV(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HLV.BU instruction. +template +static execute_status execute_HLV_BU(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hlv.bu"); + return execute_HLV(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HLV.H instruction. +template +static execute_status execute_HLV_H(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hlv.h"); + return execute_HLV(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HLV.HU instruction. +template +static execute_status execute_HLV_HU(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hlv.hu"); + return execute_HLV(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HLV.W instruction. +template +static execute_status execute_HLV_W(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hlv.w"); + return execute_HLV(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HLV.WU instruction. +template +static execute_status execute_HLV_WU(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hlv.wu"); + return execute_HLV(a, pc, mcycle, insn); +} + +template +static inline execute_status execute_HLVX(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + return execute_HLV(a, pc, mcycle, insn, PTE_XWR_X_SHIFT); +} + +/// \brief Implementation of the HLVX.HU instruction. +template +static execute_status execute_HLVX_HU(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hlvx.hu"); + return execute_HLVX(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HLVX.WU instruction. +template +static execute_status execute_HLVX_WU(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hlvx.wu"); + return execute_HLVX(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HLV.D instruction. +template +static execute_status execute_HLV_D(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hlv.d"); + return execute_HLV(a, pc, mcycle, insn); +} + +template +static inline execute_status execute_HSV(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + uint8_t access_mode = NOM_U; + auto hv_allowed = check_HV_insn_allowed(a, access_mode, pc, insn); + if (unlikely(hv_allowed != execute_status::success)) { + return hv_allowed; + } + + const uint64_t vaddr = a.read_x(insn_get_rs1(insn)); + const uint64_t val = a.read_x(insn_get_rs2(insn)); + auto [status, new_pc] = write_virtual_memory_slow(a, pc, mcycle, vaddr, access_mode, val); + pc = new_pc; + if (status) { + return advance_to_next_insn(a, pc); + } else { + return advance_to_raised_exception(a, pc); + } +} + +/// \brief Implementation of the HSV.B instruction. +template +static execute_status execute_HSV_B(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hsv.b"); + return execute_HSV(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HSV.H instruction. +template +static execute_status execute_HSV_H(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hsv.h"); + return execute_HSV(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HSV.W instruction. +template +static execute_status execute_HSV_W(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hsv.w"); + return execute_HSV(a, pc, mcycle, insn); +} + +/// \brief Implementation of the HSV.D instruction. +template +static execute_status execute_HSV_D(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + dump_insn(a, pc, insn, "hsv.d"); + return execute_HSV(a, pc, mcycle, insn); +} + template static FORCE_INLINE execute_status execute_SRLI_SRAI(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { switch (static_cast(insn_get_funct7_sr1(insn))) { @@ -3467,20 +4669,131 @@ static FORCE_INLINE execute_status execute_SRLW_DIVUW_SRAW(STATE_ACCESS &a, uint } template -static FORCE_INLINE execute_status execute_privileged(STATE_ACCESS &a, uint64_t &pc, uint64_t &mcycle, uint32_t insn) { - switch (static_cast(insn)) { - case insn_privileged::ECALL: +static FORCE_INLINE execute_status execute_privileged_E(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + switch (static_cast(insn_get_rs2(insn))) { + case insn_E_rs2::ECALL: return execute_ECALL(a, pc, insn); - case insn_privileged::EBREAK: + case insn_E_rs2::EBREAK: return execute_EBREAK(a, pc, insn); - case insn_privileged::SRET: - return execute_SRET(a, pc, insn); - case insn_privileged::MRET: - return execute_MRET(a, pc, insn); - case insn_privileged::WFI: + default: + return raise_illegal_insn_exception(a, pc, insn); + } +} + +template +static inline execute_status execute_privileged_WFI_SRET(STATE_ACCESS &a, uint64_t &pc, uint64_t &mcycle, + uint32_t insn) { + switch (static_cast(insn_get_rs2(insn))) { + case insn_WFI_SRET_rs2::WFI: return execute_WFI(a, pc, mcycle, insn); + case insn_WFI_SRET_rs2::SRET: + return execute_SRET(a, pc, insn); + default: + return raise_illegal_insn_exception(a, pc, insn); + } +} + +template +static inline execute_status execute_privileged_SFENCE_INVAL(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { + switch (static_cast(insn_get_rs2(insn))) { + case insn_SFENCE_INVAL_rs2::SFENCE_W_INVAL: + return execute_SFENCE_W_INVAL(a, pc, insn); + case insn_SFENCE_INVAL_rs2::SFENCE_INVAL_IR: + return execute_SFENCE_INVAL_IR(a, pc, insn); + default: + return raise_illegal_insn_exception(a, pc, insn); + } +} + +template +static inline execute_status execute_privileged_HLV_B(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + switch (static_cast(insn_get_rs2(insn))) { + case insn_HLV_B_rs2::HLV_B: + return execute_HLV_B(a, pc, mcycle, insn); + case insn_HLV_B_rs2::HLV_BU: + return execute_HLV_BU(a, pc, mcycle, insn); + default: + return raise_illegal_insn_exception(a, pc, insn); + } +} + +template +static inline execute_status execute_privileged_HLV_H(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + switch (static_cast(insn_get_rs2(insn))) { + case insn_HLV_H_rs2::HLV_H: + return execute_HLV_H(a, pc, mcycle, insn); + case insn_HLV_H_rs2::HLV_HU: + return execute_HLV_HU(a, pc, mcycle, insn); + case insn_HLV_H_rs2::HLVX_HU: + return execute_HLVX_HU(a, pc, mcycle, insn); default: + return raise_illegal_insn_exception(a, pc, insn); + } +} + +template +static inline execute_status execute_privileged_HLV_W(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + switch (static_cast(insn_get_rs2(insn))) { + case insn_HLV_W_rs2::HLV_W: + return execute_HLV_W(a, pc, mcycle, insn); + case insn_HLV_W_rs2::HLV_WU: + return execute_HLV_WU(a, pc, mcycle, insn); + case insn_HLV_W_rs2::HLVX_WU: + return execute_HLVX_WU(a, pc, mcycle, insn); + default: + return raise_illegal_insn_exception(a, pc, insn); + } +} + +template +static inline execute_status execute_privileged(STATE_ACCESS &a, uint64_t &pc, uint64_t &mcycle, uint32_t insn) { + switch (static_cast(insn_get_funct7(insn))) { + case insn_privileged_funct7::E: + return execute_privileged_E(a, pc, insn); + case insn_privileged_funct7::WFI_SRET: + return execute_privileged_WFI_SRET(a, pc, mcycle, insn); + case insn_privileged_funct7::SFENCE_INVAL: + return execute_privileged_SFENCE_INVAL(a, pc, insn); + case insn_privileged_funct7::MRET: + return execute_MRET(a, pc, insn); + case insn_privileged_funct7::SFENCE_VMA: return execute_SFENCE_VMA(a, pc, insn); + case insn_privileged_funct7::SINVAL_VMA: + return execute_SINVAL_VMA(a, pc, insn); + case insn_privileged_funct7::HFENCE_VVMA: + return execute_HFENCE_VVMA(a, pc, insn); + case insn_privileged_funct7::HFENCE_GVMA: + return execute_HFENCE_GVMA(a, pc, insn); + case insn_privileged_funct7::HINVAL_VVMA: + return execute_HINVAL_VVMA(a, pc, insn); + case insn_privileged_funct7::HINVAL_GVMA: + return execute_HINVAL_GVMA(a, pc, insn); + default: + return raise_illegal_insn_exception(a, pc, insn); + } +} + +template +static inline execute_status execute_hv_store_load(STATE_ACCESS &a, uint64_t &pc, uint64_t mcycle, uint32_t insn) { + switch (static_cast(insn_get_funct7(insn))) { + case insn_privileged_funct7::HLV_B: + return execute_privileged_HLV_B(a, pc, mcycle, insn); + case insn_privileged_funct7::HLV_H: + return execute_privileged_HLV_H(a, pc, mcycle, insn); + case insn_privileged_funct7::HLV_W: + return execute_privileged_HLV_W(a, pc, mcycle, insn); + case insn_privileged_funct7::HSV_B: + return execute_HSV_B(a, pc, mcycle, insn); + case insn_privileged_funct7::HSV_H: + return execute_HSV_H(a, pc, mcycle, insn); + case insn_privileged_funct7::HSV_W: + return execute_HSV_W(a, pc, mcycle, insn); + case insn_privileged_funct7::HLV_D: + return execute_HLV_D(a, pc, mcycle, insn); + case insn_privileged_funct7::HSV_D: + return execute_HSV_D(a, pc, mcycle, insn); + default: + return raise_illegal_insn_exception(a, pc, insn); } } @@ -5026,7 +6339,7 @@ static FORCE_INLINE execute_status execute_C_MV(STATE_ACCESS &a, uint64_t &pc, u template static FORCE_INLINE execute_status execute_C_EBREAK(STATE_ACCESS &a, uint64_t &pc, uint32_t insn) { dump_insn(a, pc, insn, "c.ebreak"); - pc = raise_exception(a, pc, MCAUSE_BREAKPOINT, pc); + pc = raise_exception(a, pc, MCAUSE_BREAKPOINT, pc, 0); return advance_to_raised_exception(a, pc); } @@ -5172,6 +6485,12 @@ static FORCE_INLINE execute_status execute_insn(STATE_ACCESS &a, uint64_t &pc, u if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); } + if (a.read_iflags_VRT()) { + auto vsstatus = a.read_vsstatus(); + if (unlikely((vsstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { + return raise_virtual_insn_exception(a, pc, insn); + } + } switch (c_funct3) { case insn_c_funct3::C_FLD: return execute_C_FLD(a, pc, mcycle, insn); @@ -5326,6 +6645,8 @@ static FORCE_INLINE execute_status execute_insn(STATE_ACCESS &a, uint64_t &pc, u return execute_SRLW_DIVUW_SRAW(a, pc, insn); case insn_funct3_00000_opcode::privileged: return execute_privileged(a, pc, mcycle, insn); + case insn_funct3_00000_opcode::hv_store_load: + return execute_hv_store_load(a, pc, mcycle, insn); default: { // Here we are sure that the next instruction, at best, can only be a floating point instruction, // or, at worst, an illegal instruction. @@ -5335,6 +6656,12 @@ static FORCE_INLINE execute_status execute_insn(STATE_ACCESS &a, uint64_t &pc, u if (unlikely((a.read_mstatus() & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { return raise_illegal_insn_exception(a, pc, insn); } + if (a.read_iflags_VRT()) { + auto vsstatus = a.read_vsstatus(); + if (unlikely((vsstatus & MSTATUS_FS_MASK) == MSTATUS_FS_OFF)) { + return raise_virtual_insn_exception(a, pc, insn); + } + } switch (funct3_00000_opcode) { case insn_funct3_00000_opcode::FSW: return execute_FSW(a, pc, mcycle, insn); @@ -5405,9 +6732,14 @@ template static FORCE_INLINE fetch_status fetch_translate_pc_slow(STATE_ACCESS &a, uint64_t &pc, uint64_t vaddr, unsigned char **phptr) { uint64_t paddr{}; + auto access_mode = get_current_memory_access_mode(a); // Walk page table and obtain the physical address - if (unlikely(!translate_virtual_address(a, &paddr, vaddr, PTE_XWR_X_SHIFT))) { - pc = raise_exception(a, pc, MCAUSE_FETCH_PAGE_FAULT, vaddr); + const uint8_t cause = + translate_virtual_address(a, &paddr, vaddr, access_mode, PTE_XWR_X_SHIFT); + if (unlikely(cause)) { + // When a guest-page-fault trap is taken into HS-mode, htval is written with the guest + // physical address that faulted, shifted right by 2 bits. + pc = raise_exception(a, pc, cause, vaddr, paddr >> 2); return fetch_status::exception; } // Walk memory map to find the range that contains the physical address @@ -5415,7 +6747,7 @@ static FORCE_INLINE fetch_status fetch_translate_pc_slow(STATE_ACCESS &a, uint64 // We only execute directly from RAM (as in "random access memory") // If the range is not memory or not executable, this as a PMA violation if (unlikely(!pma.get_istart_M() || !pma.get_istart_X())) { - pc = raise_exception(a, pc, MCAUSE_INSN_ACCESS_FAULT, vaddr); + pc = raise_exception(a, pc, MCAUSE_INSN_ACCESS_FAULT, vaddr, 0); return fetch_status::exception; } unsigned char *hpage = a.template replace_tlb_entry(vaddr, paddr, pma); diff --git a/src/jsonrpc-virtual-machine.cpp b/src/jsonrpc-virtual-machine.cpp index 45c237e79..cb04dff9f 100644 --- a/src/jsonrpc-virtual-machine.cpp +++ b/src/jsonrpc-virtual-machine.cpp @@ -624,6 +624,150 @@ void jsonrpc_virtual_machine::do_write_senvcfg(uint64_t val) { write_csr(csr::senvcfg, val); } +uint64_t jsonrpc_virtual_machine::do_read_hstatus(void) const { + return read_csr(csr::hstatus); +} + +void jsonrpc_virtual_machine::do_write_hstatus(uint64_t val) { + write_csr(csr::hstatus, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_hideleg(void) const { + return read_csr(csr::hideleg); +} + +void jsonrpc_virtual_machine::do_write_hideleg(uint64_t val) { + write_csr(csr::hideleg, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_hedeleg(void) const { + return read_csr(csr::hedeleg); +} + +void jsonrpc_virtual_machine::do_write_hedeleg(uint64_t val) { + write_csr(csr::hedeleg, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_hip(void) const { + return read_csr(csr::hip); +} + +void jsonrpc_virtual_machine::do_write_hip(uint64_t val) { + write_csr(csr::hip, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_hvip(void) const { + return read_csr(csr::hvip); +} + +void jsonrpc_virtual_machine::do_write_hvip(uint64_t val) { + write_csr(csr::hvip, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_hie(void) const { + return read_csr(csr::hie); +} + +void jsonrpc_virtual_machine::do_write_hie(uint64_t val) { + write_csr(csr::hie, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_hgatp(void) const { + return read_csr(csr::hgatp); +} + +void jsonrpc_virtual_machine::do_write_hgatp(uint64_t val) { + write_csr(csr::hgatp, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_htimedelta(void) const { + return read_csr(csr::htimedelta); +} + +void jsonrpc_virtual_machine::do_write_htimedelta(uint64_t val) { + write_csr(csr::htimedelta, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_htval(void) const { + return read_csr(csr::htval); +} + +void jsonrpc_virtual_machine::do_write_htval(uint64_t val) { + write_csr(csr::htval, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_vsepc(void) const { + return read_csr(csr::vsepc); +} + +void jsonrpc_virtual_machine::do_write_vsepc(uint64_t val) { + write_csr(csr::vsepc, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_vsstatus(void) const { + return read_csr(csr::vsstatus); +} + +void jsonrpc_virtual_machine::do_write_vsstatus(uint64_t val) { + write_csr(csr::vsstatus, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_vscause(void) const { + return read_csr(csr::vscause); +} + +void jsonrpc_virtual_machine::do_write_vscause(uint64_t val) { + write_csr(csr::vscause, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_vstval(void) const { + return read_csr(csr::vstval); +} + +void jsonrpc_virtual_machine::do_write_vstval(uint64_t val) { + write_csr(csr::vstval, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_vstvec(void) const { + return read_csr(csr::vstvec); +} + +void jsonrpc_virtual_machine::do_write_vstvec(uint64_t val) { + write_csr(csr::vstvec, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_vsscratch(void) const { + return read_csr(csr::vsscratch); +} + +void jsonrpc_virtual_machine::do_write_vsscratch(uint64_t val) { + write_csr(csr::vsscratch, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_vsatp(void) const { + return read_csr(csr::vsatp); +} + +void jsonrpc_virtual_machine::do_write_vsatp(uint64_t val) { + write_csr(csr::vsatp, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_vsip(void) const { + return read_csr(csr::vsip); +} + +void jsonrpc_virtual_machine::do_write_vsip(uint64_t val) { + write_csr(csr::vsip, val); +} + +uint64_t jsonrpc_virtual_machine::do_read_vsie(void) const { + return read_csr(csr::vsie); +} + +void jsonrpc_virtual_machine::do_write_vsie(uint64_t val) { + write_csr(csr::vsie, val); +} + uint64_t jsonrpc_virtual_machine::do_read_ilrsc(void) const { return read_csr(csr::ilrsc); } diff --git a/src/jsonrpc-virtual-machine.h b/src/jsonrpc-virtual-machine.h index f96f8b5f0..096eb9782 100644 --- a/src/jsonrpc-virtual-machine.h +++ b/src/jsonrpc-virtual-machine.h @@ -133,6 +133,44 @@ class jsonrpc_virtual_machine final : public i_virtual_machine { void do_write_scounteren(uint64_t val) override; uint64_t do_read_senvcfg(void) const override; void do_write_senvcfg(uint64_t val) override; + uint64_t do_read_hstatus(void) const override; + void do_write_hstatus(uint64_t val) override; + uint64_t do_read_hideleg(void) const override; + void do_write_hideleg(uint64_t val) override; + uint64_t do_read_hedeleg(void) const override; + void do_write_hedeleg(uint64_t val) override; + uint64_t do_read_hip(void) const override; + void do_write_hip(uint64_t val) override; + uint64_t do_read_hvip(void) const override; + void do_write_hvip(uint64_t val) override; + uint64_t do_read_hie(void) const override; + void do_write_hie(uint64_t val) override; + uint64_t do_read_hgatp(void) const override; + void do_write_hgatp(uint64_t val) override; + uint64_t do_read_henvcfg(void) const override; + void do_write_henvcfg(uint64_t val) override; + uint64_t do_read_htimedelta(void) const override; + void do_write_htimedelta(uint64_t val) override; + uint64_t do_read_htval(void) const override; + void do_write_htval(uint64_t val) override; + uint64_t do_read_vsepc(void) const override; + void do_write_vsepc(uint64_t val) override; + uint64_t do_read_vsstatus(void) const override; + void do_write_vsstatus(uint64_t val) override; + uint64_t do_read_vscause(void) const override; + void do_write_vscause(uint64_t val) override; + uint64_t do_read_vstval(void) const override; + void do_write_vstval(uint64_t val) override; + uint64_t do_read_vstvec(void) const override; + void do_write_vstvec(uint64_t val) override; + uint64_t do_read_vsscratch(void) const override; + void do_write_vsscratch(uint64_t val) override; + uint64_t do_read_vsatp(void) const override; + void do_write_vsatp(uint64_t val) override; + uint64_t do_read_vsip(void) const override; + void do_write_vsip(uint64_t val) override; + uint64_t do_read_vsie(void) const override; + void do_write_vsie(uint64_t val) override; uint64_t do_read_ilrsc(void) const override; void do_write_ilrsc(uint64_t val) override; uint64_t do_read_iflags(void) const override; diff --git a/src/machine-c-api.cpp b/src/machine-c-api.cpp index 0808d5b7b..6f756cb50 100644 --- a/src/machine-c-api.cpp +++ b/src/machine-c-api.cpp @@ -1144,6 +1144,25 @@ IMPL_MACHINE_READ_WRITE(stval) IMPL_MACHINE_READ_WRITE(satp) IMPL_MACHINE_READ_WRITE(scounteren) IMPL_MACHINE_READ_WRITE(senvcfg) +IMPL_MACHINE_READ_WRITE(hstatus) +IMPL_MACHINE_READ_WRITE(hideleg) +IMPL_MACHINE_READ_WRITE(hedeleg) +IMPL_MACHINE_READ_WRITE(hip) +IMPL_MACHINE_READ_WRITE(hvip) +IMPL_MACHINE_READ_WRITE(hie) +IMPL_MACHINE_READ_WRITE(hgatp) +IMPL_MACHINE_READ_WRITE(henvcfg) +IMPL_MACHINE_READ_WRITE(htimedelta) +IMPL_MACHINE_READ_WRITE(htval) +IMPL_MACHINE_READ_WRITE(vsepc) +IMPL_MACHINE_READ_WRITE(vsstatus) +IMPL_MACHINE_READ_WRITE(vscause) +IMPL_MACHINE_READ_WRITE(vstval) +IMPL_MACHINE_READ_WRITE(vstvec) +IMPL_MACHINE_READ_WRITE(vsscratch) +IMPL_MACHINE_READ_WRITE(vsatp) +IMPL_MACHINE_READ_WRITE(vsie) +IMPL_MACHINE_READ_WRITE(vsip) IMPL_MACHINE_READ_WRITE(ilrsc) IMPL_MACHINE_READ_WRITE(iflags) IMPL_MACHINE_READ_WRITE(htif_tohost) @@ -1161,8 +1180,8 @@ IMPL_MACHINE_READ_WRITE(uarch_pc) IMPL_MACHINE_READ(uarch_ram_length) // clang-format-on -uint64_t cm_packed_iflags(int PRV, int X, int Y, int H) { - return cartesi::machine_state::packed_iflags(PRV, X, Y, H); +uint64_t cm_packed_iflags(int VRT, int PRV, int X, int Y, int H) { + return cartesi::machine_state::packed_iflags(VRT, PRV, X, Y, H); } int cm_read_iflags_Y(const cm_machine *m, bool *val, char **err_msg) try { diff --git a/src/machine-c-api.h b/src/machine-c-api.h index 7c411e89f..b3f068674 100644 --- a/src/machine-c-api.h +++ b/src/machine-c-api.h @@ -128,6 +128,24 @@ typedef enum { // NOLINT(modernize-use-using) CM_PROC_SATP, CM_PROC_SCOUNTEREN, CM_PROC_SENVCFG, + CM_PROC_HSTATUS, + CM_PROC_HIDELEG, + CM_PROC_HEDELEG, + CM_PROC_HIE, + CM_PROC_HIP, + CM_PROC_HVIP, + CM_PROC_HGATP, + CM_PROC_HTIMEDELTA, + CM_PROC_HTVAL, + CM_PROC_VSEPC, + CM_PROC_VSSTATUS, + CM_PROC_VSCAUSE, + CM_PROC_VSTVAL, + CM_PROC_VSTVEC, + CM_PROC_VSSCRATCH, + CM_PROC_VSATP, + CM_PROC_VSIE, + CM_PROC_VSIP, CM_PROC_ILRSC, CM_PROC_IFLAGS, CM_PROC_CLINT_MTIMECMP, @@ -181,6 +199,25 @@ typedef struct { // NOLINT(modernize-use-using) uint64_t satp; ///< Value of satp CSR uint64_t scounteren; ///< Value of scounteren CSR uint64_t senvcfg; ///< Value of senvcfg CSR + uint64_t hstatus; ///< Value of hstatus CSR + uint64_t hideleg; ///< Value of hideleg CSR + uint64_t hedeleg; ///< Value of hedeleg CSR + uint64_t hie; ///< Value of hie CSR + uint64_t hip; ///< Value of hip CSR + uint64_t hvip; ///< Value of hvip CSR + uint64_t hgatp; ///< Value of hgatp CSR + uint64_t henvcfg; ///< Value of henvcfg CSR + uint64_t htimedelta; ///< Value of htimedelta CSR + uint64_t htval; ///< Value of htval CSR + uint64_t vsepc; ///< Value of vsepc CSR + uint64_t vsstatus; ///< Value of vsstatus CSR + uint64_t vscause; ///< Value of vscause CSR + uint64_t vstval; ///< Value of vstval CSR + uint64_t vstvec; ///< Value of vstvec CSR + uint64_t vsscratch; ///< Value of vsscratch CSR + uint64_t vsatp; ///< Value of vsatp CSR + uint64_t vsip; ///< Value of vsip CSR + uint64_t vsie; ///< Value of vsie CSR uint64_t ilrsc; ///< Value of ilrsc CSR uint64_t iflags; ///< Value of iflags CSR } cm_processor_config; @@ -1215,6 +1252,348 @@ CM_API int cm_read_senvcfg(const cm_machine *m, uint64_t *val, char **err_msg); /// \returns 0 for success, non zero code for error CM_API int cm_write_senvcfg(cm_machine *m, uint64_t val, char **err_msg); +/// \brief Reads the value of the hstatus register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_hstatus(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the hstatus register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_hstatus(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the hideleg register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_hideleg(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the hideleg register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_hideleg(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the hedeleg register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_hedeleg(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the hedeleg register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_hedeleg(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the hip register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_hip(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the hip register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_hip(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the hvip register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_hvip(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the hvip register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_hvip(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the hie register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_hie(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the hie register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_hie(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the hgatp register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_hgatp(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the hgatp register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_hgatp(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the henvcfg register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_henvcfg(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the henvcfg register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_henvcfg(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the htimedelta register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_htimedelta(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the htimedelta register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_htimedelta(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the htval register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_htval(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the htval register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_htval(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the vsepc register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_vsepc(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the vsepc register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_vsepc(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the vsstatus register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_vsstatus(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the vsstatus register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_vsstatus(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the vscause register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_vscause(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the vscause register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_vscause(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the vstval register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_vstval(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the vstval register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_vstval(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the vstvec register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_vstvec(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the vstvec register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_vstvec(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the vsscratch register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_vsscratch(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the vsscratch register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_vsscratch(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the vsatp register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_vsatp(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the vsatp register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_vsatp(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the vsie register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_vsie(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the vsie register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_vsie(cm_machine *m, uint64_t val, char **err_msg); + +/// \brief Reads the value of the vsip register. +/// \param m Pointer to valid machine instance +/// \param val Receives value of the register. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_read_vsip(const cm_machine *m, uint64_t *val, char **err_msg); + +/// \brief Writes the value of the vsip register. +/// \param m Pointer to valid machine instance +/// \param val New register value. +/// \param err_msg Receives the error message if function execution fails +/// or NULL in case of successfull function execution. In case of failure error_msg +/// must be deleted by the function caller using cm_delete_error_message +/// \returns 0 for success, non zero code for error +CM_API int cm_write_vsip(cm_machine *m, uint64_t val, char **err_msg); + /// \brief Reads the value of the ilrsc register. /// \param m Pointer to valid machine instance /// \param val Receives value of the register. @@ -1247,7 +1626,7 @@ CM_API int cm_read_iflags(const cm_machine *m, uint64_t *val, char **err_msg); /// \brief Returns packed iflags from its component fields. /// \param val Receives value of the register. -CM_API uint64_t cm_packed_iflags(int PRV, int X, int Y, int H); +CM_API uint64_t cm_packed_iflags(int VRT, int PRV, int X, int Y, int H); /// \brief Reads the value of the iflags register. /// \param m Pointer to valid machine instance diff --git a/src/machine-config.h b/src/machine-config.h index c05986fcd..f147b8477 100644 --- a/src/machine-config.h +++ b/src/machine-config.h @@ -63,6 +63,25 @@ struct processor_config final { uint64_t satp{SATP_INIT}; ///< Value of satp CSR uint64_t scounteren{SCOUNTEREN_INIT}; ///< Value of scounteren CSR uint64_t senvcfg{SENVCFG_INIT}; ///< Value of senvcfg CSR + uint64_t hstatus{HSTATUS_INIT}; ///< Value of hstatus CSR + uint64_t hideleg{HIDELEG_INIT}; ///< Value of hideleg CSR + uint64_t hedeleg{HEDELEG_INIT}; ///< Value of hedeleg CSR + uint64_t hie{HIE_INIT}; ///< Value of hie CSR + uint64_t hip{HIP_INIT}; ///< Value of hip CSR + uint64_t hvip{HVIP_INIT}; ///< Value of hvip CSR + uint64_t hgatp{HGATP_INIT}; ///< Value of hgatp CSR + uint64_t henvcfg{HENVCFG_INIT}; ///< Value of henvcfg CSR + uint64_t htimedelta{HTIMEDELTA_INIT}; ///< Value of htimedelta CSR + uint64_t htval{HTVAL_INIT}; ///< Value of htval CSR + uint64_t vsepc{VSEPC_INIT}; ///< Value of vsepc CSR + uint64_t vsstatus{VSSTATUS_INIT}; ///< Value of vsstatus CSR + uint64_t vscause{VSCAUSE_INIT}; ///< Value of vscause CSR + uint64_t vstval{VSTVAL_INIT}; ///< Value of vstval CSR + uint64_t vstvec{VSTVEC_INIT}; ///< Value of vstvec CSR + uint64_t vsscratch{VSSCRATCH_INIT}; ///< Value of vsscratch CSR + uint64_t vsatp{VSATP_INIT}; ///< Value of vsatp CSR + uint64_t vsip{VSIP_INIT}; ///< Value of vsip CSR + uint64_t vsie{VSIE_INIT}; ///< Value of vsie CSR uint64_t ilrsc{ILRSC_INIT}; ///< Value of ilrsc CSR uint64_t iflags{IFLAGS_INIT}; ///< Value of iflags CSR }; diff --git a/src/machine-state.h b/src/machine-state.h index c042a6ee1..53b226cde 100644 --- a/src/machine-state.h +++ b/src/machine-state.h @@ -38,7 +38,8 @@ namespace cartesi { struct unpacked_iflags { - uint8_t PRV; ///< Privilege level. + bool V; ///< Virtual mode. + uint8_t NOM; ///< Nominal privilege level. bool X; ///< CPU has yielded with automatic reset. bool Y; ///< CPU has yielded with manual reset. bool H; ///< CPU has been permanently halted. @@ -93,6 +94,27 @@ struct machine_state { uint64_t scounteren; ///< CSR scounteren. uint64_t senvcfg; ///< CSR senvcfg. + uint64_t hstatus; ///< CSR hstatus. + uint64_t hideleg; ///< CSR hideleg. + uint64_t hedeleg; ///< CSR hedeleg. + uint64_t hip; ///< CSR hip. + uint64_t hvip; ///< CSR hvip. + uint64_t hie; ///< CSR hie. + uint64_t hgatp; ///< CSR hgatp. + uint64_t henvcfg; ///< CSR henvcfg. + uint64_t htimedelta; ///< CSR htimedelta. + uint64_t htval; ///< CSR htval. + + uint64_t vsepc; ///< CSR vsepc. + uint64_t vsstatus; ///< CSR vsstatus. + uint64_t vscause; ///< CSR vscause. + uint64_t vstval; ///< CSR vstval. + uint64_t vstvec; ///< CSR vstvec. + uint64_t vsscratch; ///< CSR vsscratch. + uint64_t vsatp; ///< CSR vsatp. + uint64_t vsie; ///< CSR vsie. + uint64_t vsip; ///< CSR vsip. + // Cartesi-specific state uint64_t ilrsc; ///< Cartesi-specific CSR ilrsc (For LR/SC instructions). @@ -133,26 +155,29 @@ struct machine_state { /// \brief Reads the value of the iflags register. /// \returns The value of the register. uint64_t read_iflags(void) const { - return packed_iflags(iflags.PRV, iflags.X, iflags.Y, iflags.H); + return packed_iflags(iflags.V, iflags.NOM, iflags.X, iflags.Y, iflags.H); } /// \brief Reads the value of the iflags register. /// \param val New register value. void write_iflags(uint64_t val) { - iflags.H = (val >> IFLAGS_H_SHIFT) & 1; - iflags.Y = (val >> IFLAGS_Y_SHIFT) & 1; - iflags.X = (val >> IFLAGS_X_SHIFT) & 1; - iflags.PRV = (val >> IFLAGS_PRV_SHIFT) & 3; + iflags.H = (val & IFLAGS_H_MASK) >> IFLAGS_H_SHIFT; + iflags.Y = (val & IFLAGS_Y_MASK) >> IFLAGS_Y_SHIFT; + iflags.X = (val & IFLAGS_X_MASK) >> IFLAGS_X_SHIFT; + iflags.NOM = (val & IFLAGS_NOM_MASK) >> IFLAGS_NOM_SHIFT; + iflags.V = (val & IFLAGS_V_MASK) >> IFLAGS_V_SHIFT; } /// \brief Packs iflags into the CSR value - /// \param PRV privilege level + /// \param V virtual mode + /// \param NOM nominal privilege level /// \param I Waiting for interrupts flag /// \param Y Yielded flag /// \param H Halted flag /// \returns Packed iflags - static uint64_t packed_iflags(int PRV, int X, int Y, int H) { - return (PRV << IFLAGS_PRV_SHIFT) | (X << IFLAGS_X_SHIFT) | (Y << IFLAGS_Y_SHIFT) | (H << IFLAGS_H_SHIFT); + static uint64_t packed_iflags(int V, int NOM, int X, int Y, int H) { + return (V << IFLAGS_V_SHIFT) | (NOM << IFLAGS_NOM_SHIFT) | (X << IFLAGS_X_SHIFT) | (Y << IFLAGS_Y_SHIFT) | + (H << IFLAGS_H_SHIFT); } }; diff --git a/src/machine.cpp b/src/machine.cpp index c48271340..de22c0f23 100644 --- a/src/machine.cpp +++ b/src/machine.cpp @@ -344,6 +344,25 @@ machine::machine(const machine_config &c, const machine_runtime_config &r) : write_satp(m_c.processor.satp); write_scounteren(m_c.processor.scounteren); write_senvcfg(m_c.processor.senvcfg); + write_hstatus(m_c.processor.hstatus); + write_hedeleg(m_c.processor.hedeleg); + write_hideleg(m_c.processor.hideleg); + write_hie(m_c.processor.hie); + write_hip(m_c.processor.hip); + write_hvip(m_c.processor.hvip); + write_hgatp(m_c.processor.hgatp); + write_henvcfg(m_c.processor.henvcfg); + write_htimedelta(m_c.processor.htimedelta); + write_htval(m_c.processor.htval); + write_vsepc(m_c.processor.vsepc); + write_vsstatus(m_c.processor.vsstatus); + write_vscause(m_c.processor.vscause); + write_vstval(m_c.processor.vstval); + write_vstvec(m_c.processor.vstvec); + write_vsscratch(m_c.processor.vsscratch); + write_vsatp(m_c.processor.vsatp); + write_vsie(m_c.processor.vsie); + write_vsip(m_c.processor.vsip); write_ilrsc(m_c.processor.ilrsc); write_iflags(m_c.processor.iflags); @@ -532,6 +551,25 @@ machine_config machine::get_serialization_config(void) const { c.processor.satp = read_satp(); c.processor.scounteren = read_scounteren(); c.processor.senvcfg = read_senvcfg(); + c.processor.hstatus = read_hstatus(); + c.processor.hideleg = read_hideleg(); + c.processor.hedeleg = read_hedeleg(); + c.processor.hie = read_hie(); + c.processor.hip = read_hip(); + c.processor.hvip = read_hvip(); + c.processor.hgatp = read_hgatp(); + c.processor.henvcfg = read_henvcfg(); + c.processor.htimedelta = read_htimedelta(); + c.processor.htval = read_htval(); + c.processor.vsepc = read_vsepc(); + c.processor.vsstatus = read_vsstatus(); + c.processor.vscause = read_vscause(); + c.processor.vstval = read_vstval(); + c.processor.vstvec = read_vstvec(); + c.processor.vsscratch = read_vsscratch(); + c.processor.vsatp = read_vsatp(); + c.processor.vsie = read_vsie(); + c.processor.vsip = read_vsip(); c.processor.ilrsc = read_ilrsc(); c.processor.iflags = read_iflags(); // Copy current CLINT state to config @@ -724,9 +762,9 @@ machine::~machine() { (void) fprintf(stderr, "fence.i: %" PRIu64 "\n", m_s.stats.fence_i); (void) fprintf(stderr, "fence.vma: %" PRIu64 "\n", m_s.stats.fence_vma); (void) fprintf(stderr, "max asid: %" PRIu64 "\n", m_s.stats.max_asid); - (void) fprintf(stderr, "User mode: %" PRIu64 "\n", m_s.stats.priv_level[PRV_U]); - (void) fprintf(stderr, "Supervisor mode: %" PRIu64 "\n", m_s.stats.priv_level[PRV_S]); - (void) fprintf(stderr, "Machine mode: %" PRIu64 "\n", m_s.stats.priv_level[PRV_M]); + (void) fprintf(stderr, "User mode: %" PRIu64 "\n", m_s.stats.priv_level[NOM_U]); + (void) fprintf(stderr, "Supervisor mode: %" PRIu64 "\n", m_s.stats.priv_level[NOM_S]); + (void) fprintf(stderr, "Machine mode: %" PRIu64 "\n", m_s.stats.priv_level[NOM_M]); (void) fprintf(stderr, "tlb code hit ratio: %.4f\n", TLB_HIT_RATIO(m_s, tlb_cmiss, tlb_chit)); (void) fprintf(stderr, "tlb read hit ratio: %.4f\n", TLB_HIT_RATIO(m_s, tlb_rmiss, tlb_rhit)); @@ -993,6 +1031,158 @@ void machine::write_senvcfg(uint64_t val) { m_s.senvcfg = val; } +uint64_t machine::read_hstatus(void) const { + return m_s.hstatus; +} + +void machine::write_hstatus(uint64_t val) { + m_s.hstatus = val; +} + +uint64_t machine::read_hideleg(void) const { + return m_s.hideleg; +} + +void machine::write_hideleg(uint64_t val) { + m_s.hideleg = val; +} + +uint64_t machine::read_hedeleg(void) const { + return m_s.hedeleg; +} + +void machine::write_hedeleg(uint64_t val) { + m_s.hedeleg = val; +} + +uint64_t machine::read_hie(void) const { + return m_s.hie; +} + +void machine::write_hie(uint64_t val) { + m_s.hie = val; +} + +uint64_t machine::read_hip(void) const { + return m_s.hip; +} + +void machine::write_hip(uint64_t val) { + m_s.hip = val; +} + +uint64_t machine::read_hvip(void) const { + return m_s.hvip; +} + +void machine::write_hvip(uint64_t val) { + m_s.hvip = val; +} + +uint64_t machine::read_hgatp(void) const { + return m_s.hgatp; +} + +void machine::write_hgatp(uint64_t val) { + m_s.hgatp = val; +} + +uint64_t machine::read_henvcfg(void) const { + return m_s.henvcfg; +} + +void machine::write_henvcfg(uint64_t val) { + m_s.henvcfg = val; +} + +uint64_t machine::read_htimedelta(void) const { + return m_s.htimedelta; +} + +void machine::write_htimedelta(uint64_t val) { + m_s.htimedelta = val; +} + +uint64_t machine::read_htval(void) const { + return m_s.htval; +} + +void machine::write_htval(uint64_t val) { + m_s.htval = val; +} + +uint64_t machine::read_vsepc(void) const { + return m_s.vsepc; +} + +void machine::write_vsepc(uint64_t val) { + m_s.vsepc = val; +} + +uint64_t machine::read_vsstatus(void) const { + return m_s.vsstatus; +} + +void machine::write_vsstatus(uint64_t val) { + m_s.vsstatus = val; +} + +uint64_t machine::read_vscause(void) const { + return m_s.vscause; +} + +void machine::write_vscause(uint64_t val) { + m_s.vscause = val; +} + +uint64_t machine::read_vstval(void) const { + return m_s.vstval; +} + +void machine::write_vstval(uint64_t val) { + m_s.vstval = val; +} + +uint64_t machine::read_vstvec(void) const { + return m_s.vstvec; +} + +void machine::write_vstvec(uint64_t val) { + m_s.vstvec = val; +} + +uint64_t machine::read_vsscratch(void) const { + return m_s.vsscratch; +} + +void machine::write_vsscratch(uint64_t val) { + m_s.vsscratch = val; +} + +uint64_t machine::read_vsatp(void) const { + return m_s.vsatp; +} + +void machine::write_vsatp(uint64_t val) { + m_s.vsatp = val; +} + +uint64_t machine::read_vsie(void) const { + return m_s.vsie; +} + +void machine::write_vsie(uint64_t val) { + m_s.vsie = val; +} + +uint64_t machine::read_vsip(void) const { + return m_s.vsip; +} + +void machine::write_vsip(uint64_t val) { + m_s.vsip = val; +} + uint64_t machine::read_ilrsc(void) const { return m_s.ilrsc; } @@ -1131,6 +1321,44 @@ uint64_t machine::read_csr(csr r) const { return read_scounteren(); case csr::senvcfg: return read_senvcfg(); + case csr::hstatus: + return read_hstatus(); + case csr::hideleg: + return read_hideleg(); + case csr::hedeleg: + return read_hedeleg(); + case csr::hie: + return read_hie(); + case csr::hip: + return read_hip(); + case csr::hvip: + return read_hvip(); + case csr::hgatp: + return read_hgatp(); + case csr::henvcfg: + return read_henvcfg(); + case csr::htimedelta: + return read_htimedelta(); + case csr::htval: + return read_htval(); + case csr::vsepc: + return read_vsepc(); + case csr::vsstatus: + return read_vsstatus(); + case csr::vscause: + return read_vscause(); + case csr::vstval: + return read_vstval(); + case csr::vstvec: + return read_vstvec(); + case csr::vsscratch: + return read_vsscratch(); + case csr::vsatp: + return read_vsatp(); + case csr::vsie: + return read_vsie(); + case csr::vsip: + return read_vsip(); case csr::ilrsc: return read_ilrsc(); case csr::iflags: @@ -1213,6 +1441,44 @@ void machine::write_csr(csr csr, uint64_t value) { return write_scounteren(value); case csr::senvcfg: return write_senvcfg(value); + case csr::hstatus: + return write_hstatus(value); + case csr::hideleg: + return write_hideleg(value); + case csr::hedeleg: + return write_hedeleg(value); + case csr::hie: + return write_hie(value); + case csr::hip: + return write_hip(value); + case csr::hvip: + return write_hvip(value); + case csr::hgatp: + return write_hgatp(value); + case csr::henvcfg: + return write_henvcfg(value); + case csr::htimedelta: + return write_htimedelta(value); + case csr::htval: + return write_htval(value); + case csr::vsepc: + return write_vsepc(value); + case csr::vsstatus: + return write_vsstatus(value); + case csr::vscause: + return write_vscause(value); + case csr::vstvec: + return write_vstvec(value); + case csr::vstval: + return write_vstval(value); + case csr::vsscratch: + return write_vsscratch(value); + case csr::vsatp: + return write_vsatp(value); + case csr::vsie: + return write_vsie(value); + case csr::vsip: + return write_vsip(value); case csr::ilrsc: return write_ilrsc(value); case csr::iflags: @@ -1306,6 +1572,44 @@ uint64_t machine::get_csr_address(csr csr) { return shadow_state_get_csr_abs_addr(shadow_state_csr::scounteren); case csr::senvcfg: return shadow_state_get_csr_abs_addr(shadow_state_csr::senvcfg); + case csr::hstatus: + return shadow_state_get_csr_abs_addr(shadow_state_csr::hstatus); + case csr::hideleg: + return shadow_state_get_csr_abs_addr(shadow_state_csr::hideleg); + case csr::hedeleg: + return shadow_state_get_csr_abs_addr(shadow_state_csr::hedeleg); + case csr::hie: + return shadow_state_get_csr_abs_addr(shadow_state_csr::hie); + case csr::hip: + return shadow_state_get_csr_abs_addr(shadow_state_csr::hip); + case csr::hvip: + return shadow_state_get_csr_abs_addr(shadow_state_csr::hvip); + case csr::hgatp: + return shadow_state_get_csr_abs_addr(shadow_state_csr::hgatp); + case csr::henvcfg: + return shadow_state_get_csr_abs_addr(shadow_state_csr::henvcfg); + case csr::htimedelta: + return shadow_state_get_csr_abs_addr(shadow_state_csr::htimedelta); + case csr::htval: + return shadow_state_get_csr_abs_addr(shadow_state_csr::htval); + case csr::vsepc: + return shadow_state_get_csr_abs_addr(shadow_state_csr::vsepc); + case csr::vsstatus: + return shadow_state_get_csr_abs_addr(shadow_state_csr::vsstatus); + case csr::vscause: + return shadow_state_get_csr_abs_addr(shadow_state_csr::vscause); + case csr::vstval: + return shadow_state_get_csr_abs_addr(shadow_state_csr::vstval); + case csr::vstvec: + return shadow_state_get_csr_abs_addr(shadow_state_csr::vstvec); + case csr::vsscratch: + return shadow_state_get_csr_abs_addr(shadow_state_csr::vsscratch); + case csr::vsatp: + return shadow_state_get_csr_abs_addr(shadow_state_csr::vsatp); + case csr::vsie: + return shadow_state_get_csr_abs_addr(shadow_state_csr::vsie); + case csr::vsip: + return shadow_state_get_csr_abs_addr(shadow_state_csr::vsip); case csr::ilrsc: return shadow_state_get_csr_abs_addr(shadow_state_csr::ilrsc); case csr::iflags: @@ -1336,7 +1640,19 @@ uint64_t machine::get_csr_address(csr csr) { } uint8_t machine::read_iflags_PRV(void) const { - return m_s.iflags.PRV; + return m_s.iflags.NOM; +} + +bool machine::read_iflags_VRT(void) const { + return m_s.iflags.V; +} + +void machine::reset_iflags_VRT(void) { + m_s.iflags.V = false; +} + +void machine::set_iflags_VRT(void) { + m_s.iflags.V = true; } bool machine::read_iflags_Y(void) const { @@ -1738,6 +2054,19 @@ void machine::write_memory(uint64_t address, const unsigned char *data, size_t l memcpy(pma.get_memory().get_host_memory() + (address - pma.get_start()), data, length); } +static uint8_t get_access_mode(uint8_t priv, bool virt, uint64_t mstatus) { + if ((mstatus & MSTATUS_MPRV_MASK) && !virt) { + priv = (mstatus & MSTATUS_MPP_MASK) >> MSTATUS_MPP_SHIFT; + virt = (mstatus & MSTATUS_MPV_MASK) >> MSTATUS_MPV_SHIFT; + } + + uint8_t access_mode = priv; + if (virt) { + access_mode |= ACCESS_MODE_V_MASK; + } + return access_mode; +} + void machine::read_virtual_memory(uint64_t vaddr_start, unsigned char *data, uint64_t length) { state_access a(*this); if (length == 0) { @@ -1752,9 +2081,17 @@ void machine::read_virtual_memory(uint64_t vaddr_start, unsigned char *data, uin // copy page by page, because we need to perform address translation again for each page for (uint64_t vaddr_page = vaddr_page_start; vaddr_page < vaddr_page_limit; vaddr_page += PMA_PAGE_SIZE) { uint64_t paddr_page = 0; - if (!translate_virtual_address(a, &paddr_page, vaddr_page, PTE_XWR_R_SHIFT)) { - throw std::invalid_argument{"page fault"}; + + auto priv = read_iflags_PRV(); + auto virt = read_iflags_VRT(); + auto mstatus = read_mstatus(); + auto access_mode = get_access_mode(priv, virt, mstatus); + const uint8_t cause = translate_virtual_address(a, &paddr_page, + vaddr_page, access_mode, PTE_XWR_R_SHIFT); + if (cause) { + throw std::runtime_error{"page fault"}; } + uint64_t paddr = paddr_page; uint64_t vaddr = vaddr_page; uint64_t chunklen = std::min(PMA_PAGE_SIZE, vaddr_limit - vaddr); @@ -1783,10 +2120,16 @@ void machine::write_virtual_memory(uint64_t vaddr_start, const unsigned char *da // copy page by page, because we need to perform address translation again for each page for (uint64_t vaddr_page = vaddr_page_start; vaddr_page < vaddr_page_limit; vaddr_page += PMA_PAGE_SIZE) { uint64_t paddr_page = 0; + auto priv = read_iflags_PRV(); + auto virt = read_iflags_VRT(); + auto mstatus = read_mstatus(); + auto access_mode = get_access_mode(priv, virt, mstatus); // perform address translation using read access mode, // so we can write any reachable virtual memory range - if (!translate_virtual_address(a, &paddr_page, vaddr_page, PTE_XWR_R_SHIFT)) { - throw std::invalid_argument{"page fault"}; + const uint8_t cause = translate_virtual_address(a, &paddr_page, + vaddr_page, access_mode, PTE_XWR_R_SHIFT); + if (cause) { + throw std::runtime_error{"page fault"}; } uint64_t paddr = paddr_page; uint64_t vaddr = vaddr_page; diff --git a/src/machine.h b/src/machine.h index b2d1cb0df..a90371de6 100644 --- a/src/machine.h +++ b/src/machine.h @@ -162,6 +162,25 @@ class machine final { satp, scounteren, senvcfg, + hstatus, + hideleg, + hedeleg, + hie, + hip, + hvip, + hgatp, + henvcfg, + htimedelta, + htval, + vsepc, + vsstatus, + vscause, + vstval, + vstvec, + vsscratch, + vsatp, + vsie, + vsip, ilrsc, iflags, clint_mtimecmp, @@ -608,13 +627,165 @@ class machine final { /// \param value New register value. void write_ilrsc(uint64_t value); + /// \brief Writes the value of the hstatus register. + /// \param val New register value. + void write_hstatus(uint64_t val); + + /// \brief Reads the value of the hstatus register. + /// \returns The value of the register. + uint64_t read_hstatus(void) const; + + /// \brief Writes the value of the hideleg register. + /// \param val New register value. + void write_hideleg(uint64_t val); + + /// \brief Reads the value of the hideleg register. + /// \returns The value of the register. + uint64_t read_hideleg(void) const; + + /// \brief Writes the value of the hedeleg register. + /// \param val New register value. + void write_hedeleg(uint64_t val); + + /// \brief Reads the value of the hedeleg register. + /// \returns The value of the register. + uint64_t read_hedeleg(void) const; + + /// \brief Writes the value of the hie register. + /// \param val New register value. + void write_hie(uint64_t val); + + /// \brief Reads the value of the hie register. + /// \returns The value of the register. + uint64_t read_hie(void) const; + + /// \brief Writes the value of the hip register. + /// \param val New register value. + void write_hip(uint64_t val); + + /// \brief Reads the value of the hip register. + /// \returns The value of the register. + uint64_t read_hip(void) const; + + /// \brief Writes the value of the hvip register. + /// \param val New register value. + void write_hvip(uint64_t val); + + /// \brief Reads the value of the hvip register. + /// \returns The value of the register. + uint64_t read_hvip(void) const; + + /// \brief Writes the value of the hgatp register. + /// \param val New register value. + void write_hgatp(uint64_t val); + + /// \brief Reads the value of the hgatp register. + /// \returns The value of the register. + uint64_t read_hgatp(void) const; + + /// \brief Writes the value of the henvcfg register. + /// \param val New register value. + void write_henvcfg(uint64_t val); + + /// \brief Reads the value of the henvcfg register. + /// \returns The value of the register. + uint64_t read_henvcfg(void) const; + + /// \brief Writes the value of the htimedelta register. + /// \param val New register value. + void write_htimedelta(uint64_t val); + + /// \brief Reads the value of the htimedelta register. + /// \returns The value of the register. + uint64_t read_htimedelta(void) const; + + /// \brief Writes the value of the htval register. + /// \param val New register value. + void write_htval(uint64_t val); + + /// \brief Reads the value of the htval register. + /// \returns The value of the register. + uint64_t read_htval(void) const; + + /// \brief Writes the value of the vsepc register. + /// \param val New register value. + void write_vsepc(uint64_t val); + + /// \brief Reads the value of the vsepc register. + /// \returns The value of the register. + uint64_t read_vsepc(void) const; + + /// \brief Writes the value of the vsstatus register. + /// \param val New register value. + void write_vsstatus(uint64_t val); + + /// \brief Reads the value of the vsstatus register. + /// \returns The value of the register. + uint64_t read_vsstatus(void) const; + + /// \brief Writes the value of the vscause register. + /// \param val New register value. + void write_vscause(uint64_t val); + + /// \brief Reads the value of the vscause register. + /// \returns The value of the register. + uint64_t read_vscause(void) const; + + /// \brief Writes the value of the vstval register. + /// \param val New register value. + void write_vstval(uint64_t val); + + /// \brief Reads the value of the vstval register. + /// \returns The value of the register. + uint64_t read_vstval(void) const; + + /// \brief Writes the value of the vstvec register. + /// \param val New register value. + void write_vstvec(uint64_t val); + + /// \brief Reads the value of the vstvec register. + /// \returns The value of the register. + uint64_t read_vstvec(void) const; + + /// \brief Writes the value of the vsscratch register. + /// \param val New register value. + void write_vsscratch(uint64_t val); + + /// \brief Reads the value of the vsscratch register. + /// \returns The value of the register. + uint64_t read_vsscratch(void) const; + + /// \brief Writes the value of the vsatp register. + /// \param val New register value. + void write_vsatp(uint64_t val); + + /// \brief Reads the value of the vsatp register. + /// \returns The value of the register. + uint64_t read_vsatp(void) const; + + /// \brief Writes the value of the vsie register. + /// \param val New register value. + void write_vsie(uint64_t val); + + /// \brief Reads the value of the vsie register. + /// \returns The value of the register. + uint64_t read_vsie(void) const; + + /// \brief Writes the value of the vsip register. + /// \param val New register value. + void write_vsip(uint64_t val); + + /// \brief Reads the value of the vsip register. + /// \returns The value of the register. + uint64_t read_vsip(void) const; + /// \brief Reads the value of the iflags register. /// \returns The value of the register. uint64_t read_iflags(void) const; /// \brief Returns packed iflags from its component fields. /// \returns The value of the register. - uint64_t packed_iflags(int PRV, int Y, int H); + uint64_t packed_iflags(int V, int NOM, int Y, int H); /// \brief Reads the value of the iflags register. /// \param value New register value. @@ -712,6 +883,16 @@ class machine final { /// \returns The field value. uint8_t read_iflags_PRV(void) const; + /// \brief Checks the value of the iflags_VRT field. + /// \returns The field value. + bool read_iflags_VRT(void) const; + + /// \brief Sets the iflags_VRT flag. + void set_iflags_VRT(void); + + /// \brief Resets the iflags_VRT flag. + void reset_iflags_VRT(void); + /// \brief Sets the iflags_H flag. void set_iflags_H(void); diff --git a/src/protobuf-util.cpp b/src/protobuf-util.cpp index ba22f7818..5d65b81dc 100644 --- a/src/protobuf-util.cpp +++ b/src/protobuf-util.cpp @@ -152,6 +152,25 @@ void set_proto_machine_config(const machine_config &c, CartesiMachine::MachineCo proto_p->set_satp(c.processor.satp); proto_p->set_scounteren(c.processor.scounteren); proto_p->set_senvcfg(c.processor.senvcfg); + proto_p->set_hstatus(c.processor.hstatus); + proto_p->set_hedeleg(c.processor.hedeleg); + proto_p->set_hideleg(c.processor.hideleg); + proto_p->set_hie(c.processor.hie); + proto_p->set_hip(c.processor.hip); + proto_p->set_hvip(c.processor.hvip); + proto_p->set_hgatp(c.processor.hgatp); + proto_p->set_henvcfg(c.processor.henvcfg); + proto_p->set_htimedelta(c.processor.htimedelta); + proto_p->set_htval(c.processor.htval); + proto_p->set_vsepc(c.processor.vsepc); + proto_p->set_vsstatus(c.processor.vsstatus); + proto_p->set_vscause(c.processor.vscause); + proto_p->set_vstval(c.processor.vstval); + proto_p->set_vstvec(c.processor.vstvec); + proto_p->set_vsscratch(c.processor.vsscratch); + proto_p->set_vsatp(c.processor.vsatp); + proto_p->set_vsie(c.processor.vsie); + proto_p->set_vsip(c.processor.vsip); proto_p->set_ilrsc(c.processor.ilrsc); proto_p->set_iflags(c.processor.iflags); for (const auto &f : c.flash_drive) { @@ -648,6 +667,63 @@ processor_config get_proto_processor_config(const CartesiMachine::ProcessorConfi if (proto_p.has_senvcfg()) { p.senvcfg = proto_p.senvcfg(); } + if (proto_p.has_hstatus()) { + p.hstatus = proto_p.hstatus(); + } + if (proto_p.has_hideleg()) { + p.hideleg = proto_p.hideleg(); + } + if (proto_p.has_hedeleg()) { + p.hedeleg = proto_p.hedeleg(); + } + if (proto_p.has_hie()) { + p.hie = proto_p.hie(); + } + if (proto_p.has_hip()) { + p.hip = proto_p.hip(); + } + if (proto_p.has_hvip()) { + p.hvip = proto_p.hvip(); + } + if (proto_p.has_hgatp()) { + p.hgatp = proto_p.hgatp(); + } + if (proto_p.has_henvcfg()) { + p.henvcfg = proto_p.henvcfg(); + } + if (proto_p.has_htimedelta()) { + p.htimedelta = proto_p.htimedelta(); + } + if (proto_p.has_htval()) { + p.htval = proto_p.htval(); + } + if (proto_p.has_vsepc()) { + p.vsepc = proto_p.vsepc(); + } + if (proto_p.has_vsstatus()) { + p.vsstatus = proto_p.vsstatus(); + } + if (proto_p.has_vscause()) { + p.vscause = proto_p.vscause(); + } + if (proto_p.has_vstval()) { + p.vstval = proto_p.vstval(); + } + if (proto_p.has_vstvec()) { + p.vstvec = proto_p.vstvec(); + } + if (proto_p.has_vsscratch()) { + p.vsscratch = proto_p.vsscratch(); + } + if (proto_p.has_vsatp()) { + p.vsatp = proto_p.vsatp(); + } + if (proto_p.has_vsie()) { + p.vsie = proto_p.vsie(); + } + if (proto_p.has_vsip()) { + p.vsip = proto_p.vsip(); + } if (proto_p.has_ilrsc()) { p.ilrsc = proto_p.ilrsc(); } diff --git a/src/riscv-constants.h b/src/riscv-constants.h index 4749d3d63..d98ee6b1e 100644 --- a/src/riscv-constants.h +++ b/src/riscv-constants.h @@ -41,21 +41,31 @@ enum REG_COUNT { X_REG_COUNT = 32, F_REG_COUNT = 32, UARCH_X_REG_COUNT = 32 }; /// \brief MIP shifts enum MIP_shifts { MIP_SSIP_SHIFT = 1, + MIP_VSSIP_SHIFT = 2, MIP_MSIP_SHIFT = 3, MIP_STIP_SHIFT = 5, + MIP_VSTIP_SHIFT = 6, MIP_MTIP_SHIFT = 7, MIP_SEIP_SHIFT = 9, - MIP_MEIP_SHIFT = 11 + MIP_VSEIP_SHIFT = 10, + MIP_MEIP_SHIFT = 11, + MIP_SGEIP_SHIFT = 12 }; /// \brief MIP masks enum MIP_masks : uint64_t { - MIP_SSIP_MASK = UINT64_C(1) << MIP_SSIP_SHIFT, ///< Supervisor software interrupt - MIP_MSIP_MASK = UINT64_C(1) << MIP_MSIP_SHIFT, ///< Machine software interrupt - MIP_STIP_MASK = UINT64_C(1) << MIP_STIP_SHIFT, ///< Supervisor timer interrupt - MIP_MTIP_MASK = UINT64_C(1) << MIP_MTIP_SHIFT, ///< Machine timer interrupt - MIP_SEIP_MASK = UINT64_C(1) << MIP_SEIP_SHIFT, ///< Supervisor external interrupt - MIP_MEIP_MASK = UINT64_C(1) << MIP_MEIP_SHIFT ///< Machine external interrupt + MIP_SSIP_MASK = UINT64_C(1) << MIP_SSIP_SHIFT, ///< Supervisor software interrupt + MIP_VSSIP_MASK = UINT64_C(1) << MIP_VSSIP_SHIFT, ///< VS-level software interrupt + MIP_MSIP_MASK = UINT64_C(1) << MIP_MSIP_SHIFT, ///< Machine software interrupt + MIP_STIP_MASK = UINT64_C(1) << MIP_STIP_SHIFT, ///< Supervisor timer interrupt + MIP_VSTIP_MASK = UINT64_C(1) << MIP_VSTIP_SHIFT, ///< VS-level timer interrupt + MIP_MTIP_MASK = UINT64_C(1) << MIP_MTIP_SHIFT, ///< Machine timer interrupt + MIP_SEIP_MASK = UINT64_C(1) << MIP_SEIP_SHIFT, ///< Supervisor external interrupt + MIP_VSEIP_MASK = UINT64_C(1) << MIP_VSEIP_SHIFT, ///< VS-level external interrupt + MIP_MEIP_MASK = UINT64_C(1) << MIP_MEIP_SHIFT, ///< Machine external interrupt + MIP_SGEIP_MASK = + UINT64_C(1) << MIP_SGEIP_SHIFT, ///< Interrupt-pending bit for guest external interrupts at HS-level + MIP_VS_MASK = MIP_VSEIP_MASK | MIP_VSTIP_MASK | MIP_VSSIP_MASK, ///< VS-mode masks }; /// \brief mcause for exceptions @@ -68,13 +78,18 @@ enum MCAUSE_constants : uint64_t { MCAUSE_LOAD_ACCESS_FAULT = 0x5, ///< Load access fault MCAUSE_STORE_AMO_ADDRESS_MISALIGNED = 0x6, ///< Store/AMO address misaligned MCAUSE_STORE_AMO_ACCESS_FAULT = 0x7, ///< Store/AMO access fault - MCAUSE_ECALL_BASE = 0x8, ///< Environment call (+0: from U-mode, +1: from S-mode, +3: from M-mode) - MCAUSE_USER_ECALL = 0x8, ///< Environment call from U-mode - MCAUSE_SUPERVISOR_ECALL = 0x9, ///< Environment call from S-mode - MCAUSE_MACHINE_ECALL = 0xb, ///< Environment call from M-mode - MCAUSE_FETCH_PAGE_FAULT = 0xc, ///< Instruction page fault - MCAUSE_LOAD_PAGE_FAULT = 0xd, ///< Load page fault - MCAUSE_STORE_AMO_PAGE_FAULT = 0xf, ///< Store/AMO page fault + MCAUSE_ECALL_BASE = 0x8, ///< Environment call (+0: U/VU-mode, +1: HS-mode, +2: VS-mode, +3: M-mode) + MCAUSE_USER_ECALL = 0x8, ///< Environment call from U-mode + MCAUSE_HSUPERVISOR_ECALL = 0x9, ///< Environment call from HS-mode + MCAUSE_VSUPERVISOR_ECALL = 0xa, ///< Environment call from VS-mode + MCAUSE_MACHINE_ECALL = 0xb, ///< Environment call from M-mode + MCAUSE_FETCH_PAGE_FAULT = 0xc, ///< Instruction page fault + MCAUSE_LOAD_PAGE_FAULT = 0xd, ///< Load page fault + MCAUSE_STORE_AMO_PAGE_FAULT = 0xf, ///< Store/AMO page fault + MCAUSE_INSTRUCTION_GUEST_PAGE_FAULT = 0x14, ///< Instruction guest-page fault + MCAUSE_LOAD_GUEST_PAGE_FAULT = 0x15, ///< Load guest-page fault + MCAUSE_VIRTUAL_INSTRUCTION = 0x16, ///< Virtual instruction + MCAUSE_STORE_AMO_GUEST_PAGE_FAULT = 0x17, ///< Store/AMO guest-page fault MCAUSE_INTERRUPT_FLAG = UINT64_C(1) << (XLEN - 1) ///< Interrupt flag }; @@ -90,28 +105,43 @@ enum MCAUSE_masks : uint64_t { MCAUSE_STORE_AMO_ADDRESS_MISALIGNED_MASK = UINT64_C(1) << MCAUSE_STORE_AMO_ADDRESS_MISALIGNED, MCAUSE_STORE_AMO_ACCESS_FAULT_MASK = UINT64_C(1) << MCAUSE_STORE_AMO_ACCESS_FAULT, MCAUSE_USER_ECALL_MASK = UINT64_C(1) << MCAUSE_USER_ECALL, - MCAUSE_SUPERVISOR_ECALL_MASK = UINT64_C(1) << MCAUSE_SUPERVISOR_ECALL, + MCAUSE_HSUPERVISOR_ECALL_MASK = UINT64_C(1) << MCAUSE_HSUPERVISOR_ECALL, + MCAUSE_VSUPERVISOR_ECALL_MASK = UINT64_C(1) << MCAUSE_VSUPERVISOR_ECALL, MCAUSE_MACHINE_ECALL_MASK = UINT64_C(1) << MCAUSE_MACHINE_ECALL, MCAUSE_FETCH_PAGE_FAULT_MASK = UINT64_C(1) << MCAUSE_FETCH_PAGE_FAULT, MCAUSE_LOAD_PAGE_FAULT_MASK = UINT64_C(1) << MCAUSE_LOAD_PAGE_FAULT, - MCAUSE_STORE_AMO_PAGE_FAULT_MASK = UINT64_C(1) << MCAUSE_STORE_AMO_PAGE_FAULT + MCAUSE_STORE_AMO_PAGE_FAULT_MASK = UINT64_C(1) << MCAUSE_STORE_AMO_PAGE_FAULT, + MCAUSE_INSTRUCTION_GUEST_PAGE_FAULT_MASK = UINT64_C(1) << MCAUSE_INSTRUCTION_GUEST_PAGE_FAULT, + MCAUSE_LOAD_GUEST_PAGE_FAULT_MASK = UINT64_C(1) << MCAUSE_LOAD_GUEST_PAGE_FAULT, + MCAUSE_VIRTUAL_INSTRUCTION_MASK = UINT64_C(1) << MCAUSE_VIRTUAL_INSTRUCTION, + MCAUSE_STORE_AMO_GUEST_PAGE_FAULT_MASK = UINT64_C(1) << MCAUSE_STORE_AMO_GUEST_PAGE_FAULT, }; -/// \brief medeleg read/write masks -enum MEDELEG_RW_masks : uint64_t { - MEDELEG_W_MASK = MCAUSE_INSN_ADDRESS_MISALIGNED_MASK | MCAUSE_INSN_ACCESS_FAULT_MASK | MCAUSE_ILLEGAL_INSN_MASK | - MCAUSE_BREAKPOINT_MASK | MCAUSE_LOAD_ADDRESS_MISALIGNED_MASK | MCAUSE_LOAD_ACCESS_FAULT_MASK | - MCAUSE_STORE_AMO_ADDRESS_MISALIGNED_MASK | MCAUSE_STORE_AMO_ACCESS_FAULT_MASK | MCAUSE_USER_ECALL_MASK | - MCAUSE_SUPERVISOR_ECALL_MASK | MCAUSE_FETCH_PAGE_FAULT_MASK | MCAUSE_LOAD_PAGE_FAULT_MASK | - MCAUSE_STORE_AMO_PAGE_FAULT_MASK +/// \ Nominal privilege mode constants +enum NOM_constants : uint8_t { + NOM_U = 0, ///< User mode + NOM_S = 1, ///< Supervisor mode + NOM_RSVD = 2, ///< Reserved + NOM_M = 3 ///< Machine mode }; -/// \brief Privilege modes -enum PRV_constants : uint8_t { - PRV_U = 0, ///< User mode - PRV_S = 1, ///< Supervisor mode - PRV_HS = 2, ///< Hypervisor-extended supervisor mode - PRV_M = 3 ///< Machine mode +/// \brief Access mode shifts +enum MODE_shifts : uint8_t { ACCESS_MODE_NOM_SHIFT = 0, ACCESS_MODE_V_SHIFT = 2 }; + +/// \brief Access mode masks +enum MODE_masks : uint8_t { + ACCESS_MODE_NOM_MASK = 3 << ACCESS_MODE_NOM_SHIFT, + ACCESS_MODE_V_MASK = 1 << ACCESS_MODE_V_SHIFT +}; + +/// \brief Access type constants +enum ACCESS_TYPE_constants : uint8_t { ACCESS_TYPE_FETCH = 0, ACCESS_TYPE_STORE = 1, ACCESS_TYPE_LOAD = 2 }; + +/// \brief Translation constants +enum TRANSLATION_constants : uint8_t { + TRANSLATION_HS = 1, ///< one-stage address translation: supervisor virtual -> supervisor physical + TRANSLATION_G = 2, ///< two-stage address translation: guest physical -> supervisor physical + TRANSLATION_VS = 3 ///< two-stage address translation: guest virtual -> guest physical }; /// \brief misa shifts @@ -124,6 +154,7 @@ enum MISA_shifts { MISA_EXT_F_SHIFT = ('F' - 'A'), MISA_EXT_D_SHIFT = ('D' - 'A'), MISA_EXT_C_SHIFT = ('C' - 'A'), + MISA_EXT_H_SHIFT = ('H' - 'A'), MISA_MXL_SHIFT = (XLEN - 2) }; @@ -138,6 +169,7 @@ enum MISA_masks : uint64_t { MISA_EXT_F_MASK = UINT64_C(1) << MISA_EXT_F_SHIFT, ///< Single-precision floating-point extension MISA_EXT_D_MASK = UINT64_C(1) << MISA_EXT_D_SHIFT, ///< Double-precision floating-point extension MISA_EXT_C_MASK = UINT64_C(1) << MISA_EXT_C_SHIFT, ///< Compressed extension + MISA_EXT_H_MASK = UINT64_C(1) << MISA_EXT_H_SHIFT, ///< Hypervisor extension }; /// \brief misa constants @@ -166,6 +198,8 @@ enum MSTATUS_shifts { MSTATUS_SXL_SHIFT = 34, MSTATUS_SBE_SHIFT = 36, MSTATUS_MBE_SHIFT = 37, + MSTATUS_GVA_SHIFT = 38, + MSTATUS_MPV_SHIFT = 39, MSTATUS_SD_SHIFT = XLEN - 1 }; @@ -192,23 +226,28 @@ enum MSTATUS_masks : uint64_t { MSTATUS_SXL_MASK = UINT64_C(3) << MSTATUS_SXL_SHIFT, MSTATUS_SBE_MASK = UINT64_C(1) << MSTATUS_SBE_SHIFT, MSTATUS_MBE_MASK = UINT64_C(1) << MSTATUS_MBE_SHIFT, - MSTATUS_SD_MASK = UINT64_C(1) << MSTATUS_SD_SHIFT, MSTATUS_FS_OFF = UINT64_C(0) << MSTATUS_FS_SHIFT, MSTATUS_FS_INITIAL = UINT64_C(1) << MSTATUS_FS_SHIFT, MSTATUS_FS_CLEAN = UINT64_C(2) << MSTATUS_FS_SHIFT, - MSTATUS_FS_DIRTY = UINT64_C(3) << MSTATUS_FS_SHIFT + MSTATUS_FS_DIRTY = UINT64_C(3) << MSTATUS_FS_SHIFT, + + MSTATUS_GVA_MASK = UINT64_C(1) << MSTATUS_GVA_SHIFT, + MSTATUS_MPV_MASK = UINT64_C(1) << MSTATUS_MPV_SHIFT, + MSTATUS_SD_MASK = UINT64_C(1) << MSTATUS_SD_SHIFT }; /// \brief mstatus read-write masks enum MSTATUS_RW_masks : uint64_t { MSTATUS_W_MASK = (MSTATUS_SIE_MASK | MSTATUS_MIE_MASK | MSTATUS_SPIE_MASK | MSTATUS_MPIE_MASK | MSTATUS_SPP_MASK | MSTATUS_MPP_MASK | MSTATUS_FS_MASK | MSTATUS_MPRV_MASK | MSTATUS_SUM_MASK | MSTATUS_MXR_MASK | - MSTATUS_TVM_MASK | MSTATUS_TW_MASK | MSTATUS_TSR_MASK), ///< Write mask for mstatus - MSTATUS_R_MASK = (MSTATUS_SIE_MASK | MSTATUS_MIE_MASK | MSTATUS_SPIE_MASK | MSTATUS_UBE_MASK | MSTATUS_MPIE_MASK | - MSTATUS_SPP_MASK | MSTATUS_MPP_MASK | MSTATUS_FS_MASK | MSTATUS_VS_MASK | MSTATUS_MPRV_MASK | MSTATUS_SUM_MASK | - MSTATUS_MXR_MASK | MSTATUS_TVM_MASK | MSTATUS_TW_MASK | MSTATUS_TSR_MASK | MSTATUS_UXL_MASK | MSTATUS_SXL_MASK | - MSTATUS_SBE_MASK | MSTATUS_MBE_MASK | MSTATUS_SD_MASK) ///< Read mask for mstatus + MSTATUS_TVM_MASK | MSTATUS_TW_MASK | MSTATUS_TSR_MASK | MSTATUS_GVA_MASK | + MSTATUS_MPV_MASK), ///< Write mask for mstatus + MSTATUS_R_MASK = (MSTATUS_SIE_MASK | MSTATUS_MIE_MASK | MSTATUS_SPIE_MASK | MSTATUS_MPIE_MASK | MSTATUS_SPP_MASK | + MSTATUS_MPP_MASK | MSTATUS_FS_MASK | MSTATUS_VS_MASK | MSTATUS_MPRV_MASK | MSTATUS_SUM_MASK | MSTATUS_MXR_MASK | + MSTATUS_TVM_MASK | MSTATUS_TW_MASK | MSTATUS_TSR_MASK | MSTATUS_UXL_MASK | MSTATUS_SXL_MASK | MSTATUS_SBE_MASK | + MSTATUS_MBE_MASK | MSTATUS_SD_MASK | MSTATUS_GVA_MASK | MSTATUS_MPV_MASK | MSTATUS_UBE_MASK | + MSTATUS_XS_MASK) ///< Read mask for mstatus }; /// \brief sstatus read/write masks @@ -238,9 +277,113 @@ enum SATP_modes : uint64_t { SATP_MODE_SV57 = 10, }; +/// \brief masks that allow to check for zero bits in the incoming G-stage translation addresses +enum SVX_zero_bits_masks : uint64_t { + SV39X4_ZERO_MASK = ~((UINT64_C(1) << 41) - 1), + SV48X4_ZERO_MASK = ~((UINT64_C(1) << 50) - 1), +}; + /// \brief ASID masks enum ASID_masks : uint64_t { ASID_R_MASK = (UINT64_C(1) << ASIDLEN) - 1, ASID_MAX_MASK = (UINT64_C(1) << ASIDMAX) - 1 }; +/// \brief vsstatus read/write masks +enum VSSTATUS_rw_masks : uint64_t { + VSSTATUS_W_MASK = (MSTATUS_SIE_MASK | MSTATUS_SPIE_MASK | MSTATUS_SPP_MASK | MSTATUS_FS_MASK | MSTATUS_SUM_MASK | + MSTATUS_MXR_MASK), ///< Write mask for vsstatus + VSSTATUS_R_MASK = (MSTATUS_SIE_MASK | MSTATUS_SPIE_MASK | MSTATUS_UBE_MASK | MSTATUS_SPP_MASK | MSTATUS_VS_MASK | + MSTATUS_FS_MASK | MSTATUS_XS_MASK | MSTATUS_SUM_MASK | MSTATUS_MXR_MASK | MSTATUS_UXL_MASK | + MSTATUS_SD_MASK) ///< Read mask for vsstatus +}; + +/// \brief hstatus shifts +enum HSTATUS_shifts { + HSTATUS_VSBE_SHIFT = 5, + HSTATUS_GVA_SHIFT = 6, + HSTATUS_SPV_SHIFT = 7, + HSTATUS_SPVP_SHIFT = 8, + HSTATUS_HU_SHIFT = 9, + HSTATUS_VGEIN_SHIFT = 12, + HSTATUS_VTVM_SHIFT = 20, + HSTATUS_VTW_SHIFT = 21, + HSTATUS_VTSR_SHIFT = 22, + HSTATUS_VSXL_SHIFT = 32, +}; + +/// \brief hstatus masks +enum HSTATUS_masks : uint64_t { + HSTATUS_VSBE_MASK = UINT64_C(1) << HSTATUS_VSBE_SHIFT, + HSTATUS_GVA_MASK = UINT64_C(1) << HSTATUS_GVA_SHIFT, + HSTATUS_SPV_MASK = UINT64_C(1) << HSTATUS_SPV_SHIFT, + HSTATUS_SPVP_MASK = UINT64_C(1) << HSTATUS_SPVP_SHIFT, + HSTATUS_HU_MASK = UINT64_C(1) << HSTATUS_HU_SHIFT, + HSTATUS_VGEIN_MASK = UINT64_C(63) << HSTATUS_VGEIN_SHIFT, + HSTATUS_VTVM_MASK = UINT64_C(1) << HSTATUS_VTVM_SHIFT, + HSTATUS_VTW_MASK = UINT64_C(1) << HSTATUS_VTW_SHIFT, + HSTATUS_VTSR_MASK = UINT64_C(1) << HSTATUS_VTSR_SHIFT, + HSTATUS_VSXL_MASK = UINT64_C(3) << HSTATUS_VSXL_SHIFT +}; + +/// \brief hstatus read/write masks +enum HSTATUS_RW_masks : uint64_t { + HSTATUS_W_MASK = (HSTATUS_GVA_MASK | HSTATUS_SPV_MASK | HSTATUS_SPVP_MASK | HSTATUS_HU_MASK | HSTATUS_VTVM_MASK | + HSTATUS_VTW_MASK | HSTATUS_VTSR_MASK), + HSTATUS_R_MASK = (HSTATUS_VSBE_MASK | HSTATUS_GVA_MASK | HSTATUS_SPV_MASK | HSTATUS_SPVP_MASK | HSTATUS_HU_MASK | + HSTATUS_VGEIN_MASK | HSTATUS_VTVM_MASK | HSTATUS_VTW_MASK | HSTATUS_VTSR_MASK | HSTATUS_VSXL_MASK), +}; + +enum XIE_XIP_STANDARD_BITS_MASK : uint64_t { STANDARD_BITS_MASK = (UINT64_C(1) << 16) - 1 }; + +/// \brief sip & sie read/write masks +enum SIX_RW_masks : uint64_t { + SIE_RW_MASK = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK, + SIP_RW_MASK = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK, +}; + +/// \brief vsip & vsie read/write masks +enum VSIX_RW_masks : uint64_t { + VSIE_RW_MASK = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK, + VSIP_RW_MASK = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK, +}; + +/// \brief mie & mip read/write masks +enum MIX_RW_masks : uint64_t { + MIP_VIRTUAL_RW_MASK = VSIP_RW_MASK << 1, + MIE_VIRTUAL_RW_MASK = VSIE_RW_MASK << 1, + MIE_W_MASK = MIP_MSIP_MASK | MIP_MTIP_MASK | MIP_MEIP_MASK | MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK | + MIE_VIRTUAL_RW_MASK, + MIP_W_MASK = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK | MIP_VIRTUAL_RW_MASK, +}; + +/// \brief hie, hip & hvip read/write masks +enum HIX_RW_masks : uint64_t { + HIX_MASK = MIP_VSEIP_MASK | MIP_VSTIP_MASK | MIP_VSSIP_MASK | MIP_SGEIP_MASK, + HIE_W_MASK = MIP_VSEIP_MASK | MIP_VSTIP_MASK | MIP_VSSIP_MASK | MIP_SGEIP_MASK, + HIE_R_MASK = MIP_VSEIP_MASK | MIP_VSTIP_MASK | MIP_VSSIP_MASK, + HIP_W_MASK = MIP_VSSIP_MASK, + HIP_R_MASK = MIP_VSEIP_MASK | MIP_VSTIP_MASK | MIP_VSSIP_MASK | MIP_SGEIP_MASK, + HVIP_RW_MASK = MIP_VSEIP_MASK | MIP_VSTIP_MASK | MIP_VSSIP_MASK, +}; + +/// \brief hideleg & hedeleg write masks +enum HXDELEG_RW_masks : uint64_t { + HIDELEG_W_MASK = MIP_VSEIP_MASK | MIP_VSTIP_MASK | MIP_VSSIP_MASK, + HEDELEG_W_MASK = MCAUSE_INSN_ADDRESS_MISALIGNED_MASK | MCAUSE_INSN_ACCESS_FAULT_MASK | MCAUSE_ILLEGAL_INSN_MASK | + MCAUSE_BREAKPOINT_MASK | MCAUSE_LOAD_ADDRESS_MISALIGNED_MASK | MCAUSE_LOAD_ACCESS_FAULT_MASK | + MCAUSE_STORE_AMO_ADDRESS_MISALIGNED_MASK | MCAUSE_STORE_AMO_ACCESS_FAULT_MASK | MCAUSE_USER_ECALL_MASK | + MCAUSE_FETCH_PAGE_FAULT_MASK | MCAUSE_LOAD_PAGE_FAULT_MASK | MCAUSE_STORE_AMO_PAGE_FAULT_MASK, +}; + +/// \brief mideleg & medeleg write masks +enum MXDELEG_RW_masks : uint64_t { + MIDELEG_W_MASK = MIP_SSIP_MASK | MIP_STIP_MASK | MIP_SEIP_MASK, + MEDELEG_W_MASK = MCAUSE_INSN_ADDRESS_MISALIGNED_MASK | MCAUSE_INSN_ACCESS_FAULT_MASK | MCAUSE_ILLEGAL_INSN_MASK | + MCAUSE_BREAKPOINT_MASK | MCAUSE_LOAD_ADDRESS_MISALIGNED_MASK | MCAUSE_LOAD_ACCESS_FAULT_MASK | + MCAUSE_STORE_AMO_ADDRESS_MISALIGNED_MASK | MCAUSE_STORE_AMO_ACCESS_FAULT_MASK | MCAUSE_USER_ECALL_MASK | + MCAUSE_HSUPERVISOR_ECALL_MASK | MCAUSE_VSUPERVISOR_ECALL_MASK | MCAUSE_FETCH_PAGE_FAULT_MASK | + MCAUSE_LOAD_PAGE_FAULT_MASK | MCAUSE_STORE_AMO_PAGE_FAULT_MASK | MCAUSE_INSTRUCTION_GUEST_PAGE_FAULT_MASK | + MCAUSE_LOAD_GUEST_PAGE_FAULT_MASK | MCAUSE_VIRTUAL_INSTRUCTION_MASK | MCAUSE_STORE_AMO_GUEST_PAGE_FAULT_MASK, +}; + /// \brief Page-table entry shifts enum PTE_shifts { PTE_XWR_R_SHIFT = 0, @@ -418,13 +561,20 @@ enum COUNTEREN_rw_masks : uint64_t { }; /// \brief Cartesi-specific iflags shifts -enum IFLAGS_shifts { IFLAGS_H_SHIFT = 0, IFLAGS_Y_SHIFT = 1, IFLAGS_X_SHIFT = 2, IFLAGS_PRV_SHIFT = 3 }; +enum IFLAGS_shifts { + IFLAGS_H_SHIFT = 0, + IFLAGS_Y_SHIFT = 1, + IFLAGS_X_SHIFT = 2, + IFLAGS_NOM_SHIFT = 3, + IFLAGS_V_SHIFT = 5 +}; enum IFLAGS_masks : uint64_t { IFLAGS_H_MASK = UINT64_C(1) << IFLAGS_H_SHIFT, IFLAGS_Y_MASK = UINT64_C(1) << IFLAGS_Y_SHIFT, IFLAGS_X_MASK = UINT64_C(1) << IFLAGS_X_SHIFT, - IFLAGS_PRV_MASK = UINT64_C(3) << IFLAGS_PRV_SHIFT + IFLAGS_NOM_MASK = UINT64_C(3) << IFLAGS_NOM_SHIFT, + IFLAGS_V_MASK = UINT64_C(1) << IFLAGS_V_SHIFT, }; /// \brief Initial values for Cartesi machines @@ -451,7 +601,7 @@ enum CARTESI_init : uint64_t { MIE_INIT = UINT64_C(0), ///< Initial value for mie MIP_INIT = UINT64_C(0), ///< Initial value for mip MEDELEG_INIT = UINT64_C(0), ///< Initial value for medeleg - MIDELEG_INIT = UINT64_C(0), ///< Initial value for mideleg + MIDELEG_INIT = UINT64_C(0x444), ///< Initial value for mideleg MCOUNTEREN_INIT = UINT64_C(0), ///< Initial value for mcounteren STVEC_INIT = UINT64_C(0), ///< Initial value for stvec SSCRATCH_INIT = UINT64_C(0), ///< Initial value for sscratch @@ -461,7 +611,7 @@ enum CARTESI_init : uint64_t { SATP_INIT = UINT64_C(0), ///< Initial value for satp SCOUNTEREN_INIT = UINT64_C(0), ///< Initial value for scounteren ILRSC_INIT = UINT64_C(-1), ///< Initial value for ilrsc - IFLAGS_INIT = static_cast(PRV_M) << IFLAGS_PRV_SHIFT, ///< Initial value for iflags + IFLAGS_INIT = static_cast(NOM_M) << IFLAGS_NOM_SHIFT, ///< Initial value for iflags MTIMECMP_INIT = UINT64_C(0), ///< Initial value for mtimecmp FROMHOST_INIT = UINT64_C(0), ///< Initial value for fromhost TOHOST_INIT = UINT64_C(0), ///< Initial value for tohost @@ -469,6 +619,25 @@ enum CARTESI_init : uint64_t { SENVCFG_INIT = UINT64_C(0), ///< Initial value for senvcfg UARCH_PC_INIT = UINT64_C(0x70000000), ///< Initial value for microarchitecture pc UARCH_CYCLE_INIT = UINT64_C(0), ///< Initial value for microarchitecture cycle + HSTATUS_INIT = (MISA_MXL_VALUE << HSTATUS_VSXL_SHIFT), ///< Initial value for hstatus + HIDELEG_INIT = UINT64_C(0), ///< Initial value for hideleg + HEDELEG_INIT = UINT64_C(0), ///< Initial value for hedeleg + HIP_INIT = UINT64_C(0), ///< Initial value for hip + HVIP_INIT = UINT64_C(0), ///< Initial value for hvip + HIE_INIT = UINT64_C(0), ///< Initial value for hie + HGATP_INIT = UINT64_C(0), ///< Initial value for hgatp + HENVCFG_INIT = UINT64_C(0), ///< Initial value for henvcfg + HTIMEDELTA_INIT = UINT64_C(0), ///< Initial value for htimedelta + HTVAL_INIT = UINT64_C(0), ///< Initial value for htval + VSEPC_INIT = UINT64_C(0), ///< Initial value for vsepc + VSSTATUS_INIT = (MISA_MXL_VALUE << MSTATUS_UXL_SHIFT), ///< Initial value for vsstatus + VSCAUSE_INIT = UINT64_C(0), ///< Initial value for vscause + VSTVAL_INIT = UINT64_C(0), ///< Initial value for vstval + VSTVEC_INIT = UINT64_C(0), ///< Initial value for vstvec + VSSCRATCH_INIT = UINT64_C(0), ///< Initial value for vsscratch + VSATP_INIT = UINT64_C(0), ///< Initial value for vsatp + VSIP_INIT = UINT64_C(0), ///< Initial value for vsip + VSIE_INIT = UINT64_C(0), ///< Initial value for vsie MHARTID_INIT = UINT64_C(0), ///< Initial mhartid FDTADDR_INIT = PMA_DTB_START, ///< Initial FDT address @@ -555,6 +724,8 @@ enum class CSR_address : uint32_t { mcause = 0x342, mtval = 0x343, mip = 0x344, + mtinst = 0x34a, + mtval2 = 0x34b, mcycle = 0xb00, minstret = 0xb02, @@ -619,6 +790,33 @@ enum class CSR_address : uint32_t { mhpmevent30 = 0x33e, mhpmevent31 = 0x33f, + hstatus = 0x600, + hedeleg = 0x602, + hideleg = 0x603, + hie = 0x604, + hcounteren = 0x606, + hgeie = 0x607, + + htval = 0x643, + hip = 0x644, + hvip = 0x645, + htinst = 0x64a, + hgeip = 0xe12, + + henvcfg = 0x60a, + hgatp = 0x680, + htimedelta = 0x605, + + vsstatus = 0x200, + vsie = 0x204, + vstvec = 0x205, + vsscratch = 0x240, + vsepc = 0x241, + vscause = 0x242, + vstval = 0x243, + vsip = 0x244, + vsatp = 0x280, + tselect = 0x7a0, tdata1 = 0x7a1, tdata2 = 0x7a2, @@ -794,6 +992,7 @@ enum class insn_funct3_00000_opcode : uint32_t { ADDW_MULW_SUBW = 0b000000000111011, SRLW_DIVUW_SRAW = 0b101000000111011, privileged = 0b000000001110011, + hv_store_load = 0b100000001110011, }; /// \brief The result of insn >> 26 (6 most significant bits of funct7) can be @@ -918,13 +1117,65 @@ enum insn_ADDW_MULW_SUBW_funct7 : uint32_t { ADDW = 0b0000000, MULW = 0b0000001, /// \brief funct7 constants for SRLW, DIVUW, SRAW instructions enum insn_SRLW_DIVUW_SRAW_funct7 : uint32_t { SRLW = 0b0000000, DIVUW = 0b0000001, SRAW = 0b0100000 }; -/// \brief Privileged instructions, except for SFENCE.VMA, have no parameters -enum class insn_privileged : uint32_t { - ECALL = 0b00000000000000000000000001110011, - EBREAK = 0b00000000000100000000000001110011, - SRET = 0b00010000001000000000000001110011, - MRET = 0b00110000001000000000000001110011, - WFI = 0b00010000010100000000000001110011 +/// \brief funct7 constants for ECALL, EBREAK, WFI, SRET, MRET, SFENCE, HL instructions +enum class insn_privileged_funct7 : uint32_t { + E = 0b0000000, // ecall & ebreak + WFI_SRET = 0b0001000, // wfi & sret + MRET = 0b0011000, + SFENCE_VMA = 0b0001001, + SINVAL_VMA = 0b0001011, + SFENCE_INVAL = 0b0001100, // sfence.w.inval & sfence.inval.ir + HFENCE_VVMA = 0b0010001, + HFENCE_GVMA = 0b0110001, + HINVAL_VVMA = 0b0010011, + HINVAL_GVMA = 0b0110011, + HLV_B = 0b0110000, // hlv.b & hlv.bu + HLV_H = 0b0110010, // hlv.h & hlv.hu & hlvx.hu + HLV_W = 0b0110100, // hlv.w & hlvx.wu + HSV_B = 0b0110001, + HSV_H = 0b0110011, + HSV_W = 0b0110101, + HLV_WU = 0b0110100, + HLV_D = 0b0110110, + HSV_D = 0b0110111, +}; + +/// \brief rs2 constants for ECALL and EBREAK instructions +enum class insn_E_rs2 : uint32_t { + ECALL = 0b00000, + EBREAK = 0b00001, +}; + +/// \brief rs2 constants for WFI and SRET instructions +enum class insn_WFI_SRET_rs2 : uint32_t { + WFI = 0b00101, + SRET = 0b00010, +}; + +/// \brief rs2 constants for SFENCE_INVAL instructions +enum class insn_SFENCE_INVAL_rs2 : uint32_t { + SFENCE_W_INVAL = 0b00000, + SFENCE_INVAL_IR = 0b00001, +}; + +/// \brief rs2 constants for HLV_B instructions +enum class insn_HLV_B_rs2 : uint32_t { + HLV_B = 0b00000, + HLV_BU = 0b00001, +}; + +/// \brief rs2 constants for HLV_H instructions +enum class insn_HLV_H_rs2 : uint32_t { + HLV_H = 0b00000, + HLV_HU = 0b00001, + HLVX_HU = 0b00011, +}; + +/// \brief rs2 constants for HLV_W instructions +enum class insn_HLV_W_rs2 : uint32_t { + HLV_W = 0b00000, + HLV_WU = 0b00001, + HLVX_WU = 0b00011, }; /// \brief funct2 constants for FMADD, FMSUB, FNMADD, FMNSUB instructions diff --git a/src/shadow-state-factory.cpp b/src/shadow-state-factory.cpp index 8cbb4f00d..f6d2cb413 100644 --- a/src/shadow-state-factory.cpp +++ b/src/shadow-state-factory.cpp @@ -83,6 +83,25 @@ static bool shadow_state_peek(const pma_entry &pma, const machine &m, uint64_t p s->satp = m.read_satp(); s->scounteren = m.read_scounteren(); s->senvcfg = m.read_senvcfg(); + s->hstatus = m.read_hstatus(); + s->hideleg = m.read_hideleg(); + s->hedeleg = m.read_hedeleg(); + s->hie = m.read_hie(); + s->hip = m.read_hip(); + s->hvip = m.read_hvip(); + s->hgatp = m.read_hgatp(); + s->henvcfg = m.read_henvcfg(); + s->htimedelta = m.read_htimedelta(); + s->htval = m.read_htval(); + s->vsepc = m.read_vsepc(); + s->vsstatus = m.read_vsstatus(); + s->vscause = m.read_vscause(); + s->vstval = m.read_vstval(); + s->vstvec = m.read_vstvec(); + s->vsscratch = m.read_vsscratch(); + s->vsatp = m.read_vsatp(); + s->vsie = m.read_vsie(); + s->vsip = m.read_vsip(); s->ilrsc = m.read_ilrsc(); s->iflags = m.read_iflags(); s->clint_mtimecmp = m.read_clint_mtimecmp(); diff --git a/src/shadow-state.h b/src/shadow-state.h index 99e37d1b7..8625a5850 100644 --- a/src/shadow-state.h +++ b/src/shadow-state.h @@ -61,6 +61,25 @@ struct shadow_state { uint64_t satp; uint64_t scounteren; uint64_t senvcfg; + uint64_t hstatus; + uint64_t hideleg; + uint64_t hedeleg; + uint64_t hie; + uint64_t hip; + uint64_t hvip; + uint64_t hgatp; + uint64_t henvcfg; + uint64_t htimedelta; + uint64_t htval; + uint64_t vsepc; + uint64_t vsstatus; + uint64_t vscause; + uint64_t vstval; + uint64_t vstvec; + uint64_t vsscratch; + uint64_t vsatp; + uint64_t vsie; + uint64_t vsip; uint64_t ilrsc; uint64_t iflags; uint64_t clint_mtimecmp; @@ -110,6 +129,25 @@ enum class shadow_state_csr { satp = offsetof(shadow_state, satp), scounteren = offsetof(shadow_state, scounteren), senvcfg = offsetof(shadow_state, senvcfg), + hstatus = offsetof(shadow_state, hstatus), + hideleg = offsetof(shadow_state, hideleg), + hedeleg = offsetof(shadow_state, hedeleg), + hie = offsetof(shadow_state, hie), + hip = offsetof(shadow_state, hip), + hvip = offsetof(shadow_state, hvip), + hgatp = offsetof(shadow_state, hgatp), + henvcfg = offsetof(shadow_state, henvcfg), + htimedelta = offsetof(shadow_state, htimedelta), + htval = offsetof(shadow_state, htval), + vsepc = offsetof(shadow_state, vsepc), + vsstatus = offsetof(shadow_state, vsstatus), + vscause = offsetof(shadow_state, vscause), + vstval = offsetof(shadow_state, vstval), + vstvec = offsetof(shadow_state, vstvec), + vsscratch = offsetof(shadow_state, vsscratch), + vsatp = offsetof(shadow_state, vsatp), + vsie = offsetof(shadow_state, vsie), + vsip = offsetof(shadow_state, vsip), ilrsc = offsetof(shadow_state, ilrsc), iflags = offsetof(shadow_state, iflags), clint_mtimecmp = offsetof(shadow_state, clint_mtimecmp), diff --git a/src/state-access.h b/src/state-access.h index 11a97585c..9bcfa4a71 100644 --- a/src/state-access.h +++ b/src/state-access.h @@ -315,6 +315,158 @@ class state_access : public i_state_access { m_m.get_state().scounteren = val; } + uint64_t do_read_hstatus(void) const { + return m_m.get_state().hstatus; + } + + void do_write_hstatus(uint64_t val) { + m_m.get_state().hstatus = val; + } + + uint64_t do_read_hideleg(void) const { + return m_m.get_state().hideleg; + } + + void do_write_hideleg(uint64_t val) { + m_m.get_state().hideleg = val; + } + + uint64_t do_read_hedeleg(void) const { + return m_m.get_state().hedeleg; + } + + void do_write_hedeleg(uint64_t val) { + m_m.get_state().hedeleg = val; + } + + uint64_t do_read_hie(void) const { + return m_m.get_state().hie; + } + + void do_write_hie(uint64_t val) { + m_m.get_state().hie = val; + } + + uint64_t do_read_hip(void) const { + return m_m.get_state().hip; + } + + void do_write_hip(uint64_t val) { + m_m.get_state().hip = val; + } + + uint64_t do_read_hvip(void) const { + return m_m.get_state().hvip; + } + + void do_write_hvip(uint64_t val) { + m_m.get_state().hvip = val; + } + + uint64_t do_read_hgatp(void) const { + return m_m.get_state().hgatp; + } + + void do_write_hgatp(uint64_t val) { + m_m.get_state().hgatp = val; + } + + uint64_t do_read_henvcfg(void) const { + return m_m.get_state().henvcfg; + } + + void do_write_henvcfg(uint64_t val) { + m_m.get_state().henvcfg = val; + } + + uint64_t do_read_htimedelta(void) const { + return m_m.get_state().htimedelta; + } + + void do_write_htimedelta(uint64_t val) { + m_m.get_state().htimedelta = val; + } + + uint64_t do_read_htval(void) const { + return m_m.get_state().htval; + } + + void do_write_htval(uint64_t val) { + m_m.get_state().htval = val; + } + + uint64_t do_read_vsepc(void) const { + return m_m.get_state().vsepc; + } + + void do_write_vsepc(uint64_t val) { + m_m.get_state().vsepc = val; + } + + uint64_t do_read_vsstatus(void) const { + return m_m.get_state().vsstatus; + } + + void do_write_vsstatus(uint64_t val) { + m_m.get_state().vsstatus = val; + } + + uint64_t do_read_vscause(void) const { + return m_m.get_state().vscause; + } + + void do_write_vscause(uint64_t val) { + m_m.get_state().vscause = val; + } + + uint64_t do_read_vstval(void) const { + return m_m.get_state().vstval; + } + + void do_write_vstval(uint64_t val) { + m_m.get_state().vstval = val; + } + + uint64_t do_read_vstvec(void) const { + return m_m.get_state().vstvec; + } + + void do_write_vstvec(uint64_t val) { + m_m.get_state().vstvec = val; + } + + uint64_t do_read_vsscratch(void) const { + return m_m.get_state().vsscratch; + } + + void do_write_vsscratch(uint64_t val) { + m_m.get_state().vsscratch = val; + } + + uint64_t do_read_vsatp(void) const { + return m_m.get_state().vsatp; + } + + void do_write_vsatp(uint64_t val) { + m_m.get_state().vsatp = val; + } + + uint64_t do_read_vsie(void) const { + return m_m.get_state().vsie; + } + + void do_write_vsie(uint64_t val) { + m_m.get_state().vsie = val; + } + + uint64_t do_read_vsip(void) const { + return m_m.get_state().vsip; + } + + void do_write_vsip(uint64_t val) { + m_m.get_state().vsip = val; + } + uint64_t do_read_ilrsc(void) const { return m_m.get_state().ilrsc; } @@ -356,11 +508,23 @@ class state_access : public i_state_access { } uint8_t do_read_iflags_PRV(void) const { - return m_m.get_state().iflags.PRV; + return m_m.get_state().iflags.NOM; } void do_write_iflags_PRV(uint8_t val) { - m_m.get_state().iflags.PRV = val; + m_m.get_state().iflags.NOM = val; + } + + void do_set_iflags_VRT(void) { + m_m.get_state().iflags.V = true; + } + + void do_reset_iflags_VRT(void) { + m_m.get_state().iflags.V = false; + } + + bool do_read_iflags_VRT(void) const { + return m_m.get_state().iflags.V; } uint64_t do_read_clint_mtimecmp(void) const { diff --git a/src/test-machine-c-api.cpp b/src/test-machine-c-api.cpp index 2ff68aaf5..fd836f890 100644 --- a/src/test-machine-c-api.cpp +++ b/src/test-machine-c-api.cpp @@ -411,7 +411,12 @@ bool operator==(const cm_processor_config &lhs, const cm_processor_config &rhs) lhs.mcounteren == rhs.mcounteren && lhs.menvcfg == rhs.menvcfg && lhs.stvec == rhs.stvec && lhs.sscratch == rhs.sscratch && lhs.sepc == rhs.sepc && lhs.scause == rhs.scause && lhs.stval == rhs.stval && lhs.satp == rhs.satp && lhs.scounteren == rhs.scounteren && lhs.senvcfg == rhs.senvcfg && - lhs.ilrsc == rhs.ilrsc && lhs.iflags == rhs.iflags; + lhs.hstatus == rhs.hstatus && lhs.hideleg == rhs.hideleg && lhs.hedeleg == rhs.hedeleg && lhs.hie == rhs.hie && + lhs.hip == rhs.hip && lhs.hvip == rhs.hvip && lhs.hgatp == rhs.hgatp && lhs.henvcfg == rhs.henvcfg && + lhs.htimedelta == rhs.htimedelta && lhs.htval == rhs.htval && lhs.vsepc == rhs.vsepc && + lhs.vsstatus == rhs.vsstatus && lhs.vscause == rhs.vscause && lhs.vstval == rhs.vstval && + lhs.vstvec == rhs.vstvec && lhs.vsscratch == rhs.vsscratch && lhs.vsatp == rhs.vsatp && lhs.vsie == rhs.vsie && + lhs.vsip == rhs.vsip && lhs.ilrsc == rhs.ilrsc && lhs.iflags == rhs.iflags; } bool operator==(const cm_ram_config &lhs, const cm_ram_config &rhs) { @@ -1085,6 +1090,25 @@ CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, stval) CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, satp) CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, scounteren) CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, senvcfg) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, hstatus) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, hideleg) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, hedeleg) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, hie) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, hip) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, hvip) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, hgatp) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, henvcfg) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, htimedelta) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, htval) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, vsepc) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, vsstatus) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, vscause) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, vstval) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, vstvec) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, vsscratch) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, vsatp) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, vsie) +CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, vsip) CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, ilrsc) CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, iflags) CHECK_READER_FAILS_ON_nullptr_MACHINE(uint64_t, htif_tohost) @@ -1140,6 +1164,25 @@ CHECK_WRITER_FAILS_ON_nullptr_MACHINE(stval) CHECK_WRITER_FAILS_ON_nullptr_MACHINE(satp) CHECK_WRITER_FAILS_ON_nullptr_MACHINE(scounteren) CHECK_WRITER_FAILS_ON_nullptr_MACHINE(senvcfg) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(hstatus) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(hideleg) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(hedeleg) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(hie) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(hip) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(hvip) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(hgatp) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(henvcfg) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(htimedelta) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(htval) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(vsepc) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(vsstatus) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(vscause) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(vstval) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(vstvec) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(vsscratch) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(vsatp) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(vsip) +CHECK_WRITER_FAILS_ON_nullptr_MACHINE(vsie) CHECK_WRITER_FAILS_ON_nullptr_MACHINE(ilrsc) CHECK_WRITER_FAILS_ON_nullptr_MACHINE(iflags) CHECK_WRITER_FAILS_ON_nullptr_MACHINE(htif_tohost) @@ -1193,6 +1236,25 @@ CHECK_REGISTER_READ_WRITE(stval) CHECK_REGISTER_READ_WRITE(satp) CHECK_REGISTER_READ_WRITE(scounteren) CHECK_REGISTER_READ_WRITE(senvcfg) +CHECK_REGISTER_READ_WRITE(hstatus) +CHECK_REGISTER_READ_WRITE(hideleg) +CHECK_REGISTER_READ_WRITE(hedeleg) +CHECK_REGISTER_READ_WRITE(hie) +CHECK_REGISTER_READ_WRITE(hip) +CHECK_REGISTER_READ_WRITE(hvip) +CHECK_REGISTER_READ_WRITE(hgatp) +CHECK_REGISTER_READ_WRITE(henvcfg) +CHECK_REGISTER_READ_WRITE(htimedelta) +CHECK_REGISTER_READ_WRITE(htval) +CHECK_REGISTER_READ_WRITE(vsepc) +CHECK_REGISTER_READ_WRITE(vsstatus) +CHECK_REGISTER_READ_WRITE(vscause) +CHECK_REGISTER_READ_WRITE(vstval) +CHECK_REGISTER_READ_WRITE(vstvec) +CHECK_REGISTER_READ_WRITE(vsscratch) +CHECK_REGISTER_READ_WRITE(vsatp) +CHECK_REGISTER_READ_WRITE(vsip) +CHECK_REGISTER_READ_WRITE(vsie) CHECK_REGISTER_READ_WRITE(ilrsc) CHECK_REGISTER_READ_WRITE(htif_tohost) CHECK_REGISTER_READ_WRITE(htif_fromhost) @@ -1230,9 +1292,9 @@ BOOST_AUTO_TEST_CASE_NOLINT(set_iflags_h_null_machine_test) { } BOOST_AUTO_TEST_CASE_NOLINT(packed_iflags_test) { - uint64_t iflags = cm_packed_iflags(0, 0, 0, 0); + uint64_t iflags = cm_packed_iflags(0, 0, 0, 0, 0); BOOST_CHECK_EQUAL(0, iflags); - iflags = cm_packed_iflags(1, 1, 1, 1); + iflags = cm_packed_iflags(1, 1, 1, 1, 1); BOOST_CHECK_EQUAL(0xf, iflags); } diff --git a/src/tests/machine-bind.lua b/src/tests/machine-bind.lua index 65bb2bbe5..679b55dff 100755 --- a/src/tests/machine-bind.lua +++ b/src/tests/machine-bind.lua @@ -235,8 +235,27 @@ local function get_cpu_csr_test_values() satp = 0x2c0, scounteren = 0x2c8, senvcfg = 0x2d0, + hstatus = 0x2d8, + hideleg = 0x2e0, + hedeleg = 0x2e8, + hie = 0x2f0, + hip = 0x2f8, + hvip = 0x300, + hgatp = 0x308, + henvcfg = 0x310, + htimedelta = 0x318, + htval = 0x320, + vsepc = 0x328, + vsstatus = 0x330, + vscause = 0x338, + vstval = 0x340, + vstvec = 0x348, + vsscratch = 0x350, + vsatp = 0x358, + vsie = 0x360, + vsip = 0x368, fcsr = 0x61, - ilrsc = 0x2e0, + ilrsc = 0x370, } end @@ -505,7 +524,7 @@ do_test("should have expected values", function(machine) test_config(initial_config) assert(initial_config.processor.pc == 0x200, "wrong pc reg initial config value") assert(initial_config.processor.ilrsc == 0x2e0, "wrong ilrsc reg initial config value") - assert(initial_config.processor.mstatus == 0x230, "wrong mstatus reg initial config value") + assert(initial_config.processor.mstatus == 0x368, "wrong mstatus reg initial config value") assert(initial_config.clint.mtimecmp == 0, "wrong clint mtimecmp initial config value") assert(initial_config.htif.fromhost == 0, "wrong htif fromhost initial config value") assert(initial_config.htif.tohost == 0, "wrong htif tohost initial config value") diff --git a/src/translate-virtual-address.h b/src/translate-virtual-address.h index b333becdf..8394e92c1 100644 --- a/src/translate-virtual-address.h +++ b/src/translate-virtual-address.h @@ -88,52 +88,81 @@ static inline bool read_ram_uint64(STATE_ACCESS &a, uint64_t paddr, uint64_t *pv return true; } -/// \brief Walk the page table and translate a virtual address to the corresponding physical address -/// \tparam STATE_ACCESS Class of machine state accessor object. -/// \tparam UPDATE_PTE Whether PTE entries can be modified during the translation. -/// \param a Machine state accessor object. -/// \param vaddr Virtual address -/// \param ppaddr Pointer to physical address. -/// \param xwr_shift Encodes the access mode by the shift to the XWR triad (PTE_XWR_R_SHIFT, -/// PTE_XWR_R_SHIFT, or PTE_XWR_R_SHIFT) -/// \details This function is outlined to minimize host CPU code cache pressure. -/// \returns True if succeeded, false otherwise. -template -static NO_INLINE bool translate_virtual_address(STATE_ACCESS &a, uint64_t *ppaddr, uint64_t vaddr, int xwr_shift) { - auto priv = a.read_iflags_PRV(); - const uint64_t mstatus = a.read_mstatus(); - - // When MPRV is set, data loads and stores use privilege in MPP - // instead of the current privilege level (code access is unaffected) - if (xwr_shift != PTE_XWR_X_SHIFT && (mstatus & MSTATUS_MPRV_MASK)) { - priv = (mstatus & MSTATUS_MPP_MASK) >> MSTATUS_MPP_SHIFT; +namespace details { +template +static constexpr uint8_t to_mcause_code() { + if constexpr (TRANSLATION_MODE == TRANSLATION_G) { + if constexpr (ACCESS_TYPE == ACCESS_TYPE_LOAD) { + return MCAUSE_LOAD_GUEST_PAGE_FAULT; + } else if constexpr (ACCESS_TYPE == ACCESS_TYPE_STORE) { + return MCAUSE_STORE_AMO_GUEST_PAGE_FAULT; + } else { + return MCAUSE_INSTRUCTION_GUEST_PAGE_FAULT; + } + } else { + if constexpr (ACCESS_TYPE == ACCESS_TYPE_LOAD) { + return MCAUSE_LOAD_PAGE_FAULT; + } else if constexpr (ACCESS_TYPE == ACCESS_TYPE_STORE) { + return MCAUSE_STORE_AMO_PAGE_FAULT; + } else { + return MCAUSE_FETCH_PAGE_FAULT; + } } +} - // The satp register is considered active when the effective privilege mode is S-mode or U-mode. - // Executions of the address-translation algorithm may only begin using a given value of satp when - // satp is active. - if (unlikely(priv > PRV_S)) { - // We are in M-mode (or in HS-mode if Hypervisor extension is active) - *ppaddr = vaddr; - return true; +template +static uint8_t translate_virtual_address(STATE_ACCESS &a, uint64_t *ppaddr, uint64_t vaddr, int xwr_shift, + uint8_t priv) { + bool mxr = false; + bool sum = false; + uint64_t atp = 0; + uint8_t widenbits = 0; + if constexpr (TRANSLATION_MODE == TRANSLATION_HS) { + mxr = a.read_mstatus() & MSTATUS_MXR_MASK; + sum = a.read_mstatus() & MSTATUS_SUM_MASK; + atp = a.read_satp(); + } else if constexpr (TRANSLATION_MODE == TRANSLATION_G) { + mxr = a.read_mstatus() & MSTATUS_MXR_MASK; + atp = a.read_hgatp(); + widenbits = 2; + } else { // translation == TRANSLATION_VS + mxr = (a.read_vsstatus() & MSTATUS_MXR_MASK) || (a.read_mstatus() & MSTATUS_MXR_MASK); + sum = a.read_vsstatus() & MSTATUS_SUM_MASK; + atp = a.read_vsatp(); } - const uint64_t satp = a.read_satp(); - - const uint64_t mode = satp >> SATP_MODE_SHIFT; + const uint64_t mode = atp >> SATP_MODE_SHIFT; switch (mode) { case SATP_MODE_BARE: // Bare: No translation or protection *ppaddr = vaddr; - return true; + return 0; case SATP_MODE_SV39: // Sv39: Page-based 39-bit virtual addressing case SATP_MODE_SV48: // Sv48: Page-based 48-bit virtual addressing case SATP_MODE_SV57: // Sv57: Page-based 57-bit virtual addressing break; default: // Unsupported mode - return false; + // when a guest-page-fault occurs, return the guest physical address that faulted + if constexpr (TRANSLATION_MODE == TRANSLATION_G) { + *ppaddr = vaddr; + } + return to_mcause_code(); } // Here we know we are in sv39, sv48 or sv57 modes + // for Sv39x4 address bits 63:41 must all be zeros, or else a guest-page-fault exception occurs. + // for Sv48x4 & Sv57x4 address bits 63:50 must all be zeros, or else a guest-page-fault exception occurs. + if constexpr (TRANSLATION_MODE == TRANSLATION_G) { + uint64_t zero_bits = 0; + if (mode == SATP_MODE_SV39) { + zero_bits = vaddr & SV39X4_ZERO_MASK; + } else { + zero_bits = vaddr & SV48X4_ZERO_MASK; + } + if (zero_bits != 0) { + return to_mcause_code(); + } + } + // Page table hierarchy of sv39 has 3 levels, sv48 has 4 levels, // and sv57 has 5 levels // ??D It doesn't seem like restricting to one or the other will @@ -144,30 +173,56 @@ static NO_INLINE bool translate_virtual_address(STATE_ACCESS &a, uint64_t *ppadd // The least significant 12 bits of vaddr are the page offset // Then come levels virtual page numbers (VPN) // The rest of vaddr must be filled with copies of the - // most significant bit in VPN[levels] + // most significant bit in VPN[levels] (does not apply to G translation) // Hence, the use of arithmetic shifts here const int vaddr_bits = XLEN - (LOG2_PAGE_SIZE + levels * LOG2_VPN_SIZE); - if (unlikely((static_cast(vaddr << vaddr_bits) >> vaddr_bits) != static_cast(vaddr))) { - return false; + if constexpr (TRANSLATION_MODE != TRANSLATION_G) { + if (unlikely((static_cast(vaddr << vaddr_bits) >> vaddr_bits) != static_cast(vaddr))) { + return to_mcause_code(); + } } // Initialize pte_addr with the base address for the root page table - uint64_t pte_addr = (satp & SATP_PPN_MASK) << LOG2_PAGE_SIZE; + uint64_t pte_addr = (atp & SATP_PPN_MASK) << LOG2_PAGE_SIZE; for (int i = 0; i < levels; i++) { + int vpn_bits = LOG2_VPN_SIZE; + if (i == 0) + vpn_bits += widenbits; + const uint64_t vpn_mask = (1 << vpn_bits) - 1; + // Mask out VPN[levels-i-1] - const int vaddr_shift = LOG2_PAGE_SIZE + LOG2_VPN_SIZE * (levels - 1 - i); - const uint64_t vpn = (vaddr >> vaddr_shift) & VPN_MASK; + const int vaddr_shift = LOG2_PAGE_SIZE + vpn_bits * (levels - 1 - i); + const uint64_t vpn = (vaddr >> vaddr_shift) & vpn_mask; // Add offset to find physical address of page table entry pte_addr += vpn << LOG2_PTE_SIZE; //??D we can probably save this shift here + if constexpr (TRANSLATION_MODE == TRANSLATION_VS) { + // The spec says: + // ``` + // When V=1, memory accesses that would normally bypass address translation are subject to G-stage address + // translation alone. + // ``` + // Thus, here we apply G-stage translation to the PTE address. + uint64_t pte_addr_g = 0; + // For G-stage address translation, all memory accesses (including those made to access data structures + // for VS-stage address translation) are considered to be user-level accesses, as though executed in U-mode. + // Thus, passing NOM_U here. + int ret = translate_virtual_address(a, + &pte_addr_g, pte_addr, xwr_shift, NOM_U); + if (ret) { + *ppaddr = pte_addr; + return to_mcause_code(); + } + pte_addr = pte_addr_g; + } // Read page table entry from physical memory uint64_t pte = 0; if (unlikely(!read_ram_uint64(a, pte_addr, &pte))) { - return false; + break; } // The OS can mark page table entries as invalid, // but these entries shouldn't be reached during page lookups if (unlikely(!(pte & PTE_V_MASK))) { - return false; + break; } // Bits 60–54 are reserved for future standard use and must be zeroed // by software for forward compatibility. If any of these bits are set, @@ -177,7 +232,7 @@ static NO_INLINE bool translate_virtual_address(STATE_ACCESS &a, uint64_t *ppadd // If Svnapot is not implemented, bit 63 remains reserved and must be zeroed // by software for forward compatibility, or else a page-fault exception is raised. if (unlikely(pte & (PTE_60_54_MASK | PTE_PBMT_MASK | PTE_N_MASK))) { - return false; + break; } // Clear all flags in least significant bits, then shift back to multiple of page size to form physical address. const uint64_t ppn = (pte & PTE_PPN_MASK) << (LOG2_PAGE_SIZE - PTE_PPN_SHIFT); @@ -187,39 +242,41 @@ static NO_INLINE bool translate_virtual_address(STATE_ACCESS &a, uint64_t *ppadd if (xwr != 0) { // These protection bit combinations are reserved for future use if (unlikely(xwr == PTE_XWR_W_MASK || xwr == (PTE_XWR_W_MASK | PTE_XWR_X_MASK))) { - return false; + break; } - // (We know we are not PRV_M if we reached here) - if (priv == PRV_S) { - if ((pte & PTE_U_MASK)) { + + if (priv == NOM_S) { + if (pte & PTE_U_MASK) { // S-mode can never execute instructions from user pages, regardless of the state of SUM if (unlikely(xwr_shift == PTE_XWR_X_SHIFT)) { - return false; + break; } // If SUM is not set, forbid S-mode code from accessing U-mode memory - if (unlikely(!(mstatus & MSTATUS_SUM_MASK))) { - return false; + if (unlikely(!sum)) { + break; } } } else { // Forbid U-mode code from accessing S-mode memory if (unlikely(!(pte & PTE_U_MASK))) { - return false; + break; } } + // MXR allows read access to execute-only pages - if (mstatus & MSTATUS_MXR_MASK) { + if (mxr) { // Set R bit if X bit is set xwr |= (xwr >> PTE_XWR_X_SHIFT); } + // Check protection bits against requested access if (unlikely(((xwr >> xwr_shift) & 1) == 0)) { - return false; + break; } // Check page, megapage, and gigapage alignment const uint64_t vaddr_mask = (UINT64_C(1) << vaddr_shift) - 1; if (unlikely(ppn & vaddr_mask)) { - return false; + break; } // Decide if we need to update access bits in pte if constexpr (UPDATE_PTE) { @@ -229,20 +286,68 @@ static NO_INLINE bool translate_virtual_address(STATE_ACCESS &a, uint64_t *ppadd update_pte |= PTE_D_MASK; // Set dirty bit } if (pte != update_pte) { - write_ram_uint64(a, pte_addr, update_pte); // Can't fail since read succeeded earlier + write_ram_uint64(a, pte_addr, update_pte); // Can't fail since read succeeded earlier } } // Add page offset in vaddr to ppn to form physical address *ppaddr = (vaddr & vaddr_mask) | (ppn & ~vaddr_mask); - return true; + return 0; // xwr == 0 means we have a pointer to the start of the next page table } else { pte_addr = ppn; } } - return false; + + // when a guest-page-fault occurs, return the guest physical address that faulted + if constexpr (TRANSLATION_MODE == TRANSLATION_G) { + *ppaddr = vaddr; + } + return to_mcause_code(); } +} // namespace details +/// \brief Walk the page table and translate a virtual address to the corresponding physical address +/// \tparam STATE_ACCESS Class of machine state accessor object. +/// \tparam ACCESS_TYPE memory access type (fetch, store or load). +/// \param a Machine state accessor object. +/// \param ppaddr Pointer to physical address. +/// \param vaddr Virtual address +/// \param access_mode Encoded mode (virtual/non-virtual) and privilege of the memory access. +/// \param xwr_shift Encodes the access mode by the shift to the XWR triad (PTE_XWR_R_SHIFT, +/// PTE_XWR_W_SHIFT, or PTE_XWR_X_SHIFT) +/// \returns 0 if succeeded, the corresponding MCAUSE code otherwise. Please note that 0 is +/// repurposed here to be equivalent as success because the function will never return misaligned causes. +template +static uint8_t translate_virtual_address(STATE_ACCESS &a, uint64_t *ppaddr, uint64_t vaddr, uint8_t access_mode, + int xwr_shift) { + const uint8_t priv = (access_mode & ACCESS_MODE_NOM_MASK) >> ACCESS_MODE_NOM_SHIFT; + const uint8_t virt = (access_mode & ACCESS_MODE_V_MASK) >> ACCESS_MODE_V_SHIFT; + + // M-mode code does not use virtual memory + if (unlikely(priv == NOM_M)) { + *ppaddr = vaddr; + return 0; + } + + if (unlikely(virt)) { + uint64_t guest_paddr = 0; + int ret = details::translate_virtual_address(a, + &guest_paddr, vaddr, xwr_shift, priv); + if (unlikely(ret)) { + return ret; + } + // for G-stage translation guest-page-fault exceptions are raised instead of regular page-fault exceptions + // + // for G-stage address translation, all memory accesses (including those made to access data structures + // for VS-stage address translation) are considered to be user-level accesses, as though executed in U-mode. + // thus, passing NOM_U here. + return details::translate_virtual_address(a, ppaddr, + guest_paddr, xwr_shift, NOM_U); + } else { + return details::translate_virtual_address(a, ppaddr, + vaddr, xwr_shift, priv); + } +} } // namespace cartesi #endif /* end of include guard: TRANSLATE_VIRTUAL_ADDRESS_H */ diff --git a/src/uarch-bridge.h b/src/uarch-bridge.h index edc4435bb..eaa16f930 100644 --- a/src/uarch-bridge.h +++ b/src/uarch-bridge.h @@ -51,7 +51,7 @@ class uarch_bridge { switch (static_cast(paddr)) { case shadow_state_csr::uarch_halt_flag: if (data != uarch_halt_flag_halt_value) { - throw std::runtime_error("invalid value written microarchitecture halt flag"); + throw std::runtime_error("invalid value written microarchitecture halt flag"); } return uarch_halt(us); case shadow_state_csr::pc: @@ -129,6 +129,63 @@ class uarch_bridge { case shadow_state_csr::senvcfg: s.senvcfg = data; return; + case shadow_state_csr::hstatus: + s.hstatus = data; + return; + case shadow_state_csr::hideleg: + s.hideleg = data; + return; + case shadow_state_csr::hedeleg: + s.hedeleg = data; + return; + case shadow_state_csr::hie: + s.hie = data; + return; + case shadow_state_csr::hip: + s.hip = data; + return; + case shadow_state_csr::hvip: + s.hvip = data; + return; + case shadow_state_csr::hgatp: + s.hgatp = data; + return; + case shadow_state_csr::henvcfg: + s.henvcfg = data; + return; + case shadow_state_csr::htimedelta: + s.htimedelta = data; + return; + case shadow_state_csr::htval: + s.htval = data; + return; + case shadow_state_csr::vsepc: + s.vsepc = data; + return; + case shadow_state_csr::vsstatus: + s.vsstatus = data; + return; + case shadow_state_csr::vscause: + s.vscause = data; + return; + case shadow_state_csr::vstval: + s.vstval = data; + return; + case shadow_state_csr::vstvec: + s.vstvec = data; + return; + case shadow_state_csr::vsscratch: + s.vsscratch = data; + return; + case shadow_state_csr::vsatp: + s.vsatp = data; + return; + case shadow_state_csr::vsie: + s.vsie = data; + return; + case shadow_state_csr::vsip: + s.vsip = data; + return; case shadow_state_csr::ilrsc: s.ilrsc = data; return; @@ -240,6 +297,44 @@ class uarch_bridge { return s.scounteren; case shadow_state_csr::senvcfg: return s.senvcfg; + case shadow_state_csr::hstatus: + return s.hstatus; + case shadow_state_csr::hideleg: + return s.hideleg; + case shadow_state_csr::hedeleg: + return s.hedeleg; + case shadow_state_csr::hie: + return s.hie; + case shadow_state_csr::hip: + return s.hip; + case shadow_state_csr::hvip: + return s.hvip; + case shadow_state_csr::hgatp: + return s.hgatp; + case shadow_state_csr::henvcfg: + return s.henvcfg; + case shadow_state_csr::htimedelta: + return s.htimedelta; + case shadow_state_csr::htval: + return s.htval; + case shadow_state_csr::vsepc: + return s.vsepc; + case shadow_state_csr::vsstatus: + return s.vsstatus; + case shadow_state_csr::vscause: + return s.vscause; + case shadow_state_csr::vstval: + return s.vstval; + case shadow_state_csr::vstvec: + return s.vstvec; + case shadow_state_csr::vsscratch: + return s.vsscratch; + case shadow_state_csr::vsatp: + return s.vsatp; + case shadow_state_csr::vsie: + return s.vsie; + case shadow_state_csr::vsip: + return s.vsip; case shadow_state_csr::ilrsc: return s.ilrsc; case shadow_state_csr::iflags: @@ -334,6 +429,44 @@ class uarch_bridge { return "scounteren"; case shadow_state_csr::senvcfg: return "senvcfg"; + case shadow_state_csr::hstatus: + return "hstatus"; + case shadow_state_csr::hideleg: + return "hideleg"; + case shadow_state_csr::hedeleg: + return "hedeleg"; + case shadow_state_csr::hie: + return "hie"; + case shadow_state_csr::hip: + return "hip"; + case shadow_state_csr::hvip: + return "hvip"; + case shadow_state_csr::hgatp: + return "hgatp"; + case shadow_state_csr::henvcfg: + return "henvcfg"; + case shadow_state_csr::htimedelta: + return "htimedelta"; + case shadow_state_csr::htval: + return "htval"; + case shadow_state_csr::vsepc: + return "vsepc"; + case shadow_state_csr::vsstatus: + return "vsstatus"; + case shadow_state_csr::vscause: + return "vscause"; + case shadow_state_csr::vstval: + return "vstval"; + case shadow_state_csr::vstvec: + return "vstvec"; + case shadow_state_csr::vsscratch: + return "vsscratch"; + case shadow_state_csr::vsatp: + return "vsatp"; + case shadow_state_csr::vsie: + return "vsie"; + case shadow_state_csr::vsip: + return "vsip"; case shadow_state_csr::ilrsc: return "ilrsc"; case shadow_state_csr::iflags: diff --git a/src/virtual-machine.cpp b/src/virtual-machine.cpp index eb2ebcc70..fbb8b02d6 100644 --- a/src/virtual-machine.cpp +++ b/src/virtual-machine.cpp @@ -304,6 +304,158 @@ void virtual_machine::do_write_senvcfg(uint64_t val) { return m_machine->write_senvcfg(val); } +uint64_t virtual_machine::do_read_hstatus(void) const { + return m_machine->read_hstatus(); +} + +void virtual_machine::do_write_hstatus(uint64_t val) { + return m_machine->write_hstatus(val); +} + +uint64_t virtual_machine::do_read_hideleg(void) const { + return m_machine->read_hideleg(); +} + +void virtual_machine::do_write_hideleg(uint64_t val) { + return m_machine->write_hideleg(val); +} + +uint64_t virtual_machine::do_read_hedeleg(void) const { + return m_machine->read_hedeleg(); +} + +void virtual_machine::do_write_hedeleg(uint64_t val) { + return m_machine->write_hedeleg(val); +} + +uint64_t virtual_machine::do_read_hie(void) const { + return m_machine->read_hie(); +} + +void virtual_machine::do_write_hie(uint64_t val) { + return m_machine->write_hie(val); +} + +uint64_t virtual_machine::do_read_hip(void) const { + return m_machine->read_hip(); +} + +void virtual_machine::do_write_hip(uint64_t val) { + return m_machine->write_hip(val); +} + +uint64_t virtual_machine::do_read_hvip(void) const { + return m_machine->read_hvip(); +} + +void virtual_machine::do_write_hvip(uint64_t val) { + return m_machine->write_hvip(val); +} + +uint64_t virtual_machine::do_read_hgatp(void) const { + return m_machine->read_hgatp(); +} + +void virtual_machine::do_write_hgatp(uint64_t val) { + return m_machine->write_hgatp(val); +} + +uint64_t virtual_machine::do_read_henvcfg(void) const { + return m_machine->read_henvcfg(); +} + +void virtual_machine::do_write_henvcfg(uint64_t val) { + return m_machine->write_henvcfg(val); +} + +uint64_t virtual_machine::do_read_htimedelta(void) const { + return m_machine->read_htimedelta(); +} + +void virtual_machine::do_write_htimedelta(uint64_t val) { + return m_machine->write_htimedelta(val); +} + +uint64_t virtual_machine::do_read_htval(void) const { + return m_machine->read_htval(); +} + +void virtual_machine::do_write_htval(uint64_t val) { + return m_machine->write_htval(val); +} + +uint64_t virtual_machine::do_read_vsepc(void) const { + return m_machine->read_vsepc(); +} + +void virtual_machine::do_write_vsepc(uint64_t val) { + return m_machine->write_vsepc(val); +} + +uint64_t virtual_machine::do_read_vsstatus(void) const { + return m_machine->read_vsstatus(); +} + +void virtual_machine::do_write_vsstatus(uint64_t val) { + return m_machine->write_vsstatus(val); +} + +uint64_t virtual_machine::do_read_vscause(void) const { + return m_machine->read_vscause(); +} + +void virtual_machine::do_write_vscause(uint64_t val) { + return m_machine->write_vscause(val); +} + +uint64_t virtual_machine::do_read_vstval(void) const { + return m_machine->read_vstval(); +} + +void virtual_machine::do_write_vstval(uint64_t val) { + return m_machine->write_vstval(val); +} + +uint64_t virtual_machine::do_read_vstvec(void) const { + return m_machine->read_vstvec(); +} + +void virtual_machine::do_write_vstvec(uint64_t val) { + return m_machine->write_vstvec(val); +} + +uint64_t virtual_machine::do_read_vsscratch(void) const { + return m_machine->read_vsscratch(); +} + +void virtual_machine::do_write_vsscratch(uint64_t val) { + return m_machine->write_vsscratch(val); +} + +uint64_t virtual_machine::do_read_vsatp(void) const { + return m_machine->read_vsatp(); +} + +void virtual_machine::do_write_vsatp(uint64_t val) { + return m_machine->write_vsatp(val); +} + +uint64_t virtual_machine::do_read_vsie(void) const { + return m_machine->read_vsie(); +} + +void virtual_machine::do_write_vsie(uint64_t val) { + return m_machine->write_vsie(val); +} + +uint64_t virtual_machine::do_read_vsip(void) const { + return m_machine->read_vsip(); +} + +void virtual_machine::do_write_vsip(uint64_t val) { + return m_machine->write_vsip(val); +} + uint64_t virtual_machine::do_read_ilrsc(void) const { return m_machine->read_ilrsc(); } diff --git a/src/virtual-machine.h b/src/virtual-machine.h index da5249ad2..7e70051e0 100644 --- a/src/virtual-machine.h +++ b/src/virtual-machine.h @@ -109,6 +109,44 @@ class virtual_machine : public i_virtual_machine { void do_write_scounteren(uint64_t val) override; uint64_t do_read_senvcfg(void) const override; void do_write_senvcfg(uint64_t val) override; + uint64_t do_read_hstatus(void) const override; + void do_write_hstatus(uint64_t val) override; + uint64_t do_read_hideleg(void) const override; + void do_write_hideleg(uint64_t val) override; + uint64_t do_read_hedeleg(void) const override; + void do_write_hedeleg(uint64_t val) override; + uint64_t do_read_hie(void) const override; + void do_write_hie(uint64_t val) override; + uint64_t do_read_hip(void) const override; + void do_write_hip(uint64_t val) override; + uint64_t do_read_hvip(void) const override; + void do_write_hvip(uint64_t val) override; + uint64_t do_read_hgatp(void) const override; + void do_write_hgatp(uint64_t val) override; + uint64_t do_read_henvcfg(void) const override; + void do_write_henvcfg(uint64_t val) override; + uint64_t do_read_htimedelta(void) const override; + void do_write_htimedelta(uint64_t val) override; + uint64_t do_read_htval(void) const override; + void do_write_htval(uint64_t val) override; + uint64_t do_read_vsepc(void) const override; + void do_write_vsepc(uint64_t val) override; + uint64_t do_read_vsstatus(void) const override; + void do_write_vsstatus(uint64_t val) override; + uint64_t do_read_vscause(void) const override; + void do_write_vscause(uint64_t val) override; + uint64_t do_read_vstval(void) const override; + void do_write_vstval(uint64_t val) override; + uint64_t do_read_vstvec(void) const override; + void do_write_vstvec(uint64_t val) override; + uint64_t do_read_vsscratch(void) const override; + void do_write_vsscratch(uint64_t val) override; + uint64_t do_read_vsatp(void) const override; + void do_write_vsatp(uint64_t val) override; + uint64_t do_read_vsie(void) const override; + void do_write_vsie(uint64_t val) override; + uint64_t do_read_vsip(void) const override; + void do_write_vsip(uint64_t val) override; uint64_t do_read_ilrsc(void) const override; void do_write_ilrsc(uint64_t val) override; uint64_t do_read_iflags(void) const override; diff --git a/uarch/uarch-machine-state-access.h b/uarch/uarch-machine-state-access.h index 7a563e97f..d5acd93b8 100644 --- a/uarch/uarch-machine-state-access.h +++ b/uarch/uarch-machine-state-access.h @@ -395,6 +395,158 @@ class uarch_machine_state_access : public i_state_access(shadow_state_get_csr_abs_addr(shadow_state_csr::hstatus)); + } + + void do_write_hstatus(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hstatus), val); + } + + uint64_t do_read_hideleg(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hideleg)); + } + + void do_write_hideleg(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hideleg), val); + } + + uint64_t do_read_hedeleg(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hedeleg)); + } + + void do_write_hedeleg(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hedeleg), val); + } + + uint64_t do_read_hip(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hip)); + } + + void do_write_hip(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hip), val); + } + + uint64_t do_read_hvip(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hvip)); + } + + void do_write_hvip(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hvip), val); + } + + uint64_t do_read_hie(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hie)); + } + + void do_write_hie(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hie), val); + } + + uint64_t do_read_hgatp(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hgatp)); + } + + void do_write_hgatp(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::hgatp), val); + } + + uint64_t do_read_henvcfg(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::henvcfg)); + } + + void do_write_henvcfg(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::henvcfg), val); + } + + uint64_t do_read_htimedelta(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::htimedelta)); + } + + void do_write_htimedelta(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::htimedelta), val); + } + + uint64_t do_read_htval(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::htval)); + } + + void do_write_htval(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::htval), val); + } + + uint64_t do_read_vsepc(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsepc)); + } + + void do_write_vsepc(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsepc), val); + } + + uint64_t do_read_vsstatus(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsstatus)); + } + + void do_write_vsstatus(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsstatus), val); + } + + uint64_t do_read_vscause(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vscause)); + } + + void do_write_vscause(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vscause), val); + } + + uint64_t do_read_vstval(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vstval)); + } + + void do_write_vstval(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vstval), val); + } + + uint64_t do_read_vstvec(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vstvec)); + } + + void do_write_vstvec(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vstvec), val); + } + + uint64_t do_read_vsscratch(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsscratch)); + } + + void do_write_vsscratch(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsscratch), val); + } + + uint64_t do_read_vsatp(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsatp)); + } + + void do_write_vsatp(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsatp), val); + } + + uint64_t do_read_vsie(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsie)); + } + + void do_write_vsie(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsie), val); + } + + uint64_t do_read_vsip(void) { + return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsip)); + } + + void do_write_vsip(uint64_t val) { + raw_write_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::vsip), val); + } + uint64_t do_read_ilrsc(void) { return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::ilrsc)); } @@ -456,21 +608,37 @@ class uarch_machine_state_access : public i_state_access> IFLAGS_PRV_SHIFT; + return (iflags & IFLAGS_NOM_MASK) >> IFLAGS_NOM_SHIFT; } void do_write_iflags_PRV(uint8_t val) { auto old_iflags = read_iflags(); auto new_iflags = - (old_iflags & (~IFLAGS_PRV_MASK)) | ((static_cast(val) << IFLAGS_PRV_SHIFT) & IFLAGS_PRV_MASK); + (old_iflags & (~IFLAGS_NOM_MASK)) | ((static_cast(val) << IFLAGS_NOM_SHIFT) & IFLAGS_NOM_MASK); write_iflags(new_iflags); } uint64_t do_read_clint_mtimecmp(void) { return raw_read_memory(shadow_state_get_csr_abs_addr(shadow_state_csr::clint_mtimecmp)); - } void do_write_clint_mtimecmp(uint64_t val) {