diff --git a/include/isa.h b/include/isa.h index d34dad8cc..de581693d 100644 --- a/include/isa.h +++ b/include/isa.h @@ -45,7 +45,8 @@ void isa_hostcall(uint32_t id, rtlreg_t *dest, const rtlreg_t *src1, // memory enum { MMU_DIRECT, MMU_TRANSLATE, MMU_DYNAMIC }; -enum { MEM_TYPE_IFETCH, MEM_TYPE_READ, MEM_TYPE_WRITE, MEM_TYPE_IFETCH_READ, MEM_TYPE_WRITE_READ, IFDEF(CONFIG_RV_MBMC, MEM_TYPE_BM_READ) }; // The second to last and the third to last are prepared for PTW. +enum { MEM_TYPE_IFETCH, MEM_TYPE_READ, MEM_TYPE_WRITE, MEM_TYPE_IFETCH_READ, + MEM_TYPE_WRITE_READ, MEM_TYPE_READ_EXEC, IFDEF(CONFIG_RV_MBMC, MEM_TYPE_BM_READ) }; enum { MEM_RET_OK, MEM_RET_FAIL}; #ifndef isa_mmu_state int isa_mmu_state(); diff --git a/include/memory/host-tlb.h b/include/memory/host-tlb.h index 7a8c8a246..51df9304a 100644 --- a/include/memory/host-tlb.h +++ b/include/memory/host-tlb.h @@ -19,7 +19,7 @@ #include struct Decode; -word_t hosttlb_read(struct Decode *s, vaddr_t vaddr, int len, int type); +word_t hosttlb_read(struct Decode *s, vaddr_t vaddr, int len, int type, int trap_type); void hosttlb_write(struct Decode *s, vaddr_t vaddr, int len, word_t data); void hosttlb_init(); void hosttlb_flush(vaddr_t vaddr); diff --git a/src/isa/riscv64/system/mmu.c b/src/isa/riscv64/system/mmu.c index 0d911b442..d5771d7bf 100644 --- a/src/isa/riscv64/system/mmu.c +++ b/src/isa/riscv64/system/mmu.c @@ -1007,7 +1007,10 @@ bool pmpcfg_check_permission(uint8_t pmpcfg,int type,int out_mode) { return true; } else { - if (type == MEM_TYPE_READ || type == MEM_TYPE_IFETCH_READ + if (type == MEM_TYPE_READ_EXEC) { + return (pmpcfg & PMP_R) && (pmpcfg & PMP_X); + } + else if (type == MEM_TYPE_READ || type == MEM_TYPE_IFETCH_READ || type == MEM_TYPE_WRITE_READ) return pmpcfg & PMP_R; else if (type == MEM_TYPE_WRITE) @@ -1074,7 +1077,10 @@ bool pmptable_check_permission(word_t offset, word_t root_table_base, int type, #define W_BIT 0x2 #define X_BIT 0x4 /* Check permission */ - if (type == MEM_TYPE_READ || type == MEM_TYPE_IFETCH_READ + if (type == MEM_TYPE_READ_EXEC) { + return (perm & R_BIT) && (perm & X_BIT); + } + else if (type == MEM_TYPE_READ || type == MEM_TYPE_IFETCH_READ || type == MEM_TYPE_WRITE_READ) { return perm & R_BIT; } @@ -1176,6 +1182,7 @@ bool isa_pmp_check_permission(paddr_t addr, int len, int type, int out_mode) { return (mode == MODE_M && !(cfg & PMP_L)) || + (type == MEM_TYPE_READ_EXEC && (cfg & PMP_R) && (cfg & PMP_X)) || ((type == MEM_TYPE_READ || type == MEM_TYPE_IFETCH_READ || type == MEM_TYPE_WRITE_READ) && (cfg & PMP_R)) || (type == MEM_TYPE_WRITE && (cfg & PMP_W)) || @@ -1285,6 +1292,7 @@ bool isa_pma_check_permission(paddr_t addr, int len, int type) { return false; } return + (type == MEM_TYPE_READ_EXEC && (cfg & PMA_R) && (cfg & PMA_X)) || ((type == MEM_TYPE_READ || type == MEM_TYPE_IFETCH_READ || type == MEM_TYPE_WRITE_READ) && (cfg & PMA_R)) || (type == MEM_TYPE_WRITE && (cfg & PMA_W)) || diff --git a/src/memory/host-tlb.c b/src/memory/host-tlb.c index 559e2d328..d202bfc73 100644 --- a/src/memory/host-tlb.c +++ b/src/memory/host-tlb.c @@ -72,14 +72,14 @@ static paddr_t va2pa(struct Decode *s, vaddr_t vaddr, int len, int type) { } __attribute__((noinline)) -static word_t hosttlb_read_slowpath(struct Decode *s, vaddr_t vaddr, int len, int type) { - paddr_t paddr = va2pa(s, vaddr, len, type); - word_t data = paddr_read(paddr, len, type, type, cpu.mode, vaddr); +static word_t hosttlb_read_slowpath(struct Decode *s, vaddr_t vaddr, int len, int type, int trap_type) { + paddr_t paddr = va2pa(s, vaddr, len, trap_type); + word_t data = paddr_read(paddr, len, type, trap_type, cpu.mode, vaddr); if ( MUXDEF(CONFIG_RV_MBMC, isa_bmc_check_permission(paddr, len, 0, 0), true) && likely(in_pmem(paddr)) ) { - HostTLBEntry *e = type == MEM_TYPE_IFETCH ? + HostTLBEntry *e = trap_type == MEM_TYPE_IFETCH ? &hostxtlb[hosttlb_idx(vaddr)] : &hostrtlb[hosttlb_idx(vaddr)]; #ifdef CONFIG_USE_SPARSEMM e->offset = (uint8_t *)(paddr - vaddr); @@ -110,21 +110,21 @@ static void hosttlb_write_slowpath(struct Decode *s, vaddr_t vaddr, int len, wor } } -word_t hosttlb_read(struct Decode *s, vaddr_t vaddr, int len, int type) { +word_t hosttlb_read(struct Decode *s, vaddr_t vaddr, int len, int type, int trap_type) { Logm("hosttlb_reading " FMT_WORD, vaddr); #ifdef CONFIG_RVH extern bool has_two_stage_translation(); if(has_two_stage_translation()){ - paddr_t paddr = va2pa(s, vaddr, len, type); - return paddr_read(paddr, len, type, type, cpu.mode, vaddr); + paddr_t paddr = va2pa(s, vaddr, len, trap_type); + return paddr_read(paddr, len, type, trap_type, cpu.mode, vaddr); } #endif vaddr_t gvpn = hosttlb_vpn(vaddr); - HostTLBEntry *e = type == MEM_TYPE_IFETCH ? + HostTLBEntry *e = trap_type == MEM_TYPE_IFETCH ? &hostxtlb[hosttlb_idx(vaddr)] : &hostrtlb[hosttlb_idx(vaddr)]; if (unlikely(e->gvpn != gvpn)) { Logm("Host TLB slow path"); - return hosttlb_read_slowpath(s, vaddr, len, type); + return hosttlb_read_slowpath(s, vaddr, len, type, trap_type); } else { Logm("Host TLB fast path"); #ifdef CONFIG_USE_SPARSEMM diff --git a/src/memory/paddr.c b/src/memory/paddr.c index eb0948613..8d598c05b 100644 --- a/src/memory/paddr.c +++ b/src/memory/paddr.c @@ -268,13 +268,23 @@ bool check_paddr(paddr_t addr, int len, int type, int trap_type, int mode, vaddr return true; } +// "type" selects which permission bits PMP/PMA check (e.g. R, W, X, or R+X). +// "trap_type" selects which fault class is raised on failure (EX_LAF, EX_SAF, +// or EX_IAF). For most callers they are the same value. They differ when the +// access semantics and the fault class don't naturally align: +// - MEM_TYPE_IFETCH_READ / MEM_TYPE_WRITE_READ: PTW reads a PTE. +// type checks R only; trap_type may be the original access intent +// (e.g. IFETCH or WRITE) so the correct fault class is raised. +// - MEM_TYPE_READ_EXEC: HLVX load. +// type = READ_EXEC (checks R+X), trap_type = READ (raises EX_LAF). word_t paddr_read(paddr_t addr, int len, int type, int trap_type, int mode, vaddr_t vaddr) { IFDEF(CONFIG_SHARE, hardware_error_check(vaddr);) __attribute__((unused)) int cross_page_load = (mode & CROSS_PAGE_LD_FLAG) != 0; mode &= ~CROSS_PAGE_LD_FLAG; - assert(type == MEM_TYPE_READ || type == MEM_TYPE_IFETCH_READ || type == MEM_TYPE_IFETCH || type == MEM_TYPE_WRITE_READ); + assert(type == MEM_TYPE_READ || type == MEM_TYPE_READ_EXEC || type == MEM_TYPE_IFETCH_READ || + type == MEM_TYPE_IFETCH || type == MEM_TYPE_WRITE_READ); if (cpu.pbmt != 0) { isa_mmio_misalign_data_addr_check(addr, vaddr, len, MEM_TYPE_READ, cross_page_load); } diff --git a/src/memory/vaddr.c b/src/memory/vaddr.c index 5def46a5e..384fbb955 100644 --- a/src/memory/vaddr.c +++ b/src/memory/vaddr.c @@ -46,21 +46,21 @@ static paddr_t vaddr_trans_and_check_exception(vaddr_t vaddr, int len, int type, return paddr; } -static word_t vaddr_read_cross_page(vaddr_t addr, int len, int type, bool needTranslate) { +static word_t vaddr_read_cross_page(vaddr_t addr, int len, int type, int trap_type, bool needTranslate) { vaddr_t vaddr = addr; word_t data = 0; int i; for (i = 0; i < len; i ++, vaddr ++) { paddr_t paddr = vaddr; if (needTranslate) { - paddr_t mmu_ret = isa_mmu_translate(vaddr, 1, type); + paddr_t mmu_ret = isa_mmu_translate(vaddr, 1, trap_type); int ret = mmu_ret & PAGE_MASK; if (ret != MEM_RET_OK) return 0; paddr = (mmu_ret & ~PAGE_MASK) | (vaddr & PAGE_MASK); } #ifdef CONFIG_MULTICORE_DIFF - if (type == MEM_TYPE_IFETCH) { + if (trap_type == MEM_TYPE_IFETCH) { if (!isa_pmp_check_permission(paddr, 1, MEM_TYPE_IFETCH, cpu.mode) || !isa_pma_check_permission(paddr, 1, MEM_TYPE_IFETCH)) { Log("pmp or pma check failed when ifetch"); @@ -69,9 +69,9 @@ static word_t vaddr_read_cross_page(vaddr_t addr, int len, int type, bool needTr longjmp_exception(EX_IAF); } } - word_t byte = (type == MEM_TYPE_IFETCH ? golden_pmem_read(paddr, 1) : paddr_read(paddr, 1, type, type, cpu.mode | CROSS_PAGE_LD_FLAG, vaddr)); + word_t byte = (trap_type == MEM_TYPE_IFETCH ? golden_pmem_read(paddr, 1) : paddr_read(paddr, 1, type, trap_type, cpu.mode | CROSS_PAGE_LD_FLAG, vaddr)); #else - word_t byte = paddr_read(paddr, 1, type, type, cpu.mode | CROSS_PAGE_LD_FLAG, vaddr); + word_t byte = paddr_read(paddr, 1, type, trap_type, cpu.mode | CROSS_PAGE_LD_FLAG, vaddr); #endif data |= byte << (i << 3); } @@ -131,14 +131,14 @@ static void vaddr_write_cross_page(vaddr_t addr, int len, word_t data, bool need #ifndef ENABLE_HOSTTLB __attribute__((noinline)) -static word_t vaddr_mmu_read(struct Decode *s, vaddr_t addr, int len, int type) { +static word_t vaddr_mmu_read(struct Decode *s, vaddr_t addr, int len, int type, int trap_type) { vaddr_t vaddr = addr; - paddr_t pg_base = isa_mmu_translate(addr, len, type); + paddr_t pg_base = isa_mmu_translate(addr, len, trap_type); int ret = pg_base & PAGE_MASK; if (ret == MEM_RET_OK) { addr = pg_base | (addr & PAGE_MASK); #ifdef CONFIG_MULTICORE_DIFF - if (type == MEM_TYPE_IFETCH) { + if (trap_type == MEM_TYPE_IFETCH) { if (!isa_pmp_check_permission(addr, len, MEM_TYPE_IFETCH, cpu.mode) || !isa_pma_check_permission(addr, len, MEM_TYPE_IFETCH)) { Log("pmp or pma check failed when ifetch"); @@ -147,9 +147,9 @@ static word_t vaddr_mmu_read(struct Decode *s, vaddr_t addr, int len, int type) longjmp_exception(EX_IAF); } } - word_t rdata = (type == MEM_TYPE_IFETCH ? golden_pmem_read(addr, len) : paddr_read(addr, len, type, type, cpu.mode, vaddr)); + word_t rdata = (trap_type == MEM_TYPE_IFETCH ? golden_pmem_read(addr, len) : paddr_read(addr, len, type, trap_type, cpu.mode, vaddr)); #else - word_t rdata = paddr_read(addr, len, type, type, cpu.mode, vaddr); + word_t rdata = paddr_read(addr, len, type, trap_type, cpu.mode, vaddr); #endif // CONFIG_MULTICORE_DIFF ref_log_cpu("mmu_read: vaddr 0x%lx, paddr 0x%lx, rdata 0x%lx", vaddr, addr, rdata); @@ -173,39 +173,43 @@ static void vaddr_mmu_write(struct Decode *s, vaddr_t addr, int len, word_t data #endif // ENABLE_HOSTTLB -static inline word_t vaddr_read_internal(void *s, vaddr_t addr, int len, int type, int mmu_mode) { +static inline word_t vaddr_read_internal(void *s, vaddr_t addr, int len, int trap_type, int mmu_mode) { #ifdef CONFIG_RVH + bool is_hlvx = false; // check whether here is a hlvx instruction // when inst fetch or vaddr_read_safe (for examine memory), s is NULL if (s != NULL) { extern int rvh_hlvx_check(struct Decode *s, int type); - rvh_hlvx_check((Decode*)s, type); + is_hlvx = rvh_hlvx_check((Decode*)s, trap_type); } + int type = (trap_type == MEM_TYPE_READ && is_hlvx) ? MEM_TYPE_READ_EXEC : trap_type; +#else + int type = trap_type; #endif - addr = get_effective_address(addr, type); + addr = get_effective_address(addr, trap_type); bool is_cross_page = false; void isa_misalign_data_addr_check(vaddr_t vaddr, int len, int type); - if (type != MEM_TYPE_IFETCH) { - isa_misalign_data_addr_check(addr, len, type); + if (trap_type != MEM_TYPE_IFETCH) { + isa_misalign_data_addr_check(addr, len, trap_type); is_cross_page = ((addr & PAGE_MASK) + len) > PAGE_SIZE && len != 1; } if (unlikely(mmu_mode == MMU_DYNAMIC || mmu_mode == MMU_TRANSLATE)) { Logm("Checking mmu when MMU_DYN"); - mmu_mode = isa_mmu_check(addr, len, type); + mmu_mode = isa_mmu_check(addr, len, trap_type); } if (is_cross_page) { - return vaddr_read_cross_page(addr, len, type, mmu_mode == MMU_DYNAMIC || mmu_mode == MMU_TRANSLATE); + return vaddr_read_cross_page(addr, len, type, trap_type, mmu_mode == MMU_DYNAMIC || mmu_mode == MMU_TRANSLATE); } if (mmu_mode == MMU_DIRECT) { Logm("Paddr reading directly"); - return paddr_read(addr, len, type, type, cpu.mode, addr); + return paddr_read(addr, len, type, trap_type, cpu.mode, addr); } - return MUXDEF(ENABLE_HOSTTLB, hosttlb_read, vaddr_mmu_read) ((struct Decode *)s, addr, len, type); + return MUXDEF(ENABLE_HOSTTLB, hosttlb_read, vaddr_mmu_read) ((struct Decode *)s, addr, len, type, trap_type); return 0; }