Hypervisor forward progress: prevent ITLB miss fault PTW thrashing DC…

…ache (#3004)

src/main/scala/rocket/Frontend.scala

@@ -58,6 +58,7 @@ class FrontendIO(implicit p: Parameters) extends CoreBundle()(p) {
   val flush_icache = Bool(OUTPUT)
   val npc = UInt(INPUT, width = vaddrBitsExtended)
   val perf = new FrontendPerfEvents().asInput
+  val progress = Bool(OUTPUT)
 }
 
 class Frontend(val icacheParams: ICacheParams, staticIdForMetadataUseOnly: Int)(implicit p: Parameters) extends LazyModule {
@@ -147,6 +148,14 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
     s2_tlb_resp := tlb.io.resp
   }
 
+  val recent_progress_counter_init = 3.U
+  val recent_progress_counter = RegInit(recent_progress_counter_init)
+  val recent_progress = recent_progress_counter > 0
+  when(io.ptw.req.fire && recent_progress) { recent_progress_counter := recent_progress_counter - 1 }
+  when(io.cpu.progress) { recent_progress_counter := recent_progress_counter_init }
+
+  val s2_kill_speculative_tlb_refill = s2_speculative && !recent_progress
+
   io.ptw <> tlb.io.ptw
   tlb.io.req.valid := s1_valid && !s2_replay
   tlb.io.req.bits.vaddr := s1_pc
@@ -155,7 +164,7 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
   tlb.io.req.bits.prv := io.ptw.status.prv
   tlb.io.req.bits.v := io.ptw.status.v
   tlb.io.sfence := io.cpu.sfence
-  tlb.io.kill := !s2_valid
+  tlb.io.kill := !s2_valid || s2_kill_speculative_tlb_refill
 
   icache.io.req.valid := s0_valid
   icache.io.req.bits.addr := io.cpu.npc
@@ -168,13 +177,13 @@ class FrontendModule(outer: Frontend) extends LazyModuleImp(outer)
   icache.io.s2_cacheable := s2_tlb_resp.cacheable
   icache.io.s2_prefetch := s2_tlb_resp.prefetchable && !io.ptw.customCSRs.asInstanceOf[RocketCustomCSRs].disableICachePrefetch
 
-  fq.io.enq.valid := RegNext(s1_valid) && s2_valid && (icache.io.resp.valid || !s2_tlb_resp.miss && icache.io.s2_kill)
+  fq.io.enq.valid := RegNext(s1_valid) && s2_valid && (icache.io.resp.valid || (s2_kill_speculative_tlb_refill && s2_tlb_resp.miss) || (!s2_tlb_resp.miss && icache.io.s2_kill))
   fq.io.enq.bits.pc := s2_pc
   io.cpu.npc := alignPC(Mux(io.cpu.req.valid, io.cpu.req.bits.pc, npc))
 
   fq.io.enq.bits.data := icache.io.resp.bits.data
   fq.io.enq.bits.mask := UInt((1 << fetchWidth)-1) << s2_pc.extract(log2Ceil(fetchWidth)+log2Ceil(coreInstBytes)-1, log2Ceil(coreInstBytes))
-  fq.io.enq.bits.replay := icache.io.resp.bits.replay || icache.io.s2_kill && !icache.io.resp.valid && !s2_xcpt
+  fq.io.enq.bits.replay := (icache.io.resp.bits.replay || icache.io.s2_kill && !icache.io.resp.valid && !s2_xcpt) || (s2_kill_speculative_tlb_refill && s2_tlb_resp.miss)
   fq.io.enq.bits.btb := s2_btb_resp_bits
   fq.io.enq.bits.btb.taken := s2_btb_taken
   fq.io.enq.bits.xcpt := s2_tlb_resp

src/main/scala/rocket/RocketCore.scala

@@ -839,6 +839,7 @@ class Rocket(tile: RocketTile)(implicit p: Parameters) extends CoreModule()(p)
     imem_might_request_reg := ex_pc_valid || mem_pc_valid || io.ptw.customCSRs.disableICacheClockGate
     imem_might_request_reg
   }
+  io.imem.progress := RegNext(wb_reg_valid && !replay_wb_common)
   io.imem.sfence.valid := wb_reg_valid && wb_reg_sfence
   io.imem.sfence.bits.rs1 := wb_reg_mem_size(0)
   io.imem.sfence.bits.rs2 := wb_reg_mem_size(1)