From b7a724410b3f1e51a8c5984a9cad3afcd7d57c9a Mon Sep 17 00:00:00 2001
From: Blaise Tine <tinebp@iam-ssh1.research.intel-research.net>
Date: Thu, 3 Dec 2020 07:30:19 -0800
Subject: [PATCH] update DRAM simulation - reduce the latency of duplicate
 requests (simulate DRAM cache)

---
 driver/opae/vlsim/opae_sim.cpp | 24 ++++++++++++++++--------
 hw/rtl/VX_config.vh            |  2 +-
 hw/simulate/simulator.cpp      | 29 +++++++++++++++++++----------
 hw/simulate/simulator.h        |  1 +
 4 files changed, 37 insertions(+), 19 deletions(-)

diff --git a/driver/opae/vlsim/opae_sim.cpp b/driver/opae/vlsim/opae_sim.cpp
index 0a4bae40..4b8cb468 100644
--- a/driver/opae/vlsim/opae_sim.cpp
+++ b/driver/opae/vlsim/opae_sim.cpp
@@ -245,15 +245,17 @@ void opae_sim::sTxPort_bus() {
 }
   
 void opae_sim::avs_bus() {
-  // schedule DRAM read responses
+  // update DRAM responses schedule
+  for (auto& rsp : dram_reads_) {
+    if (rsp.cycles_left > 0)
+      rsp.cycles_left -= 1;
+  }
+
+  // schedule DRAM responses in FIFO order
   std::list<dram_rd_req_t>::iterator dram_rd_it(dram_reads_.end());
-  for (auto it = dram_reads_.begin(), ie = dram_reads_.end(); it != ie; ++it) {
-    if (it->cycles_left > 0) {
-      it->cycles_left -= 1;
-    }
-    if ((it != ie) && (it->cycles_left == 0)) {
-      dram_rd_it = it;
-    }
+  if (!dram_reads_.empty() 
+   && (0 == dram_reads_.begin()->cycles_left)) {
+      dram_rd_it = dram_reads_.begin();
   }
 
   // send DRAM response  
@@ -304,6 +306,12 @@ void opae_sim::avs_bus() {
       dram_req.addr = vortex_afu_->avs_address;
       ram_.read(vortex_afu_->avs_address * CACHE_BLOCK_SIZE, CACHE_BLOCK_SIZE, dram_req.block.data());      
       dram_req.cycles_left = DRAM_LATENCY;
+      for (auto& rsp : dram_reads_) {
+        if (dram_req.addr == rsp.addr) {
+          dram_req.cycles_left = rsp.cycles_left;
+          break;
+        }
+      }
       dram_reads_.emplace_back(dram_req);
       /*printf("%0ld: [sim] DRAM Rd Req: addr=%x, pending={", timestamp, dram_req.addr * CACHE_BLOCK_SIZE);
       for (auto& req : dram_reads_) {
diff --git a/hw/rtl/VX_config.vh b/hw/rtl/VX_config.vh
index e7eaa03e..f5034091 100644
--- a/hw/rtl/VX_config.vh
+++ b/hw/rtl/VX_config.vh
@@ -16,7 +16,7 @@
 `endif
 
 `ifndef NUM_THREADS
-`define NUM_THREADS 4
+`define NUM_THREADS 8
 `endif
 
 `ifndef NUM_BARRIERS
diff --git a/hw/simulate/simulator.cpp b/hw/simulate/simulator.cpp
index 45fc9056..26b4a6d7 100644
--- a/hw/simulate/simulator.cpp
+++ b/hw/simulate/simulator.cpp
@@ -127,15 +127,17 @@ void Simulator::eval_dram_bus() {
     return;
   }
 
-  // schedule DRAM responses
+  // update DRAM responses schedule
+  for (auto& rsp : dram_rsp_vec_) {
+    if (rsp.cycles_left > 0)
+      rsp.cycles_left -= 1;
+  }
+
+  // schedule DRAM responses in FIFO order
   std::list<dram_req_t>::iterator dram_rsp_it(dram_rsp_vec_.end());
-  for (auto it = dram_rsp_vec_.begin(), ie = dram_rsp_vec_.end(); it != ie; ++it) {
-    if (it->cycles_left > 0) {
-      it->cycles_left -= 1;
-    }
-    if ((dram_rsp_it == ie) && (it->cycles_left == 0)) {
-      dram_rsp_it = it;
-    }
+  if (!dram_rsp_vec_.empty() 
+   && (0 == dram_rsp_vec_.begin()->cycles_left)) {
+      dram_rsp_it = dram_rsp_vec_.begin();
   }
 
   // send DRAM response  
@@ -180,9 +182,16 @@ void Simulator::eval_dram_bus() {
         }
       } else {
         dram_req_t dram_req;        
-        dram_req.tag = vortex_->dram_req_tag;   
+        dram_req.tag  = vortex_->dram_req_tag;   
+        dram_req.addr = vortex_->dram_req_addr;
         ram_->read(vortex_->dram_req_addr * GLOBAL_BLOCK_SIZE, GLOBAL_BLOCK_SIZE, dram_req.block.data());
-        dram_req.cycles_left = DRAM_LATENCY;     
+        dram_req.cycles_left = DRAM_LATENCY;
+        for (auto& rsp : dram_rsp_vec_) {
+          if (dram_req.addr == rsp.addr) {
+            dram_req.cycles_left = rsp.cycles_left;
+            break;
+          }
+        }     
         dram_rsp_vec_.emplace_back(dram_req);
       } 
     }    
diff --git a/hw/simulate/simulator.h b/hw/simulate/simulator.h
index a1613d98..2532b99a 100644
--- a/hw/simulate/simulator.h
+++ b/hw/simulate/simulator.h
@@ -51,6 +51,7 @@ private:
   typedef struct {
     int cycles_left;  
     std::array<uint8_t, GLOBAL_BLOCK_SIZE> block;
+    uint32_t addr;
     uint32_t tag;
   } dram_req_t;