Vortex 2.0 changes:

+ Microarchitecture optimizations + 64-bit support + Xilinx FPGA support + LLVM-16 support + Refactoring and quality control fixes minor update minor update minor update minor update minor update minor update cleanup cleanup cache bindings and memory perf refactory minor update minor update hw unit tests fixes minor update minor update minor update minor update minor update minor udpate minor update minor update minor update minor update minor update minor update minor update minor updates minor updates minor update minor update minor update minor update minor update minor update minor updates minor updates minor updates minor updates minor update minor update
2023-10-19 20:51:22 -07:00
parent d69a64c32c
commit c1e168fdbe
1309 changed files with 247412 additions and 311463 deletions
--- a/hw/unittest/.gitignore
+++ b/hw/unittest/.gitignore
@@ -0,0 +1 @@
+*/obj_dir/*
--- a/hw/unittest/Makefile
+++ b/hw/unittest/Makefile
@@ -0,0 +1,17 @@
+all:
+	$(MAKE) -C cache
+	$(MAKE) -C generic_queue
+	$(MAKE) -C mem_streamer
+	$(MAKE) -C top_modules
+
+run:
+	$(MAKE) -C cache run
+	$(MAKE) -C generic_queue run
+	$(MAKE) -C mem_streamer run
+	$(MAKE) -C top_modules run
+
+clean:
+	$(MAKE) -C cache clean
+	$(MAKE) -C generic_queue clean
+	$(MAKE) -C mem_streamer clean
+	$(MAKE) -C top_modules clean
--- a/hw/unittest/VX_divide_tb.v
+++ b/hw/unittest/VX_divide_tb.v
@@ -0,0 +1,172 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+`timescale 1ns/1ps
+
+module VX_tb_divide();
+
+`ifdef TRACE
+    initial
+    begin
+    $dumpfile("trace.vcd");
+    $dumpvars(0,test);
+    end
+`endif
+
+    reg clk;
+    reg rst;
+
+    reg [31:0] numer, denom;
+
+    wire [31:0] o_div[0:7], o_rem[0:7];
+
+    for (genvar i = 0; i < 8; ++i) begin
+        VX_divide#(
+            .WIDTHN(32),
+            .WIDTHD(32),
+            .WIDTHQ(32),
+            .WIDTHR(32),
+            .PIPELINE(i)
+        ) div(
+            .clock(clk),
+            .aclr(rst),
+            .clken(1'b1),
+            .numer(numer),
+            .denom(denom),
+            .quotient(o_div[i]),
+            .remainder(o_rem[i])
+        );
+    end
+
+    initial begin
+        clk = 0; rst = 0;
+
+        numer = 56;
+        denom = 11;
+
+        $display("56 / 11 #0");
+        if (o_div[0] != 5 || o_rem[0] != 1) begin
+            $display("PIPE0: div=", o_div[0], " rem=", o_rem[0]);
+            $display("expected 5,1 EXITING");
+            $finish();
+        end
+
+        if (o_div[1] != 1'bx || o_rem[1] != 1'bx) begin
+            $display("PIPE1: div=", o_div[1], " rem=", o_rem[1]);
+            $display("expected x,x EXITING");
+            $finish();
+        end
+
+        if (o_div[2] != 1'bx || o_rem[2] != 1'bx) begin
+            $display("PIPE2: div=", o_div[2], " rem=", o_rem[2]);
+            $display("expected x,x EXITING");
+            $finish();
+        end
+
+        if (o_div[3] != 1'bx || o_rem[3] != 1'bx) begin
+            $display("PIPE3: div=", o_div[3], " rem=", o_rem[3]);
+            $display("expected x,x EXITING");
+            $finish();
+        end
+
+        #2;
+
+        $display("56 / 11 #2");
+         if (o_div[0] != 5 || o_rem[0] != 1) begin
+            $display("PIPE0: div=", o_div[0], " rem=", o_rem[0]);
+            $display("expected 5,1, EXITING");
+            $finish();
+        end
+
+        if (o_div[1] != 5 || o_rem[1] != 1) begin
+            $display("PIPE1: div=", o_div[1], " rem=", o_rem[1]);
+            $display("expected 5,1 EXITING");
+            $finish();
+        end
+
+        if (o_div[2] != 1'bx || o_rem[2] != 1'bx) begin
+            $display("PIPE2: div=", o_div[2], " rem=", o_rem[2]);
+            $display("expected x,x EXITING");
+            $finish();
+        end
+
+        if (o_div[3] != 1'bx || o_rem[3] != 1'bx) begin
+            $display("PIPE3: div=", o_div[3], " rem=", o_rem[3]);
+            $display("expected x,x EXITING");
+            $finish();
+        end
+
+        #2;
+
+        $display("56 / 11 #4");
+        if (o_div[0] != 5 || o_rem[0] != 1) begin
+            $display("PIPE0: div=", o_div[0], " rem=", o_rem[0]);
+            $display("expected 5,1 EXITING");
+            $finish();
+        end
+
+        if (o_div[1] != 5 || o_rem[1] != 1) begin
+            $display("PIPE1: div=", o_div[1], " rem=", o_rem[1]);
+            $display("expected 5,1 EXITING");
+            $finish();
+        end
+
+        if (o_div[2] != 5 || o_rem[2] != 1) begin
+            $display("PIPE2: div=", o_div[2], " rem=", o_rem[2]);
+            $display("expected 5,1 EXITING");
+            $finish();
+        end
+
+        if (o_div[3] != 1'bx || o_rem[3] != 1'bx) begin
+            $display("PIPE3: div=", o_div[3], " rem=", o_rem[3]);
+            $display("expected x,x EXITING");
+            $finish();
+        end
+
+        #2;
+
+        $display("56 / 11 #6");
+
+        if (o_div[0] != 5 || o_rem[0] != 1) begin
+            $display("PIPE0: div=", o_div[0], " rem=", o_rem[0]);
+            $display("expected 5,1 EXITING");
+            $finish();
+        end
+
+        if (o_div[1] != 5 || o_rem[1] != 1) begin
+            $display("PIPE1: div=", o_div[1], " rem=", o_rem[1]);
+            $display("expected 5,1 EXITING");
+            $finish();
+        end
+
+        if (o_div[2] != 5 || o_rem[2] != 1) begin
+            $display("PIPE2: div=", o_div[2], " rem=", o_rem[2]);
+            $display("expected 5,1 EXITING");
+            $finish();
+        end
+
+        if (o_div[3] != 5 || o_rem[3] != 1) begin
+            $display("PIPE3: div=", o_div[3], " rem=", o_rem[3]);
+            $display("expected 5,1 EXITING");
+            $finish();
+        end
+
+        $display("PASS");
+
+        $finish();
+    end
+
+    always #1
+        clk = !clk;
+
+endmodule
--- a/hw/unittest/cache/Makefile
+++ b/hw/unittest/cache/Makefile
@@ -0,0 +1,77 @@
+DESTDIR ?= .
+RTL_DIR = ../../rtl
+DPI_DIR = ../../dpi
+
+CONFIGS +=
+PARAMS +=
+
+CXXFLAGS += -std=c++11 -Wall -Wextra -Wfatal-errors -Wno-array-bounds
+CXXFLAGS += -fPIC -Wno-maybe-uninitialized
+CXXFLAGS += -I../../.. -I../../common -I../../../../sim/common
+CXXFLAGS += $(CONFIGS)
+
+LDFLAGS += 
+
+# control RTL debug tracing states
+DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE_BANK 
+DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE_MSHR
+DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE_TAG
+DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE_DATA
+
+DBG_FLAGS += -DDEBUG_LEVEL=$(DEBUG) -DVCD_OUTPUT $(DBG_TRACE_FLAGS)
+
+RTL_PKGS = $(RTL_DIR)/VX_gpu_pkg.sv
+
+RTL_INCLUDE = -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/mem -I$(RTL_DIR)/cache
+
+SRCS = cachesim.cpp testbench.cpp
+SRCS += $(DPI_DIR)/util_dpi.cpp
+
+TOP = VX_cache_top
+
+VL_FLAGS = --exe
+VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic
+VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO
+VL_FLAGS += --x-initial unique --x-assign unique
+VL_FLAGS += -DSIMULATION
+VL_FLAGS += $(CONFIGS)
+VL_FLAGS += $(PARAMS)
+VL_FLAGS += $(RTL_INCLUDE)
+VL_FLAGS += $(RTL_PKGS)
+VL_FLAGS += --cc $(TOP) --top-module $(TOP)
+
+# Enable Verilator multithreaded simulation
+THREADS ?= $(shell python -c 'import multiprocessing as mp; print(mp.cpu_count())')
+VL_FLAGS += -j $(THREADS)
+#VL_FLAGS += --threads $(THREADS)
+
+# Debugigng
+ifdef DEBUG
+	VL_FLAGS += --trace --trace-structs $(DBG_FLAGS)
+	CXXFLAGS += -g -O0 $(DBG_FLAGS)
+else    
+	VL_FLAGS += -DNDEBUG
+	CXXFLAGS += -O2 -DNDEBUG
+endif
+
+# Enable perf counters
+ifdef PERF
+	VL_FLAGS += -DPERF_ENABLE
+	CXXFLAGS += -DPERF_ENABLE
+endif
+
+PROJECT = cache
+
+all: $(DESTDIR)/$(PROJECT)
+	
+$(DESTDIR)/$(PROJECT): $(SRCS)
+	verilator --build $(VL_FLAGS) $^ -CFLAGS '$(CXXFLAGS)' -o ../$@
+
+run: $(DESTDIR)/$(PROJECT)
+	$(DESTDIR)/$(PROJECT)
+
+waves: trace.vcd
+	gtkwave -o trace.vcd
+
+clean:
+	rm -rf obj_dir $(DESTDIR)/$(PROJECT)
--- a/hw/unittest/cache/cachesim.cpp
+++ b/hw/unittest/cache/cachesim.cpp
@@ -0,0 +1,353 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cachesim.h"
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <vector>
+#include <bitset>
+
+#ifndef TRACE_START_TIME
+#define TRACE_START_TIME 0ull
+#endif
+
+#ifndef TRACE_STOP_TIME
+#define TRACE_STOP_TIME -1ull
+#endif
+
+static uint64_t timestamp = 0;
+static bool trace_enabled = false;
+static uint64_t trace_start_time = TRACE_START_TIME;
+static uint64_t trace_stop_time  = TRACE_STOP_TIME;
+
+double sc_time_stamp() { 
+  return timestamp;
+}
+
+bool sim_trace_enabled() {
+  if (timestamp >= trace_start_time 
+   && timestamp < trace_stop_time)
+    return true;
+  return trace_enabled;
+}
+
+void sim_trace_enable(bool enable) {
+  trace_enabled = enable;
+}
+
+CacheSim::CacheSim() {
+  // force random values for uninitialized signals  
+  Verilated::randReset(2);
+
+  ram_ = nullptr;
+  cache_ = new VVX_cache_top();
+
+  mem_rsp_active_ = false;
+  snp_req_active_ = false;
+
+#ifdef VCD_OUTPUT
+  Verilated::traceEverOn(true);
+  trace_ = new VerilatedVcdC;
+  cache_->trace(trace_, 99);
+  trace_->open("trace.vcd");
+#endif
+}
+
+CacheSim::~CacheSim() {
+#ifdef VCD_OUTPUT
+  trace_->close();
+#endif
+  delete cache_;
+  //need to delete the req and rsp vectors
+}
+
+void CacheSim::attach_ram(RAM* ram) {
+  ram_ = ram;
+  mem_rsp_vec_.clear();
+}
+
+void CacheSim::reset() {
+#ifndef NDEBUG
+  std::cout << timestamp << ": [sim] reset()" << std::endl;
+#endif
+
+  cache_->reset = 1;
+  this->step();
+  cache_->reset = 0;
+  this->step();
+
+  mem_rsp_vec_.clear();
+  //clear req and rsp vecs
+  
+}
+
+void CacheSim::step() {
+  //std::cout << timestamp << ": [sim] step()" << std::endl;
+  //toggle clock
+  cache_->clk = 0;
+  this->eval();
+
+  cache_->clk = 1;
+  this->eval();
+
+  //handle core and memory reqs and rsps
+  this->eval_reqs();
+  this->eval_rsps();
+  this->eval_mem_bus();
+  timestamp++;
+}
+
+void CacheSim::eval() {
+  cache_->eval();
+#ifdef VCD_OUTPUT
+  trace_->dump(timestamp);
+#endif
+  ++timestamp;
+}
+
+void CacheSim::run(){
+//#ifndef NDEBUG
+  
+//#endif
+  this->step();
+
+  int valid = 300; 
+  int stalls = 20 + 10; 
+  
+  while (valid > -1) {
+
+      this->step();
+      display_miss(); 
+      if(cache_->core_rsp_valid){
+        get_core_rsp();
+      }
+      
+      if(!cache_->core_req_valid && !cache_->core_rsp_valid){
+        valid--; 
+       
+      }
+      stalls--; 
+      if (stalls == 20){
+          //stall_mem(); 
+          //send_snoop_req(); 
+          stalls--; 
+      }
+  }
+}
+
+void CacheSim::clear_req(){
+  cache_->core_req_valid = 0; 
+}
+
+void CacheSim::send_req(core_req_t *req){
+  core_req_vec_.push(req);
+  unsigned int *data = new unsigned int[4];
+  core_rsp_vec_.insert(std::pair<unsigned int, unsigned int*>(req->tag, data));
+}
+
+bool CacheSim::get_core_req_ready(){
+  return cache_->core_req_ready; 
+}
+
+bool CacheSim::get_core_rsp_ready(){
+  return cache_->core_rsp_ready; 
+}
+
+void CacheSim::eval_reqs(){
+  //check to see if cache is accepting reqs
+  if(!core_req_vec_.empty() && cache_->core_req_ready){
+    core_req_t *req = core_req_vec_.front();
+
+    cache_->core_req_valid = req->valid;
+    cache_->core_req_rw = req->rw; 
+    cache_->core_req_byteen = req->byteen;
+
+    cache_->core_req_addr[0] = req->addr[0];
+    cache_->core_req_addr[1] = req->addr[1];
+    cache_->core_req_addr[2] = req->addr[2];
+    cache_->core_req_addr[3] = req->addr[3];
+
+    cache_->core_req_data[0] = req->data[0];
+    cache_->core_req_data[1] = req->data[1];
+    cache_->core_req_data[2] = req->data[2];
+    cache_->core_req_data[3] = req->data[3];
+
+    cache_->core_req_tag = req->tag; 
+
+    core_req_vec_.pop();
+   
+  } else {
+    clear_req();
+  }
+}
+
+void CacheSim::eval_rsps(){
+  //check to see if a request has been responded to
+  if (cache_->core_rsp_valid){
+      core_rsp_vec_.at(cache_->core_rsp_tag)[0] = cache_->core_rsp_data[0];
+      core_rsp_vec_.at(cache_->core_rsp_tag)[1] = cache_->core_rsp_data[1];
+      core_rsp_vec_.at(cache_->core_rsp_tag)[2] = cache_->core_rsp_data[2];
+      core_rsp_vec_.at(cache_->core_rsp_tag)[3] = cache_->core_rsp_data[3];
+  }
+}
+
+void CacheSim::stall_mem(){
+  cache_->mem_req_ready = 0;
+}
+
+void CacheSim::send_snoop_req(){
+    /*cache_->snp_req_valid = 1;
+    cache_->snp_req_addr = 0x12222222;
+    cache_->snp_req_invalidate = 1; 
+    cache_->snp_req_tag = 0xff; */
+}
+
+void CacheSim::eval_mem_bus() {
+  if (ram_ == nullptr) {
+    cache_->mem_req_ready = 0;
+    return;
+  }
+
+  // schedule memory responses
+  int dequeue_index = -1;
+  for (int i = 0; i < mem_rsp_vec_.size(); i++) {
+    if (mem_rsp_vec_[i].cycles_left > 0) {
+      mem_rsp_vec_[i].cycles_left -= 1;
+    }
+    if ((dequeue_index == -1) 
+     && (mem_rsp_vec_[i].cycles_left == 0)) {
+      dequeue_index = i;
+    }
+  }
+
+  // send memory response  
+  if (mem_rsp_active_
+   && cache_->mem_rsp_valid 
+   && cache_->mem_rsp_ready) {
+    mem_rsp_active_ = false;
+  }
+  if (!mem_rsp_active_) {
+    if (dequeue_index != -1) { //time to respond to the request
+      cache_->mem_rsp_valid = 1;
+
+      //copy data from the rsp queue to the cache module
+      memcpy(cache_->mem_rsp_data.data(), mem_rsp_vec_[dequeue_index].data, MEM_BLOCK_SIZE);
+
+      cache_->mem_rsp_tag = mem_rsp_vec_[dequeue_index].tag;    
+      free(mem_rsp_vec_[dequeue_index].data); //take data out of the queue
+      mem_rsp_vec_.erase(mem_rsp_vec_.begin() + dequeue_index);
+      mem_rsp_active_ = true;
+    } else {
+      cache_->mem_rsp_valid = 0;
+    }
+  }
+
+  // handle memory stalls
+  bool mem_stalled = false;
+#ifdef ENABLE_MEM_STALLS
+  if (0 == ((timestamp/2) % MEM_STALLS_MODULO)) { 
+    mem_stalled = true;
+  } else
+  if (mem_rsp_vec_.size() >= MEM_RQ_SIZE) {
+    mem_stalled = true;
+  }
+#endif
+
+  // process memory requests
+  if (!mem_stalled) {
+    if (cache_->mem_req_valid) {
+      if (cache_->mem_req_rw) { //write = 1
+        uint64_t byteen = cache_->mem_req_byteen;
+        uint64_t base_addr = (cache_->mem_req_addr * MEM_BLOCK_SIZE);
+        uint8_t* data = reinterpret_cast<uint8_t*>(cache_->mem_req_data.data());
+        for (int i = 0; i < MEM_BLOCK_SIZE; i++) {
+          if ((byteen >> i) & 0x1) {            
+            (*ram_)[base_addr + i] = data[i];
+          }
+        }
+      } else {
+        mem_req_t mem_req;
+        mem_req.cycles_left = MEM_LATENCY;     
+        mem_req.data = (uint8_t*)malloc(MEM_BLOCK_SIZE);
+        mem_req.tag = cache_->mem_req_tag;
+        ram_->read(cache_->mem_req_addr * MEM_BLOCK_SIZE, MEM_BLOCK_SIZE, mem_req.data);
+        mem_rsp_vec_.push_back(mem_req);
+      } 
+    }    
+  }
+
+  cache_->mem_req_ready = ~mem_stalled;
+}
+
+bool CacheSim::assert_equal(unsigned int* data, unsigned int tag){
+  int check = 0;
+  unsigned int *rsp = core_rsp_vec_.at(tag);
+  for (int i = 0; i < 4; ++i){
+    for (int j = 0; j < 4; ++j){
+      if (data[i] == rsp[j]){
+        check++;
+      }
+    }
+  }
+
+  return check; 
+
+}
+
+//DEBUG
+
+void CacheSim::display_miss(){
+  //int i = (unsigned int)cache_->miss_vec; 
+  //std::bitset<8> x(i); 
+  //if (i) std::cout << "Miss Vec " << x << std::endl;
+  //std::cout << "Miss Vec 0" << cache_->miss_vec[0] << std::endl;
+}
+
+void CacheSim::get_core_req(unsigned int (&rsp)[4]){
+  rsp[0] = cache_->core_rsp_data[0];
+  rsp[1] = cache_->core_rsp_data[1];
+  rsp[2] = cache_->core_rsp_data[2];
+  rsp[3] = cache_->core_rsp_data[3];
+
+  //std::cout << std::hex << "core_rsp_valid: " << cache_->core_rsp_valid << std::endl;
+  //std::cout << std::hex << "core_rsp_data: " << cache_->core_rsp_data << std::endl;
+  //std::cout << std::hex << "core_rsp_tag: " << cache_->core_rsp_tag << std::endl; 
+}
+
+void CacheSim::get_core_rsp(){
+  //std::cout << cache_->genblk5_BRA_0_KET_->bank->is_fill_in_pipe<< std::endl; 
+  char check = cache_->core_rsp_valid;
+  std::cout << std::hex << "core_rsp_valid: " << (unsigned int) check << std::endl;
+  std::cout << std::hex << "core_rsp_data[0]: " << cache_->core_rsp_data[0] << std::endl;
+  std::cout << std::hex << "core_rsp_data[1]: " << cache_->core_rsp_data[1] << std::endl;
+  std::cout << std::hex << "core_rsp_data[2]: " << cache_->core_rsp_data[2] << std::endl;
+  std::cout << std::hex << "core_rsp_data[3]: " << cache_->core_rsp_data[3] << std::endl;
+  std::cout << std::hex << "core_rsp_tag: " << cache_->core_rsp_tag << std::endl; 
+}
+
+void CacheSim::get_mem_req(){
+  std::cout << std::hex << "mem_req_valid: " << cache_->mem_req_valid << std::endl;
+  std::cout << std::hex << "mem_req_rw: " << cache_->mem_req_rw << std::endl;
+  std::cout << std::hex << "mem_req_byteen: " << cache_->mem_req_byteen << std::endl;
+  std::cout << std::hex << "mem_req_addr: " << cache_->mem_req_addr << std::endl;
+  std::cout << std::hex << "mem_req_data: " << cache_->mem_req_data << std::endl; 
+  std::cout << std::hex << "mem_req_tag: " << cache_->mem_req_tag << std::endl;
+}
+
+void CacheSim::get_mem_rsp(){
+  std::cout << std::hex << "mem_rsp_valid: " << cache_->mem_rsp_valid << std::endl;
+  std::cout << std::hex << "mem_rsp_data: " << cache_->mem_rsp_data << std::endl; 
+  std::cout << std::hex << "mem_rsp_tag: " << cache_->mem_rsp_tag << std::endl;
+  std::cout << std::hex << "mem_rsp_ready: " << cache_->mem_rsp_ready << std::endl;
+}
--- a/hw/unittest/cache/cachesim.h
+++ b/hw/unittest/cache/cachesim.h
@@ -0,0 +1,105 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include "VVX_cache_top.h"
+#include "VVX_cache_top__Syms.h"
+#include "verilated.h"
+
+#ifdef VCD_OUTPUT
+#include <verilated_vcd_c.h>
+#endif
+
+#include <VX_config.h>
+#include "ram.h"
+#include <ostream>
+#include <vector>
+#include <queue>
+
+#define ENABLE_MEM_STALLS
+#define MEM_LATENCY 100
+#define MEM_RQ_SIZE 16
+#define MEM_STALLS_MODULO 16
+#define MEM_BLOCK_SIZE 16
+
+typedef struct {
+  int cycles_left;
+  uint8_t *data;
+  unsigned tag;
+} mem_req_t;
+
+typedef struct {
+  char valid;
+  char rw; 
+  unsigned byteen;
+  unsigned *addr;
+  unsigned *data;
+  unsigned int tag;
+} core_req_t;
+
+class CacheSim {
+public:
+  
+  CacheSim();
+  virtual ~CacheSim();
+  
+  bool busy();  
+
+  void reset();
+  void step();
+  void wait(uint32_t cycles); 
+  void attach_ram(RAM* ram);
+  void run();  //run until all reqs are empty
+ 
+  //req/rsp
+  void send_req(core_req_t *req);
+  void clear_req(); 
+  void stall_mem();
+  void send_snoop_req();
+  void send_snp_fwd_in(); 
+
+  //assert funcs
+  bool assert_equal(unsigned int* data, unsigned int tag);
+
+  //debug funcs
+  void get_mem_req();
+  void get_core_req(unsigned int (&rsp)[4]);
+  void get_core_rsp();
+  bool get_core_req_ready();
+  bool get_core_rsp_ready();
+  void get_mem_rsp();
+  void display_miss();
+
+private:  
+
+  void eval();  
+  void eval_reqs(); 
+  void eval_rsps();
+  void eval_mem_bus();
+  
+  std::queue<core_req_t*> core_req_vec_; 
+  std::vector<mem_req_t> mem_rsp_vec_;
+  std::map<unsigned int, unsigned int*> core_rsp_vec_;
+  int mem_rsp_active_;
+
+  uint32_t snp_req_active_;
+  uint32_t snp_req_size_;
+  uint32_t pending_snp_reqs_;
+
+  VVX_cache_top *cache_;
+  RAM *ram_;
+#ifdef VCD_OUTPUT
+  VerilatedVcdC *trace_;
+#endif
+};
--- a/hw/unittest/cache/ram.h
+++ b/hw/unittest/cache/ram.h
@@ -0,0 +1,77 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <stdio.h>
+#include <stdint.h>
+
+class RAM {
+private:
+
+  mutable uint8_t *mem_[(1 << 12)];      
+
+  uint8_t *get(uint32_t address) const {
+    uint32_t block_addr   = address >> 20;
+    uint32_t block_offset = address & 0x000FFFFF;
+    if (mem_[block_addr] == NULL) {
+      mem_[block_addr] = new uint8_t[(1 << 20)];
+    }
+    return mem_[block_addr] + block_offset;
+  }
+
+public:
+
+  RAM() {
+    for (uint32_t i = 0; i < (1 << 12); i++) {
+      mem_[i] = NULL;
+    }
+  }
+
+  ~RAM() {
+    this->clear();
+  }
+
+  size_t size() const {
+    return (1ull << 32);
+  }
+
+  void clear() {
+    for (uint32_t i = 0; i < (1 << 12); i++) {
+      if (mem_[i]) {
+        delete [] mem_[i];
+        mem_[i] = NULL;
+      }
+    }
+  }
+
+  void read(uint32_t address, uint32_t length, uint8_t *data) const {
+    for (unsigned i = 0; i < length; i++) {
+      data[i] = *this->get(address + i);
+    }
+  }
+
+  void write(uint32_t address, uint32_t length, const uint8_t *data) {
+    for (unsigned i = 0; i < length; i++) {
+      *this->get(address + i) = data[i];
+    }
+  }
+
+  uint8_t& operator[](uint32_t address) {
+    return *get(address);
+  }
+
+  const uint8_t& operator[](uint32_t address) const {
+    return *get(address);
+  }
+};
--- a/hw/unittest/cache/testbench.cpp
+++ b/hw/unittest/cache/testbench.cpp
@@ -0,0 +1,248 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "cachesim.h"
+#include <iostream>
+#include <fstream>
+#include <iomanip>
+
+#define VCD_OUTPUT 1
+
+
+int REQ_RSP(CacheSim *sim){ //verified
+  unsigned int addr[4] = {0x12222222, 0xabbbbbbb, 0xcddddddd, 0xe4444444};
+  unsigned int data[4] = {0xffffffff, 0x11111111, 0x22222222, 0x33333333};
+  unsigned int rsp[4] = {0,0,0,0};
+  char responded = 0;
+  //write req
+  core_req_t* write = new core_req_t;
+  write->valid = 0xf;
+  write->rw = 0xf;
+  write->byteen = 0xffff;
+  write->addr = addr; 
+  write->data = data;
+  write->tag = 0xff;
+
+  //read req
+  core_req_t* read = new core_req_t;
+  read->valid = 0xf;
+  read->rw = 0;
+  read->byteen = 0xffff;
+  read->addr = addr;
+  read->data = addr; 
+  read->tag = 0xff;
+  
+  // reset the device
+  sim->reset();
+
+  //queue reqs
+  sim->send_req(write);
+  sim->send_req(read);
+
+  sim->run();
+
+  int check = sim->assert_equal(data, write->tag);
+  
+  if (check == 4) return 1; 
+
+  return 0;
+}
+
+int HIT_1(CacheSim *sim){
+  unsigned int addr[4] = {0x12222222, 0xabbbbbbb, 0xcddddddd, 0xe4444444};
+  unsigned int data[4] = {0xffffffff, 0x11111111, 0x22222222, 0x33333333};
+  unsigned int rsp[4] = {0,0,0,0};
+  char responded = 0;
+  //write req
+  core_req_t* write = new core_req_t;
+  write->valid = 0xf;
+  write->rw = 0xf;
+  write->byteen = 0xffff;
+  write->addr = addr; 
+  write->data = data;
+  write->tag = 0x11;
+
+  //read req
+  core_req_t* read = new core_req_t;
+  read->valid = 0xf;
+  read->rw = 0;
+  read->byteen = 0xffff;
+  read->addr = addr;
+  read->data = addr; 
+  read->tag = 0x22;
+  
+  // reset the device
+  sim->reset();
+
+  //queue reqs
+  sim->send_req(write);
+  sim->send_req(read);
+
+  sim->run();
+
+  bool check = sim->assert_equal(data, write->tag);
+  
+  return check;
+}
+
+int MISS_1(CacheSim *sim){
+  unsigned int addr1[4] = {0x12222222, 0xabbbbbbb, 0xcddddddd, 0xe4444444};
+  unsigned int addr2[4] = {0x12229222, 0xabbbb4bb, 0xcddd47dd, 0xe4423544};
+  unsigned int addr3[4] = {0x12223332, 0xabb454bb, 0xcdddeefd, 0xe4447744};
+  unsigned int data[4] = {0xffffffff, 0x11111111, 0x22222222, 0x33333333};
+  unsigned int rsp[4] = {0,0,0,0};
+  char responded = 0;
+  //write req
+  core_req_t* write = new core_req_t;
+  write->valid = 0xf;
+  write->rw = 0xf;
+  write->byteen = 0xffff;
+  write->addr = addr1; 
+  write->data = data;
+  write->tag = 0xff;
+
+  //read req
+  core_req_t* read1 = new core_req_t;
+  read1->valid = 0xf;
+  read1->rw = 0;
+  read1->byteen = 0xffff;
+  read1->addr = addr1;
+  read1->data = data; 
+  read1->tag = 0xff;
+
+  core_req_t* read2 = new core_req_t;
+  read2->valid = 0xf;
+  read2->rw = 0;
+  read2->byteen = 0xffff;
+  read2->addr = addr2;
+  read2->data = data; 
+  read2->tag = 0xff;
+  
+  core_req_t* read3 = new core_req_t;
+  read3->valid = 0xf;
+  read3->rw = 0;
+  read3->byteen = 0xffff;
+  read3->addr = addr3;
+  read3->data = data; 
+  read3->tag = 0xff;
+  
+  // reset the device
+  sim->reset();
+
+  //queue reqs
+  sim->send_req(write);
+  sim->send_req(read1);
+  sim->send_req(read2);
+  sim->send_req(read3);
+
+  sim->run();
+
+  bool check = sim->assert_equal(data, write->tag);
+  
+  return check;
+}
+int FLUSH(CacheSim *sim){
+  unsigned int addr[4] = {0x12222222, 0xabbbbbbb, 0xcddddddd, 0xe4444444};
+  unsigned int data[4] = {0xffffffff, 0x11111111, 0x22222222, 0x33333333};
+  unsigned int rsp[4] = {0,0,0,0};
+  char responded = 0;
+  //write req
+  core_req_t* write = new core_req_t;
+  write->valid = 0xf;
+  write->rw = 0xf;
+  write->byteen = 0xffff;
+  write->addr = addr; 
+  write->data = data;
+  write->tag = 0xff;
+
+  //read req
+  core_req_t* read = new core_req_t;
+  read->valid = 0xf;
+  read->rw = 0;
+  read->byteen = 0xffff;
+  read->addr = addr;
+  read->data = addr; 
+  read->tag = 0xff;
+  
+  // reset the device
+  sim->reset();
+
+  //queue reqs
+  sim->send_req(write);
+  sim->send_req(read);
+
+  sim->run();
+
+  bool check = sim->assert_equal(data, write->tag);
+  
+  return check;
+}
+
+
+int BACK_PRESSURE(CacheSim *sim){
+  //happens whenever the core is stalled or memory is stalled
+  unsigned int addr[4] = {0x12222222, 0xabbbbbbb, 0xcddddddd, 0xe4444444};
+  unsigned int data[4] = {0xffffffff, 0x11111111, 0x22222222, 0x33333333};
+  unsigned int rsp[4] = {0,0,0,0};
+  char responded = 0;
+  
+  //write req
+  core_req_t* write = new core_req_t;
+  write->valid = 0xf;
+  write->rw = 0xf;
+  write->byteen = 0xffff;
+  write->addr = addr; 
+  write->data = data;
+  write->tag = 0xff;
+
+  //read req
+  core_req_t* read = new core_req_t;
+  read->valid = 0xf;
+  read->rw = 0;
+  read->byteen = 0xffff;
+  read->addr = addr;
+  read->data = addr; 
+  read->tag = 0xff;
+  
+  // reset the device
+  sim->reset();
+
+  //queue reqs
+  for (int i = 0; i < 10; i++){
+    sim->send_req(write);
+  }
+  sim->send_req(read); 
+
+  sim->run();
+
+  bool check = sim->assert_equal(data, write->tag);
+  
+  return check;
+}
+
+
+int main(int argc, char **argv)
+{
+  //init
+  RAM ram;
+	CacheSim cachesim;
+  cachesim.attach_ram(&ram);
+  int check = REQ_RSP(&cachesim);
+  if(check){
+    std::cout << "PASSED" << std::endl;
+  } else {
+    std::cout << "FAILED" << std::endl;
+  }
+  
+	return 0;
+}
--- a/hw/unittest/common/vl_simulator.h
+++ b/hw/unittest/common/vl_simulator.h
@@ -0,0 +1,94 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <array>
+#include <cstdint>
+#include "verilated.h"
+
+#ifdef VCD_OUTPUT
+#include <verilated_vcd_c.h>	// Trace file format header
+#endif
+
+template <typename T>
+class vl_simulator {
+private:
+
+  T top_;
+#ifdef VCD_OUTPUT
+  VerilatedVcdC tfp_;
+#endif
+
+public:
+
+  vl_simulator() {
+    top_.clk = 0;
+    top_.reset = 0;
+  #ifdef VCD_OUTPUT
+    Verilated::traceEverOn(true);
+    top_.trace(&tfp_, 99);
+    tfp_.open("trace.vcd");
+  #endif
+  }
+
+  ~vl_simulator() {
+  #ifdef VCD_OUTPUT
+    tfp_.close();
+  #endif
+    top_.final();
+  }
+
+  uint64_t reset(uint64_t ticks) {
+    top_.reset = 1;
+    ticks = this->step(ticks, 2);
+    top_.reset = 0;
+    return ticks;
+  }
+
+  uint64_t step(uint64_t ticks, uint32_t count = 1) {
+    while (count--) {
+      top_.eval();
+    #ifdef VCD_OUTPUT
+      tfp_.dump(ticks);
+    #endif
+      top_.clk = !top_.clk;
+      ++ticks;
+    }
+    return ticks;
+  }
+
+  T* operator->() {
+    return &top_;
+  }
+};
+
+template <typename... Args>
+void vl_setw(uint32_t* sig, Args&&... args) {
+  std::array<uint32_t, sizeof... (Args)> arr{static_cast<uint32_t>(std::forward<Args>(args))...};
+  for (size_t i = 0; i < sizeof... (Args); ++i) {
+    sig[i] = arr[i];
+  }
+}
+
+template <typename... Args>
+int  vl_cmpw(const uint32_t* sig, Args&&... args) {
+  std::array<uint32_t, sizeof... (Args)> arr{static_cast<uint32_t>(std::forward<Args>(args))...};
+  for (size_t i = 0; i < sizeof... (Args); ++i) {
+    if (sig[i] < arr[i])
+      return -1;
+    if (sig[i] > arr[i])
+      return 1;
+  }
+  return 0;
+}
--- a/hw/unittest/generic_queue/Makefile
+++ b/hw/unittest/generic_queue/Makefile
@@ -0,0 +1,65 @@
+DESTDIR ?= .
+RTL_DIR = ../../rtl
+DPI_DIR = ../../dpi
+
+CONFIGS +=
+PARAMS +=
+
+CXXFLAGS += -std=c++11 -Wall -Wextra -Wfatal-errors -Wno-array-bounds
+CXXFLAGS += -fPIC -Wno-maybe-uninitialized
+CXXFLAGS += -I../../.. -I../../common -I../../../../sim/common
+CXXFLAGS += $(CONFIGS)
+
+LDFLAGS +=
+
+DBG_FLAGS += -DDEBUG_LEVEL=$(DEBUG) -DVCD_OUTPUT $(DBG_TRACE_FLAGS)
+
+RTL_PKGS +=
+
+RTL_INCLUDE = -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs
+
+SRCS = main.cpp
+SRCS += $(DPI_DIR)/util_dpi.cpp
+
+TOP = VX_fifo_queue
+
+VL_FLAGS = --exe
+VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic
+VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO
+VL_FLAGS += --x-initial unique --x-assign unique
+VL_FLAGS += -DSIMULATION
+VL_FLAGS += $(CONFIGS)
+VL_FLAGS += $(PARAMS)
+VL_FLAGS += $(RTL_INCLUDE)
+VL_FLAGS += $(RTL_PKGS)
+VL_FLAGS += --cc $(TOP) --top-module $(TOP)
+
+# Enable Verilator multithreaded simulation
+THREADS ?= $(shell python -c 'import multiprocessing as mp; print(mp.cpu_count())')
+VL_FLAGS += -j $(THREADS)
+#VL_FLAGS += --threads $(THREADS)
+
+# Debugigng
+ifdef DEBUG
+	VL_FLAGS += --trace --trace-structs $(DBG_FLAGS)
+	CXXFLAGS += -g -O0 $(DBG_FLAGS)
+else    
+	VL_FLAGS += -DNDEBUG
+	CXXFLAGS += -O2 -DNDEBUG
+endif
+
+PROJECT = generic_queue
+
+all: $(DESTDIR)/$(PROJECT)
+	
+$(DESTDIR)/$(PROJECT): $(SRCS)
+	verilator --build $(VL_FLAGS) $^ -CFLAGS '$(CXXFLAGS)' -o ../$@
+
+run: $(DESTDIR)/$(PROJECT)
+	$(DESTDIR)/$(PROJECT)
+
+waves: trace.vcd
+	gtkwave -o trace.vcd
+
+clean:
+	rm -rf obj_dir $(DESTDIR)/$(PROJECT)
--- a/hw/unittest/generic_queue/main.cpp
+++ b/hw/unittest/generic_queue/main.cpp
@@ -0,0 +1,128 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "vl_simulator.h"
+#include "VVX_fifo_queue.h"
+#include <iostream>
+
+#define MAX_TICKS 20
+
+#ifndef TRACE_START_TIME
+#define TRACE_START_TIME 0ull
+#endif
+
+#ifndef TRACE_STOP_TIME
+#define TRACE_STOP_TIME -1ull
+#endif
+
+#define CHECK(x)                                  \
+   do {                                           \
+     if (x)                                       \
+       break;                                     \
+     std::cout << "FAILED: " << #x << std::endl;  \
+	   std::abort();			                          \
+   } while (false)
+
+static uint64_t timestamp = 0;
+static bool trace_enabled = false;
+static uint64_t trace_start_time = TRACE_START_TIME;
+static uint64_t trace_stop_time  = TRACE_STOP_TIME;
+
+double sc_time_stamp() { 
+  return timestamp;
+}
+
+bool sim_trace_enabled() {
+  if (timestamp >= trace_start_time 
+   && timestamp < trace_stop_time)
+    return true;
+  return trace_enabled;
+}
+
+void sim_trace_enable(bool enable) {
+  trace_enabled = enable;
+}
+
+using Device = VVX_fifo_queue;
+
+int main(int argc, char **argv) {
+  // Initialize Verilators variables
+  Verilated::commandArgs(argc, argv);
+
+  vl_simulator<Device> sim;
+
+  // run test
+  timestamp = sim.reset(0);
+  while (timestamp < MAX_TICKS) {
+    switch (timestamp) {
+    case 0:
+      // initial values
+      sim->pop  = 0;
+      sim->push = 0;
+      timestamp = sim.step(timestamp, 2);
+      break;
+    case 2:
+      // Verify outputs    
+      CHECK(sim->full == 0x0);
+      CHECK(sim->empty == 0x1);
+      // push 0xa
+      sim->pop     = 0;
+      sim->push    = 1;
+      sim->data_in = 0xa;
+      break;
+    case 4:
+      // verify outputs    
+      CHECK(sim->data_out == 0xa);
+      CHECK(sim->full == 0x0);
+      CHECK(sim->empty == 0x0);
+      // push 0xb
+      sim->pop     = 0;
+      sim->push    = 1;
+      sim->data_in = 0xb;
+      break;
+    case 6:
+      // verify outputs    
+      CHECK(sim->data_out == 0xa);
+      CHECK(sim->full == 0x1);
+      CHECK(sim->empty == 0x0);
+      // pop
+      sim->pop  = 1;
+      sim->push = 0;
+      break;
+    case 8:
+      // verify outputs    
+      CHECK(sim->data_out == 0xb);
+      CHECK(sim->full == 0x0);
+      CHECK(sim->empty == 0x0);
+      // pop
+      sim->pop  = 1;
+      sim->push = 0;
+      break;
+    case 10:
+      // verify outputs    
+      CHECK(sim->full == 0x0);
+      CHECK(sim->empty == 0x1);
+      sim->pop  = 0;
+      sim->push = 0;
+      break;
+    }
+
+    // advance clock
+    timestamp = sim.step(timestamp, 2);
+  }
+
+  std::cout << "PASSED!" << std::endl;
+  std::cout << "Simulation time: " << std::dec << timestamp/2 << " cycles" << std::endl;
+
+  return 0;
+}
--- a/hw/unittest/mem_streamer/Makefile
+++ b/hw/unittest/mem_streamer/Makefile
@@ -0,0 +1,65 @@
+DESTDIR ?= .
+RTL_DIR = ../../rtl
+DPI_DIR = ../../dpi
+
+CONFIGS +=
+PARAMS +=
+
+CXXFLAGS += -std=c++11 -Wall -Wextra -Wfatal-errors -Wno-array-bounds
+CXXFLAGS += -fPIC -Wno-maybe-uninitialized
+CXXFLAGS += -I../../.. -I../../common -I../../../../sim/common
+CXXFLAGS += $(CONFIGS)
+
+LDFLAGS +=
+
+DBG_FLAGS += -DDEBUG_LEVEL=$(DEBUG) -DVCD_OUTPUT $(DBG_TRACE_FLAGS)
+
+RTL_PKGS +=
+
+RTL_INCLUDE = -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs
+
+SRCS = memsim.cpp ram.cpp
+SRCS += $(DPI_DIR)/util_dpi.cpp
+
+TOP = VX_mem_scheduler
+
+VL_FLAGS = --exe
+VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic
+VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO
+VL_FLAGS += --x-initial unique --x-assign unique
+VL_FLAGS += -DSIMULATION
+VL_FLAGS += $(CONFIGS)
+VL_FLAGS += $(PARAMS)
+VL_FLAGS += $(RTL_INCLUDE)
+VL_FLAGS += $(RTL_PKGS)
+VL_FLAGS += --cc $(TOP) --top-module $(TOP)
+
+# Enable Verilator multithreaded simulation
+THREADS ?= $(shell python -c 'import multiprocessing as mp; print(mp.cpu_count())')
+VL_FLAGS += -j $(THREADS)
+#VL_FLAGS += --threads $(THREADS)
+
+# Debugigng
+ifdef DEBUG
+	VL_FLAGS += --trace --trace-structs $(DBG_FLAGS)
+	CXXFLAGS += -g -O0 $(DBG_FLAGS)
+else    
+	VL_FLAGS += -DNDEBUG
+	CXXFLAGS += -O2 -DNDEBUG
+endif
+
+PROJECT = mem_streamer
+
+all: $(DESTDIR)/$(PROJECT)
+	
+$(DESTDIR)/$(PROJECT): $(SRCS)
+	verilator --build $(VL_FLAGS) $^ -CFLAGS '$(CXXFLAGS)' -o ../$@
+
+run: $(DESTDIR)/$(PROJECT)
+	$(DESTDIR)/$(PROJECT)
+
+waves: trace.vcd
+	gtkwave -o trace.vcd
+
+clean:
+	rm -rf obj_dir $(DESTDIR)/$(PROJECT)
--- a/hw/unittest/mem_streamer/memsim.cpp
+++ b/hw/unittest/mem_streamer/memsim.cpp
@@ -0,0 +1,166 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <random>
+#include "memsim.h"
+#include "ram.h"
+
+#ifndef TRACE_START_TIME
+#define TRACE_START_TIME 0ull
+#endif
+
+#ifndef TRACE_STOP_TIME
+#define TRACE_STOP_TIME -1ull
+#endif
+
+static bool trace_enabled = false;
+static uint64_t trace_start_time = 0;
+static uint64_t trace_stop_time = -1ull;
+static uint64_t timestamp = 0;
+
+double sc_time_stamp() { 
+  	return timestamp;
+}
+
+bool sim_trace_enabled() {
+	if (timestamp >= trace_start_time 
+	&& timestamp < trace_stop_time)
+			return true;
+	return trace_enabled;
+}
+
+void sim_trace_enable (bool enable) {
+  	trace_enabled = enable;
+}
+
+int generate_rand (int min, int max) {
+	int range = max - min + 1;
+	return rand() % range + min;
+}
+
+int generate_rand_mask (int mask) {
+	int result = 0;
+	int m = mask;
+	for (int i = 0; i < 4; i++) {
+		int bit = m & 0b1;
+		int rand_bit = generate_rand (0, bit);
+		result |= (rand_bit << i);
+		m = m >> 1;
+	}
+	return result;
+}
+
+MemSim::MemSim() {
+	msu_ = new VVX_mem_scheduler();
+
+	// Enable tracing
+	Verilated::traceEverOn(true);
+
+#ifdef VCD_OUTPUT
+  	Verilated::traceEverOn(true);
+  	trace_ = new VerilatedVcdC;
+  	cache_->trace(trace_, 99);
+  	race_->open("trace.vcd");
+#endif
+}
+
+MemSim::~MemSim() {
+#ifdef VCD_OUTPUT
+	trace_->close();
+#endif
+	delete msu_;
+}
+
+void MemSim::eval() {
+	msu_->eval();
+#ifdef VCD_OUTPUT
+	trace_->dump(timestamp++);
+#endif
+}
+
+void MemSim::step() {
+	msu_->clk = 0;
+	this->eval();
+
+	msu_->clk = 1;
+	this->eval();
+}
+
+void MemSim::reset() {
+	msu_->reset = 1;
+	this->step();
+
+	msu_->reset = 0;
+	this->step();
+}
+
+void MemSim::attach_core() {
+	if (msu_->req_ready) {
+		msu_->req_valid 	= generate_rand(0, 1);
+		msu_->req_rw 		= generate_rand(0, 1);
+		msu_->req_mask 		= generate_rand(0b0001, 0b1111);
+		msu_->req_byteen 	= 0b1;
+		msu_->req_addr 		= generate_rand(0, 0x10000000);
+		msu_->req_data 		= generate_rand(0x60000000, 0x80000000);
+		msu_->req_tag 		= generate_rand(0x00, 0xFF);
+	}
+	msu_->rsp_ready = true;
+}
+
+void MemSim::attach_ram (RAM *ram) {
+
+	req_t req;
+	req.valid 			= msu_->mem_req_valid;
+	req.rw 				= msu_->mem_req_rw;
+	req.byteen			= msu_->mem_req_byteen;
+	req.addr 			= msu_->mem_req_addr;
+	req.data 			= msu_->mem_req_data;
+	req.tag 			= msu_->mem_req_tag;
+	msu_->mem_req_ready = ram->is_ready();
+
+	ram->insert_req(req);
+
+	rsp_t rsp;
+	rsp = ram->schedule_rsp();
+
+	msu_->mem_rsp_valid = rsp.valid;
+	msu_->mem_rsp_data 	= rsp.data;
+	msu_->mem_rsp_tag 	= rsp.tag;
+	rsp.ready 			= msu_->mem_rsp_ready;
+	std::cout<<"MEMSIM: mem_rsp_ready: "<<rsp.ready<<"\n";
+
+	ram->halt_rsp(rsp);
+}
+
+void MemSim::run(RAM *ram) {
+	this->reset();
+
+	while (sc_time_stamp() < SIM_TIME) {
+		this->step();
+		std::cout<<"========================="<<"\n";
+		std::cout<<"Cycle: "<<sc_time_stamp()<<"\n";
+		this->attach_core();
+		this->attach_ram(ram);
+	}
+}
+
+int main (int argc, char** argv, char** env) {
+    Verilated::commandArgs(argc, argv);
+
+	MemSim memsim;
+	RAM ram;	
+
+	memsim.run(&ram);
+
+	return 0;
+}
--- a/hw/unittest/mem_streamer/memsim.h
+++ b/hw/unittest/mem_streamer/memsim.h
@@ -0,0 +1,49 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <unordered_map>
+#include <vector>
+#include <verilated.h>
+#include <verilated_vcd_c.h>
+#include "VVX_mem_scheduler.h"
+#include "VVX_mem_scheduler__Syms.h"
+#include "ram.h"
+
+#define SIM_TIME 5000
+
+int generate_rand (int min, int max);
+int generate_rand_mask (int mask);
+
+class MemSim {
+public:
+    MemSim();
+    virtual ~MemSim();
+
+    void run(RAM *ram);
+
+private:
+    VVX_mem_scheduler *msu_;
+#ifdef VCD_OUTPUT
+    VerilatedVcdC *trace_;
+#endif
+
+    void eval();
+    void step();
+    void reset();
+
+    void attach_core();
+    void attach_ram(RAM *ram);
+};
--- a/hw/unittest/mem_streamer/ram.cpp
+++ b/hw/unittest/mem_streamer/ram.cpp
@@ -0,0 +1,123 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "ram.h"
+#include "memsim.h"
+
+RAM::RAM() {
+
+    ram_.clear();
+    is_rsp_active_ = false;
+    is_rsp_stall_ = false;
+}
+
+bool RAM::check_duplicate_req(req_t req) {
+    for(int i = 0; i < ram_.size(); i++) {
+        if (ram_[i].addr == req.addr) {
+            std::cout<<"RAM: Duplicate entry. Do not insert..."<<std::endl;
+            return true;
+        }
+    }
+    return false;
+}
+
+int RAM::simulate_cycle_delay() {
+
+    std::cout<<"RAM: # entries: "<<ram_.size()<<std::endl;
+
+    int dequeue_index = -1;
+
+    for (int i = 0; i < ram_.size(); i++) {
+        if (!is_rsp_stall_) {
+            if (ram_[i].cycles_left > 0) {
+                ram_[i].cycles_left -= 1;
+            }
+        }
+
+        std::cout<<"RAM: # cycles left: "<<ram_[i].cycles_left<<std::endl;
+
+        if (ram_[i].cycles_left == 0) {
+            dequeue_index = i;
+        }
+    }
+    return dequeue_index;
+}
+
+void RAM::insert_req(req_t req) {
+    if ( !(this->check_duplicate_req(req)) && req.valid && !req.rw) {
+        req_t r;
+        r.valid     = req.valid;
+        r.rw        = req.rw;
+        r.byteen    = req.byteen;
+        r.addr      = req.addr;
+        r.data      = req.data;
+        r.tag       = req.tag & 0b11;
+
+        // Store metadata
+        r.cycles_left = MEM_LATENCY;
+
+        std::cout<<"RAM: Insert entry... "<<std::endl;
+        std::cout<<"Write? : "<<req.rw<<std::endl;
+        ram_.push_back(r);
+    }
+}
+
+uint8_t RAM::is_ready() {    
+    // return generate_rand(0b1000, 0b1111);
+    return 0b1111;
+}
+
+rsp_t RAM::schedule_rsp() {
+    rsp_t rsp;
+    int dequeue_index = this->simulate_cycle_delay();
+
+    if (!is_rsp_active_) {
+        if (dequeue_index != -1) {
+
+            std::cout<<"RAM: Scheduling response... "<<std::endl;
+
+            is_rsp_active_ = true;
+            rsp.valid   = 1;
+            rsp.mask    = generate_rand_mask(ram_[dequeue_index].valid);
+            rsp.data    = generate_rand(0x20000000, 0x30000000);
+            rsp.tag     = ram_[dequeue_index].tag;
+
+            std::cout<<std::hex;
+            std::cout<<"RAM: Response mask: "<<+rsp.mask<<" | Required mask: "<<+ram_[dequeue_index].valid<<std::endl;
+
+            ram_[dequeue_index].rsp_sent_mask = rsp.mask;
+            ram_[dequeue_index].valid = ram_[dequeue_index].valid & ~ram_[dequeue_index].rsp_sent_mask;
+
+            if (0 == ram_[dequeue_index].valid) {
+                ram_.erase(ram_.begin() + dequeue_index);
+                is_rsp_stall_ = false;
+                std::cout<<"RAM: Clear entry... "<<std::endl;
+            } else {
+                is_rsp_stall_ = true;
+                std::cout<<"RAM: Stall... "<<std::endl;
+            }
+        } else {
+            rsp.valid = false;
+        }
+    } 
+    
+    return rsp;
+}
+
+// Schedule response for only one cycle
+void RAM::halt_rsp(rsp_t rsp) {
+    if (is_rsp_active_ && rsp.valid && rsp.ready) {
+        std::cout<<"RAM: Halt response..."<<std::endl;
+        is_rsp_active_ = false;
+    }
+}
--- a/hw/unittest/mem_streamer/ram.h
+++ b/hw/unittest/mem_streamer/ram.h
@@ -0,0 +1,64 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <iostream>
+#include <vector>
+
+#define MEM_LATENCY 4
+
+typedef struct {
+    uint8_t     valid;
+    bool        rw;
+    uint8_t     byteen;
+    uint32_t    addr;
+    uint32_t    data;
+    uint8_t     tag;
+    uint8_t     ready;
+
+    // Metadata
+    uint8_t     rsp_sent_mask;
+    double      cycles_left;
+} req_t;
+
+typedef struct {
+    bool        valid;
+    uint8_t     mask;
+    uint32_t    data;
+    uint8_t     tag;
+    bool        ready;
+} rsp_t;
+
+class RAM {
+
+    private:
+        std::vector<req_t> ram_;
+
+        bool is_rsp_active_;
+        bool is_rsp_stall_;
+
+        bool    check_duplicate_req(req_t req);
+        int     simulate_cycle_delay();
+    
+    public:
+        RAM();
+        
+        uint8_t is_ready();
+        void    insert_req(req_t req);
+        rsp_t   schedule_rsp();
+        void    halt_rsp(rsp_t rsp);
+
+};
+
+//////////////////////////////////////////////////////
--- a/hw/unittest/top_modules/Makefile
+++ b/hw/unittest/top_modules/Makefile
@@ -0,0 +1,68 @@
+DESTDIR ?= .
+RTL_DIR = ../../rtl
+DPI_DIR = ../../dpi
+
+CONFIGS +=
+PARAMS +=
+
+CXXFLAGS += -std=c++11 -Wall -Wextra -Wfatal-errors -Wno-array-bounds
+CXXFLAGS += -fPIC -Wno-maybe-uninitialized
+CXXFLAGS += -I../../.. -I../../common -I../../../../sim/common
+CXXFLAGS += $(CONFIGS)
+
+LDFLAGS += 
+
+DBG_FLAGS += -DDEBUG_LEVEL=$(DEBUG) -DVCD_OUTPUT $(DBG_TRACE_FLAGS)
+
+RTL_PKGS = $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv
+
+RTL_INCLUDE = -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/mem -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/core
+
+SRCS = main.cpp
+SRCS += $(DPI_DIR)/util_dpi.cpp
+
+VL_FLAGS = --exe
+VL_FLAGS += --language 1800-2009 --assert -Wall -Wpedantic
+VL_FLAGS += -Wno-DECLFILENAME -Wno-REDEFMACRO
+VL_FLAGS += --x-initial unique --x-assign unique
+VL_FLAGS += -DSIMULATION
+VL_FLAGS += $(CONFIGS)
+VL_FLAGS += $(PARAMS)
+VL_FLAGS += $(RTL_INCLUDE)
+VL_FLAGS += $(RTL_PKGS)
+
+# Enable Verilator multithreaded simulation
+THREADS ?= $(shell python -c 'import multiprocessing as mp; print(mp.cpu_count())')
+VL_FLAGS += -j $(THREADS)
+#VL_FLAGS += --threads $(THREADS)
+
+# Debugigng
+ifdef DEBUG
+	VL_FLAGS += --trace --trace-structs $(DBG_FLAGS)
+	CXXFLAGS += -g -O0 $(DBG_FLAGS)
+else    
+	VL_FLAGS += -DNDEBUG
+	CXXFLAGS += -O2 -DNDEBUG
+endif
+
+# Enable perf counters
+ifdef PERF
+	VL_FLAGS += -DPERF_ENABLE
+	CXXFLAGS += -DPERF_ENABLE
+endif
+
+PROJECT = top_modules
+
+all: build
+	
+build: $(SRCS)
+	verilator --build $(VL_FLAGS) --cc VX_cache_cluster_top --top-module VX_cache_cluster_top $^ -CFLAGS '$(CXXFLAGS)'
+	verilator --build $(VL_FLAGS) --cc VX_cache_top --top-module VX_cache_top $^ -CFLAGS '$(CXXFLAGS)'
+	verilator --build $(VL_FLAGS) --cc VX_core_top --top-module VX_core_top $^ -CFLAGS '$(CXXFLAGS)'
+
+run: 
+
+waves: 
+
+clean:
+	rm -rf obj_dir
--- a/hw/unittest/top_modules/main.cpp
+++ b/hw/unittest/top_modules/main.cpp
@@ -0,0 +1,49 @@
+// Copyright © 2019-2023
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "vl_simulator.h"
+
+#ifndef TRACE_START_TIME
+#define TRACE_START_TIME 0ull
+#endif
+
+#ifndef TRACE_STOP_TIME
+#define TRACE_STOP_TIME -1ull
+#endif
+
+static uint64_t timestamp = 0;
+static bool trace_enabled = false;
+static uint64_t trace_start_time = TRACE_START_TIME;
+static uint64_t trace_stop_time  = TRACE_STOP_TIME;
+
+double sc_time_stamp() { 
+  return timestamp;
+}
+
+bool sim_trace_enabled() {
+  if (timestamp >= trace_start_time 
+   && timestamp < trace_stop_time)
+    return true;
+  return trace_enabled;
+}
+
+void sim_trace_enable(bool enable) {
+  trace_enabled = enable;
+}
+
+int main(int argc, char **argv) {
+  // Initialize Verilators variables
+  Verilated::commandArgs(argc, argv);
+
+  return 0;
+}