From c023cf06883cd5581d012162add5e49038ac5800 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 23 Jun 2020 14:56:46 -0700
Subject: [PATCH 01/74] Rough initial implementation of diplomatic multiclock

---
 .../chipyard/src/main/scala/ChipTop.scala     | 134 ++++++------------
 .../chipyard/src/main/scala/Clocks.scala      | 115 +++++++++++++++
 .../src/main/scala/ConfigFragments.scala      |   7 +
 .../chipyard/src/main/scala/IOBinders.scala   |   4 +-
 .../chipyard/src/main/scala/Subsystem.scala   |  24 +++-
 .../chipyard/src/main/scala/TestHarness.scala |   9 +-
 .../main/scala/config/AbstractConfig.scala    |   1 +
 .../main/scala/config/TracegenConfigs.scala   |   5 +
 .../main/scala/config/TutorialConfigs.scala   |   4 +
 9 files changed, 207 insertions(+), 96 deletions(-)
 create mode 100644 generators/chipyard/src/main/scala/Clocks.scala

diff --git a/generators/chipyard/src/main/scala/ChipTop.scala b/generators/chipyard/src/main/scala/ChipTop.scala
index d0b4df02..0227feb8 100644
--- a/generators/chipyard/src/main/scala/ChipTop.scala
+++ b/generators/chipyard/src/main/scala/ChipTop.scala
@@ -4,8 +4,10 @@ import chisel3._
 
 import scala.collection.mutable.{ArrayBuffer}
 
+import freechips.rocketchip.prci.{ClockGroupIdentityNode, ClockSinkParameters, ClockSinkNode, ClockGroup}
+import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
 import freechips.rocketchip.config.{Parameters, Field}
-import freechips.rocketchip.diplomacy.{LazyModule}
+import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, LazyRawModuleImp, LazyModuleImpLike}
 import freechips.rocketchip.util.{ResetCatchAndSync}
 import chipyard.config.ConfigValName._
 import chipyard.iobinders.{IOBinders, TestHarnessFunction, IOBinderTuple}
@@ -14,108 +16,62 @@ import barstools.iocell.chisel._
 
 case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => LazyModule(new DigitalTop()(p)))
 
-/**
-  * Chipyard provides three baseline, top-level reset schemes, set using the
-  * [[GlobalResetSchemeKey]] in a Parameters instance. These are:
-  *
-  * 1) Synchronous: The input coming to the chip is synchronous to the provided
-  *    clocks and will be used without modification as a synchronous reset.
-  *    This is safe only for use in FireSim and SW simulation.
-  *
-  * 2) Asynchronous: The input reset is asynchronous to the input clock, but it
-  *    is caught and synchronized to that clock before it is dissemenated.
-  *    Thus, downsteam modules will be emitted with synchronously reset state
-  *    elements.
-  *
-  * 3) Asynchronous Full: The input reset is asynchronous to the input clock,
-  *    and is used globally as an async reset. Downstream modules will be emitted
-  *    with asynchronously reset state elements.
-  *
-  */
-sealed trait GlobalResetScheme {
-  def pinIsAsync: Boolean
-}
-sealed trait HasAsyncInput { self: GlobalResetScheme =>
-  def pinIsAsync = true
-}
-
-sealed trait HasSyncInput { self: GlobalResetScheme =>
-  def pinIsAsync = false
-}
-
-case object GlobalResetSynchronous extends GlobalResetScheme with HasSyncInput
-case object GlobalResetAsynchronous extends GlobalResetScheme with HasAsyncInput
-case object GlobalResetAsynchronousFull extends GlobalResetScheme with HasAsyncInput
-case object GlobalResetSchemeKey extends Field[GlobalResetScheme](GlobalResetSynchronous)
-
 
 /**
  * The base class used for building chips. This constructor instantiates a module specified by the BuildSystem parameter,
- * named "system", which is an instance of DigitalTop by default. The default clock and reset for "system" are set by two
- * wires, "systemClock" and "systemReset", which are intended to be driven by traits mixed-in with this base class.
+ * named "system", which is an instance of DigitalTop by default. The diplomatic clocks of System, as well as its implicit clock,
+ * is aggregated into the clockGroupNode. The parameterized functions controlled by ChipyardClockKey and GlobalResetSchemeKey
+ * drive clock and reset generation
  */
-abstract class BaseChipTop()(implicit val p: Parameters) extends RawModule with HasTestHarnessFunctions {
 
+class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunctions {
   // A publicly accessible list of IO cells (useful for a floorplanning tool, for example)
   val iocells = ArrayBuffer.empty[IOCell]
   // A list of functions to call in the test harness
   val harnessFunctions = ArrayBuffer.empty[TestHarnessFunction]
-  // The system clock
-  // These are given so that IOCell can use DataMirror and generate ports with
-  // the right flow (Input/Output)
-  val systemClock = Wire(Input(Clock()))
-  val systemReset = Wire(Input(Reset()))
 
   // The system module specified by BuildSystem
   val lSystem = p(BuildSystem)(p).suggestName("system")
-  val system = withClockAndReset(systemClock, systemReset) { Module(lSystem.module) }
 
-  // Call all of the IOBinders and provide them with a default clock and reset
-  withClockAndReset(systemClock, systemReset) {
-    // Call each IOBinder on both the lazyModule instance and the module
-    // instance. Generally, an IOBinder PF should only be defined on one, so
-    // this should not lead to two invocations.
-    val (_ports, _iocells, _harnessFunctions) = p(IOBinders).values.flatMap(f => f(lSystem) ++ f(system)).unzip3
-    // We ignore _ports for now...
-    iocells ++= _iocells.flatten
-    harnessFunctions ++= _harnessFunctions.flatten
+  // The systemClockSinkNode provides the implicit clock and reset for the System
+  private val systemClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
+
+  // clockGroupNode provides a single node which aggregates all clock groups in the design
+  val clockGroupNode = ClockGroupIdentityNode()
+
+  // If the specified system has diplomatic clocks, connect it to our clockGroupNode
+  if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
+    lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode :*= clockGroupNode }
   }
+  // Connect the system implicit clock node to the clockGroupNode
+  systemClockSinkNode := ClockGroup() := clockGroupNode
 
+  // Drive the entire diplomatic clock network using this configured Key
+  clockGroupNode :*=* p(ChipyardClockKey)(this)
+
+  // NOTE: Making this a LazyRawModule is moderately dangerous, as anonymous children
+  // of ChipTop (ex: ClockGroup) do not receive clock or reset.
+  // However. anonymous children of ChipTop should not need an implicit Clock or Reset
+  // anyways, they probably need to be explicitly clocked.
+  lazy val module: LazyModuleImpLike = new LazyRawModuleImp(this) {
+    // These become the implicit clock and reset to the System
+    val system_clock = systemClockSinkNode.in.head._1.clock
+    val system_reset = systemClockSinkNode.in.head._1.reset
+
+    // The implicit clock and reset for the system is also, by convention, used for all the IOBinders
+    // TODO: This may not be the right thing to do in all cases
+    withClockAndReset(system_clock, system_reset) {
+      val (_ports, _iocells, _harnessFunctions) = p(IOBinders).values.flatMap(f => f(lSystem) ++ f(lSystem.module)).unzip3
+      // We ignore _ports for now...
+      iocells ++= _iocells.flatten
+      harnessFunctions ++= _harnessFunctions.flatten
+    }
+
+    // Connect the implicit clock/reset, if present
+    lSystem.module match { case l: LazyModuleImp => {
+      l.clock := system_clock
+      l.reset := system_reset
+    }}
+  }
 }
 
-/**
- * A simple clock and reset implementation that punches out clock and reset ports with the same
- * names as the implicit clock and reset for standard Module classes. Three basic reset schemes 
- * are provided. See [[GlobalResetScheme]].
- */
-trait HasChipTopSimpleClockAndReset { this: BaseChipTop =>
-
-  val (clock, systemClockIO) = IOCell.generateIOFromSignal(systemClock, Some("iocell_clock"))
-  val (reset, systemResetIO) = p(GlobalResetSchemeKey) match {
-    case GlobalResetSynchronous  =>
-      IOCell.generateIOFromSignal(systemReset, Some("iocell_reset"))
-    case GlobalResetAsynchronousFull =>
-      IOCell.generateIOFromSignal(systemReset, Some("iocell_reset"), abstractResetAsAsync = true)
-    case GlobalResetAsynchronous =>
-      val asyncResetCore = Wire(Input(AsyncReset()))
-      systemReset := ResetCatchAndSync(systemClock, asyncResetCore.asBool)
-      IOCell.generateIOFromSignal(asyncResetCore, Some("iocell_reset"), abstractResetAsAsync = true)
-  }
-
-  iocells ++= systemClockIO
-  iocells ++= systemResetIO
-
-  // Add a TestHarnessFunction that connects clock and reset
-  harnessFunctions += { (th: TestHarness) => {
-    // Connect clock; it's not done implicitly with RawModule
-    clock := th.clock
-    // Connect reset; it's not done implicitly with RawModule
-    // Note that we need to use dutReset, not harnessReset
-    reset := th.dutReset
-    Nil
-  } }
-
-}
-
-class ChipTop()(implicit p: Parameters) extends BaseChipTop()(p)
-  with HasChipTopSimpleClockAndReset
diff --git a/generators/chipyard/src/main/scala/Clocks.scala b/generators/chipyard/src/main/scala/Clocks.scala
new file mode 100644
index 00000000..b9508d96
--- /dev/null
+++ b/generators/chipyard/src/main/scala/Clocks.scala
@@ -0,0 +1,115 @@
+package chipyard
+
+import chisel3._
+
+import scala.collection.mutable.{ArrayBuffer}
+
+import freechips.rocketchip.prci._
+import freechips.rocketchip.subsystem.{BaseSubsystem}
+import freechips.rocketchip.config.{Parameters, Field}
+import freechips.rocketchip.diplomacy.{OutwardNodeHandle, InModuleBody}
+import freechips.rocketchip.util.{ResetCatchAndSync}
+import chipyard.config.ConfigValName._
+
+import barstools.iocell.chisel._
+
+import ChipyardClockDrivers._
+
+case object ChipyardClockKey extends Field[ClockInstantiationFn](simpleTestHarnessClock)
+
+
+/**
+  * Chipyard provides three baseline, top-level reset schemes, set using the
+  * [[GlobalResetSchemeKey]] in a Parameters instance. These are:
+  *
+  * 1) Synchronous: The input coming to the chip is synchronous to the provided
+  *    clocks and will be used without modification as a synchronous reset.
+  *    This is safe only for use in FireSim and SW simulation.
+  *
+  * 2) Asynchronous: The input reset is asynchronous to the input clock, but it
+  *    is caught and synchronized to that clock before it is dissemenated.
+  *    Thus, downsteam modules will be emitted with synchronously reset state
+  *    elements.
+  *
+  * 3) Asynchronous Full: The input reset is asynchronous to the input clock,
+  *    and is used globally as an async reset. Downstream modules will be emitted
+  *    with asynchronously reset state elements.
+  *
+  */
+sealed trait GlobalResetScheme {
+  def pinIsAsync: Boolean
+}
+sealed trait HasAsyncInput { self: GlobalResetScheme =>
+  def pinIsAsync = true
+}
+
+sealed trait HasSyncInput { self: GlobalResetScheme =>
+  def pinIsAsync = false
+}
+
+case object GlobalResetSynchronous extends GlobalResetScheme with HasSyncInput
+case object GlobalResetAsynchronous extends GlobalResetScheme with HasAsyncInput
+case object GlobalResetAsynchronousFull extends GlobalResetScheme with HasAsyncInput
+case object GlobalResetSchemeKey extends Field[GlobalResetScheme](GlobalResetSynchronous)
+
+/**
+ * A simple reset implementation that punches out reset ports
+ * for standard Module classes. Three basic reset schemes
+ * are provided. See [[GlobalResetScheme]].
+ */
+object GenerateReset {
+  def apply(chiptop: ChipTop, clock: Clock): Reset = {
+    implicit val p = chiptop.p
+    // this needs directionality so generateIOFromSignal works
+    val reset_wire = Wire(Input(Reset()))
+    val (reset_io, resetIOCell) = p(GlobalResetSchemeKey) match {
+      case GlobalResetSynchronous =>
+        IOCell.generateIOFromSignal(reset_wire, Some("iocell_reset"))
+      case GlobalResetAsynchronousFull =>
+        IOCell.generateIOFromSignal(reset_wire, Some("iocell_reset"), abstractResetAsAsync = true)
+      case GlobalResetAsynchronous => {
+        val async_reset_wire = Wire(Input(AsyncReset()))
+        reset_wire := ResetCatchAndSync(clock, async_reset_wire.asBool())
+        IOCell.generateIOFromSignal(async_reset_wire, Some("iocell_reset"), abstractResetAsAsync = true)
+      }
+    }
+    reset_io.suggestName("reset")
+    chiptop.iocells ++= resetIOCell
+    chiptop.harnessFunctions += ((th: TestHarness) => {
+      reset_io := th.dutReset
+      Nil
+    })
+    reset_wire
+  }
+}
+
+object ChipyardClockDrivers {
+  type ClockInstantiationFn = ChipTop => OutwardNodeHandle[ClockGroupSourceParameters, ClockGroupSinkParameters, ClockGroupEdgeParameters, ClockGroupBundle]
+
+  // A simple clock provider, for testing. All clocks in system are aggregated into one,
+  // and are driven by directly punching out to the TestHarness clock
+  val simpleTestHarnessClock: ClockInstantiationFn = { chiptop =>
+    implicit val p = chiptop.p
+    val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+    InModuleBody {
+      // this needs directionality so generateIOFromSignal works
+      val clock_wire = Wire(Input(Clock()))
+      val reset_wire = GenerateReset(chiptop, clock_wire)
+      val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, Some("iocell_clock"))
+      chiptop.iocells ++= clockIOCell
+
+      clock_io.suggestName("clock")
+
+      simpleClockGroupSourceNode.out.unzip._1.flatMap(_.member).map { o =>
+        o.clock := clock_wire
+        o.reset := reset_wire
+      }
+
+      chiptop.harnessFunctions += ((th: TestHarness) => {
+        clock_io := th.clock
+        Nil
+      })
+    }
+    ClockGroupAggregator() := simpleClockGroupSourceNode
+  }
+}
diff --git a/generators/chipyard/src/main/scala/ConfigFragments.scala b/generators/chipyard/src/main/scala/ConfigFragments.scala
index 024f5695..ac70f8e5 100644
--- a/generators/chipyard/src/main/scala/ConfigFragments.scala
+++ b/generators/chipyard/src/main/scala/ConfigFragments.scala
@@ -143,3 +143,10 @@ class WithRocketDCacheScratchpad extends Config((site, here, up) => {
   }
 })
 
+// The default RocketChip BaseSubsystem drives its diplomatic clock graph
+// with the implicit clocks of Subsystem. Don't do that, instead we extend
+// the diplomacy graph upwards into the ChipTop, where we connect it to
+// our clock drivers
+class WithNoSubsystemDrivenClocks extends Config((site, here, up) => {
+  case SubsystemDriveAsyncClockGroupsKey => None
+})
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 115723ab..3f986757 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -201,14 +201,14 @@ object AddIOCells {
   }
 
   def axi4(io: Seq[AXI4Bundle], node: AXI4SlaveNode, name: String): Seq[(AXI4Bundle, AXI4EdgeParameters, Seq[IOCell])] = {
-    io.zip(node.in).zipWithIndex.map{ case ((mem_axi4, (_, edge)), i) => {
+    io.zip(node.edges.in).zipWithIndex.map{ case ((mem_axi4, edge), i) => {
       val (port, ios) = IOCell.generateIOFromSignal(mem_axi4, Some(s"iocell_${name}_axi4_slave_${i}"))
       port.suggestName(s"${name}_axi4_slave_${i}")
       (port, edge, ios)
     }}
   }
   def axi4(io: Seq[AXI4Bundle], node: AXI4MasterNode, name: String): Seq[(AXI4Bundle, AXI4EdgeParameters, Seq[IOCell])] = {
-    io.zip(node.out).zipWithIndex.map{ case ((mem_axi4, (_, edge)), i) => {
+    io.zip(node.edges.out).zipWithIndex.map{ case ((mem_axi4, edge), i) => {
       //val (port, ios) = IOCell.generateIOFromSignal(mem_axi4, Some(s"iocell_${name}_axi4_master_${i}"))
       val port = IO(Flipped(AXI4Bundle(edge.bundle)))
       val ios = IOCell.generateFromSignal(mem_axi4, port, Some(s"iocell_${name}_axi4_master_${i}"))
diff --git a/generators/chipyard/src/main/scala/Subsystem.scala b/generators/chipyard/src/main/scala/Subsystem.scala
index 3ca5ab11..50714cc4 100644
--- a/generators/chipyard/src/main/scala/Subsystem.scala
+++ b/generators/chipyard/src/main/scala/Subsystem.scala
@@ -8,6 +8,7 @@ package chipyard
 import chisel3._
 import chisel3.internal.sourceinfo.{SourceInfo}
 
+import freechips.rocketchip.prci._
 import freechips.rocketchip.config.{Field, Parameters}
 import freechips.rocketchip.devices.tilelink._
 import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp, ExportDebug}
@@ -48,6 +49,10 @@ trait CanHaveHTIF { this: BaseSubsystem =>
 }
 
 
+// Controls whether tiles are driven by implicit subsystem clock, or by
+// diplomatic clock graph
+case object UseDiplomaticTileClocks extends Field[Boolean](false)
+
 class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
   with HasTiles
   with CanHaveHTIF
@@ -56,6 +61,19 @@ class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
     case r: RocketTile => r.module.core.rocketImpl.coreMonitorBundle
     case b: BoomTile => b.module.core.coreMonitorBundle
   }.toList
+
+  // TODO: In the future, RC tiles may extend ClockDomain. When that happens,
+  // we won't need to manually create this clock node and connect it to the
+  // tiles' implicit clocks
+
+  val tilesClockSinkNode = if (p(UseDiplomaticTileClocks)) {
+    val node = ClockSinkNode(List(ClockSinkParameters()))
+    node := ClockGroup()(p, ValName("chipyard_tiles")) := asyncClockGroupsNode
+    Some(node)
+  } else {
+    None
+  }
+
   override lazy val module = new ChipyardSubsystemModuleImp(this)
 }
 
@@ -64,11 +82,15 @@ class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends Bas
   with HasResetVectorWire
   with HasTilesModuleImp
 {
-
   for (i <- 0 until outer.tiles.size) {
     val wire = tile_inputs(i)
     wire.hartid := outer.hartIdList(i).U
     wire.reset_vector := global_reset_vector
+
+    outer.tilesClockSinkNode.map( n => {
+      outer.tiles(i).module.clock := n.in.head._1.clock
+      outer.tiles(i).module.reset := n.in.head._1.reset
+    })
   }
 
   // create file with core params
diff --git a/generators/chipyard/src/main/scala/TestHarness.scala b/generators/chipyard/src/main/scala/TestHarness.scala
index a82d3a33..9344cadf 100644
--- a/generators/chipyard/src/main/scala/TestHarness.scala
+++ b/generators/chipyard/src/main/scala/TestHarness.scala
@@ -8,10 +8,10 @@ import chipyard.iobinders.{TestHarnessFunction}
 import chipyard.config.ConfigValName._
 
 // -------------------------------
-// BOOM and/or Rocket Test Harness
+// Chipyard Test Harness
 // -------------------------------
 
-case object BuildTop extends Field[Parameters => HasTestHarnessFunctions]((p: Parameters) => Module(new ChipTop()(p)))
+case object BuildTop extends Field[Parameters => LazyModule with HasTestHarnessFunctions]((p: Parameters) => LazyModule(new ChipTop()(p)))
 
 trait HasTestHarnessFunctions {
   val harnessFunctions: Seq[TestHarnessFunction]
@@ -22,13 +22,14 @@ class TestHarness(implicit val p: Parameters) extends Module {
     val success = Output(Bool())
   })
 
-  val dut = p(BuildTop)(p)
+  val ldut = p(BuildTop)(p)
+  val dut = Module(ldut.module)
   io.success := false.B
 
   // dutReset assignment can be overridden via a harnessFunction, but by default it is just reset
   val dutReset = WireDefault(if (p(GlobalResetSchemeKey).pinIsAsync) reset.asAsyncReset else reset)
 
-  dut.harnessFunctions.foreach(_(this))
+  ldut.harnessFunctions.foreach(_(this))
 
   def success = io.success
   def harnessReset = this.reset.asBool
diff --git a/generators/chipyard/src/main/scala/config/AbstractConfig.scala b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
index 22f64925..2b9473ed 100644
--- a/generators/chipyard/src/main/scala/config/AbstractConfig.scala
+++ b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
@@ -17,6 +17,7 @@ class AbstractConfig extends Config(
   new chipyard.config.WithBootROM ++                             // use default bootrom
   new chipyard.config.WithUART ++                                // add a UART
   new chipyard.config.WithL2TLBs(1024) ++                        // use L2 TLBs
+  new chipyard.config.WithNoSubsystemDrivenClocks ++             // drive the subsystem diplomatic clocks from ChipTop instead of using implicit clocks
   new freechips.rocketchip.subsystem.WithNoMMIOPort ++           // no top-level MMIO master port (overrides default set in rocketchip)
   new freechips.rocketchip.subsystem.WithNoSlavePort ++          // no top-level MMIO slave port (overrides default set in rocketchip)
   new freechips.rocketchip.subsystem.WithInclusiveCache ++       // use Sifive L2 cache
diff --git a/generators/chipyard/src/main/scala/config/TracegenConfigs.scala b/generators/chipyard/src/main/scala/config/TracegenConfigs.scala
index e8aeeb29..47d567fb 100644
--- a/generators/chipyard/src/main/scala/config/TracegenConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/TracegenConfigs.scala
@@ -7,6 +7,7 @@ class TraceGenConfig extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 0, nSets = 16, nWays = 2) }) ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
@@ -15,6 +16,7 @@ class NonBlockingTraceGenConfig extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 2, nSets = 16, nWays = 2) }) ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
@@ -23,6 +25,7 @@ class BoomTraceGenConfig extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithBoomTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 8, nSets = 16, nWays = 2) }) ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
@@ -32,6 +35,7 @@ class NonBlockingTraceGenL2Config extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithL2TraceGen()(List.fill(2)(DCacheParams(nMSHRs = 2, nSets = 16, nWays = 4))) ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
@@ -41,6 +45,7 @@ class NonBlockingTraceGenL2RingConfig extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithL2TraceGen()(List.fill(2)(DCacheParams(nMSHRs = 2, nSets = 16, nWays = 4))) ++
   new testchipip.WithRingSystemBus ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
diff --git a/generators/chipyard/src/main/scala/config/TutorialConfigs.scala b/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
index 56e6362b..8872ed5e 100644
--- a/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
@@ -31,6 +31,7 @@ class TutorialStarterConfig extends Config(
   new testchipip.WithTSI ++                  // Add a TSI (Test Serial Interface)  widget to bring-up the core
   new chipyard.config.WithBootROM ++         // Use the Chipyard BootROM
   new chipyard.config.WithUART ++            // Add a UART
+  new chipyard.config.WithNoSubsystemDrivenClocks ++ // Don't drive the subsystem clocks from within the subsystem
 
   // CUSTOMIZE THE CORE
   // Uncomment out one (or multiple) of the lines below, and choose
@@ -65,6 +66,7 @@ class TutorialMMIOConfig extends Config(
   new testchipip.WithTSI ++
   new chipyard.config.WithBootROM ++
   new chipyard.config.WithUART ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
 
   // Attach either a TileLink or AXI4 version of GCD
   // Uncomment one of the below lines
@@ -92,6 +94,7 @@ class TutorialSha3Config extends Config(
   new testchipip.WithTSI ++
   new chipyard.config.WithBootROM ++
   new chipyard.config.WithUART ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
 
   // Uncomment this line once you added SHA3 to the build.sbt, and cloned the SHA3 repo
   // new sha3.WithSha3Accel ++
@@ -117,6 +120,7 @@ class TutorialSha3BlackBoxConfig extends Config(
   new testchipip.WithTSI ++
   new chipyard.config.WithBootROM ++
   new chipyard.config.WithUART ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
 
   // Uncomment these lines once SHA3 is integrated
   // new sha3.WithSha3BlackBox ++ // Specify we want the Black-box verilog version of Sha3 Ctrl

From 56e1aeb4007b2b2d252c2296fd2095ecf11320ec Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 7 Jul 2020 17:18:10 -0700
Subject: [PATCH 02/74] Support FireSim diplomatic multiclock

---
 .../chipyard/src/main/scala/ChipTop.scala     |  23 +--
 .../chipyard/src/main/scala/Clocks.scala      |  86 +++++++---
 .../src/main/scala/ConfigFragments.scala      |  18 +--
 .../chipyard/src/main/scala/IOBinders.scala   |  51 +++---
 .../chipyard/src/main/scala/Subsystem.scala   |  54 ++-----
 .../chipyard/src/main/scala/TestHarness.scala |  21 ++-
 .../src/main/scala/config/RocketConfigs.scala |   8 +
 .../src/main/scala/BridgeBinders.scala        |   2 +-
 .../firechip/src/main/scala/FireSim.scala     | 147 ++++++++++++++----
 .../src/main/scala/FireSimMulticlockPOC.scala | 107 -------------
 .../src/main/scala/TargetConfigs.scala        |  12 +-
 .../src/test/scala/ScalaTestSuite.scala       |   4 +-
 12 files changed, 273 insertions(+), 260 deletions(-)
 delete mode 100644 generators/firechip/src/main/scala/FireSimMulticlockPOC.scala

diff --git a/generators/chipyard/src/main/scala/ChipTop.scala b/generators/chipyard/src/main/scala/ChipTop.scala
index 0227feb8..ff3e2aed 100644
--- a/generators/chipyard/src/main/scala/ChipTop.scala
+++ b/generators/chipyard/src/main/scala/ChipTop.scala
@@ -9,12 +9,11 @@ import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGr
 import freechips.rocketchip.config.{Parameters, Field}
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, LazyRawModuleImp, LazyModuleImpLike}
 import freechips.rocketchip.util.{ResetCatchAndSync}
-import chipyard.config.ConfigValName._
 import chipyard.iobinders.{IOBinders, TestHarnessFunction, IOBinderTuple}
 
 import barstools.iocell.chisel._
 
-case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => LazyModule(new DigitalTop()(p)))
+case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => new DigitalTop()(p))
 
 
 /**
@@ -31,23 +30,15 @@ class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunc
   val harnessFunctions = ArrayBuffer.empty[TestHarnessFunction]
 
   // The system module specified by BuildSystem
-  val lSystem = p(BuildSystem)(p).suggestName("system")
+  val lSystem = LazyModule(p(BuildSystem)(p)).suggestName("system")
 
   // The systemClockSinkNode provides the implicit clock and reset for the System
-  private val systemClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
+  val systemClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
+  val systemClockGroup = LazyModule(new ClockGroup("system_clock"))
+  systemClockSinkNode := systemClockGroup.node
 
-  // clockGroupNode provides a single node which aggregates all clock groups in the design
-  val clockGroupNode = ClockGroupIdentityNode()
-
-  // If the specified system has diplomatic clocks, connect it to our clockGroupNode
-  if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
-    lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode :*= clockGroupNode }
-  }
-  // Connect the system implicit clock node to the clockGroupNode
-  systemClockSinkNode := ClockGroup() := clockGroupNode
-
-  // Drive the entire diplomatic clock network using this configured Key
-  clockGroupNode :*=* p(ChipyardClockKey)(this)
+  // Generate Clocks and Reset
+  p(ChipyardClockKey)(this)
 
   // NOTE: Making this a LazyRawModule is moderately dangerous, as anonymous children
   // of ChipTop (ex: ClockGroup) do not receive clock or reset.
diff --git a/generators/chipyard/src/main/scala/Clocks.scala b/generators/chipyard/src/main/scala/Clocks.scala
index b9508d96..9c71ed96 100644
--- a/generators/chipyard/src/main/scala/Clocks.scala
+++ b/generators/chipyard/src/main/scala/Clocks.scala
@@ -5,19 +5,13 @@ import chisel3._
 import scala.collection.mutable.{ArrayBuffer}
 
 import freechips.rocketchip.prci._
-import freechips.rocketchip.subsystem.{BaseSubsystem}
+import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
 import freechips.rocketchip.config.{Parameters, Field}
-import freechips.rocketchip.diplomacy.{OutwardNodeHandle, InModuleBody}
-import freechips.rocketchip.util.{ResetCatchAndSync}
-import chipyard.config.ConfigValName._
+import freechips.rocketchip.diplomacy.{OutwardNodeHandle, InModuleBody, LazyModule}
+import freechips.rocketchip.util.{ResetCatchAndSync, Pow2ClockDivider}
 
 import barstools.iocell.chisel._
 
-import ChipyardClockDrivers._
-
-case object ChipyardClockKey extends Field[ClockInstantiationFn](simpleTestHarnessClock)
-
-
 /**
   * Chipyard provides three baseline, top-level reset schemes, set using the
   * [[GlobalResetSchemeKey]] in a Parameters instance. These are:
@@ -75,7 +69,7 @@ object GenerateReset {
     }
     reset_io.suggestName("reset")
     chiptop.iocells ++= resetIOCell
-    chiptop.harnessFunctions += ((th: TestHarness) => {
+    chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
       reset_io := th.dutReset
       Nil
     })
@@ -83,33 +77,89 @@ object GenerateReset {
   }
 }
 
-object ChipyardClockDrivers {
-  type ClockInstantiationFn = ChipTop => OutwardNodeHandle[ClockGroupSourceParameters, ClockGroupSinkParameters, ClockGroupEdgeParameters, ClockGroupBundle]
 
+case object ChipyardClockKey extends Field[ChipTop => Unit](ClockDrivers.harnessClock)
+
+
+
+object ClockDrivers {
   // A simple clock provider, for testing. All clocks in system are aggregated into one,
   // and are driven by directly punching out to the TestHarness clock
-  val simpleTestHarnessClock: ClockInstantiationFn = { chiptop =>
+  val harnessClock: ChipTop => Unit = { chiptop =>
     implicit val p = chiptop.p
     val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+    val clockAggregator = LazyModule(new ClockGroupAggregator("clocks"))
+
+    // Aggregate all 3 possible clock groups with the clockAggregator
+    chiptop.systemClockGroup.node := clockAggregator.node
+    if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
+      chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := clockAggregator.node }
+    }
+    chiptop.lSystem match {
+      case l: ChipyardSubsystem => l.tileClockGroupNode := clockAggregator.node
+      case _ =>
+    }
+
+
+    clockAggregator.node := simpleClockGroupSourceNode
     InModuleBody {
       // this needs directionality so generateIOFromSignal works
       val clock_wire = Wire(Input(Clock()))
       val reset_wire = GenerateReset(chiptop, clock_wire)
       val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, Some("iocell_clock"))
       chiptop.iocells ++= clockIOCell
-
       clock_io.suggestName("clock")
 
       simpleClockGroupSourceNode.out.unzip._1.flatMap(_.member).map { o =>
         o.clock := clock_wire
         o.reset := reset_wire
       }
-
-      chiptop.harnessFunctions += ((th: TestHarness) => {
-        clock_io := th.clock
+      chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
+        clock_io := th.harnessClock
         Nil
       })
     }
-    ClockGroupAggregator() := simpleClockGroupSourceNode
+  }
+
+  val harnessMultiClock: ChipTop => Unit = { chiptop =>
+    implicit val p = chiptop.p
+    val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters(), ClockGroupSourceParameters()))
+    val uncoreClockAggregator = LazyModule(new ClockGroupAggregator("uncore_clocks"))
+
+    // Aggregate only the uncoreclocks
+    chiptop.systemClockGroup.node := uncoreClockAggregator.node
+    if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
+      chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := uncoreClockAggregator.node }
+    }
+
+    uncoreClockAggregator.node := simpleClockGroupSourceNode
+    chiptop.lSystem match {
+      case l: ChipyardSubsystem => l.tileClockGroupNode := simpleClockGroupSourceNode
+      case _ => throw new Exception("MultiClock assumes ChipyardSystem")
+    }
+
+    InModuleBody {
+      // this needs directionality so generateIOFromSignal works
+      val clock_wire = Wire(Input(Clock()))
+      val reset_wire = GenerateReset(chiptop, clock_wire)
+      val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, Some("iocell_clock"))
+      chiptop.iocells ++= clockIOCell
+      clock_io.suggestName("clock")
+      val div_clock = Pow2ClockDivider(clock_wire, 2)
+
+      simpleClockGroupSourceNode.out(0)._1.member.map { o =>
+        o.clock := div_clock
+        o.reset := ResetCatchAndSync(div_clock, reset_wire.asBool)
+      }
+      simpleClockGroupSourceNode.out(1)._1.member.map { o =>
+        o.clock := clock_wire
+        o.reset := reset_wire
+      }
+      chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
+        clock_io := th.harnessClock
+        Nil
+      })
+    }
+
   }
 }
diff --git a/generators/chipyard/src/main/scala/ConfigFragments.scala b/generators/chipyard/src/main/scala/ConfigFragments.scala
index ac70f8e5..60ccc999 100644
--- a/generators/chipyard/src/main/scala/ConfigFragments.scala
+++ b/generators/chipyard/src/main/scala/ConfigFragments.scala
@@ -25,15 +25,8 @@ import sifive.blocks.devices.gpio._
 import sifive.blocks.devices.uart._
 import sifive.blocks.devices.spi._
 
-import chipyard.{BuildTop, BuildSystem}
+import chipyard.{BuildTop, BuildSystem, ClockDrivers, ChipyardClockKey}
 
-/**
- * TODO: Why do we need this?
- */
-object ConfigValName {
-  implicit val valName = ValName("TestHarness")
-}
-import ConfigValName._
 
 // -----------------------
 // Common Config Fragments
@@ -73,7 +66,7 @@ class WithL2TLBs(entries: Int) extends Config((site, here, up) => {
 })
 
 class WithTracegenSystem extends Config((site, here, up) => {
-  case BuildSystem => (p: Parameters) => LazyModule(new TraceGenSystem()(p))
+  case BuildSystem => (p: Parameters) => new TraceGenSystem()(p)
 })
 
 /**
@@ -103,7 +96,8 @@ class WithMultiRoCCHwacha(harts: Int*) extends Config((site, here, up) => {
   case MultiRoCCKey => {
     up(MultiRoCCKey, site) ++ harts.distinct.map{ i =>
       (i -> Seq((p: Parameters) => {
-        LazyModule(new Hwacha()(p)).suggestName("hwacha")
+        val hwacha = LazyModule(new Hwacha()(p))
+        hwacha
       }))
     }
   }
@@ -150,3 +144,7 @@ class WithRocketDCacheScratchpad extends Config((site, here, up) => {
 class WithNoSubsystemDrivenClocks extends Config((site, here, up) => {
   case SubsystemDriveAsyncClockGroupsKey => None
 })
+
+class WithTileMultiClock extends Config((site, here, up) => {
+  case ChipyardClockKey => ClockDrivers.harnessMultiClock
+})
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 3f986757..84ef5269 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -41,7 +41,7 @@ import scala.reflect.{ClassTag}
 
 // DOC include start: IOBinders
 // This type describes a function callable on the TestHarness instance. Its return type is unused.
-type TestHarnessFunction = (chipyard.TestHarness) => Seq[Any]
+type TestHarnessFunction = (chipyard.HasHarnessUtils) => Seq[Any]
 // IOBinders will return a Seq of this tuple, which contains three fields:
 //  1. A Seq containing all IO ports created by the IOBinder function
 //  2. A Seq containing all IO cell modules created by the IOBinder function
@@ -228,7 +228,7 @@ object AddIOCells {
 class WithGPIOTiedOff extends OverrideIOBinder({
   (system: HasPeripheryGPIOModuleImp) => {
     val (ports2d, ioCells2d) = AddIOCells.gpio(system.gpio)
-    val harnessFn = (th: chipyard.TestHarness) => { ports2d.flatten.foreach(_ <> AnalogConst(0)); Nil }
+    val harnessFn = (th: HasHarnessUtils) => { ports2d.flatten.foreach(_ <> AnalogConst(0)); Nil }
     Seq((ports2d.flatten, ioCells2d.flatten, Some(harnessFn)))
   }
 })
@@ -237,7 +237,7 @@ class WithGPIOTiedOff extends OverrideIOBinder({
 class WithUARTAdapter extends OverrideIOBinder({
   (system: HasPeripheryUARTModuleImp) => {
     val (ports, ioCells2d) = AddIOCells.uart(system.uart)
-    val harnessFn = (th: chipyard.TestHarness) => { UARTAdapter.connect(ports)(system.p); Nil }
+    val harnessFn = (th: HasHarnessUtils) => { UARTAdapter.connect(ports)(system.p); Nil }
     Seq((ports, ioCells2d.flatten, Some(harnessFn)))
   }
 })
@@ -245,7 +245,7 @@ class WithUARTAdapter extends OverrideIOBinder({
 class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideIOBinder({
   (system: HasPeripherySPIFlashModuleImp) => {
     val (ports, ioCells2d) = AddIOCells.spi(system.qspi, "qspi")
-    val harnessFn = (th: chipyard.TestHarness) => { SimSPIFlashModel.connect(ports, th.reset, rdOnly)(system.p); Nil }
+    val harnessFn = (th: HasHarnessUtils) => { SimSPIFlashModel.connect(ports, th.harnessReset, rdOnly)(system.p); Nil }
     Seq((ports, ioCells2d.flatten, Some(harnessFn)))
   }
 })
@@ -253,8 +253,9 @@ class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideIOBinder({
 class WithSimBlockDevice extends OverrideIOBinder({
   (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev =>
     val (port, ios) = AddIOCells.blockDev(bdev)
-    val harnessFn = (th: chipyard.TestHarness) => {
-      SimBlockDevice.connect(th.clock, th.reset.asBool, Some(port))(system.p)
+    val harnessFn = (th: HasHarnessUtils) => {
+      // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock
+      SimBlockDevice.connect(th.harnessClock, th.harnessReset.asBool, Some(port))(system.p)
       Nil
     }
     Seq((Seq(port), ios, Some(harnessFn)))
@@ -264,7 +265,7 @@ class WithSimBlockDevice extends OverrideIOBinder({
 class WithBlockDeviceModel extends OverrideIOBinder({
   (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev =>
     val (port, ios) = AddIOCells.blockDev(bdev)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessUtils) => {
       BlockDeviceModel.connect(Some(port))(system.p)
       Nil
     }
@@ -287,7 +288,7 @@ class WithSimAXIMem extends OverrideIOBinder({
     val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem")
     // TODO: we are inlining the connectMem method of SimAXIMem because
     //   it takes in a dut rather than seq of axi4 ports
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessUtils) => {
       peiTuples.map { case (port, edge, ios) =>
         val mem = LazyModule(new SimAXIMem(edge, size = p(ExtMem).get.master.size))
         Module(mem.module).suggestName("mem")
@@ -304,14 +305,15 @@ class WithBlackBoxSimMem extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {
     implicit val p: Parameters = GetSystemParameters(system)
     val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem")
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessUtils) => {
       peiTuples.map { case (port, edge, ios) =>
         val memSize = p(ExtMem).get.master.size
         val lineSize = p(CacheBlockBytes)
         val mem = Module(new SimDRAM(memSize, lineSize, edge.bundle))
         mem.io.axi <> port
-        mem.io.clock := th.clock
-        mem.io.reset := th.reset
+        // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock
+        mem.io.clock := th.harnessClock
+        mem.io.reset := th.harnessReset
       }
       Nil
     }
@@ -323,7 +325,7 @@ class WithSimAXIMMIO extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MMIOPort) => {
     implicit val p: Parameters = GetSystemParameters(system)
     val peiTuples = AddIOCells.axi4(system.mmio_axi4, system.mmioAXI4Node, "mmio_mem")
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessUtils) => {
       peiTuples.zipWithIndex.map { case ((port, edge, ios), i) =>
         val mmio_mem = LazyModule(new SimAXIMem(edge, size = 4096))
         Module(mmio_mem.module).suggestName(s"mmio_mem_${i}")
@@ -343,7 +345,7 @@ class WithTieOffInterrupts extends OverrideIOBinder({
   (system: HasExtInterruptsModuleImp) => {
     val (port, ioCells) = IOCell.generateIOFromSignal(system.interrupts, Some("iocell_interrupts"))
     port.suggestName("interrupts")
-    val harnessFn = (th: chipyard.TestHarness) => { port := 0.U; Nil }
+    val harnessFn = (th: HasHarnessUtils) => { port := 0.U; Nil }
     Seq((Seq(port), ioCells, Some(harnessFn)))
   }
 })
@@ -351,7 +353,7 @@ class WithTieOffInterrupts extends OverrideIOBinder({
 class WithTieOffL2FBusAXI extends OverrideIOBinder({
   (system: CanHaveSlaveAXI4Port) => {
     val peiTuples = AddIOCells.axi4(system.l2_frontend_bus_axi4, system.l2FrontendAXI4Node, "l2_fbus")
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessUtils) => {
       peiTuples.zipWithIndex.map { case ((port, edge, ios), i) =>
         port := DontCare // tieoff doesn't completely tie-off, for some reason
         port.tieoff()
@@ -366,13 +368,14 @@ class WithTiedOffDebug extends OverrideIOBinder({
   (system: HasPeripheryDebugModuleImp) => {
     val (psdPort, resetctrlOpt, debugPortOpt, ioCells) =
       AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessUtils) => {
       Debug.tieoffDebug(debugPortOpt, resetctrlOpt, Some(psdPort))(system.p)
       // tieoffDebug doesn't actually tie everything off :/
       debugPortOpt.foreach { d =>
-        d.clockeddmi.foreach({ cdmi => cdmi.dmi.req.bits := DontCare; cdmi.dmiClock := th.clock })
+        // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock
+        d.clockeddmi.foreach({ cdmi => cdmi.dmi.req.bits := DontCare; cdmi.dmiClock := th.harnessClock })
         d.dmactiveAck := DontCare
-        d.clock := th.clock
+        d.clock := th.harnessClock
       }
       Nil
     }
@@ -384,11 +387,11 @@ class WithSimDebug extends OverrideIOBinder({
   (system: HasPeripheryDebugModuleImp) => {
     val (psdPort, resetctrlPortOpt, debugPortOpt, ioCells) =
       AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessUtils) => {
       val dtm_success = Wire(Bool())
-      Debug.connectDebug(debugPortOpt, resetctrlPortOpt, psdPort, th.clock, th.harnessReset, dtm_success)(system.p)
+      Debug.connectDebug(debugPortOpt, resetctrlPortOpt, psdPort, th.harnessClock, th.harnessReset.asBool, dtm_success)(system.p)
       when (dtm_success) { th.success := true.B }
-      th.dutReset := th.harnessReset | debugPortOpt.map { debug => AsyncResetReg(debug.ndreset).asBool }.getOrElse(false.B)
+      th.dutReset := th.harnessReset.asBool | debugPortOpt.map { debug => AsyncResetReg(debug.ndreset).asBool }.getOrElse(false.B)
       Nil
     }
     Seq((Seq(psdPort) ++ debugPortOpt.toSeq, ioCells, Some(harnessFn)))
@@ -398,7 +401,7 @@ class WithSimDebug extends OverrideIOBinder({
 class WithTiedOffSerial extends OverrideIOBinder({
   (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial =>
     val (port, ioCells) = AddIOCells.serial(serial)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessUtils) => {
       SerialAdapter.tieoff(port)
       Nil
     }
@@ -409,8 +412,8 @@ class WithTiedOffSerial extends OverrideIOBinder({
 class WithSimSerial extends OverrideIOBinder({
   (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial =>
     val (port, ioCells) = AddIOCells.serial(serial)
-    val harnessFn = (th: chipyard.TestHarness) => {
-      val ser_success = SerialAdapter.connectSimSerial(port, th.clock, th.harnessReset)
+    val harnessFn = (th: HasHarnessUtils) => {
+      val ser_success = SerialAdapter.connectSimSerial(port, th.harnessClock, th.harnessReset)
       when (ser_success) { th.success := true.B }
       Nil
     }
@@ -422,7 +425,7 @@ class WithTraceGenSuccessBinder extends OverrideIOBinder({
   (system: TraceGenSystemModuleImp) => {
     val (successPort, ioCells) = IOCell.generateIOFromSignal(system.success, Some("iocell_success"))
     successPort.suggestName("success")
-    val harnessFn = (th: chipyard.TestHarness) => { when (successPort) { th.success := true.B }; Nil }
+    val harnessFn = (th: HasHarnessUtils) => { when (successPort) { th.success := true.B }; Nil }
     Seq((Seq(successPort), ioCells, Some(harnessFn)))
   }
 })
diff --git a/generators/chipyard/src/main/scala/Subsystem.scala b/generators/chipyard/src/main/scala/Subsystem.scala
index 50714cc4..65ceddc9 100644
--- a/generators/chipyard/src/main/scala/Subsystem.scala
+++ b/generators/chipyard/src/main/scala/Subsystem.scala
@@ -11,7 +11,7 @@ import chisel3.internal.sourceinfo.{SourceInfo}
 import freechips.rocketchip.prci._
 import freechips.rocketchip.config.{Field, Parameters}
 import freechips.rocketchip.devices.tilelink._
-import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp, ExportDebug}
+import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp}
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.diplomaticobjectmodel.model.{OMInterrupt}
 import freechips.rocketchip.diplomaticobjectmodel.logicaltree.{RocketTileLogicalTreeNode, LogicalModuleTree}
@@ -25,59 +25,24 @@ import freechips.rocketchip.amba.axi4._
 import boom.common.{BoomTile}
 
 
-import testchipip.{DromajoHelper, CanHavePeripherySerial, SerialKey}
-
-
-trait CanHaveHTIF { this: BaseSubsystem =>
-  // Advertise HTIF if system can communicate with fesvr
-  if (this match {
-    case _: CanHavePeripherySerial if p(SerialKey) => true
-    case _: HasPeripheryDebug if p(ExportDebug).protocols.nonEmpty => true
-    case _ => false
-  }) {
-    ResourceBinding {
-      val htif = new Device {
-        def describe(resources: ResourceBindings): Description = {
-          val compat = resources("compat").map(_.value)
-          Description("htif", Map(
-            "compatible" -> compat))
-        }
-      }
-      Resource(htif, "compat").bind(ResourceString("ucb,htif0"))
-    }
-  }
-}
-
-
-// Controls whether tiles are driven by implicit subsystem clock, or by
-// diplomatic clock graph
-case object UseDiplomaticTileClocks extends Field[Boolean](false)
+import testchipip.{DromajoHelper}
 
 class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
   with HasTiles
-  with CanHaveHTIF
 {
   def coreMonitorBundles = tiles.map {
     case r: RocketTile => r.module.core.rocketImpl.coreMonitorBundle
     case b: BoomTile => b.module.core.coreMonitorBundle
   }.toList
 
-  // TODO: In the future, RC tiles may extend ClockDomain. When that happens,
-  // we won't need to manually create this clock node and connect it to the
-  // tiles' implicit clocks
-
-  val tilesClockSinkNode = if (p(UseDiplomaticTileClocks)) {
-    val node = ClockSinkNode(List(ClockSinkParameters()))
-    node := ClockGroup()(p, ValName("chipyard_tiles")) := asyncClockGroupsNode
-    Some(node)
-  } else {
-    None
-  }
+  val tileClockSinkNode = ClockSinkNode(List(ClockSinkParameters()))
+  val tileClockGroup = LazyModule(new ClockGroup("tile_clock"))
+  val tileClockGroupNode = tileClockGroup.node
+  tileClockSinkNode := tileClockGroupNode
 
   override lazy val module = new ChipyardSubsystemModuleImp(this)
 }
 
-
 class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
   with HasResetVectorWire
   with HasTilesModuleImp
@@ -87,10 +52,9 @@ class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends Bas
     wire.hartid := outer.hartIdList(i).U
     wire.reset_vector := global_reset_vector
 
-    outer.tilesClockSinkNode.map( n => {
-      outer.tiles(i).module.clock := n.in.head._1.clock
-      outer.tiles(i).module.reset := n.in.head._1.reset
-    })
+
+    outer.tiles(i).module.clock := outer.tileClockSinkNode.in.head._1.clock
+    outer.tiles(i).module.reset := outer.tileClockSinkNode.in.head._1.reset
   }
 
   // create file with core params
diff --git a/generators/chipyard/src/main/scala/TestHarness.scala b/generators/chipyard/src/main/scala/TestHarness.scala
index 9344cadf..92b1dd29 100644
--- a/generators/chipyard/src/main/scala/TestHarness.scala
+++ b/generators/chipyard/src/main/scala/TestHarness.scala
@@ -5,34 +5,41 @@ import chisel3._
 import freechips.rocketchip.diplomacy.{LazyModule}
 import freechips.rocketchip.config.{Field, Parameters}
 import chipyard.iobinders.{TestHarnessFunction}
-import chipyard.config.ConfigValName._
 
 // -------------------------------
 // Chipyard Test Harness
 // -------------------------------
 
-case object BuildTop extends Field[Parameters => LazyModule with HasTestHarnessFunctions]((p: Parameters) => LazyModule(new ChipTop()(p)))
+case object BuildTop extends Field[Parameters => LazyModule with HasTestHarnessFunctions]((p: Parameters) => new ChipTop()(p))
 
 trait HasTestHarnessFunctions {
   val harnessFunctions: Seq[TestHarnessFunction]
 }
 
-class TestHarness(implicit val p: Parameters) extends Module {
+trait HasHarnessUtils {
+  val harnessClock: Clock
+  val harnessReset: Reset
+  val dutReset: Reset
+  val success: Bool
+}
+
+class TestHarness(implicit val p: Parameters) extends Module with HasHarnessUtils {
   val io = IO(new Bundle {
     val success = Output(Bool())
   })
 
-  val ldut = p(BuildTop)(p)
+  val ldut = LazyModule(p(BuildTop)(p)).suggestName("ChipTop")
   val dut = Module(ldut.module)
   io.success := false.B
 
+  val harnessClock = clock
+  val harnessReset = WireInit(reset)
+  val success = io.success
+
   // dutReset assignment can be overridden via a harnessFunction, but by default it is just reset
   val dutReset = WireDefault(if (p(GlobalResetSchemeKey).pinIsAsync) reset.asAsyncReset else reset)
 
   ldut.harnessFunctions.foreach(_(this))
 
-  def success = io.success
-  def harnessReset = this.reset.asBool
-
 }
 
diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
index 8e6e4867..af0b06b7 100644
--- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
@@ -183,3 +183,11 @@ class MMIORocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithDefaultSlavePort ++ // add default external slave port
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
+
+// NOTE: This config doesn't work yet because SimWidgets in the TestHarness
+// always get the TestHarness clock. The Tiles and Uncore receive the correct clocks
+class MultiClockRocketConfig extends Config(
+  new chipyard.config.WithTileMultiClock ++                       // Put the Tile on its own clock domain
+  new freechips.rocketchip.subsystem.WithRationalRocketTiles ++   // Add rational crossings between RocketTile and uncore
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)
diff --git a/generators/firechip/src/main/scala/BridgeBinders.scala b/generators/firechip/src/main/scala/BridgeBinders.scala
index eba57451..b59d477d 100644
--- a/generators/firechip/src/main/scala/BridgeBinders.scala
+++ b/generators/firechip/src/main/scala/BridgeBinders.scala
@@ -58,7 +58,7 @@ class WithBlockDeviceBridge extends OverrideIOBinder({
 class WithFASEDBridge extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    (system.mem_axi4 zip system.memAXI4Node.in).foreach({ case (axi4, (_, edge)) =>
+    (system.mem_axi4 zip system.memAXI4Node.edges.in).foreach({ case (axi4, edge) =>
       val nastiKey = NastiParameters(axi4.r.bits.data.getWidth,
                                      axi4.ar.bits.addr.getWidth,
                                      axi4.ar.bits.id.getWidth)
diff --git a/generators/firechip/src/main/scala/FireSim.scala b/generators/firechip/src/main/scala/FireSim.scala
index a4cea5ec..d35dd7b6 100644
--- a/generators/firechip/src/main/scala/FireSim.scala
+++ b/generators/firechip/src/main/scala/FireSim.scala
@@ -3,13 +3,17 @@
 package firesim.firesim
 
 import chisel3._
+import chisel3.experimental.{IO}
 
+import freechips.rocketchip.prci._
+import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
 import freechips.rocketchip.config.{Field, Config, Parameters}
-import freechips.rocketchip.diplomacy.{LazyModule}
+import freechips.rocketchip.diplomacy.{LazyModule, InModuleBody}
+import freechips.rocketchip.util.{ResetCatchAndSync}
 
-import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge}
+import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge, RationalClock}
 
-import chipyard.{BuildSystem}
+import chipyard.{BuildSystem, BuildTop, HasHarnessUtils, ChipyardSubsystem, ChipyardClockKey, ChipTop}
 import chipyard.iobinders.{IOBinders}
 
 // Determines the number of times to instantiate the DUT in the harness.
@@ -20,6 +24,16 @@ class WithNumNodes(n: Int) extends Config((pname, site, here) => {
   case NumNodes => n
 })
 
+// Note, the main prerequisite for supporting an additional clock domain in a
+// FireSim simulation is to supply an additional clock parameter
+// (RationalClock) to the clock bridge (RationalClockBridge). The bridge
+// produces a vector of clocks, based on the provided parameter list, which you
+// may use freely without further modifications to your target design.
+case class FireSimClockParameters(additionalClocks: Seq[RationalClock]) {
+  def numClocks(): Int = additionalClocks.size + 1
+}
+case object FireSimClockKey extends Field[FireSimClockParameters](FireSimClockParameters(Seq()))
+
 // Hacky: Set before each node is generated. Ideally we'd give IO binders
 // accesses to the the Harness's parameters instance. We could then alter that.
 object NodeIdx {
@@ -28,33 +42,108 @@ object NodeIdx {
   def apply(): Int = idx
 }
 
-class FireSim(implicit val p: Parameters) extends RawModule {
-  freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary())
-  val clockBridge = Module(new RationalClockBridge)
-  val clock = clockBridge.io.clocks.head
-  val reset = WireInit(false.B)
-  withClockAndReset(clock, reset) {
-    // Instantiate multiple instances of the DUT to implement supernode
-    val targets = Seq.fill(p(NumNodes)) {
-      // It's not a RC bump without some hacks...
-      // Copy the AsyncClockGroupsKey to generate a fresh node on each
-      // instantiation of the dut, otherwise the initial instance will be
-      // reused across each node
-      import freechips.rocketchip.subsystem.AsyncClockGroupsKey
-      val lazyModule = p(BuildSystem)(p.alterPartial({
-        case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy
-      }))
-      (lazyModule, Module(lazyModule.module))
-    }
+class WithFireSimSimpleClocks extends Config((site, here, up) => {
+  case ChipyardClockKey => { chiptop: ChipTop =>
+    implicit val p = chiptop.p
+    val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+    val clockAggregator = LazyModule(new ClockGroupAggregator("clocks"))
 
-    val peekPokeBridge = PeekPokeBridge(clock, reset)
-    // A Seq of partial functions that will instantiate the right bridge only
-    // if that Mixin trait is present in the target's LazyModule class instance
-    //
-    // Apply each partial function to each DUT instance
-    for ((lazyModule, module) <- targets) {
-      p(IOBinders).values.foreach(f => f(lazyModule) ++ f(module))
-      NodeIdx.increment()
+    // Aggregate all 3 possible clock groups with the clockAggregator
+    chiptop.systemClockGroup.node := clockAggregator.node
+    if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
+      chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := clockAggregator.node }
+    }
+    chiptop.lSystem match { case l: ChipyardSubsystem => l.tileClockGroupNode := clockAggregator.node }
+
+    clockAggregator.node := simpleClockGroupSourceNode
+    InModuleBody {
+      val clock      = IO(Input(Clock())).suggestName("clock")
+      val reset      = IO(Input(Reset())).suggestName("reset")
+
+      simpleClockGroupSourceNode.out.unzip._1.flatMap(_.member).map { o =>
+        o.clock := clock
+        o.reset := reset
+      }
+
+      chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
+        clock := th.harnessClock
+        reset := th.harnessReset
+        Nil
+      })
     }
   }
+})
+
+class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Config((site, here, up) => {
+  case FireSimClockKey => FireSimClockParameters(Seq(RationalClock("TileDomain", multiplier, divisor)))
+  case ChipyardClockKey => { chiptop: ChipTop =>
+    implicit val p = chiptop.p
+    val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters(), ClockGroupSourceParameters()))
+    val uncoreClockAggregator = LazyModule(new ClockGroupAggregator("uncore_clocks"))
+
+    // Aggregate only the uncoreclocks
+    chiptop.systemClockGroup.node := uncoreClockAggregator.node
+    if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
+      chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := uncoreClockAggregator.node }
+    }
+
+    uncoreClockAggregator.node := simpleClockGroupSourceNode
+    chiptop.lSystem match {
+      case l: ChipyardSubsystem => l.tileClockGroupNode := simpleClockGroupSourceNode
+      case _ => throw new Exception("MultiClock assumes ChipyardSystem")
+    }
+
+    InModuleBody {
+      val uncore_clock = IO(Input(Clock())).suggestName("uncore_clock")
+      val tile_clock   = IO(Input(Clock())).suggestName("tile_clock")
+      val reset        = IO(Input(Reset())).suggestName("reset")
+
+      simpleClockGroupSourceNode.out(0)._1.member.map { o =>
+        o.clock := uncore_clock
+        o.reset := reset
+      }
+
+      simpleClockGroupSourceNode.out(1)._1.member.map { o =>
+        o.clock := tile_clock
+        o.reset := ResetCatchAndSync(tile_clock, reset.asBool)
+      }
+
+      chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
+        uncore_clock := th.harnessClock
+        reset        := th.harnessReset
+        th match {
+          case f: FireSim => tile_clock := f.additionalClocks(0)
+          case _ => throw new Exception("FireSimMultiClock must be used with FireSim")
+        }
+        Nil
+      })
+    }
+  }
+})
+
+class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessUtils {
+  freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary())
+  val clockBridge = Module(new RationalClockBridge(p(FireSimClockKey).additionalClocks:_*))
+  val harnessClock = clockBridge.io.clocks.head // This is the reference clock
+  val additionalClocks = clockBridge.io.clocks.tail
+  val harnessReset = WireInit(false.B)
+  val peekPokeBridge = PeekPokeBridge(harnessClock, harnessReset)
+  val dutReset = false.B // unused (if used, its a bug)
+  val success = false.B // unused (if used, its a bug)
+
+  // Instantiate multiple instances of the DUT to implement supernode
+  for (i <- 0 until p(NumNodes)) {
+    // It's not a RC bump without some hacks...
+    // Copy the AsyncClockGroupsKey to generate a fresh node on each
+    // instantiation of the dut, otherwise the initial instance will be
+    // reused across each node
+    import freechips.rocketchip.subsystem.AsyncClockGroupsKey
+    val lazyModule = LazyModule(p(BuildTop)(p.alterPartial({
+      case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy
+    })))
+    val module = Module(lazyModule.module)
+    require(lazyModule.harnessFunctions.size == 1, "There should only be 1 harness function to connect clock+reset")
+    lazyModule.harnessFunctions.foreach(_(this))
+    NodeIdx.increment()
+  }
 }
diff --git a/generators/firechip/src/main/scala/FireSimMulticlockPOC.scala b/generators/firechip/src/main/scala/FireSimMulticlockPOC.scala
deleted file mode 100644
index bf0e0e26..00000000
--- a/generators/firechip/src/main/scala/FireSimMulticlockPOC.scala
+++ /dev/null
@@ -1,107 +0,0 @@
-//See LICENSE for license details.
-
-package firesim.firesim
-
-import chisel3._
-
-import freechips.rocketchip.config.{Field, Config, Parameters}
-import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, RationalCrossing}
-import freechips.rocketchip.subsystem._
-import freechips.rocketchip.util.{ResetCatchAndSync}
-
-import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge, RationalClock}
-import firesim.configs._
-
-import boom.common.{WithRationalBoomTiles}
-
-import chipyard.{BuildSystem, DigitalTop, DigitalTopModule}
-import chipyard.config.ConfigValName._
-import chipyard.iobinders.{IOBinders}
-
-// WIP! This file is a sketch of one means of defining a multiclock target-design
-// that can be simulated in FireSim, pending a canonicalized form in Chipyard.
-//
-// Note, the main prerequisite for supporting an additional clock domain in a
-// FireSim simulation is to supply an additional clock parameter
-// (RationalClock) to the clock bridge (RationalClockBridge). The bridge
-// produces a vector of clocks, based on the provided parameter list, which you
-// may use freely without further modifications to your target design.
-
-case class FireSimClockParameters(additionalClocks: Seq[RationalClock]) {
-  def numClocks(): Int = additionalClocks.size + 1
-}
-case object FireSimClockKey extends Field[FireSimClockParameters](FireSimClockParameters(Seq()))
-
-trait HasAdditionalClocks extends LazyModuleImp {
-  val clocks = IO(Vec(p(FireSimClockKey).numClocks, Input(Clock())))
-}
-
-// Presupposes only 1 or 2 clocks.
-trait HasFireSimClockingImp extends HasAdditionalClocks {
-  val outer: HasTiles
-  val (tileClock, tileReset) = p(FireSimClockKey).additionalClocks.headOption match {
-    case Some(RationalClock(_, numer, denom)) if numer != denom => (clocks(1), ResetCatchAndSync(clocks(1), reset.toBool))
-    case None => (clocks.head, reset)
-  }
-
-  outer.tiles.foreach({ case tile =>
-    tile.module.clock := tileClock
-    tile.module.reset := tileReset
-  })
-}
-
-// Config Fragment
-class WithSingleRationalTileDomain(multiplier: Int, divisor: Int) extends Config(
-  new WithRationalRocketTiles ++
-  new WithRationalBoomTiles ++
-  new Config((site, here, up) => {
-    case FireSimClockKey => FireSimClockParameters(Seq(RationalClock("TileDomain", multiplier, divisor)))
-  })
-)
-
-class HalfRateUncore extends WithSingleRationalTileDomain(2,1)
-
-class WithFiresimMulticlockTop extends Config((site, here, up) => {
-  case BuildSystem => (p: Parameters) => LazyModule(new FiresimMulticlockTop()(p)).suggestName("system")
-})
-
-// Complete Config
-class FireSimQuadRocketMulticlockConfig extends Config(
-  new HalfRateUncore ++
-  new WithFiresimMulticlockTop ++
-  new FireSimQuadRocketConfig)
-
-// Top Definition
-class FiresimMulticlockTop(implicit p: Parameters) extends chipyard.DigitalTop
-{
-  override lazy val module = new FiresimMulticlockTopModule(this)
-}
-
-
-class FiresimMulticlockTopModule[+L <: DigitalTop](l: L) extends chipyard.DigitalTopModule(l) with HasFireSimClockingImp
-
-// Harness Definition
-class FireSimMulticlockPOC(implicit val p: Parameters) extends RawModule {
-  freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary())
-  val clockBridge = Module(new RationalClockBridge(p(FireSimClockKey).additionalClocks:_*))
-  val refClock = clockBridge.io.clocks.head
-  val reset = WireInit(false.B)
-  withClockAndReset(refClock, reset) {
-    // Instantiate multiple instances of the DUT to implement supernode
-    val targets = Seq.fill(p(NumNodes)) {
-      val lazyModule = p(BuildSystem)(p)
-      (lazyModule, Module(lazyModule.module))
-    }
-    val peekPokeBridge = PeekPokeBridge(refClock, reset)
-    // A Seq of partial functions that will instantiate the right bridge only
-    // if that Mixin trait is present in the target's class instance
-    //
-    // Apply each partial function to each DUT instance
-    for ((lazyModule, module) <- targets) {
-      p(IOBinders).values.foreach(f => f(lazyModule) ++ f(module))
-    }
-    targets.collect({ case (_, t: HasAdditionalClocks) => t.clocks := clockBridge.io.clocks })
-  }
-}
-
-
diff --git a/generators/firechip/src/main/scala/TargetConfigs.scala b/generators/firechip/src/main/scala/TargetConfigs.scala
index 2828580a..d9ba92ec 100644
--- a/generators/firechip/src/main/scala/TargetConfigs.scala
+++ b/generators/firechip/src/main/scala/TargetConfigs.scala
@@ -22,7 +22,6 @@ import testchipip.WithRingSystemBus
 
 import firesim.bridges._
 import firesim.configs._
-import chipyard.config.ConfigValName._
 
 class WithBootROM extends Config((site, here, up) => {
   case BootROMParams => {
@@ -67,6 +66,8 @@ class WithNVDLASmall extends nvidia.blocks.dla.WithNVDLA("small")
 
 // Tweaks that are generally applied to all firesim configs
 class WithFireSimConfigTweaks extends Config(
+  // Required*: Uses FireSim ClockBridge and PeekPokeBridge to drive the system with a single clock/reset
+  new WithFireSimSimpleClocks ++
   // Required*: When using FireSim-as-top to provide a correct path to the target bootrom source
   new WithBootROM ++
   // Optional*: Removing this will require target-software changes to properly capture UART output
@@ -170,3 +171,12 @@ class FireSimArianeConfig extends Config(
   new WithDefaultMemModel ++
   new WithFireSimConfigTweaks ++
   new chipyard.ArianeConfig)
+
+
+class FireSimMulticlockRocketConfig extends Config(
+  new WithFireSimRationalTileDomain(2, 1) ++
+  new WithDefaultFireSimBridges ++
+  new WithDefaultMemModel ++
+  new WithFireSimConfigTweaks ++
+  new chipyard.MultiClockRocketConfig)
+
diff --git a/generators/firechip/src/test/scala/ScalaTestSuite.scala b/generators/firechip/src/test/scala/ScalaTestSuite.scala
index f92a7960..ea1627b7 100644
--- a/generators/firechip/src/test/scala/ScalaTestSuite.scala
+++ b/generators/firechip/src/test/scala/ScalaTestSuite.scala
@@ -106,8 +106,8 @@ class BoomF1Tests extends FireSimTestSuite("FireSim", "DDR3FRFCFSLLC4MB_FireSimL
 class RocketNICF1Tests extends FireSimTestSuite("FireSim", "WithNIC_DDR3FRFCFSLLC4MB_FireSimRocketConfig", "BaseF1Config")
 // Multiclock tests
 class RocketMulticlockF1Tests extends FireSimTestSuite(
-  "FireSimMulticlockPOC",
-  "FireSimQuadRocketMulticlockConfig",
+  "FireSim",
+  "FireSimMulticlockRocketConfig",
   "WithSynthAsserts_BaseF1Config")
 
 class ArianeF1Tests extends FireSimTestSuite("FireSim", "WithNIC_DDR3FRFCFSLLC4MB_FireSimArianeConfig", "BaseF1Config")

From 578ae6fca2f521982ab74f5045a972fe7dd0fccd Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 3 Aug 2020 14:23:08 -0700
Subject: [PATCH 03/74] Bump to July 2020 rocketchip

---
 .circleci/do-rtl-build.sh                     |  2 +-
 generators/ariane                             |  2 +-
 generators/boom                               |  2 +-
 .../chipyard/src/main/scala/ChipTop.scala     | 17 ++--
 .../chipyard/src/main/scala/Clocks.scala      | 77 +++++++++++--------
 .../src/main/scala/ConfigFragments.scala      | 10 +--
 .../chipyard/src/main/scala/Subsystem.scala   | 18 +----
 .../chipyard/src/main/scala/System.scala      |  5 +-
 .../chipyard/src/main/scala/TestHarness.scala |  2 +-
 .../src/main/scala/config/RocketConfigs.scala | 11 ++-
 .../firechip/src/main/scala/FireSim.scala     | 71 ++++++++++-------
 .../src/main/scala/TargetConfigs.scala        | 14 ++--
 generators/hwacha                             |  2 +-
 generators/rocket-chip                        |  2 +-
 generators/testchipip                         |  2 +-
 .../tracegen/src/main/scala/System.scala      | 10 +--
 generators/tracegen/src/main/scala/Tile.scala | 10 ++-
 tools/chisel3                                 |  2 +-
 tools/firrtl                                  |  2 +-
 19 files changed, 142 insertions(+), 119 deletions(-)

diff --git a/.circleci/do-rtl-build.sh b/.circleci/do-rtl-build.sh
index a7c8ad50..3973026f 100755
--- a/.circleci/do-rtl-build.sh
+++ b/.circleci/do-rtl-build.sh
@@ -59,7 +59,7 @@ run "export RISCV=\"$TOOLS_DIR\"; \
      export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \
      export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \
      make -C $REMOTE_SIM_DIR clean; \
-     make -j$REMOTE_MAKE_NPROC -C $REMOTE_SIM_DIR JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" ${mapping[$1]}"
+     make -j$REMOTE_MAKE_NPROC -C $REMOTE_SIM_DIR FIRRTL_LOGLEVEL=info JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" ${mapping[$1]}"
 run "rm -rf $REMOTE_CHIPYARD_DIR/project"
 
 # copy back the final build
diff --git a/generators/ariane b/generators/ariane
index 0ed91074..3a2eed60 160000
--- a/generators/ariane
+++ b/generators/ariane
@@ -1 +1 @@
-Subproject commit 0ed9107485281545bf5abf2a042dface55e740bf
+Subproject commit 3a2eed602faac24e58a530db429f23f11810aae9
diff --git a/generators/boom b/generators/boom
index 859c6055..dc22cacf 160000
--- a/generators/boom
+++ b/generators/boom
@@ -1 +1 @@
-Subproject commit 859c60553b0cd2e84ee586ad6de25223baefb722
+Subproject commit dc22cacf71fe88b95f3393d622f53648bf0440bd
diff --git a/generators/chipyard/src/main/scala/ChipTop.scala b/generators/chipyard/src/main/scala/ChipTop.scala
index ff3e2aed..f7b94d2b 100644
--- a/generators/chipyard/src/main/scala/ChipTop.scala
+++ b/generators/chipyard/src/main/scala/ChipTop.scala
@@ -32,10 +32,8 @@ class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunc
   // The system module specified by BuildSystem
   val lSystem = LazyModule(p(BuildSystem)(p)).suggestName("system")
 
-  // The systemClockSinkNode provides the implicit clock and reset for the System
-  val systemClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
-  val systemClockGroup = LazyModule(new ClockGroup("system_clock"))
-  systemClockSinkNode := systemClockGroup.node
+  // The implicitClockSinkNode provides the implicit clock and reset for the System
+  val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
 
   // Generate Clocks and Reset
   p(ChipyardClockKey)(this)
@@ -46,12 +44,13 @@ class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunc
   // anyways, they probably need to be explicitly clocked.
   lazy val module: LazyModuleImpLike = new LazyRawModuleImp(this) {
     // These become the implicit clock and reset to the System
-    val system_clock = systemClockSinkNode.in.head._1.clock
-    val system_reset = systemClockSinkNode.in.head._1.reset
+    val implicit_clock = implicitClockSinkNode.in.head._1.clock
+    val implicit_reset = implicitClockSinkNode.in.head._1.reset
+
 
     // The implicit clock and reset for the system is also, by convention, used for all the IOBinders
     // TODO: This may not be the right thing to do in all cases
-    withClockAndReset(system_clock, system_reset) {
+    withClockAndReset(implicit_clock, implicit_reset) {
       val (_ports, _iocells, _harnessFunctions) = p(IOBinders).values.flatMap(f => f(lSystem) ++ f(lSystem.module)).unzip3
       // We ignore _ports for now...
       iocells ++= _iocells.flatten
@@ -60,8 +59,8 @@ class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunc
 
     // Connect the implicit clock/reset, if present
     lSystem.module match { case l: LazyModuleImp => {
-      l.clock := system_clock
-      l.reset := system_reset
+      l.clock := implicit_clock
+      l.reset := implicit_reset
     }}
   }
 }
diff --git a/generators/chipyard/src/main/scala/Clocks.scala b/generators/chipyard/src/main/scala/Clocks.scala
index 9c71ed96..7f181bb5 100644
--- a/generators/chipyard/src/main/scala/Clocks.scala
+++ b/generators/chipyard/src/main/scala/Clocks.scala
@@ -83,59 +83,65 @@ case object ChipyardClockKey extends Field[ChipTop => Unit](ClockDrivers.harness
 
 
 object ClockDrivers {
-  // A simple clock provider, for testing. All clocks in system are aggregated into one,
-  // and are driven by directly punching out to the TestHarness clock
+  // A simple clock provider, for testing
   val harnessClock: ChipTop => Unit = { chiptop =>
     implicit val p = chiptop.p
-    val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
-    val clockAggregator = LazyModule(new ClockGroupAggregator("clocks"))
 
-    // Aggregate all 3 possible clock groups with the clockAggregator
-    chiptop.systemClockGroup.node := clockAggregator.node
-    if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
-      chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := clockAggregator.node }
-    }
-    chiptop.lSystem match {
-      case l: ChipyardSubsystem => l.tileClockGroupNode := clockAggregator.node
-      case _ =>
+    val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
+    chiptop.implicitClockSinkNode := implicitClockSourceNode
+
+    // Drive the diplomaticclock graph of the DigitalTop (if present)
+    val simpleClockGroupSourceNode = chiptop.lSystem match {
+      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
+        val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+        l.asyncClockGroupsNode := n
+        Some(n)
+      }
+      case _ => None
     }
 
-
-    clockAggregator.node := simpleClockGroupSourceNode
     InModuleBody {
-      // this needs directionality so generateIOFromSignal works
+      //this needs directionality so generateIOFromSignal works
       val clock_wire = Wire(Input(Clock()))
       val reset_wire = GenerateReset(chiptop, clock_wire)
       val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, Some("iocell_clock"))
       chiptop.iocells ++= clockIOCell
       clock_io.suggestName("clock")
 
-      simpleClockGroupSourceNode.out.unzip._1.flatMap(_.member).map { o =>
+      implicitClockSourceNode.out.unzip._1.map { o =>
         o.clock := clock_wire
         o.reset := reset_wire
       }
+
+      simpleClockGroupSourceNode.map { n => n.out.unzip._1.map { out: ClockGroupBundle =>
+        out.member.data.foreach { o =>
+          o.clock := clock_wire
+          o.reset := reset_wire
+        }
+      }}
+
       chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
         clock_io := th.harnessClock
         Nil
       })
     }
+
   }
 
-  val harnessMultiClock: ChipTop => Unit = { chiptop =>
+
+  val harnessDividedClock: ChipTop => Unit = { chiptop =>
     implicit val p = chiptop.p
-    val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters(), ClockGroupSourceParameters()))
-    val uncoreClockAggregator = LazyModule(new ClockGroupAggregator("uncore_clocks"))
 
-    // Aggregate only the uncoreclocks
-    chiptop.systemClockGroup.node := uncoreClockAggregator.node
-    if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
-      chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := uncoreClockAggregator.node }
-    }
+    val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
+    chiptop.implicitClockSinkNode := implicitClockSourceNode
 
-    uncoreClockAggregator.node := simpleClockGroupSourceNode
-    chiptop.lSystem match {
-      case l: ChipyardSubsystem => l.tileClockGroupNode := simpleClockGroupSourceNode
-      case _ => throw new Exception("MultiClock assumes ChipyardSystem")
+    val simpleClockGroupSourceNode = chiptop.lSystem match {
+      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
+        val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+        l.asyncClockGroupsNode := n
+        Some(n)
+      }
+      case _ => throw new Exception("Harness multiclock assumes BaseSubsystem")
     }
 
     InModuleBody {
@@ -147,14 +153,19 @@ object ClockDrivers {
       clock_io.suggestName("clock")
       val div_clock = Pow2ClockDivider(clock_wire, 2)
 
-      simpleClockGroupSourceNode.out(0)._1.member.map { o =>
+      implicitClockSourceNode.out.unzip._1.map { o =>
         o.clock := div_clock
-        o.reset := ResetCatchAndSync(div_clock, reset_wire.asBool)
-      }
-      simpleClockGroupSourceNode.out(1)._1.member.map { o =>
-        o.clock := clock_wire
         o.reset := reset_wire
       }
+
+      simpleClockGroupSourceNode.map { n => n.out.unzip._1.map { out: ClockGroupBundle =>
+        out.member.elements.map { case (name, data) =>
+          // This is mega hacks, how are you actually supposed to do this?
+          data.clock := (if (name.contains("core")) clock_wire else div_clock)
+          data.reset := reset_wire
+        }
+      }}
+
       chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
         clock_io := th.harnessClock
         Nil
diff --git a/generators/chipyard/src/main/scala/ConfigFragments.scala b/generators/chipyard/src/main/scala/ConfigFragments.scala
index dd17b0a1..704b849a 100644
--- a/generators/chipyard/src/main/scala/ConfigFragments.scala
+++ b/generators/chipyard/src/main/scala/ConfigFragments.scala
@@ -6,12 +6,13 @@ import chisel3.util.{log2Up}
 import freechips.rocketchip.config.{Field, Parameters, Config}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.diplomacy.{LazyModule, ValName}
-import freechips.rocketchip.devices.tilelink.BootROMParams
+import freechips.rocketchip.devices.tilelink.{BootROMLocated}
 import freechips.rocketchip.devices.debug.{Debug}
 import freechips.rocketchip.groundtest.{GroundTestSubsystem}
 import freechips.rocketchip.tile._
 import freechips.rocketchip.rocket.{RocketCoreParams, MulDivParams, DCacheParams, ICacheParams}
 import freechips.rocketchip.util.{AsyncResetReg}
+import freechips.rocketchip.prci._
 
 import testchipip._
 import tracegen.{TraceGenSystem}
@@ -33,8 +34,7 @@ import chipyard.{BuildTop, BuildSystem, ClockDrivers, ChipyardClockKey, TestSuit
 // -----------------------
 
 class WithBootROM extends Config((site, here, up) => {
-  case BootROMParams => BootROMParams(
-    contentFileName = s"./bootrom/bootrom.rv${site(XLen)}.img")
+  case BootROMLocated(x) => up(BootROMLocated(x), site).map(_.copy(contentFileName = s"./bootrom/bootrom.rv${site(XLen)}.img"))
 })
 
 // DOC include start: gpio config fragment
@@ -159,6 +159,6 @@ class WithNoSubsystemDrivenClocks extends Config((site, here, up) => {
   case SubsystemDriveAsyncClockGroupsKey => None
 })
 
-class WithTileMultiClock extends Config((site, here, up) => {
-  case ChipyardClockKey => ClockDrivers.harnessMultiClock
+class WithTileDividedClock extends Config((site, here, up) => {
+  case ChipyardClockKey => ClockDrivers.harnessDividedClock
 })
diff --git a/generators/chipyard/src/main/scala/Subsystem.scala b/generators/chipyard/src/main/scala/Subsystem.scala
index 65ceddc9..3184bcaa 100644
--- a/generators/chipyard/src/main/scala/Subsystem.scala
+++ b/generators/chipyard/src/main/scala/Subsystem.scala
@@ -35,32 +35,16 @@ class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
     case b: BoomTile => b.module.core.coreMonitorBundle
   }.toList
 
-  val tileClockSinkNode = ClockSinkNode(List(ClockSinkParameters()))
-  val tileClockGroup = LazyModule(new ClockGroup("tile_clock"))
-  val tileClockGroupNode = tileClockGroup.node
-  tileClockSinkNode := tileClockGroupNode
-
   override lazy val module = new ChipyardSubsystemModuleImp(this)
 }
 
 class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
-  with HasResetVectorWire
   with HasTilesModuleImp
 {
-  for (i <- 0 until outer.tiles.size) {
-    val wire = tile_inputs(i)
-    wire.hartid := outer.hartIdList(i).U
-    wire.reset_vector := global_reset_vector
-
-
-    outer.tiles(i).module.clock := outer.tileClockSinkNode.in.head._1.clock
-    outer.tiles(i).module.reset := outer.tileClockSinkNode.in.head._1.reset
-  }
-
   // create file with core params
   ElaborationArtefacts.add("""core.config""", outer.tiles.map(x => x.module.toString).mkString("\n"))
   // Generate C header with relevant information for Dromajo
   // This is included in the `dromajo_params.h` header file
-  DromajoHelper.addArtefacts()
+  DromajoHelper.addArtefacts(InSubsystem)
 }
 
diff --git a/generators/chipyard/src/main/scala/System.scala b/generators/chipyard/src/main/scala/System.scala
index b0ae8a44..bd20ddc7 100644
--- a/generators/chipyard/src/main/scala/System.scala
+++ b/generators/chipyard/src/main/scala/System.scala
@@ -26,8 +26,10 @@ class ChipyardSystem(implicit p: Parameters) extends ChipyardSubsystem
   with CanHaveMasterAXI4MemPort
   with CanHaveMasterAXI4MMIOPort
   with CanHaveSlaveAXI4Port
-  with HasPeripheryBootROM
 {
+
+  val bootROM  = p(BootROMLocated(location)).map { BootROM.attach(_, this, CBUS) }
+  val maskROMs = p(MaskROMLocated(location)).map { MaskROM.attach(_, this, CBUS) }
   override lazy val module = new ChipyardSystemModule(this)
 }
 
@@ -37,5 +39,4 @@ class ChipyardSystem(implicit p: Parameters) extends ChipyardSubsystem
 class ChipyardSystemModule[+L <: ChipyardSystem](_outer: L) extends ChipyardSubsystemModuleImp(_outer)
   with HasRTCModuleImp
   with HasExtInterruptsModuleImp
-  with HasPeripheryBootROMModuleImp
   with DontTouch
diff --git a/generators/chipyard/src/main/scala/TestHarness.scala b/generators/chipyard/src/main/scala/TestHarness.scala
index 92b1dd29..70802783 100644
--- a/generators/chipyard/src/main/scala/TestHarness.scala
+++ b/generators/chipyard/src/main/scala/TestHarness.scala
@@ -28,7 +28,7 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessUtil
     val success = Output(Bool())
   })
 
-  val ldut = LazyModule(p(BuildTop)(p)).suggestName("ChipTop")
+  val ldut = LazyModule(p(BuildTop)(p)).suggestName("chiptop")
   val dut = Module(ldut.module)
   io.success := false.B
 
diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
index f7ce5a84..e0587454 100644
--- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
@@ -187,8 +187,15 @@ class MMIORocketConfig extends Config(
 
 // NOTE: This config doesn't work yet because SimWidgets in the TestHarness
 // always get the TestHarness clock. The Tiles and Uncore receive the correct clocks
-class MultiClockRocketConfig extends Config(
-  new chipyard.config.WithTileMultiClock ++                       // Put the Tile on its own clock domain
+class DividedClockRocketConfig extends Config(
+  new chipyard.config.WithTileDividedClock ++                       // Put the Tile on its own clock domain
   new freechips.rocketchip.subsystem.WithRationalRocketTiles ++   // Add rational crossings between RocketTile and uncore
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
+
+
+class TestClockRocketConfig extends Config(
+  //new chipyard.config.WithTileMultiClock ++                       // Put the Tile on its own clock domain
+  new freechips.rocketchip.subsystem.WithAsynchronousRocketTiles(8, 3) ++   // Add rational crossings between RocketTile and uncore
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)
diff --git a/generators/firechip/src/main/scala/FireSim.scala b/generators/firechip/src/main/scala/FireSim.scala
index d35dd7b6..8f1de11d 100644
--- a/generators/firechip/src/main/scala/FireSim.scala
+++ b/generators/firechip/src/main/scala/FireSim.scala
@@ -45,26 +45,36 @@ object NodeIdx {
 class WithFireSimSimpleClocks extends Config((site, here, up) => {
   case ChipyardClockKey => { chiptop: ChipTop =>
     implicit val p = chiptop.p
-    val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
-    val clockAggregator = LazyModule(new ClockGroupAggregator("clocks"))
 
-    // Aggregate all 3 possible clock groups with the clockAggregator
-    chiptop.systemClockGroup.node := clockAggregator.node
-    if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
-      chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := clockAggregator.node }
+    val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
+    chiptop.implicitClockSinkNode := implicitClockSourceNode
+
+    // Drive the diplomaticclock graph of the DigitalTop (if present)
+    val simpleClockGroupSourceNode = chiptop.lSystem match {
+      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
+        val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+        l.asyncClockGroupsNode := n
+        Some(n)
+      }
+      case _ => None
     }
-    chiptop.lSystem match { case l: ChipyardSubsystem => l.tileClockGroupNode := clockAggregator.node }
 
-    clockAggregator.node := simpleClockGroupSourceNode
     InModuleBody {
-      val clock      = IO(Input(Clock())).suggestName("clock")
-      val reset      = IO(Input(Reset())).suggestName("reset")
+      val clock = IO(Input(Clock())).suggestName("clock")
+      val reset = IO(Input(Reset())).suggestName("reset")
 
-      simpleClockGroupSourceNode.out.unzip._1.flatMap(_.member).map { o =>
+      implicitClockSourceNode.out.unzip._1.map { o =>
         o.clock := clock
         o.reset := reset
       }
 
+      simpleClockGroupSourceNode.map { n => n.out.unzip._1.map { out: ClockGroupBundle =>
+        out.member.data.foreach { o =>
+          o.clock := clock
+          o.reset := reset
+        }
+      }}
+
       chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
         clock := th.harnessClock
         reset := th.harnessReset
@@ -78,19 +88,18 @@ class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Confi
   case FireSimClockKey => FireSimClockParameters(Seq(RationalClock("TileDomain", multiplier, divisor)))
   case ChipyardClockKey => { chiptop: ChipTop =>
     implicit val p = chiptop.p
-    val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters(), ClockGroupSourceParameters()))
-    val uncoreClockAggregator = LazyModule(new ClockGroupAggregator("uncore_clocks"))
 
-    // Aggregate only the uncoreclocks
-    chiptop.systemClockGroup.node := uncoreClockAggregator.node
-    if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
-      chiptop.lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode := uncoreClockAggregator.node }
-    }
+    val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
+    chiptop.implicitClockSinkNode := implicitClockSourceNode
 
-    uncoreClockAggregator.node := simpleClockGroupSourceNode
-    chiptop.lSystem match {
-      case l: ChipyardSubsystem => l.tileClockGroupNode := simpleClockGroupSourceNode
-      case _ => throw new Exception("MultiClock assumes ChipyardSystem")
+    // Drive the diplomaticclock graph of the DigitalTop (if present)
+    val simpleClockGroupSourceNode = chiptop.lSystem match {
+      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
+        val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+        l.asyncClockGroupsNode := n
+        Some(n)
+      }
+      case _ => None
     }
 
     InModuleBody {
@@ -98,15 +107,23 @@ class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Confi
       val tile_clock   = IO(Input(Clock())).suggestName("tile_clock")
       val reset        = IO(Input(Reset())).suggestName("reset")
 
-      simpleClockGroupSourceNode.out(0)._1.member.map { o =>
+      implicitClockSourceNode.out.unzip._1.map { o =>
         o.clock := uncore_clock
         o.reset := reset
       }
 
-      simpleClockGroupSourceNode.out(1)._1.member.map { o =>
-        o.clock := tile_clock
-        o.reset := ResetCatchAndSync(tile_clock, reset.asBool)
-      }
+      simpleClockGroupSourceNode.map { n => n.out.unzip._1.map { out: ClockGroupBundle =>
+        out.member.elements.map { case (name, data) =>
+          // This is mega hacks, how are you actually supposed to do this?
+          if (name.contains("core")) {
+            data.clock := tile_clock
+            data.reset := ResetCatchAndSync(tile_clock, reset.asBool)
+          } else {
+            data.clock := uncore_clock
+            data.clock := reset
+          }
+        }
+      }}
 
       chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
         uncore_clock := th.harnessClock
diff --git a/generators/firechip/src/main/scala/TargetConfigs.scala b/generators/firechip/src/main/scala/TargetConfigs.scala
index c734ac8c..66a20bce 100644
--- a/generators/firechip/src/main/scala/TargetConfigs.scala
+++ b/generators/firechip/src/main/scala/TargetConfigs.scala
@@ -10,7 +10,7 @@ import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.rocket.DCacheParams
 import freechips.rocketchip.subsystem._
-import freechips.rocketchip.devices.tilelink.BootROMParams
+import freechips.rocketchip.devices.tilelink.{BootROMLocated, BootROMParams}
 import freechips.rocketchip.devices.debug.{DebugModuleParams, DebugModuleKey}
 import freechips.rocketchip.diplomacy.LazyModule
 import testchipip.{BlockDeviceKey, BlockDeviceConfig, SerialKey, TracePortKey, TracePortParams}
@@ -24,16 +24,16 @@ import firesim.bridges._
 import firesim.configs._
 
 class WithBootROM extends Config((site, here, up) => {
-  case BootROMParams => {
+  case BootROMLocated(x) => {
     val chipyardBootROM = new File(s"./generators/testchipip/bootrom/bootrom.rv${site(XLen)}.img")
     val firesimBootROM = new File(s"./target-rtl/chipyard/generators/testchipip/bootrom/bootrom.rv${site(XLen)}.img")
 
-     val bootROMPath = if (chipyardBootROM.exists()) {
+    val bootROMPath = if (chipyardBootROM.exists()) {
       chipyardBootROM.getAbsolutePath()
     } else {
       firesimBootROM.getAbsolutePath()
     }
-    BootROMParams(contentFileName = bootROMPath)
+    up(BootROMLocated(x), site).map(_.copy(contentFileName = bootROMPath))
   }
 })
 
@@ -188,11 +188,13 @@ class FireSimArianeConfig extends Config(
   new WithFireSimConfigTweaks ++
   new chipyard.ArianeConfig)
 
-
+//**********************************************************************************
+//* Multiclock Configurations
+//*********************************************************************************/
 class FireSimMulticlockRocketConfig extends Config(
   new WithFireSimRationalTileDomain(2, 1) ++
   new WithDefaultFireSimBridges ++
   new WithDefaultMemModel ++
   new WithFireSimConfigTweaks ++
-  new chipyard.MultiClockRocketConfig)
+  new chipyard.DividedClockRocketConfig)
 
diff --git a/generators/hwacha b/generators/hwacha
index a989b697..e29b65db 160000
--- a/generators/hwacha
+++ b/generators/hwacha
@@ -1 +1 @@
-Subproject commit a989b69759137802b4c39e9ddebb90427455fb79
+Subproject commit e29b65db86e4486ebdfd4f39d1265df83a2d7d9d
diff --git a/generators/rocket-chip b/generators/rocket-chip
index 653efa99..6eb1a3de 160000
--- a/generators/rocket-chip
+++ b/generators/rocket-chip
@@ -1 +1 @@
-Subproject commit 653efa99a27dc155bd4b4706a7e71c5c930f62b1
+Subproject commit 6eb1a3de082e27c752d9e4c1ae971c693cc192eb
diff --git a/generators/testchipip b/generators/testchipip
index 8b5c89a5..3bfd710c 160000
--- a/generators/testchipip
+++ b/generators/testchipip
@@ -1 +1 @@
-Subproject commit 8b5c89a5f7120e64a7ac5ce5210165426a58f3de
+Subproject commit 3bfd710ce36817038aae5d11848aec9a3c0c705f
diff --git a/generators/tracegen/src/main/scala/System.scala b/generators/tracegen/src/main/scala/System.scala
index ca3572d7..83f6a5e4 100644
--- a/generators/tracegen/src/main/scala/System.scala
+++ b/generators/tracegen/src/main/scala/System.scala
@@ -12,6 +12,10 @@ class TraceGenSystem(implicit p: Parameters) extends BaseSubsystem
     with CanHaveMasterAXI4MemPort {
 
   def coreMonitorBundles = Nil
+  val tileStatusNodes = tiles.collect {
+    case t: GroundTestTile => t.statusNode.makeSink()
+    case t: BoomTraceGenTile => t.statusNode.makeSink()
+  }
   override lazy val module = new TraceGenSystemModuleImp(this)
 }
 
@@ -20,12 +24,8 @@ class TraceGenSystemModuleImp(outer: TraceGenSystem)
 {
   val success = IO(Output(Bool()))
 
-  outer.tiles.zipWithIndex.map { case(t, i) => t.module.constants.hartid := i.U }
+  val status = dontTouch(DebugCombiner(outer.tileStatusNodes.map(_.bundle)))
 
-  val status = dontTouch(DebugCombiner(outer.tiles.collect {
-    case t: GroundTestTile => t.module.status
-    case t: BoomTraceGenTile => t.module.status
-  }))
   success := outer.tileCeaseSinkNode.in.head._1.asUInt.andR
 
 }
diff --git a/generators/tracegen/src/main/scala/Tile.scala b/generators/tracegen/src/main/scala/Tile.scala
index 1ddf0d84..63d68e50 100644
--- a/generators/tracegen/src/main/scala/Tile.scala
+++ b/generators/tracegen/src/main/scala/Tile.scala
@@ -3,7 +3,7 @@ package tracegen
 import chisel3._
 import chisel3.util._
 import freechips.rocketchip.config.Parameters
-import freechips.rocketchip.diplomacy.{SimpleDevice, LazyModule, SynchronousCrossing, ClockCrossingType}
+import freechips.rocketchip.diplomacy.{SimpleDevice, LazyModule, SynchronousCrossing, ClockCrossingType, BundleBridgeSource}
 import freechips.rocketchip.groundtest._
 import freechips.rocketchip.rocket._
 import freechips.rocketchip.rocket.constants.{MemoryOpConstants}
@@ -206,11 +206,13 @@ class BoomTraceGenTile private(
   val cpuDevice: SimpleDevice = new SimpleDevice("groundtest", Nil)
   val intOutwardNode: IntOutwardNode = IntIdentityNode()
   val slaveNode: TLInwardNode = TLIdentityNode()
+  val statusNode = BundleBridgeSource(() => new GroundTestStatus)
 
   val boom_params = p.alterMap(Map(TileKey -> BoomTileParams(
     dcache=params.dcache,
     core=BoomCoreParams(nPMPs=0, numLdqEntries=32, numStqEntries=32, useVM=false))))
-  val dcache = LazyModule(new BoomNonBlockingDCache(hartId)(boom_params))
+  val dcache = LazyModule(new BoomNonBlockingDCache(staticIdForMetadataUseOnly)(boom_params))
+
 
   val masterNode: TLOutwardNode = TLIdentityNode() := visibilityNode := dcache.node
 
@@ -220,11 +222,11 @@ class BoomTraceGenTile private(
 class BoomTraceGenTileModuleImp(outer: BoomTraceGenTile)
   extends BaseTileModuleImp(outer){
 
-  val status = IO(new GroundTestStatus)
+  val status = outer.statusNode.bundle
   val halt_and_catch_fire = None
 
   val tracegen = Module(new TraceGenerator(outer.params.traceParams))
-  tracegen.io.hartid := constants.hartid
+  tracegen.io.hartid := outer.hartIdSinkNode.bundle
 
   val ptw = Module(new DummyPTW(1))
   val lsu = Module(new LSU()(outer.boom_params, outer.dcache.module.edge))
diff --git a/tools/chisel3 b/tools/chisel3
index 21ea734d..cc2971fe 160000
--- a/tools/chisel3
+++ b/tools/chisel3
@@ -1 +1 @@
-Subproject commit 21ea734d809395962a8d3195a76377f6e44308f3
+Subproject commit cc2971feb15d4bc8cb4a8138b5a095ccbc92dcc3
diff --git a/tools/firrtl b/tools/firrtl
index 7c6f58d9..c07da8a5 160000
--- a/tools/firrtl
+++ b/tools/firrtl
@@ -1 +1 @@
-Subproject commit 7c6f58d986e67b3d0662a4cd6654a68f9cc52cf9
+Subproject commit c07da8a581789b88f7e6ffc98c8e810565034ad9

From abc75e9b9551c209390ef67deb2586f4c03a61a0 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 5 Aug 2020 15:00:28 -0700
Subject: [PATCH 04/74] Fix Reset bug

---
 .../chipyard/src/main/scala/Subsystem.scala   | 25 +++++++++++++++++--
 .../firechip/src/main/scala/FireSim.scala     |  2 +-
 generators/tracegen/src/main/scala/Tile.scala |  2 +-
 3 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/generators/chipyard/src/main/scala/Subsystem.scala b/generators/chipyard/src/main/scala/Subsystem.scala
index 3184bcaa..7f089ce1 100644
--- a/generators/chipyard/src/main/scala/Subsystem.scala
+++ b/generators/chipyard/src/main/scala/Subsystem.scala
@@ -11,7 +11,7 @@ import chisel3.internal.sourceinfo.{SourceInfo}
 import freechips.rocketchip.prci._
 import freechips.rocketchip.config.{Field, Parameters}
 import freechips.rocketchip.devices.tilelink._
-import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp}
+import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp, ExportDebug}
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.diplomaticobjectmodel.model.{OMInterrupt}
 import freechips.rocketchip.diplomaticobjectmodel.logicaltree.{RocketTileLogicalTreeNode, LogicalModuleTree}
@@ -25,10 +25,31 @@ import freechips.rocketchip.amba.axi4._
 import boom.common.{BoomTile}
 
 
-import testchipip.{DromajoHelper}
+import testchipip.{DromajoHelper, CanHavePeripherySerial, SerialKey}
+
+trait CanHaveHTIF { this: BaseSubsystem =>
+  // Advertise HTIF if system can communicate with fesvr
+  if (this match {
+    case _: CanHavePeripherySerial if p(SerialKey) => true
+    case _: HasPeripheryDebug if p(ExportDebug).protocols.nonEmpty => true
+    case _ => false
+  }) {
+    ResourceBinding {
+      val htif = new Device {
+        def describe(resources: ResourceBindings): Description = {
+          val compat = resources("compat").map(_.value)
+          Description("htif", Map(
+            "compatible" -> compat))
+        }
+      }
+      Resource(htif, "compat").bind(ResourceString("ucb,htif0"))
+    }
+  }
+}
 
 class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
   with HasTiles
+  with CanHaveHTIF
 {
   def coreMonitorBundles = tiles.map {
     case r: RocketTile => r.module.core.rocketImpl.coreMonitorBundle
diff --git a/generators/firechip/src/main/scala/FireSim.scala b/generators/firechip/src/main/scala/FireSim.scala
index 8f1de11d..e807e840 100644
--- a/generators/firechip/src/main/scala/FireSim.scala
+++ b/generators/firechip/src/main/scala/FireSim.scala
@@ -120,7 +120,7 @@ class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Confi
             data.reset := ResetCatchAndSync(tile_clock, reset.asBool)
           } else {
             data.clock := uncore_clock
-            data.clock := reset
+            data.reset := reset
           }
         }
       }}
diff --git a/generators/tracegen/src/main/scala/Tile.scala b/generators/tracegen/src/main/scala/Tile.scala
index 63d68e50..5ff9af56 100644
--- a/generators/tracegen/src/main/scala/Tile.scala
+++ b/generators/tracegen/src/main/scala/Tile.scala
@@ -210,7 +210,7 @@ class BoomTraceGenTile private(
 
   val boom_params = p.alterMap(Map(TileKey -> BoomTileParams(
     dcache=params.dcache,
-    core=BoomCoreParams(nPMPs=0, numLdqEntries=32, numStqEntries=32, useVM=false))))
+    core=BoomCoreParams(nPMPs=0, numLdqEntries=16, numStqEntries=16, useVM=false))))
   val dcache = LazyModule(new BoomNonBlockingDCache(staticIdForMetadataUseOnly)(boom_params))
 
 

From 8867c3241c9f3ddeed483f7fa6f35ac44631f42f Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Mon, 17 Aug 2020 17:07:29 -0700
Subject: [PATCH 05/74] Add .swo to .gitignore | Update docs

---
 .gitignore                                  |  1 +
 docs/Simulation/Software-RTL-Simulation.rst | 49 +++++++--------------
 2 files changed, 18 insertions(+), 32 deletions(-)

diff --git a/.gitignore b/.gitignore
index 35d9b2d8..a85d0dd2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@ target
 *.stamp
 *.vcd
 *.swp
+*.swo
 *.log
 *#
 *~
diff --git a/docs/Simulation/Software-RTL-Simulation.rst b/docs/Simulation/Software-RTL-Simulation.rst
index 28ae223e..ca852d36 100644
--- a/docs/Simulation/Software-RTL-Simulation.rst
+++ b/docs/Simulation/Software-RTL-Simulation.rst
@@ -40,8 +40,7 @@ For a proprietry VCS simulation, enter the ``sims/vcs`` directory
     # Enter VCS directory
     cd sims/vcs
 
-
-.. _sim-default:
+.. _sw-sim-help:
 
 Simulating The Default Example
 -------------------------------
@@ -62,12 +61,6 @@ For instance, to run one of the riscv-tools assembly tests.
 
 .. Note:: In a VCS simulator, the simulator name will be ``simv-chipyard-RocketConfig`` instead of ``simulator-chipyard-RocketConfig``.
 
-The makefiles have a ``run-binary`` rule that simplifies running the simulation executable. It adds many of the common command line options for you and redirects the output to a file.
-
-.. code-block:: shell
-
-    make run-binary BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/isa/rv64ui-p-simple
-
 Alternatively, we can run a pre-packaged suite of RISC-V assembly or benchmark tests, by adding the make target ``run-asm-tests`` or ``run-bmark-tests``.
 For example:
 
@@ -82,6 +75,22 @@ For example:
 
 .. _sw-sim-custom:
 
+Makefile Variables and Commands
+-------------------------------
+You can get a list of useful Makefile variables and commands available from the Verilator or VCS directories. simply run ``make help``:
+
+.. code-block:: shell
+
+    # Enter Verilator directory
+    cd sims/verilator
+    make help
+
+    # Enter VCS directory
+    cd sims/vcs
+    make help
+
+.. _sim-default:
+
 Simulating A Custom Project
 -------------------------------
 
@@ -132,29 +141,6 @@ All ``make`` targets that can be applied to the default example, can also be app
 Finally, in the ``generated-src/<...>-<package>-<config>/`` directory resides all of the collateral and Verilog source files for the build/simulation.
 Specifically, the SoC top-level (``TOP``) Verilog file is denoted with ``*.top.v`` while the ``TestHarness`` file is denoted with ``*.harness.v``.
 
-Fast Memory Loading
--------------------
-
-The simulator loads the program binary over a simulated serial line. This can be quite slow if there is a lot of static data, so the simulator also allows data to be loaded from a file directly into the DRAM model.
-
-.. code-block:: shell
-
-    make run-binary BINARY=test.riscv LOADMEM=testdata.hex LOADMEM_ADDR=81000000
-
-The ``.hex`` file should be a text file with a hexadecimal number on each line.
-
-.. code-block:: text
-
-    deadbeef
-    0123
-
-Each line uses little-endian order, so this file would produce the bytes "ef be ad de 01 23". ``LOADMEM_ADDR`` specifies which address in memory (in hexadecimal) to write the first byte to. The default is 0x81000000.
-
-A special target that facilitates automatically generating a hex file for an entire elf RISC-V exectuable and then running the simulator with the appropriate flags is also available.
-
-.. code-block:: shell
-
-    make run-binary-hex BINARY=test.riscv
 
 Generating Waveforms
 -----------------------
@@ -166,4 +152,3 @@ An open-source vcd-capable waveform viewer is `GTKWave <http://gtkwave.sourcefor
 
 For a VCS simulation, this will generate a vpd file (this is a proprietary waveform representation format used by Synopsys) that can be loaded to vpd-supported waveform viewers.
 If you have Synopsys licenses, we recommend using the DVE waveform viewer.
-

From d402825e7f23a883049a76d6404f7402bf7986e5 Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Mon, 17 Aug 2020 17:15:05 -0700
Subject: [PATCH 06/74] Change  eval. strategy

---
 .circleci/config.yml                          |  2 +-
 scripts/build-toolchains.sh                   | 22 +++++++++----------
 scripts/build-util.sh                         |  2 +-
 .../init-submodules-no-riscv-tools-nolog.sh   | 16 +++++++++-----
 4 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b6556850..88a20f24 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -6,7 +6,7 @@ version: 2.1
 parameters:
     tools-cache-version:
         type: string
-        default: "v5"
+        default: "v6"
 
 # default execution env.s
 executors:
diff --git a/scripts/build-toolchains.sh b/scripts/build-toolchains.sh
index 62a2f74b..2685872e 100755
--- a/scripts/build-toolchains.sh
+++ b/scripts/build-toolchains.sh
@@ -6,8 +6,8 @@
 set -e
 set -o pipefail
 
-RDIR=$(pwd)
-CHIPYARD_DIR="${CHIPYARD_DIR:-$(git rev-parse --show-toplevel)}"
+DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+CHIPYARD_DIR="$(dirname "$DIR")"
 
 usage() {
     echo "usage: ${0} [OPTIONS] [riscv-tools | esp-tools | ec2fast]"
@@ -134,18 +134,16 @@ SRCDIR="$(pwd)/toolchains" module_all qemu --prefix="${RISCV}" --target-list=ris
 git submodule update --init $CHIPYARD_DIR/tools/dromajo/dromajo-src
 make -C $CHIPYARD_DIR/tools/dromajo/dromajo-src/src
 
-cd "$RDIR"
-
 # create specific env.sh
-{
-    echo "# auto-generated by build-toolchains.sh"
-    echo "export CHIPYARD_TOOLCHAIN_SOURCED=1"
-    echo "export RISCV=$(printf '%q' "$RISCV")"
-    echo "export PATH=\${RISCV}/bin:\${PATH}"
-    echo "export LD_LIBRARY_PATH=\${RISCV}/lib\${LD_LIBRARY_PATH:+":\${LD_LIBRARY_PATH}"}"
-} > env-$TOOLCHAIN.sh
+cat > "$CHIPYARD_DIR/env-$TOOLCHAIN.sh" <<EOF
+# auto-generated by build-toolchains.sh
+export CHIPYARD_TOOLCHAIN_SOURCED=1
+export RISCV=$(printf '%q' "$RISCV")
+export PATH=\${RISCV}/bin:\${PATH}
+export LD_LIBRARY_PATH=\${RISCV}/lib\${LD_LIBRARY_PATH:+":\${LD_LIBRARY_PATH}"}
+EOF
 
 # create general env.sh
 echo "# line auto-generated by build-toolchains.sh" >> env.sh
-echo "source \$( realpath \$(dirname "\${BASH_SOURCE[0]:-\${\(%\):-%x}}") )/env-$TOOLCHAIN.sh" >> env.sh
+echo "source $(printf '%q' "$CHIPYARD_DIR/env-$TOOLCHAIN.sh")" >> env.sh
 echo "Toolchain Build Complete!"
diff --git a/scripts/build-util.sh b/scripts/build-util.sh
index 10a7cb7b..892b274d 100644
--- a/scripts/build-util.sh
+++ b/scripts/build-util.sh
@@ -15,7 +15,7 @@ case ${ncpu} in
 esac
 
 # Allow user to override MAKE
-[ -n "${MAKE}" ] || MAKE=$(command -v gnumake || command -v gmake || command -v make)
+[ -n "${MAKE:+x}" ] || MAKE=$(command -v gnumake || command -v gmake || command -v make)
 readonly MAKE
 
 
diff --git a/scripts/init-submodules-no-riscv-tools-nolog.sh b/scripts/init-submodules-no-riscv-tools-nolog.sh
index d91e89b9..cede5e47 100755
--- a/scripts/init-submodules-no-riscv-tools-nolog.sh
+++ b/scripts/init-submodules-no-riscv-tools-nolog.sh
@@ -17,10 +17,11 @@ if [ "$MINGIT" != "$(echo -e "$MINGIT\n$MYGIT" | sort -V | head -n1)" ]; then
   false
 fi
 
-RDIR=$(git rev-parse --show-toplevel)
+DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+CHIPYARD_DIR="$(dirname "$DIR")"
 
 # Ignore toolchain submodules
-cd "$RDIR"
+cd "$CHIPYARD_DIR"
 for name in toolchains/*-tools/*/ ; do
     git config submodule."${name%/}".update none
 done
@@ -71,8 +72,11 @@ git config submodule.sims/firesim.update none
 git submodule update --init software/firemarshal
 
 # Configure firemarshal to know where our firesim installation is
-if [ ! -f $RDIR/software/firemarshal/marshal-config.yaml ]; then
-  echo "firesim-dir: '../../sims/firesim/'" > $RDIR/software/firemarshal/marshal-config.yaml
+if [ ! -f ./software/firemarshal/marshal-config.yaml ]; then
+  echo "firesim-dir: '../../sims/firesim/'" > ./software/firemarshal/marshal-config.yaml
 fi
-echo "# line auto-generated by init-submodules-no-riscv-tools.sh" >> $RDIR/env.sh
-echo "PATH=\$( realpath \$(dirname "\${BASH_SOURCE[0]:-\${\(%\):-%x}}") )/software/firemarshal:\$PATH" >> $RDIR/env.sh
+
+echo "# line auto-generated by init-submodules-no-riscv-tools.sh" >> env.sh
+echo '__DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"' >> env.sh
+echo "PATH=\$__DIR/bin:\$PATH" >> env.sh
+echo "PATH=\$__DIR/software/firemarshal:\$PATH" >> env.sh

From b007d79820c13c5e93d3cc43315879a9d577fdf7 Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Mon, 17 Aug 2020 20:28:05 -0700
Subject: [PATCH 07/74] Add help section to makefiles + Reorganize

---
 common.mk                |  55 ++++++++----
 sims/vcs/Makefile        | 111 +++++++++++++++++++++++--
 sims/verilator/Makefile  | 175 +++++++++++++++++++++++++++++----------
 tools/dromajo/dromajo.mk |   3 +-
 variables.mk             |  60 +++++++++-----
 5 files changed, 316 insertions(+), 88 deletions(-)

diff --git a/common.mk b/common.mk
index 43615a92..87d4047c 100644
--- a/common.mk
+++ b/common.mk
@@ -3,22 +3,46 @@
 #########################################################################################
 SHELL=/bin/bash
 
-
 ifndef RISCV
 $(error RISCV is unset. You must set RISCV yourself, or through the Chipyard auto-generated env file)
 else
 $(info Running with RISCV=$(RISCV))
 endif
 
+#########################################################################################
+# specify user-interface variables
+#########################################################################################
+HELP_COMPILATION_VARIABLES += \
+"   EXTRA_GENERATOR_REQS   = requirements needed for the main generator" \
+"   EXTRA_SIM_CFLAGS       = CFLAGS for building simulators" \
+"   EXTRA_SIM_CXXFLAGS     = CXXFLAGS for building simulators" \
+"   EXTRA_SIM_LDFLAGS      = LDFLAGS for building simulators" \
+"   EXTRA_SIM_SOURCES      = simulation sources needed for simulator" \
+"   EXTRA_SIM_REQS         = requirements to build the simulator"
+
+EXTRA_GENERATOR_REQS ?=
+EXTRA_SIM_CXXFLAGS   ?=
+EXTRA_SIM_CFLAGS     ?=
+EXTRA_SIM_LDFLAGS    ?=
+EXTRA_SIM_SOURCES    ?=
+EXTRA_SIM_REQS       ?=
+
+#----------------------------------------------------------------------------
+HELP_SIMULATION_VARIABLES += \
+"   EXTRA_SIM_FLAGS        = runtime simulation flags (passed within +permissive)"
+
+EXTRA_SIM_FLAGS ?=
+
+#----------------------------------------------------------------------------
+HELP_COMMANDS += \
+"   run-binary             = run [./$(shell basename $(sim))] and log instructions to file" \
+"   run-binary-fast        = run [./$(shell basename $(sim))] and don't log instructions" \
+"   run-binary-debug       = run [./$(shell basename $(sim_debug))] and log instructions and waveform to files" \
+"   verilog                = generate intermediate verilog files from chisel elaboration and firrtl passes"
 
 #########################################################################################
-# extra make variables/rules from subprojects
-#
-# EXTRA_GENERATOR_REQS - requirements needed for the main generator
-# EXTRA_SIM_FLAGS - runtime simulation flags
-# EXTRA_SIM_CC_FLAGS - cc flags for simulators
-# EXTRA_SIM_SOURCES - simulation sources needed for simulator
-# EXTRA_SIM_REQS - requirements to build the simulator
+# include additional subproject make fragments
+# see HELP_COMPILATION_VARIABLES
 #########################################################################################
 include $(base_dir)/generators/ariane/ariane.mk
 include $(base_dir)/generators/tracegen/tracegen.mk
@@ -55,7 +79,6 @@ $(FIRRTL_TEST_JAR): $(call lookup_srcs,$(CHIPYARD_FIRRTL_DIR),scala)
 	cp -p $(CHIPYARD_FIRRTL_DIR)/utils/bin/firrtl-test.jar $@
 	touch $@
 
-
 #########################################################################################
 # Bloop Project Definitions
 #########################################################################################
@@ -139,19 +162,19 @@ verilog: $(sim_vsrcs)
 #########################################################################################
 # helper rules to run simulations
 #########################################################################################
-.PHONY: run-binary run-binary-fast run-binary-debug run-fast
+.PHONY: run-binary run-binary-fast
+.PHONY: run-binary-debug
+.PHONY: run-fast
+
+# run normal binary with hardware-logged insn dissassembly
 run-binary: $(output_dir) $(sim)
 	(set -o pipefail && $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(PERMISSIVE_OFF) $(BINARY) </dev/null 2> >(spike-dasm > $(sim_out_name).out) | tee $(sim_out_name).log)
 
-#########################################################################################
-# helper rules to run simulator as fast as possible
-#########################################################################################
+# run simulator as fast as possible (no insn disassembly)
 run-binary-fast: $(output_dir) $(sim)
 	(set -o pipefail && $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(PERMISSIVE_OFF) $(BINARY) </dev/null | tee $(sim_out_name).log)
 
-#########################################################################################
-# helper rules to run simulator with as much debug info as possible
-#########################################################################################
+# run simulator with as much debug info as possible
 run-binary-debug: $(output_dir) $(sim_debug)
 	(set -o pipefail && $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(WAVEFORM_FLAG) $(PERMISSIVE_OFF) $(BINARY) </dev/null 2> >(spike-dasm > $(sim_out_name).out) | tee $(sim_out_name).log)
 
diff --git a/sims/vcs/Makefile b/sims/vcs/Makefile
index 14ebae59..46b3a267 100644
--- a/sims/vcs/Makefile
+++ b/sims/vcs/Makefile
@@ -25,7 +25,10 @@ sim_prefix = simv
 sim = $(sim_dir)/$(sim_prefix)-$(MODEL_PACKAGE)-$(CONFIG)
 sim_debug = $(sim_dir)/$(sim_prefix)-$(MODEL_PACKAGE)-$(CONFIG)-debug
 
-include $(base_dir)/vcs.mk
+PERMISSIVE_ON=+permissive
+PERMISSIVE_OFF=+permissive-off
+
+WAVEFORM_FLAG=+vcdplusfile=$(sim_out_name).vpd
 
 .PHONY: default debug
 default: $(sim)
@@ -36,22 +39,97 @@ debug: $(sim_debug)
 #########################################################################################
 include $(base_dir)/common.mk
 
+#########################################################################################
+# verilator-specific user-interface variables and commands
+#########################################################################################
+HELP_COMPILATION_VARIABLES +=
+HELP_COMMANDS += \
+"   default                = compiles non-debug simulator [./$(shell basename $(sim))]" \
+"   debug                  = compiles debug simulator [./$(shell basename $(sim_debug))]" \
+"   clean                  = remove all debug/non-debug simulators and intermediate files" \
+"   clean-sim              = removes non-debug simulator and verilator-generated files" \
+"   clean-sim-debug        = removes debug simulator and verilator-generated files"
+
 #########################################################################################
 # vcs binary and arguments
 #########################################################################################
 VCS = vcs -full64
 
-VCS_OPTS = -notice -line $(VCS_CC_OPTS) $(VCS_NONCC_OPTS) $(VCS_DEFINE_OPTS) $(EXTRA_SIM_SOURCES)
+PREPROC_DEFINES = \
+	+define+VCS \
+	+define+CLOCK_PERIOD=1.0 \
+	+define+PRINTF_COND=$(TB).printf_cond \
+	+define+STOP_COND=!$(TB).reset \
+	+define+RANDOMIZE_MEM_INIT \
+	+define+RANDOMIZE_REG_INIT \
+	+define+RANDOMIZE_GARBAGE_ASSIGN \
+	+define+RANDOMIZE_INVALID_ASSIGN
+
+VCS_NONCC_OPTS = \
+	-notice \
+	-line \
+	+lint=all,noVCDE,noONGS,noUI \
+	-timescale=1ns/1ps \
+	-quiet \
+	-q \
+	+rad \
+	+vcs+lic+wait \
+	+vc+list \
+	-error=noZMMCM \
+	-error=PCWM-L \
+	-sverilog +systemverilogext+.sv+.svi+.svh+.svt -assert svaext +libext+.sv \
+	+v2k +verilog2001ext+.v95+.vt+.vp +libext+.v \
+	+incdir+$(build_dir) \
+	$(PREPROC_DEFINES) \
+	-f $(sim_common_files) \
+	$(sim_vsrcs)
+
+#----------------------------------------------------------------------------------------
+# gcc configuration/optimization
+#----------------------------------------------------------------------------------------
+# -flto slows down compilation on small-memory and breaks on firesim-manager
+CMODE := -O3 -fbranch-probabilities -march=native
+
+VCS_CXXFLAGS = \
+	$(CXXFLAGS) \
+	$(CMODE) \
+	-I$(VCS_HOME)/include \
+	-I$(RISCV)/include \
+	-I$(dramsim_dir) \
+	-std=c++11 \
+	$(EXTRA_SIM_CXXFLAGS)
+
+VCS_LDFLAGS =	\
+	$(LDFLAGS) \
+	$(CMODE) \
+	-L$(RISCV)/lib \
+	-Wl,-rpath,$(RISCV)/lib \
+	-L$(sim_dir) \
+	-L$(dramsim_dir) \
+	-lfesvr \
+	-ldramsim \
+	$(EXTRA_SIM_LDFLAGS)
+
+VCS_CC_OPTS = \
+	-CFLAGS "$(VCS_CXXFLAGS)" \
+	-LDFLAGS "$(VCS_LDFLAGS)"
+
+#----------------------------------------------------------------------------------------
+# full vcs+gcc opts
+#----------------------------------------------------------------------------------------
+VCS_OPTS = $(VCS_CC_OPTS) $(VCS_NONCC_OPTS)
 
 #########################################################################################
 # vcs simulator rules
 #########################################################################################
 $(sim): $(sim_vsrcs) $(sim_common_files) $(dramsim_lib) $(EXTRA_SIM_REQS)
-	rm -rf csrc && $(VCS) $(VCS_OPTS) -o $@
+	rm -rf csrc && $(VCS) $(VCS_OPTS) $(EXTRA_SIM_SOURCES) -o $@ \
+	-debug_pp
 
 $(sim_debug): $(sim_vsrcs) $(sim_common_files) $(dramsim_lib) $(EXTRA_SIM_REQS)
-	rm -rf csrc && $(VCS) $(VCS_OPTS) -o $@ \
-	+define+DEBUG
+	rm -rf csrc && $(VCS) $(VCS_OPTS) $(EXTRA_SIM_SOURCES) -o $@ \
+	+define+DEBUG \
+	-debug_pp
 
 #########################################################################################
 # create a vcs vpd rule
@@ -60,9 +138,26 @@ $(sim_debug): $(sim_vsrcs) $(sim_common_files) $(dramsim_lib) $(EXTRA_SIM_REQS)
 $(output_dir)/%.vpd: $(output_dir)/% $(sim_debug)
 	(set -o pipefail && $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(VERBOSE_FLAGS) +vcdplusfile=$@ $(PERMISSIVE_OFF) $< </dev/null 2> >(spike-dasm > $<.out) | tee $<.log)
 
+$(output_dir)/none.vpd: $(sim_debug)
+	mkdir -p $(output_dir)
+	(set -o pipefail && $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(VERBOSE_FLAGS) +vcdplusfile=$@ $(PERMISSIVE_OFF) none </dev/null 2> >(spike-dasm > $(output_dir)/none.out) | tee $(output_dir)/none.log)
+
 #########################################################################################
-# general cleanup rule
+# general cleanup rules
 #########################################################################################
-.PHONY: clean
+.PHONY: clean clean-sim clean-sim-debug
 clean:
-	rm -rf $(gen_dir) csrc $(sim_prefix)-* ucli.key vc_hdrs.h
+	rm -rf $(gen_dir) $(sim_prefix)-*
+
+clean-sim:
+	rm -rf csrc/ $(sim) ucli.key vc_hdrs.h
+
+clean-sim-debug:
+	rm -rf csrc/ $(sim_debug) ucli.key vc_hdrs.h
+
+#########################################################################################
+# print help text
+#########################################################################################
+.PHONY: help
+help:
+	@for line in $(HELP_LINES); do echo "$$line"; done
diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 3d676efd..07ece1c5 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -22,7 +22,7 @@ include $(base_dir)/variables.mk
 sim_name = verilator
 
 #########################################################################################
-# vcs simulator types and rules
+# verilator simulator types and rules
 #########################################################################################
 sim_prefix = simulator
 sim = $(sim_dir)/$(sim_prefix)-$(MODEL_PACKAGE)-$(CONFIG)
@@ -47,67 +47,141 @@ debug: $(sim_debug)
 include $(base_dir)/common.mk
 
 #########################################################################################
-# verilator binary and flags
+# verilator-specific user-interface variables and commands
+#########################################################################################
+HELP_COMPILATION_VARIABLES += \
+"   VERILATOR_PROFILE      = 'none' if no verilator profiling (default)" \
+"                            'all' if full verilator runtime profiling" \
+"                            'threads' if runtime thread profiling only" \
+"   VERILATOR_FST_MODE     = enable FST waveform instead of VCD. use with debug build"
+
+HELP_COMMANDS += \
+"   default                = compiles non-debug simulator [./$(shell basename $(sim))]" \
+"   debug                  = compiles debug simulator [./$(shell basename $(sim_debug))]" \
+"   clean                  = remove all debug/non-debug simulators and intermediate files" \
+"   clean-sim              = removes non-debug simulator and verilator-generated files" \
+"   clean-sim-debug        = removes debug simulator and verilator-generated files"
+
+#########################################################################################
+# verilator/cxx binary and flags
 #########################################################################################
 VERILATOR := verilator --cc --exe
 
-CXXFLAGS := \
-	$(CXXFLAGS) -O1 -std=c++11 \
-	-I$(RISCV)/include \
-	-I$(dramsim_dir) \
-	-D__STDC_FORMAT_MACROS \
-	$(EXTRA_SIM_CC_FLAGS)
+#----------------------------------------------------------------------------------------
+# user configs
+#----------------------------------------------------------------------------------------
+VERILATOR_PROFILE ?= none
+RUNTIME_PROFILING_CFLAGS := $(if $(filter $(VERILATOR_PROFILE),all),-g -pg,)
+RUNTIME_PROFILING_VFLAGS := $(if $(filter $(VERILATOR_PROFILE),all),\
+                              --prof-threads --prof-cfuncs,\
+                              $(if $(filter $(VERILATOR_PROFILE),threads),\
+								--prof-threads,))
 
-LDFLAGS := \
-	$(LDFLAGS) \
-	-L$(sim_dir) \
-	-lpthread
+VERILATOR_FST_MODE ?= 0
+TRACING_OPTS := $(if $(filter $(VERILATOR_FST_MODE),0),\
+	                  --trace,--trace-fst --trace-threads 1)
+TRACING_CFLAGS := $(if $(filter $(VERILATOR_FST_MODE),0),,-DCY_FST_TRACE)
 
-VERILATOR_CC_OPTS = \
+#----------------------------------------------------------------------------------------
+# verilation configuration/optimization
+#----------------------------------------------------------------------------------------
+# we initially had --noassert for performance, but several modules use
+# assertions, including dramsim, so we enable --assert by default
+VMODE := \
 	-O3 \
-	-CFLAGS "$(CXXFLAGS) -DTEST_HARNESS=V$(VLOG_MODEL) -DVERILATOR" \
-	-CFLAGS "-I$(build_dir) -include $(build_dir)/$(long_name).plusArgs -include $(build_dir)/verilator.h" \
-	-LDFLAGS "$(LDFLAGS)" \
-	$(RISCV)/lib/libfesvr.a \
-	$(dramsim_lib)
+	--x-assign fast \
+	--x-initial fast \
+	--assert \
+	--output-split 10000 \
+	--output-split-cfuncs 10000
 
-# default flags added for ariane
-ARIANE_VERILATOR_FLAGS = \
+# default flags added for ariane (-Wno-fatal needed for -Wall to not cause
+# a crash, since 1000s of warnings are generated)
+VERILOG_IP_VERILATOR_FLAGS := \
 	--unroll-count 256 \
-	-Werror-PINMISSING \
-	-Werror-IMPLICIT \
 	-Wno-PINCONNECTEMPTY \
 	-Wno-ASSIGNDLY \
 	-Wno-DECLFILENAME \
 	-Wno-UNUSED \
 	-Wno-UNOPTFLAT \
 	-Wno-BLKANDNBLK \
-	-Wno-style \
-	-Wall
+	-Wno-fatal
 
-# normal flags used for chipyard builds (that are incompatible with ariane)
-CHIPYARD_VERILATOR_FLAGS = \
-	--assert
+# normal flags used for chipyard builds (that are incompatible with vlog ip aka ariane)
+CHIPYARD_VERILATOR_FLAGS :=
+
+# options dependent on whether ariane/NVDLA or chipyard is used
+# NOTE: defer the evaluation of this until it is used!
+PLATFORM_OPTS = $(shell \
+	if grep -qiP "module\s+(Ariane|NVDLA)" $(build_dir)/*.*v; \
+	then echo "$(VERILOG_IP_VERILATOR_FLAGS)"; \
+	else echo "$(CHIPYARD_VERILATOR_FLAGS)"; fi)
 
 # Use --timescale to approximate timescale behavior of pre-4.034
 TIMESCALE_OPTS := $(shell verilator --version | perl -lne 'if (/(\d.\d+)/ && $$1 >= 4.034) { print "--timescale 1ns/1ps"; }')
-VERILATOR_NONCC_OPTS = \
-	$(TIMESCALE_OPTS) \
-	--top-module $(VLOG_MODEL) \
-	--vpi \
-	-Wno-fatal \
-	$(shell if ! grep -iq "module.*ariane" $(build_dir)/*.*v; then echo "$(CHIPYARD_VERILATOR_FLAGS)"; else echo "$(ARIANE_VERILATOR_FLAGS)"; fi) \
-	--output-split 10000 \
-	--output-split-cfuncs 100 \
-	--max-num-width 1048576 \
-	-f $(sim_common_files) \
-	$(sim_vsrcs)
 
-VERILATOR_DEFINES = \
+# see: https://github.com/ucb-bar/riscv-mini/issues/31
+MAX_WIDTH_OPTS = $(shell verilator --version | perl -lne 'if (/(\d.\d+)/ && $$1 > 4.016) { print "--max-num-width 1048576"; }')
+
+PREPROC_DEFINES := \
 	+define+PRINTF_COND=\$$c\(\"verbose\",\"\&\&\"\,\"done_reset\"\) \
 	+define+STOP_COND=\$$c\(\"done_reset\"\)
 
-VERILATOR_OPTS = $(VERILATOR_CC_OPTS) $(VERILATOR_NONCC_OPTS) $(VERILATOR_DEFINES) $(EXTRA_SIM_SOURCES)
+VERILATOR_NONCC_OPTS = \
+	$(RUNTIME_PROFILING_VFLAGS) \
+	$(VMODE) \
+	$(PLATFORM_OPTS) \
+	$(TIMESCALE_OPTS) \
+	$(MAX_WIDTH_OPTS) \
+	$(PREPROC_DEFINES) \
+	--top-module $(VLOG_MODEL) \
+	--vpi \
+	-f $(sim_common_files) \
+	$(sim_vsrcs)
+
+#----------------------------------------------------------------------------------------
+# gcc configuration/optimization
+#----------------------------------------------------------------------------------------
+# -flto slows down compilation on small-memory and breaks on firesim-manager
+CMODE := -O3 -fbranch-probabilities -march=native
+
+VERILATOR_CXXFLAGS = \
+	$(CXXFLAGS) \
+	$(RUNTIME_PROFILING_CFLAGS) \
+	$(TRACING_CFLAGS) \
+	$(CMODE) \
+	-std=c++11 \
+	-D__STDC_FORMAT_MACROS \
+	-DTEST_HARNESS=V$(VLOG_MODEL) \
+	-DVERILATOR \
+	-I$(RISCV)/include \
+	-I$(dramsim_dir) \
+	-I$(build_dir) \
+	-include $(build_dir)/$(long_name).plusArgs \
+	-include $(build_dir)/verilator.h \
+	$(EXTRA_SIM_CXXFLAGS)
+
+VERILATOR_LDFLAGS =	\
+	$(LDFLAGS) \
+	$(RUNTIME_PROFILING_CFLAGS) \
+	$(CMODE) \
+	-L$(RISCV)/lib \
+	-Wl,-rpath,$(RISCV)/lib \
+	-L$(sim_dir) \
+	-L$(dramsim_dir) \
+	-lfesvr \
+	-lpthread \
+	-ldramsim \
+	$(EXTRA_SIM_LDFLAGS)
+
+VERILATOR_CC_OPTS = \
+	-CFLAGS "$(VERILATOR_CXXFLAGS)" \
+	-LDFLAGS "$(VERILATOR_LDFLAGS)"
+
+#----------------------------------------------------------------------------------------
+# full verilator+gcc opts
+#----------------------------------------------------------------------------------------
+VERILATOR_OPTS = $(VERILATOR_CC_OPTS) $(VERILATOR_NONCC_OPTS)
 
 #########################################################################################
 # verilator build paths and file names
@@ -127,13 +201,13 @@ model_mk_debug = $(model_dir_debug)/V$(VLOG_MODEL).mk
 $(model_mk): $(sim_vsrcs) $(sim_common_files) $(EXTRA_SIM_REQS)
 	rm -rf $(model_dir)
 	mkdir -p $(model_dir)
-	$(VERILATOR) $(VERILATOR_OPTS) -o $(sim) -Mdir $(model_dir) -CFLAGS "-include $(model_header)"
+	$(VERILATOR) $(VERILATOR_OPTS) $(EXTRA_SIM_SOURCES) -o $(sim) -Mdir $(model_dir) -CFLAGS "-include $(model_header)"
 	touch $@
 
 $(model_mk_debug): $(sim_vsrcs) $(sim_common_files) $(EXTRA_SIM_REQS)
 	rm -rf $(model_dir_debug)
 	mkdir -p $(model_dir_debug)
-	$(VERILATOR) $(VERILATOR_OPTS) -o $(sim_debug) --trace -Mdir $(model_dir_debug) -CFLAGS "-include $(model_header_debug)"
+	$(VERILATOR) $(VERILATOR_OPTS) $(EXTRA_SIM_SOURCES) -o $(sim_debug) --trace -Mdir $(model_dir_debug) -CFLAGS "-include $(model_header_debug)"
 	touch $@
 
 #########################################################################################
@@ -155,8 +229,21 @@ $(output_dir)/%.vpd: $(output_dir)/% $(sim_debug)
 	(set -o pipefail && $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) -v$@.vcd $(PERMISSIVE_OFF) $< </dev/null 2> >(spike-dasm > $<.out) | tee $<.log)
 
 #########################################################################################
-# general cleanup rule
+# general cleanup rules
 #########################################################################################
-.PHONY: clean
+.PHONY: clean clean-sim clean-sim-debug
 clean:
 	rm -rf $(gen_dir) $(sim_prefix)-*
+
+clean-sim:
+	rm -rf $(model_dir) $(sim)
+
+clean-sim-debug:
+	rm -rf $(model_dir_debug) $(sim_debug)
+
+#########################################################################################
+# print help text
+#########################################################################################
+.PHONY: help
+help:
+	@for line in $(HELP_LINES); do echo "$$line"; done
diff --git a/tools/dromajo/dromajo.mk b/tools/dromajo/dromajo.mk
index 067faa2f..4ac17764 100644
--- a/tools/dromajo/dromajo.mk
+++ b/tools/dromajo/dromajo.mk
@@ -49,7 +49,8 @@ ifdef ENABLE_DROMAJO
 EXTRA_SIM_FLAGS += $(DROMAJO_FLAGS)
 
 # CC flags needed for all simulations
-EXTRA_SIM_CC_FLAGS += -I$(DROMAJO_DIR)
+EXTRA_SIM_CFLAGS += -I$(DROMAJO_DIR)
+EXTRA_SIM_CXXFLAGS += -I$(DROMAJO_DIR)
 
 # sourced needed for simulation
 EXTRA_SIM_SOURCES += $(DROMAJO_LIB)
diff --git a/variables.mk b/variables.mk
index d7eccb49..e41648d4 100644
--- a/variables.mk
+++ b/variables.mk
@@ -1,23 +1,45 @@
 #########################################################################################
 # makefile variables shared across multiple makefiles
+# - to use the help text, your Makefile should have a 'help' target that just
+#   prints all the HELP_LINES
 #########################################################################################
+HELP_COMPILATION_VARIABLES =
+HELP_PROJECT_VARIABLES = \
+"   SUB_PROJECT            = use the specific subproject default variables [$(SUB_PROJECT)]" \
+"   SBT_PROJECT            = the SBT project that you should find the classes/packages in [$(SBT_PROJECT)]" \
+"   MODEL                  = the top level module of the project in Chisel (normally the harness) [$(MODEL)]" \
+"   VLOG_MODEL             = the top level module of the project in Firrtl/Verilog (normally the harness) [$(VLOG_MODEL)]" \
+"   MODEL_PACKAGE          = the scala package to find the MODEL in [$(MODEL_PACKAGE)]" \
+"   CONFIG                 = the configuration class to give the parameters for the project [$(CONFIG)]" \
+"   CONFIG_PACKAGE         = the scala package to find the CONFIG class [$(CONFIG_PACKAGE)]" \
+"   GENERATOR_PACKAGE      = the scala package to find the Generator class in [$(GENERATOR_PACKAGE)]" \
+"   TB                     = wrapper over the TestHarness needed to simulate in a verilog simulator [$(TB)]" \
+"   TOP                    = top level module of the project (normally the module instantiated by the harness) [$(TOP)]"
 
-#########################################################################################
-# variables to invoke the generator
-# descriptions:
-#   SBT_PROJECT = the SBT project that you should find the classes/packages in
-#   MODEL = the top level module of the project in Chisel (normally the harness)
-#   VLOG_MODEL = the top level module of the project in Firrtl/Verilog (normally the harness)
-#   MODEL_PACKAGE = the scala package to find the MODEL in
-#   CONFIG = the configuration class to give the parameters for the project
-#   CONFIG_PACKAGE = the scala package to find the CONFIG class
-#   GENERATOR_PACKAGE = the scala package to find the Generator class in
-#   TB = wrapper over the TestHarness needed to simulate in a verilog simulator
-#   TOP = top level module of the project (normally the module instantiated by the harness)
-#
-# project specific:
-# 	SUB_PROJECT = use the specific subproject default variables
-#########################################################################################
+HELP_SIMULATION_VARIABLES = \
+"   BINARY                 = riscv binary that the simulator will run" \
+"   VERBOSE_FLAGS          = flags used when doing verbose simulation [$(VERBOSE_FLAGS)]"
+
+HELP_COMMANDS = \
+"   help                   = display this help"
+
+HELP_LINES = "" \
+	" design specifier variables:" \
+	" ---------------------------" \
+	$(HELP_PROJECT_VARIABLES) \
+	"" \
+	" compilation variables:" \
+	" ----------------------" \
+	$(HELP_COMPILATION_VARIABLES) \
+	"" \
+	" simulation variables:" \
+	" ---------------------" \
+	$(HELP_SIMULATION_VARIABLES) \
+	"" \
+	" some useful general commands:" \
+	" -----------------" \
+	$(HELP_COMMANDS) \
+	""
 
 #########################################################################################
 # subproject overrides
@@ -140,15 +162,15 @@ override SCALA_BUILDTOOL_DEPS += $(BLOOP_CONFIG_DIR)/TIMESTAMP
 # 1) the sed removes a leading {file:<path>} that sometimes needs to be
 #    provided to SBT when a project but not for bloop.
 # 2) Generally, one could could pass '--' to indicate all remaining arguments are
-# 	 destined for the scala Main, however a bug in Bloop's argument parsing causes the
+#       destined for the scala Main, however a bug in Bloop's argument parsing causes the
 #    --nailgun-port argument to be lost in this case. Workaround this by prefixing
 #    every main-destined argument with "--args"
 define run_scala_main
-	cd $(base_dir) && bloop --nailgun-port $(BLOOP_NAILGUN_PORT) run $(shell echo $(1) | sed 's/{.*}//') --main $(2)  $(addprefix --args ,$3)
+       cd $(base_dir) && bloop --nailgun-port $(BLOOP_NAILGUN_PORT) run $(shell echo $(1) | sed 's/{.*}//') --main $(2)  $(addprefix --args ,$3)
 endef
 else
 define run_scala_main
-	cd $(base_dir) && $(SBT) "project $(1)" "runMain $(2) $(3)"
+       cd $(base_dir) && $(SBT) "project $(1)" "runMain $(2) $(3)"
 endef
 endif
 

From d82e7dbed5d46acce92e029d925875ebe8eaddbf Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Tue, 18 Aug 2020 10:40:45 -0700
Subject: [PATCH 08/74] Cleanup more

---
 docs/Simulation/Software-RTL-Simulation.rst | 28 +++++++++++++++++++++
 sims/vcs/Makefile                           |  9 +------
 variables.mk                                |  6 ++---
 3 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/docs/Simulation/Software-RTL-Simulation.rst b/docs/Simulation/Software-RTL-Simulation.rst
index ca852d36..9219f5a3 100644
--- a/docs/Simulation/Software-RTL-Simulation.rst
+++ b/docs/Simulation/Software-RTL-Simulation.rst
@@ -61,6 +61,11 @@ For instance, to run one of the riscv-tools assembly tests.
 
 .. Note:: In a VCS simulator, the simulator name will be ``simv-chipyard-RocketConfig`` instead of ``simulator-chipyard-RocketConfig``.
 
+The makefiles have a ``run-binary`` rule that simplifies running the simulation executable. It adds many of the common command line options for you and redirects the output to a file.
+
+.. code-block:: shell
+    make run-binary BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/isa/rv64ui-p-simple
+
 Alternatively, we can run a pre-packaged suite of RISC-V assembly or benchmark tests, by adding the make target ``run-asm-tests`` or ``run-bmark-tests``.
 For example:
 
@@ -141,6 +146,29 @@ All ``make`` targets that can be applied to the default example, can also be app
 Finally, in the ``generated-src/<...>-<package>-<config>/`` directory resides all of the collateral and Verilog source files for the build/simulation.
 Specifically, the SoC top-level (``TOP``) Verilog file is denoted with ``*.top.v`` while the ``TestHarness`` file is denoted with ``*.harness.v``.
 
+Fast Memory Loading
+-------------------
+
+The simulator loads the program binary over a simulated serial line. This can be quite slow if there is a lot of static data, so the simulator also allows data to be loaded from a file directly into the DRAM model.
+
+.. code-block:: shell
+
+    make run-binary BINARY=test.riscv LOADMEM=testdata.hex LOADMEM_ADDR=81000000
+
+The ``.hex`` file should be a text file with a hexadecimal number on each line.
+
+.. code-block:: text
+
+    deadbeef
+    0123
+
+Each line uses little-endian order, so this file would produce the bytes "ef be ad de 01 23". ``LOADMEM_ADDR`` specifies which address in memory (in hexadecimal) to write the first byte to. The default is 0x81000000.
+
+A special target that facilitates automatically generating a hex file for an entire elf RISC-V exectuable and then running the simulator with the appropriate flags is also available.
+
+.. code-block:: shell
+
+    make run-binary-hex BINARY=test.riscv
 
 Generating Waveforms
 -----------------------
diff --git a/sims/vcs/Makefile b/sims/vcs/Makefile
index 46b3a267..766350ff 100644
--- a/sims/vcs/Makefile
+++ b/sims/vcs/Makefile
@@ -25,10 +25,7 @@ sim_prefix = simv
 sim = $(sim_dir)/$(sim_prefix)-$(MODEL_PACKAGE)-$(CONFIG)
 sim_debug = $(sim_dir)/$(sim_prefix)-$(MODEL_PACKAGE)-$(CONFIG)-debug
 
-PERMISSIVE_ON=+permissive
-PERMISSIVE_OFF=+permissive-off
-
-WAVEFORM_FLAG=+vcdplusfile=$(sim_out_name).vpd
+include $(base_dir)/vcs.mk
 
 .PHONY: default debug
 default: $(sim)
@@ -138,10 +135,6 @@ $(sim_debug): $(sim_vsrcs) $(sim_common_files) $(dramsim_lib) $(EXTRA_SIM_REQS)
 $(output_dir)/%.vpd: $(output_dir)/% $(sim_debug)
 	(set -o pipefail && $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(VERBOSE_FLAGS) +vcdplusfile=$@ $(PERMISSIVE_OFF) $< </dev/null 2> >(spike-dasm > $<.out) | tee $<.log)
 
-$(output_dir)/none.vpd: $(sim_debug)
-	mkdir -p $(output_dir)
-	(set -o pipefail && $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(VERBOSE_FLAGS) +vcdplusfile=$@ $(PERMISSIVE_OFF) none </dev/null 2> >(spike-dasm > $(output_dir)/none.out) | tee $(output_dir)/none.log)
-
 #########################################################################################
 # general cleanup rules
 #########################################################################################
diff --git a/variables.mk b/variables.mk
index e41648d4..e347ccf0 100644
--- a/variables.mk
+++ b/variables.mk
@@ -162,15 +162,15 @@ override SCALA_BUILDTOOL_DEPS += $(BLOOP_CONFIG_DIR)/TIMESTAMP
 # 1) the sed removes a leading {file:<path>} that sometimes needs to be
 #    provided to SBT when a project but not for bloop.
 # 2) Generally, one could could pass '--' to indicate all remaining arguments are
-#       destined for the scala Main, however a bug in Bloop's argument parsing causes the
+#    destined for the scala Main, however a bug in Bloop's argument parsing causes the
 #    --nailgun-port argument to be lost in this case. Workaround this by prefixing
 #    every main-destined argument with "--args"
 define run_scala_main
-       cd $(base_dir) && bloop --nailgun-port $(BLOOP_NAILGUN_PORT) run $(shell echo $(1) | sed 's/{.*}//') --main $(2)  $(addprefix --args ,$3)
+	cd $(base_dir) && bloop --nailgun-port $(BLOOP_NAILGUN_PORT) run $(shell echo $(1) | sed 's/{.*}//') --main $(2)  $(addprefix --args ,$3)
 endef
 else
 define run_scala_main
-       cd $(base_dir) && $(SBT) "project $(1)" "runMain $(2) $(3)"
+	cd $(base_dir) && $(SBT) "project $(1)" "runMain $(2) $(3)"
 endef
 endif
 

From c3749ce88f5f498fb0b9e19703ae2664aa4a8492 Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Tue, 18 Aug 2020 10:41:35 -0700
Subject: [PATCH 09/74] Add space

---
 docs/Simulation/Software-RTL-Simulation.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/Simulation/Software-RTL-Simulation.rst b/docs/Simulation/Software-RTL-Simulation.rst
index 9219f5a3..af568ef4 100644
--- a/docs/Simulation/Software-RTL-Simulation.rst
+++ b/docs/Simulation/Software-RTL-Simulation.rst
@@ -64,6 +64,7 @@ For instance, to run one of the riscv-tools assembly tests.
 The makefiles have a ``run-binary`` rule that simplifies running the simulation executable. It adds many of the common command line options for you and redirects the output to a file.
 
 .. code-block:: shell
+
     make run-binary BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/isa/rv64ui-p-simple
 
 Alternatively, we can run a pre-packaged suite of RISC-V assembly or benchmark tests, by adding the make target ``run-asm-tests`` or ``run-bmark-tests``.

From 4e7b9d195fef6c036469281891403af6bfb7f4da Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Tue, 18 Aug 2020 10:45:11 -0700
Subject: [PATCH 10/74] Dedup default simulation rules

---
 sims/vcs/Makefile       | 11 -----------
 sims/verilator/Makefile |  7 -------
 variables.mk            |  8 +++++++-
 3 files changed, 7 insertions(+), 19 deletions(-)

diff --git a/sims/vcs/Makefile b/sims/vcs/Makefile
index 766350ff..3f0c0792 100644
--- a/sims/vcs/Makefile
+++ b/sims/vcs/Makefile
@@ -36,17 +36,6 @@ debug: $(sim_debug)
 #########################################################################################
 include $(base_dir)/common.mk
 
-#########################################################################################
-# verilator-specific user-interface variables and commands
-#########################################################################################
-HELP_COMPILATION_VARIABLES +=
-HELP_COMMANDS += \
-"   default                = compiles non-debug simulator [./$(shell basename $(sim))]" \
-"   debug                  = compiles debug simulator [./$(shell basename $(sim_debug))]" \
-"   clean                  = remove all debug/non-debug simulators and intermediate files" \
-"   clean-sim              = removes non-debug simulator and verilator-generated files" \
-"   clean-sim-debug        = removes debug simulator and verilator-generated files"
-
 #########################################################################################
 # vcs binary and arguments
 #########################################################################################
diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 07ece1c5..e2f85495 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -55,13 +55,6 @@ HELP_COMPILATION_VARIABLES += \
 "                            'threads' if runtime thread profiling only" \
 "   VERILATOR_FST_MODE     = enable FST waveform instead of VCD. use with debug build"
 
-HELP_COMMANDS += \
-"   default                = compiles non-debug simulator [./$(shell basename $(sim))]" \
-"   debug                  = compiles debug simulator [./$(shell basename $(sim_debug))]" \
-"   clean                  = remove all debug/non-debug simulators and intermediate files" \
-"   clean-sim              = removes non-debug simulator and verilator-generated files" \
-"   clean-sim-debug        = removes debug simulator and verilator-generated files"
-
 #########################################################################################
 # verilator/cxx binary and flags
 #########################################################################################
diff --git a/variables.mk b/variables.mk
index e347ccf0..1522aa2e 100644
--- a/variables.mk
+++ b/variables.mk
@@ -20,8 +20,14 @@ HELP_SIMULATION_VARIABLES = \
 "   BINARY                 = riscv binary that the simulator will run" \
 "   VERBOSE_FLAGS          = flags used when doing verbose simulation [$(VERBOSE_FLAGS)]"
 
+# include default simulation rules
 HELP_COMMANDS = \
-"   help                   = display this help"
+"   help                   = display this help" \
+"   default                = compiles non-debug simulator [./$(shell basename $(sim))]" \
+"   debug                  = compiles debug simulator [./$(shell basename $(sim_debug))]" \
+"   clean                  = remove all debug/non-debug simulators and intermediate files" \
+"   clean-sim              = removes non-debug simulator and simulator-generated files" \
+"   clean-sim-debug        = removes debug simulator and simulator-generated files"
 
 HELP_LINES = "" \
 	" design specifier variables:" \

From 3b991f3ed7c27b2d6b98fdf14ba8521263719e88 Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Tue, 18 Aug 2020 11:14:01 -0700
Subject: [PATCH 11/74] Move vcs flags to vcs.mk | Misc. cleanup

---
 common.mk                |  2 --
 sims/vcs/Makefile        | 70 ++--------------------------------------
 sims/verilator/Makefile  |  1 -
 tools/dromajo/dromajo.mk |  1 -
 vcs.mk                   | 38 ++++++++++++++++++----
 vlsi/Makefile            | 10 ++++--
 6 files changed, 42 insertions(+), 80 deletions(-)

diff --git a/common.mk b/common.mk
index 87d4047c..be787a9d 100644
--- a/common.mk
+++ b/common.mk
@@ -14,7 +14,6 @@ endif
 #########################################################################################
 HELP_COMPILATION_VARIABLES += \
 "   EXTRA_GENERATOR_REQS   = requirements needed for the main generator" \
-"   EXTRA_SIM_CFLAGS       = CFLAGS for building simulators" \
 "   EXTRA_SIM_CXXFLAGS     = CXXFLAGS for building simulators" \
 "   EXTRA_SIM_LDFLAGS      = LDFLAGS for building simulators" \
 "   EXTRA_SIM_SOURCES      = simulation sources needed for simulator" \
@@ -22,7 +21,6 @@ HELP_COMPILATION_VARIABLES += \
 
 EXTRA_GENERATOR_REQS ?=
 EXTRA_SIM_CXXFLAGS   ?=
-EXTRA_SIM_CFLAGS     ?=
 EXTRA_SIM_LDFLAGS    ?=
 EXTRA_SIM_SOURCES    ?=
 EXTRA_SIM_REQS       ?=
diff --git a/sims/vcs/Makefile b/sims/vcs/Makefile
index 3f0c0792..efe0dc3c 100644
--- a/sims/vcs/Makefile
+++ b/sims/vcs/Makefile
@@ -41,81 +41,17 @@ include $(base_dir)/common.mk
 #########################################################################################
 VCS = vcs -full64
 
-PREPROC_DEFINES = \
-	+define+VCS \
-	+define+CLOCK_PERIOD=1.0 \
-	+define+PRINTF_COND=$(TB).printf_cond \
-	+define+STOP_COND=!$(TB).reset \
-	+define+RANDOMIZE_MEM_INIT \
-	+define+RANDOMIZE_REG_INIT \
-	+define+RANDOMIZE_GARBAGE_ASSIGN \
-	+define+RANDOMIZE_INVALID_ASSIGN
-
-VCS_NONCC_OPTS = \
-	-notice \
-	-line \
-	+lint=all,noVCDE,noONGS,noUI \
-	-timescale=1ns/1ps \
-	-quiet \
-	-q \
-	+rad \
-	+vcs+lic+wait \
-	+vc+list \
-	-error=noZMMCM \
-	-error=PCWM-L \
-	-sverilog +systemverilogext+.sv+.svi+.svh+.svt -assert svaext +libext+.sv \
-	+v2k +verilog2001ext+.v95+.vt+.vp +libext+.v \
-	+incdir+$(build_dir) \
-	$(PREPROC_DEFINES) \
-	-f $(sim_common_files) \
-	$(sim_vsrcs)
-
-#----------------------------------------------------------------------------------------
-# gcc configuration/optimization
-#----------------------------------------------------------------------------------------
-# -flto slows down compilation on small-memory and breaks on firesim-manager
-CMODE := -O3 -fbranch-probabilities -march=native
-
-VCS_CXXFLAGS = \
-	$(CXXFLAGS) \
-	$(CMODE) \
-	-I$(VCS_HOME)/include \
-	-I$(RISCV)/include \
-	-I$(dramsim_dir) \
-	-std=c++11 \
-	$(EXTRA_SIM_CXXFLAGS)
-
-VCS_LDFLAGS =	\
-	$(LDFLAGS) \
-	$(CMODE) \
-	-L$(RISCV)/lib \
-	-Wl,-rpath,$(RISCV)/lib \
-	-L$(sim_dir) \
-	-L$(dramsim_dir) \
-	-lfesvr \
-	-ldramsim \
-	$(EXTRA_SIM_LDFLAGS)
-
-VCS_CC_OPTS = \
-	-CFLAGS "$(VCS_CXXFLAGS)" \
-	-LDFLAGS "$(VCS_LDFLAGS)"
-
-#----------------------------------------------------------------------------------------
-# full vcs+gcc opts
-#----------------------------------------------------------------------------------------
-VCS_OPTS = $(VCS_CC_OPTS) $(VCS_NONCC_OPTS)
+VCS_OPTS = $(VCS_CC_OPTS) $(VCS_NONCC_OPTS) $(PREPROC_DEFINES)
 
 #########################################################################################
 # vcs simulator rules
 #########################################################################################
 $(sim): $(sim_vsrcs) $(sim_common_files) $(dramsim_lib) $(EXTRA_SIM_REQS)
-	rm -rf csrc && $(VCS) $(VCS_OPTS) $(EXTRA_SIM_SOURCES) -o $@ \
-	-debug_pp
+	rm -rf csrc && $(VCS) $(VCS_OPTS) $(EXTRA_SIM_SOURCES) -o $@
 
 $(sim_debug): $(sim_vsrcs) $(sim_common_files) $(dramsim_lib) $(EXTRA_SIM_REQS)
 	rm -rf csrc && $(VCS) $(VCS_OPTS) $(EXTRA_SIM_SOURCES) -o $@ \
-	+define+DEBUG \
-	-debug_pp
+	+define+DEBUG
 
 #########################################################################################
 # create a vcs vpd rule
diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index e2f85495..2cba66ab 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -157,7 +157,6 @@ VERILATOR_CXXFLAGS = \
 VERILATOR_LDFLAGS =	\
 	$(LDFLAGS) \
 	$(RUNTIME_PROFILING_CFLAGS) \
-	$(CMODE) \
 	-L$(RISCV)/lib \
 	-Wl,-rpath,$(RISCV)/lib \
 	-L$(sim_dir) \
diff --git a/tools/dromajo/dromajo.mk b/tools/dromajo/dromajo.mk
index 4ac17764..2069b394 100644
--- a/tools/dromajo/dromajo.mk
+++ b/tools/dromajo/dromajo.mk
@@ -49,7 +49,6 @@ ifdef ENABLE_DROMAJO
 EXTRA_SIM_FLAGS += $(DROMAJO_FLAGS)
 
 # CC flags needed for all simulations
-EXTRA_SIM_CFLAGS += -I$(DROMAJO_DIR)
 EXTRA_SIM_CXXFLAGS += -I$(DROMAJO_DIR)
 
 # sourced needed for simulation
diff --git a/vcs.mk b/vcs.mk
index 93e75c19..28ed0f51 100644
--- a/vcs.mk
+++ b/vcs.mk
@@ -17,9 +17,37 @@ VCS_CC_OPTS = \
 	-CC "-std=c++11" \
 	-CC "$(EXTRA_SIM_CC_FLAGS)"
 
+#----------------------------------------------------------------------------------------
+# gcc configuration/optimization
+#----------------------------------------------------------------------------------------
+# -flto slows down compilation on small-memory and breaks on firesim-manager
+CMODE := -O3 -fbranch-probabilities -march=native
+
+VCS_CXXFLAGS = \
+	$(CXXFLAGS) \
+	$(CMODE) \
+	-I$(RISCV)/include \
+	-I$(dramsim_dir) \
+	-std=c++11 \
+	$(EXTRA_SIM_CXXFLAGS)
+
+VCS_LDFLAGS =	\
+	$(LDFLAGS) \
+	-L$(RISCV)/lib \
+	-Wl,-rpath,$(RISCV)/lib \
+	-L$(sim_dir) \
+	-L$(dramsim_dir) \
+	-lfesvr \
+	-ldramsim \
+	$(EXTRA_SIM_LDFLAGS)
+
+VCS_CC_OPTS = \
+	-CFLAGS "$(VCS_CXXFLAGS)" \
+	-LDFLAGS "$(VCS_LDFLAGS)"
+
 VCS_NONCC_OPTS = \
-	$(dramsim_lib) \
-	$(RISCV)/lib/libfesvr.a \
+	-notice \
+	-line \
 	+lint=all,noVCDE,noONGS,noUI \
 	-error=PCWM-L \
 	-error=noZMMCM \
@@ -27,7 +55,6 @@ VCS_NONCC_OPTS = \
 	-quiet \
 	-q \
 	+rad \
-	+v2k \
 	+vcs+lic+wait \
 	+vc+list \
 	-f $(sim_common_files) \
@@ -35,10 +62,9 @@ VCS_NONCC_OPTS = \
 	+v2k +verilog2001ext+.v95+.vt+.vp +libext+.v \
 	-debug_pp \
 	+incdir+$(build_dir) \
-	$(sim_vsrcs) \
-	+libext+.v
+	$(sim_vsrcs)
 
-VCS_DEFINE_OPTS = \
+PREPROC_DEFINES = \
 	+define+VCS \
 	+define+CLOCK_PERIOD=$(CLOCK_PERIOD) \
 	+define+RESET_DELAY=$(RESET_DELAY) \
diff --git a/vlsi/Makefile b/vlsi/Makefile
index a9e3d3a5..3bbca7e0 100644
--- a/vlsi/Makefile
+++ b/vlsi/Makefile
@@ -115,12 +115,16 @@ $(SIM_CONF): $(VLSI_RTL) $(HARNESS_FILE) $(HARNESS_SMEMS_FILE) $(sim_common_file
 	done
 	echo "  options_meta: 'append'" >> $@
 	echo "  defines:" >> $@
-	for x in $(subst +define+,,$(VCS_DEFINE_OPTS)); do \
+	for x in $(subst +define+,,$(PREPROC_DEFINES)); do \
 		echo '    - "'$$x'"' >> $@; \
 	done
 	echo "  defines_meta: 'append'" >> $@
-	echo "  compiler_opts:" >> $@
-	for x in $(filter-out "",$(filter-out -CC,$(VCS_CC_OPTS))); do \
+	echo "  compiler_cc_opts:" >> $@
+	for x in $(filter-out "",$(VCS_CXXFLAGS)); do \
+		echo '    - "'$$x'"' >> $@; \
+	done
+	echo "  compiler_ld_opts:" >> $@
+	for x in $(filter-out "",$(VCS_LDFLAGS)); do \
 		echo '    - "'$$x'"' >> $@; \
 	done
 	echo "  compiler_opts_meta: 'append'" >> $@

From 15f508bcbf9b5dda5de5db74bf0319a31221defd Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Tue, 18 Aug 2020 11:36:48 -0700
Subject: [PATCH 12/74] First pass at updating hammer submodules

---
 vcs.mk                       | 6 ------
 vlsi/hammer                  | 2 +-
 vlsi/hammer-synopsys-plugins | 2 +-
 3 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/vcs.mk b/vcs.mk
index 28ed0f51..9367cca9 100644
--- a/vcs.mk
+++ b/vcs.mk
@@ -11,12 +11,6 @@ endif
 CLOCK_PERIOD ?= 1.0
 RESET_DELAY ?= 777.7
 
-VCS_CC_OPTS = \
-	-CC "-I$(RISCV)/include" \
-	-CC "-I$(dramsim_dir)" \
-	-CC "-std=c++11" \
-	-CC "$(EXTRA_SIM_CC_FLAGS)"
-
 #----------------------------------------------------------------------------------------
 # gcc configuration/optimization
 #----------------------------------------------------------------------------------------
diff --git a/vlsi/hammer b/vlsi/hammer
index cbc907df..56739c32 160000
--- a/vlsi/hammer
+++ b/vlsi/hammer
@@ -1 +1 @@
-Subproject commit cbc907dfe8005a8d72f1b2fb7b414ad9dbfe14b1
+Subproject commit 56739c32a81c82bdb065613cc3aa878f1ca40d4c
diff --git a/vlsi/hammer-synopsys-plugins b/vlsi/hammer-synopsys-plugins
index e5ec0da8..bd0e5c90 160000
--- a/vlsi/hammer-synopsys-plugins
+++ b/vlsi/hammer-synopsys-plugins
@@ -1 +1 @@
-Subproject commit e5ec0da8ad471b075de62989001b282e537416d0
+Subproject commit bd0e5c90126cb517928082517cf9af24ec27bfbb

From 8743f3ab95ca65d96f9482949a2dcb8fbd04523e Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Tue, 18 Aug 2020 11:43:40 -0700
Subject: [PATCH 13/74] Bump hammer

---
 vlsi/hammer | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vlsi/hammer b/vlsi/hammer
index 56739c32..da9c84f3 160000
--- a/vlsi/hammer
+++ b/vlsi/hammer
@@ -1 +1 @@
-Subproject commit 56739c32a81c82bdb065613cc3aa878f1ca40d4c
+Subproject commit da9c84f35c1b2df0feaba11395ba4e4570ab5b67

From aca96a7f4d2e89c36f9731a6542b43fbb9744428 Mon Sep 17 00:00:00 2001
From: Jacob Gadikian <jacobgadikian@gmail.com>
Date: Wed, 19 Aug 2020 01:49:15 +0700
Subject: [PATCH 14/74] Update ubuntu-req.sh (#645)

Use more cores when we can, and fewer when we can't use more
---
 scripts/ubuntu-req.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/ubuntu-req.sh b/scripts/ubuntu-req.sh
index f72c48f4..a4697174 100755
--- a/scripts/ubuntu-req.sh
+++ b/scripts/ubuntu-req.sh
@@ -24,4 +24,4 @@ sudo apt-get install -y device-tree-compiler
 git clone http://git.veripool.org/git/verilator
 cd verilator
 git checkout v4.034
-autoconf && ./configure && make -j16 && sudo make install
+autoconf && ./configure && make -j$(nproc) && sudo make install

From 9bc9d48fe467c67fbd558b413a022c50031605f9 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Tue, 18 Aug 2020 21:23:13 -0700
Subject: [PATCH 15/74] Fix vlsi/Makefile opts meta

---
 vlsi/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/vlsi/Makefile b/vlsi/Makefile
index 3bbca7e0..fe63c346 100644
--- a/vlsi/Makefile
+++ b/vlsi/Makefile
@@ -123,11 +123,12 @@ $(SIM_CONF): $(VLSI_RTL) $(HARNESS_FILE) $(HARNESS_SMEMS_FILE) $(sim_common_file
 	for x in $(filter-out "",$(VCS_CXXFLAGS)); do \
 		echo '    - "'$$x'"' >> $@; \
 	done
+	echo "  compiler_cc_opts_meta: 'append'" >> $@
 	echo "  compiler_ld_opts:" >> $@
 	for x in $(filter-out "",$(VCS_LDFLAGS)); do \
 		echo '    - "'$$x'"' >> $@; \
 	done
-	echo "  compiler_opts_meta: 'append'" >> $@
+	echo "  compiler_ls_opts_meta: 'append'" >> $@
 	echo "  execution_flags_prepend: ['$(PERMISSIVE_ON)']" >> $@
 	echo "  execution_flags_append: ['$(PERMISSIVE_OFF)']" >> $@
 	echo "  execution_flags:" >> $@

From d020058d1994df92e853064b1fa8c7bfa2a04c90 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Tue, 18 Aug 2020 23:35:03 -0700
Subject: [PATCH 16/74] Split up LDFLAGS into LDFLAGS and -l

---
 vcs.mk                       | 4 +++-
 vlsi/hammer-synopsys-plugins | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/vcs.mk b/vcs.mk
index 9367cca9..daed8af4 100644
--- a/vcs.mk
+++ b/vcs.mk
@@ -35,9 +35,11 @@ VCS_LDFLAGS =	\
 	-ldramsim \
 	$(EXTRA_SIM_LDFLAGS)
 
+# vcs requires LDFLAGS to not include library names (i.e. -l needs to be separate)
 VCS_CC_OPTS = \
 	-CFLAGS "$(VCS_CXXFLAGS)" \
-	-LDFLAGS "$(VCS_LDFLAGS)"
+	-LDFLAGS "$(filter-out -l%,$(VCS_LDFLAGS))" \
+	$(filter -l%,$(VCS_LDFLAGS))
 
 VCS_NONCC_OPTS = \
 	-notice \
diff --git a/vlsi/hammer-synopsys-plugins b/vlsi/hammer-synopsys-plugins
index bd0e5c90..706e3f6e 160000
--- a/vlsi/hammer-synopsys-plugins
+++ b/vlsi/hammer-synopsys-plugins
@@ -1 +1 @@
-Subproject commit bd0e5c90126cb517928082517cf9af24ec27bfbb
+Subproject commit 706e3f6e8dbba8f2e36b009daaf6cf9596ab68e9

From d4af9e16600a67cfeb5c18b464968651ab63f432 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Wed, 19 Aug 2020 10:48:43 -0700
Subject: [PATCH 17/74] Fix separator

---
 variables.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/variables.mk b/variables.mk
index 1522aa2e..be022e12 100644
--- a/variables.mk
+++ b/variables.mk
@@ -43,7 +43,7 @@ HELP_LINES = "" \
 	$(HELP_SIMULATION_VARIABLES) \
 	"" \
 	" some useful general commands:" \
-	" -----------------" \
+	" -----------------------------" \
 	$(HELP_COMMANDS) \
 	""
 

From de4e311dd341080202b43254750075abbe1a1dd6 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Wed, 19 Aug 2020 11:21:35 -0700
Subject: [PATCH 18/74] Remove extra CMODE flags

---
 sims/verilator/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 2cba66ab..f71f9de5 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -136,7 +136,7 @@ VERILATOR_NONCC_OPTS = \
 # gcc configuration/optimization
 #----------------------------------------------------------------------------------------
 # -flto slows down compilation on small-memory and breaks on firesim-manager
-CMODE := -O3 -fbranch-probabilities -march=native
+CMODE := -O3
 
 VERILATOR_CXXFLAGS = \
 	$(CXXFLAGS) \

From 0dd2197477a1e578925dcb2bebd7f65b7bdf39f4 Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.gonzalez@berkeley.edu>
Date: Wed, 19 Aug 2020 21:48:19 -0700
Subject: [PATCH 19/74] Fix meta key

---
 vlsi/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vlsi/Makefile b/vlsi/Makefile
index fe63c346..c0a6fdd3 100644
--- a/vlsi/Makefile
+++ b/vlsi/Makefile
@@ -128,7 +128,7 @@ $(SIM_CONF): $(VLSI_RTL) $(HARNESS_FILE) $(HARNESS_SMEMS_FILE) $(sim_common_file
 	for x in $(filter-out "",$(VCS_LDFLAGS)); do \
 		echo '    - "'$$x'"' >> $@; \
 	done
-	echo "  compiler_ls_opts_meta: 'append'" >> $@
+	echo "  compiler_ld_opts_meta: 'append'" >> $@
 	echo "  execution_flags_prepend: ['$(PERMISSIVE_ON)']" >> $@
 	echo "  execution_flags_append: ['$(PERMISSIVE_OFF)']" >> $@
 	echo "  execution_flags:" >> $@

From b7d9472b4ad6a6ba3ad85b3845de812e1be3cfe7 Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.gonzalez@berkeley.edu>
Date: Wed, 19 Aug 2020 22:10:18 -0700
Subject: [PATCH 20/74] Cleanup help commands

---
 common.mk               | 22 +++++++++++++++-------
 sims/vcs/Makefile       |  7 -------
 sims/verilator/Makefile |  7 -------
 variables.mk            |  2 +-
 4 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/common.mk b/common.mk
index be787a9d..83b74287 100644
--- a/common.mk
+++ b/common.mk
@@ -13,11 +13,11 @@ endif
 # specify user-interface variables
 #########################################################################################
 HELP_COMPILATION_VARIABLES += \
-"   EXTRA_GENERATOR_REQS   = requirements needed for the main generator" \
-"   EXTRA_SIM_CXXFLAGS     = CXXFLAGS for building simulators" \
-"   EXTRA_SIM_LDFLAGS      = LDFLAGS for building simulators" \
-"   EXTRA_SIM_SOURCES      = simulation sources needed for simulator" \
-"   EXTRA_SIM_REQS         = requirements to build the simulator"
+"   EXTRA_GENERATOR_REQS   = additional make requirements needed for the main generator" \
+"   EXTRA_SIM_CXXFLAGS     = additional CXXFLAGS for building simulators" \
+"   EXTRA_SIM_LDFLAGS      = additional LDFLAGS for building simulators" \
+"   EXTRA_SIM_SOURCES      = additional simulation sources needed for simulator" \
+"   EXTRA_SIM_REQS         = additional make requirements to build the simulator"
 
 EXTRA_GENERATOR_REQS ?=
 EXTRA_SIM_CXXFLAGS   ?=
@@ -27,7 +27,7 @@ EXTRA_SIM_REQS       ?=
 
 #----------------------------------------------------------------------------
 HELP_SIMULATION_VARIABLES += \
-"   EXTRA_SIM_FLAGS        = runtime simulation flags (passed within +permissive)"
+"   EXTRA_SIM_FLAGS        = additional runtime simulation flags (passed within +permissive)"
 
 EXTRA_SIM_FLAGS ?=
 
@@ -36,7 +36,8 @@ HELP_COMMANDS += \
 "   run-binary             = run [./$(shell basename $(sim))] and log instructions to file" \
 "   run-binary-fast        = run [./$(shell basename $(sim))] and don't log instructions" \
 "   run-binary-debug       = run [./$(shell basename $(sim_debug))] and log instructions and waveform to files" \
-"   verilog                = generate intermediate verilog files from chisel elaboration and firrtl passes"
+"   verilog                = generate intermediate verilog files from chisel elaboration and firrtl passes" \
+"   run-tests              = run all assembly and benchmark tests"
 
 #########################################################################################
 # include additional subproject make fragments
@@ -232,6 +233,13 @@ dramsim_lib = $(dramsim_dir)/libdramsim.a
 $(dramsim_lib):
 	$(MAKE) -C $(dramsim_dir) $(notdir $@)
 
+#########################################################################################
+# print help text
+#########################################################################################
+.PHONY: help
+help:
+	@for line in $(HELP_LINES); do echo "$$line"; done
+
 #########################################################################################
 # Implicit rule handling
 #########################################################################################
diff --git a/sims/vcs/Makefile b/sims/vcs/Makefile
index efe0dc3c..ac792e63 100644
--- a/sims/vcs/Makefile
+++ b/sims/vcs/Makefile
@@ -72,10 +72,3 @@ clean-sim:
 
 clean-sim-debug:
 	rm -rf csrc/ $(sim_debug) ucli.key vc_hdrs.h
-
-#########################################################################################
-# print help text
-#########################################################################################
-.PHONY: help
-help:
-	@for line in $(HELP_LINES); do echo "$$line"; done
diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index f71f9de5..9965001d 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -232,10 +232,3 @@ clean-sim:
 
 clean-sim-debug:
 	rm -rf $(model_dir_debug) $(sim_debug)
-
-#########################################################################################
-# print help text
-#########################################################################################
-.PHONY: help
-help:
-	@for line in $(HELP_LINES); do echo "$$line"; done
diff --git a/variables.mk b/variables.mk
index be022e12..7f3d1eee 100644
--- a/variables.mk
+++ b/variables.mk
@@ -17,7 +17,7 @@ HELP_PROJECT_VARIABLES = \
 "   TOP                    = top level module of the project (normally the module instantiated by the harness) [$(TOP)]"
 
 HELP_SIMULATION_VARIABLES = \
-"   BINARY                 = riscv binary that the simulator will run" \
+"   BINARY                 = riscv elf binary that the simulator will run when using the run-binary* targets" \
 "   VERBOSE_FLAGS          = flags used when doing verbose simulation [$(VERBOSE_FLAGS)]"
 
 # include default simulation rules

From af61c533dab801301d00eb8afd7ebf064b64b156 Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Wed, 19 Aug 2020 22:15:55 -0700
Subject: [PATCH 21/74] Merge .PHONY variables

---
 common.mk | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/common.mk b/common.mk
index 83b74287..4aa0a903 100644
--- a/common.mk
+++ b/common.mk
@@ -161,9 +161,7 @@ verilog: $(sim_vsrcs)
 #########################################################################################
 # helper rules to run simulations
 #########################################################################################
-.PHONY: run-binary run-binary-fast
-.PHONY: run-binary-debug
-.PHONY: run-fast
+.PHONY: run-binary run-binary-fast run-binary-debug run-fast
 
 # run normal binary with hardware-logged insn dissassembly
 run-binary: $(output_dir) $(sim)

From 4f3319dc010304c74078dead4f87d5b7b653c6cc Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Wed, 19 Aug 2020 22:16:45 -0700
Subject: [PATCH 22/74] Revert make clean for VCS

---
 sims/vcs/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/vcs/Makefile b/sims/vcs/Makefile
index ac792e63..a2896380 100644
--- a/sims/vcs/Makefile
+++ b/sims/vcs/Makefile
@@ -65,7 +65,7 @@ $(output_dir)/%.vpd: $(output_dir)/% $(sim_debug)
 #########################################################################################
 .PHONY: clean clean-sim clean-sim-debug
 clean:
-	rm -rf $(gen_dir) $(sim_prefix)-*
+	rm -rf $(gen_dir) csrc $(sim_prefix)-* ucli.key vc_hdrs.h
 
 clean-sim:
 	rm -rf csrc/ $(sim) ucli.key vc_hdrs.h

From e5158cbe4c4d9fd7554961a1e26291a08eb4e58b Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Wed, 19 Aug 2020 22:32:47 -0700
Subject: [PATCH 23/74] Rename some variables

---
 sims/verilator/Makefile | 10 ++++++----
 vcs.mk                  |  4 ++--
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 9965001d..52c78eb6 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -80,7 +80,7 @@ TRACING_CFLAGS := $(if $(filter $(VERILATOR_FST_MODE),0),,-DCY_FST_TRACE)
 #----------------------------------------------------------------------------------------
 # we initially had --noassert for performance, but several modules use
 # assertions, including dramsim, so we enable --assert by default
-VMODE := \
+VERILATOR_OPT_FLAGS := \
 	-O3 \
 	--x-assign fast \
 	--x-initial fast \
@@ -92,6 +92,8 @@ VMODE := \
 # a crash, since 1000s of warnings are generated)
 VERILOG_IP_VERILATOR_FLAGS := \
 	--unroll-count 256 \
+	-Werror-PINMISSING \
+	-Werror-IMPLICIT \
 	-Wno-PINCONNECTEMPTY \
 	-Wno-ASSIGNDLY \
 	-Wno-DECLFILENAME \
@@ -122,7 +124,7 @@ PREPROC_DEFINES := \
 
 VERILATOR_NONCC_OPTS = \
 	$(RUNTIME_PROFILING_VFLAGS) \
-	$(VMODE) \
+	$(VERILATOR_OPT_FLAGS) \
 	$(PLATFORM_OPTS) \
 	$(TIMESCALE_OPTS) \
 	$(MAX_WIDTH_OPTS) \
@@ -136,13 +138,13 @@ VERILATOR_NONCC_OPTS = \
 # gcc configuration/optimization
 #----------------------------------------------------------------------------------------
 # -flto slows down compilation on small-memory and breaks on firesim-manager
-CMODE := -O3
+CXX_OPT_FLAGS := -O3
 
 VERILATOR_CXXFLAGS = \
 	$(CXXFLAGS) \
 	$(RUNTIME_PROFILING_CFLAGS) \
 	$(TRACING_CFLAGS) \
-	$(CMODE) \
+	$(CXX_OPT_FLAGS) \
 	-std=c++11 \
 	-D__STDC_FORMAT_MACROS \
 	-DTEST_HARNESS=V$(VLOG_MODEL) \
diff --git a/vcs.mk b/vcs.mk
index daed8af4..ef5761df 100644
--- a/vcs.mk
+++ b/vcs.mk
@@ -15,11 +15,11 @@ RESET_DELAY ?= 777.7
 # gcc configuration/optimization
 #----------------------------------------------------------------------------------------
 # -flto slows down compilation on small-memory and breaks on firesim-manager
-CMODE := -O3 -fbranch-probabilities -march=native
+CXX_OPT_FLAGS := -O3 -fbranch-probabilities -march=native
 
 VCS_CXXFLAGS = \
 	$(CXXFLAGS) \
-	$(CMODE) \
+	$(CXX_OPT_FLAGS) \
 	-I$(RISCV)/include \
 	-I$(dramsim_dir) \
 	-std=c++11 \

From 9087b58cf08c722902c9c10d139f3f6db8f01db3 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Wed, 19 Aug 2020 22:35:02 -0700
Subject: [PATCH 24/74] Add testbench to description

---
 variables.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/variables.mk b/variables.mk
index 7f3d1eee..b187a23d 100644
--- a/variables.mk
+++ b/variables.mk
@@ -13,7 +13,7 @@ HELP_PROJECT_VARIABLES = \
 "   CONFIG                 = the configuration class to give the parameters for the project [$(CONFIG)]" \
 "   CONFIG_PACKAGE         = the scala package to find the CONFIG class [$(CONFIG_PACKAGE)]" \
 "   GENERATOR_PACKAGE      = the scala package to find the Generator class in [$(GENERATOR_PACKAGE)]" \
-"   TB                     = wrapper over the TestHarness needed to simulate in a verilog simulator [$(TB)]" \
+"   TB                     = testbench wrapper over the TestHarness needed to simulate in a verilog simulator [$(TB)]" \
 "   TOP                    = top level module of the project (normally the module instantiated by the harness) [$(TOP)]"
 
 HELP_SIMULATION_VARIABLES = \

From e77e610ee4920873b44398fb32e09dddcf30ebca Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Wed, 19 Aug 2020 23:42:41 -0700
Subject: [PATCH 25/74] Fix Hammer Sim

---
 vcs.mk | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/vcs.mk b/vcs.mk
index ef5761df..a282aa44 100644
--- a/vcs.mk
+++ b/vcs.mk
@@ -14,8 +14,7 @@ RESET_DELAY ?= 777.7
 #----------------------------------------------------------------------------------------
 # gcc configuration/optimization
 #----------------------------------------------------------------------------------------
-# -flto slows down compilation on small-memory and breaks on firesim-manager
-CXX_OPT_FLAGS := -O3 -fbranch-probabilities -march=native
+CXX_OPT_FLAGS := -O3
 
 VCS_CXXFLAGS = \
 	$(CXXFLAGS) \

From 9d3c14034dac9daa763ff9cad82326624d3ae5fb Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Thu, 20 Aug 2020 10:40:21 -0700
Subject: [PATCH 26/74] Revert Verilator external IP flags

---
 sims/verilator/Makefile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 52c78eb6..685b7939 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -88,8 +88,7 @@ VERILATOR_OPT_FLAGS := \
 	--output-split 10000 \
 	--output-split-cfuncs 10000
 
-# default flags added for ariane (-Wno-fatal needed for -Wall to not cause
-# a crash, since 1000s of warnings are generated)
+# default flags added for external IP (ariane/NVDLA)
 VERILOG_IP_VERILATOR_FLAGS := \
 	--unroll-count 256 \
 	-Werror-PINMISSING \
@@ -100,12 +99,13 @@ VERILOG_IP_VERILATOR_FLAGS := \
 	-Wno-UNUSED \
 	-Wno-UNOPTFLAT \
 	-Wno-BLKANDNBLK \
-	-Wno-fatal
+	-Wno-style \
+	-Wall
 
 # normal flags used for chipyard builds (that are incompatible with vlog ip aka ariane)
 CHIPYARD_VERILATOR_FLAGS :=
 
-# options dependent on whether ariane/NVDLA or chipyard is used
+# options dependent on whether external IP (ariane/NVDLA) or just chipyard is used
 # NOTE: defer the evaluation of this until it is used!
 PLATFORM_OPTS = $(shell \
 	if grep -qiP "module\s+(Ariane|NVDLA)" $(build_dir)/*.*v; \

From 435bfac45e6fa073923eca003cb9e8236074ad2f Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Thu, 20 Aug 2020 16:10:14 -0700
Subject: [PATCH 27/74] Fix NVDLA/Ariane builds

---
 sims/verilator/Makefile | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 685b7939..10b28cb7 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -84,15 +84,12 @@ VERILATOR_OPT_FLAGS := \
 	-O3 \
 	--x-assign fast \
 	--x-initial fast \
-	--assert \
 	--output-split 10000 \
 	--output-split-cfuncs 10000
 
 # default flags added for external IP (ariane/NVDLA)
 VERILOG_IP_VERILATOR_FLAGS := \
 	--unroll-count 256 \
-	-Werror-PINMISSING \
-	-Werror-IMPLICIT \
 	-Wno-PINCONNECTEMPTY \
 	-Wno-ASSIGNDLY \
 	-Wno-DECLFILENAME \
@@ -102,8 +99,9 @@ VERILOG_IP_VERILATOR_FLAGS := \
 	-Wno-style \
 	-Wall
 
-# normal flags used for chipyard builds (that are incompatible with vlog ip aka ariane)
-CHIPYARD_VERILATOR_FLAGS :=
+# normal flags used for chipyard builds (that are incompatible with vlog ip aka ariane/NVDLA)
+CHIPYARD_VERILATOR_FLAGS := \
+	--assert
 
 # options dependent on whether external IP (ariane/NVDLA) or just chipyard is used
 # NOTE: defer the evaluation of this until it is used!
@@ -126,6 +124,7 @@ VERILATOR_NONCC_OPTS = \
 	$(RUNTIME_PROFILING_VFLAGS) \
 	$(VERILATOR_OPT_FLAGS) \
 	$(PLATFORM_OPTS) \
+	-Wno-fatal \
 	$(TIMESCALE_OPTS) \
 	$(MAX_WIDTH_OPTS) \
 	$(PREPROC_DEFINES) \

From fdda4cf8f5f6704c6814a2cdb48bfae7ec1b258d Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Thu, 20 Aug 2020 16:21:39 -0700
Subject: [PATCH 28/74] Update the cfuncs split

---
 sims/verilator/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 10b28cb7..295729b7 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -85,7 +85,7 @@ VERILATOR_OPT_FLAGS := \
 	--x-assign fast \
 	--x-initial fast \
 	--output-split 10000 \
-	--output-split-cfuncs 10000
+	--output-split-cfuncs 100
 
 # default flags added for external IP (ariane/NVDLA)
 VERILOG_IP_VERILATOR_FLAGS := \

From 51d8d403be75c87e4cd20f83aac90e487f3caa88 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Thu, 20 Aug 2020 23:17:25 -0700
Subject: [PATCH 29/74] bump hammer/hammer-synopsys plugins

---
 vlsi/hammer                  | 2 +-
 vlsi/hammer-synopsys-plugins | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/vlsi/hammer b/vlsi/hammer
index da9c84f3..bed4d340 160000
--- a/vlsi/hammer
+++ b/vlsi/hammer
@@ -1 +1 @@
-Subproject commit da9c84f35c1b2df0feaba11395ba4e4570ab5b67
+Subproject commit bed4d34094fa4c72db37a0066050c475eb5e37b2
diff --git a/vlsi/hammer-synopsys-plugins b/vlsi/hammer-synopsys-plugins
index 706e3f6e..f8a79222 160000
--- a/vlsi/hammer-synopsys-plugins
+++ b/vlsi/hammer-synopsys-plugins
@@ -1 +1 @@
-Subproject commit 706e3f6e8dbba8f2e36b009daaf6cf9596ab68e9
+Subproject commit f8a7922220c70b6905b37ab30bda6c791b594792

From 425b8ce8503c40b3ee61af60b135134a6f322bd0 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Thu, 20 Aug 2020 23:37:17 -0700
Subject: [PATCH 30/74] Add support for multi-threaded verilator

---
 common.mk                                     |  16 ++-
 docs/Simulation/Software-RTL-Simulation.rst   |  15 +++
 .../src/main/resources/csrc/emulator.cc       | 104 +++++++++++-------
 scripts/numa_prefix                           |  67 +++++++++++
 sims/verilator/Makefile                       |   7 +-
 5 files changed, 161 insertions(+), 48 deletions(-)
 create mode 100755 scripts/numa_prefix

diff --git a/common.mk b/common.mk
index 4aa0a903..89ebbea3 100644
--- a/common.mk
+++ b/common.mk
@@ -27,9 +27,13 @@ EXTRA_SIM_REQS       ?=
 
 #----------------------------------------------------------------------------
 HELP_SIMULATION_VARIABLES += \
-"   EXTRA_SIM_FLAGS        = additional runtime simulation flags (passed within +permissive)"
+"   EXTRA_SIM_FLAGS        = additional runtime simulation flags (passed within +permissive)" \
+"   NUMACTL                = set to '1' to wrap simulator in the appropriate numactl command"
 
 EXTRA_SIM_FLAGS ?=
+NUMACTL         ?= 0
+
+NUMA_PREFIX = $(if $(filter $(NUMACTL),0),,$(shell $(base_dir)/scripts/numa_prefix))
 
 #----------------------------------------------------------------------------
 HELP_COMMANDS += \
@@ -165,15 +169,15 @@ verilog: $(sim_vsrcs)
 
 # run normal binary with hardware-logged insn dissassembly
 run-binary: $(output_dir) $(sim)
-	(set -o pipefail && $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(PERMISSIVE_OFF) $(BINARY) </dev/null 2> >(spike-dasm > $(sim_out_name).out) | tee $(sim_out_name).log)
+	(set -o pipefail && $(NUMA_PREFIX) $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(PERMISSIVE_OFF) $(BINARY) </dev/null 2> >(spike-dasm > $(sim_out_name).out) | tee $(sim_out_name).log)
 
 # run simulator as fast as possible (no insn disassembly)
 run-binary-fast: $(output_dir) $(sim)
-	(set -o pipefail && $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(PERMISSIVE_OFF) $(BINARY) </dev/null | tee $(sim_out_name).log)
+	(set -o pipefail && $(NUMA_PREFIX) $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(PERMISSIVE_OFF) $(BINARY) </dev/null | tee $(sim_out_name).log)
 
 # run simulator with as much debug info as possible
 run-binary-debug: $(output_dir) $(sim_debug)
-	(set -o pipefail && $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(WAVEFORM_FLAG) $(PERMISSIVE_OFF) $(BINARY) </dev/null 2> >(spike-dasm > $(sim_out_name).out) | tee $(sim_out_name).log)
+	(set -o pipefail && $(NUMA_PREFIX) $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(WAVEFORM_FLAG) $(PERMISSIVE_OFF) $(BINARY) </dev/null 2> >(spike-dasm > $(sim_out_name).out) | tee $(sim_out_name).log)
 
 run-fast: run-asm-tests-fast run-bmark-tests-fast
 
@@ -209,10 +213,10 @@ $(output_dir)/%: $(RISCV)/riscv64-unknown-elf/share/riscv-tests/isa/% $(output_d
 	ln -sf $< $@
 
 $(output_dir)/%.run: $(output_dir)/% $(sim)
-	(set -o pipefail && $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(PERMISSIVE_OFF) $< </dev/null | tee $<.log) && touch $@
+	(set -o pipefail && $(NUMA_PREFIX) $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(PERMISSIVE_OFF) $< </dev/null | tee $<.log) && touch $@
 
 $(output_dir)/%.out: $(output_dir)/% $(sim)
-	(set -o pipefail && $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(PERMISSIVE_OFF) $< </dev/null 2> >(spike-dasm > $@) | tee $<.log)
+	(set -o pipefail && $(NUMA_PREFIX) $(sim) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) $(PERMISSIVE_OFF) $< </dev/null 2> >(spike-dasm > $@) | tee $<.log)
 
 #########################################################################################
 # include build/project specific makefrags made from the generator
diff --git a/docs/Simulation/Software-RTL-Simulation.rst b/docs/Simulation/Software-RTL-Simulation.rst
index af568ef4..6d7fb2b8 100644
--- a/docs/Simulation/Software-RTL-Simulation.rst
+++ b/docs/Simulation/Software-RTL-Simulation.rst
@@ -181,3 +181,18 @@ An open-source vcd-capable waveform viewer is `GTKWave <http://gtkwave.sourcefor
 
 For a VCS simulation, this will generate a vpd file (this is a proprietary waveform representation format used by Synopsys) that can be loaded to vpd-supported waveform viewers.
 If you have Synopsys licenses, we recommend using the DVE waveform viewer.
+
+.. _sw-sim-verilator-opts:
+
+Additional Verilator Options
+-------------------------------
+
+When building the verilator simulator there are some additional options:
+
+.. code-block:: shell
+
+   make VERILATOR_THREADS=8
+
+The ``VERILATOR_THREADS=<num>`` option enables the compiled Verilator simulator to use ``<num>`` parallel threads.
+On a multi-socket machine, you will want to make sure all threads are on the same socket by using ``numactl``.
+You can also just use the ``numa_prefix`` wrapper, which is a simple wrapper around ``numactl`` that runs your verilated simulator like this: ``$(numa_prefix) ./simulator-<name> <simulator-args>``.
diff --git a/generators/utilities/src/main/resources/csrc/emulator.cc b/generators/utilities/src/main/resources/csrc/emulator.cc
index 27a8aa4a..dc8827a9 100644
--- a/generators/utilities/src/main/resources/csrc/emulator.cc
+++ b/generators/utilities/src/main/resources/csrc/emulator.cc
@@ -1,11 +1,15 @@
 // See LICENSE.SiFive for license details.
 // See LICENSE.Berkeley for license details.
 
-#include "verilated.h"
 #if VM_TRACE
 #include <memory>
+#if CY_FST_TRACE
+#include "verilated_fst_c.h"
+#else
+#include "verilated.h"
 #include "verilated_vcd_c.h"
-#endif
+#endif // CY_FST_TRACE
+#endif // VM_TRACE
 #include <fesvr/dtm.h>
 #include <fesvr/tsi.h>
 #include "remote_bitbang.h"
@@ -16,6 +20,8 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <getopt.h>
+// needed for s_vpi_vlog_info, which is needed for multithreading
+#include <vpi_user.h>
 
 // For option parsing, which is split across this file, Verilog, and
 // FESVR's HTIF, a few external files must be pulled in. The list of
@@ -35,6 +41,7 @@
 extern tsi_t* tsi;
 extern dtm_t* dtm;
 extern remote_bitbang_t * jtag;
+extern int dramsim;
 
 static uint64_t trace_count = 0;
 bool verbose = false;
@@ -50,6 +57,18 @@ double sc_time_stamp()
   return trace_count;
 }
 
+// need to pull htif_argc/htif_argv out here so the thread that calls tick()
+// for the HTIF device can initialize properly with the cmdline args. this
+// was pulled out here for multithreading to work
+static int htif_argc;
+static char **htif_argv = NULL;
+extern "C" int vpi_get_vlog_info(s_vpi_vlog_info *vlog_info_s)
+{
+  vlog_info_s->argc = htif_argc;
+  vlog_info_s->argv = htif_argv;
+  return 1;
+}
+
 static void usage(const char * program_name)
 {
   printf("Usage: %s [EMULATOR OPTION]... [VERILOG PLUSARG]... [HOST OPTION]... BINARY [TARGET OPTION]...\n",
@@ -113,34 +132,34 @@ int main(int argc, char** argv)
   // Port numbers are 16 bit unsigned integers. 
   uint16_t rbb_port = 0;
 #if VM_TRACE
+  const char* vcdfile_name = NULL; 
   FILE * vcdfile = NULL;
   uint64_t start = 0;
 #endif
   int verilog_plusargs_legal = 1;
 
-  opterr = 1;
+  dramsim = 0;
 
   while (1) {
     static struct option long_options[] = {
-      {"cycle-count",     no_argument,       0, 'c' },
-      {"help",            no_argument,       0, 'h' },
-      {"max-cycles",      required_argument, 0, 'm' },
-      {"seed",            required_argument, 0, 's' },
-      {"rbb-port",        required_argument, 0, 'r' },
-      {"verbose",         no_argument,       0, 'V' },
-      {"permissive",      no_argument,       0, 'p' },
-      {"permissive-off",  no_argument,       0, 'o' },
+      {"cycle-count", no_argument,       0, 'c' },
+      {"help",        no_argument,       0, 'h' },
+      {"max-cycles",  required_argument, 0, 'm' },
+      {"seed",        required_argument, 0, 's' },
+      {"rbb-port",    required_argument, 0, 'r' },
+      {"verbose",     no_argument,       0, 'V' },
+      {"dramsim",     no_argument,       0, 'D' },
 #if VM_TRACE
-      {"vcd",             required_argument, 0, 'v' },
-      {"dump-start",      required_argument, 0, 'x' },
+      {"vcd",         required_argument, 0, 'v' },
+      {"dump-start",  required_argument, 0, 'x' },
 #endif
       HTIF_LONG_OPTIONS
     };
     int option_index = 0;
 #if VM_TRACE
-    int c = getopt_long(argc, argv, "-chm:s:r:v:Vx:po", long_options, &option_index);
+    int c = getopt_long(argc, argv, "-chm:s:r:v:Vx:D", long_options, &option_index);
 #else
-    int c = getopt_long(argc, argv, "-chm:s:r:Vpo", long_options, &option_index);
+    int c = getopt_long(argc, argv, "-chm:s:r:VD", long_options, &option_index);
 #endif
     if (c == -1) break;
  retry:
@@ -153,10 +172,10 @@ int main(int argc, char** argv)
       case 's': random_seed = atoi(optarg); break;
       case 'r': rbb_port = atoi(optarg);    break;
       case 'V': verbose = true;             break;
-      case 'p': opterr = 0;                 break;
-      case 'o': opterr = 1;                 break;
+      case 'D': dramsim = 1;                break;
 #if VM_TRACE
       case 'v': {
+        vcdfile_name = optarg;
         vcdfile = strcmp(optarg, "-") == 0 ? stdout : fopen(optarg, "w");
         if (!vcdfile) {
           std::cerr << "Unable to open " << optarg << " for VCD write\n";
@@ -188,10 +207,8 @@ int main(int argc, char** argv)
 #endif
         else if (arg.substr(0, 12) == "+cycle-count")
           c = 'c';
-        else if (arg == "+permissive")
-          c = 'p';
-        else if (arg == "+permissive-off")
-          c = 'o';
+        else if (arg == "+dramsim")
+          c = 'D';
         // If we don't find a legacy '+' EMULATOR argument, it still could be
         // a VERILOG_PLUSARG and not an error.
         else if (verilog_plusargs_legal) {
@@ -223,13 +240,9 @@ int main(int argc, char** argv)
             }
             htif_option++;
           }
-          if(opterr) {
-            std::cerr << argv[0] << ": invalid plus-arg (Verilog or HTIF) \""
-                      << arg << "\"\n";
-            c = '?';
-          } else {
-            c = 'p';
-          }
+          std::cerr << argv[0] << ": invalid plus-arg (Verilog or HTIF) \""
+                    << arg << "\"\n";
+          c = '?';
         }
         goto retry;
       }
@@ -251,6 +264,10 @@ done_processing:
     usage(argv[0]);
     return 1;
   }
+  htif_argc = 1 + argc - optind;
+  htif_argv = (char **) malloc((htif_argc) * sizeof (char *));
+  htif_argv[0] = argv[0];
+  for (int i = 1; optind < argc;) htif_argv[i++] = argv[optind++];
 
   if (verbose)
     fprintf(stderr, "using random seed %u\n", random_seed);
@@ -264,17 +281,17 @@ done_processing:
 
 #if VM_TRACE
   Verilated::traceEverOn(true); // Verilator must compute traced signals
+#if CY_FST_TRACE
+  std::unique_ptr<VerilatedFstC> tfp(new VerilatedFstC);
+#else
   std::unique_ptr<VerilatedVcdFILE> vcdfd(new VerilatedVcdFILE(vcdfile));
   std::unique_ptr<VerilatedVcdC> tfp(new VerilatedVcdC(vcdfd.get()));
-  if (vcdfile) {
+#endif // CY_FST_TRACE
+  if (vcdfile_name) {
     tile->trace(tfp.get(), 99);  // Trace 99 levels of hierarchy
-    tfp->open("");
+    tfp->open(vcdfile_name);
   }
-#endif
-
-  jtag = new remote_bitbang_t(rbb_port);
-  dtm = new dtm_t(argc, argv);
-  tsi = new tsi_t(argc, argv);
+#endif // VM_TRACE
 
   signal(SIGTERM, handle_sigterm);
 
@@ -304,8 +321,7 @@ done_processing:
   tile->reset = 0;
   done_reset = true;
 
-  while (!dtm->done() && !jtag->done() && !tsi->done() &&
-         !tile->io_success && trace_count < max_cycles) {
+  do {
     tile->clock = 0;
     tile->eval();
 #if VM_TRACE
@@ -322,6 +338,13 @@ done_processing:
 #endif
     trace_count++;
   }
+  // for verilator multithreading. need to do 1 loop before checking if
+  // tsi exists, since tsi is created by verilated thread on the first 
+  // serial_tick. 
+  while ((!dtm || !dtm->done()) && 
+         (!jtag || !jtag->done()) && 
+         (!tsi || !tsi->done()) &&
+         !tile->io_success && trace_count < max_cycles);
 
 #if VM_TRACE
   if (tfp)
@@ -330,17 +353,17 @@ done_processing:
     fclose(vcdfile);
 #endif
 
-  if (dtm->exit_code())
+  if (dtm && dtm->exit_code())
   {
     fprintf(stderr, "*** FAILED *** via dtm (code = %d, seed %d) after %ld cycles\n", dtm->exit_code(), random_seed, trace_count);
     ret = dtm->exit_code();
   }
-  else if (tsi->exit_code())
+  else if (tsi && tsi->exit_code())
   {
     fprintf(stderr, "*** FAILED *** (code = %d, seed %d) after %ld cycles\n", tsi->exit_code(), random_seed, trace_count);
     ret = tsi->exit_code();
   }
-  else if (jtag->exit_code())
+  else if (jtag && jtag->exit_code())
   {
     fprintf(stderr, "*** FAILED *** via jtag (code = %d, seed %d) after %ld cycles\n", jtag->exit_code(), random_seed, trace_count);
     ret = jtag->exit_code();
@@ -359,5 +382,6 @@ done_processing:
   if (tsi) delete tsi;
   if (jtag) delete jtag;
   if (tile) delete tile;
+  if (htif_argv) free(htif_argv);
   return ret;
 }
diff --git a/scripts/numa_prefix b/scripts/numa_prefix
new file mode 100755
index 00000000..af2d2eb1
--- /dev/null
+++ b/scripts/numa_prefix
@@ -0,0 +1,67 @@
+#!/usr/bin/env perl
+
+#============================================================================
+# - really simple script, which just prints out the numactl cmd to
+#   prefix before your actual command. it determines this based on free
+#   memory size attached to every node.
+# - when you run this on a machine without `numactl`, the output is empty,
+#   so `$(numa_prefix) <cmd> <args>` turns in to `<cmd> <args>`.
+# - when the machine has `numactl` installed, regardless of the socket-count
+#   on the machine, the resulting command is:
+#   `numactl -m <socket> -C <core-id list> -- <cmd> <args>`
+# - example output from `numactl -H` on a 2 socket machine:
+#     available: 2 nodes (0-1)
+#     node 0 cpus: 0 2 4 6 8 10 12 14 16 18 20 22
+#     node 0 size: 131026 MB
+#     node 0 free: 7934 MB
+#     node 1 cpus: 1 3 5 7 9 11 13 15 17 19 21 23
+#     node 1 size: 65536 MB
+#     node 1 free: 429 MB
+#     node distances:
+#     node   0   1
+#       0:  10  20
+#       1:  20  10
+#============================================================================
+
+use strict;
+use warnings;
+
+my $path = `which numactl`;
+if(length($path) > 0) {
+  my ($head_line, @rest) = map {chomp; $_} `numactl -H`;
+
+  if($head_line =~ /available: (\d+) nodes/) {
+    my $node_count     = $1;
+    my $best_node_id   = undef
+    my $best_cpus      = undef;
+    my $best_free_size = undef;
+
+		# loop through available nodes, selecting the node with the most free mem
+    foreach my $num (1..$node_count) {
+      my $cpus_line     = shift(@rest);
+      my $mem_size_line = shift(@rest);
+      my $mem_free_line = shift(@rest);
+
+      if($cpus_line =~ /node (\d+) cpus: (\d.*\d)$/) {
+        my ($node_id, $cpus) = ($1, $2);
+        $cpus =~ s/\s+/,/g;
+
+        if($mem_free_line =~ /node $node_id free: (\d+) \S+$/) {
+          my $free_size = $1;
+          if(!defined($best_free_size) || ($free_size > $best_free_size)) {
+            $best_node_id   = $node_id;
+            $best_cpus      = $cpus;
+            $best_free_size = $free_size;
+          }
+        } else {
+          die("malformed mem-free line: $mem_free_line\n");
+        }
+      } else {
+        die("malformed cpus line: $cpus_line\n");
+      }
+    }
+    print("numactl -m $best_node_id -C $best_cpus --");
+  } else {
+    die("malformed head line: $head_line\n");
+  }
+}
diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 295729b7..5ce75683 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -70,6 +70,9 @@ RUNTIME_PROFILING_VFLAGS := $(if $(filter $(VERILATOR_PROFILE),all),\
                               $(if $(filter $(VERILATOR_PROFILE),threads),\
 								--prof-threads,))
 
+VERILATOR_THREADS ?= 1
+RUNTIME_THREADS := --threads $(VERILATOR_THREADS) --threads-dpi all
+
 VERILATOR_FST_MODE ?= 0
 TRACING_OPTS := $(if $(filter $(VERILATOR_FST_MODE),0),\
 	                  --trace,--trace-fst --trace-threads 1)
@@ -122,6 +125,7 @@ PREPROC_DEFINES := \
 
 VERILATOR_NONCC_OPTS = \
 	$(RUNTIME_PROFILING_VFLAGS) \
+	$(RUNTIME_THREADS) \
 	$(VERILATOR_OPT_FLAGS) \
 	$(PLATFORM_OPTS) \
 	-Wno-fatal \
@@ -157,7 +161,6 @@ VERILATOR_CXXFLAGS = \
 
 VERILATOR_LDFLAGS =	\
 	$(LDFLAGS) \
-	$(RUNTIME_PROFILING_CFLAGS) \
 	-L$(RISCV)/lib \
 	-Wl,-rpath,$(RISCV)/lib \
 	-L$(sim_dir) \
@@ -219,7 +222,7 @@ $(sim_debug): $(model_mk_debug) $(dramsim_lib)
 $(output_dir)/%.vpd: $(output_dir)/% $(sim_debug)
 	rm -f $@.vcd && mkfifo $@.vcd
 	vcd2vpd $@.vcd $@ > /dev/null &
-	(set -o pipefail && $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) -v$@.vcd $(PERMISSIVE_OFF) $< </dev/null 2> >(spike-dasm > $<.out) | tee $<.log)
+	(set -o pipefail && $(NUMA_PREFIX) $(sim_debug) $(PERMISSIVE_ON) $(SIM_FLAGS) $(EXTRA_SIM_FLAGS) $(SEED_FLAG) $(VERBOSE_FLAGS) -v$@.vcd $(PERMISSIVE_OFF) $< </dev/null 2> >(spike-dasm > $<.out) | tee $<.log)
 
 #########################################################################################
 # general cleanup rules

From 446631543b1ae93db282b20de4076d0ede54f4dd Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Fri, 21 Aug 2020 11:20:45 -0700
Subject: [PATCH 31/74] Use TRACING_OPTS in debug Verilator

---
 sims/verilator/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 5ce75683..f945cb01 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -203,7 +203,7 @@ $(model_mk): $(sim_vsrcs) $(sim_common_files) $(EXTRA_SIM_REQS)
 $(model_mk_debug): $(sim_vsrcs) $(sim_common_files) $(EXTRA_SIM_REQS)
 	rm -rf $(model_dir_debug)
 	mkdir -p $(model_dir_debug)
-	$(VERILATOR) $(VERILATOR_OPTS) $(EXTRA_SIM_SOURCES) -o $(sim_debug) --trace -Mdir $(model_dir_debug) -CFLAGS "-include $(model_header_debug)"
+	$(VERILATOR) $(VERILATOR_OPTS) $(EXTRA_SIM_SOURCES) -o $(sim_debug) $(TRACING_OPTS) -Mdir $(model_dir_debug) -CFLAGS "-include $(model_header_debug)"
 	touch $@
 
 #########################################################################################

From c9791ccbdfda3bd9a688026a2f659422a6d09fc7 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Fri, 21 Aug 2020 12:06:18 -0700
Subject: [PATCH 32/74] Update docs | Revert/Update emulator.cc

---
 docs/Simulation/Software-RTL-Simulation.rst   |  7 +-
 .../src/main/resources/csrc/emulator.cc       | 69 +++++++++++--------
 2 files changed, 43 insertions(+), 33 deletions(-)

diff --git a/docs/Simulation/Software-RTL-Simulation.rst b/docs/Simulation/Software-RTL-Simulation.rst
index 6d7fb2b8..32164f6d 100644
--- a/docs/Simulation/Software-RTL-Simulation.rst
+++ b/docs/Simulation/Software-RTL-Simulation.rst
@@ -191,8 +191,9 @@ When building the verilator simulator there are some additional options:
 
 .. code-block:: shell
 
-   make VERILATOR_THREADS=8
+   make VERILATOR_THREADS=8 NUMACTL=1
 
 The ``VERILATOR_THREADS=<num>`` option enables the compiled Verilator simulator to use ``<num>`` parallel threads.
-On a multi-socket machine, you will want to make sure all threads are on the same socket by using ``numactl``.
-You can also just use the ``numa_prefix`` wrapper, which is a simple wrapper around ``numactl`` that runs your verilated simulator like this: ``$(numa_prefix) ./simulator-<name> <simulator-args>``.
+On a multi-socket machine, you will want to make sure all threads are on the same socket by using ``NUMACTL=1`` to enable ``numactl``.
+By enabling this, you will use CHipyard's ``numa_prefix`` wrapper, which is a simple wrapper around ``numactl`` that runs your verilated simulator like this: ``$(numa_prefix) ./simulator-<name> <simulator-args>``.
+Note that both these flags are mutually exclusive, you can use either independently (though it makes sense to use ``NUMACTL`` just with ``VERILATOR_THREADS=8`` during a Verilator simulation).
diff --git a/generators/utilities/src/main/resources/csrc/emulator.cc b/generators/utilities/src/main/resources/csrc/emulator.cc
index dc8827a9..e92c4688 100644
--- a/generators/utilities/src/main/resources/csrc/emulator.cc
+++ b/generators/utilities/src/main/resources/csrc/emulator.cc
@@ -41,7 +41,6 @@
 extern tsi_t* tsi;
 extern dtm_t* dtm;
 extern remote_bitbang_t * jtag;
-extern int dramsim;
 
 static uint64_t trace_count = 0;
 bool verbose = false;
@@ -129,37 +128,38 @@ int main(int argc, char** argv)
   uint64_t max_cycles = -1;
   int ret = 0;
   bool print_cycles = false;
-  // Port numbers are 16 bit unsigned integers. 
+  // Port numbers are 16 bit unsigned integers.
   uint16_t rbb_port = 0;
 #if VM_TRACE
-  const char* vcdfile_name = NULL; 
+  const char* vcdfile_name = NULL;
   FILE * vcdfile = NULL;
   uint64_t start = 0;
 #endif
   int verilog_plusargs_legal = 1;
 
-  dramsim = 0;
+  opterr = 1;
 
   while (1) {
     static struct option long_options[] = {
-      {"cycle-count", no_argument,       0, 'c' },
-      {"help",        no_argument,       0, 'h' },
-      {"max-cycles",  required_argument, 0, 'm' },
-      {"seed",        required_argument, 0, 's' },
-      {"rbb-port",    required_argument, 0, 'r' },
-      {"verbose",     no_argument,       0, 'V' },
-      {"dramsim",     no_argument,       0, 'D' },
+      {"cycle-count",     no_argument,       0, 'c' },
+      {"help",            no_argument,       0, 'h' },
+      {"max-cycles",      required_argument, 0, 'm' },
+      {"seed",            required_argument, 0, 's' },
+      {"rbb-port",        required_argument, 0, 'r' },
+      {"verbose",         no_argument,       0, 'V' },
+      {"permissive",      no_argument,       0, 'p' },
+      {"permissive-off",  no_argument,       0, 'o' },
 #if VM_TRACE
-      {"vcd",         required_argument, 0, 'v' },
-      {"dump-start",  required_argument, 0, 'x' },
+      {"vcd",             required_argument, 0, 'v' },
+      {"dump-start",      required_argument, 0, 'x' },
 #endif
       HTIF_LONG_OPTIONS
     };
     int option_index = 0;
 #if VM_TRACE
-    int c = getopt_long(argc, argv, "-chm:s:r:v:Vx:D", long_options, &option_index);
+    int c = getopt_long(argc, argv, "-chm:s:r:v:Vx:po", long_options, &option_index);
 #else
-    int c = getopt_long(argc, argv, "-chm:s:r:VD", long_options, &option_index);
+    int c = getopt_long(argc, argv, "-chm:s:r:Vpo", long_options, &option_index);
 #endif
     if (c == -1) break;
  retry:
@@ -172,7 +172,8 @@ int main(int argc, char** argv)
       case 's': random_seed = atoi(optarg); break;
       case 'r': rbb_port = atoi(optarg);    break;
       case 'V': verbose = true;             break;
-      case 'D': dramsim = 1;                break;
+      case 'p': opterr = 0;                 break;
+      case 'o': opterr = 1;                 break;
 #if VM_TRACE
       case 'v': {
         vcdfile_name = optarg;
@@ -207,8 +208,10 @@ int main(int argc, char** argv)
 #endif
         else if (arg.substr(0, 12) == "+cycle-count")
           c = 'c';
-        else if (arg == "+dramsim")
-          c = 'D';
+        else if (arg == "+permissive")
+          c = 'p';
+        else if (arg == "+permissive-off")
+          c = 'o';
         // If we don't find a legacy '+' EMULATOR argument, it still could be
         // a VERILOG_PLUSARG and not an error.
         else if (verilog_plusargs_legal) {
@@ -240,9 +243,13 @@ int main(int argc, char** argv)
             }
             htif_option++;
           }
-          std::cerr << argv[0] << ": invalid plus-arg (Verilog or HTIF) \""
-                    << arg << "\"\n";
-          c = '?';
+          if(opterr) {
+            std::cerr << argv[0] << ": invalid plus-arg (Verilog or HTIF) \""
+                      << arg << "\"\n";
+            c = '?';
+          } else {
+            c = 'p';
+          }
         }
         goto retry;
       }
@@ -264,10 +271,10 @@ done_processing:
     usage(argv[0]);
     return 1;
   }
-  htif_argc = 1 + argc - optind;
-  htif_argv = (char **) malloc((htif_argc) * sizeof (char *));
-  htif_argv[0] = argv[0];
-  for (int i = 1; optind < argc;) htif_argv[i++] = argv[optind++];
+
+  // set argc/v in vpi_get_vlog_info
+  htif_argc = argc;
+  htif_argv = argv;
 
   if (verbose)
     fprintf(stderr, "using random seed %u\n", random_seed);
@@ -293,6 +300,9 @@ done_processing:
   }
 #endif // VM_TRACE
 
+  // RocketChip currently only supports RBB port 0, so this needs to stay here
+  jtag = new remote_bitbang_t(rbb_port);
+
   signal(SIGTERM, handle_sigterm);
 
   bool dump;
@@ -339,10 +349,10 @@ done_processing:
     trace_count++;
   }
   // for verilator multithreading. need to do 1 loop before checking if
-  // tsi exists, since tsi is created by verilated thread on the first 
-  // serial_tick. 
-  while ((!dtm || !dtm->done()) && 
-         (!jtag || !jtag->done()) && 
+  // tsi exists, since tsi is created by verilated thread on the first
+  // serial_tick.
+  while ((!dtm || !dtm->done()) &&
+         (!jtag || !jtag->done()) &&
          (!tsi || !tsi->done()) &&
          !tile->io_success && trace_count < max_cycles);
 
@@ -382,6 +392,5 @@ done_processing:
   if (tsi) delete tsi;
   if (jtag) delete jtag;
   if (tile) delete tile;
-  if (htif_argv) free(htif_argv);
   return ret;
 }

From 2168813da018f52a8781ee3e0e851bea5207a597 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Fri, 21 Aug 2020 13:56:14 -0700
Subject: [PATCH 33/74] Add help string | Fix emulator CC to not conflict with
 --vpi

---
 .../src/main/resources/csrc/emulator.cc        | 18 ------------------
 sims/verilator/Makefile                        |  1 +
 2 files changed, 1 insertion(+), 18 deletions(-)

diff --git a/generators/utilities/src/main/resources/csrc/emulator.cc b/generators/utilities/src/main/resources/csrc/emulator.cc
index e92c4688..5e0ea38b 100644
--- a/generators/utilities/src/main/resources/csrc/emulator.cc
+++ b/generators/utilities/src/main/resources/csrc/emulator.cc
@@ -20,8 +20,6 @@
 #include <stdlib.h>
 #include <unistd.h>
 #include <getopt.h>
-// needed for s_vpi_vlog_info, which is needed for multithreading
-#include <vpi_user.h>
 
 // For option parsing, which is split across this file, Verilog, and
 // FESVR's HTIF, a few external files must be pulled in. The list of
@@ -56,18 +54,6 @@ double sc_time_stamp()
   return trace_count;
 }
 
-// need to pull htif_argc/htif_argv out here so the thread that calls tick()
-// for the HTIF device can initialize properly with the cmdline args. this
-// was pulled out here for multithreading to work
-static int htif_argc;
-static char **htif_argv = NULL;
-extern "C" int vpi_get_vlog_info(s_vpi_vlog_info *vlog_info_s)
-{
-  vlog_info_s->argc = htif_argc;
-  vlog_info_s->argv = htif_argv;
-  return 1;
-}
-
 static void usage(const char * program_name)
 {
   printf("Usage: %s [EMULATOR OPTION]... [VERILOG PLUSARG]... [HOST OPTION]... BINARY [TARGET OPTION]...\n",
@@ -272,10 +258,6 @@ done_processing:
     return 1;
   }
 
-  // set argc/v in vpi_get_vlog_info
-  htif_argc = argc;
-  htif_argv = argv;
-
   if (verbose)
     fprintf(stderr, "using random seed %u\n", random_seed);
 
diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index f945cb01..3cc7c18b 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -53,6 +53,7 @@ HELP_COMPILATION_VARIABLES += \
 "   VERILATOR_PROFILE      = 'none' if no verilator profiling (default)" \
 "                            'all' if full verilator runtime profiling" \
 "                            'threads' if runtime thread profiling only" \
+"   VERILATOR_THREADS      = how many threads the simulator will use (default 1)" \
 "   VERILATOR_FST_MODE     = enable FST waveform instead of VCD. use with debug build"
 
 #########################################################################################

From fa97359516670693e2c9c1fb1c339631a2c6bbae Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Fri, 21 Aug 2020 16:14:07 -0700
Subject: [PATCH 34/74] Cleanup VCS's csrc directory | Fix small doc typo

---
 docs/Simulation/Software-RTL-Simulation.rst |  2 +-
 sims/vcs/Makefile                           | 18 +++++++++++++-----
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/docs/Simulation/Software-RTL-Simulation.rst b/docs/Simulation/Software-RTL-Simulation.rst
index 32164f6d..d952c62c 100644
--- a/docs/Simulation/Software-RTL-Simulation.rst
+++ b/docs/Simulation/Software-RTL-Simulation.rst
@@ -195,5 +195,5 @@ When building the verilator simulator there are some additional options:
 
 The ``VERILATOR_THREADS=<num>`` option enables the compiled Verilator simulator to use ``<num>`` parallel threads.
 On a multi-socket machine, you will want to make sure all threads are on the same socket by using ``NUMACTL=1`` to enable ``numactl``.
-By enabling this, you will use CHipyard's ``numa_prefix`` wrapper, which is a simple wrapper around ``numactl`` that runs your verilated simulator like this: ``$(numa_prefix) ./simulator-<name> <simulator-args>``.
+By enabling this, you will use Chipyard's ``numa_prefix`` wrapper, which is a simple wrapper around ``numactl`` that runs your verilated simulator like this: ``$(numa_prefix) ./simulator-<name> <simulator-args>``.
 Note that both these flags are mutually exclusive, you can use either independently (though it makes sense to use ``NUMACTL`` just with ``VERILATOR_THREADS=8`` during a Verilator simulation).
diff --git a/sims/vcs/Makefile b/sims/vcs/Makefile
index a2896380..2c44ae6e 100644
--- a/sims/vcs/Makefile
+++ b/sims/vcs/Makefile
@@ -43,14 +43,22 @@ VCS = vcs -full64
 
 VCS_OPTS = $(VCS_CC_OPTS) $(VCS_NONCC_OPTS) $(PREPROC_DEFINES)
 
+#########################################################################################
+# vcs build paths
+#########################################################################################
+model_dir = $(build_dir)/$(long_name)
+model_dir_debug = $(build_dir)/$(long_name).debug
+
 #########################################################################################
 # vcs simulator rules
 #########################################################################################
 $(sim): $(sim_vsrcs) $(sim_common_files) $(dramsim_lib) $(EXTRA_SIM_REQS)
-	rm -rf csrc && $(VCS) $(VCS_OPTS) $(EXTRA_SIM_SOURCES) -o $@
+	rm -rf $(model_dir)
+	$(VCS) $(VCS_OPTS) $(EXTRA_SIM_SOURCES) -o $@ -Mdir=$(model_dir)
 
 $(sim_debug): $(sim_vsrcs) $(sim_common_files) $(dramsim_lib) $(EXTRA_SIM_REQS)
-	rm -rf csrc && $(VCS) $(VCS_OPTS) $(EXTRA_SIM_SOURCES) -o $@ \
+	rm -rf $(model_dir_debug)
+	$(VCS) $(VCS_OPTS) $(EXTRA_SIM_SOURCES) -o $@ -Mdir=$(model_dir_debug)  \
 	+define+DEBUG
 
 #########################################################################################
@@ -65,10 +73,10 @@ $(output_dir)/%.vpd: $(output_dir)/% $(sim_debug)
 #########################################################################################
 .PHONY: clean clean-sim clean-sim-debug
 clean:
-	rm -rf $(gen_dir) csrc $(sim_prefix)-* ucli.key vc_hdrs.h
+	rm -rf $(gen_dir) $(sim_prefix)-* ucli.key
 
 clean-sim:
-	rm -rf csrc/ $(sim) ucli.key vc_hdrs.h
+	rm -rf $(model_dir) $(build_dir)/vc_hdrs.h $(sim) $(sim).daidir ucli.key
 
 clean-sim-debug:
-	rm -rf csrc/ $(sim_debug) ucli.key vc_hdrs.h
+	rm -rf $(model_dir_debug) $(build_dir)/vc_hdrs.h $(sim_debug) $(sim_debug).daidir ucli.key

From 06747e6cffa6beec1b573d7da28735041e7def55 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Fri, 21 Aug 2020 17:27:50 -0700
Subject: [PATCH 35/74] Dedup simulation flags

---
 sims/common-sim-flags.mk | 24 ++++++++++++++++++++++++
 sims/verilator/Makefile  | 24 ++++--------------------
 vcs.mk                   | 21 +++------------------
 3 files changed, 31 insertions(+), 38 deletions(-)
 create mode 100644 sims/common-sim-flags.mk

diff --git a/sims/common-sim-flags.mk b/sims/common-sim-flags.mk
new file mode 100644
index 00000000..1c73a510
--- /dev/null
+++ b/sims/common-sim-flags.mk
@@ -0,0 +1,24 @@
+#----------------------------------------------------------------------------------------
+# common gcc configuration/optimization
+#----------------------------------------------------------------------------------------
+SIM_OPT_CXXFLAGS := -O3
+
+SIM_CXXFLAGS = \
+	$(CXXFLAGS) \
+	$(SIM_OPT_CXXFLAGS) \
+	-std=c++11 \
+	-I$(RISCV)/include \
+	-I$(dramsim_dir) \
+	-I$(build_dir) \
+	$(EXTRA_SIM_CXXFLAGS)
+
+SIM_LDFLAGS = \
+	$(LDFLAGS) \
+	-L$(RISCV)/lib \
+	-Wl,-rpath,$(RISCV)/lib \
+	-L$(sim_dir) \
+	-L$(dramsim_dir) \
+	-lfesvr \
+	-lpthread \
+	-ldramsim \
+	$(EXTRA_SIM_LDFLAGS)
diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile
index 3cc7c18b..211b5676 100644
--- a/sims/verilator/Makefile
+++ b/sims/verilator/Makefile
@@ -141,35 +141,19 @@ VERILATOR_NONCC_OPTS = \
 #----------------------------------------------------------------------------------------
 # gcc configuration/optimization
 #----------------------------------------------------------------------------------------
-# -flto slows down compilation on small-memory and breaks on firesim-manager
-CXX_OPT_FLAGS := -O3
+include $(base_dir)/sims/common-sim-flags.mk
 
 VERILATOR_CXXFLAGS = \
-	$(CXXFLAGS) \
+	$(SIM_CXXFLAGS) \
 	$(RUNTIME_PROFILING_CFLAGS) \
 	$(TRACING_CFLAGS) \
-	$(CXX_OPT_FLAGS) \
-	-std=c++11 \
 	-D__STDC_FORMAT_MACROS \
 	-DTEST_HARNESS=V$(VLOG_MODEL) \
 	-DVERILATOR \
-	-I$(RISCV)/include \
-	-I$(dramsim_dir) \
-	-I$(build_dir) \
 	-include $(build_dir)/$(long_name).plusArgs \
-	-include $(build_dir)/verilator.h \
-	$(EXTRA_SIM_CXXFLAGS)
+	-include $(build_dir)/verilator.h
 
-VERILATOR_LDFLAGS =	\
-	$(LDFLAGS) \
-	-L$(RISCV)/lib \
-	-Wl,-rpath,$(RISCV)/lib \
-	-L$(sim_dir) \
-	-L$(dramsim_dir) \
-	-lfesvr \
-	-lpthread \
-	-ldramsim \
-	$(EXTRA_SIM_LDFLAGS)
+VERILATOR_LDFLAGS = $(SIM_LDFLAGS)
 
 VERILATOR_CC_OPTS = \
 	-CFLAGS "$(VERILATOR_CXXFLAGS)" \
diff --git a/vcs.mk b/vcs.mk
index a282aa44..c3ba6da6 100644
--- a/vcs.mk
+++ b/vcs.mk
@@ -14,25 +14,10 @@ RESET_DELAY ?= 777.7
 #----------------------------------------------------------------------------------------
 # gcc configuration/optimization
 #----------------------------------------------------------------------------------------
-CXX_OPT_FLAGS := -O3
+include $(base_dir)/sims/common-sim-flags.mk
 
-VCS_CXXFLAGS = \
-	$(CXXFLAGS) \
-	$(CXX_OPT_FLAGS) \
-	-I$(RISCV)/include \
-	-I$(dramsim_dir) \
-	-std=c++11 \
-	$(EXTRA_SIM_CXXFLAGS)
-
-VCS_LDFLAGS =	\
-	$(LDFLAGS) \
-	-L$(RISCV)/lib \
-	-Wl,-rpath,$(RISCV)/lib \
-	-L$(sim_dir) \
-	-L$(dramsim_dir) \
-	-lfesvr \
-	-ldramsim \
-	$(EXTRA_SIM_LDFLAGS)
+VCS_CXXFLAGS = $(SIM_CXXFLAGS)
+VCS_LDFLAGS = $(SIM_LDFLAGS)
 
 # vcs requires LDFLAGS to not include library names (i.e. -l needs to be separate)
 VCS_CC_OPTS = \

From 03457bf9348b6c5a6b423c55e953bf245c4c5d68 Mon Sep 17 00:00:00 2001
From: abejgonzalez <abe.j.gonza@gmail.com>
Date: Fri, 21 Aug 2020 17:44:29 -0700
Subject: [PATCH 36/74] Remove unneeded libpthread in simulations

---
 sims/common-sim-flags.mk | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sims/common-sim-flags.mk b/sims/common-sim-flags.mk
index 1c73a510..766dd0d3 100644
--- a/sims/common-sim-flags.mk
+++ b/sims/common-sim-flags.mk
@@ -19,6 +19,5 @@ SIM_LDFLAGS = \
 	-L$(sim_dir) \
 	-L$(dramsim_dir) \
 	-lfesvr \
-	-lpthread \
 	-ldramsim \
 	$(EXTRA_SIM_LDFLAGS)

From 543136db8c639873de65242ab1b70b1a3307bf1f Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Mon, 24 Aug 2020 11:04:28 -0700
Subject: [PATCH 37/74] Port numa_prefix to python2 [ci skip]

---
 scripts/numa_prefix | 85 +++++++++++++++++++++++++--------------------
 1 file changed, 47 insertions(+), 38 deletions(-)

diff --git a/scripts/numa_prefix b/scripts/numa_prefix
index af2d2eb1..e4cc2aec 100755
--- a/scripts/numa_prefix
+++ b/scripts/numa_prefix
@@ -1,4 +1,4 @@
-#!/usr/bin/env perl
+#!/usr/bin/env python
 
 #============================================================================
 # - really simple script, which just prints out the numactl cmd to
@@ -23,45 +23,54 @@
 #       1:  20  10
 #============================================================================
 
-use strict;
-use warnings;
+import subprocess
+import re
+import sys
 
-my $path = `which numactl`;
-if(length($path) > 0) {
-  my ($head_line, @rest) = map {chomp; $_} `numactl -H`;
+which_proc = subprocess.Popen(["which", "numactl"], stdout=subprocess.PIPE)
+out, err = which_proc.communicate()
 
-  if($head_line =~ /available: (\d+) nodes/) {
-    my $node_count     = $1;
-    my $best_node_id   = undef
-    my $best_cpus      = undef;
-    my $best_free_size = undef;
+if out != "":
+    numactl_proc = subprocess.Popen(["numactl", "-H"], stdout=subprocess.PIPE)
+    out, err = numactl_proc.communicate()
 
-		# loop through available nodes, selecting the node with the most free mem
-    foreach my $num (1..$node_count) {
-      my $cpus_line     = shift(@rest);
-      my $mem_size_line = shift(@rest);
-      my $mem_free_line = shift(@rest);
+    lines = out.split("\n")
+    line_idx = 0
 
-      if($cpus_line =~ /node (\d+) cpus: (\d.*\d)$/) {
-        my ($node_id, $cpus) = ($1, $2);
-        $cpus =~ s/\s+/,/g;
+    head_line = lines[line_idx]
+    line_idx += 1
+    node_match = re.match(r"^ *available: +(\d+) nodes", head_line)
+    if node_match:
+        avail_nodes = node_match.group(1)
+        best_node_id = ""
+        best_cpus = ""
+        best_free_size = 0
 
-        if($mem_free_line =~ /node $node_id free: (\d+) \S+$/) {
-          my $free_size = $1;
-          if(!defined($best_free_size) || ($free_size > $best_free_size)) {
-            $best_node_id   = $node_id;
-            $best_cpus      = $cpus;
-            $best_free_size = $free_size;
-          }
-        } else {
-          die("malformed mem-free line: $mem_free_line\n");
-        }
-      } else {
-        die("malformed cpus line: $cpus_line\n");
-      }
-    }
-    print("numactl -m $best_node_id -C $best_cpus --");
-  } else {
-    die("malformed head line: $head_line\n");
-  }
-}
+        # loop through available nodes, selecting the node with the most free mem
+        for i in avail_nodes:
+            cpu_line      = lines[line_idx]
+            mem_size_line = lines[line_idx + 1]
+            mem_free_line = lines[line_idx + 2]
+            line_idx += 3
+
+            cpu_match = re.match(r"^ *node (\d+) cpus: (\d.*\d)$", cpu_line)
+            if cpu_match:
+                node_id = cpu_match.group(1)
+                cpus = cpu_match.group(2).replace(" ", ",")
+
+                mem_free_match = re.match(r"^ *node " + node_id + " free: (\d+) \S+$", mem_free_line)
+                if mem_free_match:
+                    free_size = mem_free_match.group(1)
+                    if int(free_size) > int(best_free_size):
+                        best_node_id = node_id
+                        best_cpus = cpus
+                        best_free_size = free_size
+                else:
+                    sys.exit("[ERROR] Malformed mem free line: " + mem_free_line)
+
+            else:
+                sys.exit("[ERROR] Malformed cpus line: " + cpu_line)
+
+        sys.stdout.write("numactl -m " + best_node_id + " -C " + best_cpus + " --")
+    else:
+        sys.exit("[ERROR] Malformed head line: " + head_line)

From f8a0757eee51404652b2a0432e780b88ff795b4c Mon Sep 17 00:00:00 2001
From: Abraham Gonzalez <abe.j.gonza@gmail.com>
Date: Mon, 24 Aug 2020 11:24:28 -0700
Subject: [PATCH 38/74] Remove extra mem_size line [ci skip]

---
 scripts/numa_prefix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/numa_prefix b/scripts/numa_prefix
index e4cc2aec..b9521cbd 100755
--- a/scripts/numa_prefix
+++ b/scripts/numa_prefix
@@ -49,7 +49,7 @@ if out != "":
         # loop through available nodes, selecting the node with the most free mem
         for i in avail_nodes:
             cpu_line      = lines[line_idx]
-            mem_size_line = lines[line_idx + 1]
+            # mem. size unused. skip and use mem. free
             mem_free_line = lines[line_idx + 2]
             line_idx += 3
 

From e275a4589090d924e4c829c927a4fe0b87eb7335 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 24 Aug 2020 17:51:48 -0700
Subject: [PATCH 39/74] Address PR comments

---
 .../chipyard/src/main/scala/ChipTop.scala     |  4 +-
 .../chipyard/src/main/scala/Clocks.scala      | 12 +++---
 .../src/main/scala/ConfigFragments.scala      |  7 ++--
 .../chipyard/src/main/scala/IOBinders.scala   | 39 ++++++++++---------
 .../chipyard/src/main/scala/TestHarness.scala | 12 +++---
 .../src/main/scala/config/RocketConfigs.scala |  5 ---
 .../firechip/src/main/scala/FireSim.scala     | 16 ++++----
 vlsi/Makefile                                 |  2 +-
 8 files changed, 49 insertions(+), 48 deletions(-)

diff --git a/generators/chipyard/src/main/scala/ChipTop.scala b/generators/chipyard/src/main/scala/ChipTop.scala
index f7b94d2b..cf71987b 100644
--- a/generators/chipyard/src/main/scala/ChipTop.scala
+++ b/generators/chipyard/src/main/scala/ChipTop.scala
@@ -19,7 +19,7 @@ case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters)
 /**
  * The base class used for building chips. This constructor instantiates a module specified by the BuildSystem parameter,
  * named "system", which is an instance of DigitalTop by default. The diplomatic clocks of System, as well as its implicit clock,
- * is aggregated into the clockGroupNode. The parameterized functions controlled by ChipyardClockKey and GlobalResetSchemeKey
+ * is aggregated into the clockGroupNode. The parameterized functions controlled by ClockingSchemeKey and GlobalResetSchemeKey
  * drive clock and reset generation
  */
 
@@ -36,7 +36,7 @@ class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunc
   val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
 
   // Generate Clocks and Reset
-  p(ChipyardClockKey)(this)
+  p(ClockingSchemeKey)(this)
 
   // NOTE: Making this a LazyRawModule is moderately dangerous, as anonymous children
   // of ChipTop (ex: ClockGroup) do not receive clock or reset.
diff --git a/generators/chipyard/src/main/scala/Clocks.scala b/generators/chipyard/src/main/scala/Clocks.scala
index 7f181bb5..3fa349b5 100644
--- a/generators/chipyard/src/main/scala/Clocks.scala
+++ b/generators/chipyard/src/main/scala/Clocks.scala
@@ -69,7 +69,7 @@ object GenerateReset {
     }
     reset_io.suggestName("reset")
     chiptop.iocells ++= resetIOCell
-    chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
+    chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
       reset_io := th.dutReset
       Nil
     })
@@ -78,11 +78,11 @@ object GenerateReset {
 }
 
 
-case object ChipyardClockKey extends Field[ChipTop => Unit](ClockDrivers.harnessClock)
+case object ClockingSchemeKey extends Field[ChipTop => Unit](ClockingSchemeGenerators.harnessClock)
 
 
 
-object ClockDrivers {
+object ClockingSchemeGenerators {
   // A simple clock provider, for testing
   val harnessClock: ChipTop => Unit = { chiptop =>
     implicit val p = chiptop.p
@@ -120,7 +120,7 @@ object ClockDrivers {
         }
       }}
 
-      chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
+      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
         clock_io := th.harnessClock
         Nil
       })
@@ -132,6 +132,8 @@ object ClockDrivers {
   val harnessDividedClock: ChipTop => Unit = { chiptop =>
     implicit val p = chiptop.p
 
+    require(false, "Divided clock is broken until we fix passing onchip clocks to TestHarness objects")
+
     val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
     chiptop.implicitClockSinkNode := implicitClockSourceNode
 
@@ -166,7 +168,7 @@ object ClockDrivers {
         }
       }}
 
-      chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
+      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
         clock_io := th.harnessClock
         Nil
       })
diff --git a/generators/chipyard/src/main/scala/ConfigFragments.scala b/generators/chipyard/src/main/scala/ConfigFragments.scala
index 704b849a..6336c05a 100644
--- a/generators/chipyard/src/main/scala/ConfigFragments.scala
+++ b/generators/chipyard/src/main/scala/ConfigFragments.scala
@@ -26,7 +26,7 @@ import sifive.blocks.devices.gpio._
 import sifive.blocks.devices.uart._
 import sifive.blocks.devices.spi._
 
-import chipyard.{BuildTop, BuildSystem, ClockDrivers, ChipyardClockKey, TestSuitesKey, TestSuiteHelper}
+import chipyard.{BuildTop, BuildSystem, ClockingSchemeGenerators, ClockingSchemeKey, TestSuitesKey, TestSuiteHelper}
 
 
 // -----------------------
@@ -98,7 +98,7 @@ class WithMultiRoCCHwacha(harts: Int*) extends Config(
     case MultiRoCCKey => {
       up(MultiRoCCKey, site) ++ harts.distinct.map{ i =>
         (i -> Seq((p: Parameters) => {
-          val hwacha = LazyModule(new Hwacha()(p)).suggestName("hwacha")
+          val hwacha = LazyModule(new Hwacha()(p))
           hwacha
         }))
       }
@@ -160,5 +160,6 @@ class WithNoSubsystemDrivenClocks extends Config((site, here, up) => {
 })
 
 class WithTileDividedClock extends Config((site, here, up) => {
-  case ChipyardClockKey => ClockDrivers.harnessDividedClock
+  case ClockingSchemeKey => ClockingSchemeGenerators.harnessDividedClock
 })
+
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 84ef5269..6438801a 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -41,7 +41,7 @@ import scala.reflect.{ClassTag}
 
 // DOC include start: IOBinders
 // This type describes a function callable on the TestHarness instance. Its return type is unused.
-type TestHarnessFunction = (chipyard.HasHarnessUtils) => Seq[Any]
+type TestHarnessFunction = (chipyard.HasHarnessSignalReferences) => Seq[Any]
 // IOBinders will return a Seq of this tuple, which contains three fields:
 //  1. A Seq containing all IO ports created by the IOBinder function
 //  2. A Seq containing all IO cell modules created by the IOBinder function
@@ -228,7 +228,7 @@ object AddIOCells {
 class WithGPIOTiedOff extends OverrideIOBinder({
   (system: HasPeripheryGPIOModuleImp) => {
     val (ports2d, ioCells2d) = AddIOCells.gpio(system.gpio)
-    val harnessFn = (th: HasHarnessUtils) => { ports2d.flatten.foreach(_ <> AnalogConst(0)); Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { ports2d.flatten.foreach(_ <> AnalogConst(0)); Nil }
     Seq((ports2d.flatten, ioCells2d.flatten, Some(harnessFn)))
   }
 })
@@ -237,7 +237,7 @@ class WithGPIOTiedOff extends OverrideIOBinder({
 class WithUARTAdapter extends OverrideIOBinder({
   (system: HasPeripheryUARTModuleImp) => {
     val (ports, ioCells2d) = AddIOCells.uart(system.uart)
-    val harnessFn = (th: HasHarnessUtils) => { UARTAdapter.connect(ports)(system.p); Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { UARTAdapter.connect(ports)(system.p); Nil }
     Seq((ports, ioCells2d.flatten, Some(harnessFn)))
   }
 })
@@ -245,7 +245,7 @@ class WithUARTAdapter extends OverrideIOBinder({
 class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideIOBinder({
   (system: HasPeripherySPIFlashModuleImp) => {
     val (ports, ioCells2d) = AddIOCells.spi(system.qspi, "qspi")
-    val harnessFn = (th: HasHarnessUtils) => { SimSPIFlashModel.connect(ports, th.harnessReset, rdOnly)(system.p); Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { SimSPIFlashModel.connect(ports, th.harnessReset, rdOnly)(system.p); Nil }
     Seq((ports, ioCells2d.flatten, Some(harnessFn)))
   }
 })
@@ -253,7 +253,7 @@ class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideIOBinder({
 class WithSimBlockDevice extends OverrideIOBinder({
   (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev =>
     val (port, ios) = AddIOCells.blockDev(bdev)
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock
       SimBlockDevice.connect(th.harnessClock, th.harnessReset.asBool, Some(port))(system.p)
       Nil
@@ -265,7 +265,7 @@ class WithSimBlockDevice extends OverrideIOBinder({
 class WithBlockDeviceModel extends OverrideIOBinder({
   (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev =>
     val (port, ios) = AddIOCells.blockDev(bdev)
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       BlockDeviceModel.connect(Some(port))(system.p)
       Nil
     }
@@ -288,7 +288,7 @@ class WithSimAXIMem extends OverrideIOBinder({
     val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem")
     // TODO: we are inlining the connectMem method of SimAXIMem because
     //   it takes in a dut rather than seq of axi4 ports
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       peiTuples.map { case (port, edge, ios) =>
         val mem = LazyModule(new SimAXIMem(edge, size = p(ExtMem).get.master.size))
         Module(mem.module).suggestName("mem")
@@ -305,7 +305,7 @@ class WithBlackBoxSimMem extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {
     implicit val p: Parameters = GetSystemParameters(system)
     val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem")
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       peiTuples.map { case (port, edge, ios) =>
         val memSize = p(ExtMem).get.master.size
         val lineSize = p(CacheBlockBytes)
@@ -325,7 +325,7 @@ class WithSimAXIMMIO extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MMIOPort) => {
     implicit val p: Parameters = GetSystemParameters(system)
     val peiTuples = AddIOCells.axi4(system.mmio_axi4, system.mmioAXI4Node, "mmio_mem")
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       peiTuples.zipWithIndex.map { case ((port, edge, ios), i) =>
         val mmio_mem = LazyModule(new SimAXIMem(edge, size = 4096))
         Module(mmio_mem.module).suggestName(s"mmio_mem_${i}")
@@ -345,7 +345,7 @@ class WithTieOffInterrupts extends OverrideIOBinder({
   (system: HasExtInterruptsModuleImp) => {
     val (port, ioCells) = IOCell.generateIOFromSignal(system.interrupts, Some("iocell_interrupts"))
     port.suggestName("interrupts")
-    val harnessFn = (th: HasHarnessUtils) => { port := 0.U; Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { port := 0.U; Nil }
     Seq((Seq(port), ioCells, Some(harnessFn)))
   }
 })
@@ -353,7 +353,7 @@ class WithTieOffInterrupts extends OverrideIOBinder({
 class WithTieOffL2FBusAXI extends OverrideIOBinder({
   (system: CanHaveSlaveAXI4Port) => {
     val peiTuples = AddIOCells.axi4(system.l2_frontend_bus_axi4, system.l2FrontendAXI4Node, "l2_fbus")
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       peiTuples.zipWithIndex.map { case ((port, edge, ios), i) =>
         port := DontCare // tieoff doesn't completely tie-off, for some reason
         port.tieoff()
@@ -364,18 +364,19 @@ class WithTieOffL2FBusAXI extends OverrideIOBinder({
   }
 })
 
+// TODO we need to rethink what "Tie-off-debug" means. The current system punches out
+// excessive IOs.
 class WithTiedOffDebug extends OverrideIOBinder({
   (system: HasPeripheryDebugModuleImp) => {
     val (psdPort, resetctrlOpt, debugPortOpt, ioCells) =
       AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p)
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       Debug.tieoffDebug(debugPortOpt, resetctrlOpt, Some(psdPort))(system.p)
       // tieoffDebug doesn't actually tie everything off :/
       debugPortOpt.foreach { d =>
-        // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock
         d.clockeddmi.foreach({ cdmi => cdmi.dmi.req.bits := DontCare; cdmi.dmiClock := th.harnessClock })
         d.dmactiveAck := DontCare
-        d.clock := th.harnessClock
+        d.clock := th.harnessClock // TODO fix: This should be driven from within the chip
       }
       Nil
     }
@@ -383,11 +384,13 @@ class WithTiedOffDebug extends OverrideIOBinder({
   }
 })
 
+// TODO we need to rethink what this does. The current system punches out excessive IOs.
+// Some of the debug clock/reset should be driven from on-chip
 class WithSimDebug extends OverrideIOBinder({
   (system: HasPeripheryDebugModuleImp) => {
     val (psdPort, resetctrlPortOpt, debugPortOpt, ioCells) =
       AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p)
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       val dtm_success = Wire(Bool())
       Debug.connectDebug(debugPortOpt, resetctrlPortOpt, psdPort, th.harnessClock, th.harnessReset.asBool, dtm_success)(system.p)
       when (dtm_success) { th.success := true.B }
@@ -401,7 +404,7 @@ class WithSimDebug extends OverrideIOBinder({
 class WithTiedOffSerial extends OverrideIOBinder({
   (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial =>
     val (port, ioCells) = AddIOCells.serial(serial)
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       SerialAdapter.tieoff(port)
       Nil
     }
@@ -412,7 +415,7 @@ class WithTiedOffSerial extends OverrideIOBinder({
 class WithSimSerial extends OverrideIOBinder({
   (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial =>
     val (port, ioCells) = AddIOCells.serial(serial)
-    val harnessFn = (th: HasHarnessUtils) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       val ser_success = SerialAdapter.connectSimSerial(port, th.harnessClock, th.harnessReset)
       when (ser_success) { th.success := true.B }
       Nil
@@ -425,7 +428,7 @@ class WithTraceGenSuccessBinder extends OverrideIOBinder({
   (system: TraceGenSystemModuleImp) => {
     val (successPort, ioCells) = IOCell.generateIOFromSignal(system.success, Some("iocell_success"))
     successPort.suggestName("success")
-    val harnessFn = (th: HasHarnessUtils) => { when (successPort) { th.success := true.B }; Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { when (successPort) { th.success := true.B }; Nil }
     Seq((Seq(successPort), ioCells, Some(harnessFn)))
   }
 })
diff --git a/generators/chipyard/src/main/scala/TestHarness.scala b/generators/chipyard/src/main/scala/TestHarness.scala
index 70802783..b296e328 100644
--- a/generators/chipyard/src/main/scala/TestHarness.scala
+++ b/generators/chipyard/src/main/scala/TestHarness.scala
@@ -16,14 +16,14 @@ trait HasTestHarnessFunctions {
   val harnessFunctions: Seq[TestHarnessFunction]
 }
 
-trait HasHarnessUtils {
-  val harnessClock: Clock
-  val harnessReset: Reset
-  val dutReset: Reset
-  val success: Bool
+trait HasHarnessSignalReferences {
+  def harnessClock: Clock
+  def harnessReset: Reset
+  def dutReset: Reset
+  def success: Bool
 }
 
-class TestHarness(implicit val p: Parameters) extends Module with HasHarnessUtils {
+class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSignalReferences {
   val io = IO(new Bundle {
     val success = Output(Bool())
   })
diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
index e0587454..07033609 100644
--- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
@@ -194,8 +194,3 @@ class DividedClockRocketConfig extends Config(
   new chipyard.config.AbstractConfig)
 
 
-class TestClockRocketConfig extends Config(
-  //new chipyard.config.WithTileMultiClock ++                       // Put the Tile on its own clock domain
-  new freechips.rocketchip.subsystem.WithAsynchronousRocketTiles(8, 3) ++   // Add rational crossings between RocketTile and uncore
-  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
-  new chipyard.config.AbstractConfig)
diff --git a/generators/firechip/src/main/scala/FireSim.scala b/generators/firechip/src/main/scala/FireSim.scala
index e807e840..158674a0 100644
--- a/generators/firechip/src/main/scala/FireSim.scala
+++ b/generators/firechip/src/main/scala/FireSim.scala
@@ -13,7 +13,7 @@ import freechips.rocketchip.util.{ResetCatchAndSync}
 
 import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge, RationalClock}
 
-import chipyard.{BuildSystem, BuildTop, HasHarnessUtils, ChipyardSubsystem, ChipyardClockKey, ChipTop}
+import chipyard.{BuildSystem, BuildTop, HasHarnessSignalReferences, ChipyardSubsystem, ClockingSchemeKey, ChipTop}
 import chipyard.iobinders.{IOBinders}
 
 // Determines the number of times to instantiate the DUT in the harness.
@@ -43,7 +43,7 @@ object NodeIdx {
 }
 
 class WithFireSimSimpleClocks extends Config((site, here, up) => {
-  case ChipyardClockKey => { chiptop: ChipTop =>
+  case ClockingSchemeKey => { chiptop: ChipTop =>
     implicit val p = chiptop.p
 
     val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
@@ -75,7 +75,7 @@ class WithFireSimSimpleClocks extends Config((site, here, up) => {
         }
       }}
 
-      chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
+      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
         clock := th.harnessClock
         reset := th.harnessReset
         Nil
@@ -86,7 +86,7 @@ class WithFireSimSimpleClocks extends Config((site, here, up) => {
 
 class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Config((site, here, up) => {
   case FireSimClockKey => FireSimClockParameters(Seq(RationalClock("TileDomain", multiplier, divisor)))
-  case ChipyardClockKey => { chiptop: ChipTop =>
+  case ClockingSchemeKey => { chiptop: ChipTop =>
     implicit val p = chiptop.p
 
     val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
@@ -125,7 +125,7 @@ class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Confi
         }
       }}
 
-      chiptop.harnessFunctions += ((th: HasHarnessUtils) => {
+      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
         uncore_clock := th.harnessClock
         reset        := th.harnessReset
         th match {
@@ -138,15 +138,15 @@ class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Confi
   }
 })
 
-class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessUtils {
+class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSignalReferences {
   freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary())
   val clockBridge = Module(new RationalClockBridge(p(FireSimClockKey).additionalClocks:_*))
   val harnessClock = clockBridge.io.clocks.head // This is the reference clock
   val additionalClocks = clockBridge.io.clocks.tail
   val harnessReset = WireInit(false.B)
   val peekPokeBridge = PeekPokeBridge(harnessClock, harnessReset)
-  val dutReset = false.B // unused (if used, its a bug)
-  val success = false.B // unused (if used, its a bug)
+  def dutReset = { require(false, "dutReset should not be used in Firesim"); false.B }
+  def success = { require(false, "success should not be used in Firesim"); false.B }
 
   // Instantiate multiple instances of the DUT to implement supernode
   for (i <- 0 until p(NumNodes)) {
diff --git a/vlsi/Makefile b/vlsi/Makefile
index a9e3d3a5..da9f2193 100644
--- a/vlsi/Makefile
+++ b/vlsi/Makefile
@@ -34,7 +34,7 @@ INPUT_CONFS        ?= $(if $(filter $(tech_name),nangate45),\
                         example-asap7.yml)
 HAMMER_EXEC        ?= example-vlsi
 VLSI_TOP           ?= $(TOP)
-VLSI_HARNESS_DUT_NAME ?= dut
+VLSI_HARNESS_DUT_NAME ?= chiptop
 VLSI_OBJ_DIR       ?= $(vlsi_dir)/build
 ifneq ($(CUSTOM_VLOG),)
 	OBJ_DIR          ?= $(VLSI_OBJ_DIR)/custom-$(VLSI_TOP)

From 5a25ee5206d44a7794aacf4540088fec52248dfa Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Thu, 27 Aug 2020 08:08:08 +0000
Subject: [PATCH 40/74] Bump Firesim for new AGFIs

---
 sims/firesim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/firesim b/sims/firesim
index b13e7529..05edd6be 160000
--- a/sims/firesim
+++ b/sims/firesim
@@ -1 +1 @@
-Subproject commit b13e75296c44b1f3fa987d15df6a595668842dfe
+Subproject commit 05edd6be8c0464ea53a664a2164d3eba6a7f62aa

From 239b6b6e095aff7075b25c91c45f35a9bb9a113f Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Thu, 27 Aug 2020 13:00:43 -0700
Subject: [PATCH 41/74] Bump testchipip

---
 generators/testchipip | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generators/testchipip b/generators/testchipip
index add28f0b..1e7373f6 160000
--- a/generators/testchipip
+++ b/generators/testchipip
@@ -1 +1 @@
-Subproject commit add28f0b61dd337c1d4fa36b48569166743cb1d5
+Subproject commit 1e7373f6398c198e2dee2bcf692917ec2ac21b53

From 933df4e05c284a127a743f6fc81865939bfaee49 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Thu, 27 Aug 2020 23:27:24 -0700
Subject: [PATCH 42/74] Whitelist firemarshal's dev branch for commit-on-master
 check

---
 .circleci/check-commit.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.circleci/check-commit.sh b/.circleci/check-commit.sh
index 98c770ad..7824b943 100755
--- a/.circleci/check-commit.sh
+++ b/.circleci/check-commit.sh
@@ -82,7 +82,12 @@ search
 
 submodules=("coremark" "firemarshal" "nvdla-workload" "spec2017")
 dir="software"
-branches=("master")
+if [ "$CIRCLE_BRANCH" == "master" ] || [ "$CIRCLE_BRANCH" == "dev" ]
+then
+    branches=("master")
+else
+    branches=("master" "dev")
+fi
 search
 
 submodules=("DRAMSim2" "axe" "barstools" "chisel-testers" "dsptools" "firrtl-interpreter" "torture" "treadle")

From 27b78f4de21d390cbe6deb24cc2743a63b4a10ca Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Thu, 27 Aug 2020 23:48:01 -0700
Subject: [PATCH 43/74] Only punch realistic subset of DebugIO through chiptop

---
 .../chipyard/src/main/scala/IOBinders.scala   | 131 ++++++++++++------
 1 file changed, 90 insertions(+), 41 deletions(-)

diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 6438801a..1c81f535 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -7,6 +7,7 @@ import chisel3.experimental.{Analog, IO}
 import freechips.rocketchip.config.{Field, Config, Parameters}
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImpLike}
 import freechips.rocketchip.devices.debug._
+import freechips.rocketchip.jtag.{JTAGIO}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.system.{SimAXIMem}
 import freechips.rocketchip.amba.axi4.{AXI4Bundle, AXI4SlaveNode, AXI4MasterNode, AXI4EdgeParameters}
@@ -163,30 +164,65 @@ object AddIOCells {
   }
 
   /**
-   * Add IO cells to a debug module and name the IO ports.
-   * @param psd A PSDIO bundle
+   * Add IO cells to a debug module and name the IO ports, for debug IO which must go off-chip
+   * For on-chip debug IO, drive them appropriately
+   * @param system A BaseSubsystem that might have a debug module
    * @param resetctrlOpt An optional ResetCtrlIO bundle
    * @param debugOpt An optional DebugIO bundle
-   * @return Returns a tuple3 of (Top-level PSDIO IO; Optional top-level DebugIO IO; a list of IOCell module references)
+   * @return Returns a tuple2 of (Generated debug io ports, Generated IOCells)
    */
-  def debug(psd: PSDIO, resetctrlOpt: Option[ResetCtrlIO], debugOpt: Option[DebugIO])(implicit p: Parameters):
-      (PSDIO, Option[ResetCtrlIO], Option[DebugIO], Seq[IOCell]) = {
-    val (psdPort, psdIOs) = IOCell.generateIOFromSignal(
-      psd, Some("iocell_psd"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
-    val debugTuple = debugOpt.map(d =>
-      IOCell.generateIOFromSignal(d, Some("iocell_debug"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync))
-    val debugPortOpt: Option[DebugIO] = debugTuple.map(_._1)
-    val debugIOs: Seq[IOCell] = debugTuple.map(_._2).toSeq.flatten
-    debugPortOpt.foreach(_.suggestName("debug"))
+  def debug(system: HasPeripheryDebugModuleImp)(implicit p: Parameters): (Seq[Bundle], Seq[IOCell]) = {
+    system.debug.map { debug =>
 
-    val resetctrlTuple = resetctrlOpt.map(d =>
-      IOCell.generateIOFromSignal(d, Some("iocell_resetctrl"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync))
-    val resetctrlPortOpt: Option[ResetCtrlIO] = resetctrlTuple.map(_._1)
-    val resetctrlIOs: Seq[IOCell] = resetctrlTuple.map(_._2).toSeq.flatten
-    resetctrlPortOpt.foreach(_.suggestName("resetctrl"))
+      // We never use the PSDIO, so tie it off on-chip
+      system.psd.psd.foreach { _ <> 0.U.asTypeOf(new PSDTestMode) }
 
-    psdPort.suggestName("psd")
-    (psdPort, resetctrlPortOpt, debugPortOpt, psdIOs ++ debugIOs ++ resetctrlIOs)
+      // Set resetCtrlOpt with the system reset
+      system.resetctrl.map { rcio => rcio.hartIsInReset.map { _ := system.reset.asBool } }
+
+      system.debug.map { d =>
+        // Tie off extTrigger
+        d.extTrigger.foreach { t =>
+          t.in.req := false.B
+          t.out.ack := t.out.req
+        }
+
+        // Tie off disableDebug
+        d.disableDebug.foreach { d => d := false.B }
+
+        // Drive JTAG on-chip IOs
+        d.systemjtag.map { j =>
+          j.reset := system.reset
+          j.mfr_id := system.p(JtagDTMKey).idcodeManufId.U(11.W)
+          j.part_number := system.p(JtagDTMKey).idcodePartNum.U(16.W)
+          j.version := system.p(JtagDTMKey).idcodeVersion.U(4.W)
+        }
+      }
+
+
+      // Connect DebugClockAndReset to system implicit clock. TODO this should use the clock of the bus the debug module is attached to
+      Debug.connectDebugClockAndReset(Some(debug), system.clock)(system.p)
+
+      // Add IOCells for the DMI/JTAG/APB ports
+
+      val dmiTuple = debug.clockeddmi.map { d =>
+        IOCell.generateIOFromSignal(d, Some("iocell_dmi"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
+      }
+      dmiTuple.map(_._1).foreach(_.suggestName("dmi"))
+
+      val jtagTuple = debug.systemjtag.map { j =>
+        IOCell.generateIOFromSignal(j.jtag, Some("iocell_jtag"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
+      }
+      jtagTuple.map(_._1).foreach(_.suggestName("jtag"))
+
+      val apbTuple = debug.apb.map { a =>
+        IOCell.generateIOFromSignal(a, Some("iocell_apb"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
+      }
+      apbTuple.map(_._1).foreach(_.suggestName("apb"))
+
+      val allTuples = (dmiTuple ++ jtagTuple ++ apbTuple).toSeq
+      (allTuples.map(_._1).toSeq, allTuples.flatMap(_._2).toSeq)
+    }.getOrElse((Nil, Nil))
   }
 
   /**
@@ -364,40 +400,53 @@ class WithTieOffL2FBusAXI extends OverrideIOBinder({
   }
 })
 
-// TODO we need to rethink what "Tie-off-debug" means. The current system punches out
-// excessive IOs.
-class WithTiedOffDebug extends OverrideIOBinder({
+class WithSimDebug extends OverrideIOBinder({
   (system: HasPeripheryDebugModuleImp) => {
-    val (psdPort, resetctrlOpt, debugPortOpt, ioCells) =
-      AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p)
+    val (ports, iocells) = AddIOCells.debug(system)(system.p)
     val harnessFn = (th: HasHarnessSignalReferences) => {
-      Debug.tieoffDebug(debugPortOpt, resetctrlOpt, Some(psdPort))(system.p)
-      // tieoffDebug doesn't actually tie everything off :/
-      debugPortOpt.foreach { d =>
-        d.clockeddmi.foreach({ cdmi => cdmi.dmi.req.bits := DontCare; cdmi.dmiClock := th.harnessClock })
-        d.dmactiveAck := DontCare
-        d.clock := th.harnessClock // TODO fix: This should be driven from within the chip
+      val dtm_success = Wire(Bool())
+      when (dtm_success) { th.success := true.B }
+      ports.map {
+        case d: ClockedDMIIO =>
+          val dtm = Module(new SimDTM()(system.p)).connect(th.harnessClock, th.harnessReset.asBool, d, dtm_success)
+        case j: JTAGIO =>
+          val jtag = Module(new SimJTAG(tickDelay=3)).connect(j, th.harnessClock, th.harnessReset.asBool, ~(th.harnessReset.asBool), dtm_success)
+        case _ =>
+          require(false, "We only support DMI or JTAG simulated debug connections")
       }
       Nil
     }
-    Seq((Seq(psdPort) ++ resetctrlOpt ++ debugPortOpt.toSeq, Nil, Some(harnessFn)))
+    Seq((ports, iocells, Some(harnessFn)))
   }
 })
 
-// TODO we need to rethink what this does. The current system punches out excessive IOs.
-// Some of the debug clock/reset should be driven from on-chip
-class WithSimDebug extends OverrideIOBinder({
+class WithTiedOffDebug extends OverrideIOBinder({
   (system: HasPeripheryDebugModuleImp) => {
-    val (psdPort, resetctrlPortOpt, debugPortOpt, ioCells) =
-      AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p)
+    val (ports, iocells) = AddIOCells.debug(system)(system.p)
     val harnessFn = (th: HasHarnessSignalReferences) => {
-      val dtm_success = Wire(Bool())
-      Debug.connectDebug(debugPortOpt, resetctrlPortOpt, psdPort, th.harnessClock, th.harnessReset.asBool, dtm_success)(system.p)
-      when (dtm_success) { th.success := true.B }
-      th.dutReset := th.harnessReset.asBool | debugPortOpt.map { debug => AsyncResetReg(debug.ndreset).asBool }.getOrElse(false.B)
+      ports.map {
+        case d: ClockedDMIIO =>
+          d.dmi.req.valid := false.B
+          d.dmi.req.bits  := DontCare
+          d.dmi.resp.ready := true.B
+          d.dmiClock := th.harnessClock
+          d.dmiReset := th.harnessReset
+        case j: JTAGIO =>
+          j.TCK := true.B.asClock
+          j.TMS := true.B
+          j.TDI := true.B
+          j.TRSTn.foreach { r => r := true.B }
+        case a: ClockedAPBBundle =>
+          a.tieoff()
+          a.clock := false.B.asClock
+          a.reset := true.B.asAsyncReset
+          a.psel := false.B
+          a.penable := false.B
+        case _ => require(false)
+      }
       Nil
     }
-    Seq((Seq(psdPort) ++ debugPortOpt.toSeq, ioCells, Some(harnessFn)))
+    Seq((ports, iocells, Some(harnessFn)))
   }
 })
 

From 5705f2645fac94a4cb416594d3cd4c85d5d56f7e Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 28 Aug 2020 14:21:03 -0700
Subject: [PATCH 44/74] Bump toolchains

---
 toolchains/esp-tools/riscv-isa-sim   | 2 +-
 toolchains/esp-tools/riscv-tests     | 2 +-
 toolchains/riscv-tools/riscv-isa-sim | 2 +-
 toolchains/riscv-tools/riscv-openocd | 2 +-
 toolchains/riscv-tools/riscv-tests   | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/toolchains/esp-tools/riscv-isa-sim b/toolchains/esp-tools/riscv-isa-sim
index a1ff6b03..aa332c6a 160000
--- a/toolchains/esp-tools/riscv-isa-sim
+++ b/toolchains/esp-tools/riscv-isa-sim
@@ -1 +1 @@
-Subproject commit a1ff6b03f7f630a06327798238256973568e3837
+Subproject commit aa332c6a9a5ec77a9b97cdb4a1978ad394b17f1e
diff --git a/toolchains/esp-tools/riscv-tests b/toolchains/esp-tools/riscv-tests
index f1370d05..e116930c 160000
--- a/toolchains/esp-tools/riscv-tests
+++ b/toolchains/esp-tools/riscv-tests
@@ -1 +1 @@
-Subproject commit f1370d054389fc83974fc820985b5c51693b8f9d
+Subproject commit e116930c7d4a30fc2a1378417089a089e9e4cad0
diff --git a/toolchains/riscv-tools/riscv-isa-sim b/toolchains/riscv-tools/riscv-isa-sim
index 8d860c19..acd953af 160000
--- a/toolchains/riscv-tools/riscv-isa-sim
+++ b/toolchains/riscv-tools/riscv-isa-sim
@@ -1 +1 @@
-Subproject commit 8d860c190640e19e0f23a21d2479b4a36d13d342
+Subproject commit acd953afd2f52d64e2264c2c7c713dc0ad614406
diff --git a/toolchains/riscv-tools/riscv-openocd b/toolchains/riscv-tools/riscv-openocd
index 7c82a7b9..cbb15587 160000
--- a/toolchains/riscv-tools/riscv-openocd
+++ b/toolchains/riscv-tools/riscv-openocd
@@ -1 +1 @@
-Subproject commit 7c82a7b9d5b7d8b71e0a66826705ec141db718c3
+Subproject commit cbb15587dc782ac8ade7ae252e7b760cfba4a178
diff --git a/toolchains/riscv-tools/riscv-tests b/toolchains/riscv-tools/riscv-tests
index 249796ce..19bfdab4 160000
--- a/toolchains/riscv-tools/riscv-tests
+++ b/toolchains/riscv-tools/riscv-tests
@@ -1 +1 @@
-Subproject commit 249796cec94d75ff10ca034153e206a319e87158
+Subproject commit 19bfdab48c2a6da4a2c67d5779757da7b073811d

From 20013d1348ba67f30c4dc6421f675516726e5a8e Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 28 Aug 2020 14:21:24 -0700
Subject: [PATCH 45/74] Add DTM based bringup to regressions

---
 .circleci/config.yml   | 38 +++++++++++++++++++-------------------
 .circleci/defaults.sh  |  1 +
 .circleci/run-tests.sh |  3 +++
 3 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 88a20f24..eac8504a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -199,6 +199,11 @@ jobs:
         steps:
             - prepare-rtl:
                 project-key: "chipyard-rocket"
+    prepare-chipyard-dmirocket:
+        executor: main-env
+        steps:
+            - prepare-rtl:
+                project-key: "chipyard-dmirocket"
     prepare-chipyard-sha3:
         executor: main-env
         steps:
@@ -225,11 +230,6 @@ jobs:
         steps:
             - prepare-rtl:
                 project-key: "chipyard-boom"
-    prepare-rocketchip:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "rocketchip"
     prepare-chipyard-blkdev:
         executor: main-env
         steps:
@@ -297,6 +297,11 @@ jobs:
         steps:
             - run-tests:
                 project-key: "chipyard-rocket"
+    chipyard-dmirocket-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                project-key: "chipyard-dmirocket"
     chipyard-sha3-run-tests:
         executor: main-env
         steps:
@@ -323,11 +328,6 @@ jobs:
         steps:
             - run-tests:
                 project-key: "chipyard-boom"
-    rocketchip-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "rocketchip"
     chipyard-hwacha-run-tests:
         executor: main-env
         steps:
@@ -451,6 +451,11 @@ workflows:
                     - install-riscv-toolchain
                     - install-verilator
 
+            - prepare-chipyard-dmirocket:
+                requires:
+                    - install-riscv-toolchain
+                    - install-verilator
+
             - prepare-chipyard-sha3:
                 requires:
                     - install-riscv-toolchain
@@ -476,11 +481,6 @@ workflows:
                     - install-riscv-toolchain
                     - install-verilator
 
-            - prepare-rocketchip:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
             - prepare-chipyard-blkdev:
                 requires:
                     - install-riscv-toolchain
@@ -547,6 +547,10 @@ workflows:
                 requires:
                     - prepare-chipyard-rocket
 
+            - chipyard-dmirocket-run-tests:
+                requires:
+                    - prepare-chipyard-dmirocket
+
             - chipyard-sha3-run-tests:
                 requires:
                     - prepare-chipyard-sha3
@@ -567,10 +571,6 @@ workflows:
                 requires:
                     - prepare-chipyard-boom
 
-            - rocketchip-run-tests:
-                requires:
-                    - prepare-rocketchip
-
             - chipyard-hwacha-run-tests:
                 requires:
                     - prepare-chipyard-hwacha
diff --git a/.circleci/defaults.sh b/.circleci/defaults.sh
index 7cb8c1e2..7ffb1d3c 100755
--- a/.circleci/defaults.sh
+++ b/.circleci/defaults.sh
@@ -48,6 +48,7 @@ LOCAL_FIRESIM_DIR=$LOCAL_CHIPYARD_DIR/sims/firesim/sim
 # key value store to get the build strings
 declare -A mapping
 mapping["chipyard-rocket"]=""
+mapping["chipyard-dmirocket"]=" CONFIG=dmiRocketConfig"
 mapping["chipyard-sha3"]=" CONFIG=Sha3RocketConfig"
 mapping["chipyard-streaming-fir"]=" CONFIG=StreamingFIRRocketConfig"
 mapping["chipyard-streaming-passthrough"]=" CONFIG=StreamingPassthroughRocketConfig"
diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh
index 08b95e68..3e7b0285 100755
--- a/.circleci/run-tests.sh
+++ b/.circleci/run-tests.sh
@@ -32,6 +32,9 @@ case $1 in
     chipyard-rocket)
         run_bmark ${mapping[$1]}
         ;;
+    chipyard-dmirocket)
+        run_bmark ${mapping[$1]}
+        ;;
     chipyard-boom)
         run_bmark ${mapping[$1]}
         ;;

From 17239c56f80aa6b2f9c2b8407a09ea22769da7ab Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 28 Aug 2020 14:36:09 -0700
Subject: [PATCH 46/74] Update AddIOCells.debug comment

---
 generators/chipyard/src/main/scala/IOBinders.scala | 2 --
 1 file changed, 2 deletions(-)

diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 1c81f535..7f4daea3 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -167,8 +167,6 @@ object AddIOCells {
    * Add IO cells to a debug module and name the IO ports, for debug IO which must go off-chip
    * For on-chip debug IO, drive them appropriately
    * @param system A BaseSubsystem that might have a debug module
-   * @param resetctrlOpt An optional ResetCtrlIO bundle
-   * @param debugOpt An optional DebugIO bundle
    * @return Returns a tuple2 of (Generated debug io ports, Generated IOCells)
    */
   def debug(system: HasPeripheryDebugModuleImp)(implicit p: Parameters): (Seq[Bundle], Seq[IOCell]) = {

From c8448cc3e105c38eac54d1fc78545b8897cbad44 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Sun, 30 Aug 2020 18:10:52 -0700
Subject: [PATCH 47/74] Bore out a bus clock to drive DebugIO from ChipTop

---
 .../chipyard/src/main/scala/IOBinders.scala      | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 7f4daea3..f13a6882 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -2,6 +2,7 @@ package chipyard
 package object iobinders {
 
 import chisel3._
+import chisel3.util.experimental.{BoringUtils}
 import chisel3.experimental.{Analog, IO}
 
 import freechips.rocketchip.config.{Field, Config, Parameters}
@@ -171,12 +172,19 @@ object AddIOCells {
    */
   def debug(system: HasPeripheryDebugModuleImp)(implicit p: Parameters): (Seq[Bundle], Seq[IOCell]) = {
     system.debug.map { debug =>
+      val tlbus = system.outer.asInstanceOf[BaseSubsystem].locateTLBusWrapper(p(ExportDebug).slaveWhere)
+      val debug_clock = Wire(Clock()).suggestName("debug_clock")
+      val debug_reset = Wire(Reset()).suggestName("debug_reset")
+      debug_clock := false.B.asClock
+      debug_reset := false.B
+      BoringUtils.bore(tlbus.module.clock, Seq(debug_clock))
+      BoringUtils.bore(tlbus.module.reset, Seq(debug_reset))
 
       // We never use the PSDIO, so tie it off on-chip
       system.psd.psd.foreach { _ <> 0.U.asTypeOf(new PSDTestMode) }
 
       // Set resetCtrlOpt with the system reset
-      system.resetctrl.map { rcio => rcio.hartIsInReset.map { _ := system.reset.asBool } }
+      system.resetctrl.map { rcio => rcio.hartIsInReset.map { _ := debug_reset.asBool } }
 
       system.debug.map { d =>
         // Tie off extTrigger
@@ -190,7 +198,7 @@ object AddIOCells {
 
         // Drive JTAG on-chip IOs
         d.systemjtag.map { j =>
-          j.reset := system.reset
+          j.reset := debug_reset
           j.mfr_id := system.p(JtagDTMKey).idcodeManufId.U(11.W)
           j.part_number := system.p(JtagDTMKey).idcodePartNum.U(16.W)
           j.version := system.p(JtagDTMKey).idcodeVersion.U(4.W)
@@ -199,7 +207,9 @@ object AddIOCells {
 
 
       // Connect DebugClockAndReset to system implicit clock. TODO this should use the clock of the bus the debug module is attached to
-      Debug.connectDebugClockAndReset(Some(debug), system.clock)(system.p)
+
+
+      Debug.connectDebugClockAndReset(Some(debug), debug_clock)(system.p)
 
       // Add IOCells for the DMI/JTAG/APB ports
 

From 4b304623208b4c0b55f4c2f0f280f2c30011acf4 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 2 Sep 2020 20:19:27 -0700
Subject: [PATCH 48/74] Change default IO set to JTAG+Serial, instead of
 JTAG+DMI

---
 docs/Advanced-Concepts/Chip-Communication.rst | 33 +------------------
 .../src/main/scala/ConfigFragments.scala      |  9 ++++-
 .../chipyard/src/main/scala/IOBinders.scala   | 22 ++++---------
 .../main/scala/config/AbstractConfig.scala    |  3 +-
 .../src/main/scala/config/ArianeConfigs.scala |  2 +-
 .../src/main/scala/config/RocketConfigs.scala | 12 ++-----
 .../main/scala/config/TutorialConfigs.scala   | 19 ++++++++---
 7 files changed, 34 insertions(+), 66 deletions(-)

diff --git a/docs/Advanced-Concepts/Chip-Communication.rst b/docs/Advanced-Concepts/Chip-Communication.rst
index e36805ec..84bfc5fb 100644
--- a/docs/Advanced-Concepts/Chip-Communication.rst
+++ b/docs/Advanced-Concepts/Chip-Communication.rst
@@ -130,38 +130,7 @@ Using the JTAG Interface
 ------------------------
 
 The main way to use JTAG with a Rocket Chip based system is to instantiate the Debug Transfer Module (DTM)
-and configure it to use a JTAG interface (by default the DTM is setup to use the DMI interface mentioned above).
-
-Creating a DTM+JTAG Config
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-First, a DTM config must be created for the system that you want to create.
-This step is similar to the DMI simulation section within the :ref:`Starting the TSI or DMI Simulation` section.
-The configuration is very similar to a DMI-based configuration. The main difference
-is the addition of the ``WithJtagDTM`` config fragment that configures the instantiated DTM to use the JTAG protocol as the
-bringup method.
-
-.. literalinclude:: ../../generators/chipyard/src/main/scala/config/RocketConfigs.scala
-    :language: scala
-    :start-after: DOC include start: JtagRocket
-    :end-before: DOC include end: JtagRocket
-
-Building a DTM+JTAG Simulator
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-After creating the config, call the ``make`` command like the following to build a simulator for your RTL:
-
-.. code-block:: bash
-
-    cd sims/verilator
-    # or
-    cd sims/vcs
-
-    make CONFIG=jtagRocketConfig
-
-In this example, the simulation will use the config that you previously specified, as well as set
-the other parameters that are needed to satisfy the build system. After that point, you
-should have a JTAG enabled simulator that you can attach to using OpenOCD and GDB!
+and configure it to use a JTAG interface. The default Chipyard designs configure the DTM to use JTAG. you may attach OpenOCD and GDB to any of the default JTAG-enabled designs.
 
 Debugging with JTAG
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/generators/chipyard/src/main/scala/ConfigFragments.scala b/generators/chipyard/src/main/scala/ConfigFragments.scala
index 6336c05a..cfa465e7 100644
--- a/generators/chipyard/src/main/scala/ConfigFragments.scala
+++ b/generators/chipyard/src/main/scala/ConfigFragments.scala
@@ -7,7 +7,7 @@ import freechips.rocketchip.config.{Field, Parameters, Config}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.diplomacy.{LazyModule, ValName}
 import freechips.rocketchip.devices.tilelink.{BootROMLocated}
-import freechips.rocketchip.devices.debug.{Debug}
+import freechips.rocketchip.devices.debug.{Debug, ExportDebug, DebugModuleKey, DMI}
 import freechips.rocketchip.groundtest.{GroundTestSubsystem}
 import freechips.rocketchip.tile._
 import freechips.rocketchip.rocket.{RocketCoreParams, MulDivParams, DCacheParams, ICacheParams}
@@ -163,3 +163,10 @@ class WithTileDividedClock extends Config((site, here, up) => {
   case ClockingSchemeKey => ClockingSchemeGenerators.harnessDividedClock
 })
 
+class WithDMIDTM extends Config((site, here, up) => {
+  case ExportDebug => up(ExportDebug, site).copy(protocols = Set(DMI))
+})
+
+class WithNoDebug extends Config((site, here, up) => {
+  case DebugModuleKey => None
+})
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index f13a6882..54a0d1dc 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -167,6 +167,7 @@ object AddIOCells {
   /**
    * Add IO cells to a debug module and name the IO ports, for debug IO which must go off-chip
    * For on-chip debug IO, drive them appropriately
+   * Mostly copied from rocket-chip/src/main/scala/devices/debug/Periphery.scala
    * @param system A BaseSubsystem that might have a debug module
    * @return Returns a tuple2 of (Generated debug io ports, Generated IOCells)
    */
@@ -175,27 +176,22 @@ object AddIOCells {
       val tlbus = system.outer.asInstanceOf[BaseSubsystem].locateTLBusWrapper(p(ExportDebug).slaveWhere)
       val debug_clock = Wire(Clock()).suggestName("debug_clock")
       val debug_reset = Wire(Reset()).suggestName("debug_reset")
-      debug_clock := false.B.asClock
-      debug_reset := false.B
+      debug_clock := false.B.asClock // must provide default assignment to avoid firrtl unassigned error
+      debug_reset := false.B // must provide default assignment to avoid firrtl unassigned error
       BoringUtils.bore(tlbus.module.clock, Seq(debug_clock))
       BoringUtils.bore(tlbus.module.reset, Seq(debug_reset))
 
       // We never use the PSDIO, so tie it off on-chip
       system.psd.psd.foreach { _ <> 0.U.asTypeOf(new PSDTestMode) }
-
-      // Set resetCtrlOpt with the system reset
       system.resetctrl.map { rcio => rcio.hartIsInReset.map { _ := debug_reset.asBool } }
-
       system.debug.map { d =>
         // Tie off extTrigger
         d.extTrigger.foreach { t =>
           t.in.req := false.B
           t.out.ack := t.out.req
         }
-
         // Tie off disableDebug
         d.disableDebug.foreach { d => d := false.B }
-
         // Drive JTAG on-chip IOs
         d.systemjtag.map { j =>
           j.reset := debug_reset
@@ -204,15 +200,9 @@ object AddIOCells {
           j.version := system.p(JtagDTMKey).idcodeVersion.U(4.W)
         }
       }
-
-
-      // Connect DebugClockAndReset to system implicit clock. TODO this should use the clock of the bus the debug module is attached to
-
-
       Debug.connectDebugClockAndReset(Some(debug), debug_clock)(system.p)
 
       // Add IOCells for the DMI/JTAG/APB ports
-
       val dmiTuple = debug.clockeddmi.map { d =>
         IOCell.generateIOFromSignal(d, Some("iocell_dmi"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
       }
@@ -412,7 +402,7 @@ class WithSimDebug extends OverrideIOBinder({
   (system: HasPeripheryDebugModuleImp) => {
     val (ports, iocells) = AddIOCells.debug(system)(system.p)
     val harnessFn = (th: HasHarnessSignalReferences) => {
-      val dtm_success = Wire(Bool())
+      val dtm_success = WireInit(false.B)
       when (dtm_success) { th.success := true.B }
       ports.map {
         case d: ClockedDMIIO =>
@@ -437,8 +427,8 @@ class WithTiedOffDebug extends OverrideIOBinder({
           d.dmi.req.valid := false.B
           d.dmi.req.bits  := DontCare
           d.dmi.resp.ready := true.B
-          d.dmiClock := th.harnessClock
-          d.dmiReset := th.harnessReset
+          d.dmiClock := false.B.asClock
+          d.dmiReset := true.B
         case j: JTAGIO =>
           j.TCK := true.B.asClock
           j.TMS := true.B
diff --git a/generators/chipyard/src/main/scala/config/AbstractConfig.scala b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
index 2b9473ed..a925ec56 100644
--- a/generators/chipyard/src/main/scala/config/AbstractConfig.scala
+++ b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
@@ -11,13 +11,14 @@ class AbstractConfig extends Config(
   new chipyard.iobinders.WithUARTAdapter ++                      // display UART with a SimUARTAdapter
   new chipyard.iobinders.WithTieOffInterrupts ++                 // tie off top-level interrupts
   new chipyard.iobinders.WithBlackBoxSimMem ++                   // drive the master AXI4 memory with a blackbox DRAMSim model
-  new chipyard.iobinders.WithTiedOffDebug ++                     // tie off debug (since we are using SimSerial for testing)
+  new chipyard.iobinders.WithSimDebug ++                         // attach SimJTAG
   new chipyard.iobinders.WithSimSerial ++                        // drive TSI with SimSerial for testing
   new testchipip.WithTSI ++                                      // use testchipip serial offchip link
   new chipyard.config.WithBootROM ++                             // use default bootrom
   new chipyard.config.WithUART ++                                // add a UART
   new chipyard.config.WithL2TLBs(1024) ++                        // use L2 TLBs
   new chipyard.config.WithNoSubsystemDrivenClocks ++             // drive the subsystem diplomatic clocks from ChipTop instead of using implicit clocks
+  new freechips.rocketchip.subsystem.WithJtagDTM ++              // set the debug module to expose a JTAG port
   new freechips.rocketchip.subsystem.WithNoMMIOPort ++           // no top-level MMIO master port (overrides default set in rocketchip)
   new freechips.rocketchip.subsystem.WithNoSlavePort ++          // no top-level MMIO slave port (overrides default set in rocketchip)
   new freechips.rocketchip.subsystem.WithInclusiveCache ++       // use Sifive L2 cache
diff --git a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
index 7bc985aa..6bc7cf69 100644
--- a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
@@ -14,6 +14,6 @@ class ArianeConfig extends Config(
 
 class dmiArianeConfig extends Config(
   new chipyard.iobinders.WithTiedOffSerial ++          // Tie off the serial port, override default instantiation of SimSerial
-  new chipyard.iobinders.WithSimDebug ++               // add SimDebug and use it to drive simulation, override default tie-off debug
+  new chipyard.config.WithDMIDTM ++                    // have debug module expose a clocked DMI port
   new ariane.WithNArianeCores(1) ++                    // single Ariane core
   new chipyard.config.AbstractConfig)
diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
index 07033609..420ba192 100644
--- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
@@ -23,18 +23,10 @@ class GemminiRocketConfig extends Config(
   new chipyard.config.AbstractConfig)
 // DOC include end: GemminiRocketConfig
 
-// DOC include start: JtagRocket
-class jtagRocketConfig extends Config(
-  new chipyard.iobinders.WithSimDebug ++                   // add SimDebug, in addition to default SimSerial
-  new freechips.rocketchip.subsystem.WithJtagDTM ++        // sets DTM communication interface to JTAG
-  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
-  new chipyard.config.AbstractConfig)
-// DOC include end: JtagRocket
-
 // DOC include start: DmiRocket
 class dmiRocketConfig extends Config(
-  new chipyard.iobinders.WithTiedOffSerial ++          // tie-off serial, override default add SimSerial
-  new chipyard.iobinders.WithSimDebug ++               // add SimDebug, override default tie-off debug
+  new chipyard.iobinders.WithTiedOffSerial ++          // don't use serial to drive the chip, since we use DMI instead
+  new chipyard.config.WithDMIDTM ++                    // have debug module expose a clocked DMI port
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 // DOC include end: DmiRocket
diff --git a/generators/chipyard/src/main/scala/config/TutorialConfigs.scala b/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
index 8872ed5e..d501b6c0 100644
--- a/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
@@ -23,7 +23,7 @@ class TutorialStarterConfig extends Config(
   new chipyard.iobinders.WithUARTAdapter ++       // Connect a SimUART adapter to display UART on stdout
   new chipyard.iobinders.WithBlackBoxSimMem ++    // Connect simulated external memory
   new chipyard.iobinders.WithTieOffInterrupts ++  // Do not simulate external interrupts
-  new chipyard.iobinders.WithTiedOffDebug ++      // Disconnect the debug module, since we use TSI for bring-up
+  new chipyard.iobinders.WithSimDebug ++          // Connect SimJTAG (or SimDTM) widgets to debug ios
   new chipyard.iobinders.WithSimSerial ++         // Connect external SimSerial widget to drive TSI
 
   // Config fragments below this line affect hardware generation
@@ -43,13 +43,19 @@ class TutorialStarterConfig extends Config(
   // Uncomment this line, and specify a size if you want to have a L2
   // new freechips.rocketchip.subsystem.WithInclusiveCache(nBanks=1, nWays=4, capacityKB=128) ++
 
+  // Set the debug module to expose an external JTAG port
+  new freechips.rocketchip.subsystem.WithJtagDTM ++
+
   // For simpler designs, we want to minimize IOs on
   // our Top. These config fragments remove unnecessary
   // ports
   new freechips.rocketchip.subsystem.WithNoMMIOPort ++
   new freechips.rocketchip.subsystem.WithNoSlavePort ++
   new freechips.rocketchip.subsystem.WithNExtTopInterrupts(0) ++
-  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++  // hierarchical buses including mbus+l2
+
+  // Use the standard hierarchical bus topology including mbus+l2
+  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
+
   // BaseConfig configures "bare" rocketchip system
   new freechips.rocketchip.system.BaseConfig
 )
@@ -60,7 +66,7 @@ class TutorialMMIOConfig extends Config(
   new chipyard.iobinders.WithUARTAdapter ++
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTieOffInterrupts ++
-  new chipyard.iobinders.WithTiedOffDebug ++
+  new chipyard.iobinders.WithSimDebug ++
   new chipyard.iobinders.WithSimSerial ++
 
   new testchipip.WithTSI ++
@@ -76,6 +82,7 @@ class TutorialMMIOConfig extends Config(
   // For this demonstration we assume the base system is a single-core Rocket, for fast elaboration
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
+  new freechips.rocketchip.subsystem.WithJtagDTM ++
   new freechips.rocketchip.subsystem.WithNoMMIOPort ++
   new freechips.rocketchip.subsystem.WithNoSlavePort ++
   new freechips.rocketchip.subsystem.WithNExtTopInterrupts(0) ++
@@ -88,7 +95,7 @@ class TutorialSha3Config extends Config(
   new chipyard.iobinders.WithUARTAdapter ++
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTieOffInterrupts ++
-  new chipyard.iobinders.WithTiedOffDebug ++
+  new chipyard.iobinders.WithSimDebug ++
   new chipyard.iobinders.WithSimSerial ++
 
   new testchipip.WithTSI ++
@@ -101,6 +108,7 @@ class TutorialSha3Config extends Config(
 
   // For this demonstration we assume the base system is a single-core Rocket, for fast elaboration
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new freechips.rocketchip.subsystem.WithJtagDTM ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
   new freechips.rocketchip.subsystem.WithNoMMIOPort ++
   new freechips.rocketchip.subsystem.WithNoSlavePort ++
@@ -114,7 +122,7 @@ class TutorialSha3BlackBoxConfig extends Config(
   new chipyard.iobinders.WithUARTAdapter ++
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTieOffInterrupts ++
-  new chipyard.iobinders.WithTiedOffDebug ++
+  new chipyard.iobinders.WithSimDebug ++
   new chipyard.iobinders.WithSimSerial ++
 
   new testchipip.WithTSI ++
@@ -128,6 +136,7 @@ class TutorialSha3BlackBoxConfig extends Config(
 
   // For this demonstration we assume the base system is a single-core Rocket, for fast elaboration
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new freechips.rocketchip.subsystem.WithJtagDTM ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
   new freechips.rocketchip.subsystem.WithNoMMIOPort ++
   new freechips.rocketchip.subsystem.WithNoSlavePort ++

From 23e4c22a44963f03c05eb1687865ca9490c71d25 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 2 Sep 2020 23:52:55 -0700
Subject: [PATCH 49/74] Don't run find in base_dir to avoid slow filesystem
 search

---
 common.mk | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/common.mk b/common.mk
index 89ebbea3..f8bd4cf4 100644
--- a/common.mk
+++ b/common.mk
@@ -62,7 +62,8 @@ SOURCE_DIRS = $(addprefix $(base_dir)/,generators sims/firesim/sim tools/barstoo
 SCALA_SOURCES = $(call lookup_srcs,$(SOURCE_DIRS),scala)
 VLOG_SOURCES = $(call lookup_srcs,$(SOURCE_DIRS),sv) $(call lookup_srcs,$(SOURCE_DIRS),v)
 # This assumes no SBT meta-build sources
-SBT_SOURCES = $(call lookup_srcs,$(base_dir),sbt)
+SBT_SOURCE_DIRS = $(addprefix $(base_dir)/,generators sims/firesim/sim tools)
+SBT_SOURCES = $(call lookup_srcs,$(SBT_SOURCE_DIRS),sbt) $(base_dir)/build.sbt $(base_dir)/project/plugins.sbt
 
 #########################################################################################
 # jar creation variables and rules

From 3258fd8db8021407d24abb37f303e1d2a3dd34d6 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Thu, 3 Sep 2020 23:53:51 -0700
Subject: [PATCH 50/74] Remove JTAG from firesim comfigs due to @(posedge ~clk)
 issue

---
 docs/Advanced-Concepts/Chip-Communication.rst          | 3 ++-
 generators/firechip/src/main/scala/TargetConfigs.scala | 4 +++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/docs/Advanced-Concepts/Chip-Communication.rst b/docs/Advanced-Concepts/Chip-Communication.rst
index 84bfc5fb..8d63992a 100644
--- a/docs/Advanced-Concepts/Chip-Communication.rst
+++ b/docs/Advanced-Concepts/Chip-Communication.rst
@@ -130,7 +130,8 @@ Using the JTAG Interface
 ------------------------
 
 The main way to use JTAG with a Rocket Chip based system is to instantiate the Debug Transfer Module (DTM)
-and configure it to use a JTAG interface. The default Chipyard designs configure the DTM to use JTAG. you may attach OpenOCD and GDB to any of the default JTAG-enabled designs.
+and configure it to use a JTAG interface. The default Chipyard designs instantiate the DTM and configure it
+to use JTAG. You may attach OpenOCD and GDB to any of the default JTAG-enabled designs.
 
 Debugging with JTAG
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/generators/firechip/src/main/scala/TargetConfigs.scala b/generators/firechip/src/main/scala/TargetConfigs.scala
index 66a20bce..0d8cd367 100644
--- a/generators/firechip/src/main/scala/TargetConfigs.scala
+++ b/generators/firechip/src/main/scala/TargetConfigs.scala
@@ -88,7 +88,9 @@ class WithFireSimConfigTweaks extends Config(
   // Optional: Removing this will require using an initramfs under linux
   new testchipip.WithBlockDevice ++
   // Required*: Scale default baud rate with periphery bus frequency
-  new chipyard.config.WithUART(BigInt(3686400L))
+  new chipyard.config.WithUART(BigInt(3686400L)) ++
+  // Required: Do not support debug module w. JTAG until FIRRTL stops emitting @(posedge ~clock)
+  new chipyard.config.WithNoDebug
 )
 
 /*******************************************************************************

From 0f50e4d1186323c3de008de55892695039019add Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 4 Sep 2020 13:35:05 -0700
Subject: [PATCH 51/74] Split IOBinders into IOBinders and Harness Binders |
 punch out clocks to harness for simwidgets and bridges

---
 build.sbt                                     |   2 +-
 .../chipyard/src/main/scala/ChipTop.scala     |  18 +-
 .../chipyard/src/main/scala/Clocks.scala      |   5 +-
 .../chipyard/src/main/scala/DigitalTop.scala  |   1 -
 .../src/main/scala/HarnessBinders.scala       | 274 ++++++++++
 .../chipyard/src/main/scala/IOBinders.scala   | 489 +++++++-----------
 .../chipyard/src/main/scala/TestHarness.scala |  17 +-
 .../main/scala/config/AbstractConfig.scala    |  36 +-
 .../src/main/scala/config/ArianeConfigs.scala |   2 +-
 .../src/main/scala/config/BoomConfigs.scala   |   4 +-
 .../src/main/scala/config/RocketConfigs.scala |  17 +-
 .../main/scala/config/TracegenConfigs.scala   |  30 +-
 .../main/scala/config/TutorialConfigs.scala   |  96 +---
 .../src/main/scala/BridgeBinders.scala        | 150 +++---
 .../firechip/src/main/scala/FireSim.scala     |  18 +-
 generators/icenet                             |   2 +-
 generators/testchipip                         |   2 +-
 sims/firesim                                  |   2 +-
 tools/barstools                               |   2 +-
 19 files changed, 643 insertions(+), 524 deletions(-)
 create mode 100644 generators/chipyard/src/main/scala/HarnessBinders.scala

diff --git a/build.sbt b/build.sbt
index 5d642c1d..3ca021a7 100644
--- a/build.sbt
+++ b/build.sbt
@@ -211,7 +211,7 @@ lazy val midas      = ProjectRef(firesimDir, "midas")
 lazy val firesimLib = ProjectRef(firesimDir, "firesimLib")
 
 lazy val firechip = conditionalDependsOn(project in file("generators/firechip"))
-  .dependsOn(chipyard, midasTargetUtils, midas, firesimLib % "test->test;compile->compile")
+  .dependsOn(chipyard, midasTargetUtils, midas, iocell, firesimLib % "test->test;compile->compile")
   .settings(
     commonSettings,
     testGrouping in Test := isolateAllTests( (definedTests in Test).value ),
diff --git a/generators/chipyard/src/main/scala/ChipTop.scala b/generators/chipyard/src/main/scala/ChipTop.scala
index cf71987b..6ae63d57 100644
--- a/generators/chipyard/src/main/scala/ChipTop.scala
+++ b/generators/chipyard/src/main/scala/ChipTop.scala
@@ -9,7 +9,7 @@ import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGr
 import freechips.rocketchip.config.{Parameters, Field}
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, LazyRawModuleImp, LazyModuleImpLike}
 import freechips.rocketchip.util.{ResetCatchAndSync}
-import chipyard.iobinders.{IOBinders, TestHarnessFunction, IOBinderTuple}
+import chipyard.iobinders._
 
 import barstools.iocell.chisel._
 
@@ -26,11 +26,9 @@ case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters)
 class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunctions {
   // A publicly accessible list of IO cells (useful for a floorplanning tool, for example)
   val iocells = ArrayBuffer.empty[IOCell]
-  // A list of functions to call in the test harness
-  val harnessFunctions = ArrayBuffer.empty[TestHarnessFunction]
 
   // The system module specified by BuildSystem
-  val lSystem = LazyModule(p(BuildSystem)(p)).suggestName("system")
+  val lazySystem = LazyModule(p(BuildSystem)(p)).suggestName("system")
 
   // The implicitClockSinkNode provides the implicit clock and reset for the System
   val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
@@ -50,15 +48,13 @@ class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunc
 
     // The implicit clock and reset for the system is also, by convention, used for all the IOBinders
     // TODO: This may not be the right thing to do in all cases
-    withClockAndReset(implicit_clock, implicit_reset) {
-      val (_ports, _iocells, _harnessFunctions) = p(IOBinders).values.flatMap(f => f(lSystem) ++ f(lSystem.module)).unzip3
-      // We ignore _ports for now...
-      iocells ++= _iocells.flatten
-      harnessFunctions ++= _harnessFunctions.flatten
-    }
+    val (_ports, _iocells, _portMap) = ApplyIOBinders(lazySystem, p(IOBinders))
+    // We ignore _ports for now...
+    iocells ++= _iocells
+    portMap ++= _portMap
 
     // Connect the implicit clock/reset, if present
-    lSystem.module match { case l: LazyModuleImp => {
+    lazySystem.module match { case l: LazyModuleImp => {
       l.clock := implicit_clock
       l.reset := implicit_reset
     }}
diff --git a/generators/chipyard/src/main/scala/Clocks.scala b/generators/chipyard/src/main/scala/Clocks.scala
index 3fa349b5..d6b19e8d 100644
--- a/generators/chipyard/src/main/scala/Clocks.scala
+++ b/generators/chipyard/src/main/scala/Clocks.scala
@@ -91,7 +91,7 @@ object ClockingSchemeGenerators {
     chiptop.implicitClockSinkNode := implicitClockSourceNode
 
     // Drive the diplomaticclock graph of the DigitalTop (if present)
-    val simpleClockGroupSourceNode = chiptop.lSystem match {
+    val simpleClockGroupSourceNode = chiptop.lazySystem match {
       case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
         val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
         l.asyncClockGroupsNode := n
@@ -120,6 +120,7 @@ object ClockingSchemeGenerators {
         }
       }}
 
+
       chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
         clock_io := th.harnessClock
         Nil
@@ -137,7 +138,7 @@ object ClockingSchemeGenerators {
     val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
     chiptop.implicitClockSinkNode := implicitClockSourceNode
 
-    val simpleClockGroupSourceNode = chiptop.lSystem match {
+    val simpleClockGroupSourceNode = chiptop.lazySystem match {
       case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
         val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
         l.asyncClockGroupsNode := n
diff --git a/generators/chipyard/src/main/scala/DigitalTop.scala b/generators/chipyard/src/main/scala/DigitalTop.scala
index 81d0003d..ddcf66f3 100644
--- a/generators/chipyard/src/main/scala/DigitalTop.scala
+++ b/generators/chipyard/src/main/scala/DigitalTop.scala
@@ -33,7 +33,6 @@ class DigitalTop(implicit p: Parameters) extends ChipyardSystem
 class DigitalTopModule[+L <: DigitalTop](l: L) extends ChipyardSystemModule(l)
   with testchipip.CanHaveTraceIOModuleImp
   with testchipip.CanHavePeripheryBlockDeviceModuleImp
-  with testchipip.CanHavePeripherySerialModuleImp
   with sifive.blocks.devices.uart.HasPeripheryUARTModuleImp
   with sifive.blocks.devices.gpio.HasPeripheryGPIOModuleImp
   with sifive.blocks.devices.spi.HasPeripherySPIFlashModuleImp
diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala
new file mode 100644
index 00000000..54c99042
--- /dev/null
+++ b/generators/chipyard/src/main/scala/HarnessBinders.scala
@@ -0,0 +1,274 @@
+package chipyard.harness
+
+import chisel3._
+
+import freechips.rocketchip.config.{Field, Config, Parameters}
+import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImpLike}
+import freechips.rocketchip.amba.axi4.{AXI4Bundle, AXI4SlaveNode, AXI4MasterNode, AXI4EdgeParameters}
+import freechips.rocketchip.devices.debug._
+import freechips.rocketchip.jtag.{JTAGIO}
+import freechips.rocketchip.system.{SimAXIMem}
+import freechips.rocketchip.subsystem._
+
+import sifive.blocks.devices.gpio._
+import sifive.blocks.devices.uart._
+import sifive.blocks.devices.spi._
+
+import barstools.iocell.chisel._
+
+import testchipip._
+
+import chipyard.HasHarnessSignalReferences
+
+import tracegen.{TraceGenSystemModuleImp}
+import icenet.{CanHavePeripheryIceNICModuleImp, SimNetwork, NicLoopback, NICKey, NICIOvonly}
+
+import scala.reflect.{ClassTag}
+
+case object HarnessBinders extends Field[Map[String, (Any, HasHarnessSignalReferences, Seq[Data]) => Seq[Any]]](
+  Map[String, (Any, HasHarnessSignalReferences, Seq[Data]) => Seq[Any]]().withDefaultValue((t: Any, th: HasHarnessSignalReferences, d: Seq[Data]) => Nil)
+)
+
+
+object ApplyHarnessBinders {
+  def apply(th: HasHarnessSignalReferences, sys: LazyModule, map: Map[String, (Any, HasHarnessSignalReferences, Seq[Data]) => Seq[Any]], portMap: Map[String, Seq[Data]]) = {
+    val pm = portMap.withDefaultValue(Nil)
+    map.map { case (s, f) => f(sys, th, pm(s)) ++ f(sys.module, th, pm(s)) }
+  }
+}
+
+class OverrideHarnessBinder[T](fn: => (T, HasHarnessSignalReferences, Seq[Data]) => Seq[Any])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+  case HarnessBinders => up(HarnessBinders, site) + (tag.runtimeClass.toString ->
+      ((t: Any, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+        t match {
+          case system: T => fn(system, th, ports)
+          case _ => Nil
+        }
+      })
+  )
+})
+
+class ComposeHarnessBinder[T](fn: => (T, HasHarnessSignalReferences, Seq[Data]) => Seq[Any])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+  case HarnessBinders => up(HarnessBinders, site) + (tag.runtimeClass.toString ->
+      ((t: Any, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+        t match {
+          case system: T => up(HarnessBinders, site)(tag.runtimeClass.toString)(system, th, ports) ++ fn(system, th, ports)
+          case _ => Nil
+        }
+      })
+  )
+})
+
+class WithGPIOTiedOff extends OverrideHarnessBinder({
+  (system: HasPeripheryGPIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    ports.map { case p: GPIOPortIO => p <> AnalogConst(0) }
+    Nil
+  }
+})
+
+class WithUARTAdapter extends OverrideHarnessBinder({
+  (system: HasPeripheryUARTModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    UARTAdapter.connect(ports.map(_.asInstanceOf[UARTPortIO]))(system.p)
+    Nil
+  }
+})
+
+class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideHarnessBinder({
+  (system: HasPeripherySPIFlashModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    SimSPIFlashModel.connect(ports.map(_.asInstanceOf[SPIChipIO]), th.harnessReset, rdOnly)(system.p)
+    Nil
+  }
+})
+
+class WithSimBlockDevice extends OverrideHarnessBinder({
+  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val clock = WireInit(false.B.asClock)
+    ports.map {
+      case p: BlockDeviceIO => SimBlockDevice.connect(clock, th.harnessReset.asBool, Some(p))(system.p)
+      case c: Clock => clock := c
+    }
+    Nil
+  }
+})
+
+class WithBlockDeviceModel extends OverrideHarnessBinder({
+  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val clock = WireInit(false.B.asClock)
+    ports.map {
+      case p: BlockDeviceIO => withClockAndReset(clock, th.harnessReset) { BlockDeviceModel.connect(Some(p))(system.p) }
+      case c: Clock => clock := c
+    }
+    Nil
+  }
+})
+
+class WithLoopbackNIC extends OverrideHarnessBinder({
+  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val clock = WireInit(false.B.asClock)
+    ports.map {
+      case p: NICIOvonly => withClockAndReset(clock, th.harnessReset) {
+        NicLoopback.connect(Some(p), system.p(NICKey))
+      }
+      case c: Clock => clock := c
+    }
+    Nil
+  }
+})
+
+class WithSimNetwork extends OverrideHarnessBinder({
+  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val clock = WireInit(false.B.asClock)
+    ports.map {
+      case p: NICIOvonly => SimNetwork.connect(Some(p), clock, th.harnessReset.asBool)
+      case c: Clock => clock := c
+    }
+    Nil
+  }
+})
+
+class WithSimAXIMem extends OverrideHarnessBinder({
+  (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
+    val clock = WireInit(false.B.asClock)
+    ports.filter(_.isInstanceOf[Clock]).map { case p: Clock => clock := p }
+    val axi4_ports = ports.filter(_.isInstanceOf[AXI4Bundle])
+    (axi4_ports zip system.memAXI4Node.edges.in).map { case (port: AXI4Bundle, edge) =>
+      val mem = LazyModule(new SimAXIMem(edge, size=p(ExtMem).get.master.size)(p))
+      withClockAndReset(clock, th.harnessReset) {
+        Module(mem.module).suggestName("mem")
+      }
+      mem.io_axi4.head <> port
+    }
+    Nil
+  }
+})
+
+class WithBlackBoxSimMem extends OverrideHarnessBinder({
+  (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
+    val clock = WireInit(false.B.asClock)
+    ports.filter(_.isInstanceOf[Clock]).map { case p: Clock => clock := p }
+    val axi4_ports = ports.collect { case p: AXI4Bundle => p }
+    (axi4_ports zip system.memAXI4Node.edges.in).map { case (port: AXI4Bundle, edge) =>
+      val memSize = p(ExtMem).get.master.size
+      val lineSize = p(CacheBlockBytes)
+      val mem = Module(new SimDRAM(memSize, lineSize, edge.bundle)).suggestName("simdram")
+      mem.io.axi <> port
+      mem.io.clock := clock
+      mem.io.reset := th.harnessReset
+    }
+    Nil
+  }
+})
+
+class WithSimAXIMMIO extends OverrideHarnessBinder({
+  (system: CanHaveMasterAXI4MMIOPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
+    val clock = WireInit(false.B.asClock)
+    ports.filter(_.isInstanceOf[Clock]).map { case p: Clock => clock := p }
+    (ports zip system.mmioAXI4Node.edges.in).zipWithIndex.map { case ((port: AXI4Bundle, edge), i) =>
+      val mmio_mem = LazyModule(new SimAXIMem(edge, size = 4096)(p))
+      withClockAndReset(clock, th.harnessReset) {
+        Module(mmio_mem.module).suggestName(s"mmio_mem_${i}")
+      }
+      mmio_mem.io_axi4.head <> port
+    }
+    Nil
+  }
+})
+
+class WithTieOffInterrupts extends OverrideHarnessBinder({
+  (system: HasExtInterruptsModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    ports.map { case p: UInt =>  p := 0.U }
+    Nil
+  }
+})
+
+class WithTieOffL2FBusAXI extends OverrideHarnessBinder({
+  (system: CanHaveSlaveAXI4Port, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    ports.map { case p: AXI4Bundle =>
+      p := DontCare
+      p.tieoff()
+    }
+    Nil
+  }
+})
+
+class WithSimDebug extends OverrideHarnessBinder({
+  (system: HasPeripheryDebugModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    if (!ports.isEmpty) {
+      val dtm_success = Wire(Bool())
+      when (dtm_success) { th.success := true.B }
+      ports.map {
+        case d: ClockedDMIIO =>
+          val dtm = Module(new SimDTM()(system.p)).connect(th.harnessClock, th.harnessReset.asBool, d, dtm_success)
+        case j: JTAGIO =>
+          val jtag = Module(new SimJTAG(tickDelay=3)).connect(j, th.harnessClock, th.harnessReset.asBool, ~(th.harnessReset.asBool), dtm_success)
+        case _ =>
+          require(false, "We only support DMI or JTAG simulated debug connections")
+      }
+    }
+    Nil
+  }
+})
+
+class WithTiedOffDebug extends OverrideHarnessBinder({
+  (system: HasPeripheryDebugModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    ports.map {
+      case d: ClockedDMIIO =>
+        d.dmi.req.valid := false.B
+        d.dmi.req.bits  := DontCare
+        d.dmi.resp.ready := true.B
+        d.dmiClock := false.B.asClock
+        d.dmiReset := true.B
+      case j: JTAGIO =>
+        j.TCK := true.B.asClock
+        j.TMS := true.B
+        j.TDI := true.B
+        j.TRSTn.foreach { r => r := true.B }
+      case a: ClockedAPBBundle =>
+        a.tieoff()
+        a.clock := false.B.asClock
+        a.reset := true.B.asAsyncReset
+        a.psel := false.B
+        a.penable := false.B
+      case _ => require(false)
+    }
+    Nil
+  }
+})
+
+class WithTiedOffSerial extends OverrideHarnessBinder({
+  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    ports.map { case p: SerialIO => SerialAdapter.tieoff(Some(p)) }
+    Nil
+  }
+})
+
+class WithSimSerial extends OverrideHarnessBinder({
+  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val serial_clock = WireInit(false.B.asClock)
+    ports.map {
+      case p: SerialIO =>
+        val ser_success = SerialAdapter.connectSimSerial(p, serial_clock, th.harnessReset)
+        when (ser_success) { th.success := true.B }
+      case c: Clock =>
+        serial_clock := c
+    }
+    Nil
+  }
+})
+
+class WithTraceGenSuccess extends OverrideHarnessBinder({
+  (system: TraceGenSystemModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    ports.map { case p: Bool => when (p) { th.success := true.B } }
+    Nil
+  }
+})
+
+class WithSimDromajoBridge extends ComposeHarnessBinder({
+  (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    ports.map { case p: TraceOutputTop => p.traces.map(tileTrace => SimDromajoBridge(tileTrace)(system.p)) }
+    Nil
+  }
+})
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 54a0d1dc..0dad7e21 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -1,9 +1,8 @@
-package chipyard
-package object iobinders {
+package chipyard.iobinders
 
 import chisel3._
 import chisel3.util.experimental.{BoringUtils}
-import chisel3.experimental.{Analog, IO}
+import chisel3.experimental.{Analog, IO, DataMirror}
 
 import freechips.rocketchip.config.{Field, Config, Parameters}
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImpLike}
@@ -23,7 +22,9 @@ import tracegen.{TraceGenSystemModuleImp}
 import barstools.iocell.chisel._
 
 import testchipip._
-import icenet.{CanHavePeripheryIceNICModuleImp, SimNetwork, NicLoopback, NICKey}
+import icenet.{CanHavePeripheryIceNICModuleImp, SimNetwork, NicLoopback, NICKey, NICIOvonly}
+
+import chipyard.GlobalResetSchemeKey
 
 import scala.reflect.{ClassTag}
 
@@ -43,17 +44,22 @@ import scala.reflect.{ClassTag}
 
 // DOC include start: IOBinders
 // This type describes a function callable on the TestHarness instance. Its return type is unused.
-type TestHarnessFunction = (chipyard.HasHarnessSignalReferences) => Seq[Any]
-// IOBinders will return a Seq of this tuple, which contains three fields:
-//  1. A Seq containing all IO ports created by the IOBinder function
-//  2. A Seq containing all IO cell modules created by the IOBinder function
-//  3. An optional function to call inside the test harness (e.g. to connect the IOs)
-type IOBinderTuple = (Seq[Data], Seq[IOCell], Option[TestHarnessFunction])
 
-case object IOBinders extends Field[Map[String, (Any) => Seq[IOBinderTuple]]](
-  Map[String, (Any) => Seq[IOBinderTuple]]().withDefaultValue((Any) => Nil)
+
+
+case object IOBinders extends Field[Map[String, (Any) => (Seq[Data], Seq[IOCell])]](
+  Map[String, (Any) => (Seq[Data], Seq[IOCell])]().withDefaultValue((Any) => (Nil, Nil))
 )
 
+object ApplyIOBinders {
+  def apply(sys: LazyModule, map: Map[String, (Any) => (Seq[Data], Seq[IOCell])]):
+      (Iterable[Data], Iterable[IOCell], Map[String, Seq[Data]]) = {
+    val r = map.map({ case (s,f) => (f(sys), s) }) ++ map.map({ case (s,f) => (f(sys.module), s) })
+    (r.flatMap(_._1._1), r.flatMap(_._1._2), r.map { t => t._2 -> t._1._1 })
+  }
+}
+
+
 // Note: The parameters instance is accessible only through LazyModule
 // or LazyModuleImpLike. The self-type requirement in traits like
 // CanHaveMasterAXI4MemPort is insufficient to make it accessible to the IOBinder
@@ -71,12 +77,12 @@ object GetSystemParameters {
 
 // This macro overrides previous matches on some Top mixin. This is useful for
 // binders which drive IO, since those typically cannot be composed
-class OverrideIOBinder[T](fn: => (T) => Seq[IOBinderTuple])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class OverrideIOBinder[T](fn: => (T) => (Seq[Data], Seq[IOCell]))(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
   case IOBinders => up(IOBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any) => {
         t match {
           case system: T => fn(system)
-          case _ => Nil
+          case _ => (Nil, Nil)
         }
       })
   )
@@ -84,32 +90,44 @@ class OverrideIOBinder[T](fn: => (T) => Seq[IOBinderTuple])(implicit tag: ClassT
 
 // This macro composes with previous matches on some Top mixin. This is useful for
 // annotation-like binders, since those can typically be composed
-class ComposeIOBinder[T](fn: => (T) => Seq[IOBinderTuple])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class ComposeIOBinder[T](fn: => (T) => (Seq[Data], Seq[IOCell]))(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
   case IOBinders => up(IOBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any) => {
         t match {
-          case system: T => (up(IOBinders, site)(tag.runtimeClass.toString)(system)
-            ++ fn(system))
-          case _ => Nil
+          case system: T =>
+            val r = up(IOBinders, site)(tag.runtimeClass.toString)(system)
+            val h = fn(system)
+            (r._1 ++ h._1, r._2 ++ h._2)
+          case _ => (Nil, Nil)
         }
       })
   )
 })
 
+object BoreHelper {
+  def apply(name: String, source: Clock): Clock = {
+    val clock_io = IO(Output(Clock())).suggestName(name)
+    val clock_wire = Wire(Clock()).suggestName(s"chiptop_${name}")
+    dontTouch(clock_wire)
+    clock_wire := false.B.asClock // necessary for BoringUtils to work properly
+    BoringUtils.bore(source, Seq(clock_wire))
+    clock_io := clock_wire
+    clock_io
+  }
+}
+
 // DOC include end: IOBinders
 
-object AddIOCells {
-  /**
-   * Add IO cells to a SiFive GPIO devices and name the IO ports.
-   * @param gpios A Seq of GPIO port bundles
-   * @param genFn A callable function to generate a DigitalGPIOCell module to use
-   * @return Returns a tuple of (a 2D Seq of Analog IOs corresponding to individual GPIO pins; a 2D Seq of IOCell module references)
-   */
-  def gpio(gpios: Seq[GPIOPortIO], genFn: () => DigitalGPIOCell = IOCell.genericGPIO): (Seq[Seq[Analog]], Seq[Seq[IOCell]]) = {
-    gpios.zipWithIndex.map({ case (gpio, i) =>
+
+case object IOCellKey extends Field[IOCellTypeParams](GenericIOCellParams())
+
+
+class WithGPIOCells extends OverrideIOBinder({
+  (system: HasPeripheryGPIOModuleImp) => {
+    val (ports2d, cells2d) = system.gpio.zipWithIndex.map({ case (gpio, i) =>
       gpio.pins.zipWithIndex.map({ case (pin, j) =>
         val g = IO(Analog(1.W)).suggestName(s"gpio_${i}_${j}")
-        val iocell = genFn().suggestName(s"iocell_gpio_${i}_${j}")
+        val iocell = system.p(IOCellKey).gpio().suggestName(s"iocell_gpio_${i}_${j}")
         iocell.io.o := pin.o.oval
         iocell.io.oe := pin.o.oe
         iocell.io.ie := pin.o.ie
@@ -118,40 +136,35 @@ object AddIOCells {
         (g, iocell)
       }).unzip
     }).unzip
+    (ports2d.flatten, cells2d.flatten)
   }
+})
 
-  /**
-   * Add IO cells to a SiFive UART devices and name the IO ports.
-   * @param uartPins A Seq of UART port bundles
-   * @return Returns a tuple of (A Seq of top-level UARTPortIO IOs; a 2D Seq of IOCell module references)
-   */
-  def uart(uartPins: Seq[UARTPortIO]): (Seq[UARTPortIO], Seq[Seq[IOCell]]) = {
-    uartPins.zipWithIndex.map({ case (u, i) =>
-      val (port, ios) = IOCell.generateIOFromSignal(u, Some(s"iocell_uart_${i}"))
+
+class WithUARTIOCells extends OverrideIOBinder({
+  (system: HasPeripheryUARTModuleImp) => {
+    val (ports, cells2d) = system.uart.zipWithIndex.map({ case (u, i) =>
+      val (port, ios) = IOCell.generateIOFromSignal(u, Some(s"iocell_uart_${i}"), system.p(IOCellKey))
       port.suggestName(s"uart_${i}")
       (port, ios)
     }).unzip
+    (ports, cells2d.flatten)
   }
+})
 
-  /**
-   * Add IO cells to a SiFive SPI devices and name the IO ports.
-   * @param spiPins A Seq of SPI port bundles
-   * @param basename The base name for this port (defaults to "spi")
-   * @param genFn A callable function to generate a DigitalGPIOCell module to use
-   * @return Returns a tuple of (A Seq of top-level SPIChipIO IOs; a 2D Seq of IOCell module references)
-   */
-  def spi(spiPins: Seq[SPIPortIO], basename: String = "spi", genFn: () => DigitalGPIOCell = IOCell.genericGPIO): (Seq[SPIChipIO], Seq[Seq[IOCell]]) = {
-    spiPins.zipWithIndex.map({ case (s, i) =>
-      val port = IO(new SPIChipIO(s.c.csWidth)).suggestName(s"${basename}_${i}")
-      val iocellBase = s"iocell_${basename}_${i}"
+class WithSPIIOCells extends OverrideIOBinder({
+  (system: HasPeripherySPIFlashModuleImp) => {
+    val (ports, cells2d) = system.qspi.zipWithIndex.map({ case (s, i) =>
+      val port = IO(new SPIChipIO(s.c.csWidth)).suggestName(s"spi_${i}")
+      val iocellBase = s"iocell_spi_${i}"
 
       // SCK and CS are unidirectional outputs
-      val sckIOs = IOCell.generateFromSignal(s.sck, port.sck, Some(s"${iocellBase}_sck"))
-      val csIOs = IOCell.generateFromSignal(s.cs, port.cs, Some(s"${iocellBase}_cs"))
+      val sckIOs = IOCell.generateFromSignal(s.sck, port.sck, Some(s"${iocellBase}_sck"), system.p(IOCellKey))
+      val csIOs = IOCell.generateFromSignal(s.cs, port.cs, Some(s"${iocellBase}_cs"), system.p(IOCellKey))
 
       // DQ are bidirectional, so then need special treatment
       val dqIOs = s.dq.zip(port.dq).zipWithIndex.map { case ((pin, ana), j) =>
-        val iocell = genFn().suggestName(s"${iocellBase}_dq_${j}")
+        val iocell = system.p(IOCellKey).gpio().suggestName(s"${iocellBase}_dq_${j}")
         iocell.io.o := pin.o
         iocell.io.oe := pin.oe
         iocell.io.ie := true.B
@@ -162,17 +175,27 @@ object AddIOCells {
 
       (port, dqIOs ++ csIOs ++ sckIOs)
     }).unzip
+    (ports, cells2d.flatten)
   }
+})
 
-  /**
-   * Add IO cells to a debug module and name the IO ports, for debug IO which must go off-chip
-   * For on-chip debug IO, drive them appropriately
-   * Mostly copied from rocket-chip/src/main/scala/devices/debug/Periphery.scala
-   * @param system A BaseSubsystem that might have a debug module
-   * @return Returns a tuple2 of (Generated debug io ports, Generated IOCells)
-   */
-  def debug(system: HasPeripheryDebugModuleImp)(implicit p: Parameters): (Seq[Bundle], Seq[IOCell]) = {
-    system.debug.map { debug =>
+class WithExtInterruptIOCells extends OverrideIOBinder({
+  (system: HasExtInterruptsModuleImp) => {
+    if (system.outer.nExtInterrupts > 0) {
+      val (port, cells) = IOCell.generateIOFromSignal(system.interrupts, Some("iocell_interrupts"), system.p(IOCellKey))
+      port.suggestName("ext_interrupts")
+      (Seq(port), cells)
+    } else {
+      (Nil, Nil)
+    }
+  }
+})
+
+
+class WithDebugIOCells extends OverrideIOBinder({
+  (system: HasPeripheryDebugModuleImp) => {
+    system.debug.map({ debug =>
+      val p = system.p
       val tlbus = system.outer.asInstanceOf[BaseSubsystem].locateTLBusWrapper(p(ExportDebug).slaveWhere)
       val debug_clock = Wire(Clock()).suggestName("debug_clock")
       val debug_reset = Wire(Reset()).suggestName("debug_reset")
@@ -204,288 +227,130 @@ object AddIOCells {
 
       // Add IOCells for the DMI/JTAG/APB ports
       val dmiTuple = debug.clockeddmi.map { d =>
-        IOCell.generateIOFromSignal(d, Some("iocell_dmi"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
+        IOCell.generateIOFromSignal(d, Some("iocell_dmi"), p(IOCellKey), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
       }
       dmiTuple.map(_._1).foreach(_.suggestName("dmi"))
 
       val jtagTuple = debug.systemjtag.map { j =>
-        IOCell.generateIOFromSignal(j.jtag, Some("iocell_jtag"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
+        IOCell.generateIOFromSignal(j.jtag, Some("iocell_jtag"), p(IOCellKey), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
       }
       jtagTuple.map(_._1).foreach(_.suggestName("jtag"))
 
       val apbTuple = debug.apb.map { a =>
-        IOCell.generateIOFromSignal(a, Some("iocell_apb"), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
+        IOCell.generateIOFromSignal(a, Some("iocell_apb"), p(IOCellKey), abstractResetAsAsync = p(GlobalResetSchemeKey).pinIsAsync)
       }
       apbTuple.map(_._1).foreach(_.suggestName("apb"))
 
       val allTuples = (dmiTuple ++ jtagTuple ++ apbTuple).toSeq
       (allTuples.map(_._1).toSeq, allTuples.flatMap(_._2).toSeq)
-    }.getOrElse((Nil, Nil))
+    }).getOrElse((Nil, Nil))
   }
+})
 
-  /**
-   * Add IO cells to a serial module and name the IO ports.
-   * @param serial A SerialIO bundle
-   * @return Returns a tuple of (Top-level SerialIO IO; a list of IOCell module references)
-   */
-  def serial(serial: SerialIO): (SerialIO, Seq[IOCell]) = {
-    val (port, ios) = IOCell.generateIOFromSignal(serial, Some("iocell_serial"))
+class WithSerialIOCells extends OverrideIOBinder({
+  (system: CanHavePeripherySerial) => system.serial.map({ s =>
+    val sys = system.asInstanceOf[BaseSubsystem]
+    val (port, cells) = IOCell.generateIOFromSignal(s, Some("iocell_serial"), sys.p(IOCellKey))
+    val serial_clock = Wire(Output(Clock())).suggestName("chiptop_serial_clock")
+    serial_clock := false.B.asClock // necessary for BoringUtils to work properly
+    dontTouch(serial_clock)
+    BoringUtils.bore(sys.fbus.module.clock, Seq(serial_clock))
+    val (serial_clock_io, serial_clock_cell) = IOCell.generateIOFromSignal(serial_clock, Some("serial_clock"), sys.p(IOCellKey))
+    serial_clock_io.suggestName("serial_clock")
     port.suggestName("serial")
-    (port, ios)
-  }
-
-  def axi4(io: Seq[AXI4Bundle], node: AXI4SlaveNode, name: String): Seq[(AXI4Bundle, AXI4EdgeParameters, Seq[IOCell])] = {
-    io.zip(node.edges.in).zipWithIndex.map{ case ((mem_axi4, edge), i) => {
-      val (port, ios) = IOCell.generateIOFromSignal(mem_axi4, Some(s"iocell_${name}_axi4_slave_${i}"))
-      port.suggestName(s"${name}_axi4_slave_${i}")
-      (port, edge, ios)
-    }}
-  }
-  def axi4(io: Seq[AXI4Bundle], node: AXI4MasterNode, name: String): Seq[(AXI4Bundle, AXI4EdgeParameters, Seq[IOCell])] = {
-    io.zip(node.edges.out).zipWithIndex.map{ case ((mem_axi4, edge), i) => {
-      //val (port, ios) = IOCell.generateIOFromSignal(mem_axi4, Some(s"iocell_${name}_axi4_master_${i}"))
-      val port = IO(Flipped(AXI4Bundle(edge.bundle)))
-      val ios = IOCell.generateFromSignal(mem_axi4, port, Some(s"iocell_${name}_axi4_master_${i}"))
-      port.suggestName(s"${name}_axi4_master_${i}")
-      (port, edge, ios)
-    }}
-  }
-
-  def blockDev(bdev: BlockDeviceIO): (BlockDeviceIO, Seq[IOCell]) = {
-    val (port, ios) = IOCell.generateIOFromSignal(bdev, Some("iocell_bdev"))
-    port.suggestName("bdev")
-    (port, ios)
-  }
-}
-
-// DOC include start: WithGPIOTiedOff
-class WithGPIOTiedOff extends OverrideIOBinder({
-  (system: HasPeripheryGPIOModuleImp) => {
-    val (ports2d, ioCells2d) = AddIOCells.gpio(system.gpio)
-    val harnessFn = (th: HasHarnessSignalReferences) => { ports2d.flatten.foreach(_ <> AnalogConst(0)); Nil }
-    Seq((ports2d.flatten, ioCells2d.flatten, Some(harnessFn)))
-  }
-})
-// DOC include end: WithGPIOTiedOff
-
-class WithUARTAdapter extends OverrideIOBinder({
-  (system: HasPeripheryUARTModuleImp) => {
-    val (ports, ioCells2d) = AddIOCells.uart(system.uart)
-    val harnessFn = (th: HasHarnessSignalReferences) => { UARTAdapter.connect(ports)(system.p); Nil }
-    Seq((ports, ioCells2d.flatten, Some(harnessFn)))
-  }
+    (Seq(port, serial_clock_io), cells ++ serial_clock_cell)
+  }).getOrElse((Nil, Nil))
 })
 
-class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideIOBinder({
-  (system: HasPeripherySPIFlashModuleImp) => {
-    val (ports, ioCells2d) = AddIOCells.spi(system.qspi, "qspi")
-    val harnessFn = (th: HasHarnessSignalReferences) => { SimSPIFlashModel.connect(ports, th.harnessReset, rdOnly)(system.p); Nil }
-    Seq((ports, ioCells2d.flatten, Some(harnessFn)))
-  }
-})
 
-class WithSimBlockDevice extends OverrideIOBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev =>
-    val (port, ios) = AddIOCells.blockDev(bdev)
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock
-      SimBlockDevice.connect(th.harnessClock, th.harnessReset.asBool, Some(port))(system.p)
-      Nil
-    }
-    Seq((Seq(port), ios, Some(harnessFn)))
-  }.getOrElse(Nil)
-})
-
-class WithBlockDeviceModel extends OverrideIOBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev =>
-    val (port, ios) = AddIOCells.blockDev(bdev)
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      BlockDeviceModel.connect(Some(port))(system.p)
-      Nil
-    }
-    Seq((Seq(port), ios, Some(harnessFn)))
-  }.getOrElse(Nil)
-})
-
-class WithLoopbackNIC extends OverrideIOBinder({
-  (system: CanHavePeripheryIceNICModuleImp) => system.connectNicLoopback(); Nil
-})
-
-class WithSimNIC extends OverrideIOBinder({
-  (system: CanHavePeripheryIceNICModuleImp) => system.connectSimNetwork(system.clock, system.reset.asBool); Nil
-})
-
-// DOC include start: WithSimAXIMem
-class WithSimAXIMem extends OverrideIOBinder({
+class WithAXI4MemPunchthrough extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {
-    implicit val p: Parameters = GetSystemParameters(system)
-    val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem")
-    // TODO: we are inlining the connectMem method of SimAXIMem because
-    //   it takes in a dut rather than seq of axi4 ports
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      peiTuples.map { case (port, edge, ios) =>
-        val mem = LazyModule(new SimAXIMem(edge, size = p(ExtMem).get.master.size))
-        Module(mem.module).suggestName("mem")
-        mem.io_axi4.head <> port
-      }
-      Nil
+    val clock = if (!system.mem_axi4.isEmpty) {
+      Some(BoreHelper("axi4_mem_clock", system.asInstanceOf[BaseSubsystem].mbus.module.clock))
+    } else {
+      None
     }
-    Seq((peiTuples.map(_._1), peiTuples.flatMap(_._3), Some(harnessFn)))
-  }
-})
-// DOC include end: WithSimAXIMem
-
-class WithBlackBoxSimMem extends OverrideIOBinder({
-  (system: CanHaveMasterAXI4MemPort) => {
-    implicit val p: Parameters = GetSystemParameters(system)
-    val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem")
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      peiTuples.map { case (port, edge, ios) =>
-        val memSize = p(ExtMem).get.master.size
-        val lineSize = p(CacheBlockBytes)
-        val mem = Module(new SimDRAM(memSize, lineSize, edge.bundle))
-        mem.io.axi <> port
-        // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock
-        mem.io.clock := th.harnessClock
-        mem.io.reset := th.harnessReset
-      }
-      Nil
-    }
-    Seq((peiTuples.map(_._1), peiTuples.flatMap(_._3), Some(harnessFn)))
+    val ports = system.mem_axi4.map({ m =>
+      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName("axi4_mem")
+      p <> m
+      p
+    })
+    (ports ++ clock, Nil)
   }
 })
 
-class WithSimAXIMMIO extends OverrideIOBinder({
+class WithAXI4MMIOPunchthrough extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MMIOPort) => {
-    implicit val p: Parameters = GetSystemParameters(system)
-    val peiTuples = AddIOCells.axi4(system.mmio_axi4, system.mmioAXI4Node, "mmio_mem")
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      peiTuples.zipWithIndex.map { case ((port, edge, ios), i) =>
-        val mmio_mem = LazyModule(new SimAXIMem(edge, size = 4096))
-        Module(mmio_mem.module).suggestName(s"mmio_mem_${i}")
-        mmio_mem.io_axi4.head <> port
-      }
-      Nil
+    val clock = if (!system.mmio_axi4.isEmpty) {
+      Some(BoreHelper("axi4_mmio_clock", system.asInstanceOf[BaseSubsystem].mbus.module.clock))
+    } else {
+      None
     }
-    Seq((peiTuples.map(_._1), peiTuples.flatMap(_._3), Some(harnessFn)))
+    val ports = system.mmio_axi4.map({ m =>
+      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName("axi4_mmio")
+      p <> m
+      p
+    })
+    (ports ++ clock, Nil)
   }
 })
 
+class WithL2FBusAXI4Punchthrough extends OverrideIOBinder({
+  (system: CanHaveSlaveAXI4Port) => {
+    val port = system.l2_frontend_bus_axi4.map { m =>
+      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName("axi4_fbus")
+      p <> m
+      p
+    }
+    (port, Nil)
+  }
+})
+
+class WithBlockDeviceIOPunchthrough extends OverrideIOBinder({
+  (system: CanHavePeripheryBlockDeviceModuleImp) => {
+    val ports = system.bdev.map({ bdev =>
+      val p = IO(new BlockDeviceIO()(system.p)).suggestName("blockdev")
+      val clock = BoreHelper("blkdev_clk", system.outer.controller.get.module.clock)
+      p <> bdev
+      Seq(p, clock)
+    }).getOrElse(Nil)
+    (ports, Nil)
+  }
+})
+
+class WithNICIOPunchthrough extends OverrideIOBinder({
+  (system: CanHavePeripheryIceNICModuleImp) => {
+    val port = system.net.map({ n =>
+      val p = IO(new NICIOvonly).suggestName("nic")
+      val clock = BoreHelper("nic_clk", system.outer.icenicOpt.get.module.clock)
+      p <> n
+      Seq(p, clock)
+    }).getOrElse(Nil)
+    (port.toSeq, Nil)
+  }
+})
+
+class WithTraceGenSuccessPunchthrough extends OverrideIOBinder({
+  (system: TraceGenSystemModuleImp) => {
+    val success = IO(Output(Bool())).suggestName("success")
+    success := system.success
+    (Seq(success), Nil)
+  }
+})
+
+class WithTraceIOPunchthrough extends OverrideIOBinder({
+  (system: CanHaveTraceIOModuleImp) => {
+    val ports = system.traceIO.map { t =>
+      val trace = IO(DataMirror.internal.chiselTypeClone[TraceOutputTop](t)).suggestName("trace")
+      trace <> t
+      trace
+    }
+    (ports.toSeq, Nil)
+  }
+})
+
+
 class WithDontTouchPorts extends OverrideIOBinder({
-  (system: DontTouch) => system.dontTouchPorts(); Nil
+  (system: DontTouch) => system.dontTouchPorts(); (Nil, Nil)
 })
 
-class WithTieOffInterrupts extends OverrideIOBinder({
-  (system: HasExtInterruptsModuleImp) => {
-    val (port, ioCells) = IOCell.generateIOFromSignal(system.interrupts, Some("iocell_interrupts"))
-    port.suggestName("interrupts")
-    val harnessFn = (th: HasHarnessSignalReferences) => { port := 0.U; Nil }
-    Seq((Seq(port), ioCells, Some(harnessFn)))
-  }
-})
-
-class WithTieOffL2FBusAXI extends OverrideIOBinder({
-  (system: CanHaveSlaveAXI4Port) => {
-    val peiTuples = AddIOCells.axi4(system.l2_frontend_bus_axi4, system.l2FrontendAXI4Node, "l2_fbus")
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      peiTuples.zipWithIndex.map { case ((port, edge, ios), i) =>
-        port := DontCare // tieoff doesn't completely tie-off, for some reason
-        port.tieoff()
-      }
-      Nil
-    }
-    Seq((peiTuples.map(_._1), peiTuples.flatMap(_._3), Some(harnessFn)))
-  }
-})
-
-class WithSimDebug extends OverrideIOBinder({
-  (system: HasPeripheryDebugModuleImp) => {
-    val (ports, iocells) = AddIOCells.debug(system)(system.p)
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      val dtm_success = WireInit(false.B)
-      when (dtm_success) { th.success := true.B }
-      ports.map {
-        case d: ClockedDMIIO =>
-          val dtm = Module(new SimDTM()(system.p)).connect(th.harnessClock, th.harnessReset.asBool, d, dtm_success)
-        case j: JTAGIO =>
-          val jtag = Module(new SimJTAG(tickDelay=3)).connect(j, th.harnessClock, th.harnessReset.asBool, ~(th.harnessReset.asBool), dtm_success)
-        case _ =>
-          require(false, "We only support DMI or JTAG simulated debug connections")
-      }
-      Nil
-    }
-    Seq((ports, iocells, Some(harnessFn)))
-  }
-})
-
-class WithTiedOffDebug extends OverrideIOBinder({
-  (system: HasPeripheryDebugModuleImp) => {
-    val (ports, iocells) = AddIOCells.debug(system)(system.p)
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      ports.map {
-        case d: ClockedDMIIO =>
-          d.dmi.req.valid := false.B
-          d.dmi.req.bits  := DontCare
-          d.dmi.resp.ready := true.B
-          d.dmiClock := false.B.asClock
-          d.dmiReset := true.B
-        case j: JTAGIO =>
-          j.TCK := true.B.asClock
-          j.TMS := true.B
-          j.TDI := true.B
-          j.TRSTn.foreach { r => r := true.B }
-        case a: ClockedAPBBundle =>
-          a.tieoff()
-          a.clock := false.B.asClock
-          a.reset := true.B.asAsyncReset
-          a.psel := false.B
-          a.penable := false.B
-        case _ => require(false)
-      }
-      Nil
-    }
-    Seq((ports, iocells, Some(harnessFn)))
-  }
-})
-
-class WithTiedOffSerial extends OverrideIOBinder({
-  (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial =>
-    val (port, ioCells) = AddIOCells.serial(serial)
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      SerialAdapter.tieoff(port)
-      Nil
-    }
-    Seq((Seq(port), ioCells, Some(harnessFn)))
-  }).getOrElse(Nil)
-})
-
-class WithSimSerial extends OverrideIOBinder({
-  (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial =>
-    val (port, ioCells) = AddIOCells.serial(serial)
-    val harnessFn = (th: HasHarnessSignalReferences) => {
-      val ser_success = SerialAdapter.connectSimSerial(port, th.harnessClock, th.harnessReset)
-      when (ser_success) { th.success := true.B }
-      Nil
-    }
-    Seq((Seq(port), ioCells, Some(harnessFn)))
-  }).getOrElse(Nil)
-})
-
-class WithTraceGenSuccessBinder extends OverrideIOBinder({
-  (system: TraceGenSystemModuleImp) => {
-    val (successPort, ioCells) = IOCell.generateIOFromSignal(system.success, Some("iocell_success"))
-    successPort.suggestName("success")
-    val harnessFn = (th: HasHarnessSignalReferences) => { when (successPort) { th.success := true.B }; Nil }
-    Seq((Seq(successPort), ioCells, Some(harnessFn)))
-  }
-})
-
-class WithSimDromajoBridge extends ComposeIOBinder({
-   (system: CanHaveTraceIOModuleImp) => {
-     system.traceIO match { case Some(t) => t.traces.map(tileTrace => SimDromajoBridge(tileTrace)(system.p)) }
-     Nil
-   }
-})
-
-
-} /* end package object */
diff --git a/generators/chipyard/src/main/scala/TestHarness.scala b/generators/chipyard/src/main/scala/TestHarness.scala
index b296e328..67cf03bf 100644
--- a/generators/chipyard/src/main/scala/TestHarness.scala
+++ b/generators/chipyard/src/main/scala/TestHarness.scala
@@ -1,19 +1,22 @@
 package chipyard
 
 import chisel3._
-
+import scala.collection.mutable.{ArrayBuffer}
 import freechips.rocketchip.diplomacy.{LazyModule}
 import freechips.rocketchip.config.{Field, Parameters}
-import chipyard.iobinders.{TestHarnessFunction}
+
+import chipyard.harness.{ApplyHarnessBinders, HarnessBinders}
 
 // -------------------------------
 // Chipyard Test Harness
 // -------------------------------
 
-case object BuildTop extends Field[Parameters => LazyModule with HasTestHarnessFunctions]((p: Parameters) => new ChipTop()(p))
+case object BuildTop extends Field[Parameters => LazyModule]((p: Parameters) => new ChipTop()(p))
 
 trait HasTestHarnessFunctions {
-  val harnessFunctions: Seq[TestHarnessFunction]
+  val lazySystem: LazyModule
+  val harnessFunctions = ArrayBuffer.empty[HasHarnessSignalReferences => Seq[Any]]
+  val portMap = scala.collection.mutable.Map[String, Seq[Data]]()
 }
 
 trait HasHarnessSignalReferences {
@@ -39,7 +42,9 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSign
   // dutReset assignment can be overridden via a harnessFunction, but by default it is just reset
   val dutReset = WireDefault(if (p(GlobalResetSchemeKey).pinIsAsync) reset.asAsyncReset else reset)
 
-  ldut.harnessFunctions.foreach(_(this))
-
+  ldut match { case d: HasTestHarnessFunctions =>
+    d.harnessFunctions.foreach(_(this))
+    ApplyHarnessBinders(this, d.lazySystem, p(HarnessBinders), d.portMap.toMap)
+  }
 }
 
diff --git a/generators/chipyard/src/main/scala/config/AbstractConfig.scala b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
index a925ec56..950cb4b4 100644
--- a/generators/chipyard/src/main/scala/config/AbstractConfig.scala
+++ b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
@@ -5,14 +5,40 @@ import freechips.rocketchip.config.{Config}
 // --------------
 // Chipyard abstract ("base") configuration
 // NOTE: This configuration is NOT INSTANTIABLE, as it defines a empty system with no tiles
+//
+// The default set of IOBinders instantiate IOcells and ChipTop IOs for digital IO bundles.
+// The default set of HarnessBinders instantiate TestHarness hardware for interacting with ChipTop IOs
 // --------------
 
 class AbstractConfig extends Config(
-  new chipyard.iobinders.WithUARTAdapter ++                      // display UART with a SimUARTAdapter
-  new chipyard.iobinders.WithTieOffInterrupts ++                 // tie off top-level interrupts
-  new chipyard.iobinders.WithBlackBoxSimMem ++                   // drive the master AXI4 memory with a blackbox DRAMSim model
-  new chipyard.iobinders.WithSimDebug ++                         // attach SimJTAG
-  new chipyard.iobinders.WithSimSerial ++                        // drive TSI with SimSerial for testing
+  // The HarnessBinders control generation of hardware in the TestHarness
+  new chipyard.harness.WithUARTAdapter ++                       // add UART adapter to display UART on stdout, if uart is present
+  new chipyard.harness.WithBlackBoxSimMem ++                    // add SimDRAM DRAM model for axi4 backing memory, if axi4 mem is enabled
+  new chipyard.harness.WithSimDebug ++                          // add SimJTAG or SimDTM adapters if debug module is enabled
+  new chipyard.harness.WithSimSerial ++                         // add SimSerial adapter for HTIF, if serial port is present
+  new chipyard.harness.WithGPIOTiedOff ++                       // tie-off chiptop GPIOs, if GPIOs are present
+  new chipyard.harness.WithSimSPIFlashModel ++                  // add simulated SPI flash memory, if SPI is enabled
+  new chipyard.harness.WithSimAXIMMIO ++                        // add SimAXIMem for axi4 mmio port, if enabled
+  new chipyard.harness.WithTieOffInterrupts ++                  // tie-off interrupt ports, if present
+  new chipyard.harness.WithTieOffL2FBusAXI ++                   // tie-off external AXI4 master, if present
+
+  // The IOBinders instantiate ChipTop IOs to match desired digital IOs
+  // IOCells are generated for "Chip-like" IOs, while simulation-only IOs are directly punched through
+  new chipyard.iobinders.WithAXI4MemPunchthrough ++
+  new chipyard.iobinders.WithAXI4MMIOPunchthrough ++
+  new chipyard.iobinders.WithL2FBusAXI4Punchthrough ++
+  new chipyard.iobinders.WithBlockDeviceIOPunchthrough ++
+  new chipyard.iobinders.WithNICIOPunchthrough ++
+  new chipyard.iobinders.WithSerialIOCells ++
+  new chipyard.iobinders.WithDebugIOCells ++
+  new chipyard.iobinders.WithUARTIOCells ++
+  new chipyard.iobinders.WithGPIOCells ++
+  new chipyard.iobinders.WithUARTIOCells ++
+  new chipyard.iobinders.WithSPIIOCells ++
+  new chipyard.iobinders.WithTraceIOPunchthrough ++
+  new chipyard.iobinders.WithExtInterruptIOCells ++
+
+
   new testchipip.WithTSI ++                                      // use testchipip serial offchip link
   new chipyard.config.WithBootROM ++                             // use default bootrom
   new chipyard.config.WithUART ++                                // add a UART
diff --git a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
index 6bc7cf69..47e7c15b 100644
--- a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
@@ -13,7 +13,7 @@ class ArianeConfig extends Config(
   new chipyard.config.AbstractConfig)
 
 class dmiArianeConfig extends Config(
-  new chipyard.iobinders.WithTiedOffSerial ++          // Tie off the serial port, override default instantiation of SimSerial
+  new chipyard.harness.WithTiedOffSerial ++          // Tie off the serial port, override default instantiation of SimSerial
   new chipyard.config.WithDMIDTM ++                    // have debug module expose a clocked DMI port
   new ariane.WithNArianeCores(1) ++                    // single Ariane core
   new chipyard.config.AbstractConfig)
diff --git a/generators/chipyard/src/main/scala/config/BoomConfigs.scala b/generators/chipyard/src/main/scala/config/BoomConfigs.scala
index 575e1f98..9e1f558a 100644
--- a/generators/chipyard/src/main/scala/config/BoomConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/BoomConfigs.scala
@@ -33,13 +33,13 @@ class HwachaLargeBoomConfig extends Config(
   new chipyard.config.AbstractConfig)
 
 class LoopbackNICLargeBoomConfig extends Config(
-  new chipyard.iobinders.WithLoopbackNIC ++                      // drive NIC IOs with loopback
+  new chipyard.harness.WithLoopbackNIC ++                        // drive NIC IOs with loopback
   new icenet.WithIceNIC ++                                       // build a NIC
   new boom.common.WithNLargeBooms(1) ++
   new chipyard.config.AbstractConfig)
 
 class DromajoBoomConfig extends Config(
-  new chipyard.iobinders.WithSimDromajoBridge ++                 // attach Dromajo
+  new chipyard.harness.WithSimDromajoBridge ++                   // attach Dromajo
   new chipyard.config.WithTraceIO ++                             // enable the traceio
   new boom.common.WithNSmallBooms(1) ++
   new chipyard.config.AbstractConfig)
diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
index 420ba192..a8686961 100644
--- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
@@ -25,7 +25,7 @@ class GemminiRocketConfig extends Config(
 
 // DOC include start: DmiRocket
 class dmiRocketConfig extends Config(
-  new chipyard.iobinders.WithTiedOffSerial ++          // don't use serial to drive the chip, since we use DMI instead
+  new chipyard.harness.WithTiedOffSerial ++          // don't use serial to drive the chip, since we use DMI instead
   new chipyard.config.WithDMIDTM ++                    // have debug module expose a clocked DMI port
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
@@ -46,37 +46,36 @@ class GCDAXI4BlackBoxRocketConfig extends Config(
 // DOC include end: GCDAXI4BlackBoxRocketConfig
 
 class LargeSPIFlashROMRocketConfig extends Config(
-  new chipyard.iobinders.WithSimSPIFlashModel(true) ++      // add the SPI flash model in the harness (read-only)
+  new chipyard.harness.WithSimSPIFlashModel(true) ++      // add the SPI flash model in the harness (read-only)
   new chipyard.config.WithSPIFlash ++                       // add the SPI flash controller
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class SmallSPIFlashRocketConfig extends Config(
-  new chipyard.iobinders.WithSimSPIFlashModel(false) ++     // add the SPI flash model in the harness (writeable)
+  new chipyard.harness.WithSimSPIFlashModel(false) ++     // add the SPI flash model in the harness (writeable)
   new chipyard.config.WithSPIFlash(0x100000) ++             // add the SPI flash controller (1 MiB)
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class SimAXIRocketConfig extends Config(
-  new chipyard.iobinders.WithSimAXIMem ++                   // drive the master AXI4 memory with a SimAXIMem, a 1-cycle magic memory, instead of default SimDRAM
+  new chipyard.harness.WithSimAXIMem ++                   // drive the master AXI4 memory with a SimAXIMem, a 1-cycle magic memory, instead of default SimDRAM
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class SimBlockDeviceRocketConfig extends Config(
-  new chipyard.iobinders.WithSimBlockDevice ++             // drive block-device IOs with SimBlockDevice
+  new chipyard.harness.WithSimBlockDevice ++             // drive block-device IOs with SimBlockDevice
   new testchipip.WithBlockDevice ++                        // add block-device module to peripherybus
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class BlockDeviceModelRocketConfig extends Config(
-  new chipyard.iobinders.WithBlockDeviceModel ++           // drive block-device IOs with a BlockDeviceModel
+  new chipyard.harness.WithBlockDeviceModel ++           // drive block-device IOs with a BlockDeviceModel
   new testchipip.WithBlockDevice ++                        // add block-device module to periphery bus
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 // DOC include start: GPIORocketConfig
 class GPIORocketConfig extends Config(
-  new chipyard.iobinders.WithGPIOTiedOff ++                // tie off GPIO inputs into the top
   new chipyard.config.WithGPIO ++                          // add GPIOs to the peripherybus
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
@@ -111,7 +110,7 @@ class InitZeroRocketConfig extends Config(
 // DOC include end: InitZeroRocketConfig
 
 class LoopbackNICRocketConfig extends Config(
-  new chipyard.iobinders.WithLoopbackNIC ++                        // drive NIC IOs with loopback
+  new chipyard.harness.WithLoopbackNIC ++                        // drive NIC IOs with loopback
   new icenet.WithIceNIC ++                                         // add an IceNIC
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
@@ -170,8 +169,6 @@ class LargeNVDLARocketConfig extends Config(
   new chipyard.config.AbstractConfig)
 
 class MMIORocketConfig extends Config(
-  new chipyard.iobinders.WithTieOffL2FBusAXI ++ // Tie-off the incoming MMIO port
-  new chipyard.iobinders.WithSimAXIMMIO ++ // Attach a simulated memory to the outwards MMIO port
   new freechips.rocketchip.subsystem.WithDefaultMMIOPort ++ // add default external master port
   new freechips.rocketchip.subsystem.WithDefaultSlavePort ++ // add default external slave port
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
diff --git a/generators/chipyard/src/main/scala/config/TracegenConfigs.scala b/generators/chipyard/src/main/scala/config/TracegenConfigs.scala
index 47d567fb..28bc5dcd 100644
--- a/generators/chipyard/src/main/scala/config/TracegenConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/TracegenConfigs.scala
@@ -4,8 +4,10 @@ import freechips.rocketchip.config.{Config}
 import freechips.rocketchip.rocket.{DCacheParams}
 
 class TraceGenConfig extends Config(
-  new chipyard.iobinders.WithBlackBoxSimMem ++
-  new chipyard.iobinders.WithTraceGenSuccessBinder ++
+  new chipyard.harness.WithBlackBoxSimMem ++
+  new chipyard.harness.WithTraceGenSuccess ++
+  new chipyard.iobinders.WithAXI4MemPunchthrough ++
+  new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
   new chipyard.config.WithTracegenSystem ++
   new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 0, nSets = 16, nWays = 2) }) ++
@@ -13,8 +15,10 @@ class TraceGenConfig extends Config(
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
 
 class NonBlockingTraceGenConfig extends Config(
-  new chipyard.iobinders.WithBlackBoxSimMem ++
-  new chipyard.iobinders.WithTraceGenSuccessBinder ++
+  new chipyard.harness.WithBlackBoxSimMem ++
+  new chipyard.harness.WithTraceGenSuccess ++
+  new chipyard.iobinders.WithAXI4MemPunchthrough ++
+  new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
   new chipyard.config.WithTracegenSystem ++
   new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 2, nSets = 16, nWays = 2) }) ++
@@ -22,8 +26,10 @@ class NonBlockingTraceGenConfig extends Config(
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
 
 class BoomTraceGenConfig extends Config(
-  new chipyard.iobinders.WithBlackBoxSimMem ++
-  new chipyard.iobinders.WithTraceGenSuccessBinder ++
+  new chipyard.harness.WithBlackBoxSimMem ++
+  new chipyard.harness.WithTraceGenSuccess ++
+  new chipyard.iobinders.WithAXI4MemPunchthrough ++
+  new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
   new chipyard.config.WithTracegenSystem ++
   new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithBoomTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 8, nSets = 16, nWays = 2) }) ++
@@ -32,8 +38,10 @@ class BoomTraceGenConfig extends Config(
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
 
 class NonBlockingTraceGenL2Config extends Config(
-  new chipyard.iobinders.WithBlackBoxSimMem ++
-  new chipyard.iobinders.WithTraceGenSuccessBinder ++
+  new chipyard.harness.WithBlackBoxSimMem ++
+  new chipyard.harness.WithTraceGenSuccess ++
+  new chipyard.iobinders.WithAXI4MemPunchthrough ++
+  new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
   new chipyard.config.WithTracegenSystem ++
   new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithL2TraceGen()(List.fill(2)(DCacheParams(nMSHRs = 2, nSets = 16, nWays = 4))) ++
@@ -42,8 +50,10 @@ class NonBlockingTraceGenL2Config extends Config(
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
 
 class NonBlockingTraceGenL2RingConfig extends Config(
-  new chipyard.iobinders.WithBlackBoxSimMem ++
-  new chipyard.iobinders.WithTraceGenSuccessBinder ++
+  new chipyard.harness.WithBlackBoxSimMem ++
+  new chipyard.harness.WithTraceGenSuccess ++
+  new chipyard.iobinders.WithAXI4MemPunchthrough ++
+  new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
   new chipyard.config.WithTracegenSystem ++
   new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithL2TraceGen()(List.fill(2)(DCacheParams(nMSHRs = 2, nSets = 16, nWays = 4))) ++
diff --git a/generators/chipyard/src/main/scala/config/TutorialConfigs.scala b/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
index d501b6c0..e7adbdc1 100644
--- a/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
@@ -12,27 +12,16 @@ import freechips.rocketchip.config.{Config}
 
 // This file was originally developed for the cancelled ASPLOS-2020
 // Chipyard tutorial. While the configs here work, the corresponding
-// slideware has not yet been created
+// slideware has not yet been created.
 
 // NOTE: Configs should be read bottom-up, since they are applied bottom-up
 
+// NOTE: The TutorialConfigs build off of the AbstractConfig defined in AbstractConfig.scala
+//       Users should try to understand the functionality of the AbstractConfig before proceeding
+//       with the TutorialConfigs below
+
 // Tutorial Phase 1: Configure the cores, caches
 class TutorialStarterConfig extends Config(
-  // IOBinders specify how to connect to IOs in our TestHarness
-  // These config fragments do not affect
-  new chipyard.iobinders.WithUARTAdapter ++       // Connect a SimUART adapter to display UART on stdout
-  new chipyard.iobinders.WithBlackBoxSimMem ++    // Connect simulated external memory
-  new chipyard.iobinders.WithTieOffInterrupts ++  // Do not simulate external interrupts
-  new chipyard.iobinders.WithSimDebug ++          // Connect SimJTAG (or SimDTM) widgets to debug ios
-  new chipyard.iobinders.WithSimSerial ++         // Connect external SimSerial widget to drive TSI
-
-  // Config fragments below this line affect hardware generation
-  // of the Top
-  new testchipip.WithTSI ++                  // Add a TSI (Test Serial Interface)  widget to bring-up the core
-  new chipyard.config.WithBootROM ++         // Use the Chipyard BootROM
-  new chipyard.config.WithUART ++            // Add a UART
-  new chipyard.config.WithNoSubsystemDrivenClocks ++ // Don't drive the subsystem clocks from within the subsystem
-
   // CUSTOMIZE THE CORE
   // Uncomment out one (or multiple) of the lines below, and choose
   // how many cores you want.
@@ -43,36 +32,11 @@ class TutorialStarterConfig extends Config(
   // Uncomment this line, and specify a size if you want to have a L2
   // new freechips.rocketchip.subsystem.WithInclusiveCache(nBanks=1, nWays=4, capacityKB=128) ++
 
-  // Set the debug module to expose an external JTAG port
-  new freechips.rocketchip.subsystem.WithJtagDTM ++
-
-  // For simpler designs, we want to minimize IOs on
-  // our Top. These config fragments remove unnecessary
-  // ports
-  new freechips.rocketchip.subsystem.WithNoMMIOPort ++
-  new freechips.rocketchip.subsystem.WithNoSlavePort ++
-  new freechips.rocketchip.subsystem.WithNExtTopInterrupts(0) ++
-
-  // Use the standard hierarchical bus topology including mbus+l2
-  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
-
-  // BaseConfig configures "bare" rocketchip system
-  new freechips.rocketchip.system.BaseConfig
+  new chipyard.config.AbstractConfig
 )
 
-
 // Tutorial Phase 2: Integrate a TileLink or AXI4 MMIO device
 class TutorialMMIOConfig extends Config(
-  new chipyard.iobinders.WithUARTAdapter ++
-  new chipyard.iobinders.WithBlackBoxSimMem ++
-  new chipyard.iobinders.WithTieOffInterrupts ++
-  new chipyard.iobinders.WithSimDebug ++
-  new chipyard.iobinders.WithSimSerial ++
-
-  new testchipip.WithTSI ++
-  new chipyard.config.WithBootROM ++
-  new chipyard.config.WithUART ++
-  new chipyard.config.WithNoSubsystemDrivenClocks ++
 
   // Attach either a TileLink or AXI4 version of GCD
   // Uncomment one of the below lines
@@ -81,66 +45,26 @@ class TutorialMMIOConfig extends Config(
 
   // For this demonstration we assume the base system is a single-core Rocket, for fast elaboration
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
-  new freechips.rocketchip.subsystem.WithInclusiveCache ++
-  new freechips.rocketchip.subsystem.WithJtagDTM ++
-  new freechips.rocketchip.subsystem.WithNoMMIOPort ++
-  new freechips.rocketchip.subsystem.WithNoSlavePort ++
-  new freechips.rocketchip.subsystem.WithNExtTopInterrupts(0) ++
-  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
-  new freechips.rocketchip.system.BaseConfig
+  new chipyard.config.AbstractConfig
 )
 
-// Tutorial Phase 3: Integrate a SHA3 RoCC accelerator
+// // Tutorial Phase 3: Integrate a SHA3 RoCC accelerator
 class TutorialSha3Config extends Config(
-  new chipyard.iobinders.WithUARTAdapter ++
-  new chipyard.iobinders.WithBlackBoxSimMem ++
-  new chipyard.iobinders.WithTieOffInterrupts ++
-  new chipyard.iobinders.WithSimDebug ++
-  new chipyard.iobinders.WithSimSerial ++
-
-  new testchipip.WithTSI ++
-  new chipyard.config.WithBootROM ++
-  new chipyard.config.WithUART ++
-  new chipyard.config.WithNoSubsystemDrivenClocks ++
-
   // Uncomment this line once you added SHA3 to the build.sbt, and cloned the SHA3 repo
   // new sha3.WithSha3Accel ++
 
   // For this demonstration we assume the base system is a single-core Rocket, for fast elaboration
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
-  new freechips.rocketchip.subsystem.WithJtagDTM ++
-  new freechips.rocketchip.subsystem.WithInclusiveCache ++
-  new freechips.rocketchip.subsystem.WithNoMMIOPort ++
-  new freechips.rocketchip.subsystem.WithNoSlavePort ++
-  new freechips.rocketchip.subsystem.WithNExtTopInterrupts(0) ++
-  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
-  new freechips.rocketchip.system.BaseConfig
+  new chipyard.config.AbstractConfig
 )
 
 // Tutorial Phase 4: Integrate a Black-box verilog version of the SHA3 RoCC accelerator
 class TutorialSha3BlackBoxConfig extends Config(
-  new chipyard.iobinders.WithUARTAdapter ++
-  new chipyard.iobinders.WithBlackBoxSimMem ++
-  new chipyard.iobinders.WithTieOffInterrupts ++
-  new chipyard.iobinders.WithSimDebug ++
-  new chipyard.iobinders.WithSimSerial ++
-
-  new testchipip.WithTSI ++
-  new chipyard.config.WithBootROM ++
-  new chipyard.config.WithUART ++
-  new chipyard.config.WithNoSubsystemDrivenClocks ++
-
   // Uncomment these lines once SHA3 is integrated
   // new sha3.WithSha3BlackBox ++ // Specify we want the Black-box verilog version of Sha3 Ctrl
   // new sha3.WithSha3Accel ++
 
   // For this demonstration we assume the base system is a single-core Rocket, for fast elaboration
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
-  new freechips.rocketchip.subsystem.WithJtagDTM ++
-  new freechips.rocketchip.subsystem.WithInclusiveCache ++
-  new freechips.rocketchip.subsystem.WithNoMMIOPort ++
-  new freechips.rocketchip.subsystem.WithNoSlavePort ++
-  new freechips.rocketchip.subsystem.WithNExtTopInterrupts(0) ++
-  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
-  new freechips.rocketchip.system.BaseConfig
+  new chipyard.config.AbstractConfig
 )
diff --git a/generators/firechip/src/main/scala/BridgeBinders.scala b/generators/firechip/src/main/scala/BridgeBinders.scala
index b59d477d..0eb61a05 100644
--- a/generators/firechip/src/main/scala/BridgeBinders.scala
+++ b/generators/firechip/src/main/scala/BridgeBinders.scala
@@ -8,13 +8,13 @@ import chisel3.experimental.annotate
 import freechips.rocketchip.config.{Field, Config, Parameters}
 import freechips.rocketchip.diplomacy.{LazyModule}
 import freechips.rocketchip.devices.debug.{Debug, HasPeripheryDebugModuleImp}
+import freechips.rocketchip.amba.axi4.{AXI4Bundle}
 import freechips.rocketchip.subsystem.{CanHaveMasterAXI4MemPort, HasExtInterruptsModuleImp, BaseSubsystem, HasTilesModuleImp}
 import freechips.rocketchip.tile.{RocketTile}
-import sifive.blocks.devices.uart.HasPeripheryUARTModuleImp
-import sifive.blocks.devices.gpio.{HasPeripheryGPIOModuleImp}
+import sifive.blocks.devices.uart._
 
-import testchipip.{CanHavePeripherySerialModuleImp, CanHavePeripheryBlockDeviceModuleImp}
-import icenet.CanHavePeripheryIceNICModuleImp
+import testchipip._
+import icenet.{CanHavePeripheryIceNICModuleImp, SimNetwork, NicLoopback, NICKey, NICIOvonly}
 
 import junctions.{NastiKey, NastiParameters}
 import midas.models.{FASEDBridge, AXI4EdgeSummary, CompleteConfig}
@@ -25,73 +25,116 @@ import tracegen.{TraceGenSystemModuleImp}
 import ariane.ArianeTile
 
 import boom.common.{BoomTile}
-
-import chipyard.iobinders.{IOBinders, OverrideIOBinder, ComposeIOBinder, GetSystemParameters}
-import testchipip.{CanHaveTraceIOModuleImp}
+import barstools.iocell.chisel._
+import chipyard.iobinders.{IOBinders, OverrideIOBinder, ComposeIOBinder, GetSystemParameters, IOCellKey}
+import chipyard.{HasHarnessSignalReferences}
+import chipyard.harness._
 
 object MainMemoryConsts {
   val regionNamePrefix = "MainMemory"
   def globalName = s"${regionNamePrefix}_${NodeIdx()}"
 }
 
-class WithSerialBridge extends OverrideIOBinder({
-  (system: CanHavePeripherySerialModuleImp) =>
-    system.serial.foreach(s => SerialBridge(system.clock, s, MainMemoryConsts.globalName)(system.p)); Nil
+trait Unsupported {
+  require(false, "We do not support this IOCell type")
+}
+
+class FireSimAnalogIOCell extends RawModule with AnalogIOCell with Unsupported
+class FireSimDigitalGPIOCell extends RawModule with DigitalGPIOCell with Unsupported
+class FireSimDigitalInIOCell extends RawModule with DigitalInIOCell { io.i := io.pad }
+class FireSimDigitalOutIOCell extends RawModule with DigitalOutIOCell { io.pad := io.o }
+
+case class FireSimIOCellParams() extends IOCellTypeParams {
+  def analog() = Module(new FireSimAnalogIOCell)
+  def gpio()   = Module(new FireSimDigitalGPIOCell)
+  def input()  = Module(new FireSimDigitalInIOCell)
+  def output() = Module(new FireSimDigitalOutIOCell)
+}
+
+class WithFireSimIOCellModels extends Config((site, here, up) => {
+  case IOCellKey => FireSimIOCellParams()
 })
 
-class WithNICBridge extends OverrideIOBinder({
-  (system: CanHavePeripheryIceNICModuleImp) =>
-    system.net.foreach(n => NICBridge(system.clock, n)(system.p)); Nil
+class WithSerialBridge extends OverrideHarnessBinder({
+  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val clock = ports.collectFirst({case c: Clock => c})
+    val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
+    ports.filter(_.isInstanceOf[SerialIO]).map {
+      case s: SerialIO => withClockAndReset(clock.get, th.harnessReset) {
+        SerialBridge(clock.get, s, MainMemoryConsts.globalName)(p)
+      }
+      case _ =>
+    }
+    Nil
+  }
 })
 
-class WithUARTBridge extends OverrideIOBinder({
-  (system: HasPeripheryUARTModuleImp) =>
-    system.uart.foreach(u => UARTBridge(system.clock, u)(system.p)); Nil
+class WithNICBridge extends OverrideHarnessBinder({
+  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val clock = ports.collectFirst({case c: Clock => c})
+    ports.map {
+      case p: NICIOvonly => withClockAndReset(clock.get, th.harnessReset) { NICBridge(clock.get, p)(system.p) }
+      case _ =>
+    }
+    Nil
+  }
 })
 
-class WithBlockDeviceBridge extends OverrideIOBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp) =>
-    system.bdev.foreach(b => BlockDevBridge(system.clock, b, system.reset.toBool)(system.p)); Nil
+class WithUARTBridge extends OverrideHarnessBinder({
+  (system: HasPeripheryUARTModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) =>
+    ports.map { case p: UARTPortIO => UARTBridge(th.harnessClock, p)(system.p) }; Nil
 })
 
+class WithBlockDeviceBridge extends OverrideHarnessBinder({
+  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    val clock = ports.collectFirst({case c: Clock => c})
+    ports.map {
+      case p: BlockDeviceIO => BlockDevBridge(clock.get, p, th.harnessReset.toBool)(system.p)
+      case _ =>
+    }
+    Nil
+  }
+})
 
-class WithFASEDBridge extends OverrideIOBinder({
-  (system: CanHaveMasterAXI4MemPort) => {
+class WithFASEDBridge extends OverrideHarnessBinder({
+  (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    (system.mem_axi4 zip system.memAXI4Node.edges.in).foreach({ case (axi4, edge) =>
+    val clock = ports.collectFirst({case c: Clock => c})
+    val axi4_ports = ports.collect { case p: AXI4Bundle => p }
+    (axi4_ports zip system.memAXI4Node.edges.in).map { case (axi4: AXI4Bundle, edge) =>
       val nastiKey = NastiParameters(axi4.r.bits.data.getWidth,
                                      axi4.ar.bits.addr.getWidth,
                                      axi4.ar.bits.id.getWidth)
       system match {
-        case s: BaseSubsystem => FASEDBridge(s.module.clock, axi4, s.module.reset.toBool,
+        case s: BaseSubsystem => FASEDBridge(clock.get, axi4, th.harnessReset.asBool,
           CompleteConfig(p(firesim.configs.MemModelKey),
                          nastiKey,
                          Some(AXI4EdgeSummary(edge)),
                          Some(MainMemoryConsts.globalName)))
         case _ => throw new Exception("Attempting to attach FASED Bridge to misconfigured design")
       }
-    })
+    }
     Nil
   }
 })
 
-class WithTracerVBridge extends ComposeIOBinder({
-  (system: CanHaveTraceIOModuleImp) =>
-    system.traceIO.foreach(_.traces.map(tileTrace => TracerVBridge(tileTrace)(system.p))); Nil
-})
-
-
-
-class WithDromajoBridge extends ComposeIOBinder({
-  (system: CanHaveTraceIOModuleImp) => {
-    system.traceIO.foreach(_.traces.map(tileTrace => DromajoBridge(tileTrace)(system.p))); Nil
+class WithTracerVBridge extends ComposeHarnessBinder({
+  (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    ports.map { case p: TraceOutputTop => p.traces.map(tileTrace =>
+      withClockAndReset(tileTrace.clock, tileTrace.reset) { TracerVBridge(tileTrace)(system.p) }
+    )}
   }
 })
 
+class WithDromajoBridge extends ComposeHarnessBinder({
+  (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) =>
+    ports.map { case p: TraceOutputTop => p.traces.map(tileTrace => DromajoBridge(tileTrace)(system.p)) }; Nil
+})
 
-class WithTraceGenBridge extends OverrideIOBinder({
-  (system: TraceGenSystemModuleImp) =>
-    GroundTestBridge(system.clock, system.success)(system.p); Nil
+
+class WithTraceGenBridge extends OverrideHarnessBinder({
+  (system: TraceGenSystemModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) =>
+    ports.map { case p: Bool => GroundTestBridge(th.harnessClock, p)(system.p) }; Nil
 })
 
 class WithFireSimMultiCycleRegfile extends ComposeIOBinder({
@@ -105,52 +148,25 @@ class WithFireSimMultiCycleRegfile extends ComposeIOBinder({
         val core = b.module.core
         core.iregfile match {
           case irf: boom.exu.RegisterFileSynthesizable => annotate(MemModelAnnotation(irf.regfile))
-          case _ => Nil
         }
         if (core.fp_pipeline != null) core.fp_pipeline.fregfile match {
           case frf: boom.exu.RegisterFileSynthesizable => annotate(MemModelAnnotation(frf.regfile))
-          case _ => Nil
         }
       }
       case _ =>
     }
-    Nil
+    (Nil, Nil)
   }
 })
 
-class WithTiedOffSystemGPIO extends OverrideIOBinder({
-  (system: HasPeripheryGPIOModuleImp) =>
-    system.gpio.foreach(_.pins.foreach(_.i.ival := false.B)); Nil
-})
-
-class WithTiedOffSystemDebug extends OverrideIOBinder({
-  (system: HasPeripheryDebugModuleImp) => {
-    Debug.tieoffDebug(system.debug, system.resetctrl, Some(system.psd))(system.p)
-    // tieoffDebug doesn't actually tie everything off :/
-    system.debug.foreach { d =>
-      d.clockeddmi.foreach({ cdmi => cdmi.dmi.req.bits := DontCare })
-      d.dmactiveAck := DontCare
-    }
-    Nil
-  }
-})
-
-class WithTiedOffSystemInterrupts extends OverrideIOBinder({
-  (system: HasExtInterruptsModuleImp) =>
-    system.interrupts := 0.U; Nil
-})
-
-
 // Shorthand to register all of the provided bridges above
 class WithDefaultFireSimBridges extends Config(
-  new WithTiedOffSystemGPIO ++
-  new WithTiedOffSystemDebug ++
-  new WithTiedOffSystemInterrupts ++
   new WithSerialBridge ++
   new WithNICBridge ++
   new WithUARTBridge ++
   new WithBlockDeviceBridge ++
   new WithFASEDBridge ++
   new WithFireSimMultiCycleRegfile ++
-  new WithTracerVBridge
+  new WithTracerVBridge ++
+  new WithFireSimIOCellModels
 )
diff --git a/generators/firechip/src/main/scala/FireSim.scala b/generators/firechip/src/main/scala/FireSim.scala
index 158674a0..d2ef4e60 100644
--- a/generators/firechip/src/main/scala/FireSim.scala
+++ b/generators/firechip/src/main/scala/FireSim.scala
@@ -13,8 +13,9 @@ import freechips.rocketchip.util.{ResetCatchAndSync}
 
 import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge, RationalClock}
 
-import chipyard.{BuildSystem, BuildTop, HasHarnessSignalReferences, ChipyardSubsystem, ClockingSchemeKey, ChipTop}
-import chipyard.iobinders.{IOBinders}
+import chipyard._
+import chipyard.harness._
+import chipyard.iobinders._
 
 // Determines the number of times to instantiate the DUT in the harness.
 // Subsumes legacy supernode support
@@ -50,7 +51,7 @@ class WithFireSimSimpleClocks extends Config((site, here, up) => {
     chiptop.implicitClockSinkNode := implicitClockSourceNode
 
     // Drive the diplomaticclock graph of the DigitalTop (if present)
-    val simpleClockGroupSourceNode = chiptop.lSystem match {
+    val simpleClockGroupSourceNode = chiptop.lazySystem match {
       case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
         val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
         l.asyncClockGroupsNode := n
@@ -93,7 +94,7 @@ class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Confi
     chiptop.implicitClockSinkNode := implicitClockSourceNode
 
     // Drive the diplomaticclock graph of the DigitalTop (if present)
-    val simpleClockGroupSourceNode = chiptop.lSystem match {
+    val simpleClockGroupSourceNode = chiptop.lazySystem match {
       case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
         val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
         l.asyncClockGroupsNode := n
@@ -159,8 +160,13 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna
       case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy
     })))
     val module = Module(lazyModule.module)
-    require(lazyModule.harnessFunctions.size == 1, "There should only be 1 harness function to connect clock+reset")
-    lazyModule.harnessFunctions.foreach(_(this))
+    lazyModule match { case d: HasTestHarnessFunctions =>
+      require(d.harnessFunctions.size == 1, "There should only be 1 harness function to connect clock+reset")
+      d.harnessFunctions.foreach(_(this))
+      ApplyHarnessBinders(this, d.lazySystem, p(HarnessBinders), d.portMap.toMap)
+    }
+    
+
     NodeIdx.increment()
   }
 }
diff --git a/generators/icenet b/generators/icenet
index 705ca506..bb23c81f 160000
--- a/generators/icenet
+++ b/generators/icenet
@@ -1 +1 @@
-Subproject commit 705ca50690383aa589dc560a5e7c152af04c46ad
+Subproject commit bb23c81fcfbdfde6767c5b7daa95f8f9436fb7db
diff --git a/generators/testchipip b/generators/testchipip
index 1e7373f6..6f4e7ae2 160000
--- a/generators/testchipip
+++ b/generators/testchipip
@@ -1 +1 @@
-Subproject commit 1e7373f6398c198e2dee2bcf692917ec2ac21b53
+Subproject commit 6f4e7ae2c94e867f08cd0d408a106fba2f389de2
diff --git a/sims/firesim b/sims/firesim
index 05edd6be..069c3c11 160000
--- a/sims/firesim
+++ b/sims/firesim
@@ -1 +1 @@
-Subproject commit 05edd6be8c0464ea53a664a2164d3eba6a7f62aa
+Subproject commit 069c3c111d46a3f3d20b4bd6f17aaa9b23487cc7
diff --git a/tools/barstools b/tools/barstools
index aa1c90c4..ba681676 160000
--- a/tools/barstools
+++ b/tools/barstools
@@ -1 +1 @@
-Subproject commit aa1c90c4ccb73c2c379550f3296892cc81e8a195
+Subproject commit ba681676f338af158023c99b4c802009aa0b601b

From b613c14f1c4cee96ee5abf181ee19df54494fd29 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 4 Sep 2020 20:03:12 -0700
Subject: [PATCH 52/74] Fix remaining HarnessBinders bugs

---
 .circleci/defaults.sh                            |  4 ++--
 .../chipyard/src/main/scala/HarnessBinders.scala |  5 ++++-
 .../chipyard/src/main/scala/IOBinders.scala      | 16 ++++++++--------
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/.circleci/defaults.sh b/.circleci/defaults.sh
index 7ffb1d3c..703737cd 100755
--- a/.circleci/defaults.sh
+++ b/.circleci/defaults.sh
@@ -61,8 +61,8 @@ mapping["chipyard-ariane"]=" CONFIG=ArianeConfig"
 mapping["chipyard-spiflashread"]=" CONFIG=LargeSPIFlashROMRocketConfig"
 mapping["chipyard-spiflashwrite"]=" CONFIG=SmallSPIFlashRocketConfig"
 mapping["chipyard-mmios"]=" CONFIG=MMIORocketConfig verilog"
-mapping["tracegen"]=" CONFIG=NonBlockingTraceGenL2Config TOP=TraceGenSystem"
-mapping["tracegen-boom"]=" CONFIG=BoomTraceGenConfig TOP=TraceGenSystem"
+mapping["tracegen"]=" CONFIG=NonBlockingTraceGenL2Config"
+mapping["tracegen-boom"]=" CONFIG=BoomTraceGenConfig"
 mapping["chipyard-nvdla"]=" CONFIG=SmallNVDLARocketConfig"
 mapping["firesim"]="SCALA_TEST=firesim.firesim.RocketNICF1Tests"
 mapping["firesim-multiclock"]="SCALA_TEST=firesim.firesim.RocketMulticlockF1Tests"
diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala
index 54c99042..89017172 100644
--- a/generators/chipyard/src/main/scala/HarnessBinders.scala
+++ b/generators/chipyard/src/main/scala/HarnessBinders.scala
@@ -240,7 +240,10 @@ class WithTiedOffDebug extends OverrideHarnessBinder({
 
 class WithTiedOffSerial extends OverrideHarnessBinder({
   (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    ports.map { case p: SerialIO => SerialAdapter.tieoff(Some(p)) }
+    ports.map {
+      case p: SerialIO => SerialAdapter.tieoff(Some(p))
+      case _ =>
+    }
     Nil
   }
 })
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 0dad7e21..acd57207 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -270,8 +270,8 @@ class WithAXI4MemPunchthrough extends OverrideIOBinder({
     } else {
       None
     }
-    val ports = system.mem_axi4.map({ m =>
-      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName("axi4_mem")
+    val ports = system.mem_axi4.zipWithIndex.map({ case (m, i) =>
+      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName(s"axi4_mem_${i}")
       p <> m
       p
     })
@@ -286,8 +286,8 @@ class WithAXI4MMIOPunchthrough extends OverrideIOBinder({
     } else {
       None
     }
-    val ports = system.mmio_axi4.map({ m =>
-      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName("axi4_mmio")
+    val ports = system.mmio_axi4.zipWithIndex.map({ case (m, i) =>
+      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName(s"axi4_mmio_${i}")
       p <> m
       p
     })
@@ -297,11 +297,11 @@ class WithAXI4MMIOPunchthrough extends OverrideIOBinder({
 
 class WithL2FBusAXI4Punchthrough extends OverrideIOBinder({
   (system: CanHaveSlaveAXI4Port) => {
-    val port = system.l2_frontend_bus_axi4.map { m =>
-      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName("axi4_fbus")
-      p <> m
+    val port = system.l2_frontend_bus_axi4.zipWithIndex.map({ case (m, i) =>
+      val p = IO(Flipped(DataMirror.internal.chiselTypeClone[AXI4Bundle](m))).suggestName(s"axi4_fbus_${i}")
+      m <> p
       p
-    }
+    })
     (port, Nil)
   }
 })

From 9eb88c55fca24af5bcc8e1c63647e545fbde006b Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 4 Sep 2020 23:07:23 -0700
Subject: [PATCH 53/74] Fix FireSim submodule

---
 sims/firesim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/firesim b/sims/firesim
index 069c3c11..7cb10dcb 160000
--- a/sims/firesim
+++ b/sims/firesim
@@ -1 +1 @@
-Subproject commit 069c3c111d46a3f3d20b4bd6f17aaa9b23487cc7
+Subproject commit 7cb10dcb5eac4f2dc35ca8ee2bb0e1629fedd19b

From ab21c53a42a4bd313c86a84b9c67abe7c77ee69d Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 4 Sep 2020 23:49:40 -0700
Subject: [PATCH 54/74] Add documentation on HarnessBinders

---
 docs/Advanced-Concepts/Debugging-BOOM.rst     | 13 +++--
 docs/Advanced-Concepts/Top-Testharness.rst    |  6 +-
 docs/Customization/IOBinders.rst              | 58 ++++++++++---------
 .../src/main/scala/HarnessBinders.scala       |  2 +
 .../chipyard/src/main/scala/IOBinders.scala   | 13 +----
 5 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/docs/Advanced-Concepts/Debugging-BOOM.rst b/docs/Advanced-Concepts/Debugging-BOOM.rst
index 1e61804c..fd41a9d7 100644
--- a/docs/Advanced-Concepts/Debugging-BOOM.rst
+++ b/docs/Advanced-Concepts/Debugging-BOOM.rst
@@ -12,11 +12,14 @@ to verify functionality.
 Setting up Dromajo Co-simulation
 --------------------------------------
 
-Dromajo co-simulation is setup to work when two config fragments are added to a BOOM config.
-First, a ``chipyard.config.WithTraceIO`` config fragment must be added so that BOOM's traceport is enabled.
-Second, a ``chipyard.iobinders.WithSimDromajoBridge`` config fragment must be added to
-connect the Dromajo co-simulator to the traceport.
-Once both config fragments are added Dromajo should be enabled.
+Dromajo co-simulation is setup to work when three config fragments are added to a BOOM config.
+
+ * A ``chipyard.config.WithTraceIO`` config fragment must be added so that BOOM's traceport is enabled.
+ * A ``chipyard.iobinders.WithTraceIOPunchthrough`` config fragment must be added to add the ``TraceIO`` to the ``ChipTop``
+ * A ``chipyard.harness.WithSimDromajoBridge`` config fragment must be added to instantiate a Dromajo cosimulator in the ``TestHarness`` and connect it to the ``ChipTop``'s ``TraceIO``
+
+
+Once all config fragments are added Dromajo should be enabled.
 
 To build/run Dromajo with a BOOM design, run your configuration the following make commands:
 
diff --git a/docs/Advanced-Concepts/Top-Testharness.rst b/docs/Advanced-Concepts/Top-Testharness.rst
index ebd5b370..43a8a338 100644
--- a/docs/Advanced-Concepts/Top-Testharness.rst
+++ b/docs/Advanced-Concepts/Top-Testharness.rst
@@ -14,9 +14,9 @@ ChipTop/DUT
 ``ChipTop`` is the top-level module that instantiates the ``System`` submodule, usually an instance of the concrete class ``DigitalTop``.
 The vast majority of the design resides in the ``System``.
 Other components that exist inside the ``ChipTop`` layer are generally IO cells, clock receivers and multiplexers, reset synchronizers, and other analog IP that needs to exist outside of the ``System``.
-The ``IOBinders`` are responsible for instantiating the IO cells and defining the test harness collateral that connects to the top-level ports.
-Most of these types of devices can be instantiated using custom ``IOBinders``, so the provided ``ChipTop`` and ``ChipTopCaughtReset`` classes are sufficient.
-However, if needed, the ``BaseChipTop`` abstract class can be extended for building more custom ``ChipTop`` designs.
+The ``IOBinders`` are responsible for instantiating the IO cells for ``ChipTop`` IO that correspond to IO of the ``System``.
+The ``HarnessBinders`` are responsible for instantiating test harness collateral that connects to the ``ChipTop`` ports.
+Most types of devices and testing collateral can be instantiated using custom ``IOBinders`` and ``HarnessBinders``.
 
 
 System/DigitalTop
diff --git a/docs/Customization/IOBinders.rst b/docs/Customization/IOBinders.rst
index 6332d07a..55920cd3 100644
--- a/docs/Customization/IOBinders.rst
+++ b/docs/Customization/IOBinders.rst
@@ -1,41 +1,45 @@
+IOBinders and HarnessBinders
+============================
+
+In Chipyard we use special ``Parameters`` keys, ``IOBinders`` and ``HarnessBinders`` to bridge the gap between digital system IOs and TestHarness collateral.
+
 IOBinders
 =========
 
-In Chipyard we use a special ``Parameters`` key, ``IOBinders`` to instantiate IO cells in the ``ChipTop`` layer and determine what modules to bind to the IOs of a ``ChipTop`` in the ``TestHarness``.
+The ``IOBinder`` functions are responsible for instantiating IO cells and IOPorts in the ``ChipTop`` layer.
+
+``IOBinders`` are typically defined using the ``OverrideIOBinder`` or ``ComposeIOBinder`` macros. An ``IOBInder`` consists of a function matching ``Systems`` with a given trait that generates IO ports and IOCells, and returns a list of generated ports and cells.
+
+For example, the ``WithUARTIOCells`` IOBinder specifies will, for any ``System`` that might have UART ports (``HasPeripheryUARTModuleIMP``, generate ports within the ``ChipTop`` (``ports``) as well as IOCells with the appropriate type and direction (``cells2d``). This function returns a the list of generated ports, and the list of generate IOCells. The list of generated ports is passed to the ``HarnessBinders`` such that they can be connected to ``TestHarness`` devices.
+
 
 .. literalinclude:: ../../generators/chipyard/src/main/scala/IOBinders.scala
    :language: scala
-   :start-after: DOC include start: IOBinders
-   :end-before: DOC include end: IOBinders
+   :start-after: DOC include start: WithUARTIOCells
+   :end-before: DOC include end: WithUARTIOCells
 
+HarnessBinders
+==============
 
-This special key solves the problem of duplicating test-harnesses for each different ``System`` type.
-You could just as well create a custom harness module that attaches IOs explicitly.
-Instead, the ``IOBinders`` key provides a map from Scala traits to attachment behaviors.
-Each ``IOBinder`` returns a tuple of three values: the list of ``ChipTop`` ports created by the ``IOBinder``, the list of all IO cell modules instantiated by the ``IOBinder``, and an optional function to be called inside the test harness.
-This function is responsible for instantiating logic inside the ``TestHarness`` to appropriately drive the ``ChipTop`` IO ports created by the ``IOBinder``.
-Conveniently, because the ``IOBinder`` is generating the port, it may also use the port inside this function, which prevents the ``BaseChipTop`` code from ever needing to access the port ``val``, thus having the ``IOBinder`` house all port specific code.
-This scheme prevents the need to have two separate binder functions for each ``System`` trait.
-When creating custom ``IOBinders`` it is important to use ``suggestName`` to name ports; otherwise Chisel will raise an exception trying to name the IOs.
-The example ``IOBinders`` demonstrate this.
+The ``HarnessBinder`` functions determine what modules to bind to the IOs of a ``ChipTop`` in the ``TestHarness``. The ``HarnessBinder`` interface is designed to be reused across various simulation modes, enabling decoupling of the target design from simulation and testing concerns.
 
-As an example, the ``WithGPIOTiedOff`` IOBinder creates IO cells for the GPIO module(s) instantiated in the ``System``, then punches out new ``Analog`` ports for each one.
-The test harness simply ties these off, but additional logic could be inserted to perform some kind of test in the ``TestHarness``.
+ * For SW RTL simulations, the default set of ``HarnessBinders`` instantiate software-simulated models of various devices, for example external memory or UART, and connect those models to the IOs of the ``ChipTop``.
+ * For FireSim simulations, FireSim-specific ``HarnessBinders`` instantiate ``Bridges``, which faciliate cycle-accurate simulation across the simulated chip's IOs. See the FireSim documentation for more details.
+ * In the future, a Chipyard FPGA prototyping flow may use ``HarnessBinders`` to connect ``ChipTop`` IOs to other devices or IOs in the FPGA harness.
 
-.. literalinclude:: ../../generators/chipyard/src/main/scala/IOBinders.scala
+For FireSim simulations, the ``HarnessBinder`` attach ``Bridge`` modules (See the FireSim documentation for more details). 
+
+Like ``IOBinders``, ``HarnessBinders`` are defined using macros (``OverrideHarnessBinder, ComposeHarnessBinder``), and matches ``Systems`` with a given trait. However, ``HarnessBinders`` are also passed a reference to the ``TestHarness`` (``th: HasHarnessSignalReferences``) and the list of ports generated by the corresponding ``IOBinder`` (``ports: Seq[Data]``).
+
+For exmaple, the ``WithUARTAdapter`` will connect the UART SW display adapter to the ports generated by the ``WithUARTIOCells`` described earlier, if those ports are present.
+
+.. literalinclude:: ../../generators/chipyard/src/main/scala/HarnessBinders.scala
    :language: scala
-   :start-after: DOC include start: WithGPIOTiedOff
-   :end-before: DOC include end: WithGPIOTiedOff
+   :start-after: DOC include start: WithUARTAdapter
+   :end-before: DOC include end: WithUARTAdapter
 
+The ``IOBinder`` and ``HarnesBinder`` system is designed to enable decoupling of concerns between target design and simulation ssystem.
 
-``IOBinders`` also do not need to create ports. Some ``IOBinders`` can simply insert circuitry inside the ``ChipTop`` layer.
-For example, the ``WithSimAXIMemTiedOff`` IOBinder specifies that any ``System`` which matches ``CanHaveMasterAXI4MemPortModuleImp`` will have a ``SimAXIMem`` connected inside ``ChipTop``.
+For a given set of chip IOs, there may be not only multiple simulation platforms ("harnesses", so-to-speak), but also multiple simulation strategies. For example, the choice of whether to connect the backing AXI4 memory port to a accurate DRAM model (``SimDRAM``) or a simple simulated memory model (``SimAXIMem``) is isolated in ``HarnessBinders``, and does not affect target RTL generation.
 
-.. literalinclude:: ../../generators/chipyard/src/main/scala/IOBinders.scala
-   :language: scala
-   :start-after: DOC include start: WithSimAXIMem
-   :end-before: DOC include end: WithSimAXIMem
-
-These classes are all ``Config`` objects, which can be mixed into the configs to specify IO connection behaviors.
-
-There are two macros for generating these ``Config``s. ``OverrideIOBinder`` overrides any existing behaviors set for a particular IO in the ``Config`` object. This macro is frequently used because typically top-level IOs drive or are driven by only one source, so a composition of ``IOBinders`` does not make sense. The ``ComposeIOBinder`` macro provides the functionality of not overriding existing behaviors.
+Similarly, for a given simulation platform and strategy, there may be multiple strategies for generating the chip IOs. This target-design configuration is isolated in the ``IOBinders``.
diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala
index 89017172..a26b5104 100644
--- a/generators/chipyard/src/main/scala/HarnessBinders.scala
+++ b/generators/chipyard/src/main/scala/HarnessBinders.scala
@@ -66,12 +66,14 @@ class WithGPIOTiedOff extends OverrideHarnessBinder({
   }
 })
 
+// DOC include start: WithUARTAdapter
 class WithUARTAdapter extends OverrideHarnessBinder({
   (system: HasPeripheryUARTModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
     UARTAdapter.connect(ports.map(_.asInstanceOf[UARTPortIO]))(system.p)
     Nil
   }
 })
+// DOC include end: WithUARTAdapter
 
 class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideHarnessBinder({
   (system: HasPeripherySPIFlashModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index acd57207..db5c7008 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -40,13 +40,6 @@ import scala.reflect.{ClassTag}
 
 // You can add your own binder by adding a new (key, fn) pair, typically by using
 // the OverrideIOBinder or ComposeIOBinder macros
-
-
-// DOC include start: IOBinders
-// This type describes a function callable on the TestHarness instance. Its return type is unused.
-
-
-
 case object IOBinders extends Field[Map[String, (Any) => (Seq[Data], Seq[IOCell])]](
   Map[String, (Any) => (Seq[Data], Seq[IOCell])]().withDefaultValue((Any) => (Nil, Nil))
 )
@@ -59,7 +52,6 @@ object ApplyIOBinders {
   }
 }
 
-
 // Note: The parameters instance is accessible only through LazyModule
 // or LazyModuleImpLike. The self-type requirement in traits like
 // CanHaveMasterAXI4MemPort is insufficient to make it accessible to the IOBinder
@@ -116,8 +108,6 @@ object BoreHelper {
   }
 }
 
-// DOC include end: IOBinders
-
 
 case object IOCellKey extends Field[IOCellTypeParams](GenericIOCellParams())
 
@@ -140,7 +130,7 @@ class WithGPIOCells extends OverrideIOBinder({
   }
 })
 
-
+// DOC include start: WithUARTIOCells
 class WithUARTIOCells extends OverrideIOBinder({
   (system: HasPeripheryUARTModuleImp) => {
     val (ports, cells2d) = system.uart.zipWithIndex.map({ case (u, i) =>
@@ -151,6 +141,7 @@ class WithUARTIOCells extends OverrideIOBinder({
     (ports, cells2d.flatten)
   }
 })
+// DOC include end: WithUARTIOCells
 
 class WithSPIIOCells extends OverrideIOBinder({
   (system: HasPeripherySPIFlashModuleImp) => {

From 927244bf2e724c01b42a59dc297942ba12bdaa39 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Sat, 5 Sep 2020 11:16:55 -0700
Subject: [PATCH 55/74] DTM only supports HTIF in DMI mode

---
 generators/chipyard/src/main/scala/Subsystem.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generators/chipyard/src/main/scala/Subsystem.scala b/generators/chipyard/src/main/scala/Subsystem.scala
index 7f089ce1..92753db5 100644
--- a/generators/chipyard/src/main/scala/Subsystem.scala
+++ b/generators/chipyard/src/main/scala/Subsystem.scala
@@ -31,7 +31,7 @@ trait CanHaveHTIF { this: BaseSubsystem =>
   // Advertise HTIF if system can communicate with fesvr
   if (this match {
     case _: CanHavePeripherySerial if p(SerialKey) => true
-    case _: HasPeripheryDebug if p(ExportDebug).protocols.nonEmpty => true
+    case _: HasPeripheryDebug if p(ExportDebug).dmi => true
     case _ => false
   }) {
     ResourceBinding {

From 7ed02a7d385921a22e88fa1e77dc411d16a732c8 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 7 Sep 2020 11:36:37 -0700
Subject: [PATCH 56/74] Fix Typos

---
 docs/Customization/IOBinders.rst                     | 12 ++++++------
 generators/chipyard/src/main/scala/TestHarness.scala |  6 +++---
 .../src/main/scala/config/TutorialConfigs.scala      |  2 +-
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/docs/Customization/IOBinders.rst b/docs/Customization/IOBinders.rst
index 55920cd3..adfac22b 100644
--- a/docs/Customization/IOBinders.rst
+++ b/docs/Customization/IOBinders.rst
@@ -10,7 +10,7 @@ The ``IOBinder`` functions are responsible for instantiating IO cells and IOPort
 
 ``IOBinders`` are typically defined using the ``OverrideIOBinder`` or ``ComposeIOBinder`` macros. An ``IOBInder`` consists of a function matching ``Systems`` with a given trait that generates IO ports and IOCells, and returns a list of generated ports and cells.
 
-For example, the ``WithUARTIOCells`` IOBinder specifies will, for any ``System`` that might have UART ports (``HasPeripheryUARTModuleIMP``, generate ports within the ``ChipTop`` (``ports``) as well as IOCells with the appropriate type and direction (``cells2d``). This function returns a the list of generated ports, and the list of generate IOCells. The list of generated ports is passed to the ``HarnessBinders`` such that they can be connected to ``TestHarness`` devices.
+For example, the ``WithUARTIOCells`` IOBinder will, for any ``System`` that might have UART ports (``HasPeripheryUARTModuleImp``, generate ports within the ``ChipTop`` (``ports``) as well as IOCells with the appropriate type and direction (``cells2d``). This function returns a the list of generated ports, and the list of generated IOCells. The list of generated ports is passed to the ``HarnessBinders`` such that they can be connected to ``TestHarness`` devices.
 
 
 .. literalinclude:: ../../generators/chipyard/src/main/scala/IOBinders.scala
@@ -21,15 +21,15 @@ For example, the ``WithUARTIOCells`` IOBinder specifies will, for any ``System``
 HarnessBinders
 ==============
 
-The ``HarnessBinder`` functions determine what modules to bind to the IOs of a ``ChipTop`` in the ``TestHarness``. The ``HarnessBinder`` interface is designed to be reused across various simulation modes, enabling decoupling of the target design from simulation and testing concerns.
+The ``HarnessBinder`` functions determine what modules to bind to the IOs of a ``ChipTop`` in the ``TestHarness``. The ``HarnessBinder`` interface is designed to be reused across various simulation/implementation modes, enabling decoupling of the target design from simulation and testing concerns.
 
  * For SW RTL simulations, the default set of ``HarnessBinders`` instantiate software-simulated models of various devices, for example external memory or UART, and connect those models to the IOs of the ``ChipTop``.
  * For FireSim simulations, FireSim-specific ``HarnessBinders`` instantiate ``Bridges``, which faciliate cycle-accurate simulation across the simulated chip's IOs. See the FireSim documentation for more details.
  * In the future, a Chipyard FPGA prototyping flow may use ``HarnessBinders`` to connect ``ChipTop`` IOs to other devices or IOs in the FPGA harness.
 
-For FireSim simulations, the ``HarnessBinder`` attach ``Bridge`` modules (See the FireSim documentation for more details). 
+For FireSim simulations, the ``HarnessBinder`` attaches ``Bridge`` modules (See the FireSim documentation for more details). 
 
-Like ``IOBinders``, ``HarnessBinders`` are defined using macros (``OverrideHarnessBinder, ComposeHarnessBinder``), and matches ``Systems`` with a given trait. However, ``HarnessBinders`` are also passed a reference to the ``TestHarness`` (``th: HasHarnessSignalReferences``) and the list of ports generated by the corresponding ``IOBinder`` (``ports: Seq[Data]``).
+Like ``IOBinders``, ``HarnessBinders`` are defined using macros (``OverrideHarnessBinder, ComposeHarnessBinder``), and match ``Systems`` with a given trait. However, ``HarnessBinders`` are also passed a reference to the ``TestHarness`` (``th: HasHarnessSignalReferences``) and the list of ports generated by the corresponding ``IOBinder`` (``ports: Seq[Data]``).
 
 For exmaple, the ``WithUARTAdapter`` will connect the UART SW display adapter to the ports generated by the ``WithUARTIOCells`` described earlier, if those ports are present.
 
@@ -38,8 +38,8 @@ For exmaple, the ``WithUARTAdapter`` will connect the UART SW display adapter to
    :start-after: DOC include start: WithUARTAdapter
    :end-before: DOC include end: WithUARTAdapter
 
-The ``IOBinder`` and ``HarnesBinder`` system is designed to enable decoupling of concerns between target design and simulation ssystem.
+The ``IOBinder`` and ``HarnessBinder`` system is designed to enable decoupling of concerns between the target design and the simulation system.
 
-For a given set of chip IOs, there may be not only multiple simulation platforms ("harnesses", so-to-speak), but also multiple simulation strategies. For example, the choice of whether to connect the backing AXI4 memory port to a accurate DRAM model (``SimDRAM``) or a simple simulated memory model (``SimAXIMem``) is isolated in ``HarnessBinders``, and does not affect target RTL generation.
+For a given set of chip IOs, there may be not only multiple simulation platforms ("harnesses", so-to-speak), but also multiple simulation strategies. For example, the choice of whether to connect the backing AXI4 memory port to an accurate DRAM model (``SimDRAM``) or a simple simulated memory model (``SimAXIMem``) is isolated in ``HarnessBinders``, and does not affect target RTL generation.
 
 Similarly, for a given simulation platform and strategy, there may be multiple strategies for generating the chip IOs. This target-design configuration is isolated in the ``IOBinders``.
diff --git a/generators/chipyard/src/main/scala/TestHarness.scala b/generators/chipyard/src/main/scala/TestHarness.scala
index 67cf03bf..2faff565 100644
--- a/generators/chipyard/src/main/scala/TestHarness.scala
+++ b/generators/chipyard/src/main/scala/TestHarness.scala
@@ -31,8 +31,8 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSign
     val success = Output(Bool())
   })
 
-  val ldut = LazyModule(p(BuildTop)(p)).suggestName("chiptop")
-  val dut = Module(ldut.module)
+  val lazyDut = LazyModule(p(BuildTop)(p)).suggestName("chiptop")
+  val dut = Module(lazyDut.module)
   io.success := false.B
 
   val harnessClock = clock
@@ -42,7 +42,7 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSign
   // dutReset assignment can be overridden via a harnessFunction, but by default it is just reset
   val dutReset = WireDefault(if (p(GlobalResetSchemeKey).pinIsAsync) reset.asAsyncReset else reset)
 
-  ldut match { case d: HasTestHarnessFunctions =>
+  lazyDut match { case d: HasTestHarnessFunctions =>
     d.harnessFunctions.foreach(_(this))
     ApplyHarnessBinders(this, d.lazySystem, p(HarnessBinders), d.portMap.toMap)
   }
diff --git a/generators/chipyard/src/main/scala/config/TutorialConfigs.scala b/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
index e7adbdc1..3c64958f 100644
--- a/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/TutorialConfigs.scala
@@ -48,7 +48,7 @@ class TutorialMMIOConfig extends Config(
   new chipyard.config.AbstractConfig
 )
 
-// // Tutorial Phase 3: Integrate a SHA3 RoCC accelerator
+// Tutorial Phase 3: Integrate a SHA3 RoCC accelerator
 class TutorialSha3Config extends Config(
   // Uncomment this line once you added SHA3 to the build.sbt, and cloned the SHA3 repo
   // new sha3.WithSha3Accel ++

From b4e270219d06f5be568bbc62df362a9c29f98822 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 7 Sep 2020 14:02:31 -0700
Subject: [PATCH 57/74] Bump firesim

---
 sims/firesim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/firesim b/sims/firesim
index 7cb10dcb..522125da 160000
--- a/sims/firesim
+++ b/sims/firesim
@@ -1 +1 @@
-Subproject commit 7cb10dcb5eac4f2dc35ca8ee2bb0e1629fedd19b
+Subproject commit 522125da5d546544e3ce4492b6f105eeb93fcdf6

From 11a9ad24286e0588e7e7a613d81ac3d8552fbe14 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 8 Sep 2020 15:52:09 -0700
Subject: [PATCH 58/74] Address code review comments

---
 build.sbt                                     |  2 +-
 docs/Customization/IOBinders.rst              |  6 +--
 .../chipyard/src/main/scala/ChipTop.scala     |  3 +-
 .../chipyard/src/main/scala/Clocks.scala      |  1 -
 .../src/main/scala/HarnessBinders.scala       | 36 ++++++++-----
 .../chipyard/src/main/scala/IOBinders.scala   | 20 +++++--
 .../src/main/scala/config/ArianeConfigs.scala |  2 +-
 .../src/main/scala/config/RocketConfigs.scala | 54 +++++++++----------
 .../main/scala/config/TracegenConfigs.scala   | 44 ++++-----------
 9 files changed, 81 insertions(+), 87 deletions(-)

diff --git a/build.sbt b/build.sbt
index 3ca021a7..5d642c1d 100644
--- a/build.sbt
+++ b/build.sbt
@@ -211,7 +211,7 @@ lazy val midas      = ProjectRef(firesimDir, "midas")
 lazy val firesimLib = ProjectRef(firesimDir, "firesimLib")
 
 lazy val firechip = conditionalDependsOn(project in file("generators/firechip"))
-  .dependsOn(chipyard, midasTargetUtils, midas, iocell, firesimLib % "test->test;compile->compile")
+  .dependsOn(chipyard, midasTargetUtils, midas, firesimLib % "test->test;compile->compile")
   .settings(
     commonSettings,
     testGrouping in Test := isolateAllTests( (definedTests in Test).value ),
diff --git a/docs/Customization/IOBinders.rst b/docs/Customization/IOBinders.rst
index adfac22b..ff180bcd 100644
--- a/docs/Customization/IOBinders.rst
+++ b/docs/Customization/IOBinders.rst
@@ -8,7 +8,7 @@ IOBinders
 
 The ``IOBinder`` functions are responsible for instantiating IO cells and IOPorts in the ``ChipTop`` layer.
 
-``IOBinders`` are typically defined using the ``OverrideIOBinder`` or ``ComposeIOBinder`` macros. An ``IOBInder`` consists of a function matching ``Systems`` with a given trait that generates IO ports and IOCells, and returns a list of generated ports and cells.
+``IOBinders`` are typically defined using the ``OverrideIOBinder`` or ``ComposeIOBinder`` macros. An ``IOBinder`` consists of a function matching ``Systems`` with a given trait that generates IO ports and IOCells, and returns a list of generated ports and cells.
 
 For example, the ``WithUARTIOCells`` IOBinder will, for any ``System`` that might have UART ports (``HasPeripheryUARTModuleImp``, generate ports within the ``ChipTop`` (``ports``) as well as IOCells with the appropriate type and direction (``cells2d``). This function returns a the list of generated ports, and the list of generated IOCells. The list of generated ports is passed to the ``HarnessBinders`` such that they can be connected to ``TestHarness`` devices.
 
@@ -23,12 +23,10 @@ HarnessBinders
 
 The ``HarnessBinder`` functions determine what modules to bind to the IOs of a ``ChipTop`` in the ``TestHarness``. The ``HarnessBinder`` interface is designed to be reused across various simulation/implementation modes, enabling decoupling of the target design from simulation and testing concerns.
 
- * For SW RTL simulations, the default set of ``HarnessBinders`` instantiate software-simulated models of various devices, for example external memory or UART, and connect those models to the IOs of the ``ChipTop``.
+ * For SW RTL or GL simulations, the default set of ``HarnessBinders`` instantiate software-simulated models of various devices, for example external memory or UART, and connect those models to the IOs of the ``ChipTop``.
  * For FireSim simulations, FireSim-specific ``HarnessBinders`` instantiate ``Bridges``, which faciliate cycle-accurate simulation across the simulated chip's IOs. See the FireSim documentation for more details.
  * In the future, a Chipyard FPGA prototyping flow may use ``HarnessBinders`` to connect ``ChipTop`` IOs to other devices or IOs in the FPGA harness.
 
-For FireSim simulations, the ``HarnessBinder`` attaches ``Bridge`` modules (See the FireSim documentation for more details). 
-
 Like ``IOBinders``, ``HarnessBinders`` are defined using macros (``OverrideHarnessBinder, ComposeHarnessBinder``), and match ``Systems`` with a given trait. However, ``HarnessBinders`` are also passed a reference to the ``TestHarness`` (``th: HasHarnessSignalReferences``) and the list of ports generated by the corresponding ``IOBinder`` (``ports: Seq[Data]``).
 
 For exmaple, the ``WithUARTAdapter`` will connect the UART SW display adapter to the ports generated by the ``WithUARTIOCells`` described earlier, if those ports are present.
diff --git a/generators/chipyard/src/main/scala/ChipTop.scala b/generators/chipyard/src/main/scala/ChipTop.scala
index 6ae63d57..dfe08780 100644
--- a/generators/chipyard/src/main/scala/ChipTop.scala
+++ b/generators/chipyard/src/main/scala/ChipTop.scala
@@ -46,8 +46,7 @@ class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunc
     val implicit_reset = implicitClockSinkNode.in.head._1.reset
 
 
-    // The implicit clock and reset for the system is also, by convention, used for all the IOBinders
-    // TODO: This may not be the right thing to do in all cases
+    // Note: IOBinders cannot rely on the implicit clock/reset, as this is a LazyRawModuleImp
     val (_ports, _iocells, _portMap) = ApplyIOBinders(lazySystem, p(IOBinders))
     // We ignore _ports for now...
     iocells ++= _iocells
diff --git a/generators/chipyard/src/main/scala/Clocks.scala b/generators/chipyard/src/main/scala/Clocks.scala
index d6b19e8d..a1ff1b0f 100644
--- a/generators/chipyard/src/main/scala/Clocks.scala
+++ b/generators/chipyard/src/main/scala/Clocks.scala
@@ -120,7 +120,6 @@ object ClockingSchemeGenerators {
         }
       }}
 
-
       chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
         clock_io := th.harnessClock
         Nil
diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala
index a26b5104..694d45da 100644
--- a/generators/chipyard/src/main/scala/HarnessBinders.scala
+++ b/generators/chipyard/src/main/scala/HarnessBinders.scala
@@ -69,7 +69,7 @@ class WithGPIOTiedOff extends OverrideHarnessBinder({
 // DOC include start: WithUARTAdapter
 class WithUARTAdapter extends OverrideHarnessBinder({
   (system: HasPeripheryUARTModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    UARTAdapter.connect(ports.map(_.asInstanceOf[UARTPortIO]))(system.p)
+    UARTAdapter.connect(ports.map({case p: UARTPortIO => p}))(system.p)
     Nil
   }
 })
@@ -77,7 +77,7 @@ class WithUARTAdapter extends OverrideHarnessBinder({
 
 class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideHarnessBinder({
   (system: HasPeripherySPIFlashModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    SimSPIFlashModel.connect(ports.map(_.asInstanceOf[SPIChipIO]), th.harnessReset, rdOnly)(system.p)
+    SimSPIFlashModel.connect(ports.map({case p: SPIChipIO => p}), th.harnessReset, rdOnly)(system.p)
     Nil
   }
 })
@@ -132,8 +132,11 @@ class WithSimAXIMem extends OverrideHarnessBinder({
   (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
     val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
     val clock = WireInit(false.B.asClock)
-    ports.filter(_.isInstanceOf[Clock]).map { case p: Clock => clock := p }
-    val axi4_ports = ports.filter(_.isInstanceOf[AXI4Bundle])
+    ports.map {
+      case p: Clock => clock := p
+      case _ =>
+    }
+    val axi4_ports = ports.collect { case p: AXI4Bundle => p }
     (axi4_ports zip system.memAXI4Node.edges.in).map { case (port: AXI4Bundle, edge) =>
       val mem = LazyModule(new SimAXIMem(edge, size=p(ExtMem).get.master.size)(p))
       withClockAndReset(clock, th.harnessReset) {
@@ -149,7 +152,10 @@ class WithBlackBoxSimMem extends OverrideHarnessBinder({
   (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
     val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
     val clock = WireInit(false.B.asClock)
-    ports.filter(_.isInstanceOf[Clock]).map { case p: Clock => clock := p }
+    ports.map {
+      case p: Clock => clock := p
+      case _ =>
+    }
     val axi4_ports = ports.collect { case p: AXI4Bundle => p }
     (axi4_ports zip system.memAXI4Node.edges.in).map { case (port: AXI4Bundle, edge) =>
       val memSize = p(ExtMem).get.master.size
@@ -167,11 +173,15 @@ class WithSimAXIMMIO extends OverrideHarnessBinder({
   (system: CanHaveMasterAXI4MMIOPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
     val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
     val clock = WireInit(false.B.asClock)
-    ports.filter(_.isInstanceOf[Clock]).map { case p: Clock => clock := p }
-    (ports zip system.mmioAXI4Node.edges.in).zipWithIndex.map { case ((port: AXI4Bundle, edge), i) =>
-      val mmio_mem = LazyModule(new SimAXIMem(edge, size = 4096)(p))
+    ports.map {
+       case p: Clock => clock := p
+       case _ =>
+    }
+    val axi4_ports = ports.collect { case p: AXI4Bundle => p }
+    (axi4_ports zip system.mmioAXI4Node.edges.in).map { case (port: AXI4Bundle, edge) =>
+      val mmio_mem = LazyModule(new SimAXIMem(edge, size = p(ExtBus).get.size)(p))
       withClockAndReset(clock, th.harnessReset) {
-        Module(mmio_mem.module).suggestName(s"mmio_mem_${i}")
+        Module(mmio_mem.module).suggestName("mmio_mem")
       }
       mmio_mem.io_axi4.head <> port
     }
@@ -188,9 +198,11 @@ class WithTieOffInterrupts extends OverrideHarnessBinder({
 
 class WithTieOffL2FBusAXI extends OverrideHarnessBinder({
   (system: CanHaveSlaveAXI4Port, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    ports.map { case p: AXI4Bundle =>
-      p := DontCare
-      p.tieoff()
+    ports.map {
+      case p: AXI4Bundle =>
+        p := DontCare
+        p.tieoff()
+      case c: Clock =>
     }
     Nil
   }
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index db5c7008..dfcbd8ba 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -47,8 +47,13 @@ case object IOBinders extends Field[Map[String, (Any) => (Seq[Data], Seq[IOCell]
 object ApplyIOBinders {
   def apply(sys: LazyModule, map: Map[String, (Any) => (Seq[Data], Seq[IOCell])]):
       (Iterable[Data], Iterable[IOCell], Map[String, Seq[Data]]) = {
-    val r = map.map({ case (s,f) => (f(sys), s) }) ++ map.map({ case (s,f) => (f(sys.module), s) })
-    (r.flatMap(_._1._1), r.flatMap(_._1._2), r.map { t => t._2 -> t._1._1 })
+    val lzy = map.map({ case (s,f) => s -> f(sys) })
+    val imp = map.map({ case (s,f) => s -> f(sys.module) })
+
+    val ports: Iterable[Data] = lzy.values.map(_._1).flatten ++ imp.values.map(_._1).flatten
+    val cells: Iterable[IOCell] = lzy.values.map(_._2).flatten ++ imp.values.map(_._2).flatten
+    val portMap: Map[String, Seq[Data]] = map.keys.map(k => k -> (lzy(k)._1 ++ imp(k)._1)).toMap
+    (ports, cells, portMap)
   }
 }
 
@@ -287,13 +292,18 @@ class WithAXI4MMIOPunchthrough extends OverrideIOBinder({
 })
 
 class WithL2FBusAXI4Punchthrough extends OverrideIOBinder({
-  (system: CanHaveSlaveAXI4Port) => {
-    val port = system.l2_frontend_bus_axi4.zipWithIndex.map({ case (m, i) =>
+   (system: CanHaveSlaveAXI4Port) => {
+    val clock = if (!system.l2_frontend_bus_axi4.isEmpty) {
+      Some(BoreHelper("axi4_fbus_clock", system.asInstanceOf[BaseSubsystem].fbus.module.clock))
+    } else {
+      None
+    }
+    val ports = system.l2_frontend_bus_axi4.zipWithIndex.map({ case (m, i) =>
       val p = IO(Flipped(DataMirror.internal.chiselTypeClone[AXI4Bundle](m))).suggestName(s"axi4_fbus_${i}")
       m <> p
       p
     })
-    (port, Nil)
+    (ports ++ clock, Nil)
   }
 })
 
diff --git a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
index 47e7c15b..9f701331 100644
--- a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
@@ -13,7 +13,7 @@ class ArianeConfig extends Config(
   new chipyard.config.AbstractConfig)
 
 class dmiArianeConfig extends Config(
-  new chipyard.harness.WithTiedOffSerial ++          // Tie off the serial port, override default instantiation of SimSerial
+  new chipyard.harness.WithTiedOffSerial ++            // Tie off the serial port, override default instantiation of SimSerial
   new chipyard.config.WithDMIDTM ++                    // have debug module expose a clocked DMI port
   new ariane.WithNArianeCores(1) ++                    // single Ariane core
   new chipyard.config.AbstractConfig)
diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
index a8686961..16d298fb 100644
--- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
@@ -12,21 +12,21 @@ class RocketConfig extends Config(
 
 class HwachaRocketConfig extends Config(
   new chipyard.config.WithHwachaTest ++
-  new hwacha.DefaultHwachaConfig ++                        // use Hwacha vector accelerator
+  new hwacha.DefaultHwachaConfig ++                              // use Hwacha vector accelerator
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 // DOC include start: GemminiRocketConfig
 class GemminiRocketConfig extends Config(
-  new gemmini.DefaultGemminiConfig ++                        // use Gemmini systolic array GEMM accelerator
+  new gemmini.DefaultGemminiConfig ++                            // use Gemmini systolic array GEMM accelerator
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 // DOC include end: GemminiRocketConfig
 
 // DOC include start: DmiRocket
 class dmiRocketConfig extends Config(
-  new chipyard.harness.WithTiedOffSerial ++          // don't use serial to drive the chip, since we use DMI instead
-  new chipyard.config.WithDMIDTM ++                    // have debug module expose a clocked DMI port
+  new chipyard.harness.WithTiedOffSerial ++                      // don't use serial to drive the chip, since we use DMI instead
+  new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 // DOC include end: DmiRocket
@@ -40,53 +40,53 @@ class GCDTLRocketConfig extends Config(
 
 // DOC include start: GCDAXI4BlackBoxRocketConfig
 class GCDAXI4BlackBoxRocketConfig extends Config(
-  new chipyard.example.WithGCD(useAXI4=true, useBlackBox=true) ++          // Use GCD blackboxed verilog, connect by AXI4->Tilelink
+  new chipyard.example.WithGCD(useAXI4=true, useBlackBox=true) ++            // Use GCD blackboxed verilog, connect by AXI4->Tilelink
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 // DOC include end: GCDAXI4BlackBoxRocketConfig
 
 class LargeSPIFlashROMRocketConfig extends Config(
-  new chipyard.harness.WithSimSPIFlashModel(true) ++      // add the SPI flash model in the harness (read-only)
+  new chipyard.harness.WithSimSPIFlashModel(true) ++        // add the SPI flash model in the harness (read-only)
   new chipyard.config.WithSPIFlash ++                       // add the SPI flash controller
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class SmallSPIFlashRocketConfig extends Config(
-  new chipyard.harness.WithSimSPIFlashModel(false) ++     // add the SPI flash model in the harness (writeable)
+  new chipyard.harness.WithSimSPIFlashModel(false) ++       // add the SPI flash model in the harness (writeable)
   new chipyard.config.WithSPIFlash(0x100000) ++             // add the SPI flash controller (1 MiB)
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class SimAXIRocketConfig extends Config(
-  new chipyard.harness.WithSimAXIMem ++                   // drive the master AXI4 memory with a SimAXIMem, a 1-cycle magic memory, instead of default SimDRAM
+  new chipyard.harness.WithSimAXIMem ++                     // drive the master AXI4 memory with a SimAXIMem, a 1-cycle magic memory, instead of default SimDRAM
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class SimBlockDeviceRocketConfig extends Config(
-  new chipyard.harness.WithSimBlockDevice ++             // drive block-device IOs with SimBlockDevice
-  new testchipip.WithBlockDevice ++                        // add block-device module to peripherybus
+  new chipyard.harness.WithSimBlockDevice ++                // drive block-device IOs with SimBlockDevice
+  new testchipip.WithBlockDevice ++                         // add block-device module to peripherybus
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class BlockDeviceModelRocketConfig extends Config(
-  new chipyard.harness.WithBlockDeviceModel ++           // drive block-device IOs with a BlockDeviceModel
-  new testchipip.WithBlockDevice ++                        // add block-device module to periphery bus
+  new chipyard.harness.WithBlockDeviceModel ++              // drive block-device IOs with a BlockDeviceModel
+  new testchipip.WithBlockDevice ++                         // add block-device module to periphery bus
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 // DOC include start: GPIORocketConfig
 class GPIORocketConfig extends Config(
-  new chipyard.config.WithGPIO ++                          // add GPIOs to the peripherybus
+  new chipyard.config.WithGPIO ++                           // add GPIOs to the peripherybus
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 // DOC include end: GPIORocketConfig
 
 class QuadRocketConfig extends Config(
-  new freechips.rocketchip.subsystem.WithNBigCores(4) ++   // quad-core (4 RocketTiles)
+  new freechips.rocketchip.subsystem.WithNBigCores(4) ++    // quad-core (4 RocketTiles)
   new chipyard.config.AbstractConfig)
 
 class RV32RocketConfig extends Config(
-  new freechips.rocketchip.subsystem.WithRV32 ++           // set RocketTiles to be 32-bit
+  new freechips.rocketchip.subsystem.WithRV32 ++            // set RocketTiles to be 32-bit
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
@@ -104,14 +104,14 @@ class Sha3RocketConfig extends Config(
 
 // DOC include start: InitZeroRocketConfig
 class InitZeroRocketConfig extends Config(
-  new chipyard.example.WithInitZero(0x88000000L, 0x1000L) ++                // add InitZero
+  new chipyard.example.WithInitZero(0x88000000L, 0x1000L) ++   // add InitZero
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 // DOC include end: InitZeroRocketConfig
 
 class LoopbackNICRocketConfig extends Config(
-  new chipyard.harness.WithLoopbackNIC ++                        // drive NIC IOs with loopback
-  new icenet.WithIceNIC ++                                         // add an IceNIC
+  new chipyard.harness.WithLoopbackNIC ++                      // drive NIC IOs with loopback
+  new icenet.WithIceNIC ++                                     // add an IceNIC
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
@@ -126,8 +126,8 @@ class ScratchpadOnlyRocketConfig extends Config(
 // DOC include end: l1scratchpadrocket
 
 class L1ScratchpadRocketConfig extends Config(
-  new chipyard.config.WithRocketICacheScratchpad ++   // use rocket ICache scratchpad
-  new chipyard.config.WithRocketDCacheScratchpad ++   // use rocket DCache scratchpad
+  new chipyard.config.WithRocketICacheScratchpad ++         // use rocket ICache scratchpad
+  new chipyard.config.WithRocketDCacheScratchpad ++         // use rocket DCache scratchpad
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
@@ -141,35 +141,35 @@ class MbusScratchpadRocketConfig extends Config(
 
 // DOC include start: RingSystemBusRocket
 class RingSystemBusRocketConfig extends Config(
-  new testchipip.WithRingSystemBus ++ // Ring-topology system bus
+  new testchipip.WithRingSystemBus ++                       // Ring-topology system bus
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 // DOC include end: RingSystemBusRocket
 
 class StreamingPassthroughRocketConfig extends Config(
-  new chipyard.example.WithStreamingPassthrough ++ // use top with tilelink-controlled streaming passthrough
+  new chipyard.example.WithStreamingPassthrough ++          // use top with tilelink-controlled streaming passthrough
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 // DOC include start: StreamingFIRRocketConfig
 class StreamingFIRRocketConfig extends Config (
-  new chipyard.example.WithStreamingFIR ++ // use top with tilelink-controlled streaming FIR
+  new chipyard.example.WithStreamingFIR ++                  // use top with tilelink-controlled streaming FIR
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 // DOC include end: StreamingFIRRocketConfig
 
 class SmallNVDLARocketConfig extends Config(
-  new nvidia.blocks.dla.WithNVDLA("small") ++ // add a small NVDLA
+  new nvidia.blocks.dla.WithNVDLA("small") ++               // add a small NVDLA
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class LargeNVDLARocketConfig extends Config(
-  new nvidia.blocks.dla.WithNVDLA("large", true) ++ // add a large NVDLA with synth. rams
+  new nvidia.blocks.dla.WithNVDLA("large", true) ++         // add a large NVDLA with synth. rams
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
 class MMIORocketConfig extends Config(
-  new freechips.rocketchip.subsystem.WithDefaultMMIOPort ++ // add default external master port
+  new freechips.rocketchip.subsystem.WithDefaultMMIOPort ++  // add default external master port
   new freechips.rocketchip.subsystem.WithDefaultSlavePort ++ // add default external slave port
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
@@ -177,7 +177,7 @@ class MMIORocketConfig extends Config(
 // NOTE: This config doesn't work yet because SimWidgets in the TestHarness
 // always get the TestHarness clock. The Tiles and Uncore receive the correct clocks
 class DividedClockRocketConfig extends Config(
-  new chipyard.config.WithTileDividedClock ++                       // Put the Tile on its own clock domain
+  new chipyard.config.WithTileDividedClock ++                     // Put the Tile on its own clock domain
   new freechips.rocketchip.subsystem.WithRationalRocketTiles ++   // Add rational crossings between RocketTile and uncore
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
diff --git a/generators/chipyard/src/main/scala/config/TracegenConfigs.scala b/generators/chipyard/src/main/scala/config/TracegenConfigs.scala
index 28bc5dcd..78cb6851 100644
--- a/generators/chipyard/src/main/scala/config/TracegenConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/TracegenConfigs.scala
@@ -3,61 +3,37 @@ package chipyard
 import freechips.rocketchip.config.{Config}
 import freechips.rocketchip.rocket.{DCacheParams}
 
-class TraceGenConfig extends Config(
+class AbstractTraceGenConfig extends Config(
   new chipyard.harness.WithBlackBoxSimMem ++
   new chipyard.harness.WithTraceGenSuccess ++
   new chipyard.iobinders.WithAXI4MemPunchthrough ++
   new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
   new chipyard.config.WithTracegenSystem ++
   new chipyard.config.WithNoSubsystemDrivenClocks ++
-  new tracegen.WithTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 0, nSets = 16, nWays = 2) }) ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
 
+
+class TraceGenConfig extends Config(
+  new tracegen.WithTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 0, nSets = 16, nWays = 2) }) ++
+  new AbstractTraceGenConfig)
+
 class NonBlockingTraceGenConfig extends Config(
-  new chipyard.harness.WithBlackBoxSimMem ++
-  new chipyard.harness.WithTraceGenSuccess ++
-  new chipyard.iobinders.WithAXI4MemPunchthrough ++
-  new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
-  new chipyard.config.WithTracegenSystem ++
-  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 2, nSets = 16, nWays = 2) }) ++
-  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
-  new freechips.rocketchip.groundtest.GroundTestBaseConfig)
+  new AbstractTraceGenConfig)
 
 class BoomTraceGenConfig extends Config(
-  new chipyard.harness.WithBlackBoxSimMem ++
-  new chipyard.harness.WithTraceGenSuccess ++
-  new chipyard.iobinders.WithAXI4MemPunchthrough ++
-  new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
-  new chipyard.config.WithTracegenSystem ++
-  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithBoomTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 8, nSets = 16, nWays = 2) }) ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
-  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
-  new freechips.rocketchip.groundtest.GroundTestBaseConfig)
+  new AbstractTraceGenConfig)
 
 class NonBlockingTraceGenL2Config extends Config(
-  new chipyard.harness.WithBlackBoxSimMem ++
-  new chipyard.harness.WithTraceGenSuccess ++
-  new chipyard.iobinders.WithAXI4MemPunchthrough ++
-  new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
-  new chipyard.config.WithTracegenSystem ++
-  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithL2TraceGen()(List.fill(2)(DCacheParams(nMSHRs = 2, nSets = 16, nWays = 4))) ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
-  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
-  new freechips.rocketchip.groundtest.GroundTestBaseConfig)
+  new AbstractTraceGenConfig)
 
 class NonBlockingTraceGenL2RingConfig extends Config(
-  new chipyard.harness.WithBlackBoxSimMem ++
-  new chipyard.harness.WithTraceGenSuccess ++
-  new chipyard.iobinders.WithAXI4MemPunchthrough ++
-  new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
-  new chipyard.config.WithTracegenSystem ++
-  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithL2TraceGen()(List.fill(2)(DCacheParams(nMSHRs = 2, nSets = 16, nWays = 4))) ++
   new testchipip.WithRingSystemBus ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
-  new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
-  new freechips.rocketchip.groundtest.GroundTestBaseConfig)
+  new AbstractTraceGenConfig)

From 8f9574fd7980f38320bfac8095ce9d3d31d94a70 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 8 Sep 2020 21:52:50 -0700
Subject: [PATCH 59/74] Clean up passing ports from IOBinders to HarnessBinders

---
 .../src/main/scala/HarnessBinders.scala       | 175 +++++++-----------
 .../chipyard/src/main/scala/IOBinders.scala   | 103 +++++------
 2 files changed, 114 insertions(+), 164 deletions(-)

diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala
index 694d45da..f427d44e 100644
--- a/generators/chipyard/src/main/scala/HarnessBinders.scala
+++ b/generators/chipyard/src/main/scala/HarnessBinders.scala
@@ -1,6 +1,7 @@
 package chipyard.harness
 
 import chisel3._
+import chisel3.experimental.{Analog}
 
 import freechips.rocketchip.config.{Field, Config, Parameters}
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImpLike}
@@ -19,6 +20,7 @@ import barstools.iocell.chisel._
 import testchipip._
 
 import chipyard.HasHarnessSignalReferences
+import chipyard.iobinders.ClockedIO
 
 import tracegen.{TraceGenSystemModuleImp}
 import icenet.{CanHavePeripheryIceNICModuleImp, SimNetwork, NicLoopback, NICKey, NICIOvonly}
@@ -33,26 +35,29 @@ case object HarnessBinders extends Field[Map[String, (Any, HasHarnessSignalRefer
 object ApplyHarnessBinders {
   def apply(th: HasHarnessSignalReferences, sys: LazyModule, map: Map[String, (Any, HasHarnessSignalReferences, Seq[Data]) => Seq[Any]], portMap: Map[String, Seq[Data]]) = {
     val pm = portMap.withDefaultValue(Nil)
-    map.map { case (s, f) => f(sys, th, pm(s)) ++ f(sys.module, th, pm(s)) }
+    map.map { case (s, f) => f(sys, th, pm(s)) ++ f(sys.module, th, pm(s))
+    }
   }
 }
 
-class OverrideHarnessBinder[T](fn: => (T, HasHarnessSignalReferences, Seq[Data]) => Seq[Any])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class OverrideHarnessBinder[T, S <: Data](fn: => (T, HasHarnessSignalReferences, Seq[S]) => Seq[Any])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
   case HarnessBinders => up(HarnessBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+        val pts = ports.map(_.asInstanceOf[S])
         t match {
-          case system: T => fn(system, th, ports)
+          case system: T => fn(system, th, pts)
           case _ => Nil
         }
       })
   )
 })
 
-class ComposeHarnessBinder[T](fn: => (T, HasHarnessSignalReferences, Seq[Data]) => Seq[Any])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class ComposeHarnessBinder[T, S <: Data](fn: => (T, HasHarnessSignalReferences, Seq[S]) => Seq[Any])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
   case HarnessBinders => up(HarnessBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+        val pts = ports.map(_.asInstanceOf[S])
         t match {
-          case system: T => up(HarnessBinders, site)(tag.runtimeClass.toString)(system, th, ports) ++ fn(system, th, ports)
+          case system: T => up(HarnessBinders, site)(tag.runtimeClass.toString)(system, th, pts) ++ fn(system, th, pts)
           case _ => Nil
         }
       })
@@ -60,109 +65,83 @@ class ComposeHarnessBinder[T](fn: => (T, HasHarnessSignalReferences, Seq[Data])
 })
 
 class WithGPIOTiedOff extends OverrideHarnessBinder({
-  (system: HasPeripheryGPIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    ports.map { case p: GPIOPortIO => p <> AnalogConst(0) }
+  (system: HasPeripheryGPIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[Analog]) => {
+    ports.foreach { _ <> AnalogConst(0) }
     Nil
   }
 })
 
 // DOC include start: WithUARTAdapter
 class WithUARTAdapter extends OverrideHarnessBinder({
-  (system: HasPeripheryUARTModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    UARTAdapter.connect(ports.map({case p: UARTPortIO => p}))(system.p)
+  (system: HasPeripheryUARTModuleImp, th: HasHarnessSignalReferences, ports: Seq[UARTPortIO]) => {
+    UARTAdapter.connect(ports)(system.p)
     Nil
   }
 })
 // DOC include end: WithUARTAdapter
 
 class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideHarnessBinder({
-  (system: HasPeripherySPIFlashModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    SimSPIFlashModel.connect(ports.map({case p: SPIChipIO => p}), th.harnessReset, rdOnly)(system.p)
+  (system: HasPeripherySPIFlashModuleImp, th: HasHarnessSignalReferences, ports: Seq[SPIChipIO]) => {
+    SimSPIFlashModel.connect(ports, th.harnessReset, rdOnly)(system.p)
     Nil
   }
 })
 
 class WithSimBlockDevice extends OverrideHarnessBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    val clock = WireInit(false.B.asClock)
-    ports.map {
-      case p: BlockDeviceIO => SimBlockDevice.connect(clock, th.harnessReset.asBool, Some(p))(system.p)
-      case c: Clock => clock := c
-    }
+  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
+    ports.map { p => SimBlockDevice.connect(p.clock, th.harnessReset.asBool, Some(p.bits))(system.p) }
     Nil
   }
 })
 
 class WithBlockDeviceModel extends OverrideHarnessBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    val clock = WireInit(false.B.asClock)
-    ports.map {
-      case p: BlockDeviceIO => withClockAndReset(clock, th.harnessReset) { BlockDeviceModel.connect(Some(p))(system.p) }
-      case c: Clock => clock := c
-    }
+  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
+    ports.map { p => withClockAndReset(p.clock, th.harnessReset) { BlockDeviceModel.connect(Some(p.bits))(system.p) } }
     Nil
   }
 })
 
 class WithLoopbackNIC extends OverrideHarnessBinder({
-  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    val clock = WireInit(false.B.asClock)
-    ports.map {
-      case p: NICIOvonly => withClockAndReset(clock, th.harnessReset) {
-        NicLoopback.connect(Some(p), system.p(NICKey))
+  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
+    ports.map { p =>
+      withClockAndReset(p.clock, th.harnessReset) {
+        NicLoopback.connect(Some(p.bits), system.p(NICKey))
       }
-      case c: Clock => clock := c
     }
     Nil
   }
 })
 
 class WithSimNetwork extends OverrideHarnessBinder({
-  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    val clock = WireInit(false.B.asClock)
-    ports.map {
-      case p: NICIOvonly => SimNetwork.connect(Some(p), clock, th.harnessReset.asBool)
-      case c: Clock => clock := c
-    }
+  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
+    ports.map { p => SimNetwork.connect(Some(p.bits), p.clock, th.harnessReset.asBool) }
     Nil
   }
 })
 
 class WithSimAXIMem extends OverrideHarnessBinder({
-  (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+  (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[AXI4Bundle]]) => {
     val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
-    val clock = WireInit(false.B.asClock)
-    ports.map {
-      case p: Clock => clock := p
-      case _ =>
-    }
-    val axi4_ports = ports.collect { case p: AXI4Bundle => p }
-    (axi4_ports zip system.memAXI4Node.edges.in).map { case (port: AXI4Bundle, edge) =>
+    (ports zip system.memAXI4Node.edges.in).map { case (port, edge) =>
       val mem = LazyModule(new SimAXIMem(edge, size=p(ExtMem).get.master.size)(p))
-      withClockAndReset(clock, th.harnessReset) {
+      withClockAndReset(port.clock, th.harnessReset) {
         Module(mem.module).suggestName("mem")
       }
-      mem.io_axi4.head <> port
+      mem.io_axi4.head <> port.bits
     }
     Nil
   }
 })
 
 class WithBlackBoxSimMem extends OverrideHarnessBinder({
-  (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+  (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[AXI4Bundle]]) => {
     val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
-    val clock = WireInit(false.B.asClock)
-    ports.map {
-      case p: Clock => clock := p
-      case _ =>
-    }
-    val axi4_ports = ports.collect { case p: AXI4Bundle => p }
-    (axi4_ports zip system.memAXI4Node.edges.in).map { case (port: AXI4Bundle, edge) =>
+    (ports zip system.memAXI4Node.edges.in).map { case (port, edge) =>
       val memSize = p(ExtMem).get.master.size
       val lineSize = p(CacheBlockBytes)
       val mem = Module(new SimDRAM(memSize, lineSize, edge.bundle)).suggestName("simdram")
-      mem.io.axi <> port
-      mem.io.clock := clock
+      mem.io.axi <> port.bits
+      mem.io.clock := port.clock
       mem.io.reset := th.harnessReset
     }
     Nil
@@ -170,57 +149,44 @@ class WithBlackBoxSimMem extends OverrideHarnessBinder({
 })
 
 class WithSimAXIMMIO extends OverrideHarnessBinder({
-  (system: CanHaveMasterAXI4MMIOPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+  (system: CanHaveMasterAXI4MMIOPort, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[AXI4Bundle]]) => {
     val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
-    val clock = WireInit(false.B.asClock)
-    ports.map {
-       case p: Clock => clock := p
-       case _ =>
-    }
-    val axi4_ports = ports.collect { case p: AXI4Bundle => p }
-    (axi4_ports zip system.mmioAXI4Node.edges.in).map { case (port: AXI4Bundle, edge) =>
+    (ports zip system.mmioAXI4Node.edges.in).map { case (port, edge) =>
       val mmio_mem = LazyModule(new SimAXIMem(edge, size = p(ExtBus).get.size)(p))
-      withClockAndReset(clock, th.harnessReset) {
+      withClockAndReset(port.clock, th.harnessReset) {
         Module(mmio_mem.module).suggestName("mmio_mem")
       }
-      mmio_mem.io_axi4.head <> port
+      mmio_mem.io_axi4.head <> port.bits
     }
     Nil
   }
 })
 
 class WithTieOffInterrupts extends OverrideHarnessBinder({
-  (system: HasExtInterruptsModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    ports.map { case p: UInt =>  p := 0.U }
+  (system: HasExtInterruptsModuleImp, th: HasHarnessSignalReferences, ports: Seq[UInt]) => {
+    ports.foreach { _ := 0.U }
     Nil
   }
 })
 
 class WithTieOffL2FBusAXI extends OverrideHarnessBinder({
-  (system: CanHaveSlaveAXI4Port, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    ports.map {
-      case p: AXI4Bundle =>
-        p := DontCare
-        p.tieoff()
-      case c: Clock =>
-    }
+  (system: CanHaveSlaveAXI4Port, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[AXI4Bundle]]) => {
+    ports.foreach({ p => p := DontCare; p.bits.tieoff() })
     Nil
   }
 })
 
 class WithSimDebug extends OverrideHarnessBinder({
   (system: HasPeripheryDebugModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    if (!ports.isEmpty) {
-      val dtm_success = Wire(Bool())
-      when (dtm_success) { th.success := true.B }
-      ports.map {
-        case d: ClockedDMIIO =>
-          val dtm = Module(new SimDTM()(system.p)).connect(th.harnessClock, th.harnessReset.asBool, d, dtm_success)
-        case j: JTAGIO =>
-          val jtag = Module(new SimJTAG(tickDelay=3)).connect(j, th.harnessClock, th.harnessReset.asBool, ~(th.harnessReset.asBool), dtm_success)
-        case _ =>
-          require(false, "We only support DMI or JTAG simulated debug connections")
-      }
+    ports.map {
+      case d: ClockedDMIIO =>
+        val dtm_success = WireInit(false.B)
+        when (dtm_success) { th.success := true.B }
+        val dtm = Module(new SimDTM()(system.p)).connect(th.harnessClock, th.harnessReset.asBool, d, dtm_success)
+      case j: JTAGIO =>
+        val dtm_success = WireInit(false.B)
+        when (dtm_success) { th.success := true.B }
+        val jtag = Module(new SimJTAG(tickDelay=3)).connect(j, th.harnessClock, th.harnessReset.asBool, ~(th.harnessReset.asBool), dtm_success)
     }
     Nil
   }
@@ -229,63 +195,56 @@ class WithSimDebug extends OverrideHarnessBinder({
 class WithTiedOffDebug extends OverrideHarnessBinder({
   (system: HasPeripheryDebugModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
     ports.map {
+      case j: JTAGIO =>
+        j.TCK := true.B.asClock
+        j.TMS := true.B
+        j.TDI := true.B
+        j.TRSTn.foreach { r => r := true.B }
       case d: ClockedDMIIO =>
         d.dmi.req.valid := false.B
         d.dmi.req.bits  := DontCare
         d.dmi.resp.ready := true.B
         d.dmiClock := false.B.asClock
         d.dmiReset := true.B
-      case j: JTAGIO =>
-        j.TCK := true.B.asClock
-        j.TMS := true.B
-        j.TDI := true.B
-        j.TRSTn.foreach { r => r := true.B }
       case a: ClockedAPBBundle =>
         a.tieoff()
         a.clock := false.B.asClock
         a.reset := true.B.asAsyncReset
         a.psel := false.B
         a.penable := false.B
-      case _ => require(false)
     }
     Nil
   }
 })
 
+
 class WithTiedOffSerial extends OverrideHarnessBinder({
-  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    ports.map {
-      case p: SerialIO => SerialAdapter.tieoff(Some(p))
-      case _ =>
-    }
+  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+    ports.map { p => SerialAdapter.tieoff(Some(p.bits)) }
     Nil
   }
 })
 
 class WithSimSerial extends OverrideHarnessBinder({
-  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    val serial_clock = WireInit(false.B.asClock)
-    ports.map {
-      case p: SerialIO =>
-        val ser_success = SerialAdapter.connectSimSerial(p, serial_clock, th.harnessReset)
-        when (ser_success) { th.success := true.B }
-      case c: Clock =>
-        serial_clock := c
+  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+    ports.map { p =>
+      val ser_success = SerialAdapter.connectSimSerial(p.bits, p.clock, th.harnessReset)
+      when (ser_success) { th.success := true.B }
     }
     Nil
   }
 })
 
 class WithTraceGenSuccess extends OverrideHarnessBinder({
-  (system: TraceGenSystemModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    ports.map { case p: Bool => when (p) { th.success := true.B } }
+  (system: TraceGenSystemModuleImp, th: HasHarnessSignalReferences, ports: Seq[Bool]) => {
+    ports.map { p => when (p) { th.success := true.B } }
     Nil
   }
 })
 
 class WithSimDromajoBridge extends ComposeHarnessBinder({
-  (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    ports.map { case p: TraceOutputTop => p.traces.map(tileTrace => SimDromajoBridge(tileTrace)(system.p)) }
+  (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[TraceOutputTop]) => {
+    ports.map { p => p.traces.map(tileTrace => SimDromajoBridge(tileTrace)(system.p)) }
     Nil
   }
 })
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index dfcbd8ba..969ad571 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -74,7 +74,7 @@ object GetSystemParameters {
 
 // This macro overrides previous matches on some Top mixin. This is useful for
 // binders which drive IO, since those typically cannot be composed
-class OverrideIOBinder[T](fn: => (T) => (Seq[Data], Seq[IOCell]))(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class OverrideIOBinder[T, S <: Data](fn: => (T) => (Seq[S], Seq[IOCell]))(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
   case IOBinders => up(IOBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any) => {
         t match {
@@ -87,7 +87,7 @@ class OverrideIOBinder[T](fn: => (T) => (Seq[Data], Seq[IOCell]))(implicit tag:
 
 // This macro composes with previous matches on some Top mixin. This is useful for
 // annotation-like binders, since those can typically be composed
-class ComposeIOBinder[T](fn: => (T) => (Seq[Data], Seq[IOCell]))(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class ComposeIOBinder[T, S <: Data](fn: => (T) => (Seq[S], Seq[IOCell]))(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
   case IOBinders => up(IOBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any) => {
         t match {
@@ -116,6 +116,11 @@ object BoreHelper {
 
 case object IOCellKey extends Field[IOCellTypeParams](GenericIOCellParams())
 
+class ClockedIO[T <: Data](gen: T) extends Bundle {
+  val clock = Output(Clock())
+  val bits = gen
+  override def cloneType: this.type = (new ClockedIO(DataMirror.internal.chiselTypeClone[T](gen))).asInstanceOf[this.type]
+}
 
 class WithGPIOCells extends OverrideIOBinder({
   (system: HasPeripheryGPIOModuleImp) => {
@@ -131,14 +136,15 @@ class WithGPIOCells extends OverrideIOBinder({
         (g, iocell)
       }).unzip
     }).unzip
-    (ports2d.flatten, cells2d.flatten)
+    val ports: Seq[Analog] = ports2d.flatten
+    (ports, cells2d.flatten)
   }
 })
 
 // DOC include start: WithUARTIOCells
 class WithUARTIOCells extends OverrideIOBinder({
   (system: HasPeripheryUARTModuleImp) => {
-    val (ports, cells2d) = system.uart.zipWithIndex.map({ case (u, i) =>
+    val (ports: Seq[UARTPortIO], cells2d) = system.uart.zipWithIndex.map({ case (u, i) =>
       val (port, ios) = IOCell.generateIOFromSignal(u, Some(s"iocell_uart_${i}"), system.p(IOCellKey))
       port.suggestName(s"uart_${i}")
       (port, ios)
@@ -150,7 +156,7 @@ class WithUARTIOCells extends OverrideIOBinder({
 
 class WithSPIIOCells extends OverrideIOBinder({
   (system: HasPeripherySPIFlashModuleImp) => {
-    val (ports, cells2d) = system.qspi.zipWithIndex.map({ case (s, i) =>
+    val (ports: Seq[SPIChipIO], cells2d) = system.qspi.zipWithIndex.map({ case (s, i) =>
       val port = IO(new SPIChipIO(s.c.csWidth)).suggestName(s"spi_${i}")
       val iocellBase = s"iocell_spi_${i}"
 
@@ -178,7 +184,7 @@ class WithSPIIOCells extends OverrideIOBinder({
 class WithExtInterruptIOCells extends OverrideIOBinder({
   (system: HasExtInterruptsModuleImp) => {
     if (system.outer.nExtInterrupts > 0) {
-      val (port, cells) = IOCell.generateIOFromSignal(system.interrupts, Some("iocell_interrupts"), system.p(IOCellKey))
+      val (port: UInt, cells) = IOCell.generateIOFromSignal(system.interrupts, Some("iocell_interrupts"), system.p(IOCellKey))
       port.suggestName("ext_interrupts")
       (Seq(port), cells)
     } else {
@@ -246,94 +252,79 @@ class WithDebugIOCells extends OverrideIOBinder({
 class WithSerialIOCells extends OverrideIOBinder({
   (system: CanHavePeripherySerial) => system.serial.map({ s =>
     val sys = system.asInstanceOf[BaseSubsystem]
-    val (port, cells) = IOCell.generateIOFromSignal(s, Some("iocell_serial"), sys.p(IOCellKey))
-    val serial_clock = Wire(Output(Clock())).suggestName("chiptop_serial_clock")
-    serial_clock := false.B.asClock // necessary for BoringUtils to work properly
-    dontTouch(serial_clock)
-    BoringUtils.bore(sys.fbus.module.clock, Seq(serial_clock))
-    val (serial_clock_io, serial_clock_cell) = IOCell.generateIOFromSignal(serial_clock, Some("serial_clock"), sys.p(IOCellKey))
-    serial_clock_io.suggestName("serial_clock")
+    val clocked_serial = Wire(new ClockedIO(DataMirror.internal.chiselTypeClone[SerialIO](s))).suggestName("serial_wire")
+    clocked_serial.clock := BoreHelper("serial_clock", sys.fbus.module.clock)
+    clocked_serial.bits <> s
+    val (port, cells) = IOCell.generateIOFromSignal(clocked_serial, Some("serial"), sys.p(IOCellKey))
     port.suggestName("serial")
-    (Seq(port, serial_clock_io), cells ++ serial_clock_cell)
+    (Seq(port), cells)
   }).getOrElse((Nil, Nil))
 })
 
 
 class WithAXI4MemPunchthrough extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {
-    val clock = if (!system.mem_axi4.isEmpty) {
-      Some(BoreHelper("axi4_mem_clock", system.asInstanceOf[BaseSubsystem].mbus.module.clock))
-    } else {
-      None
-    }
-    val ports = system.mem_axi4.zipWithIndex.map({ case (m, i) =>
-      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName(s"axi4_mem_${i}")
-      p <> m
+    val ports: Seq[ClockedIO[AXI4Bundle]] = system.mem_axi4.zipWithIndex.map({ case (m, i) =>
+      val p = IO(new ClockedIO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m))).suggestName(s"axi4_mem_${i}")
+      p.bits <> m
+      p.clock := BoreHelper("axi4_mem_clock", system.asInstanceOf[BaseSubsystem].mbus.module.clock)
       p
     })
-    (ports ++ clock, Nil)
+    (ports, Nil)
   }
 })
 
 class WithAXI4MMIOPunchthrough extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MMIOPort) => {
-    val clock = if (!system.mmio_axi4.isEmpty) {
-      Some(BoreHelper("axi4_mmio_clock", system.asInstanceOf[BaseSubsystem].mbus.module.clock))
-    } else {
-      None
-    }
-    val ports = system.mmio_axi4.zipWithIndex.map({ case (m, i) =>
-      val p = IO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)).suggestName(s"axi4_mmio_${i}")
-      p <> m
+    val ports: Seq[ClockedIO[AXI4Bundle]] = system.mmio_axi4.zipWithIndex.map({ case (m, i) =>
+      val p = IO(new ClockedIO(DataMirror.internal.chiselTypeClone[AXI4Bundle](m))).suggestName(s"axi4_mmio_${i}")
+      p.bits <> m
+      p.clock := BoreHelper("axi4_mmio_clock", system.asInstanceOf[BaseSubsystem].mbus.module.clock)
       p
     })
-    (ports ++ clock, Nil)
+    (ports, Nil)
   }
 })
 
 class WithL2FBusAXI4Punchthrough extends OverrideIOBinder({
    (system: CanHaveSlaveAXI4Port) => {
-    val clock = if (!system.l2_frontend_bus_axi4.isEmpty) {
-      Some(BoreHelper("axi4_fbus_clock", system.asInstanceOf[BaseSubsystem].fbus.module.clock))
-    } else {
-      None
-    }
-    val ports = system.l2_frontend_bus_axi4.zipWithIndex.map({ case (m, i) =>
-      val p = IO(Flipped(DataMirror.internal.chiselTypeClone[AXI4Bundle](m))).suggestName(s"axi4_fbus_${i}")
-      m <> p
+    val ports: Seq[ClockedIO[AXI4Bundle]] = system.l2_frontend_bus_axi4.zipWithIndex.map({ case (m, i) =>
+      val p = IO(new ClockedIO(Flipped(DataMirror.internal.chiselTypeClone[AXI4Bundle](m)))).suggestName(s"axi4_fbus_${i}")
+      m <> p.bits
+      p.clock := BoreHelper("axi4_fbus_clock", system.asInstanceOf[BaseSubsystem].fbus.module.clock)
       p
     })
-    (ports ++ clock, Nil)
+    (ports, Nil)
   }
 })
 
 class WithBlockDeviceIOPunchthrough extends OverrideIOBinder({
   (system: CanHavePeripheryBlockDeviceModuleImp) => {
-    val ports = system.bdev.map({ bdev =>
-      val p = IO(new BlockDeviceIO()(system.p)).suggestName("blockdev")
-      val clock = BoreHelper("blkdev_clk", system.outer.controller.get.module.clock)
-      p <> bdev
-      Seq(p, clock)
-    }).getOrElse(Nil)
+    val ports: Seq[ClockedIO[BlockDeviceIO]] = system.bdev.map({ bdev =>
+      val p = IO(new ClockedIO(new BlockDeviceIO()(system.p))).suggestName("blockdev")
+      p.clock := BoreHelper("blkdev_clk", system.outer.controller.get.module.clock)
+      p.bits <> bdev
+      p
+    }).toSeq
     (ports, Nil)
   }
 })
 
 class WithNICIOPunchthrough extends OverrideIOBinder({
   (system: CanHavePeripheryIceNICModuleImp) => {
-    val port = system.net.map({ n =>
-      val p = IO(new NICIOvonly).suggestName("nic")
-      val clock = BoreHelper("nic_clk", system.outer.icenicOpt.get.module.clock)
-      p <> n
-      Seq(p, clock)
-    }).getOrElse(Nil)
-    (port.toSeq, Nil)
+    val ports: Seq[ClockedIO[NICIOvonly]] = system.net.map({ n =>
+      val p = IO(new ClockedIO(new NICIOvonly)).suggestName("nic")
+      p.clock := BoreHelper("nic_clk", system.outer.icenicOpt.get.module.clock)
+      p.bits <> n
+      p
+    }).toSeq
+    (ports, Nil)
   }
 })
 
 class WithTraceGenSuccessPunchthrough extends OverrideIOBinder({
   (system: TraceGenSystemModuleImp) => {
-    val success = IO(Output(Bool())).suggestName("success")
+    val success: Bool = IO(Output(Bool())).suggestName("success")
     success := system.success
     (Seq(success), Nil)
   }
@@ -341,7 +332,7 @@ class WithTraceGenSuccessPunchthrough extends OverrideIOBinder({
 
 class WithTraceIOPunchthrough extends OverrideIOBinder({
   (system: CanHaveTraceIOModuleImp) => {
-    val ports = system.traceIO.map { t =>
+    val ports: Option[TraceOutputTop] = system.traceIO.map { t =>
       val trace = IO(DataMirror.internal.chiselTypeClone[TraceOutputTop](t)).suggestName("trace")
       trace <> t
       trace

From facef464e69e786323e22bedc3ed8a60ea1429c6 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 9 Sep 2020 00:15:02 -0700
Subject: [PATCH 60/74] Update BridgeBinders | fix runtime HarnessBinder port
 type checks

---
 .../src/main/scala/HarnessBinders.scala       | 13 ++--
 .../src/main/scala/BridgeBinders.scala        | 66 ++++++++-----------
 2 files changed, 35 insertions(+), 44 deletions(-)

diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala
index f427d44e..90365b39 100644
--- a/generators/chipyard/src/main/scala/HarnessBinders.scala
+++ b/generators/chipyard/src/main/scala/HarnessBinders.scala
@@ -35,15 +35,15 @@ case object HarnessBinders extends Field[Map[String, (Any, HasHarnessSignalRefer
 object ApplyHarnessBinders {
   def apply(th: HasHarnessSignalReferences, sys: LazyModule, map: Map[String, (Any, HasHarnessSignalReferences, Seq[Data]) => Seq[Any]], portMap: Map[String, Seq[Data]]) = {
     val pm = portMap.withDefaultValue(Nil)
-    map.map { case (s, f) => f(sys, th, pm(s)) ++ f(sys.module, th, pm(s))
-    }
+    map.map { case (s, f) => f(sys, th, pm(s)) ++ f(sys.module, th, pm(s)) }
   }
 }
 
-class OverrideHarnessBinder[T, S <: Data](fn: => (T, HasHarnessSignalReferences, Seq[S]) => Seq[Any])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class OverrideHarnessBinder[T, S <: Data](fn: => (T, HasHarnessSignalReferences, Seq[S]) => Seq[Any])(implicit tag: ClassTag[T], ptag: ClassTag[S]) extends Config((site, here, up) => {
   case HarnessBinders => up(HarnessBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-        val pts = ports.map(_.asInstanceOf[S])
+        val pts = ports.collect({case p: S => p})
+        require (pts.length == ports.length, s"Port type mismatch between IOBinder and HarnessBinder: ${ptag}")
         t match {
           case system: T => fn(system, th, pts)
           case _ => Nil
@@ -52,10 +52,11 @@ class OverrideHarnessBinder[T, S <: Data](fn: => (T, HasHarnessSignalReferences,
   )
 })
 
-class ComposeHarnessBinder[T, S <: Data](fn: => (T, HasHarnessSignalReferences, Seq[S]) => Seq[Any])(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class ComposeHarnessBinder[T, S <: Data](fn: => (T, HasHarnessSignalReferences, Seq[S]) => Seq[Any])(implicit tag: ClassTag[T], ptag: ClassTag[S]) extends Config((site, here, up) => {
   case HarnessBinders => up(HarnessBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-        val pts = ports.map(_.asInstanceOf[S])
+        val pts = ports.collect({case p: S => p})
+        require (pts.length == ports.length, s"Port type mismatch between IOBinder and HarnessBinder: ${ptag}")
         t match {
           case system: T => up(HarnessBinders, site)(tag.runtimeClass.toString)(system, th, pts) ++ fn(system, th, pts)
           case _ => Nil
diff --git a/generators/firechip/src/main/scala/BridgeBinders.scala b/generators/firechip/src/main/scala/BridgeBinders.scala
index 0eb61a05..5dcddcfa 100644
--- a/generators/firechip/src/main/scala/BridgeBinders.scala
+++ b/generators/firechip/src/main/scala/BridgeBinders.scala
@@ -26,7 +26,7 @@ import ariane.ArianeTile
 
 import boom.common.{BoomTile}
 import barstools.iocell.chisel._
-import chipyard.iobinders.{IOBinders, OverrideIOBinder, ComposeIOBinder, GetSystemParameters, IOCellKey}
+import chipyard.iobinders.{ClockedIO, IOBinders, OverrideIOBinder, ComposeIOBinder, GetSystemParameters, IOCellKey}
 import chipyard.{HasHarnessSignalReferences}
 import chipyard.harness._
 
@@ -56,57 +56,44 @@ class WithFireSimIOCellModels extends Config((site, here, up) => {
 })
 
 class WithSerialBridge extends OverrideHarnessBinder({
-  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    val clock = ports.collectFirst({case c: Clock => c})
-    val p: Parameters = chipyard.iobinders.GetSystemParameters(system)
-    ports.filter(_.isInstanceOf[SerialIO]).map {
-      case s: SerialIO => withClockAndReset(clock.get, th.harnessReset) {
-        SerialBridge(clock.get, s, MainMemoryConsts.globalName)(p)
+  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+    ports.map { p =>
+      withClockAndReset(p.clock, th.harnessReset) {
+        SerialBridge(p.clock, p.bits, MainMemoryConsts.globalName)(GetSystemParameters(system))
       }
-      case _ =>
     }
     Nil
   }
 })
 
 class WithNICBridge extends OverrideHarnessBinder({
-  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    val clock = ports.collectFirst({case c: Clock => c})
-    ports.map {
-      case p: NICIOvonly => withClockAndReset(clock.get, th.harnessReset) { NICBridge(clock.get, p)(system.p) }
-      case _ =>
-    }
+  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
+    ports.map { p => withClockAndReset(p.clock, th.harnessReset) { NICBridge(p.clock, p.bits)(system.p) } }
     Nil
   }
 })
 
 class WithUARTBridge extends OverrideHarnessBinder({
-  (system: HasPeripheryUARTModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) =>
-    ports.map { case p: UARTPortIO => UARTBridge(th.harnessClock, p)(system.p) }; Nil
+  (system: HasPeripheryUARTModuleImp, th: HasHarnessSignalReferences, ports: Seq[UARTPortIO]) =>
+    ports.map { p => UARTBridge(th.harnessClock, p)(system.p) }; Nil
 })
 
 class WithBlockDeviceBridge extends OverrideHarnessBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    val clock = ports.collectFirst({case c: Clock => c})
-    ports.map {
-      case p: BlockDeviceIO => BlockDevBridge(clock.get, p, th.harnessReset.toBool)(system.p)
-      case _ =>
-    }
+  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
+    ports.map { p => BlockDevBridge(p.clock, p.bits, th.harnessReset.toBool)(system.p) }
     Nil
   }
 })
 
 class WithFASEDBridge extends OverrideHarnessBinder({
-  (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+  (system: CanHaveMasterAXI4MemPort, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[AXI4Bundle]]) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    val clock = ports.collectFirst({case c: Clock => c})
-    val axi4_ports = ports.collect { case p: AXI4Bundle => p }
-    (axi4_ports zip system.memAXI4Node.edges.in).map { case (axi4: AXI4Bundle, edge) =>
-      val nastiKey = NastiParameters(axi4.r.bits.data.getWidth,
-                                     axi4.ar.bits.addr.getWidth,
-                                     axi4.ar.bits.id.getWidth)
+    (ports zip system.memAXI4Node.edges.in).map { case (axi4, edge) =>
+      val nastiKey = NastiParameters(axi4.bits.r.bits.data.getWidth,
+                                     axi4.bits.ar.bits.addr.getWidth,
+                                     axi4.bits.ar.bits.id.getWidth)
       system match {
-        case s: BaseSubsystem => FASEDBridge(clock.get, axi4, th.harnessReset.asBool,
+        case s: BaseSubsystem => FASEDBridge(axi4.clock, axi4.bits, th.harnessReset.asBool,
           CompleteConfig(p(firesim.configs.MemModelKey),
                          nastiKey,
                          Some(AXI4EdgeSummary(edge)),
@@ -119,22 +106,25 @@ class WithFASEDBridge extends OverrideHarnessBinder({
 })
 
 class WithTracerVBridge extends ComposeHarnessBinder({
-  (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
-    ports.map { case p: TraceOutputTop => p.traces.map(tileTrace =>
-      withClockAndReset(tileTrace.clock, tileTrace.reset) { TracerVBridge(tileTrace)(system.p) }
-    )}
+  (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[TraceOutputTop]) => {
+    ports.map { p =>
+      p.traces.map(
+        tileTrace => withClockAndReset(tileTrace.clock, tileTrace.reset) { TracerVBridge(tileTrace)(system.p) }
+      )
+    }
+    Nil
   }
 })
 
 class WithDromajoBridge extends ComposeHarnessBinder({
-  (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) =>
-    ports.map { case p: TraceOutputTop => p.traces.map(tileTrace => DromajoBridge(tileTrace)(system.p)) }; Nil
+  (system: CanHaveTraceIOModuleImp, th: HasHarnessSignalReferences, ports: Seq[TraceOutputTop]) =>
+    ports.map { p => p.traces.map(tileTrace => DromajoBridge(tileTrace)(system.p)) }; Nil
 })
 
 
 class WithTraceGenBridge extends OverrideHarnessBinder({
-  (system: TraceGenSystemModuleImp, th: HasHarnessSignalReferences, ports: Seq[Data]) =>
-    ports.map { case p: Bool => GroundTestBridge(th.harnessClock, p)(system.p) }; Nil
+  (system: TraceGenSystemModuleImp, th: HasHarnessSignalReferences, ports: Seq[Bool]) =>
+    ports.map { p => GroundTestBridge(th.harnessClock, p)(system.p) }; Nil
 })
 
 class WithFireSimMultiCycleRegfile extends ComposeIOBinder({

From a5385c0a54d9291ba0ec444c58042fdcc387ee98 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Thu, 10 Sep 2020 23:20:37 -0700
Subject: [PATCH 61/74] Update testchipip/icenet to use rocket-chip Located API

---
 .../chipyard/src/main/scala/DigitalTop.scala  |  2 -
 .../src/main/scala/HarnessBinders.scala       | 28 ++++++-----
 .../chipyard/src/main/scala/IOBinders.scala   | 48 +++++++++----------
 .../src/main/scala/BridgeBinders.scala        | 14 +++---
 .../src/main/scala/TargetConfigs.scala        |  2 +-
 generators/icenet                             |  2 +-
 generators/testchipip                         |  2 +-
 7 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/generators/chipyard/src/main/scala/DigitalTop.scala b/generators/chipyard/src/main/scala/DigitalTop.scala
index ddcf66f3..a065b6be 100644
--- a/generators/chipyard/src/main/scala/DigitalTop.scala
+++ b/generators/chipyard/src/main/scala/DigitalTop.scala
@@ -32,11 +32,9 @@ class DigitalTop(implicit p: Parameters) extends ChipyardSystem
 
 class DigitalTopModule[+L <: DigitalTop](l: L) extends ChipyardSystemModule(l)
   with testchipip.CanHaveTraceIOModuleImp
-  with testchipip.CanHavePeripheryBlockDeviceModuleImp
   with sifive.blocks.devices.uart.HasPeripheryUARTModuleImp
   with sifive.blocks.devices.gpio.HasPeripheryGPIOModuleImp
   with sifive.blocks.devices.spi.HasPeripherySPIFlashModuleImp
-  with icenet.CanHavePeripheryIceNICModuleImp
   with chipyard.example.CanHavePeripheryGCDModuleImp
   with freechips.rocketchip.util.DontTouch
 // DOC include end: DigitalTop
diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala
index 90365b39..06a3af5d 100644
--- a/generators/chipyard/src/main/scala/HarnessBinders.scala
+++ b/generators/chipyard/src/main/scala/HarnessBinders.scala
@@ -20,10 +20,10 @@ import barstools.iocell.chisel._
 import testchipip._
 
 import chipyard.HasHarnessSignalReferences
-import chipyard.iobinders.ClockedIO
+import chipyard.iobinders.GetSystemParameters
 
 import tracegen.{TraceGenSystemModuleImp}
-import icenet.{CanHavePeripheryIceNICModuleImp, SimNetwork, NicLoopback, NICKey, NICIOvonly}
+import icenet.{CanHavePeripheryIceNIC, SimNetwork, NicLoopback, NICKey, NICIOvonly}
 
 import scala.reflect.{ClassTag}
 
@@ -89,24 +89,27 @@ class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideHarnessBinder
 })
 
 class WithSimBlockDevice extends OverrideHarnessBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
-    ports.map { p => SimBlockDevice.connect(p.clock, th.harnessReset.asBool, Some(p.bits))(system.p) }
+  (system: CanHavePeripheryBlockDevice, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
+    implicit val p: Parameters = GetSystemParameters(system)
+    ports.map { b => SimBlockDevice.connect(b.clock, th.harnessReset.asBool, Some(b.bits)) }
     Nil
   }
 })
 
 class WithBlockDeviceModel extends OverrideHarnessBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
-    ports.map { p => withClockAndReset(p.clock, th.harnessReset) { BlockDeviceModel.connect(Some(p.bits))(system.p) } }
+  (system: CanHavePeripheryBlockDevice, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
+    implicit val p: Parameters = GetSystemParameters(system)
+    ports.map { b => withClockAndReset(b.clock, th.harnessReset) { BlockDeviceModel.connect(Some(b.bits)) } }
     Nil
   }
 })
 
 class WithLoopbackNIC extends OverrideHarnessBinder({
-  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
-    ports.map { p =>
-      withClockAndReset(p.clock, th.harnessReset) {
-        NicLoopback.connect(Some(p.bits), system.p(NICKey))
+  (system: CanHavePeripheryIceNIC, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
+    implicit val p: Parameters = GetSystemParameters(system)
+    ports.map { n =>
+      withClockAndReset(n.clock, th.harnessReset) {
+        NicLoopback.connect(Some(n.bits), p(NICKey))
       }
     }
     Nil
@@ -114,8 +117,9 @@ class WithLoopbackNIC extends OverrideHarnessBinder({
 })
 
 class WithSimNetwork extends OverrideHarnessBinder({
-  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
-    ports.map { p => SimNetwork.connect(Some(p.bits), p.clock, th.harnessReset.asBool) }
+  (system: CanHavePeripheryIceNIC, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
+    implicit val p: Parameters = GetSystemParameters(system)
+    ports.map { n => SimNetwork.connect(Some(n.bits), n.clock, th.harnessReset.asBool) }
     Nil
   }
 })
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 969ad571..78233118 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -4,7 +4,7 @@ import chisel3._
 import chisel3.util.experimental.{BoringUtils}
 import chisel3.experimental.{Analog, IO, DataMirror}
 
-import freechips.rocketchip.config.{Field, Config, Parameters}
+import freechips.rocketchip.config._
 import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImpLike}
 import freechips.rocketchip.devices.debug._
 import freechips.rocketchip.jtag.{JTAGIO}
@@ -22,7 +22,7 @@ import tracegen.{TraceGenSystemModuleImp}
 import barstools.iocell.chisel._
 
 import testchipip._
-import icenet.{CanHavePeripheryIceNICModuleImp, SimNetwork, NicLoopback, NICKey, NICIOvonly}
+import icenet.{CanHavePeripheryIceNIC, SimNetwork, NicLoopback, NICKey, NICIOvonly}
 
 import chipyard.GlobalResetSchemeKey
 
@@ -43,15 +43,15 @@ import scala.reflect.{ClassTag}
 case object IOBinders extends Field[Map[String, (Any) => (Seq[Data], Seq[IOCell])]](
   Map[String, (Any) => (Seq[Data], Seq[IOCell])]().withDefaultValue((Any) => (Nil, Nil))
 )
-
 object ApplyIOBinders {
   def apply(sys: LazyModule, map: Map[String, (Any) => (Seq[Data], Seq[IOCell])]):
       (Iterable[Data], Iterable[IOCell], Map[String, Seq[Data]]) = {
     val lzy = map.map({ case (s,f) => s -> f(sys) })
     val imp = map.map({ case (s,f) => s -> f(sys.module) })
+    val unzipped = (lzy.values ++ imp.values).unzip
 
-    val ports: Iterable[Data] = lzy.values.map(_._1).flatten ++ imp.values.map(_._1).flatten
-    val cells: Iterable[IOCell] = lzy.values.map(_._2).flatten ++ imp.values.map(_._2).flatten
+    val ports: Iterable[Data] = unzipped._1.flatten
+    val cells: Iterable[IOCell] = unzipped._2.flatten
     val portMap: Map[String, Seq[Data]] = map.keys.map(k => k -> (lzy(k)._1 ++ imp(k)._1)).toMap
     (ports, cells, portMap)
   }
@@ -72,13 +72,17 @@ object GetSystemParameters {
   }
 }
 
+class IOBinder(f: (View, View, View) => PartialFunction[Any, Any]) extends Config(f)
+
 // This macro overrides previous matches on some Top mixin. This is useful for
 // binders which drive IO, since those typically cannot be composed
-class OverrideIOBinder[T, S <: Data](fn: => (T) => (Seq[S], Seq[IOCell]))(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class OverrideIOBinder[T, S <: Data](fn: => (T) => (Seq[S], Seq[IOCell]))(implicit tag: ClassTag[T]) extends IOBinder((site, here, up) => {
   case IOBinders => up(IOBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any) => {
         t match {
-          case system: T => fn(system)
+          case system: T =>
+            val (ports, cells) = fn(system)
+            (ports, cells)
           case _ => (Nil, Nil)
         }
       })
@@ -87,14 +91,16 @@ class OverrideIOBinder[T, S <: Data](fn: => (T) => (Seq[S], Seq[IOCell]))(implic
 
 // This macro composes with previous matches on some Top mixin. This is useful for
 // annotation-like binders, since those can typically be composed
-class ComposeIOBinder[T, S <: Data](fn: => (T) => (Seq[S], Seq[IOCell]))(implicit tag: ClassTag[T]) extends Config((site, here, up) => {
+class ComposeIOBinder[T, S <: Data](fn: => (T) => (Seq[S], Seq[IOCell]))(implicit tag: ClassTag[T]) extends IOBinder((site, here, up) => {
   case IOBinders => up(IOBinders, site) + (tag.runtimeClass.toString ->
       ((t: Any) => {
         t match {
           case system: T =>
             val r = up(IOBinders, site)(tag.runtimeClass.toString)(system)
             val h = fn(system)
-            (r._1 ++ h._1, r._2 ++ h._2)
+            val ports = r._1 ++ h._1
+            val cells = r._2 ++ h._2
+            (ports, cells)
           case _ => (Nil, Nil)
         }
       })
@@ -116,11 +122,6 @@ object BoreHelper {
 
 case object IOCellKey extends Field[IOCellTypeParams](GenericIOCellParams())
 
-class ClockedIO[T <: Data](gen: T) extends Bundle {
-  val clock = Output(Clock())
-  val bits = gen
-  override def cloneType: this.type = (new ClockedIO(DataMirror.internal.chiselTypeClone[T](gen))).asInstanceOf[this.type]
-}
 
 class WithGPIOCells extends OverrideIOBinder({
   (system: HasPeripheryGPIOModuleImp) => {
@@ -252,10 +253,7 @@ class WithDebugIOCells extends OverrideIOBinder({
 class WithSerialIOCells extends OverrideIOBinder({
   (system: CanHavePeripherySerial) => system.serial.map({ s =>
     val sys = system.asInstanceOf[BaseSubsystem]
-    val clocked_serial = Wire(new ClockedIO(DataMirror.internal.chiselTypeClone[SerialIO](s))).suggestName("serial_wire")
-    clocked_serial.clock := BoreHelper("serial_clock", sys.fbus.module.clock)
-    clocked_serial.bits <> s
-    val (port, cells) = IOCell.generateIOFromSignal(clocked_serial, Some("serial"), sys.p(IOCellKey))
+    val (port, cells) = IOCell.generateIOFromSignal(s.getWrappedValue, Some("serial"), sys.p(IOCellKey))
     port.suggestName("serial")
     (Seq(port), cells)
   }).getOrElse((Nil, Nil))
@@ -299,11 +297,10 @@ class WithL2FBusAXI4Punchthrough extends OverrideIOBinder({
 })
 
 class WithBlockDeviceIOPunchthrough extends OverrideIOBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp) => {
+  (system: CanHavePeripheryBlockDevice) => {
     val ports: Seq[ClockedIO[BlockDeviceIO]] = system.bdev.map({ bdev =>
-      val p = IO(new ClockedIO(new BlockDeviceIO()(system.p))).suggestName("blockdev")
-      p.clock := BoreHelper("blkdev_clk", system.outer.controller.get.module.clock)
-      p.bits <> bdev
+      val p = IO(new ClockedIO(new BlockDeviceIO()(GetSystemParameters(system)))).suggestName("blockdev")
+      p <> bdev
       p
     }).toSeq
     (ports, Nil)
@@ -311,11 +308,10 @@ class WithBlockDeviceIOPunchthrough extends OverrideIOBinder({
 })
 
 class WithNICIOPunchthrough extends OverrideIOBinder({
-  (system: CanHavePeripheryIceNICModuleImp) => {
-    val ports: Seq[ClockedIO[NICIOvonly]] = system.net.map({ n =>
+  (system: CanHavePeripheryIceNIC) => {
+    val ports: Seq[ClockedIO[NICIOvonly]] = system.icenicOpt.map({ n =>
       val p = IO(new ClockedIO(new NICIOvonly)).suggestName("nic")
-      p.clock := BoreHelper("nic_clk", system.outer.icenicOpt.get.module.clock)
-      p.bits <> n
+      p <> n
       p
     }).toSeq
     (ports, Nil)
diff --git a/generators/firechip/src/main/scala/BridgeBinders.scala b/generators/firechip/src/main/scala/BridgeBinders.scala
index 5dcddcfa..f42a1adf 100644
--- a/generators/firechip/src/main/scala/BridgeBinders.scala
+++ b/generators/firechip/src/main/scala/BridgeBinders.scala
@@ -14,7 +14,7 @@ import freechips.rocketchip.tile.{RocketTile}
 import sifive.blocks.devices.uart._
 
 import testchipip._
-import icenet.{CanHavePeripheryIceNICModuleImp, SimNetwork, NicLoopback, NICKey, NICIOvonly}
+import icenet.{CanHavePeripheryIceNIC, SimNetwork, NicLoopback, NICKey, NICIOvonly}
 
 import junctions.{NastiKey, NastiParameters}
 import midas.models.{FASEDBridge, AXI4EdgeSummary, CompleteConfig}
@@ -26,7 +26,7 @@ import ariane.ArianeTile
 
 import boom.common.{BoomTile}
 import barstools.iocell.chisel._
-import chipyard.iobinders.{ClockedIO, IOBinders, OverrideIOBinder, ComposeIOBinder, GetSystemParameters, IOCellKey}
+import chipyard.iobinders.{IOBinders, OverrideIOBinder, ComposeIOBinder, GetSystemParameters, IOCellKey}
 import chipyard.{HasHarnessSignalReferences}
 import chipyard.harness._
 
@@ -67,8 +67,9 @@ class WithSerialBridge extends OverrideHarnessBinder({
 })
 
 class WithNICBridge extends OverrideHarnessBinder({
-  (system: CanHavePeripheryIceNICModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
-    ports.map { p => withClockAndReset(p.clock, th.harnessReset) { NICBridge(p.clock, p.bits)(system.p) } }
+  (system: CanHavePeripheryIceNIC, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[NICIOvonly]]) => {
+    val p: Parameters = GetSystemParameters(system)
+    ports.map { n => withClockAndReset(n.clock, th.harnessReset) { NICBridge(n.clock, n.bits)(p) } }
     Nil
   }
 })
@@ -79,8 +80,9 @@ class WithUARTBridge extends OverrideHarnessBinder({
 })
 
 class WithBlockDeviceBridge extends OverrideHarnessBinder({
-  (system: CanHavePeripheryBlockDeviceModuleImp, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
-    ports.map { p => BlockDevBridge(p.clock, p.bits, th.harnessReset.toBool)(system.p) }
+  (system: CanHavePeripheryBlockDevice, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[BlockDeviceIO]]) => {
+    implicit val p: Parameters = GetSystemParameters(system)
+    ports.map { b => BlockDevBridge(b.clock, b.bits, th.harnessReset.toBool) }
     Nil
   }
 })
diff --git a/generators/firechip/src/main/scala/TargetConfigs.scala b/generators/firechip/src/main/scala/TargetConfigs.scala
index 0d8cd367..2dede960 100644
--- a/generators/firechip/src/main/scala/TargetConfigs.scala
+++ b/generators/firechip/src/main/scala/TargetConfigs.scala
@@ -13,7 +13,7 @@ import freechips.rocketchip.subsystem._
 import freechips.rocketchip.devices.tilelink.{BootROMLocated, BootROMParams}
 import freechips.rocketchip.devices.debug.{DebugModuleParams, DebugModuleKey}
 import freechips.rocketchip.diplomacy.LazyModule
-import testchipip.{BlockDeviceKey, BlockDeviceConfig, SerialKey, TracePortKey, TracePortParams}
+import testchipip.{BlockDeviceKey, BlockDeviceConfig, TracePortKey, TracePortParams}
 import sifive.blocks.devices.uart.{PeripheryUARTKey, UARTParams}
 import scala.math.{min, max}
 
diff --git a/generators/icenet b/generators/icenet
index bb23c81f..277a9080 160000
--- a/generators/icenet
+++ b/generators/icenet
@@ -1 +1 @@
-Subproject commit bb23c81fcfbdfde6767c5b7daa95f8f9436fb7db
+Subproject commit 277a9080fe92afb25d942f3207252e0ddea1b864
diff --git a/generators/testchipip b/generators/testchipip
index 6f4e7ae2..70df93e5 160000
--- a/generators/testchipip
+++ b/generators/testchipip
@@ -1 +1 @@
-Subproject commit 6f4e7ae2c94e867f08cd0d408a106fba2f389de2
+Subproject commit 70df93e5867e6676e62bb23b11b9086471fdecf7

From d2b42cee2ce7e5d22d3480ad7e6b9470a2507652 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Sat, 12 Sep 2020 23:31:54 -0700
Subject: [PATCH 62/74] Bump testchipip

---
 generators/testchipip | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/generators/testchipip b/generators/testchipip
index 70df93e5..a86c827c 160000
--- a/generators/testchipip
+++ b/generators/testchipip
@@ -1 +1 @@
-Subproject commit 70df93e5867e6676e62bb23b11b9086471fdecf7
+Subproject commit a86c827ca6e4e9d8320117ef1223da0ff752d064

From be0c04123245a722bc9bd18484fd2cb5d40c1c07 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Sun, 13 Sep 2020 06:36:37 +0000
Subject: [PATCH 63/74] Bump Firesim

---
 sims/firesim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/firesim b/sims/firesim
index 522125da..c1cd3e5e 160000
--- a/sims/firesim
+++ b/sims/firesim
@@ -1 +1 @@
-Subproject commit 522125da5d546544e3ce4492b6f105eeb93fcdf6
+Subproject commit c1cd3e5e7013b30f30508c7f47ff13180949eafe

From 6c5bce5430002e60a676a9089f7d5822523e13ff Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Sun, 13 Sep 2020 11:59:16 -0700
Subject: [PATCH 64/74] Support Tilelink over serial

---
 .circleci/config.yml                          | 19 ++++++++++
 .circleci/defaults.sh                         |  1 +
 .circleci/run-tests.sh                        |  3 ++
 .../chipyard/src/main/scala/DigitalTop.scala  |  3 +-
 .../src/main/scala/HarnessBinders.scala       | 36 ++++++++++++++++---
 .../chipyard/src/main/scala/IOBinders.scala   | 17 ++++++---
 .../chipyard/src/main/scala/Subsystem.scala   |  4 +--
 .../main/scala/config/AbstractConfig.scala    |  7 ++--
 .../src/main/scala/config/ArianeConfigs.scala |  2 +-
 .../src/main/scala/config/RocketConfigs.scala | 10 ++++--
 .../src/main/scala/BridgeBinders.scala        |  2 +-
 .../src/main/scala/TargetConfigs.scala        |  4 +--
 generators/testchipip                         |  2 +-
 sims/firesim                                  |  2 +-
 14 files changed, 90 insertions(+), 22 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index eac8504a..1b239354 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -204,6 +204,11 @@ jobs:
         steps:
             - prepare-rtl:
                 project-key: "chipyard-dmirocket"
+    prepare-chipyard-tlserial:
+        executor: main-env
+        steps:
+            - prepare-rtl:
+                project-key: "chipyard-tlserial"
     prepare-chipyard-sha3:
         executor: main-env
         steps:
@@ -302,6 +307,11 @@ jobs:
         steps:
             - run-tests:
                 project-key: "chipyard-dmirocket"
+    chipyard-tlserial-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                project-key: "chipyard-tlserial"
     chipyard-sha3-run-tests:
         executor: main-env
         steps:
@@ -456,6 +466,11 @@ workflows:
                     - install-riscv-toolchain
                     - install-verilator
 
+            - prepare-chipyard-tlserial:
+                requires:
+                    - install-riscv-toolchain
+                    - install-verilator
+
             - prepare-chipyard-sha3:
                 requires:
                     - install-riscv-toolchain
@@ -551,6 +566,10 @@ workflows:
                 requires:
                     - prepare-chipyard-dmirocket
 
+            - chipyard-tlserial-run-tests:
+                requires:
+                    - prepare-chipyard-tlserial
+
             - chipyard-sha3-run-tests:
                 requires:
                     - prepare-chipyard-sha3
diff --git a/.circleci/defaults.sh b/.circleci/defaults.sh
index 703737cd..2a744e89 100755
--- a/.circleci/defaults.sh
+++ b/.circleci/defaults.sh
@@ -49,6 +49,7 @@ LOCAL_FIRESIM_DIR=$LOCAL_CHIPYARD_DIR/sims/firesim/sim
 declare -A mapping
 mapping["chipyard-rocket"]=""
 mapping["chipyard-dmirocket"]=" CONFIG=dmiRocketConfig"
+mapping["chipyard-tlserial"]=" CONFIG=TLSerialRocketConfig"
 mapping["chipyard-sha3"]=" CONFIG=Sha3RocketConfig"
 mapping["chipyard-streaming-fir"]=" CONFIG=StreamingFIRRocketConfig"
 mapping["chipyard-streaming-passthrough"]=" CONFIG=StreamingPassthroughRocketConfig"
diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh
index 3e7b0285..cee04dd3 100755
--- a/.circleci/run-tests.sh
+++ b/.circleci/run-tests.sh
@@ -35,6 +35,9 @@ case $1 in
     chipyard-dmirocket)
         run_bmark ${mapping[$1]}
         ;;
+    chipyard-tlserial)
+        run_bmark ${mapping[$1]}
+        ;;
     chipyard-boom)
         run_bmark ${mapping[$1]}
         ;;
diff --git a/generators/chipyard/src/main/scala/DigitalTop.scala b/generators/chipyard/src/main/scala/DigitalTop.scala
index a065b6be..f967b8fc 100644
--- a/generators/chipyard/src/main/scala/DigitalTop.scala
+++ b/generators/chipyard/src/main/scala/DigitalTop.scala
@@ -16,7 +16,8 @@ class DigitalTop(implicit p: Parameters) extends ChipyardSystem
   with testchipip.CanHaveTraceIO // Enables optionally adding trace IO
   with testchipip.CanHaveBackingScratchpad // Enables optionally adding a backing scratchpad
   with testchipip.CanHavePeripheryBlockDevice // Enables optionally adding the block device
-  with testchipip.CanHavePeripherySerial // Enables optionally adding the TSI serial-adapter and port
+  with testchipip.CanHavePeripheryTLSerial // Enables optionally adding the tilelink-over-serial backing memory
+  with testchipip.CanHavePeripheryTSISerial // Enables optionally adding the TSI serial-adapter and port
   with sifive.blocks.devices.uart.HasPeripheryUART // Enables optionally adding the sifive UART
   with sifive.blocks.devices.gpio.HasPeripheryGPIO // Enables optionally adding the sifive GPIOs
   with sifive.blocks.devices.spi.HasPeripherySPIFlash // Enables optionally adding the sifive SPI flash controller
diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala
index 06a3af5d..c1b57f72 100644
--- a/generators/chipyard/src/main/scala/HarnessBinders.scala
+++ b/generators/chipyard/src/main/scala/HarnessBinders.scala
@@ -223,15 +223,15 @@ class WithTiedOffDebug extends OverrideHarnessBinder({
 })
 
 
-class WithTiedOffSerial extends OverrideHarnessBinder({
-  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+class WithTiedOffTSISerial extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTSISerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
     ports.map { p => SerialAdapter.tieoff(Some(p.bits)) }
     Nil
   }
 })
 
-class WithSimSerial extends OverrideHarnessBinder({
-  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+class WithSimTSISerial extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTSISerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
     ports.map { p =>
       val ser_success = SerialAdapter.connectSimSerial(p.bits, p.clock, th.harnessReset)
       when (ser_success) { th.success := true.B }
@@ -240,6 +240,34 @@ class WithSimSerial extends OverrideHarnessBinder({
   }
 })
 
+class WithSimTLSerial(withHarnessSerialAdapter: Boolean = false) extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTLSerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+    implicit val p = chipyard.iobinders.GetSystemParameters(system)
+    ports.map({ port =>
+      withClockAndReset(port.clock, th.harnessReset) {
+        val lRam = LazyModule(new SerialRAM(
+          p(SerialTLKey).get.width,
+          p(SerialTLKey).get.memParams.master.size,
+          p(SerialTLKey).get.memParams.master.base,
+          managerEdge = system.serdesser.get.managerNode.edges.in(0),
+          clientEdge = system.serdesser.get.clientNode.edges.out(0)
+        ))
+        val ram = Module(lRam.module)
+        ram.io.ser <> port.bits
+
+        require(lRam.serdesser.module.mergedParams == system.serdesser.get.module.mergedParams,
+          "Mismatch between chip-side diplomatic params and testram diplomatic params")
+        if (withHarnessSerialAdapter) {
+          val success = SerialAdapter.connectSimSerial(Some(ram.io.tsi_ser), port.clock, th.harnessReset.asBool)
+          when (success) { th.success := true.B }
+        } else {
+          SerialAdapter.tieoff(Some(ram.io.tsi_ser))
+        }
+      }
+    })
+  }
+})
+
 class WithTraceGenSuccess extends OverrideHarnessBinder({
   (system: TraceGenSystemModuleImp, th: HasHarnessSignalReferences, ports: Seq[Bool]) => {
     ports.map { p => when (p) { th.success := true.B } }
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 78233118..03138cc1 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -250,11 +250,20 @@ class WithDebugIOCells extends OverrideIOBinder({
   }
 })
 
-class WithSerialIOCells extends OverrideIOBinder({
-  (system: CanHavePeripherySerial) => system.serial.map({ s =>
+class WithSerialTSIIOCells extends OverrideIOBinder({
+  (system: CanHavePeripheryTSISerial) => system.serial_tsi.map({ s =>
     val sys = system.asInstanceOf[BaseSubsystem]
-    val (port, cells) = IOCell.generateIOFromSignal(s.getWrappedValue, Some("serial"), sys.p(IOCellKey))
-    port.suggestName("serial")
+    val (port, cells) = IOCell.generateIOFromSignal(s.getWrappedValue, Some("serial_tsi"), sys.p(IOCellKey))
+    port.suggestName("serial_tsi")
+    (Seq(port), cells)
+  }).getOrElse((Nil, Nil))
+})
+
+class WithSerialTLIOCells extends OverrideIOBinder({
+  (system: CanHavePeripheryTLSerial) => system.serial_tl.map({ s =>
+    val sys = system.asInstanceOf[BaseSubsystem]
+    val (port, cells) = IOCell.generateIOFromSignal(s.getWrappedValue, Some("serial_tl"), sys.p(IOCellKey))
+    port.suggestName("serial_tl")
     (Seq(port), cells)
   }).getOrElse((Nil, Nil))
 })
diff --git a/generators/chipyard/src/main/scala/Subsystem.scala b/generators/chipyard/src/main/scala/Subsystem.scala
index 7f089ce1..35c41e0c 100644
--- a/generators/chipyard/src/main/scala/Subsystem.scala
+++ b/generators/chipyard/src/main/scala/Subsystem.scala
@@ -25,12 +25,12 @@ import freechips.rocketchip.amba.axi4._
 import boom.common.{BoomTile}
 
 
-import testchipip.{DromajoHelper, CanHavePeripherySerial, SerialKey}
+import testchipip.{DromajoHelper, CanHavePeripheryTSISerial, SerialTSIKey}
 
 trait CanHaveHTIF { this: BaseSubsystem =>
   // Advertise HTIF if system can communicate with fesvr
   if (this match {
-    case _: CanHavePeripherySerial if p(SerialKey) => true
+    case _: CanHavePeripheryTSISerial if p(SerialTSIKey).nonEmpty => true
     case _: HasPeripheryDebug if p(ExportDebug).protocols.nonEmpty => true
     case _ => false
   }) {
diff --git a/generators/chipyard/src/main/scala/config/AbstractConfig.scala b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
index 950cb4b4..aed55d02 100644
--- a/generators/chipyard/src/main/scala/config/AbstractConfig.scala
+++ b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
@@ -15,7 +15,7 @@ class AbstractConfig extends Config(
   new chipyard.harness.WithUARTAdapter ++                       // add UART adapter to display UART on stdout, if uart is present
   new chipyard.harness.WithBlackBoxSimMem ++                    // add SimDRAM DRAM model for axi4 backing memory, if axi4 mem is enabled
   new chipyard.harness.WithSimDebug ++                          // add SimJTAG or SimDTM adapters if debug module is enabled
-  new chipyard.harness.WithSimSerial ++                         // add SimSerial adapter for HTIF, if serial port is present
+  new chipyard.harness.WithSimTSISerial ++                      // add SimSerial adapter for HTIF, if serial port is present
   new chipyard.harness.WithGPIOTiedOff ++                       // tie-off chiptop GPIOs, if GPIOs are present
   new chipyard.harness.WithSimSPIFlashModel ++                  // add simulated SPI flash memory, if SPI is enabled
   new chipyard.harness.WithSimAXIMMIO ++                        // add SimAXIMem for axi4 mmio port, if enabled
@@ -29,7 +29,8 @@ class AbstractConfig extends Config(
   new chipyard.iobinders.WithL2FBusAXI4Punchthrough ++
   new chipyard.iobinders.WithBlockDeviceIOPunchthrough ++
   new chipyard.iobinders.WithNICIOPunchthrough ++
-  new chipyard.iobinders.WithSerialIOCells ++
+  new chipyard.iobinders.WithSerialTSIIOCells ++
+  new chipyard.iobinders.WithSerialTLIOCells ++
   new chipyard.iobinders.WithDebugIOCells ++
   new chipyard.iobinders.WithUARTIOCells ++
   new chipyard.iobinders.WithGPIOCells ++
@@ -39,7 +40,7 @@ class AbstractConfig extends Config(
   new chipyard.iobinders.WithExtInterruptIOCells ++
 
 
-  new testchipip.WithTSI ++                                      // use testchipip serial offchip link
+  new testchipip.WithSerialTSI ++                                // use testchipip serial offchip link
   new chipyard.config.WithBootROM ++                             // use default bootrom
   new chipyard.config.WithUART ++                                // add a UART
   new chipyard.config.WithL2TLBs(1024) ++                        // use L2 TLBs
diff --git a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
index 9f701331..b7c347dc 100644
--- a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
@@ -13,7 +13,7 @@ class ArianeConfig extends Config(
   new chipyard.config.AbstractConfig)
 
 class dmiArianeConfig extends Config(
-  new chipyard.harness.WithTiedOffSerial ++            // Tie off the serial port, override default instantiation of SimSerial
+  new chipyard.harness.WithTiedOffTSISerial ++         // Tie off the serial port, override default instantiation of SimSerial
   new chipyard.config.WithDMIDTM ++                    // have debug module expose a clocked DMI port
   new ariane.WithNArianeCores(1) ++                    // single Ariane core
   new chipyard.config.AbstractConfig)
diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
index 16d298fb..9f2a97de 100644
--- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
@@ -25,7 +25,7 @@ class GemminiRocketConfig extends Config(
 
 // DOC include start: DmiRocket
 class dmiRocketConfig extends Config(
-  new chipyard.harness.WithTiedOffSerial ++                      // don't use serial to drive the chip, since we use DMI instead
+  new chipyard.harness.WithTiedOffTSISerial ++                      // don't use serial to drive the chip, since we use DMI instead
   new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
@@ -182,4 +182,10 @@ class DividedClockRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
-
+class TLSerialRocketConfig extends Config(
+  new chipyard.harness.WithSimTLSerial(withHarnessSerialAdapter = true) ++ // add external TL backing memory, and external serial adapter
+  new testchipip.WithDefaultSerialTL ++                                    // support tilelink-over-serial
+  new testchipip.WithNoSerialTSI ++                                        // remove internal serial adapter
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new freechips.rocketchip.subsystem.WithNoMemPort ++                      // remove AXI4 backing memory
+  new chipyard.config.AbstractConfig)
diff --git a/generators/firechip/src/main/scala/BridgeBinders.scala b/generators/firechip/src/main/scala/BridgeBinders.scala
index f42a1adf..ebd629c7 100644
--- a/generators/firechip/src/main/scala/BridgeBinders.scala
+++ b/generators/firechip/src/main/scala/BridgeBinders.scala
@@ -56,7 +56,7 @@ class WithFireSimIOCellModels extends Config((site, here, up) => {
 })
 
 class WithSerialBridge extends OverrideHarnessBinder({
-  (system: CanHavePeripherySerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+  (system: CanHavePeripheryTSISerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
     ports.map { p =>
       withClockAndReset(p.clock, th.harnessReset) {
         SerialBridge(p.clock, p.bits, MainMemoryConsts.globalName)(GetSystemParameters(system))
diff --git a/generators/firechip/src/main/scala/TargetConfigs.scala b/generators/firechip/src/main/scala/TargetConfigs.scala
index 2dede960..7df0bf61 100644
--- a/generators/firechip/src/main/scala/TargetConfigs.scala
+++ b/generators/firechip/src/main/scala/TargetConfigs.scala
@@ -84,7 +84,7 @@ class WithFireSimConfigTweaks extends Config(
   // Required: Adds IO to attach SerialBridge. The SerialBridges is responsible
   // for signalling simulation termination under simulation success. This fragment can
   // be removed if you supply an auxiliary bridge that signals simulation termination
-  new testchipip.WithTSI ++
+  new testchipip.WithSerialTSI ++
   // Optional: Removing this will require using an initramfs under linux
   new testchipip.WithBlockDevice ++
   // Required*: Scale default baud rate with periphery bus frequency
@@ -131,7 +131,7 @@ class FireSimSmallSystemConfig extends Config(
   new WithoutClockGating ++
   new WithoutTLMonitors ++
   new freechips.rocketchip.subsystem.WithExtMemSize(1 << 28) ++
-  new testchipip.WithTSI ++
+  new testchipip.WithSerialTSI ++
   new testchipip.WithBlockDevice ++
   new chipyard.config.WithUART ++
   new freechips.rocketchip.subsystem.WithInclusiveCache(nWays = 2, capacityKB = 64) ++
diff --git a/generators/testchipip b/generators/testchipip
index a86c827c..6f815737 160000
--- a/generators/testchipip
+++ b/generators/testchipip
@@ -1 +1 @@
-Subproject commit a86c827ca6e4e9d8320117ef1223da0ff752d064
+Subproject commit 6f8157375451fbb32e9dae291f2c65878a2b1dcd
diff --git a/sims/firesim b/sims/firesim
index c1cd3e5e..2b52057e 160000
--- a/sims/firesim
+++ b/sims/firesim
@@ -1 +1 @@
-Subproject commit c1cd3e5e7013b30f30508c7f47ff13180949eafe
+Subproject commit 2b52057e158fd91d44c6259aa08869622a88040a

From 10625a3a6cc72a304a8608134b9d4876b9386e6a Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 14 Sep 2020 13:27:31 -0700
Subject: [PATCH 65/74] Undo regression in iocells flexibility

---
 .../src/main/scala/BridgeBinders.scala         | 18 ++++++++++++++----
 tools/barstools                                |  2 +-
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/generators/firechip/src/main/scala/BridgeBinders.scala b/generators/firechip/src/main/scala/BridgeBinders.scala
index f42a1adf..8a4d0a69 100644
--- a/generators/firechip/src/main/scala/BridgeBinders.scala
+++ b/generators/firechip/src/main/scala/BridgeBinders.scala
@@ -39,10 +39,20 @@ trait Unsupported {
   require(false, "We do not support this IOCell type")
 }
 
-class FireSimAnalogIOCell extends RawModule with AnalogIOCell with Unsupported
-class FireSimDigitalGPIOCell extends RawModule with DigitalGPIOCell with Unsupported
-class FireSimDigitalInIOCell extends RawModule with DigitalInIOCell { io.i := io.pad }
-class FireSimDigitalOutIOCell extends RawModule with DigitalOutIOCell { io.pad := io.o }
+class FireSimAnalogIOCell extends RawModule with AnalogIOCell with Unsupported {
+  val io = IO(new AnalogIOCellBundle)
+}
+class FireSimDigitalGPIOCell extends RawModule with DigitalGPIOCell with Unsupported {
+  val io = IO(new DigitalGPIOCellBundle)
+}
+class FireSimDigitalInIOCell extends RawModule with DigitalInIOCell {
+  val io = IO(new DigitalInIOCellBundle)
+  io.i := io.pad
+}
+class FireSimDigitalOutIOCell extends RawModule with DigitalOutIOCell {
+  val io = IO(new DigitalOutIOCellBundle)
+  io.pad := io.o
+}
 
 case class FireSimIOCellParams() extends IOCellTypeParams {
   def analog() = Module(new FireSimAnalogIOCell)
diff --git a/tools/barstools b/tools/barstools
index ba681676..31590a79 160000
--- a/tools/barstools
+++ b/tools/barstools
@@ -1 +1 @@
-Subproject commit ba681676f338af158023c99b4c802009aa0b601b
+Subproject commit 31590a7948db47fd16beed266c4833579acc305b

From 0d8e87126cb4070044a3a86da7fe7bdac125a4fc Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 14 Sep 2020 19:41:02 -0700
Subject: [PATCH 66/74] Deprecate support for on-chip SerialAdapter

---
 .../chipyard/src/main/scala/DigitalTop.scala  |  1 -
 .../src/main/scala/HarnessBinders.scala       | 50 +++++--------------
 .../chipyard/src/main/scala/IOBinders.scala   |  9 ----
 .../chipyard/src/main/scala/Subsystem.scala   |  4 +-
 .../main/scala/config/AbstractConfig.scala    |  6 +--
 .../src/main/scala/config/ArianeConfigs.scala |  2 +-
 .../src/main/scala/config/RocketConfigs.scala | 10 ++--
 .../src/main/scala/BridgeBinders.scala        |  5 +-
 .../src/main/scala/TargetConfigs.scala        |  4 +-
 generators/testchipip                         |  2 +-
 tools/barstools                               |  2 +-
 11 files changed, 29 insertions(+), 66 deletions(-)

diff --git a/generators/chipyard/src/main/scala/DigitalTop.scala b/generators/chipyard/src/main/scala/DigitalTop.scala
index f967b8fc..a14a3fbe 100644
--- a/generators/chipyard/src/main/scala/DigitalTop.scala
+++ b/generators/chipyard/src/main/scala/DigitalTop.scala
@@ -17,7 +17,6 @@ class DigitalTop(implicit p: Parameters) extends ChipyardSystem
   with testchipip.CanHaveBackingScratchpad // Enables optionally adding a backing scratchpad
   with testchipip.CanHavePeripheryBlockDevice // Enables optionally adding the block device
   with testchipip.CanHavePeripheryTLSerial // Enables optionally adding the tilelink-over-serial backing memory
-  with testchipip.CanHavePeripheryTSISerial // Enables optionally adding the TSI serial-adapter and port
   with sifive.blocks.devices.uart.HasPeripheryUART // Enables optionally adding the sifive UART
   with sifive.blocks.devices.gpio.HasPeripheryGPIO // Enables optionally adding the sifive GPIOs
   with sifive.blocks.devices.spi.HasPeripherySPIFlash // Enables optionally adding the sifive SPI flash controller
diff --git a/generators/chipyard/src/main/scala/HarnessBinders.scala b/generators/chipyard/src/main/scala/HarnessBinders.scala
index c1b57f72..e5cfacfb 100644
--- a/generators/chipyard/src/main/scala/HarnessBinders.scala
+++ b/generators/chipyard/src/main/scala/HarnessBinders.scala
@@ -223,47 +223,23 @@ class WithTiedOffDebug extends OverrideHarnessBinder({
 })
 
 
-class WithTiedOffTSISerial extends OverrideHarnessBinder({
-  (system: CanHavePeripheryTSISerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
-    ports.map { p => SerialAdapter.tieoff(Some(p.bits)) }
-    Nil
-  }
-})
-
-class WithSimTSISerial extends OverrideHarnessBinder({
-  (system: CanHavePeripheryTSISerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
-    ports.map { p =>
-      val ser_success = SerialAdapter.connectSimSerial(p.bits, p.clock, th.harnessReset)
-      when (ser_success) { th.success := true.B }
-    }
-    Nil
-  }
-})
-
-class WithSimTLSerial(withHarnessSerialAdapter: Boolean = false) extends OverrideHarnessBinder({
+class WithSerialAdapterTiedOff extends OverrideHarnessBinder({
   (system: CanHavePeripheryTLSerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
     implicit val p = chipyard.iobinders.GetSystemParameters(system)
     ports.map({ port =>
-      withClockAndReset(port.clock, th.harnessReset) {
-        val lRam = LazyModule(new SerialRAM(
-          p(SerialTLKey).get.width,
-          p(SerialTLKey).get.memParams.master.size,
-          p(SerialTLKey).get.memParams.master.base,
-          managerEdge = system.serdesser.get.managerNode.edges.in(0),
-          clientEdge = system.serdesser.get.clientNode.edges.out(0)
-        ))
-        val ram = Module(lRam.module)
-        ram.io.ser <> port.bits
+      val ram = SerialAdapter.connectHarnessRAM(system.serdesser.get, port, th.harnessReset)
+      SerialAdapter.tieoff(ram.module.io.tsi_ser)
+    })
+  }
+})
 
-        require(lRam.serdesser.module.mergedParams == system.serdesser.get.module.mergedParams,
-          "Mismatch between chip-side diplomatic params and testram diplomatic params")
-        if (withHarnessSerialAdapter) {
-          val success = SerialAdapter.connectSimSerial(Some(ram.io.tsi_ser), port.clock, th.harnessReset.asBool)
-          when (success) { th.success := true.B }
-        } else {
-          SerialAdapter.tieoff(Some(ram.io.tsi_ser))
-        }
-      }
+class WithSimSerial extends OverrideHarnessBinder({
+  (system: CanHavePeripheryTLSerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+    implicit val p = chipyard.iobinders.GetSystemParameters(system)
+    ports.map({ port =>
+      val ram = SerialAdapter.connectHarnessRAM(system.serdesser.get, port, th.harnessReset)
+      val success = SerialAdapter.connectSimSerial(ram.module.io.tsi_ser, port.clock, th.harnessReset.asBool)
+      when (success) { th.success := true.B }
     })
   }
 })
diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala
index 03138cc1..73a8acfd 100644
--- a/generators/chipyard/src/main/scala/IOBinders.scala
+++ b/generators/chipyard/src/main/scala/IOBinders.scala
@@ -250,15 +250,6 @@ class WithDebugIOCells extends OverrideIOBinder({
   }
 })
 
-class WithSerialTSIIOCells extends OverrideIOBinder({
-  (system: CanHavePeripheryTSISerial) => system.serial_tsi.map({ s =>
-    val sys = system.asInstanceOf[BaseSubsystem]
-    val (port, cells) = IOCell.generateIOFromSignal(s.getWrappedValue, Some("serial_tsi"), sys.p(IOCellKey))
-    port.suggestName("serial_tsi")
-    (Seq(port), cells)
-  }).getOrElse((Nil, Nil))
-})
-
 class WithSerialTLIOCells extends OverrideIOBinder({
   (system: CanHavePeripheryTLSerial) => system.serial_tl.map({ s =>
     val sys = system.asInstanceOf[BaseSubsystem]
diff --git a/generators/chipyard/src/main/scala/Subsystem.scala b/generators/chipyard/src/main/scala/Subsystem.scala
index c3ca93b7..172be692 100644
--- a/generators/chipyard/src/main/scala/Subsystem.scala
+++ b/generators/chipyard/src/main/scala/Subsystem.scala
@@ -25,12 +25,12 @@ import freechips.rocketchip.amba.axi4._
 import boom.common.{BoomTile}
 
 
-import testchipip.{DromajoHelper, CanHavePeripheryTSISerial, SerialTSIKey}
+import testchipip.{DromajoHelper, CanHavePeripheryTLSerial, SerialTLKey}
 
 trait CanHaveHTIF { this: BaseSubsystem =>
   // Advertise HTIF if system can communicate with fesvr
   if (this match {
-    case _: CanHavePeripheryTSISerial if p(SerialTSIKey).nonEmpty => true
+    case _: CanHavePeripheryTKSerial if p(SerialTLKey).nonEmpty => true
     case _: HasPeripheryDebug if p(ExportDebug).dmi => true
     case _ => false
   }) {
diff --git a/generators/chipyard/src/main/scala/config/AbstractConfig.scala b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
index aed55d02..5b356c74 100644
--- a/generators/chipyard/src/main/scala/config/AbstractConfig.scala
+++ b/generators/chipyard/src/main/scala/config/AbstractConfig.scala
@@ -14,8 +14,8 @@ class AbstractConfig extends Config(
   // The HarnessBinders control generation of hardware in the TestHarness
   new chipyard.harness.WithUARTAdapter ++                       // add UART adapter to display UART on stdout, if uart is present
   new chipyard.harness.WithBlackBoxSimMem ++                    // add SimDRAM DRAM model for axi4 backing memory, if axi4 mem is enabled
+  new chipyard.harness.WithSimSerial ++                         // add external serial-adapter and RAM
   new chipyard.harness.WithSimDebug ++                          // add SimJTAG or SimDTM adapters if debug module is enabled
-  new chipyard.harness.WithSimTSISerial ++                      // add SimSerial adapter for HTIF, if serial port is present
   new chipyard.harness.WithGPIOTiedOff ++                       // tie-off chiptop GPIOs, if GPIOs are present
   new chipyard.harness.WithSimSPIFlashModel ++                  // add simulated SPI flash memory, if SPI is enabled
   new chipyard.harness.WithSimAXIMMIO ++                        // add SimAXIMem for axi4 mmio port, if enabled
@@ -29,7 +29,6 @@ class AbstractConfig extends Config(
   new chipyard.iobinders.WithL2FBusAXI4Punchthrough ++
   new chipyard.iobinders.WithBlockDeviceIOPunchthrough ++
   new chipyard.iobinders.WithNICIOPunchthrough ++
-  new chipyard.iobinders.WithSerialTSIIOCells ++
   new chipyard.iobinders.WithSerialTLIOCells ++
   new chipyard.iobinders.WithDebugIOCells ++
   new chipyard.iobinders.WithUARTIOCells ++
@@ -39,8 +38,7 @@ class AbstractConfig extends Config(
   new chipyard.iobinders.WithTraceIOPunchthrough ++
   new chipyard.iobinders.WithExtInterruptIOCells ++
 
-
-  new testchipip.WithSerialTSI ++                                // use testchipip serial offchip link
+  new testchipip.WithDefaultSerialTL ++                          // use serialized tilelink port to external serialadapter/harnessRAM
   new chipyard.config.WithBootROM ++                             // use default bootrom
   new chipyard.config.WithUART ++                                // add a UART
   new chipyard.config.WithL2TLBs(1024) ++                        // use L2 TLBs
diff --git a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
index b7c347dc..6e75ac54 100644
--- a/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/ArianeConfigs.scala
@@ -13,7 +13,7 @@ class ArianeConfig extends Config(
   new chipyard.config.AbstractConfig)
 
 class dmiArianeConfig extends Config(
-  new chipyard.harness.WithTiedOffTSISerial ++         // Tie off the serial port, override default instantiation of SimSerial
+  new chipyard.harness.WithSerialAdapterTiedOff ++     // Tie off the serial port, override default instantiation of SimSerial
   new chipyard.config.WithDMIDTM ++                    // have debug module expose a clocked DMI port
   new ariane.WithNArianeCores(1) ++                    // single Ariane core
   new chipyard.config.AbstractConfig)
diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
index 9f2a97de..42b5f3b0 100644
--- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala
+++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala
@@ -25,7 +25,7 @@ class GemminiRocketConfig extends Config(
 
 // DOC include start: DmiRocket
 class dmiRocketConfig extends Config(
-  new chipyard.harness.WithTiedOffTSISerial ++                      // don't use serial to drive the chip, since we use DMI instead
+  new chipyard.harness.WithSerialAdapterTiedOff ++               // don't attach an external SimSerial
   new chipyard.config.WithDMIDTM ++                              // have debug module expose a clocked DMI port
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
@@ -182,10 +182,8 @@ class DividedClockRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
 
-class TLSerialRocketConfig extends Config(
-  new chipyard.harness.WithSimTLSerial(withHarnessSerialAdapter = true) ++ // add external TL backing memory, and external serial adapter
-  new testchipip.WithDefaultSerialTL ++                                    // support tilelink-over-serial
-  new testchipip.WithNoSerialTSI ++                                        // remove internal serial adapter
+class LBWIFMemoryRocketConfig extends Config(
+  new testchipip.WithSerialTLMem(isMainMemory=true) ++      // set lbwif memory base to DRAM_BASE, use as main memory
+  new freechips.rocketchip.subsystem.WithNoMemPort ++       // remove AXI4 backing memory
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
-  new freechips.rocketchip.subsystem.WithNoMemPort ++                      // remove AXI4 backing memory
   new chipyard.config.AbstractConfig)
diff --git a/generators/firechip/src/main/scala/BridgeBinders.scala b/generators/firechip/src/main/scala/BridgeBinders.scala
index 98f672a9..4943e130 100644
--- a/generators/firechip/src/main/scala/BridgeBinders.scala
+++ b/generators/firechip/src/main/scala/BridgeBinders.scala
@@ -66,10 +66,11 @@ class WithFireSimIOCellModels extends Config((site, here, up) => {
 })
 
 class WithSerialBridge extends OverrideHarnessBinder({
-  (system: CanHavePeripheryTSISerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
+  (system: CanHavePeripheryTLSerial, th: HasHarnessSignalReferences, ports: Seq[ClockedIO[SerialIO]]) => {
     ports.map { p =>
+      val ram = SerialAdapter.connectHarnessRAM(system.serdesser.get, p, th.harnessReset)
       withClockAndReset(p.clock, th.harnessReset) {
-        SerialBridge(p.clock, p.bits, MainMemoryConsts.globalName)(GetSystemParameters(system))
+        SerialBridge(p.clock, ram.module.io.tsi_ser, MainMemoryConsts.globalName)(GetSystemParameters(system))
       }
     }
     Nil
diff --git a/generators/firechip/src/main/scala/TargetConfigs.scala b/generators/firechip/src/main/scala/TargetConfigs.scala
index 7df0bf61..678afccf 100644
--- a/generators/firechip/src/main/scala/TargetConfigs.scala
+++ b/generators/firechip/src/main/scala/TargetConfigs.scala
@@ -84,7 +84,7 @@ class WithFireSimConfigTweaks extends Config(
   // Required: Adds IO to attach SerialBridge. The SerialBridges is responsible
   // for signalling simulation termination under simulation success. This fragment can
   // be removed if you supply an auxiliary bridge that signals simulation termination
-  new testchipip.WithSerialTSI ++
+  new testchipip.WithDefaultSerialTL ++
   // Optional: Removing this will require using an initramfs under linux
   new testchipip.WithBlockDevice ++
   // Required*: Scale default baud rate with periphery bus frequency
@@ -131,7 +131,7 @@ class FireSimSmallSystemConfig extends Config(
   new WithoutClockGating ++
   new WithoutTLMonitors ++
   new freechips.rocketchip.subsystem.WithExtMemSize(1 << 28) ++
-  new testchipip.WithSerialTSI ++
+  new testchipip.WithDefaultSerialTL ++
   new testchipip.WithBlockDevice ++
   new chipyard.config.WithUART ++
   new freechips.rocketchip.subsystem.WithInclusiveCache(nWays = 2, capacityKB = 64) ++
diff --git a/generators/testchipip b/generators/testchipip
index 6f815737..e845cb3f 160000
--- a/generators/testchipip
+++ b/generators/testchipip
@@ -1 +1 @@
-Subproject commit 6f8157375451fbb32e9dae291f2c65878a2b1dcd
+Subproject commit e845cb3f50bb141fec7e8291a40d0b46d5ef5c12
diff --git a/tools/barstools b/tools/barstools
index 31590a79..847f72ec 160000
--- a/tools/barstools
+++ b/tools/barstools
@@ -1 +1 @@
-Subproject commit 31590a7948db47fd16beed266c4833579acc305b
+Subproject commit 847f72eca0fa3207ab7140c07e980ac9f8cf1251

From aa8b7c15ecfed723d8f01748ebb3af6e64516cde Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Mon, 14 Sep 2020 23:04:58 -0700
Subject: [PATCH 67/74] Reduce CI redundancy by grouping builds

---
 .circleci/config.yml      | 386 ++++++++++++--------------------------
 .circleci/defaults.sh     |   9 +
 .circleci/do-rtl-build.sh |  19 +-
 3 files changed, 142 insertions(+), 272 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index eac8504a..6ed04488 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -99,11 +99,10 @@ commands:
             tools-version:
                 type: string
                 default: "riscv-tools"
+            group-key:
+                type: string
             project-key:
                 type: string
-            extra-cache-restore:
-                type: string
-                default: ""
             run-script:
                 type: string
                 default: "run-tests.sh"
@@ -115,13 +114,7 @@ commands:
                 tools-version: "<< parameters.tools-version >>"
             - restore_cache:
                 keys:
-                    - << parameters.project-key >>-{{ .Branch }}-{{ .Revision }}
-            - when:
-                condition: << parameters.extra-cache-restore >>
-                steps:
-                    - restore_cache:
-                        keys:
-                            - << parameters.extra-cache-restore >>-{{ .Branch }}-{{ .Revision }}
+                    - << parameters.group-key >>-{{ .Branch }}-{{ .Revision }}
             - run:
                 name: Run << parameters.project-key >> subproject tests
                 command: .circleci/<< parameters.run-script >> << parameters.project-key >>
@@ -194,177 +187,147 @@ jobs:
                 key: extra-tests-{{ .Branch }}-{{ .Revision }}
                 paths:
                     - "/home/riscvuser/project/tests"
-    prepare-chipyard-rocket:
+
+    prepare-chipyard-cores:
         executor: main-env
         steps:
             - prepare-rtl:
-                project-key: "chipyard-rocket"
-    prepare-chipyard-dmirocket:
+                project-key: "group-cores"
+    prepare-chipyard-peripherals:
         executor: main-env
         steps:
             - prepare-rtl:
-                project-key: "chipyard-dmirocket"
-    prepare-chipyard-sha3:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-sha3"
-    prepare-chipyard-streaming-fir:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-streaming-fir"
-    prepare-chipyard-streaming-passthrough:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-streaming-passthrough"
-    prepare-chipyard-hetero:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-hetero"
-                timeout: "240m"
-    prepare-chipyard-boom:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-boom"
-    prepare-chipyard-blkdev:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-blkdev"
-    prepare-chipyard-hwacha:
+                project-key: "group-peripherals"
+    prepare-chipyard-accels:
         executor: main-env
         steps:
             - prepare-rtl:
                 tools-version: "esp-tools"
-                project-key: "chipyard-hwacha"
-    prepare-chipyard-gemmini:
+                project-key: "group-accels"
+    prepare-chipyard-tracegen:
         executor: main-env
         steps:
             - prepare-rtl:
-                tools-version: "esp-tools"
-                project-key: "chipyard-gemmini"
-    prepare-tracegen:
+                project-key: "group-tracegen"
+    prepare-chipyard-other:
         executor: main-env
         steps:
             - prepare-rtl:
-                project-key: "tracegen"
-    prepare-tracegen-boom:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "tracegen-boom"
-    prepare-chipyard-ariane:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-ariane"
-    prepare-icenet:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "icenet"
-    prepare-testchipip:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "testchipip"
-    prepare-chipyard-nvdla:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-nvdla"
-    prepare-chipyard-spiflashwrite:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-spiflashwrite"
-    prepare-chipyard-spiflashread:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-spiflashread"
-    prepare-chipyard-mmios:
-        executor: main-env
-        steps:
-            - prepare-rtl:
-                project-key: "chipyard-mmios"
+                project-key: "group-other"
+
     chipyard-rocket-run-tests:
         executor: main-env
         steps:
             - run-tests:
+                group-key: "group-cores"
                 project-key: "chipyard-rocket"
-    chipyard-dmirocket-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "chipyard-dmirocket"
-    chipyard-sha3-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "chipyard-sha3"
-    chipyard-streaming-fir-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "chipyard-streaming-fir"
-    chipyard-streaming-passthrough-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "chipyard-streaming-passthrough"
     chipyard-hetero-run-tests:
         executor: main-env
         steps:
             - run-tests:
+                group-key: "group-cores"
                 project-key: "chipyard-hetero"
                 timeout: "15m"
     chipyard-boom-run-tests:
         executor: main-env
         steps:
             - run-tests:
+                group-key: "group-cores"
                 project-key: "chipyard-boom"
+    chipyard-ariane-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                group-key: "group-cores"
+                project-key: "chipyard-ariane"
+                timeout: "30m"
+    chipyard-dmirocket-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                group-key: "group-peripherals"
+                project-key: "chipyard-dmirocket"
+    chipyard-spiflashwrite-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                group-key: "group-peripherals"
+                project-key: "chipyard-spiflashwrite"
+    chipyard-spiflashread-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                group-key: "group-peripherals"
+                project-key: "chipyard-spiflashread"
+    chipyard-sha3-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                group-key: "group-accels"
+                project-key: "chipyard-sha3"
+    chipyard-streaming-fir-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                group-key: "group-accels"
+                project-key: "chipyard-streaming-fir"
+    chipyard-streaming-passthrough-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                group-key: "group-accels"
+                project-key: "chipyard-streaming-passthrough"
     chipyard-hwacha-run-tests:
         executor: main-env
         steps:
             - run-tests:
                 tools-version: "esp-tools"
+                group-key: "group-accels"
                 project-key: "chipyard-hwacha"
     chipyard-gemmini-run-tests:
         executor: main-env
         steps:
             - run-tests:
                 tools-version: "esp-tools"
+                group-key: "group-accels"
                 project-key: "chipyard-gemmini"
-    chipyard-spiflashwrite-run-tests:
+    chipyard-nvdla-run-tests:
         executor: main-env
         steps:
             - run-tests:
-                project-key: "chipyard-spiflashwrite"
-    chipyard-spiflashread-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "chipyard-spiflashread"
+                group-key: "group-accels"
+                project-key: "chipyard-nvdla"
     tracegen-run-tests:
         executor: main-env
         steps:
             - run-tests:
+                group-key: "group-tracegen"
                 project-key: "tracegen"
     tracegen-boom-run-tests:
         executor: main-env
         steps:
             - run-tests:
+                group-key: "group-tracegen"
                 project-key: "tracegen-boom"
+    icenet-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                group-key: "group-other"
+                project-key: "icenet"
+                timeout: "30m"
+    testchipip-run-tests:
+        executor: main-env
+        steps:
+            - run-tests:
+                group-key: "group-other"
+                project-key: "testchipip"
+                timeout: "30m"
     firesim-run-tests:
         executor: main-env
         steps:
             - run-tests:
-                extra-cache-restore: "extra-tests"
+                group-key: "extra-tests"
                 project-key: "firesim"
                 run-script: "run-firesim-scala-tests.sh"
                 timeout: "20m"
@@ -372,7 +335,7 @@ jobs:
         executor: main-env
         steps:
             - run-tests:
-                extra-cache-restore: "extra-tests"
+                group-key: "extra-tests"
                 project-key: "fireboom"
                 run-script: "run-firesim-scala-tests.sh"
                 timeout: "45m"
@@ -380,33 +343,10 @@ jobs:
         executor: main-env
         steps:
             - run-tests:
-                extra-cache-restore: "extra-tests"
+                group-key: "extra-tests"
                 project-key: "firesim-multiclock"
                 run-script: "run-firesim-scala-tests.sh"
                 timeout: "20m"
-    chipyard-ariane-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "chipyard-ariane"
-                timeout: "30m"
-    chipyard-nvdla-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "chipyard-nvdla"
-    icenet-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "icenet"
-                timeout: "30m"
-    testchipip-run-tests:
-        executor: main-env
-        steps:
-            - run-tests:
-                project-key: "testchipip"
-                timeout: "30m"
 
 # Order and dependencies of jobs to run
 workflows:
@@ -446,154 +386,83 @@ workflows:
                     - install-riscv-toolchain
 
             # Prepare the verilator builds
-            - prepare-chipyard-rocket:
+            - prepare-chipyard-cores:
                 requires:
                     - install-riscv-toolchain
                     - install-verilator
-
-            - prepare-chipyard-dmirocket:
+            - prepare-chipyard-peripherals:
                 requires:
                     - install-riscv-toolchain
                     - install-verilator
-
-            - prepare-chipyard-sha3:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-streaming-fir:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-streaming-passthrough:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-hetero:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-boom:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-blkdev:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-hwacha:
+            - prepare-chipyard-accels:
                 requires:
                     - install-esp-toolchain
                     - install-verilator
-
-            - prepare-chipyard-gemmini:
-                requires:
-                    - install-esp-toolchain
-                    - install-verilator
-
-            - prepare-tracegen:
+            - prepare-chipyard-tracegen:
                 requires:
                     - install-riscv-toolchain
                     - install-verilator
-
-            - prepare-tracegen-boom:
+            - prepare-chipyard-other:
                 requires:
                     - install-riscv-toolchain
                     - install-verilator
 
-            - prepare-chipyard-ariane:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-icenet:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-testchipip:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-nvdla:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-spiflashwrite:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-spiflashread:
-                requires:
-                    - install-riscv-toolchain
-                    - install-verilator
-
-            - prepare-chipyard-mmios:
-                requires:
-                    - install-riscv-toolchain
-
-            # Run the respective tests
-
             # Run the example tests
             - chipyard-rocket-run-tests:
                 requires:
-                    - prepare-chipyard-rocket
+                    - prepare-chipyard-cores
+            - chipyard-hetero-run-tests:
+                requires:
+                    - prepare-chipyard-cores
+            - chipyard-boom-run-tests:
+                requires:
+                    - prepare-chipyard-cores
+            - chipyard-ariane-run-tests:
+                requires:
+                    - prepare-chipyard-cores
 
             - chipyard-dmirocket-run-tests:
                 requires:
-                    - prepare-chipyard-dmirocket
+                    - prepare-chipyard-peripherals
+            - chipyard-spiflashwrite-run-tests:
+                requires:
+                    - prepare-chipyard-peripherals
+            - chipyard-spiflashread-run-tests:
+                requires:
+                    - prepare-chipyard-peripherals
 
             - chipyard-sha3-run-tests:
                 requires:
-                    - prepare-chipyard-sha3
-
+                    - prepare-chipyard-accels
             - chipyard-streaming-fir-run-tests:
                 requires:
-                    - prepare-chipyard-streaming-fir
-
+                    - prepare-chipyard-accels
             - chipyard-streaming-passthrough-run-tests:
                 requires:
-                    - prepare-chipyard-streaming-passthrough
-
-            - chipyard-hetero-run-tests:
-                requires:
-                    - prepare-chipyard-hetero
-
-            - chipyard-boom-run-tests:
-                requires:
-                    - prepare-chipyard-boom
-
+                    - prepare-chipyard-accels
             - chipyard-hwacha-run-tests:
                 requires:
-                    - prepare-chipyard-hwacha
-
+                    - prepare-chipyard-accels
             - chipyard-gemmini-run-tests:
                 requires:
-                    - prepare-chipyard-gemmini
+                    - prepare-chipyard-accels
+            - chipyard-nvdla-run-tests:
+                requires:
+                    - prepare-chipyard-accels
 
             - tracegen-run-tests:
                 requires:
-                    - prepare-tracegen
-
+                    - prepare-chipyard-tracegen
             - tracegen-boom-run-tests:
                 requires:
-                    - prepare-tracegen-boom
+                    - prepare-chipyard-tracegen
 
-            - chipyard-spiflashwrite-run-tests:
+            - icenet-run-tests:
                 requires:
-                    - prepare-chipyard-spiflashwrite
-
-            - chipyard-spiflashread-run-tests:
+                    - prepare-chipyard-other
+            - testchipip-run-tests:
                 requires:
-                    - prepare-chipyard-spiflashread
+                    - prepare-chipyard-other
 
             # Run the firesim tests
             - firesim-run-tests:
@@ -612,17 +481,4 @@ workflows:
                     - install-verilator
                     - build-extra-tests
 
-            - chipyard-ariane-run-tests:
-                requires:
-                    - prepare-chipyard-ariane
 
-            - chipyard-nvdla-run-tests:
-                requires:
-                    - prepare-chipyard-nvdla
-            - icenet-run-tests:
-                requires:
-                    - prepare-icenet
-
-            - testchipip-run-tests:
-                requires:
-                    - prepare-testchipip
diff --git a/.circleci/defaults.sh b/.circleci/defaults.sh
index 703737cd..6f74c26d 100755
--- a/.circleci/defaults.sh
+++ b/.circleci/defaults.sh
@@ -45,6 +45,14 @@ LOCAL_CHIPYARD_DIR=$LOCAL_CHECKOUT_DIR
 LOCAL_SIM_DIR=$LOCAL_CHIPYARD_DIR/sims/verilator
 LOCAL_FIRESIM_DIR=$LOCAL_CHIPYARD_DIR/sims/firesim/sim
 
+# key value store to get the build groups
+declare -A grouping
+grouping["group-cores"]="chipyard-ariane chipyard-rocket chipyard-hetero chipyard-boom"
+grouping["group-peripherals"]="chipyard-dmirocket chipyard-blkdev chipyard-spiflashread chipyard-spiflashwrite chipyard-mmios"
+grouping["group-accels"]="chipyard-nvdla chipyard-sha3 chipyard-hwacha chipyard-gemmini chipyard-streaming-fir chipyard-streaming-passthrough"
+grouping["group-tracegen"]="tracegen tracegen-boom"
+grouping["group-other"]="icenet testchipip"
+
 # key value store to get the build strings
 declare -A mapping
 mapping["chipyard-rocket"]=""
@@ -64,6 +72,7 @@ mapping["chipyard-mmios"]=" CONFIG=MMIORocketConfig verilog"
 mapping["tracegen"]=" CONFIG=NonBlockingTraceGenL2Config"
 mapping["tracegen-boom"]=" CONFIG=BoomTraceGenConfig"
 mapping["chipyard-nvdla"]=" CONFIG=SmallNVDLARocketConfig"
+
 mapping["firesim"]="SCALA_TEST=firesim.firesim.RocketNICF1Tests"
 mapping["firesim-multiclock"]="SCALA_TEST=firesim.firesim.RocketMulticlockF1Tests"
 mapping["fireboom"]="SCALA_TEST=firesim.firesim.BoomF1Tests"
diff --git a/.circleci/do-rtl-build.sh b/.circleci/do-rtl-build.sh
index 3973026f..9093b29e 100755
--- a/.circleci/do-rtl-build.sh
+++ b/.circleci/do-rtl-build.sh
@@ -31,7 +31,7 @@ run "cp -r ~/.sbt  $REMOTE_WORK_DIR"
 TOOLS_DIR=$REMOTE_RISCV_DIR
 LD_LIB_DIR=$REMOTE_RISCV_DIR/lib
 
-if [ $1 = "chipyard-gemmini" ]; then
+if [ $1 = "group-accels" ]; then
     export RISCV=$LOCAL_ESP_DIR
     export LD_LIBRARY_PATH=$LOCAL_ESP_DIR/lib
     export PATH=$RISCV/bin:$PATH
@@ -40,9 +40,7 @@ if [ $1 = "chipyard-gemmini" ]; then
     git submodule update --init --recursive gemmini-rocc-tests
     cd gemmini-rocc-tests
     ./build.sh
-fi
 
-if [ $1 = "chipyard-hwacha" ] || [ $1 = "chipyard-gemmini" ]; then
     TOOLS_DIR=$REMOTE_ESP_DIR
     LD_LIB_DIR=$REMOTE_ESP_DIR/lib
     run "mkdir -p $REMOTE_ESP_DIR"
@@ -58,12 +56,19 @@ run "export RISCV=\"$TOOLS_DIR\"; \
      export PATH=\"$REMOTE_VERILATOR_DIR/bin:\$PATH\"; \
      export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \
      export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \
-     make -C $REMOTE_SIM_DIR clean; \
-     make -j$REMOTE_MAKE_NPROC -C $REMOTE_SIM_DIR FIRRTL_LOGLEVEL=info JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" ${mapping[$1]}"
-run "rm -rf $REMOTE_CHIPYARD_DIR/project"
+     make -C $REMOTE_SIM_DIR clean;"
 
-# copy back the final build
+read -a keys <<< ${grouping[$1]}
 
+for key in "${keys[@]}"
+do
+    run "export RISCV=\"$TOOLS_DIR\"; \
+         export LD_LIBRARY_PATH=\"$LD_LIB_DIR\"; \
+         export PATH=\"$REMOTE_VERILATOR_DIR/bin:\$PATH\"; \
+         export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \
+         export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \
+         make -j$REMOTE_MAKE_NPROC -C $REMOTE_SIM_DIR FIRRTL_LOGLEVEL=info JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" ${mapping[$key]}"
+done
 
 run "rm -rf $REMOTE_CHIPYARD_DIR/project"
 

From 36ccb12560ef24eb492b93baf5c2da996badf1b8 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 16 Sep 2020 10:29:03 -0700
Subject: [PATCH 68/74] Bump testchipip

---
 generators/chipyard/src/main/scala/Subsystem.scala | 2 +-
 generators/testchipip                              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/generators/chipyard/src/main/scala/Subsystem.scala b/generators/chipyard/src/main/scala/Subsystem.scala
index 172be692..5dd6ac18 100644
--- a/generators/chipyard/src/main/scala/Subsystem.scala
+++ b/generators/chipyard/src/main/scala/Subsystem.scala
@@ -30,7 +30,7 @@ import testchipip.{DromajoHelper, CanHavePeripheryTLSerial, SerialTLKey}
 trait CanHaveHTIF { this: BaseSubsystem =>
   // Advertise HTIF if system can communicate with fesvr
   if (this match {
-    case _: CanHavePeripheryTKSerial if p(SerialTLKey).nonEmpty => true
+    case _: CanHavePeripheryTLSerial if p(SerialTLKey).nonEmpty => true
     case _: HasPeripheryDebug if p(ExportDebug).dmi => true
     case _ => false
   }) {
diff --git a/generators/testchipip b/generators/testchipip
index e845cb3f..57cac41f 160000
--- a/generators/testchipip
+++ b/generators/testchipip
@@ -1 +1 @@
-Subproject commit e845cb3f50bb141fec7e8291a40d0b46d5ef5c12
+Subproject commit 57cac41f71653ca2a04d936f36ee8c78ec9919fd

From 269af01a70d6fe6411dcc6ba5ef7f5aed5fa4e1d Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 16 Sep 2020 12:06:36 -0700
Subject: [PATCH 69/74] Bump testchipip

---
 .circleci/defaults.sh | 2 +-
 generators/testchipip | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/defaults.sh b/.circleci/defaults.sh
index 2a744e89..6053b367 100755
--- a/.circleci/defaults.sh
+++ b/.circleci/defaults.sh
@@ -49,7 +49,7 @@ LOCAL_FIRESIM_DIR=$LOCAL_CHIPYARD_DIR/sims/firesim/sim
 declare -A mapping
 mapping["chipyard-rocket"]=""
 mapping["chipyard-dmirocket"]=" CONFIG=dmiRocketConfig"
-mapping["chipyard-tlserial"]=" CONFIG=TLSerialRocketConfig"
+mapping["chipyard-tlserial"]=" CONFIG=LBWIFMemoryRocketConfig"
 mapping["chipyard-sha3"]=" CONFIG=Sha3RocketConfig"
 mapping["chipyard-streaming-fir"]=" CONFIG=StreamingFIRRocketConfig"
 mapping["chipyard-streaming-passthrough"]=" CONFIG=StreamingPassthroughRocketConfig"
diff --git a/generators/testchipip b/generators/testchipip
index 57cac41f..f2efec8e 160000
--- a/generators/testchipip
+++ b/generators/testchipip
@@ -1 +1 @@
-Subproject commit 57cac41f71653ca2a04d936f36ee8c78ec9919fd
+Subproject commit f2efec8ee7bd6988b94a6fa3615ae7f864714004

From 687430898116d98ff625fa9258ae89473dd26135 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Wed, 16 Sep 2020 15:27:43 -0700
Subject: [PATCH 70/74] Address review comments

---
 .circleci/config.yml      | 18 +++++++++---------
 .circleci/do-rtl-build.sh |  4 ----
 2 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6ed04488..48ed4883 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -73,7 +73,7 @@ commands:
             tools-version:
                 type: string
                 default: "riscv-tools"
-            project-key:
+            group-key:
                 type: string
             timeout:
                 type: string
@@ -85,11 +85,11 @@ commands:
             - setup-tools:
                 tools-version: "<< parameters.tools-version >>"
             - run:
-                name: Building << parameters.project-key >> subproject using Verilator
-                command: .circleci/<< parameters.build-script >> << parameters.project-key >>
+                name: Building << parameters.group-key >> subproject using Verilator
+                command: .circleci/<< parameters.build-script >> << parameters.group-key >>
                 no_output_timeout: << parameters.timeout >>
             - save_cache:
-                key: << parameters.project-key >>-{{ .Branch }}-{{ .Revision }}
+                key: << parameters.group-key >>-{{ .Branch }}-{{ .Revision }}
                 paths:
                     - "/home/riscvuser/project"
 
@@ -192,28 +192,28 @@ jobs:
         executor: main-env
         steps:
             - prepare-rtl:
-                project-key: "group-cores"
+                group-key: "group-cores"
     prepare-chipyard-peripherals:
         executor: main-env
         steps:
             - prepare-rtl:
-                project-key: "group-peripherals"
+                group-key: "group-peripherals"
     prepare-chipyard-accels:
         executor: main-env
         steps:
             - prepare-rtl:
                 tools-version: "esp-tools"
-                project-key: "group-accels"
+                group-key: "group-accels"
     prepare-chipyard-tracegen:
         executor: main-env
         steps:
             - prepare-rtl:
-                project-key: "group-tracegen"
+                group-key: "group-tracegen"
     prepare-chipyard-other:
         executor: main-env
         steps:
             - prepare-rtl:
-                project-key: "group-other"
+                group-key: "group-other"
 
     chipyard-rocket-run-tests:
         executor: main-env
diff --git a/.circleci/do-rtl-build.sh b/.circleci/do-rtl-build.sh
index 9093b29e..784dbc04 100755
--- a/.circleci/do-rtl-build.sh
+++ b/.circleci/do-rtl-build.sh
@@ -52,10 +52,6 @@ fi
 
 # enter the verilator directory and build the specific config on remote server
 run "export RISCV=\"$TOOLS_DIR\"; \
-     export LD_LIBRARY_PATH=\"$LD_LIB_DIR\"; \
-     export PATH=\"$REMOTE_VERILATOR_DIR/bin:\$PATH\"; \
-     export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \
-     export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \
      make -C $REMOTE_SIM_DIR clean;"
 
 read -a keys <<< ${grouping[$1]}

From aa355c7c1abb3980f28ba167b8ebac89b9255fb2 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 18 Sep 2020 10:41:59 -0700
Subject: [PATCH 71/74] Bump firesim

---
 sims/firesim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/firesim b/sims/firesim
index 2b52057e..7b2691c9 160000
--- a/sims/firesim
+++ b/sims/firesim
@@ -1 +1 @@
-Subproject commit 2b52057e158fd91d44c6259aa08869622a88040a
+Subproject commit 7b2691c935aef13fd9de0e8d95d99ad9bceaab2e

From bbf941c865f660fbf15ddac2c4f579a55628191f Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Fri, 18 Sep 2020 10:43:58 -0700
Subject: [PATCH 72/74] Bump Firesim

---
 sims/firesim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sims/firesim b/sims/firesim
index 7b2691c9..3dbe8aee 160000
--- a/sims/firesim
+++ b/sims/firesim
@@ -1 +1 @@
-Subproject commit 7b2691c935aef13fd9de0e8d95d99ad9bceaab2e
+Subproject commit 3dbe8aee3f917079e7319391bac5d23d2ba5e6de

From 6c297e3179a7e32c00dd4a0edb8ed7729b6f2c05 Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 22 Sep 2020 11:08:52 -0700
Subject: [PATCH 73/74] Fix smartelf2hex.sh creating files 64x the minimum size

---
 scripts/smartelf2hex.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/smartelf2hex.sh b/scripts/smartelf2hex.sh
index 782977ff..e6fd1f7a 100755
--- a/scripts/smartelf2hex.sh
+++ b/scripts/smartelf2hex.sh
@@ -11,4 +11,6 @@ entry_dec=`bc <<< "ibase=16;$entry_hex"`
 length_hex=`echo "$segments" | grep LOAD | tail -n 1 | tr -s [:space:] | cut -f4,6 -d' '`
 length_dec=`echo $length_hex | tr -d x | tr [:lower:] [:upper:] | tr ' ' + | sed 's/^/ibase=16;/' | sed "s/$/-$entry_hex/" | bc`
 power_2_length=`echo "x=l($length_dec)/l(2); scale=0; 2^((x+1)/1)" | bc -l`
-elf2hex 64 $power_2_length $binary $entry_dec
+width=64
+depth=$((power_2_length / width))
+elf2hex $width $depth $binary $entry_dec

From d5660c01f3ada775c671409e593826b56cd9f7bf Mon Sep 17 00:00:00 2001
From: Jerry Zhao <jerryz123@berkeley.edu>
Date: Tue, 22 Sep 2020 12:58:34 -0700
Subject: [PATCH 74/74] Bump esp-isa-sim for loadmem-fix add TLS segments to
 smartelf2hex

---
 scripts/smartelf2hex.sh            | 2 +-
 toolchains/esp-tools/riscv-isa-sim | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/smartelf2hex.sh b/scripts/smartelf2hex.sh
index e6fd1f7a..dd035690 100755
--- a/scripts/smartelf2hex.sh
+++ b/scripts/smartelf2hex.sh
@@ -8,7 +8,7 @@ binary=$1
 segments=`readelf --segments --wide $binary`
 entry_hex=`echo -e "$segments" | grep "Entry point" | cut -f3 -d' ' | sed 's/0x//' | tr [:lower:] [:upper:]`
 entry_dec=`bc <<< "ibase=16;$entry_hex"`
-length_hex=`echo "$segments" | grep LOAD | tail -n 1 | tr -s [:space:] | cut -f4,6 -d' '`
+length_hex=`echo "$segments" | grep "LOAD\|TLS" | tail -n 1 | tr -s [:space:] | cut -f4,6 -d' '`
 length_dec=`echo $length_hex | tr -d x | tr [:lower:] [:upper:] | tr ' ' + | sed 's/^/ibase=16;/' | sed "s/$/-$entry_hex/" | bc`
 power_2_length=`echo "x=l($length_dec)/l(2); scale=0; 2^((x+1)/1)" | bc -l`
 width=64
diff --git a/toolchains/esp-tools/riscv-isa-sim b/toolchains/esp-tools/riscv-isa-sim
index aa332c6a..fa94e84d 160000
--- a/toolchains/esp-tools/riscv-isa-sim
+++ b/toolchains/esp-tools/riscv-isa-sim
@@ -1 +1 @@
-Subproject commit aa332c6a9a5ec77a9b97cdb4a1978ad394b17f1e
+Subproject commit fa94e84d4ff3e23ba909a63376b294e444234752