Merge pull request #614 from ucb-bar/diplomatic-clocks

Diplomatic multiclock

.circleci/do-rtl-build.sh

@@ -59,7 +59,7 @@ run "export RISCV=\"$TOOLS_DIR\"; \
      export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \
      export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \
      make -C $REMOTE_SIM_DIR clean; \
-     make -j$REMOTE_MAKE_NPROC -C $REMOTE_SIM_DIR JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" ${mapping[$1]}"
+     make -j$REMOTE_MAKE_NPROC -C $REMOTE_SIM_DIR FIRRTL_LOGLEVEL=info JAVA_ARGS=\"$REMOTE_JAVA_ARGS\" ${mapping[$1]}"
 run "rm -rf $REMOTE_CHIPYARD_DIR/project"
 
 # copy back the final build

generators/chipyard/src/main/scala/ChipTop.scala

@@ -4,118 +4,64 @@ import chisel3._
 
 import scala.collection.mutable.{ArrayBuffer}
 
+import freechips.rocketchip.prci.{ClockGroupIdentityNode, ClockSinkParameters, ClockSinkNode, ClockGroup}
+import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
 import freechips.rocketchip.config.{Parameters, Field}
-import freechips.rocketchip.diplomacy.{LazyModule}
+import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, LazyRawModuleImp, LazyModuleImpLike}
 import freechips.rocketchip.util.{ResetCatchAndSync}
-import chipyard.config.ConfigValName._
 import chipyard.iobinders.{IOBinders, TestHarnessFunction, IOBinderTuple}
 
 import barstools.iocell.chisel._
 
-case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => LazyModule(new DigitalTop()(p)))
+case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => new DigitalTop()(p))
-
-/**
-  * Chipyard provides three baseline, top-level reset schemes, set using the
-  * [[GlobalResetSchemeKey]] in a Parameters instance. These are:
-  *
-  * 1) Synchronous: The input coming to the chip is synchronous to the provided
-  *    clocks and will be used without modification as a synchronous reset.
-  *    This is safe only for use in FireSim and SW simulation.
-  *
-  * 2) Asynchronous: The input reset is asynchronous to the input clock, but it
-  *    is caught and synchronized to that clock before it is dissemenated.
-  *    Thus, downsteam modules will be emitted with synchronously reset state
-  *    elements.
-  *
-  * 3) Asynchronous Full: The input reset is asynchronous to the input clock,
-  *    and is used globally as an async reset. Downstream modules will be emitted
-  *    with asynchronously reset state elements.
-  *
-  */
-sealed trait GlobalResetScheme {
-  def pinIsAsync: Boolean
-}
-sealed trait HasAsyncInput { self: GlobalResetScheme =>
-  def pinIsAsync = true
-}
-
-sealed trait HasSyncInput { self: GlobalResetScheme =>
-  def pinIsAsync = false
-}
-
-case object GlobalResetSynchronous extends GlobalResetScheme with HasSyncInput
-case object GlobalResetAsynchronous extends GlobalResetScheme with HasAsyncInput
-case object GlobalResetAsynchronousFull extends GlobalResetScheme with HasAsyncInput
-case object GlobalResetSchemeKey extends Field[GlobalResetScheme](GlobalResetSynchronous)
 
 
 /**
  * The base class used for building chips. This constructor instantiates a module specified by the BuildSystem parameter,
- * named "system", which is an instance of DigitalTop by default. The default clock and reset for "system" are set by two
+ * named "system", which is an instance of DigitalTop by default. The diplomatic clocks of System, as well as its implicit clock,
- * wires, "systemClock" and "systemReset", which are intended to be driven by traits mixed-in with this base class.
+ * is aggregated into the clockGroupNode. The parameterized functions controlled by ClockingSchemeKey and GlobalResetSchemeKey
+ * drive clock and reset generation
  */
-abstract class BaseChipTop()(implicit val p: Parameters) extends RawModule with HasTestHarnessFunctions {
 
+class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunctions {
   // A publicly accessible list of IO cells (useful for a floorplanning tool, for example)
   val iocells = ArrayBuffer.empty[IOCell]
   // A list of functions to call in the test harness
   val harnessFunctions = ArrayBuffer.empty[TestHarnessFunction]
-  // The system clock
-  // These are given so that IOCell can use DataMirror and generate ports with
-  // the right flow (Input/Output)
-  val systemClock = Wire(Input(Clock()))
-  val systemReset = Wire(Input(Reset()))
 
   // The system module specified by BuildSystem
-  val lSystem = p(BuildSystem)(p).suggestName("system")
+  val lSystem = LazyModule(p(BuildSystem)(p)).suggestName("system")
-  val system = withClockAndReset(systemClock, systemReset) { Module(lSystem.module) }
 
-  // Call all of the IOBinders and provide them with a default clock and reset
+  // The implicitClockSinkNode provides the implicit clock and reset for the System
-  withClockAndReset(systemClock, systemReset) {
+  val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
-    // Call each IOBinder on both the lazyModule instance and the module
+
-    // instance. Generally, an IOBinder PF should only be defined on one, so
+  // Generate Clocks and Reset
-    // this should not lead to two invocations.
+  p(ClockingSchemeKey)(this)
-    val (_ports, _iocells, _harnessFunctions) = p(IOBinders).values.flatMap(f => f(lSystem) ++ f(system)).unzip3
+
-    // We ignore _ports for now...
+  // NOTE: Making this a LazyRawModule is moderately dangerous, as anonymous children
-    iocells ++= _iocells.flatten
+  // of ChipTop (ex: ClockGroup) do not receive clock or reset.
-    harnessFunctions ++= _harnessFunctions.flatten
+  // However. anonymous children of ChipTop should not need an implicit Clock or Reset
+  // anyways, they probably need to be explicitly clocked.
+  lazy val module: LazyModuleImpLike = new LazyRawModuleImp(this) {
+    // These become the implicit clock and reset to the System
+    val implicit_clock = implicitClockSinkNode.in.head._1.clock
+    val implicit_reset = implicitClockSinkNode.in.head._1.reset
+
+
+    // The implicit clock and reset for the system is also, by convention, used for all the IOBinders
+    // TODO: This may not be the right thing to do in all cases
+    withClockAndReset(implicit_clock, implicit_reset) {
+      val (_ports, _iocells, _harnessFunctions) = p(IOBinders).values.flatMap(f => f(lSystem) ++ f(lSystem.module)).unzip3
+      // We ignore _ports for now...
+      iocells ++= _iocells.flatten
+      harnessFunctions ++= _harnessFunctions.flatten
+    }
+
+    // Connect the implicit clock/reset, if present
+    lSystem.module match { case l: LazyModuleImp => {
+      l.clock := implicit_clock
+      l.reset := implicit_reset
+    }}
   }
-
 }
 
-/**
- * A simple clock and reset implementation that punches out clock and reset ports with the same
- * names as the implicit clock and reset for standard Module classes. Three basic reset schemes 
- * are provided. See [[GlobalResetScheme]].
- */
-trait HasChipTopSimpleClockAndReset { this: BaseChipTop =>
-
-  val (clock, systemClockIO) = IOCell.generateIOFromSignal(systemClock, Some("iocell_clock"))
-  val (reset, systemResetIO) = p(GlobalResetSchemeKey) match {
-    case GlobalResetSynchronous  =>
-      IOCell.generateIOFromSignal(systemReset, Some("iocell_reset"))
-    case GlobalResetAsynchronousFull =>
-      IOCell.generateIOFromSignal(systemReset, Some("iocell_reset"), abstractResetAsAsync = true)
-    case GlobalResetAsynchronous =>
-      val asyncResetCore = Wire(Input(AsyncReset()))
-      systemReset := ResetCatchAndSync(systemClock, asyncResetCore.asBool)
-      IOCell.generateIOFromSignal(asyncResetCore, Some("iocell_reset"), abstractResetAsAsync = true)
-  }
-
-  iocells ++= systemClockIO
-  iocells ++= systemResetIO
-
-  // Add a TestHarnessFunction that connects clock and reset
-  harnessFunctions += { (th: TestHarness) => {
-    // Connect clock; it's not done implicitly with RawModule
-    clock := th.clock
-    // Connect reset; it's not done implicitly with RawModule
-    // Note that we need to use dutReset, not harnessReset
-    reset := th.dutReset
-    Nil
-  } }
-
-}
-
-class ChipTop()(implicit p: Parameters) extends BaseChipTop()(p)
-  with HasChipTopSimpleClockAndReset

generators/chipyard/src/main/scala/Clocks.scala

@@ -0,0 +1,178 @@
+package chipyard
+
+import chisel3._
+
+import scala.collection.mutable.{ArrayBuffer}
+
+import freechips.rocketchip.prci._
+import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
+import freechips.rocketchip.config.{Parameters, Field}
+import freechips.rocketchip.diplomacy.{OutwardNodeHandle, InModuleBody, LazyModule}
+import freechips.rocketchip.util.{ResetCatchAndSync, Pow2ClockDivider}
+
+import barstools.iocell.chisel._
+
+/**
+  * Chipyard provides three baseline, top-level reset schemes, set using the
+  * [[GlobalResetSchemeKey]] in a Parameters instance. These are:
+  *
+  * 1) Synchronous: The input coming to the chip is synchronous to the provided
+  *    clocks and will be used without modification as a synchronous reset.
+  *    This is safe only for use in FireSim and SW simulation.
+  *
+  * 2) Asynchronous: The input reset is asynchronous to the input clock, but it
+  *    is caught and synchronized to that clock before it is dissemenated.
+  *    Thus, downsteam modules will be emitted with synchronously reset state
+  *    elements.
+  *
+  * 3) Asynchronous Full: The input reset is asynchronous to the input clock,
+  *    and is used globally as an async reset. Downstream modules will be emitted
+  *    with asynchronously reset state elements.
+  *
+  */
+sealed trait GlobalResetScheme {
+  def pinIsAsync: Boolean
+}
+sealed trait HasAsyncInput { self: GlobalResetScheme =>
+  def pinIsAsync = true
+}
+
+sealed trait HasSyncInput { self: GlobalResetScheme =>
+  def pinIsAsync = false
+}
+
+case object GlobalResetSynchronous extends GlobalResetScheme with HasSyncInput
+case object GlobalResetAsynchronous extends GlobalResetScheme with HasAsyncInput
+case object GlobalResetAsynchronousFull extends GlobalResetScheme with HasAsyncInput
+case object GlobalResetSchemeKey extends Field[GlobalResetScheme](GlobalResetSynchronous)
+
+/**
+ * A simple reset implementation that punches out reset ports
+ * for standard Module classes. Three basic reset schemes
+ * are provided. See [[GlobalResetScheme]].
+ */
+object GenerateReset {
+  def apply(chiptop: ChipTop, clock: Clock): Reset = {
+    implicit val p = chiptop.p
+    // this needs directionality so generateIOFromSignal works
+    val reset_wire = Wire(Input(Reset()))
+    val (reset_io, resetIOCell) = p(GlobalResetSchemeKey) match {
+      case GlobalResetSynchronous =>
+        IOCell.generateIOFromSignal(reset_wire, Some("iocell_reset"))
+      case GlobalResetAsynchronousFull =>
+        IOCell.generateIOFromSignal(reset_wire, Some("iocell_reset"), abstractResetAsAsync = true)
+      case GlobalResetAsynchronous => {
+        val async_reset_wire = Wire(Input(AsyncReset()))
+        reset_wire := ResetCatchAndSync(clock, async_reset_wire.asBool())
+        IOCell.generateIOFromSignal(async_reset_wire, Some("iocell_reset"), abstractResetAsAsync = true)
+      }
+    }
+    reset_io.suggestName("reset")
+    chiptop.iocells ++= resetIOCell
+    chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
+      reset_io := th.dutReset
+      Nil
+    })
+    reset_wire
+  }
+}
+
+
+case object ClockingSchemeKey extends Field[ChipTop => Unit](ClockingSchemeGenerators.harnessClock)
+
+
+
+object ClockingSchemeGenerators {
+  // A simple clock provider, for testing
+  val harnessClock: ChipTop => Unit = { chiptop =>
+    implicit val p = chiptop.p
+
+    val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
+    chiptop.implicitClockSinkNode := implicitClockSourceNode
+
+    // Drive the diplomaticclock graph of the DigitalTop (if present)
+    val simpleClockGroupSourceNode = chiptop.lSystem match {
+      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
+        val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+        l.asyncClockGroupsNode := n
+        Some(n)
+      }
+      case _ => None
+    }
+
+    InModuleBody {
+      //this needs directionality so generateIOFromSignal works
+      val clock_wire = Wire(Input(Clock()))
+      val reset_wire = GenerateReset(chiptop, clock_wire)
+      val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, Some("iocell_clock"))
+      chiptop.iocells ++= clockIOCell
+      clock_io.suggestName("clock")
+
+      implicitClockSourceNode.out.unzip._1.map { o =>
+        o.clock := clock_wire
+        o.reset := reset_wire
+      }
+
+      simpleClockGroupSourceNode.map { n => n.out.unzip._1.map { out: ClockGroupBundle =>
+        out.member.data.foreach { o =>
+          o.clock := clock_wire
+          o.reset := reset_wire
+        }
+      }}
+
+      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
+        clock_io := th.harnessClock
+        Nil
+      })
+    }
+
+  }
+
+
+  val harnessDividedClock: ChipTop => Unit = { chiptop =>
+    implicit val p = chiptop.p
+
+    require(false, "Divided clock is broken until we fix passing onchip clocks to TestHarness objects")
+
+    val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
+    chiptop.implicitClockSinkNode := implicitClockSourceNode
+
+    val simpleClockGroupSourceNode = chiptop.lSystem match {
+      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
+        val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+        l.asyncClockGroupsNode := n
+        Some(n)
+      }
+      case _ => throw new Exception("Harness multiclock assumes BaseSubsystem")
+    }
+
+    InModuleBody {
+      // this needs directionality so generateIOFromSignal works
+      val clock_wire = Wire(Input(Clock()))
+      val reset_wire = GenerateReset(chiptop, clock_wire)
+      val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, Some("iocell_clock"))
+      chiptop.iocells ++= clockIOCell
+      clock_io.suggestName("clock")
+      val div_clock = Pow2ClockDivider(clock_wire, 2)
+
+      implicitClockSourceNode.out.unzip._1.map { o =>
+        o.clock := div_clock
+        o.reset := reset_wire
+      }
+
+      simpleClockGroupSourceNode.map { n => n.out.unzip._1.map { out: ClockGroupBundle =>
+        out.member.elements.map { case (name, data) =>
+          // This is mega hacks, how are you actually supposed to do this?
+          data.clock := (if (name.contains("core")) clock_wire else div_clock)
+          data.reset := reset_wire
+        }
+      }}
+
+      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
+        clock_io := th.harnessClock
+        Nil
+      })
+    }
+
+  }
+}

generators/chipyard/src/main/scala/ConfigFragments.scala

@@ -6,12 +6,13 @@ import chisel3.util.{log2Up}
 import freechips.rocketchip.config.{Field, Parameters, Config}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.diplomacy.{LazyModule, ValName}
-import freechips.rocketchip.devices.tilelink.BootROMParams
+import freechips.rocketchip.devices.tilelink.{BootROMLocated}
 import freechips.rocketchip.devices.debug.{Debug}
 import freechips.rocketchip.groundtest.{GroundTestSubsystem}
 import freechips.rocketchip.tile._
 import freechips.rocketchip.rocket.{RocketCoreParams, MulDivParams, DCacheParams, ICacheParams}
 import freechips.rocketchip.util.{AsyncResetReg}
+import freechips.rocketchip.prci._
 
 import testchipip._
 import tracegen.{TraceGenSystem}
@@ -25,23 +26,15 @@ import sifive.blocks.devices.gpio._
 import sifive.blocks.devices.uart._
 import sifive.blocks.devices.spi._
 
-import chipyard.{BuildTop, BuildSystem, TestSuitesKey, TestSuiteHelper}
+import chipyard.{BuildTop, BuildSystem, ClockingSchemeGenerators, ClockingSchemeKey, TestSuitesKey, TestSuiteHelper}
 
-/**
- * TODO: Why do we need this?
- */
-object ConfigValName {
-  implicit val valName = ValName("TestHarness")
-}
-import ConfigValName._
 
 // -----------------------
 // Common Config Fragments
 // -----------------------
 
 class WithBootROM extends Config((site, here, up) => {
-  case BootROMParams => BootROMParams(
+  case BootROMLocated(x) => up(BootROMLocated(x), site).map(_.copy(contentFileName = s"./bootrom/bootrom.rv${site(XLen)}.img"))
-    contentFileName = s"./bootrom/bootrom.rv${site(XLen)}.img")
 })
 
 // DOC include start: gpio config fragment
@@ -73,7 +66,7 @@ class WithL2TLBs(entries: Int) extends Config((site, here, up) => {
 })
 
 class WithTracegenSystem extends Config((site, here, up) => {
-  case BuildSystem => (p: Parameters) => LazyModule(new TraceGenSystem()(p))
+  case BuildSystem => (p: Parameters) => new TraceGenSystem()(p)
 })
 
 /**
@@ -105,7 +98,8 @@ class WithMultiRoCCHwacha(harts: Int*) extends Config(
     case MultiRoCCKey => {
       up(MultiRoCCKey, site) ++ harts.distinct.map{ i =>
         (i -> Seq((p: Parameters) => {
-          LazyModule(new Hwacha()(p)).suggestName("hwacha")
+          val hwacha = LazyModule(new Hwacha()(p))
+          hwacha
         }))
       }
     }
@@ -156,3 +150,16 @@ class WithHwachaTest extends Config((site, here, up) => {
     "SRC_EXTENSION = $(base_dir)/hwacha/$(src_path)/*.scala" + "\nDISASM_EXTENSION = --extension=hwacha"
   }
 })
+
+// The default RocketChip BaseSubsystem drives its diplomatic clock graph
+// with the implicit clocks of Subsystem. Don't do that, instead we extend
+// the diplomacy graph upwards into the ChipTop, where we connect it to
+// our clock drivers
+class WithNoSubsystemDrivenClocks extends Config((site, here, up) => {
+  case SubsystemDriveAsyncClockGroupsKey => None
+})
+
+class WithTileDividedClock extends Config((site, here, up) => {
+  case ClockingSchemeKey => ClockingSchemeGenerators.harnessDividedClock
+})
+

generators/chipyard/src/main/scala/IOBinders.scala

@@ -41,7 +41,7 @@ import scala.reflect.{ClassTag}
 
 // DOC include start: IOBinders
 // This type describes a function callable on the TestHarness instance. Its return type is unused.
-type TestHarnessFunction = (chipyard.TestHarness) => Seq[Any]
+type TestHarnessFunction = (chipyard.HasHarnessSignalReferences) => Seq[Any]
 // IOBinders will return a Seq of this tuple, which contains three fields:
 //  1. A Seq containing all IO ports created by the IOBinder function
 //  2. A Seq containing all IO cell modules created by the IOBinder function
@@ -201,14 +201,14 @@ object AddIOCells {
   }
 
   def axi4(io: Seq[AXI4Bundle], node: AXI4SlaveNode, name: String): Seq[(AXI4Bundle, AXI4EdgeParameters, Seq[IOCell])] = {
-    io.zip(node.in).zipWithIndex.map{ case ((mem_axi4, (_, edge)), i) => {
+    io.zip(node.edges.in).zipWithIndex.map{ case ((mem_axi4, edge), i) => {
       val (port, ios) = IOCell.generateIOFromSignal(mem_axi4, Some(s"iocell_${name}_axi4_slave_${i}"))
       port.suggestName(s"${name}_axi4_slave_${i}")
       (port, edge, ios)
     }}
   }
   def axi4(io: Seq[AXI4Bundle], node: AXI4MasterNode, name: String): Seq[(AXI4Bundle, AXI4EdgeParameters, Seq[IOCell])] = {
-    io.zip(node.out).zipWithIndex.map{ case ((mem_axi4, (_, edge)), i) => {
+    io.zip(node.edges.out).zipWithIndex.map{ case ((mem_axi4, edge), i) => {
       //val (port, ios) = IOCell.generateIOFromSignal(mem_axi4, Some(s"iocell_${name}_axi4_master_${i}"))
       val port = IO(Flipped(AXI4Bundle(edge.bundle)))
       val ios = IOCell.generateFromSignal(mem_axi4, port, Some(s"iocell_${name}_axi4_master_${i}"))
@@ -228,7 +228,7 @@ object AddIOCells {
 class WithGPIOTiedOff extends OverrideIOBinder({
   (system: HasPeripheryGPIOModuleImp) => {
     val (ports2d, ioCells2d) = AddIOCells.gpio(system.gpio)
-    val harnessFn = (th: chipyard.TestHarness) => { ports2d.flatten.foreach(_ <> AnalogConst(0)); Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { ports2d.flatten.foreach(_ <> AnalogConst(0)); Nil }
     Seq((ports2d.flatten, ioCells2d.flatten, Some(harnessFn)))
   }
 })
@@ -237,7 +237,7 @@ class WithGPIOTiedOff extends OverrideIOBinder({
 class WithUARTAdapter extends OverrideIOBinder({
   (system: HasPeripheryUARTModuleImp) => {
     val (ports, ioCells2d) = AddIOCells.uart(system.uart)
-    val harnessFn = (th: chipyard.TestHarness) => { UARTAdapter.connect(ports)(system.p); Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { UARTAdapter.connect(ports)(system.p); Nil }
     Seq((ports, ioCells2d.flatten, Some(harnessFn)))
   }
 })
@@ -245,7 +245,7 @@ class WithUARTAdapter extends OverrideIOBinder({
 class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideIOBinder({
   (system: HasPeripherySPIFlashModuleImp) => {
     val (ports, ioCells2d) = AddIOCells.spi(system.qspi, "qspi")
-    val harnessFn = (th: chipyard.TestHarness) => { SimSPIFlashModel.connect(ports, th.reset, rdOnly)(system.p); Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { SimSPIFlashModel.connect(ports, th.harnessReset, rdOnly)(system.p); Nil }
     Seq((ports, ioCells2d.flatten, Some(harnessFn)))
   }
 })
@@ -253,8 +253,9 @@ class WithSimSPIFlashModel(rdOnly: Boolean = true) extends OverrideIOBinder({
 class WithSimBlockDevice extends OverrideIOBinder({
   (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev =>
     val (port, ios) = AddIOCells.blockDev(bdev)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
-      SimBlockDevice.connect(th.clock, th.reset.asBool, Some(port))(system.p)
+      // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock
+      SimBlockDevice.connect(th.harnessClock, th.harnessReset.asBool, Some(port))(system.p)
       Nil
     }
     Seq((Seq(port), ios, Some(harnessFn)))
@@ -264,7 +265,7 @@ class WithSimBlockDevice extends OverrideIOBinder({
 class WithBlockDeviceModel extends OverrideIOBinder({
   (system: CanHavePeripheryBlockDeviceModuleImp) => system.bdev.map { bdev =>
     val (port, ios) = AddIOCells.blockDev(bdev)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       BlockDeviceModel.connect(Some(port))(system.p)
       Nil
     }
@@ -287,7 +288,7 @@ class WithSimAXIMem extends OverrideIOBinder({
     val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem")
     // TODO: we are inlining the connectMem method of SimAXIMem because
     //   it takes in a dut rather than seq of axi4 ports
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       peiTuples.map { case (port, edge, ios) =>
         val mem = LazyModule(new SimAXIMem(edge, size = p(ExtMem).get.master.size))
         Module(mem.module).suggestName("mem")
@@ -304,14 +305,15 @@ class WithBlackBoxSimMem extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {
     implicit val p: Parameters = GetSystemParameters(system)
     val peiTuples = AddIOCells.axi4(system.mem_axi4, system.memAXI4Node, "mem")
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       peiTuples.map { case (port, edge, ios) =>
         val memSize = p(ExtMem).get.master.size
         val lineSize = p(CacheBlockBytes)
         val mem = Module(new SimDRAM(memSize, lineSize, edge.bundle))
         mem.io.axi <> port
-        mem.io.clock := th.clock
+        // TODO: Using harness clock/reset will be incorrect when systemClock =/= harnessClock
-        mem.io.reset := th.reset
+        mem.io.clock := th.harnessClock
+        mem.io.reset := th.harnessReset
       }
       Nil
     }
@@ -323,7 +325,7 @@ class WithSimAXIMMIO extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MMIOPort) => {
     implicit val p: Parameters = GetSystemParameters(system)
     val peiTuples = AddIOCells.axi4(system.mmio_axi4, system.mmioAXI4Node, "mmio_mem")
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       peiTuples.zipWithIndex.map { case ((port, edge, ios), i) =>
         val mmio_mem = LazyModule(new SimAXIMem(edge, size = 4096))
         Module(mmio_mem.module).suggestName(s"mmio_mem_${i}")
@@ -343,7 +345,7 @@ class WithTieOffInterrupts extends OverrideIOBinder({
   (system: HasExtInterruptsModuleImp) => {
     val (port, ioCells) = IOCell.generateIOFromSignal(system.interrupts, Some("iocell_interrupts"))
     port.suggestName("interrupts")
-    val harnessFn = (th: chipyard.TestHarness) => { port := 0.U; Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { port := 0.U; Nil }
     Seq((Seq(port), ioCells, Some(harnessFn)))
   }
 })
@@ -351,7 +353,7 @@ class WithTieOffInterrupts extends OverrideIOBinder({
 class WithTieOffL2FBusAXI extends OverrideIOBinder({
   (system: CanHaveSlaveAXI4Port) => {
     val peiTuples = AddIOCells.axi4(system.l2_frontend_bus_axi4, system.l2FrontendAXI4Node, "l2_fbus")
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       peiTuples.zipWithIndex.map { case ((port, edge, ios), i) =>
         port := DontCare // tieoff doesn't completely tie-off, for some reason
         port.tieoff()
@@ -362,17 +364,19 @@ class WithTieOffL2FBusAXI extends OverrideIOBinder({
   }
 })
 
+// TODO we need to rethink what "Tie-off-debug" means. The current system punches out
+// excessive IOs.
 class WithTiedOffDebug extends OverrideIOBinder({
   (system: HasPeripheryDebugModuleImp) => {
     val (psdPort, resetctrlOpt, debugPortOpt, ioCells) =
       AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       Debug.tieoffDebug(debugPortOpt, resetctrlOpt, Some(psdPort))(system.p)
       // tieoffDebug doesn't actually tie everything off :/
       debugPortOpt.foreach { d =>
-        d.clockeddmi.foreach({ cdmi => cdmi.dmi.req.bits := DontCare; cdmi.dmiClock := th.clock })
+        d.clockeddmi.foreach({ cdmi => cdmi.dmi.req.bits := DontCare; cdmi.dmiClock := th.harnessClock })
         d.dmactiveAck := DontCare
-        d.clock := th.clock
+        d.clock := th.harnessClock // TODO fix: This should be driven from within the chip
       }
       Nil
     }
@@ -380,15 +384,17 @@ class WithTiedOffDebug extends OverrideIOBinder({
   }
 })
 
+// TODO we need to rethink what this does. The current system punches out excessive IOs.
+// Some of the debug clock/reset should be driven from on-chip
 class WithSimDebug extends OverrideIOBinder({
   (system: HasPeripheryDebugModuleImp) => {
     val (psdPort, resetctrlPortOpt, debugPortOpt, ioCells) =
       AddIOCells.debug(system.psd, system.resetctrl, system.debug)(system.p)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       val dtm_success = Wire(Bool())
-      Debug.connectDebug(debugPortOpt, resetctrlPortOpt, psdPort, th.clock, th.harnessReset, dtm_success)(system.p)
+      Debug.connectDebug(debugPortOpt, resetctrlPortOpt, psdPort, th.harnessClock, th.harnessReset.asBool, dtm_success)(system.p)
       when (dtm_success) { th.success := true.B }
-      th.dutReset := th.harnessReset | debugPortOpt.map { debug => AsyncResetReg(debug.ndreset).asBool }.getOrElse(false.B)
+      th.dutReset := th.harnessReset.asBool | debugPortOpt.map { debug => AsyncResetReg(debug.ndreset).asBool }.getOrElse(false.B)
       Nil
     }
     Seq((Seq(psdPort) ++ debugPortOpt.toSeq, ioCells, Some(harnessFn)))
@@ -398,7 +404,7 @@ class WithSimDebug extends OverrideIOBinder({
 class WithTiedOffSerial extends OverrideIOBinder({
   (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial =>
     val (port, ioCells) = AddIOCells.serial(serial)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
       SerialAdapter.tieoff(port)
       Nil
     }
@@ -409,8 +415,8 @@ class WithTiedOffSerial extends OverrideIOBinder({
 class WithSimSerial extends OverrideIOBinder({
   (system: CanHavePeripherySerialModuleImp) => system.serial.map({ serial =>
     val (port, ioCells) = AddIOCells.serial(serial)
-    val harnessFn = (th: chipyard.TestHarness) => {
+    val harnessFn = (th: HasHarnessSignalReferences) => {
-      val ser_success = SerialAdapter.connectSimSerial(port, th.clock, th.harnessReset)
+      val ser_success = SerialAdapter.connectSimSerial(port, th.harnessClock, th.harnessReset)
       when (ser_success) { th.success := true.B }
       Nil
     }
@@ -422,7 +428,7 @@ class WithTraceGenSuccessBinder extends OverrideIOBinder({
   (system: TraceGenSystemModuleImp) => {
     val (successPort, ioCells) = IOCell.generateIOFromSignal(system.success, Some("iocell_success"))
     successPort.suggestName("success")
-    val harnessFn = (th: chipyard.TestHarness) => { when (successPort) { th.success := true.B }; Nil }
+    val harnessFn = (th: HasHarnessSignalReferences) => { when (successPort) { th.success := true.B }; Nil }
     Seq((Seq(successPort), ioCells, Some(harnessFn)))
   }
 })

generators/chipyard/src/main/scala/Subsystem.scala

@@ -8,6 +8,7 @@ package chipyard
 import chisel3._
 import chisel3.internal.sourceinfo.{SourceInfo}
 
+import freechips.rocketchip.prci._
 import freechips.rocketchip.config.{Field, Parameters}
 import freechips.rocketchip.devices.tilelink._
 import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp, ExportDebug}
@@ -26,7 +27,6 @@ import boom.common.{BoomTile}
 
 import testchipip.{DromajoHelper, CanHavePeripherySerial, SerialKey}
 
-
 trait CanHaveHTIF { this: BaseSubsystem =>
   // Advertise HTIF if system can communicate with fesvr
   if (this match {
@@ -47,7 +47,6 @@ trait CanHaveHTIF { this: BaseSubsystem =>
   }
 }
 
-
 class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
   with HasTiles
   with CanHaveHTIF
@@ -56,25 +55,17 @@ class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
     case r: RocketTile => r.module.core.rocketImpl.coreMonitorBundle
     case b: BoomTile => b.module.core.coreMonitorBundle
   }.toList
+
   override lazy val module = new ChipyardSubsystemModuleImp(this)
 }
 
-
 class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends BaseSubsystemModuleImp(_outer)
-  with HasResetVectorWire
   with HasTilesModuleImp
 {
-
-  for (i <- 0 until outer.tiles.size) {
-    val wire = tile_inputs(i)
-    wire.hartid := outer.hartIdList(i).U
-    wire.reset_vector := global_reset_vector
-  }
-
   // create file with core params
   ElaborationArtefacts.add("""core.config""", outer.tiles.map(x => x.module.toString).mkString("\n"))
   // Generate C header with relevant information for Dromajo
   // This is included in the `dromajo_params.h` header file
-  DromajoHelper.addArtefacts()
+  DromajoHelper.addArtefacts(InSubsystem)
 }
 

generators/chipyard/src/main/scala/System.scala

@@ -26,8 +26,10 @@ class ChipyardSystem(implicit p: Parameters) extends ChipyardSubsystem
   with CanHaveMasterAXI4MemPort
   with CanHaveMasterAXI4MMIOPort
   with CanHaveSlaveAXI4Port
-  with HasPeripheryBootROM
 {
+
+  val bootROM  = p(BootROMLocated(location)).map { BootROM.attach(_, this, CBUS) }
+  val maskROMs = p(MaskROMLocated(location)).map { MaskROM.attach(_, this, CBUS) }
   override lazy val module = new ChipyardSystemModule(this)
 }
 
@@ -37,5 +39,4 @@ class ChipyardSystem(implicit p: Parameters) extends ChipyardSubsystem
 class ChipyardSystemModule[+L <: ChipyardSystem](_outer: L) extends ChipyardSubsystemModuleImp(_outer)
   with HasRTCModuleImp
   with HasExtInterruptsModuleImp
-  with HasPeripheryBootROMModuleImp
   with DontTouch

generators/chipyard/src/main/scala/TestHarness.scala

@@ -5,33 +5,41 @@ import chisel3._
 import freechips.rocketchip.diplomacy.{LazyModule}
 import freechips.rocketchip.config.{Field, Parameters}
 import chipyard.iobinders.{TestHarnessFunction}
-import chipyard.config.ConfigValName._
 
 // -------------------------------
-// BOOM and/or Rocket Test Harness
+// Chipyard Test Harness
 // -------------------------------
 
-case object BuildTop extends Field[Parameters => HasTestHarnessFunctions]((p: Parameters) => Module(new ChipTop()(p)))
+case object BuildTop extends Field[Parameters => LazyModule with HasTestHarnessFunctions]((p: Parameters) => new ChipTop()(p))
 
 trait HasTestHarnessFunctions {
   val harnessFunctions: Seq[TestHarnessFunction]
 }
 
-class TestHarness(implicit val p: Parameters) extends Module {
+trait HasHarnessSignalReferences {
+  def harnessClock: Clock
+  def harnessReset: Reset
+  def dutReset: Reset
+  def success: Bool
+}
+
+class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSignalReferences {
   val io = IO(new Bundle {
     val success = Output(Bool())
   })
 
-  val dut = p(BuildTop)(p)
+  val ldut = LazyModule(p(BuildTop)(p)).suggestName("chiptop")
+  val dut = Module(ldut.module)
   io.success := false.B
 
+  val harnessClock = clock
+  val harnessReset = WireInit(reset)
+  val success = io.success
+
   // dutReset assignment can be overridden via a harnessFunction, but by default it is just reset
   val dutReset = WireDefault(if (p(GlobalResetSchemeKey).pinIsAsync) reset.asAsyncReset else reset)
 
-  dut.harnessFunctions.foreach(_(this))
+  ldut.harnessFunctions.foreach(_(this))
-
-  def success = io.success
-  def harnessReset = this.reset.asBool
 
 }
 

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -17,6 +17,7 @@ class AbstractConfig extends Config(
   new chipyard.config.WithBootROM ++                             // use default bootrom
   new chipyard.config.WithUART ++                                // add a UART
   new chipyard.config.WithL2TLBs(1024) ++                        // use L2 TLBs
+  new chipyard.config.WithNoSubsystemDrivenClocks ++             // drive the subsystem diplomatic clocks from ChipTop instead of using implicit clocks
   new freechips.rocketchip.subsystem.WithNoMMIOPort ++           // no top-level MMIO master port (overrides default set in rocketchip)
   new freechips.rocketchip.subsystem.WithNoSlavePort ++          // no top-level MMIO slave port (overrides default set in rocketchip)
   new freechips.rocketchip.subsystem.WithInclusiveCache ++       // use Sifive L2 cache

generators/chipyard/src/main/scala/config/RocketConfigs.scala

@@ -184,3 +184,13 @@ class MMIORocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithDefaultSlavePort ++ // add default external slave port
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new chipyard.config.AbstractConfig)
+
+// NOTE: This config doesn't work yet because SimWidgets in the TestHarness
+// always get the TestHarness clock. The Tiles and Uncore receive the correct clocks
+class DividedClockRocketConfig extends Config(
+  new chipyard.config.WithTileDividedClock ++                       // Put the Tile on its own clock domain
+  new freechips.rocketchip.subsystem.WithRationalRocketTiles ++   // Add rational crossings between RocketTile and uncore
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new chipyard.config.AbstractConfig)
+
+

generators/chipyard/src/main/scala/config/TracegenConfigs.scala

@@ -7,6 +7,7 @@ class TraceGenConfig extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 0, nSets = 16, nWays = 2) }) ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
@@ -15,6 +16,7 @@ class NonBlockingTraceGenConfig extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 2, nSets = 16, nWays = 2) }) ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
   new freechips.rocketchip.groundtest.GroundTestBaseConfig)
@@ -23,6 +25,7 @@ class BoomTraceGenConfig extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithBoomTraceGen()(List.fill(2) { DCacheParams(nMSHRs = 8, nSets = 16, nWays = 2) }) ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
@@ -32,6 +35,7 @@ class NonBlockingTraceGenL2Config extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithL2TraceGen()(List.fill(2)(DCacheParams(nMSHRs = 2, nSets = 16, nWays = 4))) ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++
   new freechips.rocketchip.subsystem.WithCoherentBusTopology ++
@@ -41,6 +45,7 @@ class NonBlockingTraceGenL2RingConfig extends Config(
   new chipyard.iobinders.WithBlackBoxSimMem ++
   new chipyard.iobinders.WithTraceGenSuccessBinder ++
   new chipyard.config.WithTracegenSystem ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
   new tracegen.WithL2TraceGen()(List.fill(2)(DCacheParams(nMSHRs = 2, nSets = 16, nWays = 4))) ++
   new testchipip.WithRingSystemBus ++
   new freechips.rocketchip.subsystem.WithInclusiveCache ++

generators/chipyard/src/main/scala/config/TutorialConfigs.scala

@@ -31,6 +31,7 @@ class TutorialStarterConfig extends Config(
   new testchipip.WithTSI ++                  // Add a TSI (Test Serial Interface)  widget to bring-up the core
   new chipyard.config.WithBootROM ++         // Use the Chipyard BootROM
   new chipyard.config.WithUART ++            // Add a UART
+  new chipyard.config.WithNoSubsystemDrivenClocks ++ // Don't drive the subsystem clocks from within the subsystem
 
   // CUSTOMIZE THE CORE
   // Uncomment out one (or multiple) of the lines below, and choose
@@ -65,6 +66,7 @@ class TutorialMMIOConfig extends Config(
   new testchipip.WithTSI ++
   new chipyard.config.WithBootROM ++
   new chipyard.config.WithUART ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
 
   // Attach either a TileLink or AXI4 version of GCD
   // Uncomment one of the below lines
@@ -92,6 +94,7 @@ class TutorialSha3Config extends Config(
   new testchipip.WithTSI ++
   new chipyard.config.WithBootROM ++
   new chipyard.config.WithUART ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
 
   // Uncomment this line once you added SHA3 to the build.sbt, and cloned the SHA3 repo
   // new sha3.WithSha3Accel ++
@@ -117,6 +120,7 @@ class TutorialSha3BlackBoxConfig extends Config(
   new testchipip.WithTSI ++
   new chipyard.config.WithBootROM ++
   new chipyard.config.WithUART ++
+  new chipyard.config.WithNoSubsystemDrivenClocks ++
 
   // Uncomment these lines once SHA3 is integrated
   // new sha3.WithSha3BlackBox ++ // Specify we want the Black-box verilog version of Sha3 Ctrl

generators/firechip/src/main/scala/BridgeBinders.scala

@@ -58,7 +58,7 @@ class WithBlockDeviceBridge extends OverrideIOBinder({
 class WithFASEDBridge extends OverrideIOBinder({
   (system: CanHaveMasterAXI4MemPort) => {
     implicit val p: Parameters = GetSystemParameters(system)
-    (system.mem_axi4 zip system.memAXI4Node.in).foreach({ case (axi4, (_, edge)) =>
+    (system.mem_axi4 zip system.memAXI4Node.edges.in).foreach({ case (axi4, edge) =>
       val nastiKey = NastiParameters(axi4.r.bits.data.getWidth,
                                      axi4.ar.bits.addr.getWidth,
                                      axi4.ar.bits.id.getWidth)

generators/firechip/src/main/scala/FireSim.scala

@@ -3,13 +3,17 @@
 package firesim.firesim
 
 import chisel3._
+import chisel3.experimental.{IO}
 
+import freechips.rocketchip.prci._
+import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
 import freechips.rocketchip.config.{Field, Config, Parameters}
-import freechips.rocketchip.diplomacy.{LazyModule}
+import freechips.rocketchip.diplomacy.{LazyModule, InModuleBody}
+import freechips.rocketchip.util.{ResetCatchAndSync}
 
-import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge}
+import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge, RationalClock}
 
-import chipyard.{BuildSystem}
+import chipyard.{BuildSystem, BuildTop, HasHarnessSignalReferences, ChipyardSubsystem, ClockingSchemeKey, ChipTop}
 import chipyard.iobinders.{IOBinders}
 
 // Determines the number of times to instantiate the DUT in the harness.
@@ -20,6 +24,16 @@ class WithNumNodes(n: Int) extends Config((pname, site, here) => {
   case NumNodes => n
 })
 
+// Note, the main prerequisite for supporting an additional clock domain in a
+// FireSim simulation is to supply an additional clock parameter
+// (RationalClock) to the clock bridge (RationalClockBridge). The bridge
+// produces a vector of clocks, based on the provided parameter list, which you
+// may use freely without further modifications to your target design.
+case class FireSimClockParameters(additionalClocks: Seq[RationalClock]) {
+  def numClocks(): Int = additionalClocks.size + 1
+}
+case object FireSimClockKey extends Field[FireSimClockParameters](FireSimClockParameters(Seq()))
+
 // Hacky: Set before each node is generated. Ideally we'd give IO binders
 // accesses to the the Harness's parameters instance. We could then alter that.
 object NodeIdx {
@@ -28,33 +42,125 @@ object NodeIdx {
   def apply(): Int = idx
 }
 
-class FireSim(implicit val p: Parameters) extends RawModule {
+class WithFireSimSimpleClocks extends Config((site, here, up) => {
-  freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary())
+  case ClockingSchemeKey => { chiptop: ChipTop =>
-  val clockBridge = Module(new RationalClockBridge)
+    implicit val p = chiptop.p
-  val clock = clockBridge.io.clocks.head
+
-  val reset = WireInit(false.B)
+    val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
-  withClockAndReset(clock, reset) {
+    chiptop.implicitClockSinkNode := implicitClockSourceNode
-    // Instantiate multiple instances of the DUT to implement supernode
+
-    val targets = Seq.fill(p(NumNodes)) {
+    // Drive the diplomaticclock graph of the DigitalTop (if present)
-      // It's not a RC bump without some hacks...
+    val simpleClockGroupSourceNode = chiptop.lSystem match {
-      // Copy the AsyncClockGroupsKey to generate a fresh node on each
+      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
-      // instantiation of the dut, otherwise the initial instance will be
+        val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
-      // reused across each node
+        l.asyncClockGroupsNode := n
-      import freechips.rocketchip.subsystem.AsyncClockGroupsKey
+        Some(n)
-      val lazyModule = p(BuildSystem)(p.alterPartial({
+      }
-        case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy
+      case _ => None
-      }))
-      (lazyModule, Module(lazyModule.module))
     }
 
-    val peekPokeBridge = PeekPokeBridge(clock, reset)
+    InModuleBody {
-    // A Seq of partial functions that will instantiate the right bridge only
+      val clock = IO(Input(Clock())).suggestName("clock")
-    // if that Mixin trait is present in the target's LazyModule class instance
+      val reset = IO(Input(Reset())).suggestName("reset")
-    //
+
-    // Apply each partial function to each DUT instance
+      implicitClockSourceNode.out.unzip._1.map { o =>
-    for ((lazyModule, module) <- targets) {
+        o.clock := clock
-      p(IOBinders).values.foreach(f => f(lazyModule) ++ f(module))
+        o.reset := reset
-      NodeIdx.increment()
+      }
+
+      simpleClockGroupSourceNode.map { n => n.out.unzip._1.map { out: ClockGroupBundle =>
+        out.member.data.foreach { o =>
+          o.clock := clock
+          o.reset := reset
+        }
+      }}
+
+      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
+        clock := th.harnessClock
+        reset := th.harnessReset
+        Nil
+      })
     }
   }
+})
+
+class WithFireSimRationalTileDomain(multiplier: Int, divisor: Int) extends Config((site, here, up) => {
+  case FireSimClockKey => FireSimClockParameters(Seq(RationalClock("TileDomain", multiplier, divisor)))
+  case ClockingSchemeKey => { chiptop: ChipTop =>
+    implicit val p = chiptop.p
+
+    val implicitClockSourceNode = ClockSourceNode(Seq(ClockSourceParameters()))
+    chiptop.implicitClockSinkNode := implicitClockSourceNode
+
+    // Drive the diplomaticclock graph of the DigitalTop (if present)
+    val simpleClockGroupSourceNode = chiptop.lSystem match {
+      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) => {
+        val n = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+        l.asyncClockGroupsNode := n
+        Some(n)
+      }
+      case _ => None
+    }
+
+    InModuleBody {
+      val uncore_clock = IO(Input(Clock())).suggestName("uncore_clock")
+      val tile_clock   = IO(Input(Clock())).suggestName("tile_clock")
+      val reset        = IO(Input(Reset())).suggestName("reset")
+
+      implicitClockSourceNode.out.unzip._1.map { o =>
+        o.clock := uncore_clock
+        o.reset := reset
+      }
+
+      simpleClockGroupSourceNode.map { n => n.out.unzip._1.map { out: ClockGroupBundle =>
+        out.member.elements.map { case (name, data) =>
+          // This is mega hacks, how are you actually supposed to do this?
+          if (name.contains("core")) {
+            data.clock := tile_clock
+            data.reset := ResetCatchAndSync(tile_clock, reset.asBool)
+          } else {
+            data.clock := uncore_clock
+            data.reset := reset
+          }
+        }
+      }}
+
+      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
+        uncore_clock := th.harnessClock
+        reset        := th.harnessReset
+        th match {
+          case f: FireSim => tile_clock := f.additionalClocks(0)
+          case _ => throw new Exception("FireSimMultiClock must be used with FireSim")
+        }
+        Nil
+      })
+    }
+  }
+})
+
+class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSignalReferences {
+  freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary())
+  val clockBridge = Module(new RationalClockBridge(p(FireSimClockKey).additionalClocks:_*))
+  val harnessClock = clockBridge.io.clocks.head // This is the reference clock
+  val additionalClocks = clockBridge.io.clocks.tail
+  val harnessReset = WireInit(false.B)
+  val peekPokeBridge = PeekPokeBridge(harnessClock, harnessReset)
+  def dutReset = { require(false, "dutReset should not be used in Firesim"); false.B }
+  def success = { require(false, "success should not be used in Firesim"); false.B }
+
+  // Instantiate multiple instances of the DUT to implement supernode
+  for (i <- 0 until p(NumNodes)) {
+    // It's not a RC bump without some hacks...
+    // Copy the AsyncClockGroupsKey to generate a fresh node on each
+    // instantiation of the dut, otherwise the initial instance will be
+    // reused across each node
+    import freechips.rocketchip.subsystem.AsyncClockGroupsKey
+    val lazyModule = LazyModule(p(BuildTop)(p.alterPartial({
+      case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy
+    })))
+    val module = Module(lazyModule.module)
+    require(lazyModule.harnessFunctions.size == 1, "There should only be 1 harness function to connect clock+reset")
+    lazyModule.harnessFunctions.foreach(_(this))
+    NodeIdx.increment()
+  }
 }

generators/firechip/src/main/scala/FireSimMulticlockPOC.scala

@@ -1,107 +0,0 @@
-//See LICENSE for license details.
-
-package firesim.firesim
-
-import chisel3._
-
-import freechips.rocketchip.config.{Field, Config, Parameters}
-import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, RationalCrossing}
-import freechips.rocketchip.subsystem._
-import freechips.rocketchip.util.{ResetCatchAndSync}
-
-import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge, RationalClock}
-import firesim.configs._
-
-import boom.common.{WithRationalBoomTiles}
-
-import chipyard.{BuildSystem, DigitalTop, DigitalTopModule}
-import chipyard.config.ConfigValName._
-import chipyard.iobinders.{IOBinders}
-
-// WIP! This file is a sketch of one means of defining a multiclock target-design
-// that can be simulated in FireSim, pending a canonicalized form in Chipyard.
-//
-// Note, the main prerequisite for supporting an additional clock domain in a
-// FireSim simulation is to supply an additional clock parameter
-// (RationalClock) to the clock bridge (RationalClockBridge). The bridge
-// produces a vector of clocks, based on the provided parameter list, which you
-// may use freely without further modifications to your target design.
-
-case class FireSimClockParameters(additionalClocks: Seq[RationalClock]) {
-  def numClocks(): Int = additionalClocks.size + 1
-}
-case object FireSimClockKey extends Field[FireSimClockParameters](FireSimClockParameters(Seq()))
-
-trait HasAdditionalClocks extends LazyModuleImp {
-  val clocks = IO(Vec(p(FireSimClockKey).numClocks, Input(Clock())))
-}
-
-// Presupposes only 1 or 2 clocks.
-trait HasFireSimClockingImp extends HasAdditionalClocks {
-  val outer: HasTiles
-  val (tileClock, tileReset) = p(FireSimClockKey).additionalClocks.headOption match {
-    case Some(RationalClock(_, numer, denom)) if numer != denom => (clocks(1), ResetCatchAndSync(clocks(1), reset.toBool))
-    case None => (clocks.head, reset)
-  }
-
-  outer.tiles.foreach({ case tile =>
-    tile.module.clock := tileClock
-    tile.module.reset := tileReset
-  })
-}
-
-// Config Fragment
-class WithSingleRationalTileDomain(multiplier: Int, divisor: Int) extends Config(
-  new WithRationalRocketTiles ++
-  new WithRationalBoomTiles ++
-  new Config((site, here, up) => {
-    case FireSimClockKey => FireSimClockParameters(Seq(RationalClock("TileDomain", multiplier, divisor)))
-  })
-)
-
-class HalfRateUncore extends WithSingleRationalTileDomain(2,1)
-
-class WithFiresimMulticlockTop extends Config((site, here, up) => {
-  case BuildSystem => (p: Parameters) => LazyModule(new FiresimMulticlockTop()(p)).suggestName("system")
-})
-
-// Complete Config
-class FireSimQuadRocketMulticlockConfig extends Config(
-  new HalfRateUncore ++
-  new WithFiresimMulticlockTop ++
-  new FireSimQuadRocketConfig)
-
-// Top Definition
-class FiresimMulticlockTop(implicit p: Parameters) extends chipyard.DigitalTop
-{
-  override lazy val module = new FiresimMulticlockTopModule(this)
-}
-
-
-class FiresimMulticlockTopModule[+L <: DigitalTop](l: L) extends chipyard.DigitalTopModule(l) with HasFireSimClockingImp
-
-// Harness Definition
-class FireSimMulticlockPOC(implicit val p: Parameters) extends RawModule {
-  freechips.rocketchip.util.property.cover.setPropLib(new midas.passes.FireSimPropertyLibrary())
-  val clockBridge = Module(new RationalClockBridge(p(FireSimClockKey).additionalClocks:_*))
-  val refClock = clockBridge.io.clocks.head
-  val reset = WireInit(false.B)
-  withClockAndReset(refClock, reset) {
-    // Instantiate multiple instances of the DUT to implement supernode
-    val targets = Seq.fill(p(NumNodes)) {
-      val lazyModule = p(BuildSystem)(p)
-      (lazyModule, Module(lazyModule.module))
-    }
-    val peekPokeBridge = PeekPokeBridge(refClock, reset)
-    // A Seq of partial functions that will instantiate the right bridge only
-    // if that Mixin trait is present in the target's class instance
-    //
-    // Apply each partial function to each DUT instance
-    for ((lazyModule, module) <- targets) {
-      p(IOBinders).values.foreach(f => f(lazyModule) ++ f(module))
-    }
-    targets.collect({ case (_, t: HasAdditionalClocks) => t.clocks := clockBridge.io.clocks })
-  }
-}
-
-

generators/firechip/src/main/scala/TargetConfigs.scala

@@ -10,7 +10,7 @@ import freechips.rocketchip.tile._
 import freechips.rocketchip.tilelink._
 import freechips.rocketchip.rocket.DCacheParams
 import freechips.rocketchip.subsystem._
-import freechips.rocketchip.devices.tilelink.BootROMParams
+import freechips.rocketchip.devices.tilelink.{BootROMLocated, BootROMParams}
 import freechips.rocketchip.devices.debug.{DebugModuleParams, DebugModuleKey}
 import freechips.rocketchip.diplomacy.LazyModule
 import testchipip.{BlockDeviceKey, BlockDeviceConfig, SerialKey, TracePortKey, TracePortParams}
@@ -22,19 +22,18 @@ import testchipip.WithRingSystemBus
 
 import firesim.bridges._
 import firesim.configs._
-import chipyard.config.ConfigValName._
 
 class WithBootROM extends Config((site, here, up) => {
-  case BootROMParams => {
+  case BootROMLocated(x) => {
     val chipyardBootROM = new File(s"./generators/testchipip/bootrom/bootrom.rv${site(XLen)}.img")
     val firesimBootROM = new File(s"./target-rtl/chipyard/generators/testchipip/bootrom/bootrom.rv${site(XLen)}.img")
 
-     val bootROMPath = if (chipyardBootROM.exists()) {
+    val bootROMPath = if (chipyardBootROM.exists()) {
       chipyardBootROM.getAbsolutePath()
     } else {
       firesimBootROM.getAbsolutePath()
     }
-    BootROMParams(contentFileName = bootROMPath)
+    up(BootROMLocated(x), site).map(_.copy(contentFileName = bootROMPath))
   }
 })
 
@@ -67,6 +66,8 @@ class WithNVDLASmall extends nvidia.blocks.dla.WithNVDLA("small")
 
 // Tweaks that are generally applied to all firesim configs
 class WithFireSimConfigTweaks extends Config(
+  // Required*: Uses FireSim ClockBridge and PeekPokeBridge to drive the system with a single clock/reset
+  new WithFireSimSimpleClocks ++
   // Required*: When using FireSim-as-top to provide a correct path to the target bootrom source
   new WithBootROM ++
   // Optional*: Removing this will require adjusting the UART baud rate and
@@ -186,3 +187,14 @@ class FireSimArianeConfig extends Config(
   new WithDefaultMemModel ++
   new WithFireSimConfigTweaks ++
   new chipyard.ArianeConfig)
+
+//**********************************************************************************
+//* Multiclock Configurations
+//*********************************************************************************/
+class FireSimMulticlockRocketConfig extends Config(
+  new WithFireSimRationalTileDomain(2, 1) ++
+  new WithDefaultFireSimBridges ++
+  new WithDefaultMemModel ++
+  new WithFireSimConfigTweaks ++
+  new chipyard.DividedClockRocketConfig)
+

generators/firechip/src/test/scala/ScalaTestSuite.scala

@@ -106,8 +106,8 @@ class BoomF1Tests extends FireSimTestSuite("FireSim", "DDR3FRFCFSLLC4MB_FireSimL
 class RocketNICF1Tests extends FireSimTestSuite("FireSim", "WithNIC_DDR3FRFCFSLLC4MB_FireSimRocketConfig", "BaseF1Config")
 // Multiclock tests
 class RocketMulticlockF1Tests extends FireSimTestSuite(
-  "FireSimMulticlockPOC",
+  "FireSim",
-  "FireSimQuadRocketMulticlockConfig",
+  "FireSimMulticlockRocketConfig",
   "WithSynthAsserts_BaseF1Config")
 
 class ArianeF1Tests extends FireSimTestSuite("FireSim", "WithNIC_DDR3FRFCFSLLC4MB_FireSimArianeConfig", "BaseF1Config")

generators/tracegen/src/main/scala/System.scala

@@ -12,6 +12,10 @@ class TraceGenSystem(implicit p: Parameters) extends BaseSubsystem
     with CanHaveMasterAXI4MemPort {
 
   def coreMonitorBundles = Nil
+  val tileStatusNodes = tiles.collect {
+    case t: GroundTestTile => t.statusNode.makeSink()
+    case t: BoomTraceGenTile => t.statusNode.makeSink()
+  }
   override lazy val module = new TraceGenSystemModuleImp(this)
 }
 
@@ -20,12 +24,8 @@ class TraceGenSystemModuleImp(outer: TraceGenSystem)
 {
   val success = IO(Output(Bool()))
 
-  outer.tiles.zipWithIndex.map { case(t, i) => t.module.constants.hartid := i.U }
+  val status = dontTouch(DebugCombiner(outer.tileStatusNodes.map(_.bundle)))
 
-  val status = dontTouch(DebugCombiner(outer.tiles.collect {
-    case t: GroundTestTile => t.module.status
-    case t: BoomTraceGenTile => t.module.status
-  }))
   success := outer.tileCeaseSinkNode.in.head._1.asUInt.andR
 
 }

generators/tracegen/src/main/scala/Tile.scala

@@ -3,7 +3,7 @@ package tracegen
 import chisel3._
 import chisel3.util._
 import freechips.rocketchip.config.Parameters
-import freechips.rocketchip.diplomacy.{SimpleDevice, LazyModule, SynchronousCrossing, ClockCrossingType}
+import freechips.rocketchip.diplomacy.{SimpleDevice, LazyModule, SynchronousCrossing, ClockCrossingType, BundleBridgeSource}
 import freechips.rocketchip.groundtest._
 import freechips.rocketchip.rocket._
 import freechips.rocketchip.rocket.constants.{MemoryOpConstants}
@@ -206,11 +206,13 @@ class BoomTraceGenTile private(
   val cpuDevice: SimpleDevice = new SimpleDevice("groundtest", Nil)
   val intOutwardNode: IntOutwardNode = IntIdentityNode()
   val slaveNode: TLInwardNode = TLIdentityNode()
+  val statusNode = BundleBridgeSource(() => new GroundTestStatus)
 
   val boom_params = p.alterMap(Map(TileKey -> BoomTileParams(
     dcache=params.dcache,
-    core=BoomCoreParams(nPMPs=0, numLdqEntries=32, numStqEntries=32, useVM=false))))
+    core=BoomCoreParams(nPMPs=0, numLdqEntries=16, numStqEntries=16, useVM=false))))
-  val dcache = LazyModule(new BoomNonBlockingDCache(hartId)(boom_params))
+  val dcache = LazyModule(new BoomNonBlockingDCache(staticIdForMetadataUseOnly)(boom_params))
+
 
   val masterNode: TLOutwardNode = TLIdentityNode() := visibilityNode := dcache.node
 
@@ -220,11 +222,11 @@ class BoomTraceGenTile private(
 class BoomTraceGenTileModuleImp(outer: BoomTraceGenTile)
   extends BaseTileModuleImp(outer){
 
-  val status = IO(new GroundTestStatus)
+  val status = outer.statusNode.bundle
   val halt_and_catch_fire = None
 
   val tracegen = Module(new TraceGenerator(outer.params.traceParams))
-  tracegen.io.hartid := constants.hartid
+  tracegen.io.hartid := outer.hartIdSinkNode.bundle
 
   val ptw = Module(new DummyPTW(1))
   val lsu = Module(new LSU()(outer.boom_params, outer.dcache.module.edge))

vlsi/Makefile

@@ -34,7 +34,7 @@ INPUT_CONFS        ?= $(if $(filter $(tech_name),nangate45),\
                         example-asap7.yml)
 HAMMER_EXEC        ?= example-vlsi
 VLSI_TOP           ?= $(TOP)
-VLSI_HARNESS_DUT_NAME ?= dut
+VLSI_HARNESS_DUT_NAME ?= chiptop
 VLSI_OBJ_DIR       ?= $(vlsi_dir)/build
 ifneq ($(CUSTOM_VLOG),)
 	OBJ_DIR          ?= $(VLSI_OBJ_DIR)/custom-$(VLSI_TOP)