Merge pull request #900 from ucb-bar/clocking-overhaul

Switch PRCI to HarnessBinder/IOBinders

fpga/src/main/scala/arty/TestHarness.scala

@@ -7,7 +7,7 @@ import freechips.rocketchip.config.{Parameters}
 
 import sifive.fpgashells.shell.xilinx.artyshell.{ArtyShell}
 
-import chipyard.{BuildTop, HasHarnessSignalReferences, HasTestHarnessFunctions}
+import chipyard.{BuildTop, HasHarnessSignalReferences}
 import chipyard.harness.{ApplyHarnessBinders}
 import chipyard.iobinders.{HasIOBinders}
 
@@ -34,9 +34,6 @@ class ArtyFPGATestHarness(override implicit val p: Parameters) extends ArtyShell
   val dutReset = dReset
 
   // must be after HasHarnessSignalReferences assignments
-  lazyDut match { case d: HasTestHarnessFunctions =>
-    d.harnessFunctions.foreach(_(this))
-  }
   lazyDut match { case d: HasIOBinders =>
     ApplyHarnessBinders(this, d.lazySystem, d.portMap)
   }

fpga/src/main/scala/vcu118/TestHarness.scala

@@ -17,7 +17,7 @@ import sifive.blocks.devices.uart._
 import sifive.blocks.devices.spi._
 import sifive.blocks.devices.gpio._
 
-import chipyard.{HasHarnessSignalReferences, HasTestHarnessFunctions, BuildTop, ChipTop, ExtTLMem, CanHaveMasterTLMemPort, DefaultClockFrequencyKey, HasReferenceClockFreq}
+import chipyard.{HasHarnessSignalReferences, BuildTop, ChipTop, ExtTLMem, CanHaveMasterTLMemPort, DefaultClockFrequencyKey}
 import chipyard.iobinders.{HasIOBinders}
 import chipyard.harness.{ApplyHarnessBinders}
 
@@ -129,17 +129,11 @@ class VCU118FPGATestHarnessImp(_outer: VCU118FPGATestHarness) extends LazyRawMod
   childReset := buildtopReset
 
   // harness binders are non-lazy
-  _outer.topDesign match { case d: HasTestHarnessFunctions =>
-    d.harnessFunctions.foreach(_(this))
-  }
   _outer.topDesign match { case d: HasIOBinders =>
     ApplyHarnessBinders(this, d.lazySystem, d.portMap)
   }
 
   // check the top-level reference clock is equal to the default
   // non-exhaustive since you need all ChipTop clocks to equal the default
-  _outer.topDesign match {
-    case d: HasReferenceClockFreq => require(d.refClockFreqMHz == p(DefaultClockFrequencyKey))
-    case _ =>
-  }
+  require(getRefClockFreq == p(DefaultClockFrequencyKey))
 }

generators/chipyard/src/main/scala/ChipTop.scala

@@ -15,10 +15,6 @@ import barstools.iocell.chisel._
 
 case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => new DigitalTop()(p))
 
-trait HasReferenceClockFreq {
-  def refClockFreqMHz: Double
-}
-
 /**
  * The base class used for building chips. This constructor instantiates a module specified by the BuildSystem parameter,
  * named "system", which is an instance of DigitalTop by default. The diplomatic clocks of System, as well as its implicit clock,
@@ -27,31 +23,14 @@ trait HasReferenceClockFreq {
  */
 
 class ChipTop(implicit p: Parameters) extends LazyModule with BindingScope
-    with HasTestHarnessFunctions with HasReferenceClockFreq with HasIOBinders {
+    with HasIOBinders {
   // The system module specified by BuildSystem
   lazy val lazySystem = LazyModule(p(BuildSystem)(p)).suggestName("system")
 
-  // The implicitClockSinkNode provides the implicit clock and reset for the system (connected by clocking scheme)
-  val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters(name = Some("implicit_clock"))))
-
-  // Generate Clocks and Reset
-  val mvRefClkFreq = p(ClockingSchemeKey)(this)
-  def refClockFreqMHz: Double = mvRefClkFreq.getWrappedValue
-
   // NOTE: Making this a LazyRawModule is moderately dangerous, as anonymous children
   // of ChipTop (ex: ClockGroup) do not receive clock or reset.
   // However. anonymous children of ChipTop should not need an implicit Clock or Reset
   // anyways, they probably need to be explicitly clocked.
-  lazy val module: LazyModuleImpLike = new LazyRawModuleImp(this) {
-    // These become the implicit clock and reset to the System
-    val implicit_clock = implicitClockSinkNode.in.head._1.clock
-    val implicit_reset = implicitClockSinkNode.in.head._1.reset
-
-    // Connect the implicit clock/reset, if present
-    lazySystem.module match { case l: LazyModuleImp => {
-      l.clock := implicit_clock
-      l.reset := implicit_reset
-    }}
-  }
+  lazy val module: LazyModuleImpLike = new LazyRawModuleImp(this) {  }
 }
 

generators/chipyard/src/main/scala/Clocks.scala

@@ -1,128 +0,0 @@
-package chipyard
-
-import chisel3._
-
-import scala.collection.mutable.{ArrayBuffer}
-
-import freechips.rocketchip.prci._
-import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey, InstantiatesTiles}
-import freechips.rocketchip.config.{Parameters, Field, Config}
-import freechips.rocketchip.diplomacy.{ModuleValue, OutwardNodeHandle, InModuleBody, LazyModule}
-import freechips.rocketchip.util.{ResetCatchAndSync}
-
-import barstools.iocell.chisel._
-import testchipip.{TLTileResetCtrl}
-
-import chipyard.clocking._
-import chipyard.iobinders._
-
-/**
- * A simple reset implementation that punches out reset ports
- * for standard Module classes. The ChipTop reset pin is Async.
- * Synchronization is performed in the ClockGroupResetSynchronizer
- */
-object GenerateReset {
-  def apply(chiptop: ChipTop, clock: Clock): Reset = {
-    implicit val p = chiptop.p
-    // this needs directionality so generateIOFromSignal works
-    val async_reset_wire = Wire(Input(AsyncReset()))
-    val (reset_io, resetIOCell) = IOCell.generateIOFromSignal(async_reset_wire, "reset", p(IOCellKey),
-      abstractResetAsAsync = true)
-
-    chiptop.iocells ++= resetIOCell
-    chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
-      reset_io := th.dutReset
-      Nil
-    })
-    async_reset_wire
-  }
-}
-
-
-case object ClockingSchemeKey extends Field[ChipTop => ModuleValue[Double]](ClockingSchemeGenerators.dividerOnlyClockGenerator)
-/*
-  * This is a Seq of assignment functions, that accept a clock name and return an optional frequency.
-  * Functions that appear later in this seq have higher precedence that earlier ones.
-  * If no function returns a non-empty value, the value specified in
-  * [[DefaultClockFrequencyKey]] will be used.
-  */
-case object ClockFrequencyAssignersKey extends Field[Seq[(String) => Option[Double]]](Seq.empty)
-case object DefaultClockFrequencyKey extends Field[Double]()
-
-class ClockNameMatchesAssignment(name: String, fMHz: Double) extends Config((site, here, up) => {
-  case ClockFrequencyAssignersKey => up(ClockFrequencyAssignersKey, site) ++
-    Seq((cName: String) => if (cName == name) Some(fMHz) else None)
-})
-
-class ClockNameContainsAssignment(name: String, fMHz: Double) extends Config((site, here, up) => {
-  case ClockFrequencyAssignersKey => up(ClockFrequencyAssignersKey, site) ++
-    Seq((cName: String) => if (cName.contains(name)) Some(fMHz) else None)
-})
-
-object ClockingSchemeGenerators {
-  val dividerOnlyClockGenerator: ChipTop => ModuleValue[Double] = { chiptop =>
-    implicit val p = chiptop.p
-
-    // Requires existence of undriven asyncClockGroups in subsystem
-    val systemAsyncClockGroup = chiptop.lazySystem match {
-      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) =>
-        l.asyncClockGroupsNode
-    }
-
-    // Add a control register for each tile's reset
-    val resetSetter = chiptop.lazySystem match {
-      case sys: BaseSubsystem with InstantiatesTiles => Some(TLTileResetCtrl(sys))
-      case _ => None
-    }
-    val resetSetterResetProvider = resetSetter.map(_.tileResetProviderNode).getOrElse(ClockGroupEphemeralNode())
-
-    val aggregator = LazyModule(new ClockGroupAggregator("allClocks")).node
-    // provides the implicit clock to the system
-    (chiptop.implicitClockSinkNode
-      := ClockGroup()
-      := aggregator)
-    // provides the system clock (ex. the bus clocks)
-    (systemAsyncClockGroup
-      :*= ClockGroupNamePrefixer()
-      :*= aggregator)
-
-    val referenceClockSource =  ClockSourceNode(Seq(ClockSourceParameters()))
-    val dividerOnlyClkGenerator = LazyModule(new DividerOnlyClockGenerator("buildTopClockGenerator"))
-    // provides all the divided clocks (from the top-level clock)
-    (aggregator
-      := ClockGroupFrequencySpecifier(p(ClockFrequencyAssignersKey), p(DefaultClockFrequencyKey))
-      := ClockGroupResetSynchronizer()
-      := resetSetterResetProvider
-      := dividerOnlyClkGenerator.node
-      := referenceClockSource)
-
-    val asyncResetBroadcast = FixedClockBroadcast(None)
-    resetSetter.foreach(_.asyncResetSinkNode := asyncResetBroadcast)
-    val asyncResetSource = ClockSourceNode(Seq(ClockSourceParameters()))
-    asyncResetBroadcast := asyncResetSource
-
-    InModuleBody {
-      val clock_wire = Wire(Input(Clock()))
-      val reset_wire = GenerateReset(chiptop, clock_wire)
-      val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, "clock", p(IOCellKey))
-      chiptop.iocells ++= clockIOCell
-
-      referenceClockSource.out.unzip._1.map { o =>
-        o.clock := clock_wire
-        o.reset := reset_wire
-      }
-
-      asyncResetSource.out.unzip._1.map { o =>
-        o.clock := false.B.asClock // async reset broadcast network does not provide a clock
-        o.reset := reset_wire
-      }
-
-      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
-        clock_io := th.buildtopClock
-        Nil })
-
-      // return the reference frequency
-      dividerOnlyClkGenerator.module.referenceFreq
-    }
-  }
-}

generators/chipyard/src/main/scala/ConfigFragments.scala

@@ -31,6 +31,7 @@ import sifive.blocks.devices.uart._
 import sifive.blocks.devices.spi._
 
 import chipyard._
+import chipyard.clocking._
 
 // -----------------------
 // Common Config Fragments
@@ -68,8 +69,10 @@ class WithL2TLBs(entries: Int) extends Config((site, here, up) => {
   }
 })
 
+class TraceGenTop(implicit p: Parameters) extends TraceGenSystem
+  with HasChipyardPRCI
 class WithTracegenSystem extends Config((site, here, up) => {
-  case BuildSystem => (p: Parameters) => new TraceGenSystem()(p)
+  case BuildSystem => (p: Parameters) => new TraceGenTop()(p)
 })
 
 /**

generators/chipyard/src/main/scala/DigitalTop.scala

@@ -31,6 +31,7 @@ class DigitalTop(implicit p: Parameters) extends ChipyardSystem
   with chipyard.example.CanHavePeripheryStreamingFIR // Enables optionally adding the DSPTools FIR example widget
   with chipyard.example.CanHavePeripheryStreamingPassthrough // Enables optionally adding the DSPTools streaming-passthrough example widget
   with nvidia.blocks.dla.CanHavePeripheryNVDLA // Enables optionally having an NVDLA
+  with chipyard.clocking.HasChipyardPRCI // Use Chipyard reset/clock distribution
 {
   override lazy val module = new DigitalTopModule(this)
 }

generators/chipyard/src/main/scala/HarnessBinders.scala

@@ -22,7 +22,8 @@ import barstools.iocell.chisel._
 import testchipip._
 
 import chipyard.{HasHarnessSignalReferences, HarnessClockInstantiatorKey}
-import chipyard.iobinders.{GetSystemParameters, JTAGChipIO}
+import chipyard.clocking.{HasChipyardPRCI}
+import chipyard.iobinders.{GetSystemParameters, JTAGChipIO, ClockWithFreq}
 
 import tracegen.{TraceGenSystemModuleImp}
 import icenet.{CanHavePeripheryIceNIC, SimNetwork, NicLoopback, NICKey, NICIOvonly}
@@ -322,8 +323,23 @@ class WithSimDromajoBridge extends ComposeHarnessBinder({
   }
 })
 
-class WithTieOffCustomBootPin extends OverrideHarnessBinder({
+class WithCustomBootPinPlusArg extends OverrideHarnessBinder({
   (system: CanHavePeripheryCustomBootPin, th: HasHarnessSignalReferences, ports: Seq[Bool]) => {
-    ports.foreach(_ := false.B)
+    val pin = PlusArg("custom_boot_pin", width=1)
+    ports.foreach(_ := pin)
+  }
+})
+
+
+class WithClockAndResetFromHarness extends OverrideHarnessBinder({
+  (system: HasChipyardPRCI, th: HasHarnessSignalReferences, ports: Seq[Data]) => {
+    implicit val p = GetSystemParameters(system)
+    ports.map ({
+      case c: ClockWithFreq => {
+        th.setRefClockFreq(c.freqMHz)
+        c.clock := th.buildtopClock
+      }
+      case r: AsyncReset => r := th.buildtopReset.asAsyncReset
+    })
   }
 })

generators/chipyard/src/main/scala/IOBinders.scala

@@ -11,7 +11,7 @@ import freechips.rocketchip.subsystem._
 import freechips.rocketchip.system.{SimAXIMem}
 import freechips.rocketchip.amba.axi4.{AXI4Bundle, AXI4SlaveNode, AXI4MasterNode, AXI4EdgeParameters}
 import freechips.rocketchip.util._
-import freechips.rocketchip.prci.{ClockSinkNode, ClockSinkParameters}
+import freechips.rocketchip.prci._
 import freechips.rocketchip.groundtest.{GroundTestSubsystemModuleImp, GroundTestSubsystem}
 
 import sifive.blocks.devices.gpio._
@@ -23,6 +23,7 @@ import barstools.iocell.chisel._
 
 import testchipip._
 import icenet.{CanHavePeripheryIceNIC, SimNetwork, NicLoopback, NICKey, NICIOvonly}
+import chipyard.clocking.{HasChipyardPRCI, DividerOnlyClockGenerator}
 
 import scala.reflect.{ClassTag}
 
@@ -384,3 +385,45 @@ class WithDontTouchPorts extends OverrideIOBinder({
   (system: DontTouch) => system.dontTouchPorts(); (Nil, Nil)
 })
 
+class ClockWithFreq(val freqMHz: Double) extends Bundle {
+  val clock = Clock()
+}
+
+class WithDividerOnlyClockGenerator extends OverrideLazyIOBinder({
+  (system: HasChipyardPRCI) => {
+    // Connect the implicit clock
+    implicit val p = GetSystemParameters(system)
+    val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters(name = Some("implicit_clock"))))
+    system.connectImplicitClockSinkNode(implicitClockSinkNode)
+    InModuleBody {
+      val implicit_clock = implicitClockSinkNode.in.head._1.clock
+      val implicit_reset = implicitClockSinkNode.in.head._1.reset
+      system.asInstanceOf[BaseSubsystem].module match { case l: LazyModuleImp => {
+        l.clock := implicit_clock
+        l.reset := implicit_reset
+      }}
+    }
+
+    // Connect all other requested clocks
+    val referenceClockSource = ClockSourceNode(Seq(ClockSourceParameters()))
+    val dividerOnlyClockGen = LazyModule(new DividerOnlyClockGenerator("buildTopClockGenerator"))
+
+    (system.allClockGroupsNode
+      := dividerOnlyClockGen.node
+      := referenceClockSource)
+
+    InModuleBody {
+      val clock_wire = Wire(Input(new ClockWithFreq(dividerOnlyClockGen.module.referenceFreq)))
+      val reset_wire = Wire(Input(AsyncReset()))
+      val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, "clock", p(IOCellKey))
+      val (reset_io, resetIOCell) = IOCell.generateIOFromSignal(reset_wire, "reset", p(IOCellKey))
+
+      referenceClockSource.out.unzip._1.map { o =>
+        o.clock := clock_wire.clock
+        o.reset := reset_wire
+      }
+
+      (Seq(clock_io, reset_io), clockIOCell ++ resetIOCell)
+    }
+  }
+})

generators/chipyard/src/main/scala/TestHarness.scala

@@ -17,13 +17,14 @@ import chipyard.clocking.{SimplePllConfiguration, ClockDividerN}
 // -------------------------------
 
 case object BuildTop extends Field[Parameters => LazyModule]((p: Parameters) => new ChipTop()(p))
-
-trait HasTestHarnessFunctions {
-  val harnessFunctions = ArrayBuffer.empty[HasHarnessSignalReferences => Seq[Any]]
-}
+case object DefaultClockFrequencyKey extends Field[Double](100.0) // MHz
 
 trait HasHarnessSignalReferences {
+  implicit val p: Parameters
   // clock/reset of the chiptop reference clock (can be different than the implicit harness clock/reset)
+  var refClockFreq: Double = p(DefaultClockFrequencyKey)
+  def setRefClockFreq(freqMHz: Double) = { refClockFreq = freqMHz }
+  def getRefClockFreq: Double = refClockFreq
   def buildtopClock: Clock
   def buildtopReset: Reset
   def dutReset: Reset
@@ -90,25 +91,18 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSign
 
   io.success := false.B
 
-  val freqMHz = lazyDut match {
-    case d: HasReferenceClockFreq => d.refClockFreqMHz
-    case _ => p(DefaultClockFrequencyKey)
-  }
-  val refClkBundle = p(HarnessClockInstantiatorKey).requestClockBundle("buildtop_reference_clock", freqMHz * (1000 * 1000))
-
-  buildtopClock := refClkBundle.clock
-  buildtopReset := WireInit(refClkBundle.reset)
-  val dutReset = refClkBundle.reset.asAsyncReset
-
+  val dutReset = buildtopReset.asAsyncReset
   val success = io.success
 
-  lazyDut match { case d: HasTestHarnessFunctions =>
-    d.harnessFunctions.foreach(_(this))
-  }
   lazyDut match { case d: HasIOBinders =>
     ApplyHarnessBinders(this, d.lazySystem, d.portMap)
   }
 
+  val refClkBundle = p(HarnessClockInstantiatorKey).requestClockBundle("buildtop_reference_clock", getRefClockFreq * (1000 * 1000))
+
+  buildtopClock := refClkBundle.clock
+  buildtopReset := WireInit(refClkBundle.reset)
+
   val implicitHarnessClockBundle = Wire(new ClockBundle(ClockBundleParameters()))
   implicitHarnessClockBundle.clock := clock
   implicitHarnessClockBundle.reset := reset

generators/chipyard/src/main/scala/clocking/ClockGroupCombiner.scala

@@ -0,0 +1,81 @@
+package chipyard.clocking
+
+import chisel3._
+import chisel3.util._
+import chisel3.experimental.{Analog, IO}
+
+import freechips.rocketchip.config._
+import freechips.rocketchip.subsystem._
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.prci._
+import freechips.rocketchip.util._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.devices.tilelink._
+import freechips.rocketchip.regmapper._
+import freechips.rocketchip.subsystem._
+
+object ClockGroupCombiner {
+  def apply()(implicit p: Parameters, valName: ValName): ClockGroupAdapterNode = {
+    LazyModule(new ClockGroupCombiner()).node
+  }
+}
+
+case object ClockGroupCombinerKey extends Field[Seq[(String, ClockSinkParameters => Boolean)]](Nil)
+
+// All clock groups with a name containing any substring in names will be combined into a single clock group
+class WithClockGroupsCombinedByName(grouped_name: String, names: String*) extends Config((site, here, up) => {
+  case ClockGroupCombinerKey => {
+    val combiner: ClockSinkParameters => Boolean = { m => names.map(n => m.name.get.contains(n)).reduce(_||_) }
+    up(ClockGroupCombinerKey) ++ Seq((grouped_name, combiner))
+  }
+})
+
+/** This node combines sets of clock groups according to functions provided in the ClockGroupCombinerKey
+  * The ClockGroupCombinersKey contains a list of tuples of:
+  * - The name of the combined group
+  * - A function on the ClockSinkParameters, returning True if the associated clock group should be grouped by this node
+  * This node will fail if
+  * - Multiple grouping functions match a single clock group
+  * - A grouping function matches zero clock groups
+  * - A grouping function matches clock groups with different requested frequncies
+  */
+class ClockGroupCombiner(implicit p: Parameters, v: ValName) extends LazyModule {
+  val combiners = p(ClockGroupCombinerKey)
+  val sourceFn: ClockGroupSourceParameters => ClockGroupSourceParameters = { m => m }
+  val sinkFn: ClockGroupSinkParameters   => ClockGroupSinkParameters = { u =>
+    var i = 0
+    val (grouped, rest) = combiners.map(_._2).foldLeft((Seq[ClockSinkParameters](), u.members)) { case ((grouped, rest), c) =>
+      val (g, r) = rest.partition(c(_))
+      val name = combiners(i)._1
+      i = i + 1
+      require(g.size >= 1)
+      require(g.forall(_.take.get == g.head.take.get))
+      (grouped ++ Seq(ClockSinkParameters(take = g.head.take, name = Some(name))), r)
+    }
+    ClockGroupSinkParameters(
+      name = u.name,
+      members = grouped ++ rest
+    )
+  }
+
+
+  val node = ClockGroupAdapterNode(sourceFn, sinkFn)
+  lazy val module = new LazyRawModuleImp(this) {
+    (node.out zip node.in).map { case ((o, oe), (i, ie)) =>
+      {
+        val inMap = (i.member.data zip ie.sink.members).map { case (id, im) =>
+          im.name.get -> id
+        }.toMap
+        (o.member.data zip oe.sink.members).map { case (od, om) =>
+          val matches = combiners.filter(c => c._2(om))
+          require(matches.size <= 1)
+          if (matches.size == 0) {
+            od := inMap(om.name.get)
+          } else {
+            od := inMap(matches(0)._1)
+          }
+        }
+      }
+    }
+  }
+}

generators/chipyard/src/main/scala/clocking/ClockGroupNamePrefixer.scala

@@ -2,10 +2,23 @@ package chipyard.clocking
 
 import chisel3._
 
-import freechips.rocketchip.config.{Parameters}
+import freechips.rocketchip.config.{Parameters, Config, Field}
 import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.prci._
 
+case object ClockFrequencyAssignersKey extends Field[Seq[(String) => Option[Double]]](Seq.empty)
+
+class ClockNameMatchesAssignment(name: String, fMHz: Double) extends Config((site, here, up) => {
+  case ClockFrequencyAssignersKey => up(ClockFrequencyAssignersKey, site) ++
+    Seq((cName: String) => if (cName == name) Some(fMHz) else None)
+})
+
+class ClockNameContainsAssignment(name: String, fMHz: Double) extends Config((site, here, up) => {
+  case ClockFrequencyAssignersKey => up(ClockFrequencyAssignersKey, site) ++
+    Seq((cName: String) => if (cName.contains(name)) Some(fMHz) else None)
+})
+
+
 /**
   * This sort of node can be used when it is a connectivity passthrough, but modifies
   * the flow of parameters (which may result in changing the names of the underlying signals).

generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala

@@ -0,0 +1,79 @@
+package chipyard.clocking
+
+import chisel3._
+
+import scala.collection.mutable.{ArrayBuffer}
+
+import freechips.rocketchip.config.{Parameters, Field, Config}
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.devices.tilelink._
+import freechips.rocketchip.regmapper._
+import freechips.rocketchip.subsystem._
+import freechips.rocketchip.util._
+import freechips.rocketchip.tile._
+import freechips.rocketchip.prci._
+
+import testchipip.{TLTileResetCtrl}
+import chipyard.{DefaultClockFrequencyKey}
+
+case class ChipyardPRCIControlParams(
+  slaveWhere: TLBusWrapperLocation = CBUS,
+  baseAddress: BigInt = 0x100000,
+  enableTileClockGating: Boolean = true
+)
+
+
+case object ChipyardPRCIControlKey extends Field[ChipyardPRCIControlParams](ChipyardPRCIControlParams())
+
+trait HasChipyardPRCI { this: BaseSubsystem with InstantiatesTiles =>
+  require(p(SubsystemDriveAsyncClockGroupsKey).isEmpty, "Subsystem asyncClockGroups must be undriven")
+
+  implicit val n = ValName("chipyardPRCI")
+
+  val prciParams = p(ChipyardPRCIControlKey)
+
+  // Set up clock domain
+  private val tlbus = locateTLBusWrapper(prciParams.slaveWhere)
+  val prci_ctrl_domain = LazyModule(new ClockSinkDomain(name=Some("chipyard-prci-control")))
+  prci_ctrl_domain.clockNode := tlbus.fixedClockNode
+
+  // Aggregate all the clock groups into a single node
+  val aggregator = LazyModule(new ClockGroupAggregator("allClocks")).node
+  val allClockGroupsNode = ClockGroupEphemeralNode()
+
+  // There are two "sets" of clocks which must be dealt with
+
+  // 1. The implicit clock from the subsystem. RC is moving away from depending on this
+  //    clock, but some modules still use it. Since the implicit clock sink node
+  //    is created in the ChipTop (the hierarchy wrapping the subsystem), this function
+  //    is provided to allow connecting that clock to the clock aggregator. This function
+  //    should be called in the ChipTop context
+  def connectImplicitClockSinkNode(sink: ClockSinkNode) =
+    (sink
+      := ClockGroup()
+      := aggregator)
+
+  // 2. The rest of the diplomatic clocks in the subsystem are routed to this asyncClockGroupsNode
+  (asyncClockGroupsNode
+    :*= ClockGroupNamePrefixer()
+    :*= aggregator)
+
+
+  // Once all the clocks are gathered in the aggregator node, several steps remain
+  // 1. Assign frequencies to any clock groups which did not specify a frequency.
+  // 2. Combine duplicated clock groups (clock groups which physically should be in the same clock domain)
+  // 3. Synchronize reset to each clock group
+  // 4. Clock gate the clock groups corresponding to Tiles (if desired).
+  // 5. Add reset control registers to the tiles (if desired)
+  // The final clock group here contains physically distinct clock domains, which some PRCI node in a
+  // diplomatic IOBinder should drive
+  (aggregator
+    := ClockGroupFrequencySpecifier(p(ClockFrequencyAssignersKey), p(DefaultClockFrequencyKey))
+    := ClockGroupCombiner()
+    := ClockGroupResetSynchronizer()
+    := TileClockGater(prciParams.baseAddress + 0x00000, tlbus, prciParams.enableTileClockGating)
+    := TileResetSetter(prciParams.baseAddress + 0x10000, tlbus, tile_prci_domains.map(_.tile_reset_domain.clockNode.portParams(0).name.get), Nil)
+    := allClockGroupsNode)
+}
+

generators/chipyard/src/main/scala/clocking/TileClockGater.scala

@@ -0,0 +1,55 @@
+package chipyard.clocking
+
+import chisel3._
+import chisel3.util._
+import chisel3.experimental.{Analog, IO}
+
+import freechips.rocketchip.config._
+import freechips.rocketchip.subsystem._
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.prci._
+import freechips.rocketchip.util._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.devices.tilelink._
+import freechips.rocketchip.regmapper._
+import freechips.rocketchip.subsystem._
+
+/** This node adds clock gating control registers. 
+  * If deploying on a platform which does not support clock gating, deasserting the enable
+  * flag will generate the registers, preserving the same memory map and behavior, but will not 
+  * generate any gaters
+  */
+class TileClockGater(address: BigInt, beatBytes: Int, enable: Boolean)(implicit p: Parameters, valName: ValName) extends LazyModule
+{
+  val device = new SimpleDevice(s"clock-gater", Nil)
+  val clockNode = ClockGroupIdentityNode()
+  val tlNode = TLRegisterNode(Seq(AddressSet(address, 4096-1)), device, "reg/control", beatBytes=beatBytes)
+  lazy val module = new LazyModuleImp(this) {
+    val sources = clockNode.in.head._1.member.data.toSeq
+    val sinks = clockNode.out.head._1.member.elements.toSeq
+    val nSinks = sinks.size
+    val regs = (0 until nSinks).map({i =>
+      val sinkName = sinks(i)._1
+      val reg = withReset(sources(i).reset) { Module(new AsyncResetRegVec(w=1, init=1)) }
+      if (sinkName.contains("tile") && enable) {
+        println(s"ClockGate for ${sinkName} regmapped at ${(address+i*4).toString(16)}")
+        sinks(i)._2.clock := ClockGate(sources(i).clock,  reg.io.q.asBool)
+        sinks(i)._2.reset := sources(i).reset
+      } else {
+        sinks(i)._2 := sources(i)
+      }
+      reg
+    })
+    tlNode.regmap((0 until nSinks).map({i =>
+      i*4 -> Seq(RegField.rwReg(1, regs(i).io))
+    }): _*)
+  }
+}
+
+object TileClockGater {
+  def apply(address: BigInt, tlbus: TLBusWrapper, enable: Boolean)(implicit p: Parameters, v: ValName) = {
+    val gater = LazyModule(new TileClockGater(address, tlbus.beatBytes, enable))
+    tlbus.toVariableWidthSlave(Some("clock-gater")) { gater.tlNode := TLBuffer() }
+    gater.clockNode
+  }
+}

generators/chipyard/src/main/scala/clocking/TileResetSetter.scala

@@ -0,0 +1,72 @@
+package chipyard.clocking
+
+import chisel3._
+import chisel3.util._
+import chisel3.experimental.{Analog, IO}
+
+import freechips.rocketchip.config._
+import freechips.rocketchip.subsystem._
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.prci._
+import freechips.rocketchip.util._
+import freechips.rocketchip.tilelink._
+import freechips.rocketchip.devices.tilelink._
+import freechips.rocketchip.regmapper._
+import freechips.rocketchip.subsystem._
+
+// Currently only works if all tiles are already driven by independent clock groups
+// TODO: After https://github.com/chipsalliance/rocket-chip/pull/2842 is merged, we should
+// always put all tiles on independent clock groups
+class TileResetSetter(address: BigInt, beatBytes: Int, tileNames: Seq[String], initResetHarts: Seq[Int])(implicit p: Parameters)
+    extends LazyModule {
+  val device = new SimpleDevice("tile-reset-setter", Nil)
+  val tlNode = TLRegisterNode(Seq(AddressSet(address, 4096-1)), device, "reg/control", beatBytes=beatBytes)
+  val clockNode = ClockGroupIdentityNode()
+
+  lazy val module = new LazyModuleImp(this) {
+    val nTiles = p(TilesLocated(InSubsystem)).size
+    require (nTiles <= 4096 / 4)
+    val tile_async_resets = Wire(Vec(nTiles, Reset()))
+    val r_tile_resets = (0 until nTiles).map({ i =>
+      tile_async_resets(i) := true.B.asAsyncReset // Remove this line after https://github.com/chipsalliance/rocket-chip/pull/2842
+      withReset (tile_async_resets(i)) {
+        Module(new AsyncResetRegVec(w=1, init=(if (initResetHarts.contains(i)) 1 else 0)))
+      }
+    })
+    tlNode.regmap((0 until nTiles).map({ i =>
+      i * 4 -> Seq(RegField.rwReg(1, r_tile_resets(i).io)),
+    }): _*)
+
+    val tileMap = tileNames.zipWithIndex.map({ case (n, i) =>
+        n -> (tile_async_resets(i), r_tile_resets(i).io.q)
+    })
+
+    (clockNode.out zip clockNode.in).map { case ((o, _), (i, _)) =>
+      (o.member.elements zip i.member.elements).foreach { case ((name, oD), (_, iD)) =>
+        oD.clock := iD.clock
+        oD.reset := iD.reset
+        for ((n, (rIn, rOut)) <- tileMap) {
+          if (name.contains(n)) {
+            println(name, n)
+            // Async because the reset coming out of the AsyncResetRegVec is
+            // clocked to the bus this is attached to, not the clock in this
+            // clock bundle. We expect a ClockGroupResetSynchronizer downstream
+            // to synchronize the resets
+            // Also, this or enforces that the tiles come out of reset after the reset of the system
+            oD.reset := (rOut.asBool || iD.reset.asBool).asAsyncReset
+            rIn := iD.reset
+          }
+        }
+      }
+    }
+  }
+}
+
+
+object TileResetSetter {
+  def apply(address: BigInt, tlbus: TLBusWrapper, tileNames: Seq[String], initResetHarts: Seq[Int])(implicit p: Parameters, v: ValName) = {
+    val setter = LazyModule(new TileResetSetter(address, tlbus.beatBytes, tileNames, initResetHarts))
+    tlbus.toVariableWidthSlave(Some("tile-reset-setter")) { setter.tlNode := TLBuffer() }
+    setter.clockNode
+  }
+}

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -21,7 +21,8 @@ class AbstractConfig extends Config(
   new chipyard.harness.WithSimAXIMMIO ++                        // add SimAXIMem for axi4 mmio port, if enabled
   new chipyard.harness.WithTieOffInterrupts ++                  // tie-off interrupt ports, if present
   new chipyard.harness.WithTieOffL2FBusAXI ++                   // tie-off external AXI4 master, if present
-  new chipyard.harness.WithTieOffCustomBootPin ++
+  new chipyard.harness.WithCustomBootPinPlusArg ++
+  new chipyard.harness.WithClockAndResetFromHarness ++
 
   // The IOBinders instantiate ChipTop IOs to match desired digital IOs
   // IOCells are generated for "Chip-like" IOs, while simulation-only IOs are directly punched through
@@ -39,6 +40,7 @@ class AbstractConfig extends Config(
   new chipyard.iobinders.WithTraceIOPunchthrough ++
   new chipyard.iobinders.WithExtInterruptIOCells ++
   new chipyard.iobinders.WithCustomBootPin ++
+  new chipyard.iobinders.WithDividerOnlyClockGenerator ++
 
   new testchipip.WithDefaultSerialTL ++                          // use serialized tilelink port to external serialadapter/harnessRAM
   new chipyard.config.WithBootROM ++                             // use default bootrom

generators/chipyard/src/main/scala/config/TracegenConfigs.scala

@@ -6,8 +6,10 @@ import freechips.rocketchip.rocket.{DCacheParams}
 class AbstractTraceGenConfig extends Config(
   new chipyard.harness.WithBlackBoxSimMem ++
   new chipyard.harness.WithTraceGenSuccess ++
+  new chipyard.harness.WithClockAndResetFromHarness ++
   new chipyard.iobinders.WithAXI4MemPunchthrough ++
   new chipyard.iobinders.WithTraceGenSuccessPunchthrough ++
+  new chipyard.iobinders.WithDividerOnlyClockGenerator ++
   new chipyard.config.WithTracegenSystem ++
   new chipyard.config.WithNoSubsystemDrivenClocks ++
   new chipyard.config.WithPeripheryBusFrequencyAsDefault ++

generators/firechip/src/main/scala/FireSim.scala

@@ -10,7 +10,7 @@ import chisel3.experimental.{IO}
 import freechips.rocketchip.prci._
 import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
 import freechips.rocketchip.config.{Field, Config, Parameters}
-import freechips.rocketchip.diplomacy.{LazyModule, InModuleBody, ValName}
+import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, InModuleBody, ValName}
 import freechips.rocketchip.util.{ResetCatchAndSync, RecordMap}
 
 import midas.widgets.{Bridge, PeekPokeBridge, RationalClockBridge, RationalClock, ResetPulseBridge, ResetPulseBridgeParameters}
@@ -166,59 +166,69 @@ class ClockBridgeInstantiator {
 case object ClockBridgeInstantiatorKey extends Field[ClockBridgeInstantiator](new ClockBridgeInstantiator)
 case object FireSimBaseClockNameKey extends Field[String]("implicit_clock")
 
-class WithFireSimSimpleClocks extends Config((site, here, up) => {
-  case ClockingSchemeKey => { chiptop: ChipTop =>
-    implicit val p = chiptop.p
+class ClocksWithSinkParams(val params: Seq[ClockSinkParameters]) extends Bundle {
+  val clocks = Vec(params.size, Clock())
+}
+
+class WithFireSimSimpleClocks extends OverrideLazyIOBinder({
+  (system: HasChipyardPRCI) => {
+    implicit val p = GetSystemParameters(system)
     // Figure out what provides this in the chipyard scheme
     implicit val valName = ValName("FireSimClocking")
 
-    // Requires existence of undriven asyncClockGroups in subsystem
-    val systemAsyncClockGroup = chiptop.lazySystem match {
-      case l: BaseSubsystem if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) =>
-        l.asyncClockGroupsNode
+    val implicitClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters(name = Some("implicit_clock"))))
+    system.connectImplicitClockSinkNode(implicitClockSinkNode)
+    InModuleBody {
+      val implicit_clock = implicitClockSinkNode.in.head._1.clock
+      val implicit_reset = implicitClockSinkNode.in.head._1.reset
+      system.asInstanceOf[BaseSubsystem].module match { case l: LazyModuleImp => {
+        l.clock := implicit_clock
+        l.reset := implicit_reset
+      }}
     }
 
-    val aggregator = LazyModule(new ClockGroupAggregator("allClocks")).node
-    (chiptop.implicitClockSinkNode := ClockGroup() := aggregator)
-    (systemAsyncClockGroup :*= ClockGroupNamePrefixer() :*= aggregator)
-
     val inputClockSource = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
-
-    (aggregator
-      := ClockGroupResetSynchronizer()
-      := ClockGroupFrequencySpecifier(p(ClockFrequencyAssignersKey), p(DefaultClockFrequencyKey))
-      := inputClockSource)
-
+    system.allClockGroupsNode := inputClockSource
 
     InModuleBody {
       val (clockGroupBundle, clockGroupEdge) = inputClockSource.out.head
-      val input_clocks = IO(Input(RecordMap((clockGroupEdge.sink.members.map { m => (m.name.get, Clock()) }):_* )))
+      val reset_io = IO(Input(AsyncReset())).suggestName("async_reset")
+
+      val input_clocks = IO(Input(new ClocksWithSinkParams(clockGroupEdge.sink.members)))
         .suggestName("clocks")
-      val reset = IO(Input(Reset())).suggestName("reset")
 
-      (clockGroupBundle.member.data zip input_clocks.data).foreach { case (clockBundle, inputClock) =>
+      (clockGroupBundle.member.data zip input_clocks.clocks).foreach { case (clockBundle, inputClock) =>
         clockBundle.clock := inputClock
-        clockBundle.reset := reset
+        clockBundle.reset := reset_io
       }
 
-      val pllConfig = new SimplePllConfiguration("firesimBuildTopClockGenerator", clockGroupEdge.sink.members)
-      pllConfig.emitSummaries
-      val rationalClockSpecs = for ((sinkP, division) <- pllConfig.sinkDividerMap) yield {
-        RationalClock(sinkP.name.get, 1, division)
-      }
+      (Seq(reset_io, input_clocks), Nil)
+    }
+  }
+})
 
-      chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
-        reset := th.buildtopReset
-        input_clocks := p(ClockBridgeInstantiatorKey)
-          .requestClockRecordMap(BuildTopClockParameters(
+class WithFireSimHarnessClockBinder extends OverrideHarnessBinder({
+  (system: HasChipyardPRCI, th: FireSim, ports: Seq[Data]) => {
+    implicit val p = th.p
+    ports.map ({
+      case c: ClocksWithSinkParams => {
+        val pllConfig = new SimplePllConfiguration("firesimBuildTopClockGenerator", c.params)
+        pllConfig.emitSummaries
+        th.setRefClockFreq(pllConfig.referenceFreqMHz)
+        val rationalClockSpecs = for ((sinkP, division) <- pllConfig.sinkDividerMap) yield {
+          RationalClock(sinkP.name.get, 1, division)
+        }
+        val input_clocks: RecordMap[Clock] = p(ClockBridgeInstantiatorKey).requestClockRecordMap(
+          BuildTopClockParameters(
             rationalClockSpecs.toSeq,
             p(FireSimBaseClockNameKey),
             pllConfig.referenceFreqMHz * (1000 * 1000)))
-        Nil })
-
-      // return the reference frequency
-      pllConfig.referenceFreqMHz
-    }
+        (c.clocks zip c.params) map ({ case (clock, param) =>
+          clock := input_clocks(param.name.get).get
+        })
+      }
+      case r: Reset => r := th.buildtopReset.asAsyncReset
+    })
   }
 })
 
@@ -245,8 +255,6 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna
   def dutReset = { require(false, "dutReset should not be used in Firesim"); false.B }
   def success = { require(false, "success should not be used in Firesim"); false.B }
 
-  var btFreqMHz: Option[Double] = None
-
   // Instantiate multiple instances of the DUT to implement supernode
   for (i <- 0 until p(NumNodes)) {
     // It's not a RC bump without some hacks...
@@ -259,22 +267,13 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna
     })))
     val module = Module(lazyModule.module)
 
-    btFreqMHz = Some(lazyModule match {
-      case d: HasReferenceClockFreq => d.refClockFreqMHz
-      case _ => p(DefaultClockFrequencyKey)
-    })
-
-    lazyModule match { case d: HasTestHarnessFunctions =>
-      require(d.harnessFunctions.size == 1, "There should only be 1 harness function to connect clock+reset")
-      d.harnessFunctions.foreach(_(this))
-    }
     lazyModule match { case d: HasIOBinders =>
       ApplyHarnessBinders(this, d.lazySystem, d.portMap)
     }
     NodeIdx.increment()
   }
 
-  buildtopClock := p(ClockBridgeInstantiatorKey).requestClock("buildtop_reference_clock", btFreqMHz.get * (1000 * 1000))
+  buildtopClock := p(ClockBridgeInstantiatorKey).requestClock("buildtop_reference_clock", getRefClockFreq * (1000 * 1000))
 
   p(ClockBridgeInstantiatorKey).instantiateFireSimClockBridge
 }

generators/firechip/src/main/scala/TargetConfigs.scala

@@ -17,6 +17,7 @@ import testchipip.{BlockDeviceKey, BlockDeviceConfig, TracePortKey, TracePortPar
 import sifive.blocks.devices.uart.{PeripheryUARTKey, UARTParams}
 import scala.math.{min, max}
 
+import chipyard.clocking.{ChipyardPRCIControlKey}
 import icenet._
 import testchipip.WithRingSystemBus
 
@@ -40,6 +41,7 @@ class WithBootROM extends Config((site, here, up) => {
 // Disables clock-gating; doesn't play nice with our FAME-1 pass
 class WithoutClockGating extends Config((site, here, up) => {
   case DebugModuleKey => up(DebugModuleKey, site).map(_.copy(clockGate = false))
+  case ChipyardPRCIControlKey => up(ChipyardPRCIControlKey, site).copy(enableTileClockGating = false)
 })
 
 // Testing configurations
@@ -65,6 +67,7 @@ class WithFireSimDesignTweaks extends Config(
   // Required: Bake in the default FASED memory model
   new WithDefaultMemModel ++
   // Required*: Uses FireSim ClockBridge and PeekPokeBridge to drive the system with a single clock/reset
+  new WithFireSimHarnessClockBinder ++
   new WithFireSimSimpleClocks ++
   // Required*: When using FireSim-as-top to provide a correct path to the target bootrom source
   new WithBootROM ++