Update FireSim to support harness clocks | Small config renaming

generators/chipyard/src/main/scala/TestHarness.scala

@@ -95,7 +95,7 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessSign
     case d: HasReferenceClockFreq => d.refClockFreqMHz.getOrElse(p(DefaultClockFrequencyKey))
     case _ => p(DefaultClockFrequencyKey)
   }
-  val refClkBundle = p(HarnessClockInstantiatorKey).getClockBundleWire("chiptop_reference_clock", freqMHz * (1000 * 1000))
+  val refClkBundle = p(HarnessClockInstantiatorKey).getClockBundleWire("buildtop_reference_clock", freqMHz * (1000 * 1000))
 
   harnessClock := refClkBundle.clock
   harnessReset := WireInit(refClkBundle.reset)

generators/chipyard/src/main/scala/clocking/DividerOnlyClockGenerator.scala

@@ -89,6 +89,7 @@ class SimplePllConfiguration(
      ElaborationArtefacts.add(s"${name}.freq-summary", summaryString)
      println(summaryString)
    }
+   def referenceSinkParams(): ClockSinkParameters = sinkDividerMap.find(_._2 == 1).get._1
 }
 
 case class DividerOnlyClockGeneratorNode(pllName: String)(implicit valName: ValName)

generators/firechip/src/main/scala/BridgeBinders.scala

@@ -12,6 +12,8 @@ import freechips.rocketchip.devices.debug.{Debug, HasPeripheryDebugModuleImp}
 import freechips.rocketchip.amba.axi4.{AXI4Bundle}
 import freechips.rocketchip.subsystem._
 import freechips.rocketchip.tile.{RocketTile}
+import freechips.rocketchip.prci.{ClockBundle, ClockBundleParameters}
+import freechips.rocketchip.util.{ResetCatchAndSync}
 import sifive.blocks.devices.uart._
 
 import testchipip._
@@ -104,24 +106,43 @@ class WithBlockDeviceBridge extends OverrideHarnessBinder({
 })
 
 class WithAXIOverSerialTLCombinedBridges extends OverrideHarnessBinder({
-  (system: CanHavePeripheryTLSerial, th: FireSim, ports: Seq[SerialAndPassthroughClockResetIO]) => {
+  (system: CanHavePeripheryTLSerial, th: FireSim, ports: Seq[ClockedIO[SerialIO]]) => {
     implicit val p = GetSystemParameters(system)
 
     p(SerialTLKey).map({ sVal =>
-      // require having memory over the serdes link
+      // currently only the harness AXI port supports a passthrough clock
+      require(sVal.axiMemOverSerialTLParams.isDefined)
+      val axiDomainParams = sVal.axiMemOverSerialTLParams.get
       require(sVal.isMemoryDevice)
 
+      val memFreq: Double = axiDomainParams.axiClockParams match {
+        case Some(clkParams) => clkParams.clockFreqMHz * 1000000
+        case None => {
+          // get freq. from what the master of the serial link specifies
+          system.asInstanceOf[HasTileLinkLocations].locateTLBusWrapper(p(SerialTLAttachKey).masterWhere).dtsFrequency.get.toDouble
+        }
+      }
+
       ports.map({ port =>
-        val offchipNetwork = SerialAdapter.connectHarnessMultiClockAXIRAM(system.serdesser.get, port, th.harnessReset)
-        SerialBridge(port.clocked_serial.clock, offchipNetwork.module.io.tsi_ser, Some(MainMemoryConsts.globalName))
+        val axiClock = p(ClockBridgeInstantiatorKey).getClock("mem_over_serial_tl_clock", memFreq)
+        val axiClockBundle = Wire(new ClockBundle(ClockBundleParameters()))
+        axiClockBundle.clock := axiClock
+        axiClockBundle.reset := ResetCatchAndSync(axiClock, th.harnessReset.asBool)
+
+        val harnessMultiClockAXIRAM = SerialAdapter.connectHarnessMultiClockAXIRAM(
+          system.serdesser.get,
+          port,
+          axiClockBundle,
+          th.harnessReset)
+        SerialBridge(port.clock, harnessMultiClockAXIRAM.module.io.tsi_ser, Some(MainMemoryConsts.globalName))
 
         // connect SimAxiMem
-        (offchipNetwork.mem_axi4 zip offchipNetwork.memAXI4Node.edges.in).map { case (axi4, edge) =>
-          val nastiKey = NastiParameters(axi4.r.bits.data.getWidth,
-                                        axi4.ar.bits.addr.getWidth,
-                                        axi4.ar.bits.id.getWidth)
+        (harnessMultiClockAXIRAM.mem_axi4 zip harnessMultiClockAXIRAM.memNode.edges.in).map { case (axi4, edge) =>
+          val nastiKey = NastiParameters(axi4.bits.r.bits.data.getWidth,
+                                        axi4.bits.ar.bits.addr.getWidth,
+                                        axi4.bits.ar.bits.id.getWidth)
           system match {
-            case s: BaseSubsystem => FASEDBridge(port.passthrough_clock_reset.clock, axi4, port.passthrough_clock_reset.reset.asBool,
+            case s: BaseSubsystem => FASEDBridge(axi4.clock, axi4.bits, axi4.reset.asBool,
               CompleteConfig(p(firesim.configs.MemModelKey),
                             nastiKey,
                             Some(AXI4EdgeSummary(edge)),

generators/firechip/src/main/scala/FireSim.scala

@@ -2,6 +2,8 @@
 
 package firesim.firesim
 
+import scala.collection.mutable.{HashMap}
+
 import chisel3._
 import chisel3.experimental.{IO}
 
@@ -38,44 +40,104 @@ object NodeIdx {
 /**
   * Under FireSim's current multiclock implementation there can be only a
   * single clock bridge. This requires, therefore, that it  be instantiated in
-  * the harness and reused across all supernode instances. This class attempts to 
+  * the harness and reused across all supernode instances. This class attempts to
   * memoize its instantiation such that it can be referenced from within a ClockScheme function.
   */
 class ClockBridgeInstantiator {
-  private var _clockRecord: Option[RecordMap[Clock]] = None
+  private var _harnessClockMap: HashMap[String, (Double, Clock)] = HashMap.empty
 
-  def getClockRecord: RecordMap[Clock] = _clockRecord.get
+  // Assumes that the supernode implementation results in duplicated clocks
+  //   (i.e. only 1 set of clocks is generated for all BuildTop designs)
+  private var _ratClockMap: HashMap[String, (RationalClock, Clock)] = HashMap.empty
+  private var _ratRefName: Option[String] = None
 
-  def getClockRecordOrInstantiate(allClocks: Seq[RationalClock], baseClockName: String): RecordMap[Clock] = {
-    if (_clockRecord.isEmpty) {
-      require(allClocks.exists(_.name == baseClockName),
-        s"Provided base-clock name, ${baseClockName}, does not match a defined clock. Available clocks:\n " +
-        allClocks.map(_.name).mkString("\n "))
+  /**
+   * Request a clock at a particular frequency
+   *
+   * @param name An identifier for the associated clock domain
+   *
+   * @param freqRequested Freq. for the domain in Hz
+   */
+  def getClock(name: String, freqRequested: Double): Clock = {
+    val clkWire = Wire(new Clock)
+    _harnessClockMap(name) = (freqRequested, clkWire)
+    clkWire
+  }
 
-      val baseClock = allClocks.find(_.name == baseClockName).get
-      val simplified = allClocks.map { c =>
-        c.copy(multiplier = c.multiplier * baseClock.divisor, divisor = c.divisor * baseClock.multiplier)
-         .simplify
-      }
+  /**
+   * Get a RecordMap of clocks for a set of input RationalClocks
+   *
+   * @param allClocks Seq. of RationalClocks that want a clock
+   *
+   * @param baseClockName Name of domain that the allClocks is rational to
+   */
+  def getClockRecordMap(allClocks: Seq[RationalClock], baseClockName: String): RecordMap[Clock] = {
+    val ratClockRecordMapWire = Wire(RecordMap(allClocks.map { c => (c.name, Clock()) }:_*))
 
-    /**
-      * Removes clocks that have the same frequency before instantiating the
-      * clock bridge to avoid unnecessary BUFGCE use.
-      */
-      val distinct = simplified.foldLeft(Seq(RationalClock(baseClockName, 1, 1))) { case (list, candidate) =>
-        if (list.exists { clock => clock.equalFrequency(candidate) }) list else list :+ candidate
-      }
-
-      val clockBridge = Module(new RationalClockBridge(distinct))
-      val cbVecTuples = distinct.zip(clockBridge.io.clocks)
-      val outputWire = Wire(RecordMap(simplified.map { c => (c.name, Clock()) }:_*))
-      for (parameter <- simplified) {
-        val (_, cbClockField) = cbVecTuples.find(_._1.equalFrequency(parameter)).get
-        outputWire(parameter.name).get := cbClockField
-      }
-      _clockRecord = Some(outputWire)
+    _ratRefName = Some(baseClockName)
+    for (clock <- allClocks) {
+      val clkWire = Wire(new Clock)
+      _ratClockMap(clock.name) = (clock, clkWire)
+      ratClockRecordMapWire(clock.name).get := clkWire
+    }
+
+    ratClockRecordMapWire
+  }
+
+  /**
+   * Connect all clocks requested to ClockBridge
+   */
+  def instantiateFireSimDividerPLL: Unit = {
+    // Simplify the RationalClocks ratio's
+    val refRatClock = _ratClockMap.find(_._1 == _ratRefName.get).get._2._1
+    val simpleRatClocks = _ratClockMap.map { t =>
+      val ratClock = t._2._1
+      ratClock.copy(
+        multiplier = ratClock.multiplier * refRatClock.divisor,
+        divisor = ratClock.divisor * refRatClock.multiplier).simplify
+    }
+
+    // Determine all the clock dividers (harness + rational clocks)
+    //   Note: Requires that the BuildTop reference frequency is requested with proper freq.
+    val refRatClockFreq = _harnessClockMap.find(_._1 == _ratRefName.get).get._2._1
+    val refRatSinkParams = ClockSinkParameters(take=Some(ClockParameters(freqMHz=refRatClockFreq / (1000 * 1000))),name=Some(_ratRefName.get))
+    val harSinkParams = _harnessClockMap.map { case (name, (freq, bundle)) =>
+      ClockSinkParameters(take=Some(ClockParameters(freqMHz=freq / (1000 * 1000))),name=Some(name))
+    }.toSeq
+    val allSinkParams = harSinkParams :+ refRatSinkParams
+
+    // Use PLL config to determine overall div's
+    val pllConfig = new SimplePllConfiguration("firesimOverallClockBridge", allSinkParams)
+    pllConfig.emitSummaries
+
+    // Adjust all BuildTop RationalClocks with the div determined by the PLL
+    val refRatDiv = pllConfig.sinkDividerMap(refRatSinkParams)
+    val adjRefRatClocks = simpleRatClocks.map { clock =>
+      clock.copy(divisor = clock.divisor * refRatDiv).simplify
+    }
+
+    // Convert harness clocks to RationalClocks
+    val harRatClocks = harSinkParams.map { case ClockSinkParameters(_, _, _, _, clkParamsOpt, nameOpt) =>
+      RationalClock(nameOpt.get, 1, pllConfig.referenceFreqMHz.toInt / clkParamsOpt.get.freqMHz.toInt)
+    }
+
+    val allAdjRatClks = adjRefRatClocks ++ harRatClocks
+
+    // Removes clocks that have the same frequency before instantiating the
+    //   clock bridge to avoid unnecessary BUFGCE use.
+    val allDistinctRatClocks = allAdjRatClks.foldLeft(Seq(RationalClock(pllConfig.referenceSinkParams.name.get, 1, 1))) {
+      case (list, candidate) => if (list.exists { clock => clock.equalFrequency(candidate) }) list else list :+ candidate
+    }
+
+    val clockBridge = Module(new RationalClockBridge(allDistinctRatClocks))
+    val cbVecTuples = allDistinctRatClocks.zip(clockBridge.io.clocks)
+
+    // Connect all clocks (harness + BuildTop clocks)
+    for (clock <- allAdjRatClks) {
+      val (_, cbClockField) = cbVecTuples.find(_._1.equalFrequency(clock)).get
+      _ratClockMap.get(clock.name).map { case (_, clk) => clk := cbClockField }
+      _harnessClockMap.get(clock.name).map { case (_, clk) => clk := cbClockField }
     }
-    getClockRecord
   }
 }
 
@@ -117,16 +179,19 @@ class WithFireSimSimpleClocks extends Config((site, here, up) => {
         clockBundle.reset := reset
       }
 
-      val pllConfig = new SimplePllConfiguration("FireSim RationalClockBridge", clockGroupEdge.sink.members)
+      val pllConfig = new SimplePllConfiguration("firesimBuildTopClockGenerator", clockGroupEdge.sink.members)
       pllConfig.emitSummaries
       val rationalClockSpecs = for ((sinkP, division) <- pllConfig.sinkDividerMap) yield {
         RationalClock(sinkP.name.get, 1, division)
       }
 
+      // Set the reference frequency used
+      chiptop.refClockFreqMHz = Some(pllConfig.referenceFreqMHz)
+
       chiptop.harnessFunctions += ((th: HasHarnessSignalReferences) => {
         reset := th.harnessReset
         input_clocks := p(ClockBridgeInstantiatorKey)
-          .getClockRecordOrInstantiate(rationalClockSpecs.toSeq, p(FireSimBaseClockNameKey))
+          .getClockRecordMap(rationalClockSpecs.toSeq, p(FireSimBaseClockNameKey))
         Nil })
     }
   }
@@ -140,6 +205,8 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna
   def dutReset = { require(false, "dutReset should not be used in Firesim"); false.B }
   def success = { require(false, "success should not be used in Firesim"); false.B }
 
+  var btFreqMHz: Option[Double] = None
+
   // Instantiate multiple instances of the DUT to implement supernode
   for (i <- 0 until p(NumNodes)) {
     // It's not a RC bump without some hacks...
@@ -150,7 +217,15 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna
     val lazyModule = LazyModule(p(BuildTop)(p.alterPartial({
       case AsyncClockGroupsKey => p(AsyncClockGroupsKey).copy
     })))
-    val module = Module(lazyModule.module)
+    withClockAndReset(harnessClock, harnessReset) {
+      val module = Module(lazyModule.module)
+    }
+
+    btFreqMHz = Some(lazyModule match {
+      case d: HasReferenceClockFreq => d.refClockFreqMHz.getOrElse(p(DefaultClockFrequencyKey))
+      case _ => p(DefaultClockFrequencyKey)
+    })
+
     lazyModule match { case d: HasTestHarnessFunctions =>
       require(d.harnessFunctions.size == 1, "There should only be 1 harness function to connect clock+reset")
       d.harnessFunctions.foreach(_(this))
@@ -160,5 +235,8 @@ class FireSim(implicit val p: Parameters) extends RawModule with HasHarnessSigna
     }
     NodeIdx.increment()
   }
-  harnessClock := p(ClockBridgeInstantiatorKey).getClockRecord("implicit_clock").get
+
+  harnessClock := p(ClockBridgeInstantiatorKey).getClock(p(FireSimBaseClockNameKey), btFreqMHz.get * (1000 * 1000))
+
+  p(ClockBridgeInstantiatorKey).instantiateFireSimDividerPLL
 }

generators/firechip/src/main/scala/TargetConfigs.scala

@@ -59,7 +59,7 @@ class WithNIC extends icenet.WithIceNIC(inBufFlits = 8192, ctrlQueueDepth = 64)
 class WithNVDLALarge extends nvidia.blocks.dla.WithNVDLA("large")
 class WithNVDLASmall extends nvidia.blocks.dla.WithNVDLA("small")
 
-class WithFireSimConfigTweaksWithoutClocking extends Config(
+class WithFireSimDesignTweaks extends Config(
   // Required: Bake in the default FASED memory model
   new WithDefaultMemModel ++
   // Required*: Uses FireSim ClockBridge and PeekPokeBridge to drive the system with a single clock/reset
@@ -96,7 +96,7 @@ class WithFireSimConfigTweaks extends Config(
   new chipyard.config.WithAsynchrousMemoryBusCrossing ++
   new testchipip.WithAsynchronousSerialSlaveCrossing ++
   // Tweaks that are independent from multi-clock
-  new WithFireSimConfigTweaksWithoutClocking
+  new WithFireSimDesignTweaks
 )
 
 /*******************************************************************************
@@ -207,8 +207,9 @@ class FireSimMulticlockRocketConfig extends Config(
 class FireSimMulticlockAXIOverSerialConfig extends Config(
   new WithAXIOverSerialTLCombinedBridges ++ // use combined bridge to connect to axi mem over serial
   new WithDefaultFireSimBridges ++
+  new testchipip.WithBlockDevice(false) ++ // disable blockdev
   new WithDefaultMemModel ++
-  new WithFireSimConfigTweaksWithoutClocking ++ // don't inherit firesim clocking
+  new WithFireSimDesignTweaks ++ // don't inherit firesim clocking
   new chipyard.MulticlockAXIOverSerialConfig
 )