From 61a0cf91528b85308bbe60f531febd8bfed646fb Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 24 May 2023 18:38:04 -0700 Subject: [PATCH 01/24] Fix run-binary-none --- common.mk | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/common.mk b/common.mk index 50a01a4d..4501c20a 100644 --- a/common.mk +++ b/common.mk @@ -350,9 +350,7 @@ run-binary-debug: check-binary $(BINARY).run.debug run-binaries-debug: check-binaries $(addsuffix .run.debug,$(BINARIES)) %.run.debug: %.check-exists $(SIM_DEBUG_PREREQ) | $(output_dir) -ifneq (none,$*) - riscv64-unknown-elf-objdump -D $* > $(call get_sim_out_name,$*).dump -endif + if [ "$*" != "none" ]; then riscv64-unknown-elf-objdump -D $* > $(call get_sim_out_name,$*).dump ; fi (set -o pipefail && $(NUMA_PREFIX) $(sim_debug) $(PERMISSIVE_ON) $(call get_common_sim_flags,$*) $(VERBOSE_FLAGS) $(call get_waveform_flag,$(call get_sim_out_name,$*)) $(PERMISSIVE_OFF) $* >(spike-dasm > $(call get_sim_out_name,$*).out) | tee $(call get_sim_out_name,$*).log) run-fast: run-asm-tests-fast run-bmark-tests-fast From af0dce8e8a9f4c249bb93710fe329afebb0f5dbe Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 24 May 2023 18:39:08 -0700 Subject: [PATCH 02/24] Switch to PeripheryUARTTSI --- .../chipyard/src/main/scala/DigitalTop.scala | 1 + .../chipyard/src/main/scala/IOBinders.scala | 9 +++++++++ .../main/scala/config/AbstractConfig.scala | 20 +++++++++++-------- .../config/PeripheralDeviceConfigs.scala | 10 ++++++++++ .../src/main/scala/config/RocketConfigs.scala | 8 -------- .../main/scala/harness/HarnessBinders.scala | 19 ++++++------------ 6 files changed, 38 insertions(+), 29 deletions(-) diff --git a/generators/chipyard/src/main/scala/DigitalTop.scala b/generators/chipyard/src/main/scala/DigitalTop.scala index 6711eac7..c7099f97 100644 --- a/generators/chipyard/src/main/scala/DigitalTop.scala +++ b/generators/chipyard/src/main/scala/DigitalTop.scala @@ -13,6 +13,7 @@ import freechips.rocketchip.devices.tilelink._ // DOC include start: DigitalTop class DigitalTop(implicit p: Parameters) extends ChipyardSystem + with testchipip.CanHavePeripheryUARTTSI // Enables optional UART-based TSI transport with testchipip.CanHavePeripheryCustomBootPin // Enables optional custom boot pin with testchipip.CanHavePeripheryBootAddrReg // Use programmable boot address register with testchipip.CanHaveTraceIO // Enables optionally adding trace IO diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala index 8dd0020e..11dff842 100644 --- a/generators/chipyard/src/main/scala/IOBinders.scala +++ b/generators/chipyard/src/main/scala/IOBinders.scala @@ -411,6 +411,15 @@ class WithCustomBootPin extends OverrideIOBinder({ }).getOrElse((Nil, Nil)) }) +class WithUARTTSIPunchthrough extends OverrideIOBinder({ + (system: CanHavePeripheryUARTTSI) => system.uart_tsi.map({ p => + val sys = system.asInstanceOf[BaseSubsystem] + val port = IO(new UARTPortIO(p.c)) + port <> p + (Seq(port), Nil) + }).getOrElse((Nil, Nil)) +}) + class WithTLMemPunchthrough extends OverrideIOBinder({ (system: CanHaveMasterTLMemPort) => { val io_tl_mem_pins_temp = IO(DataMirror.internal.chiselTypeClone[HeterogeneousBag[TLBundle]](system.mem_tl)).suggestName("tl_slave") diff --git a/generators/chipyard/src/main/scala/config/AbstractConfig.scala b/generators/chipyard/src/main/scala/config/AbstractConfig.scala index 48f10cd9..063b21d4 100644 --- a/generators/chipyard/src/main/scala/config/AbstractConfig.scala +++ b/generators/chipyard/src/main/scala/config/AbstractConfig.scala @@ -22,25 +22,29 @@ class AbstractConfig extends Config( new chipyard.harness.WithTieOffInterrupts ++ // tie-off interrupt ports, if present new chipyard.harness.WithTieOffL2FBusAXI ++ // tie-off external AXI4 master, if present new chipyard.harness.WithCustomBootPinPlusArg ++ // drive custom-boot pin with a plusarg, if custom-boot-pin is present + new chipyard.harness.WithSimUARTToUARTTSI ++ // connect a SimUART to the UART-TSI port new chipyard.harness.WithClockAndResetFromHarness ++ // all Clock/Reset I/O in ChipTop should be driven by harnessClockInstantiator new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ // generate clocks in harness with unsynthesizable ClockSourceAtFreqMHz // The IOBinders instantiate ChipTop IOs to match desired digital IOs - // IOCells are generated for "Chip-like" IOs, while simulation-only IOs are directly punched through + // IOCells are generated for "Chip-like" IOs + new chipyard.iobinders.WithSerialTLIOCells ++ + new chipyard.iobinders.WithDebugIOCells ++ + new chipyard.iobinders.WithUARTIOCells ++ + new chipyard.iobinders.WithGPIOCells ++ + new chipyard.iobinders.WithSPIIOCells ++ + new chipyard.iobinders.WithExtInterruptIOCells ++ + new chipyard.iobinders.WithCustomBootPin ++ + // The "punchtrough" IOBInders below don't generate IOCells, as these interfaces shouldn't really be mapped to ASIC IO + // Instead, they directly pass through the DigitalTop ports to ports in the ChipTop new chipyard.iobinders.WithAXI4MemPunchthrough ++ new chipyard.iobinders.WithAXI4MMIOPunchthrough ++ new chipyard.iobinders.WithTLMemPunchthrough ++ new chipyard.iobinders.WithL2FBusAXI4Punchthrough ++ new chipyard.iobinders.WithBlockDeviceIOPunchthrough ++ new chipyard.iobinders.WithNICIOPunchthrough ++ - new chipyard.iobinders.WithSerialTLIOCells ++ - new chipyard.iobinders.WithDebugIOCells ++ - new chipyard.iobinders.WithUARTIOCells ++ - new chipyard.iobinders.WithGPIOCells ++ - new chipyard.iobinders.WithSPIIOCells ++ new chipyard.iobinders.WithTraceIOPunchthrough ++ - new chipyard.iobinders.WithExtInterruptIOCells ++ - new chipyard.iobinders.WithCustomBootPin ++ + new chipyard.iobinders.WithUARTTSIPunchthrough ++ // By default, punch out IOs to the Harness new chipyard.clocking.WithPassthroughClockGenerator ++ diff --git a/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala b/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala index 8da229b4..9eec165e 100644 --- a/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala +++ b/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala @@ -80,3 +80,13 @@ class QuadChannelRocketConfig extends Config( new freechips.rocketchip.subsystem.WithNMemoryChannels(4) ++ // 4 AXI4 channels new freechips.rocketchip.subsystem.WithNBigCores(1) ++ new chipyard.config.AbstractConfig) + +class UARTTSIRocketConfig extends Config( + new chipyard.harness.WithSerialTLTiedOff ++ + new testchipip.WithUARTTSIClient ++ + new chipyard.config.WithMemoryBusFrequency(10) ++ + new chipyard.config.WithFrontBusFrequency(10) ++ + new chipyard.config.WithPeripheryBusFrequency(10) ++ + new freechips.rocketchip.subsystem.WithNBigCores(1) ++ // single rocket-core + new chipyard.config.AbstractConfig) + diff --git a/generators/chipyard/src/main/scala/config/RocketConfigs.scala b/generators/chipyard/src/main/scala/config/RocketConfigs.scala index af6bfc1d..2bcc0089 100644 --- a/generators/chipyard/src/main/scala/config/RocketConfigs.scala +++ b/generators/chipyard/src/main/scala/config/RocketConfigs.scala @@ -19,14 +19,6 @@ class TinyRocketConfig extends Config( new freechips.rocketchip.subsystem.With1TinyCore ++ // single tiny rocket-core new chipyard.config.AbstractConfig) -class UARTTSIRocketConfig extends Config( - new chipyard.harness.WithUARTSerial ++ - new chipyard.config.WithNoUART ++ - new chipyard.config.WithMemoryBusFrequency(10) ++ - new chipyard.config.WithPeripheryBusFrequency(10) ++ - new freechips.rocketchip.subsystem.WithNBigCores(1) ++ // single rocket-core - new chipyard.config.AbstractConfig) - class SimAXIRocketConfig extends Config( new chipyard.harness.WithSimAXIMem ++ // drive the master AXI4 memory with a SimAXIMem, a 1-cycle magic memory, instead of default SimDRAM new freechips.rocketchip.subsystem.WithNBigCores(1) ++ diff --git a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala index a78c748b..69d6f6d3 100644 --- a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala +++ b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala @@ -317,21 +317,14 @@ class WithSimTSIOverSerialTL extends OverrideHarnessBinder({ } }) -class WithUARTSerial extends OverrideHarnessBinder({ - (system: CanHavePeripheryTLSerial, th: HasHarnessInstantiators, ports: Seq[ClockedIO[SerialIO]]) => { +class WithSimUARTToUARTTSI extends OverrideHarnessBinder({ + (system: CanHavePeripheryUARTTSI, th: HasHarnessInstantiators, ports: Seq[UARTPortIO]) => { implicit val p = chipyard.iobinders.GetSystemParameters(system) ports.map({ port => - val freq = p(PeripheryBusKey).dtsFrequency.get - val bits = port.bits - port.clock := th.harnessBinderClock - val ram = TSIHarness.connectRAM(system.serdesser.get, bits, th.harnessBinderReset) - val uart_to_serial = Module(new UARTToSerial(freq, UARTParams(0))) - val serial_width_adapter = Module(new SerialWidthAdapter( - 8, TSI.WIDTH)) - ram.module.io.tsi.flipConnect(serial_width_adapter.io.wide) - UARTAdapter.connect(Seq(uart_to_serial.io.uart), uart_to_serial.div) - serial_width_adapter.io.narrow.flipConnect(uart_to_serial.io.serial) - th.success := false.B + UARTAdapter.connect(Seq(port), + baudrate=port.c.initBaudRate, + clockFrequency=th.getHarnessBinderClockFreqHz.toInt, + forcePty=true) }) } }) From 889713b5b157b7f0c4758b378a373c1367023b9b Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 24 May 2023 19:15:11 -0700 Subject: [PATCH 03/24] Switch to UARTTSIIO --- fpga/src/main/scala/arty100t/Configs.scala | 4 ++- .../main/scala/arty100t/HarnessBinders.scala | 30 ++++++------------- .../chipyard/src/main/scala/IOBinders.scala | 6 ++-- .../main/scala/harness/HarnessBinders.scala | 12 ++++---- generators/testchipip | 2 +- 5 files changed, 23 insertions(+), 31 deletions(-) diff --git a/fpga/src/main/scala/arty100t/Configs.scala b/fpga/src/main/scala/arty100t/Configs.scala index 20790e12..4a0fb293 100644 --- a/fpga/src/main/scala/arty100t/Configs.scala +++ b/fpga/src/main/scala/arty100t/Configs.scala @@ -22,12 +22,14 @@ class WithNoDesignKey extends Config((site, here, up) => { }) class WithArty100TTweaks extends Config( - new chipyard.harness.WithAllClocksFromHarnessClockInstantiator ++ new WithArty100TUARTTSI ++ new WithArty100TDDRTL ++ new WithNoDesignKey ++ + new testchipip.WithUARTTSIClient ++ + new chipyard.harness.WithSerialTLTiedOff ++ new chipyard.harness.WithHarnessBinderClockFreqMHz(50) ++ new chipyard.config.WithMemoryBusFrequency(50.0) ++ + new chipyard.config.WithFrontBusFrequency(50.0) ++ new chipyard.config.WithSystemBusFrequency(50.0) ++ new chipyard.config.WithPeripheryBusFrequency(50.0) ++ new chipyard.harness.WithAllClocksFromHarnessClockInstantiator ++ diff --git a/fpga/src/main/scala/arty100t/HarnessBinders.scala b/fpga/src/main/scala/arty100t/HarnessBinders.scala index f6750d37..060e3d8d 100644 --- a/fpga/src/main/scala/arty100t/HarnessBinders.scala +++ b/fpga/src/main/scala/arty100t/HarnessBinders.scala @@ -21,29 +21,17 @@ import chipyard.iobinders.JTAGChipIO import testchipip._ class WithArty100TUARTTSI(uartBaudRate: BigInt = 115200) extends OverrideHarnessBinder({ - (system: CanHavePeripheryTLSerial, th: HasHarnessInstantiators, ports: Seq[ClockedIO[SerialIO]]) => { + (system: CanHavePeripheryUARTTSI, th: HasHarnessInstantiators, ports: Seq[UARTTSIIO]) => { implicit val p = chipyard.iobinders.GetSystemParameters(system) + require(ports.size <= 1) + val ath = th.asInstanceOf[LazyRawModuleImp].wrapper.asInstanceOf[Arty100THarness] ports.map({ port => - val ath = th.asInstanceOf[LazyRawModuleImp].wrapper.asInstanceOf[Arty100THarness] - val freq = p(PeripheryBusKey).dtsFrequency.get - val bits = port.bits - port.clock := th.harnessBinderClock - val ram = TSIHarness.connectRAM(system.serdesser.get, bits, th.harnessBinderReset) - val uart_to_serial = Module(new UARTToSerial( - freq, UARTParams(0, initBaudRate=uartBaudRate))) - val serial_width_adapter = Module(new SerialWidthAdapter( - narrowW = 8, wideW = TSI.WIDTH)) - serial_width_adapter.io.narrow.flipConnect(uart_to_serial.io.serial) - - ram.module.io.tsi.flipConnect(serial_width_adapter.io.wide) - - ath.io_uart_bb.bundle <> uart_to_serial.io.uart - ath.other_leds(1) := uart_to_serial.io.dropped - - ath.other_leds(9) := ram.module.io.tsi2tl_state(0) - ath.other_leds(10) := ram.module.io.tsi2tl_state(1) - ath.other_leds(11) := ram.module.io.tsi2tl_state(2) - ath.other_leds(12) := ram.module.io.tsi2tl_state(3) + ath.io_uart_bb.bundle <> port.uart + ath.other_leds(1) := port.dropped + ath.other_leds(9) := port.tsi2tl_state(0) + ath.other_leds(10) := port.tsi2tl_state(1) + ath.other_leds(11) := port.tsi2tl_state(2) + ath.other_leds(12) := port.tsi2tl_state(3) }) } }) diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala index 11dff842..d00067b4 100644 --- a/generators/chipyard/src/main/scala/IOBinders.scala +++ b/generators/chipyard/src/main/scala/IOBinders.scala @@ -414,9 +414,9 @@ class WithCustomBootPin extends OverrideIOBinder({ class WithUARTTSIPunchthrough extends OverrideIOBinder({ (system: CanHavePeripheryUARTTSI) => system.uart_tsi.map({ p => val sys = system.asInstanceOf[BaseSubsystem] - val port = IO(new UARTPortIO(p.c)) - port <> p - (Seq(port), Nil) + val uart_tsi = IO(new UARTTSIIO(p.uartParams)) + uart_tsi <> p + (Seq(uart_tsi), Nil) }).getOrElse((Nil, Nil)) }) diff --git a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala index 69d6f6d3..2c8a7a75 100644 --- a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala +++ b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala @@ -318,14 +318,16 @@ class WithSimTSIOverSerialTL extends OverrideHarnessBinder({ }) class WithSimUARTToUARTTSI extends OverrideHarnessBinder({ - (system: CanHavePeripheryUARTTSI, th: HasHarnessInstantiators, ports: Seq[UARTPortIO]) => { + (system: CanHavePeripheryUARTTSI, th: HasHarnessInstantiators, ports: Seq[UARTTSIIO]) => { implicit val p = chipyard.iobinders.GetSystemParameters(system) - ports.map({ port => - UARTAdapter.connect(Seq(port), - baudrate=port.c.initBaudRate, + require(ports.size <= 1) + ports.map { port => { + UARTAdapter.connect(Seq(port.uart), + baudrate=port.uartParams.initBaudRate, clockFrequency=th.getHarnessBinderClockFreqHz.toInt, forcePty=true) - }) + assert(!port.dropped) + }} } }) diff --git a/generators/testchipip b/generators/testchipip index 518a36af..e2ab39f2 160000 --- a/generators/testchipip +++ b/generators/testchipip @@ -1 +1 @@ -Subproject commit 518a36afc9b1a64d7007689824a258affd4daef3 +Subproject commit e2ab39f2777f92e8ebc467ebf9c7bf1974793e1e From db0f68453adf67d6f6e2afd50eef34ec23ca67b8 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 24 May 2023 19:17:29 -0700 Subject: [PATCH 04/24] Simplify ApplyHarnessBinders --- .../chipyard/src/main/scala/harness/HarnessBinders.scala | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala index 2c8a7a75..0c2e4c49 100644 --- a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala +++ b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala @@ -51,12 +51,8 @@ class HarnessBinder[T, S <: HasHarnessInstantiators, U <: Data](composer: ((T, S val pts = ports.collect({case p: U => p}) require (pts.length == ports.length, s"Port type mismatch between IOBinder and HarnessBinder: ${portTag}") val upfn = up(HarnessBinders, site)(systemTag.runtimeClass.toString) - th match { - case th: S => - t match { - case system: T => composer(upfn)(system, th, pts) - case _ => - } + (th, t) match { + case (th: S, system: T) => composer(upfn)(system, th, pts) case _ => } }) From 436a6cb538e0de0556129efdbc8c8505b2bd479b Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 24 May 2023 19:40:30 -0700 Subject: [PATCH 05/24] Add MultiHarnessBinder API --- .../main/scala/harness/HarnessBinders.scala | 4 +- .../harness/HasHarnessInstantiators.scala | 1 + .../scala/harness/MultiHarnesBinders.scala | 78 +++++++++++++++++++ .../src/main/scala/harness/package.scala | 17 ++++ 4 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 generators/chipyard/src/main/scala/harness/MultiHarnesBinders.scala create mode 100644 generators/chipyard/src/main/scala/harness/package.scala diff --git a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala index 0c2e4c49..db6f0cda 100644 --- a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala +++ b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala @@ -30,9 +30,7 @@ import icenet.{CanHavePeripheryIceNIC, SimNetwork, NicLoopback, NICKey, NICIOvon import scala.reflect.{ClassTag} -case object HarnessBinders extends Field[Map[String, (Any, HasHarnessInstantiators, Seq[Data]) => Unit]]( - Map[String, (Any, HasHarnessInstantiators, Seq[Data]) => Unit]().withDefaultValue((t: Any, th: HasHarnessInstantiators, d: Seq[Data]) => ()) -) +case object HarnessBinders extends Field[HarnessBinderMap](HarnessBinderMapDefault) object ApplyHarnessBinders { def apply(th: HasHarnessInstantiators, sys: LazyModule, portMap: Map[String, Seq[Data]])(implicit p: Parameters): Unit = { diff --git a/generators/chipyard/src/main/scala/harness/HasHarnessInstantiators.scala b/generators/chipyard/src/main/scala/harness/HasHarnessInstantiators.scala index b5c7e2ce..4c1186b9 100644 --- a/generators/chipyard/src/main/scala/harness/HasHarnessInstantiators.scala +++ b/generators/chipyard/src/main/scala/harness/HasHarnessInstantiators.scala @@ -86,6 +86,7 @@ trait HasHarnessInstantiators { case (d: HasIOBinders, i: Int) => ApplyHarnessBinders(this, d.lazySystem, d.portMap)(chipParameters(i)) case _ => } + ApplyMultiHarnessBinders(this, lazyDuts) } val harnessBinderClk = harnessClockInstantiator.requestClockMHz("harnessbinder_clock", getHarnessBinderClockFreqMHz) diff --git a/generators/chipyard/src/main/scala/harness/MultiHarnesBinders.scala b/generators/chipyard/src/main/scala/harness/MultiHarnesBinders.scala new file mode 100644 index 00000000..1c057dff --- /dev/null +++ b/generators/chipyard/src/main/scala/harness/MultiHarnesBinders.scala @@ -0,0 +1,78 @@ +package chipyard.harness + +import chisel3._ +import chisel3.util._ + +import org.chipsalliance.cde.config.{Field, Config, Parameters} +import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImpLike} +import freechips.rocketchip.devices.debug._ +import freechips.rocketchip.subsystem._ +import freechips.rocketchip.util._ + +import testchipip._ + +import chipyard._ +import chipyard.clocking.{HasChipyardPRCI, ClockWithFreq} +import chipyard.iobinders.{GetSystemParameters, JTAGChipIO, HasIOBinders} + +import scala.reflect.{ClassTag} + +case class MultiHarnessBinders(c0: Int, c1: Int) extends Field[MultiHarnessBinderMap](MultiHarnessBinderMapDefault) + +class MultiHarnessBinder[T0, T1, S <: HasHarnessInstantiators, U0 <: Data, U1 <: Data] + (chip0: Int, chip1: Int, fn: => (T0, T1, S, Seq[U0], Seq[U1]) => Unit) + (implicit tag0: ClassTag[T0], tag1: ClassTag[T1], thtag: ClassTag[S], ptag0: ClassTag[U0], ptag1: ClassTag[U1]) + extends Config((site, here, up) => { + // Override any HarnessBinders for chip0/chip1 + case MultiChipParameters(`chip0`) => new Config( + new OverrideHarnessBinder[T0, S, U0]((system: T0, th: S, ports: Seq[U0]) => Nil) ++ + up(MultiChipParameters(chip0)) + ) + case MultiChipParameters(`chip1`) => new Config( + new OverrideHarnessBinder[T1, S, U1]((system: T1, th: S, ports: Seq[U1]) => Nil) ++ + up(MultiChipParameters(chip1)) + ) + // Set the multiharnessbinder key + case MultiHarnessBinders(`chip0`, `chip1`) => up(MultiHarnessBinders(chip0, chip1)) + + ((tag0.runtimeClass.toString, tag1.runtimeClass.toString) -> + ((c0: Any, c1: Any, th: HasHarnessInstantiators, ports0: Seq[Data], ports1: Seq[Data]) => { + val pts0 = ports0.map(_.asInstanceOf[U0]) + val pts1 = ports1.map(_.asInstanceOf[U1]) + require(pts0.size == pts1.size) + (c0, c1, th) match { + case (c0: T0, c1: T1, th: S) => fn(c0, c1, th, pts0, pts1) + case _ => + } + }) + ) + }) + +object ApplyMultiHarnessBinders { + def apply(th: HasHarnessInstantiators, chips: Seq[LazyModule])(implicit p: Parameters): Unit = { + Seq.tabulate(chips.size, chips.size) { case (i, j) => if (i != j) { + (chips(i), chips(j)) match { + case (l0: HasIOBinders, l1: HasIOBinders) => p(MultiHarnessBinders(i, j)).foreach { + case ((s0, s1), f) => { + f(l0.lazySystem , l1.lazySystem , th, l0.portMap(s0), l1.portMap(s1)) + f(l0.lazySystem.module, l1.lazySystem.module, th, l0.portMap(s0), l1.portMap(s1)) + } + } + case _ => + } + }} + } +} + +class WithMultiChipSerialTL(chip0: Int, chip1: Int) extends MultiHarnessBinder(chip0, chip1, ( + (system0: CanHavePeripheryTLSerial, system1: CanHavePeripheryTLSerial, + th: HasHarnessInstantiators, + ports0: Seq[ClockedIO[SerialIO]], ports1: Seq[ClockedIO[SerialIO]] + ) => { + require(ports0.size == ports1.size) + (ports0 zip ports1).map { case (l, r) => + l.clock <> r.clock + require(l.bits.w == r.bits.w) + l.bits.flipConnect(r.bits) + } + } +)) diff --git a/generators/chipyard/src/main/scala/harness/package.scala b/generators/chipyard/src/main/scala/harness/package.scala new file mode 100644 index 00000000..789d1595 --- /dev/null +++ b/generators/chipyard/src/main/scala/harness/package.scala @@ -0,0 +1,17 @@ +package chipyard + +import chisel3._ +import scala.collection.immutable.ListMap + +package object harness +{ + type HarnessBinderFunction = (Any, HasHarnessInstantiators, Seq[Data]) => Unit + type HarnessBinderMap = Map[String, HarnessBinderFunction] + def HarnessBinderMapDefault: HarnessBinderMap = (new ListMap[String, HarnessBinderFunction]) + .withDefaultValue((t: Any, th: HasHarnessInstantiators, d: Seq[Data]) => ()) + + type MultiHarnessBinderFunction = (Any, Any, HasHarnessInstantiators, Seq[Data], Seq[Data]) => Unit + type MultiHarnessBinderMap = Map[(String, String), MultiHarnessBinderFunction] + def MultiHarnessBinderMapDefault: MultiHarnessBinderMap = (new ListMap[(String, String), MultiHarnessBinderFunction]) + .withDefaultValue((_: Any, _: Any, _: HasHarnessInstantiators, _: Seq[Data], _: Seq[Data]) => ()) +} From 1d50fb31156db0cc989725b5c79747e78d68a598 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 May 2023 11:29:03 -0700 Subject: [PATCH 06/24] Hang the periphery TLSerial off a OBUS --- .../src/main/scala/config/ChipConfigs.scala | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/generators/chipyard/src/main/scala/config/ChipConfigs.scala b/generators/chipyard/src/main/scala/config/ChipConfigs.scala index 1d717eae..e7c8ce9b 100644 --- a/generators/chipyard/src/main/scala/config/ChipConfigs.scala +++ b/generators/chipyard/src/main/scala/config/ChipConfigs.scala @@ -2,30 +2,37 @@ package chipyard import org.chipsalliance.cde.config.{Config} import freechips.rocketchip.diplomacy._ +import freechips.rocketchip.subsystem.{MBUS} // A simple config demonstrating how to set up a basic chip in Chipyard -class ChipLikeQuadRocketConfig extends Config( +class ChipLikeRocketConfig extends Config( //================================== // Set up TestHarness //================================== new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ // use absolute frequencies for simulations in the harness // NOTE: This only simulates properly in VCS + new chipyard.harness.WithSimAXIMemOverSerialTL ++ // Attach SimDRAM to serial-tl port //================================== // Set up tiles //================================== new freechips.rocketchip.subsystem.WithAsynchronousRocketTiles(3, 3) ++ // Add rational crossings between RocketTile and uncore - new freechips.rocketchip.subsystem.WithNBigCores(4) ++ // quad-core (4 RocketTiles) + new freechips.rocketchip.subsystem.WithNBigCores(1) ++ // 1 RocketTile //================================== // Set up I/O //================================== new testchipip.WithSerialTLWidth(4) ++ new testchipip.WithSerialTLBackingMemory ++ // Backing memory is over serial TL protocol - new chipyard.harness.WithSimAXIMemOverSerialTL ++ // Attach fast SimDRAM to TestHarness new freechips.rocketchip.subsystem.WithExtMemSize((1 << 30) * 4L) ++ // 4GB max external memory new freechips.rocketchip.subsystem.WithNMemoryChannels(1) ++ // 1 memory channel + //================================== + // Set up buses + //================================== + new testchipip.WithOffchipBusManager(MBUS) ++ + new testchipip.WithOffchipBus ++ + //================================== // Set up clock./reset //================================== From 922a4c11ada9f230cff0122cb49ddb748f8b5e4d Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 May 2023 16:37:36 -0700 Subject: [PATCH 07/24] Add tethered chip bringup example --- .../chipyard/src/main/scala/IOBinders.scala | 9 +++ .../src/main/scala/config/ChipConfigs.scala | 56 ++++++++++++++++++- .../src/main/scala/config/NoCoreConfigs.scala | 9 +++ .../config/fragments/ClockingFragments.scala | 8 +++ .../fragments/PeripheralFragments.scala | 18 +++++- .../main/scala/harness/HarnessBinders.scala | 22 +++++++- 6 files changed, 119 insertions(+), 3 deletions(-) diff --git a/generators/chipyard/src/main/scala/IOBinders.scala b/generators/chipyard/src/main/scala/IOBinders.scala index d00067b4..3c8fbeb0 100644 --- a/generators/chipyard/src/main/scala/IOBinders.scala +++ b/generators/chipyard/src/main/scala/IOBinders.scala @@ -303,6 +303,15 @@ class WithSerialTLIOCells extends OverrideIOBinder({ }).getOrElse((Nil, Nil)) }) +class WithSerialTLPunchthrough extends OverrideIOBinder({ + (system: CanHavePeripheryTLSerial) => system.serial_tl.map({ s => + val sys = system.asInstanceOf[BaseSubsystem] + val port = IO(s.getWrappedValue.cloneType) + port <> s.getWrappedValue + (Seq(port), Nil) + }).getOrElse((Nil, Nil)) +}) + class WithAXI4MemPunchthrough extends OverrideLazyIOBinder({ (system: CanHaveMasterAXI4MemPort) => { implicit val p: Parameters = GetSystemParameters(system) diff --git a/generators/chipyard/src/main/scala/config/ChipConfigs.scala b/generators/chipyard/src/main/scala/config/ChipConfigs.scala index e7c8ce9b..10c9be23 100644 --- a/generators/chipyard/src/main/scala/config/ChipConfigs.scala +++ b/generators/chipyard/src/main/scala/config/ChipConfigs.scala @@ -2,7 +2,8 @@ package chipyard import org.chipsalliance.cde.config.{Config} import freechips.rocketchip.diplomacy._ -import freechips.rocketchip.subsystem.{MBUS} +import freechips.rocketchip.subsystem.{MBUS, SBUS} +import testchipip.{OBUS} // A simple config demonstrating how to set up a basic chip in Chipyard class ChipLikeRocketConfig extends Config( @@ -43,3 +44,56 @@ class ChipLikeRocketConfig extends Config( new chipyard.config.AbstractConfig) +// A simple config demonstrating a "bringup prototype" to bringup the ChipLikeRocketconfig +class ChipBringupHostConfig extends Config( + //============================= + // Set up TestHarness for standalone-sim + // These fragments only affect the design when simulated by itself (without the ChipLikeRocketConfig) + //============================= + new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ + new chipyard.harness.WithSerialTLTiedOff ++ // when doing standalone sim, tie off the serial-tl port + new chipyard.harness.WithSimTSIToUARTTSI ++ + new chipyard.iobinders.WithSerialTLPunchthrough ++ + + //============================= + // Setup the SerialTL side on the bringup device + //============================= + new testchipip.WithSerialTLWidth(4) ++ // match width with the chip + new testchipip.WithSerialTLMem(base = 0x0, size = BigInt(1) << 48, // accessible memory of the chip + idBits = 4, isMainMemory = false) ++ + new testchipip.WithSerialTLClockDirection(provideClockFreqMHz = Some(75)) ++ // bringup board drives the clock for the serial-tl receiver on the chip, use 50MHz clock + + //============================ + // Setup bus topology on the bringup system + //============================ + new testchipip.WithOffchipBusManager(SBUS, + blockRange = AddressSet.misaligned(0x80000000L, (BigInt(1) << 30) * 4), + replicationBase = Some(BigInt(1) << 48)) ++ + new testchipip.WithOffchipBus ++ // offchip bus + + //============================= + // Set up memory on the bringup system + //============================= + new freechips.rocketchip.subsystem.WithExtMemSize((1 << 30) * 4L) ++ // match what the chip believes + + //============================= + // Generate the TSI-over-UART side of the bringup system + //============================= + new testchipip.WithUARTTSIClient(initBaudRate = BigInt(921600)) ++ // nonstandard baud rate to improve performance + + //============================= + // Set up clocks of the bringup system + //============================= + new chipyard.clocking.WithPassthroughClockGenerator ++ // pass all the clocks through, since this isn't a chip + new chipyard.config.WithFrontBusFrequency(75.0) ++ + new chipyard.config.WithMemoryBusFrequency(75.0) ++ + new chipyard.config.WithPeripheryBusFrequency(75.0) ++ + + // Base is the no-cores config + new chipyard.NoCoresConfig) + +class TetheredChipLikeRocketConfig extends Config( + new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ // use absolute freqs for sims in the harness + new chipyard.harness.WithMultiChipSerialTL(0, 1) ++ + new chipyard.harness.WithMultiChip(0, new ChipLikeRocketConfig) ++ + new chipyard.harness.WithMultiChip(1, new ChipBringupHostConfig)) diff --git a/generators/chipyard/src/main/scala/config/NoCoreConfigs.scala b/generators/chipyard/src/main/scala/config/NoCoreConfigs.scala index 10be8aca..19e9a7e0 100644 --- a/generators/chipyard/src/main/scala/config/NoCoreConfigs.scala +++ b/generators/chipyard/src/main/scala/config/NoCoreConfigs.scala @@ -4,6 +4,15 @@ import org.chipsalliance.cde.config.{Config} // A empty config with no cores. Useful for testing class NoCoresConfig extends Config( + new testchipip.WithNoBootAddrReg ++ + new testchipip.WithNoCustomBootPin ++ + new chipyard.config.WithNoCLINT ++ + new chipyard.config.WithNoBootROM ++ + new chipyard.config.WithBroadcastManager ++ + new chipyard.config.WithNoUART ++ + new chipyard.config.WithNoTileClockGaters ++ + new chipyard.config.WithNoTileResetSetters ++ + new chipyard.config.WithNoBusErrorDevices ++ new chipyard.config.WithNoDebug ++ new chipyard.config.WithNoPLIC ++ new chipyard.config.AbstractConfig) diff --git a/generators/chipyard/src/main/scala/config/fragments/ClockingFragments.scala b/generators/chipyard/src/main/scala/config/fragments/ClockingFragments.scala index 1f7bb943..509b2a73 100644 --- a/generators/chipyard/src/main/scala/config/fragments/ClockingFragments.scala +++ b/generators/chipyard/src/main/scala/config/fragments/ClockingFragments.scala @@ -106,3 +106,11 @@ class WithControlBusFrequency(freqMHz: Double) extends Config((site, here, up) = class WithRationalMemoryBusCrossing extends WithSbusToMbusCrossingType(RationalCrossing(Symmetric)) class WithAsynchrousMemoryBusCrossing extends WithSbusToMbusCrossingType(AsynchronousCrossing()) + +class WithNoTileClockGaters extends Config((site, here, up) => { + case ChipyardPRCIControlKey => up(ChipyardPRCIControlKey).copy(enableTileClockGating = false) +}) + +class WithNoTileResetSetters extends Config((site, here, up) => { + case ChipyardPRCIControlKey => up(ChipyardPRCIControlKey).copy(enableTileResetSetting = false) +}) diff --git a/generators/chipyard/src/main/scala/config/fragments/PeripheralFragments.scala b/generators/chipyard/src/main/scala/config/fragments/PeripheralFragments.scala index 3b607ae0..85606494 100644 --- a/generators/chipyard/src/main/scala/config/fragments/PeripheralFragments.scala +++ b/generators/chipyard/src/main/scala/config/fragments/PeripheralFragments.scala @@ -5,7 +5,7 @@ import chisel3._ import chisel3.util.{log2Up} import org.chipsalliance.cde.config.{Config} -import freechips.rocketchip.devices.tilelink.{BootROMLocated, PLICKey} +import freechips.rocketchip.devices.tilelink.{BootROMLocated, PLICKey, CLINTKey} import freechips.rocketchip.devices.debug.{Debug, ExportDebug, DebugModuleKey, DMI} import freechips.rocketchip.stage.phases.TargetDirKey import freechips.rocketchip.subsystem._ @@ -75,3 +75,19 @@ class WithNoPLIC extends Config((site, here, up) => { class WithDebugModuleAbstractDataWords(words: Int = 16) extends Config((site, here, up) => { case DebugModuleKey => up(DebugModuleKey).map(_.copy(nAbstractDataWords=words)) }) + +class WithNoCLINT extends Config((site, here, up) => { + case CLINTKey => None +}) + +class WithNoBootROM extends Config((site, here, up) => { + case BootROMLocated(_) => None +}) + +class WithNoBusErrorDevices extends Config((site, here, up) => { + case SystemBusKey => up(SystemBusKey).copy(errorDevice = None) + case ControlBusKey => up(ControlBusKey).copy(errorDevice = None) + case PeripheryBusKey => up(PeripheryBusKey).copy(errorDevice = None) + case MemoryBusKey => up(MemoryBusKey).copy(errorDevice = None) + case FrontBusKey => up(FrontBusKey).copy(errorDevice = None) +}) diff --git a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala index db6f0cda..4cb85aba 100644 --- a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala +++ b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala @@ -290,7 +290,9 @@ class WithSerialTLTiedOff extends OverrideHarnessBinder({ implicit val p = chipyard.iobinders.GetSystemParameters(system) ports.map({ port => val bits = port.bits - port.clock := false.B.asClock + if (DataMirror.directionOf(port.clock) == Direction.Input) { + port.clock := false.B.asClock + } port.bits.out.ready := false.B port.bits.in.valid := false.B port.bits.in.bits := DontCare @@ -325,6 +327,24 @@ class WithSimUARTToUARTTSI extends OverrideHarnessBinder({ } }) +class WithSimTSIToUARTTSI extends OverrideHarnessBinder({ + (system: CanHavePeripheryUARTTSI, th: HasHarnessInstantiators, ports: Seq[UARTTSIIO]) => { + implicit val p = chipyard.iobinders.GetSystemParameters(system) + require(ports.size <= 1) + ports.map({ port => + val freq = th.getHarnessBinderClockFreqHz.toInt + val uart_to_serial = Module(new UARTToSerial(freq, port.uartParams)) + val serial_width_adapter = Module(new SerialWidthAdapter(8, TSI.WIDTH)) + val success = SimTSI.connect(Some(TSIIO(serial_width_adapter.io.wide)), th.harnessBinderClock, th.harnessBinderReset) + when (success) { th.success := true.B } + assert(!uart_to_serial.io.dropped) + serial_width_adapter.io.narrow.flipConnect(uart_to_serial.io.serial) + uart_to_serial.io.uart.rxd := port.uart.txd + port.uart.rxd := uart_to_serial.io.uart.txd + }) + } +}) + class WithTraceGenSuccess extends OverrideHarnessBinder({ (system: TraceGenSystemModuleImp, th: HasHarnessInstantiators, ports: Seq[Bool]) => { From 68402ab99f34d041e569b4a1b8dff3ac6230e557 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 May 2023 17:21:42 -0700 Subject: [PATCH 08/24] Bump rocket-chip-blocks --- generators/sifive-blocks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generators/sifive-blocks b/generators/sifive-blocks index 534d3b74..abf129a3 160000 --- a/generators/sifive-blocks +++ b/generators/sifive-blocks @@ -1 +1 @@ -Subproject commit 534d3b74a0f22e67198aa361ae987042ee56dead +Subproject commit abf129a33bf3d73dbc017f34862038b6e722b8ed From 8a46de0d9dfb982f49a82b22972dc60347b82af4 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 May 2023 23:41:42 -0700 Subject: [PATCH 09/24] Add BringupHostConfig and TetheredChip dual-chip sim configs --- .../src/main/scala/config/ChipConfigs.scala | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/generators/chipyard/src/main/scala/config/ChipConfigs.scala b/generators/chipyard/src/main/scala/config/ChipConfigs.scala index 10c9be23..c7c6cae2 100644 --- a/generators/chipyard/src/main/scala/config/ChipConfigs.scala +++ b/generators/chipyard/src/main/scala/config/ChipConfigs.scala @@ -48,44 +48,42 @@ class ChipLikeRocketConfig extends Config( class ChipBringupHostConfig extends Config( //============================= // Set up TestHarness for standalone-sim - // These fragments only affect the design when simulated by itself (without the ChipLikeRocketConfig) //============================= - new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ - new chipyard.harness.WithSerialTLTiedOff ++ // when doing standalone sim, tie off the serial-tl port - new chipyard.harness.WithSimTSIToUARTTSI ++ - new chipyard.iobinders.WithSerialTLPunchthrough ++ + new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ // Generate absolute frequencies + new chipyard.harness.WithSerialTLTiedOff ++ // when doing standalone sim, tie off the serial-tl port + new chipyard.harness.WithSimTSIToUARTTSI ++ // Attach SimTSI-over-UART to the UART-TSI port + new chipyard.iobinders.WithSerialTLPunchthrough ++ // Don't generate IOCells for the serial TL (this design maps to FPGA) //============================= // Setup the SerialTL side on the bringup device //============================= new testchipip.WithSerialTLWidth(4) ++ // match width with the chip - new testchipip.WithSerialTLMem(base = 0x0, size = BigInt(1) << 48, // accessible memory of the chip - idBits = 4, isMainMemory = false) ++ + new testchipip.WithSerialTLMem(base = 0x0, size = 0x80000000L, // accessible memory of the chip that doesn't come from the tethered host + idBits = 4, isMainMemory = false) ++ // This assumes off-chip mem starts at 0x8000_0000 new testchipip.WithSerialTLClockDirection(provideClockFreqMHz = Some(75)) ++ // bringup board drives the clock for the serial-tl receiver on the chip, use 50MHz clock //============================ // Setup bus topology on the bringup system //============================ - new testchipip.WithOffchipBusManager(SBUS, - blockRange = AddressSet.misaligned(0x80000000L, (BigInt(1) << 30) * 4), - replicationBase = Some(BigInt(1) << 48)) ++ + new testchipip.WithOffchipBusManager(SBUS, // offchip bus hans off the SBUS + blockRange = AddressSet.misaligned(0x80000000L, (BigInt(1) << 30) * 4)) ++ // offchip bus should not see the main memory of the testchip, since that can be accesses directly through this system new testchipip.WithOffchipBus ++ // offchip bus //============================= // Set up memory on the bringup system //============================= - new freechips.rocketchip.subsystem.WithExtMemSize((1 << 30) * 4L) ++ // match what the chip believes + new freechips.rocketchip.subsystem.WithExtMemSize((1 << 30) * 4L) ++ // match what the chip believes the max size should be //============================= // Generate the TSI-over-UART side of the bringup system //============================= - new testchipip.WithUARTTSIClient(initBaudRate = BigInt(921600)) ++ // nonstandard baud rate to improve performance + new testchipip.WithUARTTSIClient(initBaudRate = BigInt(921600)) ++ // nonstandard baud rate to improve performance //============================= // Set up clocks of the bringup system //============================= - new chipyard.clocking.WithPassthroughClockGenerator ++ // pass all the clocks through, since this isn't a chip - new chipyard.config.WithFrontBusFrequency(75.0) ++ + new chipyard.clocking.WithPassthroughClockGenerator ++ // pass all the clocks through, since this isn't a chip + new chipyard.config.WithFrontBusFrequency(75.0) ++ // run all buses of this system at 75 MHz new chipyard.config.WithMemoryBusFrequency(75.0) ++ new chipyard.config.WithPeripheryBusFrequency(75.0) ++ @@ -93,7 +91,7 @@ class ChipBringupHostConfig extends Config( new chipyard.NoCoresConfig) class TetheredChipLikeRocketConfig extends Config( - new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ // use absolute freqs for sims in the harness - new chipyard.harness.WithMultiChipSerialTL(0, 1) ++ + new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ // use absolute freqs for sims in the harness + new chipyard.harness.WithMultiChipSerialTL(0, 1) ++ // connect the serial-tl ports of the chips together new chipyard.harness.WithMultiChip(0, new ChipLikeRocketConfig) ++ new chipyard.harness.WithMultiChip(1, new ChipBringupHostConfig)) From ec7310dad04fcd20964857c3e9bad610fa912440 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 May 2023 23:52:14 -0700 Subject: [PATCH 10/24] Dedup run-tests CI script --- .github/scripts/run-tests.sh | 76 ++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/.github/scripts/run-tests.sh b/.github/scripts/run-tests.sh index 5ed6aab4..dd42a4a2 100755 --- a/.github/scripts/run-tests.sh +++ b/.github/scripts/run-tests.sh @@ -12,11 +12,11 @@ source $SCRIPT_DIR/defaults.sh DISABLE_SIM_PREREQ="BREAK_SIM_PREREQ=1" run_bmark () { - make run-bmark-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ $@ + make run-bmark-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} $@ } run_asm () { - make run-asm-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ $@ + make run-asm-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} $@ } run_both () { @@ -25,35 +25,41 @@ run_both () { } run_tracegen () { - make tracegen -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ $@ + make tracegen -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} $@ } +run_binary () { + make run-binary-fast -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} $@ + case $1 in chipyard-rocket) - run_bmark ${mapping[$1]} + run_bmark make -C $LOCAL_CHIPYARD_DIR/tests - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary LOADMEM=1 BINARY=$LOCAL_CHIPYARD_DIR/tests/hello.riscv - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary BINARY=$LOCAL_CHIPYARD_DIR/tests/hello.riscv + # Test run-binary with and without loadmem + run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/hello.riscv LOADMEM=1 + run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/hello.riscv ;; chipyard-dmirocket) + # Test checkpoint-restore $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -i 10000 - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch + run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch ;; chipyard-boom) - run_bmark ${mapping[$1]} + run_bmark ;; chipyard-dmiboom) + # Test checkpoint-restore $LOCAL_CHIPYARD_DIR/scripts/generate-ckpt.sh -b $RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -i 10000 - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch + run_binary LOADARCH=$PWD/dhrystone.riscv.0x80000000.10000.loadarch ;; chipyard-spike) - run_bmark ${mapping[$1]} + run_bmark ;; chipyard-hetero) - run_bmark ${mapping[$1]} + run_bmark ;; rocketchip) - run_bmark ${mapping[$1]} + run_bmark ;; chipyard-hwacha) make run-rv64uv-p-asm-tests -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} @@ -62,74 +68,70 @@ case $1 in GEMMINI_SOFTWARE_DIR=$LOCAL_SIM_DIR/../../generators/gemmini/software/gemmini-rocc-tests rm -rf $GEMMINI_SOFTWARE_DIR/riscv-tests cd $LOCAL_SIM_DIR - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary-fast BINARY=$GEMMINI_SOFTWARE_DIR/build/bareMetalC/aligned-baremetal - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary-fast BINARY=$GEMMINI_SOFTWARE_DIR/build/bareMetalC/raw_hazard-baremetal - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary-fast BINARY=$GEMMINI_SOFTWARE_DIR/build/bareMetalC/mvin_mvout-baremetal + run_binary BINARY=$GEMMINI_SOFTWARE_DIR/build/bareMetalC/aligned-baremetal + run_binary BINARY=$GEMMINI_SOFTWARE_DIR/build/bareMetalC/raw_hazard-baremetal + run_binary BINARY=$GEMMINI_SOFTWARE_DIR/build/bareMetalC/mvin_mvout-baremetal ;; chipyard-sha3) (cd $LOCAL_CHIPYARD_DIR/generators/sha3/software && ./build.sh) - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary-fast BINARY=$LOCAL_CHIPYARD_DIR/generators/sha3/software/tests/bare/sha3-rocc.riscv + run_binary BINARY=$LOCAL_CHIPYARD_DIR/generators/sha3/software/tests/bare/sha3-rocc.riscv ;; chipyard-mempress) (cd $LOCAL_CHIPYARD_DIR/generators/mempress/software/src && make) - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary-fast BINARY=$LOCAL_CHIPYARD_DIR/generators/mempress/software/src/mempress-rocc.riscv + run_binary BINARY=$LOCAL_CHIPYARD_DIR/generators/mempress/software/src/mempress-rocc.riscv ;; chipyard-manymmioaccels) make -C $LOCAL_CHIPYARD_DIR/tests # test streaming-passthrough - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary-fast BINARY=$LOCAL_CHIPYARD_DIR/tests/streaming-passthrough.riscv + run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/streaming-passthrough.riscv # test streaming-fir - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary-fast BINARY=$LOCAL_CHIPYARD_DIR/tests/streaming-fir.riscv + run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/streaming-fir.riscv # test nvdla - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} BINARY=$LOCAL_CHIPYARD_DIR/tests/nvdla.riscv run-binary-fast + run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/nvdla.riscv # test fft - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} BINARY=$LOCAL_CHIPYARD_DIR/tests/fft.riscv run-binary-fast - + run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/fft.riscv ;; chipyard-manyperipherals) - # SPI Flash read tests, then bmark tests - + # SPI Flash read tests make -C $LOCAL_CHIPYARD_DIR/tests - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} BINARY=$LOCAL_CHIPYARD_DIR/tests/spiflashread.riscv run-binary-fast - - run_bmark ${mapping[$1]} + run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/spiflashread.riscv ;; chipyard-spiflashwrite) make -C $LOCAL_CHIPYARD_DIR/tests - make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} BINARY=$LOCAL_CHIPYARD_DIR/tests/spiflashwrite.riscv run-binary-fast + run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/spiflashwrite.riscv [[ "`xxd $LOCAL_CHIPYARD_DIR/tests/spiflash.img | grep 1337\ 00ff\ aa55\ face | wc -l`" == "6" ]] || false ;; tracegen) - run_tracegen ${mapping[$1]} + run_tracegen ;; tracegen-boom) - run_tracegen ${mapping[$1]} + run_tracegen ;; chipyard-cva6) - make run-binary-fast -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/multiply.riscv + run_binary BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/multiply.riscv ;; chipyard-ibex) # Ibex cannot run the riscv-tests binaries for some reason - # make run-binary-fast -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/isa/rv32ui-p-simple + # run_binary BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/isa/rv32ui-p-simple ;; chipyard-sodor) - run_asm ${mapping[$1]} + run_asm ;; chipyard-constellation) - make run-binary-hex BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} + run_binary LOADMEM=1 BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/dhrystone.riscv ;; icenet) - make run-binary-fast BINARY=none -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} + run_binary BINARY=none ;; testchipip) - make run-binary-fast BINARY=none -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} + run_binary BINARY=none ;; constellation) - make run-binary-fast BINARY=none -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} + run_binary BINARY=none ;; *) echo "No set of tests for $1. Did you spell it right?" From dda57705950c43648a583ec4d68587c5cd001da5 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 May 2023 23:55:15 -0700 Subject: [PATCH 11/24] Add Tethered config to CI --- .github/scripts/defaults.sh | 5 +++-- .github/scripts/run-tests.sh | 3 +++ .github/workflows/chipyard-run-tests.yml | 24 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/scripts/defaults.sh b/.github/scripts/defaults.sh index d9dd5f96..2d471e9e 100755 --- a/.github/scripts/defaults.sh +++ b/.github/scripts/defaults.sh @@ -29,7 +29,7 @@ REMOTE_COURSIER_CACHE=$REMOTE_WORK_DIR/.coursier-cache # key value store to get the build groups declare -A grouping grouping["group-cores"]="chipyard-cva6 chipyard-ibex chipyard-rocket chipyard-hetero chipyard-boom chipyard-sodor chipyard-digitaltop chipyard-multiclock-rocket chipyard-nomem-scratchpad chipyard-spike chipyard-clone" -grouping["group-peripherals"]="chipyard-dmirocket chipyard-dmiboom chipyard-spiflashwrite chipyard-mmios chipyard-nocores chipyard-manyperipherals chipyard-chiplike" +grouping["group-peripherals"]="chipyard-dmirocket chipyard-dmiboom chipyard-spiflashwrite chipyard-mmios chipyard-nocores chipyard-manyperipherals chipyard-chiplike chipyard-tethered" grouping["group-accels"]="chipyard-mempress chipyard-sha3 chipyard-hwacha chipyard-gemmini chipyard-manymmioaccels" grouping["group-constellation"]="chipyard-constellation" grouping["group-tracegen"]="tracegen tracegen-boom" @@ -54,7 +54,8 @@ mapping["chipyard-cva6"]=" CONFIG=CVA6Config" mapping["chipyard-ibex"]=" CONFIG=IbexConfig" mapping["chipyard-spiflashwrite"]=" CONFIG=SmallSPIFlashRocketConfig EXTRA_SIM_FLAGS='+spiflash0=${LOCAL_CHIPYARD_DIR}/tests/spiflash.img'" mapping["chipyard-manyperipherals"]=" CONFIG=ManyPeripheralsRocketConfig EXTRA_SIM_FLAGS='+spiflash0=${LOCAL_CHIPYARD_DIR}/tests/spiflash.img'" -mapping["chipyard-chiplike"]=" CONFIG=ChipLikeQuadRocketConfig MODEL=FlatTestHarness MODEL_PACKAGE=chipyard.example verilog" +mapping["chipyard-chiplike"]=" CONFIG=ChipLikeRocketConfig MODEL=FlatTestHarness MODEL_PACKAGE=chipyard.example verilog" +mapping["chipyard-tethered"]=" CONFIG=TetheredChipLikeRocketConfig" mapping["chipyard-cloneboom"]=" CONFIG=Cloned64MegaBoomConfig verilog" mapping["chipyard-nocores"]=" CONFIG=NoCoresConfig verilog" mapping["tracegen"]=" CONFIG=NonBlockingTraceGenL2Config" diff --git a/.github/scripts/run-tests.sh b/.github/scripts/run-tests.sh index dd42a4a2..e97e68fe 100755 --- a/.github/scripts/run-tests.sh +++ b/.github/scripts/run-tests.sh @@ -105,6 +105,9 @@ case $1 in run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/spiflashwrite.riscv [[ "`xxd $LOCAL_CHIPYARD_DIR/tests/spiflash.img | grep 1337\ 00ff\ aa55\ face | wc -l`" == "6" ]] || false ;; + chipyard-tethered) + make -C $LOCAL_CHIPYARD_DIR/tests + run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/hello.riscv LOADMEM=1 EXTRA_SIM_FLAGS="+cflush_addr=0x2010200" tracegen) run_tracegen ;; diff --git a/.github/workflows/chipyard-run-tests.yml b/.github/workflows/chipyard-run-tests.yml index d357637c..f8ca8e11 100644 --- a/.github/workflows/chipyard-run-tests.yml +++ b/.github/workflows/chipyard-run-tests.yml @@ -672,6 +672,29 @@ jobs: group-key: "group-peripherals" project-key: "chipyard-manyperipherals" + chipyard-tethered-run-tests: + name: chipyard-tethered-run-tests + needs: prepare-chipyard-peripherals + runs-on: self-hosted + steps: + - name: Delete old checkout + run: | + ls -alh . + rm -rf ${{ github.workspace }}/* || true + rm -rf ${{ github.workspace }}/.* || true + ls -alh . + - name: Checkout + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env + - name: Run tests + uses: ./.github/actions/run-tests + with: + group-key: "group-peripherals" + project-key: "chipyard-tethered" + chipyard-sha3-run-tests: name: chipyard-sha3-run-tests needs: prepare-chipyard-accels @@ -971,6 +994,7 @@ jobs: chipyard-dmirocket-run-tests, chipyard-spiflashwrite-run-tests, chipyard-manyperipherals-run-tests, + chipyard-tethered-run-tests, chipyard-sha3-run-tests, chipyard-gemmini-run-tests, chipyard-manymmioaccels-run-tests, From 38966fe8309cc50881bc45daf48ac7a9fa772cac Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Fri, 26 May 2023 00:05:22 -0700 Subject: [PATCH 12/24] Bump testchipip --- generators/testchipip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generators/testchipip b/generators/testchipip index e2ab39f2..3ecea861 160000 --- a/generators/testchipip +++ b/generators/testchipip @@ -1 +1 @@ -Subproject commit e2ab39f2777f92e8ebc467ebf9c7bf1974793e1e +Subproject commit 3ecea8610d1d9d7dedbf30bebe645e624f5d28d6 From bd49068eba4e2352e774688f93dc33d1c102da07 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Fri, 26 May 2023 09:28:25 -0700 Subject: [PATCH 13/24] Fix ManyPeripheralsRocketConfig' --- .../src/main/scala/config/PeripheralDeviceConfigs.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala b/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala index 9eec165e..db766a1e 100644 --- a/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala +++ b/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala @@ -65,13 +65,15 @@ class dmiRocketConfig extends Config( // DOC include end: DmiRocket class ManyPeripheralsRocketConfig extends Config( + new testchipip.WithBlockDevice ++ // add block-device module to peripherybus + new testchipip.WithOffchipBusManager(MBUS) ++ + new testchipip.WithOffchipBus ++ + new testchipip.WithSerialTLMem(isMainMemory=true) ++ // set lbwif memory base to DRAM_BASE, use as main memory new chipyard.harness.WithSimSPIFlashModel(true) ++ // add the SPI flash model in the harness (read-only) new chipyard.harness.WithSimBlockDevice ++ // drive block-device IOs with SimBlockDevice new chipyard.config.WithSPIFlash ++ // add the SPI flash controller new freechips.rocketchip.subsystem.WithDefaultMMIOPort ++ // add default external master port new freechips.rocketchip.subsystem.WithDefaultSlavePort ++ // add default external slave port - new testchipip.WithBlockDevice ++ // add block-device module to peripherybus - new testchipip.WithSerialTLMem(isMainMemory=true) ++ // set lbwif memory base to DRAM_BASE, use as main memory new freechips.rocketchip.subsystem.WithNoMemPort ++ // remove AXI4 backing memory new freechips.rocketchip.subsystem.WithNBigCores(1) ++ new chipyard.config.AbstractConfig) From 17135368a643ccbef699b63df0b80de041c1da55 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Fri, 26 May 2023 09:33:35 -0700 Subject: [PATCH 14/24] Rename offchipbus manager to client --- generators/chipyard/src/main/scala/config/ChipConfigs.scala | 2 +- .../src/main/scala/config/PeripheralDeviceConfigs.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generators/chipyard/src/main/scala/config/ChipConfigs.scala b/generators/chipyard/src/main/scala/config/ChipConfigs.scala index c7c6cae2..e4acefd5 100644 --- a/generators/chipyard/src/main/scala/config/ChipConfigs.scala +++ b/generators/chipyard/src/main/scala/config/ChipConfigs.scala @@ -65,7 +65,7 @@ class ChipBringupHostConfig extends Config( //============================ // Setup bus topology on the bringup system //============================ - new testchipip.WithOffchipBusManager(SBUS, // offchip bus hans off the SBUS + new testchipip.WithOffchipBusClient(SBUS, // offchip bus hans off the SBUS blockRange = AddressSet.misaligned(0x80000000L, (BigInt(1) << 30) * 4)) ++ // offchip bus should not see the main memory of the testchip, since that can be accesses directly through this system new testchipip.WithOffchipBus ++ // offchip bus diff --git a/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala b/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala index db766a1e..7ad1f17d 100644 --- a/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala +++ b/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala @@ -66,7 +66,7 @@ class dmiRocketConfig extends Config( class ManyPeripheralsRocketConfig extends Config( new testchipip.WithBlockDevice ++ // add block-device module to peripherybus - new testchipip.WithOffchipBusManager(MBUS) ++ + new testchipip.WithOffchipBusClient(MBUS) ++ new testchipip.WithOffchipBus ++ new testchipip.WithSerialTLMem(isMainMemory=true) ++ // set lbwif memory base to DRAM_BASE, use as main memory new chipyard.harness.WithSimSPIFlashModel(true) ++ // add the SPI flash model in the harness (read-only) From 43daa427192cb0a2c1cfc610781f768a41076721 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Fri, 26 May 2023 09:34:44 -0700 Subject: [PATCH 15/24] Fix comment in ChipConfigs --- generators/chipyard/src/main/scala/config/ChipConfigs.scala | 6 +++--- .../src/main/scala/config/PeripheralDeviceConfigs.scala | 5 +++-- generators/testchipip | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/generators/chipyard/src/main/scala/config/ChipConfigs.scala b/generators/chipyard/src/main/scala/config/ChipConfigs.scala index e4acefd5..c85ef593 100644 --- a/generators/chipyard/src/main/scala/config/ChipConfigs.scala +++ b/generators/chipyard/src/main/scala/config/ChipConfigs.scala @@ -31,7 +31,7 @@ class ChipLikeRocketConfig extends Config( //================================== // Set up buses //================================== - new testchipip.WithOffchipBusManager(MBUS) ++ + new testchipip.WithOffchipBusClient(MBUS) ++ new testchipip.WithOffchipBus ++ //================================== @@ -65,8 +65,8 @@ class ChipBringupHostConfig extends Config( //============================ // Setup bus topology on the bringup system //============================ - new testchipip.WithOffchipBusClient(SBUS, // offchip bus hans off the SBUS - blockRange = AddressSet.misaligned(0x80000000L, (BigInt(1) << 30) * 4)) ++ // offchip bus should not see the main memory of the testchip, since that can be accesses directly through this system + new testchipip.WithOffchipBusClient(SBUS, // offchip bus hangs off the SBUS + blockRange = AddressSet.misaligned(0x80000000L, (BigInt(1) << 30) * 4)) ++ // offchip bus should not see the main memory of the testchip, since that can be accessed directly new testchipip.WithOffchipBus ++ // offchip bus //============================= diff --git a/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala b/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala index 7ad1f17d..aaa1dbe6 100644 --- a/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala +++ b/generators/chipyard/src/main/scala/config/PeripheralDeviceConfigs.scala @@ -2,6 +2,7 @@ package chipyard import org.chipsalliance.cde.config.{Config} import freechips.rocketchip.diplomacy.{AsynchronousCrossing} +import freechips.rocketchip.subsystem.{MBUS} // --------------------------------------------------------- // Configs which add non-default peripheral devices or ports @@ -66,8 +67,8 @@ class dmiRocketConfig extends Config( class ManyPeripheralsRocketConfig extends Config( new testchipip.WithBlockDevice ++ // add block-device module to peripherybus - new testchipip.WithOffchipBusClient(MBUS) ++ - new testchipip.WithOffchipBus ++ + new testchipip.WithOffchipBusClient(MBUS) ++ // OBUS provides backing memory to the MBUS + new testchipip.WithOffchipBus ++ // OBUS must exist for serial-tl to master off-chip memory new testchipip.WithSerialTLMem(isMainMemory=true) ++ // set lbwif memory base to DRAM_BASE, use as main memory new chipyard.harness.WithSimSPIFlashModel(true) ++ // add the SPI flash model in the harness (read-only) new chipyard.harness.WithSimBlockDevice ++ // drive block-device IOs with SimBlockDevice diff --git a/generators/testchipip b/generators/testchipip index 3ecea861..456223c9 160000 --- a/generators/testchipip +++ b/generators/testchipip @@ -1 +1 @@ -Subproject commit 3ecea8610d1d9d7dedbf30bebe645e624f5d28d6 +Subproject commit 456223c9164596f7c57e114b69b2584e0dcf2901 From 3db7976db62018dc7a70a5550f75947a39a5c072 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Fri, 26 May 2023 23:02:55 -0700 Subject: [PATCH 16/24] Fix typo in CI run-tests.sh --- .github/scripts/run-tests.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/scripts/run-tests.sh b/.github/scripts/run-tests.sh index e97e68fe..2c3a6af7 100755 --- a/.github/scripts/run-tests.sh +++ b/.github/scripts/run-tests.sh @@ -10,13 +10,14 @@ SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" source $SCRIPT_DIR/defaults.sh DISABLE_SIM_PREREQ="BREAK_SIM_PREREQ=1" +MAPPING_FLAGS=${mapping[$1]} run_bmark () { - make run-bmark-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} $@ + make run-bmark-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ $MAPPING_FLAGS $@ } run_asm () { - make run-asm-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} $@ + make run-asm-tests-fast -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ $MAPPING_FLAGS $@ } run_both () { @@ -25,11 +26,12 @@ run_both () { } run_tracegen () { - make tracegen -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} $@ + make tracegen -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ $MAPPING_FLAGS $@ } run_binary () { - make run-binary-fast -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} $@ + make run-binary-fast -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ $MAPPING_FLAGS $@ +} case $1 in chipyard-rocket) @@ -62,7 +64,7 @@ case $1 in run_bmark ;; chipyard-hwacha) - make run-rv64uv-p-asm-tests -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} + make run-rv64uv-p-asm-tests -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ $MAPPING_FLAGS ;; chipyard-gemmini) GEMMINI_SOFTWARE_DIR=$LOCAL_SIM_DIR/../../generators/gemmini/software/gemmini-rocc-tests @@ -108,6 +110,7 @@ case $1 in chipyard-tethered) make -C $LOCAL_CHIPYARD_DIR/tests run_binary BINARY=$LOCAL_CHIPYARD_DIR/tests/hello.riscv LOADMEM=1 EXTRA_SIM_FLAGS="+cflush_addr=0x2010200" + ;; tracegen) run_tracegen ;; From 592f1897e9e85e58043cfd853d4f9baba170dc14 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 28 May 2023 19:43:25 -0700 Subject: [PATCH 17/24] drive the top-level AsyncReset pin directly from reference reset --- generators/chipyard/src/main/scala/harness/HarnessBinders.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala index 4cb85aba..681b4cb2 100644 --- a/generators/chipyard/src/main/scala/harness/HarnessBinders.scala +++ b/generators/chipyard/src/main/scala/harness/HarnessBinders.scala @@ -394,7 +394,7 @@ class WithClockAndResetFromHarness extends OverrideHarnessBinder({ val clock = th.harnessClockInstantiator.requestClockMHz(s"clock_${c.freqMHz.toInt}MHz", c.freqMHz) c.clock := clock } - case r: AsyncReset => r := th.harnessBinderReset.asAsyncReset + case r: AsyncReset => r := th.referenceReset.asAsyncReset }) } }) From bb14a1f17344eb6aedfae054488169c88a4c3d98 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 1 Jun 2023 18:43:34 -0700 Subject: [PATCH 18/24] Fix NoCoresConfig with no PRCI features --- .../src/main/scala/clocking/ClockBinders.scala | 6 +++--- .../src/main/scala/clocking/HasChipyardPRCI.scala | 14 ++++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/generators/chipyard/src/main/scala/clocking/ClockBinders.scala b/generators/chipyard/src/main/scala/clocking/ClockBinders.scala index 9fa6f32b..3c2bab04 100644 --- a/generators/chipyard/src/main/scala/clocking/ClockBinders.scala +++ b/generators/chipyard/src/main/scala/clocking/ClockBinders.scala @@ -38,9 +38,9 @@ class WithPLLSelectorDividerClockGenerator extends OverrideLazyIOBinder({ val clockSelector = system.prci_ctrl_domain { LazyModule(new TLClockSelector(baseAddress + 0x30000, tlbus.beatBytes)) } val pllCtrl = system.prci_ctrl_domain { LazyModule(new FakePLLCtrl (baseAddress + 0x40000, tlbus.beatBytes)) } - clockDivider.tlNode := system.prci_ctrl_bus - clockSelector.tlNode := system.prci_ctrl_bus - pllCtrl.tlNode := system.prci_ctrl_bus + clockDivider.tlNode := system.prci_ctrl_bus.get + clockSelector.tlNode := system.prci_ctrl_bus.get + pllCtrl.tlNode := system.prci_ctrl_bus.get system.allClockGroupsNode := clockDivider.clockNode := clockSelector.clockNode diff --git a/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala b/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala index 71010c66..7422c7ad 100644 --- a/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala +++ b/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala @@ -21,7 +21,9 @@ case class ChipyardPRCIControlParams( baseAddress: BigInt = 0x100000, enableTileClockGating: Boolean = true, enableTileResetSetting: Boolean = true -) +) { + def generatePRCIXBar = enableTileClockGating || enableTileResetSetting +} case object ChipyardPRCIControlKey extends Field[ChipyardPRCIControlParams](ChipyardPRCIControlParams()) @@ -36,13 +38,13 @@ trait HasChipyardPRCI { this: BaseSubsystem with InstantiatesTiles => val prci_ctrl_domain = LazyModule(new ClockSinkDomain(name=Some("chipyard-prci-control"))) prci_ctrl_domain.clockNode := tlbus.fixedClockNode - val prci_ctrl_bus = prci_ctrl_domain { TLXbar() } - tlbus.coupleTo("prci_ctrl") { (prci_ctrl_bus + val prci_ctrl_bus = Option.when(prciParams.generatePRCIXBar) { prci_ctrl_domain { TLXbar() } } + prci_ctrl_bus.foreach(xbar => tlbus.coupleTo("prci_ctrl") { (xbar := TLFIFOFixer(TLFIFOFixer.all) := TLFragmenter(tlbus.beatBytes, tlbus.blockBytes) := TLBuffer() := _) - } + }) // Aggregate all the clock groups into a single node val aggregator = LazyModule(new ClockGroupAggregator("allClocks")).node @@ -82,13 +84,13 @@ trait HasChipyardPRCI { this: BaseSubsystem with InstantiatesTiles => val resetSynchronizer = prci_ctrl_domain { ClockGroupResetSynchronizer() } val tileClockGater = Option.when(prciParams.enableTileClockGating) { prci_ctrl_domain { val clock_gater = LazyModule(new TileClockGater(prciParams.baseAddress + 0x00000, tlbus.beatBytes)) - clock_gater.tlNode := prci_ctrl_bus + clock_gater.tlNode := prci_ctrl_bus.get clock_gater } } val tileResetSetter = Option.when(prciParams.enableTileResetSetting) { prci_ctrl_domain { val reset_setter = LazyModule(new TileResetSetter(prciParams.baseAddress + 0x10000, tlbus.beatBytes, tile_prci_domains.map(_.tile_reset_domain.clockNode.portParams(0).name.get), Nil)) - reset_setter.tlNode := prci_ctrl_bus + reset_setter.tlNode := prci_ctrl_bus.get reset_setter } } From 56e5421b8ee3d1aa1467c27952f1925061f109a4 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 5 Jun 2023 09:58:29 -0700 Subject: [PATCH 19/24] Fix verilator random seed flag --- sims/verilator/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sims/verilator/Makefile b/sims/verilator/Makefile index 8cf51e23..0341ff7d 100644 --- a/sims/verilator/Makefile +++ b/sims/verilator/Makefile @@ -32,7 +32,7 @@ include $(base_dir)/sims/common-sim-flags.mk # If verilator seed unspecified, verilator uses srand as random seed ifdef RANDOM_SEED -SEED_FLAG=+verilator+seed+I$(RANDOM_SEED) +SEED_FLAG=+verilator+seed+$(RANDOM_SEED) else SEED_FLAG= endif From 4d22af17a275addd740308284492212c0214142d Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 7 Jun 2023 23:38:00 -0700 Subject: [PATCH 20/24] Move ResetStretcher to testchipip --- .../main/scala/clocking/TLClockSelector.scala | 17 ----------------- generators/testchipip | 2 +- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/generators/chipyard/src/main/scala/clocking/TLClockSelector.scala b/generators/chipyard/src/main/scala/clocking/TLClockSelector.scala index 06821ead..27870428 100644 --- a/generators/chipyard/src/main/scala/clocking/TLClockSelector.scala +++ b/generators/chipyard/src/main/scala/clocking/TLClockSelector.scala @@ -13,23 +13,6 @@ import freechips.rocketchip.util.ElaborationArtefacts import testchipip._ -object ResetStretcher { - def apply(clock: Clock, reset: Reset, cycles: Int): Reset = { - withClockAndReset(clock, reset) { - val n = log2Ceil(cycles) - val count = Module(new AsyncResetRegVec(w=n, init=0)) - val resetout = Module(new AsyncResetRegVec(w=1, init=1)) - count.io.en := resetout.io.q - count.io.d := count.io.q + 1.U - resetout.io.en := resetout.io.q - resetout.io.d := count.io.q < (cycles-1).U - - resetout.io.q.asBool - } - } -} - - case class ClockSelNode()(implicit valName: ValName) extends MixedNexusNode(ClockImp, ClockGroupImp)( dFn = { d => ClockGroupSourceParameters() }, diff --git a/generators/testchipip b/generators/testchipip index 90401326..978e53e0 160000 --- a/generators/testchipip +++ b/generators/testchipip @@ -1 +1 @@ -Subproject commit 904013261858f74140558522eaea2419dd466c79 +Subproject commit 978e53e0033f8311fce55c7233e805d6375fcc91 From 517820146149b9904b726325b25777da3a25403c Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 8 Jun 2023 10:39:15 -0700 Subject: [PATCH 21/24] Add no-reset-sync config option to disable synchronizers for simulators which don't handle async reset properly --- .../main/scala/clocking/HasChipyardPRCI.scala | 28 +++++++++++++++++-- .../src/main/scala/config/ChipConfigs.scala | 10 +++++++ .../config/fragments/ClockingFragments.scala | 4 +++ generators/testchipip | 2 +- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala b/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala index 7422c7ad..5fb8f6ef 100644 --- a/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala +++ b/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala @@ -14,13 +14,14 @@ import freechips.rocketchip.util._ import freechips.rocketchip.tile._ import freechips.rocketchip.prci._ -import testchipip.{TLTileResetCtrl} +import testchipip.{TLTileResetCtrl, ClockGroupFakeResetSynchronizer} case class ChipyardPRCIControlParams( slaveWhere: TLBusWrapperLocation = CBUS, baseAddress: BigInt = 0x100000, enableTileClockGating: Boolean = true, - enableTileResetSetting: Boolean = true + enableTileResetSetting: Boolean = true, + enableResetSynchronizers: Boolean = true // this should only be disable to work around verilator async-reset initialziation problems ) { def generatePRCIXBar = enableTileClockGating || enableTileResetSetting } @@ -81,7 +82,9 @@ trait HasChipyardPRCI { this: BaseSubsystem with InstantiatesTiles => // diplomatic IOBinder should drive val frequencySpecifier = ClockGroupFrequencySpecifier(p(ClockFrequencyAssignersKey)) val clockGroupCombiner = ClockGroupCombiner() - val resetSynchronizer = prci_ctrl_domain { ClockGroupResetSynchronizer() } + val resetSynchronizer = prci_ctrl_domain { + if (prciParams.enableResetSynchronizers) ClockGroupResetSynchronizer() else ClockGroupFakeResetSynchronizer() + } val tileClockGater = Option.when(prciParams.enableTileClockGating) { prci_ctrl_domain { val clock_gater = LazyModule(new TileClockGater(prciParams.baseAddress + 0x00000, tlbus.beatBytes)) clock_gater.tlNode := prci_ctrl_bus.get @@ -94,6 +97,25 @@ trait HasChipyardPRCI { this: BaseSubsystem with InstantiatesTiles => reset_setter } } + if (!prciParams.enableResetSynchronizers) { + println(Console.RED + s""" + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +WARNING: + +DISABLING THE RESET SYNCHRONIZERS RESULTS IN +A BROKEN DESIGN THAT WILL NOT BEHAVE +PROPERLY AS ASIC OR FPGA. + +THESE SHOULD ONLY BE DISABLED TO WORK AROUND +LIMITATIONS IN ASYNC RESET INITIALIZATION IN +RTL SIMULATORS. + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +""" + Console.RESET) + } + (aggregator := frequencySpecifier := clockGroupCombiner diff --git a/generators/chipyard/src/main/scala/config/ChipConfigs.scala b/generators/chipyard/src/main/scala/config/ChipConfigs.scala index c8564a26..c1e641b0 100644 --- a/generators/chipyard/src/main/scala/config/ChipConfigs.scala +++ b/generators/chipyard/src/main/scala/config/ChipConfigs.scala @@ -95,3 +95,13 @@ class TetheredChipLikeRocketConfig extends Config( new chipyard.harness.WithMultiChipSerialTL(0, 1) ++ // connect the serial-tl ports of the chips together new chipyard.harness.WithMultiChip(0, new ChipLikeRocketConfig) ++ new chipyard.harness.WithMultiChip(1, new ChipBringupHostConfig)) + + +// Verilator does not initialize some of the async-reset reset-synchronizer +// flops properly, so this config disables them. +// This config should only be used for verilator simulations +class VerilatorCITetheredChipLikeRocketConfig extends Config( + new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ // use absolute freqs for sims in the harness + new chipyard.harness.WithMultiChipSerialTL(0, 1) ++ // connect the serial-tl ports of the chips together + new chipyard.harness.WithMultiChip(0, new chipyard.config.WithNoResetSynchronizers ++ new ChipLikeRocketConfig) ++ + new chipyard.harness.WithMultiChip(1, new ChipBringupHostConfig)) diff --git a/generators/chipyard/src/main/scala/config/fragments/ClockingFragments.scala b/generators/chipyard/src/main/scala/config/fragments/ClockingFragments.scala index 509b2a73..2da9fbf2 100644 --- a/generators/chipyard/src/main/scala/config/fragments/ClockingFragments.scala +++ b/generators/chipyard/src/main/scala/config/fragments/ClockingFragments.scala @@ -114,3 +114,7 @@ class WithNoTileClockGaters extends Config((site, here, up) => { class WithNoTileResetSetters extends Config((site, here, up) => { case ChipyardPRCIControlKey => up(ChipyardPRCIControlKey).copy(enableTileResetSetting = false) }) + +class WithNoResetSynchronizers extends Config((site, here, up) => { + case ChipyardPRCIControlKey => up(ChipyardPRCIControlKey).copy(enableResetSynchronizers = false) +}) diff --git a/generators/testchipip b/generators/testchipip index 978e53e0..bc43b99c 160000 --- a/generators/testchipip +++ b/generators/testchipip @@ -1 +1 @@ -Subproject commit 978e53e0033f8311fce55c7233e805d6375fcc91 +Subproject commit bc43b99cfcd60c7dab4659cdbd993eaca8f3cf95 From d13c54bcd1ab0b3b2c270e67951f05494327f632 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 8 Jun 2023 10:39:54 -0700 Subject: [PATCH 22/24] Use VerilatorCI config for TetheredChipRocketConfig --- .github/scripts/defaults.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/defaults.sh b/.github/scripts/defaults.sh index 85e4cc1b..872484b6 100755 --- a/.github/scripts/defaults.sh +++ b/.github/scripts/defaults.sh @@ -55,7 +55,7 @@ mapping["chipyard-ibex"]=" CONFIG=IbexConfig" mapping["chipyard-spiflashwrite"]=" CONFIG=SmallSPIFlashRocketConfig EXTRA_SIM_FLAGS='+spiflash0=${LOCAL_CHIPYARD_DIR}/tests/spiflash.img'" mapping["chipyard-manyperipherals"]=" CONFIG=ManyPeripheralsRocketConfig EXTRA_SIM_FLAGS='+spiflash0=${LOCAL_CHIPYARD_DIR}/tests/spiflash.img'" mapping["chipyard-chiplike"]=" CONFIG=ChipLikeRocketConfig MODEL=FlatTestHarness MODEL_PACKAGE=chipyard.example verilog" -mapping["chipyard-tethered"]=" CONFIG=TetheredChipLikeRocketConfig" +mapping["chipyard-tethered"]=" CONFIG=VerilatorCITetheredChipLikeRocketConfig" mapping["chipyard-cloneboom"]=" CONFIG=Cloned64MegaBoomConfig verilog" mapping["chipyard-nocores"]=" CONFIG=NoCoresConfig verilog" mapping["tracegen"]=" CONFIG=NonBlockingTraceGenL2Config" From 984ea24650e0308783819e7fd334c3a0ac555ca2 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 5 Jul 2023 14:55:03 -0700 Subject: [PATCH 23/24] Apply suggestions from code review Co-authored-by: Abraham Gonzalez --- .../chipyard/src/main/scala/clocking/HasChipyardPRCI.scala | 4 ++-- .../chipyard/src/main/scala/config/AbstractConfig.scala | 4 ++-- generators/chipyard/src/main/scala/config/ChipConfigs.scala | 2 +- .../{MultiHarnesBinders.scala => MultiHarnessBinders.scala} | 0 4 files changed, 5 insertions(+), 5 deletions(-) rename generators/chipyard/src/main/scala/harness/{MultiHarnesBinders.scala => MultiHarnessBinders.scala} (100%) diff --git a/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala b/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala index 5fb8f6ef..906a631d 100644 --- a/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala +++ b/generators/chipyard/src/main/scala/clocking/HasChipyardPRCI.scala @@ -21,7 +21,7 @@ case class ChipyardPRCIControlParams( baseAddress: BigInt = 0x100000, enableTileClockGating: Boolean = true, enableTileResetSetting: Boolean = true, - enableResetSynchronizers: Boolean = true // this should only be disable to work around verilator async-reset initialziation problems + enableResetSynchronizers: Boolean = true // this should only be disabled to work around verilator async-reset initialization problems ) { def generatePRCIXBar = enableTileClockGating || enableTileResetSetting } @@ -110,7 +110,7 @@ PROPERLY AS ASIC OR FPGA. THESE SHOULD ONLY BE DISABLED TO WORK AROUND LIMITATIONS IN ASYNC RESET INITIALIZATION IN -RTL SIMULATORS. +RTL SIMULATORS, NAMELY VERILATOR. !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! """ + Console.RESET) diff --git a/generators/chipyard/src/main/scala/config/AbstractConfig.scala b/generators/chipyard/src/main/scala/config/AbstractConfig.scala index 218b435c..80b59efe 100644 --- a/generators/chipyard/src/main/scala/config/AbstractConfig.scala +++ b/generators/chipyard/src/main/scala/config/AbstractConfig.scala @@ -22,7 +22,7 @@ class AbstractConfig extends Config( new chipyard.harness.WithTieOffInterrupts ++ // tie-off interrupt ports, if present new chipyard.harness.WithTieOffL2FBusAXI ++ // tie-off external AXI4 master, if present new chipyard.harness.WithCustomBootPinPlusArg ++ // drive custom-boot pin with a plusarg, if custom-boot-pin is present - new chipyard.harness.WithSimUARTToUARTTSI ++ // connect a SimUART to the UART-TSI port + new chipyard.harness.WithSimUARTToUARTTSI ++ // connect a SimUART to the UART-TSI port new chipyard.harness.WithClockAndResetFromHarness ++ // all Clock/Reset I/O in ChipTop should be driven by harnessClockInstantiator new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++ // generate clocks in harness with unsynthesizable ClockSourceAtFreqMHz @@ -35,7 +35,7 @@ class AbstractConfig extends Config( new chipyard.iobinders.WithSPIIOCells ++ new chipyard.iobinders.WithExtInterruptIOCells ++ new chipyard.iobinders.WithCustomBootPin ++ - // The "punchtrough" IOBInders below don't generate IOCells, as these interfaces shouldn't really be mapped to ASIC IO + // The "punchthrough" IOBInders below don't generate IOCells, as these interfaces shouldn't really be mapped to ASIC IO // Instead, they directly pass through the DigitalTop ports to ports in the ChipTop new chipyard.iobinders.WithAXI4MemPunchthrough ++ new chipyard.iobinders.WithAXI4MMIOPunchthrough ++ diff --git a/generators/chipyard/src/main/scala/config/ChipConfigs.scala b/generators/chipyard/src/main/scala/config/ChipConfigs.scala index c1e641b0..f4bee2af 100644 --- a/generators/chipyard/src/main/scala/config/ChipConfigs.scala +++ b/generators/chipyard/src/main/scala/config/ChipConfigs.scala @@ -60,7 +60,7 @@ class ChipBringupHostConfig extends Config( new testchipip.WithSerialTLWidth(4) ++ // match width with the chip new testchipip.WithSerialTLMem(base = 0x0, size = 0x80000000L, // accessible memory of the chip that doesn't come from the tethered host idBits = 4, isMainMemory = false) ++ // This assumes off-chip mem starts at 0x8000_0000 - new testchipip.WithSerialTLClockDirection(provideClockFreqMHz = Some(75)) ++ // bringup board drives the clock for the serial-tl receiver on the chip, use 50MHz clock + new testchipip.WithSerialTLClockDirection(provideClockFreqMHz = Some(75)) ++ // bringup board drives the clock for the serial-tl receiver on the chip, use 75MHz clock //============================ // Setup bus topology on the bringup system diff --git a/generators/chipyard/src/main/scala/harness/MultiHarnesBinders.scala b/generators/chipyard/src/main/scala/harness/MultiHarnessBinders.scala similarity index 100% rename from generators/chipyard/src/main/scala/harness/MultiHarnesBinders.scala rename to generators/chipyard/src/main/scala/harness/MultiHarnessBinders.scala From 16b7c14e4c065a5e0ff6ac4c380eb0dccf1100dc Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 9 Jul 2023 15:22:04 -0700 Subject: [PATCH 24/24] Bump testchipip --- generators/testchipip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generators/testchipip b/generators/testchipip index 68d5c565..19522315 160000 --- a/generators/testchipip +++ b/generators/testchipip @@ -1 +1 @@ -Subproject commit 68d5c565413eac9b1e3bb3e7ff4d477d0cd2512e +Subproject commit 1952231569c939a9a5e47fa2eef8168405d0136d