Rough initial implementation of diplomatic multiclock

generators/chipyard/src/main/scala/ChipTop.scala

@@ -4,8 +4,11 @@ import chisel3._
 
 import scala.collection.mutable.{ArrayBuffer}
 
+import freechips.rocketchip.prci._
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.subsystem.{BaseSubsystem, SubsystemDriveAsyncClockGroupsKey}
 import freechips.rocketchip.config.{Parameters, Field}
-import freechips.rocketchip.diplomacy.{LazyModule}
+import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, LazyRawModuleImp, LazyModuleImpLike}
 import freechips.rocketchip.util.{ResetCatchAndSync}
 import chipyard.config.ConfigValName._
 import chipyard.iobinders.{IOBinders, TestHarnessFunction, IOBinderTuple}
@@ -14,108 +17,58 @@ import barstools.iocell.chisel._
 
 case object BuildSystem extends Field[Parameters => LazyModule]((p: Parameters) => LazyModule(new DigitalTop()(p)))
 
-/**
-  * Chipyard provides three baseline, top-level reset schemes, set using the
-  * [[GlobalResetSchemeKey]] in a Parameters instance. These are:
-  *
-  * 1) Synchronous: The input coming to the chip is synchronous to the provided
-  *    clocks and will be used without modification as a synchronous reset.
-  *    This is safe only for use in FireSim and SW simulation.
-  *
-  * 2) Asynchronous: The input reset is asynchronous to the input clock, but it
-  *    is caught and synchronized to that clock before it is dissemenated.
-  *    Thus, downsteam modules will be emitted with synchronously reset state
-  *    elements.
-  *
-  * 3) Asynchronous Full: The input reset is asynchronous to the input clock,
-  *    and is used globally as an async reset. Downstream modules will be emitted
-  *    with asynchronously reset state elements.
-  *
-  */
-sealed trait GlobalResetScheme {
-  def pinIsAsync: Boolean
-}
-sealed trait HasAsyncInput { self: GlobalResetScheme =>
-  def pinIsAsync = true
-}
-
-sealed trait HasSyncInput { self: GlobalResetScheme =>
-  def pinIsAsync = false
-}
-
-case object GlobalResetSynchronous extends GlobalResetScheme with HasSyncInput
-case object GlobalResetAsynchronous extends GlobalResetScheme with HasAsyncInput
-case object GlobalResetAsynchronousFull extends GlobalResetScheme with HasAsyncInput
-case object GlobalResetSchemeKey extends Field[GlobalResetScheme](GlobalResetSynchronous)
-
 
 /**
  * The base class used for building chips. This constructor instantiates a module specified by the BuildSystem parameter,
- * named "system", which is an instance of DigitalTop by default. The default clock and reset for "system" are set by two
- * wires, "systemClock" and "systemReset", which are intended to be driven by traits mixed-in with this base class.
+ * named "system", which is an instance of DigitalTop by default. The diplomatic clocks of System, as well as its implicit clock,
+ * is aggregated into the clockGroupNode. The parameterized functions controlled by ChipyardClockKey and GlobalResetSchemeKey
+ * drive clock and reset generation
  */
-abstract class BaseChipTop()(implicit val p: Parameters) extends RawModule with HasTestHarnessFunctions {
 
+class ChipTop(implicit p: Parameters) extends LazyModule with HasTestHarnessFunctions {
   // A publicly accessible list of IO cells (useful for a floorplanning tool, for example)
   val iocells = ArrayBuffer.empty[IOCell]
   // A list of functions to call in the test harness
   val harnessFunctions = ArrayBuffer.empty[TestHarnessFunction]
-  // The system clock
-  // These are given so that IOCell can use DataMirror and generate ports with
-  // the right flow (Input/Output)
-  val systemClock = Wire(Input(Clock()))
-  val systemReset = Wire(Input(Reset()))
 
   // The system module specified by BuildSystem
   val lSystem = p(BuildSystem)(p).suggestName("system")
-  val system = withClockAndReset(systemClock, systemReset) { Module(lSystem.module) }
-
-  // Call all of the IOBinders and provide them with a default clock and reset
-  withClockAndReset(systemClock, systemReset) {
-    // Call each IOBinder on both the lazyModule instance and the module
-    // instance. Generally, an IOBinder PF should only be defined on one, so
-    // this should not lead to two invocations.
-    val (_ports, _iocells, _harnessFunctions) = p(IOBinders).values.flatMap(f => f(lSystem) ++ f(system)).unzip3
-    // We ignore _ports for now...
-    iocells ++= _iocells.flatten
-    harnessFunctions ++= _harnessFunctions.flatten
-  }
 
-}
-
-/**
- * A simple clock and reset implementation that punches out clock and reset ports with the same
- * names as the implicit clock and reset for standard Module classes. Three basic reset schemes 
- * are provided. See [[GlobalResetScheme]].
- */
-trait HasChipTopSimpleClockAndReset { this: BaseChipTop =>
-
-  val (clock, systemClockIO) = IOCell.generateIOFromSignal(systemClock, Some("iocell_clock"))
-  val (reset, systemResetIO) = p(GlobalResetSchemeKey) match {
-    case GlobalResetSynchronous  =>
-      IOCell.generateIOFromSignal(systemReset, Some("iocell_reset"))
-    case GlobalResetAsynchronousFull =>
-      IOCell.generateIOFromSignal(systemReset, Some("iocell_reset"), abstractResetAsAsync = true)
-    case GlobalResetAsynchronous =>
-      val asyncResetCore = Wire(Input(AsyncReset()))
-      systemReset := ResetCatchAndSync(systemClock, asyncResetCore.asBool)
-      IOCell.generateIOFromSignal(asyncResetCore, Some("iocell_reset"), abstractResetAsAsync = true)
-  }
-
-  iocells ++= systemClockIO
-  iocells ++= systemResetIO
+  // The systemClockSinkNode provides the implicit clock and reset for the System
+  private val systemClockSinkNode = ClockSinkNode(Seq(ClockSinkParameters()))
 
-  // Add a TestHarnessFunction that connects clock and reset
-  harnessFunctions += { (th: TestHarness) => {
-    // Connect clock; it's not done implicitly with RawModule
-    clock := th.clock
-    // Connect reset; it's not done implicitly with RawModule
-    // Note that we need to use dutReset, not harnessReset
-    reset := th.dutReset
-    Nil
-  } }
+  // clockGroupNode provides a single node which aggregates all clock groups in the design
+  val clockGroupNode = ClockGroupIdentityNode()
 
+  // If the specified system has diplomatic clocks, connect it to our clockGroupNode
+  if (p(SubsystemDriveAsyncClockGroupsKey).isEmpty) {
+    lSystem match { case l: BaseSubsystem => l.asyncClockGroupsNode :*= clockGroupNode }
+  }
+  // Connect the system implicit clock node to the clockGroupNode
+  systemClockSinkNode := ClockGroup() := clockGroupNode
+
+  // Drive the entire diplomatic clock network using this configured Key
+  clockGroupNode :*=* p(ChipyardClockKey)(this)
+
+  lazy val module: LazyModuleImpLike = new LazyRawModuleImp(this) {
+    // These become the implicit clock and reset to the System
+    val system_clock = systemClockSinkNode.in.head._1.clock
+    val system_reset = systemClockSinkNode.in.head._1.reset
+
+    // The implicit clock and reset for the system is also, by convention, used for all the IOBinders
+    // TODO: This may not be the right thing to do in all cases
+    withClockAndReset(system_clock, system_reset) {
+      val (_ports, _iocells, _harnessFunctions) = p(IOBinders).values.flatMap(f => f(lSystem) ++ f(lSystem.module)).unzip3
+      // We ignore _ports for now...
+      iocells ++= _iocells.flatten
+      harnessFunctions ++= _harnessFunctions.flatten
+    }
+
+    // Connect the implicit clock/reset, if present
+    lSystem.module match { case l: LazyModuleImp => {
+      l.clock := system_clock
+      l.reset := system_reset
+    }}
+  }
 }
 
-class ChipTop()(implicit p: Parameters) extends BaseChipTop()(p)
-  with HasChipTopSimpleClockAndReset

generators/chipyard/src/main/scala/Clocks.scala

@@ -0,0 +1,115 @@
+package chipyard
+
+import chisel3._
+
+import scala.collection.mutable.{ArrayBuffer}
+
+import freechips.rocketchip.prci._
+import freechips.rocketchip.diplomacy._
+import freechips.rocketchip.subsystem.{BaseSubsystem}
+import freechips.rocketchip.config.{Parameters, Field}
+import freechips.rocketchip.diplomacy.{LazyModule, LazyModuleImp, LazyRawModuleImp, LazyModuleImpLike}
+import freechips.rocketchip.util.{ResetCatchAndSync}
+import chipyard.config.ConfigValName._
+import chipyard.iobinders.{IOBinders, TestHarnessFunction, IOBinderTuple}
+
+import barstools.iocell.chisel._
+
+import ChipyardClockDrivers._
+
+case object ChipyardClockKey extends Field[ClockInstantiationFn](simpleTestHarnessClock)
+
+
+/**
+  * Chipyard provides three baseline, top-level reset schemes, set using the
+  * [[GlobalResetSchemeKey]] in a Parameters instance. These are:
+  *
+  * 1) Synchronous: The input coming to the chip is synchronous to the provided
+  *    clocks and will be used without modification as a synchronous reset.
+  *    This is safe only for use in FireSim and SW simulation.
+  *
+  * 2) Asynchronous: The input reset is asynchronous to the input clock, but it
+  *    is caught and synchronized to that clock before it is dissemenated.
+  *    Thus, downsteam modules will be emitted with synchronously reset state
+  *    elements.
+  *
+  * 3) Asynchronous Full: The input reset is asynchronous to the input clock,
+  *    and is used globally as an async reset. Downstream modules will be emitted
+  *    with asynchronously reset state elements.
+  *
+  */
+sealed trait GlobalResetScheme {
+  def pinIsAsync: Boolean
+}
+sealed trait HasAsyncInput { self: GlobalResetScheme =>
+  def pinIsAsync = true
+}
+
+sealed trait HasSyncInput { self: GlobalResetScheme =>
+  def pinIsAsync = false
+}
+
+case object GlobalResetSynchronous extends GlobalResetScheme with HasSyncInput
+case object GlobalResetAsynchronous extends GlobalResetScheme with HasAsyncInput
+case object GlobalResetAsynchronousFull extends GlobalResetScheme with HasAsyncInput
+case object GlobalResetSchemeKey extends Field[GlobalResetScheme](GlobalResetSynchronous)
+
+/**
+ * A simple reset implementation that punches out reset ports
+ * for standard Module classes. Three basic reset schemes
+ * are provided. See [[GlobalResetScheme]].
+ */
+object GenerateReset {
+  def apply(chiptop: ChipTop, clock: Clock): Reset = {
+    implicit val p = chiptop.p
+    val reset_wire = Wire(Input(Reset()))
+    val (reset_io, resetIOCell) = p(GlobalResetSchemeKey) match {
+      case GlobalResetSynchronous =>
+        IOCell.generateIOFromSignal(reset_wire, Some("iocell_reset"))
+      case GlobalResetAsynchronousFull =>
+        IOCell.generateIOFromSignal(reset_wire, Some("iocell_reset"), abstractResetAsAsync = true)
+      case GlobalResetAsynchronous => {
+        val async_reset_wire = Wire(Input(AsyncReset()))
+        reset_wire := ResetCatchAndSync(clock, async_reset_wire.asBool())
+        IOCell.generateIOFromSignal(async_reset_wire, Some("iocell_reset"), abstractResetAsAsync = true)
+      }
+    }
+    reset_io.suggestName("reset")
+    chiptop.iocells ++= resetIOCell
+    chiptop.harnessFunctions += ((th: TestHarness) => {
+      reset_io := th.dutReset
+      Nil
+    })
+    reset_wire
+  }
+}
+
+object ChipyardClockDrivers {
+  type ClockInstantiationFn = ChipTop => OutwardNodeHandle[ClockGroupSourceParameters, ClockGroupSinkParameters, ClockGroupEdgeParameters, ClockGroupBundle]
+
+  // A simple clock provider, for testing. All clocks in system are aggregated into one,
+  // and are driven by directly punching out to the TestHarness clock
+  val simpleTestHarnessClock: ClockInstantiationFn = { chiptop =>
+    implicit val p = chiptop.p
+    val simpleClockGroupSourceNode = ClockGroupSourceNode(Seq(ClockGroupSourceParameters()))
+    InModuleBody {
+      val clock_wire = Wire(Input(Clock())) // this needs directionality so generateIOFromSignal works
+      val reset_wire = GenerateReset(chiptop, clock_wire)
+      val (clock_io, clockIOCell) = IOCell.generateIOFromSignal(clock_wire, Some("iocell_clock"))
+      chiptop.iocells ++= clockIOCell
+
+      clock_io.suggestName("clock")
+
+      simpleClockGroupSourceNode.out.unzip._1.flatMap(_.member).map { o =>
+        o.clock := clock_wire
+        o.reset := reset_wire
+      }
+
+      chiptop.harnessFunctions += ((th: TestHarness) => {
+        clock_io := th.clock
+        Nil
+      })
+    }
+    ClockGroupAggregator() := simpleClockGroupSourceNode
+  }
+}

generators/chipyard/src/main/scala/ConfigFragments.scala

@@ -143,3 +143,10 @@ class WithRocketDCacheScratchpad extends Config((site, here, up) => {
   }
 })
 
+// The default RocketChip BaseSubsystem drives its diplomatic clock graph
+// with the implicit clocks of Subsystem. Don't do that, instead we extend
+// the diplomacy graph upwards into the ChipTop, where we connect it to
+// our clock drivers
+class WithNoSubsystemDrivenClocks extends Config((site, here, up) => {
+  case SubsystemDriveAsyncClockGroupsKey => None
+})

generators/chipyard/src/main/scala/IOBinders.scala

@@ -201,14 +201,14 @@ object AddIOCells {
   }
 
   def axi4(io: Seq[AXI4Bundle], node: AXI4SlaveNode, name: String): Seq[(AXI4Bundle, AXI4EdgeParameters, Seq[IOCell])] = {
-    io.zip(node.in).zipWithIndex.map{ case ((mem_axi4, (_, edge)), i) => {
+    io.zip(node.edges.in).zipWithIndex.map{ case ((mem_axi4, (_, edge)), i) => {
       val (port, ios) = IOCell.generateIOFromSignal(mem_axi4, Some(s"iocell_${name}_axi4_slave_${i}"))
       port.suggestName(s"${name}_axi4_slave_${i}")
       (port, edge, ios)
     }}
   }
   def axi4(io: Seq[AXI4Bundle], node: AXI4MasterNode, name: String): Seq[(AXI4Bundle, AXI4EdgeParameters, Seq[IOCell])] = {
-    io.zip(node.out).zipWithIndex.map{ case ((mem_axi4, (_, edge)), i) => {
+    io.zip(node.edges.out).zipWithIndex.map{ case ((mem_axi4, (_, edge)), i) => {
       //val (port, ios) = IOCell.generateIOFromSignal(mem_axi4, Some(s"iocell_${name}_axi4_master_${i}"))
       val port = IO(Flipped(AXI4Bundle(edge.bundle)))
       val ios = IOCell.generateFromSignal(mem_axi4, port, Some(s"iocell_${name}_axi4_master_${i}"))

generators/chipyard/src/main/scala/Subsystem.scala

@@ -8,6 +8,7 @@ package chipyard
 import chisel3._
 import chisel3.internal.sourceinfo.{SourceInfo}
 
+import freechips.rocketchip.prci._
 import freechips.rocketchip.config.{Field, Parameters}
 import freechips.rocketchip.devices.tilelink._
 import freechips.rocketchip.devices.debug.{HasPeripheryDebug, HasPeripheryDebugModuleImp, ExportDebug}
@@ -48,6 +49,10 @@ trait CanHaveHTIF { this: BaseSubsystem =>
 }
 
 
+// Controls whether tiles are driven by implicit subsystem clock, or by
+// diplomatic clock graph
+case object UseDiplomaticTileClocks extends Field[Boolean](false)
+
 class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
   with HasTiles
   with CanHaveHTIF
@@ -56,6 +61,19 @@ class ChipyardSubsystem(implicit p: Parameters) extends BaseSubsystem
     case r: RocketTile => r.module.core.rocketImpl.coreMonitorBundle
     case b: BoomTile => b.module.core.coreMonitorBundle
   }.toList
+
+  // TODO: In the future, RC tiles may extend ClockDomain. When that happens,
+  // we won't need to manually create this clock node and connect it to the
+  // tiles' implicit clocks
+
+  val tilesClockSinkNode = if (p(UseDiplomaticTileClocks)) {
+    val node = ClockSinkNode(List(ClockSinkParameters()))
+    node := ClockGroup()(p, ValName("chipyard_tiles")) := asyncClockGroupsNode
+    Some(node)
+  } else {
+    None
+  }
+
   override lazy val module = new ChipyardSubsystemModuleImp(this)
 }
 
@@ -64,11 +82,15 @@ class ChipyardSubsystemModuleImp[+L <: ChipyardSubsystem](_outer: L) extends Bas
   with HasResetVectorWire
   with HasTilesModuleImp
 {
-
   for (i <- 0 until outer.tiles.size) {
     val wire = tile_inputs(i)
     wire.hartid := outer.hartIdList(i).U
     wire.reset_vector := global_reset_vector
+
+    outer.tilesClockSinkNode.map( n => {
+      outer.tiles(i).module.clock := n.in.head._1.clock
+      outer.tiles(i).module.reset := n.in.head._1.reset
+    })
   }
 
   // create file with core params

generators/chipyard/src/main/scala/TestHarness.scala

@@ -8,10 +8,10 @@ import chipyard.iobinders.{TestHarnessFunction}
 import chipyard.config.ConfigValName._
 
 // -------------------------------
-// BOOM and/or Rocket Test Harness
+// Chipyard Test Harness
 // -------------------------------
 
-case object BuildTop extends Field[Parameters => HasTestHarnessFunctions]((p: Parameters) => Module(new ChipTop()(p)))
+case object BuildTop extends Field[Parameters => LazyModule with HasTestHarnessFunctions]((p: Parameters) => LazyModule(new ChipTop()(p)))
 
 trait HasTestHarnessFunctions {
   val harnessFunctions: Seq[TestHarnessFunction]
@@ -22,13 +22,14 @@ class TestHarness(implicit val p: Parameters) extends Module {
     val success = Output(Bool())
   })
 
-  val dut = p(BuildTop)(p)
+  val ldut = p(BuildTop)(p)
+  val dut = Module(ldut.module)
   io.success := false.B
 
   // dutReset assignment can be overridden via a harnessFunction, but by default it is just reset
   val dutReset = WireDefault(if (p(GlobalResetSchemeKey).pinIsAsync) reset.asAsyncReset else reset)
 
-  dut.harnessFunctions.foreach(_(this))
+  ldut.harnessFunctions.foreach(_(this))
 
   def success = io.success
   def harnessReset = this.reset.asBool

generators/chipyard/src/main/scala/config/AbstractConfig.scala

@@ -17,6 +17,7 @@ class AbstractConfig extends Config(
   new chipyard.config.WithBootROM ++                             // use default bootrom
   new chipyard.config.WithUART ++                                // add a UART
   new chipyard.config.WithL2TLBs(1024) ++                        // use L2 TLBs
+  new chipyard.config.WithNoSubsystemDrivenClocks ++             // drive the subsystem diplomatic clocks from ChipTop instead of using implicit clocks
   new freechips.rocketchip.subsystem.WithNoMMIOPort ++           // no top-level MMIO master port (overrides default set in rocketchip)
   new freechips.rocketchip.subsystem.WithNoSlavePort ++          // no top-level MMIO slave port (overrides default set in rocketchip)
   new freechips.rocketchip.subsystem.WithInclusiveCache ++       // use Sifive L2 cache