Merge pull request #1054 from ucb-bar/use-published-deps

Use Published Chisel, FIRRTL, Treadle, FIRRTLInterpreter packages instead of building from source.

.circleci/do-rtl-build.sh

@@ -21,9 +21,6 @@ cd $LOCAL_CHIPYARD_DIR
 ./scripts/init-submodules-no-riscv-tools.sh
 ./scripts/init-fpga.sh
 
-# replace the workspace dir with a local dir so you can copy around
-sed -i -E 's/(workspace=).*(\/tools)/\1$PWD\2/g' .sbtopts
-
 # set stricthostkeychecking to no (must happen before rsync)
 run "echo \"Ping $SERVER\""
 

.circleci/run-firesim-scala-tests.sh

@@ -29,9 +29,6 @@ mkdir -p $local_firesim_sysroot
 ./scripts/build-libdwarf.sh $local_firesim_sysroot
 cd $LOCAL_CHIPYARD_DIR
 
-# replace the workspace dir with a local dir so you can copy around
-sed -i -E 's/(workspace=).*(\/tools)/\1$PWD\2/g' .sbtopts
-
 make -C $LOCAL_CHIPYARD_DIR/tools/dromajo/dromajo-src/src
 
 # set stricthostkeychecking to no (must happen before rsync)

.gitignore

@@ -20,4 +20,3 @@ tags
 env-riscv-tools.sh
 env-esp-tools.sh
 .bsp/
-.sbtopts

.gitmodules

@@ -7,12 +7,6 @@
 [submodule "barstools"]
 	path = tools/barstools
 	url = https://github.com/ucb-bar/barstools.git
-[submodule "tools/chisel3"]
-	path = tools/chisel3
-	url = https://github.com/freechipsproject/chisel3.git
-[submodule "tools/firrtl"]
-	path = tools/firrtl
-	url = https://github.com/freechipsproject/firrtl
 [submodule "tools/torture"]
 	path = tools/torture
 	url = https://github.com/ucb-bar/riscv-torture.git
@@ -77,15 +71,9 @@
 [submodule "tools/chisel-testers"]
 	path = tools/chisel-testers
 	url = https://github.com/freechipsproject/chisel-testers.git
-[submodule "tools/treadle"]
-	path = tools/treadle
-	url = https://github.com/freechipsproject/treadle.git
 [submodule "generators/sha3"]
 	path = generators/sha3
 	url = https://github.com/ucb-bar/sha3.git
-[submodule "tools/firrtl-interpreter"]
-	path = tools/firrtl-interpreter
-	url = https://github.com/freechipsproject/firrtl-interpreter.git
 [submodule "vlsi/hammer-cadence-plugins"]
 	path = vlsi/hammer-cadence-plugins
 	url = https://github.com/ucb-bar/hammer-cadence-plugins.git

build.sbt

@@ -17,14 +17,7 @@ lazy val commonSettings = Seq(
   unmanagedBase := (chipyardRoot / unmanagedBase).value,
   allDependencies := {
     // drop specific maven dependencies in subprojects in favor of Chipyard's version
-    val dropDeps = Seq(
-      ("edu.berkeley.cs", "firrtl"),
-      ("edu.berkeley.cs", "chisel3"),
-      ("edu.berkeley.cs", "rocketchip"),
-      ("edu.berkeley.cs", "chisel-iotesters"),
-      ("edu.berkeley.cs", "treadle"),
-      ("edu.berkeley.cs", "firrtl-interpreter"))
-
+    val dropDeps = Seq(("edu.berkeley.cs", "rocketchip"))
     allDependencies.value.filterNot { dep =>
       dropDeps.contains((dep.organization, dep.name))
     }
@@ -67,34 +60,33 @@ def isolateAllTests(tests: Seq[TestDefinition]) = tests map { test =>
   new Group(test.name, Seq(test), SubProcess(options))
 } toSeq
 
+val chiselVersion = "3.4.4"
+
+lazy val chiselSettings = Seq(
+  libraryDependencies ++= Seq("edu.berkeley.cs" %% "chisel3" % chiselVersion),
+  addCompilerPlugin("edu.berkeley.cs" % "chisel3-plugin" % chiselVersion cross CrossVersion.full))
+
+val firrtlVersion = "1.4.4"
+
+lazy val firrtlSettings = Seq(libraryDependencies ++= Seq("edu.berkeley.cs" %% "firrtl" % firrtlVersion))
+
+// In some projects we override the default versions of Chisel and friends.
+// This map captures the expected defaults used by projects under Chipyard.
+lazy val chipyardMandatedVersions = Map(
+  "chisel-iotesters" -> "1.5.4",
+  "firrtl-interpreter" -> "1.4.4",
+  "treadle" -> "1.3.4",
+  "chisel3" -> chiselVersion,
+  "firrtl" -> firrtlVersion
+)
+
 // Subproject definitions begin
 
 // -- Rocket Chip --
 
-// This needs to stay in sync with the chisel3 and firrtl git submodules
-val chiselVersion = "3.4.1"
-lazy val chiselRef = ProjectRef(workspaceDirectory / "chisel3", "chisel")
-lazy val chiselLib = "edu.berkeley.cs" %% "chisel3" % chiselVersion
-lazy val chiselLibDeps = (chiselRef / Keys.libraryDependencies)
-// While not built from source, *must* be in sync with the chisel3 git submodule
-// Building from source requires extending sbt-sriracha or a similar plugin and
-//   keeping scalaVersion in sync with chisel3 to the minor version
-lazy val chiselPluginLib = "edu.berkeley.cs" % "chisel3-plugin" % chiselVersion cross CrossVersion.full
-
-val firrtlVersion = "1.4.1"
-lazy val firrtlRef = ProjectRef(workspaceDirectory / "firrtl", "firrtl")
-lazy val firrtlLib = "edu.berkeley.cs" %% "firrtl" % firrtlVersion
-val firrtlLibDeps = settingKey[Seq[sbt.librarymanagement.ModuleID]]("FIRRTL Library Dependencies sans antlr4")
-Global / firrtlLibDeps := {
-  // drop antlr4 compile dep. but keep antlr4-runtime dep. (compile needs the plugin to be setup)
-  (firrtlRef / Keys.libraryDependencies).value.filterNot(_.name == "antlr4")
-}
-
- // Rocket-chip dependencies (subsumes making RC a RootProject)
+// Rocket-chip dependencies (subsumes making RC a RootProject)
 lazy val hardfloat  = (project in rocketChipDir / "hardfloat")
-  .sourceDependency(chiselRef, chiselLib)
-  .settings(addCompilerPlugin(chiselPluginLib))
-  .settings(libraryDependencies ++= chiselLibDeps.value)
+  .settings(chiselSettings)
   .dependsOn(midasTargetUtils)
   .settings(commonSettings)
   .settings(
@@ -126,11 +118,9 @@ lazy val rocketConfig = (project in rocketChipDir / "api-config-chipsalliance/bu
   )
 
 lazy val rocketchip = freshProject("rocketchip", rocketChipDir)
-  .sourceDependency(chiselRef, chiselLib)
-  .settings(addCompilerPlugin(chiselPluginLib))
-  .settings(libraryDependencies ++= chiselLibDeps.value)
   .dependsOn(hardfloat, rocketMacros, rocketConfig)
   .settings(commonSettings)
+  .settings(chiselSettings)
   .settings(
     libraryDependencies ++= Seq(
       "org.scala-lang" % "scala-reflect" % scalaVersion.value,
@@ -145,29 +135,19 @@ lazy val rocketchip = freshProject("rocketchip", rocketChipDir)
   )
 lazy val rocketLibDeps = (rocketchip / Keys.libraryDependencies)
 
+
 // -- Chipyard-managed External Projects --
 
-lazy val firrtl_interpreter = (project in file("tools/firrtl-interpreter"))
-  .sourceDependency(firrtlRef, firrtlLib)
-  .settings(commonSettings)
-  .settings(libraryDependencies ++= (Global / firrtlLibDeps).value)
-lazy val firrtlInterpreterLibDeps = (firrtl_interpreter / Keys.libraryDependencies)
-
-lazy val treadle = (project in file("tools/treadle"))
-  .sourceDependency(firrtlRef, firrtlLib)
-  .settings(commonSettings)
-  .settings(libraryDependencies ++= (Global / firrtlLibDeps).value)
-lazy val treadleLibDeps = (treadle / Keys.libraryDependencies)
-
+// Because we're not using a release version of iotesters to work around a
+// scala test version problem, override it's libdeps to prevent using snapshots
 lazy val chisel_testers = (project in file("tools/chisel-testers"))
-  .sourceDependency(chiselRef, chiselLib)
-  .settings(addCompilerPlugin(chiselPluginLib))
-  .settings(libraryDependencies ++= chiselLibDeps.value)
-  .dependsOn(firrtl_interpreter, treadle)
-  .settings(libraryDependencies ++= firrtlInterpreterLibDeps.value)
-  .settings(libraryDependencies ++= treadleLibDeps.value)
-  .settings(commonSettings)
-lazy val chiselTestersLibDeps = (chisel_testers / Keys.libraryDependencies)
+  .settings(chiselSettings)
+  .settings(
+    allDependencies := allDependencies.value.map {
+      case dep if chipyardMandatedVersions.isDefinedAt(dep.name) =>
+        dep.organization %% dep.name % chipyardMandatedVersions(dep.name)
+      case o => o
+    })
 
 // -- Normal Projects --
 
@@ -179,11 +159,9 @@ lazy val testchipip = (project in file("generators/testchipip"))
   .dependsOn(rocketchip, sifive_blocks)
   .settings(libraryDependencies ++= rocketLibDeps.value)
   .settings(commonSettings)
-lazy val testchipipLib = "edu.berkeley.cs" %% "testchipip" % "1.0-020719-SNAPSHOT"
 
 lazy val chipyard = (project in file("generators/chipyard"))
-  .sourceDependency(testchipip, testchipipLib)
-  .dependsOn(rocketchip, boom, hwacha, sifive_blocks, sifive_cache, iocell,
+  .dependsOn(testchipip, rocketchip, boom, hwacha, sifive_blocks, sifive_cache, iocell,
     sha3, // On separate line to allow for cleaner tutorial-setup patches
     dsptools, `rocket-dsp-utils`,
     gemmini, icenet, tracegen, cva6, nvdla, sodor, ibex)
@@ -191,14 +169,12 @@ lazy val chipyard = (project in file("generators/chipyard"))
   .settings(commonSettings)
 
 lazy val tracegen = (project in file("generators/tracegen"))
-  .sourceDependency(testchipip, testchipipLib)
-  .dependsOn(rocketchip, sifive_cache, boom)
+  .dependsOn(testchipip, rocketchip, sifive_cache, boom)
   .settings(libraryDependencies ++= rocketLibDeps.value)
   .settings(commonSettings)
 
 lazy val icenet = (project in file("generators/icenet"))
-  .sourceDependency(testchipip, testchipipLib)
-  .dependsOn(rocketchip)
+  .dependsOn(testchipip, rocketchip)
   .settings(libraryDependencies ++= rocketLibDeps.value)
   .settings(commonSettings)
 
@@ -208,8 +184,7 @@ lazy val hwacha = (project in file("generators/hwacha"))
   .settings(commonSettings)
 
 lazy val boom = (project in file("generators/boom"))
-  .sourceDependency(testchipip, testchipipLib)
-  .dependsOn(rocketchip)
+  .dependsOn(testchipip, rocketchip)
   .settings(libraryDependencies ++= rocketLibDeps.value)
   .settings(commonSettings)
 
@@ -231,14 +206,11 @@ lazy val sodor = (project in file("generators/riscv-sodor"))
 lazy val sha3 = (project in file("generators/sha3"))
   .dependsOn(rocketchip, chisel_testers, midasTargetUtils)
   .settings(libraryDependencies ++= rocketLibDeps.value)
-  .settings(libraryDependencies ++= chiselTestersLibDeps.value)
   .settings(commonSettings)
 
 lazy val gemmini = (project in file("generators/gemmini"))
-  .sourceDependency(testchipip, testchipipLib)
-  .dependsOn(rocketchip, chisel_testers)
+  .dependsOn(testchipip, rocketchip, chisel_testers)
   .settings(libraryDependencies ++= rocketLibDeps.value)
-  .settings(libraryDependencies ++= chiselTestersLibDeps.value)
   .settings(commonSettings)
 
 lazy val nvdla = (project in file("generators/nvdla"))
@@ -247,39 +219,31 @@ lazy val nvdla = (project in file("generators/nvdla"))
   .settings(commonSettings)
 
 lazy val iocell = (project in file("./tools/barstools/iocell/"))
-  .sourceDependency(chiselRef, chiselLib)
-  .settings(addCompilerPlugin(chiselPluginLib))
-  .settings(libraryDependencies ++= chiselLibDeps.value)
+  .settings(chiselSettings)
   .settings(commonSettings)
 
 lazy val tapeout = (project in file("./tools/barstools/tapeout/"))
   .dependsOn(chisel_testers, chipyard) // must depend on chipyard to get scala resources
-  .settings(libraryDependencies ++= chiselTestersLibDeps.value)
   .settings(commonSettings)
 
 lazy val mdf = (project in file("./tools/barstools/mdf/scalalib/"))
   .settings(commonSettings)
 
 lazy val barstoolsMacros = (project in file("./tools/barstools/macros/"))
-  .sourceDependency(chiselRef, chiselLib)
-  .settings(addCompilerPlugin(chiselPluginLib))
-  .settings(libraryDependencies ++= chiselLibDeps.value)
-  .dependsOn(firrtl_interpreter, mdf, chisel_testers)
-  .settings(libraryDependencies ++= chiselTestersLibDeps.value)
-  .settings(libraryDependencies ++= firrtlInterpreterLibDeps.value)
+  .dependsOn(mdf)
   .enablePlugins(sbtassembly.AssemblyPlugin)
+  .settings(firrtlSettings)
   .settings(commonSettings)
 
 lazy val dsptools = freshProject("dsptools", file("./tools/dsptools"))
   .dependsOn(chisel_testers)
-  .settings(libraryDependencies ++= chiselTestersLibDeps.value)
   .settings(
     commonSettings,
     libraryDependencies ++= Seq(
+      "org.scalatest" %% "scalatest" % "3.2.+" % "test",
       "org.typelevel" %% "spire" % "0.16.2",
       "org.scalanlp" %% "breeze" % "1.1",
       "junit" % "junit" % "4.13" % "test",
-      "org.scalatest" %% "scalatest" % "3.0.+" % "test",
       "org.scalacheck" %% "scalacheck" % "1.14.3" % "test",
   ))
 
@@ -313,7 +277,6 @@ lazy val midas      = ProjectRef(firesimDir, "midas")
 lazy val firesimLib = ProjectRef(firesimDir, "firesimLib")
 
 lazy val firechip = (project in file("generators/firechip"))
-  .sourceDependency(testchipip, testchipipLib)
   .dependsOn(chipyard, midasTargetUtils, midas, firesimLib % "test->test;compile->compile")
   .settings(
     commonSettings,

docs/Advanced-Concepts/Managing-Published-Scala-Dependencies.rst

@@ -0,0 +1,54 @@
+Managing Published Scala Dependencies
+=====================================
+
+In preparation for Chisel 3.5, in Chipyard 1.5 Chisel, FIRRTL, the FIRRTL
+interpreter, and Treadle, were transitioned from being built-from-source to
+managed as published dependencies. Their submodules have been removed.
+Switching between published versions can be achieved by changing the versions
+specified in Chipyard's ``build.sbt``.
+
+Lists of available artifacts can be using search.maven.org or mvnrepository.org:
+
+- `Chisel3 <https://mvnrepository.com/artifact/edu.berkeley.cs/chisel3>`_
+- `FIRRTL <https://mvnrepository.com/artifact/edu.berkeley.cs/firrtl>`_
+- `FIRRTL Interpreter <https://mvnrepository.com/artifact/edu.berkeley.cs/firrtl-interpreter>`_
+- `Treadle <https://mvnrepository.com/artifact/edu.berkeley.cs/treadle>`_
+
+
+Publishing Local Changes
+-------------------------
+
+Under the new system, the simplest means to make custom source modifications to the packages
+above is to run ``sbt +publishLocal`` from within a locally modified clone of each
+of their respective repositories. This will post your custom variant
+to your local ivy2 repository, which can generally be found at ``~/.ivy2``.  See the `SBT
+documentation <https://www.scala-sbt.org/1.x/docs/Publishing.html#Publishing+locally>`_
+for more detail.
+
+In practice, this will require the following steps:
+
+#. Check out and modify the desired projects.
+#. Take note of, or modify, the versions of each projects (in
+   their ``build.sbt``). If you're cloning from ``master`` generally
+   these will have default versions of ``1.X-SNAPSHOT``, where ``X`` is
+   not-yet-released next major version. You can modify the version string, to say ``1.X-<MYSUFFIX>``, to uniquely identify your
+   change.
+#. Call ``sbt +publishLocal`` in each subproject. You may need to rebuild other
+   published dependencies. SBT will be clear about what it is publishing and
+   where it is putting it. The ``+`` is generally necessary and ensures that
+   all cross versions of the package are published.
+#. Update the Chisel or FIRRTL version in Chipyard's ``build.sbt`` to match the
+   versions of your locally published packages.
+#. Use Chipyard as you would normally. Now when you call out to make in
+   Chipyard you should see SBT resolving dependencies to the locally
+   published instances in your local ivy2 repository.
+#. When you're finished, consider removing your locally published packages (by
+   removing the appropriate directory in your ivy2 repository) to prevent
+   accidentally reusing them in the future.
+
+A final word of caution: packages you publish to your local ivy repository will
+be visible to other projects you may be building on your system. For example,
+if you locally publish Chisel 3.5.0, other projects that depend on Chisel 3.5.0
+will preferentially use the locally published variant over the version
+available on Maven (the "real" 3.5.0).  Take care to note versions you are
+publishing and remove locally published versions once you are done with them.

docs/Advanced-Concepts/index.rst

@@ -15,4 +15,5 @@ They expect you to know about Chisel, Parameters, configs, etc.
    Resources
    CDEs
    Harness-Clocks
+   Managing-Published-Scala-Dependencies
 

project/plugins.sbt

@@ -11,5 +11,4 @@ addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.2")
 addSbtPlugin("com.github.gseitz" % "sbt-protobuf" % "0.6.3")
 addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.21")
 addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "0.6.1")
-addSbtPlugin("com.eed3si9n" % "sbt-sriracha" % "0.1.0")
 addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.2.5" )

scripts/init-submodules-no-riscv-tools-nolog.sh

@@ -88,5 +88,3 @@ echo "# line auto-generated by init-submodules-no-riscv-tools.sh" >> env.sh
 echo '__DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]:-${(%):-%x}}")")"' >> env.sh
 echo "PATH=\$__DIR/bin:\$PATH" >> env.sh
 echo "PATH=\$__DIR/software/firemarshal:\$PATH" >> env.sh
-echo "-Dsbt.sourcemode=true" > .sbtopts
-echo "-Dsbt.workspace=$CHIPYARD_DIR/tools"  >> .sbtopts

scripts/tutorial-patches/build.sbt.patch

@@ -1,30 +1,28 @@
 diff --git a/build.sbt b/build.sbt
-index b1f7e004..f39c3712 100644
+index 2187fe12..2319fc95 100644
 --- a/build.sbt
 +++ b/build.sbt
-@@ -184,7 +184,7 @@ lazy val testchipipLib = "edu.berkeley.cs" %% "testchipip" % "1.0-020719-SNAPSHO
+@@ -162,7 +162,7 @@ lazy val testchipip = (project in file("generators/testchipip"))
+ 
  lazy val chipyard = (project in file("generators/chipyard"))
-   .sourceDependency(testchipip, testchipipLib)
-   .dependsOn(rocketchip, boom, hwacha, sifive_blocks, sifive_cache, iocell,
+   .dependsOn(testchipip, rocketchip, boom, hwacha, sifive_blocks, sifive_cache, iocell,
 -    sha3, // On separate line to allow for cleaner tutorial-setup patches
 +    //sha3, // On separate line to allow for cleaner tutorial-setup patches
      dsptools, `rocket-dsp-utils`,
      gemmini, icenet, tracegen, cva6, nvdla, sodor, ibex)
    .settings(libraryDependencies ++= rocketLibDeps.value)
-@@ -228,11 +228,11 @@ lazy val sodor = (project in file("generators/riscv-sodor"))
+@@ -203,10 +203,10 @@ lazy val sodor = (project in file("generators/riscv-sodor"))
    .settings(libraryDependencies ++= rocketLibDeps.value)
    .settings(commonSettings)
  
 -lazy val sha3 = (project in file("generators/sha3"))
 -  .dependsOn(rocketchip, chisel_testers, midasTargetUtils)
 -  .settings(libraryDependencies ++= rocketLibDeps.value)
--  .settings(libraryDependencies ++= chiselTestersLibDeps.value)
 -  .settings(commonSettings)
 +//lazy val sha3 = (project in file("generators/sha3"))
 +//  .dependsOn(rocketchip, chisel_testers, midasTargetUtils)
 +//  .settings(libraryDependencies ++= rocketLibDeps.value)
-+//  .settings(libraryDependencies ++= chiselTestersLibDeps.value)
 +//  .settings(commonSettings)
  
  lazy val gemmini = (project in file("generators/gemmini"))
-   .sourceDependency(testchipip, testchipipLib)
+   .dependsOn(testchipip, rocketchip, chisel_testers)

variables.mk

@@ -6,7 +6,8 @@
 HELP_COMPILATION_VARIABLES = \
 "   JAVA_HEAP_SIZE    = if overridden, set the default java heap size (default is 8G)" \
 "   JAVA_TOOL_OPTIONS = if overridden, set underlying java tool options (default sets misc. sizes and tmp dir)" \
-"   SBT_OPTS          = if overridden, set underlying sbt options (default uses options in .sbtopts)" \
+"   SBT_OPTS          = set additional sbt command line options (these take the form -Dsbt.<option>=<setting>) " \
+"                       See https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html\#Command+Line+Options" \
 "   SBT_BIN           = if overridden, used to invoke sbt (default is to invoke sbt by sbt-launch.jar)" \
 "   FIRRTL_LOGLEVEL   = if overridden, set firrtl log level (default is error)"
 
@@ -164,16 +165,6 @@ export JAVA_TOOL_OPTIONS ?= -Xmx$(JAVA_HEAP_SIZE) -Xss8M -XX:MaxPermSize=256M -D
 #########################################################################################
 # default sbt launch command
 #########################################################################################
-# by default build chisel3/firrtl and other subprojects from source
-SBT_OPTS_FILE := $(base_dir)/.sbtopts
-ifneq (,$(wildcard $(SBT_OPTS_FILE)))
-override SBT_OPTS += $(subst $$PWD,$(base_dir),$(shell cat $(SBT_OPTS_FILE)))
-endif
-# Workaround: Specify a firrtl version in system properties so that Treadle uses a
-# compatible version of FIRRTL and not 1.5-SNAPSHOT (which is the default
-# specified in it's build.sbt, and is not overridden by Chipyard's build.sbt)
-override SBT_OPTS += -DfirrtlVersion=1.4.1
-
 SCALA_BUILDTOOL_DEPS = $(SBT_SOURCES)
 
 SBT_THIN_CLIENT_TIMESTAMP = $(base_dir)/project/target/active.json