Gemmini Integration (#356)

* gemmini submodule

* fix build.sbt

* firechip gemmini config

* bump gemmini

* bump gemmini

* bump gemmini

* fix hwacha typo

* start gemmini docs

* bump gemmini

* gemmini docs

* Update Gemmini RST. Add quick-build instructions to Gemmini RST

* start gemmini CI

* bump gemmini

* gemmini CI fixes

* bump gemmini

* fix simulator name in gemmini CI

* cleanup gemmini CI

* bump esp-isa-sim to include gemmini

* update gemmini docs

* [ci skip] fix gemmini docs typos

* Update Gemmini.rst

Add instructions on building Gemmini programs, or writing your own programs.

* Changed order of VCS and Verilator in Gemmini docs

* Remove "make your own tests" from Gemmini README

* bump gemmini

* try to fix midasexamples CI

.circleci/check-commit.sh

@@ -48,7 +48,7 @@ search () {
     done
 }
 
-submodules=("boom" "hwacha" "icenet" "sha3" "rocket-chip" "sifive-blocks" "sifive-cache" "testchipip")
+submodules=("boom" "hwacha" "icenet" "sha3" "rocket-chip" "sifive-blocks" "sifive-cache" "testchipip" "gemmini")
 dir="generators"
 if [ "$CIRCLE_BRANCH" == "master" ] || [ "$CIRCLE_BRANCH" == "dev" ]
 then

.circleci/config.yml

@@ -288,6 +288,35 @@ jobs:
                 key: hwacha-{{ .Branch }}-{{ .Revision }}
                 paths:
                     - "/home/riscvuser/project"
+    prepare-gemmini:
+        docker:
+            - image: riscvboom/riscvboom-images:0.0.12
+        environment:
+            JVM_OPTS: -Xmx3200m # Customize the JVM maximum heap limit
+            TERM: dumb
+        steps:
+            - add_ssh_keys:
+                fingerprints:
+                    - "3e:c3:02:5b:ed:64:8c:b7:b0:04:43:bc:83:43:73:1e"
+            - checkout
+            - run:
+                name: Create hash of toolchains
+                command: |
+                    .circleci/create-hash.sh
+            - restore_cache:
+                keys:
+                    - esp-tools-installed-v2-{{ checksum "../esp-tools.hash" }}
+            - restore_cache:
+                keys:
+                    - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
+            - run:
+                name: Building the gemmini subproject using Verilator
+                command: .circleci/do-rtl-build.sh gemmini
+                no_output_timeout: 120m
+            - save_cache:
+                key: gemmini-{{ .Branch }}-{{ .Revision }}
+                paths:
+                    - "/home/riscvuser/project"
     prepare-tracegen:
         docker:
             - image: riscvboom/riscvboom-images:0.0.12
@@ -516,6 +545,30 @@ jobs:
             - run:
                 name: Run hwacha tests
                 command: .circleci/run-tests.sh hwacha
+    gemmini-run-tests:
+        docker:
+            - image: riscvboom/riscvboom-images:0.0.12
+        environment:
+            JVM_OPTS: -Xmx3200m # Customize the JVM maximum heap limit
+            TERM: dumb
+        steps:
+            - checkout
+            - run:
+                name: Create hash of toolchains
+                command: |
+                    .circleci/create-hash.sh
+            - restore_cache:
+                keys:
+                    - esp-tools-installed-v2-{{ checksum "../esp-tools.hash" }}
+            - restore_cache:
+                keys:
+                    - gemmini-{{ .Branch }}-{{ .Revision }}
+            - restore_cache:
+                keys:
+                    - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }}
+            - run:
+                name: Run gemmini tests
+                command: .circleci/run-tests.sh gemmini
     tracegen-run-tests:
         docker:
             - image: riscvboom/riscvboom-images:0.0.12
@@ -660,6 +713,11 @@ workflows:
                     - install-esp-toolchain
                     - install-verilator
 
+            - prepare-gemmini:
+                requires:
+                    - install-esp-toolchain
+                    - install-verilator
+
             - prepare-tracegen:
                 requires:
                     - install-riscv-toolchain
@@ -704,6 +762,10 @@ workflows:
                 requires:
                     - prepare-hwacha
 
+            - gemmini-run-tests:
+                requires:
+                    - prepare-gemmini
+
             - tracegen-run-tests:
                 requires:
                     - prepare-tracegen

.circleci/defaults.sh

@@ -47,6 +47,7 @@ mapping["boom"]="SUB_PROJECT=example CONFIG=SmallBoomConfig"
 mapping["rocketchip"]="SUB_PROJECT=rocketchip"
 mapping["blockdevrocketchip"]="SUB_PROJECT=example CONFIG=SimBlockDeviceRocketConfig TOP=TopWithBlockDevice"
 mapping["hwacha"]="SUB_PROJECT=example CONFIG=HwachaRocketConfig"
+mapping["gemmini"]="SUB_PROJECT=example CONFIG=GemminiRocketConfig"
 mapping["tracegen"]="SUB_PROJECT=tracegen CONFIG=NonBlockingTraceGenL2Config"
 mapping["firesim"]="DESIGN=FireSim TARGET_CONFIG=DDR3FRFCFSLLC4MB_FireSimRocketChipConfig PLATFORM_CONFIG=BaseF1Config"
 mapping["fireboom"]="DESIGN=FireSim TARGET_CONFIG=DDR3FRFCFSLLC4MB_FireSimBoomConfig PLATFORM_CONFIG=BaseF1Config"

.circleci/do-firesim-build.sh

@@ -34,7 +34,7 @@ TOOLS_DIR=$REMOTE_RISCV_DIR
 LD_LIB_DIR=$REMOTE_RISCV_DIR/lib
 VERILATOR_BIN_DIR=$REMOTE_VERILATOR_DIR/install/bin
 
-if [ $1 = "hwacha" ]; then
+if [ $1 = "hwacha" ] || [ $1 = "gemmini" ]; then
     TOOLS_DIR=$REMOTE_ESP_DIR
     LD_LIB_DIR=$REMOTE_ESP_DIR/lib
     run "mkdir -p $REMOTE_ESP_DIR"

.circleci/do-rtl-build.sh

@@ -32,7 +32,7 @@ run "cp -r ~/.sbt  $REMOTE_WORK_DIR"
 
 TOOLS_DIR=$REMOTE_RISCV_DIR
 LD_LIB_DIR=$REMOTE_RISCV_DIR/lib
-if [ $1 = "hwacha" ]; then
+if [ $1 = "hwacha" ] || [ $1 = "gemmini" ]; then
     TOOLS_DIR=$REMOTE_ESP_DIR
     LD_LIB_DIR=$REMOTE_ESP_DIR/lib
     run "mkdir -p $REMOTE_ESP_DIR"

.circleci/run-midasexamples-tests.sh

@@ -12,7 +12,7 @@ trap clean EXIT
 
 cd $LOCAL_CHIPYARD_DIR
 ./scripts/init-submodules-no-riscv-tools.sh
-cd sims/firesim/sim/midas && git submodule update --init
+cd sims/firesim/sim/midas
 
 # set stricthostkeychecking to no (must happen before rsync)
 run "echo \"Ping $SERVER\""

.circleci/run-tests.sh

@@ -50,6 +50,18 @@ case $1 in
         export PATH=$RISCV/bin:$PATH
         make run-rv64uv-p-asm-tests -j$NPROC -C $LOCAL_SIM_DIR VERILATOR_INSTALL_DIR=$LOCAL_VERILATOR_DIR ${mapping[$1]}
         ;;
+    gemmini)
+        export RISCV=$LOCAL_ESP_DIR
+        export LD_LIBRARY_PATH=$LOCAL_ESP_DIR/lib
+        export PATH=$RISCV/bin:$PATH
+        GEMMINI_SOFTWARE_DIR=$LOCAL_SIM_DIR/../../generators/gemmini/software/gemmini-rocc-tests
+        cd $GEMMINI_SOFTWARE_DIR
+        ./build.sh
+        cd $LOCAL_SIM_DIR
+        $LOCAL_SIM_DIR/simulator-example-GemminiRocketConfig $GEMMINI_SOFTWARE_DIR/build/bareMetalC/aligned-baremetal
+        $LOCAL_SIM_DIR/simulator-example-GemminiRocketConfig $GEMMINI_SOFTWARE_DIR/build/bareMetalC/raw_hazard-baremetal
+        $LOCAL_SIM_DIR/simulator-example-GemminiRocketConfig $GEMMINI_SOFTWARE_DIR/build/bareMetalC/mvin_mvout-baremetal
+        ;;
     tracegen)
         run_tracegen ${mapping[$1]}
         ;;

.gitmodules

@@ -104,3 +104,6 @@
 [submodule "software/coremark"]
 	path = software/coremark
 	url = https://github.com/ucb-bar/coremark-workload.git
+[submodule "generators/gemmini"]
+	path = generators/gemmini
+	url = https://github.com/ucb-bar/gemmini

build.sbt

@@ -123,7 +123,7 @@ lazy val testchipip = (project in file("generators/testchipip"))
   .settings(commonSettings)
 
 lazy val example = conditionalDependsOn(project in file("generators/example"))
-  .dependsOn(boom, hwacha, sifive_blocks, sifive_cache, utilities, sha3)
+  .dependsOn(boom, hwacha, sifive_blocks, sifive_cache, utilities, sha3, gemmini)
   .settings(commonSettings)
 
 lazy val tracegen = conditionalDependsOn(project in file("generators/tracegen"))
@@ -150,6 +150,10 @@ lazy val sha3 = (project in file("generators/sha3"))
   .dependsOn(rocketchip, chisel_testers, midasTargetUtils)
   .settings(commonSettings)
 
+lazy val gemmini = (project in file("generators/gemmini"))
+  .dependsOn(rocketchip, chisel_testers, testchipip)
+  .settings(commonSettings)
+
 lazy val tapeout = conditionalDependsOn(project in file("./tools/barstools/tapeout/"))
   .dependsOn(chisel_testers, example)
   .settings(commonSettings)

docs/Chipyard-Basics/Initial-Repo-Setup.rst

@@ -21,6 +21,8 @@ After cloning this repo, you will need to initialize all of the submodules.
     cd chipyard
     ./scripts/init-submodules-no-riscv-tools.sh
 
+.. _build-toolchains:
+
 Building a Toolchain
 ------------------------
 

docs/Generators/Gemmini.rst

@@ -0,0 +1,90 @@
+Gemmini
+====================================
+
+The Gemmini project is developing a systolic-array based matrix multiplication unit generator for the investigation of software/hardware implications of such integrated SoC accelerators. It is inspired by recent trends in machine learning accelerators for edge and mobile SoCs.
+
+Gemmini is implemented as a RoCC accelerator with non-standard RISC-V custom instructions. The Gemmini unit uses the RoCC port of a Rocket or BOOM `tile`, and by default connects to the memory system through the System Bus (i.e., directly to the L2 cache). 
+
+To add a Gemmini unit to an SoC, you should add the ``gemmini.DefaultGemminiConfig`` config mixin to the SoC configurations. To change the configuration of the Gemmini accelerator unit, you can write a custom configuration to replace the ``DefaultGemminiConfig``, which you can view under `generators/gemmini/src/main/scala/configs.scala <https://github.com/ucb-bar/gemmini/blob/master/src/main/scala/gemmini/configs.scala>`__ to see the possible configuration parameters.
+
+The example Chipyard config includes the following example SoC configuration which includes Gemmini:
+
+.. literalinclude:: ../../generators/example/src/main/scala/RocketConfigs.scala
+    :language: scala
+    :start-after: DOC include start: GemminiRocketConfig
+    :end-before: DOC include end: GemminiRocketConfig
+
+To build a simulation of this example Chipyard config, run the following commands:
+
+.. code-block:: shell
+
+    cd sims/verilator # or "cd sims/vcs"
+    make CONFIG=GemminiRocketConfig
+
+.. image:: ../_static/images/gemmini-system.png
+
+Generator Parameters
+--------------------------
+
+Major parameters of interest include:
+
+* Systolic array dimensions (``tileRows``, ``tileColumns``, ``meshRows``, ``meshColumns``): The systolic array is composed of a 2-level hierarchy, in which each tile is fully combinational, while a mesh of tiles has pipeline registers between each tile.
+
+.. image:: ../_static/images/gemmini-systolic-array.png
+
+* Dataflow parameters (``dataflow``): Determine whether the systolic array in Gemmini is output-stationary or weight-stationary, or whether it supports both dataflows so that programmers may choose between them at runtime.
+
+* Scratchpad and accumulator memory parameters (``sp_banks``, ``sp_capacity``, ``acc_capacity``): Determine the properties of the Gemmini scratchpad memory: overall capacity of the scratchpad or accumulators (in KiB), and the number of banks the scratchpad is divided into.
+
+* Type parameters (``inputType``, ``outputType``, ``accType``): Determine the data-types flowing through different parts of a Gemmini accelerator. For example, ``inputType`` may be an 8-bit fixed-point number, while ``accType``, which determines the type of partial accumulations in a matrix multiplication, may be a 32-bit integer. ``outputType`` only determines the type of the data passed between two processing elements (PEs); for example, an 8-bit multiplication may produce a 16-bit result which must be shared between PEs in a systolic array.
+
+* Access-execute queue parameters (``ld_queue_length``, ``st_queue_length``, ``ex_queue_length``, ``rob_entries``): To implement access-execute decoupling, a Gemmini accelerator has a load instruction queue, a store instruction queue, and an execute instruction queue. The relative sizes of these queue determine the level of access-execute decoupling. Gemmini also implements a reorder buffer (ROB) - the number of entries in the ROB determines possible dependency management limitations.
+
+* DMA parameters (``dma_maxbytes``, ``dma_buswidth``, ``mem_pipeline``): Gemmini implements a DMA to move data from main memory to the Gemmini scratchpad, and from the Gemmini accumulators to main memory. The size of these DMA transactions is determined by the DMA parameters. These DMA parameters are tightly coupled with Rocket Chip SoC system parameters: in particular ``dma_buswidth`` is associated with the ``SystemBusKey`` ``beatBytes`` parameter, and ``dma_maxbytes`` is associated with ``cacheblockbytes`` Rocket Chip parameters.
+
+Software
+------------------
+
+The Gemmini non-standard ISA extension is specified in the `Gemmini repository <https://github.com/ucb-bar/gemmini/blob/master/README.md>`__.
+The ISA includes configuration instructions, data movement instructions (from main memory to the Gemmini scratchpad, and from the Gemmini accumulators to main memory), and matrix multiplication execution instructions. 
+
+Since Gemmini instructions are not exposed through the GNU binutils assembler, several C macros are provided in order to construct the instruction encodings to call these instructions.
+
+The Gemmini generator includes a C matrix multiplication library which wraps the calls to the custom Gemmini instructions.
+The ``software`` directory of the generator includes the aforementioned library and macros, as well as bare-metal tests, and some FireMarshal workloads to run the tests in a Linux environment. In particular, the matrix multiplication C library can be found in the ``software/gemmini-rocc-tests/include/gemmini.h`` file. 
+
+The Gemmini generator generates a C header file based on the generator parameters. This header files gets compiled together with the matrix multiplication library to tune library performance. The generated header file can be found under ``software/gemmini-rocc-tests/include/gemmini_params.h``
+
+Build and Run Gemmini Tests
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To build Gemmini tests:
+
+.. code-block:: shell
+
+    cd generators/gemmini/software/gemmini-rocc-tests/
+    ./build.sh
+    
+Afterwards, the test binaries will be found in ``generators/gemmini/software/gemmini-rocc-tests/build``. Binaries whose names end in ``-baremetal`` are meant to be run in a bare-metal environment, while binaries whose names end in ``-linux`` are meant to run in a Linux environment. You can run the tests either on a cycle-accurate RTL simulator, or on a (much faster) functional ISA simulator called Spike.
+
+The Gemmini generator implements a custom non-standard version of Spike. This implementation is found within the ``esp-tools`` Spike implementation, together with the Hwacha vector accelerator non-standard ISA-extension. In order to use this version of Spike, please make sure to build the ``esp-tools`` software toolchain, as described in :ref:`build-toolchains`.
+
+In order to run Spike with the gemmini functional model, you will need to use the ``--extension=gemmini`` flag. For example:
+
+.. code-block:: shell
+
+    spike --extension=gemmini <some/gemmini/baremetal/test>
+
+Spike is built by default without a commit log. However, if you would like to add detailed functional log of gemmini operation to the spike model, you can rebuild spike manually (based on the instructions in the ``esp-tools/riscv-isa-sim/README`` file), with the ``--enable-gemminicommitlog`` option added to the ``configure`` step.
+
+Alternative SoC Configs
+--------------------------
+
+The Gemmini generator includes additional alternative SoC configs (configs that are not in the Chipyard example project). 
+If you would like to build one of these alternative SoC configurations which are defined in within the Gemmini project repository, you can run the following commands. These commands are similar to the one required when building a simulation from the example project, but they specify that the location of the configs are in the Gemmini subproject, as opposed to the Chipyard example project:
+
+.. code-block:: shell
+
+    cd sims/verilator # or "cd sims/vcs"
+    make CONFIG=GemminiAcceleratorConfig CONFIG_PACKAGE=gemmini MODEL_PACKAGE=freechips.rocketchip.system GENERATOR_PACKAGE=freechips.rocketchip.system TOP=ExampleRocketSystem
+

docs/Generators/Hwacha.rst

@@ -11,5 +11,5 @@ To add the Hwacha vector unit to an SoC, you should add the ``hwacha.DefaultHwac
 
 To change the configuration of the Hwacha vector unit, you can write a custom configuration to replace the ``DefaultHwachaConfig``. You can view the ``DefaultHwachaConfig`` under `generators/hwacha/src/main/scala/configs.scala <https://github.com/ucb-bar/hwacha/blob/master/src/main/scala/configs.scala>`__ to see the possible configuration parameters.
  
-Since Hwacha implements a non-standard RISC-V extension, it requires a unique software toolchain to be able to compile and asseble its vector instructions.
+Since Hwacha implements a non-standard RISC-V extension, it requires a unique software toolchain to be able to compile and assemble its vector instructions.
 To install the Hwacha toolchain, run the ``./scripts/build-toolchains.sh esp-tools`` command within the root Chipyard directory. This may take a while, and it will install the ``esp-tools-install`` directory within your Chipyard root directory. ``esp-tools`` is a fork of ``riscv-tools`` (formerly a collection of relevant software RISC-V tools) that was enhanced with additional non-standard vector instructions. However, due to the upstreaming of the equivalent RISC-V toolchains, ``esp-tools`` may not be up-to-date with the latest mainline version of the tools included in it.

docs/Generators/index.rst

@@ -22,6 +22,7 @@ so changes to the generators themselves will automatically be used when building
    Rocket
    BOOM
    Hwacha
+   Gemmini
    IceNet
    TestChipIP
    SiFive-Generators

generators/example/src/main/scala/RocketConfigs.scala

@@ -23,6 +23,16 @@ class HwachaRocketConfig extends Config(
   new freechips.rocketchip.subsystem.WithNBigCores(1) ++
   new freechips.rocketchip.system.BaseConfig)
 
+// DOC include start: GemminiRocketConfig
+class GemminiRocketConfig extends Config(
+  new WithTop ++
+  new WithBootROM ++
+  new freechips.rocketchip.subsystem.WithInclusiveCache ++
+  new gemmini.DefaultGemminiConfig ++                        // use Gemmini systolic array GEMM accelerator
+  new freechips.rocketchip.subsystem.WithNBigCores(1) ++
+  new freechips.rocketchip.system.BaseConfig)
+// DOC include end: GemminiRocketConfig
+
 class RoccRocketConfig extends Config(
   new WithTop ++
   new WithBootROM ++

generators/firechip/src/main/scala/TargetConfigs.scala

@@ -208,6 +208,18 @@ class FireSimRocketBoomConfig extends Config(
   new FireSimBoomConfig
 )
 
+//**********************************************************************************
+//* Gemmini Configurations
+//*********************************************************************************/
+
+// Gemmini systolic accelerator default config
+class FireSimRocketChipGemminiL2Config extends Config(
+  new WithInclusiveCache ++
+  new gemmini.DefaultGemminiConfig ++
+  new WithNBigCores(1) ++
+  new FireSimRocketChipConfig)
+
+
 //**********************************************************************************
 //* Supernode Configurations
 //*********************************************************************************/