diff --git a/.circleci/check-commit.sh b/.circleci/check-commit.sh index 9f8c701b..ed3093fd 100755 --- a/.circleci/check-commit.sh +++ b/.circleci/check-commit.sh @@ -48,7 +48,7 @@ search () { done } -submodules=("boom" "hwacha" "icenet" "sha3" "rocket-chip" "sifive-blocks" "sifive-cache" "testchipip") +submodules=("boom" "hwacha" "icenet" "sha3" "rocket-chip" "sifive-blocks" "sifive-cache" "testchipip" "gemmini") dir="generators" if [ "$CIRCLE_BRANCH" == "master" ] || [ "$CIRCLE_BRANCH" == "dev" ] then diff --git a/.circleci/config.yml b/.circleci/config.yml index bb5b3a50..75f13068 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -288,6 +288,35 @@ jobs: key: hwacha-{{ .Branch }}-{{ .Revision }} paths: - "/home/riscvuser/project" + prepare-gemmini: + docker: + - image: riscvboom/riscvboom-images:0.0.12 + environment: + JVM_OPTS: -Xmx3200m # Customize the JVM maximum heap limit + TERM: dumb + steps: + - add_ssh_keys: + fingerprints: + - "3e:c3:02:5b:ed:64:8c:b7:b0:04:43:bc:83:43:73:1e" + - checkout + - run: + name: Create hash of toolchains + command: | + .circleci/create-hash.sh + - restore_cache: + keys: + - esp-tools-installed-v2-{{ checksum "../esp-tools.hash" }} + - restore_cache: + keys: + - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }} + - run: + name: Building the gemmini subproject using Verilator + command: .circleci/do-rtl-build.sh gemmini + no_output_timeout: 120m + - save_cache: + key: gemmini-{{ .Branch }}-{{ .Revision }} + paths: + - "/home/riscvuser/project" prepare-tracegen: docker: - image: riscvboom/riscvboom-images:0.0.12 @@ -516,6 +545,30 @@ jobs: - run: name: Run hwacha tests command: .circleci/run-tests.sh hwacha + gemmini-run-tests: + docker: + - image: riscvboom/riscvboom-images:0.0.12 + environment: + JVM_OPTS: -Xmx3200m # Customize the JVM maximum heap limit + TERM: dumb + steps: + - checkout + - run: + name: Create hash of toolchains + command: | + .circleci/create-hash.sh + - restore_cache: + keys: + - esp-tools-installed-v2-{{ checksum "../esp-tools.hash" }} + - restore_cache: + keys: + - gemmini-{{ .Branch }}-{{ .Revision }} + - restore_cache: + keys: + - verilator-installed-v3-{{ checksum "sims/verilator/verilator.mk" }} + - run: + name: Run gemmini tests + command: .circleci/run-tests.sh gemmini tracegen-run-tests: docker: - image: riscvboom/riscvboom-images:0.0.12 @@ -660,6 +713,11 @@ workflows: - install-esp-toolchain - install-verilator + - prepare-gemmini: + requires: + - install-esp-toolchain + - install-verilator + - prepare-tracegen: requires: - install-riscv-toolchain @@ -704,6 +762,10 @@ workflows: requires: - prepare-hwacha + - gemmini-run-tests: + requires: + - prepare-gemmini + - tracegen-run-tests: requires: - prepare-tracegen diff --git a/.circleci/defaults.sh b/.circleci/defaults.sh index ffd45ffb..87a761d9 100755 --- a/.circleci/defaults.sh +++ b/.circleci/defaults.sh @@ -47,6 +47,7 @@ mapping["boom"]="SUB_PROJECT=example CONFIG=SmallBoomConfig" mapping["rocketchip"]="SUB_PROJECT=rocketchip" mapping["blockdevrocketchip"]="SUB_PROJECT=example CONFIG=SimBlockDeviceRocketConfig TOP=TopWithBlockDevice" mapping["hwacha"]="SUB_PROJECT=example CONFIG=HwachaRocketConfig" +mapping["gemmini"]="SUB_PROJECT=example CONFIG=GemminiRocketConfig" mapping["tracegen"]="SUB_PROJECT=tracegen CONFIG=NonBlockingTraceGenL2Config" mapping["firesim"]="DESIGN=FireSim TARGET_CONFIG=DDR3FRFCFSLLC4MB_FireSimRocketChipConfig PLATFORM_CONFIG=BaseF1Config" mapping["fireboom"]="DESIGN=FireSim TARGET_CONFIG=DDR3FRFCFSLLC4MB_FireSimBoomConfig PLATFORM_CONFIG=BaseF1Config" diff --git a/.circleci/do-firesim-build.sh b/.circleci/do-firesim-build.sh index 5eb42e56..8114355f 100755 --- a/.circleci/do-firesim-build.sh +++ b/.circleci/do-firesim-build.sh @@ -34,7 +34,7 @@ TOOLS_DIR=$REMOTE_RISCV_DIR LD_LIB_DIR=$REMOTE_RISCV_DIR/lib VERILATOR_BIN_DIR=$REMOTE_VERILATOR_DIR/install/bin -if [ $1 = "hwacha" ]; then +if [ $1 = "hwacha" ] || [ $1 = "gemmini" ]; then TOOLS_DIR=$REMOTE_ESP_DIR LD_LIB_DIR=$REMOTE_ESP_DIR/lib run "mkdir -p $REMOTE_ESP_DIR" diff --git a/.circleci/do-rtl-build.sh b/.circleci/do-rtl-build.sh index 50f9ce94..dd667dfb 100755 --- a/.circleci/do-rtl-build.sh +++ b/.circleci/do-rtl-build.sh @@ -32,7 +32,7 @@ run "cp -r ~/.sbt $REMOTE_WORK_DIR" TOOLS_DIR=$REMOTE_RISCV_DIR LD_LIB_DIR=$REMOTE_RISCV_DIR/lib -if [ $1 = "hwacha" ]; then +if [ $1 = "hwacha" ] || [ $1 = "gemmini" ]; then TOOLS_DIR=$REMOTE_ESP_DIR LD_LIB_DIR=$REMOTE_ESP_DIR/lib run "mkdir -p $REMOTE_ESP_DIR" diff --git a/.circleci/run-midasexamples-tests.sh b/.circleci/run-midasexamples-tests.sh index 383a75ff..7d31ac11 100755 --- a/.circleci/run-midasexamples-tests.sh +++ b/.circleci/run-midasexamples-tests.sh @@ -12,7 +12,7 @@ trap clean EXIT cd $LOCAL_CHIPYARD_DIR ./scripts/init-submodules-no-riscv-tools.sh -cd sims/firesim/sim/midas && git submodule update --init +cd sims/firesim/sim/midas # set stricthostkeychecking to no (must happen before rsync) run "echo \"Ping $SERVER\"" diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 138e2785..97789b4c 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -50,6 +50,18 @@ case $1 in export PATH=$RISCV/bin:$PATH make run-rv64uv-p-asm-tests -j$NPROC -C $LOCAL_SIM_DIR VERILATOR_INSTALL_DIR=$LOCAL_VERILATOR_DIR ${mapping[$1]} ;; + gemmini) + export RISCV=$LOCAL_ESP_DIR + export LD_LIBRARY_PATH=$LOCAL_ESP_DIR/lib + export PATH=$RISCV/bin:$PATH + GEMMINI_SOFTWARE_DIR=$LOCAL_SIM_DIR/../../generators/gemmini/software/gemmini-rocc-tests + cd $GEMMINI_SOFTWARE_DIR + ./build.sh + cd $LOCAL_SIM_DIR + $LOCAL_SIM_DIR/simulator-example-GemminiRocketConfig $GEMMINI_SOFTWARE_DIR/build/bareMetalC/aligned-baremetal + $LOCAL_SIM_DIR/simulator-example-GemminiRocketConfig $GEMMINI_SOFTWARE_DIR/build/bareMetalC/raw_hazard-baremetal + $LOCAL_SIM_DIR/simulator-example-GemminiRocketConfig $GEMMINI_SOFTWARE_DIR/build/bareMetalC/mvin_mvout-baremetal + ;; tracegen) run_tracegen ${mapping[$1]} ;; diff --git a/.gitmodules b/.gitmodules index 14c593bb..7e6fca6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -104,3 +104,6 @@ [submodule "software/coremark"] path = software/coremark url = https://github.com/ucb-bar/coremark-workload.git +[submodule "generators/gemmini"] + path = generators/gemmini + url = https://github.com/ucb-bar/gemmini diff --git a/build.sbt b/build.sbt index e252cfa9..2f27fdc1 100644 --- a/build.sbt +++ b/build.sbt @@ -123,7 +123,7 @@ lazy val testchipip = (project in file("generators/testchipip")) .settings(commonSettings) lazy val example = conditionalDependsOn(project in file("generators/example")) - .dependsOn(boom, hwacha, sifive_blocks, sifive_cache, utilities, sha3) + .dependsOn(boom, hwacha, sifive_blocks, sifive_cache, utilities, sha3, gemmini) .settings(commonSettings) lazy val tracegen = conditionalDependsOn(project in file("generators/tracegen")) @@ -150,6 +150,10 @@ lazy val sha3 = (project in file("generators/sha3")) .dependsOn(rocketchip, chisel_testers, midasTargetUtils) .settings(commonSettings) +lazy val gemmini = (project in file("generators/gemmini")) + .dependsOn(rocketchip, chisel_testers, testchipip) + .settings(commonSettings) + lazy val tapeout = conditionalDependsOn(project in file("./tools/barstools/tapeout/")) .dependsOn(chisel_testers, example) .settings(commonSettings) diff --git a/docs/Chipyard-Basics/Initial-Repo-Setup.rst b/docs/Chipyard-Basics/Initial-Repo-Setup.rst index ad5f1abd..1ab06c7a 100644 --- a/docs/Chipyard-Basics/Initial-Repo-Setup.rst +++ b/docs/Chipyard-Basics/Initial-Repo-Setup.rst @@ -21,6 +21,8 @@ After cloning this repo, you will need to initialize all of the submodules. cd chipyard ./scripts/init-submodules-no-riscv-tools.sh +.. _build-toolchains: + Building a Toolchain ------------------------ diff --git a/docs/Generators/Gemmini.rst b/docs/Generators/Gemmini.rst new file mode 100644 index 00000000..d90e6224 --- /dev/null +++ b/docs/Generators/Gemmini.rst @@ -0,0 +1,90 @@ +Gemmini +==================================== + +The Gemmini project is developing a systolic-array based matrix multiplication unit generator for the investigation of software/hardware implications of such integrated SoC accelerators. It is inspired by recent trends in machine learning accelerators for edge and mobile SoCs. + +Gemmini is implemented as a RoCC accelerator with non-standard RISC-V custom instructions. The Gemmini unit uses the RoCC port of a Rocket or BOOM `tile`, and by default connects to the memory system through the System Bus (i.e., directly to the L2 cache). + +To add a Gemmini unit to an SoC, you should add the ``gemmini.DefaultGemminiConfig`` config mixin to the SoC configurations. To change the configuration of the Gemmini accelerator unit, you can write a custom configuration to replace the ``DefaultGemminiConfig``, which you can view under `generators/gemmini/src/main/scala/configs.scala `__ to see the possible configuration parameters. + +The example Chipyard config includes the following example SoC configuration which includes Gemmini: + +.. literalinclude:: ../../generators/example/src/main/scala/RocketConfigs.scala + :language: scala + :start-after: DOC include start: GemminiRocketConfig + :end-before: DOC include end: GemminiRocketConfig + +To build a simulation of this example Chipyard config, run the following commands: + +.. code-block:: shell + + cd sims/verilator # or "cd sims/vcs" + make CONFIG=GemminiRocketConfig + +.. image:: ../_static/images/gemmini-system.png + +Generator Parameters +-------------------------- + +Major parameters of interest include: + +* Systolic array dimensions (``tileRows``, ``tileColumns``, ``meshRows``, ``meshColumns``): The systolic array is composed of a 2-level hierarchy, in which each tile is fully combinational, while a mesh of tiles has pipeline registers between each tile. + +.. image:: ../_static/images/gemmini-systolic-array.png + +* Dataflow parameters (``dataflow``): Determine whether the systolic array in Gemmini is output-stationary or weight-stationary, or whether it supports both dataflows so that programmers may choose between them at runtime. + +* Scratchpad and accumulator memory parameters (``sp_banks``, ``sp_capacity``, ``acc_capacity``): Determine the properties of the Gemmini scratchpad memory: overall capacity of the scratchpad or accumulators (in KiB), and the number of banks the scratchpad is divided into. + +* Type parameters (``inputType``, ``outputType``, ``accType``): Determine the data-types flowing through different parts of a Gemmini accelerator. For example, ``inputType`` may be an 8-bit fixed-point number, while ``accType``, which determines the type of partial accumulations in a matrix multiplication, may be a 32-bit integer. ``outputType`` only determines the type of the data passed between two processing elements (PEs); for example, an 8-bit multiplication may produce a 16-bit result which must be shared between PEs in a systolic array. + +* Access-execute queue parameters (``ld_queue_length``, ``st_queue_length``, ``ex_queue_length``, ``rob_entries``): To implement access-execute decoupling, a Gemmini accelerator has a load instruction queue, a store instruction queue, and an execute instruction queue. The relative sizes of these queue determine the level of access-execute decoupling. Gemmini also implements a reorder buffer (ROB) - the number of entries in the ROB determines possible dependency management limitations. + +* DMA parameters (``dma_maxbytes``, ``dma_buswidth``, ``mem_pipeline``): Gemmini implements a DMA to move data from main memory to the Gemmini scratchpad, and from the Gemmini accumulators to main memory. The size of these DMA transactions is determined by the DMA parameters. These DMA parameters are tightly coupled with Rocket Chip SoC system parameters: in particular ``dma_buswidth`` is associated with the ``SystemBusKey`` ``beatBytes`` parameter, and ``dma_maxbytes`` is associated with ``cacheblockbytes`` Rocket Chip parameters. + +Software +------------------ + +The Gemmini non-standard ISA extension is specified in the `Gemmini repository `__. +The ISA includes configuration instructions, data movement instructions (from main memory to the Gemmini scratchpad, and from the Gemmini accumulators to main memory), and matrix multiplication execution instructions. + +Since Gemmini instructions are not exposed through the GNU binutils assembler, several C macros are provided in order to construct the instruction encodings to call these instructions. + +The Gemmini generator includes a C matrix multiplication library which wraps the calls to the custom Gemmini instructions. +The ``software`` directory of the generator includes the aforementioned library and macros, as well as bare-metal tests, and some FireMarshal workloads to run the tests in a Linux environment. In particular, the matrix multiplication C library can be found in the ``software/gemmini-rocc-tests/include/gemmini.h`` file. + +The Gemmini generator generates a C header file based on the generator parameters. This header files gets compiled together with the matrix multiplication library to tune library performance. The generated header file can be found under ``software/gemmini-rocc-tests/include/gemmini_params.h`` + +Build and Run Gemmini Tests +^^^^^^^^^^^^^^^^^^^^^^^^^ + +To build Gemmini tests: + +.. code-block:: shell + + cd generators/gemmini/software/gemmini-rocc-tests/ + ./build.sh + +Afterwards, the test binaries will be found in ``generators/gemmini/software/gemmini-rocc-tests/build``. Binaries whose names end in ``-baremetal`` are meant to be run in a bare-metal environment, while binaries whose names end in ``-linux`` are meant to run in a Linux environment. You can run the tests either on a cycle-accurate RTL simulator, or on a (much faster) functional ISA simulator called Spike. + +The Gemmini generator implements a custom non-standard version of Spike. This implementation is found within the ``esp-tools`` Spike implementation, together with the Hwacha vector accelerator non-standard ISA-extension. In order to use this version of Spike, please make sure to build the ``esp-tools`` software toolchain, as described in :ref:`build-toolchains`. + +In order to run Spike with the gemmini functional model, you will need to use the ``--extension=gemmini`` flag. For example: + +.. code-block:: shell + + spike --extension=gemmini + +Spike is built by default without a commit log. However, if you would like to add detailed functional log of gemmini operation to the spike model, you can rebuild spike manually (based on the instructions in the ``esp-tools/riscv-isa-sim/README`` file), with the ``--enable-gemminicommitlog`` option added to the ``configure`` step. + +Alternative SoC Configs +-------------------------- + +The Gemmini generator includes additional alternative SoC configs (configs that are not in the Chipyard example project). +If you would like to build one of these alternative SoC configurations which are defined in within the Gemmini project repository, you can run the following commands. These commands are similar to the one required when building a simulation from the example project, but they specify that the location of the configs are in the Gemmini subproject, as opposed to the Chipyard example project: + +.. code-block:: shell + + cd sims/verilator # or "cd sims/vcs" + make CONFIG=GemminiAcceleratorConfig CONFIG_PACKAGE=gemmini MODEL_PACKAGE=freechips.rocketchip.system GENERATOR_PACKAGE=freechips.rocketchip.system TOP=ExampleRocketSystem + diff --git a/docs/Generators/Hwacha.rst b/docs/Generators/Hwacha.rst index 240d6fc9..ac4f4fdf 100644 --- a/docs/Generators/Hwacha.rst +++ b/docs/Generators/Hwacha.rst @@ -11,5 +11,5 @@ To add the Hwacha vector unit to an SoC, you should add the ``hwacha.DefaultHwac To change the configuration of the Hwacha vector unit, you can write a custom configuration to replace the ``DefaultHwachaConfig``. You can view the ``DefaultHwachaConfig`` under `generators/hwacha/src/main/scala/configs.scala `__ to see the possible configuration parameters. -Since Hwacha implements a non-standard RISC-V extension, it requires a unique software toolchain to be able to compile and asseble its vector instructions. +Since Hwacha implements a non-standard RISC-V extension, it requires a unique software toolchain to be able to compile and assemble its vector instructions. To install the Hwacha toolchain, run the ``./scripts/build-toolchains.sh esp-tools`` command within the root Chipyard directory. This may take a while, and it will install the ``esp-tools-install`` directory within your Chipyard root directory. ``esp-tools`` is a fork of ``riscv-tools`` (formerly a collection of relevant software RISC-V tools) that was enhanced with additional non-standard vector instructions. However, due to the upstreaming of the equivalent RISC-V toolchains, ``esp-tools`` may not be up-to-date with the latest mainline version of the tools included in it. diff --git a/docs/Generators/index.rst b/docs/Generators/index.rst index 5aa097b6..5bb5d491 100644 --- a/docs/Generators/index.rst +++ b/docs/Generators/index.rst @@ -22,6 +22,7 @@ so changes to the generators themselves will automatically be used when building Rocket BOOM Hwacha + Gemmini IceNet TestChipIP SiFive-Generators diff --git a/docs/_static/images/gemmini-system.png b/docs/_static/images/gemmini-system.png new file mode 100644 index 00000000..df44708f Binary files /dev/null and b/docs/_static/images/gemmini-system.png differ diff --git a/docs/_static/images/gemmini-systolic-array.png b/docs/_static/images/gemmini-systolic-array.png new file mode 100644 index 00000000..6586065a Binary files /dev/null and b/docs/_static/images/gemmini-systolic-array.png differ diff --git a/generators/example/src/main/scala/RocketConfigs.scala b/generators/example/src/main/scala/RocketConfigs.scala index 615111df..377b3995 100644 --- a/generators/example/src/main/scala/RocketConfigs.scala +++ b/generators/example/src/main/scala/RocketConfigs.scala @@ -23,6 +23,16 @@ class HwachaRocketConfig extends Config( new freechips.rocketchip.subsystem.WithNBigCores(1) ++ new freechips.rocketchip.system.BaseConfig) +// DOC include start: GemminiRocketConfig +class GemminiRocketConfig extends Config( + new WithTop ++ + new WithBootROM ++ + new freechips.rocketchip.subsystem.WithInclusiveCache ++ + new gemmini.DefaultGemminiConfig ++ // use Gemmini systolic array GEMM accelerator + new freechips.rocketchip.subsystem.WithNBigCores(1) ++ + new freechips.rocketchip.system.BaseConfig) +// DOC include end: GemminiRocketConfig + class RoccRocketConfig extends Config( new WithTop ++ new WithBootROM ++ diff --git a/generators/firechip/src/main/scala/TargetConfigs.scala b/generators/firechip/src/main/scala/TargetConfigs.scala index 4541578d..f7961c14 100644 --- a/generators/firechip/src/main/scala/TargetConfigs.scala +++ b/generators/firechip/src/main/scala/TargetConfigs.scala @@ -208,6 +208,18 @@ class FireSimRocketBoomConfig extends Config( new FireSimBoomConfig ) +//********************************************************************************** +//* Gemmini Configurations +//*********************************************************************************/ + +// Gemmini systolic accelerator default config +class FireSimRocketChipGemminiL2Config extends Config( + new WithInclusiveCache ++ + new gemmini.DefaultGemminiConfig ++ + new WithNBigCores(1) ++ + new FireSimRocketChipConfig) + + //********************************************************************************** //* Supernode Configurations //*********************************************************************************/ diff --git a/generators/gemmini b/generators/gemmini new file mode 160000 index 00000000..16fda885 --- /dev/null +++ b/generators/gemmini @@ -0,0 +1 @@ +Subproject commit 16fda88555be77df41a6c0687f64810417843dae diff --git a/toolchains/esp-tools/riscv-isa-sim b/toolchains/esp-tools/riscv-isa-sim index 0ffa02e5..5965f8fc 160000 --- a/toolchains/esp-tools/riscv-isa-sim +++ b/toolchains/esp-tools/riscv-isa-sim @@ -1 +1 @@ -Subproject commit 0ffa02e5b4ca57ec44684119a1a9a31b3871857b +Subproject commit 5965f8fcdbe931d6f8acabdea53c1e3a7b54e3f4