Update Radiance for Blackwell testcase support

- Update RadianceConfigs.scala with Blackwell configurations
- Update Verilator Makefile with optimized build flags
- Update submodules: radiance (Blackwell implementation), gemmini (params update)
- Update build flags and gitignore

.gitignore

@@ -28,3 +28,7 @@ project/project/
 .sbt
 .classpath_cache/
 .vscode/
+**/*.o
+**/*.fir
+**/*.d
+test_run_dir/

generators/chipyard/src/main/scala/config/RadianceConfigs.scala

@@ -26,6 +26,7 @@ class WithRadBootROM(address: BigInt = 0x10000, size: Int = 0x10000, hang: BigIn
 class VirgoConfig extends RadianceClusterConfig
 class VirgoFP16Config extends RadianceFP16ClusterConfig
 class VirgoHopperConfig extends Radiance4CFP16ClusterConfig
+class VirgoBlackwellConfig extends RadianceBlackwellClusterConfig
 class VirgoFlashConfig extends RadianceClusterConfig
 class VirgoSynConfig extends RadianceClusterSynConfig
 class VirgoFP16SynConfig extends RadianceFP16ClusterSynConfig
@@ -34,7 +35,7 @@ class VirgoHopperSynConfig extends Radiance4CFP16ClusterSynConfig
 class RadianceBaseConfig extends Config(
   // NOTE: when changing these, remember to change NUM_CORES/THREADS/WARPS in
   // the verilog source as well!
-  new radiance.subsystem.WithSimtConfig(nWarps = 8, nCoreLanes = 8, nMemLanes = 8, nSrcIds = 32) ++
+  new radiance.subsystem.WithSimtConfig(nWarps = 4, nCoreLanes = 4, nMemLanes = 4, nSrcIds = 32) ++
   new chipyard.config.WithSystemBusWidth(bitWidth = 256) ++
   new WithExtMemSize(BigInt("80000000", 16)) ++
   new WithRadBootROM() ++
@@ -56,7 +57,7 @@ class RadianceBaseConfig extends Config(
 class RadianceFP16ClusterConfig extends Config(
   new radiance.subsystem.WithRadianceGemmini(location = InCluster(0), dim = 16, accSizeInKB = 32, tileSize = (8, 4, 8), dataType = RadianceGemminiDataType.FP16) ++
   new radiance.subsystem.WithRadianceCores(8, location = InCluster(0), tensorCoreFP16 = true, tensorCoreDecoupled = false, useVxCache = false) ++
-  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 128 << 10, numBanks = 4, numWords = 16) ++
+  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 32 << 10, numBanks = 4, numWords = 16) ++
   new radiance.subsystem.WithCoalescer(nNewSrcIds = 16) ++
   new radiance.subsystem.WithVortexL1Banks(nBanks = 8) ++
   new radiance.subsystem.WithRadianceCluster(0) ++
@@ -65,7 +66,7 @@ class RadianceFP16ClusterConfig extends Config(
 class Radiance8B8WFP16ClusterConfig extends Config(
   new radiance.subsystem.WithRadianceGemmini(location = InCluster(0), dim = 16, accSizeInKB = 32, tileSize = (8, 4, 8), dataType = RadianceGemminiDataType.FP16) ++
   new radiance.subsystem.WithRadianceCores(8, location = InCluster(0), tensorCoreFP16 = true, tensorCoreDecoupled = false, useVxCache = false) ++
-  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 128 << 10, numBanks = 8, numWords = 8) ++
+  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 32 << 10, numBanks = 8, numWords = 8) ++
   new radiance.subsystem.WithCoalescer(nNewSrcIds = 16) ++
   new radiance.subsystem.WithVortexL1Banks(nBanks = 8) ++
   new radiance.subsystem.WithRadianceCluster(0) ++
@@ -74,12 +75,21 @@ class Radiance8B8WFP16ClusterConfig extends Config(
 class Radiance4CFP16ClusterConfig extends Config(
   new radiance.subsystem.WithRadianceGemmini(location = InCluster(0), dim = 16, accSizeInKB = 32, tileSize = (8, 4, 8), dataType = RadianceGemminiDataType.FP16) ++
   new radiance.subsystem.WithRadianceCores(4, location = InCluster(0), tensorCoreFP16 = true, tensorCoreDecoupled = true, useVxCache = false) ++
-  // new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 128 << 10, numBanks = 4, numWords = 16,
+  // new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 32 << 10, numBanks = 4, numWords = 16,
   //                                              memType = radiance.subsystem.TwoReadOneWrite,
   //                                              serializeUnaligned = radiance.subsystem.CoreSerialized) ++
   // NOTE: Hopper Tensor Core does not work with 16-word config due to the
   // address alignment requirement
-  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 128 << 10, numBanks = 4, numWords = 8) ++
+  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 32 << 10, numBanks = 4, numWords = 8) ++
+  new radiance.subsystem.WithCoalescer(nNewSrcIds = 16) ++
+  new radiance.subsystem.WithVortexL1Banks(nBanks = 8) ++
+  new radiance.subsystem.WithRadianceCluster(0) ++
+  new RadianceBaseConfig)
+
+class RadianceBlackwellClusterConfig extends Config(
+  new radiance.subsystem.WithRadianceGemmini(location = InCluster(0), dim = 16, accSizeInKB = 32, tileSize = (8, 4, 8), dataType = RadianceGemminiDataType.FP16) ++
+  new radiance.subsystem.WithRadianceCores(1, location = InCluster(0), tensorCoreFP16 = true, tensorCoreDecoupled = false, tensorCoreBlackwell = true, startupAddress = BigInt("80000000", 16), useVxCache = false) ++
+  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 32 << 10, numBanks = 4, numWords = 8) ++
   new radiance.subsystem.WithCoalescer(nNewSrcIds = 16) ++
   new radiance.subsystem.WithVortexL1Banks(nBanks = 8) ++
   new radiance.subsystem.WithRadianceCluster(0) ++
@@ -90,7 +100,7 @@ class RadianceClusterConfig extends Config(
   new radiance.subsystem.WithRadianceGemmini(location = InCluster(0), dim = 8, accSizeInKB = 16, tileSize = 8) ++
   new radiance.subsystem.WithRadianceCores(4, location = InCluster(0), tensorCoreFP16 = false, tensorCoreDecoupled = false, useVxCache = false) ++
   // new radiance.subsystem.WithRadianceFrameBuffer(x"ff018000", 16, 0x8000, x"ff011000", "fb0") ++
-  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 256 << 10/*KBytes*/, numBanks = 8, numWords = 8,
+  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 32 << 10/*KBytes*/, numBanks = 8, numWords = 8,
                                                // memType = radiance.subsystem.TwoReadOneWrite,
                                                serializeUnaligned = radiance.subsystem.CoreSerialized) ++
   new radiance.subsystem.WithCoalescer(nNewSrcIds = 16) ++
@@ -123,7 +133,7 @@ class RadianceBigLittleClusterConfig extends Config(
   new radiance.subsystem.WithRadianceGemmini(location = InCluster(0), dim = 4, accSizeInKB = 16, tileSize = 16) ++
   new radiance.subsystem.WithRadianceGemmini(location = InCluster(0), dim = 8, accSizeInKB = 16, tileSize = 8) ++
   new radiance.subsystem.WithRadianceCores(2, location = InCluster(0), useVxCache = false) ++
-  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 64 << 10, numBanks = 4, numWords = 8) ++
+  new radiance.subsystem.WithRadianceSharedMem(address = x"ff000000", size = 32 << 10, numBanks = 4, numWords = 8) ++
   new radiance.subsystem.WithCoalescer(nNewSrcIds = 16) ++
   new radiance.subsystem.WithVortexL1Banks(nBanks = 8)++
   new radiance.subsystem.WithRadianceCluster(0) ++

sims/.gitignore

@@ -1,2 +1,2 @@
 *.bin
-*.bin.*
+*.bin.*

sims/common-sim-flags.mk

@@ -1,7 +1,7 @@
 #----------------------------------------------------------------------------------------
 # common gcc configuration/optimization
 #----------------------------------------------------------------------------------------
-SIM_OPT_CXXFLAGS := -O3
+SIM_OPT_CXXFLAGS := -O0
 LRISCV=-lriscv
 
 export USE_CHISEL6=1

sims/vcs/scripts/compile_designs.sh

@@ -16,4 +16,6 @@ make CONFIG=VirgoFP16Config debug
 echo -e "\nCompiling hopper & virgo designs"
 make CONFIG=VirgoHopperConfig
 make CONFIG=VirgoHopperConfig debug
+echo -e "\nCompiling flash designs"
+make CONFIG=VirgoFlashConfig
 echo -e "\nCompilation completed"

sims/vcs/scripts/env.sh

@@ -1,6 +1,4 @@
+#!/bin/bash
 
-echo "!!!!!!!!!!!!!!!1!!!!!!!!!!!"
+ENV_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-echo "need to update kernels path"
+export KERNELS_PATH="$(realpath ${ENV_SCRIPT_DIR}/../../../../virgo-kernels/kernels)"
-echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!"
-
-export KERNELS_PATH="/scratch/yrh/vxsw/kernels"

sims/vcs/scripts/pprint

@@ -156,7 +156,7 @@ def main():
             print(translated_line, end='')
             sys.stdout.flush()
 
-    print("\033[s", end='')
+    print("\033[2J\033[s", end='')
     print("\033[" + str(lineno) + "H\033[2K\033[1m" + run_label, "DONE", "\033[0m", end='')
     print("\033[u", end='', flush=True)
 

sims/vcs/scripts/run_sims.sh

@@ -38,6 +38,8 @@ check_exists "simv-chipyard.harness-VirgoHopperConfig-debug"
 check_exists "$KERNELS_PATH/sgemm_tcore/kernel.radiance.gemm.tcore.volta.dim256.elf"
 check_exists "$KERNELS_PATH/sgemm_tcore/kernel.radiance.gemm.tcore.hopper.dim512.elf"
 check_exists "$KERNELS_PATH/sgemm_gemmini_dma/kernel.radiance.gemm.virgo.hopper.dim1024.elf"
+check_exists "$KERNELS_PATH/flash_attention/kernel.radiance.flash.ampere.seqlen1024.headdim64.elf"
+check_exists "$KERNELS_PATH/flash_attention/kernel.radiance.flash.virgo.seqlen1024.headdim64.elf"
 
 echo "Simulations will be started in parallel in 5 seconds. Please do not Ctrl+C as it kills all subprocesses."
 
@@ -54,6 +56,9 @@ for dim in "${dims[@]}"; do
     suffix=""
 done
 
+start_run VirgoFlashConfig flash_attention/kernel.radiance.flash.ampere.seqlen1024.headdim64.elf  "flash ampere" ""
+start_run VirgoFlashConfig flash_attention/kernel.radiance.flash.virgo.seqlen1024.headdim64.elf   "flash virgo " ""
+
 wait
 
 echo "All simulations have finished!"

sims/vcs/scripts/sanity.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+set -e
+
+echoerr() { echo "$@" 1>&2; }
+
+CURRENT_DIR="${PWD##*/}"
+if [[ "$CURRENT_DIR" != "vcs" ]]; then
+    echoerr "Error: This script must be run from chipyard/sims/vcs."
+    exit 1
+fi
+
+source ./scripts/env.sh > /dev/null
+
+check_exists() {
+    if ! [ -f "$1" ]; then
+        echo "Error: looked for file $1 that does not exist."
+        exit 1
+    fi
+}
+
+echo "Checking if all kernels have been compiled.."
+
+dims=(256 512 1024)
+for dim in "${dims[@]}"; do
+    check_exists "$KERNELS_PATH/sgemm_tcore/kernel.radiance.gemm.tcore.volta.dim${dim}.elf"
+    check_exists "$KERNELS_PATH/sgemm_tcore/kernel.radiance.gemm.tcore.ampere.dim${dim}.elf"
+    check_exists "$KERNELS_PATH/sgemm_tcore/kernel.radiance.gemm.tcore.hopper.dim${dim}.elf"
+    check_exists "$KERNELS_PATH/sgemm_gemmini_dma/kernel.radiance.gemm.virgo.hopper.dim${dim}.elf"
+done
+
+check_exists "$KERNELS_PATH/flash_attention/kernel.radiance.flash.ampere.seqlen1024.headdim64.elf"
+check_exists "$KERNELS_PATH/flash_attention/kernel.radiance.flash.virgo.seqlen1024.headdim64.elf"
+
+echo "Checking if all simulation binaries have been compiled.."
+
+check_exists "simv-chipyard.harness-VirgoFP16Config"
+check_exists "simv-chipyard.harness-VirgoFP16Config-debug"
+check_exists "simv-chipyard.harness-VirgoHopperConfig"
+check_exists "simv-chipyard.harness-VirgoHopperConfig-debug"
+check_exists "simv-chipyard.harness-VirgoFlashConfig"
+
+echo "Sanity check passed!"

sims/vcs/scripts/utilization.flash.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+echoerr() { echo "$@" 1>&2; }
+
+CURRENT_DIR="${PWD##*/}"
+if [[ "$CURRENT_DIR" != "vcs" ]]; then
+    echoerr "Error: This script must be run from chipyard/sims/vcs."
+    exit 1
+fi
+
+source ./scripts/env.sh > /dev/null
+
+rm -f /tmp/markers.log
+runtime() {
+    log_path="output/chipyard.harness.TestHarness.$1/kernel.radiance.$2.log"
+    check_exists "${log_path}"
+    if [ -z "$(tail -n10 ${log_path} | rg 'finish called')" ]; then
+        echo "$3,0"
+        echoerr "$3 run is not complete"
+        return
+    fi
+    rg "(e0d0a013|be90a013)" ${log_path} > /tmp/markers.log
+    echo -n "$3,"
+    cycles=$(python3 ./scripts/runtime_fast.py /tmp/markers.log)
+    echo "$cycles"
+    util=$(echo "scale=2; 64 * 64 * 64 * 100 / $cycles" | bc)
+    echoerr "$3 cycles: $cycles, hw utilization: ${util}%"
+    rm -f /tmp/markers.log
+}
+
+check_exists() {
+    if ! [ -f "$1" ]; then
+        echoerr "Error: looked for file $1 that does not exist."
+        exit 1
+    fi
+}
+
+echo ",cycles"
+runtime VirgoFlashConfig flash.virgo.seqlen1024.headdim64  "virgo"
+runtime VirgoFlashConfig flash.ampere.seqlen1024.headdim64 "ampere"

sims/verilator/Makefile

@@ -91,6 +91,7 @@ RUNTIME_PROFILING_VFLAGS := $(if $(filter $(VERILATOR_PROFILE),all),\
 
 VERILATOR_THREADS ?= 1
 RUNTIME_THREADS := --threads $(VERILATOR_THREADS) --threads-dpi all
+VERILATOR_MAKE_JOBS ?= 4
 
 USE_FST ?= 0
 TRACING_OPTS := $(if $(filter $(USE_FST),0),\
@@ -103,12 +104,16 @@ get_waveform_flag = +vcdfile=$(1).$(if $(filter $(USE_FST),0),vcd,fst)
 #----------------------------------------------------------------------------------------
 # we initially had --noassert for performance, but several modules use
 # assertions, including dramsim, so we enable --assert by default
+VERILATOR_OUTPUT_SPLIT ?= 10000
+VERILATOR_OUTPUT_SPLIT_CFUNCS ?= 100
+
 VERILATOR_OPT_FLAGS ?= \
-	-O3 \
+	-O0 \
 	--x-assign fast \
 	--x-initial fast \
-	--output-split 10000 \
+	--output-split $(VERILATOR_OUTPUT_SPLIT) \
-	--output-split-cfuncs 100
+	--output-split-cfuncs $(VERILATOR_OUTPUT_SPLIT_CFUNCS) \
+	-DPRINTF_COND_=1
 
 # default flags added for external IP (cva6/NVDLA)
 VERILOG_IP_VERILATOR_FLAGS := \
@@ -164,6 +169,7 @@ VERILATOR_NONCC_OPTS = \
 VERILATOR_CXXFLAGS = \
 	$(SIM_CXXFLAGS) \
 	$(RUNTIME_PROFILING_CFLAGS) \
+	-fno-inline \
 	-DVERILATOR
 
 VERILATOR_LDFLAGS = $(SIM_LDFLAGS)
@@ -208,10 +214,10 @@ $(model_mk_debug): $(sim_common_files) $(EXTRA_SIM_REQS)
 # invoke make to make verilator sim rules
 #########################################################################################
 $(sim): $(model_mk) $(dramsim_lib)
-	$(MAKE) VM_PARALLEL_BUILDS=1 -C $(model_dir) -f V$(TB).mk
+	$(MAKE) -j $(VERILATOR_MAKE_JOBS) VM_PARALLEL_BUILDS=1 -C $(model_dir) -f V$(TB).mk
 
 $(sim_debug): $(model_mk_debug) $(dramsim_lib)
-	$(MAKE) VM_PARALLEL_BUILDS=1 -C $(model_dir_debug) -f V$(TB).mk
+	$(MAKE) -j $(VERILATOR_MAKE_JOBS) VM_PARALLEL_BUILDS=1 -C $(model_dir_debug) -f V$(TB).mk
 
 #########################################################################################
 # create a verilator vpd rule

variables.mk

@@ -285,6 +285,10 @@ get_out_name = $(subst $() $(),_,$(notdir $(basename $(1))))
 LOADMEM ?=
 LOADARCH ?=
 
+ifeq ($(CONFIG),VirgoBlackwellConfig)
+override LOADMEM = 1
+endif
+
 ifneq ($(LOADARCH),)
 override BINARY = $(addsuffix /mem.elf,$(LOADARCH))
 override BINARIES = $(addsuffix /mem.elf,$(LOADARCH))