From 086cc1f74f83110c7824365f584b7a796f620e0b Mon Sep 17 00:00:00 2001 From: Ella Schwarz Date: Sun, 21 Nov 2021 18:30:21 -0800 Subject: [PATCH] Address PR comments for Ibex --- .circleci/build-extra-tests.sh | 11 +++ .circleci/build-toolchains.sh | 18 ++++ .circleci/clean-old-files.sh | 29 ++++++ .circleci/create-hash.sh | 23 +++++ .circleci/do-rtl-build.sh | 95 +++++++++++++++++++ .circleci/install-verilator.sh | 25 +++++ .circleci/run-firesim-scala-tests.sh | 62 ++++++++++++ .circleci/run-tests.sh | 2 +- .github/README.md | 137 --------------------------- .github/scripts/run-tests.sh | 2 +- generators/ibex | 2 +- 11 files changed, 266 insertions(+), 140 deletions(-) create mode 100755 .circleci/build-extra-tests.sh create mode 100755 .circleci/build-toolchains.sh create mode 100755 .circleci/clean-old-files.sh create mode 100755 .circleci/create-hash.sh create mode 100755 .circleci/do-rtl-build.sh create mode 100755 .circleci/install-verilator.sh create mode 100755 .circleci/run-firesim-scala-tests.sh delete mode 100644 .github/README.md diff --git a/.circleci/build-extra-tests.sh b/.circleci/build-extra-tests.sh new file mode 100755 index 00000000..e38b50fe --- /dev/null +++ b/.circleci/build-extra-tests.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# turn echo on and error on earliest command +set -ex + +# get shared variables +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +source $SCRIPT_DIR/defaults.sh + +make -C $LOCAL_CHIPYARD_DIR/tests clean +make -C $LOCAL_CHIPYARD_DIR/tests diff --git a/.circleci/build-toolchains.sh b/.circleci/build-toolchains.sh new file mode 100755 index 00000000..160b6f5a --- /dev/null +++ b/.circleci/build-toolchains.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# create the riscv tools/esp tools binaries +# passed in as + +# turn echo on and error on earliest command +set -ex + +# get shared variables +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +source $SCRIPT_DIR/defaults.sh + +if [ ! -d "$HOME/$1-install" ]; then + cd $HOME + + # init all submodules including the tools + CHIPYARD_DIR="$LOCAL_CHIPYARD_DIR" NPROC=$CI_MAKE_NPROC $LOCAL_CHIPYARD_DIR/scripts/build-toolchains.sh $1 +fi diff --git a/.circleci/clean-old-files.sh b/.circleci/clean-old-files.sh new file mode 100755 index 00000000..5824c4b7 --- /dev/null +++ b/.circleci/clean-old-files.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# clean directories that are older than 14 days +# argument is used as the directory to look in + +age () { + local AGE_SEC + local CUR_SEC + local DIFF_SEC + local SEC_PER_DAY + + SEC_PER_DAY=86400 + + CUR_SEC=$(date +%s) + AGE_SEC=$(stat -c %Y -- "$1") + DIFF_SEC=$(expr $CUR_SEC - $AGE_SEC) + + echo $(expr $DIFF_SEC / $SEC_PER_DAY) +} + +for d in $1/*/ ; do + DIR_AGE="$(age $d)" + if [ $DIR_AGE -ge 14 ]; then + echo "Deleting $d since is it $DIR_AGE old" + rm -rf $d + else + echo "Keep $d since it is $DIR_AGE old" + fi +done diff --git a/.circleci/create-hash.sh b/.circleci/create-hash.sh new file mode 100755 index 00000000..7a8915a5 --- /dev/null +++ b/.circleci/create-hash.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# get the hash of riscv-tools + +# turn echo on and error on earliest command +set -ex +set -o pipefail + +# get shared variables +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +source $SCRIPT_DIR/defaults.sh + +# enter bhd repo +cd $LOCAL_CHIPYARD_DIR + +# Use normalized output of git-submodule status as hashfile +for tools in 'riscv-tools' 'esp-tools' ; do + git submodule status "toolchains/${tools}" 'toolchains/libgloss' 'toolchains/qemu' | + while read -r line ; do + echo "${line#[!0-9a-f]}" + done > "${HOME}/${tools}.hash" +done +echo "Hashfile for riscv-tools and esp-tools created in $HOME" diff --git a/.circleci/do-rtl-build.sh b/.circleci/do-rtl-build.sh new file mode 100755 index 00000000..95f3c903 --- /dev/null +++ b/.circleci/do-rtl-build.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# create the different verilator builds +# usage: +# do-rtl-build.sh sim +# run rtl build for simulations and copy back results +# do-rtl-build.sh fpga +# run rtl build for fpga and don't copy back results + +# turn echo on and error on earliest command +set -ex + +# get shared variables +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +source $SCRIPT_DIR/defaults.sh + +# call clean on exit +trap clean EXIT + +cd $LOCAL_CHIPYARD_DIR +./scripts/init-submodules-no-riscv-tools.sh +./scripts/init-fpga.sh + +# replace the workspace dir with a local dir so you can copy around +sed -i -E 's/(workspace=).*(\/tools)/\1$PWD\2/g' .sbtopts + +# set stricthostkeychecking to no (must happen before rsync) +run "echo \"Ping $SERVER\"" + +clean + +# copy over riscv/esp-tools, and chipyard to remote +run "mkdir -p $REMOTE_CHIPYARD_DIR" +copy $LOCAL_CHIPYARD_DIR/ $SERVER:$REMOTE_CHIPYARD_DIR + +run "cp -r ~/.ivy2 $REMOTE_WORK_DIR" +run "cp -r ~/.sbt $REMOTE_WORK_DIR" + +TOOLS_DIR=$REMOTE_RISCV_DIR +LD_LIB_DIR=$REMOTE_RISCV_DIR/lib + +if [ $1 = "group-accels" ]; then + export RISCV=$LOCAL_ESP_DIR + export LD_LIBRARY_PATH=$LOCAL_ESP_DIR/lib + export PATH=$RISCV/bin:$PATH + GEMMINI_SOFTWARE_DIR=$LOCAL_SIM_DIR/../../generators/gemmini/software/gemmini-rocc-tests + cd $LOCAL_SIM_DIR/../../generators/gemmini/software + git submodule update --init --recursive gemmini-rocc-tests + cd gemmini-rocc-tests + ./build.sh + + TOOLS_DIR=$REMOTE_ESP_DIR + LD_LIB_DIR=$REMOTE_ESP_DIR/lib + run "mkdir -p $REMOTE_ESP_DIR" + copy $LOCAL_ESP_DIR/ $SERVER:$REMOTE_ESP_DIR +else + run "mkdir -p $REMOTE_RISCV_DIR" + copy $LOCAL_RISCV_DIR/ $SERVER:$REMOTE_RISCV_DIR +fi + +# choose what make dir to use +case $2 in + "sim") + REMOTE_MAKE_DIR=$REMOTE_SIM_DIR + ;; + "fpga") + REMOTE_MAKE_DIR=$REMOTE_FPGA_DIR + ;; +esac + +# enter the verilator directory and build the specific config on remote server +run "export RISCV=\"$TOOLS_DIR\"; \ + make -C $REMOTE_MAKE_DIR clean;" + +read -a keys <<< ${grouping[$1]} + +# need to set the PATH to use the new verilator (with the new verilator root) +for key in "${keys[@]}" +do + run "export RISCV=\"$TOOLS_DIR\"; \ + export LD_LIBRARY_PATH=\"$LD_LIB_DIR\"; \ + export PATH=\"$REMOTE_VERILATOR_DIR/bin:\$PATH\"; \ + export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \ + export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \ + make -j$REMOTE_MAKE_NPROC -C $REMOTE_MAKE_DIR FIRRTL_LOGLEVEL=info JAVA_TOOL_OPTIONS=\"$REMOTE_JAVA_OPTS\" SBT_OPTS=\"$REMOTE_SBT_OPTS\" ${mapping[$key]}" +done + +run "rm -rf $REMOTE_CHIPYARD_DIR/project" + +# choose to copy back results +if [ $2 = "sim" ]; then + # copy back the final build + mkdir -p $LOCAL_CHIPYARD_DIR + copy $SERVER:$REMOTE_CHIPYARD_DIR/ $LOCAL_CHIPYARD_DIR +fi diff --git a/.circleci/install-verilator.sh b/.circleci/install-verilator.sh new file mode 100755 index 00000000..2170768a --- /dev/null +++ b/.circleci/install-verilator.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# move verilator to the remote server + +# turn echo on and error on earliest command +set -ex + +# get shared variables +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +source $SCRIPT_DIR/defaults.sh + +# clean older directories (delete prior directories related to this branch also) +run_script $LOCAL_CHIPYARD_DIR/.circleci/clean-old-files.sh $CI_DIR +run "rm -rf $REMOTE_PREFIX*" + +# set stricthostkeychecking to no (must happen before rsync) +run "echo \"Ping $SERVER\"" + +run "git clone http://git.veripool.org/git/verilator $REMOTE_VERILATOR_DIR; \ + cd $REMOTE_VERILATOR_DIR; \ + git checkout $VERILATOR_VERSION; \ + autoconf; \ + export VERILATOR_ROOT=$REMOTE_VERILATOR_DIR; \ + ./configure; \ + make -j$REMOTE_MAKE_NPROC;" diff --git a/.circleci/run-firesim-scala-tests.sh b/.circleci/run-firesim-scala-tests.sh new file mode 100755 index 00000000..0a74adb8 --- /dev/null +++ b/.circleci/run-firesim-scala-tests.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# create the different verilator builds +# argument is the make command string + +# turn echo on and error on earliest command +set -ex + +# get shared variables +SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" +source $SCRIPT_DIR/defaults.sh + +# call clean on exit +trap clean EXIT + +# Directory locations for handling firesim-local installations of libelf/libdwarf +# This would generally be handled by build-setup.sh/firesim-setup.sh +firesim_sysroot=lib-install +local_firesim_sysroot=$LOCAL_FIRESIM_DIR/$firesim_sysroot +remote_firesim_sysroot=$REMOTE_FIRESIM_DIR/$firesim_sysroot + +cd $LOCAL_CHIPYARD_DIR +./scripts/init-submodules-no-riscv-tools.sh +cd $LOCAL_CHIPYARD_DIR/sims/firesim/sim/firesim-lib/src/main/cc/lib +git submodule update --init elfutils libdwarf +cd $LOCAL_CHIPYARD_DIR/sims/firesim +mkdir -p $local_firesim_sysroot +./scripts/build-libelf.sh $local_firesim_sysroot +./scripts/build-libdwarf.sh $local_firesim_sysroot +cd $LOCAL_CHIPYARD_DIR + +# replace the workspace dir with a local dir so you can copy around +sed -i -E 's/(workspace=).*(\/tools)/\1$PWD\2/g' .sbtopts + +make -C $LOCAL_CHIPYARD_DIR/tools/dromajo/dromajo-src/src + +# set stricthostkeychecking to no (must happen before rsync) +run "echo \"Ping $SERVER\"" + +clean + +# copy over riscv/esp-tools, and chipyard to remote +run "mkdir -p $REMOTE_CHIPYARD_DIR" +run "mkdir -p $REMOTE_RISCV_DIR" +copy $LOCAL_CHIPYARD_DIR/ $SERVER:$REMOTE_CHIPYARD_DIR +copy $LOCAL_RISCV_DIR/ $SERVER:$REMOTE_RISCV_DIR + +run "cp -r ~/.ivy2 $REMOTE_WORK_DIR" +run "cp -r ~/.sbt $REMOTE_WORK_DIR" + +TOOLS_DIR=$REMOTE_RISCV_DIR + +LD_LIB_DIR=$remote_firesim_sysroot/lib:$REMOTE_RISCV_DIR/lib + +# Run Firesim Scala Tests +run "export RISCV=\"$TOOLS_DIR\"; \ + export LD_LIBRARY_PATH=\"$LD_LIB_DIR\"; \ + export FIRESIM_ENV_SOURCED=1; \ + export PATH=\"$REMOTE_VERILATOR_DIR/bin:\$PATH\"; \ + export VERILATOR_ROOT=\"$REMOTE_VERILATOR_DIR\"; \ + export COURSIER_CACHE=\"$REMOTE_WORK_DIR/.coursier-cache\"; \ + make -C $REMOTE_FIRESIM_DIR JAVA_TOOL_OPTIONS=\"$REMOTE_JAVA_OPTS\" SBT_OPTS=\"$REMOTE_SBT_OPTS\" testOnly ${mapping[$1]}" diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 051deefd..461843fd 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -101,7 +101,7 @@ case $1 in run_asm ${mapping[$1]} ;; chipyard-ibex) - run_bmark ${mapping[$1]} + run_bmark ${mapping[$1]} #TODO: Find 32-bit test ;; chipyard-nvdla) make -C $LOCAL_CHIPYARD_DIR/tests diff --git a/.github/README.md b/.github/README.md deleted file mode 100644 index 89fcc3f6..00000000 --- a/.github/README.md +++ /dev/null @@ -1,137 +0,0 @@ -Chipyard Continuous Integration (CI) -=========== - -Website: https://gihub.com/gh/ucb-bar/chipyard/actions - -GitHub Actions Brief Explanation ---------------------------- - -CI is executed by Github Actions (GA). GA is controlled by `.yml` files in the `.github/workflows/` directory. -In our case we have just one workflow named `chipyard-rocket-run-tests.yml`. -It defines a number of `jobs` within it that do particular tasks. -All jobs in the workflow must pass for the CI run to be successful. -In general, a job is run in parallel with others unless it depends on some other job. -The dependency of one job on the completion of another is specified via the `needs` field. - -For example: -```yaml - prepare-chipyard-cores: - name: prepare-chipyard-cores - needs: [make-keys, setup-complete] -``` -This specifies that the `prepare-chipyard-cores` job needs the both the `make-keys` and the `setup-complete` steps to -be completed before it can run. - -Chipyard runs its CI using a docker image created from `dockerfiles/Dockerfile`. -See its [README](../dockerfiles/README.md) for more details. - -Finally, within each job's `steps:` section, the steps are run sequentially and state persists throughout a job. -So when you run something like `checkout` the next step has the checked out code. - -[Composite Actions](https://docs.github.com/en/actions/creating-actions) (CA) allow for limited subroutine like code re-use within GA. -We use both community created and our own Composite Actions in our CI process. CA capabilities are changing rapidly. -Nesting of composite actions was only recently unveiled. There is a lot of room for more code reuse, in particular -we specify things over and over like docker image tag and checkout commands. - -One use of CA: our process relies on caching to avoid running time-consuming and intensive tasks more often than necessary. - -The following is an example of using the cache@v2 composite action. A step `uses: actions/cache@v2` which take as parameters the -path that contains the data to be cached and a key. Paths can have multiple targets. -The following step can look at the result of the cache operation, if there was cache miss, then we run the command that -will generate the data to be cached. The caching of the generated data is implicit. ->Note: GA cache documentation suggests using the yml level `if: steps.cache-primes.outputs.cache-hit != 'true'` to -> determine whether to run the data generation command. -> At the time of this writing the if construct has a bug and will not run correctly within a composite action. The use -> of a bash based if is a hack found on stackoverflow -```yaml - - uses: actions/cache@v2 - id: rtl-build-id - with: - path: | - sims/verilator - sims/firesim/sim - generators/gemmini/software/gemmini-rocc-tests - key: ${{ inputs.group-key }}-${{ github.ref }}-${{ github.sha }} - - name: run rtl build script if not cached - run: | - if [[ "${{ steps.rtl-build-id.outputs.cache-hit }}" != 'true' ]]; then - echo "Cache miss on ${{ inputs.group-key }}-${{ github.ref }}-${{ github.sha }}" - ./.github/scripts/${{ inputs.build-script }} ${{ inputs.group-key }} ${{ inputs.build-type }} - else - echo "cache hit do not prepare rtl" - fi - shell: bash -``` - -Our own composite actions are defined in the `.github/actions//action.yml` - -.github/scripts directory -------------------- - -This directory contains most the collateral for the Chipyard CI to work. -The following is included in `.github/scripts/: directory - - `build-toolchains.sh` # build either riscv-tools or esp-tools - `create-hash.sh` # create hashes of riscv-tools/esp-tools to use as hash keys - `do-rtl-build.sh` # use verilator to build a sim executable (remotely) - `defaults.sh` # default variables used - `check-commit.sh` # check that submodule commits are valid - `build-extra-tests.sh` # build default chipyard tests located in tests/ - `clean-old-files.sh` # clean up build server files - `do-fpga-rtl-build.sh` # similar to `do-rtl-build` but using fpga/ - `install-verilator.sh` # install verilator on build server - `run-firesim-scala-tests.sh` # run firesim scala tests - `run-tests.sh # run tests for a specific set of designs - -How things are set up for Chipyard ---------------------------------- - -The steps for CI to run are as follows. -1. Build the toolchains in parallel (note: `esp-tools` is currently not used in the run). -The docker image sets up the `PATH` and `RISCV` variable so that `riscv-tools` is the default (currently the `env.sh` script that is created at tool build is unused). -2. Create the simulator binary. -This requires the `riscv-tools` for `fesvr` and `verilator` to be able to build the binary. -This stores all collateral for the tests (srcs, generated-srcs, sim binary, etc) to run "out of the gate" in the next job (make needs everything or else it will run again). -3. Finally, run the desired tests. - -Other CI Setup --------------- - -To get the CI to work correctly you need to create the following GH Repository Secrets - -| Secret | Value | -| -------| ------------- | -| BUILDSERVER | the hostname of the remote build server (likely be a millennium machine) | -| BUILDUSER | the login to use on the build server | -| BUILDDIR | the directory to use on the build server | -| SERVERKEY | a private key to access the build server | - -The default.sh script defines the following, -```bash -CI_DIR = /path/to/where/you/want/to/store/remote/files -```` -but in the future this should likely be a GH Secret too. - -The scripts also construct (repeatedly) a SERVER env using the above secrets -```bash -SERVER = ${{ secrets.BUILDUSER }}@${{ secrets.BUILDSERVER }} -``` - -Additionally, you need to add under the "PERMISSIONS" "SSH Permissions" section a private key that is on the build server that you are using. -After adding a private key, it will show a fingerprint that should be added under the jobs that need to be run. - -Note: On the remote server you need to have the `*.pub` key file added to the `authorized_keys` file. - -Additional Work ---------------- -- It would be nice to add the ability to re-run just parts of the workflow. [See Workflows Hacks](https://github.com/jaredpalmer/razzle/blob/f8305c26997bae8ef0f5dfa52540d842451b4090/.github/workflows/examples.yml) - - -Notes on CIRCLE CI ------------------- -This code is heavily based on the origin [CircleCI]() work. There a quite a few differences -- CCI supports workflow level variables, in GA we must define thiing like `BUILDSERVER: ${{ secrets.BUILDSERVER }}` in every job -- CCI allows a much larger cache. The entire CY directory with toolchains and RTL could be cached, with GA there is a 5Gb total cache limit -- GA support more parallel jobs 20 vs 4 -- GA seems to allow much longer run times -- \ No newline at end of file diff --git a/.github/scripts/run-tests.sh b/.github/scripts/run-tests.sh index 58a0e143..e5bdf99c 100755 --- a/.github/scripts/run-tests.sh +++ b/.github/scripts/run-tests.sh @@ -102,7 +102,7 @@ case $1 in make run-binary-fast -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} BINARY=$RISCV/riscv64-unknown-elf/share/riscv-tests/benchmarks/multiply.riscv ;; chipyard-ibex) - run_bmark ${mapping[$1]} + run_bmark ${mapping[$1]} #TODO: Find 32-bit test ;; chipyard-sodor) run_asm ${mapping[$1]} diff --git a/generators/ibex b/generators/ibex index d08e847e..bdf41a05 160000 --- a/generators/ibex +++ b/generators/ibex @@ -1 +1 @@ -Subproject commit d08e847ead100895c81c0ab3a3513a4c17823ef1 +Subproject commit bdf41a05484e57afa4ed20b35beff80320cfdd02