From 1de35a6af4057d8ba9216a8e9119f1072edff6ca Mon Sep 17 00:00:00 2001 From: abejgonzalez Date: Thu, 28 Apr 2022 15:10:46 -0700 Subject: [PATCH] Use conda + Update initial setup docs --- .github/CI_README.md | 20 +- .github/actions/cleanup-conda/action.yml | 28 ++ .github/actions/create-conda-env/action.yml | 18 + .github/actions/git-workaround/action.yml | 15 + .github/actions/prepare-rtl/action.yml | 16 +- .github/actions/run-tests/action.yml | 17 +- .github/actions/toolchain-build/action.yml | 51 -- .github/scripts/build-extra-tests.sh | 4 - .github/scripts/build-toolchains.sh | 21 - .github/scripts/check-commit.sh | 27 -- .github/scripts/create-hash.sh | 20 - .github/scripts/defaults.sh | 13 +- .github/scripts/remote-do-rtl-build.sh | 11 - .github/scripts/remote-install-verilator.sh | 22 - .../scripts/remote-run-firesim-scala-tests.sh | 13 +- .github/scripts/run-tests.sh | 13 - .github/workflows/chipyard-run-tests.yml | 448 ++++++++++-------- .gitmodules | 37 -- .readthedocs.yml | 7 +- dockerfiles/Dockerfile | 57 +-- docs/Chipyard-Basics/Initial-Repo-Setup.rst | 125 +++-- docs/Customization/Boot-Process.rst | 2 +- .../FPGA-Accelerated-Simulation.rst | 5 +- docs/conf.py | 16 +- docs/requirements.txt | 5 - scripts/build-toolchains.sh | 227 --------- scripts/centos-req.sh | 33 -- scripts/check-tracegen.sh | 11 +- scripts/conda-requirements.yaml | 57 +++ .../init-submodules-no-riscv-tools-nolog.sh | 17 +- scripts/install-conda.sh | 178 +++++++ scripts/ubuntu-req.sh | 33 -- toolchains/esp-tools/riscv-gnu-toolchain | 1 - toolchains/esp-tools/riscv-isa-sim | 1 - toolchains/esp-tools/riscv-pk | 1 - toolchains/esp-tools/riscv-tests | 1 - toolchains/libgloss | 1 - toolchains/qemu | 1 - toolchains/riscv-tools/riscv-gnu-toolchain | 1 - .../riscv-tools/riscv-gnu-toolchain-prebuilt | 1 - toolchains/riscv-tools/riscv-isa-sim | 1 - toolchains/riscv-tools/riscv-openocd | 1 - toolchains/riscv-tools/riscv-pk | 1 - toolchains/riscv-tools/riscv-tests | 1 - tools/dromajo/dromajo.mk | 2 +- variables.mk | 2 +- 46 files changed, 742 insertions(+), 841 deletions(-) create mode 100644 .github/actions/cleanup-conda/action.yml create mode 100644 .github/actions/create-conda-env/action.yml create mode 100644 .github/actions/git-workaround/action.yml delete mode 100644 .github/actions/toolchain-build/action.yml delete mode 100755 .github/scripts/build-toolchains.sh delete mode 100755 .github/scripts/create-hash.sh delete mode 100755 .github/scripts/remote-install-verilator.sh delete mode 100644 docs/requirements.txt delete mode 100755 scripts/build-toolchains.sh delete mode 100755 scripts/centos-req.sh create mode 100644 scripts/conda-requirements.yaml create mode 100755 scripts/install-conda.sh delete mode 100755 scripts/ubuntu-req.sh delete mode 160000 toolchains/esp-tools/riscv-gnu-toolchain delete mode 160000 toolchains/esp-tools/riscv-isa-sim delete mode 160000 toolchains/esp-tools/riscv-pk delete mode 160000 toolchains/esp-tools/riscv-tests delete mode 160000 toolchains/libgloss delete mode 160000 toolchains/qemu delete mode 160000 toolchains/riscv-tools/riscv-gnu-toolchain delete mode 160000 toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt delete mode 160000 toolchains/riscv-tools/riscv-isa-sim delete mode 160000 toolchains/riscv-tools/riscv-openocd delete mode 160000 toolchains/riscv-tools/riscv-pk delete mode 160000 toolchains/riscv-tools/riscv-tests diff --git a/.github/CI_README.md b/.github/CI_README.md index 2f70449e..c3066f79 100644 --- a/.github/CI_README.md +++ b/.github/CI_README.md @@ -22,7 +22,7 @@ For example: This specifies that the `prepare-chipyard-cores` job needs the both the `make-keys` and the `setup-complete` steps to be completed before it can run. -Chipyard runs its CI using a docker image created from `dockerfiles/Dockerfile`. +Chipyard runs its CI using a docker image created from `dockerfiles/Dockerfile` and on Berkeley's millennium machines. See its [README](../dockerfiles/README.md) for more details. Finally, within each job's `steps:` section, the steps are run sequentially and state persists throughout a job. @@ -71,9 +71,7 @@ Our own composite actions are defined in the `.github/actions//actio This directory contains most the collateral for the Chipyard CI to work. The following is included in `.github/scripts/: directory - `build-toolchains.sh` # build either riscv-tools or esp-tools - `create-hash.sh` # create hashes of riscv-tools/esp-tools to use as hash keys - `remote-do-rtl-build.sh` # use verilator to build a sim executable (remotely) + `remote-do-rtl-build.sh` # use verilator to build a sim executable (remotely) `defaults.sh` # default variables used `check-commit.sh` # check that submodule commits are valid `build-extra-tests.sh` # build default chipyard tests located in tests/ @@ -101,21 +99,9 @@ To get the CI to work correctly you need to create the following GH Repository S | Secret | Value | | -------| ------------- | -| BUILDSERVER | the hostname of the remote build server (likely be a millennium machine) | -| BUILDUSER | the login to use on the build server | | BUILDDIR | the directory to use on the build server | -| SERVERKEY | a private key to access the build server | -The main workflow also constructs and places in the environment a SERVER and a work directyory on that server env using the above secrets. -The SERVER is constructed like this: -```bash -SERVER = ${{ secrets.BUILDUSER }}@${{ secrets.BUILDSERVER }} -``` - -Additionally, you need to add under the "PERMISSIONS" "SSH Permissions" section a private key that is on the build server that you are using. -After adding a private key, it will show a fingerprint that should be added under the jobs that need to be run. - -Note: On the remote server you need to have the `*.pub` key file added to the `authorized_keys` file. +Additionally, you need to install conda on the build servers that exist Notes on CIRCLE CI ------------------ diff --git a/.github/actions/cleanup-conda/action.yml b/.github/actions/cleanup-conda/action.yml new file mode 100644 index 00000000..74dac2ef --- /dev/null +++ b/.github/actions/cleanup-conda/action.yml @@ -0,0 +1,28 @@ +name: cleanup-conda +description: 'Remove extra conda environments' + +runs: + using: "composite" + steps: + - name: Remove extra conda environments + run: | + CONDA_REMOVE_NAMES=$(conda env list | awk '{print $1}' | tail -n +3 | grep "${{ env.conda-env-name-no-time }}" || true) + if [ -z "$CONDA_REMOVE_NAMES" ]; then + echo "No matching conda environments for ${{ env.conda-env-name-no-time }}. Skip removal." + else + echo "Removing $CONDA_REMOVE_NAMES conda environments." + for env in $CONDA_REMOVE_NAMES; do + conda env remove -n $env + done + fi + conda env list | awk '{print $1}' | tail -n +4 | while read envname; do + ENV_DATE=$(echo $envname | sed "s/cy-[[:digit:]]\+-\(.*\)-\(riscv\|esp\)-tools/\1/") + NUM_DIFF=$(( ( $(date +%s) - $(date --date="$ENV_DATE" +%s) )/(60*60*24) )) + if (( $NUM_DIFF > 7 )); then + echo "Removing $envname since it is $NUM_DIFF days old." + conda env remove -n $envname + else + echo "Skipping removal of $envname since it is $NUM_DIFF days old." + fi + done + shell: bash -leo pipefail {0} diff --git a/.github/actions/create-conda-env/action.yml b/.github/actions/create-conda-env/action.yml new file mode 100644 index 00000000..61bd0dd5 --- /dev/null +++ b/.github/actions/create-conda-env/action.yml @@ -0,0 +1,18 @@ +name: create-conda-env +description: 'Create conda environments if they dont exist' + +runs: + using: "composite" + steps: + - name: Create conda environments + run: | + if conda env list | grep -q "envs/${{ env.conda-env-name-no-time }}"; then + echo "Using pre-existing conda environments with prefix ${{ env.conda-env-name-no-time }}" + else + echo "Creating a conda environment for each toolchain" + conda env create -f ./scripts/conda-requirements.yaml -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools + conda install -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools -c ucb-bar -y riscv-tools + conda env create -f ./scripts/conda-requirements.yaml -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-esp-tools + conda install -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-esp-tools -c ucb-bar -y esp-tools + fi + shell: bash -leo pipefail {0} diff --git a/.github/actions/git-workaround/action.yml b/.github/actions/git-workaround/action.yml new file mode 100644 index 00000000..c9d923d3 --- /dev/null +++ b/.github/actions/git-workaround/action.yml @@ -0,0 +1,15 @@ +name: git-workaround +description: 'Workaround https://github.com/actions/checkout/issues/766' + +runs: + using: "composite" + steps: + - name: Workaround + run: | + if git config --global -l | grep -q "safe.directory=$GITHUB_WORKSPACE"; then + echo "Skip adding safe directory" + else + echo "Add $GITHUB_WORKSPACE to global git config" + git config --global --add safe.directory "$GITHUB_WORKSPACE" + fi + shell: bash -leo pipefail {0} diff --git a/.github/actions/prepare-rtl/action.yml b/.github/actions/prepare-rtl/action.yml index 25098bfc..74a04037 100644 --- a/.github/actions/prepare-rtl/action.yml +++ b/.github/actions/prepare-rtl/action.yml @@ -13,28 +13,30 @@ inputs: description: type of build required: false default: "sim" + toolchain: + description: toolchain to use + required: false + default: "riscv-tools" runs: using: "composite" steps: - - name: Build RISC-V toolchains - uses: ./.github/actions/toolchain-build - - - uses: actions/cache@v2 + - uses: actions/cache@v3 id: rtl-build-id with: path: | sims/verilator sims/firesim/sim generators/gemmini/software/gemmini-rocc-tests - key: ${{ inputs.group-key }}-${{ github.ref }}-${{ github.sha }} + key: ${{ inputs.group-key }}-${{ github.sha }} - name: Run RTL build if not cached run: | + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }} if [[ "${{ steps.rtl-build-id.outputs.cache-hit }}" != 'true' ]]; then - echo "Cache miss on ${{ inputs.group-key }}-${{ github.ref }}-${{ github.sha }}" + echo "Cache miss on ${{ inputs.group-key }}-${{ github.sha }}" ./.github/scripts/${{ inputs.build-script }} ${{ inputs.group-key }} ${{ inputs.build-type }} else echo "Cache hit do not rebuild RTL for ${{ inputs.group-key }}" fi - shell: bash + shell: bash -leo pipefail {0} diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 05cc6498..6803376e 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -12,20 +12,29 @@ inputs: description: rtl build script to use required: false default: "run-tests.sh" + toolchain: + description: toolchain to use + required: false + default: "riscv-tools" runs: using: "composite" steps: - name: Init submodules (since only the RTL is cached) - run: ./scripts/init-submodules-no-riscv-tools.sh --skip-validate - shell: bash + run: | + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }} + ./scripts/init-submodules-no-riscv-tools.sh --skip-validate + shell: bash -leo pipefail {0} # Note: You shouldn't need the other inputs since it shouldn't build RTL from scratch - name: Build RTL uses: ./.github/actions/prepare-rtl with: group-key: ${{ inputs.group-key }} + toolchain: ${{ inputs.toolchain }} - name: Run RTL tests - run: ./.github/scripts/${{ inputs.run-script }} ${{ inputs.project-key }} - shell: bash + run: | + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }} + ./.github/scripts/${{ inputs.run-script }} ${{ inputs.project-key }} + shell: bash -leo pipefail {0} diff --git a/.github/actions/toolchain-build/action.yml b/.github/actions/toolchain-build/action.yml deleted file mode 100644 index c3492054..00000000 --- a/.github/actions/toolchain-build/action.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: toolchain-build -description: 'Build/cache both toolchains' - -runs: - using: "composite" - steps: - - name: Generate hashes - run: .github/scripts/create-hash.sh - shell: bash - - # since "hashFiles" function differs on self-hosted vs GH-A machines - # make sure to cache the GH-A hashFiles result so that self-hosted can use it - - run: | - echo "${{ hashFiles('**/riscv-tools.hash') }}" > riscv-tools.hashFilesOutput - echo "${{ hashFiles('**/esp-tools.hash') }}" > esp-tools.hashFilesOutput - shell: bash - - - name: Cache hashFiles outputs - uses: actions/cache@v2 - with: - path: | - riscv-tools.hashFilesOutput - esp-tools.hashFilesOutput - key: hashFiles-${{ github.sha }} - - - name: Generate cache keys based off hashFiles output - id: genkey - run: | - echo "::set-output name=riscv-tools-cache-key::$(cat riscv-tools.hashFilesOutput)" - echo "::set-output name=esp-tools-cache-key::$(cat esp-tools.hashFilesOutput)" - shell: bash - - - name: Cache riscv-tools - uses: actions/cache@v2 - with: - path: riscv-tools-install - key: riscv-tools-installed-${{ env.tools-cache-version }}-${{ steps.genkey.outputs.riscv-tools-cache-key }} - - - name: Cache esp-tools - uses: actions/cache@v2 - with: - path: esp-tools-install - key: esp-tools-installed-${{ env.tools-cache-version }}-${{ steps.genkey.outputs.esp-tools-cache-key }} - - - name: Build RISC-V toolchain if not cached - run: ./.github/scripts/build-toolchains.sh riscv-tools - shell: bash - - - name: Build ESP RISC-V toolchain if not cached - run: ./.github/scripts/build-toolchains.sh esp-tools - shell: bash diff --git a/.github/scripts/build-extra-tests.sh b/.github/scripts/build-extra-tests.sh index a77f5482..e38b50fe 100755 --- a/.github/scripts/build-extra-tests.sh +++ b/.github/scripts/build-extra-tests.sh @@ -7,9 +7,5 @@ set -ex SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" source $SCRIPT_DIR/defaults.sh -export RISCV="$GITHUB_WORKSPACE/riscv-tools-install" -export LD_LIBRARY_PATH="$RISCV/lib" -export PATH="$RISCV/bin:$PATH" - make -C $LOCAL_CHIPYARD_DIR/tests clean make -C $LOCAL_CHIPYARD_DIR/tests diff --git a/.github/scripts/build-toolchains.sh b/.github/scripts/build-toolchains.sh deleted file mode 100755 index 1f23c408..00000000 --- a/.github/scripts/build-toolchains.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# create the riscv tools/esp tools binaries -# passed in as - -# turn echo on and error on earliest command -set -ex - -# get shared variables -SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" -source $SCRIPT_DIR/defaults.sh - -if [ ! -d "$HOME/$1-install" ]; then - cd $HOME - - # init all submodules including the tools - CHIPYARD_DIR="$LOCAL_CHIPYARD_DIR" NPROC=$CI_MAKE_NPROC $LOCAL_CHIPYARD_DIR/scripts/build-toolchains.sh $1 - - # de-init the toolchain area to save on space (forced to ignore local changes) - git submodule deinit --force $LOCAL_CHIPYARD_DIR/toolchains/$1 -fi diff --git a/.github/scripts/check-commit.sh b/.github/scripts/check-commit.sh index 27abacd4..510b4f59 100755 --- a/.github/scripts/check-commit.sh +++ b/.github/scripts/check-commit.sh @@ -52,28 +52,6 @@ dir="generators" branches=("master" "main" "dev") search -submodules=("riscv-gnu-toolchain" "riscv-isa-sim" "riscv-pk" "riscv-tests") -dir="toolchains/esp-tools" -branches=("master") -search - - -submodules=("riscv-gnu-toolchain" "riscv-isa-sim" "riscv-pk" "riscv-tests") -dir="toolchains/riscv-tools" -branches=("master") -search - -# riscv-openocd doesn't use its master branch -submodules=("riscv-openocd") -dir="toolchains/riscv-tools" -branches=("riscv") -search - -submodules=("qemu" "libgloss") -dir="toolchains" -branches=("master") -search - submodules=("coremark" "firemarshal" "nvdla-workload" "spec2017") dir="software" branches=("master" "dev") @@ -84,11 +62,6 @@ dir="tools" branches=("master" "dev") search -submodules=("dromajo-src") -dir="tools/dromajo" -branches=("master") -search - submodules=("firesim") dir="sims" branches=("master" "main" "dev" "1.13.x") diff --git a/.github/scripts/create-hash.sh b/.github/scripts/create-hash.sh deleted file mode 100755 index f64c0696..00000000 --- a/.github/scripts/create-hash.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# get the hash of riscv-tools - -# turn echo on and error on earliest command -set -ex -set -o pipefail - -# get shared variables -SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" -source $SCRIPT_DIR/defaults.sh - -# Use normalized output of git-submodule status as hashfile -for tools in 'riscv-tools' 'esp-tools' ; do - git submodule status "toolchains/${tools}" 'toolchains/libgloss' 'toolchains/qemu' | - while read -r line ; do - echo "${line#[!0-9a-f]}" - done > "${tools}.hash" -done -echo "Hashfile for riscv-tools and esp-tools created in $PWD" diff --git a/.github/scripts/defaults.sh b/.github/scripts/defaults.sh index 8022a470..e196b994 100755 --- a/.github/scripts/defaults.sh +++ b/.github/scripts/defaults.sh @@ -5,18 +5,11 @@ CI_MAKE_NPROC=8 # chosen based on a 24c system shared with 1 other project REMOTE_MAKE_NPROC=4 -# verilator version -VERILATOR_VERSION=v4.034 - -HOME=$GITHUB_WORKSPACE - # remote variables # CI_DIR is defined externally based on the GH repository secret BUILDDIR REMOTE_PREFIX=$CI_DIR/${GITHUB_REPOSITORY#*/}-${GITHUB_REF_NAME//\//-} REMOTE_WORK_DIR=$GITHUB_WORKSPACE -REMOTE_RISCV_DIR=$GITHUB_WORKSPACE/riscv-tools-install -REMOTE_ESP_DIR=$GITHUB_WORKSPACE/esp-tools-install REMOTE_CHIPYARD_DIR=$GITHUB_WORKSPACE REMOTE_SIM_DIR=$REMOTE_CHIPYARD_DIR/sims/verilator REMOTE_FIRESIM_DIR=$REMOTE_CHIPYARD_DIR/sims/firesim/sim @@ -24,13 +17,9 @@ REMOTE_FPGA_DIR=$REMOTE_CHIPYARD_DIR/fpga REMOTE_JAVA_OPTS="-Xmx10G -Xss8M" # Disable the supershell to greatly improve the readability of SBT output when captured by Circle CI REMOTE_SBT_OPTS="-Dsbt.ivy.home=$REMOTE_WORK_DIR/.ivy2 -Dsbt.supershell=false -Dsbt.global.base=$REMOTE_WORK_DIR/.sbt -Dsbt.boot.directory=$REMOTE_WORK_DIR/.sbt/boot" -REMOTE_VERILATOR_DIR=$REMOTE_PREFIX-$GITHUB_SHA-verilator-install # local variables (aka within the docker container) -LOCAL_CHECKOUT_DIR=$HOME/project -LOCAL_RISCV_DIR=$HOME/riscv-tools-install -LOCAL_ESP_DIR=$HOME/esp-tools-install -LOCAL_CHIPYARD_DIR=$HOME +LOCAL_CHIPYARD_DIR=$GITHUB_WORKSPACE LOCAL_SIM_DIR=$LOCAL_CHIPYARD_DIR/sims/verilator LOCAL_FIRESIM_DIR=$LOCAL_CHIPYARD_DIR/sims/firesim/sim diff --git a/.github/scripts/remote-do-rtl-build.sh b/.github/scripts/remote-do-rtl-build.sh index 1f4251b1..80252511 100755 --- a/.github/scripts/remote-do-rtl-build.sh +++ b/.github/scripts/remote-do-rtl-build.sh @@ -18,13 +18,7 @@ cd $REMOTE_CHIPYARD_DIR ./scripts/init-submodules-no-riscv-tools.sh --skip-validate ./scripts/init-fpga.sh -TOOLS_DIR=$REMOTE_RISCV_DIR -LD_LIB_DIR=$REMOTE_RISCV_DIR/lib - if [ $1 = "group-accels" ]; then - export RISCV=$REMOTE_ESP_DIR - export LD_LIBRARY_PATH=$REMOTE_ESP_DIR/lib - export PATH=$RISCV/bin:$PATH pushd $REMOTE_CHIPYARD_DIR/generators/gemmini/software git submodule update --init --recursive gemmini-rocc-tests pushd gemmini-rocc-tests @@ -44,7 +38,6 @@ case $2 in esac # enter the verilator directory and build the specific config on remote server -export RISCV=$TOOLS_DIR make -C $REMOTE_MAKE_DIR clean read -a keys <<< ${grouping[$1]} @@ -52,10 +45,6 @@ read -a keys <<< ${grouping[$1]} # need to set the PATH to use the new verilator (with the new verilator root) for key in "${keys[@]}" do - export RISCV=$TOOLS_DIR - export LD_LIBRARY_PATH=$LD_LIB_DIR - export PATH=$REMOTE_VERILATOR_DIR/bin:$PATH - export VERILATOR_ROOT=$REMOTE_VERILATOR_DIR export COURSIER_CACHE=$REMOTE_WORK_DIR/.coursier-cache make -j$REMOTE_MAKE_NPROC -C $REMOTE_MAKE_DIR FIRRTL_LOGLEVEL=info JAVA_OPTS="$REMOTE_JAVA_OPTS" SBT_OPTS="$REMOTE_SBT_OPTS" ${mapping[$key]} done diff --git a/.github/scripts/remote-install-verilator.sh b/.github/scripts/remote-install-verilator.sh deleted file mode 100755 index b244614e..00000000 --- a/.github/scripts/remote-install-verilator.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# install verilator - -# turn echo on and error on earliest command -set -ex - -# get shared variables -SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" -source $SCRIPT_DIR/defaults.sh - -# clean older directories (delete prior directories related to this branch also) -$SCRIPT_DIR/clean-old-files.sh $CI_DIR -rm -rf $REMOTE_PREFIX* - -git clone http://git.veripool.org/git/verilator $REMOTE_VERILATOR_DIR -cd $REMOTE_VERILATOR_DIR -git checkout $VERILATOR_VERSION -autoconf -export VERILATOR_ROOT=$REMOTE_VERILATOR_DIR -./configure -make -j$REMOTE_MAKE_NPROC diff --git a/.github/scripts/remote-run-firesim-scala-tests.sh b/.github/scripts/remote-run-firesim-scala-tests.sh index 1b75d29f..31f20763 100755 --- a/.github/scripts/remote-run-firesim-scala-tests.sh +++ b/.github/scripts/remote-run-firesim-scala-tests.sh @@ -10,10 +10,6 @@ set -ex SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" source $SCRIPT_DIR/defaults.sh -export RISCV="$REMOTE_RISCV_DIR" -export LD_LIBRARY_PATH="$RISCV/lib" -export PATH="$RISCV/bin:$PATH" - # Directory locations for handling firesim-local installations of libelf/libdwarf # This would generally be handled by build-setup.sh/firesim-setup.sh REMOTE_FIRESIM_SYSROOT=$REMOTE_FIRESIM_DIR/lib-install @@ -30,15 +26,8 @@ cd $REMOTE_CHIPYARD_DIR make -C $REMOTE_CHIPYARD_DIR/tools/dromajo/dromajo-src/src -TOOLS_DIR=$REMOTE_RISCV_DIR - -LD_LIB_DIR=$REMOTE_FIRESIM_SYSROOT/lib:$REMOTE_RISCV_DIR/lib - # Run Firesim Scala Tests -export RISCV=$TOOLS_DIR -export LD_LIBRARY_PATH=$LD_LIB_DIR +export LD_LIBRARY_PATH=$REMOTE_FIRESIM_SYSROOT/lib:$LD_LIBRARY_PATH export FIRESIM_ENV_SOURCED=1; -export PATH=$REMOTE_VERILATOR_DIR/bin:$PATH -export VERILATOR_ROOT=$REMOTE_VERILATOR_DIR export COURSIER_CACHE=$REMOTE_WORK_DIR/.coursier-cache make -C $REMOTE_FIRESIM_DIR JAVA_OPTS="$REMOTE_JAVA_OPTS" SBT_OPTS="$REMOTE_SBT_OPTS" testOnly ${mapping[$1]} diff --git a/.github/scripts/run-tests.sh b/.github/scripts/run-tests.sh index bc344c1a..d97f05dc 100755 --- a/.github/scripts/run-tests.sh +++ b/.github/scripts/run-tests.sh @@ -9,10 +9,6 @@ set -ex SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )" source $SCRIPT_DIR/defaults.sh -export RISCV="$GITHUB_WORKSPACE/riscv-tools-install" -export LD_LIBRARY_PATH="$RISCV/lib" -export PATH="$RISCV/bin:$PATH" - DISABLE_SIM_PREREQ="BREAK_SIM_PREREQ=1" run_bmark () { @@ -52,15 +48,9 @@ case $1 in run_bmark ${mapping[$1]} ;; chipyard-hwacha) - export RISCV=$LOCAL_ESP_DIR - export LD_LIBRARY_PATH=$LOCAL_ESP_DIR/lib - export PATH=$RISCV/bin:$PATH make run-rv64uv-p-asm-tests -j$CI_MAKE_NPROC -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} ;; chipyard-gemmini) - export RISCV=$LOCAL_ESP_DIR - export LD_LIBRARY_PATH=$LOCAL_ESP_DIR/lib - export PATH=$RISCV/bin:$PATH GEMMINI_SOFTWARE_DIR=$LOCAL_SIM_DIR/../../generators/gemmini/software/gemmini-rocc-tests rm -rf $GEMMINI_SOFTWARE_DIR/riscv-tests cd $LOCAL_SIM_DIR @@ -69,9 +59,6 @@ case $1 in make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary-fast BINARY=$GEMMINI_SOFTWARE_DIR/build/bareMetalC/mvin_mvout-baremetal ;; chipyard-sha3) - export RISCV=$LOCAL_ESP_DIR - export LD_LIBRARY_PATH=$LOCAL_ESP_DIR/lib - export PATH=$RISCV/bin:$PATH (cd $LOCAL_CHIPYARD_DIR/generators/sha3/software && ./build.sh) make -C $LOCAL_SIM_DIR $DISABLE_SIM_PREREQ ${mapping[$1]} run-binary-fast BINARY=$LOCAL_CHIPYARD_DIR/generators/sha3/software/tests/bare/sha3-rocc.riscv ;; diff --git a/.github/workflows/chipyard-run-tests.yml b/.github/workflows/chipyard-run-tests.yml index d41efd65..ec2a8f52 100644 --- a/.github/workflows/chipyard-run-tests.yml +++ b/.github/workflows/chipyard-run-tests.yml @@ -7,13 +7,17 @@ on: - main - '1.[0-9]*.x' +defaults: + run: + shell: bash -leo pipefail {0} + env: - tools-cache-version: v14 - BUILDSERVER: ${{ secrets.BUILDSERVER }} - BUILDUSER: ${{ secrets.BUILDUSER }} - SERVER: ${{ secrets.BUILDUSER }}@${{ secrets.BUILDSERVER }} + tools-cache-version: v17 CI_DIR: ${{ secrets.BUILDDIR }} JVM_OPTS: -Xmx3200m # Customize the JVM maximum heap limit + conda-env-name-no-time: cy-${{ github.run_id }} + workflow-timestamp: ${{ github.event.pull_request.updated_at }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: cancel-prior-workflows: @@ -36,7 +40,9 @@ jobs: needs-rtl: ${{ steps.filter.outputs.all_count != steps.filter.outputs.skip-rtl_count }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround - uses: dorny/paths-filter@v2 id: filter with: @@ -62,7 +68,7 @@ jobs: if: needs.change-filters.outputs.needs-rtl == 'true' runs-on: ubuntu-latest container: - image: ucbbar/chipyard-ci-image:554b436 + image: ucbbar/chipyard-ci-image:59dad4bc options: --entrypoint /bin/bash steps: - name: Delete old checkout @@ -72,9 +78,15 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Check commits of each submodule - run: .github/scripts/check-commit.sh + run: | + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools + .github/scripts/check-commit.sh tutorial-setup-check: name: tutorial-setup-check @@ -82,7 +94,7 @@ jobs: if: needs.change-filters.outputs.needs-rtl == 'true' runs-on: ubuntu-latest container: - image: ucbbar/chipyard-ci-image:554b436 + image: ucbbar/chipyard-ci-image:59dad4bc options: --entrypoint /bin/bash steps: - name: Delete old checkout @@ -92,16 +104,22 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Check that the tutorial-setup patches apply - run: scripts/tutorial-setup.sh + run: | + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools + scripts/tutorial-setup.sh documentation-check: name: documentation-check needs: change-filters runs-on: ubuntu-latest container: - image: ucbbar/chipyard-ci-image:554b436 + image: ucbbar/chipyard-ci-image:59dad4bc options: --entrypoint /bin/bash steps: - name: Delete old checkout @@ -111,44 +129,26 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Check that documentation builds with no warnings/errors run: | - sudo apt-get update -y - sudo apt-get install -y python3-pip - sudo pip3 install -r docs/requirements.txt + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools make -C docs html - name: Show error log from sphinx if failed if: ${{ failure() }} run: cat /tmp/sphinx-err*.log - install-toolchains: - name: install-toolchains - needs: change-filters - if: needs.change-filters.outputs.needs-rtl == 'true' - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash - steps: - - name: Delete old checkout - run: | - ls -alh . - rm -rf ${{ github.workspace }}/* || true - rm -rf ${{ github.workspace }}/.* || true - ls -alh . - - name: Checkout - uses: actions/checkout@v2 - - name: Build RISC-V toolchains - uses: ./.github/actions/toolchain-build - build-extra-tests: name: build-extra-tests - needs: [change-filters, install-toolchains] + needs: [change-filters] if: needs.change-filters.outputs.needs-rtl == 'true' runs-on: ubuntu-latest container: - image: ucbbar/chipyard-ci-image:554b436 + image: ucbbar/chipyard-ci-image:59dad4bc options: --entrypoint /bin/bash steps: - name: Delete old checkout @@ -158,24 +158,28 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 - - name: Build RISC-V toolchains - uses: ./.github/actions/toolchain-build + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Generate keys id: genkey run: | - echo "::set-output name=extra-tests-cache-key::extra-tests-${{ github.ref }}-${{ github.sha }}" - - uses: actions/cache@v2 + echo "::set-output name=extra-tests-cache-key::extra-tests-${{ github.sha }}" + - uses: actions/cache@v3 id: build-extra-tools-cache with: path: extra-tests-install key: ${{ steps.genkey.outputs.extra-tests-cache-key }} restore-keys: ${{ steps.genkey.outputs.extra-tests-cache-key }} - name: Build extra tests - run: .github/scripts/build-extra-tests.sh + run: | + conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools + .github/scripts/build-extra-tests.sh - install-verilator-knight: - name: install-verilator-knight + create-conda-env-knight: + name: create-conda-env-knight needs: [change-filters, cancel-prior-workflows] if: needs.change-filters.outputs.needs-rtl == 'true' runs-on: knight @@ -187,12 +191,16 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 - - name: Build verilator on knight CI machine - run: .github/scripts/remote-install-verilator.sh + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Cleanup conda + uses: ./.github/actions/cleanup-conda + - name: Create conda env + uses: ./.github/actions/create-conda-env - install-verilator-ferry: - name: install-verilator-ferry + create-conda-env-ferry: + name: create-conda-env-ferry needs: [change-filters, cancel-prior-workflows] if: needs.change-filters.outputs.needs-rtl == 'true' runs-on: ferry @@ -204,16 +212,20 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 - - name: Build verilator on ferry CI machine - run: .github/scripts/remote-install-verilator.sh + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Cleanup conda + uses: ./.github/actions/cleanup-conda + - name: Create conda env + uses: ./.github/actions/create-conda-env # Sentinel job to simplify how we specify which that basic setup is complete # # When adding new prep jobs, please add them to `needs` below setup-complete: name: setup-complete - needs: [install-toolchains, install-verilator-knight, install-verilator-ferry, build-extra-tests] + needs: [create-conda-env-knight, create-conda-env-ferry, build-extra-tests] runs-on: ubuntu-latest steps: - name: Set up complete @@ -233,7 +245,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Build RTL on self-hosted uses: ./.github/actions/prepare-rtl with: @@ -251,7 +267,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Build RTL on self-hosted uses: ./.github/actions/prepare-rtl with: @@ -269,11 +289,16 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Build RTL on self-hosted uses: ./.github/actions/prepare-rtl with: group-key: "group-accels" + toolchain: "esp-tools" prepare-chipyard-tracegen: name: prepare-chipyard-tracegen @@ -287,7 +312,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Build RTL on self-hosted uses: ./.github/actions/prepare-rtl with: @@ -305,7 +334,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Build RTL on self-hosted uses: ./.github/actions/prepare-rtl with: @@ -323,7 +356,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Build RTL on self-hosted uses: ./.github/actions/prepare-rtl with: @@ -335,10 +372,7 @@ jobs: chipyard-rocket-run-tests: name: chipyard-rocket-run-tests needs: prepare-chipyard-cores - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -347,7 +381,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -357,10 +395,7 @@ jobs: chipyard-hetero-run-tests: name: chipyard-hetero-run-tests needs: prepare-chipyard-cores - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -369,7 +404,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -379,10 +418,7 @@ jobs: chipyard-boom-run-tests: name: chipyard-boom-run-tests needs: prepare-chipyard-cores - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -391,7 +427,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -401,10 +441,7 @@ jobs: chipyard-cva6-run-tests: name: chipyard-cva6-run-tests needs: prepare-chipyard-cores - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -413,7 +450,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -423,10 +464,7 @@ jobs: chipyard-ibex-run-tests: name: chipyard-ibex-run-tests needs: prepare-chipyard-cores - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -435,7 +473,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -445,10 +487,7 @@ jobs: chipyard-sodor-run-tests: name: chipyard-sodor-run-tests needs: prepare-chipyard-cores - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -457,7 +496,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -467,10 +510,7 @@ jobs: chipyard-fftgenerator-run-tests: name: chipyard-fftgenerator-run-tests needs: prepare-chipyard-accels - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -479,7 +519,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -489,10 +533,7 @@ jobs: chipyard-dmirocket-run-tests: name: chipyard-dmirocket-run-tests needs: prepare-chipyard-peripherals - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -501,7 +542,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -511,10 +556,7 @@ jobs: chipyard-spiflashwrite-run-tests: name: chipyard-spiflashwrite-run-tests needs: prepare-chipyard-peripherals - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -523,7 +565,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -533,10 +579,7 @@ jobs: chipyard-spiflashread-run-tests: name: chipyard-spiflashread-run-tests needs: prepare-chipyard-peripherals - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -545,7 +588,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -555,10 +602,7 @@ jobs: chipyard-lbwif-run-tests: name: chipyard-lbwif-run-tests needs: prepare-chipyard-peripherals - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -567,7 +611,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -577,10 +625,7 @@ jobs: chipyard-sha3-run-tests: name: chipyard-sha3-run-tests needs: prepare-chipyard-accels - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -589,20 +634,22 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: group-key: "group-accels" project-key: "chipyard-sha3" + toolchain: "esp-tools" chipyard-streaming-fir-run-tests: name: chipyard-streaming-fir-run-tests needs: prepare-chipyard-accels - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -611,7 +658,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -621,10 +672,7 @@ jobs: chipyard-streaming-passthrough-run-tests: name: chipyard-streaming-passthrough-run-tests needs: prepare-chipyard-accels - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -633,7 +681,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -643,10 +695,7 @@ jobs: chipyard-hwacha-run-tests: name: chipyard-hwacha-run-tests needs: prepare-chipyard-accels - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -655,20 +704,22 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: group-key: "group-accels" project-key: "chipyard-hwacha" + toolchain: "esp-tools" chipyard-gemmini-run-tests: name: chipyard-gemmini-run-tests needs: prepare-chipyard-accels - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -677,20 +728,22 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: group-key: "group-accels" project-key: "chipyard-gemmini" + toolchain: "esp-tools" chipyard-nvdla-run-tests: name: chipyard-nvdla-run-tests needs: prepare-chipyard-accels - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -699,7 +752,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -709,10 +766,7 @@ jobs: tracegen-boom-run-tests: name: tracegen-boom-run-tests needs: prepare-chipyard-tracegen - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -721,7 +775,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -731,10 +789,7 @@ jobs: tracegen-run-tests: name: tracegen-run-tests needs: prepare-chipyard-tracegen - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -743,7 +798,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -753,10 +812,7 @@ jobs: icenet-run-tests: name: icenet-run-tests needs: prepare-chipyard-other - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -765,7 +821,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: @@ -775,10 +835,7 @@ jobs: testchipip-run-tests: name: testchipip-run-tests needs: prepare-chipyard-other - runs-on: ubuntu-latest - container: - image: ucbbar/chipyard-ci-image:554b436 - options: --entrypoint /bin/bash + runs-on: self-hosted steps: - name: Delete old checkout run: | @@ -787,33 +844,40 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests uses: ./.github/actions/run-tests with: group-key: "group-other" project-key: "testchipip" -# TODO: Determine why elfutils is failing here -# firesim-run-tests: -# name: firesim-run-tests -# needs: setup-complete -# runs-on: self-hosted -# steps: -# - name: Delete old checkout -# run: | -# ls -alh . -# rm -rf ${{ github.workspace }}/* || true -# rm -rf ${{ github.workspace }}/.* || true -# ls -alh . -# - name: Checkout -# uses: actions/checkout@v2 -# - name: Run tests on self-hosted -# uses: ./.github/actions/run-tests -# with: -# group-key: "extra-tests" -# project-key: "firesim" -# run-script: "remote-run-firesim-scala-tests.sh" + firesim-run-tests: + name: firesim-run-tests + needs: setup-complete + runs-on: self-hosted + steps: + - name: Delete old checkout + run: | + ls -alh . + rm -rf ${{ github.workspace }}/* || true + rm -rf ${{ github.workspace }}/.* || true + ls -alh . + - name: Checkout + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env + - name: Run tests on self-hosted + uses: ./.github/actions/run-tests + with: + group-key: "extra-tests" + project-key: "firesim" + run-script: "remote-run-firesim-scala-tests.sh" fireboom-run-tests: name: fireboom-run-tests @@ -827,7 +891,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests on self-hosted uses: ./.github/actions/run-tests with: @@ -847,7 +915,11 @@ jobs: rm -rf ${{ github.workspace }}/.* || true ls -alh . - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - name: Create conda env + uses: ./.github/actions/create-conda-env - name: Run tests on self-hosted uses: ./.github/actions/run-tests with: diff --git a/.gitmodules b/.gitmodules index 11ca0d5a..01aa80c0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -28,40 +28,6 @@ [submodule "generators/block-inclusivecache-sifive"] path = generators/sifive-cache url = https://github.com/sifive/block-inclusivecache-sifive.git -[submodule "toolchains/riscv-tools/riscv-gnu-toolchain"] - path = toolchains/riscv-tools/riscv-gnu-toolchain - url = https://github.com/riscv/riscv-gnu-toolchain.git -[submodule "toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt"] - path = toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt - url = https://github.com/ucb-bar/chipyard-toolchain-prebuilt.git - shallow = true -[submodule "toolchains/riscv-tools/riscv-isa-sim"] - path = toolchains/riscv-tools/riscv-isa-sim - url = https://github.com/riscv/riscv-isa-sim.git -[submodule "toolchains/riscv-tools/riscv-pk"] - path = toolchains/riscv-tools/riscv-pk - url = https://github.com/riscv/riscv-pk.git -[submodule "toolchains/riscv-tools/riscv-tests"] - path = toolchains/riscv-tools/riscv-tests - url = https://github.com/riscv/riscv-tests.git -[submodule "toolchains/riscv-tools/riscv-openocd"] - path = toolchains/riscv-tools/riscv-openocd - url = https://github.com/riscv/riscv-openocd.git -[submodule "toolchains/esp-tools/riscv-gnu-toolchain"] - path = toolchains/esp-tools/riscv-gnu-toolchain - url = https://github.com/ucb-bar/esp-gnu-toolchain.git -[submodule "toolchains/esp-tools/riscv-isa-sim"] - path = toolchains/esp-tools/riscv-isa-sim - url = https://github.com/ucb-bar/esp-isa-sim.git -[submodule "toolchains/esp-tools/riscv-pk"] - path = toolchains/esp-tools/riscv-pk - url = https://github.com/riscv/riscv-pk.git -[submodule "toolchains/esp-tools/riscv-tests"] - path = toolchains/esp-tools/riscv-tests - url = https://github.com/ucb-bar/esp-tests.git -[submodule "toolchains/libgloss"] - path = toolchains/libgloss - url = https://github.com/ucb-bar/libgloss-htif.git [submodule "vlsi/hammer"] path = vlsi/hammer url = https://github.com/ucb-bar/hammer.git @@ -83,9 +49,6 @@ [submodule "vlsi/hammer-mentor-plugins"] path = vlsi/hammer-mentor-plugins url = https://github.com/ucb-bar/hammer-mentor-plugins.git -[submodule "toolchains/qemu"] - path = toolchains/qemu - url = https://github.com/qemu/qemu.git [submodule "tools/axe"] path = tools/axe url = https://github.com/CTSRD-CHERI/axe.git diff --git a/.readthedocs.yml b/.readthedocs.yml index 4c894485..fc58869b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,13 +3,12 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.6" + python: "mambaforge-4.10" formats: all sphinx: configuration: docs/conf.py -python: - install: - - requirements: docs/requirements.txt +conda: + environment: scripts/conda-requirements.yaml diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 7011f38e..ef36e1b0 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -2,59 +2,53 @@ # BUILD BASE FOR CI -FROM ubuntu:18.04 as base +FROM ubuntu:20.04 as base ARG CHIPYARD_HASH MAINTAINER https://groups.google.com/forum/#!forum/chipyard -SHELL ["/bin/bash", "-c"] +SHELL ["/bin/bash", "-c"] # Install dependencies for ubuntu-req.sh RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - curl \ - git \ - sudo \ - ca-certificates \ - keyboard-configuration \ - console-setup \ - bc \ - unzip + curl \ + wget \ + git \ + sudo \ + ca-certificates \ + keyboard-configuration \ + console-setup \ + bc \ + unzip WORKDIR /root # Install Chipyard and run ubuntu-req.sh to install necessary dependencies RUN git clone https://github.com/ucb-bar/chipyard.git && \ cd chipyard && \ - git checkout $CHIPYARD_HASH && \ - ./scripts/ubuntu-req.sh 1>/dev/null && \ - sudo rm -rf /var/lib/apt/lists/* + git checkout $CHIPYARD_HASH -# Update PATH for RISCV toolchain (note: hardcoded for CircleCI) -ENV RISCV="/root/riscv-tools-install" -ENV LD_LIBRARY_PATH="$RISCV/lib" -ENV PATH="$RISCV/bin:$PATH" +RUN ./chipyard/scripts/install-conda.sh # BUILD IMAGE WITH TOOLCHAINS # Use above build as base FROM base as base-with-tools +SHELL ["/bin/bash", "-cl"] + +RUN conda env create -f ./chipyard/scripts/conda-requirements.yaml + +# Install riscv-tools +RUN conda install -n chipyard -c ucb-bar ucb-bar-riscv-tools + +SHELL ["/opt/conda/bin/conda", "run", "-n", "chipyard", "/bin/bash", "-cl"] + # Init submodules RUN cd chipyard && \ export MAKEFLAGS=-"j $(nproc)" && \ - ./scripts/init-submodules-no-riscv-tools.sh 1>/dev/null - -# Install riscv-tools -RUN cd chipyard && \ - export MAKEFLAGS=-"j $(nproc)" && \ - ./scripts/build-toolchains.sh riscv-tools 1>/dev/null - -# Install esp-tools -RUN cd chipyard && \ - export MAKEFLAGS=-"j $(nproc)" && \ - ./scripts/build-toolchains.sh esp-tools 1>/dev/null - + ./scripts/init-submodules-no-riscv-tools.sh --skip-validate 1>/dev/null # Set up FireMarshal. Building and cleaning br-base.json builds the underlying # buildroot image (which takes a long time) but doesn't keep all the br-base @@ -64,9 +58,8 @@ RUN cd chipyard && \ cd software/firemarshal && \ ./init-submodules.sh && \ pip3 install -r python-requirements.txt && \ - marshal build br-base.json && \ - marshal clean br-base.json - + ./marshal build br-base.json && \ + ./marshal clean br-base.json # Run script to set environment variables on entry ENTRYPOINT ["chipyard/scripts/entrypoint.sh"] diff --git a/docs/Chipyard-Basics/Initial-Repo-Setup.rst b/docs/Chipyard-Basics/Initial-Repo-Setup.rst index 2638d852..9dc9dcdc 100644 --- a/docs/Chipyard-Basics/Initial-Repo-Setup.rst +++ b/docs/Chipyard-Basics/Initial-Repo-Setup.rst @@ -6,28 +6,41 @@ Requirements Chipyard is developed and tested on Linux-based systems. -.. Warning:: It is possible to use this on macOS or other BSD-based systems, although GNU tools will need to be installed; it is also recommended to install the RISC-V toolchain from ``brew``. +.. Warning:: It is possible to use this on macOS or other BSD-based systems, although GNU tools will need to be installed; + it is also recommended to install the RISC-V toolchain from ``brew``. .. Warning:: Working under Windows is not recommended. +Running on AWS EC2 with FireSim +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In CentOS-based platforms, we recommend installing the following dependencies: +If you plan on using Chipyard alongside FireSim, you should refer to the :fsim_doc:`FireSim documentation <>`. +Specifically, you should follow the :fsim_doc:`Initial Setup/Simulation ` +section of the docs up until :fsim_doc:`Setting up the FireSim Repo `. +At that point, instead of checking out FireSim you can checkout Chipyard by following :ref:`Chipyard-Basics/Initial-Repo-Setup:Setting up the Chipyard Repo`. -.. include:: /../scripts/centos-req.sh - :code: bash +Default Requirements Installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In Ubuntu/Debian-based platforms (Ubuntu), we recommend installing the following dependencies. -These dependencies were written based on Ubuntu 16.04 LTS and 18.04 LTS - If they don't work for you, you can try out the Docker image (:ref:`Chipyard-Basics/Initial-Repo-Setup:Pre-built Docker Image`) before manually installing or removing dependencies: +In Chipyard, we use the `Conda `__ package manager to help manage system dependencies. +Conda allows users to create an "environment" that holds system dependencies like ``make``, ``gcc``, etc. -.. include:: /../scripts/ubuntu-req.sh - :code: bash +First Chipyard requires there to be Conda installed on the system. +Please refer to the `Conda installation instructions `__ on how to install Conda. +Afterwards, verify that Conda is a sufficient version (we test on version 4.12.0/4.13.0). -.. Note:: When running on an Amazon Web Services EC2 FPGA-development instance (for FireSim), FireSim includes a machine setup script that will install all of the aforementioned dependencies (and some additional ones). +.. code-block:: shell + + conda --version # must be version 4.12.0 or higher + +After Conda is installed and is on your ``PATH``, we need to install a version of ``git`` to initially checkout the repository. +For this you can use the system package manager like ``yum`` or ``apt`` to install ``git``. +This ``git`` is only used to first checkout the repository, we will later install a newer version of ``git`` with Conda. Setting up the Chipyard Repo ------------------------------------------- -Start by fetching Chipyard's sources. Run: +Start by checkout out the proper Chipyard's version. Run: .. parsed-literal:: @@ -36,40 +49,90 @@ Start by fetching Chipyard's sources. Run: # checkout latest official chipyard release # note: this may not be the latest release if the documentation version != "stable" git checkout |version| + +If you are running Chipyard alongside FireSim on AWS EC2, you should skip the :ref:`Chipyard-Basics/Initial-Repo-Setup:Conda Environment Setup` section and instead jump to :ref:`Chipyard-Basics/Initial-Repo-Setup:Fetch Chipyard Sources`. + +Conda Environment Setup +~~~~~~~~~~~~~~~~~~~~~~~ + +.. Warning:: When running on an Amazon Web Services EC2 FPGA-development instance + (for FireSim), FireSim includes a similar machine setup script that will install all + of the aforementioned dependencies (and some additional ones) and will activate the + proper conda environment. Skip this section. + +Next run the following command to create Chipyard's Conda environment. + +.. code-block:: shell + + conda env create -f scripts/conda-requirements.yml + +By running the following command you should see a "chipyard" environment listed (the default environment is called "chipyard"). + +.. code-block:: shell + + conda env list + +.. Note:: Refer to FireSim's :fsim_doc:`Conda documentation ` on more information + on how to use Conda and some of its benefits. + +Next go ahead and activate the conda environment that was setup. + +.. code-block:: shell + + conda activate chipyard + +We recommend that you add this "activate" command your ``.bashrc`` (or other environment setup file). + +Fetch Chipyard Sources +~~~~~~~~~~~~~~~~~~~~~~ + +To fetch all Chipyard sources, run the following: + +.. code-block:: shell + ./scripts/init-submodules-no-riscv-tools.sh This will initialize and checkout all of the necessary git submodules. This will also validate that you are on a tagged branch, otherwise it will prompt for confirmation. When updating Chipyard to a new version, you will also want to rerun this script to update the submodules. -Using git directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior. +Using ``git`` directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior. -.. _build-toolchains: - -Building a Toolchain +Obtaining a Toolchain ------------------------ -The `toolchains` directory contains toolchains that include a cross-compiler toolchain, frontend server, and proxy kernel, which you will need in order to compile code to RISC-V instructions and run them on your design. -Currently there are two toolchains, one for normal RISC-V programs, and another for Hwacha (``esp-tools``). -For custom installations, Each tool within the toolchains contains individual installation procedures within its README file. -To get a basic installation (which is the only thing needed for most Chipyard use-cases), just the following steps are necessary. -This will take about 20-30 minutes. You can expedite the process by setting a ``make`` environment variable to use parallel cores: ``export MAKEFLAGS=-j8``. +Currently there are two toolchains, one for normal RISC-V programs called ``riscv-tools``, and another for Hwacha/Gemmini called ``esp-tools``. +To get a basic ``riscv-tools`` installation (which is the only thing needed for most Chipyard use-cases), just the following steps are necessary. .. code-block:: shell - ./scripts/build-toolchains.sh riscv-tools # for a normal risc-v toolchain + conda install -c ucb-bar riscv-tools # for a normal risc-v toolchain -.. Note:: If you are planning to use the Hwacha vector unit, or other RoCC-based accelerators, you should build the esp-tools toolchain by adding the ``esp-tools`` argument to the script above. - If you are running on an Amazon Web Services EC2 instance, intending to use FireSim, you can also use the ``--ec2fast`` flag for an expedited installation of a pre-compiled toolchain. +.. Note:: If you are planning to use the Hwacha vector unit, or other RoCC-based accelerators, you should obtain the ``esp-tools`` toolchain by adding the ``esp-tools`` argument to the command above. -Once the script is run, a ``env.sh`` file is emitted that sets the ``PATH``, ``RISCV``, and ``LD_LIBRARY_PATH`` environment variables. -You can put this in your ``.bashrc`` or equivalent environment setup file to get the proper variables, or directly include it in your current environment: +Once the command is run, the ``PATH``, ``RISCV``, and ``LD_LIBRARY_PATH`` environment variables will be set properly. + +.. Note:: If you are a power user and would like to build your own toolchain, you can refer to the https://github.com/ucb-bar/riscv-tools-feedstock and https://github.com/ucb-bar/esp-tools-feedstock repositories (submoduled in the ``toolchains`` directory) on how to build a toolchain yourself. + +.. Note:: If can deactivate/activate a toolchain (but keep it installed) by running ``source $CONDA_PREFIX/etc/conda/deactivate.d/deactivate-${PKG_NAME}.sh`` or ``$CONDA_PREFIX/etc/conda/activate.d/activate-${PKG_NAME}.sh`` (``PKG_NAME`` for example is ``ucb-bar-riscv-tools``). This will modify the aforementioned 3 environment variables. + +Sourcing ``env.sh`` +------------------- + +Once setup is complete, an emitted ``env.sh`` file should exist in the top-level repository. +This sets up necessary environment variables needed for future Chipyard steps (needed for the ``make`` system to work properly). +You can source this file in your ``.bashrc`` or equivalent environment setup file to get the proper variables, or directly include it in your current environment: .. code-block:: shell source ./env.sh -These variables need to be set for the ``make`` system to work properly. +.. Warning:: This ``env.sh`` file should always be sourced before running any ``make`` commands. + +.. Warning:: ``env.sh`` files are generated per-Chipyard repository. + In a multi-Chipyard repository setup, it is possible to source multiple ``env.sh`` files (in any order). + However, it is recommended that the final ``env.sh`` file sourced is the ``env.sh`` located in the + Chipyard repo that you expect to run ``make`` commands in. Pre-built Docker Image ------------------------------------------- @@ -115,20 +178,22 @@ In order to upgrade between Chipyard versions, we recommend using a fresh clone Chipyard is a complex framework that depends on a mix of build systems and scripts. Specifically, it relies on git submodules, on sbt build files, and on custom written bash scripts and generated files. -For this reason, upgrading between Chipyard versions is **not** as trivial as just running ``git submodule update -recursive``. This will result in recursive cloning of large submodules that are not necessarily used within your specific Chipyard environments. Furthermore, it will not resolve the status of stale state generated files which may not be compatible between release versions. +For this reason, upgrading between Chipyard versions is **not** as trivial as just running ``git submodule update --recursive``. This will result in recursive cloning of large submodules that are not necessarily used within your specific Chipyard environments. +Furthermore, it will not resolve the status of stale state generated files which may not be compatible between release versions. -If you are an advanced git user, an alternative approach to a fresh repository clone may be to run ``git clean -dfx``, and then run the standard Chipyard setup sequence. This approach is dangerous, and **not-recommended** for users who are not deeply familiar with git, since it "blows up" the repository state and removes all untracked and modified files without warning. Hence, if you were working on custom un-committed changes, you would lose them. +If you are an advanced git user, an alternative approach to a fresh repository clone may be to run ``git clean -dfx``, and then run the standard Chipyard setup sequence. +This approach is dangerous, and **not-recommended** for users who are not deeply familiar with git, since it "blows up" the repository state and removes all untracked and modified files without warning. +Hence, if you were working on custom un-committed changes, you would lose them. If you would still like to try to perform an in-place manual version upgrade (**not-recommended**), we recommend at least trying to resolve stale state in the following areas: * Delete stale ``target`` directories generated by sbt. -* Delete jar collateral generated by FIRRTL (``lib/firrtl.jar``) - * Re-generate generated scripts and source files (for example, ``env.sh``) * Re-generating/deleting target software state (Linux kernel binaries, Linux images) within FireMarshal -This is by no means a comprehensive list of potential stale state within Chipyard. Hence, as mentioned earlier, the recommended method for a Chipyard version upgrade is a fresh clone (or a merge, and then a fresh clone). +This is by no means a comprehensive list of potential stale state within Chipyard. +Hence, as mentioned earlier, the recommended method for a Chipyard version upgrade is a fresh clone (or a merge, and then a fresh clone). diff --git a/docs/Customization/Boot-Process.rst b/docs/Customization/Boot-Process.rst index f52a7afc..a84c4e99 100644 --- a/docs/Customization/Boot-Process.rst +++ b/docs/Customization/Boot-Process.rst @@ -74,6 +74,6 @@ mode, thus starting userspace execution. The easiest way to build a BBL image that boots Linux is to use the FireMarshal tool that lives in the `firesim-software `_ repository. Directions on how to use FireMarshal can be found in the -`FireSim documentation `_. +:fsim_doc:`FireSim documentation `. Using FireMarshal, you can add custom kernel configurations and userspace software to your workload. diff --git a/docs/Simulation/FPGA-Accelerated-Simulation.rst b/docs/Simulation/FPGA-Accelerated-Simulation.rst index 86f0eb8b..1d644155 100644 --- a/docs/Simulation/FPGA-Accelerated-Simulation.rst +++ b/docs/Simulation/FPGA-Accelerated-Simulation.rst @@ -40,13 +40,12 @@ Finally, source the following environment at the root of the firesim directory: .. Note:: Every time you want to use FireSim with a fresh shell, you must source this ``sourceme-f1-manager.sh`` At this point you're ready to use FireSim with Chipyard. If you're not already -familiar with FireSim, please return to the `FireSim Docs -`__, +familiar with FireSim, please return to the :fsim_doc:`FireSim Docs `, and proceed with the rest of the tutorial. Running your Design in FireSim ------------------------------ -Converting a Chipyard config (one in ``chipyard/src/main/scala`` to run in FireSim is simple, and can be done either through the traditional configuration system or through FireSim's build-recipes scheme. +Converting a Chipyard config (one in ``chipyard/src/main/scala`` to run in FireSim is simple, and can be done either through the traditional configuration system or through FireSim's build-recipes scheme. A FireSim simulation requires 3 additional config fragments: diff --git a/docs/conf.py b/docs/conf.py index acf781f5..7bd8dcb2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,14 +32,17 @@ import subprocess # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.autodoc', +extensions = [ + 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', - 'sphinx.ext.autosectionlabel'] + 'sphinx.ext.autosectionlabel', + 'sphinx.ext.extlinks', +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -67,6 +70,8 @@ if on_rtd: for item, value in os.environ.items(): print("[READTHEDOCS] {} = {}".format(item, value)) +# default to latest for non rtd builds (this will be overridden on rtd) +rtd_version = "latest" if on_rtd: rtd_version = os.environ.get("READTHEDOCS_VERSION") if rtd_version == "latest": @@ -93,7 +98,7 @@ release = version # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -227,3 +232,8 @@ intersphinx_mapping = {'python' : ('https://docs.python.org/', None), # resolve label conflict between documents autosectionlabel_prefix_document = True + +# shorten FireSim references +extlinks = { + 'fsim_doc' : ('https://docs.fires.im/en/' + rtd_version + '/%s', 'fsim_doc %s') +} diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index ad19f37d..00000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -Sphinx==1.8.5 -Pygments==2.7.4 -sphinx-autobuild -sphinx_rtd_theme==0.2.5b1 -docutils==0.16 diff --git a/scripts/build-toolchains.sh b/scripts/build-toolchains.sh deleted file mode 100755 index 0d472464..00000000 --- a/scripts/build-toolchains.sh +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env bash - -#this script is based on the firesim build toolchains script - -# exit script if any command fails -set -e -set -o pipefail - -# On macOS, use GNU readlink from 'coreutils' package in Homebrew/MacPorts -if [ "$(uname -s)" = "Darwin" ] ; then - READLINK=greadlink -else - READLINK=readlink -fi - -# If BASH_SOURCE is undefined, we may be running under zsh, in that case -# provide a zsh-compatible alternative -DIR="$(dirname "$($READLINK -f "${BASH_SOURCE[0]:-${(%):-%x}}")")" -CHIPYARD_DIR="$(dirname "$DIR")" - -# Allow user to override MAKE -[ -n "${MAKE:+x}" ] || MAKE=$(command -v gnumake || command -v gmake || command -v make) -readonly MAKE - -usage() { - echo "usage: ${0} [OPTIONS] [riscv-tools | esp-tools | ec2fast]" - echo "" - echo "Installation Types" - echo " riscv-tools: if set, builds the riscv toolchain (this is also the default)" - echo " esp-tools: if set, builds esp-tools toolchain used for the hwacha vector accelerator" - echo " ec2fast: if set, pulls in a pre-compiled RISC-V toolchain for an EC2 manager instance" - echo "" - echo "Options" - echo " --prefix PREFIX : Install destination. If unset, defaults to $(pwd)/riscv-tools-install" - echo " or $(pwd)/esp-tools-install" - echo " --ignore-qemu : Ignore installing QEMU" - echo " --clean-after-install : Run make clean in calls to module_make and module_build" - echo " --arch -a : Architecture (e.g., rv64gc)" - echo " --help -h : Display this message" - exit "$1" -} - -error() { - echo "${0##*/}: ${1}" >&2 -} -die() { - error "$1" - exit "${2:--1}" -} - -TOOLCHAIN="riscv-tools" -EC2FASTINSTALL="false" -IGNOREQEMU="" -CLEANAFTERINSTALL="" -RISCV="" -ARCH="" - -# getopts does not support long options, and is inflexible -while [ "$1" != "" ]; -do - case $1 in - -h | --help | help ) - usage 3 ;; - -p | --prefix ) - shift - RISCV=$(realpath $1) ;; - --ignore-qemu ) - IGNOREQEMU="true" ;; - -a | --arch ) - shift - ARCH=$1 ;; - --clean-after-install ) - CLEANAFTERINSTALL="true" ;; - riscv-tools | esp-tools) - TOOLCHAIN=$1 ;; - ec2fast ) - EC2FASTINSTALL="true" ;; - * ) - error "invalid option $1" - usage 1 ;; - esac - shift -done - -if [ -z "$RISCV" ] ; then - INSTALL_DIR="$TOOLCHAIN-install" - RISCV="$(pwd)/$INSTALL_DIR" -fi - -if [ -z "$ARCH" ] ; then - XLEN=64 -elif [[ "$ARCH" =~ ^rv(32|64)((i?m?a?f?d?|g?)c?)$ ]]; then - XLEN=${BASH_REMATCH[1]} -else - error "invalid arch $ARCH" - usage 1 -fi - -echo "Installing toolchain to $RISCV" - -# install risc-v tools -export RISCV="$RISCV" - -cd "${CHIPYARD_DIR}" - -SRCDIR="$(pwd)/toolchains/${TOOLCHAIN}" -[ -d "${SRCDIR}" ] || die "unsupported toolchain: ${TOOLCHAIN}" -. ./scripts/build-util.sh - - -if [ "${EC2FASTINSTALL}" = true ] ; then - [ "${TOOLCHAIN}" = 'riscv-tools' ] || - die "unsupported precompiled toolchain: ${TOOLCHAIN}" - - echo '=> Fetching pre-built toolchain' - module=toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt - git config --unset submodule."${module}".update || : - git submodule update --init --depth 1 "${module}" - - echo '==> Verifying toolchain version hash' - # Find commit hash without initializing the submodule - hashsrc="$(git ls-tree -d HEAD "${SRCDIR}/riscv-gnu-toolchain" | { - unset IFS && read -r _ type obj _ && - test -n "${obj}" && test "${type}" = 'commit' && echo "${obj}" - }; )" || - die 'failed to obtain riscv-gnu-toolchain submodule hash' "$?" - - read -r hashbin < "${module}/HASH" || - die 'failed to obtain riscv-gnu-toolchain-prebuilt hash' "$?" - - echo "==> ${hashsrc}" - [ "${hashsrc}" = "${hashbin}" ] || - die "pre-built version mismatch: ${hashbin}" - - echo '==> Installing pre-built toolchain' - "${MAKE}" -C "${module}" DESTDIR="${RISCV}" install - git submodule deinit "${module}" || : - -else - MAKE_VER=$("${MAKE}" --version) || true - case ${MAKE_VER} in - 'GNU Make '[4-9]\.*) - ;; - 'GNU Make '[1-9][0-9]) - ;; - *) - die 'obsolete make version; need GNU make 4.x or later' - ;; - esac - - module_prepare riscv-gnu-toolchain qemu - module_build riscv-gnu-toolchain --prefix="${RISCV}" --with-cmodel=medany ${ARCH:+--with-arch=${ARCH}} - echo '==> Building GNU/Linux toolchain' - module_make riscv-gnu-toolchain linux -fi - -# disable boost explicitly for https://github.com/riscv-software-src/riscv-isa-sim/issues/834 -# since we don't have it in our requirements -module_all riscv-isa-sim --prefix="${RISCV}" --with-boost=no --with-boost-asio=no --with-boost-regex=no -# build static libfesvr library for linking into firesim driver (or others) -echo '==> Installing libfesvr static library' -module_make riscv-isa-sim libfesvr.a -cp -p "${SRCDIR}/riscv-isa-sim/build/libfesvr.a" "${RISCV}/lib/" - -CC= CXX= module_all riscv-pk --prefix="${RISCV}" --host=riscv${XLEN}-unknown-elf -module_all riscv-tests --prefix="${RISCV}/riscv${XLEN}-unknown-elf" --with-xlen=${XLEN} - -# Common tools (not in any particular toolchain dir) - -CC= CXX= SRCDIR="$(pwd)/toolchains" module_all libgloss --prefix="${RISCV}/riscv${XLEN}-unknown-elf" --host=riscv${XLEN}-unknown-elf - -if [ -z "$IGNOREQEMU" ] ; then - echo "=> Starting qemu build" - dir="$(pwd)/toolchains/qemu" - echo "==> Initializing qemu submodule" - #since we don't want to use the global config we init passing rewrite config in to the command - git -c url.https://github.com/qemu.insteadOf=https://git.qemu.org/git submodule update --init --recursive "$dir" - echo "==> Applying url-rewriting to avoid git.qemu.org" - # and once the clones exist, we recurse through them and set the rewrite - # in the local config so that any further commands by the user have the rewrite. uggh. git, why you so ugly? - git -C "$dir" config --local url.https://github.com/qemu.insteadOf https://git.qemu.org/git - git -C "$dir" submodule foreach --recursive 'git config --local url.https://github.com/qemu.insteadOf https://git.qemu.org/git' - - # check to see whether the rewrite rules are needed any more - # If you find git.qemu.org in any .gitmodules file below qemu, you still need them - # the /dev/null redirection in the submodule grepping is to quiet non-existance of further .gitmodules - ! grep -q 'git\.qemu\.org' "$dir/.gitmodules" && \ - git -C "$dir" submodule foreach --quiet --recursive '! grep -q "git\.qemu\.org" .gitmodules 2>/dev/null' && \ - echo "==> PLEASE REMOVE qemu URL-REWRITING from scripts/build-toolchains.sh. It is no longer needed!" && exit 1 - - ( - # newer version of BFD-based ld has made '-no-pie' an error because it renamed to '--no-pie' - # meanwhile, ld.gold will still accept '-no-pie' - # QEMU 5.0 still uses '-no-pie' in it's linker options - - # default LD to ld if it isn't set - if ( set +o pipefail; ${LD:-ld} -no-pie |& grep 'did you mean --no-pie' >/dev/null); then - echo "==> LD doesn't like '-no-pie'" - # LD has the problem, look for ld.gold - if type ld.gold >&/dev/null; then - echo "==> Using ld.gold to link QEMU" - export LD=ld.gold - fi - fi - - # now actually do the build - SRCDIR="$(pwd)/toolchains" module_build qemu --prefix="${RISCV}" --target-list=riscv${XLEN}-softmmu --disable-werror - ) -fi - -# make Dromajo -git submodule update --init $CHIPYARD_DIR/tools/dromajo/dromajo-src -make -C $CHIPYARD_DIR/tools/dromajo/dromajo-src/src - -# create specific env.sh -cat > "$CHIPYARD_DIR/env-$TOOLCHAIN.sh" <> env.sh -echo "source $(printf '%q' "$CHIPYARD_DIR/env-$TOOLCHAIN.sh")" >> env.sh -echo "Toolchain Build Complete!" diff --git a/scripts/centos-req.sh b/scripts/centos-req.sh deleted file mode 100755 index 8c7ac3a0..00000000 --- a/scripts/centos-req.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -ex - -sudo yum groupinstall -y "Development tools" -sudo yum install -y gmp-devel mpfr-devel libmpc-devel zlib-devel vim git java java-devel - -# Install SBT https://www.scala-sbt.org/release/docs/Installing-sbt-on-Linux.html#Red+Hat+Enterprise+Linux+and+other+RPM-based+distributions -# sudo rm -f /etc/yum.repos.d/bintray-rpm.repo -# Use rm above if sbt installed from bintray before. -curl -L https://www.scala-sbt.org/sbt-rpm.repo > sbt-rpm.repo -sudo mv sbt-rpm.repo /etc/yum.repos.d/ - -sudo yum install -y sbt texinfo gengetopt -sudo yum install -y expat-devel libusb1-devel ncurses-devel cmake "perl(ExtUtils::MakeMaker)" -# deps for poky -sudo yum install -y python38 patch diffstat texi2html texinfo subversion chrpath git wget -# deps for qemu -sudo yum install -y gtk3-devel -# deps for firemarshal -sudo yum install -y python38-pip python38-devel rsync libguestfs-tools makeinfo expat ctags -# Install GNU make 4.x (needed to cross-compile glibc 2.28+) -sudo yum install -y centos-release-scl -sudo yum install -y devtoolset-8-make -# install DTC -sudo yum install -y dtc -sudo yum install -y python - -# install verilator -git clone http://git.veripool.org/git/verilator -cd verilator -git checkout v4.034 -autoconf && ./configure && make -j$(nproc) && sudo make install diff --git a/scripts/check-tracegen.sh b/scripts/check-tracegen.sh index 947e7312..e2974fb1 100755 --- a/scripts/check-tracegen.sh +++ b/scripts/check-tracegen.sh @@ -7,17 +7,24 @@ AXE_DIR=$(realpath ${SCRIPT_DIR}/../tools/axe) ROCKET_DIR=$(realpath ${SCRIPT_DIR}/../generators/rocket-chip) TO_AXE=${ROCKET_DIR}/scripts/toaxe.py +TO_AXE_PY3=/tmp/toaxe.py AXE=${AXE_DIR}/src/axe AXE_SHRINK=${AXE_DIR}/src/axe-shrink.py +AXE_SHRINK_PY3=/tmp/axe-shrink.py + +# TODO: convert scripts to py3 in src +2to3 $TO_AXE -o /tmp -n -w +sed -i '30d' $TO_AXE_PY3 # remove import sets +2to3 $AXE_SHRINK -o /tmp -n -w PATH=$PATH:${AXE_DIR}/src grep '.*:.*#.*@' $1 > /tmp/clean-trace.txt -python2 "$TO_AXE" /tmp/clean-trace.txt > /tmp/trace.axe +python "$TO_AXE_PY3" /tmp/clean-trace.txt > /tmp/trace.axe result=$("$AXE" check wmo /tmp/trace.axe) if [ "$result" != OK ]; then - "$AXE_SHRINK" wmo /tmp/trace.axe + "$AXE_SHRINK_PY3" wmo /tmp/trace.axe else echo OK fi diff --git a/scripts/conda-requirements.yaml b/scripts/conda-requirements.yaml new file mode 100644 index 00000000..d3d7bd31 --- /dev/null +++ b/scripts/conda-requirements.yaml @@ -0,0 +1,57 @@ +name: chipyard +channels: + - conda-forge + - ucb-bar +dependencies: + # firemarshal deps + - python=3.9 + - rsync + - psutil + - doit=0.35.0 + - gitpython + - humanfriendly + - e2fsprogs + - ctags + - bison + - flex + - expat + # current version of buildroot won't build with make 4.3 https://github.com/firesim/FireMarshal/issues/236 + - make!=4.3 + - pyyaml + - unzip + - readline + - coreutils + - lzop + - qemu # from ucb-bar + + # doc requirements + - sphinx + - pygments + - sphinx-autobuild + - sphinx_rtd_theme + - docutils + + # misc. c/c++ compilers + related + - gcc==10.* # pinned for libelf/libdwarf builds + - gxx==10.* # pinned for libelf/libdwarf builds + - conda-gcc-specs + - binutils + + # rocket-chip deps + - sbt + - openjdk + - dtc + - verilator==4.034 + + # chipyard deps + - dromajo # from ucb-bar + + # other misc. deps + - ca-certificates + - vim + - gengetopt + - cmake + - git + - wget + - sed + - autoconf diff --git a/scripts/init-submodules-no-riscv-tools-nolog.sh b/scripts/init-submodules-no-riscv-tools-nolog.sh index e2379ecc..a2a04427 100755 --- a/scripts/init-submodules-no-riscv-tools-nolog.sh +++ b/scripts/init-submodules-no-riscv-tools-nolog.sh @@ -76,7 +76,6 @@ cd "$CHIPYARD_DIR" ( # Blocklist of submodules to initially skip: - # - Toolchain submodules # - Generators with huge submodules (e.g., linux sources) # - FireSim until explicitly requested # - Hammer tool plugins @@ -84,9 +83,6 @@ cd "$CHIPYARD_DIR" # Call the given subcommand (shell function) on each submodule # path to temporarily exclude during the recursive update for name in \ - toolchains/*-tools/*/ \ - toolchains/libgloss \ - toolchains/qemu \ generators/sha3 \ generators/gemmini \ sims/firesim \ @@ -136,7 +132,12 @@ if [ ! -f ./software/firemarshal/marshal-config.yaml ]; then echo "firesim-dir: '../../sims/firesim/'" > ./software/firemarshal/marshal-config.yaml fi -echo "# line auto-generated by init-submodules-no-riscv-tools.sh" >> env.sh -echo '__DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]:-${(%):-%x}}")")"' >> env.sh -echo "PATH=\$__DIR/bin:\$PATH" >> env.sh -echo "PATH=\$__DIR/software/firemarshal:\$PATH" >> env.sh +cat << EOT >> env.sh +# line auto-generated by init-submodules-no-riscv-tools.sh +__DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]:-${(%):-%x}}")")" +PATH=\$__DIR/bin:\$PATH +PATH=\$__DIR/software/firemarshal:\$PATH +if [ -z \${CONDA_DEFAULT_ENV+x} ]; then + echo "WARNING: No conda environment detected. Did you activate the conda environment (e.x. 'conda activate chipyard')?" +fi +EOT diff --git a/scripts/install-conda.sh b/scripts/install-conda.sh new file mode 100755 index 00000000..ab460f0c --- /dev/null +++ b/scripts/install-conda.sh @@ -0,0 +1,178 @@ +#!/bin/bash + +CONDA_INSTALL_PREFIX=/opt/conda +CONDA_INSTALLER_VERSION=4.12.0-0 +CONDA_INSTALLER="https://github.com/conda-forge/miniforge/releases/download/${CONDA_INSTALLER_VERSION}/Miniforge3-${CONDA_INSTALLER_VERSION}-Linux-x86_64.sh" +CONDA_CMD="conda" # some installers install mamba or micromamba +CONDA_ENV_NAME="firesim" + +DRY_RUN_OPTION="" +DRY_RUN_ECHO=() +REINSTALL_CONDA=0 + +usage() +{ + echo "Usage: $0 [options]" + echo + echo "Options:" + echo "[--help] List this help" + echo "[--prefix ] Install prefix for conda. Defaults to /opt/conda." + echo " If /bin/conda already exists, it will be used and install is skipped." + echo "[--dry-run] Pass-through to all conda commands and only print other commands." + echo " NOTE: --dry-run will still install conda to --prefix" + echo "[--reinstall-conda] Repairs a broken base environment by reinstalling." + echo " NOTE: will only reinstall conda and exit" + echo + echo "Examples:" + echo " % $0" + echo " Install into default system-wide prefix (using sudo if needed) and add install to system-wide /etc/profile.d" + echo " % $0 --prefix ~/conda" + echo " Install into $HOME/conda" +} + + +while [ $# -gt 0 ]; do + case "$1" in + --help) + usage + exit 1 + ;; + --prefix) + shift + CONDA_INSTALL_PREFIX="$1" + shift + ;; + --dry-run) + shift + DRY_RUN_OPTION="--dry-run" + DRY_RUN_ECHO=(echo "Would Run:") + ;; + --reinstall-conda) + shift + REINSTALL_CONDA=1 + ;; + *) + echo "Invalid Argument: $1" + usage + exit 1 + ;; + esac +done + +if [[ $REINSTALL_CONDA -eq 1 && -n "$DRY_RUN_OPTION" ]]; then + echo "::ERROR:: --dry-run and --reinstall-conda are mutually exclusive. Pick one or the other." +fi + +set -ex +set -o pipefail + +# uname options are not portable so do what https://www.gnu.org/software/coreutils/faq/coreutils-faq.html#uname-is-system-specific +# suggests and iteratively probe the system type +if ! type uname >&/dev/null; then + echo "::ERROR:: need 'uname' command available to determine if we support this sytem" + exit 1 +fi + +if [[ "$(uname)" != "Linux" ]]; then + echo "::ERROR:: $0 only supports 'Linux' not '$(uname)'" + exit 1 +fi + +if [[ "$(uname -mo)" != "x86_64 GNU/Linux" ]]; then + echo "::ERROR:: $0 only supports 'x86_64 GNU/Linux' not '$(uname -io)'" + exit 1 +fi + +if [[ ! -r /etc/os-release ]]; then + echo "::ERROR:: $0 depends on /etc/os-release for distro-specific setup and it doesn't exist here" + exit 1 +fi + +OS_FLAVOR=$(grep '^ID=' /etc/os-release | awk -F= '{print $2}' | tr -d '"') +OS_VERSION=$(grep '^VERSION_ID=' /etc/os-release | awk -F= '{print $2}' | tr -d '"') + +# platform-specific setup +case "$OS_FLAVOR" in + ubuntu) + ;; + centos) + ;; + *) + echo "::ERROR:: Unknown OS flavor '$OS_FLAVOR'. Unable to do platform-specific setup." + exit 1 + ;; +esac + + +# everything else is platform-agnostic and could easily be expanded to Windows and/or OSX + +SUDO="" +prefix_parent=$(dirname "$CONDA_INSTALL_PREFIX") +if [[ ! -e "$prefix_parent" ]]; then + mkdir -p "$prefix_parent" || SUDO=sudo +elif [[ ! -w "$prefix_parent" ]]; then + SUDO=sudo +fi + +if [[ -n "$SUDO" ]]; then + echo "::INFO:: using 'sudo' to install conda" + # ensure files are read-execute for everyone + umask 022 +fi + +if [[ -n "$SUDO" || "$(id -u)" == 0 ]]; then + INSTALL_TYPE=system +else + INSTALL_TYPE=user +fi + +# to enable use of sudo and avoid modifying 'secure_path' in /etc/sudoers, we specify the full path to conda +CONDA_EXE="${CONDA_INSTALL_PREFIX}/bin/$CONDA_CMD" + +if [[ -x "$CONDA_EXE" && $REINSTALL_CONDA -eq 0 ]]; then + echo "::INFO:: '$CONDA_EXE' already exists, skipping conda install" +else + wget -O install_conda.sh "$CONDA_INSTALLER" || curl -fsSLo install_conda.sh "$CONDA_INSTALLER" + if [[ $REINSTALL_CONDA -eq 1 ]]; then + conda_install_extra="-u" + echo "::INFO:: RE-installing conda to '$CONDA_INSTALL_PREFIX'" + else + conda_install_extra="" + echo "::INFO:: installing conda to '$CONDA_INSTALL_PREFIX'" + fi + # -b for non-interactive install + $SUDO bash ./install_conda.sh -b -p "$CONDA_INSTALL_PREFIX" $conda_install_extra + rm ./install_conda.sh + + # see https://conda-forge.org/docs/user/tipsandtricks.html#multiple-channels + # for more information on strict channel_priority + "${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set channel_priority strict + # By default, don't mess with people's PS1, I personally find it annoying + "${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set changeps1 false + # don't automatically activate the 'base' environment when intializing shells + "${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set auto_activate_base false + # don't automatically update conda to avoid https://github.com/conda-forge/conda-libmamba-solver-feedstock/issues/2 + "${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set auto_update_conda false + + # conda-build is a special case and must always be installed into the base environment + $SUDO "$CONDA_EXE" install $DRY_RUN_OPTION -y -n base conda-build + + # conda-libmamba-solver is a special case and must always be installed into the base environment + # see https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community + $SUDO "$CONDA_EXE" install $DRY_RUN_OPTION -y -n base conda-libmamba-solver + # Use the fast solver by default + "${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set experimental_solver libmamba + + conda_init_extra_args=() + if [[ "$INSTALL_TYPE" == system ]]; then + # if we're installing into a root-owned directory using sudo, or we're already root + # initialize conda in the system-wide rcfiles + conda_init_extra_args=(--no-user --system) + fi + $SUDO "${CONDA_EXE}" init $DRY_RUN_OPTION "${conda_init_extra_args[@]}" bash + + if [[ $REINSTALL_CONDA -eq 1 ]]; then + echo "::INFO:: Done reinstalling conda. Exiting" + exit 0 + fi +fi diff --git a/scripts/ubuntu-req.sh b/scripts/ubuntu-req.sh deleted file mode 100755 index ff5fe3ad..00000000 --- a/scripts/ubuntu-req.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -set -ex - -sudo apt-get install -y build-essential bison flex software-properties-common curl -sudo apt-get install -y libgmp-dev libmpfr-dev libmpc-dev zlib1g-dev vim default-jdk default-jre -# install sbt: https://www.scala-sbt.org/release/docs/Installing-sbt-on-Linux.html#Ubuntu+and+other+Debian-based+distributions -echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list -curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo apt-key add -sudo apt-get update -sudo apt-get install -y sbt -sudo apt-get install -y texinfo gengetopt -sudo apt-get install -y libexpat1-dev libusb-dev libncurses5-dev cmake -# deps for poky -sudo apt-get install -y python3.8 patch diffstat texi2html texinfo subversion chrpath wget -# deps for qemu -sudo apt-get install -y libgtk-3-dev gettext -# deps for firemarshal -sudo apt-get install -y python3-pip python3.8-dev rsync libguestfs-tools expat ctags -# install DTC -sudo apt-get install -y device-tree-compiler -sudo apt-get install -y python -# install git >= 2.17 -sudo add-apt-repository ppa:git-core/ppa -y -sudo apt-get update -sudo apt-get install git -y - -# install verilator -sudo apt-get install -y autoconf -git clone http://git.veripool.org/git/verilator -cd verilator -git checkout v4.034 -autoconf && ./configure && make -j$(nproc) && sudo make install diff --git a/toolchains/esp-tools/riscv-gnu-toolchain b/toolchains/esp-tools/riscv-gnu-toolchain deleted file mode 160000 index 9f532293..00000000 --- a/toolchains/esp-tools/riscv-gnu-toolchain +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9f532293985d08b0c176d96c7b650e5f433780e1 diff --git a/toolchains/esp-tools/riscv-isa-sim b/toolchains/esp-tools/riscv-isa-sim deleted file mode 160000 index 34741e07..00000000 --- a/toolchains/esp-tools/riscv-isa-sim +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 34741e07bc6b56f1762ce579537948d58e28cd5a diff --git a/toolchains/esp-tools/riscv-pk b/toolchains/esp-tools/riscv-pk deleted file mode 160000 index e8e6b3aa..00000000 --- a/toolchains/esp-tools/riscv-pk +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e8e6b3aaee44d43b48164fbd377864c3a682dbd3 diff --git a/toolchains/esp-tools/riscv-tests b/toolchains/esp-tools/riscv-tests deleted file mode 160000 index 7c3dfd32..00000000 --- a/toolchains/esp-tools/riscv-tests +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7c3dfd3205fa90104db06346e797a8eeea149207 diff --git a/toolchains/libgloss b/toolchains/libgloss deleted file mode 160000 index 04b24976..00000000 --- a/toolchains/libgloss +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 04b249764b1bc0b784b9f7ff422084f4b53a07ef diff --git a/toolchains/qemu b/toolchains/qemu deleted file mode 160000 index fdd76fec..00000000 --- a/toolchains/qemu +++ /dev/null @@ -1 +0,0 @@ -Subproject commit fdd76fecdde1ad444ff4deb7f1c4f7e4a1ef97d6 diff --git a/toolchains/riscv-tools/riscv-gnu-toolchain b/toolchains/riscv-tools/riscv-gnu-toolchain deleted file mode 160000 index 2855d823..00000000 --- a/toolchains/riscv-tools/riscv-gnu-toolchain +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2855d823a6e93d50af604264b02ced951e80de67 diff --git a/toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt b/toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt deleted file mode 160000 index 918be237..00000000 --- a/toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 918be2371723257ed6590c77d5996ed040261c85 diff --git a/toolchains/riscv-tools/riscv-isa-sim b/toolchains/riscv-tools/riscv-isa-sim deleted file mode 160000 index ce42f1b5..00000000 --- a/toolchains/riscv-tools/riscv-isa-sim +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ce42f1b55a3fa5a3a522439b5370be2256f68862 diff --git a/toolchains/riscv-tools/riscv-openocd b/toolchains/riscv-tools/riscv-openocd deleted file mode 160000 index cbb15587..00000000 --- a/toolchains/riscv-tools/riscv-openocd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cbb15587dc782ac8ade7ae252e7b760cfba4a178 diff --git a/toolchains/riscv-tools/riscv-pk b/toolchains/riscv-tools/riscv-pk deleted file mode 160000 index e8e6b3aa..00000000 --- a/toolchains/riscv-tools/riscv-pk +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e8e6b3aaee44d43b48164fbd377864c3a682dbd3 diff --git a/toolchains/riscv-tools/riscv-tests b/toolchains/riscv-tools/riscv-tests deleted file mode 160000 index 1ce128fa..00000000 --- a/toolchains/riscv-tools/riscv-tests +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1ce128fa78c24bb0ed399c647e7139322b5353a7 diff --git a/tools/dromajo/dromajo.mk b/tools/dromajo/dromajo.mk index 320aee94..0d0d6b5f 100644 --- a/tools/dromajo/dromajo.mk +++ b/tools/dromajo/dromajo.mk @@ -4,7 +4,7 @@ DROMAJO_DIR = $(base_dir)/tools/dromajo/dromajo-src/src DROMAJO_LIB_NAME = dromajo_cosim -DROMAJO_LIB = $(DROMAJO_DIR)/lib$(DROMAJO_LIB_NAME).a +DROMAJO_LIB = $(CONDA_PREFIX)/lib/lib$(DROMAJO_LIB_NAME).a # Dromajo assumes using the default bootrom DROMAJO_ROM = $(build_dir)/bootrom.rv64.img diff --git a/variables.mk b/variables.mk index 7025d89e..d5834bee 100644 --- a/variables.mk +++ b/variables.mk @@ -179,7 +179,7 @@ SBT_CLIENT_FLAG = --client endif # passes $(JAVA_TOOL_OPTIONS) from env to java -SBT_BIN ?= java -jar $(ROCKETCHIP_DIR)/sbt-launch.jar $(SBT_OPTS) +SBT_BIN ?= sbt SBT = $(SBT_BIN) $(SBT_CLIENT_FLAG) SBT_NON_THIN = $(subst $(SBT_CLIENT_FLAG),,$(SBT))