diff --git a/.github/workflows/chipyard-full-flow.yml b/.github/workflows/chipyard-full-flow.yml new file mode 100644 index 00000000..cc2168e8 --- /dev/null +++ b/.github/workflows/chipyard-full-flow.yml @@ -0,0 +1,104 @@ +name: chipyard-ci-full-flow + +on: + # run ci on pull requests targeting following branches (runs on the merge commit) + pull_request: + branches: + - main + - '1.[0-9]*.x' + +defaults: + run: + shell: bash -leo pipefail {0} + +env: + REMOTE_WORK_DIR: ${{ secrets.BUILDDIR }}/cy-ci-shared/cy-${{ github.sha }} + +jobs: + cancel-prior-workflows: + name: cancel-prior-workflows + runs-on: ubuntu-latest + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.11.0 + with: + access_token: ${{ github.token }} + + # Set up a set of boolean conditions to control which branches of the CI + # workflow will execute This is based off the conditional job execution + # example here: https://github.com/dorny/paths-filter#examples + change-filters: + name: filter-jobs-on-changes + runs-on: ubuntu-latest + # Queried by downstream jobs to determine if they should run. + outputs: + needs-rtl: ${{ steps.filter.outputs.all_count != steps.filter.outputs.skip-rtl_count }} + + steps: + - uses: actions/checkout@v3 + - name: Git workaround + uses: ./.github/actions/git-workaround + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + all: + - '**' + + # If any of the files changed match, do a doc build + docs: &docs-filter + - 'docs/**' + - '.readthedocs.yml' + + # If all files match to this filter, skip the main ci pipeline + skip-rtl: + - *docs-filter + - '**/*.md' + - '**/.gitignore' + - '.github/ISSUE_TEMPLATE/**' + + setup-repo: + name: setup-repo + needs: [change-filters, cancel-prior-workflows] + if: needs.change-filters.outputs.needs-rtl == 'true' + runs-on: ferry + steps: + - name: Delete old checkout + run: | + ls -alh . + rm -rf ${{ github.workspace }}/* || true + rm -rf ${{ github.workspace }}/.* || true + ls -alh . + - uses: actions/checkout@v3 + - name: Setup repo copy + run: | + git clone $GITHUB_WORKSPACE ${{ env.REMOTE_WORK_DIR }} + - name: Setup repo + run: | + cd ${{ env.REMOTE_WORK_DIR }} + eval "$(conda shell.bash hook)" + export MAKEFLAGS="-j32" + ./build-setup.sh -f + + run-tutorial: + name: run-tutorial + needs: [setup-repo] + runs-on: ferry + steps: + - name: Run smoke test + run: | + cd ${{ env.REMOTE_WORK_DIR }} + eval "$(conda shell.bash hook)" + source env.sh + cd sims/verilator + make verilog + + cleanup: + name: cleanup + needs: [run-tutorial] + runs-on: ferry + if: ${{ always() && contains(join(needs.*.result, ','), 'success') }} + steps: + - name: Delete repo copy and conda env + run: | + rm -rf ${{ env.REMOTE_WORK_DIR }} diff --git a/.github/workflows/chipyard-run-tests.yml b/.github/workflows/chipyard-run-tests.yml index d4351a98..0ed26ea6 100644 --- a/.github/workflows/chipyard-run-tests.yml +++ b/.github/workflows/chipyard-run-tests.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@0.9.1 + uses: styfle/cancel-workflow-action@0.11.0 with: access_token: ${{ github.token }} diff --git a/docs/Chipyard-Basics/Initial-Repo-Setup.rst b/docs/Chipyard-Basics/Initial-Repo-Setup.rst index 6e481e9c..6543afc2 100644 --- a/docs/Chipyard-Basics/Initial-Repo-Setup.rst +++ b/docs/Chipyard-Basics/Initial-Repo-Setup.rst @@ -62,27 +62,25 @@ Start by checking out the proper Chipyard version. Run: # note: this may not be the latest release if the documentation version != "stable" git checkout |version| -Next run the following script to create Chipyard's Conda environment including a pre-built RISC-V toolchain. +Next run the following script to fully setup Chipyard with a specific toolchain. There are two toolchains, one for normal RISC-V programs called ``riscv-tools`` which is the one needed for most Chipyard use-cases, and another for Hwacha/Gemmini called ``esp-tools``. Run the following script based off which compiler you would like to use. +.. Warning:: The following script will complete a "full" installation of Chipyard which may take a long time depending on the system. + Ensure that this script completes fully (no interruptions) before continuing on. + .. code-block:: shell ./build-setup.sh riscv-tools # or esp-tools -This script wraps around the conda environment initialization process and also runs the ``init-submodules-no-riscv-tools.sh`` and ``build-toolchain-extra.sh`` scripts. +This script wraps around the conda environment initialization process, initializes all submodules (with the ``init-submodules-no-riscv-tools.sh`` script), installs a toolchain, and runs other setups. +See ``./build-setup.sh --help`` for more details on what this does and how to disable parts of the setup. -The ``init-submodules-no-riscv-tools.sh`` script will initialize and checkout all of the necessary git submodules. -This will also validate that you are on a tagged branch, otherwise it will prompt for confirmation. -When updating Chipyard to a new version, you will also want to rerun this script to update the submodules. -Using ``git`` directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior. +.. Warning:: Using ``git`` directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior. -The ``build-toolchain-extra.sh`` script will install extra toolchain utilities/tests used by Chipyard. -This command builds utilities like Spike, RISC-V Proxy Kernel, libgloss, and RISC-V tests from source for a specific toolchain type. +.. Note:: By default, the ``build-setup.sh`` script installs extra toolchain utilities (RISC-V tests, PK, Spike, etc) to ``$CONDA_PREFIX/``. Thus, if you uninstall the compiler using ``conda remove`` these utilities/tests will also have to be re-installed/built. -.. Note:: By default, the ``build-toolchain-extra.sh`` script installs to ``$CONDA_PREFIX/``. Thus, if you uninstall the compiler using ``conda remove`` these utilities/tests will also have to be re-installed/built. - -.. Note:: If you already have a working conda environment setup, separate Chipyard clones can use that pre-used environment in combination with running the aforementioned scripts yourself (``init-submodules...`` and ``build-toolchain...``). +.. Note:: If you already have a working conda environment setup, separate Chipyard clones can use that pre-used environment in combination with running the aforementioned scripts yourself (``init-submodules...``, ``build-toolchain...``, etc). .. Note:: If you are a power user and would like to build your own compiler/toolchain, you can refer to the https://github.com/ucb-bar/riscv-tools-feedstock and https://github.com/ucb-bar/esp-tools-feedstock repositories (submoduled in the ``toolchains/*`` directories) on how to build the compiler yourself. diff --git a/docs/Simulation/FPGA-Accelerated-Simulation.rst b/docs/Simulation/FPGA-Accelerated-Simulation.rst index 50d48c4e..703561fe 100644 --- a/docs/Simulation/FPGA-Accelerated-Simulation.rst +++ b/docs/Simulation/FPGA-Accelerated-Simulation.rst @@ -14,14 +14,7 @@ FireSim supports running on Amazon EC2 F1 FPGA-enabled cloud instances and on lo The rest of this documentation assumes you are running on an Amazon EC2 F1 FPGA-enabled virtual instance. In order to simuate your Chipyard design using FireSim, make sure to follow the repository setup as described by :ref:`Chipyard-Basics/Initial-Repo-Setup:Initial Repository Setup`, if you have not already. - -Next, initalize FireSim as a library in Chipyard by running: - -.. code-block:: shell - - # At the root of your chipyard repo - ./scripts/firesim-setup.sh - +This setup should have setup the Chipyard repository including FireSim by running the ``./scripts/firesim-setup.sh`` script. ``firesim-setup.sh`` initializes additional submodules and then invokes FireSim's ``build-setup.sh`` script adding ``--library`` to properly initialize FireSim as a library submodule in Chipyard. You may run diff --git a/scripts/build-setup.sh b/scripts/build-setup.sh index 8d5d25a5..0fc71ad6 100755 --- a/scripts/build-setup.sh +++ b/scripts/build-setup.sh @@ -14,28 +14,45 @@ common_setup usage() { echo "Usage: ${0} [OPTIONS] [riscv-tools | esp-tools]" echo "" - echo "Helper script to initialize repository that wraps other scripts." - echo "Sets up conda environment, initializes submodules, and installs toolchain collateral." - echo "" echo "Installation Types" echo " riscv-tools: if set, builds the riscv toolchain (this is also the default)" echo " esp-tools: if set, builds esp-tools toolchain used for the hwacha vector accelerator" echo "" + echo "Helper script to fully initialize repository that wraps other scripts." + echo "By default it initializes/installs things in the following order:" + echo " 1. Conda environment" + echo " 2. Chipyard submodules" + echo " 3. Toolchain collateral (Spike, PK, tests, libgloss)" + echo " 4. Ctags" + echo " 5. Chipyard pre-compile sources" + echo " 6. FireSim" + echo " 7. FireSim pre-compile sources" + echo " 8. FireMarshal" + echo " 9. FireMarshal pre-compile default buildroot Linux sources" + echo " 10. Runs repository clean-up" + echo "" + echo "**See below for options to skip parts of the setup. Skipping parts of the setup is not guaranteed to be tested/working.**" + echo "" echo "Options" - echo " --help -h : Display this message" - echo " --unpinned-deps -ud : Use unpinned conda environment" - echo " --force -f : Skip prompt checking for tagged release/conda" - echo " --skip-validate : DEPRECATED: Same functionality as --force" - echo " --skip-conda : Skip conda env creation" - echo " --skip-toolchain-extra : Skip building extra RISC-V toolchain collateral (Spike, PK, tests, libgloos)" + echo " --help -h : Display this message" + + echo " --force -f : Skip all prompts and checks" + echo " --skip-validate : DEPRECATED: Same functionality as --force" + echo " --verbose -v : Verbose printout" + + echo " --use-unpinned-deps -ud : Use unpinned conda environment" + + echo " --skip -s N : Skip step N in the list above. Use multiple times to skip multiple steps ('-s N -s M ...')." + exit "$1" } -TOOLCHAIN="riscv-tools" -USE_PINNED_DEPS=true +TOOLCHAIN_TYPE="riscv-tools" FORCE_FLAG="" -SKIP_CONDA=false -SKIP_TOOLCHAIN=false +VERBOSE=false +VERBOSE_FLAG="" +USE_UNPINNED_DEPS=false +SKIP_LIST=() # getopts does not support long options, and is inflexible while [ "$1" != "" ]; @@ -44,15 +61,17 @@ do -h | --help ) usage 3 ;; riscv-tools | esp-tools) - TOOLCHAIN=$1 ;; - -ud | --unpinned-deps ) - USE_PINNED_DEPS=false ;; + TOOLCHAIN_TYPE=$1 ;; --force | -f | --skip-validate) FORCE_FLAG=$1 ;; - --skip-conda) - SKIP_CONDA=true ;; - --skip-toolchain-extra) - SKIP_TOOLCHAIN=true ;; + --verbose | -v) + VERBOSE_FLAG=$1 + set -x ;; + -ud | --use-unpinned-deps ) + USE_UNPINNED_DEPS=true ;; + --skip | -s) + shift + SKIP_LIST+=(${1}) ;; * ) error "invalid option $1" usage 1 ;; @@ -60,15 +79,23 @@ do shift done -if [ "$SKIP_CONDA" = false ]; then +# check if the arg is found in the SKIP_LIST +do_skip() { + local value=$1 + [[ ! " ${SKIP_LIST[*]} " =~ " ${value} " ]] +} + +{ + +if do_skip "1"; then # note: lock file must end in .conda-lock.yml - see https://github.com/conda-incubator/conda-lock/issues/154 CONDA_REQS=$RDIR/conda-reqs CONDA_LOCK_REQS=$CONDA_REQS/conda-lock-reqs - LOCKFILE=$CONDA_LOCK_REQS/conda-requirements-$TOOLCHAIN-linux-64.conda-lock.yml + LOCKFILE=$CONDA_LOCK_REQS/conda-requirements-$TOOLCHAIN_TYPE-linux-64.conda-lock.yml - if [ "$USE_PINNED_DEPS" = false ]; then + if [ "$USE_UNPINNED_DEPS" = true ]; then # auto-gen the lockfile - conda-lock -f $CONDA_REQS/chipyard.yaml -f $CONDA_REQS/$TOOLCHAIN.yaml --lockfile $LOCKFILE + conda-lock -f $CONDA_REQS/chipyard.yaml -f $CONDA_REQS/$TOOLCHAIN_TYPE.yaml --lockfile $LOCKFILE fi # use conda-lock to create env @@ -85,16 +112,63 @@ if [ -z "$FORCE_FLAG" ]; then fi fi -$RDIR/scripts/init-submodules-no-riscv-tools.sh $FORCE_FLAG - -if [ "$SKIP_TOOLCHAIN" = false ]; then - $RDIR/scripts/build-toolchain-extra.sh $FORCE_FLAG $TOOLCHAIN +if do_skip "2"; then + $RDIR/scripts/init-submodules-no-riscv-tools.sh $FORCE_FLAG fi -$RDIR/scripts/gen-tags.sh +if do_skip "3"; then + $RDIR/scripts/build-toolchain-extra.sh $FORCE_FLAG $TOOLCHAIN_TYPE +fi -cat << EOT >> env.sh +if do_skip "4"; then + $RDIR/scripts/gen-tags.sh +fi + +if do_skip "5"; then + pushd $RDIR/sims/verilator + make launch-sbt SBT_COMMAND=";project chipyard; compile" + make launch-sbt SBT_COMMAND=";project tapeout; compile" + popd +fi + +if do_skip "6"; then + $RDIR/scripts/firesim-setup.sh + $RDIR/sims/firesim/gen-tags.sh + + if do_skip "7"; then + pushd $RDIR/sims/firesim + ( + source sourceme-f1-manager.sh --skip-ssh-setup + pushd sim + make sbt SBT_COMMAND="project firechip; compile" TARGET_PROJECT=firesim + popd + ) + popd + fi +fi + +if do_skip "8"; then + pushd $RDIR/software/firemarshal + ./init-submodules.sh + + if do_skip "9"; then + source $RDIR/scripts/fix-open-files.sh + ./marshal $VERBOSE_FLAG build br-base.json + ./marshal $VERBOSE_FLAG clean br-base.json + fi + popd +fi + +if do_skip "10"; then + $RDIR/scripts/repo-clean.sh +fi + +cat <> env.sh # line auto-generated by $0 conda activate $RDIR/.conda-env source $RDIR/scripts/fix-open-files.sh EOT + +echo "Setup complete!" + +} 2>&1 | tee build-setup.log diff --git a/scripts/first-clone-setup-fast.sh b/scripts/first-clone-setup-fast.sh deleted file mode 100755 index 728fac1b..00000000 --- a/scripts/first-clone-setup-fast.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -{ - -export MAKEFLAGS=-j16 - -STARTDIR=$(git rev-parse --show-toplevel) - -./build-setup.sh riscv-tools -f - -source $STARTDIR/.conda-env/etc/profile.d/conda.sh - -source env.sh - -./scripts/firesim-setup.sh -cd sims/firesim -source sourceme-f1-manager.sh --skip-ssh-setup - -cd sim -unset MAKEFLAGS -make verilator -export MAKEFLAGS=-j16 - -cd $STARTDIR/software/firemarshal -./init-submodules.sh -marshal -v build br-base.json - -cd $STARTDIR -./scripts/repo-clean.sh - -} 2>&1 | tee first-clone-setup-fast-log diff --git a/scripts/fix-open-files.sh b/scripts/fix-open-files.sh index 79e12e7a..6e116771 100644 --- a/scripts/fix-open-files.sh +++ b/scripts/fix-open-files.sh @@ -1,10 +1,12 @@ # first, check if the system allows sufficient limits (the hard limit) HARD_LIMIT=$(ulimit -Hn) +SOFT_LIMIT=$(ulimit -Sn) REQUIRED_LIMIT=16384 if [ "$HARD_LIMIT" -lt "$REQUIRED_LIMIT" ]; then echo "WARNING: Your system does not support an open files limit (the output of 'ulimit -Sn' and 'ulimit -Hn') of at least $REQUIRED_LIMIT, which is required to workaround a bug in buildroot. You will not be able to build a Linux distro with FireMarshal until this is addressed." + echo "WARNING: HARD_LIMIT=$HARD_LIMIT SOFT_LIMIT=$SOFT_LIMIT REQUIRED_LIMIT=$REQUIRED_LIMIT" fi # in any case, set the soft limit to the same value as the hard limit