wu-arch/chipyard
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Rework build-setup | Add single-node CI

Browse Source
This commit is contained in:
abejgonzalez
2022-12-05 14:26:08 -08:00
parent 43b75640cc
commit 5996ec69a5
7 changed files with 221 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
.github/workflows/chipyard-full-flow.yml vendored Normal file
View File

@@ -0,0 +1,104 @@
name: chipyard-ci-full-flow
on:
# run ci on pull requests targeting following branches (runs on the merge commit)
pull_request:
branches:
- main
- '1.[0-9]*.x'
defaults:
run:
shell: bash -leo pipefail {0}
env:
REMOTE_WORK_DIR: ${{ secrets.BUILDDIR }}/cy-ci-shared/cy-${{ github.sha }}
jobs:
cancel-prior-workflows:
name: cancel-prior-workflows
runs-on: ubuntu-latest
steps:
- name: Cancel Previous Runs
uses: styfle/cancel-workflow-action@0.11.0
with:
access_token: ${{ github.token }}
# Set up a set of boolean conditions to control which branches of the CI
# workflow will execute This is based off the conditional job execution
# example here: https://github.com/dorny/paths-filter#examples
change-filters:
name: filter-jobs-on-changes
runs-on: ubuntu-latest
# Queried by downstream jobs to determine if they should run.
outputs:
needs-rtl: ${{ steps.filter.outputs.all_count != steps.filter.outputs.skip-rtl_count }}
steps:
- uses: actions/checkout@v3
- name: Git workaround
uses: ./.github/actions/git-workaround
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
all:
- '**'
# If any of the files changed match, do a doc build
docs: &docs-filter
- 'docs/**'
- '.readthedocs.yml'
# If all files match to this filter, skip the main ci pipeline
skip-rtl:
- *docs-filter
- '**/*.md'
- '**/.gitignore'
- '.github/ISSUE_TEMPLATE/**'
setup-repo:
name: setup-repo
needs: [change-filters, cancel-prior-workflows]
if: needs.change-filters.outputs.needs-rtl == 'true'
runs-on: ferry
steps:
- name: Delete old checkout
run: |
ls -alh .
rm -rf ${{ github.workspace }}/* || true
rm -rf ${{ github.workspace }}/.* || true
ls -alh .
- uses: actions/checkout@v3
- name: Setup repo copy
run: |
git clone $GITHUB_WORKSPACE ${{ env.REMOTE_WORK_DIR }}
- name: Setup repo
run: |
cd ${{ env.REMOTE_WORK_DIR }}
eval "$(conda shell.bash hook)"
export MAKEFLAGS="-j32"
./build-setup.sh -f
run-tutorial:
name: run-tutorial
needs: [setup-repo]
runs-on: ferry
steps:
- name: Run smoke test
run: |
cd ${{ env.REMOTE_WORK_DIR }}
eval "$(conda shell.bash hook)"
source env.sh
cd sims/verilator
make verilog
cleanup:
name: cleanup
needs: [run-tutorial]
runs-on: ferry
if: ${{ always() && contains(join(needs.*.result, ','), 'success') }}
steps:
- name: Delete repo copy and conda env
run: |
rm -rf ${{ env.REMOTE_WORK_DIR }}

2
.github/workflows/chipyard-run-tests.yml vendored
View File

@@ -25,7 +25,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Cancel Previous Runs
uses: styfle/cancel-workflow-action@0.9.1
uses: styfle/cancel-workflow-action@0.11.0
with:
access_token: ${{ github.token }}

20
docs/Chipyard-Basics/Initial-Repo-Setup.rst
View File

@@ -62,27 +62,25 @@ Start by checking out the proper Chipyard version. Run:
# note: this may not be the latest release if the documentation version != "stable"
git checkout |version|
Next run the following script to create Chipyard's Conda environment including a pre-built RISC-V toolchain.
Next run the following script to fully setup Chipyard with a specific toolchain.
There are two toolchains, one for normal RISC-V programs called ``riscv-tools`` which is the one needed for most Chipyard use-cases, and another for Hwacha/Gemmini called ``esp-tools``.
Run the following script based off which compiler you would like to use.
.. Warning:: The following script will complete a "full" installation of Chipyard which may take a long time depending on the system.
Ensure that this script completes fully (no interruptions) before continuing on.
.. code-block:: shell
./build-setup.sh riscv-tools # or esp-tools
This script wraps around the conda environment initialization process and also runs the ``init-submodules-no-riscv-tools.sh`` and ``build-toolchain-extra.sh`` scripts.
This script wraps around the conda environment initialization process, initializes all submodules (with the ``init-submodules-no-riscv-tools.sh`` script), installs a toolchain, and runs other setups.
See ``./build-setup.sh --help`` for more details on what this does and how to disable parts of the setup.
The ``init-submodules-no-riscv-tools.sh`` script will initialize and checkout all of the necessary git submodules.
This will also validate that you are on a tagged branch, otherwise it will prompt for confirmation.
When updating Chipyard to a new version, you will also want to rerun this script to update the submodules.
Using ``git`` directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior.
.. Warning:: Using ``git`` directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior.
The ``build-toolchain-extra.sh`` script will install extra toolchain utilities/tests used by Chipyard.
This command builds utilities like Spike, RISC-V Proxy Kernel, libgloss, and RISC-V tests from source for a specific toolchain type.
.. Note:: By default, the ``build-setup.sh`` script installs extra toolchain utilities (RISC-V tests, PK, Spike, etc) to ``$CONDA_PREFIX/<toolchain-type>``. Thus, if you uninstall the compiler using ``conda remove`` these utilities/tests will also have to be re-installed/built.
.. Note:: By default, the ``build-toolchain-extra.sh`` script installs to ``$CONDA_PREFIX/<toolchain-type>``. Thus, if you uninstall the compiler using ``conda remove`` these utilities/tests will also have to be re-installed/built.
.. Note:: If you already have a working conda environment setup, separate Chipyard clones can use that pre-used environment in combination with running the aforementioned scripts yourself (``init-submodules...`` and ``build-toolchain...``).
.. Note:: If you already have a working conda environment setup, separate Chipyard clones can use that pre-used environment in combination with running the aforementioned scripts yourself (``init-submodules...``, ``build-toolchain...``, etc).
.. Note:: If you are a power user and would like to build your own compiler/toolchain, you can refer to the https://github.com/ucb-bar/riscv-tools-feedstock and https://github.com/ucb-bar/esp-tools-feedstock repositories (submoduled in the ``toolchains/*`` directories) on how to build the compiler yourself.

9
docs/Simulation/FPGA-Accelerated-Simulation.rst
View File

@@ -14,14 +14,7 @@ FireSim supports running on Amazon EC2 F1 FPGA-enabled cloud instances and on lo
The rest of this documentation assumes you are running on an Amazon EC2 F1 FPGA-enabled virtual instance.
In order to simuate your Chipyard design using FireSim, make sure to follow the repository setup as described by
:ref:`Chipyard-Basics/Initial-Repo-Setup:Initial Repository Setup`, if you have not already.
Next, initalize FireSim as a library in Chipyard by running:
.. code-block:: shell
# At the root of your chipyard repo
./scripts/firesim-setup.sh
This setup should have setup the Chipyard repository including FireSim by running the ``./scripts/firesim-setup.sh`` script.
``firesim-setup.sh`` initializes additional submodules and then invokes
FireSim's ``build-setup.sh`` script adding ``--library`` to properly
initialize FireSim as a library submodule in Chipyard. You may run

134
scripts/build-setup.sh
View File

@@ -14,28 +14,45 @@ common_setup
usage() {
echo "Usage: ${0} [OPTIONS] [riscv-tools | esp-tools]"
echo ""
echo "Helper script to initialize repository that wraps other scripts."
echo "Sets up conda environment, initializes submodules, and installs toolchain collateral."
echo ""
echo "Installation Types"
echo " riscv-tools: if set, builds the riscv toolchain (this is also the default)"
echo " esp-tools: if set, builds esp-tools toolchain used for the hwacha vector accelerator"
echo ""
echo "Helper script to fully initialize repository that wraps other scripts."
echo "By default it initializes/installs things in the following order:"
echo " 1. Conda environment"
echo " 2. Chipyard submodules"
echo " 3. Toolchain collateral (Spike, PK, tests, libgloss)"
echo " 4. Ctags"
echo " 5. Chipyard pre-compile sources"
echo " 6. FireSim"
echo " 7. FireSim pre-compile sources"
echo " 8. FireMarshal"
echo " 9. FireMarshal pre-compile default buildroot Linux sources"
echo " 10. Runs repository clean-up"
echo ""
echo "**See below for options to skip parts of the setup. Skipping parts of the setup is not guaranteed to be tested/working.**"
echo ""
echo "Options"
echo " --help -h : Display this message"
echo " --unpinned-deps -ud : Use unpinned conda environment"
echo " --force -f : Skip prompt checking for tagged release/conda"
echo " --skip-validate : DEPRECATED: Same functionality as --force"
echo " --skip-conda : Skip conda env creation"
echo " --skip-toolchain-extra : Skip building extra RISC-V toolchain collateral (Spike, PK, tests, libgloos)"
echo " --help -h : Display this message"
echo " --force -f : Skip all prompts and checks"
echo " --skip-validate : DEPRECATED: Same functionality as --force"
echo " --verbose -v : Verbose printout"
echo " --use-unpinned-deps -ud : Use unpinned conda environment"
echo " --skip -s N : Skip step N in the list above. Use multiple times to skip multiple steps ('-s N -s M ...')."
exit "$1"
}
TOOLCHAIN="riscv-tools"
USE_PINNED_DEPS=true
TOOLCHAIN_TYPE="riscv-tools"
FORCE_FLAG=""
SKIP_CONDA=false
SKIP_TOOLCHAIN=false
VERBOSE=false
VERBOSE_FLAG=""
USE_UNPINNED_DEPS=false
SKIP_LIST=()
# getopts does not support long options, and is inflexible
while [ "$1" != "" ];
@@ -44,15 +61,17 @@ do
-h | --help )
usage 3 ;;
riscv-tools | esp-tools)
TOOLCHAIN=$1 ;;
-ud | --unpinned-deps )
USE_PINNED_DEPS=false ;;
TOOLCHAIN_TYPE=$1 ;;
--force | -f | --skip-validate)
FORCE_FLAG=$1 ;;
--skip-conda)
SKIP_CONDA=true ;;
--skip-toolchain-extra)
SKIP_TOOLCHAIN=true ;;
--verbose | -v)
VERBOSE_FLAG=$1
set -x ;;
-ud | --use-unpinned-deps )
USE_UNPINNED_DEPS=true ;;
--skip | -s)
shift
SKIP_LIST+=(${1}) ;;
* )
error "invalid option $1"
usage 1 ;;
@@ -60,15 +79,23 @@ do
shift
done
if [ "$SKIP_CONDA" = false ]; then
# check if the arg is found in the SKIP_LIST
do_skip() {
local value=$1
[[ ! " ${SKIP_LIST[*]} " =~ " ${value} " ]]
}
{
if do_skip "1"; then
# note: lock file must end in .conda-lock.yml - see https://github.com/conda-incubator/conda-lock/issues/154
CONDA_REQS=$RDIR/conda-reqs
CONDA_LOCK_REQS=$CONDA_REQS/conda-lock-reqs
LOCKFILE=$CONDA_LOCK_REQS/conda-requirements-$TOOLCHAIN-linux-64.conda-lock.yml
LOCKFILE=$CONDA_LOCK_REQS/conda-requirements-$TOOLCHAIN_TYPE-linux-64.conda-lock.yml
if [ "$USE_PINNED_DEPS" = false ]; then
if [ "$USE_UNPINNED_DEPS" = true ]; then
# auto-gen the lockfile
conda-lock -f $CONDA_REQS/chipyard.yaml -f $CONDA_REQS/$TOOLCHAIN.yaml --lockfile $LOCKFILE
conda-lock -f $CONDA_REQS/chipyard.yaml -f $CONDA_REQS/$TOOLCHAIN_TYPE.yaml --lockfile $LOCKFILE
fi
# use conda-lock to create env
@@ -85,16 +112,63 @@ if [ -z "$FORCE_FLAG" ]; then
fi
fi
$RDIR/scripts/init-submodules-no-riscv-tools.sh $FORCE_FLAG
if [ "$SKIP_TOOLCHAIN" = false ]; then
$RDIR/scripts/build-toolchain-extra.sh $FORCE_FLAG $TOOLCHAIN
if do_skip "2"; then
$RDIR/scripts/init-submodules-no-riscv-tools.sh $FORCE_FLAG
fi
$RDIR/scripts/gen-tags.sh
if do_skip "3"; then
$RDIR/scripts/build-toolchain-extra.sh $FORCE_FLAG $TOOLCHAIN_TYPE
fi
cat << EOT >> env.sh
if do_skip "4"; then
$RDIR/scripts/gen-tags.sh
fi
if do_skip "5"; then
pushd $RDIR/sims/verilator
make launch-sbt SBT_COMMAND=";project chipyard; compile"
make launch-sbt SBT_COMMAND=";project tapeout; compile"
popd
fi
if do_skip "6"; then
$RDIR/scripts/firesim-setup.sh
$RDIR/sims/firesim/gen-tags.sh
if do_skip "7"; then
pushd $RDIR/sims/firesim
(
source sourceme-f1-manager.sh --skip-ssh-setup
pushd sim
make sbt SBT_COMMAND="project firechip; compile" TARGET_PROJECT=firesim
popd
)
popd
fi
fi
if do_skip "8"; then
pushd $RDIR/software/firemarshal
./init-submodules.sh
if do_skip "9"; then
source $RDIR/scripts/fix-open-files.sh
./marshal $VERBOSE_FLAG build br-base.json
./marshal $VERBOSE_FLAG clean br-base.json
fi
popd
fi
if do_skip "10"; then
$RDIR/scripts/repo-clean.sh
fi
cat <<EOT >> env.sh
# line auto-generated by $0
conda activate $RDIR/.conda-env
source $RDIR/scripts/fix-open-files.sh
EOT
echo "Setup complete!"
} 2>&1 | tee build-setup.log

33
scripts/first-clone-setup-fast.sh
View File

@@ -1,33 +0,0 @@
#!/usr/bin/env bash
set -ex
{
export MAKEFLAGS=-j16
STARTDIR=$(git rev-parse --show-toplevel)
./build-setup.sh riscv-tools -f
source $STARTDIR/.conda-env/etc/profile.d/conda.sh
source env.sh
./scripts/firesim-setup.sh
cd sims/firesim
source sourceme-f1-manager.sh --skip-ssh-setup
cd sim
unset MAKEFLAGS
make verilator
export MAKEFLAGS=-j16
cd $STARTDIR/software/firemarshal
./init-submodules.sh
marshal -v build br-base.json
cd $STARTDIR
./scripts/repo-clean.sh
} 2>&1 | tee first-clone-setup-fast-log

2
scripts/fix-open-files.sh
View File

@@ -1,10 +1,12 @@
# first, check if the system allows sufficient limits (the hard limit)
HARD_LIMIT=$(ulimit -Hn)
SOFT_LIMIT=$(ulimit -Sn)
REQUIRED_LIMIT=16384
if [ "$HARD_LIMIT" -lt "$REQUIRED_LIMIT" ]; then
echo "WARNING: Your system does not support an open files limit (the output of 'ulimit -Sn' and 'ulimit -Hn') of at least $REQUIRED_LIMIT, which is required to workaround a bug in buildroot. You will not be able to build a Linux distro with FireMarshal until this is addressed."
echo "WARNING: HARD_LIMIT=$HARD_LIMIT SOFT_LIMIT=$SOFT_LIMIT REQUIRED_LIMIT=$REQUIRED_LIMIT"
fi
# in any case, set the soft limit to the same value as the hard limit