From 2f1e5e994b5c867fac6a7c90fb1dc72dd4082b2d Mon Sep 17 00:00:00 2001 From: Albert Ou Date: Mon, 23 Sep 2019 23:47:39 -0700 Subject: [PATCH] toolchains: Flatten riscv-tools submodule This allows individual components to be better maintained following the deprecation of riscv-tools. Eliminate non-essential submodules. build-static-libfesvr.sh is no longer necessary since libfesvr.a is built as part of the riscv-isa-sim build. For simplicity, only riscv-gnu-toolchain is now pre-built instead of the entirety of riscv-tools. --- .gitmodules | 22 ++- scripts/build-static-libfesvr.sh | 21 --- scripts/build-toolchains.sh | 151 ++++++------------ scripts/build-util.sh | 89 +++++++++++ scripts/init-submodules-no-riscv-tools.sh | 28 ++-- toolchains/esp-tools | 1 - toolchains/riscv-tools | 1 - toolchains/riscv-tools/riscv-gnu-toolchain | 1 + .../riscv-tools/riscv-gnu-toolchain-prebuilt | 1 + toolchains/riscv-tools/riscv-isa-sim | 1 + toolchains/riscv-tools/riscv-pk | 1 + toolchains/riscv-tools/riscv-tests | 1 + 12 files changed, 174 insertions(+), 144 deletions(-) delete mode 100755 scripts/build-static-libfesvr.sh create mode 100644 scripts/build-util.sh delete mode 160000 toolchains/esp-tools delete mode 160000 toolchains/riscv-tools create mode 160000 toolchains/riscv-tools/riscv-gnu-toolchain create mode 160000 toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt create mode 160000 toolchains/riscv-tools/riscv-isa-sim create mode 160000 toolchains/riscv-tools/riscv-pk create mode 160000 toolchains/riscv-tools/riscv-tests diff --git a/.gitmodules b/.gitmodules index 35addf76..d7691ffb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,12 +34,22 @@ [submodule "generators/block-inclusivecache-sifive"] path = generators/sifive-cache url = https://github.com/sifive/block-inclusivecache-sifive.git -[submodule "toolchains/riscv-tools"] - path = toolchains/riscv-tools - url = https://github.com/freechipsproject/rocket-tools.git -[submodule "toolchains/esp-tools"] - path = toolchains/esp-tools - url = https://github.com/ucb-bar/esp-tools.git +[submodule "toolchains/riscv-tools/riscv-gnu-toolchain"] + path = toolchains/riscv-tools/riscv-gnu-toolchain + url = https://github.com/riscv/riscv-gnu-toolchain.git +[submodule "toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt"] + path = toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt + url = https://github.com/ucb-bar/chipyard-toolchain-prebuilt.git + shallow = true +[submodule "toolchains/riscv-tools/riscv-isa-sim"] + path = toolchains/riscv-tools/riscv-isa-sim + url = https://github.com/riscv/riscv-isa-sim.git +[submodule "toolchains/riscv-tools/riscv-pk"] + path = toolchains/riscv-tools/riscv-pk + url = https://github.com/riscv/riscv-pk.git +[submodule "toolchains/riscv-tools/riscv-tests"] + path = toolchains/riscv-tools/riscv-tests + url = https://github.com/riscv/riscv-tests.git [submodule "vlsi/hammer"] path = vlsi/hammer url = https://github.com/ucb-bar/hammer.git diff --git a/scripts/build-static-libfesvr.sh b/scripts/build-static-libfesvr.sh deleted file mode 100755 index d5b7f896..00000000 --- a/scripts/build-static-libfesvr.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# This ungodly script surreptitiously builds an archive from existing fesvr objects -# Invoke from riscv-fesvr/build - -if [ "x$RISCV" = "x" ] -then - echo "Please set the RISCV environment variable to your preferred install path." - exit 1 -fi - -set -e - -objs=$(make -n -f <( - echo 'include Makefile' - echo '$(info $(value fesvr_objs))' - ) | head -n 1) - -ar rcs -o libfesvr.a $objs -cp -f libfesvr.a "${RISCV}/lib" - diff --git a/scripts/build-toolchains.sh b/scripts/build-toolchains.sh index 76680f79..734398c3 100755 --- a/scripts/build-toolchains.sh +++ b/scripts/build-toolchains.sh @@ -6,11 +6,8 @@ set -e set -o pipefail -unamestr=$(uname) RDIR=$(pwd) -: ${CHIPYARD_DIR:=$(pwd)} #default value is the PWD unless overridden - -PRECOMPILED_REPO_HASH=56a40961c98db5e8f904f15dc6efd0870bfefd9e +CHIPYARD_DIR="${CHIPYARD_DIR:-$(git rev-parse --show-toplevel)}" usage() { echo "usage: ${0} [riscv-tools | esp-tools | ec2fast]" @@ -23,18 +20,13 @@ usage() { error() { echo "${0##*/}: ${1}" >&2 } - -#taken from riscv-tools to check for open-ocd autoconf versions -check_version() { - "$1" --version | awk "NR==1 {if (\$NF>$2) {exit 0} exit 1}" || { - error "${3} requires at least ${1} version ${2}" - exit 1 - } +die() { + error "$1" + exit "${2:--1}" } TOOLCHAIN="riscv-tools" EC2FASTINSTALL="false" -FASTINSTALL="false" while getopts 'hH-:' opt ; do case $opt in @@ -64,33 +56,6 @@ elif [ -n "$1" ] ; then TOOLCHAIN="$1" fi - -if [ "$EC2FASTINSTALL" = "true" ]; then - if [ "$TOOLCHAIN" = "riscv-tools" ]; then - cd "$RDIR" - git clone https://github.com/firesim/firesim-riscv-tools-prebuilt.git - cd firesim-riscv-tools-prebuilt - git checkout "$PRECOMPILED_REPO_HASH" - PREBUILTHASH="$(cat HASH)" - git -C "${CHIPYARD_DIR}" submodule update --init "toolchains/${TOOLCHAIN}" - cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN" - GITHASH="$(git rev-parse HEAD)" - cd "$RDIR" - echo "prebuilt hash: $PREBUILTHASH" - echo "git hash: $GITHASH" - if [[ $PREBUILTHASH == $GITHASH && "$EC2FASTINSTALL" == "true" ]]; then - FASTINSTALL=true - echo "Using fast pre-compiled install for riscv-tools" - else - error 'error: hash of precompiled toolchain does not match the riscv-tools submodule hash' - exit -1 - fi - else - error "error: unsupported precompiled toolchain: ${TOOLCHAIN}" - exit -1 - fi -fi - INSTALL_DIR="$TOOLCHAIN-install" RISCV="$(pwd)/$INSTALL_DIR" @@ -98,52 +63,57 @@ RISCV="$(pwd)/$INSTALL_DIR" # install risc-v tools export RISCV="$RISCV" -if [ "$FASTINSTALL" = true ]; then - cd firesim-riscv-tools-prebuilt - ./installrelease.sh - mv distrib "$RISCV" - # copy HASH in case user wants it later - cp HASH "$RISCV" - cd "$RDIR" - rm -rf firesim-riscv-tools-prebuilt +cd "${CHIPYARD_DIR}" + +SRCDIR="$(pwd)/toolchains/${TOOLCHAIN}" +[ -d "${SRCDIR}" ] || die "unsupported toolchain: ${TOOLCHAIN}" +. ./scripts/build-util.sh + + +if [ "${EC2FASTINSTALL}" = true ] ; then + [ "${TOOLCHAIN}" = 'riscv-tools' ] || + die "unsupported precompiled toolchain: ${TOOLCHAIN}" + + echo '=> Fetching pre-built toolchain' + module=toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt + git config --unset submodule."${module}".update || : + git submodule update --init --depth 1 "${module}" + + echo '==> Verifying toolchain version hash' + # Find commit hash without initializing the submodule + hashsrc="$(git ls-tree -d HEAD "${SRCDIR}/riscv-gnu-toolchain" | { + unset IFS && read -r _ type obj _ && + test -n "${obj}" && test "${type}" = 'commit' && echo "${obj}" + }; )" || + die 'failed to obtain riscv-gnu-toolchain submodule hash' "$?" + + read -r hashbin < "${module}/HASH" || + die 'failed to obtain riscv-gnu-toolchain-prebuilt hash' "$?" + + echo "==> ${hashsrc}" + [ "${hashsrc}" = "${hashbin}" ] || + die "pre-built version mismatch: ${hashbin}" + + echo '==> Installing pre-built toolchain' + "${MAKE}" -C "${module}" DESTDIR="${RISCV}" install + git submodule deinit "${module}" || : + else - mkdir -p "$RISCV" - git -C "${CHIPYARD_DIR}" submodule update --init --recursive "toolchains/${TOOLCHAIN}" #--jobs 8 - cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN" - - # Scale number of parallel make jobs by hardware thread count - ncpu="$(getconf _NPROCESSORS_ONLN || # GNU - getconf NPROCESSORS_ONLN || # *BSD, Solaris - nproc --all || # Linux - sysctl -n hw.ncpu || # *BSD, OS X - :)" 2>/dev/null - case ${ncpu} in - ''|*[^0-9]*) ;; # Ignore non-integer values - *) export MAKEFLAGS="-j ${ncpu}" ;; - esac - - #build the actual toolchain - #./build.sh - source build.common - echo "Starting RISC-V Toolchain build process" - build_project riscv-fesvr --prefix="${RISCV}" - build_project riscv-isa-sim --prefix="${RISCV}" --with-fesvr="${RISCV}" - build_project riscv-gnu-toolchain --prefix="${RISCV}" - CC= CXX= build_project riscv-pk --prefix="${RISCV}" --host=riscv64-unknown-elf - build_project riscv-tests --prefix="${RISCV}/riscv64-unknown-elf" - echo -e "\\nRISC-V Toolchain installation completed!" - - # build static libfesvr library for linking into firesim driver (or others) - cd riscv-fesvr/build - "${CHIPYARD_DIR}/scripts/build-static-libfesvr.sh" - cd "$RDIR" - # build linux toolchain - cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN/riscv-gnu-toolchain/build" - make linux - echo -e "\\nRISC-V Linux GNU Toolchain installation completed!" - + module_prepare riscv-gnu-toolchain qemu + module_build riscv-gnu-toolchain --prefix="${RISCV}" + echo '==> Building GNU/Linux toolchain' + module_make riscv-gnu-toolchain linux fi +module_all riscv-isa-sim --prefix="${RISCV}" +# build static libfesvr library for linking into firesim driver (or others) +echo '==> Installing libfesvr static library' +module_make riscv-isa-sim libfesvr.a +cp -p "${SRCDIR}/riscv-isa-sim/build/libfesvr.a" "${RISCV}/lib/" + +CC= CXX= module_all riscv-pk --prefix="${RISCV}" --host=riscv64-unknown-elf +module_all riscv-tests --prefix="${RISCV}/riscv64-unknown-elf" + cd "$RDIR" { @@ -153,20 +123,3 @@ cd "$RDIR" echo "export LD_LIBRARY_PATH=\${RISCV}/lib\${LD_LIBRARY_PATH:+":\${LD_LIBRARY_PATH}"}" } > env.sh echo "Toolchain Build Complete!" - -if [ "$FASTINSTALL" = "false" ]; then - # commands that can't run on EC2 (specifically, OpenOCD because of autoconf version_ - # see if the instance info page exists. if not, we are not on ec2. - # this is one of the few methods that works without sudo - if wget -T 1 -t 3 -O /dev/null http://169.254.169.254/; then - echo "Skipping RISC-V OpenOCD" - else - echo "Building RISC-V OpenOCD" - cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN" - check_version automake 1.14 "OpenOCD build" - check_version autoconf 2.64 "OpenOCD build" - build_project riscv-openocd --prefix="${RISCV}" --enable-remote-bitbang --enable-jtag_vpi --disable-werror - echo -e "\\nRISC-V OpenOCD installation completed!" - cd "$RDIR" - fi -fi diff --git a/scripts/build-util.sh b/scripts/build-util.sh new file mode 100644 index 00000000..5ca38147 --- /dev/null +++ b/scripts/build-util.sh @@ -0,0 +1,89 @@ +# Derived from +# https://github.com/riscv/riscv-tools/blob/master/build.common + +[ -n "${SRCDIR}" ] || exit 1 + +# Scale number of parallel make jobs by hardware thread count +ncpu="${NPROC:-$(getconf _NPROCESSORS_ONLN || # GNU + getconf NPROCESSORS_ONLN || # *BSD, Solaris + nproc --all || # Linux + sysctl -n hw.ncpu || # *BSD, OS X + :)}" 2>/dev/null +case ${ncpu} in +''|*[!0-9]*) ;; # Ignore non-integer values +*) export MAKEFLAGS="-j ${ncpu} ${MAKEFLAGS}" ;; +esac + +MAKE=$(command -v gmake || command -v make) +readonly MAKE + + +module_prepare() ( # [ignored-submodule..] + set -e + name=$1 + shift + + dir="${SRCDIR}/${name}" + echo "=> Starting ${name} build" + echo "==> Initializing ${name} submodule" + if [ $# -gt 0 ] ; then + git submodule update --init "${dir}" + while [ -n "$1" ] ; do + git -C "${dir}" config submodule."${1}".update none + shift + done + fi + git submodule update --init --recursive "${dir}" +) + +module_run() ( # + set -e + cd "${SRCDIR}/${1}" + shift + "$@" +) + +module_make() ( # + set -e -o pipefail + cd "${SRCDIR}/${1}/build" + shift + "${MAKE}" "$@" | tee "build-${1:-make}.log" +) + +module_build() ( # [configure-arg..] + set -e -o pipefail + name=$1 + shift + + cd "${SRCDIR}/${name}" + + if [ -e build ] ; then + echo "==> Removing existing ${name}/build directory" + rm -rf build + fi + if ! [ -e configure ] ; then + echo "==> Updating autoconf files for ${name}" + find . -iname configure.ac -type f -print0 | + while read -r -d '' file ; do + mkdir -p -- "${file%/*}/m4" + done + autoreconf -i + fi + + mkdir -p build + cd build + { + export PATH="${RISCV:+${RISCV}/bin:}${PATH}" + echo "==> Configuring ${name}" + ../configure "$@" + echo "==> Building ${name}" + "${MAKE}" + echo "==> Installing ${name}" + "${MAKE}" install + } 2>&1 | tee build.log +) + +module_all() { # [configure-arg..] + module_prepare "$1" + module_build "$@" +} diff --git a/scripts/init-submodules-no-riscv-tools.sh b/scripts/init-submodules-no-riscv-tools.sh index cf3244c4..e17b67ef 100755 --- a/scripts/init-submodules-no-riscv-tools.sh +++ b/scripts/init-submodules-no-riscv-tools.sh @@ -4,15 +4,13 @@ set -e set -o pipefail -unamestr=$(uname) -RDIR=$(pwd) -scripts_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +RDIR=$(git rev-parse --show-toplevel) -# ignore riscv-tools for submodule init recursive -# you must do this globally (otherwise riscv-tools deep -# in the submodule tree will get pulled anyway -git config submodule.toolchains/riscv-tools.update none -git config submodule.toolchains/esp-tools.update none +# Ignore toolchain submodules +cd "$RDIR" +for name in toolchains/*/*/ ; do + git config submodule."${name%/}".update none +done # Disable updates to the FireSim submodule until explicitly requested git config submodule.sims/firesim.update none # Disable updates to the hammer tool plugins repos @@ -20,9 +18,10 @@ git config submodule.vlsi/hammer-cadence-plugins.update none git config submodule.vlsi/hammer-synopsys-plugins.update none git config submodule.vlsi/hammer-mentor-plugins.update none git submodule update --init --recursive #--jobs 8 -# unignore riscv-tools,catapult-shell2 globally -git config --unset submodule.toolchains/riscv-tools.update -git config --unset submodule.toolchains/esp-tools.update +# Un-ignore toolchain submodules +for name in toolchains/*/*/ ; do + git config --unset submodule."${name%/}".update +done git config --unset submodule.vlsi/hammer-cadence-plugins.update git config --unset submodule.vlsi/hammer-synopsys-plugins.update git config --unset submodule.vlsi/hammer-mentor-plugins.update @@ -30,9 +29,6 @@ git config --unset submodule.vlsi/hammer-mentor-plugins.update # Renable firesim and init only the required submodules to provide # all required scala deps, without doing a full build-setup git config --unset submodule.sims/firesim.update -cd "${scripts_dir}/../sims" -git submodule update --init firesim -cd firesim/sim -git submodule update --init midas -cd "$RDIR" +git submodule update --init sims/firesim +git -C sims/firesim submodule update --init sim/midas git config submodule.sims/firesim.update none diff --git a/toolchains/esp-tools b/toolchains/esp-tools deleted file mode 160000 index dcb6012f..00000000 --- a/toolchains/esp-tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit dcb6012f77101e793948cc90ac31b3735a9f3f6d diff --git a/toolchains/riscv-tools b/toolchains/riscv-tools deleted file mode 160000 index bce7b5e3..00000000 --- a/toolchains/riscv-tools +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bce7b5e363957c134f944769e677354467b7e4ed diff --git a/toolchains/riscv-tools/riscv-gnu-toolchain b/toolchains/riscv-tools/riscv-gnu-toolchain new file mode 160000 index 00000000..2855d823 --- /dev/null +++ b/toolchains/riscv-tools/riscv-gnu-toolchain @@ -0,0 +1 @@ +Subproject commit 2855d823a6e93d50af604264b02ced951e80de67 diff --git a/toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt b/toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt new file mode 160000 index 00000000..5e32a015 --- /dev/null +++ b/toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt @@ -0,0 +1 @@ +Subproject commit 5e32a0157f91ebfb5c7ea7113fce28bf40016fa4 diff --git a/toolchains/riscv-tools/riscv-isa-sim b/toolchains/riscv-tools/riscv-isa-sim new file mode 160000 index 00000000..9443c1db --- /dev/null +++ b/toolchains/riscv-tools/riscv-isa-sim @@ -0,0 +1 @@ +Subproject commit 9443c1dbac0301faf3a47c5e6914cc7dcb34983e diff --git a/toolchains/riscv-tools/riscv-pk b/toolchains/riscv-tools/riscv-pk new file mode 160000 index 00000000..a3e4ac61 --- /dev/null +++ b/toolchains/riscv-tools/riscv-pk @@ -0,0 +1 @@ +Subproject commit a3e4ac61d2b1ff37a22b9193b85d3b94273e80cb diff --git a/toolchains/riscv-tools/riscv-tests b/toolchains/riscv-tools/riscv-tests new file mode 160000 index 00000000..249796ce --- /dev/null +++ b/toolchains/riscv-tools/riscv-tests @@ -0,0 +1 @@ +Subproject commit 249796cec94d75ff10ca034153e206a319e87158