toolchains: Flatten riscv-tools submodule

This allows individual components to be better maintained following the
deprecation of riscv-tools.  Eliminate non-essential submodules.

build-static-libfesvr.sh is no longer necessary since libfesvr.a is
built as part of the riscv-isa-sim build.

For simplicity, only riscv-gnu-toolchain is now pre-built instead of the
entirety of riscv-tools.

.gitmodules

@@ -34,12 +34,22 @@
 [submodule "generators/block-inclusivecache-sifive"]
 	path = generators/sifive-cache
 	url = https://github.com/sifive/block-inclusivecache-sifive.git
-[submodule "toolchains/riscv-tools"]
-	path = toolchains/riscv-tools
-	url = https://github.com/freechipsproject/rocket-tools.git
-[submodule "toolchains/esp-tools"]
-	path = toolchains/esp-tools
-	url = https://github.com/ucb-bar/esp-tools.git
+[submodule "toolchains/riscv-tools/riscv-gnu-toolchain"]
+	path = toolchains/riscv-tools/riscv-gnu-toolchain
+	url = https://github.com/riscv/riscv-gnu-toolchain.git
+[submodule "toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt"]
+	path = toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt
+	url = https://github.com/ucb-bar/chipyard-toolchain-prebuilt.git
+	shallow = true
+[submodule "toolchains/riscv-tools/riscv-isa-sim"]
+	path = toolchains/riscv-tools/riscv-isa-sim
+	url = https://github.com/riscv/riscv-isa-sim.git
+[submodule "toolchains/riscv-tools/riscv-pk"]
+	path = toolchains/riscv-tools/riscv-pk
+	url = https://github.com/riscv/riscv-pk.git
+[submodule "toolchains/riscv-tools/riscv-tests"]
+	path = toolchains/riscv-tools/riscv-tests
+	url = https://github.com/riscv/riscv-tests.git
 [submodule "vlsi/hammer"]
 	path = vlsi/hammer
 	url = https://github.com/ucb-bar/hammer.git

scripts/build-static-libfesvr.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-# This ungodly script surreptitiously builds an archive from existing fesvr objects
-# Invoke from riscv-fesvr/build
-
-if [ "x$RISCV" = "x" ]
-then
-  echo "Please set the RISCV environment variable to your preferred install path."
-  exit 1
-fi
-
-set -e
-
-objs=$(make -n -f <(
-    echo 'include Makefile'
-    echo '$(info $(value fesvr_objs))'
-    ) | head -n 1)
-
-ar rcs -o libfesvr.a $objs
-cp -f libfesvr.a "${RISCV}/lib"
-

scripts/build-toolchains.sh

@@ -6,11 +6,8 @@
 set -e
 set -o pipefail
 
-unamestr=$(uname)
 RDIR=$(pwd)
-: ${CHIPYARD_DIR:=$(pwd)} #default value is the PWD unless overridden
-
-PRECOMPILED_REPO_HASH=56a40961c98db5e8f904f15dc6efd0870bfefd9e
+CHIPYARD_DIR="${CHIPYARD_DIR:-$(git rev-parse --show-toplevel)}"
 
 usage() {
     echo "usage: ${0} [riscv-tools | esp-tools | ec2fast]"
@@ -23,18 +20,13 @@ usage() {
 error() {
     echo "${0##*/}: ${1}" >&2
 }
-
-#taken from riscv-tools to check for open-ocd autoconf versions
-check_version() {
-    "$1" --version | awk "NR==1 {if (\$NF>$2) {exit 0} exit 1}" || {
-        error "${3} requires at least ${1} version ${2}"
-        exit 1
-    }
+die() {
+    error "$1"
+    exit "${2:--1}"
 }
 
 TOOLCHAIN="riscv-tools"
 EC2FASTINSTALL="false"
-FASTINSTALL="false"
 
 while getopts 'hH-:' opt ; do
     case $opt in
@@ -64,33 +56,6 @@ elif [ -n "$1" ] ; then
     TOOLCHAIN="$1"
 fi
 
-
-if [ "$EC2FASTINSTALL" = "true" ]; then
-    if [ "$TOOLCHAIN" = "riscv-tools" ]; then
-      cd "$RDIR"
-      git clone https://github.com/firesim/firesim-riscv-tools-prebuilt.git
-      cd firesim-riscv-tools-prebuilt
-      git checkout "$PRECOMPILED_REPO_HASH"
-      PREBUILTHASH="$(cat HASH)"
-      git -C "${CHIPYARD_DIR}" submodule update --init "toolchains/${TOOLCHAIN}"
-      cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN"
-      GITHASH="$(git rev-parse HEAD)"
-      cd "$RDIR"
-      echo "prebuilt hash: $PREBUILTHASH"
-      echo "git      hash: $GITHASH"
-      if [[ $PREBUILTHASH == $GITHASH && "$EC2FASTINSTALL" == "true" ]]; then
-          FASTINSTALL=true
-          echo "Using fast pre-compiled install for riscv-tools"
-      else
-          error 'error: hash of precompiled toolchain does not match the riscv-tools submodule hash'
-          exit -1
-      fi
-    else
-          error "error: unsupported precompiled toolchain: ${TOOLCHAIN}"
-          exit -1
-    fi
-fi
-
 INSTALL_DIR="$TOOLCHAIN-install"
 
 RISCV="$(pwd)/$INSTALL_DIR"
@@ -98,52 +63,57 @@ RISCV="$(pwd)/$INSTALL_DIR"
 # install risc-v tools
 export RISCV="$RISCV"
 
-if [ "$FASTINSTALL" = true ]; then
-    cd firesim-riscv-tools-prebuilt
-    ./installrelease.sh
-    mv distrib "$RISCV"
-    # copy HASH in case user wants it later
-    cp HASH "$RISCV"
-    cd "$RDIR"
-    rm -rf firesim-riscv-tools-prebuilt
+cd "${CHIPYARD_DIR}"
+
+SRCDIR="$(pwd)/toolchains/${TOOLCHAIN}"
+[ -d "${SRCDIR}" ] || die "unsupported toolchain: ${TOOLCHAIN}"
+. ./scripts/build-util.sh
+
+
+if [ "${EC2FASTINSTALL}" = true ] ; then
+    [ "${TOOLCHAIN}" = 'riscv-tools' ] ||
+        die "unsupported precompiled toolchain: ${TOOLCHAIN}"
+
+    echo '=>  Fetching pre-built toolchain'
+    module=toolchains/riscv-tools/riscv-gnu-toolchain-prebuilt
+    git config --unset submodule."${module}".update || :
+    git submodule update --init --depth 1 "${module}"
+
+    echo '==>  Verifying toolchain version hash'
+    # Find commit hash without initializing the submodule
+    hashsrc="$(git ls-tree -d HEAD "${SRCDIR}/riscv-gnu-toolchain" | {
+        unset IFS && read -r _ type obj _ &&
+        test -n "${obj}" && test "${type}" = 'commit' && echo "${obj}"
+    }; )" ||
+        die 'failed to obtain riscv-gnu-toolchain submodule hash' "$?"
+
+    read -r hashbin < "${module}/HASH" ||
+        die 'failed to obtain riscv-gnu-toolchain-prebuilt hash' "$?"
+
+    echo "==>  ${hashsrc}"
+    [ "${hashsrc}" = "${hashbin}" ] ||
+        die "pre-built version mismatch: ${hashbin}"
+
+    echo '==>  Installing pre-built toolchain'
+    "${MAKE}" -C "${module}" DESTDIR="${RISCV}" install
+    git submodule deinit "${module}" || :
+
 else
-    mkdir -p "$RISCV"
-    git -C "${CHIPYARD_DIR}" submodule update --init --recursive "toolchains/${TOOLCHAIN}" #--jobs 8
-    cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN"
-
-    # Scale number of parallel make jobs by hardware thread count
-    ncpu="$(getconf _NPROCESSORS_ONLN || # GNU
-        getconf NPROCESSORS_ONLN || # *BSD, Solaris
-        nproc --all || # Linux
-        sysctl -n hw.ncpu || # *BSD, OS X
-        :)" 2>/dev/null
-    case ${ncpu} in
-    ''|*[^0-9]*) ;; # Ignore non-integer values
-    *) export MAKEFLAGS="-j ${ncpu}" ;;
-    esac
-
-    #build the actual toolchain
-    #./build.sh
-    source build.common
-    echo "Starting RISC-V Toolchain build process"
-    build_project riscv-fesvr --prefix="${RISCV}"
-    build_project riscv-isa-sim --prefix="${RISCV}" --with-fesvr="${RISCV}"
-    build_project riscv-gnu-toolchain --prefix="${RISCV}"
-    CC= CXX= build_project riscv-pk --prefix="${RISCV}" --host=riscv64-unknown-elf
-    build_project riscv-tests --prefix="${RISCV}/riscv64-unknown-elf"
-    echo -e "\\nRISC-V Toolchain installation completed!"
-
-    # build static libfesvr library for linking into firesim driver (or others)
-    cd riscv-fesvr/build
-    "${CHIPYARD_DIR}/scripts/build-static-libfesvr.sh"
-    cd "$RDIR"
-    # build linux toolchain
-    cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN/riscv-gnu-toolchain/build"
-    make linux
-    echo -e "\\nRISC-V Linux GNU Toolchain installation completed!"
-
+    module_prepare riscv-gnu-toolchain qemu
+    module_build riscv-gnu-toolchain --prefix="${RISCV}"
+    echo '==>  Building GNU/Linux toolchain'
+    module_make riscv-gnu-toolchain linux
 fi
 
+module_all riscv-isa-sim --prefix="${RISCV}"
+# build static libfesvr library for linking into firesim driver (or others)
+echo '==>  Installing libfesvr static library'
+module_make riscv-isa-sim libfesvr.a
+cp -p "${SRCDIR}/riscv-isa-sim/build/libfesvr.a" "${RISCV}/lib/"
+
+CC= CXX= module_all riscv-pk --prefix="${RISCV}" --host=riscv64-unknown-elf
+module_all riscv-tests --prefix="${RISCV}/riscv64-unknown-elf"
+
 cd "$RDIR"
 
 {
@@ -153,20 +123,3 @@ cd "$RDIR"
     echo "export LD_LIBRARY_PATH=\${RISCV}/lib\${LD_LIBRARY_PATH:+":\${LD_LIBRARY_PATH}"}"
 } > env.sh
 echo "Toolchain Build Complete!"
-
-if [ "$FASTINSTALL" = "false" ]; then
-    # commands that can't run on EC2 (specifically, OpenOCD because of autoconf version_
-    # see if the instance info page exists. if not, we are not on ec2.
-    # this is one of the few methods that works without sudo
-    if wget -T 1 -t 3 -O /dev/null http://169.254.169.254/; then
-        echo "Skipping RISC-V OpenOCD"
-    else
-        echo "Building RISC-V OpenOCD"
-        cd "$CHIPYARD_DIR/toolchains/$TOOLCHAIN"
-        check_version automake 1.14 "OpenOCD build"
-        check_version autoconf 2.64 "OpenOCD build"
-        build_project riscv-openocd --prefix="${RISCV}" --enable-remote-bitbang --enable-jtag_vpi --disable-werror
-        echo -e "\\nRISC-V OpenOCD installation completed!"
-        cd "$RDIR"
-    fi
-fi

scripts/build-util.sh

@@ -0,0 +1,89 @@
+# Derived from
+# https://github.com/riscv/riscv-tools/blob/master/build.common
+
+[ -n "${SRCDIR}" ] || exit 1
+
+# Scale number of parallel make jobs by hardware thread count
+ncpu="${NPROC:-$(getconf _NPROCESSORS_ONLN || # GNU
+    getconf NPROCESSORS_ONLN || # *BSD, Solaris
+    nproc --all || # Linux
+    sysctl -n hw.ncpu || # *BSD, OS X
+    :)}" 2>/dev/null
+case ${ncpu} in
+''|*[!0-9]*) ;; # Ignore non-integer values
+*) export MAKEFLAGS="-j ${ncpu} ${MAKEFLAGS}" ;;
+esac
+
+MAKE=$(command -v gmake || command -v make)
+readonly MAKE
+
+
+module_prepare() ( # <submodule> [ignored-submodule..]
+    set -e
+    name=$1
+    shift
+
+    dir="${SRCDIR}/${name}"
+    echo "=>  Starting ${name} build"
+    echo "==>  Initializing ${name} submodule"
+    if [ $# -gt 0 ] ; then
+        git submodule update --init "${dir}"
+        while [ -n "$1" ] ; do
+            git -C "${dir}" config submodule."${1}".update none
+            shift
+        done
+    fi
+    git submodule update --init --recursive "${dir}"
+)
+
+module_run() ( # <submodule> <command..>
+    set -e
+    cd "${SRCDIR}/${1}"
+    shift
+    "$@"
+)
+
+module_make() ( # <submodule> <target..>
+    set -e -o pipefail
+    cd "${SRCDIR}/${1}/build"
+    shift
+    "${MAKE}" "$@" | tee "build-${1:-make}.log"
+)
+
+module_build() ( # <submodule> [configure-arg..]
+    set -e -o pipefail
+    name=$1
+    shift
+
+    cd "${SRCDIR}/${name}"
+
+    if [ -e build ] ; then
+        echo "==>  Removing existing ${name}/build directory"
+        rm -rf build
+    fi
+    if ! [ -e configure ] ; then
+        echo "==>  Updating autoconf files for ${name}"
+        find . -iname configure.ac -type f -print0 |
+        while read -r -d '' file ; do
+            mkdir -p -- "${file%/*}/m4"
+        done
+        autoreconf -i
+    fi
+
+    mkdir -p build
+    cd build
+    {
+        export PATH="${RISCV:+${RISCV}/bin:}${PATH}"
+        echo "==>  Configuring ${name}"
+        ../configure "$@"
+        echo "==>  Building ${name}"
+        "${MAKE}"
+        echo "==>  Installing ${name}"
+        "${MAKE}" install
+    } 2>&1 | tee build.log
+)
+
+module_all() { # <submodule> [configure-arg..]
+    module_prepare "$1"
+    module_build "$@"
+}

scripts/init-submodules-no-riscv-tools.sh

@@ -4,15 +4,13 @@
 set -e
 set -o pipefail
 
-unamestr=$(uname)
-RDIR=$(pwd)
-scripts_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+RDIR=$(git rev-parse --show-toplevel)
 
-# ignore riscv-tools for submodule init recursive
-# you must do this globally (otherwise riscv-tools deep
-# in the submodule tree will get pulled anyway
-git config submodule.toolchains/riscv-tools.update none
-git config submodule.toolchains/esp-tools.update none
+# Ignore toolchain submodules
+cd "$RDIR"
+for name in toolchains/*/*/ ; do
+    git config submodule."${name%/}".update none
+done
 # Disable updates to the FireSim submodule until explicitly requested
 git config submodule.sims/firesim.update none
 # Disable updates to the hammer tool plugins repos
@@ -20,9 +18,10 @@ git config submodule.vlsi/hammer-cadence-plugins.update none
 git config submodule.vlsi/hammer-synopsys-plugins.update none
 git config submodule.vlsi/hammer-mentor-plugins.update none
 git submodule update --init --recursive #--jobs 8
-# unignore riscv-tools,catapult-shell2 globally
-git config --unset submodule.toolchains/riscv-tools.update
-git config --unset submodule.toolchains/esp-tools.update
+# Un-ignore toolchain submodules
+for name in toolchains/*/*/ ; do
+    git config --unset submodule."${name%/}".update
+done
 git config --unset submodule.vlsi/hammer-cadence-plugins.update
 git config --unset submodule.vlsi/hammer-synopsys-plugins.update
 git config --unset submodule.vlsi/hammer-mentor-plugins.update
@@ -30,9 +29,6 @@ git config --unset submodule.vlsi/hammer-mentor-plugins.update
 # Renable firesim and init only the required submodules to provide
 # all required scala deps, without doing a full build-setup
 git config --unset submodule.sims/firesim.update
-cd "${scripts_dir}/../sims"
-git submodule update --init firesim
-cd firesim/sim
-git submodule update --init midas
-cd "$RDIR"
+git submodule update --init sims/firesim
+git -C sims/firesim submodule update --init sim/midas
 git config submodule.sims/firesim.update none