Rework build-setup | Add single-node CI

scripts/build-setup.sh

@@ -14,28 +14,45 @@ common_setup
 usage() {
     echo "Usage: ${0} [OPTIONS] [riscv-tools | esp-tools]"
     echo ""
-    echo "Helper script to initialize repository that wraps other scripts."
-    echo "Sets up conda environment, initializes submodules, and installs toolchain collateral."
-    echo ""
     echo "Installation Types"
     echo "  riscv-tools: if set, builds the riscv toolchain (this is also the default)"
     echo "  esp-tools: if set, builds esp-tools toolchain used for the hwacha vector accelerator"
     echo ""
+    echo "Helper script to fully initialize repository that wraps other scripts."
+    echo "By default it initializes/installs things in the following order:"
+    echo "   1. Conda environment"
+    echo "   2. Chipyard submodules"
+    echo "   3. Toolchain collateral (Spike, PK, tests, libgloss)"
+    echo "   4. Ctags"
+    echo "   5. Chipyard pre-compile sources"
+    echo "   6. FireSim"
+    echo "   7. FireSim pre-compile sources"
+    echo "   8. FireMarshal"
+    echo "   9. FireMarshal pre-compile default buildroot Linux sources"
+    echo "  10. Runs repository clean-up"
+    echo ""
+    echo "**See below for options to skip parts of the setup. Skipping parts of the setup is not guaranteed to be tested/working.**"
+    echo ""
     echo "Options"
-    echo "  --help -h              : Display this message"
-    echo "  --unpinned-deps -ud    : Use unpinned conda environment"
-    echo "  --force -f             : Skip prompt checking for tagged release/conda"
-    echo "  --skip-validate        : DEPRECATED: Same functionality as --force"
-    echo "  --skip-conda           : Skip conda env creation"
-    echo "  --skip-toolchain-extra : Skip building extra RISC-V toolchain collateral (Spike, PK, tests, libgloos)"
+    echo "  --help -h               : Display this message"
+
+    echo "  --force -f              : Skip all prompts and checks"
+    echo "  --skip-validate         : DEPRECATED: Same functionality as --force"
+    echo "  --verbose -v            : Verbose printout"
+
+    echo "  --use-unpinned-deps -ud : Use unpinned conda environment"
+
+    echo "  --skip -s N             : Skip step N in the list above. Use multiple times to skip multiple steps ('-s N -s M ...')."
+
     exit "$1"
 }
 
-TOOLCHAIN="riscv-tools"
-USE_PINNED_DEPS=true
+TOOLCHAIN_TYPE="riscv-tools"
 FORCE_FLAG=""
-SKIP_CONDA=false
-SKIP_TOOLCHAIN=false
+VERBOSE=false
+VERBOSE_FLAG=""
+USE_UNPINNED_DEPS=false
+SKIP_LIST=()
 
 # getopts does not support long options, and is inflexible
 while [ "$1" != "" ];
@@ -44,15 +61,17 @@ do
         -h | --help )
             usage 3 ;;
         riscv-tools | esp-tools)
-            TOOLCHAIN=$1 ;;
-        -ud | --unpinned-deps )
-            USE_PINNED_DEPS=false ;;
+            TOOLCHAIN_TYPE=$1 ;;
         --force | -f | --skip-validate)
             FORCE_FLAG=$1 ;;
-        --skip-conda)
-            SKIP_CONDA=true ;;
-        --skip-toolchain-extra)
-            SKIP_TOOLCHAIN=true ;;
+        --verbose | -v)
+            VERBOSE_FLAG=$1
+            set -x ;;
+        -ud | --use-unpinned-deps )
+            USE_UNPINNED_DEPS=true ;;
+        --skip | -s)
+            shift
+            SKIP_LIST+=(${1}) ;;
         * )
             error "invalid option $1"
             usage 1 ;;
@@ -60,15 +79,23 @@ do
     shift
 done
 
-if [ "$SKIP_CONDA" = false ]; then
+# check if the arg is found in the SKIP_LIST
+do_skip() {
+    local value=$1
+    [[ ! " ${SKIP_LIST[*]} " =~ " ${value} " ]]
+}
+
+{
+
+if do_skip "1"; then
     # note: lock file must end in .conda-lock.yml - see https://github.com/conda-incubator/conda-lock/issues/154
     CONDA_REQS=$RDIR/conda-reqs
     CONDA_LOCK_REQS=$CONDA_REQS/conda-lock-reqs
-    LOCKFILE=$CONDA_LOCK_REQS/conda-requirements-$TOOLCHAIN-linux-64.conda-lock.yml
+    LOCKFILE=$CONDA_LOCK_REQS/conda-requirements-$TOOLCHAIN_TYPE-linux-64.conda-lock.yml
 
-    if [ "$USE_PINNED_DEPS" = false ]; then
+    if [ "$USE_UNPINNED_DEPS" = true ]; then
         # auto-gen the lockfile
-        conda-lock -f $CONDA_REQS/chipyard.yaml -f $CONDA_REQS/$TOOLCHAIN.yaml --lockfile $LOCKFILE
+        conda-lock -f $CONDA_REQS/chipyard.yaml -f $CONDA_REQS/$TOOLCHAIN_TYPE.yaml --lockfile $LOCKFILE
     fi
 
     # use conda-lock to create env
@@ -85,16 +112,63 @@ if [ -z "$FORCE_FLAG" ]; then
     fi
 fi
 
-$RDIR/scripts/init-submodules-no-riscv-tools.sh $FORCE_FLAG
-
-if [ "$SKIP_TOOLCHAIN" = false ]; then
-    $RDIR/scripts/build-toolchain-extra.sh $FORCE_FLAG $TOOLCHAIN
+if do_skip "2"; then
+    $RDIR/scripts/init-submodules-no-riscv-tools.sh $FORCE_FLAG
 fi
 
-$RDIR/scripts/gen-tags.sh
+if do_skip "3"; then
+    $RDIR/scripts/build-toolchain-extra.sh $FORCE_FLAG $TOOLCHAIN_TYPE
+fi
 
-cat << EOT >> env.sh
+if do_skip "4"; then
+    $RDIR/scripts/gen-tags.sh
+fi
+
+if do_skip "5"; then
+    pushd $RDIR/sims/verilator
+    make launch-sbt SBT_COMMAND=";project chipyard; compile"
+    make launch-sbt SBT_COMMAND=";project tapeout; compile"
+    popd
+fi
+
+if do_skip "6"; then
+    $RDIR/scripts/firesim-setup.sh
+    $RDIR/sims/firesim/gen-tags.sh
+
+    if do_skip "7"; then
+        pushd $RDIR/sims/firesim
+        (
+            source sourceme-f1-manager.sh --skip-ssh-setup
+            pushd sim
+            make sbt SBT_COMMAND="project firechip; compile" TARGET_PROJECT=firesim
+            popd
+        )
+        popd
+    fi
+fi
+
+if do_skip "8"; then
+    pushd $RDIR/software/firemarshal
+    ./init-submodules.sh
+
+    if do_skip "9"; then
+        source $RDIR/scripts/fix-open-files.sh
+        ./marshal $VERBOSE_FLAG build br-base.json
+        ./marshal $VERBOSE_FLAG clean br-base.json
+    fi
+    popd
+fi
+
+if do_skip "10"; then
+    $RDIR/scripts/repo-clean.sh
+fi
+
+cat <<EOT >> env.sh
 # line auto-generated by $0
 conda activate $RDIR/.conda-env
 source $RDIR/scripts/fix-open-files.sh
 EOT
+
+echo "Setup complete!"
+
+} 2>&1 | tee build-setup.log