Use lockfiles for conda

.github/CI_README.md

@@ -22,7 +22,7 @@ For example:
 This specifies that the `prepare-chipyard-cores` job needs the both the `make-keys` and the `setup-complete` steps to
 be completed before it can run.
 
-Chipyard runs its CI using a docker image created from `dockerfiles/Dockerfile` and on Berkeley's millennium machines.
+Chipyard runs its CI using a docker image created from `dockerfiles/Dockerfile` and on Berkeley's compute infrastructure.
 See its [README](../dockerfiles/README.md) for more details.
 
 Finally, within each job's `steps:` section, the steps are run sequentially and state persists throughout a job.

.github/PULL_REQUEST_TEMPLATE.md

@@ -37,6 +37,7 @@ Provide a brief description of the PR immediately below this comment, if the tit
 - [ ] Did you state the type-of-change/impact?
 - [ ] Did you delete any extraneous prints/debugging code?
 - [ ] Did you mark the PR with a `changelog:` label?
+- [ ] (If applicable) Did you update the conda `.conda-lock.yml` file if you updated the conda requirements file?
 - [ ] (If applicable) Did you add documentation for the feature?
 - [ ] (If applicable) Did you add a test demonstrating the PR?
 <!-- Do this if this PR is a bugfix that should be applied to the latest release -->

.github/actions/create-conda-env/action.yml

@@ -10,12 +10,10 @@ runs:
           echo "Using pre-existing conda environments with prefix ${{ env.conda-env-name-no-time }}"
         else
           echo "Creating a conda environment for each toolchain with the toolchain installed"
-          conda env create -f ./scripts/conda-requirements.yaml -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools
-          conda install -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools -c ucb-bar -y riscv-tools
-          conda env create -f ./scripts/conda-requirements.yaml -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-esp-tools
-          conda install -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-esp-tools -c ucb-bar -y esp-tools
+          conda-lock install -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools ./conda-requirements-riscv-tools-linux-64.conda-lock.yml
+          conda-lock install -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-esp-tools ./conda-requirements-esp-tools-linux-64.conda-lock.yml
 
-          echo "Add source collateral to RISC-V area"
+          echo "Add extra toolchain collateral to RISC-V install area"
           conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools
           ./scripts/build-toolchain-extra.sh riscv-tools
           conda deactivate

.github/scripts/install-conda.sh

@@ -159,6 +159,10 @@ else
     # conda-libmamba-solver is a special case and must always be installed into the base environment
     # see https://www.anaconda.com/blog/a-faster-conda-for-a-growing-community
     $SUDO "$CONDA_EXE" install $DRY_RUN_OPTION -y -n base conda-libmamba-solver
+
+    # conda-lock is a special case and must always be installed into the base environment
+    $SUDO "$CONDA_EXE" install $DRY_RUN_OPTION -y -n base conda-lock
+
     # Use the fast solver by default
     "${DRY_RUN_ECHO[@]}" $SUDO "$CONDA_EXE" config --system --set experimental_solver libmamba
 

conda-requirements-esp-tools.yaml

@@ -0,0 +1,61 @@
+channels:
+  - ucb-bar
+  - conda-forge
+  - nodefaults
+dependencies:
+  # RISC-V ESP project compiler
+  - esp-tools
+
+  # firemarshal deps
+  - python=3.9
+  - rsync
+  - psutil
+  - doit=0.35.0
+  - gitpython
+  - humanfriendly
+  - e2fsprogs
+  - ctags
+  - bison
+  - flex
+  - expat
+  # current version of buildroot won't build with make 4.3 https://github.com/firesim/FireMarshal/issues/236
+  - make!=4.3
+  - pyyaml
+  - unzip
+  - readline
+  - coreutils
+  - lzop
+  - qemu # from ucb-bar channel - https://github.com/ucb-bar/qemu-feedstock
+
+  # doc requirements
+  - sphinx
+  - pygments
+  - sphinx-autobuild
+  - sphinx_rtd_theme
+  - docutils
+
+  # misc. c/c++ compilers + related
+  - gcc==10.* # pinned for libelf/libdwarf builds
+  - gxx==10.* # pinned for libelf/libdwarf builds
+  - conda-gcc-specs
+  - binutils
+
+  # rocket-chip deps
+  - sbt
+  - openjdk
+  - dtc
+  - verilator==4.034
+
+  # chipyard deps
+  - dromajo # from ucb-bar channel - https://github.com/riscv-boom/dromajo
+
+  # other misc. deps
+  - ca-certificates
+  - vim
+  - gengetopt
+  - cmake
+  - git
+  - wget
+  - sed
+  - autoconf
+  - conda-lock

conda-requirements-riscv-tools.yaml

@@ -3,6 +3,9 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
+  # RISC-V default compiler
+  - riscv-tools
+
   # firemarshal deps
   - python=3.9
   - rsync
@@ -55,3 +58,4 @@ dependencies:
   - wget
   - sed
   - autoconf
+  - conda-lock

dockerfiles/Dockerfile

@@ -38,17 +38,12 @@ FROM base as base-with-tools
 
 SHELL ["/bin/bash", "-cl"]
 
-RUN conda env create -f ./chipyard/scripts/conda-requirements.yaml
-
-# Install riscv-tools
-RUN conda install -n chipyard -c ucb-bar ucb-bar-riscv-tools
-
-SHELL ["/opt/conda/bin/conda", "run", "-n", "chipyard", "/bin/bash", "-cl"]
-
-# Init submodules
+# Initialize repo
 RUN cd chipyard && \
         export MAKEFLAGS=-"j $(nproc)" && \
-        ./scripts/init-submodules-no-riscv-tools.sh --skip-validate 1>/dev/null
+        ./setup.sh --env-name chipyard --skip-validate
+
+SHELL ["/opt/conda/bin/conda", "run", "-n", "chipyard", "/bin/bash", "-cl"]
 
 # Set up FireMarshal. Building and cleaning br-base.json builds the underlying
 # buildroot image (which takes a long time) but doesn't keep all the br-base

docs/Chipyard-Basics/Initial-Repo-Setup.rst

@@ -41,6 +41,13 @@ After Conda is installed and is on your ``PATH``, we need to install a version o
 For this you can use the system package manager like ``yum`` or ``apt`` to install ``git``.
 This ``git`` is only used to first checkout the repository, we will later install a newer version of ``git`` with Conda.
 
+Finally we need to install ``conda-lock`` into the ``base`` conda environment.
+This is done by the following:
+
+.. code-block:: shell
+
+    conda install -n base conda-lock
+
 Setting up the Chipyard Repo
 -------------------------------------------
 
@@ -54,23 +61,36 @@ Start by checking out the proper Chipyard's version. Run:
     # note: this may not be the latest release if the documentation version != "stable"
     git checkout |version|
 
-If you are running Chipyard alongside FireSim on AWS EC2, you should skip the :ref:`Chipyard-Basics/Initial-Repo-Setup:Conda Environment Setup` section and instead jump to :ref:`Chipyard-Basics/Initial-Repo-Setup:Fetch Chipyard Sources`.
-
-Conda Environment Setup
-~~~~~~~~~~~~~~~~~~~~~~~
-
 .. Warning:: When running on an Amazon Web Services EC2 FPGA-development instance
     (for FireSim), FireSim includes a similar machine setup script that will install all
     of the aforementioned dependencies (and some additional ones) and will activate the
-    proper conda environment. Skip this section.
+    proper conda environment. **Skip the rest of this section.**
 
-Next run the following command to create Chipyard's Conda environment.
+Next run the following script to create Chipyard's Conda environment including a pre-built RISC-V toolchain.
+There are two toolchains, one for normal RISC-V programs called ``riscv-tools`` which is the one needed for most Chipyard use-cases, and another for Hwacha/Gemmini called ``esp-tools``.
+Run the following script based off which compiler you would like to use.
 
 .. code-block:: shell
 
-    conda env create -f scripts/conda-requirements.yml
+    ./setup.sh --env-name chipyard riscv-tools # or esp-tools
 
-By running the following command you should see a "chipyard" environment listed (the default environment is called "chipyard" and can be modified with ``conda`` arguments).
+This script wraps around the conda environment initialization process and also runs the ``init-submodules-no-riscv-tools.sh`` and ``build-toolchain-extra.sh`` scripts.
+
+The ``init-subodules-no-riscv-tools.sh`` script will initialize and checkout all of the necessary git submodules.
+This will also validate that you are on a tagged branch, otherwise it will prompt for confirmation.
+When updating Chipyard to a new version, you will also want to rerun this script to update the submodules.
+Using ``git`` directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior.
+
+The ``build-toolchain-extra.sh`` script will install extra toolchain utilities/tests used by Chipyard.
+This command builds utilities like Spike, RISC-V Proxy Kernel, libgloss, and RISC-V tests from source for a specific toolchain type.
+
+.. Note:: By default, the ``build-toolchain-extra.sh`` script installs to ``$CONDA_PREFIX/<toolchain-type>``. Thus, if you uninstall the compiler using ``conda remove`` these utilities/tests will also have to be re-installed/built.
+
+.. Note:: If you already have a working conda environment setup, separate Chipyard clones can use that pre-used environment in combination with running the aforementioned scripts yourself (``init-submodules...`` and ``build-toolchain...``).
+
+.. Note:: If you are a power user and would like to build your own compiler/toolchain, you can refer to the https://github.com/ucb-bar/riscv-tools-feedstock and https://github.com/ucb-bar/esp-tools-feedstock repositories (submoduled in the ``toolchains/*`` directories) on how to build the compiler yourself.
+
+By running the following command you should see a "chipyard" environment listed.
 
 .. code-block:: shell
 
@@ -85,52 +105,10 @@ Next go ahead and activate the conda environment that was setup.
 
     conda activate chipyard
 
-We recommend that you add this "activate" command to your ``.bashrc`` (or other environment setup file).
-
-Fetch Chipyard Sources
-~~~~~~~~~~~~~~~~~~~~~~
-
-To fetch all Chipyard sources, run the following:
-
-.. code-block:: shell
-
-    ./scripts/init-submodules-no-riscv-tools.sh
-
-This will initialize and checkout all of the necessary git submodules.
-This will also validate that you are on a tagged branch, otherwise it will prompt for confirmation.
-
-When updating Chipyard to a new version, you will also want to rerun this script to update the submodules.
-Using ``git`` directly will try to initialize all submodules; this is not recommended unless you expressly desire this behavior.
-
-Obtaining a Toolchain
-------------------------
-
-Currently there are two toolchains, one for normal RISC-V programs called ``riscv-tools``, and another for Hwacha/Gemmini called ``esp-tools``.
-To get a basic ``riscv-tools`` compiler installation (which is the only thing needed for most Chipyard use-cases), just the following steps are necessary.
-
-.. code-block:: shell
-
-    conda install -c ucb-bar riscv-tools # for a normal risc-v compiler
-
-.. Note:: If you are planning to use the Hwacha vector unit, or other RoCC-based accelerators, you should obtain the ``esp-tools`` compiler by substituting the ``esp-tools`` argument to the command above.
-
 Once the command is run, the ``PATH``, ``RISCV``, and ``LD_LIBRARY_PATH`` environment variables will be set properly.
 
-.. Note:: If you are a power user and would like to build your own compiler, you can refer to the https://github.com/ucb-bar/riscv-tools-feedstock and https://github.com/ucb-bar/esp-tools-feedstock repositories (submoduled in the ``toolchains`` directory) on how to build a toolchain yourself.
-
 .. Note:: You can deactivate/activate a compiler/toolchain (but keep it installed) by running ``source $CONDA_PREFIX/etc/conda/deactivate.d/deactivate-${PKG_NAME}.sh`` or ``$CONDA_PREFIX/etc/conda/activate.d/activate-${PKG_NAME}.sh`` (``PKG_NAME`` for example is ``ucb-bar-riscv-tools``). This will modify the aforementioned 3 environment variables.
 
-Afterwards, we need to install extra toolchain utilities/tests used by Chipyard.
-This is done by the following:
-
-.. code-block:: shell
-
-    ./scripts/build-toolchain-extra.sh riscv-tools # or esp-tools respectively
-
-This command builds utilities like Spike, RISC-V Proxy Kernel, libgloss, and RISC-V tests from source for a specific toolchain type.
-
-.. Note:: By default, the ``build-toolchain-extra.sh`` script installs to ``$CONDA_PREFIX/<toolchain-type>``. Thus, if you uninstall the compiler using ``conda remove`` these utilities/tests will also have to be re-installed/built.
-
 Sourcing ``env.sh``
 -------------------
 

setup.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+# exit script if any command fails
+set -e
+set -o pipefail
+
+# On macOS, use GNU readlink from 'coreutils' package in Homebrew/MacPorts
+if [ "$(uname -s)" = "Darwin" ] ; then
+    READLINK=greadlink
+else
+    READLINK=readlink
+fi
+
+# If BASH_SOURCE is undefined, we may be running under zsh, in that case
+# provide a zsh-compatible alternative
+DIR="$(dirname "$($READLINK -f "${BASH_SOURCE[0]:-${(%):-%x}}")")"
+
+usage() {
+    echo "Usage: ${0} [OPTIONS] [riscv-tools | esp-tools]"
+    echo ""
+    echo "Helper script to initialize repository that wraps other scripts."
+    echo "Sets up conda environment, initializes submodules, and installs toolchain collateral."
+    echo ""
+    echo "Installation Types"
+    echo "  riscv-tools: if set, builds the riscv toolchain (this is also the default)"
+    echo "  esp-tools: if set, builds esp-tools toolchain used for the hwacha vector accelerator"
+    echo ""
+    echo "Options"
+    echo "  --help -h           : Display this message"
+    echo "  --env-name -en      : Conda environment name to use"
+    echo "  --unpinned-deps -ud : Use unpinned conda environment"
+    echo "  --skip-validate     : Skip prompt checking for tagged release/conda"
+    exit "$1"
+}
+
+TOOLCHAIN="riscv-tools"
+CONDA_ENV_NAME="chipyard"
+USE_PINNED_DEPS=true
+SKIP_VALIDATE_FLAG=""
+
+# getopts does not support long options, and is inflexible
+while [ "$1" != "" ];
+do
+    case $1 in
+        -h | --help )
+            usage 3 ;;
+        riscv-tools | esp-tools)
+            TOOLCHAIN=$1 ;;
+        -en | --env-name )
+            shift
+            ENV_NAME=$1 ;;
+        -ud | --unpinned-deps )
+            USE_PINNED_DEPS=false ;;
+        --skip-validate)
+            SKIP_VALIDATE_FLAG=$1 ;;
+        * )
+            error "invalid option $1"
+            usage 1 ;;
+    esac
+    shift
+done
+
+if [ "$SKIP_VALIDATE" = false ]; then
+    if [ -z ${CONDA_DEFAULT_ENV+x} ]; then
+        error "ERROR: No conda environment detected. Did you activate the conda environment (e.x. 'conda activate base')?"
+        exit 1
+    fi
+fi
+
+# note: lock file must end in .conda-lock.yml - see https://github.com/conda-incubator/conda-lock/issues/154
+LOCKFILE=$DIR/conda-requirements-$TOOLCHAIN-linux-64.conda-lock.yml
+YAMLFILE=$DIR/conda-requirements-$TOOLCHAIN.yaml
+
+if [ "$USE_PINNED_DEPS" = true ]; then
+    # use conda-lock to create env
+    conda-lock install -n $ENV_NAME $LOCKFILE
+else
+    # auto-gen the lockfile
+    conda-lock -f $YAMLFILE -p linux-64 --lockfile $LOCKFILE
+    # use conda-lock to create env
+    conda-lock install -n $ENV_NAME $LOCKFILE
+fi
+
+eval "$(conda shell.bash hook)"
+conda activate $ENV_NAME
+
+$DIR/scripts/init-submodules-no-riscv-tools.sh $SKIP_VALIDATE_FLAG
+$DIR/scripts/build-toolchain-extra.sh $SKIP_VALIDATE_FLAG $TOOLCHAIN