wu-arch/chipyard
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #1750 from ucb-bar/lean-conda

Browse Source 
Use lean conda for faster initial setup
This commit is contained in:
joonho hwangbo
2024-01-22 16:59:17 -08:00
committed by GitHub
parent de206569e6 0bc098496c
commit ccf06cc3c9
8 changed files with 5397 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
conda-reqs/chipyard-base.yaml Normal file
View File

@@ -0,0 +1,35 @@
channels:
- ucb-bar
- conda-forge
- litex-hub
- nodefaults
platforms:
- linux-64
dependencies:
# https://conda-forge.org/feedstock-outputs/
# filterable list of all conda-forge packages
# https://conda-forge.org/#contribute
# instructions on adding a recipe
# https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/pkg-specs.html#package-match-specifications
# documentation on package_spec syntax for constraining versions
- gcc<12
- gxx<12
- sysroot_linux-64=2.17 # needed to match pre-built CI XRT glibc version
- conda-gcc-specs
- binutils
# misc
- autoconf
- coreutils
- jq
- pip
- make
- git
- sbt
- openjdk=20
- dtc
- verilator==5.020
- conda-lock=1.4

33
conda-reqs/chipyard.yaml → conda-reqs/chipyard-extended.yaml
View File

@@ -8,14 +8,6 @@ platforms:
- linux-64
dependencies:
# https://conda-forge.org/feedstock-outputs/
# filterable list of all conda-forge packages
# https://conda-forge.org/#contribute
# instructions on adding a recipe
# https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/pkg-specs.html#package-match-specifications
# documentation on package_spec syntax for constraining versions
# handy tool for introspecting package relationships and file ownership
# see https://github.com/rvalieris/conda-tree
- conda-tree
@@ -23,21 +15,9 @@ dependencies:
# bundle FireSim driver with deps into installer shell-script
- constructor
- gcc<12
- gxx<12
- sysroot_linux-64=2.17 # needed to match pre-built CI XRT glibc version
- conda-gcc-specs
- binutils
# firtool handled outside of conda
# misc
- autoconf
- pre-commit
- readline
- coreutils
- jq
- pip
# firemarshal deps
- qemu # from ucb-bar channel - https://github.com/ucb-bar/qemu-feedstock
@@ -67,9 +47,7 @@ dependencies:
- findutils
- lzop
- make
- bash-completion
- sbt
- ca-certificates
- mosh
- gmp
@@ -77,8 +55,6 @@ dependencies:
- mpc
- zlib
- vim
- git
- openjdk=20
- gengetopt
- libffi
- expat
@@ -87,12 +63,6 @@ dependencies:
- cmake
- graphviz
- expect
- dtc
- verilator==5.020
- screen
- elfutils
- libdwarf-dev==0.0.0.20190110_28_ga81397fc4 # from ucb-bar channel - using mainline libdwarf-feedstock
- conda-lock=1.4
# clang-format for driver coding style enforcement.
- clang-format
@@ -109,6 +79,9 @@ dependencies:
# conda environment, pip's dependency resolution can conflict with
# conda and create broken environments. It's best to use the conda
# packages so that the environment is consistent
- screen
- elfutils
- libdwarf-dev==0.0.0.20190110_28_ga81397fc4 # from ucb-bar channel - using mainline libdwarf-feedstock
- colorama
- argcomplete
- python-graphviz

2638
conda-reqs/conda-lock-reqs/conda-requirements-esp-tools-linux-64-lean.conda-lock.yml Normal file
View File

File diff suppressed because it is too large Load Diff

49
conda-reqs/conda-lock-reqs/conda-requirements-esp-tools-linux-64.conda-lock.yml generated
View File

@@ -9,7 +9,7 @@
# To update a single package to the latest version compatible with the version constraints in the source:
# conda-lock lock --lockfile conda-requirements-esp-tools-linux-64.conda-lock.yml --update PACKAGE
# To re-solve the entire environment, e.g. after changing a version constraint in the source file:
# conda-lock -f /scratch/abejgonza/cy-circt/conda-reqs/chipyard.yaml -f /scratch/abejgonza/cy-circt/conda-reqs/docs.yaml -f /scratch/abejgonza/cy-circt/conda-reqs/esp-tools.yaml --lockfile conda-requirements-esp-tools-linux-64.conda-lock.yml
# conda-lock -f /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/chipyard-base.yaml -f /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/chipyard-extended.yaml -f /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/docs.yaml -f /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/esp-tools.yaml --lockfile conda-requirements-esp-tools-linux-64.conda-lock.yml
metadata:
channels:
- url: ucb-bar
@@ -21,13 +21,14 @@ metadata:
- url: nodefaults
used_env_vars: []
content_hash:
linux-64: 49806d76b27927ce173d3b40bc3cb7a27b124e37fca50b32d653588b3f5a2a9a
linux-64: e6f7ea5e7ade6d5e422a65ad3ec62aabcadde05413e29340a1a234e7806c2764
platforms:
- linux-64
sources:
- /scratch/abejgonza/cy-circt/conda-reqs/chipyard.yaml
- /scratch/abejgonza/cy-circt/conda-reqs/docs.yaml
- /scratch/abejgonza/cy-circt/conda-reqs/esp-tools.yaml
- /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/chipyard-base.yaml
- /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/chipyard-extended.yaml
- /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/docs.yaml
- /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/esp-tools.yaml
package:
- category: main
dependencies: {}
@@ -2157,7 +2158,7 @@ package:
version: 1.7.0
- category: main
dependencies:
python: 2.7.*|>=3.7
python: ==2.7.*|>=3.7
hash:
md5: 56febe65315cc388a5d20adf2b39a74d
sha256: 0ff5173f6b20a5c95401aa52d10dbdc05c322fc568dc7417c6f76cf68e706d16
@@ -2503,16 +2504,16 @@ package:
version: 11.4.0
- category: main
dependencies:
python: '>=3.9,<3.10.0a0'
python_abi: 3.9.* *_cp39
__unix: ''
python: '>=3.8'
hash:
md5: d4b34ce06e172f5f18c39be4aed98fd8
sha256: d259113b1a2eeea896ea3062cea1325fd5ab979a1186b209b7bdb8e066b09156
md5: 2ed1fe4b9079da97c44cfe9c2e5078fd
sha256: cd93d5d4b1d98f7ce76a8658c35de9c63e17b3a40e52f40fa2f459e0da83d0b1
manager: conda
name: humanfriendly
optional: false
platform: linux-64
url: https://conda.anaconda.org/conda-forge/linux-64/humanfriendly-10.0-py39hf3d152e_5.conda
url: https://conda.anaconda.org/conda-forge/noarch/humanfriendly-10.0-pyhd8ed1ab_6.conda
version: '10.0'
- category: main
dependencies:
@@ -3027,7 +3028,7 @@ package:
version: 0.6.6
- category: main
dependencies:
python: 2.7.*|>=3.4
python: ==2.7.*|>=3.4
hash:
md5: 076becd9e05608f8dc72757d5f3a91ff
sha256: 74c63fd03f1f1ea2b54e8bc529fd1a600aaafb24027b738d0db87909ee3a33dc
@@ -3087,16 +3088,16 @@ package:
version: 3.1.1
- category: main
dependencies:
python: '>=3.9,<3.10.0a0'
python_abi: 3.9.* *_cp39
__unix: ''
python: '>=3.8'
hash:
md5: d34b97a2386932b97c7cb80916a673e7
sha256: 42d46baeab725d3c70d22a4258549e9f0f1a72b740166cd9c3b394c4369cb306
md5: 2a7de29fb590ca14b5243c4c812c8025
sha256: a42f826e958a8d22e65b3394f437af7332610e43ee313393d1cf143f0a2d274b
manager: conda
name: pysocks
optional: false
platform: linux-64
url: https://conda.anaconda.org/conda-forge/linux-64/pysocks-1.7.1-py39hf3d152e_5.tar.bz2
url: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2
version: 1.7.1
- category: main
dependencies:
@@ -4645,14 +4646,14 @@ package:
pip: ''
python: '>=3.7,<4.0'
hash:
md5: aaafd70a6934549210865f558a0eee96
sha256: 4e695e4d83cfc4ffd557cc2bdbecc0cedb9df8831d1a415b8e8c0e5f96bc6b5f
md5: a699f404daa279b4cb41a047cf70c94a
sha256: 8cf9f34336ef7af3479a03682e171c6a2c5837cb86bf9e4c2582b75cf3a27ab9
manager: conda
name: types-awscrt
optional: false
platform: linux-64
url: https://conda.anaconda.org/conda-forge/noarch/types-awscrt-0.20.0-pyhd8ed1ab_0.conda
version: 0.20.0
url: https://conda.anaconda.org/conda-forge/noarch/types-awscrt-0.20.2-pyhd8ed1ab_0.conda
version: 0.20.2
- category: main
dependencies:
cffi: ''
@@ -5188,13 +5189,13 @@ package:
ruamel.yaml.clib: '>=0.2.0,<=0.2.7'
urllib3: '>=1.25.4,<1.27'
hash:
md5: 8376298ea81d582557485a166528a2e7
sha256: ededf952f2bb95033fea0969e01e048675e0738a5277fb145810af859be05366
md5: 42415f0a777728616a1014d6c2d03570
sha256: 94a448e74b118c6cfa22fb2ed950dd5810612ccd76626a8471b0e74595a5c221
manager: conda
name: awscli
optional: false
platform: linux-64
url: https://conda.anaconda.org/conda-forge/linux-64/awscli-2.15.12-py39hf3d152e_0.conda
url: https://conda.anaconda.org/conda-forge/linux-64/awscli-2.15.12-py39hf3d152e_1.conda
version: 2.15.12
- category: main
dependencies:

2640
conda-reqs/conda-lock-reqs/conda-requirements-riscv-tools-linux-64-lean.conda-lock.yml Normal file
View File

File diff suppressed because it is too large Load Diff

49
conda-reqs/conda-lock-reqs/conda-requirements-riscv-tools-linux-64.conda-lock.yml generated
View File

@@ -9,7 +9,7 @@
# To update a single package to the latest version compatible with the version constraints in the source:
# conda-lock lock --lockfile conda-requirements-riscv-tools-linux-64.conda-lock.yml --update PACKAGE
# To re-solve the entire environment, e.g. after changing a version constraint in the source file:
# conda-lock -f /scratch/abejgonza/cy-circt/conda-reqs/chipyard.yaml -f /scratch/abejgonza/cy-circt/conda-reqs/docs.yaml -f /scratch/abejgonza/cy-circt/conda-reqs/riscv-tools.yaml --lockfile conda-requirements-riscv-tools-linux-64.conda-lock.yml
# conda-lock -f /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/chipyard-base.yaml -f /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/chipyard-extended.yaml -f /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/docs.yaml -f /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/riscv-tools.yaml --lockfile conda-requirements-riscv-tools-linux-64.conda-lock.yml
metadata:
channels:
- url: ucb-bar
@@ -21,13 +21,14 @@ metadata:
- url: nodefaults
used_env_vars: []
content_hash:
linux-64: acaac148231e584874fde649a99f5d9224edcab7975a1aa028bfe6800fb61ec0
linux-64: eb07751579f0515bcd5e4dfff7131bf35f78430a1281125bb17f2a1c52684521
platforms:
- linux-64
sources:
- /scratch/abejgonza/cy-circt/conda-reqs/chipyard.yaml
- /scratch/abejgonza/cy-circt/conda-reqs/docs.yaml
- /scratch/abejgonza/cy-circt/conda-reqs/riscv-tools.yaml
- /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/chipyard-base.yaml
- /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/chipyard-extended.yaml
- /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/docs.yaml
- /scratch/joonho.whangbo/coding/chipyard-release-118/conda-reqs/riscv-tools.yaml
package:
- category: main
dependencies: {}
@@ -2157,7 +2158,7 @@ package:
version: 1.7.0
- category: main
dependencies:
python: 2.7.*|>=3.7
python: ==2.7.*|>=3.7
hash:
md5: 56febe65315cc388a5d20adf2b39a74d
sha256: 0ff5173f6b20a5c95401aa52d10dbdc05c322fc568dc7417c6f76cf68e706d16
@@ -2486,16 +2487,16 @@ package:
version: 11.4.0
- category: main
dependencies:
python: '>=3.10,<3.11.0a0'
python_abi: 3.10.* *_cp310
__unix: ''
python: '>=3.8'
hash:
md5: d8c02e0916d3894c9f8bbe26418c0488
sha256: 995c56f883626a41bf9f7bb67c94f61103e8876546c2886925945a27a9b28eba
md5: 2ed1fe4b9079da97c44cfe9c2e5078fd
sha256: cd93d5d4b1d98f7ce76a8658c35de9c63e17b3a40e52f40fa2f459e0da83d0b1
manager: conda
name: humanfriendly
optional: false
platform: linux-64
url: https://conda.anaconda.org/conda-forge/linux-64/humanfriendly-10.0-py310hff52083_5.conda
url: https://conda.anaconda.org/conda-forge/noarch/humanfriendly-10.0-pyhd8ed1ab_6.conda
version: '10.0'
- category: main
dependencies:
@@ -3010,7 +3011,7 @@ package:
version: 0.6.6
- category: main
dependencies:
python: 2.7.*|>=3.4
python: ==2.7.*|>=3.4
hash:
md5: 076becd9e05608f8dc72757d5f3a91ff
sha256: 74c63fd03f1f1ea2b54e8bc529fd1a600aaafb24027b738d0db87909ee3a33dc
@@ -3070,16 +3071,16 @@ package:
version: 3.1.1
- category: main
dependencies:
python: '>=3.10,<3.11.0a0'
python_abi: 3.10.* *_cp310
__unix: ''
python: '>=3.8'
hash:
md5: 378f2260e871f3ea46c6fa58d9f05277
sha256: cb6e4821234cee05acd1996cef88e40dfc2d5ab12cf12c5b1d6ed9118f7f41a7
md5: 2a7de29fb590ca14b5243c4c812c8025
sha256: a42f826e958a8d22e65b3394f437af7332610e43ee313393d1cf143f0a2d274b
manager: conda
name: pysocks
optional: false
platform: linux-64
url: https://conda.anaconda.org/conda-forge/linux-64/pysocks-1.7.1-py310hff52083_5.tar.bz2
url: https://conda.anaconda.org/conda-forge/noarch/pysocks-1.7.1-pyha2e5f31_6.tar.bz2
version: 1.7.1
- category: main
dependencies:
@@ -4646,14 +4647,14 @@ package:
pip: ''
python: '>=3.7,<4.0'
hash:
md5: aaafd70a6934549210865f558a0eee96
sha256: 4e695e4d83cfc4ffd557cc2bdbecc0cedb9df8831d1a415b8e8c0e5f96bc6b5f
md5: a699f404daa279b4cb41a047cf70c94a
sha256: 8cf9f34336ef7af3479a03682e171c6a2c5837cb86bf9e4c2582b75cf3a27ab9
manager: conda
name: types-awscrt
optional: false
platform: linux-64
url: https://conda.anaconda.org/conda-forge/noarch/types-awscrt-0.20.0-pyhd8ed1ab_0.conda
version: 0.20.0
url: https://conda.anaconda.org/conda-forge/noarch/types-awscrt-0.20.2-pyhd8ed1ab_0.conda
version: 0.20.2
- category: main
dependencies:
cffi: ''
@@ -5189,13 +5190,13 @@ package:
ruamel.yaml.clib: '>=0.2.0,<=0.2.7'
urllib3: '>=1.25.4,<1.27'
hash:
md5: 04ce380558b5922cd12afc44b98a4352
sha256: 9daa78cbca08953aad8c8bb5d207b719fd7db7d75976facd6e2ec81dd0c4c1f1
md5: 5ef7b8dcc094f40102601e5e2e145da1
sha256: dce1ad33a64336a692bd6230becb27fb1d282bb8903e82c2dad1056858c69f63
manager: conda
name: awscli
optional: false
platform: linux-64
url: https://conda.anaconda.org/conda-forge/linux-64/awscli-2.15.12-py310hff52083_0.conda
url: https://conda.anaconda.org/conda-forge/linux-64/awscli-2.15.12-py310hff52083_1.conda
version: 2.15.12
- category: main
dependencies:

12
scripts/build-setup.sh
View File

@@ -38,6 +38,7 @@ usage() {
echo " --help -h : Display this message"
echo " --verbose -v : Verbose printout"
echo " --use-unpinned-deps -ud : Use unpinned conda environment"
echo " --use-lean-conda : Install a leaner version of the repository (Smaller conda env, no FireSim, no FireMarshal)"
echo " --skip -s N : Skip step N in the list above. Use multiple times to skip multiple steps ('-s N -s M ...')."
echo " --skip-conda : Skip Conda initialization (step 1)"
@@ -57,6 +58,7 @@ TOOLCHAIN_TYPE="riscv-tools"
VERBOSE=false
VERBOSE_FLAG=""
USE_UNPINNED_DEPS=false
USE_LEAN_CONDA=false
SKIP_LIST=()
# getopts does not support long options, and is inflexible
@@ -70,6 +72,9 @@ do
--verbose | -v)
VERBOSE_FLAG=$1
set -x ;;
--use-lean-conda)
USE_LEAN_CONDA=true
SKIP_LIST+=(4 6 7 8 9) ;;
-ud | --use-unpinned-deps )
USE_UNPINNED_DEPS=true ;;
--skip | -s)
@@ -161,13 +166,18 @@ if run_step "1"; then
CONDA_REQS=$CYDIR/conda-reqs
CONDA_LOCK_REQS=$CONDA_REQS/conda-lock-reqs
# must match with the file generated by generate-conda-lockfile.sh
LOCKFILE=$CONDA_LOCK_REQS/conda-requirements-$TOOLCHAIN_TYPE-linux-64.conda-lock.yml
if [ "$USE_LEAN_CONDA" = false ]; then
LOCKFILE=$CONDA_LOCK_REQS/conda-requirements-$TOOLCHAIN_TYPE-linux-64.conda-lock.yml
else
LOCKFILE=$CONDA_LOCK_REQS/conda-requirements-$TOOLCHAIN_TYPE-linux-64-lean.conda-lock.yml
fi
if [ "$USE_UNPINNED_DEPS" = true ]; then
# auto-gen the lockfiles
$CYDIR/scripts/generate-conda-lockfiles.sh
exit_if_last_command_failed
fi
echo "Using lockfile: $LOCKFILE"
# use conda-lock to create env
conda-lock install --conda $(which conda) -p $CYDIR/.conda-env $LOCKFILE &&

27
scripts/generate-conda-lockfiles.sh
View File

@@ -16,11 +16,24 @@ for TOOLCHAIN_TYPE in riscv-tools esp-tools; do
rm -rf $LOCKFILE
conda-lock \
--no-mamba \
--no-micromamba \
-f "$REQS_DIR/chipyard.yaml" \
-f "$REQS_DIR/docs.yaml" \
-f "$REQS_DIR/$TOOLCHAIN_TYPE.yaml" \
-p linux-64 \
--lockfile $LOCKFILE
--no-mamba \
--no-micromamba \
-f "$REQS_DIR/chipyard-base.yaml" \
-f "$REQS_DIR/chipyard-extended.yaml" \
-f "$REQS_DIR/docs.yaml" \
-f "$REQS_DIR/$TOOLCHAIN_TYPE.yaml" \
-p linux-64 \
--lockfile $LOCKFILE
LOCKFILE=$REQS_DIR/conda-lock-reqs/conda-requirements-$TOOLCHAIN_TYPE-linux-64-lean.conda-lock.yml
rm -rf $LOCKFILE
conda-lock \
--no-mamba \
--no-micromamba \
-f "$REQS_DIR/chipyard-base.yaml" \
-f "$REQS_DIR/docs.yaml" \
-f "$REQS_DIR/$TOOLCHAIN_TYPE.yaml" \
-p linux-64 \
--lockfile $LOCKFILE
done