diff --git a/common.mk b/common.mk index 5c1f2461..4aeb44aa 100644 --- a/common.mk +++ b/common.mk @@ -231,26 +231,22 @@ $(SFC_MFC_TARGETS) &: $(FIRRTL_FILE) $(FINAL_ANNO_FILE) $(SFC_LEVEL) $(EXTRA_FIR $(SED) -i 's/.*/& /' $(MFC_SMEMS_CONF) # need trailing space for SFC macrocompiler # DOC include end: FirrtlCompiler -$(TOP_MODS_FILELIST) $(MODEL_MODS_FILELIST) $(ALL_MODS_FILELIST) $(BB_MODS_FILELIST) $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) &: $(MFC_MODEL_HRCHY_JSON) $(MFC_FILELIST) $(MFC_BB_MODS_FILELIST) - $(base_dir)/scripts/split-module-files.py \ +$(TOP_MODS_FILELIST) $(MODEL_MODS_FILELIST) $(ALL_MODS_FILELIST) $(BB_MODS_FILELIST) $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) &: $(MFC_MODEL_HRCHY_JSON) $(MFC_TOP_HRCHY_JSON) $(MFC_FILELIST) $(MFC_BB_MODS_FILELIST) + $(base_dir)/scripts/uniqify-module-names.py \ --model-hier-json $(MFC_MODEL_HRCHY_JSON) \ + --top-hier-json $(MFC_TOP_HRCHY_JSON) \ + --in-all-filelist $(MFC_FILELIST) \ --dut $(TOP) \ + --model $(MODEL) \ + --target-dir $(GEN_COLLATERAL_DIR) \ --out-dut-filelist $(TOP_MODS_FILELIST) \ --out-model-filelist $(MODEL_MODS_FILELIST) \ - --in-all-filelist $(MFC_FILELIST) \ - --target-dir $(GEN_COLLATERAL_DIR) + --out-model-hier-json $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) \ + --gcpath $(GEN_COLLATERAL_DIR) $(SED) -e 's;^;$(GEN_COLLATERAL_DIR)/;' $(MFC_BB_MODS_FILELIST) > $(BB_MODS_FILELIST) $(SED) -i 's/\.\///' $(TOP_MODS_FILELIST) $(SED) -i 's/\.\///' $(MODEL_MODS_FILELIST) $(SED) -i 's/\.\///' $(BB_MODS_FILELIST) - $(base_dir)/scripts/uniqify-module-names.py \ - --top-filelist $(TOP_MODS_FILELIST) \ - --mod-filelist $(MODEL_MODS_FILELIST) \ - --gen-collateral-path $(GEN_COLLATERAL_DIR) \ - --model-hier-json $(MFC_MODEL_HRCHY_JSON) \ - --out-model-hier-json $(MFC_MODEL_HRCHY_JSON_UNIQUIFIED) \ - --dut $(TOP) \ - --model $(MODEL) sort -u $(TOP_MODS_FILELIST) $(MODEL_MODS_FILELIST) $(BB_MODS_FILELIST) > $(ALL_MODS_FILELIST) $(TOP_BB_MODS_FILELIST) $(MODEL_BB_MODS_FILELIST) &: $(BB_MODS_FILELIST) $(MFC_TOP_HRCHY_JSON) $(FINAL_ANNO_FILE) diff --git a/scripts/split-module-files.py b/scripts/split-module-files.py deleted file mode 100755 index 64fe5707..00000000 --- a/scripts/split-module-files.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 - -import json -import argparse -from typing import List, Optional - -# Schema of json emitted by circt -""" -{ - "instance_name": "TestHarness", - "module_name": "TestHarness", - "instances": [ - { - "instance_name": "chiptop", - "module_name": "ChipTop", - "instances": [ - { - "instance_name": "system", - "module_name": "DigitalTop", - "instances": [ ] - }, ... - ] - }, - { - "instance_name": "simdram", - "module_name": "SimDRAM", - "instances": [] - }, - ] -} -""" - -def get_modules(js: dict) -> List[str]: - if 'instances' not in js: - return js['module_name'] - else: - mods = [] - for mod in js['instances']: - mods.extend(get_modules(mod)) - return [js['module_name']] + mods - -def find_mod_by_name(js: dict, name: str) -> Optional[List[dict]]: - if 'instances' not in js: - return None - else: - mods = [] - for mod in js['instances']: - if mod['module_name'] == name: - mods.append(mod) - other_mods = find_mod_by_name(mod, name) - if other_mods is not None: - mods.extend(other_mods) - return mods - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Convert CIRCT (firtool) hierarchy JSON into DUT and test harness filelists') - parser.add_argument('--model-hier-json', type=str, required=True, help='Path to hierarchy JSON emitted by firtool. Must include DUT as a module.') - parser.add_argument('--dut', type=str, required=True, help='Name of the DUT module.') - parser.add_argument('--out-dut-filelist', type=str, required=True, help='Path to output filelist including all modules under the DUT.') - parser.add_argument('--out-model-filelist', type=str, required=True, help='Path to output filelist including all modules under the top-most module but not modules under the DUT.') - parser.add_argument('--in-all-filelist', type=str, required=True, help='Path to input filelist that has all modules (relative paths).') - parser.add_argument('--target-dir', type=str, required=True, help='Path to where module sources are located (combined with --in-all-filelist gives the absolute path to module sources).') - args = parser.parse_args() - - with open(args.model_hier_json) as f: - j = json.load(f) - - dut_tops = find_mod_by_name(j, args.dut) - assert dut_tops is not None - assert len(dut_tops) == 1 - dut_top = dut_tops[0] - - dut_mods = set(get_modules(dut_top)) - model_mods = set(get_modules(j)) - dut_mods - both_mods = dut_mods.intersection(model_mods) - - assert len(both_mods) == 0 - - with open(args.out_dut_filelist, 'w') as df, \ - open(args.in_all_filelist) as fl: - # add paths that correspond to modules to output file - for path in fl: - writeOut = False - for dm in dut_mods: - if dm in path: - writeOut = True - break - - # prepend the target directory to get filelist with absolute paths - if writeOut: - if not args.target_dir in path: - df.write(f"{args.target_dir}/{path}") - else: - df.write(f"{path}") - - with open(args.out_model_filelist, 'w') as df, \ - open(args.in_all_filelist) as fl: - # add paths that correspond to modules to output file - for path in fl: - writeOut = False - for dm in model_mods: - if dm in path: - writeOut = True - break - - # prepend the target directory to get filelist with absolute paths - if writeOut: - if not args.target_dir in path: - df.write(f"{args.target_dir}/{path}") - else: - df.write(f"{path}") diff --git a/scripts/uniqify-module-names.py b/scripts/uniqify-module-names.py index a6255a8c..87b56b63 100755 --- a/scripts/uniqify-module-names.py +++ b/scripts/uniqify-module-names.py @@ -5,19 +5,21 @@ import argparse import shutil import os import datetime - +import sys parser = argparse.ArgumentParser(description="") -parser.add_argument("--top-filelist", type=str, required=True, help="Abs path to ..top.f") -parser.add_argument("--mod-filelist", type=str, required=True, help="Abs path to ..model.f") -parser.add_argument("--gen-collateral-path", dest="gcpath", type=str, required=True, help="Abs path to the gen-collateral directory") parser.add_argument("--model-hier-json", type=str, required=True, help="Path to hierarchy JSON emitted by firtool. Must include DUT as a module.") -parser.add_argument("--out-model-hier-json", type=str, required=True, help="Path to updated hierarchy JSON emitted by this script.") +parser.add_argument("--top-hier-json", type=str, required=True, help="Path to hierarchy JSON emitted by firtool. Must include DUT as a module.") +parser.add_argument('--in-all-filelist', type=str, required=True, help='Path to input filelist that has all modules (relative paths).') parser.add_argument("--dut", type=str, required=True, help="Name of the DUT module.") parser.add_argument("--model", type=str, required=True, help="Name of the Model module.") +parser.add_argument('--target-dir', type=str, required=True, help='Path to where module sources are located (combined with --in-all-filelist gives the absolute path to module sources).') +parser.add_argument('--out-dut-filelist', type=str, required=True, help='Path to output filelist including all modules under the DUT.') +parser.add_argument('--out-model-filelist', type=str, required=True, help='Path to output filelist including all modules under the MODEL.') +parser.add_argument("--out-model-hier-json", type=str, required=True, help="Path to updated hierarchy JSON emitted by this script.") +parser.add_argument("--gcpath", type=str, required=True, help="Path to gen-collateral") args = parser.parse_args() - MODEL_SFX=args.model + "_UNIQUIFIED" def bash(cmd): @@ -40,9 +42,6 @@ def get_filelist(filelist): print(f"Something is wrong about this line '{line}'") return fnames -def update_filelist(cur_file, new_file): - bash(f"echo \"{args.gcpath}/{new_file}\" >> {os.path.join(args.gcpath, args.mod_filelist)}") - def generate_copy(c, sfx): (cur_name, ext) = os.path.splitext(c) new_name = cur_name + "_" + sfx @@ -55,46 +54,167 @@ def generate_copy(c, sfx): bash(f"sed -i s/\"module {cur_name}\"/\"module {new_name}\"/ {new_file}") return new_file -def dfs_update_modules(tree, common_fnames, visited, top_fnames): +def dfs_update_modules(tree, common_fnames, visited, ext_dict): # List of direct submodules to update childs_to_update = list() for child in tree['instances']: # We don't have to change stuff that are under the dut - if (child['module_name'] == args.dut) or (child['module_name'] in visited): + if (child['module_name'] == args.dut): continue - if dfs_update_modules(child, common_fnames, visited, top_fnames): + if dfs_update_modules(child, common_fnames, visited, ext_dict): childs_to_update.append(child['module_name']) - if (child['module_name'] + ".sv") in common_fnames: + if (child['module_name']) in common_fnames: child['module_name'] = child['module_name'] + "_" + MODEL_SFX cur_module = tree['module_name'] - cur_file = cur_module + ".sv" new_file = None # cur_file is in the common list, or is a ancestor of of them, generate a new file - if (cur_file in common_fnames) or len(childs_to_update) > 0: - new_file = generate_copy(cur_file, MODEL_SFX) - update_filelist(cur_file, os.path.basename(new_file)) - - for submodule_name in childs_to_update: - if (submodule_name + ".sv") in common_fnames: - bash(f"sed -i s/\"{submodule_name}\"/\"{submodule_name}_{MODEL_SFX}\"/ {new_file}") + if (cur_module in common_fnames) or len(childs_to_update) > 0: + new_file = 1 visited.add(cur_module) return (new_file is not None) +def bfs_update(tree, common_fnames, ext_dict, filelist): + q = [(tree['instance_name'], tree['module_name'], tree['instances'], None)] + + updated_submodule = set() + + while len(q) != 0: + front = q[0] + q.pop(0) + (inst, mod, child, parent) = front + + try: + cur_file = mod + "." + ext_dict[mod] + except: + cur_file = mod + ".sv" + + mod_updated = False + + # if the module is common, make a copy & update its instance in its parent + if mod in common_fnames: + mod_updated = True + new_file = generate_copy(cur_file, MODEL_SFX) + filelist.append(new_file) + if parent is not None and ((parent, mod) not in updated_submodule): + print(mod, parent) + parent_file = os.path.join(args.gcpath, parent + "." + ext_dict[parent]) + bash(f"sed -i s/\"{mod}\"/\"{mod}_{MODEL_SFX}\"/ {parent_file}") + updated_submodule.add((parent, mod)) + else: + filelist.append(cur_file) + + # set the parent module name + new_mod = mod + if mod_updated: + new_mod = mod + "_" + MODEL_SFX + ext_dict[new_mod] = ext_dict[mod] + + # traverse its children + for c in child: + if c['module_name'] != args.dut: + q.append((c['instance_name'], c['module_name'], c['instances'], new_mod)) + +def bfs_collect_modules(tree, child_to_ignore = None): + q = [(tree['instance_name'], tree['module_name'], tree['instances'])] + + modules = list() + while len(q) != 0: + front = q[0] + q.pop(0) + + (inst, mod, child) = front + modules.append(mod) + for c in child: + if c['module_name'] != child_to_ignore: + print(c['module_name']) + q.append((c['instance_name'], c['module_name'], c['instances'])) + return modules + +def write_filelist(modules, out_file): + with open(out_file, "w") as df, \ + open(args.in_all_filelist) as fl: + # add paths that correspond to modules to output file + for path in fl: + writeOut = False + for dm in modules: + if dm in path: + writeOut = True + break + + # prepend the target directory to get filelist with absolute paths + if writeOut: + if not args.target_dir in path: + df.write(f"{args.target_dir}/{path}") + else: + df.write(f"{path}") + +def write_filelist_model(modules, out_file): + with open(out_file, "w") as df: + for m in modules: + if not args.target_dir in m: + df.write(f"{args.target_dir}/{m}\n") + else: + df.write(f"{m}\n") + +def get_file_ext(all_filelist): + ext_dict = dict() + with open(args.in_all_filelist) as fl: + for path in fl: + fname = os.path.basename(path) + (module, ext) = fname.strip().split(".") + ext_dict[module] = ext + return ext_dict + def main(): - top_fnames = set(get_filelist(args.top_filelist)) - mod_fnames = set(get_filelist(args.mod_filelist)) - common_fnames = top_fnames.intersection(mod_fnames) + with open(args.model_hier_json) as imhj: + imhj_data = json.load(imhj) + modules_under_model = set(bfs_collect_modules(imhj_data, child_to_ignore=args.dut)) + + for x in modules_under_model: + print(f"model only {x}") + + with open(args.top_hier_json) as imhj: + imhj_data = json.load(imhj) + modules_under_top = set(bfs_collect_modules(imhj_data)) + + for x in modules_under_top: + print(f"top only {x}") + + common_modules = modules_under_top.intersection(modules_under_model) + + print(f"modules under top {len(modules_under_top)}") + print(f"modules under model {len(modules_under_model)}") + print(f"modules under both {len(common_modules)}") + print(f"total modules {len(modules_under_top) + len(modules_under_model) - len(common_modules)}") + + write_filelist(modules_under_top, args.out_dut_filelist) + + ext_dict = get_file_ext(args.in_all_filelist) + print(f"total modules in filelist {len(ext_dict)}") + + for x in common_modules: + print(f"common {x}") + +# for x in common_fnames: +# print(f"common_fnames {x}") + with open(args.model_hier_json) as imhj: imhj_data = json.load(imhj) with open(args.out_model_hier_json, "w+") as out_file: visited = set() - dfs_update_modules(imhj_data, common_fnames, visited, top_fnames) + filelist = list() + bfs_update(imhj_data, common_modules, ext_dict, filelist) + dfs_update_modules(imhj_data, common_modules, visited, ext_dict) json.dump(imhj_data, out_file, indent=2) + updated_modules_under_model = set(bfs_collect_modules(imhj_data, child_to_ignore=args.dut)) + print(filelist) + write_filelist_model(set(filelist), args.out_model_filelist) + if __name__ == "__main__": main()