diff --git a/AMSS_NCKU_Program.py b/AMSS_NCKU_Program.py index 2d777cd..8aa5d01 100755 --- a/AMSS_NCKU_Program.py +++ b/AMSS_NCKU_Program.py @@ -174,11 +174,14 @@ import generate_macrodef generate_macrodef.generate_macrodef_h() print( " AMSS-NCKU macro file macrodef.h has been generated. " ) -generate_macrodef.generate_macrodef_fh() -print( " AMSS-NCKU macro file macrodef.fh has been generated. " ) - - -################################################################## +generate_macrodef.generate_macrodef_fh() +print( " AMSS-NCKU macro file macrodef.fh has been generated. " ) + +generate_macrodef.generate_build_config() +print( " AMSS-NCKU build config AMSS_NCKU_build.mk has been generated. " ) + + +################################################################## # Compile the AMSS-NCKU program according to user requirements @@ -217,11 +220,13 @@ shutil.copytree(AMSS_NCKU_source_path, AMSS_NCKU_source_copy) # Copy the generated macro files into the AMSS_NCKU source folder -macrodef_h_path = os.path.join(File_directory, "macrodef.h") -macrodef_fh_path = os.path.join(File_directory, "macrodef.fh") - -shutil.copy2(macrodef_h_path, AMSS_NCKU_source_copy) -shutil.copy2(macrodef_fh_path, AMSS_NCKU_source_copy) +macrodef_h_path = os.path.join(File_directory, "macrodef.h") +macrodef_fh_path = os.path.join(File_directory, "macrodef.fh") +build_config_path = os.path.join(File_directory, "AMSS_NCKU_build.mk") + +shutil.copy2(macrodef_h_path, AMSS_NCKU_source_copy) +shutil.copy2(macrodef_fh_path, AMSS_NCKU_source_copy) +shutil.copy2(build_config_path, AMSS_NCKU_source_copy) # Notes on copying files: # shutil.copy2 preserves file metadata such as modification time. diff --git a/AMSS_NCKU_source/makefile b/AMSS_NCKU_source/makefile index 231d31e..8c82faa 100644 --- a/AMSS_NCKU_source/makefile +++ b/AMSS_NCKU_source/makefile @@ -2,7 +2,31 @@ include makefile.inc +-include AMSS_NCKU_build.mk + +ABE_TYPE ?= $(shell awk '/^[[:space:]]*\#define[[:space:]]+ABEtype/ {print $$3; exit}' macrodef.h 2>/dev/null) + +ifeq ($(USE_TRANSFER_CACHE),auto) +ifeq ($(ABE_TYPE),0) +EFFECTIVE_USE_TRANSFER_CACHE = 1 +else +EFFECTIVE_USE_TRANSFER_CACHE = 0 +endif +else +EFFECTIVE_USE_TRANSFER_CACHE = $(USE_TRANSFER_CACHE) +endif + ifeq ($(USE_CXX_ESCALAR_KERNEL),1) +ifeq ($(ABE_TYPE),1) +EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 1 +else +EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 0 +endif +else +EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 0 +endif + +ifeq ($(EFFECTIVE_USE_CXX_ESCALAR_KERNEL),1) ifeq ($(USE_CXX_KERNELS),0) $(error USE_CXX_ESCALAR_KERNEL=1 requires USE_CXX_KERNELS=1 because bssn_escalar_rhs_c.C reuses the C BSSN kernel) endif @@ -13,8 +37,8 @@ endif ## 0 : fallback to Neville path POLINT6_USE_BARY ?= 1 POLINT6_FLAG = -DPOLINT6_USE_BARYCENTRIC=$(POLINT6_USE_BARY) -TRANSFER_CACHE_FLAG = -DBSSN_USE_TRANSFER_CACHE=$(USE_TRANSFER_CACHE) -ESCALAR_KERNEL_FLAG = -DBSSN_USE_ESCALAR_C_KERNEL=$(USE_CXX_ESCALAR_KERNEL) +TRANSFER_CACHE_FLAG = -DBSSN_USE_TRANSFER_CACHE=$(EFFECTIVE_USE_TRANSFER_CACHE) +ESCALAR_KERNEL_FLAG = -DBSSN_USE_ESCALAR_C_KERNEL=$(EFFECTIVE_USE_CXX_ESCALAR_KERNEL) ## ABE build flags selected by PGO_MODE (set in makefile.inc, default: opt) ## make -> opt (PGO-guided, maximum performance) @@ -98,7 +122,7 @@ CFILES = else # C++ mode (default): C rewrite of bssn/bssn-escalar rhs and helper kernels CFILES = bssn_rhs_c.o fderivs_c.o fdderivs_c.o kodiss_c.o lopsided_c.o lopsided_kodis_c.o -ifeq ($(USE_CXX_ESCALAR_KERNEL),1) +ifeq ($(EFFECTIVE_USE_CXX_ESCALAR_KERNEL),1) CFILES += bssn_escalar_rhs_c.o endif endif diff --git a/AMSS_NCKU_source/makefile.inc b/AMSS_NCKU_source/makefile.inc index 5df430f..353c1bf 100755 --- a/AMSS_NCKU_source/makefile.inc +++ b/AMSS_NCKU_source/makefile.inc @@ -55,9 +55,10 @@ USE_CXX_KERNELS ?= 1 USE_CXX_ESCALAR_KERNEL ?= 1 ## Cached transfer switch -## 1 : enable cached Sync/Restrict/OutBd transfer on evolution hot paths -## 0 (default) : keep the original uncached transfer path for precision-safe runs -USE_TRANSFER_CACHE ?= 0 +## auto (default): enable for BSSN vacuum, keep other paths on the safe uncached path +## 1 : force cached Sync/Restrict/OutBd transfer on evolution hot paths +## 0 : force the original uncached transfer path +USE_TRANSFER_CACHE ?= auto ## RK4 kernel implementation switch ## 1 (default) : use C/C++ rewrite of rungekutta4_rout (for optimization experiments) diff --git a/BSSN_BUILD_CONFIG_MIGRATION.md b/BSSN_BUILD_CONFIG_MIGRATION.md new file mode 100644 index 0000000..46a7ffb --- /dev/null +++ b/BSSN_BUILD_CONFIG_MIGRATION.md @@ -0,0 +1,211 @@ +# BSSN Build Config Migration + +This note records the build-configuration fix needed when replacing +`AMSS_NCKU_Input.py` or `generate_macrodef.py` with a newer upstream version. + +## Problem + +`AMSS_NCKU_source/macrodef.h` is not the authoritative file used by normal +runs. `AMSS_NCKU_Program.py` first generates macro files under +`input_data.File_directory`, copies `AMSS_NCKU_source` to +`/AMSS_NCKU_source_copy`, then copies the generated macro files +into that copied source tree and compiles there. + +Therefore, makefile logic must not depend only on the stale +`AMSS_NCKU_source/macrodef.h`. The actual equation path must be passed to the +copied build tree from the same generation step that creates `macrodef.h`. + +The performance regression was caused by compiling/linking the +`BSSN-EScalar` C wrapper into BSSN vacuum builds. For BSSN vacuum (`ABEtype=0`), +the build must use: + +```make +BSSN_USE_TRANSFER_CACHE=1 +BSSN_USE_ESCALAR_C_KERNEL=0 +``` + +and must not link `bssn_escalar_rhs_c.o`. + +## Required Migration Steps + +### 1. Add an ABE type helper in `generate_macrodef.py` + +Add a helper that maps `input_data.Equation_Class` to the numeric `ABEtype`. +Use the same mapping as `macrodef.h`: + +```python +def get_abe_type(): + if ( input_data.Equation_Class == "BSSN" ): + return 0 + elif ( input_data.Equation_Class == "BSSN-EScalar" ): + return 1 + elif ( input_data.Equation_Class == "BSSN-EM" ): + return 3 + elif ( input_data.Equation_Class == "Z4C" ): + return 2 + else: + raise ValueError("Equation_Class setting error!!!") +``` + +Update `generate_macrodef_h()` to print `#define ABEtype {get_abe_type()}` +instead of duplicating the if/elif mapping. + +### 2. Generate a makefile fragment + +In `generate_macrodef.py`, add: + +```python +def generate_build_config(): + file1 = open(os.path.join(input_data.File_directory, "AMSS_NCKU_build.mk"), "w") + print("# Generated by generate_macrodef.py; do not edit manually.", file=file1) + print(f"ABE_TYPE := {get_abe_type()}", file=file1) + file1.close() +``` + +This file is the build-time authority for the equation path. + +### 3. Call and copy the generated build config + +In `AMSS_NCKU_Program.py`, after generating `macrodef.h` and `macrodef.fh`, call: + +```python +generate_macrodef.generate_build_config() +print(" AMSS-NCKU build config AMSS_NCKU_build.mk has been generated. ") +``` + +When copying generated files into `AMSS_NCKU_source_copy`, also copy: + +```python +build_config_path = os.path.join(File_directory, "AMSS_NCKU_build.mk") +shutil.copy2(build_config_path, AMSS_NCKU_source_copy) +``` + +### 4. Make the source makefile consume the generated config + +At the top of `AMSS_NCKU_source/makefile`, after `include makefile.inc`, add: + +```make +-include AMSS_NCKU_build.mk + +ABE_TYPE ?= $(shell awk '/^[[:space:]]*\#define[[:space:]]+ABEtype/ {print $$3; exit}' macrodef.h 2>/dev/null) +``` + +The generated `AMSS_NCKU_build.mk` is used during normal Python-driven builds. +The fallback keeps manual source-tree builds usable. + +### 5. Gate path-specific build options by `ABE_TYPE` + +Use effective build switches: + +```make +ifeq ($(USE_TRANSFER_CACHE),auto) +ifeq ($(ABE_TYPE),0) +EFFECTIVE_USE_TRANSFER_CACHE = 1 +else +EFFECTIVE_USE_TRANSFER_CACHE = 0 +endif +else +EFFECTIVE_USE_TRANSFER_CACHE = $(USE_TRANSFER_CACHE) +endif + +ifeq ($(USE_CXX_ESCALAR_KERNEL),1) +ifeq ($(ABE_TYPE),1) +EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 1 +else +EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 0 +endif +else +EFFECTIVE_USE_CXX_ESCALAR_KERNEL = 0 +endif + +TRANSFER_CACHE_FLAG = -DBSSN_USE_TRANSFER_CACHE=$(EFFECTIVE_USE_TRANSFER_CACHE) +ESCALAR_KERNEL_FLAG = -DBSSN_USE_ESCALAR_C_KERNEL=$(EFFECTIVE_USE_CXX_ESCALAR_KERNEL) +``` + +Only add `bssn_escalar_rhs_c.o` when the effective EScalar C kernel switch is +enabled: + +```make +ifeq ($(EFFECTIVE_USE_CXX_ESCALAR_KERNEL),1) +CFILES += bssn_escalar_rhs_c.o +endif +``` + +### 6. Use safe transfer-cache default + +In `AMSS_NCKU_source/makefile.inc`, keep: + +```make +USE_TRANSFER_CACHE ?= auto +``` + +With the effective switch logic above, this enables cached transfer for BSSN +vacuum while keeping non-BSSN paths on the uncached path by default. + +## Verification Checklist + +Run these checks after migrating: + +```bash +python3 -c "import generate_macrodef; generate_macrodef.generate_build_config()" +cat GW150914/AMSS_NCKU_build.mk +``` + +For BSSN, the generated file should contain: + +```make +ABE_TYPE := 0 +``` + +Dry-run the copied or source makefile: + +```bash +make -n -B INTERP_LB_MODE=off ABE | grep -E 'BSSN_USE_TRANSFER_CACHE|BSSN_USE_ESCALAR_C_KERNEL|bssn_escalar_rhs_c' +``` + +Expected BSSN result: + +```text +-DBSSN_USE_TRANSFER_CACHE=1 -DBSSN_USE_ESCALAR_C_KERNEL=0 +``` + +and no `bssn_escalar_rhs_c.o` in the final link command. + +Run the full workflow: + +```bash +python3 AMSS_NCKU_Program.py +``` + +For the 10-step BSSN test, compare coordinate output: + +```bash +python3 - <<'PY' +from pathlib import Path +old = Path('../GW150914-06457/AMSS_NCKU_output/bssn_BH.dat') +new = Path('GW150914/AMSS_NCKU_output/bssn_BH.dat') + +def rows(path): + out = [] + for line in path.read_text().splitlines(): + if not line.strip() or line.lstrip().startswith('#'): + continue + out.append([float(x) for x in line.split()]) + return out + +ro, rn = rows(old), rows(new) +n = min(len(ro), len(rn)) +max_abs = 0.0 +for i in range(n): + for a, b in zip(ro[i], rn[i]): + max_abs = max(max_abs, abs(a - b)) +print(f"old_rows={len(ro)} new_rows={len(rn)} compared_rows={n}") +print(f"max_abs_diff={max_abs:.17g}") +PY +``` + +For the validated migration, the first 10 rows matched exactly: + +```text +max_abs_diff=0 +``` diff --git a/generate_macrodef.py b/generate_macrodef.py index 864bcce..a89ce97 100755 --- a/generate_macrodef.py +++ b/generate_macrodef.py @@ -12,6 +12,37 @@ import os import AMSS_NCKU_Input as input_data ## import program input file +################################################################## + +def get_abe_type(): + if ( input_data.Equation_Class == "BSSN" ): + return 0 + elif ( input_data.Equation_Class == "BSSN-EScalar" ): + return 1 + elif ( input_data.Equation_Class == "BSSN-EM" ): + return 3 + elif ( input_data.Equation_Class == "Z4C" ): + return 2 + else: + raise ValueError("Equation_Class setting error!!!") + + +################################################################## + +## Generate the makefile fragment used by the copied source tree. +## The source-tree macrodef.h is not authoritative because macro files +## are regenerated under File_directory for each run. + +def generate_build_config(): + + file1 = open( os.path.join(input_data.File_directory, "AMSS_NCKU_build.mk"), "w") + + print( "# Generated by generate_macrodef.py; do not edit manually.", file=file1 ) + print( f"ABE_TYPE := {get_abe_type()}", file=file1 ) + + file1.close() + + ################################################################## ## Generate the macro file macrodef.h according to user settings @@ -58,19 +89,10 @@ def generate_macrodef_h(): # 2: Z4c vacuum # 3: coupled to Maxwell field - if ( input_data.Equation_Class == "BSSN" ): - print( "#define ABEtype 0", file=file1 ) - print( file=file1 ) - elif ( input_data.Equation_Class == "BSSN-EScalar" ): - print( "#define ABEtype 1", file=file1 ) - print( file=file1 ) - elif ( input_data.Equation_Class == "BSSN-EM" ): - print( "#define ABEtype 3", file=file1 ) - print( file=file1 ) - elif ( input_data.Equation_Class == "Z4C" ): - print( "#define ABEtype 2", file=file1 ) - print( file=file1 ) - else: + try: + print( f"#define ABEtype {get_abe_type()}", file=file1 ) + print( file=file1 ) + except ValueError: print( "Equation_Class setting error!!!" ) print() print( "# Equation type #define ABEtype setting error!!!", file=file1 )