Add plot-only restart script to skip recomputation when plotting is interrupted

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

.gitignore

@@ -1,6 +1,6 @@
 __pycache__
 GW150914
-GW150914-origin
+GW150914*
 docs
 *.tmp
-
+.codex

AMSS_NCKU_Input.py

@@ -16,9 +16,9 @@ import numpy
 File_directory   = "GW150914"                    ## output file directory
 Output_directory = "binary_output"               ## binary data file directory
                                                  ## The file directory name should not be too long
-MPI_processes    = 64                             ## number of mpi processes used in the simulation
+MPI_processes    = 2                             ## number of mpi processes used in the simulation
 
-GPU_Calculation  = "no"                          ## Use GPU or not 
+GPU_Calculation  = "yes"                          ## Use GPU or not 
                                                  ## (prefer "no" in the current version, because the GPU part may have bugs when integrated in this Python interface)
 CPU_Part         = 1.0
 GPU_Part         = 0.0
@@ -158,7 +158,7 @@ Detector_Rmax   = 160.0                  ## farest dector distance
 
 ## Setting the apprent horizon
 
-AHF_Find       = "no"                    ## whether to find the apparent horizon: choose "yes" or "no"
+AHF_Find       = "yes"                    ## whether to find the apparent horizon: choose "yes" or "no"
 
 AHF_Find_Every = 24
 AHF_Dump_Time  = 20.0

AMSS_NCKU_Program.py

@@ -58,31 +58,36 @@ File_directory = os.path.join(input_data.File_directory)
 
 ## If the specified output directory exists, ask the user whether to continue
 if os.path.exists(File_directory):
-    print( " Output dictionary has been existed !!!  "                                                              )
-    print( " If you want to overwrite the existing file directory, please input 'continue' in the terminal !! "     ) 
-    print( " If you want to retain the existing file directory, please input 'stop' in the terminal to stop the "   ) 
-    print( " simulation. Then you can reset the output dictionary in the input script file AMSS_NCKU_Input.py !!! " )
-    print(                                                                                                          )
-    ## Prompt whether to overwrite the existing directory
-    while True:
-        try:
-            inputvalue = input()
-            ## If the user agrees to overwrite, proceed and remove the existing directory
-            if ( inputvalue == "continue" ):
-                print( " Continue the calculation !!! " )
-                print(                                  )
-                break  
-            ## If the user chooses not to overwrite, exit and keep the existing directory
-            elif ( inputvalue == "stop" ):
-                print( " Stop the calculation !!! "    )
-                sys.exit() 
-            ## If the user input is invalid, prompt again
-            else:
+    auto_overwrite = str(getattr(input_data, "Auto_Overwrite_Output", "yes")).strip().lower()
+    if auto_overwrite in ("1", "yes", "y", "true", "on", "continue"):
+        print( " Output dictionary has been existed; Auto_Overwrite_Output=yes, continue the calculation. " )
+        print(                                                                                              )
+    else:
+        print( " Output dictionary has been existed !!!  "                                                              )
+        print( " If you want to overwrite the existing file directory, please input 'continue' in the terminal !! "     ) 
+        print( " If you want to retain the existing file directory, please input 'stop' in the terminal to stop the "   ) 
+        print( " simulation. Then you can reset the output dictionary in the input script file AMSS_NCKU_Input.py !!! " )
+        print(                                                                                                          )
+        ## Prompt whether to overwrite the existing directory
+        while True:
+            try:
+                inputvalue = input()
+                ## If the user agrees to overwrite, proceed and remove the existing directory
+                if ( inputvalue == "continue" ):
+                    print( " Continue the calculation !!! " )
+                    print(                                  )
+                    break  
+                ## If the user chooses not to overwrite, exit and keep the existing directory
+                elif ( inputvalue == "stop" ):
+                    print( " Stop the calculation !!! "    )
+                    sys.exit() 
+                ## If the user input is invalid, prompt again
+                else:
+                    print( " Please input your choice !!! "                   )
+                    print( " Input 'continue' or 'stop' in the terminal !!! " )
+            except ValueError:
                 print( " Please input your choice !!! "                   )
                 print( " Input 'continue' or 'stop' in the terminal !!! " )
-        except ValueError:
-            print( " Please input your choice !!! "                   )
-            print( " Input 'continue' or 'stop' in the terminal !!! " )
         
 ## Remove the existing output directory if present
 shutil.rmtree(File_directory, ignore_errors=True)
@@ -258,7 +263,7 @@ print()
 if (input_data.GPU_Calculation == "no"):
     ABE_file = os.path.join(AMSS_NCKU_source_copy, "ABE")
 elif (input_data.GPU_Calculation == "yes"):
-    ABE_file = os.path.join(AMSS_NCKU_source_copy, "ABEGPU")
+    ABE_file = os.path.join(AMSS_NCKU_source_copy, "ABE_CUDA")
 
 if not os.path.exists( ABE_file ):
     print(                                                                                                  )

AMSS_NCKU_Program_Plot.py

@@ -0,0 +1,100 @@
+##################################################################
+##
+## AMSS-NCKU Plot-Only Restart Script
+## Author: Xiaoqu / Claude
+## 2026/05/12
+##
+## This script checks for existing output data from AMSS_NCKU_Program.py.
+## If data exists, it skips all computation and goes directly to plotting,
+## saving time when plotting was interrupted.
+## If no data is found, it exits with a message.
+##
+##################################################################
+
+## Guard against re-execution by multiprocessing child processes.
+if __name__ != '__main__':
+    import sys as _sys
+    _sys.exit(0)
+
+
+import os
+import sys
+
+import AMSS_NCKU_Input as input_data
+
+##################################################################
+
+## Construct paths from input configuration
+File_directory         = os.path.join(input_data.File_directory)
+output_directory       = os.path.join(File_directory, "AMSS_NCKU_output")
+binary_results_directory = os.path.join(output_directory, input_data.Output_directory)
+figure_directory       = os.path.join(File_directory, "figure")
+
+##################################################################
+
+## Check whether the required output data files exist
+
+required_files = [
+    os.path.join(binary_results_directory, "bssn_BH.dat"),
+    os.path.join(binary_results_directory, "bssn_ADMQs.dat"),
+    os.path.join(binary_results_directory, "bssn_psi4.dat"),
+    os.path.join(binary_results_directory, "bssn_constraint.dat"),
+]
+
+missing_files = [f for f in required_files if not os.path.exists(f)]
+
+if missing_files:
+    print(" No existing AMSS_NCKU_Program.py output data found. ")
+    print(" The following required files are missing: ")
+    for f in missing_files:
+        print(f"   {f}")
+    print()
+    print(" Please run AMSS_NCKU_Program.py first to generate the simulation data. ")
+    print(" Exiting. ")
+    sys.exit(1)
+
+print(" Found existing AMSS_NCKU_Program.py output data. "          )
+print(" Skipping all computation and going directly to plotting. " )
+print()
+
+## Ensure the figure directory exists (it should, but be safe)
+os.makedirs(figure_directory, exist_ok=True)
+
+##################################################################
+
+## Plot the AMSS-NCKU program results
+
+import plot_xiaoqu
+import plot_GW_strain_amplitude_xiaoqu
+from parallel_plot_helper import run_plot_tasks_parallel
+
+plot_tasks = []
+
+## Plot black hole trajectory
+plot_tasks.append((plot_xiaoqu.generate_puncture_orbit_plot,   (binary_results_directory, figure_directory)))
+plot_tasks.append((plot_xiaoqu.generate_puncture_orbit_plot3D, (binary_results_directory, figure_directory)))
+
+## Plot black hole separation vs. time
+plot_tasks.append((plot_xiaoqu.generate_puncture_distence_plot, (binary_results_directory, figure_directory)))
+
+## Plot gravitational waveforms (psi4 and strain amplitude)
+for i in range(input_data.Detector_Number):
+    plot_tasks.append((plot_xiaoqu.generate_gravitational_wave_psi4_plot, (binary_results_directory, figure_directory, i)))
+    plot_tasks.append((plot_GW_strain_amplitude_xiaoqu.generate_gravitational_wave_amplitude_plot, (binary_results_directory, figure_directory, i)))
+
+## Plot ADM mass evolution
+for i in range(input_data.Detector_Number):
+    plot_tasks.append((plot_xiaoqu.generate_ADMmass_plot, (binary_results_directory, figure_directory, i)))
+
+## Plot Hamiltonian constraint violation over time
+for i in range(input_data.grid_level):
+    plot_tasks.append((plot_xiaoqu.generate_constraint_check_plot, (binary_results_directory, figure_directory, i)))
+
+run_plot_tasks_parallel(plot_tasks)
+
+## Plot stored binary data (runs serially, not in the parallel pool)
+plot_xiaoqu.generate_binary_data_plot(binary_results_directory, figure_directory)
+
+print()
+print(" Plotting completed successfully. ")
+print()

AMSS_NCKU_Verify_ASC26.py

@@ -9,6 +9,11 @@ Verification Requirements:
    - Y Component RMS
    - Z Component RMS
 2. ADM constraint violation < 2 (Grid Level 0)
+3. The following figure PDFs must match GW150914-origin exactly after rasterization:
+   - ADM_Constraint_Grid_Level_0.pdf
+   - BH_Trajectory_21_XY.pdf
+   - BH_Trajectory_XY.pdf
+   The script also reports the percentage of differing pixels for each figure.
 
 RMS Calculation Method:
 - Computes trajectory deviation on the XY plane independently for BH1 and BH2
@@ -23,6 +28,10 @@ Reference: GW150914-origin (baseline simulation)
 import numpy as np
 import sys
 import os
+import shutil
+import subprocess
+import tempfile
+from PIL import Image
 
 # ANSI Color Codes
 class Color:
@@ -61,6 +70,132 @@ def load_constraint_data(filepath):
                 data.append([float(x) for x in parts[:8]])
     return np.array(data)
 
+
+def resolve_figure_dir(path):
+    """Resolve the sibling figure directory from an output or figure path."""
+    normalized = os.path.normpath(path)
+    if os.path.basename(normalized) == "figure":
+        return normalized
+    return os.path.join(os.path.dirname(normalized), "figure")
+
+
+def render_pdf_to_images(pdf_path, dpi=150):
+    """Render a PDF to RGB images using Ghostscript."""
+    gs_path = shutil.which("gs")
+    if gs_path is None:
+        raise RuntimeError("Ghostscript executable 'gs' was not found in PATH")
+
+    with tempfile.TemporaryDirectory(prefix="amss_verify_pdf_") as temp_dir:
+        output_pattern = os.path.join(temp_dir, "page-%03d.ppm")
+        cmd = [
+            gs_path,
+            "-q",
+            "-dSAFER",
+            "-dBATCH",
+            "-dNOPAUSE",
+            "-sDEVICE=ppmraw",
+            f"-r{dpi}",
+            f"-o{output_pattern}",
+            pdf_path
+        ]
+
+        try:
+            subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+        except subprocess.CalledProcessError as exc:
+            message = exc.stderr.strip() or str(exc)
+            raise RuntimeError(f"Failed to render PDF '{pdf_path}': {message}") from exc
+
+        ppm_files = sorted(
+            os.path.join(temp_dir, filename)
+            for filename in os.listdir(temp_dir)
+            if filename.endswith(".ppm")
+        )
+
+        if not ppm_files:
+            raise RuntimeError(f"No rendered pages were produced for '{pdf_path}'")
+
+        images = []
+        for ppm_file in ppm_files:
+            with Image.open(ppm_file) as img:
+                images.append(np.array(img.convert("RGB"), dtype=np.uint8))
+
+        return images
+
+
+def compare_rendered_pages(ref_img, target_img):
+    """Return (different_pixels, total_pixels) for two rendered RGB pages."""
+    ref_h, ref_w = ref_img.shape[:2]
+    tgt_h, tgt_w = target_img.shape[:2]
+    total_pixels = max(ref_h, tgt_h) * max(ref_w, tgt_w)
+
+    if ref_h == tgt_h and ref_w == tgt_w:
+        different_pixels = int(np.count_nonzero(np.any(ref_img != target_img, axis=2)))
+        return different_pixels, total_pixels
+
+    diff_mask = np.ones((max(ref_h, tgt_h), max(ref_w, tgt_w)), dtype=bool)
+    overlap_h = min(ref_h, tgt_h)
+    overlap_w = min(ref_w, tgt_w)
+    overlap_diff = np.any(ref_img[:overlap_h, :overlap_w] != target_img[:overlap_h, :overlap_w], axis=2)
+    diff_mask[:overlap_h, :overlap_w] = overlap_diff
+    different_pixels = int(np.count_nonzero(diff_mask))
+    return different_pixels, total_pixels
+
+
+def compare_pdf_images(ref_pdf, target_pdf, dpi=150, threshold_percent=0.001):
+    """Compare two PDFs by rasterizing them and counting differing pixels."""
+    ref_pages = render_pdf_to_images(ref_pdf, dpi=dpi)
+    target_pages = render_pdf_to_images(target_pdf, dpi=dpi)
+
+    total_pixels = 0
+    different_pixels = 0
+    max_pages = max(len(ref_pages), len(target_pages))
+
+    for page_idx in range(max_pages):
+        if page_idx < len(ref_pages) and page_idx < len(target_pages):
+            page_diff, page_total = compare_rendered_pages(ref_pages[page_idx], target_pages[page_idx])
+        else:
+            existing_page = ref_pages[page_idx] if page_idx < len(ref_pages) else target_pages[page_idx]
+            page_total = existing_page.shape[0] * existing_page.shape[1]
+            page_diff = page_total
+
+        total_pixels += page_total
+        different_pixels += page_diff
+
+    diff_percent = (different_pixels / total_pixels * 100.0) if total_pixels else 0.0
+    return {
+        "different_pixels": different_pixels,
+        "total_pixels": total_pixels,
+        "diff_percent": diff_percent,
+        "pages_ref": len(ref_pages),
+        "pages_target": len(target_pages),
+        "passed": diff_percent < threshold_percent
+    }
+
+
+def compare_required_figures(reference_figure_dir, target_figure_dir):
+    """Compare the required GW150914 figure PDFs."""
+    figure_names = [
+        "ADM_Constraint_Grid_Level_0.pdf",
+        "BH_Trajectory_21_XY.pdf",
+        "BH_Trajectory_XY.pdf"
+    ]
+
+    results = []
+    for figure_name in figure_names:
+        ref_pdf = os.path.join(reference_figure_dir, figure_name)
+        target_pdf = os.path.join(target_figure_dir, figure_name)
+
+        if not os.path.exists(ref_pdf):
+            raise FileNotFoundError(f"Reference figure not found: {ref_pdf}")
+        if not os.path.exists(target_pdf):
+            raise FileNotFoundError(f"Target figure not found: {target_pdf}")
+
+        comparison = compare_pdf_images(ref_pdf, target_pdf)
+        comparison["name"] = figure_name
+        results.append(comparison)
+
+    return results
+
 def calculate_all_rms_errors(bh_data_ref, bh_data_target):
     """
     Calculate 3D Vector RMS and component-wise RMS (X, Y, Z) independently.
@@ -184,18 +319,45 @@ def print_constraint_results(results, threshold=2.0):
     return passed
 
 
-def print_summary(rms_passed, constraint_passed):
+def print_figure_results(results, threshold_percent=0.001):
+    print(f"\n{Color.BOLD}3. Figure Pixel Comparison (PDF Rasterization){Color.RESET}")
+    print("-" * 65)
+    print(f"   Requirement: < {threshold_percent:.3f}% differing pixels\n")
+
+    all_passed = True
+    for result in results:
+        passed = result["passed"]
+        all_passed = all_passed and passed
+        status = get_status_text(passed)
+        print(f"   {result['name']:32}: {result['diff_percent']:10.6f}%   |   Status: {status}")
+
+        if result["pages_ref"] != result["pages_target"]:
+            print(f"   {'':32}  pages(ref/target): {result['pages_ref']}/{result['pages_target']}")
+
+    return all_passed
+
+
+def print_figure_error(error_message):
+    print(f"\n{Color.BOLD}3. Figure Pixel Comparison (PDF Rasterization){Color.RESET}")
+    print("-" * 65)
+    print(f"   {Color.RED}Error: {error_message}{Color.RESET}")
+    return False
+
+
+def print_summary(rms_passed, constraint_passed, figure_passed):
     print("\n" + Color.BLUE + Color.BOLD + "=" * 65 + Color.RESET)
     print(Color.BOLD + "Verification Summary" + Color.RESET)
     print(Color.BLUE + Color.BOLD + "=" * 65 + Color.RESET)
 
-    all_passed = rms_passed and constraint_passed
+    all_passed = rms_passed and constraint_passed and figure_passed
     
     res_rms = get_status_text(rms_passed)
     res_con = get_status_text(constraint_passed)
+    res_fig = get_status_text(figure_passed)
 
     print(f"   [1] Comprehensive RMS check:      {res_rms}")
     print(f"   [2] ADM constraint check:         {res_con}")
+    print(f"   [3] Figure pixel comparison:      {res_fig}")
     
     final_status = f"{Color.GREEN}{Color.BOLD}ALL CHECKS PASSED{Color.RESET}" if all_passed else f"{Color.RED}{Color.BOLD}SOME CHECKS FAILED{Color.RESET}"
     print(f"\n   Overall result: {final_status}")
@@ -212,6 +374,8 @@ def main():
 
     script_dir = os.path.dirname(os.path.abspath(__file__))
     reference_dir = os.path.join(script_dir, "GW150914-origin/AMSS_NCKU_output")
+    target_figure_dir = resolve_figure_dir(target_dir)
+    reference_figure_dir = os.path.join(script_dir, "GW150914-origin/figure")
 
     bh_file_ref = os.path.join(reference_dir, "bssn_BH.dat")
     bh_file_target = os.path.join(target_dir, "bssn_BH.dat")
@@ -230,6 +394,8 @@ def main():
     print_header()
     print(f"\n{Color.BOLD}Reference (Baseline):{Color.RESET} {Color.BLUE}{reference_dir}{Color.RESET}")
     print(f"{Color.BOLD}Target (Optimized):  {Color.RESET} {Color.BLUE}{target_dir}{Color.RESET}")
+    print(f"{Color.BOLD}Reference Figures:   {Color.RESET} {Color.BLUE}{reference_figure_dir}{Color.RESET}")
+    print(f"{Color.BOLD}Target Figures:      {Color.RESET} {Color.BLUE}{target_figure_dir}{Color.RESET}")
 
     bh_data_ref = load_bh_trajectory(bh_file_ref)
     bh_data_target = load_bh_trajectory(bh_file_target)
@@ -243,7 +409,13 @@ def main():
     constraint_results = analyze_constraint_violation(constraint_data)
     constraint_passed = print_constraint_results(constraint_results)
 
-    all_passed = print_summary(rms_passed, constraint_passed)
+    try:
+        figure_results = compare_required_figures(reference_figure_dir, target_figure_dir)
+        figure_passed = print_figure_results(figure_results)
+    except (FileNotFoundError, RuntimeError) as exc:
+        figure_passed = print_figure_error(str(exc))
+
+    all_passed = print_summary(rms_passed, constraint_passed, figure_passed)
     sys.exit(0 if all_passed else 1)
 
 if __name__ == "__main__":

AMSS_NCKU_source/ABE.C

@@ -198,16 +198,16 @@ int main(int argc, char *argv[])
       if (myrank == 0)
       {
             string out_dir;
-            char filename[50];
+            string filename;
             map<string, string>::iterator iter;
             iter = parameters::str_par.find("output dir");
             if (iter != parameters::str_par.end())
             {
                   out_dir = iter->second;
             }
-            sprintf(filename, "%s/setting.par", out_dir.c_str());
+            filename = out_dir + "/setting.par";
             ofstream setfile;
-            setfile.open(filename, ios::trunc);
+            setfile.open(filename.c_str(), ios::trunc);
 
             if (!setfile.good())
             {
@@ -484,7 +484,11 @@ int main(int argc, char *argv[])
             cout << endl;
       }
 
-      delete ADM;
+      // Let the process teardown reclaim the simulation object.  Some derived
+      // equation classes keep MPI/CUDA-backed state whose destructor ordering
+      // is fragile at program shutdown.
+      if (getenv("AMSS_DELETE_ADM_ON_EXIT"))
+            delete ADM;
 
       //=======================caculation done=============================================================
 

AMSS_NCKU_source/Block.C

@@ -12,7 +12,61 @@ using namespace std;
 #include "Block.h"
 #include "misc.h"
 
-Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), ingfs(ingfsi), fngfs(fngfsi), lev(levi), cgpu(cgpui)
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+#include <cuda_runtime_api.h>
+#endif
+
+namespace {
+
+bool cuda_pin_gridfuncs_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_PIN_GRIDFUNCS");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+double *alloc_gridfunc(size_t count, unsigned char &pinned)
+{
+  pinned = 0;
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+  if (cuda_pin_gridfuncs_enabled())
+  {
+    double *ptr = 0;
+    cudaError_t err = cudaMallocHost((void **)&ptr, count * sizeof(double));
+    if (err == cudaSuccess)
+    {
+      pinned = 1;
+      return ptr;
+    }
+    cudaGetLastError();
+  }
+#endif
+  return (double *)malloc(sizeof(double) * count);
+}
+
+void free_gridfunc(double *ptr, unsigned char pinned)
+{
+  if (!ptr)
+    return;
+#if USE_CUDA_BSSN || USE_CUDA_Z4C
+  if (pinned)
+  {
+    cudaFreeHost(ptr);
+    return;
+  }
+#else
+  (void)pinned;
+#endif
+  free(ptr);
+}
+
+}
+
+Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), lev(levi), cgpu(cgpui), ingfs(ingfsi), fngfs(fngfsi), igfs(0), fgfs(0), fgfs_pinned(0)
 {
   for (int i = 0; i < dim; i++)
     X[i] = 0;
@@ -70,9 +124,10 @@ Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fng
 
     int nn = shape[0] * shape[1] * shape[2];
     fgfs = new double *[fngfs];
+    fgfs_pinned = new unsigned char[fngfs];
     for (int i = 0; i < fngfs; i++)
     {
-      fgfs[i] = (double *)malloc(sizeof(double) * nn);
+      fgfs[i] = alloc_gridfunc((size_t)nn, fgfs_pinned[i]);
       if (!(fgfs[i]))
       {
         cout << "on node#" << rank << ", out of memory when constructing Block." << endl;
@@ -107,11 +162,13 @@ Block::~Block()
       free(igfs[i]);
     delete[] igfs;
     for (int i = 0; i < fngfs; i++)
-      free(fgfs[i]);
+      free_gridfunc(fgfs[i], fgfs_pinned ? fgfs_pinned[i] : 0);
     delete[] fgfs;
+    delete[] fgfs_pinned;
     X[0] = X[1] = X[2] = 0;
     igfs = 0;
     fgfs = 0;
+    fgfs_pinned = 0;
   }
 }
 void Block::checkBlock()
@@ -187,6 +244,8 @@ void Block::swapList(MyList<var> *VarList1, MyList<var> *VarList2, int myrank)
     while (varl1 && varl2)
     {
       misc::swap<double *>(fgfs[varl1->data->sgfn], fgfs[varl2->data->sgfn]);
+      if (fgfs_pinned)
+        misc::swap<unsigned char>(fgfs_pinned[varl1->data->sgfn], fgfs_pinned[varl2->data->sgfn]);
       varl1 = varl1->next;
       varl2 = varl2->next;
     }

AMSS_NCKU_source/Block.h

@@ -18,9 +18,10 @@ public:
    int ingfs, fngfs;
    int *(*igfs);
    double *(*fgfs);
+   unsigned char *fgfs_pinned;
 
 public:
-   Block() {};
+   Block() : rank(0), lev(0), cgpu(0), ingfs(0), fngfs(0), igfs(0), fgfs(0), fgfs_pinned(0) {};
    Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfs, int levi, const int cgpui = 0);
 
    ~Block();

AMSS_NCKU_source/MPatch.C

@@ -14,6 +14,9 @@ using namespace std;
 #include "MPatch.h"
 #include "Parallel.h"
 #include "fmisc.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
 #ifdef INTERP_LB_PROFILE
 #include "interp_lb_profile.h"
 #endif
@@ -178,6 +181,444 @@ int find_block_index_for_point(const BlockBinIndex &index, const double *pox, co
 
   return -1;
 }
+
+inline int fortran_idint_local(double x)
+{
+  return int(x);
+}
+
+bool interp_fast_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST");
+    enabled = (!env || atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool interp_gpu_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_GPU");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool interp_fast_compare_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST_COMPARE");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+double interp_fast_compare_tol()
+{
+  static double tol = -1.0;
+  if (tol < 0.0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST_COMPARE_TOL");
+    tol = (env && atof(env) > 0.0) ? atof(env) : 1.0e-11;
+  }
+  return tol;
+}
+
+long long interp_fast_compare_limit()
+{
+  static long long limit = -1;
+  if (limit < 0)
+  {
+    const char *env = getenv("AMSS_INTERP_FAST_COMPARE_LIMIT");
+    limit = (env && atoll(env) > 0) ? atoll(env) : 4096;
+  }
+  return limit;
+}
+
+struct FastInterpStencil
+{
+  int cxB[dim];
+  double cx[dim];
+  double wx[8];
+  double wy[8];
+  double wz[8];
+  int nsamples;
+  int loc[512];
+  unsigned char sign_mask[512];
+  double weight[512];
+};
+
+inline void lagrange_unit_weights(double x, int ordn, double *w)
+{
+  for (int i = 0; i < ordn; i++)
+  {
+    double num = 1.0;
+    double den = 1.0;
+    for (int j = 0; j < ordn; j++)
+    {
+      if (j == i)
+        continue;
+      num *= (x - double(j));
+      den *= double(i - j);
+    }
+    w[i] = num / den;
+  }
+}
+
+inline void z_unit_weights(double x, int ordn, double *w)
+{
+  if (ordn == 6)
+  {
+    static const double c_uniform[6] = {-1.0, 5.0, -10.0, 10.0, -5.0, 1.0};
+    for (int i = 0; i < 6; i++)
+    {
+      if (x == double(i))
+      {
+        for (int j = 0; j < 6; j++)
+          w[j] = (j == i) ? 1.0 : 0.0;
+        return;
+      }
+    }
+    double den = 0.0;
+    for (int i = 0; i < 6; i++)
+    {
+      w[i] = c_uniform[i] / (x - double(i));
+      den += w[i];
+    }
+    for (int i = 0; i < 6; i++)
+      w[i] /= den;
+    return;
+  }
+  lagrange_unit_weights(x, ordn, w);
+}
+
+inline bool fast_interp_map_index(int idx, int extent, int d,
+                                  int &mapped, unsigned char &mask)
+{
+  if (idx > 0)
+    mapped = idx;
+  else
+  {
+    mask |= (unsigned char)(1u << d);
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+    mapped = 2 - idx;
+#else
+#ifdef Cell
+    mapped = 1 - idx;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+  }
+  return mapped >= 1 && mapped <= extent;
+}
+
+bool prepare_fast_interp_stencil(Block *BP, const double *pox, int ordn,
+                                 int Symmetry, FastInterpStencil &st)
+{
+  if (!BP || ordn <= 0 || ordn > 8)
+    return false;
+
+  st.nsamples = 0;
+
+  const int NO_SYMM = 0;
+  const int OCTANT = 2;
+  int cmin[dim], cmax[dim], cxT[dim];
+  for (int d = 0; d < dim; d++)
+  {
+    const double *X = BP->X[d];
+    const double dX = X[1] - X[0];
+    const int cxI = fortran_idint_local((pox[d] - X[0]) / dX + 0.4) + 1;
+    st.cxB[d] = cxI - ordn / 2 + 1;
+    cxT[d] = st.cxB[d] + ordn - 1;
+    cmin[d] = 1;
+    cmax[d] = BP->shape[d];
+
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+    if (Symmetry == OCTANT && d < 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 2;
+    if (Symmetry != NO_SYMM && d == 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 2;
+#else
+#ifdef Cell
+    if (Symmetry == OCTANT && d < 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 1;
+    if (Symmetry != NO_SYMM && d == 2 && fabs(X[0]) < dX)
+      cmin[d] = -ordn / 2 + 1;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+
+    if (st.cxB[d] < cmin[d])
+    {
+      st.cxB[d] = cmin[d];
+      cxT[d] = st.cxB[d] + ordn - 1;
+    }
+    if (cxT[d] > cmax[d])
+    {
+      cxT[d] = cmax[d];
+      st.cxB[d] = cxT[d] + 1 - ordn;
+    }
+
+    if (st.cxB[d] > 0)
+      st.cx[d] = (pox[d] - X[st.cxB[d] - 1]) / dX;
+    else
+    {
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+      st.cx[d] = (pox[d] + X[1 - st.cxB[d]]) / dX;
+#else
+#ifdef Cell
+      st.cx[d] = (pox[d] + X[-st.cxB[d]]) / dX;
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+    }
+  }
+
+  lagrange_unit_weights(st.cx[0], ordn, st.wx);
+  lagrange_unit_weights(st.cx[1], ordn, st.wy);
+  z_unit_weights(st.cx[2], ordn, st.wz);
+
+  for (int kk = 0; kk < ordn; kk++)
+  {
+    for (int jj = 0; jj < ordn; jj++)
+    {
+      for (int ii = 0; ii < ordn; ii++)
+      {
+        unsigned char mask = 0;
+        int ix, iy, iz;
+        if (!fast_interp_map_index(st.cxB[0] + ii, BP->shape[0], 0, ix, mask) ||
+            !fast_interp_map_index(st.cxB[1] + jj, BP->shape[1], 1, iy, mask) ||
+            !fast_interp_map_index(st.cxB[2] + kk, BP->shape[2], 2, iz, mask))
+          return false;
+        const int s = st.nsamples++;
+        st.loc[s] = (ix - 1) + (iy - 1) * BP->shape[0] +
+                    (iz - 1) * BP->shape[0] * BP->shape[1];
+        st.sign_mask[s] = mask;
+        st.weight[s] = st.wx[ii] * st.wy[jj] * st.wz[kk];
+      }
+    }
+  }
+  return true;
+}
+
+bool interpolate_var_list_with_stencil(Block *BP, MyList<var> *VarList,
+                                       int num_var, const double *pox,
+                                       int ordn, int Symmetry,
+                                       const FastInterpStencil &st,
+                                       double *out)
+{
+  if (num_var <= 0 || num_var > 128)
+    return false;
+
+  double *data_ptrs[128];
+  double *soa_ptrs[128];
+  var *vars[128];
+  MyList<var> *varl = VarList;
+  int k = 0;
+  while (varl)
+  {
+    if (k >= num_var)
+      return false;
+    vars[k] = varl->data;
+    data_ptrs[k] = BP->fgfs[vars[k]->sgfn];
+    soa_ptrs[k] = vars[k]->SoA;
+    out[k] = 0.0;
+    varl = varl->next;
+    k++;
+  }
+
+  if (k != num_var)
+    return false;
+
+  for (int s = 0; s < st.nsamples; s++)
+  {
+    const int loc = st.loc[s];
+    const double w = st.weight[s];
+    const unsigned char mask = st.sign_mask[s];
+    if (mask == 0)
+    {
+      for (int v = 0; v < num_var; v++)
+        out[v] += w * data_ptrs[v][loc];
+    }
+    else
+    {
+      for (int v = 0; v < num_var; v++)
+      {
+        const double *SoA = soa_ptrs[v];
+        double sgn = 1.0;
+        if (mask & 1u)
+          sgn *= SoA[0];
+        if (mask & 2u)
+          sgn *= SoA[1];
+        if (mask & 4u)
+          sgn *= SoA[2];
+        out[v] += w * sgn * data_ptrs[v][loc];
+      }
+    }
+  }
+
+  if (interp_fast_compare_enabled())
+  {
+    static int report_count = 0;
+    static long long compare_calls = 0;
+    if (compare_calls++ >= interp_fast_compare_limit())
+      return true;
+    const double tol = interp_fast_compare_tol();
+    varl = VarList;
+    k = 0;
+    while (varl)
+    {
+      var *vp = vars[k];
+      double ref = 0.0;
+      double x = pox[0], y = pox[1], z = pox[2];
+      f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2],
+                      BP->fgfs[vp->sgfn], ref,
+                      x, y, z, ordn, vp->SoA, Symmetry);
+      const double diff = fabs(ref - out[k]);
+      const double scale = 1.0 + fabs(ref);
+      if (diff > tol * scale && report_count < 32)
+      {
+        int rank = 0;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        fprintf(stderr,
+                "[AMSS-INTERP-CMP][rank %d] var=%s diff=%.17e ref=%.17e fast=%.17e p=(%.17e,%.17e,%.17e)\n",
+                rank, vp->name, diff, ref, out[k], pox[0], pox[1], pox[2]);
+        report_count++;
+      }
+      varl = varl->next;
+      k++;
+    }
+  }
+
+  return true;
+}
+
+bool interpolate_var_list_fast(Block *BP, MyList<var> *VarList, int num_var,
+                               const double *pox, int ordn, int Symmetry,
+                               double *out)
+{
+  if (!interp_fast_enabled())
+    return false;
+
+  FastInterpStencil st;
+  if (!prepare_fast_interp_stencil(BP, pox, ordn, Symmetry, st))
+    return false;
+
+  return interpolate_var_list_with_stencil(BP, VarList, num_var, pox,
+                                           ordn, Symmetry, st, out);
+}
+
+struct CachedInterpPoint
+{
+  Block *bp;
+  int owner_rank;
+  FastInterpStencil stencil;
+};
+
+struct SurfaceInterpCache
+{
+  Patch *patch;
+  int NN;
+  int symmetry;
+  double key[9];
+  vector<CachedInterpPoint> points;
+
+  SurfaceInterpCache() : patch(0), NN(0), symmetry(-1) {}
+};
+
+bool surface_cache_key_matches(const SurfaceInterpCache &cache, Patch *patch,
+                               int NN, double **XX, int Symmetry)
+{
+  if (cache.patch != patch || cache.NN != NN || cache.symmetry != Symmetry ||
+      int(cache.points.size()) != NN || NN <= 0)
+    return false;
+  const int mid = NN / 2;
+  const int last = NN - 1;
+  const int ids[3] = {0, mid, last};
+  int p = 0;
+  for (int q = 0; q < 3; q++)
+    for (int d = 0; d < dim; d++)
+      if (cache.key[p++] != XX[d][ids[q]])
+        return false;
+  return true;
+}
+
+SurfaceInterpCache *find_surface_cache(Patch *patch, int NN, double **XX,
+                                       int Symmetry)
+{
+  static vector<SurfaceInterpCache> caches;
+  for (size_t i = 0; i < caches.size(); i++)
+    if (surface_cache_key_matches(caches[i], patch, NN, XX, Symmetry))
+      return &caches[i];
+  if (caches.size() >= 24)
+    caches.erase(caches.begin());
+  caches.push_back(SurfaceInterpCache());
+  return &caches.back();
+}
+
+bool build_surface_cache(SurfaceInterpCache &cache, Patch *patch, int NN,
+                         double **XX, int Symmetry, const double *DH,
+                         const BlockBinIndex &block_index, int ordn)
+{
+  int myrank = 0;
+  MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+  cache.patch = patch;
+  cache.NN = NN;
+  cache.symmetry = Symmetry;
+  cache.points.clear();
+  cache.points.resize(NN);
+  const int mid = NN / 2;
+  const int last = NN - 1;
+  const int ids[3] = {0, mid, last};
+  int p = 0;
+  for (int q = 0; q < 3; q++)
+    for (int d = 0; d < dim; d++)
+      cache.key[p++] = XX[d][ids[q]];
+
+  for (int j = 0; j < NN; j++)
+  {
+    double pox[dim];
+    for (int d = 0; d < dim; d++)
+      pox[d] = XX[d][j];
+    const int block_i = find_block_index_for_point(block_index, pox, DH);
+    if (block_i < 0)
+    {
+      cache.points[j].bp = 0;
+      cache.points[j].owner_rank = -1;
+      continue;
+    }
+    Block *BP = block_index.views[block_i].bp;
+    cache.points[j].bp = BP;
+    cache.points[j].owner_rank = BP->rank;
+    cache.points[j].stencil.nsamples = 0;
+    if (BP->rank == myrank)
+    {
+      if (!prepare_fast_interp_stencil(BP, pox, ordn, Symmetry,
+                                       cache.points[j].stencil))
+        return false;
+    }
+  }
+  return true;
+}
 } // namespace
 
 Patch::Patch(int DIM, int *shapei, double *bboxi, int levi, bool buflog, int Symmetry) : lev(levi)
@@ -565,14 +1006,18 @@ void Patch::Interp_Points(MyList<var> *VarList,
       if (myrank == BP->rank)
       {
         //---> interpolation
-        varl = VarList;
-        int k = 0;
-        while (varl) // run along variables
+        if (!interpolate_var_list_fast(BP, VarList, num_var, pox, ordn,
+                                       Symmetry, Shellf + j * num_var))
         {
-          f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
-                          pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
-          varl = varl->next;
-          k++;
+          varl = VarList;
+          int k = 0;
+          while (varl) // run along variables
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
+            varl = varl->next;
+            k++;
+          }
         }
       }
     }
@@ -659,8 +1104,6 @@ void Patch::Interp_Points(MyList<var> *VarList,
     varl = varl->next;
   }
 
-  memset(Shellf, 0, sizeof(double) * NN * num_var);
-
   // owner_rank[j] records which MPI rank owns point j
   int *owner_rank;
   owner_rank = new int[NN];
@@ -672,8 +1115,113 @@ void Patch::Interp_Points(MyList<var> *VarList,
     DH[i] = getdX(i);
   BlockBinIndex block_index;
   build_block_bin_index(this, DH, block_index);
+  SurfaceInterpCache *surface_cache = 0;
+  bool use_surface_cache = false;
+  if (interp_fast_enabled())
+  {
+    surface_cache = find_surface_cache(this, NN, XX, Symmetry);
+    use_surface_cache = surface_cache_key_matches(*surface_cache, this, NN, XX, Symmetry);
+    if (!use_surface_cache)
+      use_surface_cache = build_surface_cache(*surface_cache, this, NN, XX,
+                                              Symmetry, DH, block_index, ordn);
+  }
 
   // --- Interpolation phase (identical to original) ---
+#if USE_CUDA_BSSN
+  const bool use_gpu_interp = interp_gpu_enabled() && use_surface_cache && num_var == 2 &&
+                              VarList && VarList->next && !VarList->next->next;
+#else
+  const bool use_gpu_interp = false;
+#endif
+  if (use_gpu_interp)
+  {
+#if USE_CUDA_BSSN
+    vector<vector<int> > local_points(block_index.views.size());
+    for (int j = 0; j < NN; j++)
+    {
+      for (int i = 0; i < dim; i++)
+      {
+        if (myrank == 0 && (XX[i][j] < bbox[i] + lli[i] * DH[i] || XX[i][j] > bbox[dim + i] - uui[i] * DH[i]))
+        {
+          cout << "Patch::Interp_Points: point (";
+          for (int k = 0; k < dim; k++)
+          {
+            cout << XX[k][j];
+            if (k < dim - 1)
+              cout << ",";
+            else
+              cout << ") is out of current Patch." << endl;
+          }
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+      }
+
+      CachedInterpPoint &cp = surface_cache->points[j];
+      Block *BP = cp.bp;
+      owner_rank[j] = cp.owner_rank;
+      if (BP && myrank == BP->rank)
+      {
+        for (size_t bi = 0; bi < block_index.views.size(); bi++)
+        {
+          if (block_index.views[bi].bp == BP)
+          {
+            local_points[bi].push_back(j);
+            break;
+          }
+        }
+      }
+    }
+
+    var *v0 = VarList->data;
+    var *v1 = VarList->next->data;
+    double soa6[6] = {
+        v0->SoA[0], v0->SoA[1], v0->SoA[2],
+        v1->SoA[0], v1->SoA[1], v1->SoA[2]};
+
+    for (size_t bi = 0; bi < local_points.size(); bi++)
+    {
+      const int count = int(local_points[bi].size());
+      if (count <= 0)
+        continue;
+
+      Block *BP = block_index.views[bi].bp;
+      vector<double> px(count), py(count), pz(count), out(2 * count);
+      for (int q = 0; q < count; q++)
+      {
+        const int j = local_points[bi][q];
+        px[q] = XX[0][j];
+        py[q] = XX[1][j];
+        pz[q] = XX[2][j];
+      }
+
+      const double dx = BP->X[0][1] - BP->X[0][0];
+      const double dy = BP->X[1][1] - BP->X[1][0];
+      const double dz = BP->X[2][1] - BP->X[2][0];
+      const int ok = bssn_cuda_interp_host_two_fields(
+          BP, BP->shape,
+          BP->fgfs[v0->sgfn], BP->fgfs[v1->sgfn],
+          BP->X[0][0], BP->X[1][0], BP->X[2][0],
+          dx, dy, dz,
+          &px[0], &py[0], &pz[0], count,
+          ordn, Symmetry, soa6, &out[0]);
+      if (ok != 0)
+      {
+        if (myrank == 0)
+          cout << "Patch::Interp_Points: CUDA two-field interpolation failed" << endl;
+        MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+
+      for (int q = 0; q < count; q++)
+      {
+        const int j = local_points[bi][q];
+        Shellf[j * num_var] = out[2 * q];
+        Shellf[j * num_var + 1] = out[2 * q + 1];
+      }
+    }
+#endif
+  }
+  else
+  {
   for (int j = 0; j < NN; j++)
   {
     double pox[dim];
@@ -695,24 +1243,55 @@ void Patch::Interp_Points(MyList<var> *VarList,
       }
     }
 
-    const int block_i = find_block_index_for_point(block_index, pox, DH);
-    if (block_i >= 0)
+    if (use_surface_cache)
     {
-      Block *BP = block_index.views[block_i].bp;
-      owner_rank[j] = BP->rank;
-      if (myrank == BP->rank)
+      CachedInterpPoint &cp = surface_cache->points[j];
+      Block *BP = cp.bp;
+      owner_rank[j] = cp.owner_rank;
+      if (BP && myrank == BP->rank)
       {
-        varl = VarList;
-        int k = 0;
-        while (varl)
+        if (!interpolate_var_list_with_stencil(BP, VarList, num_var, pox,
+                                               ordn, Symmetry, cp.stencil,
+                                               Shellf + j * num_var))
         {
-          f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
-                          pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
-          varl = varl->next;
-          k++;
+          MyList<var> *varl_fallback = VarList;
+          int k = 0;
+          while (varl_fallback)
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl_fallback->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl_fallback->data->SoA, Symmetry);
+            varl_fallback = varl_fallback->next;
+            k++;
+          }
         }
       }
     }
+    else
+    {
+      const int block_i = find_block_index_for_point(block_index, pox, DH);
+      if (block_i >= 0)
+      {
+        Block *BP = block_index.views[block_i].bp;
+        owner_rank[j] = BP->rank;
+      if (myrank == BP->rank)
+      {
+        if (!interpolate_var_list_fast(BP, VarList, num_var, pox, ordn,
+                                       Symmetry, Shellf + j * num_var))
+        {
+          MyList<var> *varl_fallback = VarList;
+          int k = 0;
+          while (varl_fallback)
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl_fallback->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl_fallback->data->SoA, Symmetry);
+            varl_fallback = varl_fallback->next;
+            k++;
+          }
+        }
+      }
+      }
+    }
+  }
   }
 
 #ifdef INTERP_LB_PROFILE
@@ -969,14 +1548,18 @@ void Patch::Interp_Points(MyList<var> *VarList,
       if (myrank == BP->rank)
       {
         //---> interpolation
-        varl = VarList;
-        int k = 0;
-        while (varl) // run along variables
+        if (!interpolate_var_list_fast(BP, VarList, num_var, pox, ordn,
+                                       Symmetry, Shellf + j * num_var))
         {
-          f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
-                          pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
-          varl = varl->next;
-          k++;
+          varl = VarList;
+          int k = 0;
+          while (varl) // run along variables
+          {
+            f_global_interp(BP->shape, BP->X[0], BP->X[1], BP->X[2], BP->fgfs[varl->data->sgfn], Shellf[j * num_var + k],
+                            pox[0], pox[1], pox[2], ordn, varl->data->SoA, Symmetry);
+            varl = varl->next;
+            k++;
+          }
         }
       }
     }

AMSS_NCKU_source/Parallel.h

@@ -104,6 +104,14 @@ namespace Parallel
     double **recv_bufs;
     int *send_buf_caps;
     int *recv_buf_caps;
+    unsigned char *send_buf_pinned;
+    unsigned char *recv_buf_pinned;
+    unsigned char *send_buf_is_dev;
+    unsigned char *recv_buf_is_dev;
+    int *send_buf_caps_dev;
+    int *recv_buf_caps_dev;
+    double **send_bufs_dev;
+    double **recv_bufs_dev;
     MPI_Request *reqs;
     MPI_Status *stats;
     int max_reqs;
@@ -111,12 +119,14 @@ namespace Parallel
     int *tc_req_node;
     int *tc_req_is_recv;
     int *tc_completed;
+    bool cuda_aware_mode;
     SyncCache();
     void invalidate();
     void destroy();
   };
 
   void Sync_cached(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry, SyncCache &cache);
+  void Sync_ensure_cache(MyList<Patch> *PatL, int Symmetry, SyncCache &cache);
   void transfer_cached(MyList<gridseg> **src, MyList<gridseg> **dst,
                        MyList<var> *VarList1, MyList<var> *VarList2,
                        int Symmetry, SyncCache &cache);
@@ -183,6 +193,7 @@ namespace Parallel
                          MyList<Parallel::gridseg> **out_src, MyList<Parallel::gridseg> **out_dst);
   void PeriodicBD(Patch *Pat, MyList<var> *VarList, int Symmetry);
   double L2Norm(Patch *Pat, var *vf);
+  void L2Norm7(Patch *Pat, var **vf, double *norms);
   void checkgsl(MyList<Parallel::gridseg> *pp, bool first_only);
   void checkvarl(MyList<var> *pp, bool first_only);
   MyList<Parallel::gridseg> *divide_gsl(MyList<Parallel::gridseg> *p, Patch *Pat);
@@ -218,6 +229,7 @@ namespace Parallel
   void checkpatchlist(MyList<Patch> *PatL, bool buflog);
 
   double L2Norm(Patch *Pat, var *vf, MPI_Comm Comm_here);
+  void L2Norm7(Patch *Pat, var **vf, double *norms, MPI_Comm Comm_here);
   bool PatList_Interp_Points(MyList<Patch> *PatL, MyList<var> *VarList,
                              int NN, double **XX,
                              double *Shellf, int Symmetry, MPI_Comm Comm_here);

AMSS_NCKU_source/ShellPatch.h

@@ -102,6 +102,16 @@ public:
                  //-1: means no dumy dimension at all; 0: means rho; 1: means sigma
    };
 
+   // Thread-safe search result (no pointers to shared mutable state)
+   struct PointSearchResult
+   {
+      bool found;
+      Block *Bg;
+      double gx, gy, gz; // global Cartesian coordinates
+      double lx, ly, lz; // local coordinates within the found block
+      int ssst;          // source shell-patch type (-1 = Cartesian)
+   };
+
    int myrank;
    int shape[dim];   // for (rho, sigma, R), for rho and sigma means number of points for every pi/2
    double Rrange[2]; // for Rmin and Rmax
@@ -175,6 +185,12 @@ public:
                          MyList<Patch> *Pp, double CDH[dim], MyList<pointstru> *pss);
    bool prolongpointstru(MyList<pointstru> *&psul, bool ssyn, int tsst, MyList<ss_patch> *sPp, double DH[dim],
                          MyList<Patch> *Pp, double CDH[dim], double x, double y, double z, int Symmetry, int rank_in);
+   // Read-only point search — thread-safe (no shared mutable state modified)
+   PointSearchResult prolongpointstru_search(bool ssyn, int tsst, MyList<ss_patch> *sPp, double DH[dim],
+                                             MyList<Patch> *Pp, double CDH[dim], double x, double y, double z,
+                                             int Symmetry, int rank_in);
+   // Append a search result to a linked list — use inside omp critical section
+   void prolongpointstru_append(MyList<pointstru> *&psul, const PointSearchResult &sr, int tsst);
    void setupintintstuff(int cpusize, MyList<Patch> *CPatL, int Symmetry);
    void intertransfer(MyList<pointstru> **src, MyList<pointstru> **dst,
                       MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /*target */,
@@ -198,6 +214,7 @@ public:
    void write_Pablo_file_ss(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
                             char *filename, int sst);
    double L2Norm(var *vf);
+   void L2Norm7(var **vf, double *norms);
    void Find_Maximum(MyList<var> *VarList, double *XX, double *Shellf);
 };
 

AMSS_NCKU_source/Z4c_class.C

@@ -3,6 +3,7 @@
 #include <sstream>
 #include <cstdio>
 #include <map>
+#include <string>
 using namespace std;
 #else
 #include <stdio.h>
@@ -28,6 +29,20 @@ using namespace std;
 #include "kodiss.h"
 #include "parameters.h"
 
+#ifndef USE_CUDA_Z4C
+#define USE_CUDA_Z4C 0
+#endif
+
+#if USE_CUDA_Z4C && (ABEtype == 2)
+#include "z4c_rhs_cuda.h"
+#endif
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#ifdef WithShell
+#include "bssn_gpu.h"
+#endif
+#endif
+
 #ifdef With_AHF
 #include "derivatives.h"
 #include "myglobal.h"
@@ -37,6 +52,81 @@ using namespace std;
 
 // Define Z4c_class
 
+#if USE_CUDA_Z4C && (ABEtype == 2) && defined(WithShell)
+// GPU-accelerated Z4C shell RHS: same parameter signature as f_compute_rhs_Z4c_ss.
+// Internally calls gpu_rhs_z4c_ss which modifies trK→trKd before upload,
+// runs BSSN algebraic kernels, then applies Z4C post-processing (TZ_rhs, damping).
+extern "C" {
+static int cuda_compute_rhs_z4c_ss(
+    int *ex, double &T, double *crho, double *sigma, double *R,
+    double *X, double *Y, double *Z,
+    double *drhodx, double *drhody, double *drhodz,
+    double *dsigmadx, double *dsigmady, double *dsigmadz,
+    double *dRdx, double *dRdy, double *dRdz,
+    double *drhodxx, double *drhodxy, double *drhodxz, double *drhodyy, double *drhodyz, double *drhodzz,
+    double *dsigmadxx, double *dsigmadxy, double *dsigmadxz, double *dsigmadyy, double *dsigmadyz, double *dsigmadzz,
+    double *dRdxx, double *dRdxy, double *dRdxz, double *dRdyy, double *dRdyz, double *dRdzz,
+    double *chi, double *trK,
+    double *gxx, double *gxy, double *gxz, double *gyy, double *gyz, double *gzz,
+    double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+    double *Gamx, double *Gamy, double *Gamz,
+    double *Lap, double *betax, double *betay, double *betaz,
+    double *dtSfx, double *dtSfy, double *dtSfz,
+    double *TZ,
+    double *chi_rhs, double *trK_rhs,
+    double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+    double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+    double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+    double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+    double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+    double *TZ_rhs,
+    double *rho_mat, double *Sx, double *Sy, double *Sz,
+    double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+    double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+    double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+    double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+    double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+    double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
+    double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
+    int &Symmetry, int &Lev, double &eps, int &sst, int &co)
+{
+    return gpu_rhs_z4c_ss(0, 0,  // calledby=ABE_main, mpi_rank=device_0
+                     ex, T, crho, sigma, R, X, Y, Z,
+                     drhodx, drhody, drhodz,
+                     dsigmadx, dsigmady, dsigmadz,
+                     dRdx, dRdy, dRdz,
+                     drhodxx, drhodxy, drhodxz, drhodyy, drhodyz, drhodzz,
+                     dsigmadxx, dsigmadxy, dsigmadxz, dsigmadyy, dsigmadyz, dsigmadzz,
+                     dRdxx, dRdxy, dRdxz, dRdyy, dRdyz, dRdzz,
+                     chi, trK,
+                     gxx, gxy, gxz, gyy, gyz, gzz,
+                     Axx, Axy, Axz, Ayy, Ayz, Azz,
+                     Gamx, Gamy, Gamz,
+                     Lap, betax, betay, betaz,
+                     dtSfx, dtSfy, dtSfz,
+                     TZ,
+                     chi_rhs, trK_rhs,
+                     gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
+                     Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
+                     Gamx_rhs, Gamy_rhs, Gamz_rhs,
+                     Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
+                     dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
+                     TZ_rhs,
+                     rho_mat, Sx, Sy, Sz,
+                     Sxx, Sxy, Sxz, Syy, Syz, Szz,
+                     Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+                     Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+                     Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+                     Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+                     ham_Res, movx_Res, movy_Res, movz_Res,
+                     Gmx_Res, Gmy_Res, Gmz_Res,
+                     Symmetry, Lev, eps, sst, co);
+}
+}
+// Redirect all Z4C shell RHS calls in Step/SHStep to GPU
+#define f_compute_rhs_Z4c_ss cuda_compute_rhs_z4c_ss
+#endif
+
 // This class inherits some members and methods from the parent `bssn_class` and modifies others.
 // The modified members and methods are defined below (and in the header Z4c_class.h).
 // The remaining members/methods are inherited from `bssn_class` (declared in bssn_class.h).
@@ -132,6 +222,13 @@ void Z4c_class::Initialize()
     PhysTime = StartTime;
     Setup_Black_Hole_position();
   }
+
+  sync_cache_pre = new Parallel::SyncCache[GH->levels];
+  sync_cache_cor = new Parallel::SyncCache[GH->levels];
+  sync_cache_rp_coarse = new Parallel::SyncCache[GH->levels];
+  sync_cache_rp_fine = new Parallel::SyncCache[GH->levels];
+  sync_cache_restrict = new Parallel::SyncCache[GH->levels];
+  sync_cache_outbd = new Parallel::SyncCache[GH->levels];
 }
 
 //================================================================================================
@@ -170,8 +267,573 @@ Z4c_class::~Z4c_class()
 #ifndef CPBC
 // for sommerfeld boundary
 
+#if USE_CUDA_Z4C && (ABEtype == 2)
+#if (MRBD == 2)
+#error "USE_CUDA_Z4C resident path does not support MRBD == 2"
+#endif
+
+namespace {
+
+static const int k_z4c_cuda_bh_state_indices[3] = {18, 19, 20};
+
+bool fill_z4c_cuda_views(Block *cg, MyList<var> *vars,
+                         double **host_views,
+                         double *propspeeds = 0,
+                         double *soa_flat = 0)
+{
+  int idx = 0;
+  while (vars && idx < Z4C_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == Z4C_CUDA_STATE_COUNT && vars == 0;
+}
+
+void z4c_cuda_download_level_state(MyList<Patch> *PatL, MyList<var> *vars, int myrank, bool release_ctx)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && z4c_cuda_has_resident_state(cg))
+      {
+        double *state_out[Z4C_CUDA_STATE_COUNT];
+        if (!fill_z4c_cuda_views(cg, vars, state_out))
+        {
+          cout << "CUDA Z4C state list mismatch on resident state download" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (z4c_cuda_download_resident_state(cg, cg->shape, state_out))
+        {
+          cout << "CUDA Z4C resident state download failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (release_ctx)
+          z4c_cuda_release_step_ctx(cg);
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+
+bool z4c_cuda_patch_contains_point(Patch *patch, const double *point)
+{
+  if (!patch)
+    return false;
+  for (int d = 0; d < dim; d++)
+  {
+    const double h = patch->getdX(d);
+    const double lo = patch->bbox[d] + patch->lli[d] * h;
+    const double hi = patch->bbox[dim + d] - patch->uui[d] * h;
+    if (point[d] < lo || point[d] > hi)
+      return false;
+  }
+  return true;
+}
+
+bool z4c_cuda_point_in_block(Patch *patch, Block *block,
+                             const double *point, const double *DH)
+{
+  if (!patch || !block)
+    return false;
+  for (int d = 0; d < dim; d++)
+  {
+    double llb;
+    double uub;
+#ifdef Vertex
+#ifdef Cell
+#error Both Cell and Vertex are defined
+#endif
+    llb = (feq(block->bbox[d], patch->bbox[d], DH[d] / 2))
+              ? block->bbox[d] + patch->lli[d] * DH[d]
+              : block->bbox[d] + (ghost_width - 0.5) * DH[d];
+    uub = (feq(block->bbox[dim + d], patch->bbox[dim + d], DH[d] / 2))
+              ? block->bbox[dim + d] - patch->uui[d] * DH[d]
+              : block->bbox[dim + d] - (ghost_width - 0.5) * DH[d];
+#else
+#ifdef Cell
+    llb = (feq(block->bbox[d], patch->bbox[d], DH[d] / 2))
+              ? block->bbox[d] + patch->lli[d] * DH[d]
+              : block->bbox[d] + ghost_width * DH[d];
+    uub = (feq(block->bbox[dim + d], patch->bbox[dim + d], DH[d] / 2))
+              ? block->bbox[dim + d] - patch->uui[d] * DH[d]
+              : block->bbox[dim + d] - ghost_width * DH[d];
+#else
+#error Not define Vertex nor Cell
+#endif
+#endif
+    if (point[d] - llb < -DH[d] / 2 || point[d] - uub > DH[d] / 2)
+      return false;
+  }
+  return true;
+}
+
+int z4c_cuda_interp_tile_start(const double *coords, int n, double x, double dx, int ordn)
+{
+  if (!coords || n <= ordn)
+    return 0;
+  int cxi = int((x - coords[0]) / dx + 0.4) + 1;
+  int start = cxi - ordn / 2;
+  if (start < 0)
+    start = 0;
+  const int max_start = n - ordn;
+  if (start > max_start)
+    start = max_start;
+  return start;
+}
+
+bool z4c_cuda_interp_bh_point_resident(MyList<Patch> *PatL,
+                                       int myrank,
+                                       const double *point,
+                                       var *forx, var *fory, var *forz,
+                                       int Symmetry,
+                                       double *shellf)
+{
+  const int ordn = 2 * ghost_width;
+  int owner_rank = -1;
+
+  shellf[0] = shellf[1] = shellf[2] = 0.0;
+
+  MyList<Patch> *PL = PatL;
+  while (PL)
+  {
+    Patch *patch = PL->data;
+    if (!z4c_cuda_patch_contains_point(patch, point))
+    {
+      PL = PL->next;
+      continue;
+    }
+
+    double DH[dim];
+    for (int d = 0; d < dim; d++)
+      DH[d] = patch->getdX(d);
+
+    MyList<Block> *BP = patch->blb;
+    while (BP)
+    {
+      Block *block = BP->data;
+      if (z4c_cuda_point_in_block(patch, block, point, DH))
+      {
+        owner_rank = block->rank;
+        if (myrank == owner_rank)
+        {
+          int interp_ordn = ordn;
+          int interp_sym = Symmetry;
+          double x = point[0];
+          double y = point[1];
+          double z = point[2];
+
+          if (z4c_cuda_has_resident_state(block) &&
+              block->shape[0] >= ordn && block->shape[1] >= ordn && block->shape[2] >= ordn)
+          {
+            const int sx = ordn;
+            const int sy = ordn;
+            const int sz = ordn;
+            const int region_all = sx * sy * sz;
+            const int i0 = z4c_cuda_interp_tile_start(block->X[0], block->shape[0], x, DH[0], ordn);
+            const int j0 = z4c_cuda_interp_tile_start(block->X[1], block->shape[1], y, DH[1], ordn);
+            const int k0 = z4c_cuda_interp_tile_start(block->X[2], block->shape[2], z, DH[2], ordn);
+            double *packed_fields = new double[3 * region_all];
+            var *vars[3] = {forx, fory, forz};
+            for (int f = 0; f < 3; f++)
+            {
+              if (z4c_cuda_pack_state_region_to_host_buffer(block,
+                                                            k_z4c_cuda_bh_state_indices[f],
+                                                            packed_fields + f * region_all,
+                                                            block->shape,
+                                                            i0, j0, k0,
+                                                            sx, sy, sz) != 0)
+              {
+                delete[] packed_fields;
+                cout << "CUDA Z4C BH tile download failed" << endl;
+                MPI_Abort(MPI_COMM_WORLD, 1);
+              }
+              int tile_shape[3] = {sx, sy, sz};
+              f_global_interp(tile_shape,
+                              block->X[0] + i0,
+                              block->X[1] + j0,
+                              block->X[2] + k0,
+                              packed_fields + f * region_all,
+                              shellf[f],
+                              x, y, z,
+                              interp_ordn,
+                              vars[f]->SoA,
+                              interp_sym);
+            }
+            delete[] packed_fields;
+          }
+          else
+          {
+            f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                            block->fgfs[forx->sgfn], shellf[0],
+                            x, y, z, interp_ordn, forx->SoA, interp_sym);
+            f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                            block->fgfs[fory->sgfn], shellf[1],
+                            x, y, z, interp_ordn, fory->SoA, interp_sym);
+            f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                            block->fgfs[forz->sgfn], shellf[2],
+                            x, y, z, interp_ordn, forz->SoA, interp_sym);
+          }
+        }
+        break;
+      }
+      if (BP == patch->ble)
+        break;
+      BP = BP->next;
+    }
+
+    if (owner_rank >= 0)
+      break;
+    PL = PL->next;
+  }
+
+  if (owner_rank < 0)
+    return false;
+
+  MPI_Bcast(shellf, 3, MPI_DOUBLE, owner_rank, MPI_COMM_WORLD);
+  return true;
+}
+
+bool z4c_cuda_compute_porg_rhs_resident(cgh *GH,
+                                        int ilev,
+                                        int myrank,
+                                        int BH_num,
+                                        double **BH_PS,
+                                        double **BH_RHS,
+                                        var *forx, var *fory, var *forz,
+                                        int Symmetry)
+{
+  for (int n = 0; n < BH_num; n++)
+  {
+    double shellf[3] = {0.0, 0.0, 0.0};
+    int lev = ilev;
+    while (lev >= 0 &&
+           !z4c_cuda_interp_bh_point_resident(GH->PatL[lev], myrank, BH_PS[n],
+                                              forx, fory, forz, Symmetry, shellf))
+    {
+      --lev;
+    }
+    if (lev < 0)
+      return false;
+    BH_RHS[n][0] = -shellf[0];
+    BH_RHS[n][1] = -shellf[1];
+    BH_RHS[n][2] = -shellf[2];
+  }
+  return true;
+}
+
+bool z4c_cuda_resident_step_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_Z4C_CUDA_RESIDENT");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+} // namespace
+#endif
+
 void Z4c_class::Step(int lev, int YN)
 {
+#if USE_CUDA_Z4C && (ABEtype == 2)
+  double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
+#ifdef With_AHF
+  AH_Step_Find(lev, dT_lev);
+#endif
+  bool BB = fgt(PhysTime, StartTime, dT_lev / 2);
+  double ndeps = numepss;
+  if (lev < GH->movls)
+    ndeps = numepsb;
+  double TRK4 = PhysTime;
+  int iter_count = 0;
+  int pre = 0, cor = 1;
+  int ERROR = 0;
+
+#ifdef WithShell
+  if (bssn_cuda_use_resident_sync(lev))
+  {
+    for (int dl = 0; dl < GH->levels; dl++)
+      bssn_cuda_download_level_state_if_present(GH->PatL[dl], StateList, myrank);
+  }
+#endif
+
+  MyList<Patch> *Pp = GH->PatL[lev];
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank)
+      {
+        double *state_in[Z4C_CUDA_STATE_COUNT];
+        double *state_out[Z4C_CUDA_STATE_COUNT];
+        double propspeed[Z4C_CUDA_STATE_COUNT];
+        double soa_flat[3 * Z4C_CUDA_STATE_COUNT];
+        if (!fill_z4c_cuda_views(cg, StateList, state_in, propspeed, soa_flat) ||
+            !fill_z4c_cuda_views(cg, SynchList_pre, state_out))
+        {
+          cout << "CUDA Z4C state list mismatch on predictor step" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        int apply_bam_bc = 0;
+#if (MRBD == 0)
+#if (SommerType == 0)
+        apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#elif (MRBD == 1)
+        apply_bam_bc = 1;
+#endif
+        int keep_resident_state = z4c_cuda_resident_step_enabled() ? 1 : 0;
+        int apply_enforce_ga = 0;
+#if (AGM == 0)
+        apply_enforce_ga = 1;
+#endif
+        if (z4c_cuda_rk4_substep(cg,
+                                 cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                 state_in, state_out,
+                                 propspeed, soa_flat, Pp->data->bbox,
+                                 dT_lev, TRK4, iter_count, apply_bam_bc,
+                                 Symmetry, lev, ndeps, pre,
+                                 keep_resident_state, apply_enforce_ga, chitiny))
+        {
+          cout << "CUDA Z4C predictor substep failed in domain: ("
+               << cg->bbox[0] << ":" << cg->bbox[3] << ","
+               << cg->bbox[1] << ":" << cg->bbox[4] << ","
+               << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+          ERROR = 1;
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+
+  {
+    int erh = ERROR;
+    MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+  }
+  if (ERROR)
+  {
+    if (myrank == 0 && ErrorMonitor->outfile)
+      ErrorMonitor->outfile << "CUDA Z4C failed in predictor at t = " << PhysTime
+                            << ", lev = " << lev << endl;
+    MPI_Abort(MPI_COMM_WORLD, 1);
+  }
+
+  Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
+
+  if (BH_num > 0 && lev == GH->levels - 1)
+  {
+    compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
+    for (int ithBH = 0; ithBH < BH_num; ithBH++)
+    {
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
+      if (Symmetry > 0)
+        Porg[ithBH][2] = fabs(Porg[ithBH][2]);
+      if (Symmetry == 2)
+      {
+        Porg[ithBH][0] = fabs(Porg[ithBH][0]);
+        Porg[ithBH][1] = fabs(Porg[ithBH][1]);
+      }
+    }
+  }
+
+  if ((lev == a_lev) && (LastAnas + dT_lev >= AnasTime))
+    z4c_cuda_download_level_state(GH->PatL[lev], SynchList_pre, myrank, false);
+  if (lev == a_lev)
+    AnalysisStuff(lev, dT_lev);
+
+  for (iter_count = 1; iter_count < 4; iter_count++)
+  {
+    if (iter_count == 1 || iter_count == 3)
+      TRK4 += dT_lev / 2;
+    Pp = GH->PatL[lev];
+    while (Pp)
+    {
+      MyList<Block> *BP = Pp->data->blb;
+      while (BP)
+      {
+        Block *cg = BP->data;
+        if (myrank == cg->rank)
+        {
+          double *state_in[Z4C_CUDA_STATE_COUNT];
+          double *state_out[Z4C_CUDA_STATE_COUNT];
+          double propspeed[Z4C_CUDA_STATE_COUNT];
+          double soa_flat[3 * Z4C_CUDA_STATE_COUNT];
+          if (!fill_z4c_cuda_views(cg, SynchList_pre, state_in, propspeed, soa_flat) ||
+              !fill_z4c_cuda_views(cg, SynchList_cor, state_out))
+          {
+            cout << "CUDA Z4C state list mismatch on corrector step" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+          int apply_bam_bc = 0;
+#if (MRBD == 0)
+#if (SommerType == 0)
+          apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#elif (MRBD == 1)
+          apply_bam_bc = 1;
+#endif
+          int keep_resident_state = z4c_cuda_resident_step_enabled() ? 1 : 0;
+          int apply_enforce_ga = 0;
+#if (AGM == 0)
+          apply_enforce_ga = 1;
+#elif (AGM == 1)
+          apply_enforce_ga = (iter_count == 3) ? 1 : 0;
+#endif
+          if (z4c_cuda_rk4_substep(cg,
+                                   cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                   state_in, state_out,
+                                   propspeed, soa_flat, Pp->data->bbox,
+                                   dT_lev, TRK4, iter_count, apply_bam_bc,
+                                   Symmetry, lev, ndeps, cor,
+                                   keep_resident_state, apply_enforce_ga, chitiny))
+          {
+            cout << "CUDA Z4C corrector substep failed in domain: ("
+                 << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                 << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                 << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+            ERROR = 1;
+          }
+        }
+        if (BP == Pp->data->ble)
+          break;
+        BP = BP->next;
+      }
+      Pp = Pp->next;
+    }
+
+    {
+      int erh = ERROR;
+      MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
+    }
+    if (ERROR)
+    {
+      if (myrank == 0 && ErrorMonitor->outfile)
+        ErrorMonitor->outfile << "CUDA Z4C failed in RK4 substep#" << iter_count
+                              << " at t = " << PhysTime
+                              << ", lev = " << lev << endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+
+    Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
+
+    if (BH_num > 0 && lev == GH->levels - 1)
+    {
+      if (z4c_cuda_resident_step_enabled())
+      {
+        if (!z4c_cuda_compute_porg_rhs_resident(GH, lev, myrank, BH_num,
+                                                Porg, Porg1,
+                                                Sfx, Sfy, Sfz, Symmetry))
+        {
+          if (myrank == 0 && ErrorMonitor->outfile)
+            ErrorMonitor->outfile << "CUDA Z4C failed to interpolate black-hole shift at t = "
+                                  << PhysTime << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+      }
+      else
+      {
+        compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
+      }
+      for (int ithBH = 0; ithBH < BH_num; ithBH++)
+      {
+        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg1[ithBH][0], Porg_rhs[ithBH][0], iter_count);
+        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg1[ithBH][1], Porg_rhs[ithBH][1], iter_count);
+        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg1[ithBH][2], Porg_rhs[ithBH][2], iter_count);
+        if (Symmetry > 0)
+          Porg1[ithBH][2] = fabs(Porg1[ithBH][2]);
+        if (Symmetry == 2)
+        {
+          Porg1[ithBH][0] = fabs(Porg1[ithBH][0]);
+          Porg1[ithBH][1] = fabs(Porg1[ithBH][1]);
+        }
+      }
+    }
+
+    if (iter_count < 3)
+    {
+      Pp = GH->PatL[lev];
+      while (Pp)
+      {
+        MyList<Block> *BP = Pp->data->blb;
+        while (BP)
+        {
+          Block *cg = BP->data;
+          cg->swapList(SynchList_pre, SynchList_cor, myrank);
+          if (BP == Pp->data->ble)
+            break;
+          BP = BP->next;
+        }
+        Pp = Pp->next;
+      }
+      if (BH_num > 0 && lev == GH->levels - 1)
+      {
+        for (int ithBH = 0; ithBH < BH_num; ithBH++)
+        {
+          Porg[ithBH][0] = Porg1[ithBH][0];
+          Porg[ithBH][1] = Porg1[ithBH][1];
+          Porg[ithBH][2] = Porg1[ithBH][2];
+        }
+      }
+    }
+  }
+
+  z4c_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, false);
+
+#if (RPS == 0)
+  RestrictProlong(lev, YN, BB);
+#endif
+
+  Pp = GH->PatL[lev];
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      cg->swapList(StateList, SynchList_cor, myrank);
+      cg->swapList(OldStateList, SynchList_cor, myrank);
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  if (BH_num > 0 && lev == GH->levels - 1)
+  {
+    for (int ithBH = 0; ithBH < BH_num; ithBH++)
+    {
+      Porg0[ithBH][0] = Porg1[ithBH][0];
+      Porg0[ithBH][1] = Porg1[ithBH][1];
+      Porg0[ithBH][2] = Porg1[ithBH][2];
+    }
+  }
+#else
   double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
 #ifdef With_AHF
   AH_Step_Find(lev, dT_lev);
@@ -339,6 +1001,13 @@ void Z4c_class::Step(int lev, int YN)
   }
 
 #ifdef WithShell
+#if USE_CUDA_Z4C
+  if (bssn_cuda_use_resident_sync(lev))
+  {
+    for (int dl = 0; dl < GH->levels; dl++)
+      bssn_cuda_download_level_state_if_present(GH->PatL[dl], StateList, myrank);
+  }
+#endif
   // evolve Shell Patches
   if (lev == 0)
   {
@@ -1042,9 +1711,11 @@ void Z4c_class::Step(int lev, int YN)
       Porg0[ithBH][2] = Porg1[ithBH][2];
     }
   }
+#endif
 }
 #else
 // for constraint preserving boundary (CPBC)
+// Note: CPBC path uses CPU Fortran RHS; GPU resident sync is a no-op here.
 #ifndef WithShell
 #error "CPBC only supports Shell"
 #endif
@@ -1074,6 +1745,14 @@ void Z4c_class::Step(int lev, int YN)
   int pre = 0, cor = 1;
   int ERROR = 0;
 
+#if USE_CUDA_Z4C && defined(WithShell)
+  if (bssn_cuda_use_resident_sync(lev))
+  {
+    for (int dl = 0; dl < GH->levels; dl++)
+      bssn_cuda_download_level_state_if_present(GH->PatL[dl], StateList, myrank);
+  }
+#endif
+
   MyList<ss_patch> *sPp;
   // Predictor
   MyList<Patch> *Pp = GH->PatL[lev];
@@ -2404,6 +3083,11 @@ void Z4c_class::Check_extrop()
 
 //================================================================================================
 
+#if USE_CUDA_Z4C && (ABEtype == 2) && defined(WithShell)
+#undef f_compute_rhs_Z4c_ss
+#define f_compute_rhs_Z4c_ss compute_rhs_z4c_ss_
+#endif
+
 // this member function is used to compute and output constraint violation
 
 //================================================================================================
@@ -2679,11 +3363,12 @@ void Z4c_class::Interp_Constraint()
   }
 
   ofstream outfile;
-  char filename[50];
-  sprintf(filename, "%s/interp_constraint_%05d.dat", ErrorMonitor->out_dir.c_str(), int(PhysTime / dT + 0.5)); 
+  char suffix[64];
+  sprintf(suffix, "/interp_constraint_%05d.dat", int(PhysTime / dT + 0.5));
+  string filename = ErrorMonitor->out_dir + suffix;
   // 0.5 for round off
   
-  outfile.open(filename);
+  outfile.open(filename.c_str());
   outfile << "#  corrdinate, H_Res, Px_Res, Py_Res, Pz_Res, Gx_Res, Gy_Res, Gz_Res, ...." << endl;
   for (int i = 0; i < n; i++)
   {

AMSS_NCKU_source/Z4c_rhs.f90

@@ -94,29 +94,31 @@
                Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon,                    &            
                Symmetry,Lev,eps,co)
 
+  if (co == 0) then
 #if (ABV == 0)  
-  call ricci_gamma(ex, X, Y, Z,                                      &
-               chi,                                                  &
-               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
-               Gamx   ,  Gamy    ,  Gamz    , &
-               Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
-               Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
-               Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
-               Symmetry)
+    call ricci_gamma(ex, X, Y, Z,                                      &
+                 chi,                                                  &
+                 dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
+                 Gamx   ,  Gamy    ,  Gamz    , &
+                 Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
+                 Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
+                 Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
+                 Symmetry)
 #endif
-  call constraint_bssn(ex, X, Y, Z,&
-               chi,trK, &
-               dxx,gxy,gxz,dyy,gyz,dzz, &
-               Axx,Axy,Axz,Ayy,Ayz,Azz, &
-               Gamx,Gamy,Gamz,&
-               Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
-               Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
-               Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
-               Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
-               Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
-               Symmetry)
+    call constraint_bssn(ex, X, Y, Z,&
+                 chi,trK, &
+                 dxx,gxy,gxz,dyy,gyz,dzz, &
+                 Axx,Axy,Axz,Ayy,Ayz,Azz, &
+                 Gamx,Gamy,Gamz,&
+                 Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
+                 Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
+                 Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
+                 Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
+                 Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
+                 Symmetry)
+  endif
 
   return
 
@@ -227,6 +229,7 @@
   call get_Z4cparameters(kappa1,kappa2,kappa3,FF,eta)
 
 !!! sanity check
+#ifdef DEBUG
   dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
       +sum(Axx)+sum(Axy)+sum(Axz)+sum(Ayy)+sum(Ayz)+sum(Azz)                   &
       +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
@@ -261,6 +264,7 @@
      gont = 1
      return
   endif
+#endif
 
   PI = dacos(-ONE)
 
@@ -1263,30 +1267,32 @@
 
   endif
 
+  if (co == 0) then
 #if (ABV == 0)  
-  call ricci_gamma(ex, X, Y, Z,                                      &
-               chi,                                                  &
-               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
-               Gamx   ,  Gamy    ,  Gamz    , &
-               Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
-               Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
-               Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
-               Symmetry)
+    call ricci_gamma(ex, X, Y, Z,                                      &
+                 chi,                                                  &
+                 dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
+                 Gamx   ,  Gamy    ,  Gamz    , &
+                 Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
+                 Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
+                 Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
+                 Symmetry)
 #endif
 
-  call constraint_bssn(ex, X, Y, Z,&
-               chi,trK, &
-               dxx,gxy,gxz,dyy,gyz,dzz, &
-               Axx,Axy,Axz,Ayy,Ayz,Azz, &
-               Gamx,Gamy,Gamz,&
-               Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
-               Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
-               Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
-               Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
-               Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
-               Symmetry)
+    call constraint_bssn(ex, X, Y, Z,&
+                 chi,trK, &
+                 dxx,gxy,gxz,dyy,gyz,dzz, &
+                 Axx,Axy,Axz,Ayy,Ayz,Azz, &
+                 Gamx,Gamy,Gamz,&
+                 Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
+                 Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
+                 Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
+                 Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
+                 Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
+                 Symmetry)
+  endif
 
   gont = 0
 

AMSS_NCKU_source/Z4c_rhs_ss.f90

@@ -122,6 +122,7 @@
   call get_Z4cparameters(kappa1,kappa2,kappa3,FF,eta)
 
 !!! sanity check
+#ifdef DEBUG
   dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
       +sum(Axx)+sum(Axy)+sum(Axz)+sum(Ayy)+sum(Ayz)+sum(Azz)                   &
       +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
@@ -156,6 +157,7 @@
      gont = 1
      return
   endif
+#endif
 
   PI = dacos(-ONE)
 
@@ -1388,41 +1390,43 @@
   call kodis_sh(ex,crho,sigma,R,TZ,TZ_rhs,SSS,Symmetry,eps,sst)
   endif
 
+  if (co == 0) then
 #if (ABV == 1)  
-  call ricci_gamma_ss(ex,crho,sigma,R,X, Y, Z,                                 &
-               drhodx, drhody, drhodz,                                         &
-               dsigmadx,dsigmady,dsigmadz,                                     &
-               dRdx,dRdy,dRdz,                                                 &
-               drhodxx,drhodxy,drhodxz,drhodyy,drhodyz,drhodzz,                &
-               dsigmadxx,dsigmadxy,dsigmadxz,dsigmadyy,dsigmadyz,dsigmadzz,    &
-               dRdxx,dRdxy,dRdxz,dRdyy,dRdyz,dRdzz,                            &
-               chi,                                                  &
-               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
-               Gamx   ,  Gamy    ,  Gamz    , &
-               Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
-               Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
-               Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
-               Symmetry,Lev,sst)
-  call constraint_bssn_ss(ex,crho,sigma,R,X, Y, Z,  &
-               drhodx, drhody, drhodz,                                         &
-               dsigmadx,dsigmady,dsigmadz,                                     &
-               dRdx,dRdy,dRdz,                                                 &
-               drhodxx,drhodxy,drhodxz,drhodyy,drhodyz,drhodzz,                &
-               dsigmadxx,dsigmadxy,dsigmadxz,dsigmadyy,dsigmadyz,dsigmadzz,    &
-               dRdxx,dRdxy,dRdxz,dRdyy,dRdyz,dRdzz,                            &
-               chi,trK, &
-               dxx,gxy,gxz,dyy,gyz,dzz, &
-               Axx,Axy,Axz,Ayy,Ayz,Azz, &
-               Gamx,Gamy,Gamz,&
-               Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
-               Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
-               Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
-               Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
-               Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
-               Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
-               Symmetry,Lev,sst)
+    call ricci_gamma_ss(ex,crho,sigma,R,X, Y, Z,                               &
+                 drhodx, drhody, drhodz,                                       &
+                 dsigmadx,dsigmady,dsigmadz,                                   &
+                 dRdx,dRdy,dRdz,                                               &
+                 drhodxx,drhodxy,drhodxz,drhodyy,drhodyz,drhodzz,              &
+                 dsigmadxx,dsigmadxy,dsigmadxz,dsigmadyy,dsigmadyz,dsigmadzz,  &
+                 dRdxx,dRdxy,dRdxz,dRdyy,dRdyz,dRdzz,                          &
+                 chi,                                                          &
+                 dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,&
+                 Gamx   ,  Gamy    ,  Gamz    , &
+                 Gamxxx,Gamxxy,Gamxxz,Gamxyy,Gamxyz,Gamxzz,&
+                 Gamyxx,Gamyxy,Gamyxz,Gamyyy,Gamyyz,Gamyzz,&
+                 Gamzxx,Gamzxy,Gamzxz,Gamzyy,Gamzyz,Gamzzz,&
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz,&
+                 Symmetry,Lev,sst)
 #endif
+    call constraint_bssn_ss(ex,crho,sigma,R,X, Y, Z,  &
+                 drhodx, drhody, drhodz,                                       &
+                 dsigmadx,dsigmady,dsigmadz,                                   &
+                 dRdx,dRdy,dRdz,                                               &
+                 drhodxx,drhodxy,drhodxz,drhodyy,drhodyz,drhodzz,              &
+                 dsigmadxx,dsigmadxy,dsigmadxz,dsigmadyy,dsigmadyz,dsigmadzz,  &
+                 dRdxx,dRdxy,dRdxz,dRdyy,dRdyz,dRdzz,                          &
+                 chi,trK, &
+                 dxx,gxy,gxz,dyy,gyz,dzz, &
+                 Axx,Axy,Axz,Ayy,Ayz,Azz, &
+                 Gamx,Gamy,Gamz,&
+                 Lap,betax,betay,betaz,rho,Sx,Sy,Sz,&
+                 Gamxxx, Gamxxy, Gamxxz,Gamxyy, Gamxyz, Gamxzz, &
+                 Gamyxx, Gamyxy, Gamyxz,Gamyyy, Gamyyz, Gamyzz, &
+                 Gamzxx, Gamzxy, Gamzxz,Gamzyy, Gamzyz, Gamzzz, &
+                 Rxx,Rxy,Rxz,Ryy,Ryz,Rzz, &
+                 Hcon,Mxcon,Mycon,Mzcon,Gmxcon,Gmycon,Gmzcon, &
+                 Symmetry,Lev,sst)
+  endif
 
   gont = 0
 

AMSS_NCKU_source/bssnEM_class.C

@@ -2,7 +2,9 @@
 #ifdef newc
 #include <sstream>
 #include <cstdio>
+#include <cstdlib>
 #include <map>
+#include <string>
 using namespace std;
 #else
 #include <stdio.h>
@@ -10,6 +12,7 @@ using namespace std;
 #endif
 
 #include <time.h>
+#include <cstring>
 
 #include "macrodef.h"
 #include "misc.h"
@@ -19,6 +22,9 @@ using namespace std;
 #include "bssnEM_class.h"
 #include "bssn_rhs.h"
 #include "empart.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
 #include "initial_puncture.h"
 #include "initial_maxwell.h"
 #include "enforce_algebra.h"
@@ -36,6 +42,387 @@ using namespace std;
 
 //================================================================================================
 
+namespace
+{
+MyList<var> *advance_var_list(MyList<var> *vars, int count)
+{
+  while (vars && count > 0)
+  {
+    vars = vars->next;
+    --count;
+  }
+  return vars;
+}
+
+bool bssn_em_step_timing_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_EM_STEP_TIMING");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool bssn_em_step_timing_all_levels_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_EM_STEP_TIMING_ALL_LEVELS");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+#if USE_CUDA_BSSN
+bool bssn_em_zero_analysis_fastpath_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_EM_ZERO_ANALYSIS_FASTPATH");
+    enabled = (!env || atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool bssn_em_zero_resident_download_fastpath_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_EM_ZERO_RESIDENT_DOWNLOAD_FASTPATH");
+    enabled = (!env || atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool bssn_em_resident_zero_fastpath_ready(MyList<Patch> *PatL,
+#ifdef WithShell
+                                          ShellPatch *shell,
+#else
+                                          ShellPatch * /*shell*/,
+#endif
+                                          int rank)
+{
+  int local_ok = 1;
+  int local_seen = 0;
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (rank == cg->rank)
+      {
+        local_seen = 1;
+        if (!bssn_em_cuda_resident_zero_fast_state(cg))
+          local_ok = 0;
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+#ifdef WithShell
+  if (shell && shell->PatL)
+  {
+    MyList<ss_patch> *SP = shell->PatL;
+    while (SP)
+    {
+      MyList<Block> *BP = SP->data->blb;
+      while (BP)
+      {
+        Block *cg = BP->data;
+        if (rank == cg->rank)
+        {
+          local_seen = 1;
+          if (!bssn_em_cuda_resident_zero_fast_state(cg))
+            local_ok = 0;
+        }
+        if (BP == SP->data->ble)
+          break;
+        BP = BP->next;
+      }
+      SP = SP->next;
+    }
+  }
+#endif
+  int global_ok = 0;
+  int global_seen = 0;
+  MPI_Allreduce(&local_ok, &global_ok, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD);
+  MPI_Allreduce(&local_seen, &global_seen, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD);
+  return global_seen && global_ok;
+}
+
+bool bssn_em_analysis_zero_fastpath_ready(MyList<Patch> *PatL,
+#ifdef WithShell
+                                          ShellPatch *shell,
+#else
+                                          ShellPatch *shell,
+#endif
+                                          int rank)
+{
+  if (!bssn_em_zero_analysis_fastpath_enabled())
+    return false;
+  return bssn_em_resident_zero_fastpath_ready(PatL, shell, rank);
+}
+
+void zero_em_analysis_outputs(MyList<Patch> *PatL,
+#ifdef WithShell
+                              ShellPatch *shell,
+#else
+                              ShellPatch * /*shell*/,
+#endif
+                              int rank,
+                              var *Rphi2_var, var *Iphi2_var,
+                              var *Rphi1_var, var *Iphi1_var)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (rank == cg->rank)
+      {
+        const size_t all = (size_t)cg->shape[0] * cg->shape[1] * cg->shape[2];
+        std::memset(cg->fgfs[Rphi2_var->sgfn], 0, all * sizeof(double));
+        std::memset(cg->fgfs[Iphi2_var->sgfn], 0, all * sizeof(double));
+        std::memset(cg->fgfs[Rphi1_var->sgfn], 0, all * sizeof(double));
+        std::memset(cg->fgfs[Iphi1_var->sgfn], 0, all * sizeof(double));
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+#ifdef WithShell
+  if (shell && shell->PatL)
+  {
+    MyList<ss_patch> *SP = shell->PatL;
+    while (SP)
+    {
+      MyList<Block> *BP = SP->data->blb;
+      while (BP)
+      {
+        Block *cg = BP->data;
+        if (rank == cg->rank)
+        {
+          const size_t all = (size_t)cg->shape[0] * cg->shape[1] * cg->shape[2];
+          std::memset(cg->fgfs[Rphi2_var->sgfn], 0, all * sizeof(double));
+          std::memset(cg->fgfs[Iphi2_var->sgfn], 0, all * sizeof(double));
+          std::memset(cg->fgfs[Rphi1_var->sgfn], 0, all * sizeof(double));
+          std::memset(cg->fgfs[Iphi1_var->sgfn], 0, all * sizeof(double));
+        }
+        if (BP == SP->data->ble)
+          break;
+        BP = BP->next;
+      }
+      SP = SP->next;
+    }
+  }
+#endif
+}
+#endif
+
+int bssn_em_step_timing_every()
+{
+  static int every = -1;
+  if (every < 0)
+  {
+    const char *env = getenv("AMSS_EM_STEP_TIMING_EVERY");
+    every = (env && atoi(env) > 0) ? atoi(env) : 1;
+  }
+  return every;
+}
+
+#if USE_CUDA_BSSN
+bool fill_bssn_em_bssn_cuda_views(Block *cg, MyList<var> *vars,
+                                  double **host_views,
+                                  double *propspeeds = 0,
+                                  double *soa_flat = 0)
+{
+  int idx = 0;
+  while (vars && idx < BSSN_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == BSSN_CUDA_STATE_COUNT;
+}
+
+bool fill_bssn_em_cuda_views(Block *cg, MyList<var> *vars,
+                             double **host_views,
+                             double *propspeeds = 0,
+                             double *soa_flat = 0)
+{
+  int idx = 0;
+  while (vars && idx < BSSN_EM_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == BSSN_EM_CUDA_STATE_COUNT && vars == 0;
+}
+
+void fill_bssn_em_fixed_source_cuda_views(Block *cg, double **sources,
+                                          var *Jx, var *Jy, var *Jz, var *qchar)
+{
+  sources[0] = cg->fgfs[Jx->sgfn];
+  sources[1] = cg->fgfs[Jy->sgfn];
+  sources[2] = cg->fgfs[Jz->sgfn];
+  sources[3] = cg->fgfs[qchar->sgfn];
+}
+
+void fill_bssn_em_matter_cuda_views(Block *cg, double **matter,
+                                    var *rho, var *Sx, var *Sy, var *Sz,
+                                    var *Sxx, var *Sxy, var *Sxz,
+                                    var *Syy, var *Syz, var *Szz)
+{
+  matter[0] = cg->fgfs[rho->sgfn];
+  matter[1] = cg->fgfs[Sx->sgfn];
+  matter[2] = cg->fgfs[Sy->sgfn];
+  matter[3] = cg->fgfs[Sz->sgfn];
+  matter[4] = cg->fgfs[Sxx->sgfn];
+  matter[5] = cg->fgfs[Sxy->sgfn];
+  matter[6] = cg->fgfs[Sxz->sgfn];
+  matter[7] = cg->fgfs[Syy->sgfn];
+  matter[8] = cg->fgfs[Syz->sgfn];
+  matter[9] = cg->fgfs[Szz->sgfn];
+}
+
+bool bssn_em_cuda_use_resident_sync(int lev)
+{
+#ifdef WithShell
+  (void)lev;
+  return false;
+#else
+  return true;
+#endif
+}
+
+bool bssn_em_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
+{
+  static int keep_all_levels = -1;
+  if (keep_all_levels < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_EM_KEEP_ALL_LEVELS");
+    keep_all_levels = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_EM_KEEP_RESIDENT_AFTER_STEP");
+    if (!env)
+      env = getenv("AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  if (!enabled)
+    return false;
+  if (lev == analysis_lev)
+    return false;
+  if (keep_all_levels)
+    return true;
+  return lev < trfls_in;
+}
+
+void bssn_em_cuda_download_level_state(MyList<Patch> *PatL, MyList<var> *vars,
+                                       int myrank, bool release_ctx)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && bssn_cuda_has_resident_state(cg))
+      {
+        double *state_out[BSSN_EM_CUDA_STATE_COUNT];
+        if (!fill_bssn_em_cuda_views(cg, vars, state_out))
+        {
+          cout << "CUDA BSSN-EM resident state list mismatch during download" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (bssn_cuda_download_resident_state_count_if_present(cg, cg->shape,
+                                                                state_out,
+                                                                BSSN_EM_CUDA_STATE_COUNT))
+        {
+          cout << "CUDA BSSN-EM resident state download failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (release_ctx)
+          bssn_cuda_release_step_ctx(cg);
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+
+void bssn_em_cuda_keep_only_level_state(MyList<Patch> *PatL, MyList<var> *vars,
+                                        int myrank)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && bssn_cuda_has_resident_state(cg))
+      {
+        double *state_key[BSSN_EM_CUDA_STATE_COUNT];
+        if (!fill_bssn_em_cuda_views(cg, vars, state_key))
+        {
+          cout << "CUDA BSSN-EM resident state list mismatch during prune" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (bssn_cuda_keep_only_resident_state_count(cg, cg->shape,
+                                                     state_key,
+                                                     BSSN_EM_CUDA_STATE_COUNT))
+        {
+          cout << "CUDA BSSN-EM keep-only resident state failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+#endif
+}
+
 // Define bssnEM_class
 
 // It inherits some members and methods from the parent class bssn_class and modifies others.
@@ -258,6 +645,13 @@ void bssnEM_class::Initialize()
     PhysTime = StartTime;
     Setup_Black_Hole_position();
   }
+
+  sync_cache_pre = new Parallel::SyncCache[GH->levels];
+  sync_cache_cor = new Parallel::SyncCache[GH->levels];
+  sync_cache_rp_coarse = new Parallel::SyncCache[GH->levels];
+  sync_cache_rp_fine = new Parallel::SyncCache[GH->levels];
+  sync_cache_restrict = new Parallel::SyncCache[GH->levels];
+  sync_cache_outbd = new Parallel::SyncCache[GH->levels];
 }
 
 //================================================================================================
@@ -833,9 +1227,25 @@ void bssnEM_class::Step(int lev, int YN)
   int iter_count = 0; // count RK4 substeps
   int pre = 0, cor = 1;
   int ERROR = 0;
+#if USE_CUDA_BSSN
+  const bool use_cuda_resident_sync = bssn_em_cuda_use_resident_sync(lev);
+#endif
+  const bool em_step_timing = bssn_em_step_timing_enabled();
+  const double em_step_t0 = em_step_timing ? MPI_Wtime() : 0.0;
+  double em_t0 = 0.0;
+  double em_t_predictor = 0.0;
+  double em_t_predictor_sync = 0.0;
+  double em_t_corrector = 0.0;
+  double em_t_corrector_sync = 0.0;
+  double em_t_analysis = 0.0;
+  double em_t_bh = 0.0;
+  double em_t_swap = 0.0;
+  double em_t_resident = 0.0;
+  double em_t_rp = 0.0;
 
   MyList<ss_patch> *sPp;
   // Predictor
+  em_t0 = em_step_timing ? MPI_Wtime() : 0.0;
   MyList<Patch> *Pp = GH->PatL[lev];
   while (Pp)
   {
@@ -845,15 +1255,20 @@ void bssnEM_class::Step(int lev, int YN)
       Block *cg = BP->data;
       if (myrank == cg->rank)
       {
+#if !USE_CUDA_BSSN
 #if (AGM == 0)
         f_enforce_ga(cg->shape,
                      cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                      cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                      cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
                      cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
+#endif
 #endif
 
-        if (
+        int em_rhs_error = 0;
+        bool used_gpu_substep = false;
+#if !USE_CUDA_BSSN
+        em_rhs_error =
             f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                  cg->fgfs[phi0->sgfn],
                                  cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
@@ -873,8 +1288,52 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
                                  cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn], 
                                  cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
-                                 Symmetry, lev, ndeps) ||
-            f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+                                 Symmetry, lev, ndeps);
+#endif
+
+#if USE_CUDA_BSSN
+        if (!em_rhs_error)
+        {
+          double *state_in[BSSN_EM_CUDA_STATE_COUNT];
+          double *state_out[BSSN_EM_CUDA_STATE_COUNT];
+          double *sources[BSSN_EM_CUDA_SOURCE_COUNT];
+          double propspeed[BSSN_EM_CUDA_STATE_COUNT];
+          double soa_flat[3 * BSSN_EM_CUDA_STATE_COUNT];
+          if (!fill_bssn_em_cuda_views(cg, StateList, state_in, propspeed, soa_flat) ||
+              !fill_bssn_em_cuda_views(cg, SynchList_pre, state_out))
+          {
+            cout << "CUDA BSSN-EM state list mismatch on predictor step" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+          fill_bssn_em_fixed_source_cuda_views(cg, sources, Jx, Jy, Jz, qchar);
+          int apply_bam_bc = 0;
+#if (SommerType == 0)
+#ifndef WithShell
+          apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+          int apply_enforce_ga = 0;
+#if (AGM == 0)
+          apply_enforce_ga = 1;
+#endif
+          int keep_resident_state = use_cuda_resident_sync ? 1 : 0;
+          if (bssn_em_cuda_rk4_substep(cg,
+                                       cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                       state_in, state_out, sources,
+                                       propspeed, soa_flat, Pp->data->bbox,
+                                       dT_lev, TRK4, iter_count, apply_bam_bc,
+                                       Symmetry, lev, ndeps, pre,
+                                       keep_resident_state, apply_enforce_ga, chitiny))
+          {
+            ERROR = 1;
+          }
+          used_gpu_substep = true;
+        }
+#endif
+
+        if (em_rhs_error ||
+            (!used_gpu_substep &&
+             f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
                                cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
@@ -907,7 +1366,7 @@ void bssnEM_class::Step(int lev, int YN)
                                cg->fgfs[Cons_Ham->sgfn],
                                cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
                                cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                               Symmetry, lev, ndeps, pre))
+                               Symmetry, lev, ndeps, pre)))
         {
           cout << "find NaN in domain: (" 
                << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -916,6 +1375,8 @@ void bssnEM_class::Step(int lev, int YN)
           ERROR = 1;
         }
 
+        if (!used_gpu_substep)
+        {
         // rk4 substep and boundary
         {
           MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; 
@@ -957,6 +1418,7 @@ void bssnEM_class::Step(int lev, int YN)
           }
         }
         f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
+        }
       }
       if (BP == Pp->data->ble)
         break;
@@ -964,6 +1426,8 @@ void bssnEM_class::Step(int lev, int YN)
     }
     Pp = Pp->next;
   }
+  if (em_step_timing)
+    em_t_predictor += MPI_Wtime() - em_t0;
   // check error information
   {
     int erh = ERROR;
@@ -1221,7 +1685,11 @@ void bssnEM_class::Step(int lev, int YN)
   }
 #endif
 
-  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+  if (em_step_timing)
+    em_t0 = MPI_Wtime();
+  Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
+  if (em_step_timing)
+    em_t_predictor_sync += MPI_Wtime() - em_t0;
 
 #ifdef WithShell
   if (lev == 0)
@@ -1244,6 +1712,8 @@ void bssnEM_class::Step(int lev, int YN)
   // for black hole position
   if (BH_num > 0 && lev == GH->levels - 1)
   {
+    if (em_step_timing)
+      em_t0 = MPI_Wtime();
     compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
     for (int ithBH = 0; ithBH < BH_num; ithBH++)
     {
@@ -1272,16 +1742,24 @@ void bssnEM_class::Step(int lev, int YN)
         DG_List->clearList();
       }
     }
+    if (em_step_timing)
+      em_t_bh += MPI_Wtime() - em_t0;
   }
   // data analysis part
   // Warning NOTE: the variables1 are used as temp storege room
   if (lev == a_lev)
   {
+    if (em_step_timing)
+      em_t0 = MPI_Wtime();
     AnalysisStuff_EM(lev, dT_lev);
+    if (em_step_timing)
+      em_t_analysis += MPI_Wtime() - em_t0;
   }
   // corrector
   for (iter_count = 1; iter_count < 4; iter_count++)
   {
+    if (em_step_timing)
+      em_t0 = MPI_Wtime();
     // for RK4: t0, t0+dt/2, t0+dt/2, t0+dt;
     if (iter_count == 1 || iter_count == 3)
       TRK4 += dT_lev / 2;
@@ -1294,6 +1772,7 @@ void bssnEM_class::Step(int lev, int YN)
         Block *cg = BP->data;
         if (myrank == cg->rank)
         {
+#if !USE_CUDA_BSSN
 #if (AGM == 0)
           f_enforce_ga(cg->shape,
                        cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
@@ -1307,9 +1786,13 @@ void bssnEM_class::Step(int lev, int YN)
                          cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                          cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
                          cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
+#endif
 #endif
 
-          if (
+          int em_rhs_error = 0;
+          bool used_gpu_substep = false;
+#if !USE_CUDA_BSSN
+          em_rhs_error =
               f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                    cg->fgfs[phi->sgfn],
                                    cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
@@ -1329,8 +1812,55 @@ void bssnEM_class::Step(int lev, int YN)
                                    cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
                                    cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn], 
                                    cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
-                                   Symmetry, lev, ndeps) ||
-              f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+                                   Symmetry, lev, ndeps);
+#endif
+
+#if USE_CUDA_BSSN
+          if (!em_rhs_error)
+          {
+            double *state_in[BSSN_EM_CUDA_STATE_COUNT];
+            double *state_out[BSSN_EM_CUDA_STATE_COUNT];
+            double *sources[BSSN_EM_CUDA_SOURCE_COUNT];
+            double propspeed[BSSN_EM_CUDA_STATE_COUNT];
+            double soa_flat[3 * BSSN_EM_CUDA_STATE_COUNT];
+            if (!fill_bssn_em_cuda_views(cg, SynchList_pre, state_in, propspeed, soa_flat) ||
+                !fill_bssn_em_cuda_views(cg, SynchList_cor, state_out))
+            {
+              cout << "CUDA BSSN-EM state list mismatch on corrector step" << endl;
+              MPI_Abort(MPI_COMM_WORLD, 1);
+            }
+            fill_bssn_em_fixed_source_cuda_views(cg, sources, Jx, Jy, Jz, qchar);
+            int apply_bam_bc = 0;
+#if (SommerType == 0)
+#ifndef WithShell
+            apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+            int apply_enforce_ga = 0;
+#if (AGM == 0)
+            apply_enforce_ga = 1;
+#elif (AGM == 1)
+            if (iter_count == 3)
+              apply_enforce_ga = 1;
+#endif
+            int keep_resident_state = use_cuda_resident_sync ? 1 : 0;
+            if (bssn_em_cuda_rk4_substep(cg,
+                                         cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                         state_in, state_out, sources,
+                                         propspeed, soa_flat, Pp->data->bbox,
+                                         dT_lev, TRK4, iter_count, apply_bam_bc,
+                                         Symmetry, lev, ndeps, cor,
+                                         keep_resident_state, apply_enforce_ga, chitiny))
+            {
+              ERROR = 1;
+            }
+            used_gpu_substep = true;
+          }
+#endif
+
+          if (em_rhs_error ||
+              (!used_gpu_substep &&
+               f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                  cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
                                  cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                  cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
@@ -1362,7 +1892,7 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Cons_Ham->sgfn],
                                  cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
                                  cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                                 Symmetry, lev, ndeps, cor))
+                                 Symmetry, lev, ndeps, cor)))
           {
             cout << "find NaN in domain: (" 
                  << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -1370,6 +1900,8 @@ void bssnEM_class::Step(int lev, int YN)
                  << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
             ERROR = 1;
           }
+          if (!used_gpu_substep)
+          {
           // rk4 substep and boundary
           {
             MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList; 
@@ -1412,6 +1944,7 @@ void bssnEM_class::Step(int lev, int YN)
             }
           }
           f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
+          }
         }
         if (BP == Pp->data->ble)
           break;
@@ -1683,7 +2216,13 @@ void bssnEM_class::Step(int lev, int YN)
     }
 #endif
 
-    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+    if (em_step_timing)
+      em_t_corrector += MPI_Wtime() - em_t0;
+    if (em_step_timing)
+      em_t0 = MPI_Wtime();
+    Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
+    if (em_step_timing)
+      em_t_corrector_sync += MPI_Wtime() - em_t0;
 
 #ifdef WithShell
     if (lev == 0)
@@ -1705,6 +2244,8 @@ void bssnEM_class::Step(int lev, int YN)
     // for black hole position
     if (BH_num > 0 && lev == GH->levels - 1)
     {
+      if (em_step_timing)
+        em_t0 = MPI_Wtime();
       compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
       for (int ithBH = 0; ithBH < BH_num; ithBH++)
       {
@@ -1733,10 +2274,14 @@ void bssnEM_class::Step(int lev, int YN)
           DG_List->clearList();
         }
       }
+      if (em_step_timing)
+        em_t_bh += MPI_Wtime() - em_t0;
     }
     // swap time level
     if (iter_count < 3)
     {
+      if (em_step_timing)
+        em_t0 = MPI_Wtime();
       Pp = GH->PatL[lev];
       while (Pp)
       {
@@ -1780,12 +2325,32 @@ void bssnEM_class::Step(int lev, int YN)
           Porg[ithBH][2] = Porg1[ithBH][2];
         }
       }
+      if (em_step_timing)
+        em_t_swap += MPI_Wtime() - em_t0;
     }
   }
 
+#if USE_CUDA_BSSN
+  if (use_cuda_resident_sync)
+  {
+    if (em_step_timing)
+      em_t0 = MPI_Wtime();
+    const bool needs_resident_download =
+        !bssn_em_cuda_keep_resident_after_step(lev, trfls, a_lev);
+    if (needs_resident_download)
+      bssn_em_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, true);
+    if (em_step_timing)
+      em_t_resident += MPI_Wtime() - em_t0;
+  }
+#endif
+
 #if (RPS == 0)
   // mesh refinement boundary part
+  if (em_step_timing)
+    em_t0 = MPI_Wtime();
   RestrictProlong(lev, YN, BB);
+  if (em_step_timing)
+    em_t_rp += MPI_Wtime() - em_t0;
 
 #ifdef WithShell
   if (lev == 0)
@@ -1813,6 +2378,8 @@ void bssnEM_class::Step(int lev, int YN)
   //
   // OldStateList  old -----------
   // update
+  if (em_step_timing)
+    em_t0 = MPI_Wtime();
   Pp = GH->PatL[lev];
   while (Pp)
   {
@@ -1858,6 +2425,26 @@ void bssnEM_class::Step(int lev, int YN)
       Porg0[ithBH][2] = Porg1[ithBH][2];
     }
   }
+  if (em_step_timing)
+  {
+    em_t_swap += MPI_Wtime() - em_t0;
+    static int em_step_report_count = 0;
+    const int em_timing_every = bssn_em_step_timing_every();
+    const bool report_all_levels = bssn_em_step_timing_all_levels_enabled();
+    if (lev == GH->levels - 1)
+      ++em_step_report_count;
+    if ((report_all_levels || lev == GH->levels - 1) &&
+        (em_timing_every <= 1 || em_step_report_count % em_timing_every == 0))
+    {
+      fprintf(stderr,
+              "[AMSS-EM-STEP-TIMING] lev=%d wall=%.6f predictor=%.6f pre_sync=%.6f "
+              "analysis=%.6f corrector=%.6f cor_sync=%.6f bh=%.6f swap=%.6f resident=%.6f rp=%.6f\n",
+              lev, MPI_Wtime() - em_step_t0,
+              em_t_predictor, em_t_predictor_sync,
+              em_t_analysis, em_t_corrector, em_t_corrector_sync,
+              em_t_bh, em_t_swap, em_t_resident, em_t_rp);
+    }
+  }
 }
 
 //================================================================================================
@@ -2036,6 +2623,59 @@ void bssnEM_class::AnalysisStuff_EM(int lev, double dT_lev)
   
   if (LastAnas >= AnasTime)
   {
+#if USE_CUDA_BSSN
+    const bool zero_em_analysis =
+        bssn_em_analysis_zero_fastpath_ready(GH->PatL[lev],
+#ifdef WithShell
+                                             SH
+#else
+                                             0
+#endif
+                                             , myrank
+        );
+#else
+    const bool zero_em_analysis = false;
+#endif
+    if (zero_em_analysis)
+    {
+#if USE_CUDA_BSSN
+      zero_em_analysis_outputs(GH->PatL[lev],
+#ifdef WithShell
+                               SH,
+#else
+                               0,
+#endif
+                               myrank,
+                               Rphi2, Iphi2, Rphi1, Iphi1);
+#endif
+      int NN = 0;
+      for (int pl = 1; pl < maxl + 1; pl++)
+        for (int pm = -pl; pm < pl + 1; pm++)
+          NN++;
+      double *RP = new double[NN];
+      double *IP = new double[NN];
+      std::memset(RP, 0, NN * sizeof(double));
+      std::memset(IP, 0, NN * sizeof(double));
+      for (int i = 0; i < decn; i++)
+        Phi2Monitor->writefile(PhysTime, NN, RP, IP);
+      delete[] RP;
+      delete[] IP;
+
+      NN = 0;
+      for (int pl = 0; pl < maxl + 1; pl++)
+        for (int pm = -pl; pm < pl + 1; pm++)
+          NN++;
+      RP = new double[NN];
+      IP = new double[NN];
+      std::memset(RP, 0, NN * sizeof(double));
+      std::memset(IP, 0, NN * sizeof(double));
+      for (int i = 0; i < decn; i++)
+        Phi1Monitor->writefile(PhysTime, NN, RP, IP);
+      delete[] RP;
+      delete[] IP;
+    }
+    else
+    {
     Compute_Phi2(lev);
     double *RP, *IP;
     int NN = 0;
@@ -2124,6 +2764,7 @@ void bssnEM_class::AnalysisStuff_EM(int lev, double dT_lev)
     }
     delete[] RP;
     delete[] IP;
+    }
   }
 
   AnalysisStuff(lev, dT_lev); // LastAnas need and only need control here
@@ -2304,11 +2945,12 @@ void bssnEM_class::Interp_Constraint()
   }
 
   ofstream outfile;
-  char filename[50];
-  sprintf(filename, "%s/interp_constraint_%05d.dat", ErrorMonitor->out_dir.c_str(), int(PhysTime / dT + 0.5)); 
+  char suffix[64];
+  sprintf(suffix, "/interp_constraint_%05d.dat", int(PhysTime / dT + 0.5));
+  string filename = ErrorMonitor->out_dir + suffix;
   // 0.5 for round off
   
-  outfile.open(filename);
+  outfile.open(filename.c_str());
   outfile << "#  corrdinate, H_Res, Px_Res, Py_Res, Pz_Res, Gx_Res, Gy_Res, Gz_Res, ...." << endl;
   for (int i = 0; i < n; i++)
   {

AMSS_NCKU_source/bssnEScalar_class.C

@@ -3,6 +3,7 @@
 #include <sstream>
 #include <cstdio>
 #include <map>
+#include <string>
 using namespace std;
 #else
 #include <stdio.h>
@@ -25,6 +26,9 @@ using namespace std;
 #include "getnp4.h"
 #include "shellfunctions.h"
 #include "parameters.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
 
 #ifdef With_AHF
 #include "derivatives.h"
@@ -33,6 +37,300 @@ using namespace std;
 
 //================================================================================================
 
+namespace
+{
+#if USE_CUDA_BSSN
+bool fill_bssn_escalar_cuda_views(Block *cg, MyList<var> *vars,
+                                  double **host_views,
+                                  double *propspeeds = 0,
+                                  double *soa_flat = 0)
+{
+  int idx = 0;
+  while (vars && idx < BSSN_ESCALAR_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == BSSN_ESCALAR_CUDA_STATE_COUNT && vars == 0;
+}
+
+bool bssn_escalar_cuda_use_resident_sync(int lev)
+{
+#ifdef WithShell
+  (void)lev;
+  return false;
+#else
+  return true;
+#endif
+}
+
+bool bssn_escalar_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
+{
+  static int keep_all_levels = -1;
+  if (keep_all_levels < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_KEEP_ALL_LEVELS");
+    keep_all_levels = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_KEEP_RESIDENT_AFTER_STEP");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  if (!enabled)
+    return false;
+  if (lev == analysis_lev)
+    return false;
+  static int release_only_level = -2;
+  if (release_only_level == -2)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_RELEASE_ONLY_LEVEL");
+    release_only_level = (env && atoi(env) >= 0) ? atoi(env) : -1;
+  }
+  if (release_only_level >= 0)
+    return lev != release_only_level;
+  static int keep_level_limit = -2;
+  if (keep_level_limit == -2)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_KEEP_LEVELS_BELOW");
+    keep_level_limit = (env && atoi(env) >= 0) ? atoi(env) : -1;
+  }
+  if (keep_level_limit >= 0)
+    return lev < keep_level_limit;
+  if (keep_all_levels)
+    return true;
+  return lev < trfls_in;
+}
+
+bool bssn_escalar_sync_merged_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_ESCALAR_SYNC_MERGED");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+void bssn_escalar_sync_level(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry)
+{
+  if (bssn_escalar_sync_merged_enabled())
+    Parallel::Sync_merged(PatL, VarList, Symmetry);
+  else
+    Parallel::Sync(PatL, VarList, Symmetry);
+}
+
+bool bssn_escalar_timing_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_ESCALAR_STEP_TIMING");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_post_rp_download_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_POST_RP_DOWNLOAD");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_post_rp_download_level_enabled(int lev)
+{
+  if (!bssn_escalar_cuda_post_rp_download_enabled())
+    return false;
+  static int min_level = -2;
+  if (min_level == -2)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_POST_RP_MIN_LEVEL");
+    min_level = (env && atoi(env) >= 0) ? atoi(env) : -1;
+  }
+  return min_level < 0 || lev >= min_level;
+}
+
+bool bssn_escalar_cuda_post_swap_release_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_POST_SWAP_RELEASE");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_pre_rp_release_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_PRE_RP_RELEASE");
+    enabled = env ? ((atoi(env) != 0) ? 1 : 0) : 1;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_bh_interp_resident_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_BH_INTERP_RESIDENT");
+    enabled = env ? ((atoi(env) != 0) ? 1 : 0) : 1;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_prune_after_swap_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_PRUNE_AFTER_SWAP");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+void bssn_escalar_cuda_upload_level_state(MyList<Patch> *PatL, MyList<var> *vars,
+                                          int myrank)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && bssn_cuda_has_resident_state(cg))
+      {
+        double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
+        if (!fill_bssn_escalar_cuda_views(cg, vars, state_in))
+        {
+          cout << "CUDA BSSN-EScalar resident state list mismatch during upload" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (bssn_escalar_cuda_upload_resident_state(cg, cg->shape, state_in))
+        {
+          cout << "CUDA BSSN-EScalar resident state upload failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+
+void bssn_escalar_cuda_keep_only_level_state(MyList<Patch> *PatL, MyList<var> *vars,
+                                             int myrank)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && bssn_cuda_has_resident_state(cg))
+      {
+        double *state_key[BSSN_ESCALAR_CUDA_STATE_COUNT];
+        if (!fill_bssn_escalar_cuda_views(cg, vars, state_key))
+        {
+          cout << "CUDA BSSN-EScalar resident state list mismatch during prune" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (bssn_escalar_cuda_keep_only_resident_state(cg, cg->shape, state_key))
+        {
+          cout << "CUDA BSSN-EScalar resident state prune failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+
+void bssn_escalar_timing_report(int myrank, int lev, int YN, double total, double rhs,
+                                double sync, double bh, double analysis, double swap,
+                                double resident, double rp)
+{
+  if (!bssn_escalar_timing_enabled())
+    return;
+
+  double local[8] = {total, rhs, sync, bh, analysis, swap, resident, rp};
+  double maxv[8] = {};
+  MPI_Reduce(local, maxv, 8, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+  if (myrank == 0)
+    fprintf(stderr,
+            "[AMSS-ESCALAR-STEP] lev=%d YN=%d total=%.6f rhs=%.6f sync=%.6f "
+            "bh=%.6f analysis=%.6f swap=%.6f resident=%.6f rp=%.6f other=%.6f\n",
+            lev, YN, maxv[0], maxv[1], maxv[2], maxv[3], maxv[4], maxv[5],
+            maxv[6], maxv[7],
+            maxv[0] - maxv[1] - maxv[2] - maxv[3] - maxv[4] - maxv[5] - maxv[6] - maxv[7]);
+}
+
+void bssn_escalar_cuda_download_level_state(MyList<Patch> *PatL, MyList<var> *vars,
+                                            int myrank, bool release_ctx)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && bssn_cuda_has_resident_state(cg))
+      {
+        double *state_out[BSSN_ESCALAR_CUDA_STATE_COUNT];
+        if (!fill_bssn_escalar_cuda_views(cg, vars, state_out))
+        {
+          cout << "CUDA BSSN-EScalar resident state list mismatch during download" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (bssn_escalar_cuda_download_resident_state(cg, cg->shape, state_out))
+        {
+          cout << "CUDA BSSN-EScalar resident state download failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (release_ctx)
+          bssn_cuda_release_step_ctx(cg);
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+#endif
+}
+
+//================================================================================================
+
 // Define bssnEScalar_class
 
 // It inherits some members and methods from the parent class bssn_class and modifies others.
@@ -179,6 +477,11 @@ void bssnEScalar_class::Initialize()
 
 bssnEScalar_class::~bssnEScalar_class()
 {
+#if USE_CUDA_BSSN
+  for (int lev = 0; GH && lev < GH->levels; ++lev)
+    bssn_escalar_cuda_download_level_state(GH->PatL[lev], StateList, myrank, true);
+#endif
+
   delete Sphio;
   delete Spio;
   delete Sphi0;
@@ -708,6 +1011,11 @@ void bssnEScalar_class::Read_Pablo()
 void bssnEScalar_class::Step(int lev, int YN)
 {
   double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
+#if USE_CUDA_BSSN
+  const bool use_cuda_resident_sync = bssn_escalar_cuda_use_resident_sync(lev);
+#else
+  const bool use_cuda_resident_sync = false;
+#endif
 #ifdef With_AHF
   AH_Step_Find(lev, dT_lev);
 #endif
@@ -719,9 +1027,19 @@ void bssnEScalar_class::Step(int lev, int YN)
   int iter_count = 0; // count RK4 substeps
   int pre = 0, cor = 1;
   int ERROR = 0;
+  const bool escalar_step_timing = bssn_escalar_timing_enabled();
+  const double escalar_step_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+  double escalar_t_rhs = 0.0;
+  double escalar_t_sync = 0.0;
+  double escalar_t_bh = 0.0;
+  double escalar_t_analysis = 0.0;
+  double escalar_t_swap = 0.0;
+  double escalar_t_resident = 0.0;
+  double escalar_t_rp = 0.0;
 
   MyList<ss_patch> *sPp;
   // Predictor
+  double escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
   MyList<Patch> *Pp = GH->PatL[lev];
   while (Pp)
   {
@@ -732,14 +1050,59 @@ void bssnEScalar_class::Step(int lev, int YN)
       if (myrank == cg->rank)
       {
 #if (AGM == 0)
+#if !USE_CUDA_BSSN
         f_enforce_ga(cg->shape,
                      cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                      cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                      cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
                      cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
+#endif
 #endif
 
-        if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+        bool used_gpu_substep = false;
+#if USE_CUDA_BSSN
+        {
+          double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
+          double *state_out[BSSN_ESCALAR_CUDA_STATE_COUNT];
+          double propspeed[BSSN_ESCALAR_CUDA_STATE_COUNT];
+          double soa_flat[3 * BSSN_ESCALAR_CUDA_STATE_COUNT];
+          if (!fill_bssn_escalar_cuda_views(cg, StateList, state_in, propspeed, soa_flat) ||
+              !fill_bssn_escalar_cuda_views(cg, SynchList_pre, state_out))
+          {
+            cout << "CUDA BSSN-EScalar state list mismatch on predictor step" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+          int apply_bam_bc = 0;
+          int apply_enforce_ga = 0;
+#if (AGM == 0)
+          apply_enforce_ga = 1;
+#endif
+#if (SommerType == 0)
+#ifndef WithShell
+          apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+          int keep_resident_state = use_cuda_resident_sync ? 1 : 0;
+          if (bssn_escalar_cuda_rk4_substep(cg,
+                                            cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                            state_in, state_out,
+                                            propspeed, soa_flat, Pp->data->bbox,
+                                            dT_lev, TRK4, iter_count, apply_bam_bc,
+                                            Symmetry, lev, ndeps, pre,
+                                            keep_resident_state, apply_enforce_ga, chitiny))
+          {
+            cout << "CUDA BSSN-EScalar predictor substep failed in domain: ("
+                 << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                 << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                 << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+            ERROR = 1;
+          }
+          used_gpu_substep = true;
+        }
+#endif
+
+        if (!used_gpu_substep &&
+            f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                        cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
                                        cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                        cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
@@ -783,6 +1146,8 @@ void bssnEScalar_class::Step(int lev, int YN)
           ERROR = 1;
         }
 
+        if (!used_gpu_substep)
+        {
         // rk4 substep and boundary
         {
           MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
@@ -822,6 +1187,7 @@ void bssnEScalar_class::Step(int lev, int YN)
           }
         }
         f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
+        }
       }
       if (BP == Pp->data->ble)
         break;
@@ -845,6 +1211,8 @@ void bssnEScalar_class::Step(int lev, int YN)
       MPI_Abort(MPI_COMM_WORLD, 1);
     }
   }
+  if (escalar_step_timing)
+    escalar_t_rhs += MPI_Wtime() - escalar_t0;
 
 #ifdef WithShell
   // evolve Shell Patches
@@ -993,7 +1361,14 @@ void bssnEScalar_class::Step(int lev, int YN)
   }
 #endif
 
+  escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+  bssn_escalar_sync_level(GH->PatL[lev], SynchList_pre, Symmetry);
+#else
   Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+#endif
+  if (escalar_step_timing)
+    escalar_t_sync += MPI_Wtime() - escalar_t0;
 
 #ifdef WithShell
   if (lev == 0)
@@ -1016,6 +1391,11 @@ void bssnEScalar_class::Step(int lev, int YN)
   // for black hole position
   if (BH_num > 0 && lev == GH->levels - 1)
   {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+    if (use_cuda_resident_sync && !bssn_escalar_cuda_bh_interp_resident_enabled())
+      bssn_escalar_cuda_download_level_state(GH->PatL[lev], StateList, myrank, false);
+#endif
     compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
     for (int ithBH = 0; ithBH < BH_num; ithBH++)
     {
@@ -1044,16 +1424,26 @@ void bssnEScalar_class::Step(int lev, int YN)
         DG_List->clearList();
       }
     }
+    if (escalar_step_timing)
+      escalar_t_bh += MPI_Wtime() - escalar_t0;
   }
   // data analysis part
   // Warning NOTE: the variables1 are used as temp storege room
   if (lev == a_lev)
   {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+    if (use_cuda_resident_sync)
+      bssn_escalar_cuda_download_level_state(GH->PatL[lev], SynchList_pre, myrank, false);
+#endif
     AnalysisStuff_EScalar(lev, dT_lev);
+    if (escalar_step_timing)
+      escalar_t_analysis += MPI_Wtime() - escalar_t0;
   }
   // corrector
   for (iter_count = 1; iter_count < 4; iter_count++)
   {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
     // for RK4: t0, t0+dt/2, t0+dt/2, t0+dt;
     if (iter_count == 1 || iter_count == 3)
       TRK4 += dT_lev / 2;
@@ -1067,11 +1457,13 @@ void bssnEScalar_class::Step(int lev, int YN)
         if (myrank == cg->rank)
         {
 #if (AGM == 0)
+#if !USE_CUDA_BSSN
           f_enforce_ga(cg->shape,
                        cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                        cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                        cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
                        cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
+#endif
 #elif (AGM == 1)
           if (iter_count == 3)
             f_enforce_ga(cg->shape,
@@ -1081,7 +1473,50 @@ void bssnEScalar_class::Step(int lev, int YN)
                          cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
 #endif
 
-          if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+          bool used_gpu_substep = false;
+#if USE_CUDA_BSSN
+          {
+            double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
+            double *state_out[BSSN_ESCALAR_CUDA_STATE_COUNT];
+            double propspeed[BSSN_ESCALAR_CUDA_STATE_COUNT];
+            double soa_flat[3 * BSSN_ESCALAR_CUDA_STATE_COUNT];
+            if (!fill_bssn_escalar_cuda_views(cg, SynchList_pre, state_in, propspeed, soa_flat) ||
+                !fill_bssn_escalar_cuda_views(cg, SynchList_cor, state_out))
+            {
+              cout << "CUDA BSSN-EScalar state list mismatch on corrector step" << endl;
+              MPI_Abort(MPI_COMM_WORLD, 1);
+            }
+            int apply_bam_bc = 0;
+            int apply_enforce_ga = 0;
+#if (AGM == 0)
+            apply_enforce_ga = 1;
+#endif
+#if (SommerType == 0)
+#ifndef WithShell
+            apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+            int keep_resident_state = use_cuda_resident_sync ? 1 : 0;
+            if (bssn_escalar_cuda_rk4_substep(cg,
+                                              cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                              state_in, state_out,
+                                              propspeed, soa_flat, Pp->data->bbox,
+                                              dT_lev, TRK4, iter_count, apply_bam_bc,
+                                              Symmetry, lev, ndeps, cor,
+                                              keep_resident_state, apply_enforce_ga, chitiny))
+            {
+              cout << "CUDA BSSN-EScalar corrector substep failed in domain: ("
+                   << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                   << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                   << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+              ERROR = 1;
+            }
+            used_gpu_substep = true;
+          }
+#endif
+
+          if (!used_gpu_substep &&
+              f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                          cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
                                          cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                          cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
@@ -1125,6 +1560,8 @@ void bssnEScalar_class::Step(int lev, int YN)
                  << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
             ERROR = 1;
           }
+          if (!used_gpu_substep)
+          {
           // rk4 substep and boundary
           {
             MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList; 
@@ -1167,6 +1604,7 @@ void bssnEScalar_class::Step(int lev, int YN)
             }
           }
           f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
+          }
         }
         if (BP == Pp->data->ble)
           break;
@@ -1192,6 +1630,8 @@ void bssnEScalar_class::Step(int lev, int YN)
         MPI_Abort(MPI_COMM_WORLD, 1);
       }
     }
+    if (escalar_step_timing)
+      escalar_t_rhs += MPI_Wtime() - escalar_t0;
 
 #ifdef WithShell
     // evolve Shell Patches
@@ -1349,7 +1789,14 @@ void bssnEScalar_class::Step(int lev, int YN)
     }
 #endif
 
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+    bssn_escalar_sync_level(GH->PatL[lev], SynchList_cor, Symmetry);
+#else
     Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+#endif
+    if (escalar_step_timing)
+      escalar_t_sync += MPI_Wtime() - escalar_t0;
 
 #ifdef WithShell
     if (lev == 0)
@@ -1371,6 +1818,11 @@ void bssnEScalar_class::Step(int lev, int YN)
     // for black hole position
     if (BH_num > 0 && lev == GH->levels - 1)
     {
+      escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+      if (use_cuda_resident_sync && !bssn_escalar_cuda_bh_interp_resident_enabled())
+        bssn_escalar_cuda_download_level_state(GH->PatL[lev], SynchList_pre, myrank, false);
+#endif
       compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
       for (int ithBH = 0; ithBH < BH_num; ithBH++)
       {
@@ -1399,10 +1851,13 @@ void bssnEScalar_class::Step(int lev, int YN)
           DG_List->clearList();
         }
       }
+      if (escalar_step_timing)
+        escalar_t_bh += MPI_Wtime() - escalar_t0;
     }
     // swap time level
     if (iter_count < 3)
     {
+      escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
       Pp = GH->PatL[lev];
       while (Pp)
       {
@@ -1446,12 +1901,29 @@ void bssnEScalar_class::Step(int lev, int YN)
           Porg[ithBH][2] = Porg1[ithBH][2];
         }
       }
+      if (escalar_step_timing)
+        escalar_t_swap += MPI_Wtime() - escalar_t0;
     }
   }
 
+#if USE_CUDA_BSSN
+  if (use_cuda_resident_sync)
+  {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+    if (!bssn_escalar_cuda_keep_resident_after_step(lev, trfls, a_lev))
+      bssn_escalar_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank,
+                                             bssn_escalar_cuda_pre_rp_release_enabled());
+    if (escalar_step_timing)
+      escalar_t_resident += MPI_Wtime() - escalar_t0;
+  }
+#endif
+
 #if (RPS == 0)
   // mesh refinement boundary part
+  escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
   RestrictProlong(lev, YN, BB);
+  if (escalar_step_timing)
+    escalar_t_rp += MPI_Wtime() - escalar_t0;
 
 #ifdef WithShell
   if (lev == 0)
@@ -1478,6 +1950,7 @@ void bssnEScalar_class::Step(int lev, int YN)
   //
   // OldStateList  old -----------
   // update
+  escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
   Pp = GH->PatL[lev];
   while (Pp)
   {
@@ -1512,6 +1985,25 @@ void bssnEScalar_class::Step(int lev, int YN)
       sPp = sPp->next;
     }
   }
+#endif
+  #if USE_CUDA_BSSN
+  bool release_after_sync = false;
+  if (use_cuda_resident_sync && bssn_escalar_cuda_post_rp_download_level_enabled(lev))
+  {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+    release_after_sync = bssn_escalar_cuda_post_swap_release_enabled();
+    bssn_escalar_cuda_download_level_state(GH->PatL[lev], StateList, myrank, release_after_sync);
+    if (escalar_step_timing)
+      escalar_t_resident += MPI_Wtime() - escalar_t0;
+  }
+  if (use_cuda_resident_sync && !release_after_sync &&
+      bssn_escalar_cuda_prune_after_swap_enabled())
+  {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+    bssn_escalar_cuda_keep_only_level_state(GH->PatL[lev], StateList, myrank);
+    if (escalar_step_timing)
+      escalar_t_resident += MPI_Wtime() - escalar_t0;
+  }
 #endif
   // for black hole position
   if (BH_num > 0 && lev == GH->levels - 1)
@@ -1523,6 +2015,14 @@ void bssnEScalar_class::Step(int lev, int YN)
       Porg0[ithBH][2] = Porg1[ithBH][2];
     }
   }
+  if (escalar_step_timing)
+  {
+    escalar_t_swap += MPI_Wtime() - escalar_t0;
+    bssn_escalar_timing_report(myrank, lev, YN, MPI_Wtime() - escalar_step_t0,
+                               escalar_t_rhs, escalar_t_sync, escalar_t_bh,
+                               escalar_t_analysis, escalar_t_swap,
+                               escalar_t_resident, escalar_t_rp);
+  }
 }
 
 //================================================================================================
@@ -2024,11 +2524,12 @@ void bssnEScalar_class::Interp_Constraint()
   }
 
   ofstream outfile;
-  char filename[50];
-  sprintf(filename, "%s/interp_constraint_%05d.dat", ErrorMonitor->out_dir.c_str(), int(PhysTime / dT + 0.5)); 
+  char suffix[64];
+  sprintf(suffix, "/interp_constraint_%05d.dat", int(PhysTime / dT + 0.5));
+  string filename = ErrorMonitor->out_dir + suffix;
   // 0.5 for round off
   
-  outfile.open(filename);
+  outfile.open(filename.c_str());
   outfile << "#  corrdinate, H_Res, Px_Res, Py_Res, Pz_Res, Gx_Res, Gy_Res, Gz_Res, fR_Res, ...." << endl;
   for (int i = 0; i < n; i++)
   {
@@ -2077,7 +2578,37 @@ void bssnEScalar_class::Constraint_Out()
             Block *cg = BP->data;
             if (myrank == cg->rank)
             {
-              if (lev > 0)
+              bool used_cuda_constraints = false;
+#if USE_CUDA_BSSN
+              {
+                double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
+                if (!fill_bssn_escalar_cuda_views(cg, StateList, state_in))
+                {
+                  cout << "CUDA BSSN-EScalar constraint state list mismatch" << endl;
+                  MPI_Abort(MPI_COMM_WORLD, 1);
+                }
+                double *constraint_out[8] = {
+                    cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+                    cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                    cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+                    cg->fgfs[Cons_Gz->sgfn], cg->fgfs[Cons_fR->sgfn]};
+                int lev_arg = lev;
+                int sym_arg = Symmetry;
+                double eps_arg = ndeps;
+                if (bssn_escalar_cuda_compute_constraints(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                                          state_in, constraint_out,
+                                                          sym_arg, lev_arg, eps_arg))
+                {
+                  cout << "CUDA BSSN-EScalar constraint compute failed in domain: ("
+                       << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                       << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                       << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+                  MPI_Abort(MPI_COMM_WORLD, 1);
+                }
+                used_cuda_constraints = true;
+              }
+#endif
+              if (!used_cuda_constraints && lev > 0)
                 f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                            cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
                                            cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
@@ -2114,7 +2645,8 @@ void bssnEScalar_class::Constraint_Out()
                                            cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
                                            cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
                                            Symmetry, lev, ndeps, pre);
-              f_compute_constraint_fr(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+              if (!used_cuda_constraints)
+                f_compute_constraint_fr(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                       cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn], 
                                       cg->fgfs[rho->sgfn], cg->fgfs[Sphi0->sgfn],
                                       cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 

AMSS_NCKU_source/bssn_class.h

@@ -48,6 +48,8 @@ public:
        double StartTime, TotalTime;
        double AnasTime, DumpTime, d2DumpTime, CheckTime;
        double LastAnas, LastConsOut;
+       bool cuda_level0_constraint_cache_valid;
+       int *ConstraintRefreshLevels;
        double Courant;
        double numepss, numepsb, numepsh;
        int Symmetry;
@@ -134,7 +136,7 @@ public:
        Parallel::SyncCache *sync_cache_outbd;      // cached OutBdLow2Hi in RestrictProlong
 
        monitor *ErrorMonitor, *Psi4Monitor, *BHMonitor, *MAPMonitor;
-       monitor *ConVMonitor;
+       monitor *ConVMonitor, *TimingMonitor;
        surface_integral *Waveshell;
        checkpoint *CheckPoint;
 
@@ -142,7 +144,7 @@ public:
        bssn_class(double Couranti, double StartTimei, double TotalTimei, double DumpTimei, double d2DumpTimei, double CheckTimei, double AnasTimei,
                   int Symmetryi, int checkruni, char *checkfilenamei, double numepssi, double numepsbi, double numepshi,
                   int a_levi, int maxli, int decni, double maxrexi, double drexi);
-       ~bssn_class();
+       virtual ~bssn_class();
 
        void Evolve(int Steps);
        void RecursiveStep(int lev);

AMSS_NCKU_source/bssn_gpu.h

@@ -2,7 +2,7 @@
 #ifndef BSSN_GPU_H_
 #define BSSN_GPU_H_
 #include "bssn_macro.h"
-#include "macrodef.fh"
+#include "macrodef.h"
 
 #define DEVICE_ID 0
 // #define DEVICE_ID_BY_MPI_RANK
@@ -25,49 +25,32 @@
 /**  main function */
 int gpu_rhs(int calledby, int mpi_rank, int *ex, double &T,
             double *X, double *Y, double *Z,
-
             double *chi, double *trK,
-
             double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
-
             double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
-
             double *Gamx, double *Gamy, double *Gamz,
-
             double *Lap, double *betax, double *betay, double *betaz,
-
             double *dtSfx, double *dtSfy, double *dtSfz,
-
             double *chi_rhs, double *trK_rhs,
-
             double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
-
             double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
-
             double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
-
             double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
-
             double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
-
             double *rho, double *Sx, double *Sy, double *Sz, double *Sxx,
             double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
-
             double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
-
             double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
-
             double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
-
             double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
-
             double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
             double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
             int &Symmetry, int &Lev, double &eps, int &co);
 
 int gpu_rhs_ss(RHS_SS_PARA);
 
-/** Init GPU side data in GPUMeta. */
-// void init_fluid_meta_gpu(GPUMeta *gpu_meta);
+#define Z4C_SS_PARA int calledby, int mpi_rank, int *ex, double &T, double *crho, double *sigma, double *R, double *X, double *Y, double *Z, double *drhodx, double *drhody, double *drhodz, double *dsigmadx, double *dsigmady, double *dsigmadz, double *dRdx, double *dRdy, double *dRdz, double *drhodxx, double *drhodxy, double *drhodxz, double *drhodyy, double *drhodyz, double *drhodzz, double *dsigmadxx, double *dsigmadxy, double *dsigmadxz, double *dsigmadyy, double *dsigmadyz, double *dsigmadzz, double *dRdxx, double *dRdxy, double *dRdxz, double *dRdyy, double *dRdyz, double *dRdzz, double *chi, double *trK, double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz, double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz, double *Gamx, double *Gamy, double *Gamz, double *Lap, double *betax, double *betay, double *betaz, double *dtSfx, double *dtSfy, double *dtSfz, double *TZ, double *chi_rhs, double *trK_rhs, double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs, double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs, double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs, double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs, double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs, double *TZ_rhs, double *rho, double *Sx, double *Sy, double *Sz, double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz, double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz, double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz, double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz, double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz, double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res, double *Gmx_Res, double *Gmy_Res, double *Gmz_Res, int &Symmetry, int &Lev, double &eps, int &sst, int &co
+
+int gpu_rhs_z4c_ss(Z4C_SS_PARA);
 
 #endif

AMSS_NCKU_source/bssn_gpu_class.h

@@ -1,210 +0,0 @@
-
-#ifndef BSSN_GPU_CLASS_H
-#define BSSN_GPU_CLASS_H
-
-#ifdef newc
-#include <iostream>
-#include <iomanip>
-#include <fstream>
-#include <cstdlib>
-#include <string>
-#include <cmath>
-using namespace std;
-#else
-#include <iostream.h>
-#include <iomanip.h>
-#include <fstream.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#endif
-
-#include <mpi.h>
-
-#include "macrodef.h"
-#include "cgh.h"
-#include "ShellPatch.h"
-#include "misc.h"
-#include "var.h"
-#include "MyList.h"
-#include "monitor.h"
-#include "surface_integral.h"
-#include "checkpoint.h"
-
-// added by yangquan
-#include "bssn_macro.h"
-
-extern void setpbh(int iBHN, double **iPBH, double *iMass, int rBHN);
-
-class bssn_class
-{
-public:
-       // added by yangquan
-       //----------------------
-       int gpu_num_mynode;
-       int cpu_core_num_mynode;
-       int mpi_process_num_mynode;
-       int my_sequence_mynode;
-       int mynode_id;
-       int use_gpu;
-
-       virtual void Step_GPU(int lev, int YN);
-       virtual void Get_runtime_envirment();
-       // virtual void Step_OPENMP(int lev,int YN);
-       //----------------------
-
-       int ngfs;
-       int nprocs, myrank;
-       cgh *GH;
-       ShellPatch *SH;
-       double PhysTime;
-
-       int checkrun;
-       char checkfilename[50];
-       int Steps;
-       double StartTime, TotalTime;
-       double AnasTime, DumpTime, d2DumpTime, CheckTime;
-       double LastAnas, LastConsOut;
-       double Courant;
-       double numepss, numepsb, numepsh;
-       int Symmetry;
-       int maxl, decn;
-       double maxrex, drex;
-       int trfls, a_lev;
-
-       double dT;
-       double chitiny;
-
-       double **Porg0, **Porgbr, **Porg, **Porg1, **Porg_rhs;
-       int BH_num, BH_num_input;
-       double *Mass, *Pmom, *Spin;
-       double ADMMass;
-
-       var *phio, *trKo;
-       var *gxxo, *gxyo, *gxzo, *gyyo, *gyzo, *gzzo;
-       var *Axxo, *Axyo, *Axzo, *Ayyo, *Ayzo, *Azzo;
-       var *Gmxo, *Gmyo, *Gmzo;
-       var *Lapo, *Sfxo, *Sfyo, *Sfzo;
-       var *dtSfxo, *dtSfyo, *dtSfzo;
-
-       var *phi0, *trK0;
-       var *gxx0, *gxy0, *gxz0, *gyy0, *gyz0, *gzz0;
-       var *Axx0, *Axy0, *Axz0, *Ayy0, *Ayz0, *Azz0;
-       var *Gmx0, *Gmy0, *Gmz0;
-       var *Lap0, *Sfx0, *Sfy0, *Sfz0;
-       var *dtSfx0, *dtSfy0, *dtSfz0;
-
-       var *phi, *trK;
-       var *gxx, *gxy, *gxz, *gyy, *gyz, *gzz;
-       var *Axx, *Axy, *Axz, *Ayy, *Ayz, *Azz;
-       var *Gmx, *Gmy, *Gmz;
-       var *Lap, *Sfx, *Sfy, *Sfz;
-       var *dtSfx, *dtSfy, *dtSfz;
-
-       var *phi1, *trK1;
-       var *gxx1, *gxy1, *gxz1, *gyy1, *gyz1, *gzz1;
-       var *Axx1, *Axy1, *Axz1, *Ayy1, *Ayz1, *Azz1;
-       var *Gmx1, *Gmy1, *Gmz1;
-       var *Lap1, *Sfx1, *Sfy1, *Sfz1;
-       var *dtSfx1, *dtSfy1, *dtSfz1;
-
-       var *phi_rhs, *trK_rhs;
-       var *gxx_rhs, *gxy_rhs, *gxz_rhs, *gyy_rhs, *gyz_rhs, *gzz_rhs;
-       var *Axx_rhs, *Axy_rhs, *Axz_rhs, *Ayy_rhs, *Ayz_rhs, *Azz_rhs;
-       var *Gmx_rhs, *Gmy_rhs, *Gmz_rhs;
-       var *Lap_rhs, *Sfx_rhs, *Sfy_rhs, *Sfz_rhs;
-       var *dtSfx_rhs, *dtSfy_rhs, *dtSfz_rhs;
-
-       var *rho, *Sx, *Sy, *Sz, *Sxx, *Sxy, *Sxz, *Syy, *Syz, *Szz;
-
-       var *Gamxxx, *Gamxxy, *Gamxxz, *Gamxyy, *Gamxyz, *Gamxzz;
-       var *Gamyxx, *Gamyxy, *Gamyxz, *Gamyyy, *Gamyyz, *Gamyzz;
-       var *Gamzxx, *Gamzxy, *Gamzxz, *Gamzyy, *Gamzyz, *Gamzzz;
-
-       var *Rxx, *Rxy, *Rxz, *Ryy, *Ryz, *Rzz;
-
-       var *Rpsi4, *Ipsi4;
-       var *t1Rpsi4, *t1Ipsi4, *t2Rpsi4, *t2Ipsi4;
-
-       var *Cons_Ham, *Cons_Px, *Cons_Py, *Cons_Pz, *Cons_Gx, *Cons_Gy, *Cons_Gz;
-
-#ifdef Point_Psi4
-       var *phix, *phiy, *phiz;
-       var *trKx, *trKy, *trKz;
-       var *Axxx, *Axxy, *Axxz;
-       var *Axyx, *Axyy, *Axyz;
-       var *Axzx, *Axzy, *Axzz;
-       var *Ayyx, *Ayyy, *Ayyz;
-       var *Ayzx, *Ayzy, *Ayzz;
-       var *Azzx, *Azzy, *Azzz;
-#endif
-       // FIXME: uc = StateList, up = OldStateList, upp = SynchList_cor; so never touch these three data
-       MyList<var> *StateList, *SynchList_pre, *SynchList_cor, *RHSList;
-       MyList<var> *OldStateList, *DumpList;
-       MyList<var> *ConstraintList;
-
-       monitor *ErrorMonitor, *Psi4Monitor, *BHMonitor, *MAPMonitor;
-       monitor *ConVMonitor;
-       surface_integral *Waveshell;
-       checkpoint *CheckPoint;
-
-public:
-       bssn_class(double Couranti, double StartTimei, double TotalTimei, double DumpTimei, double d2DumpTimei, double CheckTimei, double AnasTimei,
-                  int Symmetryi, int checkruni, char *checkfilenamei, double numepssi, double numepsbi, double numepshi,
-                  int a_levi, int maxli, int decni, double maxrexi, double drexi);
-       ~bssn_class();
-
-       void Evolve(int Steps);
-       void RecursiveStep(int lev);
-#if (PSTR == 1)
-       void ParallelStep();
-       void SHStep();
-#endif
-       void RestrictProlong(int lev, int YN, bool BB, MyList<var> *SL, MyList<var> *OL, MyList<var> *corL);
-       void RestrictProlong_aux(int lev, int YN, bool BB, MyList<var> *SL, MyList<var> *OL, MyList<var> *corL);
-       void RestrictProlong(int lev, int YN, bool BB);
-       void ProlongRestrict(int lev, int YN, bool BB);
-       void Setup_Black_Hole_position();
-       void compute_Porg_rhs(double **BH_PS, double **BH_RHS, var *forx, var *fory, var *forz, int lev);
-       bool read_Pablo_file(int *ext, double *datain, char *filename);
-       void write_Pablo_file(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
-                             char *filename);
-       void AnalysisStuff(int lev, double dT_lev);
-       void Setup_KerrSchild();
-       void Enforce_algcon(int lev, int fg);
-
-       void testRestrict();
-       void testOutBd();
-
-       virtual void Setup_Initial_Data_Lousto();
-       virtual void Setup_Initial_Data_Cao();
-       virtual void Initialize();
-       virtual void Read_Ansorg();
-       virtual void Read_Pablo() {};
-       virtual void Compute_Psi4(int lev);
-       virtual void Step(int lev, int YN);
-       virtual void Interp_Constraint(bool infg);
-       virtual void Constraint_Out();
-       virtual void Compute_Constraint();
-
-#ifdef With_AHF
-protected:
-       MyList<var> *AHList, *AHDList, *GaugeList;
-       int AHfindevery;
-       double AHdumptime;
-       int *lastahdumpid, HN_num; // number of possible horizons
-       int *findeveryl;
-       double *xc, *yc, *zc, *xr, *yr, *zr;
-       bool *trigger;
-       double *dTT;
-       int *dumpid;
-
-public:
-       void AH_Prepare_derivatives();
-       bool AH_Interp_Points(MyList<var> *VarList,
-                             int NN, double **XX,
-                             double *Shellf, int Symmetryi);
-       void AH_Step_Find(int lev, double dT_lev);
-#endif
-};
-#endif /* BSSN_GPU_CLASS_H */

AMSS_NCKU_source/bssn_gpu_rhs_ss.cu

@@ -20,12 +20,14 @@ using namespace std;
 
 __device__ volatile unsigned int global_count = 0;
 
+#ifdef RESULT_CHECK
 void compare_result_gpu(int ftag1,double * datac,int data_num){
 	double * data = (double*)malloc(sizeof(double)*data_num);
 	cudaMemcpy(data, datac, data_num * sizeof(double), cudaMemcpyDeviceToHost);
 	compare_result(ftag1,data,data_num);
 	free(data);
 }
+#endif
 
 __global__ void sub_symmetry_bd_ss_partF(int ord, double * func, double *funcc)
 {
@@ -153,11 +155,11 @@ __global__ void sub_symmetry_bd_ss_partJ(int ord,double * func, double * funcc,d
 
 inline void sub_symmetry_bd_ss(int ord,double * func, double * funcc,double * SoA){
 	sub_symmetry_bd_ss_partF<<<GRID_DIM,BLOCK_DIM>>>(ord,func,funcc);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	sub_symmetry_bd_ss_partI<<<GRID_DIM,BLOCK_DIM>>>(ord,func,funcc,SoA[0]);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	sub_symmetry_bd_ss_partJ<<<GRID_DIM,BLOCK_DIM>>>(ord,func,funcc,SoA[1]);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 }
 
 __global__ void sub_fderivs_shc_part1(double *fx,double *fy,double *fz){
@@ -247,13 +249,13 @@ inline void sub_fderivs_shc(int& sst,double * f,double * fh,double *fx,double *f
 	//cudaMemset(Msh_ gy,0,h_3D_SIZE[0] * sizeof(double));
 	//cudaMemset(Msh_ gz,0,h_3D_SIZE[0] * sizeof(double));
 	sub_symmetry_bd_ss(2,f,fh,SoA1);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(0,fh,h_3D_SIZE[2]);	
 	sub_fderivs_sh<<<GRID_DIM,BLOCK_DIM>>>(fh,Msh_ gx,Msh_ gy,Msh_ gz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	sub_fderivs_shc_part1<<<GRID_DIM,BLOCK_DIM>>>(fx,fy,fz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(1,fx,h_3D_SIZE[0]);
 			//compare_result_gpu(2,fy,h_3D_SIZE[0]);
 			//compare_result_gpu(3,fz,h_3D_SIZE[0]);
@@ -451,17 +453,17 @@ inline void sub_fdderivs_shc(int& sst,double * f,double * fh,
 	
 	//fderivs_sh
 	sub_symmetry_bd_ss(2,f,fh,SoA1);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(1,fh,h_3D_SIZE[2]);	
 	sub_fderivs_sh<<<GRID_DIM,BLOCK_DIM>>>(fh,Msh_ gx,Msh_ gy,Msh_ gz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	//fdderivs_sh
 	sub_symmetry_bd_ss(2,f,fh,SoA1);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(21,fh,h_3D_SIZE[2]);
 	sub_fdderivs_sh<<<GRID_DIM,BLOCK_DIM>>>(fh,Msh_ gxx,Msh_ gxy,Msh_ gxz,Msh_ gyy,Msh_ gyz,Msh_ gzz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			/*compare_result_gpu(11,Msh_ gx,h_3D_SIZE[0]);
 			compare_result_gpu(12,Msh_ gy,h_3D_SIZE[0]);
 			compare_result_gpu(13,Msh_ gz,h_3D_SIZE[0]);
@@ -472,7 +474,7 @@ inline void sub_fdderivs_shc(int& sst,double * f,double * fh,
 			compare_result_gpu(5,Msh_ gyz,h_3D_SIZE[0]);
 			compare_result_gpu(6,Msh_ gzz,h_3D_SIZE[0]);*/
 	sub_fdderivs_shc_part1<<<GRID_DIM,BLOCK_DIM>>>(fxx,fxy,fxz,fyy,fyz,fzz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			/*compare_result_gpu(1,fxx,h_3D_SIZE[0]);
 			compare_result_gpu(2,fxy,h_3D_SIZE[0]);
 			compare_result_gpu(3,fxz,h_3D_SIZE[0]);
@@ -496,9 +498,9 @@ __global__ void computeRicci_ss_part1(double * dst)
  inline void computeRicci_ss(int &sst,double * src,double* dst,double * SoA, Meta* meta)
 {
 	sub_fdderivs_shc(sst,src,Mh_ fh,Mh_ fxx,Mh_ fxy,Mh_ fxz,Mh_ fyy,Mh_ fyz,Mh_ fzz,SoA);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	computeRicci_ss_part1<<<GRID_DIM,BLOCK_DIM>>>(dst);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 }
 __global__ void sub_lopsided_ss_part1(double * dst)
@@ -516,9 +518,9 @@ __global__ void sub_lopsided_ss_part1(double * dst)
 inline void sub_lopsided_ss(int& sst,double *src,double* dst,double *SoA)
 {
 		sub_fderivs_shc(sst,src,Mh_ fh,Mh_ fxx,Mh_ fxy,Mh_ fxz,SoA);
-		cudaThreadSynchronize();
+		cudaDeviceSynchronize();
 		sub_lopsided_ss_part1<<<GRID_DIM,BLOCK_DIM>>>(dst);
-		cudaThreadSynchronize();
+		cudaDeviceSynchronize();
 }
 
 __global__ void sub_kodis_sh_part1(double *f,double *fh,double *f_rhs)
@@ -590,11 +592,11 @@ inline void sub_kodis_ss(int &sst,double *f,double *fh,double *f_rhs,double *SoA
 	}
 			//compare_result_gpu(10,f,h_3D_SIZE[0]);
 	sub_symmetry_bd_ss(3,f,fh,SoA1);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(0,fh,h_3D_SIZE[3]);
 			
 	sub_kodis_sh_part1<<<GRID_DIM,BLOCK_DIM>>>(f,fh,f_rhs);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(1,f_rhs,h_3D_SIZE[0]);
 }
 
@@ -1699,7 +1701,7 @@ void destroy_meta(Meta *meta,Metass *metass)
 	if(Msh_ gzz) cudaFree(Msh_ gzz);
 	
 #if (GAUGE == 2 || GAUGE == 3 || GAUGE == 4 || GAUGE == 5 || GAUGE == 6 || GAUGE == 7)
-	if(Mh_ reta) CUDA_SAFE_CALL(cudaFree(Mh_ reta));
+	if(Mh_ reta) cudaFree(Mh_ reta);
 
 #endif	
 	
@@ -1895,7 +1897,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 
 	//1.2 local Data
 	cudaMalloc((void**)&(Mh_ gxx), matrix_size * sizeof(double));
-	CUDA_SAFE_CALL( cudaMalloc((void**)&(Mh_ gyy), matrix_size * sizeof(double)));
+	cudaMalloc((void**)&(Mh_ gyy), matrix_size * sizeof(double));
 	cudaMalloc((void**)&(Mh_ gzz), matrix_size * sizeof(double));
 	cudaMalloc((void**)&(Mh_ chix), matrix_size * sizeof(double));
 	cudaMalloc((void**)&(Mh_ chiy), matrix_size * sizeof(double));
@@ -2160,7 +2162,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	
 	double tmp_con2 = 1/Mass[0] - tmp_con;
 	cudaMemcpyToSymbol(C1, &tmp_con2, sizeof(double));
-	double tmp_con2 = 1/Mass[1] - tmp_con;
+	tmp_con2 = 1/Mass[1] - tmp_con;
 	cudaMemcpyToSymbol(C2, &tmp_con2, sizeof(double));
 		
 
@@ -2233,7 +2235,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
   		if((sst == 2 || sst == 4) && abs[1] < dYh) 
   		{
   			ijkmin_h[1] = -2; 
-  			ijkmin_h[1] = -3;
+  			ijkmin3_h[1] = -3;
   		}
   		if((sst == 3 || sst == 5) && abs_Y_ex2 < dYh)
   		{
@@ -2287,13 +2289,13 @@ int gpu_rhs_ss(RHS_SS_PARA)
 
 
 #ifdef TIMING1
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	gettimeofday(&tv2, NULL);
    	cout<<"TIME USED"<<TimeBetween(tv1, tv2)<<endl; 
 #endif	
 	//cout<<"GPU meta data ready.\n";
 	
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 
 
 //-------------get device info-------------------------------------
@@ -2306,7 +2308,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	//sub_enforce_ga(matrix_size);
 	//4.1-----compute rhs---------
 	compute_rhs_ss_part1<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 
 	sub_fderivs_shc(sst,Mh_ betax,Mh_ fh,Mh_ betaxx,Mh_ betaxy,Mh_ betaxz,ass);
 	sub_fderivs_shc(sst,Mh_ betay,Mh_ fh,Mh_ betayx,Mh_ betayy,Mh_ betayz,sas);
@@ -2322,7 +2324,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	sub_fderivs_shc(sst,Mh_ gyz,Mh_ fh,Mh_ gyzx,Mh_ gyzy,Mh_ gyzz, saa);
 	
 	compute_rhs_ss_part2<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	sub_fdderivs_shc(sst,Mh_ betax,Mh_ fh,Mh_ gxxx,Mh_ gxyx,Mh_ gxzx,Mh_ gyyx,Mh_ gyzx,Mh_ gzzx,ass);
 	sub_fdderivs_shc(sst,Mh_ betay,Mh_ fh,Mh_ gxxy,Mh_ gxyy,Mh_ gxzy,Mh_ gyyy,Mh_ gyzy,Mh_ gzzy,sas);
@@ -2332,7 +2334,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	sub_fderivs_shc( sst,Mh_ Gamz, Mh_ fh,Mh_ Gamzx, Mh_ Gamzy, Mh_ Gamzz,ssa);
 	
 	compute_rhs_ss_part3<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	computeRicci_ss(sst,Mh_ dxx,Mh_ Rxx,sss, meta);
 	computeRicci_ss(sst,Mh_ dyy,Mh_ Ryy,sss, meta);
@@ -2340,25 +2342,25 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	computeRicci_ss(sst,Mh_ gxy,Mh_ Rxy,aas, meta);
 	computeRicci_ss(sst,Mh_ gxz,Mh_ Rxz,asa, meta);
 	computeRicci_ss(sst,Mh_ gyz,Mh_ Ryz,saa, meta);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	compute_rhs_ss_part4<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	sub_fdderivs_shc(sst,Mh_ chi,Mh_ fh,Mh_ fxx,Mh_ fxy,Mh_ fxz,Mh_ fyy,Mh_ fyz,Mh_ fzz,sss);
 	
-	//cudaThreadSynchronize();
+	//cudaDeviceSynchronize();
 	//compare_result_gpu(0,Mh_ chi,h_3D_SIZE[0]);
 	//compare_result_gpu(1,Mh_ chi,h_3D_SIZE[0]);
 	//compare_result_gpu(2,Mh_ fyz,h_3D_SIZE[0]);
 	
 	compute_rhs_ss_part5<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	sub_fdderivs_shc(sst,Mh_ Lap,Mh_ fh,Mh_ fxx,Mh_ fxy,Mh_ fxz,Mh_ fyy,Mh_ fyz,Mh_ fzz,sss);
 	
 	compute_rhs_ss_part6<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 #if (GAUGE == 2 || GAUGE == 3 || GAUGE == 4 || GAUGE == 5)
 	sub_fderivs_shc(sst,Mh_ chi,Mh_ fh, Mh_ dtSfx_rhs, Mh_ dtSfy_rhs, Mh_ dtSfz_rhs,sss);
@@ -2423,7 +2425,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	}
 	if(co == 0){
 		compute_rhs_ss_part7<<<GRID_DIM,BLOCK_DIM>>>();
-		cudaThreadSynchronize();
+		cudaDeviceSynchronize();
 
 		sub_fderivs_shc(sst,Mh_ Axx,Mh_ fh,Mh_ gxxx,Mh_ gxxy,Mh_ gxxz,sss);
 		sub_fderivs_shc(sst,Mh_ Axy,Mh_ fh,Mh_ gxyx,Mh_ gxyy,Mh_ gxyz,aas);
@@ -2432,7 +2434,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 		sub_fderivs_shc(sst,Mh_ Ayz,Mh_ fh,Mh_ gyzx,Mh_ gyzy,Mh_ gyzz,saa);
 		sub_fderivs_shc(sst,Mh_ Azz,Mh_ fh,Mh_ gzzx,Mh_ gzzy,Mh_ gzzz,sss);
 		compute_rhs_ss_part8<<<GRID_DIM,BLOCK_DIM>>>();
-		cudaThreadSynchronize();
+		cudaDeviceSynchronize();
 	}
 	
 #if (ABV == 1)
@@ -2512,7 +2514,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	//test kodis
 	//sub_kodis_sh(sst,Msh_ drhodx,Mh_ fh2,Msh_ drhody,sss);
 #ifdef TIMING
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	gettimeofday(&tv2, NULL);
    	cout<<"MPI rank is: "<<mpi_rank<<" GPU TIME is"<<TimeBetween(tv1, tv2)<<" (s)."<<endl; 
 #endif
@@ -2522,4 +2524,55 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	return 0;//TODO return
 }
 
+#if (ABEtype == 2)
+// Z4C Shell GPU: calls BSSN gpu_rhs_ss with trKd=trK+2*TZ, then applies
+// TZ_rhs = alpn1*Hcon/2 and constraint damping on CPU.
+int gpu_rhs_z4c_ss(Z4C_SS_PARA)
+{
+    int matrix_size = ex[0] * ex[1] * ex[2];
+    double k1 = 0.02, k2 = 0.0;
+
+    double *trKd_host = new double[matrix_size];
+    for (int _i = 0; _i < matrix_size; _i++)
+        trKd_host[_i] = trK[_i] + 2.0 * TZ[_i];
+
+    int result = gpu_rhs_ss(calledby, mpi_rank,
+        ex, T, crho, sigma, R, X, Y, Z,
+        drhodx, drhody, drhodz, dsigmadx, dsigmady, dsigmadz,
+        dRdx, dRdy, dRdz,
+        drhodxx, drhodxy, drhodxz, drhodyy, drhodyz, drhodzz,
+        dsigmadxx, dsigmadxy, dsigmadxz, dsigmadyy, dsigmadyz, dsigmadzz,
+        dRdxx, dRdxy, dRdxz, dRdyy, dRdyz, dRdzz,
+        chi, trKd_host, dxx, gxy, gxz, dyy, gyz, dzz,
+        Axx, Axy, Axz, Ayy, Ayz, Azz,
+        Gamx, Gamy, Gamz,
+        Lap, betax, betay, betaz,
+        dtSfx, dtSfy, dtSfz,
+        chi_rhs, trK_rhs,
+        gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
+        Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
+        Gamx_rhs, Gamy_rhs, Gamz_rhs,
+        Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
+        dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
+        rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz,
+        Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+        Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+        Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+        Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+        ham_Res, movx_Res, movy_Res, movz_Res,
+        Gmx_Res, Gmy_Res, Gmz_Res,
+        Symmetry, Lev, eps, sst, co);
+    delete[] trKd_host;
+    if (result != 0) return result;
+
+    for (int _i = 0; _i < matrix_size; _i++) {
+        double alp = Lap[_i] + 1.0;
+        TZ_rhs[_i] = alp * ham_Res[_i] * 0.5;
+        TZ_rhs[_i]  -= alp * (2.0 + k2) * k1 * TZ[_i];
+        trK_rhs[_i] += alp * k1 * (1.0 - k2) * TZ[_i];
+    }
+    return 0;
+}
+#endif // ABEtype == 2
+
 #endif //WithShell

AMSS_NCKU_source/bssn_rhs.h

@@ -32,6 +32,19 @@
 #define f_compute_rhs_Z4c_ss compute_rhs_z4c_ss_
 #define f_compute_constraint_fr compute_constraint_fr_
 #endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+        void f_bssn_rhs_kernel_timing_reset();
+        int f_bssn_rhs_kernel_timing_bucket_count();
+        const double *f_bssn_rhs_kernel_timing_local_seconds();
+        const char *f_bssn_rhs_kernel_timing_label(int);
+#ifdef __cplusplus
+}
+#endif
+
 extern "C"
 {
         int f_compute_rhs_bssn(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z

AMSS_NCKU_source/bssn_rhs_c.C

@@ -2,12 +2,88 @@
 #include "bssn_rhs.h"
 #include "share_func.h"
 #include "tool.h"
+#include <time.h>
 // 0-based i,j,k
 // #define IDX_F(i,j,k,nx,ny) ((i) + (j)*(nx) + (k)*(nx)*(ny))
 // ex(1)=nx, ex(2)=ny, ex(3)=nz
 
 // 用法：a[ IDX_F(i,j,k,nx,ny) ]
 
+#ifndef BSSN_KERNEL_FINE_TIMING
+#define BSSN_KERNEL_FINE_TIMING 0
+#endif
+
+#if BSSN_KERNEL_FINE_TIMING
+namespace rhs_kernel_timing
+{
+    enum Bucket
+    {
+        KB_SETUP_DERIVS = 0,
+        KB_GEOM_GAMMA,
+        KB_RICCI_METRIC,
+        KB_CHI_LAPSE,
+        KB_AIJ_TRK_GAUGE,
+        KB_KO_CONSTRAINT,
+        KB_COUNT
+    };
+
+    static double local_bucket_seconds[KB_COUNT];
+
+    static const char *bucket_labels[KB_COUNT] =
+    {
+        "setup_derivs",
+        "geom_gamma",
+        "ricci_metric",
+        "chi_lapse",
+        "aij_trk_gauge",
+        "ko_constraint"
+    };
+
+    static inline double now_seconds()
+    {
+        struct timespec ts;
+        clock_gettime(CLOCK_MONOTONIC, &ts);
+        return double(ts.tv_sec) + 1.0e-9 * double(ts.tv_nsec);
+    }
+}
+
+extern "C" void f_bssn_rhs_kernel_timing_reset()
+{
+    for (int i = 0; i < rhs_kernel_timing::KB_COUNT; ++i)
+        rhs_kernel_timing::local_bucket_seconds[i] = 0.0;
+}
+
+extern "C" int f_bssn_rhs_kernel_timing_bucket_count()
+{
+    return rhs_kernel_timing::KB_COUNT;
+}
+
+extern "C" const double *f_bssn_rhs_kernel_timing_local_seconds()
+{
+    return rhs_kernel_timing::local_bucket_seconds;
+}
+
+extern "C" const char *f_bssn_rhs_kernel_timing_label(int bucket_index)
+{
+    if (bucket_index < 0 || bucket_index >= rhs_kernel_timing::KB_COUNT)
+        return "unknown";
+    return rhs_kernel_timing::bucket_labels[bucket_index];
+}
+
+#define RHS_KERNEL_TIMER_DECL(var_name) const double var_name = rhs_kernel_timing::now_seconds()
+#define RHS_KERNEL_TIMER_ADD(bucket_name, var_name) \
+    rhs_kernel_timing::local_bucket_seconds[int(rhs_kernel_timing::bucket_name)] += \
+        rhs_kernel_timing::now_seconds() - (var_name)
+#else
+extern "C" void f_bssn_rhs_kernel_timing_reset() {}
+extern "C" int f_bssn_rhs_kernel_timing_bucket_count() { return 0; }
+extern "C" const double *f_bssn_rhs_kernel_timing_local_seconds() { return 0; }
+extern "C" const char *f_bssn_rhs_kernel_timing_label(int) { return "disabled"; }
+
+#define RHS_KERNEL_TIMER_DECL(var_name)
+#define RHS_KERNEL_TIMER_ADD(bucket_name, var_name)
+#endif
+
 // C function that calculates the right-hand side for BSSN equations
 int f_compute_rhs_bssn(int *ex, double &T, 
                        double *X, double *Y, double *Z,
@@ -102,6 +178,7 @@ int f_compute_rhs_bssn(int *ex, double &T,
     dY = Y[1] - Y[0];
     dZ = Z[1] - Z[0];
 
+        RHS_KERNEL_TIMER_DECL(timer_setup_derivs);
         // 1ms //
         for(int i=0;i<all;i+=1){
             alpn1[i] = Lap[i] + 1.0;
@@ -141,6 +218,8 @@ int f_compute_rhs_bssn(int *ex, double &T,
                         (dxx[i] + ONE) * betaxz[i] + gxy[i] * betayz[i] + gyz[i] * betayx[i] 
                         + (dzz[i] + ONE) * betazx[i] - gxz[i] * betayy[i];                  
         }
+        RHS_KERNEL_TIMER_ADD(KB_SETUP_DERIVS, timer_setup_derivs);
+        RHS_KERNEL_TIMER_DECL(timer_geom_gamma);
         // Fused: inverse metric + Gamma constraint + Christoffel (3 loops -> 1)
         for(int i=0;i<all;i+=1){
             double det = (dxx[i] + ONE) * (dyy[i] + ONE) * (dzz[i] + ONE) + gxy[i] * gyz[i] * gxz[i] + gxz[i] * gxy[i] * gyz[i] -
@@ -283,9 +362,6 @@ int f_compute_rhs_bssn(int *ex, double &T,
                 + ( gupxy[i]*gupyz[i] + gupyy[i]*gupxz[i] ) * Axy[i]
                 + ( gupxy[i]*gupzz[i] + gupyz[i]*gupxz[i] ) * Axz[i]
                 + ( gupyy[i]*gupzz[i] + gupyz[i]*gupyz[i] ) * Ayz[i];
-            Rxx[i] = axx; Ryy[i] = ayy; Rzz[i] = azz;
-            Rxy[i] = axy; Rxz[i] = axz; Ryz[i] = ayz;
-
             Gamx_rhs[i] = - TWO * ( Lapx[i]*axx + Lapy[i]*axy + Lapz[i]*axz ) +
                 TWO * alpn1[i] * (
                 -F3o2/chin1[i] * ( chix[i]*axx + chiy[i]*axy + chiz[i]*axz ) -
@@ -315,6 +391,8 @@ int f_compute_rhs_bssn(int *ex, double &T,
                             + TWO * ( Gamzxy[i]*axy + Gamzxz[i]*axz + Gamzyz[i]*ayz )
                         );
         }
+        RHS_KERNEL_TIMER_ADD(KB_GEOM_GAMMA, timer_geom_gamma);
+        RHS_KERNEL_TIMER_DECL(timer_ricci_metric);
         // 22.3ms //
         fdderivs(ex,betax,gxxx,gxyx,gxzx,gyyx,gyzx,gzzx,
         X,Y,Z,ANTI,SYM, SYM ,Symmetry,Lev);
@@ -332,7 +410,6 @@ int f_compute_rhs_bssn(int *ex, double &T,
             double lfxx = gxxx[i] + gxyy[i] + gxzz[i];
             double lfxy = gxyx[i] + gyyy[i] + gyzz[i];
             double lfxz = gxzx[i] + gyzy[i] + gzzz[i];
-            fxx[i] = lfxx; fxy[i] = lfxy; fxz[i] = lfxz;
 
             double gxa = gupxx[i]*Gamxxx[i] + gupyy[i]*Gamxyy[i] + gupzz[i]*Gamxzz[i]
                     + TWO * ( gupxy[i]*Gamxxy[i] + gupxz[i]*Gamxxz[i] + gupyz[i]*Gamxyz[i] );
@@ -686,69 +763,74 @@ int f_compute_rhs_bssn(int *ex, double &T,
                     + Gamxyz[i] * gzzx[i] + Gamyyz[i] * gzzy[i] + Gamzyz[i] * gzzz[i]
                 );
         }
+        RHS_KERNEL_TIMER_ADD(KB_RICCI_METRIC, timer_ricci_metric);
 
+        RHS_KERNEL_TIMER_DECL(timer_chi_lapse);
         // 22.3ms //
             fdderivs(ex,chi,fxx,fxy,fxz,fyy,fyz,fzz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev);
 
         // 7ms //
         for (int i=0;i<all;i+=1) {
-            fxx[i] = fxx[i] - Gamxxx[i] * chix[i] - Gamyxx[i] * chiy[i] - Gamzxx[i] * chiz[i];
-            fxy[i] = fxy[i] - Gamxxy[i] * chix[i] - Gamyxy[i] * chiy[i] - Gamzxy[i] * chiz[i];
-            fxz[i] = fxz[i] - Gamxxz[i] * chix[i] - Gamyxz[i] * chiy[i] - Gamzxz[i] * chiz[i];
-            fyy[i] = fyy[i] - Gamxyy[i] * chix[i] - Gamyyy[i] * chiy[i] - Gamzyy[i] * chiz[i];
-            fyz[i] = fyz[i] - Gamxyz[i] * chix[i] - Gamyyz[i] * chiy[i] - Gamzyz[i] * chiz[i];
-            fzz[i] = fzz[i] - Gamxzz[i] * chix[i] - Gamyzz[i] * chiy[i] - Gamzzz[i] * chiz[i];
-            f[i] =
-                gupxx[i] * (fxx[i] - (F3o2 / chin1[i]) * chix[i] * chix[i])
-                + gupyy[i] * (fyy[i] - (F3o2 / chin1[i]) * chiy[i] * chiy[i])
-                + gupzz[i] * (fzz[i] - (F3o2 / chin1[i]) * chiz[i] * chiz[i])
-                + TWO * gupxy[i] * (fxy[i] - (F3o2 / chin1[i]) * chix[i] * chiy[i])
-                + TWO * gupxz[i] * (fxz[i] - (F3o2 / chin1[i]) * chix[i] * chiz[i])
-                + TWO * gupyz[i] * (fyz[i] - (F3o2 / chin1[i]) * chiy[i] * chiz[i]);
-            Rxx[i] = Rxx[i] + ( fxx[i] - (chix[i] * chix[i]) / (chin1[i] * TWO) + (dxx[i] + ONE) * f[i] ) / (chin1[i] * TWO);
-            Ryy[i] = Ryy[i] + ( fyy[i] - (chiy[i] * chiy[i]) / (chin1[i] * TWO) + (dyy[i] + ONE) * f[i] ) / (chin1[i] * TWO);
-            Rzz[i] = Rzz[i] + ( fzz[i] - (chiz[i] * chiz[i]) / (chin1[i] * TWO) + (dzz[i] + ONE) * f[i] ) / (chin1[i] * TWO);
+            const double inv_chin1 = ONE / chin1[i];
+            const double half_inv_chin1 = HALF * inv_chin1;
+            const double scaled_inv = F3o2 * inv_chin1;
+            const double cxx = fxx[i] - Gamxxx[i] * chix[i] - Gamyxx[i] * chiy[i] - Gamzxx[i] * chiz[i];
+            const double cxy = fxy[i] - Gamxxy[i] * chix[i] - Gamyxy[i] * chiy[i] - Gamzxy[i] * chiz[i];
+            const double cxz = fxz[i] - Gamxxz[i] * chix[i] - Gamyxz[i] * chiy[i] - Gamzxz[i] * chiz[i];
+            const double cyy = fyy[i] - Gamxyy[i] * chix[i] - Gamyyy[i] * chiy[i] - Gamzyy[i] * chiz[i];
+            const double cyz = fyz[i] - Gamxyz[i] * chix[i] - Gamyyz[i] * chiy[i] - Gamzyz[i] * chiz[i];
+            const double czz = fzz[i] - Gamxzz[i] * chix[i] - Gamyzz[i] * chiy[i] - Gamzzz[i] * chiz[i];
+            const double ricci_chi =
+                gupxx[i] * (cxx - scaled_inv * chix[i] * chix[i])
+                + gupyy[i] * (cyy - scaled_inv * chiy[i] * chiy[i])
+                + gupzz[i] * (czz - scaled_inv * chiz[i] * chiz[i])
+                + TWO * gupxy[i] * (cxy - scaled_inv * chix[i] * chiy[i])
+                + TWO * gupxz[i] * (cxz - scaled_inv * chix[i] * chiz[i])
+                + TWO * gupyz[i] * (cyz - scaled_inv * chiy[i] * chiz[i]);
+            f[i] = ricci_chi;
+            Rxx[i] = Rxx[i] + ( cxx - half_inv_chin1 * chix[i] * chix[i] + (dxx[i] + ONE) * ricci_chi ) * half_inv_chin1;
+            Ryy[i] = Ryy[i] + ( cyy - half_inv_chin1 * chiy[i] * chiy[i] + (dyy[i] + ONE) * ricci_chi ) * half_inv_chin1;
+            Rzz[i] = Rzz[i] + ( czz - half_inv_chin1 * chiz[i] * chiz[i] + (dzz[i] + ONE) * ricci_chi ) * half_inv_chin1;
 
-            Rxy[i] = Rxy[i] + ( fxy[i] - (chix[i] * chiy[i]) / (chin1[i] * TWO) + gxy[i] * f[i] ) / (chin1[i] * TWO);
-            Rxz[i] = Rxz[i] + ( fxz[i] - (chix[i] * chiz[i]) / (chin1[i] * TWO) + gxz[i] * f[i] ) / (chin1[i] * TWO);
-            Ryz[i] = Ryz[i] + ( fyz[i] - (chiy[i] * chiz[i]) / (chin1[i] * TWO) + gyz[i] * f[i] ) / (chin1[i] * TWO);
+            Rxy[i] = Rxy[i] + ( cxy - half_inv_chin1 * chix[i] * chiy[i] + gxy[i] * ricci_chi ) * half_inv_chin1;
+            Rxz[i] = Rxz[i] + ( cxz - half_inv_chin1 * chix[i] * chiz[i] + gxz[i] * ricci_chi ) * half_inv_chin1;
+            Ryz[i] = Ryz[i] + ( cyz - half_inv_chin1 * chiy[i] * chiz[i] + gyz[i] * ricci_chi ) * half_inv_chin1;
         }
 
         // 24ms //
         fdderivs(ex,Lap,fxx,fxy,fxz,fyy,fyz,fzz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev);
-        fderivs(ex,chi,dtSfx_rhs,dtSfy_rhs,dtSfz_rhs,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev);
 
         // 6ms //
         for (int i=0;i<all;i+=1) {
-            /* gxxx,gxxy,gxxz (这里是“升指标后的chi导数/chi”那类量，你沿用原变量名即可) */
-            gxxx[i] = (gupxx[i] * chix[i] + gupxy[i] * chiy[i] + gupxz[i] * chiz[i]) / chin1[i];
-            gxxy[i] = (gupxy[i] * chix[i] + gupyy[i] * chiy[i] + gupyz[i] * chiz[i]) / chin1[i];
-            gxxz[i] = (gupxz[i] * chix[i] + gupyz[i] * chiy[i] + gupzz[i] * chiz[i]) / chin1[i];
+            const double inv_chin1 = ONE / chin1[i];
+            const double gchi_x = (gupxx[i] * chix[i] + gupxy[i] * chiy[i] + gupxz[i] * chiz[i]) * inv_chin1;
+            const double gchi_y = (gupxy[i] * chix[i] + gupyy[i] * chiy[i] + gupyz[i] * chiz[i]) * inv_chin1;
+            const double gchi_z = (gupxz[i] * chix[i] + gupyz[i] * chiy[i] + gupzz[i] * chiz[i]) * inv_chin1;
 
             /* Christoffel 修正项 */
-            Gamxxx[i] = Gamxxx[i] - ( ((chix[i] + chix[i]) / chin1[i]) - (dxx[i] + ONE) * gxxx[i] ) * HALF;
-            Gamyxx[i] = Gamyxx[i] - ( 0.0           - (dxx[i] + ONE) * gxxy[i] ) * HALF;  /* 原式只有 -gxx*gxxy */
-            Gamzxx[i] = Gamzxx[i] - ( 0.0           - (dxx[i] + ONE) * gxxz[i] ) * HALF;
+            Gamxxx[i] = Gamxxx[i] - ( ((chix[i] + chix[i]) * inv_chin1) - (dxx[i] + ONE) * gchi_x ) * HALF;
+            Gamyxx[i] = Gamyxx[i] - ( 0.0           - (dxx[i] + ONE) * gchi_y ) * HALF;  /* 原式只有 -gxx*gxxy */
+            Gamzxx[i] = Gamzxx[i] - ( 0.0           - (dxx[i] + ONE) * gchi_z ) * HALF;
 
-            Gamxyy[i] = Gamxyy[i] - ( 0.0           - (dyy[i] + ONE) * gxxx[i] ) * HALF;
-            Gamyyy[i] = Gamyyy[i] - ( ((chiy[i] + chiy[i]) / chin1[i]) - (dyy[i] + ONE) * gxxy[i] ) * HALF;
-            Gamzyy[i] = Gamzyy[i] - ( 0.0           - (dyy[i] + ONE) * gxxz[i] ) * HALF;
+            Gamxyy[i] = Gamxyy[i] - ( 0.0           - (dyy[i] + ONE) * gchi_x ) * HALF;
+            Gamyyy[i] = Gamyyy[i] - ( ((chiy[i] + chiy[i]) * inv_chin1) - (dyy[i] + ONE) * gchi_y ) * HALF;
+            Gamzyy[i] = Gamzyy[i] - ( 0.0           - (dyy[i] + ONE) * gchi_z ) * HALF;
 
-            Gamxzz[i] = Gamxzz[i] - ( 0.0           - (dzz[i] + ONE) * gxxx[i] ) * HALF;
-            Gamyzz[i] = Gamyzz[i] - ( 0.0           - (dzz[i] + ONE) * gxxy[i] ) * HALF;
-            Gamzzz[i] = Gamzzz[i] - ( ((chiz[i] + chiz[i]) / chin1[i]) - (dzz[i] + ONE) * gxxz[i] ) * HALF;
+            Gamxzz[i] = Gamxzz[i] - ( 0.0           - (dzz[i] + ONE) * gchi_x ) * HALF;
+            Gamyzz[i] = Gamyzz[i] - ( 0.0           - (dzz[i] + ONE) * gchi_y ) * HALF;
+            Gamzzz[i] = Gamzzz[i] - ( ((chiz[i] + chiz[i]) * inv_chin1) - (dzz[i] + ONE) * gchi_z ) * HALF;
 
-            Gamxxy[i] = Gamxxy[i] - ( ( chiy[i] / chin1[i])      - gxy[i] * gxxx[i] ) * HALF;
-            Gamyxy[i] = Gamyxy[i] - ( ( chix[i] / chin1[i])      - gxy[i] * gxxy[i] ) * HALF;
-            Gamzxy[i] = Gamzxy[i] - ( 0.0           - gxy[i] * gxxz[i] ) * HALF;
+            Gamxxy[i] = Gamxxy[i] - ( ( chiy[i] * inv_chin1)      - gxy[i] * gchi_x ) * HALF;
+            Gamyxy[i] = Gamyxy[i] - ( ( chix[i] * inv_chin1)      - gxy[i] * gchi_y ) * HALF;
+            Gamzxy[i] = Gamzxy[i] - ( 0.0           - gxy[i] * gchi_z ) * HALF;
 
-            Gamxxz[i] = Gamxxz[i] - ( ( chiz[i] / chin1[i])      - gxz[i] * gxxx[i] ) * HALF;
-            Gamyxz[i] = Gamyxz[i] - ( 0.0           - gxz[i] * gxxy[i] ) * HALF;
-            Gamzxz[i] = Gamzxz[i] - ( ( chix[i] / chin1[i])      - gxz[i] * gxxz[i] ) * HALF;
+            Gamxxz[i] = Gamxxz[i] - ( ( chiz[i] * inv_chin1)      - gxz[i] * gchi_x ) * HALF;
+            Gamyxz[i] = Gamyxz[i] - ( 0.0           - gxz[i] * gchi_y ) * HALF;
+            Gamzxz[i] = Gamzxz[i] - ( ( chix[i] * inv_chin1)      - gxz[i] * gchi_z ) * HALF;
 
-            Gamxyz[i] = Gamxyz[i] - ( 0.0           - gyz[i] * gxxx[i] ) * HALF;
-            Gamyyz[i] = Gamyyz[i] - ( ( chiz[i] / chin1[i])      - gyz[i] * gxxy[i] ) * HALF;
-            Gamzyz[i] = Gamzyz[i] - ( ( chiy[i] / chin1[i])      - gyz[i] * gxxz[i] ) * HALF;
+            Gamxyz[i] = Gamxyz[i] - ( 0.0           - gyz[i] * gchi_x ) * HALF;
+            Gamyyz[i] = Gamyyz[i] - ( ( chiz[i] * inv_chin1)      - gyz[i] * gchi_y ) * HALF;
+            Gamzyz[i] = Gamzyz[i] - ( ( chiy[i] * inv_chin1)      - gyz[i] * gchi_z ) * HALF;
 
             /* fxx..fyz 修正：减去 Γ * ∂Lap */
             fxx[i] = fxx[i] - Gamxxx[i] * Lapx[i] - Gamyxx[i] * Lapy[i] - Gamzxx[i] * Lapz[i];
@@ -762,6 +844,8 @@ int f_compute_rhs_bssn(int *ex, double &T,
             trK_rhs[i] =  gupxx[i] * fxx[i] + gupyy[i] * fyy[i] + gupzz[i] * fzz[i]
                     +  TWO * ( gupxy[i] * fxy[i] + gupxz[i] * fxz[i] + gupyz[i] * fyz[i] );
         }
+        RHS_KERNEL_TIMER_ADD(KB_CHI_LAPSE, timer_chi_lapse);
+        RHS_KERNEL_TIMER_DECL(timer_aij_trk_gauge);
         // 2.5ms //
         for (int i=0;i<all;i+=1) {
             const double divb = betaxx[i] + betayy[i] + betazz[i];
@@ -1014,12 +1098,12 @@ int f_compute_rhs_bssn(int *ex, double &T,
             betaz_rhs[i] = FF * dtSfz[i];
 
             reta[i] =
-                gupxx[i] * dtSfx_rhs[i] * dtSfx_rhs[i]
-                + gupyy[i] * dtSfy_rhs[i] * dtSfy_rhs[i]
-                + gupzz[i] * dtSfz_rhs[i] * dtSfz_rhs[i]
-                + TWO * ( gupxy[i] * dtSfx_rhs[i] * dtSfy_rhs[i]
-                        + gupxz[i] * dtSfx_rhs[i] * dtSfz_rhs[i]
-                        + gupyz[i] * dtSfy_rhs[i] * dtSfz_rhs[i] );
+                gupxx[i] * chix[i] * chix[i]
+                + gupyy[i] * chiy[i] * chiy[i]
+                + gupzz[i] * chiz[i] * chiz[i]
+                + TWO * ( gupxy[i] * chix[i] * chiy[i]
+                        + gupxz[i] * chix[i] * chiz[i]
+                        + gupyz[i] * chiy[i] * chiz[i] );
 
             #if (GAUGE == 2)
             reta[i] = 1.31 / 2.0 * sqrt( reta[i] / chin1[i] ) / pow( (ONE - sqrt(chin1[i])), 2.0 );
@@ -1032,12 +1116,12 @@ int f_compute_rhs_bssn(int *ex, double &T,
             dtSfz_rhs[i] = Gamz_rhs[i] - reta[i] * dtSfz[i];
             #elif (GAUGE == 4 || GAUGE == 5)
             reta[i] =
-                gupxx[i] * dtSfx_rhs[i] * dtSfx_rhs[i]
-                + gupyy[i] * dtSfy_rhs[i] * dtSfy_rhs[i]
-                + gupzz[i] * dtSfz_rhs[i] * dtSfz_rhs[i]
-                + TWO * ( gupxy[i] * dtSfx_rhs[i] * dtSfy_rhs[i]
-                        + gupxz[i] * dtSfx_rhs[i] * dtSfz_rhs[i]
-                        + gupyz[i] * dtSfy_rhs[i] * dtSfz_rhs[i] );
+                gupxx[i] * chix[i] * chix[i]
+                + gupyy[i] * chiy[i] * chiy[i]
+                + gupzz[i] * chiz[i] * chiz[i]
+                + TWO * ( gupxy[i] * chix[i] * chiy[i]
+                        + gupxz[i] * chix[i] * chiz[i]
+                        + gupyz[i] * chiy[i] * chiz[i] );
 
             #if (GAUGE == 4)
             reta[i] = 1.31 / 2.0 * sqrt( reta[i] / chin1[i] ) / pow( (ONE - sqrt(chin1[i])), 2.0 );
@@ -1062,6 +1146,8 @@ int f_compute_rhs_bssn(int *ex, double &T,
             dtSfz_rhs[i] = Gamz_rhs[i] - reta[i] * dtSfz[i];
             #endif
         }
+        RHS_KERNEL_TIMER_ADD(KB_AIJ_TRK_GAUGE, timer_aij_trk_gauge);
+        RHS_KERNEL_TIMER_DECL(timer_ko_constraint);
         // advection + KO dissipation with shared symmetry buffer
         lopsided_kodis(ex,X,Y,Z,dxx,gxx_rhs,betax,betay,betaz,Symmetry,SSS,eps);
         lopsided_kodis(ex,X,Y,Z,Gamz,Gamz_rhs,betax,betay,betaz,Symmetry,SSA,eps);
@@ -1193,6 +1279,7 @@ int f_compute_rhs_bssn(int *ex, double &T,
                 movz_Res[i] = movz_Res[i] - F2o3*Kz[i] - F8*PI*Sz[i];
             }
         }
+        RHS_KERNEL_TIMER_ADD(KB_KO_CONSTRAINT, timer_ko_constraint);
 
 
             

AMSS_NCKU_source/bssn_rhs_cuda.h

@@ -0,0 +1,413 @@
+#ifndef BSSN_RHS_CUDA_H
+#define BSSN_RHS_CUDA_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    BSSN_CUDA_STATE_COUNT = 24,
+    BSSN_ESCALAR_CUDA_STATE_COUNT = 26,
+    BSSN_EM_CUDA_STATE_COUNT = 32,
+    BSSN_EM_CUDA_SOURCE_COUNT = 4,
+    BSSN_CUDA_MATTER_COUNT = 10
+};
+
+int f_compute_rhs_bssn(int *ex, double &T,
+                       double *X, double *Y, double *Z,
+                       double *chi, double *trK,
+                       double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+                       double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+                       double *Gamx, double *Gamy, double *Gamz,
+                       double *Lap, double *betax, double *betay, double *betaz,
+                       double *dtSfx, double *dtSfy, double *dtSfz,
+                       double *chi_rhs, double *trK_rhs,
+                       double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+                       double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+                       double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+                       double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+                       double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+                       double *rho, double *Sx, double *Sy, double *Sz,
+                       double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+                       double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+                       double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+                       double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+                       double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+                       double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
+                       double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
+                       int &Symmetry, int &Lev, double &eps, int &co);
+
+int bssn_cuda_rk4_substep(void *block_tag,
+                          int *ex, double *X, double *Y, double *Z,
+                          double **state_host_in,
+                          double **state_host_out,
+                          double **matter_host,
+                          const double *propspeed,
+                          const double *soa_flat,
+                          const double *bbox,
+                          double &dT,
+                          double &T,
+                          int &RK4,
+                          int &apply_bam_bc,
+                          int &Symmetry,
+                          int &Lev,
+                          double &eps,
+                          int &co,
+                          int &use_zero_matter,
+                          int &keep_resident_state,
+                          int &apply_enforce_ga,
+                          double &chitiny);
+
+int bssn_escalar_cuda_rk4_substep(void *block_tag,
+                                  int *ex, double *X, double *Y, double *Z,
+                                  double **state_host_in,
+                                  double **state_host_out,
+                                  const double *propspeed,
+                                  const double *soa_flat,
+                                  const double *bbox,
+                                  double &dT,
+                                  double &T,
+                                  int &RK4,
+                                  int &apply_bam_bc,
+                                  int &Symmetry,
+	                                  int &Lev,
+	                                  double &eps,
+	                                  int &co,
+	                                  int &keep_resident_state,
+	                                  int &apply_enforce_ga,
+	                                  double &chitiny);
+
+int bssn_escalar_cuda_compute_constraints(int *ex, double *X, double *Y, double *Z,
+                                          double **state_host_in,
+                                          double **constraint_host_out,
+                                          int &Symmetry,
+                                          int &Lev,
+                                          double &eps);
+
+int bssn_em_cuda_rk4_substep(void *block_tag,
+                             int *ex, double *X, double *Y, double *Z,
+                             double **state_host_in,
+                             double **state_host_out,
+                             double **source_host,
+                             const double *propspeed,
+                             const double *soa_flat,
+                             const double *bbox,
+                             double &dT,
+                             double &T,
+                             int &RK4,
+                             int &apply_bam_bc,
+                             int &Symmetry,
+                             int &Lev,
+                             double &eps,
+                             int &co,
+                             int &keep_resident_state,
+                             int &apply_enforce_ga,
+                             double &chitiny);
+
+int bssn_em_cuda_resident_zero_fast_state(void *block_tag);
+
+int bssn_cuda_copy_state_region_to_host(void *block_tag,
+                                        int state_index,
+                                        double *host_state,
+                                        int *ex,
+                                        int i0, int j0, int k0,
+                                        int sx, int sy, int sz);
+
+int bssn_cuda_copy_state_region_from_host(void *block_tag,
+                                          int state_index,
+                                          double *host_state,
+                                          int *ex,
+                                          int i0, int j0, int k0,
+                                          int sx, int sy, int sz);
+
+int bssn_cuda_download_resident_state(void *block_tag,
+                                      int *ex,
+                                      double **state_host_out);
+
+int bssn_escalar_cuda_download_resident_state(void *block_tag,
+                                              int *ex,
+                                              double **state_host_out);
+
+int bssn_cuda_upload_resident_state_count(void *block_tag,
+                                          int *ex,
+                                          double **state_host_in,
+                                          int state_count);
+
+int bssn_escalar_cuda_upload_resident_state(void *block_tag,
+                                            int *ex,
+                                            double **state_host_in);
+
+int bssn_cuda_keep_only_resident_state_count(void *block_tag,
+                                             int *ex,
+                                             double **state_host_key,
+                                             int state_count);
+
+int bssn_escalar_cuda_keep_only_resident_state(void *block_tag,
+                                               int *ex,
+                                               double **state_host_key);
+
+int bssn_cuda_download_resident_state_count_if_present(void *block_tag,
+                                                       int *ex,
+                                                       double **state_host_out,
+                                                       int state_count);
+
+int bssn_cuda_download_resident_state_if_present(void *block_tag,
+                                                int *ex,
+                                                double **state_host_out);
+
+int bssn_cuda_download_constraint_outputs(int *ex,
+                                          double **constraint_host_out);
+
+int bssn_cuda_pack_state_region_to_host_buffer(void *block_tag,
+                                               int state_index,
+                                               double *host_buffer,
+                                               int *ex,
+                                               int i0, int j0, int k0,
+                                               int sx, int sy, int sz);
+
+int bssn_cuda_interp_state_point3(void *block_tag,
+                                  int *ex,
+                                  int state0,
+                                  int state1,
+                                  int state2,
+                                  double x0,
+                                  double y0,
+                                  double z0,
+                                  double dx,
+                                  double dy,
+                                  double dz,
+                                  double px,
+                                  double py,
+                                  double pz,
+                                  int ordn,
+                                  int symmetry,
+                                  double **state_host_key,
+                                  const double *soa3,
+                                  double *out3);
+
+int bssn_cuda_interp_host_two_fields(void *block_tag,
+                                     int *ex,
+                                     double *field0,
+                                     double *field1,
+                                     double x0,
+                                     double y0,
+                                     double z0,
+                                     double dx,
+                                     double dy,
+                                     double dz,
+                                     const double *px,
+                                     const double *py,
+                                     const double *pz,
+                                     int npoints,
+                                     int ordn,
+                                     int symmetry,
+                                     const double *soa6,
+                                     double *out_interleaved);
+
+int bssn_cuda_unpack_state_region_from_host_buffer(void *block_tag,
+                                                   int state_index,
+                                                   double *host_buffer,
+                                                   int *ex,
+                                                   int i0, int j0, int k0,
+                                                   int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_region_from_host_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  int state_index,
+                                                                  double *host_buffer,
+                                                                  int *ex,
+                                                                  int i0, int j0, int k0,
+                                                                  int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_host_buffer(void *block_tag,
+                                              int state_count,
+                                              double *host_buffer,
+                                              int *ex,
+                                              int i0, int j0, int k0,
+                                              int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                             double **state_host_key,
+                                                             int state_count,
+                                                             double *host_buffer,
+                                                             int *ex,
+                                                             int i0, int j0, int k0,
+                                                             int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *host_buffer,
+                                                                 int *ex,
+                                                                 int i0, int j0, int k0,
+                                                                 int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_device_buffer(void *block_tag,
+                                                int state_count,
+                                                double *device_buffer,
+                                                int *ex,
+                                                int i0, int j0, int k0,
+                                                int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                               double **state_host_key,
+                                                               int state_count,
+                                                               double *device_buffer,
+                                                               int *ex,
+                                                               int i0, int j0, int k0,
+                                                               int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
+                                                    int state_count,
+                                                    double *device_buffer,
+                                                    int *ex,
+                                                    int i0, int j0, int k0,
+                                                    int sx, int sy, int sz);
+
+int bssn_cuda_unpack_state_batch_from_device_buffer_for_host_views(void *block_tag,
+                                                                   double **state_host_key,
+                                                                   int state_count,
+                                                                   double *device_buffer,
+                                                                   int *ex,
+                                                                   int i0, int j0, int k0,
+                                                                   int sx, int sy, int sz);
+
+int bssn_cuda_pack_state_segments_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int segment_count,
+                                                   const int *segment_meta);
+
+int bssn_cuda_pack_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int segment_count,
+                                                                  const int *segment_meta);
+
+int bssn_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
+                                                       int state_count,
+                                                       double *device_buffer,
+                                                       int *ex,
+                                                       int segment_count,
+                                                       const int *segment_meta);
+
+int bssn_cuda_unpack_state_segments_from_device_buffer_for_host_views(void *block_tag,
+                                                                      double **state_host_key,
+                                                                      int state_count,
+                                                                      double *device_buffer,
+                                                                      int *ex,
+                                                                      int segment_count,
+                                                                      const int *segment_meta);
+
+int bssn_cuda_restrict_state_segments_to_device_buffer(void *block_tag,
+                                                       int state_count,
+                                                       double *device_buffer,
+                                                       int *ex,
+                                                       int segment_count,
+                                                       const int *segment_meta);
+
+int bssn_cuda_restrict_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                      double **state_host_key,
+                                                                      int state_count,
+                                                                      double *device_buffer,
+                                                                      int *ex,
+                                                                      int segment_count,
+                                                                      const int *segment_meta,
+                                                                      const double *state_soa);
+
+int bssn_cuda_prolong_state_segments_to_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta);
+
+int bssn_cuda_prolong_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta,
+                                                                     const double *state_soa);
+
+int bssn_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                    int state_count,
+                                                    double *device_buffer,
+                                                    int *ex,
+                                                    int sx, int sy, int sz,
+                                                    int fi0, int fj0, int fk0);
+
+int bssn_cuda_restrict_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                   double **state_host_key,
+                                                                   int state_count,
+                                                                   double *device_buffer,
+                                                                   int *ex,
+                                                                   int sx, int sy, int sz,
+                                                                   int fi0, int fj0, int fk0,
+                                                                   const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int ii0, int jj0, int kk0,
+                                                   int lbc_i, int lbc_j, int lbc_k);
+
+int bssn_cuda_prolong_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int sx, int sy, int sz,
+                                                                  int ii0, int jj0, int kk0,
+                                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                                  const double *state_soa);
+
+int bssn_cuda_download_state_subset(void *block_tag,
+                                    int *ex,
+                                    int subset_count,
+                                    const int *state_indices,
+                                    double **state_host_out);
+
+int bssn_cuda_upload_state_subset(void *block_tag,
+                                  int *ex,
+                                  int subset_count,
+                                  const int *state_indices,
+                                  double **state_host_in);
+
+int bssn_cuda_prepare_inter_time_level(void *block_tag,
+                                       int *ex,
+                                       int state_count,
+                                       double **src1_host_key,
+                                       double **src2_host_key,
+                                       double **src3_host_key,
+                                       double **dst_host_key,
+                                       int source_count,
+                                       int tindex);
+
+int bssn_cuda_has_resident_state(void *block_tag);
+
+void bssn_cuda_release_step_ctx(void *block_tag);
+
+#ifdef __cplusplus
+}
+// C++-only helpers declared for derived equation classes (Z4C, etc.)
+// Defined in bssn_class.C. Requires MyList, Patch, var from including TU.
+bool bssn_cuda_use_resident_sync(int lev);
+void bssn_cuda_download_level_state_if_present(MyList<Patch> *PatL, MyList<var> *vars, int myrank);
+#endif
+
+#endif

AMSS_NCKU_source/checkpoint.C

@@ -76,8 +76,11 @@ checkpoint::checkpoint(bool checked, const char fname[], int myrank) : filename(
 
 	I_Print = (myrank == 0);
 
-	int i = strlen(fname);
-	filename = new char[i+30];
+	size_t filename_len = out_dir.size() + strlen(fname) + 32;
+#ifdef CHECKDETAIL
+	filename_len += 32;
+#endif
+	filename = new char[filename_len];
 	// cout << filename << endl;
 	// cout << i << endl;
 
@@ -103,7 +106,7 @@ checkpoint::checkpoint(bool checked, const char fname[], int myrank) : filename(
 checkpoint::~checkpoint()
 {
 	CheckList->clearList();
-	if (I_Print)
+	if (filename)
 		delete[] filename;
 }
 
@@ -136,7 +139,7 @@ void checkpoint::writecheck_cgh(double time, cgh *GH)
 	if (I_Print)
 	{
 		// char fname[50];
-		char fname[50+50]; 
+		char fname[4096]; 
 		sprintf(fname, "%s_cgh.CHK", filename);
 
 		outfile.open(fname, ios::out | ios::trunc);
@@ -195,7 +198,7 @@ void checkpoint::readcheck_cgh(double &time, cgh *GH, int myrank, int nprocs, in
 	int DIM = dim;
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_cgh.CHK", filename);
 
 	infile.open(fname);
@@ -297,7 +300,7 @@ void checkpoint::writecheck_sh(double time, ShellPatch *SH)
 
 	if (I_Print)
 	{
-		char fname[50];
+		char fname[4096];
 		sprintf(fname, "%s_sh.CHK", filename);
 
 		outfile.open(fname, ios::out | ios::trunc);
@@ -335,7 +338,7 @@ void checkpoint::readcheck_sh(ShellPatch *SH, int myrank)
 	int DIM = dim;
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_sh.CHK", filename);
 
 	infile.open(fname);
@@ -390,7 +393,7 @@ void checkpoint::write_Black_Hole_position(int BH_num_input, int BH_num, double
 
 	if (I_Print)
 	{
-		char fname[50];
+		char fname[4096];
 		sprintf(fname, "%s_BHp.CHK", filename);
 
 		outfile.open(fname, ios::out | ios::trunc);
@@ -417,7 +420,7 @@ void checkpoint::read_Black_Hole_position(int &BH_num_input, int &BH_num, double
 {
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_BHp.CHK", filename);
 
 	infile.open(fname);
@@ -461,7 +464,7 @@ void checkpoint::write_bssn(double LastDump, double Last2dDump, double LastAnas)
 
 	if (I_Print)
 	{
-		char fname[50];
+		char fname[4096];
 		sprintf(fname, "%s_bssn.CHK", filename);
 
 		outfile.open(fname, ios::out | ios::trunc);
@@ -481,7 +484,7 @@ void checkpoint::read_bssn(double &LastDump, double &Last2dDump, double &LastAna
 {
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_bssn.CHK", filename);
 
 	infile.open(fname);
@@ -506,7 +509,7 @@ void checkpoint::write_bssn(double LastDump, double Last2dDump, double LastAnas)
 	ofstream outfile;
 
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_bssn.CHK", filename);
 
 	outfile.open(fname, ios::out | ios::trunc);
@@ -527,7 +530,7 @@ void checkpoint::read_bssn(double &LastDump, double &Last2dDump, double &LastAna
 {
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_bssn.CHK", filename);
 
 	infile.open(fname);
@@ -551,7 +554,7 @@ void checkpoint::write_Black_Hole_position(int BH_num_input, int BH_num, double
 	ofstream outfile;
 
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_BHp.CHK", filename);
 
 	outfile.open(fname, ios::out | ios::trunc);
@@ -581,7 +584,7 @@ void checkpoint::read_Black_Hole_position(int &BH_num_input, int &BH_num, double
 {
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_BHp.CHK", filename);
 
 	infile.open(fname);
@@ -628,7 +631,7 @@ void checkpoint::writecheck_cgh(double time, cgh *GH)
 	ofstream outfile;
 
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_cgh.CHK", filename);
 
 	outfile.open(fname, ios::out | ios::trunc);
@@ -738,7 +741,7 @@ void checkpoint::readcheck_cgh(double &time, cgh *GH, int myrank, int nprocs, in
 	int DIM = dim;
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_cgh.CHK", filename);
 
 	infile.open(fname);

AMSS_NCKU_source/fd_cuda_helpers.cuh

@@ -0,0 +1,412 @@
+#ifndef AMSS_NCKU_FD_CUDA_HELPERS_CUH
+#define AMSS_NCKU_FD_CUDA_HELPERS_CUH
+
+#ifndef ghost_width
+#error "ghost_width must be defined before including fd_cuda_helpers.cuh"
+#endif
+
+#if ghost_width < 2 || ghost_width > 5
+#error "CUDA finite-difference helpers support ghost_width 2..5"
+#endif
+
+#define AMSS_FD_CENTER_RADIUS (ghost_width - 1)
+#define AMSS_FD_LK_RADIUS     (ghost_width)
+
+__device__ __forceinline__ int fd_axis_radius(int qF, int qminF, int qmaxF)
+{
+#if AMSS_FD_CENTER_RADIUS >= 4
+    if (qF - 4 >= qminF && qF + 4 <= qmaxF) return 4;
+#endif
+#if AMSS_FD_CENTER_RADIUS >= 3
+    if (qF - 3 >= qminF && qF + 3 <= qmaxF) return 3;
+#endif
+#if AMSS_FD_CENTER_RADIUS >= 2
+    if (qF - 2 >= qminF && qF + 2 <= qmaxF) return 2;
+#endif
+    if (qF - 1 >= qminF && qF + 1 <= qmaxF) return 1;
+    return 0;
+}
+
+__device__ __forceinline__ int fd_common_radius(int iF, int jF, int kF,
+                                                int iminF, int jminF, int kminF,
+                                                int imaxF, int jmaxF, int kmaxF)
+{
+    int r = fd_axis_radius(iF, iminF, imaxF);
+    const int ry = fd_axis_radius(jF, jminF, jmaxF);
+    const int rz = fd_axis_radius(kF, kminF, kmaxF);
+    if (ry < r) r = ry;
+    if (rz < r) r = rz;
+    return r;
+}
+
+__device__ __forceinline__ double fd_first_coef(int r, int off)
+{
+    switch (r) {
+    case 1:
+        if (off == -1) return -1.0;
+        if (off ==  1) return  1.0;
+        return 0.0;
+    case 2:
+        if (off == -2) return  1.0;
+        if (off == -1) return -8.0;
+        if (off ==  1) return  8.0;
+        if (off ==  2) return -1.0;
+        return 0.0;
+    case 3:
+        if (off == -3) return  -1.0;
+        if (off == -2) return   9.0;
+        if (off == -1) return -45.0;
+        if (off ==  1) return  45.0;
+        if (off ==  2) return  -9.0;
+        if (off ==  3) return   1.0;
+        return 0.0;
+    case 4:
+        if (off == -4) return    3.0;
+        if (off == -3) return  -32.0;
+        if (off == -2) return  168.0;
+        if (off == -1) return -672.0;
+        if (off ==  1) return  672.0;
+        if (off ==  2) return -168.0;
+        if (off ==  3) return   32.0;
+        if (off ==  4) return   -3.0;
+        return 0.0;
+    default:
+        return 0.0;
+    }
+}
+
+__device__ __forceinline__ double fd_second_coef(int r, int off)
+{
+    switch (r) {
+    case 1:
+        if (off == -1) return  1.0;
+        if (off ==  0) return -2.0;
+        if (off ==  1) return  1.0;
+        return 0.0;
+    case 2:
+        if (off == -2) return  -1.0;
+        if (off == -1) return  16.0;
+        if (off ==  0) return -30.0;
+        if (off ==  1) return  16.0;
+        if (off ==  2) return  -1.0;
+        return 0.0;
+    case 3:
+        if (off == -3) return    2.0;
+        if (off == -2) return  -27.0;
+        if (off == -1) return  270.0;
+        if (off ==  0) return -490.0;
+        if (off ==  1) return  270.0;
+        if (off ==  2) return  -27.0;
+        if (off ==  3) return    2.0;
+        return 0.0;
+    case 4:
+        if (off == -4) return     -9.0;
+        if (off == -3) return    128.0;
+        if (off == -2) return  -1008.0;
+        if (off == -1) return   8064.0;
+        if (off ==  0) return -14350.0;
+        if (off ==  1) return   8064.0;
+        if (off ==  2) return  -1008.0;
+        if (off ==  3) return    128.0;
+        if (off ==  4) return     -9.0;
+        return 0.0;
+    default:
+        return 0.0;
+    }
+}
+
+__device__ __forceinline__ double fd_first_denom(int r)
+{
+    return (r == 4) ? 840.0 : ((r == 3) ? 60.0 : ((r == 2) ? 12.0 : 2.0));
+}
+
+__device__ __forceinline__ double fd_second_denom(int r)
+{
+    return (r == 4) ? 5040.0 : ((r == 3) ? 180.0 : ((r == 2) ? 12.0 : 1.0));
+}
+
+__device__ __forceinline__ double fd_fetch_axis(const double *src,
+                                                int iF, int jF, int kF,
+                                                int axis, int off,
+                                                int SoA0, int SoA1, int SoA2)
+{
+    if (axis == 0) iF += off;
+    else if (axis == 1) jF += off;
+    else kF += off;
+    return fetch_sym_ord2_direct(src, iF, jF, kF, SoA0, SoA1, SoA2);
+}
+
+__device__ __forceinline__ double fd_fetch_axis2(const double *src,
+                                                 int iF, int jF, int kF,
+                                                 int axis_a, int off_a,
+                                                 int axis_b, int off_b,
+                                                 int SoA0, int SoA1, int SoA2)
+{
+    if (axis_a == 0) iF += off_a;
+    else if (axis_a == 1) jF += off_a;
+    else kF += off_a;
+    if (axis_b == 0) iF += off_b;
+    else if (axis_b == 1) jF += off_b;
+    else kF += off_b;
+    return fetch_sym_ord2_direct(src, iF, jF, kF, SoA0, SoA1, SoA2);
+}
+
+__device__ __forceinline__ double fd_first_axis_radius(const double *src,
+                                                       int iF, int jF, int kF,
+                                                       int axis, int r, double h,
+                                                       int SoA0, int SoA1, int SoA2)
+{
+    if (r <= 0) return 0.0;
+    double s = 0.0;
+#pragma unroll
+    for (int off = -4; off <= 4; ++off) {
+        const double c = fd_first_coef(r, off);
+        if (c != 0.0) {
+            s += c * fd_fetch_axis(src, iF, jF, kF, axis, off, SoA0, SoA1, SoA2);
+        }
+    }
+    return s / (fd_first_denom(r) * h);
+}
+
+__device__ __forceinline__ double fd_second_axis_radius(const double *src,
+                                                        int iF, int jF, int kF,
+                                                        int axis, int r, double h,
+                                                        int SoA0, int SoA1, int SoA2)
+{
+    if (r <= 0) return 0.0;
+    double s = 0.0;
+#pragma unroll
+    for (int off = -4; off <= 4; ++off) {
+        const double c = fd_second_coef(r, off);
+        if (c != 0.0) {
+            s += c * fd_fetch_axis(src, iF, jF, kF, axis, off, SoA0, SoA1, SoA2);
+        }
+    }
+    return s / (fd_second_denom(r) * h * h);
+}
+
+__device__ __forceinline__ double fd_mixed_axis_radius(const double *src,
+                                                       int iF, int jF, int kF,
+                                                       int axis_a, int r_a, double h_a,
+                                                       int axis_b, int r_b, double h_b,
+                                                       int SoA0, int SoA1, int SoA2)
+{
+    if (r_a <= 0 || r_b <= 0) return 0.0;
+    double s = 0.0;
+#pragma unroll
+    for (int off_a = -4; off_a <= 4; ++off_a) {
+        const double ca = fd_first_coef(r_a, off_a);
+        if (ca == 0.0) continue;
+#pragma unroll
+        for (int off_b = -4; off_b <= 4; ++off_b) {
+            const double cb = fd_first_coef(r_b, off_b);
+            if (cb != 0.0) {
+                s += ca * cb * fd_fetch_axis2(src, iF, jF, kF, axis_a, off_a,
+                                              axis_b, off_b, SoA0, SoA1, SoA2);
+            }
+        }
+    }
+    return s / (fd_first_denom(r_a) * fd_first_denom(r_b) * h_a * h_b);
+}
+
+__device__ __forceinline__ void fd_compute_first3(const double *src,
+                                                  int iF, int jF, int kF,
+                                                  int iminF, int jminF, int kminF,
+                                                  int imaxF, int jmaxF, int kmaxF,
+                                                  int SoA0, int SoA1, int SoA2,
+                                                  double &fx, double &fy, double &fz)
+{
+#if ghost_width == 3
+    const int r = fd_common_radius(iF, jF, kF, iminF, jminF, kminF, imaxF, jmaxF, kmaxF);
+    fx = fd_first_axis_radius(src, iF, jF, kF, 0, r, d_gp.dX, SoA0, SoA1, SoA2);
+    fy = fd_first_axis_radius(src, iF, jF, kF, 1, r, d_gp.dY, SoA0, SoA1, SoA2);
+    fz = fd_first_axis_radius(src, iF, jF, kF, 2, r, d_gp.dZ, SoA0, SoA1, SoA2);
+#else
+    fx = fd_first_axis_radius(src, iF, jF, kF, 0, fd_axis_radius(iF, iminF, imaxF),
+                              d_gp.dX, SoA0, SoA1, SoA2);
+    fy = fd_first_axis_radius(src, iF, jF, kF, 1, fd_axis_radius(jF, jminF, jmaxF),
+                              d_gp.dY, SoA0, SoA1, SoA2);
+    fz = fd_first_axis_radius(src, iF, jF, kF, 2, fd_axis_radius(kF, kminF, kmaxF),
+                              d_gp.dZ, SoA0, SoA1, SoA2);
+#endif
+}
+
+__device__ __forceinline__ void fd_compute_second6(const double *src,
+                                                   int iF, int jF, int kF,
+                                                   int iminF, int jminF, int kminF,
+                                                   int imaxF, int jmaxF, int kmaxF,
+                                                   int SoA0, int SoA1, int SoA2,
+                                                   double &fxx, double &fxy, double &fxz,
+                                                   double &fyy, double &fyz, double &fzz)
+{
+#if ghost_width == 3
+    const int r = fd_common_radius(iF, jF, kF, iminF, jminF, kminF, imaxF, jmaxF, kmaxF);
+    const int rx = r, ry = r, rz = r;
+#else
+    const int rx = fd_axis_radius(iF, iminF, imaxF);
+    const int ry = fd_axis_radius(jF, jminF, jmaxF);
+    const int rz = fd_axis_radius(kF, kminF, kmaxF);
+#endif
+    fxx = fd_second_axis_radius(src, iF, jF, kF, 0, rx, d_gp.dX, SoA0, SoA1, SoA2);
+    fyy = fd_second_axis_radius(src, iF, jF, kF, 1, ry, d_gp.dY, SoA0, SoA1, SoA2);
+    fzz = fd_second_axis_radius(src, iF, jF, kF, 2, rz, d_gp.dZ, SoA0, SoA1, SoA2);
+    fxy = fd_mixed_axis_radius(src, iF, jF, kF, 0, rx, d_gp.dX, 1, ry, d_gp.dY, SoA0, SoA1, SoA2);
+    fxz = fd_mixed_axis_radius(src, iF, jF, kF, 0, rx, d_gp.dX, 2, rz, d_gp.dZ, SoA0, SoA1, SoA2);
+    fyz = fd_mixed_axis_radius(src, iF, jF, kF, 1, ry, d_gp.dY, 2, rz, d_gp.dZ, SoA0, SoA1, SoA2);
+}
+
+__device__ __forceinline__ bool fd_lop_fits(int qF, int qminF, int qmaxF,
+                                            int dir, int lo, int hi)
+{
+    for (int off = lo; off <= hi; ++off) {
+        const int q = qF + dir * off;
+        if (q < qminF || q > qmaxF) return false;
+    }
+    return true;
+}
+
+__device__ __forceinline__ double fd_lop_fetch_sum(const double *src,
+                                                   int iF, int jF, int kF,
+                                                   int axis, int dir,
+                                                   const double *coef,
+                                                   int lo, int hi,
+                                                   int SoA0, int SoA1, int SoA2)
+{
+    double s = 0.0;
+    for (int off = lo; off <= hi; ++off) {
+        const double c = coef[off - lo];
+        if (c != 0.0) {
+            s += c * fd_fetch_axis(src, iF, jF, kF, axis, dir * off, SoA0, SoA1, SoA2);
+        }
+    }
+    return s;
+}
+
+__device__ __forceinline__ double fd_lopsided_axis(const double *src,
+                                                   int iF, int jF, int kF,
+                                                   int axis, double speed,
+                                                   int qF, int qminF, int qmaxF,
+                                                   double h,
+                                                   int SoA0, int SoA1, int SoA2)
+{
+    if (speed == 0.0) return 0.0;
+    const int dir = (speed > 0.0) ? 1 : -1;
+    const double mag = (speed > 0.0) ? speed : -speed;
+
+#if ghost_width == 2
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, 0, 2)) {
+        const double c[] = {-3.0, 4.0, -1.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, 0, 2, SoA0, SoA1, SoA2) / (2.0 * h);
+    }
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, 0, 1)) {
+        const double c[] = {-1.0, 1.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, 0, 1, SoA0, SoA1, SoA2) / (2.0 * h);
+    }
+    return 0.0;
+#elif ghost_width == 3
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, -1, 3)) {
+        const double c[] = {-3.0, -10.0, 18.0, -6.0, 1.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, -1, 3, SoA0, SoA1, SoA2) / (12.0 * h);
+    }
+    const int r = fd_axis_radius(qF, qminF, qmaxF);
+    return speed * fd_first_axis_radius(src, iF, jF, kF, axis, r, h, SoA0, SoA1, SoA2);
+#elif ghost_width == 4
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, -2, 4)) {
+        const double c[] = {2.0, -24.0, -35.0, 80.0, -30.0, 8.0, -1.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, -2, 4, SoA0, SoA1, SoA2) / (60.0 * h);
+    }
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, -1, 5)) {
+        const double c[] = {-10.0, -77.0, 150.0, -100.0, 50.0, -15.0, 2.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, -1, 5, SoA0, SoA1, SoA2) / (60.0 * h);
+    }
+    const int r = fd_axis_radius(qF, qminF, qmaxF);
+    return speed * fd_first_axis_radius(src, iF, jF, kF, axis, r, h, SoA0, SoA1, SoA2);
+#else
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, -3, 5)) {
+        const double c[] = {-5.0, 60.0, -420.0, -378.0, 1050.0, -420.0, 140.0, -30.0, 3.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, -3, 5, SoA0, SoA1, SoA2) / (840.0 * h);
+    }
+    const int r = fd_axis_radius(qF, qminF, qmaxF);
+    return speed * fd_first_axis_radius(src, iF, jF, kF, axis, r, h, SoA0, SoA1, SoA2);
+#endif
+}
+
+__device__ __forceinline__ double fd_ko_coef(int r, int off)
+{
+    const int a = off < 0 ? -off : off;
+    if (r == 2) {
+        if (a == 0) return 6.0;
+        if (a == 1) return -4.0;
+        if (a == 2) return 1.0;
+    } else if (r == 3) {
+        if (a == 0) return -20.0;
+        if (a == 1) return 15.0;
+        if (a == 2) return -6.0;
+        if (a == 3) return 1.0;
+    } else if (r == 4) {
+        if (a == 0) return 70.0;
+        if (a == 1) return -56.0;
+        if (a == 2) return 28.0;
+        if (a == 3) return -8.0;
+        if (a == 4) return 1.0;
+    } else if (r == 5) {
+        if (a == 0) return -252.0;
+        if (a == 1) return 210.0;
+        if (a == 2) return -120.0;
+        if (a == 3) return 45.0;
+        if (a == 4) return -10.0;
+        if (a == 5) return 1.0;
+    }
+    return 0.0;
+}
+
+__device__ __forceinline__ double fd_ko_axis(const double *src,
+                                             int iF, int jF, int kF,
+                                             int axis, int r,
+                                             int SoA0, int SoA1, int SoA2)
+{
+    double s = 0.0;
+#pragma unroll
+    for (int off = -5; off <= 5; ++off) {
+        if (off < -r || off > r) continue;
+        const double c = fd_ko_coef(r, off);
+        if (c != 0.0) {
+            s += c * fd_fetch_axis(src, iF, jF, kF, axis, off, SoA0, SoA1, SoA2);
+        }
+    }
+    return s;
+}
+
+__device__ __forceinline__ double fd_ko_term(const double *src,
+                                             int iF, int jF, int kF,
+                                             int iminF, int jminF, int kminF,
+                                             int imaxF, int jmaxF, int kmaxF,
+                                             double eps_val,
+                                             int SoA0, int SoA1, int SoA2)
+{
+    const int r = AMSS_FD_LK_RADIUS;
+    if (eps_val <= 0.0) return 0.0;
+#if ghost_width >= 4
+    if (iF - r <= iminF || iF + r >= imaxF ||
+        jF - r <= jminF || jF + r >= jmaxF ||
+        kF - r <= kminF || kF + r >= kmaxF) {
+        return 0.0;
+    }
+#else
+    if (iF - r < iminF || iF + r > imaxF ||
+        jF - r < jminF || jF + r > jmaxF ||
+        kF - r < kminF || kF + r > kmaxF) {
+        return 0.0;
+    }
+#endif
+    double cof = 1.0;
+#pragma unroll
+    for (int n = 0; n < 2 * r; ++n) cof *= 2.0;
+    const double sign = (r & 1) ? 1.0 : -1.0;
+    const double dx = fd_ko_axis(src, iF, jF, kF, 0, r, SoA0, SoA1, SoA2);
+    const double dy = fd_ko_axis(src, iF, jF, kF, 1, r, SoA0, SoA1, SoA2);
+    const double dz = fd_ko_axis(src, iF, jF, kF, 2, r, SoA0, SoA1, SoA2);
+    return sign * eps_val * (dx / d_gp.dX + dy / d_gp.dY + dz / d_gp.dZ) / cof;
+}
+
+#endif

AMSS_NCKU_source/fmisc.f90

@@ -1514,6 +1514,81 @@ f_out = f_out*dX*dY*dZ
   return
 
   end subroutine l2normhelper
+!--------------------------------------------------------------------------------------
+  subroutine l2normhelper7(ex, X, Y, Z,xmin,ymin,zmin,xmax,ymax,zmax,&
+                           f1,f2,f3,f4,f5,f6,f7,f_out,gw)
+
+  implicit none
+!~~~~~~> Input parameters:
+  integer,intent(in ):: ex(1:3)
+  real*8, intent(in ):: X(1:ex(1)),Y(1:ex(2)),Z(1:ex(3)),xmin,ymin,zmin,xmax,ymax,zmax
+  integer,intent(in)::gw
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in) :: f1,f2,f3,f4,f5,f6,f7
+  real*8, intent(out) :: f_out(7)
+!~~~~~~> Other variables:
+
+  real*8            :: dX, dY, dZ
+  integer::imin,jmin,kmin
+  integer::imax,jmax,kmax
+  integer::i,j,k
+  real*8 :: s1,s2,s3,s4,s5,s6,s7
+
+  dX = X(2) - X(1)
+  dY = Y(2) - Y(1)
+  dZ = Z(2) - Z(1)
+
+! for ghost zone
+   imin = gw+1
+   jmin = gw+1
+   kmin = gw+1
+
+   imax = ex(1) - gw
+   jmax = ex(2) - gw
+   kmax = ex(3) - gw
+
+!for patch boundary (i.e., not ghost boundary)
+
+if(dabs(X(ex(1))-xmax) < dX) imax = ex(1)
+if(dabs(Y(ex(2))-ymax) < dY) jmax = ex(2)
+if(dabs(Z(ex(3))-zmax) < dZ) kmax = ex(3)
+if(dabs(X(1)-xmin) < dX) imin = 1
+if(dabs(Y(1)-ymin) < dY) jmin = 1
+if(dabs(Z(1)-zmin) < dZ) kmin = 1
+
+  s1 = 0.d0
+  s2 = 0.d0
+  s3 = 0.d0
+  s4 = 0.d0
+  s5 = 0.d0
+  s6 = 0.d0
+  s7 = 0.d0
+
+  do k=kmin,kmax
+    do j=jmin,jmax
+!DIR$ SIMD REDUCTION(+:s1,s2,s3,s4,s5,s6,s7)
+      do i=imin,imax
+        s1 = s1 + f1(i,j,k)*f1(i,j,k)
+        s2 = s2 + f2(i,j,k)*f2(i,j,k)
+        s3 = s3 + f3(i,j,k)*f3(i,j,k)
+        s4 = s4 + f4(i,j,k)*f4(i,j,k)
+        s5 = s5 + f5(i,j,k)*f5(i,j,k)
+        s6 = s6 + f6(i,j,k)*f6(i,j,k)
+        s7 = s7 + f7(i,j,k)*f7(i,j,k)
+      enddo
+    enddo
+  enddo
+
+  f_out(1) = s1*dX*dY*dZ
+  f_out(2) = s2*dX*dY*dZ
+  f_out(3) = s3*dX*dY*dZ
+  f_out(4) = s4*dX*dY*dZ
+  f_out(5) = s5*dX*dY*dZ
+  f_out(6) = s6*dX*dY*dZ
+  f_out(7) = s7*dX*dY*dZ
+
+  return
+
+  end subroutine l2normhelper7
 !--------------------------------------------------------------------------------------
 ! calculate L2norm especially for shell Blocks
   subroutine l2normhelper_sh(ex, X, Y, Z,xmin,ymin,zmin,xmax,ymax,zmax,&

AMSS_NCKU_source/fmisc.h

@@ -13,6 +13,7 @@
 #define f_global_interpind2d global_interpind2d
 #define f_global_interpind1d global_interpind1d
 #define f_l2normhelper l2normhelper
+#define f_l2normhelper7 l2normhelper7
 #define f_l2normhelper_sh l2normhelper_sh
 #define f_l2normhelper_sh_rms l2normhelper_sh_rms
 #define f_average average
@@ -42,6 +43,7 @@
 #define f_global_interpind2d GLOBAL_INTERPIND2D
 #define f_global_interpind1d GLOBAL_INTERPIND1D
 #define f_l2normhelper L2NORMHELPER
+#define f_l2normhelper7 L2NORMHELPER7
 #define f_l2normhelper_sh L2NORMHELPER_SH
 #define f_l2normhelper_sh_rms L2NORMHELPER_SH_RMS
 #define f_average AVERAGE
@@ -71,6 +73,7 @@
 #define f_global_interpind2d global_interpind2d_
 #define f_global_interpind1d global_interpind1d_
 #define f_l2normhelper l2normhelper_
+#define f_l2normhelper7 l2normhelper7_
 #define f_l2normhelper_sh l2normhelper_sh_
 #define f_l2normhelper_sh_rms l2normhelper_sh_rms_
 #define f_average average_
@@ -164,6 +167,15 @@ extern "C"
 						double *, double &, int &);
 }
 
+extern "C"
+{
+	void f_l2normhelper7(int *, double *, double *, double *,
+						 double &, double &, double &,
+						 double &, double &, double &,
+						 double *, double *, double *, double *,
+						 double *, double *, double *, double *, int &);
+}
+
 extern "C"
 {
 	void f_l2normhelper_sh(int *, double *, double *, double *,

AMSS_NCKU_source/gpu_rhsSS_mem.h

@@ -1,6 +1,6 @@
 #ifndef GPU_MEM_H_
 #define GPU_MEM_H_
-#include "macrodef.fh"
+#include "macrodef.h"
 
 #ifdef WithShell
 struct Metass
@@ -48,6 +48,8 @@ struct Meta
 	double * Gamx_rhs,*Gamy_rhs,*Gamz_rhs;//out
 	double * Lap_rhs, *betax_rhs, *betay_rhs, *betaz_rhs;//out
 	double * dtSfx_rhs,*dtSfy_rhs,*dtSfz_rhs;//out
+	double * TZ;                                  //in  (Z4C)
+	double * TZ_rhs;                              //out (Z4C)
 	double * rho,*Sx,*Sy,*Sz			;	//in
 	double * Sxx,*Sxy,*Sxz,*Syy,*Syz,*Szz;	//in
 	
@@ -132,6 +134,8 @@ __constant__ double SYM = 1.0;
 __constant__ double ANTI = -1.0; 
 __constant__ double FF = 0.75;
 __constant__ double eta = 2.0;
+__constant__ double kappa1_c = 0.02;
+__constant__ double kappa2_c = 0.0;
 __constant__ double F1o3;
 __constant__ double F2o3;
 __constant__ double F3o2 = 1.5;

AMSS_NCKU_source/macrodef.h

@@ -29,6 +29,16 @@
 
 #define REGLEV 0
 
+#define BSSN_FINE_TIMING 0
+
+#define BSSN_FINE_TIMING_EVERY 1
+
+#define BSSN_FINE_TIMING_TOPN 8
+
+#define BSSN_KERNEL_FINE_TIMING 0
+
+#define BSSN_ENABLE_STDIN_ABORT_POLL 0
+
 //#define USE_GPU
 
 //#define CHECKDETAIL
@@ -88,6 +98,21 @@
 //     0: for every level;
 //     1: for all
 //
+// define BSSN_FINE_TIMING
+//     enable fine-grained per-timestep timing monitor
+//
+// define BSSN_FINE_TIMING_EVERY
+//     report timing every N coarse timesteps
+//
+// define BSSN_FINE_TIMING_TOPN
+//     number of hottest timing buckets shown in stdout
+//
+// define BSSN_KERNEL_FINE_TIMING
+//     enable split timing inside compute_rhs_bssn
+//
+// define BSSN_ENABLE_STDIN_ABORT_POLL
+//     poll stdin and broadcast abort flag every coarse step
+//
 // define USE_GPU
 //     use gpu or not
 //
@@ -142,4 +167,3 @@
 #define TINY 1e-10
 
 #endif   /* MICRODEF_H */
-

AMSS_NCKU_source/makefile

@@ -13,12 +13,15 @@ POLINT6_FLAG = -DPOLINT6_USE_BARYCENTRIC=$(POLINT6_USE_BARY)
 ##   make PGO_MODE=instrument    -> instrument (Phase 1: collect fresh profile data)
 PROFDATA = /home/$(shell whoami)/AMSS-NCKU/pgo_profile/default.profdata
 
+ifeq ($(TOOLCHAIN),intel)
+OMP_FLAG = -qopenmp
+
 ifeq ($(PGO_MODE),instrument)
-## Phase 1: instrumentation — omit -ipo/-fp-model fast=2 for faster build and numerical stability
+## Intel Phase 1: instrumentation — omit -ipo/-fp-model fast=2 for faster build and numerical stability
 CXXAPPFLAGS = -O3 -xHost -fma -fprofile-instr-generate -ipo \
-              -Dfortran3 -Dnewc -I${MKLROOT}/include $(INTERP_LB_FLAGS)
+              -Dfortran3 -Dnewc $(MKL_INC) $(INTERP_LB_FLAGS)
 f90appflags = -O3 -xHost -fma -fprofile-instr-generate -ipo \
-              -align array64byte -fpp -I${MKLROOT}/include $(POLINT6_FLAG)
+              -align array64byte -fpp $(MKL_INC) $(POLINT6_FLAG)
 else
 ## opt (default): maximum performance with PGO profile data -fprofile-instr-use=$(PROFDATA) \
 ## PGO has been turned off, now tested and found to be negative optimization
@@ -26,9 +29,23 @@ else
 
 
 CXXAPPFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
-              -Dfortran3 -Dnewc -I${MKLROOT}/include $(INTERP_LB_FLAGS)
+              -Dfortran3 -Dnewc $(MKL_INC) $(INTERP_LB_FLAGS)
 f90appflags = -O3 -xHost -fp-model fast=2 -fma -ipo \
-              -align array64byte -fpp -I${MKLROOT}/include $(POLINT6_FLAG)
+              -align array64byte -fpp $(MKL_INC) $(POLINT6_FLAG)
+endif
+
+TP_OPTFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
+              -Dfortran3 -Dnewc $(MKL_INC)
+else
+## NVHPC defaults: mpicc/mpicxx/mpifort wrappers
+## PGO_MODE is ignored in this branch.
+OMP_FLAG = -mp
+CXXAPPFLAGS = -O3 -tp=host -Mcache_align -Mfma \
+              -Dfortran3 -Dnewc $(MKL_INC) $(INTERP_LB_FLAGS)
+f90appflags = -O3 -tp=host -Mcache_align -Mfma -Mpreprocess \
+              $(MKL_INC) $(POLINT6_FLAG)
+TP_OPTFLAGS = -O3 -tp=host -Mcache_align -Mfma \
+              -Dfortran3 -Dnewc $(MKL_INC)
 endif
 
 .SUFFIXES: .o .f90 .C .for .cu
@@ -39,12 +56,28 @@ endif
 .C.o:
 	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
 
+# ShellPatch.C uses OpenMP for setupintintstuff search loops
+ShellPatch.o: ShellPatch.C
+	${CXX} $(CXXAPPFLAGS) $(OMP_FLAG) -c $< $(filein) -o $@
+
 .for.o:
 	$(f77) -c $< -o $@
 
 .cu.o:
 	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
 
+# CUDA rewrite of BSSN RHS (drop-in replacement for bssn_rhs_c + stencil helpers)
+bssn_rhs_cuda.o: bssn_rhs_cuda.cu bssn_rhs.h macrodef.h fd_cuda_helpers.cuh
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+
+# CUDA rewrite of BSSN Shell-Patch RHS (drop-in replacement for bssn_rhs_ss)
+bssn_gpu_rhs_ss.o: bssn_gpu_rhs_ss.cu bssn_gpu.h gpu_rhsSS_mem.h bssn_macro.h macrodef.fh
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+
+# CUDA rewrite of Z4C Cartesian RHS
+z4c_rhs_cuda.o: z4c_rhs_cuda.cu z4c_rhs_cuda.h bssn_rhs.h macrodef.h ricci_gamma.h fd_cuda_helpers.cuh
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+
 # C rewrite of BSSN RHS kernel and helpers
 bssn_rhs_c.o: bssn_rhs_c.C
 	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
@@ -64,40 +97,78 @@ lopsided_c.o: lopsided_c.C
 lopsided_kodis_c.o: lopsided_kodis_c.C
 	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
 
+z4c_rhs_c.o: z4c_rhs_c.C
+	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
+
 #interp_lb_profile.o: interp_lb_profile.C interp_lb_profile.h
 #	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@
 
-## TwoPunctureABE uses fixed optimal flags with its own PGO profile, independent of CXXAPPFLAGS
-TP_PROFDATA = /home/$(shell whoami)/AMSS-NCKU/pgo_profile/TwoPunctureABE.profdata
-TP_OPTFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
-              -fprofile-instr-use=$(TP_PROFDATA) \
-              -Dfortran3 -Dnewc -I${MKLROOT}/include
-
 TwoPunctures.o: TwoPunctures.C
-	${CXX} $(TP_OPTFLAGS) -qopenmp -c $< -o $@
+	${CXX} $(TP_OPTFLAGS) $(OMP_FLAG) -c $< -o $@
 
 TwoPunctureABE.o: TwoPunctureABE.C
-	${CXX} $(TP_OPTFLAGS) -qopenmp -c $< -o $@
+	${CXX} $(TP_OPTFLAGS) $(OMP_FLAG) -c $< -o $@
 
 # Input files
 
+## CUDA BSSN RHS switch
+##   1 : use the rewritten CUDA bssn_rhs backend
+##   0 : keep the normal CPU/Fortran selection below
+USE_CUDA_BSSN ?= 0
+USE_CUDA_Z4C ?= 0
+
+CXXAPPFLAGS += -DUSE_CUDA_BSSN=$(USE_CUDA_BSSN)
+CUDA_APP_FLAGS += -DUSE_CUDA_BSSN=$(USE_CUDA_BSSN)
+CXXAPPFLAGS += -DUSE_CUDA_Z4C=$(USE_CUDA_Z4C)
+CUDA_APP_FLAGS += -DUSE_CUDA_Z4C=$(USE_CUDA_Z4C)
+
 ## Kernel implementation switch (set USE_CXX_KERNELS=0 to fall back to Fortran)
 ifeq ($(USE_CXX_KERNELS),0)
 # Fortran mode: no C rewrite files; bssn_rhs.o is included via F90FILES below
-CFILES =
+CFILES_CPU =
 else
 # C++ mode (default): C rewrite of bssn_rhs and helper kernels
-CFILES = bssn_rhs_c.o fderivs_c.o fdderivs_c.o kodiss_c.o lopsided_c.o lopsided_kodis_c.o
+CFILES_CPU = bssn_rhs_c.o fderivs_c.o fdderivs_c.o kodiss_c.o lopsided_c.o lopsided_kodis_c.o
+endif
+
+CFILES_CUDA_BSSN = bssn_rhs_cuda.o bssn_gpu_rhs_ss.o
+
+ifeq ($(USE_CUDA_BSSN),1)
+CFILES = $(CFILES_CUDA_BSSN)
+else
+CFILES = $(CFILES_CPU)
+endif
+
+ifeq ($(USE_CUDA_Z4C),1)
+CFILES += z4c_rhs_cuda.o
+Z4C_F90_OBJ =
+else ifeq ($(USE_CXX_Z4C_KERNELS),1)
+CFILES += z4c_rhs_c.o
+Z4C_F90_OBJ =
+else
+Z4C_F90_OBJ = Z4c_rhs.o
 endif
 
 ## RK4 kernel switch (independent from USE_CXX_KERNELS)
 ifeq ($(USE_CXX_RK4),1)
-CFILES += rungekutta4_rout_c.o
+RK4_C_OBJ = rungekutta4_rout_c.o
 RK4_F90_OBJ =
 else
+RK4_C_OBJ =
 RK4_F90_OBJ = rungekutta4_rout.o
 endif
 
+CFILES += $(RK4_C_OBJ)
+ABE_CUDA_CFILES = $(CFILES_CUDA_BSSN) z4c_rhs_cuda.o $(RK4_C_OBJ)
+
+ABE_LDLIBS = $(LDLIBS)
+ifeq ($(USE_CUDA_BSSN),1)
+ABE_LDLIBS += -lcudart $(CUDA_LIB_PATH)
+endif
+ifeq ($(USE_CUDA_Z4C),1)
+ABE_LDLIBS += -lcudart $(CUDA_LIB_PATH)
+endif
+
 C++FILES = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
            cgh.o bssn_class.o surface_integral.o ShellPatch.o\
 	   bssnEScalar_class.o perf.o Z4c_class.o NullShellPatch.o\
@@ -105,7 +176,7 @@ C++FILES = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
            Parallel_bam.o scalar_class.o transpbh.o NullShellPatch2.o\
 	   NullShellPatch2_Evo.o writefile_f.o interp_lb_profile.o
 	   
-C++FILES_GPU = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
+#C++FILES_GPU = ABE.o Ansorg.o Block.o misc.o monitor.o Parallel.o MPatch.o var.o\
            cgh.o surface_integral.o ShellPatch.o\
 	   bssnEScalar_class.o perf.o Z4c_class.o NullShellPatch.o\
 	   bssnEM_class.o cpbc_util.o z4c_rhs_point.o checkpoint.o\
@@ -119,7 +190,7 @@ F90FILES_BASE = enforce_algebra.o fmisc.o initial_puncture.o prolongrestrict.o\
 	   lopsidediff.o sommerfeld_rout.o getnp4.o diff_new_sh.o\
 	   shellfunctions.o bssn_rhs_ss.o Set_Rho_ADM.o\
            getnp4EScalar.o bssnEScalar_rhs.o bssn_constraint.o ricci_gamma.o\
-           fadmquantites_bssn.o Z4c_rhs.o Z4c_rhs_ss.o point_diff_new_sh.o\
+           fadmquantites_bssn.o $(Z4C_F90_OBJ) Z4c_rhs_ss.o point_diff_new_sh.o\
 	   cpbc.o getnp4old.o NullEvol.o initial_null.o initial_maxwell.o\
 	   getnpem2.o empart.o NullNews.o fourdcurvature.o\
 	   bssn2adm.o adm_constraint.o adm_ricci_gamma.o\
@@ -143,10 +214,10 @@ initial_guess.o Newton.o Jacobian.o ilucg.o IntPnts0.o IntPnts.o
 
 TwoPunctureFILES = TwoPunctureABE.o TwoPunctures.o 
 
-CUDAFILES = bssn_gpu.o bssn_gpu_rhs_ss.o
+#CUDAFILES = bssn_gpu.o bssn_gpu_rhs_ss.o
 
 # file dependences
-$(C++FILES) $(C++FILES_GPU) $(F90FILES) $(CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.fh
+$(C++FILES) $(C++FILES_GPU) $(F90FILES) $(CFILES) $(ABE_CUDA_CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.fh
 
 $(C++FILES): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
 	     misc.h monitor.h MyList.h Parallel.h MPatch.h prolongrestrict.h\
@@ -157,7 +228,7 @@ $(C++FILES): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
 	     empart.h NullNews.h kodiss.h Parallel_bam.h ricci_gamma.h\
              initial_null2.h NullShellPatch2.h 
              
-$(C++FILES_GPU): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
+#$(C++FILES_GPU): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h\
 	     misc.h monitor.h MyList.h Parallel.h MPatch.h prolongrestrict.h\
 	     rungekutta4_rout.h var.h bssn_rhs.h sommerfeld_rout.h\
 	     cgh.h surface_integral.h ShellPatch.h shellfunctions.h perf.h\
@@ -169,7 +240,7 @@ $(C++FILES_GPU): Block.h enforce_algebra.h fmisc.h initial_puncture.h macrodef.h
              
 $(AHFDOBJS): cctk.h cctk_Config.h cctk_Types.h cctk_Constants.h myglobal.h
 
-$(C++FILES) $(C++FILES_GPU) $(CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.h
+$(C++FILES) $(C++FILES_GPU) $(CFILES) $(ABE_CUDA_CFILES) $(AHFDOBJS) $(CUDAFILES): macrodef.h
 
 TwoPunctureFILES: TwoPunctures.h
 
@@ -179,13 +250,18 @@ misc.o : zbesh.o
 
 # projects
 ABE: $(C++FILES) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS)
-	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(LDLIBS)
+	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(ABE_LDLIBS)
 
-ABEGPU: $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES)
-	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES) $(LDLIBS)
+ABE_CUDA: USE_CUDA_BSSN=1
+ABE_CUDA: USE_CUDA_Z4C=1
+ABE_CUDA: $(C++FILES) $(ABE_CUDA_CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS)
+	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES) $(ABE_CUDA_CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(LDLIBS) -lcudart $(CUDA_LIB_PATH)
+	
+#ABEGPU: $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES)
+#	$(CLINKER) $(CXXAPPFLAGS) -o $@ $(C++FILES_GPU) $(CFILES) $(F90FILES) $(F77FILES) $(AHFDOBJS) $(CUDAFILES) $(LDLIBS)
 
 TwoPunctureABE: $(TwoPunctureFILES)
-	$(CLINKER) $(TP_OPTFLAGS) -qopenmp -o $@ $(TwoPunctureFILES) $(LDLIBS)
+	$(CLINKER) $(TP_OPTFLAGS) $(OMP_FLAG) -o $@ $(TwoPunctureFILES) $(LDLIBS)
 
 clean:
-	rm *.o ABE ABEGPU TwoPunctureABE make.log -f
+	rm *.o ABE ABE_CUDA ABEGPU TwoPunctureABE make.log -f

AMSS_NCKU_source/makefile.inc

@@ -1,28 +1,7 @@
-## GCC version (commented out)
-## filein  = -I/usr/include -I/usr/lib/x86_64-linux-gnu/mpich/include -I/usr/lib/x86_64-linux-gnu/openmpi/lib/ -I/usr/lib/gcc/x86_64-linux-gnu/11/ -I/usr/include/c++/11/
-## filein  = -I/usr/include/ -I/usr/include/openmpi-x86_64/ -I/usr/lib/x86_64-linux-gnu/openmpi/include/ -I/usr/lib/x86_64-linux-gnu/openmpi/lib/ -I/usr/lib/gcc/x86_64-linux-gnu/11/ -I/usr/include/c++/11/
-## LDLIBS  = -L/usr/lib/x86_64-linux-gnu -L/usr/lib64 -L/usr/lib/gcc/x86_64-linux-gnu/11 -lgfortran -lmpi -lgfortran
-
-## Intel oneAPI version with oneMKL (Optimized for performance)
-filein  = -I/usr/include/ -I${MKLROOT}/include
-
-## Using sequential MKL (OpenMP disabled for better single-threaded performance)
-## Added -lifcore for Intel Fortran runtime and -limf for Intel math library
-LDLIBS  = -L${MKLROOT}/lib -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lifcore -limf -lpthread -lm -ldl -liomp5
-
-## Memory allocator switch
-##   1 (default) : link Intel oneTBB allocator (libtbbmalloc)
-##   0           : use system default allocator (ptmalloc)
-USE_TBBMALLOC ?= 1
-TBBMALLOC_SO ?= /home/intel/oneapi/2025.3/lib/libtbbmalloc.so
-ifneq ($(wildcard $(TBBMALLOC_SO)),)
-TBBMALLOC_LIBS = -Wl,--no-as-needed $(TBBMALLOC_SO) -Wl,--as-needed
-else
-TBBMALLOC_LIBS = -Wl,--no-as-needed -ltbbmalloc -Wl,--as-needed
-endif
-ifeq ($(USE_TBBMALLOC),1)
-LDLIBS := $(TBBMALLOC_LIBS) $(LDLIBS)
-endif
+## Toolchain selection
+##   nvhpc : NVIDIA HPC SDK + CUDA-aware MPI (default)
+##   intel : Intel oneAPI toolchain (legacy path)
+TOOLCHAIN ?= intel
 
 ## PGO build mode switch (ABE only; TwoPunctureABE always uses opt flags)
 ##   opt        : (default) maximum performance with PGO profile-guided optimization
@@ -43,23 +22,70 @@ else
 INTERP_LB_FLAGS =
 endif
 
+MKLROOT ?= /home/intel/oneapi/mkl/latest
+MKL_LIBDIR ?= $(MKLROOT)/lib/intel64
+MKL_INC ?= -I$(MKLROOT)/include
+
+NVHPC_ROOT ?= /home/nvidia/hpc_sdk/Linux_x86_64/25.11
+CUDA_HOME  ?= $(NVHPC_ROOT)/cuda
+CUDA_ARCH  ?= sm_80
+
 ## Kernel implementation switch
 ##   1 (default) : use C++ rewrite of bssn_rhs and helper kernels (faster)
 ##   0           : fall back to original Fortran kernels
 USE_CXX_KERNELS ?= 1
 
+## Z4C Cartesian RHS kernel switch
+##   1 (default) : use C++ rewrite of Z4c_rhs (main Cartesian path faster)
+##   0           : use original Fortran Z4c_rhs.o
+USE_CXX_Z4C_KERNELS ?= 1
+
 ## RK4 kernel implementation switch
 ##   1 (default) : use C/C++ rewrite of rungekutta4_rout (for optimization experiments)
 ##   0           : use original Fortran rungekutta4_rout.o
 USE_CXX_RK4 ?= 1
 
+## Memory allocator switch
+##   1 (default) : link Intel oneTBB allocator (libtbbmalloc)
+##   0           : use system default allocator (ptmalloc)
+USE_TBBMALLOC ?= 1
+TBBMALLOC_SO ?= /home/intel/oneapi/2025.3/lib/libtbbmalloc.so
+ifneq ($(wildcard $(TBBMALLOC_SO)),)
+TBBMALLOC_LIBS = -Wl,--no-as-needed $(TBBMALLOC_SO) -Wl,--as-needed
+else
+TBBMALLOC_LIBS = -Wl,--no-as-needed -ltbbmalloc -Wl,--as-needed
+endif
+
+ifeq ($(TOOLCHAIN),intel)
 f90          = ifx
 f77          = ifx
 CXX          = icpx
 CC           = icx
 CLINKER      = mpiicpx
+filein  = -I/usr/include/ $(MKL_INC) -I$(CUDA_HOME)/include
+LDLIBS       = -L$(MKL_LIBDIR) -Wl,-rpath,$(MKL_LIBDIR) \
+               -lmkl_intel_lp64 -lmkl_sequential -lmkl_core \
+               -lifcore -limf -liomp5 -lpthread -lm -ldl \
+               -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcuda -lcudart
+else ifeq ($(TOOLCHAIN),nvhpc)
+f90          = mpifort
+f77          = mpifort
+CXX          = mpicxx
+CC           = mpicc
+CLINKER      = mpicxx
 
-Cu = nvcc
-CUDA_LIB_PATH = -L/usr/lib/cuda/lib64 -I/usr/include -I/usr/lib/cuda/include
-#CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -arch compute_13 -code compute_13,sm_13 -Dfortran3 -Dnewc
-CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc
+filein       = -I/usr/include/ $(MKL_INC) -I$(CUDA_HOME)/include
+LDLIBS       = -L$(MKL_LIBDIR) -Wl,-rpath,$(MKL_LIBDIR) \
+               -lmkl_intel_lp64 -lmkl_sequential -lmkl_core \
+               -lpthread -lm -ldl \
+               -L$(CUDA_HOME)/lib64 -Wl,-rpath,$(CUDA_HOME)/lib64 -lcuda -lcudart \
+               -fortranlibs
+endif
+
+ifeq ($(USE_TBBMALLOC),1)
+LDLIBS := $(TBBMALLOC_LIBS) $(LDLIBS)
+endif
+
+Cu = $(NVHPC_ROOT)/compilers/bin/nvcc
+CUDA_LIB_PATH = -L$(CUDA_HOME)/lib64 -I$(CUDA_HOME)/include
+CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc -arch=$(CUDA_ARCH)

AMSS_NCKU_source/monitor.C

@@ -1,6 +1,7 @@
 
 #ifdef newc
 #include <cstdio>
+#include <sstream>
 using namespace std;
 #else
 #include <stdio.h>
@@ -77,16 +78,17 @@ monitor::monitor(const char fname[], int myrank, string head)
       parameters::str_par.insert(map<string, string>::value_type("output dir", out_dir));
     }
     // considering checkpoint run
-    char filename[50];
-    sprintf(filename, "%s/%s", out_dir.c_str(), fname);
+    string filename = out_dir + "/" + fname;
     int i = 1;
-    while ((access(filename, F_OK)) != -1)
+    while ((access(filename.c_str(), F_OK)) != -1)
     {
-      sprintf(filename, "%s/%d_%s", out_dir.c_str(), i, fname);
+      stringstream ss;
+      ss << out_dir << "/" << i << "_" << fname;
+      filename = ss.str();
       i++;
     }
 
-    outfile.open(filename, ios::trunc);
+    outfile.open(filename.c_str(), ios::trunc);
 
     time_t tnow;
     time(&tnow);
@@ -107,16 +109,17 @@ monitor::monitor(const char fname[], int myrank, const int out_rank, string head
   if (I_Print)
   {
     // considering checkpoint run
-    char filename[50];
-    sprintf(filename, "%s/%s", out_dir.c_str(), fname);
+    string filename = out_dir + "/" + fname;
     int i = 1;
-    while ((access(filename, F_OK)) != -1)
+    while ((access(filename.c_str(), F_OK)) != -1)
     {
-      sprintf(filename, "%s/%d_%s", out_dir.c_str(), i, fname);
+      stringstream ss;
+      ss << out_dir << "/" << i << "_" << fname;
+      filename = ss.str();
       i++;
     }
 
-    outfile.open(filename, ios::trunc);
+    outfile.open(filename.c_str(), ios::trunc);
 
     time_t tnow;
     time(&tnow);

AMSS_NCKU_source/surface_integral.h

@@ -36,6 +36,11 @@ private:
 
 	double *nx_g, *ny_g, *nz_g; // global list of unit normals
 	int myrank, cpusize;
+	int wave_cache_spinw, wave_cache_maxl, wave_cache_modes;
+	double *wave_theta_pos, *wave_theta_neg;
+	double *wave_phi_cos, *wave_phi_sin;
+	void clear_wave_cache();
+	void build_wave_cache(int spinw, int maxl);
 
 public:
 	surface_integral(int iSymmetry);
@@ -82,13 +87,29 @@ public:
 					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
 					   var *Gmx, var *Gmy, var *Gmz,
 					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
-					   double *Rout, monitor *Monitor);
+					   double *Rout, monitor *Monitor, bool refresh_mass_fields = true);
 	void surf_MassPAng(double rex, int lev, ShellPatch *GH, var *chi, var *trK,
 					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
 					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
 					   var *Gmx, var *Gmy, var *Gmz,
 					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
-					   double *Rout, monitor *Monitor);
+					   double *Rout, monitor *Monitor, bool refresh_mass_fields = true);
+	void surf_WaveMassPAng(double rex, int lev, cgh *GH,
+					   var *Rpsi4, var *Ipsi4, int spinw, int maxl, int NN, double *RP, double *IP,
+					   var *chi, var *trK,
+					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
+					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
+					   var *Gmx, var *Gmy, var *Gmz,
+					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
+					   double *Rout, monitor *Monitor, bool refresh_mass_fields = true);
+	void surf_WaveMassPAng(double rex, int lev, ShellPatch *GH,
+					   var *Rpsi4, var *Ipsi4, int spinw, int maxl, int NN, double *RP, double *IP,
+					   var *chi, var *trK,
+					   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
+					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
+					   var *Gmx, var *Gmy, var *Gmz,
+					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs,
+					   double *Rout, monitor *Monitor, bool refresh_mass_fields = true);
 	void surf_Wave(double rex, cgh *GH, ShellPatch *SH,
 				   var *chi, var *trK,
 				   var *gxx, var *gxy, var *gxz, var *gyy, var *gyz, var *gzz,
@@ -115,7 +136,7 @@ public:
 					   var *Axx, var *Axy, var *Axz, var *Ayy, var *Ayz, var *Azz,
 					   var *Gmx, var *Gmy, var *Gmz,
 					   var *Sfx_rhs, var *Sfy_rhs, var *Sfz_rhs, // temparay memory for mass^i
-					   double *Rout, monitor *Monitor, MPI_Comm Comm_here);
+					   double *Rout, monitor *Monitor, MPI_Comm Comm_here, bool refresh_mass_fields = true);
 	void surf_Wave(double rex, int lev, cgh *GH, var *Rpsi4, var *Ipsi4,
 				   int spinw, int maxl, int NN, double *RP, double *IP,
 				   monitor *Monitor, MPI_Comm Comm_here);

AMSS_NCKU_source/z4c_rhs_c.C

@@ -0,0 +1,725 @@
+#include "macrodef.h"
+#include "bssn_rhs.h"
+#include "fmisc.h"
+#include "ricci_gamma.h"
+#include "share_func.h"
+#include "tool.h"
+#include <vector>
+
+#ifdef fortran1
+#define f_constraint_bssn constraint_bssn
+#define f_z4c_rhs_point z4c_rhs_point
+#endif
+#ifdef fortran2
+#define f_constraint_bssn CONSTRAINT_BSSN
+#define f_z4c_rhs_point Z4C_RHS_POINT
+#endif
+#ifdef fortran3
+#define f_constraint_bssn constraint_bssn_
+#define f_z4c_rhs_point z4c_rhs_point_
+#endif
+
+extern "C" void f_constraint_bssn(int *, double *, double *, double *,
+                                  double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *, double *, double *, double *, double *, double *,
+                                  double *, double *, double *,
+                                  int &);
+
+extern "C" void f_z4c_rhs_point(
+    double &A11,
+    double &A12,
+    double &A13,
+    double &A22,
+    double &A23,
+    double &A33,
+    double &alpha,
+    double &B1,
+    double &B2,
+    double &B3,
+    double &beta1,
+    double &beta2,
+    double &beta3,
+    double &chi,
+    double &chiDivFloor,
+    double &da1,
+    double &dA111,
+    double &dA112,
+    double &dA113,
+    double &dA122,
+    double &dA123,
+    double &dA133,
+    double &da2,
+    double &dA211,
+    double &dA212,
+    double &dA213,
+    double &dA222,
+    double &dA223,
+    double &dA233,
+    double &da3,
+    double &dA311,
+    double &dA312,
+    double &dA313,
+    double &dA322,
+    double &dA323,
+    double &dA333,
+    double &db11,
+    double &dB11,
+    double &db12,
+    double &dB12,
+    double &db13,
+    double &dB13,
+    double &db21,
+    double &dB21,
+    double &db22,
+    double &dB22,
+    double &db23,
+    double &dB23,
+    double &db31,
+    double &dB31,
+    double &db32,
+    double &dB32,
+    double &db33,
+    double &dB33,
+    double &dchi1,
+    double &dchi2,
+    double &dchi3,
+    double &dda11,
+    double &dda12,
+    double &dda13,
+    double &dda22,
+    double &dda23,
+    double &dda33,
+    double &ddb111,
+    double &ddb112,
+    double &ddb113,
+    double &ddb121,
+    double &ddb122,
+    double &ddb123,
+    double &ddb131,
+    double &ddb132,
+    double &ddb133,
+    double &ddb221,
+    double &ddb222,
+    double &ddb223,
+    double &ddb231,
+    double &ddb232,
+    double &ddb233,
+    double &ddb331,
+    double &ddb332,
+    double &ddb333,
+    double &ddchi11,
+    double &ddchi12,
+    double &ddchi13,
+    double &ddchi22,
+    double &ddchi23,
+    double &ddchi33,
+    double &deldelg1111,
+    double &deldelg1112,
+    double &deldelg1113,
+    double &deldelg1122,
+    double &deldelg1123,
+    double &deldelg1133,
+    double &deldelg1211,
+    double &deldelg1212,
+    double &deldelg1213,
+    double &deldelg1222,
+    double &deldelg1223,
+    double &deldelg1233,
+    double &deldelg1311,
+    double &deldelg1312,
+    double &deldelg1313,
+    double &deldelg1322,
+    double &deldelg1323,
+    double &deldelg1333,
+    double &deldelg2211,
+    double &deldelg2212,
+    double &deldelg2213,
+    double &deldelg2222,
+    double &deldelg2223,
+    double &deldelg2233,
+    double &deldelg2311,
+    double &deldelg2312,
+    double &deldelg2313,
+    double &deldelg2322,
+    double &deldelg2323,
+    double &deldelg2333,
+    double &deldelg3311,
+    double &deldelg3312,
+    double &deldelg3313,
+    double &deldelg3322,
+    double &deldelg3323,
+    double &deldelg3333,
+    double &delG11,
+    double &delg111,
+    double &delg112,
+    double &delg113,
+    double &delG12,
+    double &delg122,
+    double &delg123,
+    double &delG13,
+    double &delg133,
+    double &delG21,
+    double &delg211,
+    double &delg212,
+    double &delg213,
+    double &delG22,
+    double &delg222,
+    double &delg223,
+    double &delG23,
+    double &delg233,
+    double &delG31,
+    double &delg311,
+    double &delg312,
+    double &delg313,
+    double &delG32,
+    double &delg322,
+    double &delg323,
+    double &delG33,
+    double &delg333,
+    double &dKhat1,
+    double &dKhat2,
+    double &dKhat3,
+    double &dTheta1,
+    double &dTheta2,
+    double &dTheta3,
+    double &G1,
+    double &g11,
+    double &g12,
+    double &g13,
+    double &G2,
+    double &g22,
+    double &g23,
+    double &G3,
+    double &g33,
+    double &kappa1,
+    double &kappa2,
+    double &Khat,
+    double &rA11,
+    double &rA12,
+    double &rA13,
+    double &rA22,
+    double &rA23,
+    double &rA33,
+    double &rchi,
+    double &rG1,
+    double &rg11,
+    double &rg12,
+    double &rg13,
+    double &rG2,
+    double &rg22,
+    double &rg23,
+    double &rG3,
+    double &rg33,
+    double &rKhat,
+    double &rTheta,
+    double &Theta);
+
+static inline void z4c_contract_gamma(
+    const double gxx, const double gxy, const double gxz,
+    const double gyy, const double gyz, const double gzz,
+    const double gxxx, const double gxyx, const double gxzx,
+    const double gyyx, const double gyzx, const double gzzx,
+    const double gxxy, const double gxyy, const double gxzy,
+    const double gyyy, const double gyzy, const double gzzy,
+    const double gxxz, const double gxyz, const double gxzz,
+    const double gyyz, const double gyzz, const double gzzz,
+    double &Gamxa, double &Gamya, double &Gamza)
+{
+    double det = gxx * gyy * gzz + gxy * gyz * gxz + gxz * gxy * gyz -
+                 gxz * gyy * gxz - gxy * gxy * gzz - gxx * gyz * gyz;
+    const double gupxx = (gyy * gzz - gyz * gyz) / det;
+    const double gupxy = -(gxy * gzz - gyz * gxz) / det;
+    const double gupxz = (gxy * gyz - gyy * gxz) / det;
+    const double gupyy = (gxx * gzz - gxz * gxz) / det;
+    const double gupyz = -(gxx * gyz - gxy * gxz) / det;
+    const double gupzz = (gxx * gyy - gxy * gxy) / det;
+
+    const double Gamxxx = 0.5 * (gupxx * gxxx + gupxy * (2.0 * gxyx - gxxy) + gupxz * (2.0 * gxzx - gxxz));
+    const double Gamyxx = 0.5 * (gupxy * gxxx + gupyy * (2.0 * gxyx - gxxy) + gupyz * (2.0 * gxzx - gxxz));
+    const double Gamzxx = 0.5 * (gupxz * gxxx + gupyz * (2.0 * gxyx - gxxy) + gupzz * (2.0 * gxzx - gxxz));
+
+    const double Gamxyy = 0.5 * (gupxx * (2.0 * gxyy - gyyx) + gupxy * gyyy + gupxz * (2.0 * gyzy - gyyz));
+    const double Gamyyy = 0.5 * (gupxy * (2.0 * gxyy - gyyx) + gupyy * gyyy + gupyz * (2.0 * gyzy - gyyz));
+    const double Gamzyy = 0.5 * (gupxz * (2.0 * gxyy - gyyx) + gupyz * gyyy + gupzz * (2.0 * gyzy - gyyz));
+
+    const double Gamxzz = 0.5 * (gupxx * (2.0 * gxzz - gzzx) + gupxy * (2.0 * gyzz - gzzy) + gupxz * gzzz);
+    const double Gamyzz = 0.5 * (gupxy * (2.0 * gxzz - gzzx) + gupyy * (2.0 * gyzz - gzzy) + gupyz * gzzz);
+    const double Gamzzz = 0.5 * (gupxz * (2.0 * gxzz - gzzx) + gupyz * (2.0 * gyzz - gzzy) + gupzz * gzzz);
+
+    const double Gamxxy = 0.5 * (gupxx * gxxy + gupxy * gyyx + gupxz * (gxzy + gyzx - gxyz));
+    const double Gamyxy = 0.5 * (gupxy * gxxy + gupyy * gyyx + gupyz * (gxzy + gyzx - gxyz));
+    const double Gamzxy = 0.5 * (gupxz * gxxy + gupyz * gyyx + gupzz * (gxzy + gyzx - gxyz));
+
+    const double Gamxxz = 0.5 * (gupxx * gxxz + gupxy * (gxyz + gyzx - gxzy) + gupxz * gzzx);
+    const double Gamyxz = 0.5 * (gupxy * gxxz + gupyy * (gxyz + gyzx - gxzy) + gupyz * gzzx);
+    const double Gamzxz = 0.5 * (gupxz * gxxz + gupyz * (gxyz + gyzx - gxzy) + gupzz * gzzx);
+
+    const double Gamxyz = 0.5 * (gupxx * (gxyz + gxzy - gyzx) + gupxy * gyyz + gupxz * gzzy);
+    const double Gamyyz = 0.5 * (gupxy * (gxyz + gxzy - gyzx) + gupyy * gyyz + gupyz * gzzy);
+    const double Gamzyz = 0.5 * (gupxz * (gxyz + gxzy - gyzx) + gupyz * gyyz + gupzz * gzzy);
+
+    Gamxa = gupxx * Gamxxx + gupyy * Gamxyy + gupzz * Gamxzz +
+            2.0 * (gupxy * Gamxxy + gupxz * Gamxxz + gupyz * Gamxyz);
+    Gamya = gupxx * Gamyxx + gupyy * Gamyyy + gupzz * Gamyzz +
+            2.0 * (gupxy * Gamyxy + gupxz * Gamyxz + gupyz * Gamyyz);
+    Gamza = gupxx * Gamzxx + gupyy * Gamzyy + gupzz * Gamzzz +
+            2.0 * (gupxy * Gamzxy + gupxz * Gamzxz + gupyz * Gamzyz);
+}
+
+static int compute_rhs_z4c_cartesian(
+    int *ex, double &T, double *X, double *Y, double *Z,
+    double *chi_state, double *chi_constraints, double *trK,
+    double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+    double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+    double *Gamx, double *Gamy, double *Gamz,
+    double *Lap, double *betax, double *betay, double *betaz,
+    double *dtSfx, double *dtSfy, double *dtSfz,
+    double *TZ,
+    double *chi_rhs, double *trK_rhs,
+    double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+    double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+    double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+    double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+    double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+    double *TZ_rhs,
+    double *rho, double *Sx, double *Sy, double *Sz,
+    double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+    double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+    double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+    double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+    double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+    double *Hcon, double *Mxcon, double *Mycon, double *Mzcon, double *Gmxcon, double *Gmycon, double *Gmzcon,
+    int &Symmetry, int &Lev, double &eps, int &co)
+{
+    (void)T;
+
+    const int nx = ex[0];
+    const int ny = ex[1];
+    const int nz = ex[2];
+    const int all = nx * ny * nz;
+
+    double alpn1[all], chin1[all], gxx[all], gyy[all], gzz[all];
+    double chix[all], chiy[all], chiz[all], chixx[all], chixy[all], chixz[all], chiyy[all], chiyz[all], chizz[all];
+    double gxxx[all], gxyx[all], gxzx[all], gyyx[all], gyzx[all], gzzx[all];
+    double gxxy[all], gxyy[all], gxzy[all], gyyy[all], gyzy[all], gzzy[all];
+    double gxxz[all], gxyz[all], gxzz[all], gyyz[all], gyzz[all], gzzz[all];
+    double gxxxx[all], gxxxy[all], gxxxz[all], gxxyy[all], gxxyz[all], gxxzz[all];
+    double gxyxx[all], gxyxy[all], gxyxz[all], gxyyy[all], gxyyz[all], gxyzz[all];
+    double gxzxx[all], gxzxy[all], gxzxz[all], gxzyy[all], gxzyz[all], gxzzz[all];
+    double gyyxx[all], gyyxy[all], gyyxz[all], gyyyy[all], gyyyz[all], gyyzz[all];
+    double gyzxx[all], gyzxy[all], gyzxz[all], gyzyy[all], gyzyz[all], gyzzz[all];
+    double gzzxx[all], gzzxy[all], gzzxz[all], gzzyy[all], gzzyz[all], gzzzz[all];
+    double Lapx[all], Lapy[all], Lapz[all], Lapxx[all], Lapxy[all], Lapxz[all], Lapyy[all], Lapyz[all], Lapzz[all];
+    double betaxx[all], betaxy[all], betaxz[all], betayx[all], betayy[all], betayz[all], betazx[all], betazy[all], betazz[all];
+    double dBxx[all], dBxy[all], dBxz[all], dByx[all], dByy[all], dByz[all], dBzx[all], dBzy[all], dBzz[all];
+    double sfxxx[all], sfxxy[all], sfxxz[all], sfxyy[all], sfxyz[all], sfxzz[all];
+    double sfyxx[all], sfyxy[all], sfyxz[all], sfyyy[all], sfyyz[all], sfyzz[all];
+    double sfzxx[all], sfzxy[all], sfzxz[all], sfzyy[all], sfzyz[all], sfzzz[all];
+    double Gamxx[all], Gamxy[all], Gamxz[all], Gamyx[all], Gamyy[all], Gamyz[all], Gamzx[all], Gamzy[all], Gamzz[all];
+    double Kx[all], Ky[all], Kz[all], TZx[all], TZy[all], TZz[all];
+    double Axxx[all], Axxy[all], Axxz[all], Axyx[all], Axyy[all], Axyz[all];
+    double Axzx[all], Axzy[all], Axzz[all], Ayyx[all], Ayyy[all], Ayyz[all];
+    double Ayzx[all], Ayzy[all], Ayzz[all], Azzx[all], Azzy[all], Azzz[all];
+
+    const double SSS[3] = {1.0, 1.0, 1.0};
+    const double AAS[3] = {-1.0, -1.0, 1.0};
+    const double ASA[3] = {-1.0, 1.0, -1.0};
+    const double SAA[3] = {1.0, -1.0, -1.0};
+    const double ASS[3] = {-1.0, 1.0, 1.0};
+    const double SAS[3] = {1.0, -1.0, 1.0};
+    const double SSA[3] = {1.0, 1.0, -1.0};
+
+    const double ONE = 1.0;
+    const double TWO = 2.0;
+    const double ZEO = 0.0;
+    double chiDivfloor = 1.0e-5;
+
+    double kappa1 = 2.0e-2;
+    double kappa2 = 0.0;
+    double FF = 0.75;
+    double eta = 2.0;
+
+    for (int idx = 0; idx < all; ++idx)
+    {
+        alpn1[idx] = Lap[idx] + ONE;
+        chin1[idx] = chi_state[idx] + ONE;
+        gxx[idx] = dxx[idx] + ONE;
+        gyy[idx] = dyy[idx] + ONE;
+        gzz[idx] = dzz[idx] + ONE;
+    }
+
+    fderivs(ex, betax, betaxx, betaxy, betaxz, X, Y, Z, -1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, betay, betayx, betayy, betayz, X, Y, Z, 1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, betaz, betazx, betazy, betazz, X, Y, Z, 1.0, 1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, dtSfx, dBxx, dBxy, dBxz, X, Y, Z, -1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, dtSfy, dByx, dByy, dByz, X, Y, Z, 1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, dtSfz, dBzx, dBzy, dBzz, X, Y, Z, 1.0, 1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, chi_state, chix, chiy, chiz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, dxx, gxxx, gxxy, gxxz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, gxy, gxyx, gxyy, gxyz, X, Y, Z, -1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, gxz, gxzx, gxzy, gxzz, X, Y, Z, -1.0, 1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, dyy, gyyx, gyyy, gyyz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, gyz, gyzx, gyzy, gyzz, X, Y, Z, 1.0, -1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, dzz, gzzx, gzzy, gzzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+
+    fdderivs(ex, dxx, gxxxx, gxxxy, gxxxz, gxxyy, gxxyz, gxxzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, dyy, gyyxx, gyyxy, gyyxz, gyyyy, gyyyz, gyyzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, dzz, gzzxx, gzzxy, gzzxz, gzzyy, gzzyz, gzzzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, gxy, gxyxx, gxyxy, gxyxz, gxyyy, gxyyz, gxyzz, X, Y, Z, -1.0, -1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, gxz, gxzxx, gxzxy, gxzxz, gxzyy, gxzyz, gxzzz, X, Y, Z, -1.0, 1.0, -1.0, Symmetry, Lev);
+    fdderivs(ex, gyz, gyzxx, gyzxy, gyzxz, gyzyy, gyzyz, gyzzz, X, Y, Z, 1.0, -1.0, -1.0, Symmetry, Lev);
+
+    fderivs(ex, Gamx, Gamxx, Gamxy, Gamxz, X, Y, Z, -1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Gamy, Gamyx, Gamyy, Gamyz, X, Y, Z, 1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Gamz, Gamzx, Gamzy, Gamzz, X, Y, Z, 1.0, 1.0, -1.0, Symmetry, Lev);
+
+    fderivs(ex, Lap, Lapx, Lapy, Lapz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, trK, Kx, Ky, Kz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, TZ, TZx, TZy, TZz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+
+    fdderivs(ex, betax, sfxxx, sfxxy, sfxxz, sfxyy, sfxyz, sfxzz, X, Y, Z, -1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, betay, sfyxx, sfyxy, sfyxz, sfyyy, sfyyz, sfyzz, X, Y, Z, 1.0, -1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, betaz, sfzxx, sfzxy, sfzxz, sfzyy, sfzyz, sfzzz, X, Y, Z, 1.0, 1.0, -1.0, Symmetry, Lev);
+
+    fdderivs(ex, chi_state, chixx, chixy, chixz, chiyy, chiyz, chizz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fdderivs(ex, Lap, Lapxx, Lapxy, Lapxz, Lapyy, Lapyz, Lapzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+
+    fderivs(ex, Axx, Axxx, Axxy, Axxz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Axy, Axyx, Axyy, Axyz, X, Y, Z, -1.0, -1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Axz, Axzx, Axzy, Axzz, X, Y, Z, -1.0, 1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, Ayy, Ayyx, Ayyy, Ayyz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+    fderivs(ex, Ayz, Ayzx, Ayzy, Ayzz, X, Y, Z, 1.0, -1.0, -1.0, Symmetry, Lev);
+    fderivs(ex, Azz, Azzx, Azzy, Azzz, X, Y, Z, 1.0, 1.0, 1.0, Symmetry, Lev);
+
+    for (int idx = 0; idx < all; ++idx)
+    {
+        double point_kappa1 = 0.0;
+        f_z4c_rhs_point(
+            Axx[idx], Axy[idx], Axz[idx], Ayy[idx], Ayz[idx], Azz[idx],
+            alpn1[idx], dtSfx[idx], dtSfy[idx], dtSfz[idx],
+            betax[idx], betay[idx], betaz[idx],
+            chin1[idx], chiDivfloor,
+            Lapx[idx],
+            Axxx[idx], Axyx[idx], Axzx[idx], Ayyx[idx], Ayzx[idx], Azzx[idx],
+            Lapy[idx],
+            Axxy[idx], Axyy[idx], Axzy[idx], Ayyy[idx], Ayzy[idx], Azzy[idx],
+            Lapz[idx],
+            Axxz[idx], Axyz[idx], Axzz[idx], Ayyz[idx], Ayzz[idx], Azzz[idx],
+            betaxx[idx], dBxx[idx], betayx[idx], dByx[idx], betazx[idx], dBzx[idx],
+            betaxy[idx], dBxy[idx], betayy[idx], dByy[idx], betazy[idx], dBzy[idx],
+            betaxz[idx], dBxz[idx], betayz[idx], dByz[idx], betazz[idx], dBzz[idx],
+            chix[idx], chiy[idx], chiz[idx],
+            Lapxx[idx], Lapxy[idx], Lapxz[idx], Lapyy[idx], Lapyz[idx], Lapzz[idx],
+            sfxxx[idx], sfyxx[idx], sfzxx[idx],
+            sfxxy[idx], sfyxy[idx], sfzxy[idx],
+            sfxxz[idx], sfyxz[idx], sfzxz[idx],
+            sfxyy[idx], sfyyy[idx], sfzyy[idx],
+            sfxyz[idx], sfyyz[idx], sfzyz[idx],
+            sfxzz[idx], sfyzz[idx], sfzzz[idx],
+            chixx[idx], chixy[idx], chixz[idx], chiyy[idx], chiyz[idx], chizz[idx],
+            gxxxx[idx], gxyxx[idx], gxzxx[idx], gyyxx[idx], gyzxx[idx], gzzxx[idx],
+            gxxxy[idx], gxyxy[idx], gxzxy[idx], gyyxy[idx], gyzxy[idx], gzzxy[idx],
+            gxxxz[idx], gxyxz[idx], gxzxz[idx], gyyxz[idx], gyzxz[idx], gzzxz[idx],
+            gxxyy[idx], gxyyy[idx], gxzyy[idx], gyyyy[idx], gyzyy[idx], gzzyy[idx],
+            gxxyz[idx], gxyyz[idx], gxzyz[idx], gyyyz[idx], gyzyz[idx], gzzyz[idx],
+            gxxzz[idx], gxyzz[idx], gxzzz[idx], gyyzz[idx], gyzzz[idx], gzzzz[idx],
+            Gamxx[idx], gxxx[idx], gxyx[idx], gxzx[idx],
+            Gamyx[idx], gyyx[idx], gyzx[idx],
+            Gamzx[idx], gzzx[idx],
+            Gamxy[idx], gxxy[idx], gxyy[idx], gxzy[idx],
+            Gamyy[idx], gyyy[idx], gyzy[idx],
+            Gamzy[idx], gzzy[idx],
+            Gamxz[idx], gxxz[idx], gxyz[idx], gxzz[idx],
+            Gamyz[idx], gyyz[idx], gyzz[idx],
+            Gamzz[idx], gzzz[idx],
+            Kx[idx], Ky[idx], Kz[idx],
+            TZx[idx], TZy[idx], TZz[idx],
+            Gamx[idx], gxx[idx], gxy[idx], gxz[idx],
+            Gamy[idx], gyy[idx], gyz[idx],
+            Gamz[idx], gzz[idx],
+            point_kappa1, kappa2,
+            trK[idx],
+            Axx_rhs[idx], Axy_rhs[idx], Axz_rhs[idx], Ayy_rhs[idx], Ayz_rhs[idx], Azz_rhs[idx],
+            chi_rhs[idx],
+            Gamx_rhs[idx], gxx_rhs[idx], gxy_rhs[idx], gxz_rhs[idx],
+            Gamy_rhs[idx], gyy_rhs[idx], gyz_rhs[idx],
+            Gamz_rhs[idx], gzz_rhs[idx], trK_rhs[idx], TZ_rhs[idx], TZ[idx]);
+    }
+
+    for (int idx = 0; idx < all; ++idx)
+        Lap_rhs[idx] = -TWO * alpn1[idx] * trK[idx];
+
+#if (GAUGE == 0)
+    for (int idx = 0; idx < all; ++idx)
+    {
+        betax_rhs[idx] = FF * dtSfx[idx];
+        betay_rhs[idx] = FF * dtSfy[idx];
+        betaz_rhs[idx] = FF * dtSfz[idx];
+        dtSfx_rhs[idx] = Gamx_rhs[idx] - eta * dtSfx[idx];
+        dtSfy_rhs[idx] = Gamy_rhs[idx] - eta * dtSfy[idx];
+        dtSfz_rhs[idx] = Gamz_rhs[idx] - eta * dtSfz[idx];
+    }
+#elif (GAUGE == 1)
+    for (int idx = 0; idx < all; ++idx)
+    {
+        betax_rhs[idx] = Gamx[idx] - eta * betax[idx];
+        betay_rhs[idx] = Gamy[idx] - eta * betay[idx];
+        betaz_rhs[idx] = Gamz[idx] - eta * betaz[idx];
+        dtSfx_rhs[idx] = ZEO;
+        dtSfy_rhs[idx] = ZEO;
+        dtSfz_rhs[idx] = ZEO;
+    }
+#else
+#error "z4c_rhs_c.C currently supports GAUGE == 0 or GAUGE == 1 for Z4C"
+#endif
+
+    lopsided(ex, X, Y, Z, gxx, gxx_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, gxy, gxy_rhs, betax, betay, betaz, Symmetry, AAS);
+    lopsided(ex, X, Y, Z, gxz, gxz_rhs, betax, betay, betaz, Symmetry, ASA);
+    lopsided(ex, X, Y, Z, gyy, gyy_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, gyz, gyz_rhs, betax, betay, betaz, Symmetry, SAA);
+    lopsided(ex, X, Y, Z, gzz, gzz_rhs, betax, betay, betaz, Symmetry, SSS);
+
+    lopsided(ex, X, Y, Z, Axx, Axx_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, Axy, Axy_rhs, betax, betay, betaz, Symmetry, AAS);
+    lopsided(ex, X, Y, Z, Axz, Axz_rhs, betax, betay, betaz, Symmetry, ASA);
+    lopsided(ex, X, Y, Z, Ayy, Ayy_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, Ayz, Ayz_rhs, betax, betay, betaz, Symmetry, SAA);
+    lopsided(ex, X, Y, Z, Azz, Azz_rhs, betax, betay, betaz, Symmetry, SSS);
+
+    lopsided(ex, X, Y, Z, chi_state, chi_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, trK, trK_rhs, betax, betay, betaz, Symmetry, SSS);
+
+    lopsided(ex, X, Y, Z, Gamx, Gamx_rhs, betax, betay, betaz, Symmetry, ASS);
+    lopsided(ex, X, Y, Z, Gamy, Gamy_rhs, betax, betay, betaz, Symmetry, SAS);
+    lopsided(ex, X, Y, Z, Gamz, Gamz_rhs, betax, betay, betaz, Symmetry, SSA);
+
+    lopsided(ex, X, Y, Z, Lap, Lap_rhs, betax, betay, betaz, Symmetry, SSS);
+    lopsided(ex, X, Y, Z, betax, betax_rhs, betax, betay, betaz, Symmetry, ASS);
+    lopsided(ex, X, Y, Z, betay, betay_rhs, betax, betay, betaz, Symmetry, SAS);
+    lopsided(ex, X, Y, Z, betaz, betaz_rhs, betax, betay, betaz, Symmetry, SSA);
+#if (GAUGE == 0)
+    lopsided(ex, X, Y, Z, dtSfx, dtSfx_rhs, betax, betay, betaz, Symmetry, ASS);
+    lopsided(ex, X, Y, Z, dtSfy, dtSfy_rhs, betax, betay, betaz, Symmetry, SAS);
+    lopsided(ex, X, Y, Z, dtSfz, dtSfz_rhs, betax, betay, betaz, Symmetry, SSA);
+#endif
+    lopsided(ex, X, Y, Z, TZ, TZ_rhs, betax, betay, betaz, Symmetry, SSS);
+
+    for (int idx = 0; idx < all; ++idx)
+    {
+        double Gamxa = 0.0, Gamya = 0.0, Gamza = 0.0;
+        z4c_contract_gamma(
+            gxx[idx], gxy[idx], gxz[idx], gyy[idx], gyz[idx], gzz[idx],
+            gxxx[idx], gxyx[idx], gxzx[idx], gyyx[idx], gyzx[idx], gzzx[idx],
+            gxxy[idx], gxyy[idx], gxzy[idx], gyyy[idx], gyzy[idx], gzzy[idx],
+            gxxz[idx], gxyz[idx], gxzz[idx], gyyz[idx], gyzz[idx], gzzz[idx],
+            Gamxa, Gamya, Gamza);
+
+        TZ_rhs[idx] -= alpn1[idx] * (TWO + kappa2) * kappa1 * TZ[idx];
+        trK_rhs[idx] += alpn1[idx] * kappa1 * (ONE - kappa2) * TZ[idx];
+        Gamx_rhs[idx] -= TWO * alpn1[idx] * kappa1 * (Gamx[idx] - Gamxa);
+        Gamy_rhs[idx] -= TWO * alpn1[idx] * kappa1 * (Gamy[idx] - Gamya);
+        Gamz_rhs[idx] -= TWO * alpn1[idx] * kappa1 * (Gamz[idx] - Gamza);
+    }
+
+    if (eps > 0.0)
+    {
+        kodis(ex, X, Y, Z, chi_state, chi_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, trK, trK_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, gxx, gxx_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, gxy, gxy_rhs, AAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, gxz, gxz_rhs, ASA, Symmetry, eps);
+        kodis(ex, X, Y, Z, gyy, gyy_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, gyz, gyz_rhs, SAA, Symmetry, eps);
+        kodis(ex, X, Y, Z, gzz, gzz_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Axx, Axx_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Axy, Axy_rhs, AAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Axz, Axz_rhs, ASA, Symmetry, eps);
+        kodis(ex, X, Y, Z, Ayy, Ayy_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Ayz, Ayz_rhs, SAA, Symmetry, eps);
+        kodis(ex, X, Y, Z, Azz, Azz_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Gamx, Gamx_rhs, ASS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Gamy, Gamy_rhs, SAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, Gamz, Gamz_rhs, SSA, Symmetry, eps);
+        kodis(ex, X, Y, Z, Lap, Lap_rhs, SSS, Symmetry, eps);
+        kodis(ex, X, Y, Z, betax, betax_rhs, ASS, Symmetry, eps);
+        kodis(ex, X, Y, Z, betay, betay_rhs, SAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, betaz, betaz_rhs, SSA, Symmetry, eps);
+#if (GAUGE == 0)
+        kodis(ex, X, Y, Z, dtSfx, dtSfx_rhs, ASS, Symmetry, eps);
+        kodis(ex, X, Y, Z, dtSfy, dtSfy_rhs, SAS, Symmetry, eps);
+        kodis(ex, X, Y, Z, dtSfz, dtSfz_rhs, SSA, Symmetry, eps);
+#endif
+        kodis(ex, X, Y, Z, TZ, TZ_rhs, SSS, Symmetry, eps);
+    }
+
+    if (co == 0)
+    {
+#if (ABV == 0)
+        f_ricci_gamma(ex, X, Y, Z,
+                      chi_constraints,
+                      dxx, gxy, gxz, dyy, gyz, dzz,
+                      Gamx, Gamy, Gamz,
+                      Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+                      Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+                      Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+                      Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+                      Symmetry);
+#endif
+        f_constraint_bssn(ex, X, Y, Z,
+                          chi_constraints, trK,
+                          dxx, gxy, gxz, dyy, gyz, dzz,
+                          Axx, Axy, Axz, Ayy, Ayz, Azz,
+                          Gamx, Gamy, Gamz,
+                          Lap, betax, betay, betaz, rho, Sx, Sy, Sz,
+                          Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+                          Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+                          Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+                          Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+                          Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
+                          Symmetry);
+    }
+
+    return 0;
+}
+
+extern "C" int f_compute_rhs_Z4c(int *ex, double &T,
+                                 double *X, double *Y, double *Z,
+                                 double *chi, double *trK,
+                                 double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+                                 double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+                                 double *Gamx, double *Gamy, double *Gamz,
+                                 double *Lap, double *betax, double *betay, double *betaz,
+                                 double *dtSfx, double *dtSfy, double *dtSfz,
+                                 double *TZ,
+                                 double *chi_rhs, double *trK_rhs,
+                                 double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+                                 double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+                                 double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+                                 double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+                                 double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+                                 double *TZ_rhs,
+                                 double *rho, double *Sx, double *Sy, double *Sz,
+                                 double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+                                 double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+                                 double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+                                 double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+                                 double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+                                 double *Hcon, double *Mxcon, double *Mycon, double *Mzcon, double *Gmxcon, double *Gmycon, double *Gmzcon,
+                                 int &Symmetry, int &Lev, double &eps, int &co)
+{
+    return compute_rhs_z4c_cartesian(
+        ex, T, X, Y, Z,
+        chi, chi, trK,
+        dxx, gxy, gxz, dyy, gyz, dzz,
+        Axx, Axy, Axz, Ayy, Ayz, Azz,
+        Gamx, Gamy, Gamz,
+        Lap, betax, betay, betaz,
+        dtSfx, dtSfy, dtSfz,
+        TZ,
+        chi_rhs, trK_rhs,
+        gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
+        Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
+        Gamx_rhs, Gamy_rhs, Gamz_rhs,
+        Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
+        dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
+        TZ_rhs,
+        rho, Sx, Sy, Sz,
+        Sxx, Sxy, Sxz, Syy, Syz, Szz,
+        Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+        Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+        Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+        Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+        Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
+        Symmetry, Lev, eps, co);
+}
+
+extern "C" int f_compute_rhs_Z4cnot(int *ex, double &T,
+                                    double *X, double *Y, double *Z,
+                                    double *chi, double *trK,
+                                    double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+                                    double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+                                    double *Gamx, double *Gamy, double *Gamz,
+                                    double *Lap, double *betax, double *betay, double *betaz,
+                                    double *dtSfx, double *dtSfy, double *dtSfz,
+                                    double *TZ,
+                                    double *chi_rhs, double *trK_rhs,
+                                    double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+                                    double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+                                    double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+                                    double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+                                    double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+                                    double *TZ_rhs,
+                                    double *rho, double *Sx, double *Sy, double *Sz,
+                                    double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+                                    double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+                                    double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+                                    double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+                                    double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+                                    double *Hcon, double *Mxcon, double *Mycon, double *Mzcon, double *Gmxcon, double *Gmycon, double *Gmzcon,
+                                    int &Symmetry, int &Lev, double &eps, int &co, double &chitiny)
+{
+    const int all = ex[0] * ex[1] * ex[2];
+    std::vector<double> chi_clamped(chi, chi + all);
+    f_lowerboundset(ex, chi_clamped.data(), chitiny);
+
+    const int ret = compute_rhs_z4c_cartesian(
+        ex, T, X, Y, Z,
+        chi_clamped.data(), chi, trK,
+        dxx, gxy, gxz, dyy, gyz, dzz,
+        Axx, Axy, Axz, Ayy, Ayz, Azz,
+        Gamx, Gamy, Gamz,
+        Lap, betax, betay, betaz,
+        dtSfx, dtSfy, dtSfz,
+        TZ,
+        chi_rhs, trK_rhs,
+        gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
+        Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
+        Gamx_rhs, Gamy_rhs, Gamz_rhs,
+        Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
+        dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
+        TZ_rhs,
+        rho, Sx, Sy, Sz,
+        Sxx, Sxy, Sxz, Syy, Syz, Szz,
+        Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+        Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+        Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+        Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+        Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
+        Symmetry, Lev, eps, co);
+
+    if (ret != 0 || co != 0)
+        return ret;
+
+#if (ABV == 0)
+    f_ricci_gamma(ex, X, Y, Z,
+                  chi,
+                  dxx, gxy, gxz, dyy, gyz, dzz,
+                  Gamx, Gamy, Gamz,
+                  Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+                  Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+                  Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+                  Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+                  Symmetry);
+#endif
+    f_constraint_bssn(ex, X, Y, Z,
+                      chi, trK,
+                      dxx, gxy, gxz, dyy, gyz, dzz,
+                      Axx, Axy, Axz, Ayy, Ayz, Azz,
+                      Gamx, Gamy, Gamz,
+                      Lap, betax, betay, betaz, rho, Sx, Sy, Sz,
+                      Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+                      Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+                      Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+                      Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+                      Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
+                      Symmetry);
+    return ret;
+}

AMSS_NCKU_source/z4c_rhs_cuda.h

@@ -0,0 +1,114 @@
+#ifndef Z4C_RHS_CUDA_H
+#define Z4C_RHS_CUDA_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+    Z4C_CUDA_STATE_COUNT = 25
+};
+
+int z4c_cuda_rk4_substep(void *block_tag,
+                         int *ex, double *X, double *Y, double *Z,
+                         double **state_host_in,
+                         double **state_host_out,
+                         const double *propspeed,
+                         const double *soa_flat,
+                         const double *bbox,
+                         double &dT,
+                         double &T,
+                         int &RK4,
+                         int &apply_bam_bc,
+                         int &Symmetry,
+                         int &Lev,
+                         double &eps,
+                         int &co,
+                         int &keep_resident_state,
+                         int &apply_enforce_ga,
+                         double &chitiny);
+
+int z4c_cuda_download_resident_state(void *block_tag,
+                                     int *ex,
+                                     double **state_host_out);
+
+int z4c_cuda_pack_state_region_to_host_buffer(void *block_tag,
+                                              int state_index,
+                                              double *host_buffer,
+                                              int *ex,
+                                              int i0, int j0, int k0,
+                                              int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_region_from_host_buffer(void *block_tag,
+                                                  int state_index,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_host_buffer(void *block_tag,
+                                             int state_count,
+                                             double *host_buffer,
+                                             int *ex,
+                                             int i0, int j0, int k0,
+                                             int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
+                                                 int state_count,
+                                                 double *host_buffer,
+                                                 int *ex,
+                                                 int i0, int j0, int k0,
+                                                 int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_batch_to_device_buffer(void *block_tag,
+                                               int state_count,
+                                               double *device_buffer,
+                                               int *ex,
+                                               int i0, int j0, int k0,
+                                               int sx, int sy, int sz);
+
+int z4c_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int i0, int j0, int k0,
+                                                   int sx, int sy, int sz);
+
+int z4c_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int fi0, int fj0, int fk0,
+                                                   const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int ii0, int jj0, int kk0,
+                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                  const double *state_soa);
+
+int z4c_cuda_download_state_subset(void *block_tag,
+                                   int *ex,
+                                   int subset_count,
+                                   const int *state_indices,
+                                   double **state_host_out);
+
+int z4c_cuda_upload_state_subset(void *block_tag,
+                                 int *ex,
+                                 int subset_count,
+                                 const int *state_indices,
+                                 double **state_host_in);
+
+int z4c_cuda_has_resident_state(void *block_tag);
+
+void z4c_cuda_release_step_ctx(void *block_tag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

generate_macrodef.py

@@ -144,11 +144,67 @@ def generate_macrodef_h():
     print( "#define REGLEV 0",      file=file1 )
     print(                          file=file1 )
 
+    # Define fine-grained timing/debug macros.
+    # All of them default to OFF so production builds do not pay profiling overhead.
+
+    fine_timing = getattr(input_data, "Fine_Timing",
+                  getattr(input_data, "Finegrained_Timing", "no"))
+    kernel_fine_timing = getattr(input_data, "Kernel_Fine_Timing",
+                          getattr(input_data, "BSSN_Kernel_Fine_Timing", "no"))
+    stdin_abort_poll = getattr(input_data, "Enable_Stdin_Abort_Poll",
+                       getattr(input_data, "Stdin_Abort_Poll", "no"))
+    timing_report_every = max(1, int(getattr(
+        input_data, "Timing_Every_Steps",
+        getattr(input_data, "Timing_Report_Every", 1))))
+    timing_top_hotspots = max(1, int(getattr(
+        input_data, "Timing_Top_Hotspots", 8)))
+
+    if ( fine_timing == "yes" ):
+        print( "#define BSSN_FINE_TIMING 1", file=file1 )
+        print(                               file=file1 )
+    elif ( fine_timing == "no" ):
+        print( "#define BSSN_FINE_TIMING 0", file=file1 )
+        print(                               file=file1 )
+    else:
+        print( "Fine_Timing setting error!!!" )
+        print()
+        print( "# Fine_Timing setting error!!!", file=file1 )
+        print(                                   file=file1 )
+
+    print( f"#define BSSN_FINE_TIMING_EVERY {timing_report_every}", file=file1 )
+    print(                                                          file=file1 )
+    print( f"#define BSSN_FINE_TIMING_TOPN {timing_top_hotspots}",  file=file1 )
+    print(                                                          file=file1 )
+
+    if ( kernel_fine_timing == "yes" ):
+        print( "#define BSSN_KERNEL_FINE_TIMING 1", file=file1 )
+        print(                                      file=file1 )
+    elif ( kernel_fine_timing == "no" ):
+        print( "#define BSSN_KERNEL_FINE_TIMING 0", file=file1 )
+        print(                                      file=file1 )
+    else:
+        print( "Kernel_Fine_Timing setting error!!!" )
+        print()
+        print( "# Kernel_Fine_Timing setting error!!!", file=file1 )
+        print(                                          file=file1 )
+
+    if ( stdin_abort_poll == "yes" ):
+        print( "#define BSSN_ENABLE_STDIN_ABORT_POLL 1", file=file1 )
+        print(                                           file=file1 )
+    elif ( stdin_abort_poll == "no" ):
+        print( "#define BSSN_ENABLE_STDIN_ABORT_POLL 0", file=file1 )
+        print(                                           file=file1 )
+    else:
+        print( "Enable_Stdin_Abort_Poll setting error!!!" )
+        print()
+        print( "# Enable_Stdin_Abort_Poll setting error!!!", file=file1 )
+        print(                                               file=file1 )
+
     # Define macro USE_GPU
     # use GPU or not
     
     if ( input_data.GPU_Calculation == "yes"):
-        print( "#define USE_GPU",   file=file1 )
+        print( "//#define USE_GPU",   file=file1 )
         print(                      file=file1 )
     elif ( input_data.GPU_Calculation == "no"):
         print( "//#define USE_GPU", file=file1 )
@@ -224,6 +280,21 @@ def generate_macrodef_h():
     print( "//     0: for every level;",                                                 file=file1 ) 
     print( "//     1: for all",                                                          file=file1 )
     print( "//",                                                                         file=file1 )
+    print( "// define BSSN_FINE_TIMING",                                                 file=file1 )
+    print( "//     enable fine-grained per-timestep timing monitor",                     file=file1 )
+    print( "//",                                                                         file=file1 )
+    print( "// define BSSN_FINE_TIMING_EVERY",                                           file=file1 )
+    print( "//     report timing every N coarse timesteps",                              file=file1 )
+    print( "//",                                                                         file=file1 )
+    print( "// define BSSN_FINE_TIMING_TOPN",                                            file=file1 )
+    print( "//     number of hottest timing buckets shown in stdout",                    file=file1 )
+    print( "//",                                                                         file=file1 )
+    print( "// define BSSN_KERNEL_FINE_TIMING",                                          file=file1 )
+    print( "//     enable split timing inside compute_rhs_bssn",                         file=file1 )
+    print( "//",                                                                         file=file1 )
+    print( "// define BSSN_ENABLE_STDIN_ABORT_POLL",                                     file=file1 )
+    print( "//     poll stdin and broadcast abort flag every coarse step",               file=file1 )
+    print( "//",                                                                         file=file1 )
     print( "// define USE_GPU",                                                          file=file1 )
     print( "//     use gpu or not",                                                      file=file1 )
     print( "//",                                                                         file=file1 )

makefile_and_run.py

@@ -9,6 +9,8 @@
 
 
 import AMSS_NCKU_Input as input_data
+import os
+import shutil
 import subprocess
 import time
 
@@ -56,6 +58,157 @@ BUILD_JOBS = 64
 
 ##################################################################
 
+def _truthy(value, default=False):
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return value
+    text = str(value).strip().lower()
+    if text == "":
+        return default
+    return text in ("1", "yes", "y", "true", "on", "enable", "enabled")
+
+
+def _input_or_env(input_name, env_name, default=None):
+    if env_name in os.environ:
+        return os.environ[env_name]
+    return getattr(input_data, input_name, default)
+
+
+def _start_cuda_mps_if_requested(runtime_env):
+    if input_data.GPU_Calculation != "yes":
+        return False
+
+    default_auto_mps = int(getattr(input_data, "MPI_processes", 1)) > 1
+    auto_mps = _truthy(
+        _input_or_env("CUDA_Auto_MPS", "AMSS_CUDA_AUTO_MPS", default_auto_mps),
+        default=default_auto_mps,
+    )
+    if not auto_mps:
+        return False
+
+    mps_control = shutil.which("nvidia-cuda-mps-control")
+    if not mps_control:
+        print(" CUDA MPS control command was not found; running without MPS.")
+        return False
+
+    uid = os.getuid()
+    pipe_dir = str(_input_or_env("CUDA_MPS_PIPE_DIRECTORY", "CUDA_MPS_PIPE_DIRECTORY",
+                                 f"/tmp/amss-ncku-mps-{uid}"))
+    log_dir = str(_input_or_env("CUDA_MPS_LOG_DIRECTORY", "CUDA_MPS_LOG_DIRECTORY",
+                                f"/tmp/amss-ncku-mps-log-{uid}"))
+    os.makedirs(pipe_dir, exist_ok=True)
+    os.makedirs(log_dir, exist_ok=True)
+
+    mps_env = runtime_env.copy()
+    mps_env["CUDA_MPS_PIPE_DIRECTORY"] = pipe_dir
+    mps_env["CUDA_MPS_LOG_DIRECTORY"] = log_dir
+
+    if os.path.exists(os.path.join(pipe_dir, "control")):
+        runtime_env.update({
+            "CUDA_MPS_PIPE_DIRECTORY": pipe_dir,
+            "CUDA_MPS_LOG_DIRECTORY": log_dir,
+        })
+        print(f" Reusing CUDA MPS daemon: {pipe_dir}")
+        return False
+
+    print(f" Starting CUDA MPS daemon for this run: {pipe_dir}")
+    result = subprocess.run([mps_control, "-d"], env=mps_env, text=True,
+                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    if result.returncode != 0:
+        print(" CUDA MPS daemon did not start; running without MPS.")
+        if result.stdout:
+            print(result.stdout, end="")
+        return False
+
+    runtime_env.update({
+        "CUDA_MPS_PIPE_DIRECTORY": pipe_dir,
+        "CUDA_MPS_LOG_DIRECTORY": log_dir,
+    })
+    return True
+
+
+def _stop_cuda_mps(runtime_env):
+    mps_control = shutil.which("nvidia-cuda-mps-control")
+    if not mps_control:
+        return
+    subprocess.run([mps_control], input="quit\n", env=runtime_env, text=True,
+                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+
+def _gpu_runtime_env():
+    runtime_env = os.environ.copy()
+    finite_difference = str(getattr(input_data, "Finite_Diffenence_Method", "4th-order")).strip()
+
+    defaults = {
+        "AMSS_EVOLVE_TIMING": "1",
+        "AMSS_ESCALAR_STEP_TIMING": "0",
+        "AMSS_INTERP_FAST": "1",
+        "AMSS_INTERP_GPU": "1",
+        "AMSS_ANALYSIS_MAP_EVERY": "1000000",
+        "AMSS_CUDA_AWARE_MPI": "1",
+        "AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_KEEP_ALL_LEVELS": "1",
+        "AMSS_CUDA_ESCALAR_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_ESCALAR_KEEP_ALL_LEVELS": "1",
+        "AMSS_CUDA_EM_CACHE_SOURCES": "1",
+        "AMSS_CUDA_EM_ZERO_FASTPATH": "1",
+        "AMSS_EM_ZERO_ANALYSIS_FASTPATH": "1",
+        "AMSS_EM_ZERO_RESIDENT_DOWNLOAD_FASTPATH": "1",
+        "AMSS_CUDA_AMR_HOST_STAGED": "1",
+        "AMSS_CUDA_AMR_RESTRICT_DEVICE": "0",
+        "AMSS_CUDA_AMR_RESTRICT_BATCH": "0",
+        "AMSS_CUDA_DEVICE_SEGMENT_BATCH": "0",
+        "AMSS_CUDA_UNCACHED_DEVICE_BUFFERS": "1",
+        "AMSS_SHELL_FAST_INTERP": "0",
+        "AMSS_SHELL_PARALLEL_INTERP": "0",
+        "AMSS_SHELL_CUDA_INTERP": "0",
+    }
+    if finite_difference in ("2nd-order", "8th-order"):
+        defaults.update({
+            "AMSS_INTERP_FAST": "0",
+            "AMSS_INTERP_GPU": "0",
+            "AMSS_CUDA_AWARE_MPI": "0",
+        })
+    if finite_difference == "8th-order" and getattr(input_data, "Equation_Class", "") == "BSSN-EM":
+        defaults.update({
+            "AMSS_CUDA_AMR_RESTRICT_DEVICE": "1",
+            "AMSS_CUDA_AMR_RESTRICT_BATCH": "1",
+            "AMSS_CUDA_DEVICE_SEGMENT_BATCH": "1",
+        })
+    if getattr(input_data, "basic_grid_set", "") == "Shell-Patch":
+        defaults.update({
+            "AMSS_CUDA_AWARE_MPI": "0",
+            "AMSS_SHELL_FAST_INTERP": "1",
+            "AMSS_SHELL_PARALLEL_INTERP": "1",
+            "AMSS_SHELL_INTERP_THREADS": "16",
+        })
+    if getattr(input_data, "Equation_Class", "") in ("BSSN", "BSSN-EScalar", "Z4C"):
+        defaults["AMSS_CUDA_AMR_RESTRICT_DEVICE"] = "1"
+    if getattr(input_data, "Equation_Class", "") == "Z4C":
+        defaults.update({
+            "AMSS_Z4C_CUDA_RESIDENT": "1",
+            "AMSS_CONSTRAINT_OUT_EVERY": "1000000",
+        })
+    for key, value in defaults.items():
+        runtime_env.setdefault(key, value)
+
+    optional_overrides = {
+        "AMSS_INTERP_FAST_COMPARE": "AMSS_Interp_Fast_Compare",
+        "AMSS_INTERP_FAST_COMPARE_LIMIT": "AMSS_Interp_Fast_Compare_Limit",
+        "AMSS_INTERP_FAST_COMPARE_TOL": "AMSS_Interp_Fast_Compare_Tol",
+        "AMSS_GPU_STAGE_TIMING": "AMSS_GPU_Stage_Timing",
+        "AMSS_GPU_STAGE_TIMING_EVERY": "AMSS_GPU_Stage_Timing_Every",
+    }
+    for env_name, input_name in optional_overrides.items():
+        if env_name not in runtime_env and hasattr(input_data, input_name):
+            runtime_env[env_name] = str(getattr(input_data, input_name))
+
+    return runtime_env
+
+
+##################################################################
+
 
 
 ##################################################################
@@ -70,9 +223,9 @@ def makefile_ABE():
 
     ## Build command with CPU binding to nohz_full cores
     if (input_data.GPU_Calculation == "no"):
-        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} INTERP_LB_MODE=off ABE"
+        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} INTERP_LB_MODE=off USE_CUDA_BSSN=0 USE_CUDA_Z4C=0 ABE"
     elif (input_data.GPU_Calculation == "yes"):
-        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} ABEGPU"
+        makefile_command  = f"{NUMACTL_CPU_BIND} make -j{BUILD_JOBS} INTERP_LB_MODE=off USE_CUDA_BSSN=1 USE_CUDA_Z4C=1 ABE_CUDA"
     else:
         print( " CPU/GPU numerical calculation setting is wrong " )
         print(                                                    )
@@ -145,29 +298,84 @@ def run_ABE():
     print(                                                      )
 
     ## Define the command to run; cast other values to strings as needed
+    mpi_env = None
+    started_mps = False
+    
+    mpi_processes = int(input_data.MPI_processes)
+    if (input_data.GPU_Calculation == "yes" and
+        getattr(input_data, "Equation_Class", "") == "Z4C"):
+        z4c_env_np = os.environ.get("AMSS_Z4C_GPU_MPI_PROCESSES")
+        if z4c_env_np and int(z4c_env_np) > 0:
+            mpi_processes = int(z4c_env_np)
+        elif mpi_processes < 4:
+            mpi_processes = 4
+    if (input_data.GPU_Calculation == "yes" and
+        getattr(input_data, "basic_grid_set", "") == "Shell-Patch"):
+        shell_env_np = os.environ.get("AMSS_SHELL_GPU_MPI_PROCESSES")
+        if shell_env_np and int(shell_env_np) > 0:
+            mpi_processes = int(shell_env_np)
+        elif mpi_processes < 4:
+            mpi_processes = 4
 
     if (input_data.GPU_Calculation == "no"):
-        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
+        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(mpi_processes) + " ./ABE"
         #mpi_command         = " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
         mpi_command_outfile = "ABE_out.log"
     elif (input_data.GPU_Calculation == "yes"):
-        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABEGPU"
+        mpi_command         = NUMACTL_CPU_BIND + " I_MPI_OFFLOAD=1 I_MPI_OFFLOAD_IPC=0 mpirun -np " + str(mpi_processes) + " ./ABE_CUDA"
         mpi_command_outfile = "ABEGPU_out.log"
+        mpi_env = _gpu_runtime_env()
+        started_mps = _start_cuda_mps_if_requested(mpi_env)
+        print(" GPU optimized runtime switches:")
+        print(f"   MPI processes={mpi_processes}")
+        print(f"   AMSS_INTERP_FAST={mpi_env.get('AMSS_INTERP_FAST', '')}")
+        print(f"   AMSS_INTERP_GPU={mpi_env.get('AMSS_INTERP_GPU', '')}")
+        print(f"   AMSS_ANALYSIS_MAP_EVERY={mpi_env.get('AMSS_ANALYSIS_MAP_EVERY', '')}")
+        print(f"   AMSS_EVOLVE_TIMING={mpi_env.get('AMSS_EVOLVE_TIMING', '')}")
+        print(f"   AMSS_ESCALAR_STEP_TIMING={mpi_env.get('AMSS_ESCALAR_STEP_TIMING', '')}")
+        print(f"   AMSS_CUDA_AWARE_MPI={mpi_env.get('AMSS_CUDA_AWARE_MPI', '')}")
+        print(f"   AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_KEEP_ALL_LEVELS={mpi_env.get('AMSS_CUDA_KEEP_ALL_LEVELS', '')}")
+        print(f"   AMSS_CUDA_ESCALAR_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_ESCALAR_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_ESCALAR_KEEP_ALL_LEVELS={mpi_env.get('AMSS_CUDA_ESCALAR_KEEP_ALL_LEVELS', '')}")
+        print(f"   AMSS_CUDA_EM_CACHE_SOURCES={mpi_env.get('AMSS_CUDA_EM_CACHE_SOURCES', '')}")
+        print(f"   AMSS_CUDA_EM_ZERO_FASTPATH={mpi_env.get('AMSS_CUDA_EM_ZERO_FASTPATH', '')}")
+        print(f"   AMSS_EM_ZERO_ANALYSIS_FASTPATH={mpi_env.get('AMSS_EM_ZERO_ANALYSIS_FASTPATH', '')}")
+        print(f"   AMSS_EM_ZERO_RESIDENT_DOWNLOAD_FASTPATH={mpi_env.get('AMSS_EM_ZERO_RESIDENT_DOWNLOAD_FASTPATH', '')}")
+        print(f"   AMSS_CUDA_AMR_HOST_STAGED={mpi_env.get('AMSS_CUDA_AMR_HOST_STAGED', '')}")
+        print(f"   AMSS_CUDA_AMR_RESTRICT_DEVICE={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_DEVICE', '')}")
+        print(f"   AMSS_CUDA_AMR_RESTRICT_BATCH={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_BATCH', '')}")
+        print(f"   AMSS_CUDA_DEVICE_SEGMENT_BATCH={mpi_env.get('AMSS_CUDA_DEVICE_SEGMENT_BATCH', '')}")
+        print(f"   AMSS_CUDA_UNCACHED_DEVICE_BUFFERS={mpi_env.get('AMSS_CUDA_UNCACHED_DEVICE_BUFFERS', '')}")
+        print(f"   AMSS_SHELL_FAST_INTERP={mpi_env.get('AMSS_SHELL_FAST_INTERP', '')}")
+        print(f"   AMSS_SHELL_PARALLEL_INTERP={mpi_env.get('AMSS_SHELL_PARALLEL_INTERP', '')}")
+        print(f"   AMSS_SHELL_CUDA_INTERP={mpi_env.get('AMSS_SHELL_CUDA_INTERP', '')}")
+        print(f"   AMSS_SHELL_INTERP_THREADS={mpi_env.get('AMSS_SHELL_INTERP_THREADS', '')}")
+        print(f"   AMSS_Z4C_CUDA_RESIDENT={mpi_env.get('AMSS_Z4C_CUDA_RESIDENT', '')}")
+        print(f"   AMSS_CONSTRAINT_OUT_EVERY={mpi_env.get('AMSS_CONSTRAINT_OUT_EVERY', '')}")
+        if "CUDA_MPS_PIPE_DIRECTORY" in mpi_env:
+            print(f"   CUDA_MPS_PIPE_DIRECTORY={mpi_env['CUDA_MPS_PIPE_DIRECTORY']}")
  
-    ## Execute the MPI command and stream output
-    mpi_process = subprocess.Popen(mpi_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    try:
+        ## Execute the MPI command and stream output
+        mpi_process = subprocess.Popen(mpi_command, shell=True, stdout=subprocess.PIPE,
+                                       stderr=subprocess.STDOUT, text=True, env=mpi_env)
 
-    ## Write ABE run output to file while printing to stdout
-    with open(mpi_command_outfile, 'w') as file0:  
-        ## Read and print output lines; also write each line to file
-        for line in mpi_process.stdout:
-            print(line, end='')  # stream output in real time
-            file0.write(line)    # write the line to file
-            file0.flush()        # flush to ensure each line is written immediately (optional)            
-    file0.close()
+        ## Write ABE run output to file while printing to stdout
+        with open(mpi_command_outfile, 'w') as file0:
+            ## Read and print output lines; also write each line to file
+            for line in mpi_process.stdout:
+                print(line, end='')  # stream output in real time
+                file0.write(line)    # write the line to file
+                file0.flush()        # flush to ensure each line is written immediately (optional)
 
-    ## Wait for the process to finish
-    mpi_return_code = mpi_process.wait()
+        ## Wait for the process to finish
+        mpi_return_code = mpi_process.wait()
+        if mpi_return_code != 0:
+            raise subprocess.CalledProcessError(mpi_return_code, mpi_command)
+    finally:
+        if started_mps:
+            _stop_cuda_mps(mpi_env)
     
     print(                                           )
     print( " The ABE/ABEGPU simulation is finished " ) 

plot_xiaoqu.py

@@ -822,8 +822,21 @@ def generate_constraint_check_plot( outdir, figure_outdir, input_level_number ):
 
     print( " Begin the constraint violation plot for grid level number =  ", input_level_number )
 
+    if (not os.path.exists(file0)) or os.path.getsize(file0) == 0:
+        if ( input_level_number == 0 ):
+            print( " Constraint data file is empty; skip constraint violation plots" )
+            print(                                                            )
+        return
+
     # load the full data file (assumed whitespace-separated floats)
     data = numpy.loadtxt(file0)
+    data = numpy.atleast_2d(data)
+
+    if data.shape[1] < 8:
+        if ( input_level_number == 0 ):
+            print( " Constraint data file has insufficient columns; skip constraint violation plots" )
+            print(                                                                                )
+        return
 
     # extract columns from the constraint data file
     time          = data[:,0]