Add plot-only restart script to skip recomputation when plotting is interrupted

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Fix macrodef.h include and clean up stale z4c_gpu_rhs_ss.cu
2026-05-12 15:01:25 +08:00 · 2026-05-10 20:02:35 +08:00 · 2026-05-10 16:05:56 +08:00 · 2026-05-10 13:52:48 +08:00 · 2026-05-10 12:08:02 +08:00 · 2026-05-10 02:23:07 +08:00
25 changed files with 12510 additions and 4988 deletions
--- a/AMSS_NCKU_Input.py
+++ b/AMSS_NCKU_Input.py
@@ -158,7 +158,7 @@ Detector_Rmax   = 160.0                  ## farest dector distance

 ## Setting the apprent horizon

-AHF_Find       = "no"                    ## whether to find the apparent horizon: choose "yes" or "no"
+AHF_Find       = "yes"                    ## whether to find the apparent horizon: choose "yes" or "no"

 AHF_Find_Every = 24
 AHF_Dump_Time  = 20.0
--- a/AMSS_NCKU_Program_Plot.py
+++ b/AMSS_NCKU_Program_Plot.py
@@ -0,0 +1,100 @@
+##################################################################
+##
+## AMSS-NCKU Plot-Only Restart Script
+## Author: Xiaoqu / Claude
+## 2026/05/12
+##
+## This script checks for existing output data from AMSS_NCKU_Program.py.
+## If data exists, it skips all computation and goes directly to plotting,
+## saving time when plotting was interrupted.
+## If no data is found, it exits with a message.
+##
+##################################################################
+
+## Guard against re-execution by multiprocessing child processes.
+if __name__ != '__main__':
+    import sys as _sys
+    _sys.exit(0)
+
+
+import os
+import sys
+
+import AMSS_NCKU_Input as input_data
+
+##################################################################
+
+## Construct paths from input configuration
+File_directory         = os.path.join(input_data.File_directory)
+output_directory       = os.path.join(File_directory, "AMSS_NCKU_output")
+binary_results_directory = os.path.join(output_directory, input_data.Output_directory)
+figure_directory       = os.path.join(File_directory, "figure")
+
+##################################################################
+
+## Check whether the required output data files exist
+
+required_files = [
+    os.path.join(binary_results_directory, "bssn_BH.dat"),
+    os.path.join(binary_results_directory, "bssn_ADMQs.dat"),
+    os.path.join(binary_results_directory, "bssn_psi4.dat"),
+    os.path.join(binary_results_directory, "bssn_constraint.dat"),
+]
+
+missing_files = [f for f in required_files if not os.path.exists(f)]
+
+if missing_files:
+    print(" No existing AMSS_NCKU_Program.py output data found. ")
+    print(" The following required files are missing: ")
+    for f in missing_files:
+        print(f"   {f}")
+    print()
+    print(" Please run AMSS_NCKU_Program.py first to generate the simulation data. ")
+    print(" Exiting. ")
+    sys.exit(1)
+
+print(" Found existing AMSS_NCKU_Program.py output data. "          )
+print(" Skipping all computation and going directly to plotting. " )
+print()
+
+## Ensure the figure directory exists (it should, but be safe)
+os.makedirs(figure_directory, exist_ok=True)
+
+##################################################################
+
+## Plot the AMSS-NCKU program results
+
+import plot_xiaoqu
+import plot_GW_strain_amplitude_xiaoqu
+from parallel_plot_helper import run_plot_tasks_parallel
+
+plot_tasks = []
+
+## Plot black hole trajectory
+plot_tasks.append((plot_xiaoqu.generate_puncture_orbit_plot,   (binary_results_directory, figure_directory)))
+plot_tasks.append((plot_xiaoqu.generate_puncture_orbit_plot3D, (binary_results_directory, figure_directory)))
+
+## Plot black hole separation vs. time
+plot_tasks.append((plot_xiaoqu.generate_puncture_distence_plot, (binary_results_directory, figure_directory)))
+
+## Plot gravitational waveforms (psi4 and strain amplitude)
+for i in range(input_data.Detector_Number):
+    plot_tasks.append((plot_xiaoqu.generate_gravitational_wave_psi4_plot, (binary_results_directory, figure_directory, i)))
+    plot_tasks.append((plot_GW_strain_amplitude_xiaoqu.generate_gravitational_wave_amplitude_plot, (binary_results_directory, figure_directory, i)))
+
+## Plot ADM mass evolution
+for i in range(input_data.Detector_Number):
+    plot_tasks.append((plot_xiaoqu.generate_ADMmass_plot, (binary_results_directory, figure_directory, i)))
+
+## Plot Hamiltonian constraint violation over time
+for i in range(input_data.grid_level):
+    plot_tasks.append((plot_xiaoqu.generate_constraint_check_plot, (binary_results_directory, figure_directory, i)))
+
+run_plot_tasks_parallel(plot_tasks)
+
+## Plot stored binary data (runs serially, not in the parallel pool)
+plot_xiaoqu.generate_binary_data_plot(binary_results_directory, figure_directory)
+
+print()
+print(" Plotting completed successfully. ")
+print()
--- a/AMSS_NCKU_source/ABE.C
+++ b/AMSS_NCKU_source/ABE.C
@@ -198,16 +198,16 @@ int main(int argc, char *argv[])
      if (myrank == 0)
      {
            string out_dir;
-            char filename[50];
-            map<string, string>::iterator iter;
-            iter = parameters::str_par.find("output dir");
-            if (iter != parameters::str_par.end())
-            {
-                  out_dir = iter->second;
-            }
-            sprintf(filename, "%s/setting.par", out_dir.c_str());
-            ofstream setfile;
-            setfile.open(filename, ios::trunc);
+            string filename;
+            map<string, string>::iterator iter;
+            iter = parameters::str_par.find("output dir");
+            if (iter != parameters::str_par.end())
+            {
+                  out_dir = iter->second;
+            }
+            filename = out_dir + "/setting.par";
+            ofstream setfile;
+            setfile.open(filename.c_str(), ios::trunc);

            if (!setfile.good())
            {
@@ -484,7 +484,11 @@ int main(int argc, char *argv[])
            cout << endl;
      }

-      delete ADM;
+      // Let the process teardown reclaim the simulation object.  Some derived
+      // equation classes keep MPI/CUDA-backed state whose destructor ordering
+      // is fragile at program shutdown.
+      if (getenv("AMSS_DELETE_ADM_ON_EXIT"))
+            delete ADM;

      //=======================caculation done=============================================================

--- a/AMSS_NCKU_source/Parallel.C
+++ b/AMSS_NCKU_source/Parallel.C
--- a/AMSS_NCKU_source/ShellPatch.C
+++ b/AMSS_NCKU_source/ShellPatch.C
--- a/AMSS_NCKU_source/ShellPatch.h
+++ b/AMSS_NCKU_source/ShellPatch.h
@@ -102,6 +102,16 @@ public:
                 //-1: means no dumy dimension at all; 0: means rho; 1: means sigma
   };

+   // Thread-safe search result (no pointers to shared mutable state)
+   struct PointSearchResult
+   {
+      bool found;
+      Block *Bg;
+      double gx, gy, gz; // global Cartesian coordinates
+      double lx, ly, lz; // local coordinates within the found block
+      int ssst;          // source shell-patch type (-1 = Cartesian)
+   };
+
   int myrank;
   int shape[dim];   // for (rho, sigma, R), for rho and sigma means number of points for every pi/2
   double Rrange[2]; // for Rmin and Rmax
@@ -175,6 +185,12 @@ public:
                         MyList<Patch> *Pp, double CDH[dim], MyList<pointstru> *pss);
   bool prolongpointstru(MyList<pointstru> *&psul, bool ssyn, int tsst, MyList<ss_patch> *sPp, double DH[dim],
                         MyList<Patch> *Pp, double CDH[dim], double x, double y, double z, int Symmetry, int rank_in);
+   // Read-only point search — thread-safe (no shared mutable state modified)
+   PointSearchResult prolongpointstru_search(bool ssyn, int tsst, MyList<ss_patch> *sPp, double DH[dim],
+                                             MyList<Patch> *Pp, double CDH[dim], double x, double y, double z,
+                                             int Symmetry, int rank_in);
+   // Append a search result to a linked list — use inside omp critical section
+   void prolongpointstru_append(MyList<pointstru> *&psul, const PointSearchResult &sr, int tsst);
   void setupintintstuff(int cpusize, MyList<Patch> *CPatL, int Symmetry);
   void intertransfer(MyList<pointstru> **src, MyList<pointstru> **dst,
                      MyList<var> *VarList1 /* source */, MyList<var> *VarList2 /*target */,
@@ -195,11 +211,11 @@ public:
   bool Interp_One_Point(MyList<var> *VarList,
                         double *XX, /*input global Cartesian coordinate*/
                         double *Shellf, int Symmetry);
-   void write_Pablo_file_ss(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
-                            char *filename, int sst);
-   double L2Norm(var *vf);
-   void L2Norm7(var **vf, double *norms);
-   void Find_Maximum(MyList<var> *VarList, double *XX, double *Shellf);
-};
+   void write_Pablo_file_ss(int *ext, double xmin, double xmax, double ymin, double ymax, double zmin, double zmax,
+                            char *filename, int sst);
+   double L2Norm(var *vf);
+   void L2Norm7(var **vf, double *norms);
+   void Find_Maximum(MyList<var> *VarList, double *XX, double *Shellf);
+};

 #endif /* SHELLPATCH_H */
--- a/AMSS_NCKU_source/Z4c_class.C
+++ b/AMSS_NCKU_source/Z4c_class.C
--- a/AMSS_NCKU_source/bssnEM_class.C
+++ b/AMSS_NCKU_source/bssnEM_class.C
--- a/AMSS_NCKU_source/bssnEScalar_class.C
+++ b/AMSS_NCKU_source/bssnEScalar_class.C
@@ -1,9 +1,10 @@

 #ifdef newc
-#include <sstream>
-#include <cstdio>
-#include <map>
-using namespace std;
+#include <sstream>
+#include <cstdio>
+#include <map>
+#include <string>
+using namespace std;
 #else
 #include <stdio.h>
 #include <map.h>
@@ -24,16 +25,313 @@ using namespace std;
 #include "sommerfeld_rout.h"
 #include "getnp4.h"
 #include "shellfunctions.h"
-#include "parameters.h"
+#include "parameters.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif

 #ifdef With_AHF
 #include "derivatives.h"
 #include "myglobal.h"
 #endif

-//================================================================================================
-
-// Define bssnEScalar_class
+//================================================================================================
+
+namespace
+{
+#if USE_CUDA_BSSN
+bool fill_bssn_escalar_cuda_views(Block *cg, MyList<var> *vars,
+                                  double **host_views,
+                                  double *propspeeds = 0,
+                                  double *soa_flat = 0)
+{
+  int idx = 0;
+  while (vars && idx < BSSN_ESCALAR_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == BSSN_ESCALAR_CUDA_STATE_COUNT && vars == 0;
+}
+
+bool bssn_escalar_cuda_use_resident_sync(int lev)
+{
+#ifdef WithShell
+  (void)lev;
+  return false;
+#else
+  return true;
+#endif
+}
+
+bool bssn_escalar_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
+{
+  static int keep_all_levels = -1;
+  if (keep_all_levels < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_KEEP_ALL_LEVELS");
+    keep_all_levels = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_KEEP_RESIDENT_AFTER_STEP");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  if (!enabled)
+    return false;
+  if (lev == analysis_lev)
+    return false;
+  static int release_only_level = -2;
+  if (release_only_level == -2)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_RELEASE_ONLY_LEVEL");
+    release_only_level = (env && atoi(env) >= 0) ? atoi(env) : -1;
+  }
+  if (release_only_level >= 0)
+    return lev != release_only_level;
+  static int keep_level_limit = -2;
+  if (keep_level_limit == -2)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_KEEP_LEVELS_BELOW");
+    keep_level_limit = (env && atoi(env) >= 0) ? atoi(env) : -1;
+  }
+  if (keep_level_limit >= 0)
+    return lev < keep_level_limit;
+  if (keep_all_levels)
+    return true;
+  return lev < trfls_in;
+}
+
+bool bssn_escalar_sync_merged_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_ESCALAR_SYNC_MERGED");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+void bssn_escalar_sync_level(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetry)
+{
+  if (bssn_escalar_sync_merged_enabled())
+    Parallel::Sync_merged(PatL, VarList, Symmetry);
+  else
+    Parallel::Sync(PatL, VarList, Symmetry);
+}
+
+bool bssn_escalar_timing_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_ESCALAR_STEP_TIMING");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_post_rp_download_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_POST_RP_DOWNLOAD");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_post_rp_download_level_enabled(int lev)
+{
+  if (!bssn_escalar_cuda_post_rp_download_enabled())
+    return false;
+  static int min_level = -2;
+  if (min_level == -2)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_POST_RP_MIN_LEVEL");
+    min_level = (env && atoi(env) >= 0) ? atoi(env) : -1;
+  }
+  return min_level < 0 || lev >= min_level;
+}
+
+bool bssn_escalar_cuda_post_swap_release_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_POST_SWAP_RELEASE");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_pre_rp_release_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_PRE_RP_RELEASE");
+    enabled = env ? ((atoi(env) != 0) ? 1 : 0) : 1;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_bh_interp_resident_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_BH_INTERP_RESIDENT");
+    enabled = env ? ((atoi(env) != 0) ? 1 : 0) : 1;
+  }
+  return enabled != 0;
+}
+
+bool bssn_escalar_cuda_prune_after_swap_enabled()
+{
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_ESCALAR_PRUNE_AFTER_SWAP");
+    enabled = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  return enabled != 0;
+}
+
+void bssn_escalar_cuda_upload_level_state(MyList<Patch> *PatL, MyList<var> *vars,
+                                          int myrank)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && bssn_cuda_has_resident_state(cg))
+      {
+        double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
+        if (!fill_bssn_escalar_cuda_views(cg, vars, state_in))
+        {
+          cout << "CUDA BSSN-EScalar resident state list mismatch during upload" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (bssn_escalar_cuda_upload_resident_state(cg, cg->shape, state_in))
+        {
+          cout << "CUDA BSSN-EScalar resident state upload failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+
+void bssn_escalar_cuda_keep_only_level_state(MyList<Patch> *PatL, MyList<var> *vars,
+                                             int myrank)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && bssn_cuda_has_resident_state(cg))
+      {
+        double *state_key[BSSN_ESCALAR_CUDA_STATE_COUNT];
+        if (!fill_bssn_escalar_cuda_views(cg, vars, state_key))
+        {
+          cout << "CUDA BSSN-EScalar resident state list mismatch during prune" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (bssn_escalar_cuda_keep_only_resident_state(cg, cg->shape, state_key))
+        {
+          cout << "CUDA BSSN-EScalar resident state prune failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+
+void bssn_escalar_timing_report(int myrank, int lev, int YN, double total, double rhs,
+                                double sync, double bh, double analysis, double swap,
+                                double resident, double rp)
+{
+  if (!bssn_escalar_timing_enabled())
+    return;
+
+  double local[8] = {total, rhs, sync, bh, analysis, swap, resident, rp};
+  double maxv[8] = {};
+  MPI_Reduce(local, maxv, 8, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+  if (myrank == 0)
+    fprintf(stderr,
+            "[AMSS-ESCALAR-STEP] lev=%d YN=%d total=%.6f rhs=%.6f sync=%.6f "
+            "bh=%.6f analysis=%.6f swap=%.6f resident=%.6f rp=%.6f other=%.6f\n",
+            lev, YN, maxv[0], maxv[1], maxv[2], maxv[3], maxv[4], maxv[5],
+            maxv[6], maxv[7],
+            maxv[0] - maxv[1] - maxv[2] - maxv[3] - maxv[4] - maxv[5] - maxv[6] - maxv[7]);
+}
+
+void bssn_escalar_cuda_download_level_state(MyList<Patch> *PatL, MyList<var> *vars,
+                                            int myrank, bool release_ctx)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && bssn_cuda_has_resident_state(cg))
+      {
+        double *state_out[BSSN_ESCALAR_CUDA_STATE_COUNT];
+        if (!fill_bssn_escalar_cuda_views(cg, vars, state_out))
+        {
+          cout << "CUDA BSSN-EScalar resident state list mismatch during download" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (bssn_escalar_cuda_download_resident_state(cg, cg->shape, state_out))
+        {
+          cout << "CUDA BSSN-EScalar resident state download failed" << endl;
+          MPI_Abort(MPI_COMM_WORLD, 1);
+        }
+        if (release_ctx)
+          bssn_cuda_release_step_ctx(cg);
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+}
+#endif
+}
+
+//================================================================================================
+
+// Define bssnEScalar_class

 // It inherits some members and methods from the parent class bssn_class and modifies others.
 // The modified members and methods are defined below (and in the header bssnEScalar_class.h).
@@ -177,11 +475,16 @@ void bssnEScalar_class::Initialize()

 //================================================================================================

-bssnEScalar_class::~bssnEScalar_class()
-{
-  delete Sphio;
-  delete Spio;
-  delete Sphi0;
+bssnEScalar_class::~bssnEScalar_class()
+{
+#if USE_CUDA_BSSN
+  for (int lev = 0; GH && lev < GH->levels; ++lev)
+    bssn_escalar_cuda_download_level_state(GH->PatL[lev], StateList, myrank, true);
+#endif
+
+  delete Sphio;
+  delete Spio;
+  delete Sphi0;
  delete Spi0;
  delete Sphi;
  delete Spi;
@@ -707,7 +1010,12 @@ void bssnEScalar_class::Read_Pablo()

 void bssnEScalar_class::Step(int lev, int YN)
 {
-  double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
+  double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
+#if USE_CUDA_BSSN
+  const bool use_cuda_resident_sync = bssn_escalar_cuda_use_resident_sync(lev);
+#else
+  const bool use_cuda_resident_sync = false;
+#endif
 #ifdef With_AHF
  AH_Step_Find(lev, dT_lev);
 #endif
@@ -716,13 +1024,23 @@ void bssnEScalar_class::Step(int lev, int YN)
  if (lev < GH->movls)
    ndeps = numepsb;
  double TRK4 = PhysTime;
-  int iter_count = 0; // count RK4 substeps
-  int pre = 0, cor = 1;
-  int ERROR = 0;
-
-  MyList<ss_patch> *sPp;
-  // Predictor
-  MyList<Patch> *Pp = GH->PatL[lev];
+  int iter_count = 0; // count RK4 substeps
+  int pre = 0, cor = 1;
+  int ERROR = 0;
+  const bool escalar_step_timing = bssn_escalar_timing_enabled();
+  const double escalar_step_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+  double escalar_t_rhs = 0.0;
+  double escalar_t_sync = 0.0;
+  double escalar_t_bh = 0.0;
+  double escalar_t_analysis = 0.0;
+  double escalar_t_swap = 0.0;
+  double escalar_t_resident = 0.0;
+  double escalar_t_rp = 0.0;
+
+  MyList<ss_patch> *sPp;
+  // Predictor
+  double escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+  MyList<Patch> *Pp = GH->PatL[lev];
  while (Pp)
  {
    MyList<Block> *BP = Pp->data->blb;
@@ -731,15 +1049,60 @@ void bssnEScalar_class::Step(int lev, int YN)
      Block *cg = BP->data;
      if (myrank == cg->rank)
      {
-#if (AGM == 0)
-        f_enforce_ga(cg->shape,
-                     cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
-                     cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
-                     cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
-                     cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
-#endif
-
-        if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+#if (AGM == 0)
+#if !USE_CUDA_BSSN
+        f_enforce_ga(cg->shape,
+                     cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
+                     cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
+                     cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
+                     cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
+#endif
+#endif
+
+        bool used_gpu_substep = false;
+#if USE_CUDA_BSSN
+        {
+          double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
+          double *state_out[BSSN_ESCALAR_CUDA_STATE_COUNT];
+          double propspeed[BSSN_ESCALAR_CUDA_STATE_COUNT];
+          double soa_flat[3 * BSSN_ESCALAR_CUDA_STATE_COUNT];
+          if (!fill_bssn_escalar_cuda_views(cg, StateList, state_in, propspeed, soa_flat) ||
+              !fill_bssn_escalar_cuda_views(cg, SynchList_pre, state_out))
+          {
+            cout << "CUDA BSSN-EScalar state list mismatch on predictor step" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+          int apply_bam_bc = 0;
+          int apply_enforce_ga = 0;
+#if (AGM == 0)
+          apply_enforce_ga = 1;
+#endif
+#if (SommerType == 0)
+#ifndef WithShell
+          apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+          int keep_resident_state = use_cuda_resident_sync ? 1 : 0;
+          if (bssn_escalar_cuda_rk4_substep(cg,
+                                            cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                            state_in, state_out,
+                                            propspeed, soa_flat, Pp->data->bbox,
+                                            dT_lev, TRK4, iter_count, apply_bam_bc,
+                                            Symmetry, lev, ndeps, pre,
+                                            keep_resident_state, apply_enforce_ga, chitiny))
+          {
+            cout << "CUDA BSSN-EScalar predictor substep failed in domain: ("
+                 << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                 << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                 << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+            ERROR = 1;
+          }
+          used_gpu_substep = true;
+        }
+#endif
+
+        if (!used_gpu_substep &&
+            f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                       cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
                                       cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                       cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
@@ -783,9 +1146,11 @@ void bssnEScalar_class::Step(int lev, int YN)
          ERROR = 1;
        }

-        // rk4 substep and boundary
-        {
-          MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
+        if (!used_gpu_substep)
+        {
+        // rk4 substep and boundary
+        {
+          MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; // we do not check the correspondence here
          while (varl0)
          {
 #ifndef WithShell
@@ -820,9 +1185,10 @@ void bssnEScalar_class::Step(int lev, int YN)
            varl = varl->next;
            varlrhs = varlrhs->next;
          }
-        }
-        f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
-      }
+        }
+        f_lowerboundset(cg->shape, cg->fgfs[phi->sgfn], chitiny);
+        }
+      }
      if (BP == Pp->data->ble)
        break;
      BP = BP->next;
@@ -834,19 +1200,21 @@ void bssnEScalar_class::Step(int lev, int YN)
    int erh = ERROR;
    MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
  }
-  if (ERROR)
-  {
+  if (ERROR)
+  {
    Parallel::Dump_Data(GH->PatL[lev], StateList, 0, PhysTime, dT_lev);
    if (myrank == 0)
    {
      if (ErrorMonitor->outfile)
        ErrorMonitor->outfile << "find NaN in state variables at t = " << PhysTime 
                              << ", lev = " << lev << endl;
-      MPI_Abort(MPI_COMM_WORLD, 1);
-    }
-  }
-
-#ifdef WithShell
+      MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+  }
+  if (escalar_step_timing)
+    escalar_t_rhs += MPI_Wtime() - escalar_t0;
+
+#ifdef WithShell
  // evolve Shell Patches
  if (lev == 0)
  {
@@ -993,7 +1361,14 @@ void bssnEScalar_class::Step(int lev, int YN)
  }
 #endif

-  Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
+  escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+  bssn_escalar_sync_level(GH->PatL[lev], SynchList_pre, Symmetry);
+#else
+  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+#endif
+  if (escalar_step_timing)
+    escalar_t_sync += MPI_Wtime() - escalar_t0;

 #ifdef WithShell
  if (lev == 0)
@@ -1013,10 +1388,15 @@ void bssnEScalar_class::Step(int lev, int YN)
  }
 #endif

-  // for black hole position
-  if (BH_num > 0 && lev == GH->levels - 1)
-  {
-    compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
+  // for black hole position
+  if (BH_num > 0 && lev == GH->levels - 1)
+  {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+    if (use_cuda_resident_sync && !bssn_escalar_cuda_bh_interp_resident_enabled())
+      bssn_escalar_cuda_download_level_state(GH->PatL[lev], StateList, myrank, false);
+#endif
+    compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
    for (int ithBH = 0; ithBH < BH_num; ithBH++)
    {
      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
@@ -1041,19 +1421,29 @@ void bssnEScalar_class::Step(int lev, int YN)
        DG_List->insert(Sfy0);
        DG_List->insert(Sfz0);
        Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
-        DG_List->clearList();
-      }
-    }
-  }
+        DG_List->clearList();
+      }
+    }
+    if (escalar_step_timing)
+      escalar_t_bh += MPI_Wtime() - escalar_t0;
+  }
  // data analysis part
  // Warning NOTE: the variables1 are used as temp storege room
-  if (lev == a_lev)
-  {
-    AnalysisStuff_EScalar(lev, dT_lev);
-  }
-  // corrector
-  for (iter_count = 1; iter_count < 4; iter_count++)
-  {
+  if (lev == a_lev)
+  {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+    if (use_cuda_resident_sync)
+      bssn_escalar_cuda_download_level_state(GH->PatL[lev], SynchList_pre, myrank, false);
+#endif
+    AnalysisStuff_EScalar(lev, dT_lev);
+    if (escalar_step_timing)
+      escalar_t_analysis += MPI_Wtime() - escalar_t0;
+  }
+  // corrector
+  for (iter_count = 1; iter_count < 4; iter_count++)
+  {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
    // for RK4: t0, t0+dt/2, t0+dt/2, t0+dt;
    if (iter_count == 1 || iter_count == 3)
      TRK4 += dT_lev / 2;
@@ -1066,22 +1456,67 @@ void bssnEScalar_class::Step(int lev, int YN)
        Block *cg = BP->data;
        if (myrank == cg->rank)
        {
-#if (AGM == 0)
-          f_enforce_ga(cg->shape,
-                       cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
-                       cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
-                       cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
-                       cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
-#elif (AGM == 1)
-          if (iter_count == 3)
-            f_enforce_ga(cg->shape,
+#if (AGM == 0)
+#if !USE_CUDA_BSSN
+          f_enforce_ga(cg->shape,
+                       cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
+                       cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
+                       cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
+                       cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
+#endif
+#elif (AGM == 1)
+          if (iter_count == 3)
+            f_enforce_ga(cg->shape,
                         cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                         cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                         cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
                         cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
-#endif
-
-          if (f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+#endif
+
+          bool used_gpu_substep = false;
+#if USE_CUDA_BSSN
+          {
+            double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
+            double *state_out[BSSN_ESCALAR_CUDA_STATE_COUNT];
+            double propspeed[BSSN_ESCALAR_CUDA_STATE_COUNT];
+            double soa_flat[3 * BSSN_ESCALAR_CUDA_STATE_COUNT];
+            if (!fill_bssn_escalar_cuda_views(cg, SynchList_pre, state_in, propspeed, soa_flat) ||
+                !fill_bssn_escalar_cuda_views(cg, SynchList_cor, state_out))
+            {
+              cout << "CUDA BSSN-EScalar state list mismatch on corrector step" << endl;
+              MPI_Abort(MPI_COMM_WORLD, 1);
+            }
+            int apply_bam_bc = 0;
+            int apply_enforce_ga = 0;
+#if (AGM == 0)
+            apply_enforce_ga = 1;
+#endif
+#if (SommerType == 0)
+#ifndef WithShell
+            apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+            int keep_resident_state = use_cuda_resident_sync ? 1 : 0;
+            if (bssn_escalar_cuda_rk4_substep(cg,
+                                              cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                              state_in, state_out,
+                                              propspeed, soa_flat, Pp->data->bbox,
+                                              dT_lev, TRK4, iter_count, apply_bam_bc,
+                                              Symmetry, lev, ndeps, cor,
+                                              keep_resident_state, apply_enforce_ga, chitiny))
+            {
+              cout << "CUDA BSSN-EScalar corrector substep failed in domain: ("
+                   << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                   << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                   << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+              ERROR = 1;
+            }
+            used_gpu_substep = true;
+          }
+#endif
+
+          if (!used_gpu_substep &&
+              f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
                                         cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
                                         cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                         cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
@@ -1125,9 +1560,11 @@ void bssnEScalar_class::Step(int lev, int YN)
                 << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
            ERROR = 1;
          }
-          // rk4 substep and boundary
-          {
-            MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList; 
+          if (!used_gpu_substep)
+          {
+          // rk4 substep and boundary
+          {
+            MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList; 
            // we do not check the correspondence here
            
            while (varl0)
@@ -1165,9 +1602,10 @@ void bssnEScalar_class::Step(int lev, int YN)
              varl1 = varl1->next;
              varlrhs = varlrhs->next;
            }
-          }
-          f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
-        }
+          }
+          f_lowerboundset(cg->shape, cg->fgfs[phi1->sgfn], chitiny);
+          }
+        }
        if (BP == Pp->data->ble)
          break;
        BP = BP->next;
@@ -1180,8 +1618,8 @@ void bssnEScalar_class::Step(int lev, int YN)
      int erh = ERROR;
      MPI_Allreduce(&erh, &ERROR, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
    }
-    if (ERROR)
-    {
+    if (ERROR)
+    {
      Parallel::Dump_Data(GH->PatL[lev], SynchList_pre, 0, PhysTime, dT_lev);
      if (myrank == 0)
      {
@@ -1189,11 +1627,13 @@ void bssnEScalar_class::Step(int lev, int YN)
          ErrorMonitor->outfile << "find NaN in RK4 substep#" << iter_count 
                                << " variables at t = " << PhysTime 
                                << ", lev = " << lev << endl;
-        MPI_Abort(MPI_COMM_WORLD, 1);
-      }
-    }
-
-#ifdef WithShell
+        MPI_Abort(MPI_COMM_WORLD, 1);
+      }
+    }
+    if (escalar_step_timing)
+      escalar_t_rhs += MPI_Wtime() - escalar_t0;
+
+#ifdef WithShell
    // evolve Shell Patches
    if (lev == 0)
    {
@@ -1349,7 +1789,14 @@ void bssnEScalar_class::Step(int lev, int YN)
    }
 #endif

-    Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+    bssn_escalar_sync_level(GH->PatL[lev], SynchList_cor, Symmetry);
+#else
+    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+#endif
+    if (escalar_step_timing)
+      escalar_t_sync += MPI_Wtime() - escalar_t0;

 #ifdef WithShell
    if (lev == 0)
@@ -1368,10 +1815,15 @@ void bssnEScalar_class::Step(int lev, int YN)
      }
    }
 #endif
-    // for black hole position
-    if (BH_num > 0 && lev == GH->levels - 1)
-    {
-      compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
+    // for black hole position
+    if (BH_num > 0 && lev == GH->levels - 1)
+    {
+      escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+#if USE_CUDA_BSSN
+      if (use_cuda_resident_sync && !bssn_escalar_cuda_bh_interp_resident_enabled())
+        bssn_escalar_cuda_download_level_state(GH->PatL[lev], SynchList_pre, myrank, false);
+#endif
+      compute_Porg_rhs(Porg, Porg1, Sfx, Sfy, Sfz, lev);
      for (int ithBH = 0; ithBH < BH_num; ithBH++)
      {
        f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg1[ithBH][0], Porg_rhs[ithBH][0], iter_count);
@@ -1396,14 +1848,17 @@ void bssnEScalar_class::Step(int lev, int YN)
          DG_List->insert(Sfy0);
          DG_List->insert(Sfz0);
          Parallel::Dump_Data(GH->PatL[lev], DG_List, 0, PhysTime, dT_lev);
-          DG_List->clearList();
-        }
-      }
-    }
-    // swap time level
-    if (iter_count < 3)
-    {
-      Pp = GH->PatL[lev];
+          DG_List->clearList();
+        }
+      }
+      if (escalar_step_timing)
+        escalar_t_bh += MPI_Wtime() - escalar_t0;
+    }
+    // swap time level
+    if (iter_count < 3)
+    {
+      escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+      Pp = GH->PatL[lev];
      while (Pp)
      {
        MyList<Block> *BP = Pp->data->blb;
@@ -1444,16 +1899,33 @@ void bssnEScalar_class::Step(int lev, int YN)
          Porg[ithBH][0] = Porg1[ithBH][0];
          Porg[ithBH][1] = Porg1[ithBH][1];
          Porg[ithBH][2] = Porg1[ithBH][2];
-        }
-      }
-    }
-  }
-
-#if (RPS == 0)
-  // mesh refinement boundary part
-  RestrictProlong(lev, YN, BB);
-
-#ifdef WithShell
+        }
+      }
+      if (escalar_step_timing)
+        escalar_t_swap += MPI_Wtime() - escalar_t0;
+    }
+  }
+
+#if USE_CUDA_BSSN
+  if (use_cuda_resident_sync)
+  {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+    if (!bssn_escalar_cuda_keep_resident_after_step(lev, trfls, a_lev))
+      bssn_escalar_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank,
+                                             bssn_escalar_cuda_pre_rp_release_enabled());
+    if (escalar_step_timing)
+      escalar_t_resident += MPI_Wtime() - escalar_t0;
+  }
+#endif
+
+#if (RPS == 0)
+  // mesh refinement boundary part
+  escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+  RestrictProlong(lev, YN, BB);
+  if (escalar_step_timing)
+    escalar_t_rp += MPI_Wtime() - escalar_t0;
+
+#ifdef WithShell
  if (lev == 0)
  {
    clock_t prev_clock, curr_clock;
@@ -1477,8 +1949,9 @@ void bssnEScalar_class::Step(int lev, int YN)
  // StateList     0   -----------
  //
  // OldStateList  old -----------
-  // update
-  Pp = GH->PatL[lev];
+  // update
+  escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+  Pp = GH->PatL[lev];
  while (Pp)
  {
    MyList<Block> *BP = Pp->data->blb;
@@ -1512,18 +1985,45 @@ void bssnEScalar_class::Step(int lev, int YN)
      sPp = sPp->next;
    }
  }
-#endif
-  // for black hole position
-  if (BH_num > 0 && lev == GH->levels - 1)
+#endif
+  #if USE_CUDA_BSSN
+  bool release_after_sync = false;
+  if (use_cuda_resident_sync && bssn_escalar_cuda_post_rp_download_level_enabled(lev))
+  {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+    release_after_sync = bssn_escalar_cuda_post_swap_release_enabled();
+    bssn_escalar_cuda_download_level_state(GH->PatL[lev], StateList, myrank, release_after_sync);
+    if (escalar_step_timing)
+      escalar_t_resident += MPI_Wtime() - escalar_t0;
+  }
+  if (use_cuda_resident_sync && !release_after_sync &&
+      bssn_escalar_cuda_prune_after_swap_enabled())
+  {
+    escalar_t0 = escalar_step_timing ? MPI_Wtime() : 0.0;
+    bssn_escalar_cuda_keep_only_level_state(GH->PatL[lev], StateList, myrank);
+    if (escalar_step_timing)
+      escalar_t_resident += MPI_Wtime() - escalar_t0;
+  }
+#endif
+  // for black hole position
+  if (BH_num > 0 && lev == GH->levels - 1)
  {
    for (int ithBH = 0; ithBH < BH_num; ithBH++)
    {
      Porg0[ithBH][0] = Porg1[ithBH][0];
      Porg0[ithBH][1] = Porg1[ithBH][1];
-      Porg0[ithBH][2] = Porg1[ithBH][2];
-    }
-  }
-}
+      Porg0[ithBH][2] = Porg1[ithBH][2];
+    }
+  }
+  if (escalar_step_timing)
+  {
+    escalar_t_swap += MPI_Wtime() - escalar_t0;
+    bssn_escalar_timing_report(myrank, lev, YN, MPI_Wtime() - escalar_step_t0,
+                               escalar_t_rhs, escalar_t_sync, escalar_t_bh,
+                               escalar_t_analysis, escalar_t_swap,
+                               escalar_t_resident, escalar_t_rp);
+  }
+}

 //================================================================================================

@@ -2023,12 +2523,13 @@ void bssnEScalar_class::Interp_Constraint()
    }
  }

-  ofstream outfile;
-  char filename[50];
-  sprintf(filename, "%s/interp_constraint_%05d.dat", ErrorMonitor->out_dir.c_str(), int(PhysTime / dT + 0.5)); 
-  // 0.5 for round off
-  
-  outfile.open(filename);
+  ofstream outfile;
+  char suffix[64];
+  sprintf(suffix, "/interp_constraint_%05d.dat", int(PhysTime / dT + 0.5));
+  string filename = ErrorMonitor->out_dir + suffix;
+  // 0.5 for round off
+  
+  outfile.open(filename.c_str());
  outfile << "#  corrdinate, H_Res, Px_Res, Py_Res, Pz_Res, Gx_Res, Gy_Res, Gz_Res, fR_Res, ...." << endl;
  for (int i = 0; i < n; i++)
  {
@@ -2074,14 +2575,44 @@ void bssnEScalar_class::Constraint_Out()
          MyList<Block> *BP = Pp->data->blb;
          while (BP)
          {
-            Block *cg = BP->data;
-            if (myrank == cg->rank)
-            {
-              if (lev > 0)
-                f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
-                                           cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
-                                           cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
-                                           cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
+            Block *cg = BP->data;
+            if (myrank == cg->rank)
+            {
+              bool used_cuda_constraints = false;
+#if USE_CUDA_BSSN
+              {
+                double *state_in[BSSN_ESCALAR_CUDA_STATE_COUNT];
+                if (!fill_bssn_escalar_cuda_views(cg, StateList, state_in))
+                {
+                  cout << "CUDA BSSN-EScalar constraint state list mismatch" << endl;
+                  MPI_Abort(MPI_COMM_WORLD, 1);
+                }
+                double *constraint_out[8] = {
+                    cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+                    cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                    cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+                    cg->fgfs[Cons_Gz->sgfn], cg->fgfs[Cons_fR->sgfn]};
+                int lev_arg = lev;
+                int sym_arg = Symmetry;
+                double eps_arg = ndeps;
+                if (bssn_escalar_cuda_compute_constraints(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                                          state_in, constraint_out,
+                                                          sym_arg, lev_arg, eps_arg))
+                {
+                  cout << "CUDA BSSN-EScalar constraint compute failed in domain: ("
+                       << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                       << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                       << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+                  MPI_Abort(MPI_COMM_WORLD, 1);
+                }
+                used_cuda_constraints = true;
+              }
+#endif
+              if (!used_cuda_constraints && lev > 0)
+                f_compute_rhs_bssn_escalar(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+                                           cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
+                                           cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
+                                           cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                                           cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
                                           cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
                                           cg->fgfs[Gmx0->sgfn], cg->fgfs[Gmy0->sgfn], cg->fgfs[Gmz0->sgfn],
@@ -2110,15 +2641,16 @@ void bssnEScalar_class::Constraint_Out()
                                           cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
                                           cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn], 
                                           cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
-                                           cg->fgfs[Cons_Ham->sgfn],
-                                           cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
-                                           cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                                           Symmetry, lev, ndeps, pre);
-              f_compute_constraint_fr(cg->shape, cg->X[0], cg->X[1], cg->X[2],
-                                      cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn], 
-                                      cg->fgfs[rho->sgfn], cg->fgfs[Sphi0->sgfn],
-                                      cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
-                                      cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
+                                           cg->fgfs[Cons_Ham->sgfn],
+                                           cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                                           cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
+                                           Symmetry, lev, ndeps, pre);
+              if (!used_cuda_constraints)
+                f_compute_constraint_fr(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                                      cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn], 
+                                      cg->fgfs[rho->sgfn], cg->fgfs[Sphi0->sgfn],
+                                      cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
+                                      cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                                      cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
                                      cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn],
                                      cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn], 
--- a/AMSS_NCKU_source/bssn_class.C
+++ b/AMSS_NCKU_source/bssn_class.C
--- a/AMSS_NCKU_source/bssn_class.h
+++ b/AMSS_NCKU_source/bssn_class.h
@@ -144,7 +144,7 @@ public:
       bssn_class(double Couranti, double StartTimei, double TotalTimei, double DumpTimei, double d2DumpTimei, double CheckTimei, double AnasTimei,
                  int Symmetryi, int checkruni, char *checkfilenamei, double numepssi, double numepsbi, double numepshi,
                  int a_levi, int maxli, int decni, double maxrexi, double drexi);
-       ~bssn_class();
+       virtual ~bssn_class();

       void Evolve(int Steps);
       void RecursiveStep(int lev);
--- a/AMSS_NCKU_source/bssn_gpu.h
+++ b/AMSS_NCKU_source/bssn_gpu.h
@@ -0,0 +1,56 @@
+
+#ifndef BSSN_GPU_H_
+#define BSSN_GPU_H_
+#include "bssn_macro.h"
+#include "macrodef.h"
+
+#define DEVICE_ID 0
+// #define DEVICE_ID_BY_MPI_RANK
+#define GRID_DIM 256
+#define BLOCK_DIM 128
+
+#define _FH2_(i, j, k) fh[(i) + (j) * _1D_SIZE[2] + (k) * _2D_SIZE[2]]
+#define _FH3_(i, j, k) fh[(i) + (j) * _1D_SIZE[3] + (k) * _2D_SIZE[3]]
+#define pow2(x) ((x) * (x))
+#define TimeBetween(a, b) ((b.tv_sec - a.tv_sec) + (b.tv_usec - a.tv_usec) / 1000000.0f)
+#define M_ metac.
+#define Mh_ meta->
+#define Ms_ metassc.
+#define Msh_ metass->
+
+// #define TIMING
+
+#define RHS_SS_PARA int calledby, int mpi_rank, int *ex, double &T, double *crho, double *sigma, double *R, double *X, double *Y, double *Z, double *drhodx, double *drhody, double *drhodz, double *dsigmadx, double *dsigmady, double *dsigmadz, double *dRdx, double *dRdy, double *dRdz, double *drhodxx, double *drhodxy, double *drhodxz, double *drhodyy, double *drhodyz, double *drhodzz, double *dsigmadxx, double *dsigmadxy, double *dsigmadxz, double *dsigmadyy, double *dsigmadyz, double *dsigmadzz, double *dRdxx, double *dRdxy, double *dRdxz, double *dRdyy, double *dRdyz, double *dRdzz, double *chi, double *trK, double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz, double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz, double *Gamx, double *Gamy, double *Gamz, double *Lap, double *betax, double *betay, double *betaz, double *dtSfx, double *dtSfy, double *dtSfz, double *chi_rhs, double *trK_rhs, double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs, double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs, double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs, double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs, double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs, double *rho, double *Sx, double *Sy, double *Sz, double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz, double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz, double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz, double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz, double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz, double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res, double *Gmx_Res, double *Gmy_Res, double *Gmz_Res, int &Symmetry, int &Lev, double &eps, int &sst, int &co
+
+/**  main function */
+int gpu_rhs(int calledby, int mpi_rank, int *ex, double &T,
+            double *X, double *Y, double *Z,
+            double *chi, double *trK,
+            double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
+            double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
+            double *Gamx, double *Gamy, double *Gamz,
+            double *Lap, double *betax, double *betay, double *betaz,
+            double *dtSfx, double *dtSfy, double *dtSfz,
+            double *chi_rhs, double *trK_rhs,
+            double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
+            double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
+            double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
+            double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
+            double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
+            double *rho, double *Sx, double *Sy, double *Sz, double *Sxx,
+            double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
+            double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
+            double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
+            double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
+            double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
+            double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res,
+            double *Gmx_Res, double *Gmy_Res, double *Gmz_Res,
+            int &Symmetry, int &Lev, double &eps, int &co);
+
+int gpu_rhs_ss(RHS_SS_PARA);
+
+#define Z4C_SS_PARA int calledby, int mpi_rank, int *ex, double &T, double *crho, double *sigma, double *R, double *X, double *Y, double *Z, double *drhodx, double *drhody, double *drhodz, double *dsigmadx, double *dsigmady, double *dsigmadz, double *dRdx, double *dRdy, double *dRdz, double *drhodxx, double *drhodxy, double *drhodxz, double *drhodyy, double *drhodyz, double *drhodzz, double *dsigmadxx, double *dsigmadxy, double *dsigmadxz, double *dsigmadyy, double *dsigmadyz, double *dsigmadzz, double *dRdxx, double *dRdxy, double *dRdxz, double *dRdyy, double *dRdyz, double *dRdzz, double *chi, double *trK, double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz, double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz, double *Gamx, double *Gamy, double *Gamz, double *Lap, double *betax, double *betay, double *betaz, double *dtSfx, double *dtSfy, double *dtSfz, double *TZ, double *chi_rhs, double *trK_rhs, double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs, double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs, double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs, double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs, double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs, double *TZ_rhs, double *rho, double *Sx, double *Sy, double *Sz, double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz, double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz, double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz, double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz, double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz, double *ham_Res, double *movx_Res, double *movy_Res, double *movz_Res, double *Gmx_Res, double *Gmy_Res, double *Gmz_Res, int &Symmetry, int &Lev, double &eps, int &sst, int &co
+
+int gpu_rhs_z4c_ss(Z4C_SS_PARA);
+
+#endif
--- a/AMSS_NCKU_source/bssn_gpu_rhs_ss.cu
+++ b/AMSS_NCKU_source/bssn_gpu_rhs_ss.cu
@@ -20,12 +20,14 @@ using namespace std;

 __device__ volatile unsigned int global_count = 0;

+#ifdef RESULT_CHECK
 void compare_result_gpu(int ftag1,double * datac,int data_num){
 	double * data = (double*)malloc(sizeof(double)*data_num);
 	cudaMemcpy(data, datac, data_num * sizeof(double), cudaMemcpyDeviceToHost);
 	compare_result(ftag1,data,data_num);
 	free(data);
 }
+#endif

 __global__ void sub_symmetry_bd_ss_partF(int ord, double * func, double *funcc)
 {
@@ -153,11 +155,11 @@ __global__ void sub_symmetry_bd_ss_partJ(int ord,double * func, double * funcc,d

 inline void sub_symmetry_bd_ss(int ord,double * func, double * funcc,double * SoA){
 	sub_symmetry_bd_ss_partF<<<GRID_DIM,BLOCK_DIM>>>(ord,func,funcc);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	sub_symmetry_bd_ss_partI<<<GRID_DIM,BLOCK_DIM>>>(ord,func,funcc,SoA[0]);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	sub_symmetry_bd_ss_partJ<<<GRID_DIM,BLOCK_DIM>>>(ord,func,funcc,SoA[1]);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 }

 __global__ void sub_fderivs_shc_part1(double *fx,double *fy,double *fz){
@@ -247,13 +249,13 @@ inline void sub_fderivs_shc(int& sst,double * f,double * fh,double *fx,double *f
 	//cudaMemset(Msh_ gy,0,h_3D_SIZE[0] * sizeof(double));
 	//cudaMemset(Msh_ gz,0,h_3D_SIZE[0] * sizeof(double));
 	sub_symmetry_bd_ss(2,f,fh,SoA1);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(0,fh,h_3D_SIZE[2]);	
 	sub_fderivs_sh<<<GRID_DIM,BLOCK_DIM>>>(fh,Msh_ gx,Msh_ gy,Msh_ gz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	sub_fderivs_shc_part1<<<GRID_DIM,BLOCK_DIM>>>(fx,fy,fz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(1,fx,h_3D_SIZE[0]);
 			//compare_result_gpu(2,fy,h_3D_SIZE[0]);
 			//compare_result_gpu(3,fz,h_3D_SIZE[0]);
@@ -451,17 +453,17 @@ inline void sub_fdderivs_shc(int& sst,double * f,double * fh,
 	
 	//fderivs_sh
 	sub_symmetry_bd_ss(2,f,fh,SoA1);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(1,fh,h_3D_SIZE[2]);	
 	sub_fderivs_sh<<<GRID_DIM,BLOCK_DIM>>>(fh,Msh_ gx,Msh_ gy,Msh_ gz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	//fdderivs_sh
 	sub_symmetry_bd_ss(2,f,fh,SoA1);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(21,fh,h_3D_SIZE[2]);
 	sub_fdderivs_sh<<<GRID_DIM,BLOCK_DIM>>>(fh,Msh_ gxx,Msh_ gxy,Msh_ gxz,Msh_ gyy,Msh_ gyz,Msh_ gzz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			/*compare_result_gpu(11,Msh_ gx,h_3D_SIZE[0]);
 			compare_result_gpu(12,Msh_ gy,h_3D_SIZE[0]);
 			compare_result_gpu(13,Msh_ gz,h_3D_SIZE[0]);
@@ -472,7 +474,7 @@ inline void sub_fdderivs_shc(int& sst,double * f,double * fh,
 			compare_result_gpu(5,Msh_ gyz,h_3D_SIZE[0]);
 			compare_result_gpu(6,Msh_ gzz,h_3D_SIZE[0]);*/
 	sub_fdderivs_shc_part1<<<GRID_DIM,BLOCK_DIM>>>(fxx,fxy,fxz,fyy,fyz,fzz);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			/*compare_result_gpu(1,fxx,h_3D_SIZE[0]);
 			compare_result_gpu(2,fxy,h_3D_SIZE[0]);
 			compare_result_gpu(3,fxz,h_3D_SIZE[0]);
@@ -496,9 +498,9 @@ __global__ void computeRicci_ss_part1(double * dst)
 inline void computeRicci_ss(int &sst,double * src,double* dst,double * SoA, Meta* meta)
 {
 	sub_fdderivs_shc(sst,src,Mh_ fh,Mh_ fxx,Mh_ fxy,Mh_ fxz,Mh_ fyy,Mh_ fyz,Mh_ fzz,SoA);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	computeRicci_ss_part1<<<GRID_DIM,BLOCK_DIM>>>(dst);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 }
 __global__ void sub_lopsided_ss_part1(double * dst)
@@ -516,9 +518,9 @@ __global__ void sub_lopsided_ss_part1(double * dst)
 inline void sub_lopsided_ss(int& sst,double *src,double* dst,double *SoA)
 {
 		sub_fderivs_shc(sst,src,Mh_ fh,Mh_ fxx,Mh_ fxy,Mh_ fxz,SoA);
-		cudaThreadSynchronize();
+		cudaDeviceSynchronize();
 		sub_lopsided_ss_part1<<<GRID_DIM,BLOCK_DIM>>>(dst);
-		cudaThreadSynchronize();
+		cudaDeviceSynchronize();
 }

 __global__ void sub_kodis_sh_part1(double *f,double *fh,double *f_rhs)
@@ -590,11 +592,11 @@ inline void sub_kodis_ss(int &sst,double *f,double *fh,double *f_rhs,double *SoA
 	}
 			//compare_result_gpu(10,f,h_3D_SIZE[0]);
 	sub_symmetry_bd_ss(3,f,fh,SoA1);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(0,fh,h_3D_SIZE[3]);
 			
 	sub_kodis_sh_part1<<<GRID_DIM,BLOCK_DIM>>>(f,fh,f_rhs);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 			//compare_result_gpu(1,f_rhs,h_3D_SIZE[0]);
 }

@@ -1699,7 +1701,7 @@ void destroy_meta(Meta *meta,Metass *metass)
 	if(Msh_ gzz) cudaFree(Msh_ gzz);
 	
 #if (GAUGE == 2 || GAUGE == 3 || GAUGE == 4 || GAUGE == 5 || GAUGE == 6 || GAUGE == 7)
-	if(Mh_ reta) CUDA_SAFE_CALL(cudaFree(Mh_ reta));
+	if(Mh_ reta) cudaFree(Mh_ reta);

 #endif	
 	
@@ -1895,7 +1897,7 @@ int gpu_rhs_ss(RHS_SS_PARA)

 	//1.2 local Data
 	cudaMalloc((void**)&(Mh_ gxx), matrix_size * sizeof(double));
-	CUDA_SAFE_CALL( cudaMalloc((void**)&(Mh_ gyy), matrix_size * sizeof(double)));
+	cudaMalloc((void**)&(Mh_ gyy), matrix_size * sizeof(double));
 	cudaMalloc((void**)&(Mh_ gzz), matrix_size * sizeof(double));
 	cudaMalloc((void**)&(Mh_ chix), matrix_size * sizeof(double));
 	cudaMalloc((void**)&(Mh_ chiy), matrix_size * sizeof(double));
@@ -2160,7 +2162,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	
 	double tmp_con2 = 1/Mass[0] - tmp_con;
 	cudaMemcpyToSymbol(C1, &tmp_con2, sizeof(double));
-	double tmp_con2 = 1/Mass[1] - tmp_con;
+	tmp_con2 = 1/Mass[1] - tmp_con;
 	cudaMemcpyToSymbol(C2, &tmp_con2, sizeof(double));
 		

@@ -2233,7 +2235,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
  		if((sst == 2 || sst == 4) && abs[1] < dYh) 
  		{
  			ijkmin_h[1] = -2; 
-  			ijkmin_h[1] = -3;
+  			ijkmin3_h[1] = -3;
  		}
  		if((sst == 3 || sst == 5) && abs_Y_ex2 < dYh)
  		{
@@ -2287,13 +2289,13 @@ int gpu_rhs_ss(RHS_SS_PARA)


 #ifdef TIMING1
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	gettimeofday(&tv2, NULL);
   	cout<<"TIME USED"<<TimeBetween(tv1, tv2)<<endl; 
 #endif	
 	//cout<<"GPU meta data ready.\n";
 	
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();


 //-------------get device info-------------------------------------
@@ -2306,7 +2308,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	//sub_enforce_ga(matrix_size);
 	//4.1-----compute rhs---------
 	compute_rhs_ss_part1<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();

 	sub_fderivs_shc(sst,Mh_ betax,Mh_ fh,Mh_ betaxx,Mh_ betaxy,Mh_ betaxz,ass);
 	sub_fderivs_shc(sst,Mh_ betay,Mh_ fh,Mh_ betayx,Mh_ betayy,Mh_ betayz,sas);
@@ -2322,7 +2324,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	sub_fderivs_shc(sst,Mh_ gyz,Mh_ fh,Mh_ gyzx,Mh_ gyzy,Mh_ gyzz, saa);
 	
 	compute_rhs_ss_part2<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	sub_fdderivs_shc(sst,Mh_ betax,Mh_ fh,Mh_ gxxx,Mh_ gxyx,Mh_ gxzx,Mh_ gyyx,Mh_ gyzx,Mh_ gzzx,ass);
 	sub_fdderivs_shc(sst,Mh_ betay,Mh_ fh,Mh_ gxxy,Mh_ gxyy,Mh_ gxzy,Mh_ gyyy,Mh_ gyzy,Mh_ gzzy,sas);
@@ -2332,7 +2334,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	sub_fderivs_shc( sst,Mh_ Gamz, Mh_ fh,Mh_ Gamzx, Mh_ Gamzy, Mh_ Gamzz,ssa);
 	
 	compute_rhs_ss_part3<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	computeRicci_ss(sst,Mh_ dxx,Mh_ Rxx,sss, meta);
 	computeRicci_ss(sst,Mh_ dyy,Mh_ Ryy,sss, meta);
@@ -2340,25 +2342,25 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	computeRicci_ss(sst,Mh_ gxy,Mh_ Rxy,aas, meta);
 	computeRicci_ss(sst,Mh_ gxz,Mh_ Rxz,asa, meta);
 	computeRicci_ss(sst,Mh_ gyz,Mh_ Ryz,saa, meta);
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	compute_rhs_ss_part4<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	sub_fdderivs_shc(sst,Mh_ chi,Mh_ fh,Mh_ fxx,Mh_ fxy,Mh_ fxz,Mh_ fyy,Mh_ fyz,Mh_ fzz,sss);
 	
-	//cudaThreadSynchronize();
+	//cudaDeviceSynchronize();
 	//compare_result_gpu(0,Mh_ chi,h_3D_SIZE[0]);
 	//compare_result_gpu(1,Mh_ chi,h_3D_SIZE[0]);
 	//compare_result_gpu(2,Mh_ fyz,h_3D_SIZE[0]);
 	
 	compute_rhs_ss_part5<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 	sub_fdderivs_shc(sst,Mh_ Lap,Mh_ fh,Mh_ fxx,Mh_ fxy,Mh_ fxz,Mh_ fyy,Mh_ fyz,Mh_ fzz,sss);
 	
 	compute_rhs_ss_part6<<<GRID_DIM,BLOCK_DIM>>>();
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	
 #if (GAUGE == 2 || GAUGE == 3 || GAUGE == 4 || GAUGE == 5)
 	sub_fderivs_shc(sst,Mh_ chi,Mh_ fh, Mh_ dtSfx_rhs, Mh_ dtSfy_rhs, Mh_ dtSfz_rhs,sss);
@@ -2423,7 +2425,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	}
 	if(co == 0){
 		compute_rhs_ss_part7<<<GRID_DIM,BLOCK_DIM>>>();
-		cudaThreadSynchronize();
+		cudaDeviceSynchronize();

 		sub_fderivs_shc(sst,Mh_ Axx,Mh_ fh,Mh_ gxxx,Mh_ gxxy,Mh_ gxxz,sss);
 		sub_fderivs_shc(sst,Mh_ Axy,Mh_ fh,Mh_ gxyx,Mh_ gxyy,Mh_ gxyz,aas);
@@ -2432,7 +2434,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 		sub_fderivs_shc(sst,Mh_ Ayz,Mh_ fh,Mh_ gyzx,Mh_ gyzy,Mh_ gyzz,saa);
 		sub_fderivs_shc(sst,Mh_ Azz,Mh_ fh,Mh_ gzzx,Mh_ gzzy,Mh_ gzzz,sss);
 		compute_rhs_ss_part8<<<GRID_DIM,BLOCK_DIM>>>();
-		cudaThreadSynchronize();
+		cudaDeviceSynchronize();
 	}
 	
 #if (ABV == 1)
@@ -2512,7 +2514,7 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	//test kodis
 	//sub_kodis_sh(sst,Msh_ drhodx,Mh_ fh2,Msh_ drhody,sss);
 #ifdef TIMING
-	cudaThreadSynchronize();
+	cudaDeviceSynchronize();
 	gettimeofday(&tv2, NULL);
   	cout<<"MPI rank is: "<<mpi_rank<<" GPU TIME is"<<TimeBetween(tv1, tv2)<<" (s)."<<endl; 
 #endif
@@ -2522,4 +2524,55 @@ int gpu_rhs_ss(RHS_SS_PARA)
 	return 0;//TODO return
 }

+#if (ABEtype == 2)
+// Z4C Shell GPU: calls BSSN gpu_rhs_ss with trKd=trK+2*TZ, then applies
+// TZ_rhs = alpn1*Hcon/2 and constraint damping on CPU.
+int gpu_rhs_z4c_ss(Z4C_SS_PARA)
+{
+    int matrix_size = ex[0] * ex[1] * ex[2];
+    double k1 = 0.02, k2 = 0.0;
+
+    double *trKd_host = new double[matrix_size];
+    for (int _i = 0; _i < matrix_size; _i++)
+        trKd_host[_i] = trK[_i] + 2.0 * TZ[_i];
+
+    int result = gpu_rhs_ss(calledby, mpi_rank,
+        ex, T, crho, sigma, R, X, Y, Z,
+        drhodx, drhody, drhodz, dsigmadx, dsigmady, dsigmadz,
+        dRdx, dRdy, dRdz,
+        drhodxx, drhodxy, drhodxz, drhodyy, drhodyz, drhodzz,
+        dsigmadxx, dsigmadxy, dsigmadxz, dsigmadyy, dsigmadyz, dsigmadzz,
+        dRdxx, dRdxy, dRdxz, dRdyy, dRdyz, dRdzz,
+        chi, trKd_host, dxx, gxy, gxz, dyy, gyz, dzz,
+        Axx, Axy, Axz, Ayy, Ayz, Azz,
+        Gamx, Gamy, Gamz,
+        Lap, betax, betay, betaz,
+        dtSfx, dtSfy, dtSfz,
+        chi_rhs, trK_rhs,
+        gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
+        Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
+        Gamx_rhs, Gamy_rhs, Gamz_rhs,
+        Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
+        dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
+        rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz,
+        Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
+        Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
+        Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
+        Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
+        ham_Res, movx_Res, movy_Res, movz_Res,
+        Gmx_Res, Gmy_Res, Gmz_Res,
+        Symmetry, Lev, eps, sst, co);
+    delete[] trKd_host;
+    if (result != 0) return result;
+
+    for (int _i = 0; _i < matrix_size; _i++) {
+        double alp = Lap[_i] + 1.0;
+        TZ_rhs[_i] = alp * ham_Res[_i] * 0.5;
+        TZ_rhs[_i]  -= alp * (2.0 + k2) * k1 * TZ[_i];
+        trK_rhs[_i] += alp * k1 * (1.0 - k2) * TZ[_i];
+    }
+    return 0;
+}
+#endif // ABEtype == 2
+
 #endif //WithShell
--- a/AMSS_NCKU_source/bssn_rhs_cuda.cu
+++ b/AMSS_NCKU_source/bssn_rhs_cuda.cu
--- a/AMSS_NCKU_source/bssn_rhs_cuda.h
+++ b/AMSS_NCKU_source/bssn_rhs_cuda.h
@@ -7,6 +7,9 @@ extern "C" {

 enum {
    BSSN_CUDA_STATE_COUNT = 24,
+    BSSN_ESCALAR_CUDA_STATE_COUNT = 26,
+    BSSN_EM_CUDA_STATE_COUNT = 32,
+    BSSN_EM_CUDA_SOURCE_COUNT = 4,
    BSSN_CUDA_MATTER_COUNT = 10
 };

@@ -55,6 +58,54 @@ int bssn_cuda_rk4_substep(void *block_tag,
                          int &apply_enforce_ga,
                          double &chitiny);

+int bssn_escalar_cuda_rk4_substep(void *block_tag,
+                                  int *ex, double *X, double *Y, double *Z,
+                                  double **state_host_in,
+                                  double **state_host_out,
+                                  const double *propspeed,
+                                  const double *soa_flat,
+                                  const double *bbox,
+                                  double &dT,
+                                  double &T,
+                                  int &RK4,
+                                  int &apply_bam_bc,
+                                  int &Symmetry,
+	                                  int &Lev,
+	                                  double &eps,
+	                                  int &co,
+	                                  int &keep_resident_state,
+	                                  int &apply_enforce_ga,
+	                                  double &chitiny);
+
+int bssn_escalar_cuda_compute_constraints(int *ex, double *X, double *Y, double *Z,
+                                          double **state_host_in,
+                                          double **constraint_host_out,
+                                          int &Symmetry,
+                                          int &Lev,
+                                          double &eps);
+
+int bssn_em_cuda_rk4_substep(void *block_tag,
+                             int *ex, double *X, double *Y, double *Z,
+                             double **state_host_in,
+                             double **state_host_out,
+                             double **source_host,
+                             const double *propspeed,
+                             const double *soa_flat,
+                             const double *bbox,
+                             double &dT,
+                             double &T,
+                             int &RK4,
+                             int &apply_bam_bc,
+                             int &Symmetry,
+                             int &Lev,
+                             double &eps,
+                             int &co,
+                             int &keep_resident_state,
+                             int &apply_enforce_ga,
+                             double &chitiny);
+
+int bssn_em_cuda_resident_zero_fast_state(void *block_tag);
+
 int bssn_cuda_copy_state_region_to_host(void *block_tag,
                                        int state_index,
                                        double *host_state,
@@ -73,6 +124,33 @@ int bssn_cuda_download_resident_state(void *block_tag,
                                      int *ex,
                                      double **state_host_out);

+int bssn_escalar_cuda_download_resident_state(void *block_tag,
+                                              int *ex,
+                                              double **state_host_out);
+
+int bssn_cuda_upload_resident_state_count(void *block_tag,
+                                          int *ex,
+                                          double **state_host_in,
+                                          int state_count);
+
+int bssn_escalar_cuda_upload_resident_state(void *block_tag,
+                                            int *ex,
+                                            double **state_host_in);
+
+int bssn_cuda_keep_only_resident_state_count(void *block_tag,
+                                             int *ex,
+                                             double **state_host_key,
+                                             int state_count);
+
+int bssn_escalar_cuda_keep_only_resident_state(void *block_tag,
+                                               int *ex,
+                                               double **state_host_key);
+
+int bssn_cuda_download_resident_state_count_if_present(void *block_tag,
+                                                       int *ex,
+                                                       double **state_host_out,
+                                                       int state_count);
+
 int bssn_cuda_download_resident_state_if_present(void *block_tag,
                                                int *ex,
                                                double **state_host_out);
@@ -103,6 +181,7 @@ int bssn_cuda_interp_state_point3(void *block_tag,
                                  double pz,
                                  int ordn,
                                  int symmetry,
+                                  double **state_host_key,
                                  const double *soa3,
                                  double *out3);

@@ -132,6 +211,15 @@ int bssn_cuda_unpack_state_region_from_host_buffer(void *block_tag,
                                                   int i0, int j0, int k0,
                                                   int sx, int sy, int sz);

+int bssn_cuda_unpack_state_region_from_host_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  int state_index,
+                                                                  double *host_buffer,
+                                                                  int *ex,
+                                                                  int i0, int j0, int k0,
+                                                                  int sx, int sy, int sz);
+
 int bssn_cuda_pack_state_batch_to_host_buffer(void *block_tag,
                                              int state_count,
                                              double *host_buffer,
@@ -302,6 +390,7 @@ int bssn_cuda_upload_state_subset(void *block_tag,

 int bssn_cuda_prepare_inter_time_level(void *block_tag,
                                       int *ex,
+                                       int state_count,
                                       double **src1_host_key,
                                       double **src2_host_key,
                                       double **src3_host_key,
@@ -315,6 +404,10 @@ void bssn_cuda_release_step_ctx(void *block_tag);

 #ifdef __cplusplus
 }
+// C++-only helpers declared for derived equation classes (Z4C, etc.)
+// Defined in bssn_class.C. Requires MyList, Patch, var from including TU.
+bool bssn_cuda_use_resident_sync(int lev);
+void bssn_cuda_download_level_state_if_present(MyList<Patch> *PatL, MyList<var> *vars, int myrank);
 #endif

 #endif
--- a/AMSS_NCKU_source/checkpoint.C
+++ b/AMSS_NCKU_source/checkpoint.C
@@ -76,8 +76,11 @@ checkpoint::checkpoint(bool checked, const char fname[], int myrank) : filename(

 	I_Print = (myrank == 0);

-	int i = strlen(fname);
-	filename = new char[i+30];
+	size_t filename_len = out_dir.size() + strlen(fname) + 32;
+#ifdef CHECKDETAIL
+	filename_len += 32;
+#endif
+	filename = new char[filename_len];
 	// cout << filename << endl;
 	// cout << i << endl;

@@ -100,12 +103,12 @@ checkpoint::checkpoint(bool checked, const char fname[], int myrank) : filename(
                cout << " checkpoint class created " << endl;
        }
 }
-checkpoint::~checkpoint()
-{
-	CheckList->clearList();
-	if (I_Print)
-		delete[] filename;
-}
+checkpoint::~checkpoint()
+{
+	CheckList->clearList();
+	if (filename)
+		delete[] filename;
+}

 void checkpoint::addvariable(var *VV)
 {
@@ -136,7 +139,7 @@ void checkpoint::writecheck_cgh(double time, cgh *GH)
 	if (I_Print)
 	{
 		// char fname[50];
-		char fname[50+50]; 
+		char fname[4096]; 
 		sprintf(fname, "%s_cgh.CHK", filename);

 		outfile.open(fname, ios::out | ios::trunc);
@@ -195,7 +198,7 @@ void checkpoint::readcheck_cgh(double &time, cgh *GH, int myrank, int nprocs, in
 	int DIM = dim;
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_cgh.CHK", filename);

 	infile.open(fname);
@@ -297,7 +300,7 @@ void checkpoint::writecheck_sh(double time, ShellPatch *SH)

 	if (I_Print)
 	{
-		char fname[50];
+		char fname[4096];
 		sprintf(fname, "%s_sh.CHK", filename);

 		outfile.open(fname, ios::out | ios::trunc);
@@ -335,7 +338,7 @@ void checkpoint::readcheck_sh(ShellPatch *SH, int myrank)
 	int DIM = dim;
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_sh.CHK", filename);

 	infile.open(fname);
@@ -390,7 +393,7 @@ void checkpoint::write_Black_Hole_position(int BH_num_input, int BH_num, double

 	if (I_Print)
 	{
-		char fname[50];
+		char fname[4096];
 		sprintf(fname, "%s_BHp.CHK", filename);

 		outfile.open(fname, ios::out | ios::trunc);
@@ -417,7 +420,7 @@ void checkpoint::read_Black_Hole_position(int &BH_num_input, int &BH_num, double
 {
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_BHp.CHK", filename);

 	infile.open(fname);
@@ -461,7 +464,7 @@ void checkpoint::write_bssn(double LastDump, double Last2dDump, double LastAnas)

 	if (I_Print)
 	{
-		char fname[50];
+		char fname[4096];
 		sprintf(fname, "%s_bssn.CHK", filename);

 		outfile.open(fname, ios::out | ios::trunc);
@@ -481,7 +484,7 @@ void checkpoint::read_bssn(double &LastDump, double &Last2dDump, double &LastAna
 {
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_bssn.CHK", filename);

 	infile.open(fname);
@@ -506,7 +509,7 @@ void checkpoint::write_bssn(double LastDump, double Last2dDump, double LastAnas)
 	ofstream outfile;

 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_bssn.CHK", filename);

 	outfile.open(fname, ios::out | ios::trunc);
@@ -527,7 +530,7 @@ void checkpoint::read_bssn(double &LastDump, double &Last2dDump, double &LastAna
 {
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_bssn.CHK", filename);

 	infile.open(fname);
@@ -551,7 +554,7 @@ void checkpoint::write_Black_Hole_position(int BH_num_input, int BH_num, double
 	ofstream outfile;

 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_BHp.CHK", filename);

 	outfile.open(fname, ios::out | ios::trunc);
@@ -581,7 +584,7 @@ void checkpoint::read_Black_Hole_position(int &BH_num_input, int &BH_num, double
 {
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_BHp.CHK", filename);

 	infile.open(fname);
@@ -628,7 +631,7 @@ void checkpoint::writecheck_cgh(double time, cgh *GH)
 	ofstream outfile;

 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_cgh.CHK", filename);

 	outfile.open(fname, ios::out | ios::trunc);
@@ -738,7 +741,7 @@ void checkpoint::readcheck_cgh(double &time, cgh *GH, int myrank, int nprocs, in
 	int DIM = dim;
 	ifstream infile;
 	// char fname[50];
-	char fname[50+50]; 
+	char fname[4096]; 
 	sprintf(fname, "%s_cgh.CHK", filename);

 	infile.open(fname);
--- a/AMSS_NCKU_source/fd_cuda_helpers.cuh
+++ b/AMSS_NCKU_source/fd_cuda_helpers.cuh
@@ -0,0 +1,412 @@
+#ifndef AMSS_NCKU_FD_CUDA_HELPERS_CUH
+#define AMSS_NCKU_FD_CUDA_HELPERS_CUH
+
+#ifndef ghost_width
+#error "ghost_width must be defined before including fd_cuda_helpers.cuh"
+#endif
+
+#if ghost_width < 2 || ghost_width > 5
+#error "CUDA finite-difference helpers support ghost_width 2..5"
+#endif
+
+#define AMSS_FD_CENTER_RADIUS (ghost_width - 1)
+#define AMSS_FD_LK_RADIUS     (ghost_width)
+
+__device__ __forceinline__ int fd_axis_radius(int qF, int qminF, int qmaxF)
+{
+#if AMSS_FD_CENTER_RADIUS >= 4
+    if (qF - 4 >= qminF && qF + 4 <= qmaxF) return 4;
+#endif
+#if AMSS_FD_CENTER_RADIUS >= 3
+    if (qF - 3 >= qminF && qF + 3 <= qmaxF) return 3;
+#endif
+#if AMSS_FD_CENTER_RADIUS >= 2
+    if (qF - 2 >= qminF && qF + 2 <= qmaxF) return 2;
+#endif
+    if (qF - 1 >= qminF && qF + 1 <= qmaxF) return 1;
+    return 0;
+}
+
+__device__ __forceinline__ int fd_common_radius(int iF, int jF, int kF,
+                                                int iminF, int jminF, int kminF,
+                                                int imaxF, int jmaxF, int kmaxF)
+{
+    int r = fd_axis_radius(iF, iminF, imaxF);
+    const int ry = fd_axis_radius(jF, jminF, jmaxF);
+    const int rz = fd_axis_radius(kF, kminF, kmaxF);
+    if (ry < r) r = ry;
+    if (rz < r) r = rz;
+    return r;
+}
+
+__device__ __forceinline__ double fd_first_coef(int r, int off)
+{
+    switch (r) {
+    case 1:
+        if (off == -1) return -1.0;
+        if (off ==  1) return  1.0;
+        return 0.0;
+    case 2:
+        if (off == -2) return  1.0;
+        if (off == -1) return -8.0;
+        if (off ==  1) return  8.0;
+        if (off ==  2) return -1.0;
+        return 0.0;
+    case 3:
+        if (off == -3) return  -1.0;
+        if (off == -2) return   9.0;
+        if (off == -1) return -45.0;
+        if (off ==  1) return  45.0;
+        if (off ==  2) return  -9.0;
+        if (off ==  3) return   1.0;
+        return 0.0;
+    case 4:
+        if (off == -4) return    3.0;
+        if (off == -3) return  -32.0;
+        if (off == -2) return  168.0;
+        if (off == -1) return -672.0;
+        if (off ==  1) return  672.0;
+        if (off ==  2) return -168.0;
+        if (off ==  3) return   32.0;
+        if (off ==  4) return   -3.0;
+        return 0.0;
+    default:
+        return 0.0;
+    }
+}
+
+__device__ __forceinline__ double fd_second_coef(int r, int off)
+{
+    switch (r) {
+    case 1:
+        if (off == -1) return  1.0;
+        if (off ==  0) return -2.0;
+        if (off ==  1) return  1.0;
+        return 0.0;
+    case 2:
+        if (off == -2) return  -1.0;
+        if (off == -1) return  16.0;
+        if (off ==  0) return -30.0;
+        if (off ==  1) return  16.0;
+        if (off ==  2) return  -1.0;
+        return 0.0;
+    case 3:
+        if (off == -3) return    2.0;
+        if (off == -2) return  -27.0;
+        if (off == -1) return  270.0;
+        if (off ==  0) return -490.0;
+        if (off ==  1) return  270.0;
+        if (off ==  2) return  -27.0;
+        if (off ==  3) return    2.0;
+        return 0.0;
+    case 4:
+        if (off == -4) return     -9.0;
+        if (off == -3) return    128.0;
+        if (off == -2) return  -1008.0;
+        if (off == -1) return   8064.0;
+        if (off ==  0) return -14350.0;
+        if (off ==  1) return   8064.0;
+        if (off ==  2) return  -1008.0;
+        if (off ==  3) return    128.0;
+        if (off ==  4) return     -9.0;
+        return 0.0;
+    default:
+        return 0.0;
+    }
+}
+
+__device__ __forceinline__ double fd_first_denom(int r)
+{
+    return (r == 4) ? 840.0 : ((r == 3) ? 60.0 : ((r == 2) ? 12.0 : 2.0));
+}
+
+__device__ __forceinline__ double fd_second_denom(int r)
+{
+    return (r == 4) ? 5040.0 : ((r == 3) ? 180.0 : ((r == 2) ? 12.0 : 1.0));
+}
+
+__device__ __forceinline__ double fd_fetch_axis(const double *src,
+                                                int iF, int jF, int kF,
+                                                int axis, int off,
+                                                int SoA0, int SoA1, int SoA2)
+{
+    if (axis == 0) iF += off;
+    else if (axis == 1) jF += off;
+    else kF += off;
+    return fetch_sym_ord2_direct(src, iF, jF, kF, SoA0, SoA1, SoA2);
+}
+
+__device__ __forceinline__ double fd_fetch_axis2(const double *src,
+                                                 int iF, int jF, int kF,
+                                                 int axis_a, int off_a,
+                                                 int axis_b, int off_b,
+                                                 int SoA0, int SoA1, int SoA2)
+{
+    if (axis_a == 0) iF += off_a;
+    else if (axis_a == 1) jF += off_a;
+    else kF += off_a;
+    if (axis_b == 0) iF += off_b;
+    else if (axis_b == 1) jF += off_b;
+    else kF += off_b;
+    return fetch_sym_ord2_direct(src, iF, jF, kF, SoA0, SoA1, SoA2);
+}
+
+__device__ __forceinline__ double fd_first_axis_radius(const double *src,
+                                                       int iF, int jF, int kF,
+                                                       int axis, int r, double h,
+                                                       int SoA0, int SoA1, int SoA2)
+{
+    if (r <= 0) return 0.0;
+    double s = 0.0;
+#pragma unroll
+    for (int off = -4; off <= 4; ++off) {
+        const double c = fd_first_coef(r, off);
+        if (c != 0.0) {
+            s += c * fd_fetch_axis(src, iF, jF, kF, axis, off, SoA0, SoA1, SoA2);
+        }
+    }
+    return s / (fd_first_denom(r) * h);
+}
+
+__device__ __forceinline__ double fd_second_axis_radius(const double *src,
+                                                        int iF, int jF, int kF,
+                                                        int axis, int r, double h,
+                                                        int SoA0, int SoA1, int SoA2)
+{
+    if (r <= 0) return 0.0;
+    double s = 0.0;
+#pragma unroll
+    for (int off = -4; off <= 4; ++off) {
+        const double c = fd_second_coef(r, off);
+        if (c != 0.0) {
+            s += c * fd_fetch_axis(src, iF, jF, kF, axis, off, SoA0, SoA1, SoA2);
+        }
+    }
+    return s / (fd_second_denom(r) * h * h);
+}
+
+__device__ __forceinline__ double fd_mixed_axis_radius(const double *src,
+                                                       int iF, int jF, int kF,
+                                                       int axis_a, int r_a, double h_a,
+                                                       int axis_b, int r_b, double h_b,
+                                                       int SoA0, int SoA1, int SoA2)
+{
+    if (r_a <= 0 || r_b <= 0) return 0.0;
+    double s = 0.0;
+#pragma unroll
+    for (int off_a = -4; off_a <= 4; ++off_a) {
+        const double ca = fd_first_coef(r_a, off_a);
+        if (ca == 0.0) continue;
+#pragma unroll
+        for (int off_b = -4; off_b <= 4; ++off_b) {
+            const double cb = fd_first_coef(r_b, off_b);
+            if (cb != 0.0) {
+                s += ca * cb * fd_fetch_axis2(src, iF, jF, kF, axis_a, off_a,
+                                              axis_b, off_b, SoA0, SoA1, SoA2);
+            }
+        }
+    }
+    return s / (fd_first_denom(r_a) * fd_first_denom(r_b) * h_a * h_b);
+}
+
+__device__ __forceinline__ void fd_compute_first3(const double *src,
+                                                  int iF, int jF, int kF,
+                                                  int iminF, int jminF, int kminF,
+                                                  int imaxF, int jmaxF, int kmaxF,
+                                                  int SoA0, int SoA1, int SoA2,
+                                                  double &fx, double &fy, double &fz)
+{
+#if ghost_width == 3
+    const int r = fd_common_radius(iF, jF, kF, iminF, jminF, kminF, imaxF, jmaxF, kmaxF);
+    fx = fd_first_axis_radius(src, iF, jF, kF, 0, r, d_gp.dX, SoA0, SoA1, SoA2);
+    fy = fd_first_axis_radius(src, iF, jF, kF, 1, r, d_gp.dY, SoA0, SoA1, SoA2);
+    fz = fd_first_axis_radius(src, iF, jF, kF, 2, r, d_gp.dZ, SoA0, SoA1, SoA2);
+#else
+    fx = fd_first_axis_radius(src, iF, jF, kF, 0, fd_axis_radius(iF, iminF, imaxF),
+                              d_gp.dX, SoA0, SoA1, SoA2);
+    fy = fd_first_axis_radius(src, iF, jF, kF, 1, fd_axis_radius(jF, jminF, jmaxF),
+                              d_gp.dY, SoA0, SoA1, SoA2);
+    fz = fd_first_axis_radius(src, iF, jF, kF, 2, fd_axis_radius(kF, kminF, kmaxF),
+                              d_gp.dZ, SoA0, SoA1, SoA2);
+#endif
+}
+
+__device__ __forceinline__ void fd_compute_second6(const double *src,
+                                                   int iF, int jF, int kF,
+                                                   int iminF, int jminF, int kminF,
+                                                   int imaxF, int jmaxF, int kmaxF,
+                                                   int SoA0, int SoA1, int SoA2,
+                                                   double &fxx, double &fxy, double &fxz,
+                                                   double &fyy, double &fyz, double &fzz)
+{
+#if ghost_width == 3
+    const int r = fd_common_radius(iF, jF, kF, iminF, jminF, kminF, imaxF, jmaxF, kmaxF);
+    const int rx = r, ry = r, rz = r;
+#else
+    const int rx = fd_axis_radius(iF, iminF, imaxF);
+    const int ry = fd_axis_radius(jF, jminF, jmaxF);
+    const int rz = fd_axis_radius(kF, kminF, kmaxF);
+#endif
+    fxx = fd_second_axis_radius(src, iF, jF, kF, 0, rx, d_gp.dX, SoA0, SoA1, SoA2);
+    fyy = fd_second_axis_radius(src, iF, jF, kF, 1, ry, d_gp.dY, SoA0, SoA1, SoA2);
+    fzz = fd_second_axis_radius(src, iF, jF, kF, 2, rz, d_gp.dZ, SoA0, SoA1, SoA2);
+    fxy = fd_mixed_axis_radius(src, iF, jF, kF, 0, rx, d_gp.dX, 1, ry, d_gp.dY, SoA0, SoA1, SoA2);
+    fxz = fd_mixed_axis_radius(src, iF, jF, kF, 0, rx, d_gp.dX, 2, rz, d_gp.dZ, SoA0, SoA1, SoA2);
+    fyz = fd_mixed_axis_radius(src, iF, jF, kF, 1, ry, d_gp.dY, 2, rz, d_gp.dZ, SoA0, SoA1, SoA2);
+}
+
+__device__ __forceinline__ bool fd_lop_fits(int qF, int qminF, int qmaxF,
+                                            int dir, int lo, int hi)
+{
+    for (int off = lo; off <= hi; ++off) {
+        const int q = qF + dir * off;
+        if (q < qminF || q > qmaxF) return false;
+    }
+    return true;
+}
+
+__device__ __forceinline__ double fd_lop_fetch_sum(const double *src,
+                                                   int iF, int jF, int kF,
+                                                   int axis, int dir,
+                                                   const double *coef,
+                                                   int lo, int hi,
+                                                   int SoA0, int SoA1, int SoA2)
+{
+    double s = 0.0;
+    for (int off = lo; off <= hi; ++off) {
+        const double c = coef[off - lo];
+        if (c != 0.0) {
+            s += c * fd_fetch_axis(src, iF, jF, kF, axis, dir * off, SoA0, SoA1, SoA2);
+        }
+    }
+    return s;
+}
+
+__device__ __forceinline__ double fd_lopsided_axis(const double *src,
+                                                   int iF, int jF, int kF,
+                                                   int axis, double speed,
+                                                   int qF, int qminF, int qmaxF,
+                                                   double h,
+                                                   int SoA0, int SoA1, int SoA2)
+{
+    if (speed == 0.0) return 0.0;
+    const int dir = (speed > 0.0) ? 1 : -1;
+    const double mag = (speed > 0.0) ? speed : -speed;
+
+#if ghost_width == 2
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, 0, 2)) {
+        const double c[] = {-3.0, 4.0, -1.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, 0, 2, SoA0, SoA1, SoA2) / (2.0 * h);
+    }
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, 0, 1)) {
+        const double c[] = {-1.0, 1.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, 0, 1, SoA0, SoA1, SoA2) / (2.0 * h);
+    }
+    return 0.0;
+#elif ghost_width == 3
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, -1, 3)) {
+        const double c[] = {-3.0, -10.0, 18.0, -6.0, 1.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, -1, 3, SoA0, SoA1, SoA2) / (12.0 * h);
+    }
+    const int r = fd_axis_radius(qF, qminF, qmaxF);
+    return speed * fd_first_axis_radius(src, iF, jF, kF, axis, r, h, SoA0, SoA1, SoA2);
+#elif ghost_width == 4
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, -2, 4)) {
+        const double c[] = {2.0, -24.0, -35.0, 80.0, -30.0, 8.0, -1.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, -2, 4, SoA0, SoA1, SoA2) / (60.0 * h);
+    }
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, -1, 5)) {
+        const double c[] = {-10.0, -77.0, 150.0, -100.0, 50.0, -15.0, 2.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, -1, 5, SoA0, SoA1, SoA2) / (60.0 * h);
+    }
+    const int r = fd_axis_radius(qF, qminF, qmaxF);
+    return speed * fd_first_axis_radius(src, iF, jF, kF, axis, r, h, SoA0, SoA1, SoA2);
+#else
+    if (fd_lop_fits(qF, qminF, qmaxF, dir, -3, 5)) {
+        const double c[] = {-5.0, 60.0, -420.0, -378.0, 1050.0, -420.0, 140.0, -30.0, 3.0};
+        return mag * fd_lop_fetch_sum(src, iF, jF, kF, axis, dir, c, -3, 5, SoA0, SoA1, SoA2) / (840.0 * h);
+    }
+    const int r = fd_axis_radius(qF, qminF, qmaxF);
+    return speed * fd_first_axis_radius(src, iF, jF, kF, axis, r, h, SoA0, SoA1, SoA2);
+#endif
+}
+
+__device__ __forceinline__ double fd_ko_coef(int r, int off)
+{
+    const int a = off < 0 ? -off : off;
+    if (r == 2) {
+        if (a == 0) return 6.0;
+        if (a == 1) return -4.0;
+        if (a == 2) return 1.0;
+    } else if (r == 3) {
+        if (a == 0) return -20.0;
+        if (a == 1) return 15.0;
+        if (a == 2) return -6.0;
+        if (a == 3) return 1.0;
+    } else if (r == 4) {
+        if (a == 0) return 70.0;
+        if (a == 1) return -56.0;
+        if (a == 2) return 28.0;
+        if (a == 3) return -8.0;
+        if (a == 4) return 1.0;
+    } else if (r == 5) {
+        if (a == 0) return -252.0;
+        if (a == 1) return 210.0;
+        if (a == 2) return -120.0;
+        if (a == 3) return 45.0;
+        if (a == 4) return -10.0;
+        if (a == 5) return 1.0;
+    }
+    return 0.0;
+}
+
+__device__ __forceinline__ double fd_ko_axis(const double *src,
+                                             int iF, int jF, int kF,
+                                             int axis, int r,
+                                             int SoA0, int SoA1, int SoA2)
+{
+    double s = 0.0;
+#pragma unroll
+    for (int off = -5; off <= 5; ++off) {
+        if (off < -r || off > r) continue;
+        const double c = fd_ko_coef(r, off);
+        if (c != 0.0) {
+            s += c * fd_fetch_axis(src, iF, jF, kF, axis, off, SoA0, SoA1, SoA2);
+        }
+    }
+    return s;
+}
+
+__device__ __forceinline__ double fd_ko_term(const double *src,
+                                             int iF, int jF, int kF,
+                                             int iminF, int jminF, int kminF,
+                                             int imaxF, int jmaxF, int kmaxF,
+                                             double eps_val,
+                                             int SoA0, int SoA1, int SoA2)
+{
+    const int r = AMSS_FD_LK_RADIUS;
+    if (eps_val <= 0.0) return 0.0;
+#if ghost_width >= 4
+    if (iF - r <= iminF || iF + r >= imaxF ||
+        jF - r <= jminF || jF + r >= jmaxF ||
+        kF - r <= kminF || kF + r >= kmaxF) {
+        return 0.0;
+    }
+#else
+    if (iF - r < iminF || iF + r > imaxF ||
+        jF - r < jminF || jF + r > jmaxF ||
+        kF - r < kminF || kF + r > kmaxF) {
+        return 0.0;
+    }
+#endif
+    double cof = 1.0;
+#pragma unroll
+    for (int n = 0; n < 2 * r; ++n) cof *= 2.0;
+    const double sign = (r & 1) ? 1.0 : -1.0;
+    const double dx = fd_ko_axis(src, iF, jF, kF, 0, r, SoA0, SoA1, SoA2);
+    const double dy = fd_ko_axis(src, iF, jF, kF, 1, r, SoA0, SoA1, SoA2);
+    const double dz = fd_ko_axis(src, iF, jF, kF, 2, r, SoA0, SoA1, SoA2);
+    return sign * eps_val * (dx / d_gp.dX + dy / d_gp.dY + dz / d_gp.dZ) / cof;
+}
+
+#endif
--- a/AMSS_NCKU_source/gpu_rhsSS_mem.h
+++ b/AMSS_NCKU_source/gpu_rhsSS_mem.h
@@ -1,6 +1,6 @@
 #ifndef GPU_MEM_H_
 #define GPU_MEM_H_
-#include "macrodef.fh"
+#include "macrodef.h"

 #ifdef WithShell
 struct Metass
@@ -48,6 +48,8 @@ struct Meta
 	double * Gamx_rhs,*Gamy_rhs,*Gamz_rhs;//out
 	double * Lap_rhs, *betax_rhs, *betay_rhs, *betaz_rhs;//out
 	double * dtSfx_rhs,*dtSfy_rhs,*dtSfz_rhs;//out
+	double * TZ;                                  //in  (Z4C)
+	double * TZ_rhs;                              //out (Z4C)
 	double * rho,*Sx,*Sy,*Sz			;	//in
 	double * Sxx,*Sxy,*Sxz,*Syy,*Syz,*Szz;	//in
 	
@@ -132,6 +134,8 @@ __constant__ double SYM = 1.0;
 __constant__ double ANTI = -1.0; 
 __constant__ double FF = 0.75;
 __constant__ double eta = 2.0;
+__constant__ double kappa1_c = 0.02;
+__constant__ double kappa2_c = 0.0;
 __constant__ double F1o3;
 __constant__ double F2o3;
 __constant__ double F3o2 = 1.5;
--- a/AMSS_NCKU_source/makefile
+++ b/AMSS_NCKU_source/makefile
@@ -35,7 +35,6 @@ f90appflags = -O3 -xHost -fp-model fast=2 -fma -ipo \
 endif

 TP_OPTFLAGS = -O3 -xHost -fp-model fast=2 -fma -ipo \
-              -fprofile-instr-use=$(TP_PROFDATA) \
              -Dfortran3 -Dnewc $(MKL_INC)
 else
 ## NVHPC defaults: mpicc/mpicxx/mpifort wrappers
@@ -57,18 +56,26 @@ endif
 .C.o:
 	${CXX} $(CXXAPPFLAGS) -c $< $(filein) -o $@

+# ShellPatch.C uses OpenMP for setupintintstuff search loops
+ShellPatch.o: ShellPatch.C
+	${CXX} $(CXXAPPFLAGS) $(OMP_FLAG) -c $< $(filein) -o $@
+
 .for.o:
 	$(f77) -c $< -o $@

-.cu.o:
-	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
-
-# CUDA rewrite of BSSN RHS (drop-in replacement for bssn_rhs_c + stencil helpers)
-bssn_rhs_cuda.o: bssn_rhs_cuda.cu bssn_rhs.h macrodef.h
-	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+.cu.o:
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+
+# CUDA rewrite of BSSN RHS (drop-in replacement for bssn_rhs_c + stencil helpers)
+bssn_rhs_cuda.o: bssn_rhs_cuda.cu bssn_rhs.h macrodef.h fd_cuda_helpers.cuh
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)
+
+# CUDA rewrite of BSSN Shell-Patch RHS (drop-in replacement for bssn_rhs_ss)
+bssn_gpu_rhs_ss.o: bssn_gpu_rhs_ss.cu bssn_gpu.h gpu_rhsSS_mem.h bssn_macro.h macrodef.fh
+	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)

 # CUDA rewrite of Z4C Cartesian RHS
-z4c_rhs_cuda.o: z4c_rhs_cuda.cu z4c_rhs_cuda.h bssn_rhs.h macrodef.h ricci_gamma.h
+z4c_rhs_cuda.o: z4c_rhs_cuda.cu z4c_rhs_cuda.h bssn_rhs.h macrodef.h ricci_gamma.h fd_cuda_helpers.cuh
 	$(Cu) $(CUDA_APP_FLAGS) -c $< -o $@ $(CUDA_LIB_PATH)

 # C rewrite of BSSN RHS kernel and helpers
@@ -124,7 +131,7 @@ else
 CFILES_CPU = bssn_rhs_c.o fderivs_c.o fdderivs_c.o kodiss_c.o lopsided_c.o lopsided_kodis_c.o
 endif

-CFILES_CUDA_BSSN = bssn_rhs_cuda.o
+CFILES_CUDA_BSSN = bssn_rhs_cuda.o bssn_gpu_rhs_ss.o

 ifeq ($(USE_CUDA_BSSN),1)
 CFILES = $(CFILES_CUDA_BSSN)
--- a/AMSS_NCKU_source/makefile.inc
+++ b/AMSS_NCKU_source/makefile.inc
@@ -1,7 +1,7 @@
 ## Toolchain selection
 ##   nvhpc : NVIDIA HPC SDK + CUDA-aware MPI (default)
 ##   intel : Intel oneAPI toolchain (legacy path)
-TOOLCHAIN ?= nvhpc
+TOOLCHAIN ?= intel

 ## PGO build mode switch (ABE only; TwoPunctureABE always uses opt flags)
 ##   opt        : (default) maximum performance with PGO profile-guided optimization
@@ -88,4 +88,4 @@ endif

 Cu = $(NVHPC_ROOT)/compilers/bin/nvcc
 CUDA_LIB_PATH = -L$(CUDA_HOME)/lib64 -I$(CUDA_HOME)/include
-CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc -arch=$(CUDA_ARCH)
+CUDA_APP_FLAGS = -c -g -O3 --ptxas-options=-v -Dfortran3 -Dnewc -arch=$(CUDA_ARCH)
--- a/AMSS_NCKU_source/monitor.C
+++ b/AMSS_NCKU_source/monitor.C
@@ -1,7 +1,8 @@

-#ifdef newc
-#include <cstdio>
-using namespace std;
+#ifdef newc
+#include <cstdio>
+#include <sstream>
+using namespace std;
 #else
 #include <stdio.h>
 #endif
@@ -77,16 +78,17 @@ monitor::monitor(const char fname[], int myrank, string head)
      parameters::str_par.insert(map<string, string>::value_type("output dir", out_dir));
    }
    // considering checkpoint run
-    char filename[50];
-    sprintf(filename, "%s/%s", out_dir.c_str(), fname);
-    int i = 1;
-    while ((access(filename, F_OK)) != -1)
-    {
-      sprintf(filename, "%s/%d_%s", out_dir.c_str(), i, fname);
-      i++;
-    }
-
-    outfile.open(filename, ios::trunc);
+    string filename = out_dir + "/" + fname;
+    int i = 1;
+    while ((access(filename.c_str(), F_OK)) != -1)
+    {
+      stringstream ss;
+      ss << out_dir << "/" << i << "_" << fname;
+      filename = ss.str();
+      i++;
+    }
+
+    outfile.open(filename.c_str(), ios::trunc);

    time_t tnow;
    time(&tnow);
@@ -107,16 +109,17 @@ monitor::monitor(const char fname[], int myrank, const int out_rank, string head
  if (I_Print)
  {
    // considering checkpoint run
-    char filename[50];
-    sprintf(filename, "%s/%s", out_dir.c_str(), fname);
-    int i = 1;
-    while ((access(filename, F_OK)) != -1)
-    {
-      sprintf(filename, "%s/%d_%s", out_dir.c_str(), i, fname);
-      i++;
-    }
-
-    outfile.open(filename, ios::trunc);
+    string filename = out_dir + "/" + fname;
+    int i = 1;
+    while ((access(filename.c_str(), F_OK)) != -1)
+    {
+      stringstream ss;
+      ss << out_dir << "/" << i << "_" << fname;
+      filename = ss.str();
+      i++;
+    }
+
+    outfile.open(filename.c_str(), ios::trunc);

    time_t tnow;
    time(&tnow);
--- a/AMSS_NCKU_source/z4c_rhs_cuda.cu
+++ b/AMSS_NCKU_source/z4c_rhs_cuda.cu
@@ -266,6 +266,8 @@ __device__ __forceinline__ double fetch_sym_ord3_direct(const double *src,
                    + (skF - 1) * d_gp.ex[0] * d_gp.ex[1]];
 }

+#include "fd_cuda_helpers.cuh"
+
 /* ------------------------------------------------------------------ */
 /*  GPU buffer management                                              */
 /* ------------------------------------------------------------------ */
@@ -420,6 +422,7 @@ static const int k_lk_rhs_slots[BSSN_LK_FIELD_COUNT] = {
 };

 __constant__ int d_subset_state_indices[BSSN_STATE_COUNT];
+__constant__ double d_comm_state_soa[3 * BSSN_STATE_COUNT];

 static const int k_lk_soa_signs[3 * BSSN_LK_FIELD_COUNT] = {
     1,  1,  1,
@@ -727,6 +730,21 @@ static void upload_grid_params_if_needed(const GridParams &gp)
    }
 }

+static void upload_comm_state_soa(const double *state_soa, int state_count)
+{
+    double soa[3 * BSSN_STATE_COUNT];
+    for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
+        soa[3 * i + 0] = 1.0;
+        soa[3 * i + 1] = 1.0;
+        soa[3 * i + 2] = 1.0;
+    }
+    if (state_soa) {
+        const int n = (state_count < BSSN_STATE_COUNT) ? state_count : BSSN_STATE_COUNT;
+        std::memcpy(soa, state_soa, (size_t)3 * n * sizeof(double));
+    }
+    CUDA_CHECK(cudaMemcpyToSymbol(d_comm_state_soa, soa, sizeof(soa)));
+}
+
 /* ================================================================== */
 /*  A. Symmetry boundary kernels (ord=2 and ord=3)                     */
 /* ================================================================== */
@@ -1419,45 +1437,10 @@ void kern_fderivs_batched(FDerivTables tables, int field_count)
    const int jF = j0 + 1;
    const int kF = k0 + 1;

-    if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
-        (jF + 2) <= jmaxF && (jF - 2) >= jminF &&
-        (kF + 2) <= kmaxF && (kF - 2) >= kminF)
-    {
-        fx[tid] = d_gp.d12dx * (
-            fetch_sym_ord2_direct(src, iF - 2, jF, kF, SoA0, SoA1, SoA2)
-          - 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
-          + 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF, SoA0, SoA1, SoA2)
-          -       fetch_sym_ord2_direct(src, iF + 2, jF, kF, SoA0, SoA1, SoA2));
-        fy[tid] = d_gp.d12dy * (
-            fetch_sym_ord2_direct(src, iF, jF - 2, kF, SoA0, SoA1, SoA2)
-          - 8.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
-          + 8.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF, SoA0, SoA1, SoA2)
-          -       fetch_sym_ord2_direct(src, iF, jF + 2, kF, SoA0, SoA1, SoA2));
-        fz[tid] = d_gp.d12dz * (
-            fetch_sym_ord2_direct(src, iF, jF, kF - 2, SoA0, SoA1, SoA2)
-          - 8.0 * fetch_sym_ord2_direct(src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
-          + 8.0 * fetch_sym_ord2_direct(src, iF, jF, kF + 1, SoA0, SoA1, SoA2)
-          -       fetch_sym_ord2_direct(src, iF, jF, kF + 2, SoA0, SoA1, SoA2));
-    }
-    else if ((iF + 1) <= imaxF && (iF - 1) >= iminF &&
-             (jF + 1) <= jmaxF && (jF - 1) >= jminF &&
-             (kF + 1) <= kmaxF && (kF - 1) >= kminF)
-    {
-        fx[tid] = d_gp.d2dx * (
-            -fetch_sym_ord2_direct(src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
-            +fetch_sym_ord2_direct(src, iF + 1, jF, kF, SoA0, SoA1, SoA2));
-        fy[tid] = d_gp.d2dy * (
-            -fetch_sym_ord2_direct(src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
-            +fetch_sym_ord2_direct(src, iF, jF + 1, kF, SoA0, SoA1, SoA2));
-        fz[tid] = d_gp.d2dz * (
-            -fetch_sym_ord2_direct(src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
-            +fetch_sym_ord2_direct(src, iF, jF, kF + 1, SoA0, SoA1, SoA2));
-    }
-    else {
-        fx[tid] = 0.0;
-        fy[tid] = 0.0;
-        fz[tid] = 0.0;
-    }
+    fd_compute_first3(src, iF, jF, kF,
+                      iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
+                      SoA0, SoA1, SoA2,
+                      fx[tid], fy[tid], fz[tid]);
 }

 __global__ __launch_bounds__(128, 4)
@@ -1497,6 +1480,12 @@ void kern_fdderivs_batched(FDDerivTables tables, int field_count)
    const int jF = j0 + 1;
    const int kF = k0 + 1;

+#if ghost_width != 3
+    fd_compute_second6(src, iF, jF, kF,
+                       iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
+                       SoA0, SoA1, SoA2,
+                       fxx[tid], fxy[tid], fxz[tid], fyy[tid], fyz[tid], fzz[tid]);
+#else
    if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
        (jF + 2) <= jmaxF && (jF - 2) >= jminF &&
        (kF + 2) <= kmaxF && (kF - 2) >= kminF)
@@ -1624,12 +1613,43 @@ void kern_fdderivs_batched(FDDerivTables tables, int field_count)
        fxx[tid] = 0.0; fxy[tid] = 0.0; fxz[tid] = 0.0;
        fyy[tid] = 0.0; fyz[tid] = 0.0; fzz[tid] = 0.0;
    }
+#endif
 }

+static void gpu_fderivs_batch(int field_count,
+                              double *const *src_fields,
+                              double *const *fx_fields,
+                              double *const *fy_fields,
+                              double *const *fz_fields,
+                              const int *soa_signs,
+                              int all);
+static void gpu_fdderivs_batch(int field_count,
+                               double *const *src_fields,
+                               double *const *fxx_fields,
+                               double *const *fxy_fields,
+                               double *const *fxz_fields,
+                               double *const *fyy_fields,
+                               double *const *fyz_fields,
+                               double *const *fzz_fields,
+                               const int *soa_signs,
+                               int all);
+static void gpu_lopsided_kodis_single_batch(double *d_f_adv, double *d_f_ko, double *d_f_rhs,
+                                            double *d_Sfx, double *d_Sfy, double *d_Sfz,
+                                            double SoA0, double SoA1, double SoA2,
+                                            double eps_val, int all);
+
 /* symmetry_bd on GPU for ord=2, then launch fderivs kernel */
 static void gpu_fderivs(double *d_f, double *d_fx, double *d_fy, double *d_fz,
                        double SoA0, double SoA1, double SoA2, int all)
 {
+#if ghost_width != 3
+    double *src_fields[1] = {d_f};
+    double *fx_fields[1] = {d_fx};
+    double *fy_fields[1] = {d_fy};
+    double *fz_fields[1] = {d_fz};
+    const int soa_signs[3] = {(int)SoA0, (int)SoA1, (int)SoA2};
+    gpu_fderivs_batch(1, src_fields, fx_fields, fy_fields, fz_fields, soa_signs, all);
+#else
    double *fh = g_buf.d_fh2;
    const size_t nx = (size_t)g_buf.prev_nx;
    const size_t ny = (size_t)g_buf.prev_ny;
@@ -1638,6 +1658,7 @@ static void gpu_fderivs(double *d_f, double *d_fx, double *d_fy, double *d_fz,

    kern_symbd_pack_ord2<<<grid(w_pack), BLK>>>(d_f, fh, SoA0, SoA1, SoA2);
    kern_fderivs<<<grid(all), BLK>>>(fh, d_fx, d_fy, d_fz);
+#endif
 }

 /* symmetry_bd on GPU for ord=2, then launch fdderivs kernel */
@@ -1646,6 +1667,18 @@ static void gpu_fdderivs(double *d_f,
                         double *d_fyy, double *d_fyz, double *d_fzz,
                         double SoA0, double SoA1, double SoA2, int all)
 {
+#if ghost_width != 3
+    double *src_fields[1] = {d_f};
+    double *fxx_fields[1] = {d_fxx};
+    double *fxy_fields[1] = {d_fxy};
+    double *fxz_fields[1] = {d_fxz};
+    double *fyy_fields[1] = {d_fyy};
+    double *fyz_fields[1] = {d_fyz};
+    double *fzz_fields[1] = {d_fzz};
+    const int soa_signs[3] = {(int)SoA0, (int)SoA1, (int)SoA2};
+    gpu_fdderivs_batch(1, src_fields, fxx_fields, fxy_fields, fxz_fields,
+                       fyy_fields, fyz_fields, fzz_fields, soa_signs, all);
+#else
    double *fh = g_buf.d_fh2;
    const size_t nx = (size_t)g_buf.prev_nx;
    const size_t ny = (size_t)g_buf.prev_ny;
@@ -1654,6 +1687,7 @@ static void gpu_fdderivs(double *d_f,

    kern_symbd_pack_ord2<<<grid(w_pack), BLK>>>(d_f, fh, SoA0, SoA1, SoA2);
    kern_fdderivs<<<grid(all), BLK>>>(fh, d_fxx, d_fxy, d_fxz, d_fyy, d_fyz, d_fzz);
+#endif
 }

 static void gpu_fderivs_batch(int field_count,
@@ -1743,6 +1777,12 @@ void kern_phase10_ricci_batched(const double * __restrict__ gupxx,
        const int jF = j0 + 1;
        const int kF = k0 + 1;

+#if ghost_width != 3
+        fd_compute_second6(src, iF, jF, kF,
+                           iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
+                           SoA0, SoA1, SoA2,
+                           fxx, fxy, fxz, fyy, fyz, fzz);
+#else
        if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
            (jF + 2) <= jmaxF && (jF - 2) >= jminF &&
            (kF + 2) <= kmaxF && (kF - 2) >= kminF)
@@ -1866,6 +1906,7 @@ void kern_phase10_ricci_batched(const double * __restrict__ gupxx,
              - fetch_sym_ord2_direct(src, iF, jF - 1, kF + 1, SoA0, SoA1, SoA2)
              + fetch_sym_ord2_direct(src, iF, jF + 1, kF + 1, SoA0, SoA1, SoA2));
        }
+#endif
    }

    dst[tid] = gupxx[tid] * fxx + gupyy[tid] * fyy + gupzz[tid] * fzz
@@ -1930,6 +1971,16 @@ void kern_phase14_lap_chi_derivs(const double * __restrict__ Lap,
    const int jF = j0 + 1;
    const int kF = k0 + 1;

+#if ghost_width != 3
+    fd_compute_second6(Lap, iF, jF, kF,
+                       iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
+                       1, 1, 1,
+                       fxx[tid], fxy[tid], fxz[tid], fyy[tid], fyz[tid], fzz[tid]);
+    fd_compute_first3(chi, iF, jF, kF,
+                      iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
+                      1, 1, 1,
+                      chix_out[tid], chiy_out[tid], chiz_out[tid]);
+#else
    if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
        (jF + 2) <= jmaxF && (jF - 2) >= jminF &&
        (kF + 2) <= kmaxF && (kF - 2) >= kminF)
@@ -2083,6 +2134,7 @@ void kern_phase14_lap_chi_derivs(const double * __restrict__ Lap,
        fyy[tid] = 0.0; fyz[tid] = 0.0; fzz[tid] = 0.0;
        chix_out[tid] = 0.0; chiy_out[tid] = 0.0; chiz_out[tid] = 0.0;
    }
+#endif
 }

 /* Combined ord=3 advection + KO dissipation.
@@ -2094,6 +2146,11 @@ static void gpu_lopsided_kodis(double *d_f_adv, double *d_f_ko, double *d_f_rhs,
                               double SoA0, double SoA1, double SoA2,
                               double eps_val, int all)
 {
+#if ghost_width != 3
+    gpu_lopsided_kodis_single_batch(d_f_adv, d_f_ko, d_f_rhs,
+                                    d_Sfx, d_Sfy, d_Sfz,
+                                    SoA0, SoA1, SoA2, eps_val, all);
+#else
    double *fh = g_buf.d_fh3;
    const size_t nx = (size_t)g_buf.prev_nx;
    const size_t ny = (size_t)g_buf.prev_ny;
@@ -2109,6 +2166,7 @@ static void gpu_lopsided_kodis(double *d_f_adv, double *d_f_ko, double *d_f_rhs,
        }
        kern_kodis<<<grid(all), BLK>>>(fh, d_f_rhs, eps_val);
    }
+#endif
 }

 __global__ __launch_bounds__(128, 4)
@@ -2141,6 +2199,24 @@ void kern_lopsided_kodis_batched(const double * __restrict__ Sfx,
    const int jF = j0 + 1;
    const int kF = k0 + 1;

+#if ghost_width != 3
+    if (do_lopsided && i0 <= nx - 2 && j0 <= ny - 2 && k0 <= nz - 2) {
+        const double val =
+            fd_lopsided_axis(adv_src, iF, jF, kF, 0, Sfx[tid], iF, iminF, imaxF,
+                             d_gp.dX, SoA0, SoA1, SoA2)
+          + fd_lopsided_axis(adv_src, iF, jF, kF, 1, Sfy[tid], jF, jminF, jmaxF,
+                             d_gp.dY, SoA0, SoA1, SoA2)
+          + fd_lopsided_axis(adv_src, iF, jF, kF, 2, Sfz[tid], kF, kminF, kmaxF,
+                             d_gp.dZ, SoA0, SoA1, SoA2);
+        rhs[tid] += val;
+    }
+
+    if (do_kodis) {
+        rhs[tid] += fd_ko_term(ko_src, iF, jF, kF,
+                               iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
+                               eps_val, SoA0, SoA1, SoA2);
+    }
+#else
    if (do_lopsided && i0 <= nx - 2 && j0 <= ny - 2 && k0 <= nz - 2) {
        double val = 0.0;

@@ -2323,6 +2399,25 @@ void kern_lopsided_kodis_batched(const double * __restrict__ Sfx,

        rhs[tid] += (eps_val / cof) * (Dx / d_gp.dX + Dy / d_gp.dY + Dz / d_gp.dZ);
    }
+#endif
+}
+
+static void gpu_lopsided_kodis_single_batch(double *d_f_adv, double *d_f_ko, double *d_f_rhs,
+                                            double *d_Sfx, double *d_Sfy, double *d_Sfz,
+                                            double SoA0, double SoA1, double SoA2,
+                                            double eps_val, int all)
+{
+    LopsidedKodisTables tables = {};
+    tables.adv_fields[0] = d_f_adv;
+    tables.ko_fields[0] = d_f_ko;
+    tables.rhs_fields[0] = d_f_rhs;
+    tables.soa_signs[0] = (int)SoA0;
+    tables.soa_signs[1] = (int)SoA1;
+    tables.soa_signs[2] = (int)SoA2;
+
+    dim3 launch_grid((unsigned int)grid((size_t)all), 1u);
+    kern_lopsided_kodis_batched<<<launch_grid, BLK>>>(
+        d_Sfx, d_Sfy, d_Sfz, tables, eps_val, 1, eps_val > 0.0 ? 1 : 0);
 }

 static void gpu_lopsided_kodis_state_batch(double eps_val, int all)
@@ -3873,6 +3968,12 @@ void kern_phase12_13_chi_correction_fused(
        const int jF = j0 + 1;
        const int kF = k0 + 1;

+#if ghost_width != 3
+        fd_compute_second6(chi, iF, jF, kF,
+                           iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
+                           1, 1, 1,
+                           cxx, cxy, cxz, cyy, cyz, czz);
+#else
        if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
            (jF + 2) <= jmaxF && (jF - 2) >= jminF &&
            (kF + 2) <= kmaxF && (kF - 2) >= kminF)
@@ -3996,6 +4097,7 @@ void kern_phase12_13_chi_correction_fused(
              - fetch_sym_ord2_direct(chi, iF, jF - 1, kF + 1, 1, 1, 1)
              + fetch_sym_ord2_direct(chi, iF, jF + 1, kF + 1, 1, 1, 1));
        }
+#endif
    }

    const double cx = chix[tid];
@@ -4166,6 +4268,12 @@ void kern_phase15_trK_Aij_gauge(
        double fyy_v = 0.0, fyz_v = 0.0, fzz_v = 0.0;

        if (!(i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2)) {
+#if ghost_width != 3
+            fd_compute_second6(alpn1, iF, jF, kF,
+                               iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
+                               1, 1, 1,
+                               fxx_v, fxy_v, fxz_v, fyy_v, fyz_v, fzz_v);
+#else
            if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
                (jF + 2) <= jmaxF && (jF - 2) >= jminF &&
                (kF + 2) <= kmaxF && (kF - 2) >= kminF)
@@ -4289,6 +4397,7 @@ void kern_phase15_trK_Aij_gauge(
                  - fetch_sym_ord2_direct(alpn1, iF, jF - 1, kF + 1, 1, 1, 1)
                  + fetch_sym_ord2_direct(alpn1, iF, jF + 1, kF + 1, 1, 1, 1));
            }
+#endif
        }

        /* raised chi/chi */
@@ -4626,15 +4735,15 @@ static void setup_grid_params(int *ex,
    gp.imaxF = nx;
    gp.jmaxF = ny;
    gp.kmaxF = nz;
-    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) gp.kminF = -1;
-    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) gp.iminF = -1;
-    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) gp.jminF = -1;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) gp.kminF = 2 - ghost_width;
+    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) gp.iminF = 2 - ghost_width;
+    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) gp.jminF = 2 - ghost_width;
    gp.iminF3 = 1;
    gp.jminF3 = 1;
    gp.kminF3 = 1;
-    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) gp.kminF3 = -2;
-    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) gp.iminF3 = -2;
-    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) gp.jminF3 = -2;
+    if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) gp.kminF3 = 1 - ghost_width;
+    if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) gp.iminF3 = 1 - ghost_width;
+    if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) gp.jminF3 = 1 - ghost_width;
    gp.Symmetry = Symmetry;
    gp.eps = eps;
    gp.co = co;
@@ -5089,6 +5198,196 @@ __global__ void kern_unpack_state_region_batch(double * __restrict__ dst_mem,
    }
 }

+__device__ __forceinline__ double load_comm_state_cell_sym(const double * __restrict__ src_mem,
+                                                           int state_index,
+                                                           int x, int y, int z,
+                                                           int nx, int ny,
+                                                           int all)
+{
+    double s = 1.0;
+    if (x < 0) {
+        x = -x;
+        s *= d_comm_state_soa[3 * state_index + 0];
+    }
+    if (y < 0) {
+        y = -y;
+        s *= d_comm_state_soa[3 * state_index + 1];
+    }
+    if (z < 0) {
+        z = -z;
+        s *= d_comm_state_soa[3 * state_index + 2];
+    }
+    const int src = x + y * nx + z * nx * ny;
+    return s * src_mem[(size_t)state_index * all + src];
+}
+
+__global__ void kern_restrict_state_region_batch(const double * __restrict__ src_mem,
+                                                 double * __restrict__ dst,
+                                                 int nx, int ny,
+                                                 int sx, int sy, int sz,
+                                                 int fi0, int fj0, int fk0,
+                                                 int region_all,
+                                                 int state_count,
+                                                 int all)
+{
+    const int state_index = blockIdx.y;
+    if (state_index >= state_count) return;
+#if ghost_width == 5
+    const double c1 = 35.0 / 65536.0;
+    const double c2 = -405.0 / 65536.0;
+    const double c3 = 567.0 / 16384.0;
+    const double c4 = -2205.0 / 16384.0;
+    const double c5 = 19845.0 / 32768.0;
+    const int offs[10] = {-4, -3, -2, -1, 0, 1, 2, 3, 4, 5};
+    const double w[10] = {c1, c2, c3, c4, c5, c5, c4, c3, c2, c1};
+    const int nst = 10;
+#elif ghost_width == 4
+    const double c1 = -5.0 / 2048.0;
+    const double c2 = 49.0 / 2048.0;
+    const double c3 = -245.0 / 2048.0;
+    const double c4 = 1225.0 / 2048.0;
+    const int offs[8] = {-3, -2, -1, 0, 1, 2, 3, 4};
+    const double w[8] = {c1, c2, c3, c4, c4, c3, c2, c1};
+    const int nst = 8;
+#elif ghost_width == 3
+    const double c1 = 3.0 / 256.0;
+    const double c2 = -25.0 / 256.0;
+    const double c3 = 75.0 / 128.0;
+    const int offs[6] = {-2, -1, 0, 1, 2, 3};
+    const double w[6] = {c1, c2, c3, c3, c2, c1};
+    const int nst = 6;
+#else
+    const double c1 = -1.0 / 16.0;
+    const double c2 = 9.0 / 16.0;
+    const int offs[4] = {-1, 0, 1, 2};
+    const double w[4] = {c1, c2, c2, c1};
+    const int nst = 4;
+#endif
+
+    for (int local = blockIdx.x * blockDim.x + threadIdx.x;
+         local < region_all;
+         local += blockDim.x * gridDim.x)
+    {
+        const int ii = local % sx;
+        const int jj = (local / sx) % sy;
+        const int kk = local / (sx * sy);
+        const int fc_i = fi0 + 2 * ii;
+        const int fc_j = fj0 + 2 * jj;
+        const int fc_k = fk0 + 2 * kk;
+        double sum = 0.0;
+        for (int oz = 0; oz < nst; ++oz) {
+            const int z = fc_k + offs[oz];
+            const double wz = w[oz];
+            for (int oy = 0; oy < nst; ++oy) {
+                const int y = fc_j + offs[oy];
+                const double wyz = wz * w[oy];
+                for (int ox = 0; ox < nst; ++ox) {
+                    const int x = fc_i + offs[ox];
+                    sum += wyz * w[ox] *
+                           load_comm_state_cell_sym(src_mem, state_index, x, y, z, nx, ny, all);
+                }
+            }
+        }
+        dst[(size_t)state_index * region_all + local] = sum;
+    }
+}
+
+__global__ void kern_prolong_state_region_batch(const double * __restrict__ src_mem,
+                                                double * __restrict__ dst,
+                                                int nx, int ny,
+                                                int sx, int sy, int sz,
+                                                int ii0, int jj0, int kk0,
+                                                int lbc_i, int lbc_j, int lbc_k,
+                                                int region_all,
+                                                int state_count,
+                                                int all)
+{
+    const int state_index = blockIdx.y;
+    if (state_index >= state_count) return;
+#if ghost_width == 5
+    const double c1 = 13585.0 / 33554432.0;
+    const double c2 = -159885.0 / 33554432.0;
+    const double c3 = 230945.0 / 8388608.0;
+    const double c4 = -969969.0 / 8388608.0;
+    const double c5 = 14549535.0 / 16777216.0;
+    const double c6 = 4849845.0 / 16777216.0;
+    const double c7 = -692835.0 / 8388608.0;
+    const double c8 = 188955.0 / 8388608.0;
+    const double c9 = -138567.0 / 33554432.0;
+    const double c10 = 12155.0 / 33554432.0;
+    const int offs[10] = {-4, -3, -2, -1, 0, 1, 2, 3, 4, 5};
+    const double wl[10] = {c1, c2, c3, c4, c5, c6, c7, c8, c9, c10};
+    const double wr[10] = {c10, c9, c8, c7, c6, c5, c4, c3, c2, c1};
+    const int nst = 10;
+#elif ghost_width == 4
+    const double c1 = -495.0 / 262144.0;
+    const double c2 = 5005.0 / 262144.0;
+    const double c3 = -27027.0 / 262144.0;
+    const double c4 = 225225.0 / 262144.0;
+    const double c5 = 75075.0 / 262144.0;
+    const double c6 = -19305.0 / 262144.0;
+    const double c7 = 4095.0 / 262144.0;
+    const double c8 = -429.0 / 262144.0;
+    const int offs[8] = {-3, -2, -1, 0, 1, 2, 3, 4};
+    const double wl[8] = {c1, c2, c3, c4, c5, c6, c7, c8};
+    const double wr[8] = {c8, c7, c6, c5, c4, c3, c2, c1};
+    const int nst = 8;
+#elif ghost_width == 3
+    const double c1 = 77.0 / 8192.0;
+    const double c2 = -693.0 / 8192.0;
+    const double c3 = 3465.0 / 4096.0;
+    const double c4 = 1155.0 / 4096.0;
+    const double c5 = -495.0 / 8192.0;
+    const double c6 = 63.0 / 8192.0;
+    const int offs[6] = {-2, -1, 0, 1, 2, 3};
+    const double wl[6] = {c1, c2, c3, c4, c5, c6};
+    const double wr[6] = {c6, c5, c4, c3, c2, c1};
+    const int nst = 6;
+#else
+    const double c1 = -7.0 / 128.0;
+    const double c2 = 105.0 / 128.0;
+    const double c3 = 35.0 / 128.0;
+    const double c4 = -5.0 / 128.0;
+    const int offs[4] = {-1, 0, 1, 2};
+    const double wl[4] = {c1, c2, c3, c4};
+    const double wr[4] = {c4, c3, c2, c1};
+    const int nst = 4;
+#endif
+
+    for (int local = blockIdx.x * blockDim.x + threadIdx.x;
+         local < region_all;
+         local += blockDim.x * gridDim.x)
+    {
+        const int ii = local % sx;
+        const int jj = (local / sx) % sy;
+        const int kk = local / (sx * sy);
+        const int fine_i = ii0 + ii;
+        const int fine_j = jj0 + jj;
+        const int fine_k = kk0 + kk;
+        const int ci = fine_i / 2 - lbc_i;
+        const int cj = fine_j / 2 - lbc_j;
+        const int ck = fine_k / 2 - lbc_k;
+        const double *wx = ((fine_i / 2) * 2 == fine_i) ? wl : wr;
+        const double *wy = ((fine_j / 2) * 2 == fine_j) ? wl : wr;
+        const double *wz = ((fine_k / 2) * 2 == fine_k) ? wl : wr;
+        double sum = 0.0;
+        for (int oz = 0; oz < nst; ++oz) {
+            const int z = ck + offs[oz];
+            const double wzv = wz[oz];
+            for (int oy = 0; oy < nst; ++oy) {
+                const int y = cj + offs[oy];
+                const double wyz = wzv * wy[oy];
+                for (int ox = 0; ox < nst; ++ox) {
+                    const int x = ci + offs[ox];
+                    sum += wyz * wx[ox] *
+                           load_comm_state_cell_sym(src_mem, state_index, x, y, z, nx, ny, all);
+                }
+            }
+        }
+        dst[(size_t)state_index * region_all + local] = sum;
+    }
+}
+
 __global__ void kern_pack_state_subset(const double * __restrict__ src_mem,
                                       double * __restrict__ dst,
                                       int subset_count,
@@ -7511,6 +7810,60 @@ extern "C" int z4c_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
    return 0;
 }

+extern "C" int z4c_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                              int state_count,
+                                                              double *device_buffer,
+                                                              int *ex,
+                                                              int sx, int sy, int sz,
+                                                              int fi0, int fj0, int fk0,
+                                                              const double *state_soa)
+{
+    using namespace z4c_cuda;
+    init_gpu_dispatch();
+    CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
+    if (state_count <= 0 || state_count > BSSN_STATE_COUNT) return 1;
+    if (!device_buffer || sx <= 0 || sy <= 0 || sz <= 0) return 1;
+    StepContext &ctx = ensure_step_ctx(block_tag, (size_t)ex[0] * ex[1] * ex[2]);
+    const int region_all = sx * sy * sz;
+    upload_comm_state_soa(state_soa, state_count);
+    dim3 launch_grid((unsigned int)grid((size_t)region_all),
+                     (unsigned int)state_count);
+    kern_restrict_state_region_batch<<<launch_grid, BLK>>>(
+        ctx.d_state_curr_mem, device_buffer,
+        ex[0], ex[1], sx, sy, sz,
+        fi0, fj0, fk0, region_all, state_count,
+        ex[0] * ex[1] * ex[2]);
+    return 0;
+}
+
+extern "C" int z4c_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                             int state_count,
+                                                             double *device_buffer,
+                                                             int *ex,
+                                                             int sx, int sy, int sz,
+                                                             int ii0, int jj0, int kk0,
+                                                             int lbc_i, int lbc_j, int lbc_k,
+                                                             const double *state_soa)
+{
+    using namespace z4c_cuda;
+    init_gpu_dispatch();
+    CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
+    if (state_count <= 0 || state_count > BSSN_STATE_COUNT) return 1;
+    if (!device_buffer || sx <= 0 || sy <= 0 || sz <= 0) return 1;
+    StepContext &ctx = ensure_step_ctx(block_tag, (size_t)ex[0] * ex[1] * ex[2]);
+    const int region_all = sx * sy * sz;
+    upload_comm_state_soa(state_soa, state_count);
+    dim3 launch_grid((unsigned int)grid((size_t)region_all),
+                     (unsigned int)state_count);
+    kern_prolong_state_region_batch<<<launch_grid, BLK>>>(
+        ctx.d_state_curr_mem, device_buffer,
+        ex[0], ex[1], sx, sy, sz,
+        ii0, jj0, kk0, lbc_i, lbc_j, lbc_k,
+        region_all, state_count,
+        ex[0] * ex[1] * ex[2]);
+    return 0;
+}
+
 extern "C" int z4c_cuda_download_state_subset(void *block_tag,
                                              int *ex,
                                              int subset_count,
--- a/AMSS_NCKU_source/z4c_rhs_cuda.h
+++ b/AMSS_NCKU_source/z4c_rhs_cuda.h
@@ -74,6 +74,23 @@ int z4c_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
                                                   int i0, int j0, int k0,
                                                   int sx, int sy, int sz);

+int z4c_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int fi0, int fj0, int fk0,
+                                                   const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int ii0, int jj0, int kk0,
+                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                  const double *state_soa);
+
 int z4c_cuda_download_state_subset(void *block_tag,
                                   int *ex,
                                   int subset_count,
--- a/makefile_and_run.py
+++ b/makefile_and_run.py
@@ -138,18 +138,58 @@ def _stop_cuda_mps(runtime_env):

 def _gpu_runtime_env():
    runtime_env = os.environ.copy()
+    finite_difference = str(getattr(input_data, "Finite_Diffenence_Method", "4th-order")).strip()

    defaults = {
+        "AMSS_EVOLVE_TIMING": "1",
+        "AMSS_ESCALAR_STEP_TIMING": "0",
        "AMSS_INTERP_FAST": "1",
        "AMSS_INTERP_GPU": "1",
        "AMSS_ANALYSIS_MAP_EVERY": "1000000",
        "AMSS_CUDA_AWARE_MPI": "1",
        "AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP": "1",
        "AMSS_CUDA_KEEP_ALL_LEVELS": "1",
-        "AMSS_CUDA_AMR_RESTRICT_DEVICE": "1",
+        "AMSS_CUDA_ESCALAR_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_ESCALAR_KEEP_ALL_LEVELS": "1",
+        "AMSS_CUDA_EM_CACHE_SOURCES": "1",
+        "AMSS_CUDA_EM_ZERO_FASTPATH": "1",
+        "AMSS_EM_ZERO_ANALYSIS_FASTPATH": "1",
+        "AMSS_EM_ZERO_RESIDENT_DOWNLOAD_FASTPATH": "1",
+        "AMSS_CUDA_AMR_HOST_STAGED": "1",
+        "AMSS_CUDA_AMR_RESTRICT_DEVICE": "0",
        "AMSS_CUDA_AMR_RESTRICT_BATCH": "0",
        "AMSS_CUDA_DEVICE_SEGMENT_BATCH": "0",
+        "AMSS_CUDA_UNCACHED_DEVICE_BUFFERS": "1",
+        "AMSS_SHELL_FAST_INTERP": "0",
+        "AMSS_SHELL_PARALLEL_INTERP": "0",
+        "AMSS_SHELL_CUDA_INTERP": "0",
    }
+    if finite_difference in ("2nd-order", "8th-order"):
+        defaults.update({
+            "AMSS_INTERP_FAST": "0",
+            "AMSS_INTERP_GPU": "0",
+            "AMSS_CUDA_AWARE_MPI": "0",
+        })
+    if finite_difference == "8th-order" and getattr(input_data, "Equation_Class", "") == "BSSN-EM":
+        defaults.update({
+            "AMSS_CUDA_AMR_RESTRICT_DEVICE": "1",
+            "AMSS_CUDA_AMR_RESTRICT_BATCH": "1",
+            "AMSS_CUDA_DEVICE_SEGMENT_BATCH": "1",
+        })
+    if getattr(input_data, "basic_grid_set", "") == "Shell-Patch":
+        defaults.update({
+            "AMSS_CUDA_AWARE_MPI": "0",
+            "AMSS_SHELL_FAST_INTERP": "1",
+            "AMSS_SHELL_PARALLEL_INTERP": "1",
+            "AMSS_SHELL_INTERP_THREADS": "16",
+        })
+    if getattr(input_data, "Equation_Class", "") in ("BSSN", "BSSN-EScalar", "Z4C"):
+        defaults["AMSS_CUDA_AMR_RESTRICT_DEVICE"] = "1"
+    if getattr(input_data, "Equation_Class", "") == "Z4C":
+        defaults.update({
+            "AMSS_Z4C_CUDA_RESIDENT": "1",
+            "AMSS_CONSTRAINT_OUT_EVERY": "1000000",
+        })
    for key, value in defaults.items():
        runtime_env.setdefault(key, value)

@@ -261,25 +301,58 @@ def run_ABE():
    mpi_env = None
    started_mps = False
    
+    mpi_processes = int(input_data.MPI_processes)
+    if (input_data.GPU_Calculation == "yes" and
+        getattr(input_data, "Equation_Class", "") == "Z4C"):
+        z4c_env_np = os.environ.get("AMSS_Z4C_GPU_MPI_PROCESSES")
+        if z4c_env_np and int(z4c_env_np) > 0:
+            mpi_processes = int(z4c_env_np)
+        elif mpi_processes < 4:
+            mpi_processes = 4
+    if (input_data.GPU_Calculation == "yes" and
+        getattr(input_data, "basic_grid_set", "") == "Shell-Patch"):
+        shell_env_np = os.environ.get("AMSS_SHELL_GPU_MPI_PROCESSES")
+        if shell_env_np and int(shell_env_np) > 0:
+            mpi_processes = int(shell_env_np)
+        elif mpi_processes < 4:
+            mpi_processes = 4
+
    if (input_data.GPU_Calculation == "no"):
-        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
+        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(mpi_processes) + " ./ABE"
        #mpi_command         = " mpirun -np " + str(input_data.MPI_processes) + " ./ABE"
        mpi_command_outfile = "ABE_out.log"
    elif (input_data.GPU_Calculation == "yes"):
-        mpi_command         = NUMACTL_CPU_BIND + " mpirun -np " + str(input_data.MPI_processes) + " ./ABE_CUDA"
+        mpi_command         = NUMACTL_CPU_BIND + " I_MPI_OFFLOAD=1 I_MPI_OFFLOAD_IPC=0 mpirun -np " + str(mpi_processes) + " ./ABE_CUDA"
        mpi_command_outfile = "ABEGPU_out.log"
        mpi_env = _gpu_runtime_env()
        started_mps = _start_cuda_mps_if_requested(mpi_env)
        print(" GPU optimized runtime switches:")
+        print(f"   MPI processes={mpi_processes}")
        print(f"   AMSS_INTERP_FAST={mpi_env.get('AMSS_INTERP_FAST', '')}")
        print(f"   AMSS_INTERP_GPU={mpi_env.get('AMSS_INTERP_GPU', '')}")
        print(f"   AMSS_ANALYSIS_MAP_EVERY={mpi_env.get('AMSS_ANALYSIS_MAP_EVERY', '')}")
+        print(f"   AMSS_EVOLVE_TIMING={mpi_env.get('AMSS_EVOLVE_TIMING', '')}")
+        print(f"   AMSS_ESCALAR_STEP_TIMING={mpi_env.get('AMSS_ESCALAR_STEP_TIMING', '')}")
        print(f"   AMSS_CUDA_AWARE_MPI={mpi_env.get('AMSS_CUDA_AWARE_MPI', '')}")
        print(f"   AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP', '')}")
        print(f"   AMSS_CUDA_KEEP_ALL_LEVELS={mpi_env.get('AMSS_CUDA_KEEP_ALL_LEVELS', '')}")
+        print(f"   AMSS_CUDA_ESCALAR_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_ESCALAR_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_ESCALAR_KEEP_ALL_LEVELS={mpi_env.get('AMSS_CUDA_ESCALAR_KEEP_ALL_LEVELS', '')}")
+        print(f"   AMSS_CUDA_EM_CACHE_SOURCES={mpi_env.get('AMSS_CUDA_EM_CACHE_SOURCES', '')}")
+        print(f"   AMSS_CUDA_EM_ZERO_FASTPATH={mpi_env.get('AMSS_CUDA_EM_ZERO_FASTPATH', '')}")
+        print(f"   AMSS_EM_ZERO_ANALYSIS_FASTPATH={mpi_env.get('AMSS_EM_ZERO_ANALYSIS_FASTPATH', '')}")
+        print(f"   AMSS_EM_ZERO_RESIDENT_DOWNLOAD_FASTPATH={mpi_env.get('AMSS_EM_ZERO_RESIDENT_DOWNLOAD_FASTPATH', '')}")
+        print(f"   AMSS_CUDA_AMR_HOST_STAGED={mpi_env.get('AMSS_CUDA_AMR_HOST_STAGED', '')}")
        print(f"   AMSS_CUDA_AMR_RESTRICT_DEVICE={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_DEVICE', '')}")
        print(f"   AMSS_CUDA_AMR_RESTRICT_BATCH={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_BATCH', '')}")
        print(f"   AMSS_CUDA_DEVICE_SEGMENT_BATCH={mpi_env.get('AMSS_CUDA_DEVICE_SEGMENT_BATCH', '')}")
+        print(f"   AMSS_CUDA_UNCACHED_DEVICE_BUFFERS={mpi_env.get('AMSS_CUDA_UNCACHED_DEVICE_BUFFERS', '')}")
+        print(f"   AMSS_SHELL_FAST_INTERP={mpi_env.get('AMSS_SHELL_FAST_INTERP', '')}")
+        print(f"   AMSS_SHELL_PARALLEL_INTERP={mpi_env.get('AMSS_SHELL_PARALLEL_INTERP', '')}")
+        print(f"   AMSS_SHELL_CUDA_INTERP={mpi_env.get('AMSS_SHELL_CUDA_INTERP', '')}")
+        print(f"   AMSS_SHELL_INTERP_THREADS={mpi_env.get('AMSS_SHELL_INTERP_THREADS', '')}")
+        print(f"   AMSS_Z4C_CUDA_RESIDENT={mpi_env.get('AMSS_Z4C_CUDA_RESIDENT', '')}")
+        print(f"   AMSS_CONSTRAINT_OUT_EVERY={mpi_env.get('AMSS_CONSTRAINT_OUT_EVERY', '')}")
        if "CUDA_MPS_PIPE_DIRECTORY" in mpi_env:
            print(f"   CUDA_MPS_PIPE_DIRECTORY={mpi_env['CUDA_MPS_PIPE_DIRECTORY']}")
 
--- a/plot_xiaoqu.py
+++ b/plot_xiaoqu.py
@@ -808,10 +808,10 @@ def generate_ADMmass_plot( outdir, figure_outdir, detector_number_i ):

 ## Plot constraint violation for each grid level

-def generate_constraint_check_plot( outdir, figure_outdir, input_level_number ):
-
-    # path to data file
-    file0 = os.path.join(outdir, "bssn_constraint.dat")
+def generate_constraint_check_plot( outdir, figure_outdir, input_level_number ):
+
+    # path to data file
+    file0 = os.path.join(outdir, "bssn_constraint.dat")

    if ( input_level_number == 0 ):
        print(                                                   )
@@ -819,13 +819,26 @@ def generate_constraint_check_plot( outdir, figure_outdir, input_level_number ):
        print(                                                          )
        print( " corresponding data file = ", file0 )
        print(                                      )
-
-    print( " Begin the constraint violation plot for grid level number =  ", input_level_number )
-    
-    # load the full data file (assumed whitespace-separated floats)
-    data = numpy.loadtxt(file0)
-    
-    # extract columns from the constraint data file
+
+    print( " Begin the constraint violation plot for grid level number =  ", input_level_number )
+
+    if (not os.path.exists(file0)) or os.path.getsize(file0) == 0:
+        if ( input_level_number == 0 ):
+            print( " Constraint data file is empty; skip constraint violation plots" )
+            print(                                                            )
+        return
+
+    # load the full data file (assumed whitespace-separated floats)
+    data = numpy.loadtxt(file0)
+    data = numpy.atleast_2d(data)
+
+    if data.shape[1] < 8:
+        if ( input_level_number == 0 ):
+            print( " Constraint data file has insufficient columns; skip constraint violation plots" )
+            print(                                                                                )
+        return
+
+    # extract columns from the constraint data file
    time          = data[:,0]
    Constraint_H  = data[:,1]
    Constraint_Px = data[:,2]
Author	SHA1	Message	Date
CGH0S7	5d8dfaf679	Add plot-only restart script to skip recomputation when plotting is interrupted Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-12 15:01:25 +08:00
CGH0S7	24f4a45097	Fix macrodef.h include and clean up stale z4c_gpu_rhs_ss.cu Include macrodef.h (not macrodef.fh) in gpu_rhsSS_mem.h and bssn_gpu.h so that ABEtype is visible to #if guards in CUDA files. Remove the separate z4c_gpu_rhs_ss.cu (merged into bssn_gpu_rhs_ss.cu). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-10 20:02:35 +08:00
CGH0S7	f16469ea77	Simplify Z4C Shell GPU: CPU-side trKd+TZ_rhs wrapper Replace the duplicated z4c_gpu_rhs_ss.cu with a lightweight gpu_rhs_z4c_ss wrapper inside bssn_gpu_rhs_ss.cu (guarded by #if ABEtype==2). The wrapper: 1. Builds trKd = trK + 2TZ on host and passes it to gpu_rhs_ss 2. After BSSN GPU returns, computes TZ_rhs = alpn1Hcon/2 and applies kappa1/kappa2 constraint damping on CPU This avoids duplicate kernel definitions (linker errors) and keeps all shell GPU code in a single file. The CPU-side Z4C corrections are O(100K) operations — negligible vs GPU RHS time. Also remove the separate z4c_gpu_rhs_ss.cu and its build rule. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-10 16:05:56 +08:00
CGH0S7	f754aa1ec2	Add Z4C Shell-Patch GPU acceleration (Phase 3 complete) Create z4c_gpu_rhs_ss.cu (reusing BSSN shell FD/chain-rule kernels): - Uploads trKd = trK + 2TZ to GPU so existing BSSN algebraic kernels compute correct Z4C physical equations without modification - New kern_z4c_post applies TZ_rhs = alpn1 Hcon / 2, kappa1/kappa2 constraint damping, TZ advection (lopsided), and dissipation (kodis) - Adds TZ/TZ_rhs to Meta struct, alloc/upload/download/free lifecycle Add cuda_compute_rhs_z4c_ss() wrapper in Z4c_class.C matching the Fortran f_compute_rhs_Z4c_ss signature, with #define redirection for Step/SHStep call sites and #undef before analysis functions. Add z4c_gpu_rhs_ss.o to ABE_CUDA_CFILES and build rule in makefile. Add kappa1_c/kappa2_c constants to gpu_rhsSS_mem.h. Build verified with USE_CUDA_Z4C=1 + WithShell — compiles and links cleanly. All three Shell GPU files now coexist: bssn_gpu_rhs_ss.o (BSSN), z4c_gpu_rhs_ss.o (Z4C), both sharing FD/chain-rule kernels. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-10 13:52:48 +08:00
CGH0S7	c4194214c6	Enable Z4C + Shell-Patch GPU coexistence (Phase 3) Remove the compile-time #error that blocked USE_CUDA_Z4C + WithShell. Add GPU-to-CPU state sync at the start of both Z4C Step functions (non-CPBC and CPBC) so shell CPU consumers read valid field data after Cartesian GPU RHS with resident state. Move bssn_cuda_use_resident_sync and bssn_cuda_download_level_state _if_present from anonymous namespace to file scope in bssn_class.C so derived classes (Z4C) can call them. Declare both in bssn_rhs_cuda.h. Include bssn_rhs_cuda.h in Z4c_class.C. Z4C shell RHS remains on CPU (Fortran Z4c_rhs_ss.f90) pending future GPU kernel implementation. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-10 12:08:02 +08:00
CGH0S7	0ca86afd41	Use static OpenMP schedule in ShellPatch::setupintintstuff Static scheduling has lower overhead than guided for uniform workloads (grid points all have equal computational cost). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-10 02:23:07 +08:00
CGH0S7	f5bf3ab252	Add thread-safe ShellPatch::setupintintstuff with OpenMP Split prolongpointstru into search-only (prolongpointstru_search) and append-only (prolongpointstru_append) functions. The search is read-only and thread-safe; each thread builds private linked lists via prolongpointstru_append, merged after the parallel loop. This eliminates critical-section contention and delivers ~2.2x speedup: setupintintstuff: 511s -> 252s, total init: 592s -> 267s. Also add -qopenmp to ShellPatch.o compilation via makefile override rule and <omp.h> include with _OPENMP guards + fallback stubs. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-10 02:10:20 +08:00
CGH0S7	d0d3f965a6	Add diagnostic timing to Shell-Patch initialization Print MPI_Wtime breakdown of Initialize() shell setup steps and Read_Ansorg::Compute_Constraint duration. Reveals that ShellPatch::setupintintstuff() takes ~511s of the ~590s startup. The function builds interpolation tables by searching every shell grid point against all Cartesian patches — thread-safe OpenMP parallelization is blocked by shared linked-list mutations in prolongpointstru(), which would need a search/append split first. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-09 21:51:07 +08:00
CGH0S7	fbb2ed112d	Fix Compile_Constraint/analysis use CPU Fortran for shell RHS Limit GPU shell RHS redirection to Step and SHStep only via #define/#undef. Compute_Constraint, Interp_Constraint, and Constraint_Out continue using the CPU Fortran path to avoid GPU alloc-per-call overhead during initialization and analysis phases. Also: wrap compare_result_gpu in #ifdef RESULT_CHECK to avoid link error. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-09 19:25:45 +08:00
CGH0S7	bd4ce3fbf3	GPU-accelerate Shell-Patch BSSN evolution Phase 1: Enable GPU resident state for Cartesian patches in Shell mode. - Remove WithShell guard from bssn_cuda_use_resident_sync(). - Add GPU-to-CPU state sync before shell CPU consumers (SHStep, CS_Inter, inline shell RHS blocks). Phase 2: GPU-accelerate BSSN Shell Patch RHS. - Create bssn_gpu.h with RHS_SS_PARA macro and gpu_rhs_ss declaration. - Fix compilation bugs in legacy bssn_gpu_rhs_ss.cu (deprecated cudaThreadSynchronize, tmp_con2 redeclaration, ijkmin3_h typo, CUDA_SAFE_CALL, missing compare_result guard). - Add bssn_gpu_rhs_ss.o to CFILES_CUDA_BSSN with build rule. - Write cuda_compute_rhs_bssn_ss() wrapper bridging Fortran and GPU parameter conventions, redirect all shell RHS call sites via #define. Verified: 30-step Shell-Patch GPU run completes without errors/NaN. Step wall time ~4.4s (step_fn ~2.0s + RP ~0.68s + constraint ~0.70s). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-09 18:50:10 +08:00
CGH0S7	5eb49949d9	Fix AHF crash under CUDA resident-sync mode Download BSSN StateList from GPU to CPU before AHFinderDirect_find_horizons so that AH_Interp_Points reads valid field data instead of stale CPU arrays. The resident-sync path keeps canonical state on GPU; without this download the Newton iteration diverges and probes outside the computational domain. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>	2026-05-09 16:11:56 +08:00
CGH0S7	39450228f5	Accelerate Shell-Patch interpolation fast paths	2026-05-08 13:26:16 +08:00
CGH0S7	063f28b3b4	Add Shell-Patch GPU runtime fast paths	2026-05-08 09:26:36 +08:00
CGH0S7	1064a68d16	Optimize BSSN-EM 8th-order AMR transfers	2026-05-07 21:38:16 +08:00
CGH0S7	dcc83bafcb	Support 2nd and 8th order CUDA AMR paths	2026-05-07 20:31:26 +08:00
CGH0S7	c4d8d41b25	Cover Z4C CUDA AMR restrict prolong	2026-05-07 19:49:09 +08:00
CGH0S7	0076b3ca18	Optimize 6th-order CUDA AMR stencils	2026-05-07 19:22:37 +08:00
CGH0S7	9ff2f065be	Apply BSSN AMR sync default to EScalar	2026-05-07 17:12:33 +08:00
CGH0S7	2317e4abde	Fix BSSN GPU resident AMR sync default	2026-05-07 17:11:09 +08:00
CGH0S7	fea2dcc0d5	Fix BSSN-EM runtime crash	2026-05-07 16:47:55 +08:00
CGH0S7	5525465cad	Support CUDA finite-difference order selection	2026-05-07 16:28:02 +08:00
CGH0S7	96829d0441	Optimize Z4C GPU runtime defaults	2026-05-07 15:37:09 +08:00
CGH0S7	83afaf19ce	Skip zero EM resident downloads	2026-05-07 13:04:46 +08:00
CGH0S7	cb911dec06	Add EM GPU fast paths and defaults	2026-05-07 12:18:56 +08:00
CGH0S7	dd0e20d8c7	Fix BSSN-EScalar CUDA boundary and scalar KO	2026-05-06 15:44:35 +08:00
CGH0S7	ffa0d801ed	Default Python GPU runner to EScalar fast path	2026-05-06 00:12:46 +08:00
CGH0S7	ae64a22178	Complete BSSN-EScalar CUDA resident transfers	2026-05-05 23:57:42 +08:00
CGH0S7	85fe29cc2e	Optimize BSSN-EScalar CUDA path	2026-05-05 10:47:46 +08:00
ianchb	06f62dee36	Switch back to Intel toolchain as the default option Seems that Intel MPI also supports CUDA-aware by setting I_MPI_OFFLOAD to 1. Besides, I_MPI_OFFLOAD_IPC=0 is needed to avoid segfaults.	2026-05-01 21:59:13 +08:00