Stabilize EScalar CUDA fallback path

AMSS_NCKU_Input.py

@@ -31,7 +31,7 @@ GPU_Part         = 0.0
 ## Setting the physical system and numerical method
 
 Symmetry                 = "equatorial-symmetry"   ## Symmetry of System: choose equatorial-symmetry、no-symmetry、octant-symmetry
-Equation_Class           = "BSSN"                  ## Evolution Equation: choose "BSSN", "BSSN-EScalar", "BSSN-EM", "Z4C" 
+Equation_Class           = "BSSN-EScalar"                  ## Evolution Equation: choose "BSSN", "BSSN-EScalar", "BSSN-EM", "Z4C"
                                                    ## If "BSSN-EScalar" is chosen, it is necessary to set other parameters below
 Initial_Data_Method      = "Ansorg-TwoPuncture"    ## initial data method: choose "Ansorg-TwoPuncture", "Lousto-Analytical", "Cao-Analytical", "KerrSchild-Analytical"
 Time_Evolution_Method    = "runge-kutta-45"        ## time evolution method: choose "runge-kutta-45"

AMSS_NCKU_source/Z4c_class.C

@@ -1,9 +1,10 @@
 
-#ifdef newc
-#include <sstream>
-#include <cstdio>
-#include <map>
-using namespace std;
+#ifdef newc
+#include <sstream>
+#include <cstdio>
+#include <cstdlib>
+#include <map>
+using namespace std;
 #else
 #include <stdio.h>
 #include <map.h>
@@ -113,20 +114,22 @@ void Z4c_class::Initialize()
   else
     GH->compose_cgh(nprocs);
 
-#ifdef WithShell
-  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
-  if (!checkrun)
-    SH->matchcheck(GH->PatL[0]);
+#ifdef WithShell
+  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
+  if (!checkrun)
+    SH->matchcheck(GH->PatL[0]);
   SH->compose_sh(nprocs);
   SH->setupcordtrans();
   SH->Dump_xyz(0, 0, 1);
   SH->setupintintstuff(nprocs, GH->PatL[0], Symmetry);
 
-  if (checkrun)
-    CheckPoint->readcheck_sh(SH, myrank);
-#endif
-
-  double h = GH->PatL[0]->data->blb->data->getdX(0);
+  if (checkrun)
+    CheckPoint->readcheck_sh(SH, myrank);
+#endif
+
+  Initialize_Level_Runtime();
+
+  double h = GH->PatL[0]->data->blb->data->getdX(0);
   for (int i = 1; i < dim; i++)
     h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
   dT = Courant * h;
@@ -213,6 +216,35 @@ bool fill_z4c_cuda_views(Block *cg, MyList<var> *vars,
   return idx == Z4C_CUDA_STATE_COUNT && vars == 0;
 }
 
+bool z4c_cuda_keep_resident_after_step(int lev, int trfls_in, int analysis_lev)
+{
+  static int keep_all_levels = -1;
+  if (keep_all_levels < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_KEEP_ALL_LEVELS");
+    keep_all_levels = (env && atoi(env) != 0) ? 1 : 0;
+  }
+  static int enabled = -1;
+  if (enabled < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP");
+    if (env)
+      enabled = (atoi(env) != 0) ? 1 : 0;
+    else
+    {
+      env = getenv("AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP");
+      enabled = (env && atoi(env) != 0) ? 1 : 0;
+    }
+  }
+  if (!enabled)
+    return false;
+  if (lev == analysis_lev)
+    return false;
+  if (keep_all_levels)
+    return true;
+  return lev < trfls_in;
+}
+
 void z4c_cuda_download_level_state(MyList<Patch> *PatL, MyList<var> *vars, int myrank, bool release_ctx)
 {
   MyList<Patch> *Pp = PatL;
@@ -356,41 +388,57 @@ bool z4c_cuda_interp_bh_point_resident(MyList<Patch> *PatL,
           if (z4c_cuda_has_resident_state(block) &&
               block->shape[0] >= ordn && block->shape[1] >= ordn && block->shape[2] >= ordn)
           {
-            const int sx = ordn;
-            const int sy = ordn;
-            const int sz = ordn;
-            const int region_all = sx * sy * sz;
-            const int i0 = z4c_cuda_interp_tile_start(block->X[0], block->shape[0], x, DH[0], ordn);
-            const int j0 = z4c_cuda_interp_tile_start(block->X[1], block->shape[1], y, DH[1], ordn);
-            const int k0 = z4c_cuda_interp_tile_start(block->X[2], block->shape[2], z, DH[2], ordn);
-            double *packed_fields = new double[3 * region_all];
             var *vars[3] = {forx, fory, forz};
-            for (int f = 0; f < 3; f++)
+            static int use_device_bh_interp = -1;
+            if (use_device_bh_interp < 0)
             {
-              if (z4c_cuda_pack_state_region_to_host_buffer(block,
-                                                            k_z4c_cuda_bh_state_indices[f],
-                                                            packed_fields + f * region_all,
-                                                            block->shape,
-                                                            i0, j0, k0,
-                                                            sx, sy, sz) != 0)
+              const char *env = getenv("AMSS_CUDA_Z4C_BH_INTERP_DEVICE");
+              use_device_bh_interp = (env && atoi(env) != 0) ? 1 : 0;
+            }
+            bool used_device_interp = false;
+            if (use_device_bh_interp)
+            {
+              double soa3[9];
+              for (int f = 0; f < 3; f++)
               {
-                delete[] packed_fields;
-                cout << "CUDA Z4C BH tile download failed" << endl;
+                soa3[3 * f + 0] = vars[f]->SoA[0];
+                soa3[3 * f + 1] = vars[f]->SoA[1];
+                soa3[3 * f + 2] = vars[f]->SoA[2];
+              }
+              used_device_interp =
+                  (z4c_cuda_interp_state_point3(block, block->shape,
+                                                k_z4c_cuda_bh_state_indices[0],
+                                                k_z4c_cuda_bh_state_indices[1],
+                                                k_z4c_cuda_bh_state_indices[2],
+                                                block->X[0][0], block->X[1][0], block->X[2][0],
+                                                DH[0], DH[1], DH[2],
+                                                x, y, z,
+                                                interp_ordn, interp_sym,
+                                                soa3, shellf) == 0);
+            }
+            if (!used_device_interp)
+            {
+              double *shift_views[3] = {
+                  block->fgfs[forx->sgfn],
+                  block->fgfs[fory->sgfn],
+                  block->fgfs[forz->sgfn]};
+              if (z4c_cuda_download_state_subset(block, block->shape, 3,
+                                                 k_z4c_cuda_bh_state_indices,
+                                                 shift_views) != 0)
+              {
+                cout << "CUDA Z4C BH shift download failed" << endl;
                 MPI_Abort(MPI_COMM_WORLD, 1);
               }
-              int tile_shape[3] = {sx, sy, sz};
-              f_global_interp(tile_shape,
-                              block->X[0] + i0,
-                              block->X[1] + j0,
-                              block->X[2] + k0,
-                              packed_fields + f * region_all,
-                              shellf[f],
-                              x, y, z,
-                              interp_ordn,
-                              vars[f]->SoA,
-                              interp_sym);
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                              block->fgfs[forx->sgfn], shellf[0],
+                              x, y, z, interp_ordn, forx->SoA, interp_sym);
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                              block->fgfs[fory->sgfn], shellf[1],
+                              x, y, z, interp_ordn, fory->SoA, interp_sym);
+              f_global_interp(block->shape, block->X[0], block->X[1], block->X[2],
+                              block->fgfs[forz->sgfn], shellf[2],
+                              x, y, z, interp_ordn, forz->SoA, interp_sym);
             }
-            delete[] packed_fields;
           }
           else
           {
@@ -452,6 +500,117 @@ bool z4c_cuda_compute_porg_rhs_resident(cgh *GH,
   return true;
 }
 
+bool z4c_cuda_download_bh_shift_level(MyList<Patch> *PatL,
+                                      int myrank,
+                                      var *forx, var *fory, var *forz)
+{
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank && z4c_cuda_has_resident_state(cg))
+      {
+        double *fields[3] = {
+            cg->fgfs[forx->sgfn],
+            cg->fgfs[fory->sgfn],
+            cg->fgfs[forz->sgfn]};
+        if (z4c_cuda_download_state_subset(cg, cg->shape, 3,
+                                           k_z4c_cuda_bh_state_indices,
+                                           fields))
+          return false;
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  return true;
+}
+
+bool z4c_cuda_refresh_constraint_level(MyList<Patch> *PatL,
+                                       int myrank,
+                                       var *Cons_Ham, var *Cons_Px,
+                                       var *Cons_Py, var *Cons_Pz,
+                                       var *Cons_Gx, var *Cons_Gy,
+                                       var *Cons_Gz, var *TZ0,
+                                       int Symmetry, int lev, double eps)
+{
+  bool all_resident = true;
+  const int tz_index = 24;
+  MyList<Patch> *Pp = PatL;
+  while (Pp)
+  {
+    MyList<Block> *BP = Pp->data->blb;
+    while (BP)
+    {
+      Block *cg = BP->data;
+      if (myrank == cg->rank)
+      {
+        if (!z4c_cuda_has_resident_state(cg))
+        {
+          all_resident = false;
+        }
+        else
+        {
+          double *constraints[7] = {
+              cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+              cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+              cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+              cg->fgfs[Cons_Gz->sgfn]};
+          double *tz_out[1] = {cg->fgfs[TZ0->sgfn]};
+          int co = 0;
+          if (z4c_cuda_compute_constraints_resident(cg, cg->shape,
+                                                   cg->X[0], cg->X[1], cg->X[2],
+                                                   Symmetry, eps, co,
+                                                   constraints) ||
+              z4c_cuda_download_state_subset(cg, cg->shape, 1, &tz_index, tz_out))
+          {
+            cout << "CUDA Z4C resident constraint refresh failed" << endl;
+            MPI_Abort(MPI_COMM_WORLD, 1);
+          }
+        }
+      }
+      if (BP == Pp->data->ble)
+        break;
+      BP = BP->next;
+    }
+    Pp = Pp->next;
+  }
+  return all_resident;
+}
+
+long long &z4c_constraint_output_counter()
+{
+  static long long counter = 0;
+  return counter;
+}
+
+int z4c_constraint_output_every()
+{
+  static int every = -1;
+  if (every < 0)
+  {
+    const char *env = getenv("AMSS_CUDA_Z4C_CONSTRAINT_EVERY");
+    every = (env && atoi(env) > 0) ? atoi(env) : 1;
+  }
+  return every;
+}
+
+bool z4c_constraint_output_due_now()
+{
+  const int every = z4c_constraint_output_every();
+  return every <= 1 || (z4c_constraint_output_counter() % every) == 0;
+}
+
+void z4c_constraint_output_advance()
+{
+  z4c_constraint_output_counter()++;
+}
+
 } // namespace
 #endif
 
@@ -470,6 +629,34 @@ void Z4c_class::Step(int lev, int YN)
   int iter_count = 0;
   int pre = 0, cor = 1;
   int ERROR = 0;
+  const double dT_mon = dT * pow(0.5, Mymax(0, trfls));
+  const bool need_constraint_after_step =
+      (LastConsOut + dT_mon >= AnasTime) && z4c_constraint_output_due_now();
+
+  if (BH_num > 0 && lev == GH->levels - 1)
+  {
+    if (!z4c_cuda_download_bh_shift_level(GH->PatL[lev], myrank, Sfx0, Sfy0, Sfz0))
+    {
+      if (myrank == 0 && ErrorMonitor->outfile)
+        ErrorMonitor->outfile << "CUDA Z4C failed to download predictor black-hole shift at t = "
+                              << PhysTime << endl;
+      MPI_Abort(MPI_COMM_WORLD, 1);
+    }
+    compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
+    for (int ithBH = 0; ithBH < BH_num; ithBH++)
+    {
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
+      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
+      if (Symmetry > 0)
+        Porg[ithBH][2] = fabs(Porg[ithBH][2]);
+      if (Symmetry == 2)
+      {
+        Porg[ithBH][0] = fabs(Porg[ithBH][0]);
+        Porg[ithBH][1] = fabs(Porg[ithBH][1]);
+      }
+    }
+  }
 
   MyList<Patch> *Pp = GH->PatL[lev];
   while (Pp)
@@ -537,24 +724,10 @@ void Z4c_class::Step(int lev, int YN)
     MPI_Abort(MPI_COMM_WORLD, 1);
   }
 
-  Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
-
-  if (BH_num > 0 && lev == GH->levels - 1)
   {
-    compute_Porg_rhs(Porg0, Porg_rhs, Sfx0, Sfy0, Sfz0, lev);
-    for (int ithBH = 0; ithBH < BH_num; ithBH++)
-    {
-      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][0], Porg[ithBH][0], Porg_rhs[ithBH][0], iter_count);
-      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][1], Porg[ithBH][1], Porg_rhs[ithBH][1], iter_count);
-      f_rungekutta4_scalar(dT_lev, Porg0[ithBH][2], Porg[ithBH][2], Porg_rhs[ithBH][2], iter_count);
-      if (Symmetry > 0)
-        Porg[ithBH][2] = fabs(Porg[ithBH][2]);
-      if (Symmetry == 2)
-      {
-        Porg[ithBH][0] = fabs(Porg[ithBH][0]);
-        Porg[ithBH][1] = fabs(Porg[ithBH][1]);
-      }
-    }
+    Parallel::AsyncSyncState async_pre;
+    Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
+    Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry);
   }
 
   if ((lev == a_lev) && (LastAnas + dT_lev >= AnasTime))
@@ -614,6 +787,25 @@ void Z4c_class::Step(int lev, int YN)
                  << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
             ERROR = 1;
           }
+          if (!ERROR && iter_count == 3 && need_constraint_after_step)
+          {
+            double *constraints[7] = {
+                cg->fgfs[Cons_Ham->sgfn], cg->fgfs[Cons_Px->sgfn],
+                cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn],
+                cg->fgfs[Cons_Gz->sgfn]};
+            double *tz_out[1] = {cg->fgfs[TZ0->sgfn]};
+            const int tz_index = 24;
+            if (z4c_cuda_download_constraint_outputs(cg->shape, constraints) ||
+                z4c_cuda_download_state_subset(cg, cg->shape, 1, &tz_index, tz_out))
+            {
+              cout << "CUDA Z4C constraint download failed in domain: ("
+                   << cg->bbox[0] << ":" << cg->bbox[3] << ","
+                   << cg->bbox[1] << ":" << cg->bbox[4] << ","
+                   << cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
+              ERROR = 1;
+            }
+          }
         }
         if (BP == Pp->data->ble)
           break;
@@ -635,7 +827,11 @@ void Z4c_class::Step(int lev, int YN)
       MPI_Abort(MPI_COMM_WORLD, 1);
     }
 
-    Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
+    {
+      Parallel::AsyncSyncState async_cor;
+      Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
+      Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry);
+    }
 
     if (BH_num > 0 && lev == GH->levels - 1)
     {
@@ -691,7 +887,13 @@ void Z4c_class::Step(int lev, int YN)
     }
   }
 
-  z4c_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, true);
+  {
+    const bool keep_resident = z4c_cuda_keep_resident_after_step(lev, trfls, a_lev);
+    const bool need_host_after_step =
+        ((lev == a_lev) && (LastAnas + dT_lev >= AnasTime));
+    if (!keep_resident || need_host_after_step)
+      z4c_cuda_download_level_state(GH->PatL[lev], SynchList_cor, myrank, !keep_resident);
+  }
 
 #if (RPS == 0)
   RestrictProlong(lev, YN, BB);
@@ -2962,17 +3164,23 @@ void Z4c_class::Check_extrop()
 
 //================================================================================================
 
-void Z4c_class::Constraint_Out()
-{
-  // here we have to use the same variable name as in the parent class
-  LastConsOut += dT * pow(0.5, Mymax(0, trfls));
-  
-  if (LastConsOut >= AnasTime)
-  // Constraint violation
-  {
-    // recompute least the constraint data lost for moved new grid
-    for (int lev = 0; lev < GH->levels; lev++)
-    {
+void Z4c_class::Constraint_Out()
+{
+  // here we have to use the same variable name as in the parent class
+  LastConsOut += dT * pow(0.5, Mymax(0, trfls));
+  
+  if (LastConsOut >= AnasTime)
+  // Constraint violation
+  {
+#if USE_CUDA_Z4C && (ABEtype == 2)
+    bool cuda_constraints_ready = true;
+#else
+    const bool cuda_constraints_ready = false;
+#endif
+    // recompute least the constraint data lost for moved new grid
+    if (!cuda_constraints_ready)
+      for (int lev = 0; lev < GH->levels; lev++)
+    {
       // make sure the data consistent for higher levels
       if (lev > 0)
       {

AMSS_NCKU_source/bssnEM_class.C

@@ -15,10 +15,13 @@ using namespace std;
 #include "misc.h"
 #include "Ansorg.h"
 #include "fmisc.h"
-#include "Parallel.h"
-#include "bssnEM_class.h"
-#include "bssn_rhs.h"
-#include "empart.h"
+#include "Parallel.h"
+#include "bssnEM_class.h"
+#include "bssn_rhs.h"
+#if USE_CUDA_BSSN
+#include "bssn_rhs_cuda.h"
+#endif
+#include "empart.h"
 #include "initial_puncture.h"
 #include "initial_maxwell.h"
 #include "enforce_algebra.h"
@@ -32,11 +35,111 @@ using namespace std;
 #ifdef With_AHF
 #include "derivatives.h"
 #include "myglobal.h"
-#endif
-
-//================================================================================================
-
-// Define bssnEM_class
+#endif
+
+//================================================================================================
+
+#if USE_CUDA_BSSN
+namespace {
+
+bool fill_bssn_cuda_views_prefix(Block *cg, MyList<var> *vars,
+                                 double **host_views,
+                                 double *propspeeds = nullptr,
+                                 double *soa_flat = nullptr)
+{
+  int idx = 0;
+  while (vars && idx < BSSN_CUDA_STATE_COUNT)
+  {
+    host_views[idx] = cg->fgfs[vars->data->sgfn];
+    if (propspeeds)
+      propspeeds[idx] = vars->data->propspeed;
+    if (soa_flat)
+    {
+      soa_flat[3 * idx + 0] = vars->data->SoA[0];
+      soa_flat[3 * idx + 1] = vars->data->SoA[1];
+      soa_flat[3 * idx + 2] = vars->data->SoA[2];
+    }
+    vars = vars->next;
+    ++idx;
+  }
+  return idx == BSSN_CUDA_STATE_COUNT;
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b, MyList<var> *&c)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+  }
+}
+
+void skip_bssn_cuda_prefix(MyList<var> *&a, MyList<var> *&b,
+                           MyList<var> *&c, MyList<var> *&d)
+{
+  for (int i = 0; i < BSSN_CUDA_STATE_COUNT && a && b && c && d; ++i)
+  {
+    a = a->next;
+    b = b->next;
+    c = c->next;
+    d = d->next;
+  }
+}
+
+int run_bssn_em_cuda_substep(Block *cg,
+                             MyList<var> *state_in_list,
+                             MyList<var> *state_out_list,
+                             Patch *patch,
+                             double &dT_lev,
+                             double &TRK4,
+                             int &iter_count,
+                             int &Symmetry,
+                             int lev,
+                             double &ndeps,
+                             int &co,
+                             double &chitiny,
+                             var *rho, var *Sx, var *Sy, var *Sz,
+                             var *Sxx, var *Sxy, var *Sxz,
+                             var *Syy, var *Syz, var *Szz)
+{
+  double *state_in[BSSN_CUDA_STATE_COUNT];
+  double *state_out[BSSN_CUDA_STATE_COUNT];
+  double *matter[BSSN_CUDA_MATTER_COUNT] = {
+      cg->fgfs[rho->sgfn], cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
+      cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+      cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn]};
+  double propspeed[BSSN_CUDA_STATE_COUNT];
+  double soa_flat[3 * BSSN_CUDA_STATE_COUNT];
+  if (!fill_bssn_cuda_views_prefix(cg, state_in_list, state_in, propspeed, soa_flat) ||
+      !fill_bssn_cuda_views_prefix(cg, state_out_list, state_out))
+    return 1;
+
+  int apply_bam_bc = 0;
+#if (SommerType == 0)
+#ifndef WithShell
+  apply_bam_bc = (lev == 0) ? 1 : 0;
+#endif
+#endif
+  int use_zero_matter = 0;
+  int keep_resident_state = 0;
+  int apply_enforce_ga = 0;
+  return bssn_cuda_rk4_substep(cg,
+                               cg->shape, cg->X[0], cg->X[1], cg->X[2],
+                               state_in, state_out, matter,
+                               propspeed, soa_flat, patch->bbox,
+                               dT_lev, TRK4, iter_count, apply_bam_bc,
+                               Symmetry, lev, ndeps, co,
+                               use_zero_matter,
+                               keep_resident_state, apply_enforce_ga, chitiny);
+}
+
+}
+#endif
+
+//================================================================================================
+
+// Define bssnEM_class
 
 // It inherits some members and methods from the parent class bssn_class and modifies others.
 // The modified members and methods are defined below (and in the header bssnEM_class.h).
@@ -232,19 +335,21 @@ void bssnEM_class::Initialize()
   else
     GH->compose_cgh(nprocs);
 
-#ifdef WithShell
-  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
-  SH->matchcheck(GH->PatL[0]);
+#ifdef WithShell
+  SH = new ShellPatch(0, ngfs, pname, Symmetry, myrank, ErrorMonitor);
+  SH->matchcheck(GH->PatL[0]);
   SH->compose_sh(nprocs);
   SH->setupcordtrans();
   SH->Dump_xyz(0, 0, 1);
   SH->setupintintstuff(nprocs, GH->PatL[0], Symmetry);
 
-  if (checkrun)
-    CheckPoint->readcheck_sh(SH, myrank);
-#endif
-
-  double h = GH->PatL[0]->data->blb->data->getdX(0);
+  if (checkrun)
+    CheckPoint->readcheck_sh(SH, myrank);
+#endif
+
+  Initialize_Level_Runtime();
+
+  double h = GH->PatL[0]->data->blb->data->getdX(0);
   for (int i = 1; i < dim; i++)
     h = Mymin(h, GH->PatL[0]->data->blb->data->getdX(i));
   dT = Courant * h;
@@ -851,10 +956,11 @@ void bssnEM_class::Step(int lev, int YN)
                      cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                      cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
                      cg->fgfs[Ayy0->sgfn], cg->fgfs[Ayz0->sgfn], cg->fgfs[Azz0->sgfn]);
-#endif
-
-        if (
-            f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+#endif
+
+        bool used_gpu_substep = false;
+        if (
+            f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                  cg->fgfs[phi0->sgfn],
                                  cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                  cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
@@ -871,11 +977,20 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Kpsi_rhs->sgfn], cg->fgfs[Kphi_rhs->sgfn],
                                  cg->fgfs[rho->sgfn], 
                                  cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
-                                 cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn], 
-                                 cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
-                                 Symmetry, lev, ndeps) ||
-            f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
-                               cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
+                                 cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+                                 cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
+                                 Symmetry, lev, ndeps) ||
+#if USE_CUDA_BSSN
+            ((used_gpu_substep =
+                  (run_bssn_em_cuda_substep(cg, StateList, SynchList_pre, Pp->data,
+                                            dT_lev, TRK4, iter_count, Symmetry, lev,
+                                            ndeps, pre, chitiny,
+                                            rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                 ? 0
+                 : 1) ||
+#endif
+            (!used_gpu_substep && f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+                               cg->fgfs[phi0->sgfn], cg->fgfs[trK0->sgfn],
                                cg->fgfs[gxx0->sgfn], cg->fgfs[gxy0->sgfn], cg->fgfs[gxz0->sgfn], 
                                cg->fgfs[gyy0->sgfn], cg->fgfs[gyz0->sgfn], cg->fgfs[gzz0->sgfn],
                                cg->fgfs[Axx0->sgfn], cg->fgfs[Axy0->sgfn], cg->fgfs[Axz0->sgfn], 
@@ -904,10 +1019,10 @@ void bssnEM_class::Step(int lev, int YN)
                                cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
                                cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn], 
                                cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
-                               cg->fgfs[Cons_Ham->sgfn],
-                               cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
-                               cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                               Symmetry, lev, ndeps, pre))
+                               cg->fgfs[Cons_Ham->sgfn],
+                               cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                               cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
+                               Symmetry, lev, ndeps, pre)))
         {
           cout << "find NaN in domain: (" 
                << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -917,11 +1032,15 @@ void bssnEM_class::Step(int lev, int YN)
         }
 
         // rk4 substep and boundary
-        {
-          MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList; 
-          // we do not check the correspondence here
-          
-          while (varl0)
+        {
+          MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varlrhs = RHSList;
+          // we do not check the correspondence here
+#if USE_CUDA_BSSN
+          if (used_gpu_substep)
+            skip_bssn_cuda_prefix(varl0, varl, varlrhs);
+#endif
+
+          while (varl0)
           {
 #ifndef WithShell
             if (lev == 0) // sommerfeld indeed
@@ -1307,10 +1426,11 @@ void bssnEM_class::Step(int lev, int YN)
                          cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                          cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
                          cg->fgfs[Ayy->sgfn], cg->fgfs[Ayz->sgfn], cg->fgfs[Azz->sgfn]);
-#endif
-
-          if (
-              f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
+#endif
+
+          bool used_gpu_substep = false;
+          if (
+              f_compute_rhs_empart(cg->shape, cg->X[0], cg->X[1], cg->X[2],
                                    cg->fgfs[phi->sgfn],
                                    cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                    cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
@@ -1327,11 +1447,20 @@ void bssnEM_class::Step(int lev, int YN)
                                    cg->fgfs[Kpsi1->sgfn], cg->fgfs[Kphi1->sgfn],
                                    cg->fgfs[rho->sgfn], 
                                    cg->fgfs[Sx->sgfn], cg->fgfs[Sy->sgfn], cg->fgfs[Sz->sgfn],
-                                   cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn], 
-                                   cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
-                                   Symmetry, lev, ndeps) ||
-              f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
-                                 cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
+                                   cg->fgfs[Sxx->sgfn], cg->fgfs[Sxy->sgfn], cg->fgfs[Sxz->sgfn],
+                                   cg->fgfs[Syy->sgfn], cg->fgfs[Syz->sgfn], cg->fgfs[Szz->sgfn],
+                                   Symmetry, lev, ndeps) ||
+#if USE_CUDA_BSSN
+              ((used_gpu_substep =
+                    (run_bssn_em_cuda_substep(cg, SynchList_pre, SynchList_cor, Pp->data,
+                                              dT_lev, TRK4, iter_count, Symmetry, lev,
+                                              ndeps, cor, chitiny,
+                                              rho, Sx, Sy, Sz, Sxx, Sxy, Sxz, Syy, Syz, Szz) == 0))
+                   ? 0
+                   : 1) ||
+#endif
+              (!used_gpu_substep && f_compute_rhs_bssn(cg->shape, TRK4, cg->X[0], cg->X[1], cg->X[2],
+                                 cg->fgfs[phi->sgfn], cg->fgfs[trK->sgfn],
                                  cg->fgfs[gxx->sgfn], cg->fgfs[gxy->sgfn], cg->fgfs[gxz->sgfn], 
                                  cg->fgfs[gyy->sgfn], cg->fgfs[gyz->sgfn], cg->fgfs[gzz->sgfn],
                                  cg->fgfs[Axx->sgfn], cg->fgfs[Axy->sgfn], cg->fgfs[Axz->sgfn], 
@@ -1359,10 +1488,10 @@ void bssnEM_class::Step(int lev, int YN)
                                  cg->fgfs[Gamzyy->sgfn], cg->fgfs[Gamzyz->sgfn], cg->fgfs[Gamzzz->sgfn],
                                  cg->fgfs[Rxx->sgfn], cg->fgfs[Rxy->sgfn], cg->fgfs[Rxz->sgfn], 
                                  cg->fgfs[Ryy->sgfn], cg->fgfs[Ryz->sgfn], cg->fgfs[Rzz->sgfn],
-                                 cg->fgfs[Cons_Ham->sgfn],
-                                 cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
-                                 cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
-                                 Symmetry, lev, ndeps, cor))
+                                 cg->fgfs[Cons_Ham->sgfn],
+                                 cg->fgfs[Cons_Px->sgfn], cg->fgfs[Cons_Py->sgfn], cg->fgfs[Cons_Pz->sgfn],
+                                 cg->fgfs[Cons_Gx->sgfn], cg->fgfs[Cons_Gy->sgfn], cg->fgfs[Cons_Gz->sgfn],
+                                 Symmetry, lev, ndeps, cor)))
           {
             cout << "find NaN in domain: (" 
                  << cg->bbox[0] << ":" << cg->bbox[3] << "," 
@@ -1371,11 +1500,15 @@ void bssnEM_class::Step(int lev, int YN)
             ERROR = 1;
           }
           // rk4 substep and boundary
-          {
-            MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList; 
-            // we do not check the correspondence here
-            
-            while (varl0)
+          {
+            MyList<var> *varl0 = StateList, *varl = SynchList_pre, *varl1 = SynchList_cor, *varlrhs = RHSList;
+            // we do not check the correspondence here
+#if USE_CUDA_BSSN
+            if (used_gpu_substep)
+              skip_bssn_cuda_prefix(varl0, varl, varl1, varlrhs);
+#endif
+
+            while (varl0)
             {
 #ifndef WithShell
               if (lev == 0) // sommerfeld indeed

AMSS_NCKU_source/bssnEScalar_class.h

@@ -54,17 +54,21 @@ public:
      void Interp_Constraint();
      void Constraint_Out(); 
 
-protected:
-     var *Sphio, *Spio;
-     var *Sphi0, *Spi0;
+protected:
+     var *Sphio, *Spio;
+     var *Sphi0, *Spi0;
      var *Sphi,  *Spi;
      var *Sphi1, *Spi1;
      var *Sphi_rhs, *Spi_rhs;
-
-     var *Cons_fR;
-
-     monitor *MaxScalar_Monitor;
-};
+
+     var *Cons_fR;
+
+     MyList<var> *BSSNStateList, *BSSNSynchList_pre, *BSSNSynchList_cor;
+     MyList<var> *ScalarSynchList_pre, *ScalarSynchList_cor;
+     Parallel::SyncCache *sync_cache_scalar_pre, *sync_cache_scalar_cor;
+
+     monitor *MaxScalar_Monitor;
+};
 
 #endif /* BSSNESCALAR_CLASS_H */
 

AMSS_NCKU_source/bssnEScalar_rhs.f90

@@ -3,11 +3,143 @@
 !! note that the potential for scalar field in F(R) gravity
 !!       is defined in the file Set_Rho_ADM.f90
 
-#include "macrodef.fh"
-
-! rhs for scalar and GR variables
-! here we consider vacuum spacetime only
-  function compute_rhs_bssn_escalar(ex, T,X, Y, Z,                             &
+#include "macrodef.fh"
+
+! scalar RHS and stress-energy only; BSSN RHS can be supplied by CUDA.
+  function compute_rhs_bssn_escalar_matter(ex, T, X, Y, Z,                    &
+               chi    ,   trK    ,                                            &
+               dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,    &
+               Axx    ,   Axy    ,   Axz    ,   Ayy    ,   Ayz    ,   Azz,    &
+               Gamx   ,  Gamy    ,  Gamz    ,                                 &
+               Lap    ,  betax   ,  betay   ,  betaz   ,                      &
+               dtSfx  ,  dtSfy   ,  dtSfz   ,                                 &
+               Sphi   ,   Spi    ,                                            &
+               Sphi_rhs ,   Spi_rhs ,                                         &
+               rho,Sx,Sy,Sz,Sxx,Sxy,Sxz,Syy,Syz,Szz,                          &
+               Symmetry,Lev,eps) result(gont)
+  implicit none
+
+  integer,intent(in ):: ex(1:3), Symmetry,Lev
+  real*8, intent(in ):: T
+  real*8, intent(in ):: X(1:ex(1)),Y(1:ex(2)),Z(1:ex(3))
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: chi,dxx,dyy,dzz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: trK
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: gxy,gxz,gyz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Axx,Axy,Axz,Ayy,Ayz,Azz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Gamx,Gamy,Gamz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: Lap, betax, betay, betaz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: dtSfx,  dtSfy,  dtSfz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(in ) :: Sphi,Spi
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(out) :: Sphi_rhs,Spi_rhs
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: rho,Sx,Sy,Sz
+  real*8, dimension(ex(1),ex(2),ex(3)),intent(inout) :: Sxx,Sxy,Sxz,Syy,Syz,Szz
+  real*8,intent(in) :: eps
+  integer::gont
+
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gxx,gyy,gzz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: chix,chiy,chiz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: Lapx,Lapy,Lapz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: Kx,Ky,Kz,S
+  real*8, dimension(ex(1),ex(2),ex(3)) :: f,fxx,fxy,fxz,fyy,fyz,fzz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: alpn1,chin1
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gupxx,gupxy,gupxz
+  real*8, dimension(ex(1),ex(2),ex(3)) :: gupyy,gupyz,gupzz
+
+  real*8            :: dX
+  real*8, parameter :: ZEO=0.d0, ONE = 1.D0, TWO = 2.D0, HALF = 0.5D0
+  real*8, parameter :: SYM = 1.D0
+
+  dX = sum(chi)+sum(trK)+sum(dxx)+sum(gxy)+sum(gxz)+sum(dyy)+sum(gyz)+sum(dzz) &
+      +sum(Gamx)+sum(Gamy)+sum(Gamz)                                           &
+      +sum(Lap)+sum(Sphi)+sum(Spi)
+  if(dX.ne.dX) then
+     if(sum(chi).ne.sum(chi))write(*,*)"bssn_escalar_matter: find NaN in chi"
+     if(sum(trK).ne.sum(trK))write(*,*)"bssn_escalar_matter: find NaN in trk"
+     if(sum(dxx).ne.sum(dxx))write(*,*)"bssn_escalar_matter: find NaN in dxx"
+     if(sum(gxy).ne.sum(gxy))write(*,*)"bssn_escalar_matter: find NaN in gxy"
+     if(sum(gxz).ne.sum(gxz))write(*,*)"bssn_escalar_matter: find NaN in gxz"
+     if(sum(dyy).ne.sum(dyy))write(*,*)"bssn_escalar_matter: find NaN in dyy"
+     if(sum(gyz).ne.sum(gyz))write(*,*)"bssn_escalar_matter: find NaN in gyz"
+     if(sum(dzz).ne.sum(dzz))write(*,*)"bssn_escalar_matter: find NaN in dzz"
+     if(sum(Gamx).ne.sum(Gamx))write(*,*)"bssn_escalar_matter: find NaN in Gamx"
+     if(sum(Gamy).ne.sum(Gamy))write(*,*)"bssn_escalar_matter: find NaN in Gamy"
+     if(sum(Gamz).ne.sum(Gamz))write(*,*)"bssn_escalar_matter: find NaN in Gamz"
+     if(sum(Lap).ne.sum(Lap))write(*,*)"bssn_escalar_matter: find NaN in Lap"
+     if(sum(Sphi).ne.sum(Sphi))write(*,*)"bssn_escalar_matter: find NaN in Sphi"
+     if(sum(Spi).ne.sum(Spi))write(*,*)"bssn_escalar_matter: find NaN in Spi"
+     gont = 1
+     return
+  endif
+
+  alpn1 = Lap + ONE
+  chin1 = chi + ONE
+  gxx = dxx + ONE
+  gyy = dyy + ONE
+  gzz = dzz + ONE
+
+  call fderivs(ex,chi,chix,chiy,chiz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  call fderivs(ex,Lap,Lapx,Lapy,Lapz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+
+  gupzz =  gxx * gyy * gzz + gxy * gyz * gxz + gxz * gxy * gyz - &
+           gxz * gyy * gxz - gxy * gxy * gzz - gxx * gyz * gyz
+  gupxx =   ( gyy * gzz - gyz * gyz ) / gupzz
+  gupxy = - ( gxy * gzz - gyz * gxz ) / gupzz
+  gupxz =   ( gxy * gyz - gyy * gxz ) / gupzz
+  gupyy =   ( gxx * gzz - gxz * gxz ) / gupzz
+  gupyz = - ( gxx * gyz - gxy * gxz ) / gupzz
+  gupzz =   ( gxx * gyy - gxy * gxy ) / gupzz
+
+#if 1
+  Sphi_rhs = alpn1 * Spi
+  call fderivs(ex,Sphi,Kx,Ky,Kz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  call fdderivs(ex,Sphi,fxx,fxy,fxz,fyy,fyz,fzz,X,Y,Z,SYM,SYM,SYM,Symmetry,Lev)
+  Spi_rhs =   gupxx * fxx + gupyy * fyy + gupzz * fzz +           &
+             ( gupxy * fxy + gupxz * fxz + gupyz * fyz ) * TWO -  &
+             ((Gamx+(gupxx*chix+gupxy*chiy+gupxz*chiz)/TWO/chin1)*Kx  &
+            + (Gamy+(gupxy*chix+gupyy*chiy+gupyz*chiz)/TWO/chin1)*Ky  &
+            + (Gamz+(gupxz*chix+gupyz*chiy+gupzz*chiz)/TWO/chin1)*Kz)
+  Spi_rhs = Spi_rhs*alpn1 +                                      &
+            (gupxx*Lapx*Kx + gupxy*Lapx*Ky + gupxz*Lapx*Kz       &
+            +gupxy*Lapy*Kx + gupyy*Lapy*Ky + gupyz*Lapy*Kz       &
+            +gupxz*Lapz*Kx + gupyz*Lapz*Ky + gupzz*Lapz*Kz)
+
+  call frpotential(ex,Sphi,f,S)
+  Spi_rhs = Spi_rhs*chin1 + alpn1*(trK*Spi - S)
+  rho = chin1*((gupxx * Kx * Kx + gupyy * Ky * Ky + gupzz * Kz * Kz)/TWO + &
+                gupxy * Kx * Ky + gupxz * Kx * Kz + gupyz * Ky * Kz )      &
+        + Spi*Spi/TWO+f
+  Sx = -Spi*Kx
+  Sy = -Spi*Ky
+  Sz = -Spi*Kz
+  f = (rho - Spi*Spi)/chin1
+  Sxx = Kx*Kx-f*gxx
+  Sxy = Kx*Ky-f*gxy
+  Sxz = Kx*Kz-f*gxz
+  Syy = Ky*Ky-f*gyy
+  Syz = Ky*Kz-f*gyz
+  Szz = Kz*Kz-f*gzz
+#else
+  Sphi_rhs = ZEO
+  Spi_rhs = ZEO
+  rho = ZEO
+  Sx = ZEO
+  Sy = ZEO
+  Sz = ZEO
+  Sxx = ZEO
+  Sxy = ZEO
+  Sxz = ZEO
+  Syy = ZEO
+  Syz = ZEO
+  Szz = ZEO
+#endif
+
+  gont = 0
+  return
+end function compute_rhs_bssn_escalar_matter
+
+! rhs for scalar and GR variables
+! here we consider vacuum spacetime only
+  function compute_rhs_bssn_escalar(ex, T,X, Y, Z,                             &
                chi    ,   trK    ,                                             &
                dxx    ,   gxy    ,   gxz    ,   dyy    ,   gyz    ,   dzz,     &
                Axx    ,   Axy    ,   Axz    ,   Ayy    ,   Ayz    ,   Azz,     &

AMSS_NCKU_source/bssn_class.h

@@ -178,14 +178,17 @@ public:
        virtual void Initialize();
        virtual void Read_Ansorg();
        virtual void Read_Pablo() {};
-       virtual void Compute_Psi4(int lev);
-       virtual void Step(int lev, int YN);
-       virtual void Interp_Constraint(bool infg);
-       virtual void Constraint_Out();
-       virtual void Compute_Constraint();
-
-#ifdef With_AHF
-protected:
+       virtual void Compute_Psi4(int lev);
+       virtual void Step(int lev, int YN);
+       virtual void Interp_Constraint(bool infg);
+       virtual void Constraint_Out();
+       virtual void Compute_Constraint();
+
+protected:
+       void Initialize_Level_Runtime();
+
+#ifdef With_AHF
+protected:
        MyList<var> *AHList, *AHDList, *GaugeList;
        int AHfindevery;
        double AHdumptime;

AMSS_NCKU_source/bssn_rhs.h

@@ -5,8 +5,9 @@
 #ifdef fortran1
 #define f_compute_rhs_bssn compute_rhs_bssn
 #define f_compute_rhs_bssn_ss compute_rhs_bssn_ss
-#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar
-#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss
+#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar
+#define f_compute_rhs_bssn_escalar_matter compute_rhs_bssn_escalar_matter
+#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss
 #define f_compute_rhs_Z4c compute_rhs_z4c
 #define f_compute_rhs_Z4cnot compute_rhs_z4cnot
 #define f_compute_rhs_Z4c_ss compute_rhs_z4c_ss
@@ -15,8 +16,9 @@
 #ifdef fortran2
 #define f_compute_rhs_bssn COMPUTE_RHS_BSSN
 #define f_compute_rhs_bssn_ss COMPUTE_RHS_BSSN_SS
-#define f_compute_rhs_bssn_escalar COMPUTE_RHS_BSSN_ESCALAR
-#define f_compute_rhs_bssn_escalar_ss COMPUTE_RHS_BSSN_ESCALAR_SS
+#define f_compute_rhs_bssn_escalar COMPUTE_RHS_BSSN_ESCALAR
+#define f_compute_rhs_bssn_escalar_matter COMPUTE_RHS_BSSN_ESCALAR_MATTER
+#define f_compute_rhs_bssn_escalar_ss COMPUTE_RHS_BSSN_ESCALAR_SS
 #define f_compute_rhs_Z4c COMPUTE_RHS_Z4C
 #define f_compute_rhs_Z4cnot COMPUTE_RHS_Z4CNOT
 #define f_compute_rhs_Z4c_ss COMPUTE_RHS_Z4C_SS
@@ -25,8 +27,9 @@
 #ifdef fortran3
 #define f_compute_rhs_bssn compute_rhs_bssn_
 #define f_compute_rhs_bssn_ss compute_rhs_bssn_ss_
-#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar_
-#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss_
+#define f_compute_rhs_bssn_escalar compute_rhs_bssn_escalar_
+#define f_compute_rhs_bssn_escalar_matter compute_rhs_bssn_escalar_matter_
+#define f_compute_rhs_bssn_escalar_ss compute_rhs_bssn_escalar_ss_
 #define f_compute_rhs_Z4c compute_rhs_z4c_
 #define f_compute_rhs_Z4cnot compute_rhs_z4cnot_
 #define f_compute_rhs_Z4c_ss compute_rhs_z4c_ss_
@@ -96,10 +99,24 @@ extern "C"
                                   int &, int &, double &, int &, int &);
 }
 
-extern "C"
-{
-        int f_compute_rhs_bssn_escalar(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
-                                       double *, double *,                                                                                 // chi, trK
+extern "C"
+{
+        int f_compute_rhs_bssn_escalar_matter(int *, double &, double *, double *, double *,                                             // ex,T,X,Y,Z
+                                              double *, double *,                                                                        // chi, trK
+                                              double *, double *, double *, double *, double *, double *,                                // gij
+                                              double *, double *, double *, double *, double *, double *,                                // Aij
+                                              double *, double *, double *,                                                              // Gam
+                                              double *, double *, double *, double *, double *, double *, double *,                      // Gauge
+                                              double *, double *,                                                                        // Sphi, Spi
+                                              double *, double *,                                                                        // Sphi, Spi rhs
+                                              double *, double *, double *, double *, double *, double *, double *, double *, double *, double *, // stress-energy
+                                              int &, int &, double &);
+}
+
+extern "C"
+{
+        int f_compute_rhs_bssn_escalar(int *, double &, double *, double *, double *,                                                      // ex,T,X,Y,Z
+                                       double *, double *,                                                                                 // chi, trK
                                        double *, double *, double *, double *, double *, double *,                                         // gij
                                        double *, double *, double *, double *, double *, double *,                                         // Aij
                                        double *, double *, double *,                                                                       // Gam

AMSS_NCKU_source/bssn_rhs_cuda.h

@@ -55,6 +55,117 @@ int bssn_cuda_rk4_substep(void *block_tag,
                           int &apply_enforce_ga,
                           double &chitiny);
 
+int bssn_cuda_compute_escalar_matter(void *block_tag,
+                                     int *ex, double *X, double *Y, double *Z,
+                                     double **state_host_in,
+                                     double *Sphi_host,
+                                     double *Spi_host,
+                                     double *Sphi_rhs_host,
+                                     double *Spi_rhs_host,
+                                     double a2,
+                                     int &Symmetry,
+                                     int &Lev,
+                                     double &eps,
+                                     int &co,
+                                     int &apply_enforce_ga);
+
+int bssn_cuda_escalar_finalize_scalar_fields(void *block_tag,
+                                             int *ex, double *X, double *Y, double *Z,
+                                             double *Sphi_out_host,
+                                             double *Spi_out_host,
+                                             const double *propspeed,
+                                             const double *soa_flat,
+                                             const double *bbox,
+                                             double &dT,
+                                             int &RK4,
+                                             int &apply_bam_bc,
+                                             int &Symmetry,
+                                             int &Lev,
+                                             double &eps,
+                                             int &precor);
+
+int bssn_cuda_escalar_has_resident_fields(void *block_tag,
+                                          double *Sphi_host,
+                                          double *Spi_host);
+
+int bssn_cuda_escalar_has_any_resident_fields(void *block_tag);
+
+int bssn_cuda_escalar_download_fields_if_present(void *block_tag,
+                                                 int *ex,
+                                                 double *Sphi_host,
+                                                 double *Spi_host);
+
+int bssn_cuda_pack_escalar_batch_to_host_buffer(void *block_tag,
+                                                double **scalar_host_key,
+                                                double *host_buffer,
+                                                int *ex,
+                                                int i0, int j0, int k0,
+                                                int sx, int sy, int sz);
+
+int bssn_cuda_unpack_escalar_batch_from_host_buffer(void *block_tag,
+                                                    double **scalar_host_key,
+                                                    double *host_buffer,
+                                                    int *ex,
+                                                    int i0, int j0, int k0,
+                                                    int sx, int sy, int sz);
+
+int bssn_cuda_pack_escalar_batch_to_device_buffer(void *block_tag,
+                                                  double **scalar_host_key,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int i0, int j0, int k0,
+                                                  int sx, int sy, int sz);
+
+int bssn_cuda_unpack_escalar_batch_from_device_buffer(void *block_tag,
+                                                      double **scalar_host_key,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int i0, int j0, int k0,
+                                                      int sx, int sy, int sz);
+
+int bssn_cuda_restrict_escalar_batch_to_host_buffer(void *block_tag,
+                                                    double **scalar_host_key,
+                                                    double *host_buffer,
+                                                    int *ex,
+                                                    int sx, int sy, int sz,
+                                                    int fi0, int fj0, int fk0,
+                                                    const double *scalar_soa);
+
+int bssn_cuda_prolong_escalar_batch_to_host_buffer(void *block_tag,
+                                                   double **scalar_host_key,
+                                                   double *host_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int ii0, int jj0, int kk0,
+                                                   int lbc_i, int lbc_j, int lbc_k,
+                                                   const double *scalar_soa);
+
+int bssn_cuda_restrict_escalar_batch_to_device_buffer(void *block_tag,
+                                                      double **scalar_host_key,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int sx, int sy, int sz,
+                                                      int fi0, int fj0, int fk0,
+                                                      const double *scalar_soa);
+
+int bssn_cuda_prolong_escalar_batch_to_device_buffer(void *block_tag,
+                                                     double **scalar_host_key,
+                                                     double *device_buffer,
+                                                     int *ex,
+                                                     int sx, int sy, int sz,
+                                                     int ii0, int jj0, int kk0,
+                                                     int lbc_i, int lbc_j, int lbc_k,
+                                                     const double *scalar_soa);
+
+int bssn_cuda_prepare_escalar_inter_time_level(void *block_tag,
+                                               int *ex,
+                                               double **src1_host_key,
+                                               double **src2_host_key,
+                                               double **src3_host_key,
+                                               double **dst_host_key,
+                                               int source_count,
+                                               int tindex);
+
 int bssn_cuda_copy_state_region_to_host(void *block_tag,
                                         int state_index,
                                         double *host_state,
@@ -77,6 +188,9 @@ int bssn_cuda_download_resident_state_if_present(void *block_tag,
                                                 int *ex,
                                                 double **state_host_out);
 
+int bssn_cuda_resident_state_matches(void *block_tag,
+                                     double **state_host_key);
+
 int bssn_cuda_download_constraint_outputs(int *ex,
                                           double **constraint_host_out);
 
@@ -162,8 +276,44 @@ int bssn_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag
                                                                  int i0, int j0, int k0,
                                                                  int sx, int sy, int sz);
 
+int bssn_cuda_restrict_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *host_buffer,
+                                                                 int *ex,
+                                                                 int sx, int sy, int sz,
+                                                                 int fi0, int fj0, int fk0,
+                                                                 const double *state_soa);
+
+int bssn_cuda_restrict_state_batch_to_host_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *host_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int fi0, int fj0, int fk0,
+                                                  const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                                double **state_host_key,
+                                                                int state_count,
+                                                                double *host_buffer,
+                                                                int *ex,
+                                                                int sx, int sy, int sz,
+                                                                int ii0, int jj0, int kk0,
+                                                                int lbc_i, int lbc_j, int lbc_k,
+                                                                const double *state_soa);
+
+int bssn_cuda_prolong_state_batch_to_host_buffer(void *block_tag,
+                                                 int state_count,
+                                                 double *host_buffer,
+                                                 int *ex,
+                                                 int sx, int sy, int sz,
+                                                 int ii0, int jj0, int kk0,
+                                                 int lbc_i, int lbc_j, int lbc_k,
+                                                 const double *state_soa);
+
 int bssn_cuda_pack_state_batch_to_device_buffer(void *block_tag,
-                                                int state_count,
+                                                 int state_count,
                                                 double *device_buffer,
                                                 int *ex,
                                                 int i0, int j0, int k0,

AMSS_NCKU_source/macrodef.fh

@@ -13,7 +13,7 @@
 
 #define ABV 0
 
-#define EScalar_CC 2
+#define EScalar_CC  2
 
 #if 0
 

AMSS_NCKU_source/macrodef.h

@@ -10,7 +10,7 @@
 
 #define GaussInt
 
-#define ABEtype 0
+#define ABEtype 1
 
 //#define With_AHF
 #define Psi4type 0
@@ -167,3 +167,4 @@
 #define TINY 1e-10
 
 #endif   /* MICRODEF_H */
+

AMSS_NCKU_source/z4c_rhs_cuda.h

@@ -53,6 +53,14 @@ int z4c_cuda_pack_state_batch_to_host_buffer(void *block_tag,
                                              int i0, int j0, int k0,
                                              int sx, int sy, int sz);
 
+int z4c_cuda_pack_state_batch_to_host_buffer_for_host_views(void *block_tag,
+                                                            double **state_host_key,
+                                                            int state_count,
+                                                            double *host_buffer,
+                                                            int *ex,
+                                                            int i0, int j0, int k0,
+                                                            int sx, int sy, int sz);
+
 int z4c_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
                                                  int state_count,
                                                  double *host_buffer,
@@ -60,6 +68,14 @@ int z4c_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
                                                  int i0, int j0, int k0,
                                                  int sx, int sy, int sz);
 
+int z4c_cuda_unpack_state_batch_from_host_buffer_for_host_views(void *block_tag,
+                                                                double **state_host_key,
+                                                                int state_count,
+                                                                double *host_buffer,
+                                                                int *ex,
+                                                                int i0, int j0, int k0,
+                                                                int sx, int sy, int sz);
+
 int z4c_cuda_pack_state_batch_to_device_buffer(void *block_tag,
                                                int state_count,
                                                double *device_buffer,
@@ -67,6 +83,14 @@ int z4c_cuda_pack_state_batch_to_device_buffer(void *block_tag,
                                                int i0, int j0, int k0,
                                                int sx, int sy, int sz);
 
+int z4c_cuda_pack_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                              double **state_host_key,
+                                                              int state_count,
+                                                              double *device_buffer,
+                                                              int *ex,
+                                                              int i0, int j0, int k0,
+                                                              int sx, int sy, int sz);
+
 int z4c_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
                                                    int state_count,
                                                    double *device_buffer,
@@ -74,6 +98,114 @@ int z4c_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
                                                    int i0, int j0, int k0,
                                                    int sx, int sy, int sz);
 
+int z4c_cuda_unpack_state_batch_from_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int i0, int j0, int k0,
+                                                                  int sx, int sy, int sz);
+
+int z4c_cuda_pack_state_segments_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int segment_count,
+                                                  const int *segment_meta);
+
+int z4c_cuda_pack_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *device_buffer,
+                                                                 int *ex,
+                                                                 int segment_count,
+                                                                 const int *segment_meta);
+
+int z4c_cuda_unpack_state_segments_from_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta);
+
+int z4c_cuda_unpack_state_segments_from_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta);
+
+int z4c_cuda_restrict_state_segments_to_device_buffer(void *block_tag,
+                                                      int state_count,
+                                                      double *device_buffer,
+                                                      int *ex,
+                                                      int segment_count,
+                                                      const int *segment_meta,
+                                                      const double *state_soa);
+
+int z4c_cuda_restrict_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                     double **state_host_key,
+                                                                     int state_count,
+                                                                     double *device_buffer,
+                                                                     int *ex,
+                                                                     int segment_count,
+                                                                     const int *segment_meta,
+                                                                     const double *state_soa);
+
+int z4c_cuda_prolong_state_segments_to_device_buffer(void *block_tag,
+                                                     int state_count,
+                                                     double *device_buffer,
+                                                     int *ex,
+                                                     int segment_count,
+                                                     const int *segment_meta,
+                                                     const double *state_soa);
+
+int z4c_cuda_prolong_state_segments_to_device_buffer_for_host_views(void *block_tag,
+                                                                    double **state_host_key,
+                                                                    int state_count,
+                                                                    double *device_buffer,
+                                                                    int *ex,
+                                                                    int segment_count,
+                                                                    const int *segment_meta,
+                                                                    const double *state_soa);
+
+int z4c_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
+                                                   int state_count,
+                                                   double *device_buffer,
+                                                   int *ex,
+                                                   int sx, int sy, int sz,
+                                                   int fi0, int fj0, int fk0,
+                                                   const double *state_soa);
+
+int z4c_cuda_restrict_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                  double **state_host_key,
+                                                                  int state_count,
+                                                                  double *device_buffer,
+                                                                  int *ex,
+                                                                  int sx, int sy, int sz,
+                                                                  int fi0, int fj0, int fk0,
+                                                                  const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
+                                                  int state_count,
+                                                  double *device_buffer,
+                                                  int *ex,
+                                                  int sx, int sy, int sz,
+                                                  int ii0, int jj0, int kk0,
+                                                  int lbc_i, int lbc_j, int lbc_k,
+                                                  const double *state_soa);
+
+int z4c_cuda_prolong_state_batch_to_device_buffer_for_host_views(void *block_tag,
+                                                                 double **state_host_key,
+                                                                 int state_count,
+                                                                 double *device_buffer,
+                                                                 int *ex,
+                                                                 int sx, int sy, int sz,
+                                                                 int ii0, int jj0, int kk0,
+                                                                 int lbc_i, int lbc_j, int lbc_k,
+                                                                 const double *state_soa);
+
 int z4c_cuda_download_state_subset(void *block_tag,
                                    int *ex,
                                    int subset_count,
@@ -86,7 +218,36 @@ int z4c_cuda_upload_state_subset(void *block_tag,
                                  const int *state_indices,
                                  double **state_host_in);
 
+int z4c_cuda_compute_constraints_resident(void *block_tag,
+                                          int *ex, double *X, double *Y, double *Z,
+                                          int Symmetry, double eps, int co,
+                                          double **constraint_host_out);
+
+int z4c_cuda_interp_state_point3(void *block_tag,
+                                 int *ex,
+                                 int state0,
+                                 int state1,
+                                 int state2,
+                                 double x0,
+                                 double y0,
+                                 double z0,
+                                 double dx,
+                                 double dy,
+                                 double dz,
+                                 double px,
+                                 double py,
+                                 double pz,
+                                 int ordn,
+                                 int symmetry,
+                                 const double *soa3,
+                                 double *out3);
+
+int z4c_cuda_download_constraint_outputs(int *ex,
+                                         double **constraint_host_out);
+
 int z4c_cuda_has_resident_state(void *block_tag);
+int z4c_cuda_resident_state_matches(void *block_tag,
+                                    double **state_host_key);
 
 void z4c_cuda_release_step_ctx(void *block_tag);
 

makefile_and_run.py

@@ -145,11 +145,18 @@ def _gpu_runtime_env():
         "AMSS_ANALYSIS_MAP_EVERY": "1000000",
         "AMSS_CUDA_AWARE_MPI": "1",
         "AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP": "1",
+        "AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP": "1",
         "AMSS_CUDA_KEEP_ALL_LEVELS": "1",
+        "AMSS_CUDA_Z4C_AMR_DEVICE": "0",
         "AMSS_CUDA_AMR_RESTRICT_DEVICE": "1",
         "AMSS_CUDA_AMR_RESTRICT_BATCH": "0",
         "AMSS_CUDA_DEVICE_SEGMENT_BATCH": "0",
+        "AMSS_CUDA_PIN_ESCALAR_TRANSFERS": "0",
+        "AMSS_ESCALAR_GPU_RK": "0",
     }
+    if getattr(input_data, "Equation_Class", "") == "Z4C":
+        defaults["AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP"] = "0"
+        defaults["AMSS_CUDA_KEEP_ALL_LEVELS"] = "0"
     for key, value in defaults.items():
         runtime_env.setdefault(key, value)
 
@@ -276,10 +283,14 @@ def run_ABE():
         print(f"   AMSS_ANALYSIS_MAP_EVERY={mpi_env.get('AMSS_ANALYSIS_MAP_EVERY', '')}")
         print(f"   AMSS_CUDA_AWARE_MPI={mpi_env.get('AMSS_CUDA_AWARE_MPI', '')}")
         print(f"   AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_KEEP_RESIDENT_AFTER_STEP', '')}")
+        print(f"   AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP={mpi_env.get('AMSS_CUDA_Z4C_KEEP_RESIDENT_AFTER_STEP', '')}")
         print(f"   AMSS_CUDA_KEEP_ALL_LEVELS={mpi_env.get('AMSS_CUDA_KEEP_ALL_LEVELS', '')}")
+        print(f"   AMSS_CUDA_Z4C_AMR_DEVICE={mpi_env.get('AMSS_CUDA_Z4C_AMR_DEVICE', '')}")
         print(f"   AMSS_CUDA_AMR_RESTRICT_DEVICE={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_DEVICE', '')}")
         print(f"   AMSS_CUDA_AMR_RESTRICT_BATCH={mpi_env.get('AMSS_CUDA_AMR_RESTRICT_BATCH', '')}")
         print(f"   AMSS_CUDA_DEVICE_SEGMENT_BATCH={mpi_env.get('AMSS_CUDA_DEVICE_SEGMENT_BATCH', '')}")
+        print(f"   AMSS_CUDA_PIN_ESCALAR_TRANSFERS={mpi_env.get('AMSS_CUDA_PIN_ESCALAR_TRANSFERS', '')}")
+        print(f"   AMSS_ESCALAR_GPU_RK={mpi_env.get('AMSS_ESCALAR_GPU_RK', '')}")
         if "CUDA_MPS_PIPE_DIRECTORY" in mpi_env:
             print(f"   CUDA_MPS_PIPE_DIRECTORY={mpi_env['CUDA_MPS_PIPE_DIRECTORY']}")