Fallback BSSN-EScalar restrict/prolong path

.gitignore

@@ -1,6 +1,6 @@
 __pycache__
 GW150914
-GW150914-origin
+GW150914*
 docs
 *.tmp
-
+.codex

AMSS_NCKU_source/bssn_class.C

@@ -6081,6 +6081,92 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB,
 //
 // SynchList_cor  old -----------
 {
+#if (ABEtype == 1)
+#if (PSTR == 1 || PSTR == 2)
+//  stringstream a_stream;
+//  a_stream.setf(ios::left);
+#endif
+
+  if (lev > 0)
+  {
+    MyList<Patch> *Pp, *Ppc;
+    if (lev > trfls && YN == 0)
+    {
+      Pp = GH->PatL[lev - 1];
+      while (Pp)
+      {
+        if (BB)
+          Parallel::prepare_inter_time_level(Pp->data, SL, OL, corL,
+                                             SynchList_pre, 0);
+        else
+          Parallel::prepare_inter_time_level(Pp->data, SL, OL,
+                                             SynchList_pre, 0);
+        Pp = Pp->next;
+      }
+
+#if (RPB == 0)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, Symmetry);
+#elif (RPB == 1)
+      Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, GH->rsul[lev], Symmetry);
+#endif
+
+      Parallel::Sync(GH->PatL[lev - 1], SynchList_pre, Symmetry);
+
+#if (RPB == 0)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+#if (MIXOUTB == 0)
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SynchList_pre, SL, Symmetry);
+#elif (MIXOUTB == 1)
+          Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SynchList_pre, SL, Symmetry);
+#endif
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#elif (RPB == 1)
+      Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SL, GH->bdsul[lev], Symmetry);
+#endif
+    }
+    else
+    {
+#if (RPB == 0)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, Symmetry);
+#elif (RPB == 1)
+      Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, GH->rsul[lev], Symmetry);
+#endif
+
+      Parallel::Sync(GH->PatL[lev - 1], SL, Symmetry);
+
+#if (RPB == 0)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+#if (MIXOUTB == 0)
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SL, SL, Symmetry);
+#elif (MIXOUTB == 1)
+          Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SL, SL, Symmetry);
+#endif
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#elif (RPB == 1)
+      Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, GH->bdsul[lev], Symmetry);
+#endif
+    }
+
+    Parallel::Sync(GH->PatL[lev], SL, Symmetry);
+  }
+  return;
+#endif
   STEP_TIMER_DECL(timer_restrict_prolong);
 #if (PSTR == 1 || PSTR == 2)
 //  stringstream a_stream;
@@ -6123,7 +6209,11 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB,
 #endif
 
 #if (RPB == 0)
+#if (ABEtype == 1)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, Symmetry);
+#else
       Parallel::Restrict_cached(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, Symmetry, sync_cache_restrict[lev]);
+#endif
 #elif (RPB == 1)
       //       Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SL,SynchList_pre,Symmetry);
       Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, GH->rsul[lev], Symmetry);
@@ -6136,7 +6226,11 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB,
 //       misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
 #endif
 
+#if (ABEtype == 1)
+      Parallel::Sync(GH->PatL[lev - 1], SynchList_pre, Symmetry);
+#else
       Parallel::Sync_cached(GH->PatL[lev - 1], SynchList_pre, Symmetry, sync_cache_rp_coarse[lev]);
+#endif
 
 #if (PSTR == 1 || PSTR == 2)
 //       a_stream.clear();
@@ -6147,7 +6241,21 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB,
 
 #if (RPB == 0)
 #if (MIXOUTB == 0)
+#if (ABEtype == 1)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SynchList_pre, SL, Symmetry);
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#else
       Parallel::OutBdLow2Hi_cached(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SL, Symmetry, sync_cache_outbd[lev]);
+#endif
 #elif (MIXOUTB == 1)
       Parallel::OutBdLow2Himix(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SL, Symmetry);
 #endif
@@ -6174,7 +6282,11 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB,
 #endif
 
 #if (RPB == 0)
+#if (ABEtype == 1)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, Symmetry);
+#else
       Parallel::Restrict_cached(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, Symmetry, sync_cache_restrict[lev]);
+#endif
 #elif (RPB == 1)
       //       Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SL,SL,Symmetry);
       Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, GH->rsul[lev], Symmetry);
@@ -6187,7 +6299,11 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB,
 //       misc::tillherecheck(GH->Commlev[GH->mylev],GH->start_rank[GH->mylev],a_stream.str());
 #endif
 
+#if (ABEtype == 1)
+      Parallel::Sync(GH->PatL[lev - 1], SL, Symmetry);
+#else
       Parallel::Sync_cached(GH->PatL[lev - 1], SL, Symmetry, sync_cache_rp_coarse[lev]);
+#endif
 
 #if (PSTR == 1 || PSTR == 2)
 //       a_stream.clear();
@@ -6198,7 +6314,21 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB,
 
 #if (RPB == 0)
 #if (MIXOUTB == 0)
+#if (ABEtype == 1)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SL, SL, Symmetry);
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#else
       Parallel::OutBdLow2Hi_cached(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, Symmetry, sync_cache_outbd[lev]);
+#endif
 #elif (MIXOUTB == 1)
       Parallel::OutBdLow2Himix(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, Symmetry);
 #endif
@@ -6244,6 +6374,91 @@ void bssn_class::RestrictProlong_aux(int lev, int YN, bool BB,
 //
 // SynchList_cor  old -----------
 {
+#if (ABEtype == 1)
+  if (lev >= GH->levels - 1)
+    return;
+  lev = lev + 1;
+
+  if (lev > 0)
+  {
+    MyList<Patch> *Pp, *Ppc;
+    if (lev > trfls && YN == 0)
+    {
+      Pp = GH->PatL[lev - 1];
+      while (Pp)
+      {
+        if (BB)
+          Parallel::prepare_inter_time_level(Pp->data, SL, OL, corL,
+                                             SynchList_pre, 0);
+        else
+          Parallel::prepare_inter_time_level(Pp->data, SL, OL,
+                                             SynchList_pre, 0);
+        Pp = Pp->next;
+      }
+
+#if (RPB == 0)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, Symmetry);
+#elif (RPB == 1)
+      Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SynchList_pre, GH->rsul[lev], Symmetry);
+#endif
+
+      Parallel::Sync(GH->PatL[lev - 1], SynchList_pre, Symmetry);
+
+#if (RPB == 0)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+#if (MIXOUTB == 0)
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SynchList_pre, SL, Symmetry);
+#elif (MIXOUTB == 1)
+          Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SynchList_pre, SL, Symmetry);
+#endif
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#elif (RPB == 1)
+      Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SL, GH->bdsul[lev], Symmetry);
+#endif
+    }
+    else
+    {
+#if (RPB == 0)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, Symmetry);
+#elif (RPB == 1)
+      Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, GH->rsul[lev], Symmetry);
+#endif
+
+      Parallel::Sync(GH->PatL[lev - 1], SL, Symmetry);
+
+#if (RPB == 0)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+#if (MIXOUTB == 0)
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SL, SL, Symmetry);
+#elif (MIXOUTB == 1)
+          Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SL, SL, Symmetry);
+#endif
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#elif (RPB == 1)
+      Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SL, SL, GH->bdsul[lev], Symmetry);
+#endif
+    }
+
+    Parallel::Sync(GH->PatL[lev], SL, Symmetry);
+  }
+  return;
+#endif
   STEP_TIMER_DECL(timer_restrict_prolong);
   //  misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"starting RestrictProlong_aux");
 
@@ -6311,7 +6526,11 @@ void bssn_class::RestrictProlong_aux(int lev, int YN, bool BB,
 #endif
     }
 
+#if (ABEtype == 1)
+    Parallel::Sync(GH->PatL[lev], SL, Symmetry);
+#else
     Parallel::Sync_cached(GH->PatL[lev], SL, Symmetry, sync_cache_rp_fine[lev]);
+#endif
   }
   STEP_TIMER_ADD(TB_RESTRICT_PROLONG, timer_restrict_prolong);
 }
@@ -6324,8 +6543,93 @@ void bssn_class::RestrictProlong_aux(int lev, int YN, bool BB,
 
 void bssn_class::RestrictProlong(int lev, int YN, bool BB)
 {
-  STEP_TIMER_DECL(timer_restrict_prolong);
   double dT_lev = dT * pow(0.5, Mymax(lev, trfls));
+#if (ABEtype == 1)
+  if (lev > 0)
+  {
+    MyList<Patch> *Pp, *Ppc;
+    if (lev > trfls && YN == 0)
+    {
+      if (myrank == 0)
+        cout << "/=: " << GH->Lt[lev - 1] << "," << GH->Lt[lev] + dT_lev << endl;
+      Pp = GH->PatL[lev - 1];
+      while (Pp)
+      {
+        if (BB)
+          Parallel::prepare_inter_time_level(Pp->data, StateList, OldStateList, SynchList_cor,
+                                             SynchList_pre, 0);
+        else
+          Parallel::prepare_inter_time_level(Pp->data, StateList, OldStateList,
+                                             SynchList_pre, 0);
+        Pp = Pp->next;
+      }
+
+#if (RPB == 0)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, SynchList_pre, Symmetry);
+#elif (RPB == 1)
+      Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, SynchList_pre, GH->rsul[lev], Symmetry);
+#endif
+
+      Parallel::Sync(GH->PatL[lev - 1], SynchList_pre, Symmetry);
+
+#if (RPB == 0)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+#if (MIXOUTB == 0)
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SynchList_pre, SynchList_cor, Symmetry);
+#elif (MIXOUTB == 1)
+          Parallel::OutBdLow2Himix(Ppc->data, Pp->data, SynchList_pre, SynchList_cor, Symmetry);
+#endif
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#elif (RPB == 1)
+      Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SynchList_cor, GH->bdsul[lev], Symmetry);
+#endif
+    }
+    else
+    {
+      if (myrank == 0)
+        cout << "===: " << GH->Lt[lev - 1] << "," << GH->Lt[lev] + dT_lev << endl;
+#if (RPB == 0)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, StateList, Symmetry);
+#elif (RPB == 1)
+      Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, StateList, GH->rsul[lev], Symmetry);
+#endif
+
+      Parallel::Sync(GH->PatL[lev - 1], StateList, Symmetry);
+
+#if (RPB == 0)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+#if (MIXOUTB == 0)
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, StateList, SynchList_cor, Symmetry);
+#elif (MIXOUTB == 1)
+          Parallel::OutBdLow2Himix(Ppc->data, Pp->data, StateList, SynchList_cor, Symmetry);
+#endif
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#elif (RPB == 1)
+      Parallel::OutBdLow2Hi_bam(GH->PatL[lev - 1], GH->PatL[lev], StateList, SynchList_cor, GH->bdsul[lev], Symmetry);
+#endif
+    }
+
+    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+  }
+  return;
+#endif
+  STEP_TIMER_DECL(timer_restrict_prolong);
   // we assume  for fine
   // SynchList_cor 1   -----------
   //
@@ -6358,17 +6662,39 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB)
       }
 
 #if (RPB == 0)
+#if (ABEtype == 1)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, SynchList_pre, Symmetry);
+#else
       Parallel::Restrict_cached(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, SynchList_pre, Symmetry, sync_cache_restrict[lev]);
+#endif
 #elif (RPB == 1)
       //       Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_cor,SynchList_pre,Symmetry);
       Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, SynchList_pre, GH->rsul[lev], Symmetry);
 #endif
 
+#if (ABEtype == 1)
+      Parallel::Sync(GH->PatL[lev - 1], SynchList_pre, Symmetry);
+#else
       Parallel::Sync_cached(GH->PatL[lev - 1], SynchList_pre, Symmetry, sync_cache_rp_coarse[lev]);
+#endif
 
 #if (RPB == 0)
 #if (MIXOUTB == 0)
+#if (ABEtype == 1)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, SynchList_pre, SynchList_cor, Symmetry);
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#else
       Parallel::OutBdLow2Hi_cached(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SynchList_cor, Symmetry, sync_cache_outbd[lev]);
+#endif
 #elif (MIXOUTB == 1)
       Parallel::OutBdLow2Himix(GH->PatL[lev - 1], GH->PatL[lev], SynchList_pre, SynchList_cor, Symmetry);
 #endif
@@ -6382,17 +6708,39 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB)
       if (myrank == 0)
         cout << "===: " << GH->Lt[lev - 1] << "," << GH->Lt[lev] + dT_lev << endl;
 #if (RPB == 0)
+#if (ABEtype == 1)
+      Parallel::Restrict(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, StateList, Symmetry);
+#else
       Parallel::Restrict_cached(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, StateList, Symmetry, sync_cache_restrict[lev]);
+#endif
 #elif (RPB == 1)
       //       Parallel::Restrict_bam(GH->PatL[lev-1],GH->PatL[lev],SynchList_cor,StateList,Symmetry);
       Parallel::Restrict_bam(GH->PatL[lev - 1], GH->PatL[lev], SynchList_cor, StateList, GH->rsul[lev], Symmetry);
 #endif
 
+#if (ABEtype == 1)
+      Parallel::Sync(GH->PatL[lev - 1], StateList, Symmetry);
+#else
       Parallel::Sync_cached(GH->PatL[lev - 1], StateList, Symmetry, sync_cache_rp_coarse[lev]);
+#endif
 
 #if (RPB == 0)
 #if (MIXOUTB == 0)
+#if (ABEtype == 1)
+      Ppc = GH->PatL[lev - 1];
+      while (Ppc)
+      {
+        Pp = GH->PatL[lev];
+        while (Pp)
+        {
+          Parallel::OutBdLow2Hi(Ppc->data, Pp->data, StateList, SynchList_cor, Symmetry);
+          Pp = Pp->next;
+        }
+        Ppc = Ppc->next;
+      }
+#else
       Parallel::OutBdLow2Hi_cached(GH->PatL[lev - 1], GH->PatL[lev], StateList, SynchList_cor, Symmetry, sync_cache_outbd[lev]);
+#endif
 #elif (MIXOUTB == 1)
       Parallel::OutBdLow2Himix(GH->PatL[lev - 1], GH->PatL[lev], StateList, SynchList_cor, Symmetry);
 #endif
@@ -6402,7 +6750,11 @@ void bssn_class::RestrictProlong(int lev, int YN, bool BB)
 #endif
     }
 
+#if (ABEtype == 1)
+    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+#else
     Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_rp_fine[lev]);
+#endif
   }
   STEP_TIMER_ADD(TB_RESTRICT_PROLONG, timer_restrict_prolong);
 }