Disable cached sync for BSSN-EScalar

AMSS_NCKU_source/bssn_class.C

@@ -2772,7 +2772,9 @@ void bssn_class::Evolve(int Steps)
     GH->Regrid(Symmetry, BH_num, Porgbr, Porg0,
                SynchList_cor, OldStateList, StateList, SynchList_pre,
                fgt(PhysTime - dT_mon, StartTime, dT_mon / 2), ErrorMonitor);
+#if (ABEtype != 1)
     for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); }
+#endif
     STEP_TIMER_ADD(TB_REGRID, timer_regrid);
 #endif
 
@@ -3009,7 +3011,9 @@ void bssn_class::RecursiveStep(int lev)
   {
   if (ConstraintRefreshLevels)
     ConstraintRefreshLevels[lev] = 1;
+#if (ABEtype != 1)
   for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); }
+#endif
   }
   STEP_TIMER_ADD(TB_REGRID, timer_regrid_onelevel);
 #endif
@@ -3190,9 +3194,13 @@ void bssn_class::ParallelStep()
   if (GH->Regrid_Onelevel(GH->mylev, Symmetry, BH_num, Porgbr, Porg0,
                       SynchList_cor, OldStateList, StateList, SynchList_pre,
                       fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), ErrorMonitor))
+  {
+#if (ABEtype != 1)
   for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); }
 #endif
   }
+#endif
+}
 
 //================================================================================================
 
@@ -3357,7 +3365,11 @@ void bssn_class::ParallelStep()
         if (GH->Regrid_Onelevel(lev + 1, Symmetry, BH_num, Porgbr, Porg0,
                             SynchList_cor, OldStateList, StateList, SynchList_pre,
                             fgt(PhysTime - dT_levp1, StartTime, dT_levp1 / 2), ErrorMonitor))
+        {
+#if (ABEtype != 1)
         for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); }
+#endif
+        }
 
         //               a_stream.clear();
         //               a_stream.str("");
@@ -3372,7 +3384,11 @@ void bssn_class::ParallelStep()
       if (GH->Regrid_Onelevel(lev, Symmetry, BH_num, Porgbr, Porg0,
                           SynchList_cor, OldStateList, StateList, SynchList_pre,
                           fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), ErrorMonitor))
+      {
+#if (ABEtype != 1)
       for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); }
+#endif
+      }
 
       //               a_stream.clear();
       //               a_stream.str("");
@@ -3391,7 +3407,11 @@ void bssn_class::ParallelStep()
           if (GH->Regrid_Onelevel(lev - 1, Symmetry, BH_num, Porgbr, Porg0,
                               SynchList_cor, OldStateList, StateList, SynchList_pre,
                               fgt(PhysTime - dT_lev, StartTime, dT_levm1 / 2), ErrorMonitor))
+          {
+#if (ABEtype != 1)
           for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); }
+#endif
+          }
 
           //               a_stream.clear();
           //               a_stream.str("");
@@ -3407,7 +3427,11 @@ void bssn_class::ParallelStep()
           if (GH->Regrid_Onelevel(lev - 1, Symmetry, BH_num, Porgbr, Porg0,
                               SynchList_cor, OldStateList, StateList, SynchList_pre,
                               fgt(PhysTime - dT_lev, StartTime, dT_levm1 / 2), ErrorMonitor))
+          {
+#if (ABEtype != 1)
           for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); }
+#endif
+          }
 
           //               a_stream.clear();
           //               a_stream.str("");
@@ -4020,7 +4044,11 @@ void bssn_class::Step(int lev, int YN)
 
   STEP_TIMER_DECL(timer_predictor_sync);
   Parallel::AsyncSyncState async_pre;
+#if (ABEtype == 1)
+  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+#else
   Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
+#endif
 
 #ifdef WithShell
   if (lev == 0)
@@ -4039,7 +4067,9 @@ void bssn_class::Step(int lev, int YN)
     }
   }
 #endif
+#if (ABEtype != 1)
   Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry);
+#endif
   STEP_TIMER_ADD(TB_PREDICTOR_SYNC, timer_predictor_sync);
 
 #if USE_CUDA_BSSN
@@ -4455,7 +4485,11 @@ void bssn_class::Step(int lev, int YN)
 
     STEP_TIMER_DECL(timer_corrector_sync);
     Parallel::AsyncSyncState async_cor;
+#if (ABEtype == 1)
+    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+#else
     Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
+#endif
 
 #ifdef WithShell
     if (lev == 0)
@@ -4474,7 +4508,9 @@ void bssn_class::Step(int lev, int YN)
       }
     }
 #endif
+#if (ABEtype != 1)
     Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry);
+#endif
     STEP_TIMER_ADD(TB_CORRECTOR_SYNC, timer_corrector_sync);
 
 #ifdef WithShell
@@ -4965,7 +5001,11 @@ void bssn_class::Step(int lev, int YN)
 #endif
 
   Parallel::AsyncSyncState async_pre;
+#if (ABEtype == 1)
+  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+#else
   Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
+#endif
 
 #ifdef WithShell
   if (lev == 0)
@@ -4984,7 +5024,9 @@ void bssn_class::Step(int lev, int YN)
     }
   }
 #endif
+#if (ABEtype != 1)
   Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry);
+#endif
 
 #ifdef WithShell
   // Complete non-blocking error reduction and check
@@ -5310,7 +5352,11 @@ void bssn_class::Step(int lev, int YN)
 #endif
 
     Parallel::AsyncSyncState async_cor;
+#if (ABEtype == 1)
+    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+#else
     Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
+#endif
 
 #ifdef WithShell
     if (lev == 0)
@@ -5329,7 +5375,9 @@ void bssn_class::Step(int lev, int YN)
       }
     }
 #endif
+#if (ABEtype != 1)
     Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry);
+#endif
 
 #ifdef WithShell
     // Complete non-blocking error reduction and check
@@ -5721,7 +5769,11 @@ void bssn_class::Step(int lev, int YN)
 
   //   misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Predictor sync");
 
+#if (ABEtype == 1)
+  Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry);
+#else
   Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]);
+#endif
 
   // Complete non-blocking error reduction and check
   MPI_Wait(&err_req, MPI_STATUS_IGNORE);
@@ -5922,7 +5974,11 @@ void bssn_class::Step(int lev, int YN)
 
     //    misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Corrector sync");
 
+#if (ABEtype == 1)
+    Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry);
+#else
     Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]);
+#endif
 
     //    misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"after Corrector sync");