diff --git a/AMSS_NCKU_source/bssn_class.C b/AMSS_NCKU_source/bssn_class.C index 3736e98..376df07 100644 --- a/AMSS_NCKU_source/bssn_class.C +++ b/AMSS_NCKU_source/bssn_class.C @@ -2489,7 +2489,9 @@ void bssn_class::Evolve(int Steps) GH->Regrid(Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_mon, StartTime, dT_mon / 2), ErrorMonitor); +#if (ABEtype != 1) for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif STEP_TIMER_ADD(TB_REGRID, timer_regrid); #endif @@ -2730,7 +2732,9 @@ void bssn_class::RecursiveStep(int lev) { if (ConstraintRefreshLevels) ConstraintRefreshLevels[lev] = 1; +#if (ABEtype != 1) for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif } STEP_TIMER_ADD(TB_REGRID, timer_regrid_onelevel); #endif @@ -2911,7 +2915,9 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(GH->mylev, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), ErrorMonitor)) - for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#if (ABEtype != 1) + for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif #endif } @@ -3078,7 +3084,9 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(lev + 1, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_levp1, StartTime, dT_levp1 / 2), ErrorMonitor)) - for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#if (ABEtype != 1) + for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif // a_stream.clear(); // a_stream.str(""); @@ -3093,7 +3101,9 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(lev, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), ErrorMonitor)) - for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#if (ABEtype != 1) + for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif // a_stream.clear(); // a_stream.str(""); @@ -3112,7 +3122,9 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(lev - 1, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_lev, StartTime, dT_levm1 / 2), ErrorMonitor)) - for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#if (ABEtype != 1) + for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif // a_stream.clear(); // a_stream.str(""); @@ -3128,7 +3140,9 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(lev - 1, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_lev, StartTime, dT_levm1 / 2), ErrorMonitor)) - for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#if (ABEtype != 1) + for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif // a_stream.clear(); // a_stream.str(""); @@ -3659,7 +3673,11 @@ void bssn_class::Step(int lev, int YN) STEP_TIMER_DECL(timer_predictor_sync); Parallel::AsyncSyncState async_pre; +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry); +#else Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre); +#endif #ifdef WithShell if (lev == 0) @@ -3678,7 +3696,9 @@ void bssn_class::Step(int lev, int YN) } } #endif - Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry); +#if (ABEtype != 1) + Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry); +#endif #ifdef WithShell // Complete non-blocking error reduction and check @@ -4024,7 +4044,11 @@ void bssn_class::Step(int lev, int YN) STEP_TIMER_DECL(timer_corrector_sync); Parallel::AsyncSyncState async_cor; +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry); +#else Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor); +#endif #ifdef WithShell if (lev == 0) @@ -4043,7 +4067,9 @@ void bssn_class::Step(int lev, int YN) } } #endif - Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry); +#if (ABEtype != 1) + Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry); +#endif #ifdef WithShell // Complete non-blocking error reduction and check @@ -4532,7 +4558,11 @@ void bssn_class::Step(int lev, int YN) #endif Parallel::AsyncSyncState async_pre; - Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre); +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry); +#else + Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre); +#endif #ifdef WithShell if (lev == 0) @@ -4551,7 +4581,9 @@ void bssn_class::Step(int lev, int YN) } } #endif - Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry); +#if (ABEtype != 1) + Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry); +#endif #ifdef WithShell // Complete non-blocking error reduction and check @@ -4880,7 +4912,11 @@ void bssn_class::Step(int lev, int YN) #endif Parallel::AsyncSyncState async_cor; - Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor); +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry); +#else + Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor); +#endif #ifdef WithShell if (lev == 0) @@ -4899,7 +4935,9 @@ void bssn_class::Step(int lev, int YN) } } #endif - Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry); +#if (ABEtype != 1) + Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry); +#endif #ifdef WithShell // Complete non-blocking error reduction and check @@ -5291,7 +5329,11 @@ void bssn_class::Step(int lev, int YN) // misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Predictor sync"); - Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]); +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry); +#else + Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]); +#endif // Complete non-blocking error reduction and check MPI_Wait(&err_req, MPI_STATUS_IGNORE); @@ -5492,7 +5534,11 @@ void bssn_class::Step(int lev, int YN) // misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Corrector sync"); - Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]); +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry); +#else + Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]); +#endif // misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"after Corrector sync");