diff --git a/AMSS_NCKU_source/bssn_class.C b/AMSS_NCKU_source/bssn_class.C index 4a6323d..ed40deb 100644 --- a/AMSS_NCKU_source/bssn_class.C +++ b/AMSS_NCKU_source/bssn_class.C @@ -2772,7 +2772,9 @@ void bssn_class::Evolve(int Steps) GH->Regrid(Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_mon, StartTime, dT_mon / 2), ErrorMonitor); +#if (ABEtype != 1) for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif STEP_TIMER_ADD(TB_REGRID, timer_regrid); #endif @@ -3009,7 +3011,9 @@ void bssn_class::RecursiveStep(int lev) { if (ConstraintRefreshLevels) ConstraintRefreshLevels[lev] = 1; +#if (ABEtype != 1) for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif } STEP_TIMER_ADD(TB_REGRID, timer_regrid_onelevel); #endif @@ -3190,7 +3194,11 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(GH->mylev, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), ErrorMonitor)) + { +#if (ABEtype != 1) for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif + } #endif } @@ -3357,7 +3365,11 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(lev + 1, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_levp1, StartTime, dT_levp1 / 2), ErrorMonitor)) + { +#if (ABEtype != 1) for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif + } // a_stream.clear(); // a_stream.str(""); @@ -3372,7 +3384,11 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(lev, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_lev, StartTime, dT_lev / 2), ErrorMonitor)) + { +#if (ABEtype != 1) for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif + } // a_stream.clear(); // a_stream.str(""); @@ -3391,7 +3407,11 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(lev - 1, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_lev, StartTime, dT_levm1 / 2), ErrorMonitor)) + { +#if (ABEtype != 1) for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif + } // a_stream.clear(); // a_stream.str(""); @@ -3407,7 +3427,11 @@ void bssn_class::ParallelStep() if (GH->Regrid_Onelevel(lev - 1, Symmetry, BH_num, Porgbr, Porg0, SynchList_cor, OldStateList, StateList, SynchList_pre, fgt(PhysTime - dT_lev, StartTime, dT_levm1 / 2), ErrorMonitor)) + { +#if (ABEtype != 1) for (int il = 0; il < GH->levels; il++) { sync_cache_pre[il].invalidate(); sync_cache_cor[il].invalidate(); sync_cache_rp_coarse[il].invalidate(); sync_cache_rp_fine[il].invalidate(); sync_cache_restrict[il].invalidate(); sync_cache_outbd[il].invalidate(); } +#endif + } // a_stream.clear(); // a_stream.str(""); @@ -4020,7 +4044,11 @@ void bssn_class::Step(int lev, int YN) STEP_TIMER_DECL(timer_predictor_sync); Parallel::AsyncSyncState async_pre; +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry); +#else Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre); +#endif #ifdef WithShell if (lev == 0) @@ -4039,7 +4067,9 @@ void bssn_class::Step(int lev, int YN) } } #endif +#if (ABEtype != 1) Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry); +#endif STEP_TIMER_ADD(TB_PREDICTOR_SYNC, timer_predictor_sync); #if USE_CUDA_BSSN @@ -4455,7 +4485,11 @@ void bssn_class::Step(int lev, int YN) STEP_TIMER_DECL(timer_corrector_sync); Parallel::AsyncSyncState async_cor; +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry); +#else Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor); +#endif #ifdef WithShell if (lev == 0) @@ -4474,7 +4508,9 @@ void bssn_class::Step(int lev, int YN) } } #endif +#if (ABEtype != 1) Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry); +#endif STEP_TIMER_ADD(TB_CORRECTOR_SYNC, timer_corrector_sync); #ifdef WithShell @@ -4965,7 +5001,11 @@ void bssn_class::Step(int lev, int YN) #endif Parallel::AsyncSyncState async_pre; +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry); +#else Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre); +#endif #ifdef WithShell if (lev == 0) @@ -4984,7 +5024,9 @@ void bssn_class::Step(int lev, int YN) } } #endif +#if (ABEtype != 1) Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry); +#endif #ifdef WithShell // Complete non-blocking error reduction and check @@ -5310,7 +5352,11 @@ void bssn_class::Step(int lev, int YN) #endif Parallel::AsyncSyncState async_cor; +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry); +#else Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor); +#endif #ifdef WithShell if (lev == 0) @@ -5329,7 +5375,9 @@ void bssn_class::Step(int lev, int YN) } } #endif +#if (ABEtype != 1) Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry); +#endif #ifdef WithShell // Complete non-blocking error reduction and check @@ -5721,7 +5769,11 @@ void bssn_class::Step(int lev, int YN) // misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Predictor sync"); +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_pre, Symmetry); +#else Parallel::Sync_cached(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev]); +#endif // Complete non-blocking error reduction and check MPI_Wait(&err_req, MPI_STATUS_IGNORE); @@ -5922,7 +5974,11 @@ void bssn_class::Step(int lev, int YN) // misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"before Corrector sync"); +#if (ABEtype == 1) + Parallel::Sync(GH->PatL[lev], SynchList_cor, Symmetry); +#else Parallel::Sync_cached(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev]); +#endif // misc::tillherecheck(GH->Commlev[lev],GH->start_rank[lev],"after Corrector sync");