64-BitBrainstorm_2026/AMSS-NCKU
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize GPU RK4 stage sync path

Browse Source
This commit is contained in:
CGH0S7
2026-04-12 18:36:05 +08:00
parent d9287ea530
commit c8977d8356
4 changed files with 698 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

393
AMSS_NCKU_source/bssn_cuda_step.C
View File

@@ -4,7 +4,9 @@
#include <algorithm>
#include <cmath>
#include <cstring>
#include <cstdlib>
#include <iomanip>
#include <vector>
#include "bssn_class.h"
@@ -12,18 +14,172 @@
#include "bssn_gpu.h"
#include "bssn_macro.h"
namespace
{
enum StageProfileMetric
{
STAGE_PROFILE_TOTAL = 0,
STAGE_PROFILE_RHS,
STAGE_PROFILE_RUN_STAGE,
STAGE_PROFILE_RUN_STAGE_DEVICE,
STAGE_PROFILE_RUN_STAGE_HOST_FIX,
STAGE_PROFILE_LOWERBOUND,
STAGE_PROFILE_ENSURE,
STAGE_PROFILE_DOWNLOAD,
STAGE_PROFILE_CLEAR_CACHE,
STAGE_PROFILE_SYNC_START,
STAGE_PROFILE_SYNC_FINISH,
STAGE_PROFILE_REFRESH,
STAGE_PROFILE_COUNT
};
static const int kStageProfileMaxLevels = 32;
struct StageProfileStore
{
bool env_checked;
bool enabled;
int calls[kStageProfileMaxLevels];
double metric[kStageProfileMaxLevels][STAGE_PROFILE_COUNT];
};
StageProfileStore &stage_profile_store()
{
static StageProfileStore store = {};
return store;
}
bool stage_profile_enabled()
{
StageProfileStore &store = stage_profile_store();
if (!store.env_checked)
{
const char *env = getenv("AMSS_GPU_STAGE_TIMING");
store.enabled = (env && env[0] && strcmp(env, "0") != 0);
store.env_checked = true;
}
return store.enabled;
}
void stage_profile_note_call(int lev)
{
if (lev >= 0 && lev < kStageProfileMaxLevels)
stage_profile_store().calls[lev]++;
}
void stage_profile_add(int lev, StageProfileMetric metric, double seconds)
{
if (lev >= 0 && lev < kStageProfileMaxLevels)
stage_profile_store().metric[lev][metric] += seconds;
}
const char *stage_profile_metric_name(StageProfileMetric metric)
{
switch (metric)
{
case STAGE_PROFILE_TOTAL:
return "total";
case STAGE_PROFILE_RHS:
return "rhs";
case STAGE_PROFILE_RUN_STAGE:
return "run_stage";
case STAGE_PROFILE_RUN_STAGE_DEVICE:
return "run_stage_dev";
case STAGE_PROFILE_RUN_STAGE_HOST_FIX:
return "run_stage_host";
case STAGE_PROFILE_LOWERBOUND:
return "lower";
case STAGE_PROFILE_ENSURE:
return "ensure";
case STAGE_PROFILE_DOWNLOAD:
return "download";
case STAGE_PROFILE_CLEAR_CACHE:
return "clear_cache";
case STAGE_PROFILE_SYNC_START:
return "sync_start";
case STAGE_PROFILE_SYNC_FINISH:
return "sync_finish";
case STAGE_PROFILE_REFRESH:
return "refresh";
default:
return "unknown";
}
}
} // namespace
void bssn_cuda_dump_stage_profile()
{
if (!stage_profile_enabled())
return;
int myrank = 0;
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
StageProfileStore &store = stage_profile_store();
int global_calls_sum[kStageProfileMaxLevels] = {};
double global_metric_sum[kStageProfileMaxLevels][STAGE_PROFILE_COUNT] = {};
double global_metric_max[kStageProfileMaxLevels][STAGE_PROFILE_COUNT] = {};
MPI_Reduce(store.calls, global_calls_sum, kStageProfileMaxLevels, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(store.metric[0], global_metric_sum[0],
kStageProfileMaxLevels * STAGE_PROFILE_COUNT,
MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
MPI_Reduce(store.metric[0], global_metric_max[0],
kStageProfileMaxLevels * STAGE_PROFILE_COUNT,
MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
if (myrank != 0)
return;
cout << endl;
cout << " GPU stage timing summary (sum/max over MPI ranks) " << endl;
cout << " lev calls";
for (int metric = 0; metric < STAGE_PROFILE_COUNT; ++metric)
cout << " " << setw(22) << stage_profile_metric_name(static_cast<StageProfileMetric>(metric));
cout << endl;
for (int lev = 0; lev < kStageProfileMaxLevels; ++lev)
{
if (global_calls_sum[lev] == 0)
continue;
cout << setw(4) << lev << " " << setw(5) << global_calls_sum[lev];
for (int metric = 0; metric < STAGE_PROFILE_COUNT; ++metric)
{
cout << " "
<< setw(10) << setprecision(6) << fixed << global_metric_sum[lev][metric]
<< "/"
<< setw(10) << setprecision(6) << fixed << global_metric_max[lev][metric];
}
cout << endl;
}
cout << endl;
}
void bssn_class::Step_MainPath_GPU(int lev, int YN)
{
#ifdef WithShell
#error "Step_MainPath_GPU currently supports Patch grids only."
#endif
const bool profile_enabled = stage_profile_enabled();
const double step_total_begin = profile_enabled ? MPI_Wtime() : 0.0;
if (profile_enabled)
stage_profile_note_call(lev);
if (bssn_gpu_bind_process_device(myrank))
{
cerr << "GPU device bind failure on MPI rank " << myrank << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
bssn_gpu_clear_cached_device_buffers();
if (profile_enabled)
{
const double t0 = MPI_Wtime();
bssn_gpu_clear_cached_device_buffers();
stage_profile_add(lev, STAGE_PROFILE_CLEAR_CACHE, MPI_Wtime() - t0);
}
else
bssn_gpu_clear_cached_device_buffers();
setpbh(BH_num, Porg0, Mass, BH_num_input);
@@ -62,6 +218,7 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
int iter_count = 0;
int pre = 0, cor = 1;
int ERROR = 0;
const bool keep_stage_sync_on_device = (RPS == 1) && (MAPBH == 1) && (REGLEV == 0);
auto run_stage_on_block =
[&](Block *cg, Patch *patch, MyList<var> *state0_list,
@@ -71,9 +228,27 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
MyList<var> *varlb = boundary_src_list;
MyList<var> *varls = stage_data_list;
MyList<var> *varlr = rhs_list;
std::vector<const double *> batch_state0;
std::vector<double *> batch_stage;
std::vector<double *> batch_rhs;
while (varl0)
{
const bool force_host_boundary_fix = false;
const bool can_batch_device_path = (lev > 0) && !force_host_boundary_fix;
if (can_batch_device_path)
{
batch_state0.push_back(cg->fgfs[varl0->data->sgfn]);
batch_stage.push_back(cg->fgfs[varls->data->sgfn]);
batch_rhs.push_back(cg->fgfs[varlr->data->sgfn]);
varl0 = varl0->next;
varlb = varlb->next;
varls = varls->next;
varlr = varlr->next;
continue;
}
const double var_begin = profile_enabled ? MPI_Wtime() : 0.0;
if (bssn_cuda_rk4_boundary_var(cg->shape, dT_lev,
cg->X[0], cg->X[1], cg->X[2],
patch->bbox[0], patch->bbox[1], patch->bbox[2],
@@ -86,7 +261,8 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
cg->fgfs[varlr->data->sgfn],
varl0->data->propspeed,
varl0->data->SoA,
Symmetry, lev, rk_stage, false))
Symmetry, lev, rk_stage,
force_host_boundary_fix, false))
{
cerr << "GPU rk4/boundary failure: lev=" << lev
<< " rk_stage=" << rk_stage
@@ -97,18 +273,59 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
ERROR = 1;
break;
}
if (profile_enabled)
{
stage_profile_add(lev,
force_host_boundary_fix ? STAGE_PROFILE_RUN_STAGE_HOST_FIX
: STAGE_PROFILE_RUN_STAGE_DEVICE,
MPI_Wtime() - var_begin);
}
varl0 = varl0->next;
varlb = varlb->next;
varls = varls->next;
varlr = varlr->next;
}
if (!ERROR && !batch_state0.empty())
{
const double batch_begin = profile_enabled ? MPI_Wtime() : 0.0;
if (bssn_cuda_rk4_boundary_batch(cg->shape, dT_lev,
cg->X[0], cg->X[1], cg->X[2],
patch->bbox[0], patch->bbox[1], patch->bbox[2],
patch->bbox[3], patch->bbox[4], patch->bbox[5],
Symmetry,
&batch_state0[0],
&batch_stage[0],
&batch_rhs[0],
static_cast<int>(batch_state0.size()),
rk_stage, false))
{
cerr << "GPU rk4/boundary batch failure: lev=" << lev
<< " rk_stage=" << rk_stage
<< " vars=" << batch_state0.size()
<< " bbox=(" << cg->bbox[0] << ":" << cg->bbox[3] << ","
<< cg->bbox[1] << ":" << cg->bbox[4] << ","
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
else if (profile_enabled)
{
stage_profile_add(lev, STAGE_PROFILE_RUN_STAGE_DEVICE, MPI_Wtime() - batch_begin);
}
}
};
auto stage_download_var_list =
[&](Block *cg, MyList<var> *var_list) {
[&](Block *cg, MyList<var> *var_list, bool skip_unmapped) {
while (var_list)
{
if (bssn_cuda_download_buffer(cg->shape, cg->fgfs[var_list->data->sgfn]))
double *host_ptr = cg->fgfs[var_list->data->sgfn];
if (skip_unmapped && !bssn_gpu_find_device_buffer(host_ptr))
{
var_list = var_list->next;
continue;
}
if (bssn_cuda_download_buffer(cg->shape, host_ptr))
{
cerr << "GPU stage download failure: lev=" << lev
<< " var=" << var_list->data->name
@@ -123,7 +340,7 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
};
auto stage_download_patch_list =
[&](MyList<var> *var_list) {
[&](MyList<var> *var_list, bool skip_unmapped) {
MyList<Patch> *patch_it = GH->PatL[lev];
while (patch_it)
{
@@ -132,7 +349,7 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
{
Block *cg = block_it->data;
if (myrank == cg->rank)
stage_download_var_list(cg, var_list);
stage_download_var_list(cg, var_list, skip_unmapped);
if (block_it == patch_it->data->ble)
break;
@@ -341,11 +558,22 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
Block *cg = BP->data;
if (myrank == cg->rank)
{
double t0 = 0.0;
if (profile_enabled)
t0 = MPI_Wtime();
if (gpu_rhs(CALLED_BY_STEP, myrank, RHS_PARA_CALLED_FIRST_TIME))
ERROR = 1;
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_RHS, MPI_Wtime() - t0);
if (profile_enabled)
t0 = MPI_Wtime();
run_stage_on_block(cg, Pp->data, StateList, StateList, SynchList_pre, RHSList, iter_count);
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_RUN_STAGE, MPI_Wtime() - t0);
if (profile_enabled)
t0 = MPI_Wtime();
if (bssn_cuda_lowerbound(cg->shape, cg->fgfs[phi->sgfn], chitiny, false))
{
cerr << "GPU lowerbound failure: lev=" << lev
@@ -356,6 +584,8 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_LOWERBOUND, MPI_Wtime() - t0);
}
if (BP == Pp->data->ble)
break;
@@ -366,9 +596,23 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
if (!ERROR)
{
stage_download_patch_list(SynchList_pre);
if (!ERROR)
bssn_gpu_clear_cached_device_buffers();
if (!keep_stage_sync_on_device)
{
double t0 = 0.0;
if (profile_enabled)
t0 = MPI_Wtime();
stage_download_patch_list(SynchList_pre, false);
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_DOWNLOAD, MPI_Wtime() - t0);
if (!ERROR)
{
if (profile_enabled)
t0 = MPI_Wtime();
bssn_gpu_clear_cached_device_buffers();
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_CLEAR_CACHE, MPI_Wtime() - t0);
}
}
}
MPI_Request err_req_pre;
@@ -378,10 +622,35 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
}
Parallel::AsyncSyncState async_pre;
Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry, true);
if (!ERROR)
refresh_stage_device_after_sync(SynchList_pre, sync_cache_pre[lev]);
if (profile_enabled)
{
const double t0 = MPI_Wtime();
Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
stage_profile_add(lev, STAGE_PROFILE_SYNC_START, MPI_Wtime() - t0);
}
else
Parallel::Sync_start(GH->PatL[lev], SynchList_pre, Symmetry, sync_cache_pre[lev], async_pre);
if (profile_enabled)
{
const double t0 = MPI_Wtime();
Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry,
!keep_stage_sync_on_device);
stage_profile_add(lev, STAGE_PROFILE_SYNC_FINISH, MPI_Wtime() - t0);
}
else
Parallel::Sync_finish(sync_cache_pre[lev], async_pre, SynchList_pre, Symmetry,
!keep_stage_sync_on_device);
if (!ERROR && !keep_stage_sync_on_device)
{
if (profile_enabled)
{
const double t0 = MPI_Wtime();
refresh_stage_device_after_sync(SynchList_pre, sync_cache_pre[lev]);
stage_profile_add(lev, STAGE_PROFILE_REFRESH, MPI_Wtime() - t0);
}
else
refresh_stage_device_after_sync(SynchList_pre, sync_cache_pre[lev]);
}
MPI_Wait(&err_req_pre, MPI_STATUS_IGNORE);
if (ERROR)
@@ -433,12 +702,28 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
Block *cg = BP->data;
if (myrank == cg->rank)
{
double t0 = 0.0;
if (profile_enabled)
t0 = MPI_Wtime();
ensure_stage_device_var_list(cg, SynchList_pre);
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_ENSURE, MPI_Wtime() - t0);
if (profile_enabled)
t0 = MPI_Wtime();
if (gpu_rhs(CALLED_BY_STEP, myrank, RHS_PARA_CALLED_THEN))
ERROR = 1;
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_RHS, MPI_Wtime() - t0);
if (profile_enabled)
t0 = MPI_Wtime();
run_stage_on_block(cg, Pp->data, StateList, SynchList_pre, SynchList_cor, RHSList, iter_count);
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_RUN_STAGE, MPI_Wtime() - t0);
if (profile_enabled)
t0 = MPI_Wtime();
if (bssn_cuda_lowerbound(cg->shape, cg->fgfs[phi1->sgfn], chitiny, false))
{
cerr << "GPU lowerbound failure: lev=" << lev
@@ -449,6 +734,8 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
<< cg->bbox[2] << ":" << cg->bbox[5] << ")" << endl;
ERROR = 1;
}
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_LOWERBOUND, MPI_Wtime() - t0);
}
if (BP == Pp->data->ble)
@@ -460,9 +747,23 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
if (!ERROR)
{
stage_download_patch_list(SynchList_cor);
if (!ERROR)
bssn_gpu_clear_cached_device_buffers();
if (!keep_stage_sync_on_device)
{
double t0 = 0.0;
if (profile_enabled)
t0 = MPI_Wtime();
stage_download_patch_list(SynchList_cor, false);
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_DOWNLOAD, MPI_Wtime() - t0);
if (!ERROR)
{
if (profile_enabled)
t0 = MPI_Wtime();
bssn_gpu_clear_cached_device_buffers();
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_CLEAR_CACHE, MPI_Wtime() - t0);
}
}
}
MPI_Request err_req_cor;
@@ -472,10 +773,35 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
}
Parallel::AsyncSyncState async_cor;
Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry, true);
if (!ERROR && iter_count < 3)
refresh_stage_device_after_sync(SynchList_cor, sync_cache_cor[lev]);
if (profile_enabled)
{
const double t0 = MPI_Wtime();
Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
stage_profile_add(lev, STAGE_PROFILE_SYNC_START, MPI_Wtime() - t0);
}
else
Parallel::Sync_start(GH->PatL[lev], SynchList_cor, Symmetry, sync_cache_cor[lev], async_cor);
if (profile_enabled)
{
const double t0 = MPI_Wtime();
Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry,
!keep_stage_sync_on_device);
stage_profile_add(lev, STAGE_PROFILE_SYNC_FINISH, MPI_Wtime() - t0);
}
else
Parallel::Sync_finish(sync_cache_cor[lev], async_cor, SynchList_cor, Symmetry,
!keep_stage_sync_on_device);
if (!ERROR && !keep_stage_sync_on_device && iter_count < 3)
{
if (profile_enabled)
{
const double t0 = MPI_Wtime();
refresh_stage_device_after_sync(SynchList_cor, sync_cache_cor[lev]);
stage_profile_add(lev, STAGE_PROFILE_REFRESH, MPI_Wtime() - t0);
}
else
refresh_stage_device_after_sync(SynchList_cor, sync_cache_cor[lev]);
}
MPI_Wait(&err_req_cor, MPI_STATUS_IGNORE);
if (ERROR)
@@ -545,8 +871,6 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
RestrictProlong(lev, YN, BB);
#endif
bssn_gpu_clear_cached_device_buffers();
Pp = GH->PatL[lev];
while (Pp)
{
@@ -563,6 +887,28 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
Pp = Pp->next;
}
if (!ERROR && keep_stage_sync_on_device)
{
MyList<var> *final_host_lists[] = {StateList, OldStateList, SynchList_cor, SynchList_pre};
const int final_host_list_count = sizeof(final_host_lists) / sizeof(final_host_lists[0]);
for (int list_i = 0; list_i < final_host_list_count && !ERROR; ++list_i)
{
const double t0 = profile_enabled ? MPI_Wtime() : 0.0;
stage_download_patch_list(final_host_lists[list_i], true);
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_DOWNLOAD, MPI_Wtime() - t0);
}
}
if (profile_enabled)
{
const double t0 = MPI_Wtime();
bssn_gpu_clear_cached_device_buffers();
stage_profile_add(lev, STAGE_PROFILE_CLEAR_CACHE, MPI_Wtime() - t0);
}
else
bssn_gpu_clear_cached_device_buffers();
if (BH_num > 0 && lev == GH->levels - 1)
{
for (int ithBH = 0; ithBH < BH_num; ithBH++)
@@ -572,6 +918,9 @@ void bssn_class::Step_MainPath_GPU(int lev, int YN)
Porg0[ithBH][2] = Porg1[ithBH][2];
}
}
if (profile_enabled)
stage_profile_add(lev, STAGE_PROFILE_TOTAL, MPI_Wtime() - step_total_begin);
}
#endif