64-BitBrainstorm_2026/AMSS-NCKU
8
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize BSSN CUDA resident state and CUDA-aware MPI

Browse Source
This commit is contained in:
CGH0S7
2026-04-29 17:05:10 +08:00
parent a0dab90bcb
commit 22c1e7168b
7 changed files with 7461 additions and 6727 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
AMSS_NCKU_Input.py
View File

@@ -16,7 +16,7 @@ import numpy
File_directory = "GW150914" ## output file directory
Output_directory = "binary_output" ## binary data file directory
## The file directory name should not be too long
MPI_processes = 8 ## number of mpi processes used in the simulation
MPI_processes = 2 ## number of mpi processes used in the simulation
GPU_Calculation = "yes" ## Use GPU or not
## (prefer "no" in the current version, because the GPU part may have bugs when integrated in this Python interface)

65
AMSS_NCKU_source/Block.C
View File

@@ -12,7 +12,61 @@ using namespace std;
#include "Block.h"
#include "misc.h"
Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), ingfs(ingfsi), fngfs(fngfsi), lev(levi), cgpu(cgpui)
#if USE_CUDA_BSSN || USE_CUDA_Z4C
#include <cuda_runtime_api.h>
#endif
namespace {
bool cuda_pin_gridfuncs_enabled()
{
static int enabled = -1;
if (enabled < 0)
{
const char *env = getenv("AMSS_CUDA_PIN_GRIDFUNCS");
enabled = (env && atoi(env) != 0) ? 1 : 0;
}
return enabled != 0;
}
double *alloc_gridfunc(size_t count, unsigned char &pinned)
{
pinned = 0;
#if USE_CUDA_BSSN || USE_CUDA_Z4C
if (cuda_pin_gridfuncs_enabled())
{
double *ptr = 0;
cudaError_t err = cudaMallocHost((void **)&ptr, count * sizeof(double));
if (err == cudaSuccess)
{
pinned = 1;
return ptr;
}
cudaGetLastError();
}
#endif
return (double *)malloc(sizeof(double) * count);
}
void free_gridfunc(double *ptr, unsigned char pinned)
{
if (!ptr)
return;
#if USE_CUDA_BSSN || USE_CUDA_Z4C
if (pinned)
{
cudaFreeHost(ptr);
return;
}
#else
(void)pinned;
#endif
free(ptr);
}
}
Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfsi, int levi, const int cgpui) : rank(ranki), lev(levi), cgpu(cgpui), ingfs(ingfsi), fngfs(fngfsi), igfs(0), fgfs(0), fgfs_pinned(0)
{
for (int i = 0; i < dim; i++)
X[i] = 0;
@@ -70,9 +124,10 @@ Block::Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fng
int nn = shape[0] * shape[1] * shape[2];
fgfs = new double *[fngfs];
fgfs_pinned = new unsigned char[fngfs];
for (int i = 0; i < fngfs; i++)
{
fgfs[i] = (double *)malloc(sizeof(double) * nn);
fgfs[i] = alloc_gridfunc((size_t)nn, fgfs_pinned[i]);
if (!(fgfs[i]))
{
cout << "on node#" << rank << ", out of memory when constructing Block." << endl;
@@ -107,11 +162,13 @@ Block::~Block()
free(igfs[i]);
delete[] igfs;
for (int i = 0; i < fngfs; i++)
free(fgfs[i]);
free_gridfunc(fgfs[i], fgfs_pinned ? fgfs_pinned[i] : 0);
delete[] fgfs;
delete[] fgfs_pinned;
X[0] = X[1] = X[2] = 0;
igfs = 0;
fgfs = 0;
fgfs_pinned = 0;
}
}
void Block::checkBlock()
@@ -187,6 +244,8 @@ void Block::swapList(MyList<var> *VarList1, MyList<var> *VarList2, int myrank)
while (varl1 && varl2)
{
misc::swap<double *>(fgfs[varl1->data->sgfn], fgfs[varl2->data->sgfn]);
if (fgfs_pinned)
misc::swap<unsigned char>(fgfs_pinned[varl1->data->sgfn], fgfs_pinned[varl2->data->sgfn]);
varl1 = varl1->next;
varl2 = varl2->next;
}

3
AMSS_NCKU_source/Block.h
View File

@@ -18,9 +18,10 @@ public:
int ingfs, fngfs;
int *(*igfs);
double *(*fgfs);
unsigned char *fgfs_pinned;
public:
Block() {};
Block() : rank(0), lev(0), cgpu(0), ingfs(0), fngfs(0), igfs(0), fgfs(0), fgfs_pinned(0) {};
Block(int DIM, int *shapei, double *bboxi, int ranki, int ingfsi, int fngfs, int levi, const int cgpui = 0);
~Block();

471
AMSS_NCKU_source/Parallel.C
View File

@@ -23,6 +23,10 @@
namespace {
#if USE_CUDA_BSSN || USE_CUDA_Z4C
static thread_local bool s_cuda_aware_pack_active = false;
#endif
struct SyncProfileStats
{
long long start_calls;
@@ -268,6 +272,204 @@ bool cuda_direct_unpack_segment(double *buffer,
sync_profile_stats().direct_unpack_sec += MPI_Wtime() - t0;
return ok;
}
bool cuda_aware_mpi_enabled()
{
static int enabled = -1;
if (enabled < 0)
{
const char *env = getenv("AMSS_CUDA_AWARE_MPI");
enabled = (!env || atoi(env) != 0) ? 1 : 0;
}
return enabled != 0;
}
bool cuda_mpi_diag_enabled()
{
static int enabled = -1;
if (enabled < 0)
{
const char *env = getenv("AMSS_CUDA_MPI_DIAG");
enabled = (env && atoi(env) != 0) ? 1 : 0;
}
return enabled != 0 || sync_profile_enabled();
}
double *alloc_device_comm_buffer(int length)
{
if (length <= 0)
return 0;
double *ptr = 0;
cudaError_t err = cudaMalloc((void **)&ptr, (size_t)length * sizeof(double));
if (err != cudaSuccess)
{
fprintf(stderr, "Parallel: cudaMalloc failed for device comm buffer (%d doubles, err=%d)\n",
length, (int)err);
MPI_Abort(MPI_COMM_WORLD, 1);
}
return ptr;
}
void free_device_comm_buffer(double *&ptr)
{
if (!ptr)
return;
cudaFree(ptr);
ptr = 0;
}
void ensure_device_comm_buffer(double **buffers, int *caps, int idx, int length)
{
if (length <= caps[idx])
return;
free_device_comm_buffer(buffers[idx]);
buffers[idx] = alloc_device_comm_buffer(length);
if (!buffers[idx])
{
fprintf(stderr, "Parallel: failed to allocate device communication buffer (%d doubles)\n", length);
MPI_Abort(MPI_COMM_WORLD, 1);
}
caps[idx] = length;
}
bool cuda_direct_pack_segment_to_device(double *buffer,
const Parallel::gridseg *src,
const Parallel::gridseg *dst,
int state_count)
{
#if USE_CUDA_BSSN
if (state_count <= 0 || state_count > BSSN_CUDA_STATE_COUNT)
return false;
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
const int i0 = cuda_seg_begin(dst, src->Bg, 0);
const int j0 = cuda_seg_begin(dst, src->Bg, 1);
const int k0 = cuda_seg_begin(dst, src->Bg, 2);
const bool ok = bssn_cuda_pack_state_batch_to_device_buffer(
src->Bg, state_count, buffer, src->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
if (sync_profile_enabled())
sync_profile_stats().direct_pack_sec += MPI_Wtime() - t0;
return ok;
#else
(void)buffer; (void)src; (void)dst; (void)state_count;
return false;
#endif
}
bool cuda_direct_unpack_segment_from_device(double *buffer,
const Parallel::gridseg *dst,
int state_count)
{
#if USE_CUDA_BSSN
if (state_count <= 0 || state_count > BSSN_CUDA_STATE_COUNT)
return false;
const double t0 = sync_profile_enabled() ? MPI_Wtime() : 0.0;
const int i0 = cuda_seg_begin(dst, dst->Bg, 0);
const int j0 = cuda_seg_begin(dst, dst->Bg, 1);
const int k0 = cuda_seg_begin(dst, dst->Bg, 2);
const bool ok = bssn_cuda_unpack_state_batch_from_device_buffer(
dst->Bg, state_count, buffer, dst->Bg->shape,
i0, j0, k0,
dst->shape[0], dst->shape[1], dst->shape[2]) == 0;
if (sync_profile_enabled())
sync_profile_stats().direct_unpack_sec += MPI_Wtime() - t0;
return ok;
#else
(void)buffer; (void)dst; (void)state_count;
return false;
#endif
}
bool cuda_device_state_count_supported(int state_count)
{
#if USE_CUDA_BSSN
return state_count > 0 && state_count <= BSSN_CUDA_STATE_COUNT;
#else
(void)state_count;
return false;
#endif
}
bool cuda_segments_same_level(MyList<Parallel::gridseg> *src,
MyList<Parallel::gridseg> *dst,
int rank_in,
int dir,
int myrank)
{
bool has_work = false;
while (src && dst)
{
if ((dir == PACK && dst->data->Bg->rank == rank_in && src->data->Bg->rank == myrank) ||
(dir == UNPACK && src->data->Bg->rank == rank_in && dst->data->Bg->rank == myrank))
{
has_work = true;
if (!src->data || !dst->data || !src->data->Bg || !dst->data->Bg ||
src->data->Bg->lev != dst->data->Bg->lev)
return false;
}
src = src->next;
dst = dst->next;
}
return has_work;
}
bool cuda_pack_to_device_eligible(MyList<Parallel::gridseg> *src,
MyList<Parallel::gridseg> *dst,
int rank_in,
int state_count,
int myrank)
{
if (!cuda_aware_mpi_enabled() || !cuda_device_state_count_supported(state_count))
return false;
if (!cuda_segments_same_level(src, dst, rank_in, PACK, myrank))
return false;
while (src && dst)
{
if (dst->data->Bg->rank == rank_in && src->data->Bg->rank == myrank &&
!cuda_can_direct_pack(src->data, dst->data, 1))
return false;
src = src->next;
dst = dst->next;
}
return true;
}
bool cuda_recv_to_device_eligible(MyList<Parallel::gridseg> *src,
MyList<Parallel::gridseg> *dst,
int rank_in,
int state_count,
int myrank)
{
if (!cuda_aware_mpi_enabled() || !cuda_device_state_count_supported(state_count))
return false;
if (!cuda_segments_same_level(src, dst, rank_in, UNPACK, myrank))
return false;
while (src && dst)
{
if (src->data->Bg->rank == rank_in && dst->data->Bg->rank == myrank &&
!cuda_can_direct_unpack(dst->data, 1))
return false;
src = src->next;
dst = dst->next;
}
return true;
}
int data_packer_with_device_buffer(double *data,
MyList<Parallel::gridseg> *src,
MyList<Parallel::gridseg> *dst,
int rank_in,
int dir,
MyList<var> *VarLists,
MyList<var> *VarListd,
int Symmetry)
{
s_cuda_aware_pack_active = true;
int n = Parallel::data_packer(data, src, dst, rank_in, dir, VarLists, VarListd, Symmetry);
s_cuda_aware_pack_active = false;
return n;
}
#endif
} // namespace
@@ -4031,7 +4233,11 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
if (dir == PACK && cuda_state_count_direct_supported(state_count) &&
cuda_can_direct_pack(src->data, dst->data, type))
{
if (s_cuda_aware_pack_active) {
handled_by_cuda = cuda_direct_pack_segment_to_device(data + size_out, src->data, dst->data, state_count);
} else {
handled_by_cuda = cuda_direct_pack_segment(data + size_out, src->data, dst->data, state_count);
}
if (!handled_by_cuda)
{
cout << "Parallel::data_packer: CUDA direct pack failed." << endl;
@@ -4041,7 +4247,11 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
else if (dir == UNPACK && cuda_state_count_direct_supported(state_count) &&
cuda_can_direct_unpack(dst->data, type))
{
if (s_cuda_aware_pack_active) {
handled_by_cuda = cuda_direct_unpack_segment_from_device(data + size_out, dst->data, state_count);
} else {
handled_by_cuda = cuda_direct_unpack_segment(data + size_out, dst->data, state_count);
}
if (!handled_by_cuda)
{
cout << "Parallel::data_packer: CUDA direct unpack failed." << endl;
@@ -4050,6 +4260,14 @@ int Parallel::data_packer(double *data, MyList<Parallel::gridseg> *src, MyList<P
}
if (!handled_by_cuda)
{
#if USE_CUDA_BSSN || USE_CUDA_Z4C
if (s_cuda_aware_pack_active)
{
fprintf(stderr, "Parallel::data_packer: type %d not supported in CUDA-aware MPI mode. "
"This path requires host buffers but a device buffer was provided.\n", type);
MPI_Abort(MPI_COMM_WORLD, 1);
}
#endif
#endif
if (dir == PACK)
switch (type)
@@ -4620,8 +4838,12 @@ Parallel::SyncCache::SyncCache()
: valid(false), cpusize(0), combined_src(0), combined_dst(0),
send_lengths(0), recv_lengths(0), send_bufs(0), recv_bufs(0),
send_buf_caps(0), recv_buf_caps(0), send_buf_pinned(0), recv_buf_pinned(0),
send_buf_is_dev(0), recv_buf_is_dev(0),
send_buf_caps_dev(0), recv_buf_caps_dev(0),
send_bufs_dev(0), recv_bufs_dev(0),
reqs(0), stats(0), max_reqs(0),
lengths_valid(false), tc_req_node(0), tc_req_is_recv(0), tc_completed(0)
lengths_valid(false), tc_req_node(0), tc_req_is_recv(0), tc_completed(0),
cuda_aware_mode(false)
{
}
// SyncCache invalidate: free grid segment lists but keep buffers
@@ -4653,6 +4875,22 @@ void Parallel::SyncCache::destroy()
if (recv_buf_caps) delete[] recv_buf_caps;
for (int i = 0; i < cpusize; i++)
{
if (send_bufs_dev && send_bufs_dev[i])
{
#if USE_CUDA_BSSN || USE_CUDA_Z4C
free_device_comm_buffer(send_bufs_dev[i]);
#else
delete[] send_bufs_dev[i];
#endif
}
if (recv_bufs_dev && recv_bufs_dev[i])
{
#if USE_CUDA_BSSN || USE_CUDA_Z4C
free_device_comm_buffer(recv_bufs_dev[i]);
#else
delete[] recv_bufs_dev[i];
#endif
}
if (send_bufs && send_bufs[i])
{
#if USE_CUDA_BSSN || USE_CUDA_Z4C
@@ -4674,6 +4912,12 @@ void Parallel::SyncCache::destroy()
if (recv_bufs) delete[] recv_bufs;
if (send_buf_pinned) delete[] send_buf_pinned;
if (recv_buf_pinned) delete[] recv_buf_pinned;
if (send_buf_is_dev) delete[] send_buf_is_dev;
if (recv_buf_is_dev) delete[] recv_buf_is_dev;
if (send_buf_caps_dev) delete[] send_buf_caps_dev;
if (recv_buf_caps_dev) delete[] recv_buf_caps_dev;
if (send_bufs_dev) delete[] send_bufs_dev;
if (recv_bufs_dev) delete[] recv_bufs_dev;
if (reqs) delete[] reqs;
if (stats) delete[] stats;
if (tc_req_node) delete[] tc_req_node;
@@ -4682,11 +4926,15 @@ void Parallel::SyncCache::destroy()
combined_src = combined_dst = 0;
send_lengths = recv_lengths = 0;
send_buf_caps = recv_buf_caps = 0;
send_buf_caps_dev = recv_buf_caps_dev = 0;
send_bufs = recv_bufs = 0;
send_bufs_dev = recv_bufs_dev = 0;
send_buf_pinned = recv_buf_pinned = 0;
send_buf_is_dev = recv_buf_is_dev = 0;
reqs = 0; stats = 0;
tc_req_node = 0; tc_req_is_recv = 0; tc_completed = 0;
cpusize = 0; max_reqs = 0;
cuda_aware_mode = false;
}
// transfer_cached: reuse pre-allocated buffers from SyncCache
void Parallel::transfer_cached(MyList<Parallel::gridseg> **src, MyList<Parallel::gridseg> **dst,
@@ -4698,6 +4946,52 @@ void Parallel::transfer_cached(MyList<Parallel::gridseg> **src, MyList<Parallel:
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
int cpusize = cache.cpusize;
int cuda_device_sends = 0;
int cuda_device_recvs = 0;
for (int n = 0; n < cpusize; n++)
cache.send_buf_is_dev[n] = cache.recv_buf_is_dev[n] = 0;
#if USE_CUDA_BSSN || USE_CUDA_Z4C
const int state_count = cuda_state_var_count(VarList1, VarList2);
if (state_count < 0)
{
cout << "Parallel::transfer_cached: variable lists do not match." << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
if (cuda_aware_mpi_enabled())
{
for (int n = 0; n < cpusize; n++)
{
cache.send_buf_is_dev[n] = cuda_pack_to_device_eligible(src[myrank], dst[myrank], n, state_count, myrank) ? 1 : 0;
cache.recv_buf_is_dev[n] = cuda_recv_to_device_eligible(src[n], dst[n], n, state_count, myrank) ? 1 : 0;
}
cache.recv_buf_is_dev[myrank] = (cache.send_buf_is_dev[myrank] && cache.recv_buf_is_dev[myrank]) ? 1 : 0;
for (int n = 0; n < cpusize; n++)
{
cuda_device_sends += cache.send_buf_is_dev[n] ? 1 : 0;
cuda_device_recvs += cache.recv_buf_is_dev[n] ? 1 : 0;
}
if (cuda_mpi_diag_enabled())
{
static int diag_reported = 0;
int rep = diag_reported;
if (myrank == 0 && rep < 10)
{
if (__sync_bool_compare_and_swap(&diag_reported, rep, rep + 1))
fprintf(stderr, "[AMSS-CUDA-MPI][rank %d] transfer_cached: device_sends=%d "
"device_recvs=%d cuda_aware_mpi=%d\n",
myrank, cuda_device_sends, cuda_device_recvs,
cuda_aware_mpi_enabled() ? 1 : 0);
}
}
}
else
{
for (int n = 0; n < cpusize; n++)
cache.send_buf_is_dev[n] = cache.recv_buf_is_dev[n] = 0;
}
cache.cuda_aware_mode = (cuda_device_sends + cuda_device_recvs) > 0;
#endif
int req_no = 0;
int pending_recv = 0;
int node;
@@ -4713,9 +5007,17 @@ void Parallel::transfer_cached(MyList<Parallel::gridseg> **src, MyList<Parallel:
int rlength = data_packer(0, src[node], dst[node], node, UNPACK, VarList1, VarList2, Symmetry);
cache.recv_lengths[node] = rlength;
if (rlength > 0)
{
if (cache.recv_buf_is_dev[node])
{
ensure_device_comm_buffer(cache.recv_bufs_dev, cache.recv_buf_caps_dev, node, rlength);
MPI_Irecv((void *)cache.recv_bufs_dev[node], rlength, MPI_DOUBLE, node, 1, MPI_COMM_WORLD, cache.reqs + req_no);
}
else
{
ensure_comm_buffer(cache.recv_bufs, cache.recv_buf_pinned, cache.recv_buf_caps, node, rlength);
MPI_Irecv((void *)cache.recv_bufs[node], rlength, MPI_DOUBLE, node, 1, MPI_COMM_WORLD, cache.reqs + req_no);
}
req_node[req_no] = node;
req_is_recv[req_no] = 1;
req_no++;
@@ -4727,12 +5029,20 @@ void Parallel::transfer_cached(MyList<Parallel::gridseg> **src, MyList<Parallel:
int self_len = data_packer(0, src[myrank], dst[myrank], myrank, PACK, VarList1, VarList2, Symmetry);
cache.recv_lengths[myrank] = self_len;
if (self_len > 0)
{
if (cache.recv_buf_is_dev[myrank])
{
ensure_device_comm_buffer(cache.recv_bufs_dev, cache.recv_buf_caps_dev, myrank, self_len);
data_packer_with_device_buffer(cache.recv_bufs_dev[myrank], src[myrank], dst[myrank], myrank, PACK, VarList1, VarList2, Symmetry);
}
else
{
ensure_comm_buffer(cache.recv_bufs, cache.recv_buf_pinned, cache.recv_buf_caps, myrank, self_len);
data_packer(cache.recv_bufs[myrank], src[myrank], dst[myrank], myrank, PACK, VarList1, VarList2, Symmetry);
}
}
// Pack and post sends.
// Pack sends first. Device sends are posted after a single CUDA sync.
for (node = 0; node < cpusize; node++)
{
if (node == myrank) continue;
@@ -4740,9 +5050,33 @@ void Parallel::transfer_cached(MyList<Parallel::gridseg> **src, MyList<Parallel:
int slength = data_packer(0, src[myrank], dst[myrank], node, PACK, VarList1, VarList2, Symmetry);
cache.send_lengths[node] = slength;
if (slength > 0)
{
if (cache.send_buf_is_dev[node])
{
ensure_device_comm_buffer(cache.send_bufs_dev, cache.send_buf_caps_dev, node, slength);
data_packer_with_device_buffer(cache.send_bufs_dev[node], src[myrank], dst[myrank], node, PACK, VarList1, VarList2, Symmetry);
}
else
{
ensure_comm_buffer(cache.send_bufs, cache.send_buf_pinned, cache.send_buf_caps, node, slength);
data_packer(cache.send_bufs[node], src[myrank], dst[myrank], node, PACK, VarList1, VarList2, Symmetry);
}
}
}
#if USE_CUDA_BSSN || USE_CUDA_Z4C
if (cuda_device_sends > 0)
cudaDeviceSynchronize();
#endif
for (node = 0; node < cpusize; node++)
{
if (node == myrank) continue;
const int slength = cache.send_lengths[node];
if (slength > 0)
{
if (cache.send_buf_is_dev[node])
MPI_Isend((void *)cache.send_bufs_dev[node], slength, MPI_DOUBLE, node, 1, MPI_COMM_WORLD, cache.reqs + req_no);
else
MPI_Isend((void *)cache.send_bufs[node], slength, MPI_DOUBLE, node, 1, MPI_COMM_WORLD, cache.reqs + req_no);
req_node[req_no] = node;
req_is_recv[req_no] = 0;
@@ -4763,6 +5097,9 @@ void Parallel::transfer_cached(MyList<Parallel::gridseg> **src, MyList<Parallel:
if (idx >= 0 && req_is_recv[idx])
{
int recv_node_i = req_node[idx];
if (cache.recv_buf_is_dev[recv_node_i])
data_packer_with_device_buffer(cache.recv_bufs_dev[recv_node_i], src[recv_node_i], dst[recv_node_i], recv_node_i, UNPACK, VarList1, VarList2, Symmetry);
else
data_packer(cache.recv_bufs[recv_node_i], src[recv_node_i], dst[recv_node_i], recv_node_i, UNPACK, VarList1, VarList2, Symmetry);
pending_recv--;
}
@@ -4772,8 +5109,13 @@ void Parallel::transfer_cached(MyList<Parallel::gridseg> **src, MyList<Parallel:
if (req_no > 0) MPI_Waitall(req_no, cache.reqs, cache.stats);
if (self_len > 0)
{
if (cache.recv_buf_is_dev[myrank])
data_packer_with_device_buffer(cache.recv_bufs_dev[myrank], src[myrank], dst[myrank], myrank, UNPACK, VarList1, VarList2, Symmetry);
else
data_packer(cache.recv_bufs[myrank], src[myrank], dst[myrank], myrank, UNPACK, VarList1, VarList2, Symmetry);
}
}
void Parallel::Sync_ensure_cache(MyList<Patch> *PatL, int Symmetry, SyncCache &cache)
{
if (cache.valid)
@@ -4795,11 +5137,20 @@ void Parallel::Sync_ensure_cache(MyList<Patch> *PatL, int Symmetry, SyncCache &c
cache.recv_buf_caps = new int[cpusize];
cache.send_buf_pinned = new unsigned char[cpusize];
cache.recv_buf_pinned = new unsigned char[cpusize];
cache.send_buf_is_dev = new unsigned char[cpusize];
cache.recv_buf_is_dev = new unsigned char[cpusize];
cache.send_buf_caps_dev = new int[cpusize];
cache.recv_buf_caps_dev = new int[cpusize];
cache.send_bufs_dev = new double *[cpusize];
cache.recv_bufs_dev = new double *[cpusize];
for (int i = 0; i < cpusize; i++)
{
cache.send_bufs[i] = cache.recv_bufs[i] = 0;
cache.send_buf_caps[i] = cache.recv_buf_caps[i] = 0;
cache.send_buf_pinned[i] = cache.recv_buf_pinned[i] = 0;
cache.send_buf_is_dev[i] = cache.recv_buf_is_dev[i] = 0;
cache.send_buf_caps_dev[i] = cache.recv_buf_caps_dev[i] = 0;
cache.send_bufs_dev[i] = cache.recv_bufs_dev[i] = 0;
}
cache.max_reqs = 2 * cpusize;
cache.reqs = new MPI_Request[cache.max_reqs];
@@ -4900,6 +5251,47 @@ void Parallel::Sync_start(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetr
MyList<Parallel::gridseg> **src = cache.combined_src;
MyList<Parallel::gridseg> **dst = cache.combined_dst;
int cuda_device_sends = 0;
int cuda_device_recvs = 0;
for (int n = 0; n < cpusize; n++)
cache.send_buf_is_dev[n] = cache.recv_buf_is_dev[n] = 0;
#if USE_CUDA_BSSN || USE_CUDA_Z4C
const int state_count = cuda_state_var_count(VarList, VarList);
if (state_count < 0)
{
cout << "Parallel::Sync_start: variable lists do not match." << endl;
MPI_Abort(MPI_COMM_WORLD, 1);
}
if (cuda_aware_mpi_enabled())
{
for (int n = 0; n < cpusize; n++)
{
cache.send_buf_is_dev[n] = cuda_pack_to_device_eligible(src[myrank], dst[myrank], n, state_count, myrank) ? 1 : 0;
cache.recv_buf_is_dev[n] = cuda_recv_to_device_eligible(src[n], dst[n], n, state_count, myrank) ? 1 : 0;
}
cache.recv_buf_is_dev[myrank] = (cache.send_buf_is_dev[myrank] && cache.recv_buf_is_dev[myrank]) ? 1 : 0;
for (int n = 0; n < cpusize; n++)
{
cuda_device_sends += cache.send_buf_is_dev[n] ? 1 : 0;
cuda_device_recvs += cache.recv_buf_is_dev[n] ? 1 : 0;
}
if (cuda_mpi_diag_enabled())
{
static int diag_reported = 0;
int rep = diag_reported;
if (myrank == 0 && rep < 20)
{
if (__sync_bool_compare_and_swap(&diag_reported, rep, rep + 1))
fprintf(stderr, "[AMSS-CUDA-MPI][rank %d] Sync_start: device_sends=%d "
"device_recvs=%d cuda_aware_mpi=%d\n",
myrank, cuda_device_sends, cuda_device_recvs,
cuda_aware_mpi_enabled() ? 1 : 0);
}
}
}
cache.cuda_aware_mode = (cuda_device_sends + cuda_device_recvs) > 0;
#endif
for (int node = 0; node < cpusize; node++)
{
if (node == myrank)
@@ -4912,6 +5304,16 @@ void Parallel::Sync_start(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetr
rlength = cache.recv_lengths[node];
}
if (rlength > 0)
{
if (cache.recv_buf_is_dev[node])
{
ensure_device_comm_buffer(cache.recv_bufs_dev, cache.recv_buf_caps_dev, node, rlength);
state.req_node[state.req_no] = node;
state.req_is_recv[state.req_no] = 1;
state.pending_recv++;
MPI_Irecv((void *)cache.recv_bufs_dev[node], rlength, MPI_DOUBLE, node, 2, MPI_COMM_WORLD, cache.reqs + state.req_no++);
}
else
{
ensure_comm_buffer(cache.recv_bufs, cache.recv_buf_pinned, cache.recv_buf_caps, node, rlength);
state.req_node[state.req_no] = node;
@@ -4920,6 +5322,7 @@ void Parallel::Sync_start(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetr
MPI_Irecv((void *)cache.recv_bufs[node], rlength, MPI_DOUBLE, node, 2, MPI_COMM_WORLD, cache.reqs + state.req_no++);
}
}
}
for (int node = 0; node < cpusize; node++)
{
@@ -4933,11 +5336,19 @@ void Parallel::Sync_start(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetr
length = cache.recv_lengths[node];
}
if (length > 0)
{
if (cache.recv_buf_is_dev[node])
{
ensure_device_comm_buffer(cache.recv_bufs_dev, cache.recv_buf_caps_dev, node, length);
data_packer_with_device_buffer(cache.recv_bufs_dev[node], src[myrank], dst[myrank], node, PACK, VarList, VarList, Symmetry);
}
else
{
ensure_comm_buffer(cache.recv_bufs, cache.recv_buf_pinned, cache.recv_buf_caps, node, length);
data_packer(cache.recv_bufs[node], src[myrank], dst[myrank], node, PACK, VarList, VarList, Symmetry);
}
}
}
else
{
int slength;
@@ -4948,15 +5359,40 @@ void Parallel::Sync_start(MyList<Patch> *PatL, MyList<var> *VarList, int Symmetr
slength = cache.send_lengths[node];
}
if (slength > 0)
{
if (cache.send_buf_is_dev[node])
{
ensure_device_comm_buffer(cache.send_bufs_dev, cache.send_buf_caps_dev, node, slength);
data_packer_with_device_buffer(cache.send_bufs_dev[node], src[myrank], dst[myrank], node, PACK, VarList, VarList, Symmetry);
}
else
{
ensure_comm_buffer(cache.send_bufs, cache.send_buf_pinned, cache.send_buf_caps, node, slength);
data_packer(cache.send_bufs[node], src[myrank], dst[myrank], node, PACK, VarList, VarList, Symmetry);
}
}
}
}
#if USE_CUDA_BSSN || USE_CUDA_Z4C
if (cuda_device_sends > 0)
cudaDeviceSynchronize();
#endif
for (int node = 0; node < cpusize; node++)
{
if (node == myrank)
continue;
const int slength = cache.send_lengths[node];
if (slength > 0)
{
state.req_node[state.req_no] = node;
state.req_is_recv[state.req_no] = 0;
if (cache.send_buf_is_dev[node])
MPI_Isend((void *)cache.send_bufs_dev[node], slength, MPI_DOUBLE, node, 2, MPI_COMM_WORLD, cache.reqs + state.req_no++);
else
MPI_Isend((void *)cache.send_bufs[node], slength, MPI_DOUBLE, node, 2, MPI_COMM_WORLD, cache.reqs + state.req_no++);
}
}
}
cache.lengths_valid = true;
if (sync_profile_enabled())
{
@@ -4980,8 +5416,16 @@ void Parallel::Sync_finish(SyncCache &cache, AsyncSyncState &state,
double wait_sec = 0.0;
// Unpack local data first (no MPI needed)
if (cache.recv_buf_is_dev[myrank])
{
if (cache.recv_bufs_dev[myrank] && cache.recv_lengths[myrank] > 0)
data_packer_with_device_buffer(cache.recv_bufs_dev[myrank], src[myrank], dst[myrank], myrank, UNPACK, VarList, VarList, Symmetry);
}
else
{
if (cache.recv_bufs[myrank] && cache.recv_lengths[myrank] > 0)
data_packer(cache.recv_bufs[myrank], src[myrank], dst[myrank], myrank, UNPACK, VarList, VarList, Symmetry);
}
// Progressive unpack of remote receives
if (state.pending_recv > 0 && state.req_no > 0)
@@ -5001,6 +5445,9 @@ void Parallel::Sync_finish(SyncCache &cache, AsyncSyncState &state,
if (idx >= 0 && state.req_is_recv[idx])
{
int recv_node = state.req_node[idx];
if (cache.recv_buf_is_dev[recv_node])
data_packer_with_device_buffer(cache.recv_bufs_dev[recv_node], src[recv_node], dst[recv_node], recv_node, UNPACK, VarList, VarList, Symmetry);
else
data_packer(cache.recv_bufs[recv_node], src[recv_node], dst[recv_node], recv_node, UNPACK, VarList, VarList, Symmetry);
pending--;
}
@@ -6157,11 +6604,20 @@ void Parallel::Restrict_cached(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
cache.recv_buf_caps = new int[cpusize];
cache.send_buf_pinned = new unsigned char[cpusize];
cache.recv_buf_pinned = new unsigned char[cpusize];
cache.send_buf_is_dev = new unsigned char[cpusize];
cache.recv_buf_is_dev = new unsigned char[cpusize];
cache.send_buf_caps_dev = new int[cpusize];
cache.recv_buf_caps_dev = new int[cpusize];
cache.send_bufs_dev = new double *[cpusize];
cache.recv_bufs_dev = new double *[cpusize];
for (int i = 0; i < cpusize; i++)
{
cache.send_bufs[i] = cache.recv_bufs[i] = 0;
cache.send_buf_caps[i] = cache.recv_buf_caps[i] = 0;
cache.send_buf_pinned[i] = cache.recv_buf_pinned[i] = 0;
cache.send_buf_is_dev[i] = cache.recv_buf_is_dev[i] = 0;
cache.send_buf_caps_dev[i] = cache.recv_buf_caps_dev[i] = 0;
cache.send_bufs_dev[i] = cache.recv_bufs_dev[i] = 0;
}
cache.max_reqs = 2 * cpusize;
cache.reqs = new MPI_Request[cache.max_reqs];
@@ -6209,11 +6665,20 @@ void Parallel::OutBdLow2Hi_cached(MyList<Patch> *PatcL, MyList<Patch> *PatfL,
cache.recv_buf_caps = new int[cpusize];
cache.send_buf_pinned = new unsigned char[cpusize];
cache.recv_buf_pinned = new unsigned char[cpusize];
cache.send_buf_is_dev = new unsigned char[cpusize];
cache.recv_buf_is_dev = new unsigned char[cpusize];
cache.send_buf_caps_dev = new int[cpusize];
cache.recv_buf_caps_dev = new int[cpusize];
cache.send_bufs_dev = new double *[cpusize];
cache.recv_bufs_dev = new double *[cpusize];
for (int i = 0; i < cpusize; i++)
{
cache.send_bufs[i] = cache.recv_bufs[i] = 0;
cache.send_buf_caps[i] = cache.recv_buf_caps[i] = 0;
cache.send_buf_pinned[i] = cache.recv_buf_pinned[i] = 0;
cache.send_buf_is_dev[i] = cache.recv_buf_is_dev[i] = 0;
cache.send_buf_caps_dev[i] = cache.recv_buf_caps_dev[i] = 0;
cache.send_bufs_dev[i] = cache.recv_bufs_dev[i] = 0;
}
cache.max_reqs = 2 * cpusize;
cache.reqs = new MPI_Request[cache.max_reqs];

7
AMSS_NCKU_source/Parallel.h
View File

@@ -106,6 +106,12 @@ namespace Parallel
int *recv_buf_caps;
unsigned char *send_buf_pinned;
unsigned char *recv_buf_pinned;
unsigned char *send_buf_is_dev;
unsigned char *recv_buf_is_dev;
int *send_buf_caps_dev;
int *recv_buf_caps_dev;
double **send_bufs_dev;
double **recv_bufs_dev;
MPI_Request *reqs;
MPI_Status *stats;
int max_reqs;
@@ -113,6 +119,7 @@ namespace Parallel
int *tc_req_node;
int *tc_req_is_recv;
int *tc_completed;
bool cuda_aware_mode;
SyncCache();
void invalidate();
void destroy();

188
AMSS_NCKU_source/bssn_rhs_cuda.cu
View File

@@ -76,11 +76,36 @@ struct CudaProfileStats {
double output_ms;
};
enum RhsStageId {
RHS_STAGE_PREP = 0,
RHS_STAGE_DERIV1,
RHS_STAGE_METRIC,
RHS_STAGE_GAUGE_DERIV,
RHS_STAGE_GAMMA_CONTRACT,
RHS_STAGE_RICCI_DIFF,
RHS_STAGE_RICCI_FUSED,
RHS_STAGE_CHI,
RHS_STAGE_GAUGE_RHS,
RHS_STAGE_KODIS,
RHS_STAGE_CONSTRAINTS,
RHS_STAGE_COUNT
};
struct RhsStageProfileStats {
long long calls;
double ms[RHS_STAGE_COUNT];
};
static CudaProfileStats &cuda_profile_stats() {
static CudaProfileStats stats = {0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
return stats;
}
static RhsStageProfileStats &rhs_stage_profile_stats() {
static RhsStageProfileStats stats = {};
return stats;
}
static bool cuda_profile_enabled() {
static int enabled = -1;
if (enabled < 0) {
@@ -99,6 +124,24 @@ static int cuda_profile_every() {
return every;
}
static bool rhs_stage_timing_enabled() {
static int enabled = -1;
if (enabled < 0) {
const char *env = getenv("AMSS_GPU_STAGE_TIMING");
enabled = (env && atoi(env) != 0) ? 1 : 0;
}
return enabled != 0;
}
static int rhs_stage_timing_every() {
static int every = -1;
if (every < 0) {
const char *env = getenv("AMSS_GPU_STAGE_TIMING_EVERY");
every = (env && atoi(env) > 0) ? atoi(env) : cuda_profile_every();
}
return every;
}
static double cuda_profile_now_ms() {
using clock = std::chrono::steady_clock;
return std::chrono::duration<double, std::milli>(
@@ -131,6 +174,36 @@ static void cuda_profile_maybe_log() {
fflush(stderr);
}
static void rhs_stage_profile_accumulate(const double *stage_ms) {
if (!rhs_stage_timing_enabled()) return;
RhsStageProfileStats &stats = rhs_stage_profile_stats();
stats.calls++;
for (int i = 0; i < RHS_STAGE_COUNT; ++i) {
stats.ms[i] += stage_ms[i];
}
if (stats.calls <= 0 || stats.calls % rhs_stage_timing_every() != 0) return;
fprintf(stderr,
"[AMSS-CUDA-STAGE][rank %d][dev %d] calls=%lld"
" prep=%.3f deriv1=%.3f metric=%.3f gauge_deriv=%.3f"
" gamma_contract=%.3f ricci_diff=%.3f ricci_fused=%.3f"
" chi=%.3f gauge_rhs=%.3f kodis=%.3f constraints=%.3f ms\n",
g_dispatch.my_rank, g_dispatch.my_device, stats.calls,
stats.ms[RHS_STAGE_PREP] / (double)stats.calls,
stats.ms[RHS_STAGE_DERIV1] / (double)stats.calls,
stats.ms[RHS_STAGE_METRIC] / (double)stats.calls,
stats.ms[RHS_STAGE_GAUGE_DERIV] / (double)stats.calls,
stats.ms[RHS_STAGE_GAMMA_CONTRACT] / (double)stats.calls,
stats.ms[RHS_STAGE_RICCI_DIFF] / (double)stats.calls,
stats.ms[RHS_STAGE_RICCI_FUSED] / (double)stats.calls,
stats.ms[RHS_STAGE_CHI] / (double)stats.calls,
stats.ms[RHS_STAGE_GAUGE_RHS] / (double)stats.calls,
stats.ms[RHS_STAGE_KODIS] / (double)stats.calls,
stats.ms[RHS_STAGE_CONSTRAINTS] / (double)stats.calls);
fflush(stderr);
}
/* ------------------------------------------------------------------ */
/* Error checking */
/* ------------------------------------------------------------------ */
@@ -4643,6 +4716,20 @@ static void compute_patch_boundary_flags(int *ex,
static void upload_state_inputs(double **state_host, size_t all)
{
const size_t bytes = all * sizeof(double);
static int direct_upload = -1;
if (direct_upload < 0) {
const char *env = getenv("AMSS_CUDA_DIRECT_STATE_UPLOAD");
const char *pin_env = getenv("AMSS_CUDA_PIN_GRIDFUNCS");
direct_upload = env ? ((atoi(env) != 0) ? 1 : 0)
: ((pin_env && atoi(pin_env) != 0) ? 1 : 0);
}
if (direct_upload) {
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
CUDA_CHECK(cudaMemcpyAsync(g_buf.slot[k_state_input_slots[i]], state_host[i],
bytes, cudaMemcpyHostToDevice));
}
return;
}
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
std::memcpy(g_buf.h_stage + (size_t)i * all, state_host[i], bytes);
}
@@ -4697,11 +4784,24 @@ static void launch_rhs_pipeline(int all, double eps, int co)
{
const double SYM = 1.0;
const double ANTI = -1.0;
const bool stage_timing = rhs_stage_timing_enabled();
double stage_ms[RHS_STAGE_COUNT] = {};
double stage_t0 = stage_timing ? cuda_profile_now_ms() : 0.0;
#define D(s) g_buf.slot[s]
#define MARK_RHS_STAGE(stage_id) do { \
if (stage_timing) { \
cuda_profile_sync(); \
const double stage_t1 = cuda_profile_now_ms(); \
stage_ms[(stage_id)] += stage_t1 - stage_t0; \
stage_t0 = stage_t1; \
} \
} while (0)
kern_phase1_prep<<<grid(all),BLK>>>(
D(S_Lap), D(S_chi), D(S_dxx), D(S_dyy), D(S_dzz),
D(S_alpn1), D(S_chin1), D(S_gxx), D(S_gyy), D(S_gzz));
MARK_RHS_STAGE(RHS_STAGE_PREP);
{
double *src_fields[] = {
@@ -4742,6 +4842,7 @@ static void launch_rhs_pipeline(int all, double eps, int co)
src_fields, fx_fields, fy_fields, fz_fields,
soa_signs, all);
}
MARK_RHS_STAGE(RHS_STAGE_DERIV1);
kern_phase2_metric_rhs<<<grid(all),BLK>>>(
D(S_alpn1), D(S_chin1),
@@ -4799,6 +4900,7 @@ static void launch_rhs_pipeline(int all, double eps, int co)
D(S_Gamzxx), D(S_Gamzxy), D(S_Gamzxz),
D(S_Gamzyy), D(S_Gamzyz), D(S_Gamzzz),
D(S_Gamx_rhs), D(S_Gamy_rhs), D(S_Gamz_rhs));
MARK_RHS_STAGE(RHS_STAGE_METRIC);
{
double *src_fields[] = {D(S_betax), D(S_betay), D(S_betaz)};
@@ -4832,6 +4934,7 @@ static void launch_rhs_pipeline(int all, double eps, int co)
src_fields, fx_fields, fy_fields, fz_fields,
soa_signs, all);
}
MARK_RHS_STAGE(RHS_STAGE_GAUGE_DERIV);
kern_phase8_9_gamma_rhs_contract_fused<<<grid(all),BLK>>>(
D(S_gupxx), D(S_gupxy), D(S_gupxz),
@@ -4854,6 +4957,7 @@ static void launch_rhs_pipeline(int all, double eps, int co)
D(S_gxxx),D(S_gxyx),D(S_gxzx),D(S_gyyx),D(S_gyzx),D(S_gzzx),
D(S_gxxy),D(S_gxyy),D(S_gxzy),D(S_gyyy),D(S_gyzy),D(S_gzzy),
D(S_gxxz),D(S_gxyz),D(S_gxzz),D(S_gyyz),D(S_gyzz),D(S_gzzz));
MARK_RHS_STAGE(RHS_STAGE_GAMMA_CONTRACT);
{
double *src_fields[] = {D(S_dxx), D(S_dyy), D(S_dzz), D(S_gxy), D(S_gxz), D(S_gyz)};
@@ -4870,6 +4974,7 @@ static void launch_rhs_pipeline(int all, double eps, int co)
D(S_gupyy), D(S_gupyz), D(S_gupzz),
src_fields, dst_fields, soa_signs, all);
}
MARK_RHS_STAGE(RHS_STAGE_RICCI_DIFF);
kern_phase11_ricci_fused<<<grid(all),BLK>>>(
D(S_gxx),D(S_gxy),D(S_gxz),D(S_gyy),D(S_gyz),D(S_gzz),
@@ -4889,6 +4994,7 @@ static void launch_rhs_pipeline(int all, double eps, int co)
D(S_gxxz),D(S_gxyz),D(S_gxzz),D(S_gyyz),D(S_gyzz),D(S_gzzz),
D(S_Rxx),D(S_Rxy),D(S_Rxz),
D(S_Ryy),D(S_Ryz),D(S_Rzz));
MARK_RHS_STAGE(RHS_STAGE_RICCI_FUSED);
kern_phase12_13_chi_correction_fused<<<grid((size_t)all),BLK>>>(
D(S_chi), D(S_chin1),
@@ -4904,6 +5010,7 @@ static void launch_rhs_pipeline(int all, double eps, int co)
D(S_Gamzyy), D(S_Gamzyz), D(S_Gamzzz),
D(S_Rxx), D(S_Rxy), D(S_Rxz),
D(S_Ryy), D(S_Ryz), D(S_Rzz));
MARK_RHS_STAGE(RHS_STAGE_CHI);
kern_phase15_trK_Aij_gauge<<<grid(all),BLK>>>(
D(S_alpn1), D(S_chin1),
@@ -4936,8 +5043,10 @@ static void launch_rhs_pipeline(int all, double eps, int co)
D(S_betax_rhs), D(S_betay_rhs), D(S_betaz_rhs),
D(S_Gamx_rhs), D(S_Gamy_rhs), D(S_Gamz_rhs),
D(S_f_arr), D(S_S_arr));
MARK_RHS_STAGE(RHS_STAGE_GAUGE_RHS);
gpu_lopsided_kodis_state_batch(eps, all);
MARK_RHS_STAGE(RHS_STAGE_KODIS);
if (co == 0) {
{
@@ -4982,7 +5091,10 @@ static void launch_rhs_pipeline(int all, double eps, int co)
D(S_gzzx), D(S_gzzy), D(S_gzzz),
D(S_ham_Res), D(S_movx_Res), D(S_movy_Res), D(S_movz_Res));
}
MARK_RHS_STAGE(RHS_STAGE_CONSTRAINTS);
rhs_stage_profile_accumulate(stage_ms);
#undef MARK_RHS_STAGE
#undef D
}
@@ -5196,6 +5308,21 @@ static void download_resident_state(void *block_tag, int *ex, double **state_hos
const size_t all = (size_t)ex[0] * ex[1] * ex[2];
const size_t bytes = all * sizeof(double);
StepContext &ctx = ensure_step_ctx(block_tag, all);
static int direct_download = -1;
if (direct_download < 0) {
const char *env = getenv("AMSS_CUDA_DIRECT_STATE_DOWNLOAD");
const char *pin_env = getenv("AMSS_CUDA_PIN_GRIDFUNCS");
direct_download = env ? ((atoi(env) != 0) ? 1 : 0)
: ((pin_env && atoi(pin_env) != 0) ? 1 : 0);
}
if (direct_download) {
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
CUDA_CHECK(cudaMemcpyAsync(state_host_out[i], ctx.d_state_curr[i],
bytes, cudaMemcpyDeviceToHost));
}
CUDA_CHECK(cudaDeviceSynchronize());
return;
}
CUDA_CHECK(cudaMemcpy(g_buf.h_stage, ctx.d_state_curr_mem,
(size_t)BSSN_STATE_COUNT * bytes,
cudaMemcpyDeviceToHost));
@@ -5902,6 +6029,67 @@ int bssn_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
return 0;
}
static void copy_state_device_batch(void *block_tag,
int state_count,
double *device_buffer,
const int *ex,
int i0, int j0, int k0,
int sx, int sy, int sz,
int pack_not_unpack)
{
if (state_count <= 0 || state_count > BSSN_STATE_COUNT) return;
if (sx <= 0 || sy <= 0 || sz <= 0) return;
StepContext &ctx = ensure_step_ctx(block_tag, (size_t)ex[0] * ex[1] * ex[2]);
const int region_all = sx * sy * sz;
dim3 launch_grid((unsigned int)grid((size_t)region_all),
(unsigned int)state_count);
if (pack_not_unpack) {
kern_pack_state_region_batch<<<launch_grid, BLK>>>(
ctx.d_state_curr_mem, device_buffer,
ex[0], ex[1], i0, j0, k0, sx, sy, sz,
region_all, state_count,
ex[0] * ex[1] * ex[2]);
} else {
kern_unpack_state_region_batch<<<launch_grid, BLK>>>(
ctx.d_state_curr_mem, device_buffer,
ex[0], ex[1], i0, j0, k0, sx, sy, sz,
region_all, state_count,
ex[0] * ex[1] * ex[2]);
}
}
extern "C"
int bssn_cuda_pack_state_batch_to_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
int *ex,
int i0, int j0, int k0,
int sx, int sy, int sz)
{
init_gpu_dispatch();
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
copy_state_device_batch(block_tag, state_count, device_buffer, ex,
i0, j0, k0, sx, sy, sz, 1);
return 0;
}
extern "C"
int bssn_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
int *ex,
int i0, int j0, int k0,
int sx, int sy, int sz)
{
init_gpu_dispatch();
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
copy_state_device_batch(block_tag, state_count, device_buffer, ex,
i0, j0, k0, sx, sy, sz, 0);
return 0;
}
extern "C"
int bssn_cuda_download_state_subset(void *block_tag,
int *ex,

14
AMSS_NCKU_source/bssn_rhs_cuda.h
View File

@@ -104,6 +104,20 @@ int bssn_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_pack_state_batch_to_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
int *ex,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
int state_count,
double *device_buffer,
int *ex,
int i0, int j0, int k0,
int sx, int sy, int sz);
int bssn_cuda_download_state_subset(void *block_tag,
int *ex,
int subset_count,