7910 lines
356 KiB
Plaintext
7910 lines
356 KiB
Plaintext
/*
|
|
* z4c_rhs_cuda.cu - CUDA implementation of the main Cartesian Z4C RHS.
|
|
*
|
|
* This is a drop-in replacement for z4c_rhs_c.C when USE_CUDA_Z4C=1.
|
|
*/
|
|
|
|
#include <array>
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cmath>
|
|
#include <cstring>
|
|
#include <unordered_map>
|
|
#include <vector>
|
|
#include <cuda_runtime.h>
|
|
#include "macrodef.h"
|
|
#include "bssn_rhs.h"
|
|
#include "ricci_gamma.h"
|
|
#include "z4c_rhs_cuda.h"
|
|
|
|
#ifdef fortran1
|
|
#define f_constraint_bssn constraint_bssn
|
|
#endif
|
|
#ifdef fortran2
|
|
#define f_constraint_bssn CONSTRAINT_BSSN
|
|
#endif
|
|
#ifdef fortran3
|
|
#define f_constraint_bssn constraint_bssn_
|
|
#endif
|
|
|
|
namespace z4c_cuda {
|
|
|
|
/* ------------------------------------------------------------------ */
|
|
/* Multi-GPU dispatch: distribute ranks across available GPUs */
|
|
/* ------------------------------------------------------------------ */
|
|
static struct {
|
|
int num_gpus;
|
|
int my_rank;
|
|
int my_local_rank;
|
|
int my_device;
|
|
bool inited;
|
|
} g_dispatch = {0, -1, -1, -1, false};
|
|
|
|
static int env_to_int(const char *name, int fallback = -1) {
|
|
const char *v = getenv(name);
|
|
if (!v || !*v) return fallback;
|
|
return atoi(v);
|
|
}
|
|
|
|
static void init_gpu_dispatch() {
|
|
if (g_dispatch.inited) return;
|
|
cudaError_t err = cudaGetDeviceCount(&g_dispatch.num_gpus);
|
|
if (err != cudaSuccess) g_dispatch.num_gpus = 1;
|
|
if (g_dispatch.num_gpus < 1) g_dispatch.num_gpus = 1;
|
|
|
|
/* Get MPI rank from environment (set by mpirun/mpiexec). */
|
|
g_dispatch.my_rank = env_to_int("PMI_RANK",
|
|
env_to_int("OMPI_COMM_WORLD_RANK",
|
|
env_to_int("MV2_COMM_WORLD_RANK",
|
|
env_to_int("SLURM_PROCID", 0))));
|
|
|
|
/* Prefer local rank for per-node GPU mapping (avoids cross-node skew). */
|
|
g_dispatch.my_local_rank = env_to_int("OMPI_COMM_WORLD_LOCAL_RANK",
|
|
env_to_int("MV2_COMM_WORLD_LOCAL_RANK",
|
|
env_to_int("MPI_LOCALRANKID",
|
|
env_to_int("SLURM_LOCALID", -1))));
|
|
|
|
const int rank_for_map = (g_dispatch.my_local_rank >= 0)
|
|
? g_dispatch.my_local_rank : g_dispatch.my_rank;
|
|
g_dispatch.my_device = rank_for_map % g_dispatch.num_gpus;
|
|
cudaSetDevice(g_dispatch.my_device);
|
|
|
|
if (g_dispatch.my_rank == 0) {
|
|
printf("[AMSS-GPU] %d GPU(s) detected, device map uses %s rank\n",
|
|
g_dispatch.num_gpus,
|
|
(g_dispatch.my_local_rank >= 0) ? "local" : "global");
|
|
}
|
|
g_dispatch.inited = true;
|
|
}
|
|
|
|
struct CudaProfileStats {
|
|
long long calls;
|
|
double total_ms;
|
|
double state_ms;
|
|
double matter_ms;
|
|
double rhs_ms;
|
|
double bc_ms;
|
|
double finalize_ms;
|
|
double output_ms;
|
|
};
|
|
|
|
static CudaProfileStats &cuda_profile_stats() {
|
|
static CudaProfileStats stats = {0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0};
|
|
return stats;
|
|
}
|
|
|
|
static bool cuda_profile_enabled() {
|
|
static int enabled = -1;
|
|
if (enabled < 0) {
|
|
const char *env = getenv("AMSS_PROFILE_CUDA");
|
|
enabled = (env && atoi(env) != 0) ? 1 : 0;
|
|
}
|
|
return enabled != 0;
|
|
}
|
|
|
|
static int cuda_profile_every() {
|
|
static int every = -1;
|
|
if (every < 0) {
|
|
const char *env = getenv("AMSS_PROFILE_CUDA_EVERY");
|
|
every = (env && atoi(env) > 0) ? atoi(env) : 100;
|
|
}
|
|
return every;
|
|
}
|
|
|
|
static double cuda_profile_now_ms() {
|
|
using clock = std::chrono::steady_clock;
|
|
return std::chrono::duration<double, std::milli>(
|
|
clock::now().time_since_epoch()).count();
|
|
}
|
|
|
|
static void cuda_profile_sync() {
|
|
cudaError_t err = cudaDeviceSynchronize();
|
|
if (err != cudaSuccess) {
|
|
fprintf(stderr, "CUDA error %s:%d: %s\n",
|
|
__FILE__, __LINE__, cudaGetErrorString(err));
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
|
|
static void cuda_profile_maybe_log() {
|
|
if (!cuda_profile_enabled()) return;
|
|
CudaProfileStats &stats = cuda_profile_stats();
|
|
if (stats.calls <= 0 || stats.calls % cuda_profile_every() != 0) return;
|
|
fprintf(stderr,
|
|
"[AMSS-CUDA][rank %d][dev %d] calls=%lld avg_total=%.3f ms avg_state=%.3f ms avg_matter=%.3f ms avg_rhs=%.3f ms avg_bc=%.3f ms avg_finalize=%.3f ms avg_output=%.3f ms\n",
|
|
g_dispatch.my_rank, g_dispatch.my_device, stats.calls,
|
|
stats.total_ms / (double)stats.calls,
|
|
stats.state_ms / (double)stats.calls,
|
|
stats.matter_ms / (double)stats.calls,
|
|
stats.rhs_ms / (double)stats.calls,
|
|
stats.bc_ms / (double)stats.calls,
|
|
stats.finalize_ms / (double)stats.calls,
|
|
stats.output_ms / (double)stats.calls);
|
|
fflush(stderr);
|
|
}
|
|
|
|
/* ------------------------------------------------------------------ */
|
|
/* Error checking */
|
|
/* ------------------------------------------------------------------ */
|
|
#define CUDA_CHECK(call) do { \
|
|
cudaError_t err = (call); \
|
|
if (err != cudaSuccess) { \
|
|
fprintf(stderr, "CUDA error %s:%d: %s\n", \
|
|
__FILE__, __LINE__, cudaGetErrorString(err)); \
|
|
exit(EXIT_FAILURE); \
|
|
} \
|
|
} while(0)
|
|
|
|
/* ------------------------------------------------------------------ */
|
|
/* Physical / gauge constants (matching bssn_rhs_c.C) */
|
|
/* ------------------------------------------------------------------ */
|
|
static const double PI_VAL = 3.14159265358979323846;
|
|
static const double FF_VAL = 0.75;
|
|
static const double ETA_VAL = 2.0;
|
|
|
|
/* ------------------------------------------------------------------ */
|
|
/* Constant memory for grid parameters and stencil coefficients */
|
|
/* ------------------------------------------------------------------ */
|
|
struct GridParams {
|
|
int ex[3]; /* nx, ny, nz */
|
|
int all; /* nx*ny*nz */
|
|
double dX, dY, dZ;
|
|
/* fderivs coefficients */
|
|
double d12dx, d12dy, d12dz; /* 1/(12*dX) etc */
|
|
double d2dx, d2dy, d2dz; /* 1/(2*dX) etc */
|
|
/* fdderivs coefficients */
|
|
double Fdxdx, Fdydy, Fdzdz; /* 1/(12*dX^2) etc */
|
|
double Sdxdx, Sdydy, Sdzdz; /* 1/(dX^2) etc */
|
|
double Fdxdy, Fdxdz, Fdydz; /* 1/(144*dX*dY) etc */
|
|
double Sdxdy, Sdxdz, Sdydz; /* 1/(4*dX*dY) etc */
|
|
/* symmetry bounds (Fortran 1-based) */
|
|
int iminF, jminF, kminF;
|
|
int imaxF, jmaxF, kmaxF;
|
|
/* symmetry bounds for ord=3 (lopsided/kodis) */
|
|
int iminF3, jminF3, kminF3;
|
|
int Symmetry;
|
|
double eps;
|
|
int co;
|
|
/* padded sizes */
|
|
int fh2_nx, fh2_ny, fh2_nz; /* (nx+2), (ny+2), (nz+2) for ord=2 */
|
|
int fh3_nx, fh3_ny, fh3_nz; /* (nx+3), (ny+3), (nz+3) for ord=3 */
|
|
};
|
|
|
|
__constant__ GridParams d_gp;
|
|
static GridParams g_gp_host_cache = {};
|
|
static bool g_gp_host_cache_valid = false;
|
|
|
|
/* ------------------------------------------------------------------ */
|
|
/* Device indexing helpers */
|
|
/* ------------------------------------------------------------------ */
|
|
__device__ __forceinline__ int idx_ex_d(int i0, int j0, int k0) {
|
|
return i0 + j0 * d_gp.ex[0] + k0 * d_gp.ex[0] * d_gp.ex[1];
|
|
}
|
|
|
|
__device__ __forceinline__ double fetch_sym_ord2_direct(const double *src,
|
|
int iF, int jF, int kF,
|
|
int SoA0, int SoA1, int SoA2)
|
|
{
|
|
int siF = iF;
|
|
int sjF = jF;
|
|
int skF = kF;
|
|
double sign = 1.0;
|
|
|
|
if (iF <= 0) {
|
|
siF = 1 - iF;
|
|
sign *= (double)SoA0;
|
|
}
|
|
if (jF <= 0) {
|
|
sjF = 1 - jF;
|
|
sign *= (double)SoA1;
|
|
}
|
|
if (kF <= 0) {
|
|
skF = 1 - kF;
|
|
sign *= (double)SoA2;
|
|
}
|
|
|
|
return sign * src[(siF - 1)
|
|
+ (sjF - 1) * d_gp.ex[0]
|
|
+ (skF - 1) * d_gp.ex[0] * d_gp.ex[1]];
|
|
}
|
|
|
|
/* ord=2 ghost-padded: Fortran index iF -> flat index */
|
|
__device__ __forceinline__ int idx_fh2(int iF, int jF, int kF) {
|
|
return (iF + 1) + (jF + 1) * d_gp.fh2_nx + (kF + 1) * d_gp.fh2_nx * d_gp.fh2_ny;
|
|
}
|
|
|
|
/* ord=3 ghost-padded: Fortran index iF -> flat index */
|
|
__device__ __forceinline__ int idx_fh3(int iF, int jF, int kF) {
|
|
return (iF + 2) + (jF + 2) * d_gp.fh3_nx + (kF + 2) * d_gp.fh3_nx * d_gp.fh3_ny;
|
|
}
|
|
|
|
__device__ __forceinline__ double fetch_sym_ord3_direct(const double *src,
|
|
int iF, int jF, int kF,
|
|
int SoA0, int SoA1, int SoA2)
|
|
{
|
|
int siF = iF;
|
|
int sjF = jF;
|
|
int skF = kF;
|
|
double sign = 1.0;
|
|
|
|
if (iF <= 0) {
|
|
siF = 1 - iF;
|
|
sign *= (double)SoA0;
|
|
}
|
|
if (jF <= 0) {
|
|
sjF = 1 - jF;
|
|
sign *= (double)SoA1;
|
|
}
|
|
if (kF <= 0) {
|
|
skF = 1 - kF;
|
|
sign *= (double)SoA2;
|
|
}
|
|
|
|
return sign * src[(siF - 1)
|
|
+ (sjF - 1) * d_gp.ex[0]
|
|
+ (skF - 1) * d_gp.ex[0] * d_gp.ex[1]];
|
|
}
|
|
|
|
#include "fd_cuda_helpers.cuh"
|
|
|
|
/* ------------------------------------------------------------------ */
|
|
/* GPU buffer management */
|
|
/* ------------------------------------------------------------------ */
|
|
/*
|
|
* Array slot indices — all arrays live in one big cudaMalloc block.
|
|
* INPUT arrays (H2D): 39 slots
|
|
* OUTPUT arrays (D2H): 52 slots
|
|
* TEMPORARY arrays (GPU-only): ~65 slots
|
|
* Plus 2 extended arrays for ghost-padded stencils (fh_ord2, fh_ord3)
|
|
*/
|
|
|
|
/* Total number of "all"-sized slots */
|
|
#define NUM_SLOTS 320
|
|
|
|
struct GpuBuffers {
|
|
double *d_mem; /* single big allocation */
|
|
double *d_fh2; /* ghost-padded ord=2: (nx+2)*(ny+2)*(nz+2) */
|
|
double *d_fh3; /* ghost-padded ord=3: (nx+3)*(ny+3)*(nz+3) */
|
|
double *h_stage; /* host staging buffer for bulk H2D/D2H */
|
|
bool h_stage_pinned; /* true if allocated by cudaMallocHost */
|
|
double *slot[NUM_SLOTS]; /* pointers into d_mem */
|
|
size_t cap_all;
|
|
size_t cap_fh2_size;
|
|
size_t cap_fh3_size;
|
|
int prev_nx, prev_ny, prev_nz;
|
|
bool initialized;
|
|
};
|
|
|
|
static GpuBuffers g_buf = {
|
|
nullptr, nullptr, nullptr, nullptr, false, {},
|
|
0, 0, 0, 0, 0, 0, false
|
|
};
|
|
|
|
/* Slot assignments — INPUT (H2D) */
|
|
enum {
|
|
S_chi=0, S_trK, S_dxx, S_gxy, S_gxz, S_dyy, S_gyz, S_dzz,
|
|
S_Axx, S_Axy, S_Axz, S_Ayy, S_Ayz, S_Azz,
|
|
S_Gamx, S_Gamy, S_Gamz,
|
|
S_Lap, S_betax, S_betay, S_betaz,
|
|
S_dtSfx, S_dtSfy, S_dtSfz, S_TZ,
|
|
S_rho, S_Sx, S_Sy, S_Sz,
|
|
S_Sxx, S_Sxy, S_Sxz, S_Syy, S_Syz, S_Szz,
|
|
S_X, S_Y, S_Z, /* coordinate arrays — only nx/ny/nz long */
|
|
/* 37 input slots so far; X/Y/Z are special-sized */
|
|
|
|
/* OUTPUT (D2H) */
|
|
S_chi_rhs, S_trK_rhs,
|
|
S_gxx_rhs, S_gxy_rhs, S_gxz_rhs, S_gyy_rhs, S_gyz_rhs, S_gzz_rhs,
|
|
S_Axx_rhs, S_Axy_rhs, S_Axz_rhs, S_Ayy_rhs, S_Ayz_rhs, S_Azz_rhs,
|
|
S_Gamx_rhs, S_Gamy_rhs, S_Gamz_rhs,
|
|
S_Lap_rhs, S_betax_rhs, S_betay_rhs, S_betaz_rhs,
|
|
S_dtSfx_rhs, S_dtSfy_rhs, S_dtSfz_rhs, S_TZ_rhs,
|
|
S_Gamxxx, S_Gamxxy, S_Gamxxz, S_Gamxyy, S_Gamxyz, S_Gamxzz,
|
|
S_Gamyxx, S_Gamyxy, S_Gamyxz, S_Gamyyy, S_Gamyyz, S_Gamyzz,
|
|
S_Gamzxx, S_Gamzxy, S_Gamzxz, S_Gamzyy, S_Gamzyz, S_Gamzzz,
|
|
S_Rxx, S_Rxy, S_Rxz, S_Ryy, S_Ryz, S_Rzz,
|
|
S_ham_Res, S_movx_Res, S_movy_Res, S_movz_Res,
|
|
S_Gmx_Res, S_Gmy_Res, S_Gmz_Res,
|
|
|
|
/* TEMPORARY (GPU-only) */
|
|
S_gxx, S_gyy, S_gzz, /* physical metric = dxx+1 etc */
|
|
S_alpn1, S_chin1,
|
|
S_chix, S_chiy, S_chiz,
|
|
S_gxxx, S_gxyx, S_gxzx, S_gyyx, S_gyzx, S_gzzx,
|
|
S_gxxy, S_gxyy, S_gxzy, S_gyyy, S_gyzy, S_gzzy,
|
|
S_gxxz, S_gxyz, S_gxzz, S_gyyz, S_gyzz, S_gzzz,
|
|
S_Lapx, S_Lapy, S_Lapz,
|
|
S_betaxx, S_betaxy, S_betaxz,
|
|
S_betayx, S_betayy, S_betayz,
|
|
S_betazx, S_betazy, S_betazz,
|
|
S_Gamxx, S_Gamxy, S_Gamxz,
|
|
S_Gamyx, S_Gamyy_t, S_Gamyz_t,
|
|
S_Gamzx, S_Gamzy, S_Gamzz_t,
|
|
S_Kx, S_Ky, S_Kz,
|
|
S_S_arr, S_f_arr,
|
|
S_fxx, S_fxy, S_fxz, S_fyy, S_fyz, S_fzz,
|
|
S_Gamxa, S_Gamya, S_Gamza,
|
|
S_gupxx, S_gupxy, S_gupxz,
|
|
S_gupyy, S_gupyz, S_gupzz,
|
|
|
|
S_dBxx, S_dBxy, S_dBxz, S_dByx, S_dByy, S_dByz, S_dBzx, S_dBzy, S_dBzz,
|
|
S_TZx, S_TZy, S_TZz,
|
|
S_Axxx, S_Axxy, S_Axxz, S_Axyx, S_Axyy, S_Axyz,
|
|
S_Axzx, S_Axzy, S_Axzz, S_Ayyx, S_Ayyy, S_Ayyz,
|
|
S_Ayzx, S_Ayzy, S_Ayzz, S_Azzx, S_Azzy, S_Azzz,
|
|
S_chixx, S_chixy, S_chixz, S_chiyy, S_chiyz, S_chizz,
|
|
S_Lapxx, S_Lapxy, S_Lapxz, S_Lapyy, S_Lapyz, S_Lapzz,
|
|
S_gxxxx, S_gxxxy, S_gxxxz, S_gxxyy, S_gxxyz, S_gxxzz,
|
|
S_gxyxx, S_gxyxy, S_gxyxz, S_gxyyy, S_gxyyz, S_gxyzz,
|
|
S_gxzxx, S_gxzxy, S_gxzxz, S_gxzyy, S_gxzyz, S_gxzzz,
|
|
S_gyyxx, S_gyyxy, S_gyyxz, S_gyyyy, S_gyyyz, S_gyyzz,
|
|
S_gyzxx, S_gyzxy, S_gyzxz, S_gyzyy, S_gyzyz, S_gyzzz,
|
|
S_gzzxx, S_gzzxy, S_gzzxz, S_gzzyy, S_gzzyz, S_gzzzz,
|
|
S_sfxxx, S_sfxxy, S_sfxxz, S_sfxyy, S_sfxyz, S_sfxzz,
|
|
S_sfyxx, S_sfyxy, S_sfyxz, S_sfyyy, S_sfyyz, S_sfyzz,
|
|
S_sfzxx, S_sfzxy, S_sfzxz, S_sfzyy, S_sfzyz, S_sfzzz,
|
|
NUM_USED_SLOTS
|
|
};
|
|
|
|
static_assert(NUM_USED_SLOTS <= NUM_SLOTS, "Increase NUM_SLOTS");
|
|
|
|
static const int H2D_INPUT_SLOT_COUNT = (S_Szz - S_chi + 1);
|
|
static const int D2H_BASE_SLOT_COUNT = (S_Rzz - S_chi_rhs + 1);
|
|
static const int D2H_CONSTRAINT_SLOT_COUNT = (S_Gmz_Res - S_ham_Res + 1);
|
|
static const int STAGE_SLOT_COUNT =
|
|
(H2D_INPUT_SLOT_COUNT > (D2H_BASE_SLOT_COUNT + D2H_CONSTRAINT_SLOT_COUNT))
|
|
? H2D_INPUT_SLOT_COUNT
|
|
: (D2H_BASE_SLOT_COUNT + D2H_CONSTRAINT_SLOT_COUNT);
|
|
|
|
static constexpr int BSSN_STATE_COUNT = Z4C_CUDA_STATE_COUNT;
|
|
static constexpr int BSSN_MATTER_COUNT = 10;
|
|
static constexpr int BSSN_LK_FIELD_COUNT = 24;
|
|
|
|
static const int k_state_input_slots[BSSN_STATE_COUNT] = {
|
|
S_chi, S_trK, S_dxx, S_gxy, S_gxz, S_dyy, S_gyz, S_dzz,
|
|
S_Axx, S_Axy, S_Axz, S_Ayy, S_Ayz, S_Azz,
|
|
S_Gamx, S_Gamy, S_Gamz,
|
|
S_Lap, S_betax, S_betay, S_betaz,
|
|
S_dtSfx, S_dtSfy, S_dtSfz, S_TZ
|
|
};
|
|
|
|
static const int k_state_rhs_slots[BSSN_STATE_COUNT] = {
|
|
S_chi_rhs, S_trK_rhs,
|
|
S_gxx_rhs, S_gxy_rhs, S_gxz_rhs, S_gyy_rhs, S_gyz_rhs, S_gzz_rhs,
|
|
S_Axx_rhs, S_Axy_rhs, S_Axz_rhs, S_Ayy_rhs, S_Ayz_rhs, S_Azz_rhs,
|
|
S_Gamx_rhs, S_Gamy_rhs, S_Gamz_rhs,
|
|
S_Lap_rhs, S_betax_rhs, S_betay_rhs, S_betaz_rhs,
|
|
S_dtSfx_rhs, S_dtSfy_rhs, S_dtSfz_rhs, S_TZ_rhs
|
|
};
|
|
|
|
static const int k_matter_slots[BSSN_MATTER_COUNT] = {
|
|
S_rho, S_Sx, S_Sy, S_Sz, S_Sxx, S_Sxy, S_Sxz, S_Syy, S_Syz, S_Szz
|
|
};
|
|
|
|
static const int k_lk_adv_slots[BSSN_LK_FIELD_COUNT] = {
|
|
S_gxx, S_Gamz, S_gxy, S_Lap, S_gxz, S_betax, S_gyy, S_betay,
|
|
S_gyz, S_betaz, S_gzz, S_dtSfx, S_Axx, S_dtSfy, S_Axy, S_dtSfz,
|
|
S_Axz, S_Ayy, S_Ayz, S_Azz, S_chi, S_trK, S_Gamx, S_Gamy
|
|
};
|
|
|
|
static const int k_lk_ko_slots[BSSN_LK_FIELD_COUNT] = {
|
|
S_dxx, S_Gamz, S_gxy, S_Lap, S_gxz, S_betax, S_dyy, S_betay,
|
|
S_gyz, S_betaz, S_dzz, S_dtSfx, S_Axx, S_dtSfy, S_Axy, S_dtSfz,
|
|
S_Axz, S_Ayy, S_Ayz, S_Azz, S_chi, S_trK, S_Gamx, S_Gamy
|
|
};
|
|
|
|
static const int k_lk_rhs_slots[BSSN_LK_FIELD_COUNT] = {
|
|
S_gxx_rhs, S_Gamz_rhs, S_gxy_rhs, S_Lap_rhs, S_gxz_rhs, S_betax_rhs,
|
|
S_gyy_rhs, S_betay_rhs, S_gyz_rhs, S_betaz_rhs, S_gzz_rhs, S_dtSfx_rhs,
|
|
S_Axx_rhs, S_dtSfy_rhs, S_Axy_rhs, S_dtSfz_rhs, S_Axz_rhs, S_Ayy_rhs,
|
|
S_Ayz_rhs, S_Azz_rhs, S_chi_rhs, S_trK_rhs, S_Gamx_rhs, S_Gamy_rhs
|
|
};
|
|
|
|
__constant__ int d_subset_state_indices[BSSN_STATE_COUNT];
|
|
__constant__ double d_comm_state_soa[3 * BSSN_STATE_COUNT];
|
|
|
|
static const int k_lk_soa_signs[3 * BSSN_LK_FIELD_COUNT] = {
|
|
1, 1, 1,
|
|
1, 1, -1,
|
|
-1, -1, 1,
|
|
1, 1, 1,
|
|
-1, 1, -1,
|
|
-1, 1, 1,
|
|
1, 1, 1,
|
|
1, -1, 1,
|
|
1, -1, -1,
|
|
1, 1, -1,
|
|
1, 1, 1,
|
|
-1, 1, 1,
|
|
1, 1, 1,
|
|
1, -1, 1,
|
|
-1, -1, 1,
|
|
1, 1, -1,
|
|
-1, 1, -1,
|
|
1, 1, 1,
|
|
1, -1, -1,
|
|
1, 1, 1,
|
|
1, 1, 1,
|
|
1, 1, 1,
|
|
-1, 1, 1,
|
|
1, -1, 1
|
|
};
|
|
|
|
struct StepContext {
|
|
double *d_state0_mem;
|
|
double *d_accum_mem;
|
|
double *d_state_curr_mem;
|
|
double *d_state_next_mem;
|
|
double *d_matter_mem;
|
|
double *d_comm_mem;
|
|
double *h_comm_mem;
|
|
std::array<double *, BSSN_STATE_COUNT> d_state0;
|
|
std::array<double *, BSSN_STATE_COUNT> d_accum;
|
|
std::array<double *, BSSN_STATE_COUNT> d_state_curr;
|
|
std::array<double *, BSSN_STATE_COUNT> d_state_next;
|
|
std::array<double *, BSSN_MATTER_COUNT> d_matter;
|
|
size_t cap_all;
|
|
size_t cap_comm;
|
|
bool h_comm_pinned;
|
|
size_t cap_h_comm;
|
|
bool matter_ready;
|
|
bool state_ready;
|
|
|
|
StepContext()
|
|
: d_state0_mem(nullptr), d_accum_mem(nullptr),
|
|
d_state_curr_mem(nullptr), d_state_next_mem(nullptr),
|
|
d_matter_mem(nullptr), d_comm_mem(nullptr), h_comm_mem(nullptr),
|
|
cap_all(0), cap_comm(0), h_comm_pinned(false), cap_h_comm(0),
|
|
matter_ready(false), state_ready(false)
|
|
{
|
|
d_state0.fill(nullptr);
|
|
d_accum.fill(nullptr);
|
|
d_state_curr.fill(nullptr);
|
|
d_state_next.fill(nullptr);
|
|
d_matter.fill(nullptr);
|
|
}
|
|
};
|
|
|
|
struct StepAllocation {
|
|
double *d_state0_mem;
|
|
double *d_accum_mem;
|
|
double *d_state_curr_mem;
|
|
double *d_state_next_mem;
|
|
double *d_matter_mem;
|
|
double *d_comm_mem;
|
|
double *h_comm_mem;
|
|
size_t cap_all;
|
|
size_t cap_comm;
|
|
bool h_comm_pinned;
|
|
size_t cap_h_comm;
|
|
};
|
|
|
|
static std::unordered_map<void *, StepContext> g_step_ctx;
|
|
static std::vector<StepAllocation> g_step_pool;
|
|
|
|
static StepAllocation empty_step_allocation()
|
|
{
|
|
StepAllocation alloc = {
|
|
nullptr, nullptr, nullptr, nullptr, nullptr,
|
|
nullptr, nullptr,
|
|
0, 0, false, 0
|
|
};
|
|
return alloc;
|
|
}
|
|
|
|
static bool has_step_allocation(const StepAllocation &alloc)
|
|
{
|
|
return alloc.cap_all != 0;
|
|
}
|
|
|
|
static StepAllocation detach_step_allocation(StepContext &ctx)
|
|
{
|
|
StepAllocation alloc = {
|
|
ctx.d_state0_mem, ctx.d_accum_mem, ctx.d_state_curr_mem,
|
|
ctx.d_state_next_mem, ctx.d_matter_mem,
|
|
ctx.d_comm_mem, ctx.h_comm_mem,
|
|
ctx.cap_all, ctx.cap_comm, ctx.h_comm_pinned, ctx.cap_h_comm
|
|
};
|
|
ctx.d_state0_mem = nullptr;
|
|
ctx.d_accum_mem = nullptr;
|
|
ctx.d_state_curr_mem = nullptr;
|
|
ctx.d_state_next_mem = nullptr;
|
|
ctx.d_matter_mem = nullptr;
|
|
ctx.d_comm_mem = nullptr;
|
|
ctx.h_comm_mem = nullptr;
|
|
ctx.cap_all = 0;
|
|
ctx.cap_comm = 0;
|
|
ctx.h_comm_pinned = false;
|
|
ctx.cap_h_comm = 0;
|
|
ctx.matter_ready = false;
|
|
ctx.state_ready = false;
|
|
ctx.d_state0.fill(nullptr);
|
|
ctx.d_accum.fill(nullptr);
|
|
ctx.d_state_curr.fill(nullptr);
|
|
ctx.d_state_next.fill(nullptr);
|
|
ctx.d_matter.fill(nullptr);
|
|
return alloc;
|
|
}
|
|
|
|
static void attach_step_allocation(StepContext &ctx, const StepAllocation &alloc)
|
|
{
|
|
ctx.d_state0_mem = alloc.d_state0_mem;
|
|
ctx.d_accum_mem = alloc.d_accum_mem;
|
|
ctx.d_state_curr_mem = alloc.d_state_curr_mem;
|
|
ctx.d_state_next_mem = alloc.d_state_next_mem;
|
|
ctx.d_matter_mem = alloc.d_matter_mem;
|
|
ctx.d_comm_mem = alloc.d_comm_mem;
|
|
ctx.h_comm_mem = alloc.h_comm_mem;
|
|
ctx.cap_all = alloc.cap_all;
|
|
ctx.cap_comm = alloc.cap_comm;
|
|
ctx.h_comm_pinned = alloc.h_comm_pinned;
|
|
ctx.cap_h_comm = alloc.cap_h_comm;
|
|
ctx.matter_ready = false;
|
|
ctx.state_ready = false;
|
|
}
|
|
|
|
static void recycle_step_allocation(StepAllocation &alloc)
|
|
{
|
|
if (!has_step_allocation(alloc)) return;
|
|
g_step_pool.push_back(alloc);
|
|
alloc = empty_step_allocation();
|
|
}
|
|
|
|
static StepAllocation acquire_step_allocation(size_t all)
|
|
{
|
|
size_t best = g_step_pool.size();
|
|
for (size_t i = 0; i < g_step_pool.size(); ++i) {
|
|
if (g_step_pool[i].cap_all < all) continue;
|
|
if (best == g_step_pool.size() || g_step_pool[i].cap_all < g_step_pool[best].cap_all)
|
|
best = i;
|
|
}
|
|
if (best == g_step_pool.size())
|
|
return empty_step_allocation();
|
|
|
|
StepAllocation alloc = g_step_pool[best];
|
|
g_step_pool[best] = g_step_pool.back();
|
|
g_step_pool.pop_back();
|
|
return alloc;
|
|
}
|
|
|
|
static void ensure_gpu_buffers(int nx, int ny, int nz) {
|
|
size_t all = (size_t)nx * ny * nz;
|
|
size_t fh2_size = (size_t)(nx+2) * (ny+2) * (nz+2);
|
|
size_t fh3_size = (size_t)(nx+3) * (ny+3) * (nz+3);
|
|
const bool need_grow = (!g_buf.initialized)
|
|
|| (all > g_buf.cap_all)
|
|
|| (fh2_size > g_buf.cap_fh2_size)
|
|
|| (fh3_size > g_buf.cap_fh3_size);
|
|
|
|
if (need_grow) {
|
|
if (g_buf.d_mem) { cudaFree(g_buf.d_mem); g_buf.d_mem = nullptr; }
|
|
if (g_buf.d_fh2) { cudaFree(g_buf.d_fh2); g_buf.d_fh2 = nullptr; }
|
|
if (g_buf.d_fh3) { cudaFree(g_buf.d_fh3); g_buf.d_fh3 = nullptr; }
|
|
if (g_buf.h_stage) {
|
|
if (g_buf.h_stage_pinned) cudaFreeHost(g_buf.h_stage);
|
|
else free(g_buf.h_stage);
|
|
g_buf.h_stage = nullptr;
|
|
g_buf.h_stage_pinned = false;
|
|
}
|
|
|
|
CUDA_CHECK(cudaMalloc(&g_buf.d_mem, NUM_USED_SLOTS * all * sizeof(double)));
|
|
CUDA_CHECK(cudaMalloc(&g_buf.d_fh2, fh2_size * sizeof(double)));
|
|
CUDA_CHECK(cudaMalloc(&g_buf.d_fh3, fh3_size * sizeof(double)));
|
|
|
|
const size_t stage_bytes = (size_t)STAGE_SLOT_COUNT * all * sizeof(double);
|
|
cudaError_t stage_err = cudaMallocHost((void**)&g_buf.h_stage, stage_bytes);
|
|
if (stage_err == cudaSuccess) {
|
|
g_buf.h_stage_pinned = true;
|
|
} else {
|
|
g_buf.h_stage = (double *)malloc(stage_bytes);
|
|
g_buf.h_stage_pinned = false;
|
|
if (!g_buf.h_stage) {
|
|
fprintf(stderr, "Host stage allocation failed (%zu bytes)\n", stage_bytes);
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
|
|
g_buf.cap_all = all;
|
|
g_buf.cap_fh2_size = fh2_size;
|
|
g_buf.cap_fh3_size = fh3_size;
|
|
g_buf.initialized = true;
|
|
}
|
|
|
|
for (int s = 0; s < NUM_USED_SLOTS; ++s)
|
|
g_buf.slot[s] = g_buf.d_mem + s * all;
|
|
|
|
g_buf.prev_nx = nx;
|
|
g_buf.prev_ny = ny;
|
|
g_buf.prev_nz = nz;
|
|
}
|
|
|
|
static StepContext &ensure_step_ctx(void *block_tag, size_t all)
|
|
{
|
|
StepContext &ctx = g_step_ctx[block_tag];
|
|
if (ctx.cap_all < all) {
|
|
StepAllocation old_alloc = detach_step_allocation(ctx);
|
|
recycle_step_allocation(old_alloc);
|
|
|
|
StepAllocation alloc = acquire_step_allocation(all);
|
|
if (!has_step_allocation(alloc)) {
|
|
CUDA_CHECK(cudaMalloc(&alloc.d_state0_mem, BSSN_STATE_COUNT * all * sizeof(double)));
|
|
CUDA_CHECK(cudaMalloc(&alloc.d_accum_mem, BSSN_STATE_COUNT * all * sizeof(double)));
|
|
CUDA_CHECK(cudaMalloc(&alloc.d_state_curr_mem, BSSN_STATE_COUNT * all * sizeof(double)));
|
|
CUDA_CHECK(cudaMalloc(&alloc.d_state_next_mem, BSSN_STATE_COUNT * all * sizeof(double)));
|
|
CUDA_CHECK(cudaMalloc(&alloc.d_matter_mem, BSSN_MATTER_COUNT * all * sizeof(double)));
|
|
alloc.cap_all = all;
|
|
}
|
|
attach_step_allocation(ctx, alloc);
|
|
}
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
ctx.d_state0[i] = ctx.d_state0_mem + (size_t)i * all;
|
|
ctx.d_accum[i] = ctx.d_accum_mem + (size_t)i * all;
|
|
ctx.d_state_curr[i] = ctx.d_state_curr_mem + (size_t)i * all;
|
|
ctx.d_state_next[i] = ctx.d_state_next_mem + (size_t)i * all;
|
|
}
|
|
for (int i = 0; i < BSSN_MATTER_COUNT; ++i) {
|
|
ctx.d_matter[i] = ctx.d_matter_mem + (size_t)i * all;
|
|
}
|
|
return ctx;
|
|
}
|
|
|
|
static void release_step_ctx(void *block_tag)
|
|
{
|
|
auto it = g_step_ctx.find(block_tag);
|
|
if (it == g_step_ctx.end()) return;
|
|
StepAllocation alloc = detach_step_allocation(it->second);
|
|
recycle_step_allocation(alloc);
|
|
g_step_ctx.erase(it);
|
|
}
|
|
|
|
static double *ensure_step_comm_buffer(StepContext &ctx, size_t needed_doubles)
|
|
{
|
|
if (needed_doubles == 0) return nullptr;
|
|
if (ctx.cap_comm < needed_doubles) {
|
|
if (ctx.d_comm_mem) {
|
|
CUDA_CHECK(cudaFree(ctx.d_comm_mem));
|
|
ctx.d_comm_mem = nullptr;
|
|
}
|
|
CUDA_CHECK(cudaMalloc(&ctx.d_comm_mem, needed_doubles * sizeof(double)));
|
|
ctx.cap_comm = needed_doubles;
|
|
}
|
|
return ctx.d_comm_mem;
|
|
}
|
|
|
|
static double *ensure_step_host_comm_buffer(StepContext &ctx, size_t needed_doubles)
|
|
{
|
|
if (needed_doubles == 0) return nullptr;
|
|
if (ctx.cap_h_comm < needed_doubles) {
|
|
if (ctx.h_comm_mem) {
|
|
if (ctx.h_comm_pinned) cudaFreeHost(ctx.h_comm_mem);
|
|
else free(ctx.h_comm_mem);
|
|
ctx.h_comm_mem = nullptr;
|
|
ctx.h_comm_pinned = false;
|
|
}
|
|
|
|
const size_t bytes = needed_doubles * sizeof(double);
|
|
cudaError_t err = cudaMallocHost((void **)&ctx.h_comm_mem, bytes);
|
|
if (err == cudaSuccess) {
|
|
ctx.h_comm_pinned = true;
|
|
} else {
|
|
ctx.h_comm_mem = (double *)malloc(bytes);
|
|
ctx.h_comm_pinned = false;
|
|
if (!ctx.h_comm_mem) {
|
|
fprintf(stderr, "Host comm allocation failed (%zu bytes)\n", bytes);
|
|
exit(EXIT_FAILURE);
|
|
}
|
|
}
|
|
ctx.cap_h_comm = needed_doubles;
|
|
}
|
|
return ctx.h_comm_mem;
|
|
}
|
|
|
|
static void upload_grid_params_if_needed(const GridParams &gp)
|
|
{
|
|
if (!g_gp_host_cache_valid ||
|
|
std::memcmp(&g_gp_host_cache, &gp, sizeof(GridParams)) != 0) {
|
|
CUDA_CHECK(cudaMemcpyToSymbol(d_gp, &gp, sizeof(GridParams)));
|
|
g_gp_host_cache = gp;
|
|
g_gp_host_cache_valid = true;
|
|
}
|
|
}
|
|
|
|
static void upload_comm_state_soa(const double *state_soa, int state_count)
|
|
{
|
|
double soa[3 * BSSN_STATE_COUNT];
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
soa[3 * i + 0] = 1.0;
|
|
soa[3 * i + 1] = 1.0;
|
|
soa[3 * i + 2] = 1.0;
|
|
}
|
|
if (state_soa) {
|
|
const int n = (state_count < BSSN_STATE_COUNT) ? state_count : BSSN_STATE_COUNT;
|
|
std::memcpy(soa, state_soa, (size_t)3 * n * sizeof(double));
|
|
}
|
|
CUDA_CHECK(cudaMemcpyToSymbol(d_comm_state_soa, soa, sizeof(soa)));
|
|
}
|
|
|
|
/* ================================================================== */
|
|
/* A. Symmetry boundary kernels (ord=2 and ord=3) */
|
|
/* ================================================================== */
|
|
|
|
/* Step 1: Copy interior into ghost-padded array */
|
|
__global__ void kern_symbd_copy_interior_ord2(const double * __restrict__ func,
|
|
double * __restrict__ fh,
|
|
double SoA0, double SoA1, double SoA2)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int fnx = d_gp.fh2_nx, fny = d_gp.fh2_ny;
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < d_gp.all;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int i0 = tid % nx;
|
|
int j0 = (tid / nx) % ny;
|
|
int k0 = tid / (nx * ny);
|
|
int iF = i0 + 1, jF = j0 + 1, kF = k0 + 1;
|
|
fh[(iF+1) + (jF+1)*fnx + (kF+1)*fnx*fny] = func[tid];
|
|
}
|
|
}
|
|
|
|
/* Fused symmetry pack (ord=2): fill full fh from interior func in one pass. */
|
|
__global__ void kern_symbd_pack_ord2(const double * __restrict__ func,
|
|
double * __restrict__ fh,
|
|
double SoA0, double SoA1, double SoA2)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1];
|
|
const int fnx = d_gp.fh2_nx, fny = d_gp.fh2_ny, fnz = d_gp.fh2_nz;
|
|
const int total = fnx * fny * fnz;
|
|
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < total;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int ii = tid % fnx;
|
|
int jj = (tid / fnx) % fny;
|
|
int kk = tid / (fnx * fny);
|
|
|
|
int iF = ii - 1; /* -1 .. nx */
|
|
int jF = jj - 1; /* -1 .. ny */
|
|
int kF = kk - 1; /* -1 .. nz */
|
|
|
|
int siF = (iF <= 0) ? (1 - iF) : iF; /* 1..nx */
|
|
int sjF = (jF <= 0) ? (1 - jF) : jF; /* 1..ny */
|
|
int skF = (kF <= 0) ? (1 - kF) : kF; /* 1..nz */
|
|
|
|
double sign = 1.0;
|
|
if (iF <= 0) sign *= SoA0;
|
|
if (jF <= 0) sign *= SoA1;
|
|
if (kF <= 0) sign *= SoA2;
|
|
|
|
int src = (siF - 1) + (sjF - 1) * nx + (skF - 1) * nx * ny;
|
|
fh[tid] = sign * func[src];
|
|
}
|
|
}
|
|
|
|
/* Step 2: Fill i-ghosts (x-direction symmetry) */
|
|
__global__ void kern_symbd_ighost_ord2(double * __restrict__ fh, double SoA0)
|
|
{
|
|
const int ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int fnx = d_gp.fh2_nx, fny = d_gp.fh2_ny;
|
|
/* ord=2: fill iF=0 and iF=-1, i.e. ghost layers ii=0 from ii=2, ii=1 from ii=1 */
|
|
/* Fortran: do ii=0,ord-1: funcc(-ii,jF,kF) = funcc(ii+1,jF,kF)*SoA[0] */
|
|
int total = ny * nz; /* jF=1..ny, kF=1..nz */
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < total * 2; /* 2 ghost layers */
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int ii = tid / total; /* 0 or 1 */
|
|
int rem = tid % total;
|
|
int j0 = rem % ny;
|
|
int k0 = rem / ny;
|
|
int jF = j0 + 1, kF = k0 + 1;
|
|
int iF_dst = -ii; /* 0, -1 */
|
|
int iF_src = ii + 1; /* 1, 2 */
|
|
fh[(iF_dst+1) + (jF+1)*fnx + (kF+1)*fnx*fny] =
|
|
fh[(iF_src+1) + (jF+1)*fnx + (kF+1)*fnx*fny] * SoA0;
|
|
}
|
|
}
|
|
|
|
/* Step 3: Fill j-ghosts (y-direction symmetry) */
|
|
__global__ void kern_symbd_jghost_ord2(double * __restrict__ fh, double SoA1)
|
|
{
|
|
const int nx = d_gp.ex[0], nz = d_gp.ex[2];
|
|
const int fnx = d_gp.fh2_nx, fny = d_gp.fh2_ny;
|
|
/* iF ranges from -1 to nx (i.e. -ord+1 to ex1), total = nx+2 */
|
|
int irange = nx + 2;
|
|
int total = irange * nz;
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < total * 2;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int jj = tid / total;
|
|
int rem = tid % total;
|
|
int ii = rem % irange;
|
|
int k0 = rem / irange;
|
|
int iF = ii - 1; /* -1 .. nx */
|
|
int kF = k0 + 1;
|
|
int jF_dst = -jj;
|
|
int jF_src = jj + 1;
|
|
fh[(iF+1) + (jF_dst+1)*fnx + (kF+1)*fnx*fny] =
|
|
fh[(iF+1) + (jF_src+1)*fnx + (kF+1)*fnx*fny] * SoA1;
|
|
}
|
|
}
|
|
|
|
/* Step 4: Fill k-ghosts (z-direction symmetry) */
|
|
__global__ void kern_symbd_kghost_ord2(double * __restrict__ fh, double SoA2)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1];
|
|
const int fnx = d_gp.fh2_nx, fny = d_gp.fh2_ny;
|
|
int irange = nx + 2;
|
|
int jrange = ny + 2;
|
|
int total = irange * jrange;
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < total * 2;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int kk = tid / total;
|
|
int rem = tid % total;
|
|
int ii = rem % irange;
|
|
int jj = rem / irange;
|
|
int iF = ii - 1;
|
|
int jF = jj - 1;
|
|
int kF_dst = -kk;
|
|
int kF_src = kk + 1;
|
|
fh[(iF+1) + (jF+1)*fnx + (kF_dst+1)*fnx*fny] =
|
|
fh[(iF+1) + (jF+1)*fnx + (kF_src+1)*fnx*fny] * SoA2;
|
|
}
|
|
}
|
|
|
|
/* ---- ord=3 variants (for lopsided / kodis) ---- */
|
|
|
|
__global__ void kern_symbd_copy_interior_ord3(const double * __restrict__ func,
|
|
double * __restrict__ fh)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int fnx = d_gp.fh3_nx, fny = d_gp.fh3_ny;
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < d_gp.all;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int i0 = tid % nx;
|
|
int j0 = (tid / nx) % ny;
|
|
int k0 = tid / (nx * ny);
|
|
int iF = i0 + 1, jF = j0 + 1, kF = k0 + 1;
|
|
fh[(iF+2) + (jF+2)*fnx + (kF+2)*fnx*fny] = func[tid];
|
|
}
|
|
}
|
|
|
|
/* Fused symmetry pack (ord=3): fill full fh from interior func in one pass. */
|
|
__global__ void kern_symbd_pack_ord3(const double * __restrict__ func,
|
|
double * __restrict__ fh,
|
|
double SoA0, double SoA1, double SoA2)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1];
|
|
const int fnx = d_gp.fh3_nx, fny = d_gp.fh3_ny, fnz = d_gp.fh3_nz;
|
|
const int total = fnx * fny * fnz;
|
|
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < total;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int ii = tid % fnx;
|
|
int jj = (tid / fnx) % fny;
|
|
int kk = tid / (fnx * fny);
|
|
|
|
int iF = ii - 2; /* -2 .. nx */
|
|
int jF = jj - 2; /* -2 .. ny */
|
|
int kF = kk - 2; /* -2 .. nz */
|
|
|
|
int siF = (iF <= 0) ? (1 - iF) : iF; /* 1..nx */
|
|
int sjF = (jF <= 0) ? (1 - jF) : jF; /* 1..ny */
|
|
int skF = (kF <= 0) ? (1 - kF) : kF; /* 1..nz */
|
|
|
|
double sign = 1.0;
|
|
if (iF <= 0) sign *= SoA0;
|
|
if (jF <= 0) sign *= SoA1;
|
|
if (kF <= 0) sign *= SoA2;
|
|
|
|
int src = (siF - 1) + (sjF - 1) * nx + (skF - 1) * nx * ny;
|
|
fh[tid] = sign * func[src];
|
|
}
|
|
}
|
|
|
|
__global__ void kern_symbd_ighost_ord3(double * __restrict__ fh, double SoA0)
|
|
{
|
|
const int ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int fnx = d_gp.fh3_nx, fny = d_gp.fh3_ny;
|
|
int total = ny * nz;
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < total * 3;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int ii = tid / total;
|
|
int rem = tid % total;
|
|
int j0 = rem % ny;
|
|
int k0 = rem / ny;
|
|
int jF = j0 + 1, kF = k0 + 1;
|
|
int iF_dst = -ii;
|
|
int iF_src = ii + 1;
|
|
fh[(iF_dst+2) + (jF+2)*fnx + (kF+2)*fnx*fny] =
|
|
fh[(iF_src+2) + (jF+2)*fnx + (kF+2)*fnx*fny] * SoA0;
|
|
}
|
|
}
|
|
|
|
__global__ void kern_symbd_jghost_ord3(double * __restrict__ fh, double SoA1)
|
|
{
|
|
const int nx = d_gp.ex[0], nz = d_gp.ex[2];
|
|
const int fnx = d_gp.fh3_nx, fny = d_gp.fh3_ny;
|
|
int irange = nx + 3;
|
|
int total = irange * nz;
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < total * 3;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int jj = tid / total;
|
|
int rem = tid % total;
|
|
int ii = rem % irange;
|
|
int k0 = rem / irange;
|
|
int iF = ii - 2;
|
|
int kF = k0 + 1;
|
|
int jF_dst = -jj;
|
|
int jF_src = jj + 1;
|
|
fh[(iF+2) + (jF_dst+2)*fnx + (kF+2)*fnx*fny] =
|
|
fh[(iF+2) + (jF_src+2)*fnx + (kF+2)*fnx*fny] * SoA1;
|
|
}
|
|
}
|
|
|
|
__global__ void kern_symbd_kghost_ord3(double * __restrict__ fh, double SoA2)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1];
|
|
const int fnx = d_gp.fh3_nx, fny = d_gp.fh3_ny;
|
|
int irange = nx + 3;
|
|
int jrange = ny + 3;
|
|
int total = irange * jrange;
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < total * 3;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int kk = tid / total;
|
|
int rem = tid % total;
|
|
int ii = rem % irange;
|
|
int jj = rem / irange;
|
|
int iF = ii - 2;
|
|
int jF = jj - 2;
|
|
int kF_dst = -kk;
|
|
int kF_src = kk + 1;
|
|
fh[(iF+2) + (jF+2)*fnx + (kF_dst+2)*fnx*fny] =
|
|
fh[(iF+2) + (jF+2)*fnx + (kF_src+2)*fnx*fny] * SoA2;
|
|
}
|
|
}
|
|
|
|
/* ================================================================== */
|
|
/* B. Stencil kernels */
|
|
/* ================================================================== */
|
|
|
|
/* ---- First derivatives (ord=2, 4th/2nd order) ---- */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_fderivs(const double * __restrict__ fh,
|
|
double * __restrict__ fx,
|
|
double * __restrict__ fy,
|
|
double * __restrict__ fz)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const int iminF = d_gp.iminF, jminF = d_gp.jminF, kminF = d_gp.kminF;
|
|
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < d_gp.all;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int i0 = tid % nx;
|
|
int j0 = (tid / nx) % ny;
|
|
int k0 = tid / (nx * ny);
|
|
|
|
/* boundary points: leave as zero */
|
|
if (i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2) {
|
|
fx[tid] = 0.0; fy[tid] = 0.0; fz[tid] = 0.0;
|
|
continue;
|
|
}
|
|
|
|
int iF = i0 + 1, jF = j0 + 1, kF = k0 + 1;
|
|
|
|
if ((iF+2) <= imaxF && (iF-2) >= iminF &&
|
|
(jF+2) <= jmaxF && (jF-2) >= jminF &&
|
|
(kF+2) <= kmaxF && (kF-2) >= kminF)
|
|
{
|
|
fx[tid] = d_gp.d12dx * (
|
|
fh[idx_fh2(iF-2,jF,kF)] - 8.0*fh[idx_fh2(iF-1,jF,kF)]
|
|
+ 8.0*fh[idx_fh2(iF+1,jF,kF)] - fh[idx_fh2(iF+2,jF,kF)]);
|
|
fy[tid] = d_gp.d12dy * (
|
|
fh[idx_fh2(iF,jF-2,kF)] - 8.0*fh[idx_fh2(iF,jF-1,kF)]
|
|
+ 8.0*fh[idx_fh2(iF,jF+1,kF)] - fh[idx_fh2(iF,jF+2,kF)]);
|
|
fz[tid] = d_gp.d12dz * (
|
|
fh[idx_fh2(iF,jF,kF-2)] - 8.0*fh[idx_fh2(iF,jF,kF-1)]
|
|
+ 8.0*fh[idx_fh2(iF,jF,kF+1)] - fh[idx_fh2(iF,jF,kF+2)]);
|
|
}
|
|
else if ((iF+1) <= imaxF && (iF-1) >= iminF &&
|
|
(jF+1) <= jmaxF && (jF-1) >= jminF &&
|
|
(kF+1) <= kmaxF && (kF-1) >= kminF)
|
|
{
|
|
fx[tid] = d_gp.d2dx * (
|
|
-fh[idx_fh2(iF-1,jF,kF)] + fh[idx_fh2(iF+1,jF,kF)]);
|
|
fy[tid] = d_gp.d2dy * (
|
|
-fh[idx_fh2(iF,jF-1,kF)] + fh[idx_fh2(iF,jF+1,kF)]);
|
|
fz[tid] = d_gp.d2dz * (
|
|
-fh[idx_fh2(iF,jF,kF-1)] + fh[idx_fh2(iF,jF,kF+1)]);
|
|
}
|
|
else {
|
|
fx[tid] = 0.0; fy[tid] = 0.0; fz[tid] = 0.0;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* ---- Second derivatives (ord=2, 4th/2nd order) ---- */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_fdderivs(const double * __restrict__ fh,
|
|
double * __restrict__ fxx, double * __restrict__ fxy,
|
|
double * __restrict__ fxz, double * __restrict__ fyy,
|
|
double * __restrict__ fyz, double * __restrict__ fzz)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const int iminF = d_gp.iminF, jminF = d_gp.jminF, kminF = d_gp.kminF;
|
|
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < d_gp.all;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int i0 = tid % nx;
|
|
int j0 = (tid / nx) % ny;
|
|
int k0 = tid / (nx * ny);
|
|
|
|
if (i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2) {
|
|
fxx[tid]=0; fxy[tid]=0; fxz[tid]=0;
|
|
fyy[tid]=0; fyz[tid]=0; fzz[tid]=0;
|
|
continue;
|
|
}
|
|
|
|
int iF = i0+1, jF = j0+1, kF = k0+1;
|
|
|
|
if ((iF+2)<=imaxF && (iF-2)>=iminF &&
|
|
(jF+2)<=jmaxF && (jF-2)>=jminF &&
|
|
(kF+2)<=kmaxF && (kF-2)>=kminF)
|
|
{
|
|
/* 4th-order diagonal */
|
|
double c = fh[idx_fh2(iF,jF,kF)];
|
|
fxx[tid] = d_gp.Fdxdx*(
|
|
-fh[idx_fh2(iF-2,jF,kF)] + 16.0*fh[idx_fh2(iF-1,jF,kF)]
|
|
-30.0*c + 16.0*fh[idx_fh2(iF+1,jF,kF)] - fh[idx_fh2(iF+2,jF,kF)]);
|
|
fyy[tid] = d_gp.Fdydy*(
|
|
-fh[idx_fh2(iF,jF-2,kF)] + 16.0*fh[idx_fh2(iF,jF-1,kF)]
|
|
-30.0*c + 16.0*fh[idx_fh2(iF,jF+1,kF)] - fh[idx_fh2(iF,jF+2,kF)]);
|
|
fzz[tid] = d_gp.Fdzdz*(
|
|
-fh[idx_fh2(iF,jF,kF-2)] + 16.0*fh[idx_fh2(iF,jF,kF-1)]
|
|
-30.0*c + 16.0*fh[idx_fh2(iF,jF,kF+1)] - fh[idx_fh2(iF,jF,kF+2)]);
|
|
|
|
/* 4th-order cross: fxy */
|
|
{
|
|
double t_jm2 = fh[idx_fh2(iF-2,jF-2,kF)] - 8.0*fh[idx_fh2(iF-1,jF-2,kF)]
|
|
+ 8.0*fh[idx_fh2(iF+1,jF-2,kF)] - fh[idx_fh2(iF+2,jF-2,kF)];
|
|
double t_jm1 = fh[idx_fh2(iF-2,jF-1,kF)] - 8.0*fh[idx_fh2(iF-1,jF-1,kF)]
|
|
+ 8.0*fh[idx_fh2(iF+1,jF-1,kF)] - fh[idx_fh2(iF+2,jF-1,kF)];
|
|
double t_jp1 = fh[idx_fh2(iF-2,jF+1,kF)] - 8.0*fh[idx_fh2(iF-1,jF+1,kF)]
|
|
+ 8.0*fh[idx_fh2(iF+1,jF+1,kF)] - fh[idx_fh2(iF+2,jF+1,kF)];
|
|
double t_jp2 = fh[idx_fh2(iF-2,jF+2,kF)] - 8.0*fh[idx_fh2(iF-1,jF+2,kF)]
|
|
+ 8.0*fh[idx_fh2(iF+1,jF+2,kF)] - fh[idx_fh2(iF+2,jF+2,kF)];
|
|
fxy[tid] = d_gp.Fdxdy*(t_jm2 - 8.0*t_jm1 + 8.0*t_jp1 - t_jp2);
|
|
}
|
|
/* 4th-order cross: fxz */
|
|
{
|
|
double t_km2 = fh[idx_fh2(iF-2,jF,kF-2)] - 8.0*fh[idx_fh2(iF-1,jF,kF-2)]
|
|
+ 8.0*fh[idx_fh2(iF+1,jF,kF-2)] - fh[idx_fh2(iF+2,jF,kF-2)];
|
|
double t_km1 = fh[idx_fh2(iF-2,jF,kF-1)] - 8.0*fh[idx_fh2(iF-1,jF,kF-1)]
|
|
+ 8.0*fh[idx_fh2(iF+1,jF,kF-1)] - fh[idx_fh2(iF+2,jF,kF-1)];
|
|
double t_kp1 = fh[idx_fh2(iF-2,jF,kF+1)] - 8.0*fh[idx_fh2(iF-1,jF,kF+1)]
|
|
+ 8.0*fh[idx_fh2(iF+1,jF,kF+1)] - fh[idx_fh2(iF+2,jF,kF+1)];
|
|
double t_kp2 = fh[idx_fh2(iF-2,jF,kF+2)] - 8.0*fh[idx_fh2(iF-1,jF,kF+2)]
|
|
+ 8.0*fh[idx_fh2(iF+1,jF,kF+2)] - fh[idx_fh2(iF+2,jF,kF+2)];
|
|
fxz[tid] = d_gp.Fdxdz*(t_km2 - 8.0*t_km1 + 8.0*t_kp1 - t_kp2);
|
|
}
|
|
/* 4th-order cross: fyz */
|
|
{
|
|
double t_km2 = fh[idx_fh2(iF,jF-2,kF-2)] - 8.0*fh[idx_fh2(iF,jF-1,kF-2)]
|
|
+ 8.0*fh[idx_fh2(iF,jF+1,kF-2)] - fh[idx_fh2(iF,jF+2,kF-2)];
|
|
double t_km1 = fh[idx_fh2(iF,jF-2,kF-1)] - 8.0*fh[idx_fh2(iF,jF-1,kF-1)]
|
|
+ 8.0*fh[idx_fh2(iF,jF+1,kF-1)] - fh[idx_fh2(iF,jF+2,kF-1)];
|
|
double t_kp1 = fh[idx_fh2(iF,jF-2,kF+1)] - 8.0*fh[idx_fh2(iF,jF-1,kF+1)]
|
|
+ 8.0*fh[idx_fh2(iF,jF+1,kF+1)] - fh[idx_fh2(iF,jF+2,kF+1)];
|
|
double t_kp2 = fh[idx_fh2(iF,jF-2,kF+2)] - 8.0*fh[idx_fh2(iF,jF-1,kF+2)]
|
|
+ 8.0*fh[idx_fh2(iF,jF+1,kF+2)] - fh[idx_fh2(iF,jF+2,kF+2)];
|
|
fyz[tid] = d_gp.Fdydz*(t_km2 - 8.0*t_km1 + 8.0*t_kp1 - t_kp2);
|
|
}
|
|
}
|
|
else if ((iF+1)<=imaxF && (iF-1)>=iminF &&
|
|
(jF+1)<=jmaxF && (jF-1)>=jminF &&
|
|
(kF+1)<=kmaxF && (kF-1)>=kminF)
|
|
{
|
|
double c = fh[idx_fh2(iF,jF,kF)];
|
|
fxx[tid] = d_gp.Sdxdx*(fh[idx_fh2(iF-1,jF,kF)] - 2.0*c + fh[idx_fh2(iF+1,jF,kF)]);
|
|
fyy[tid] = d_gp.Sdydy*(fh[idx_fh2(iF,jF-1,kF)] - 2.0*c + fh[idx_fh2(iF,jF+1,kF)]);
|
|
fzz[tid] = d_gp.Sdzdz*(fh[idx_fh2(iF,jF,kF-1)] - 2.0*c + fh[idx_fh2(iF,jF,kF+1)]);
|
|
fxy[tid] = d_gp.Sdxdy*(fh[idx_fh2(iF-1,jF-1,kF)] - fh[idx_fh2(iF+1,jF-1,kF)]
|
|
-fh[idx_fh2(iF-1,jF+1,kF)] + fh[idx_fh2(iF+1,jF+1,kF)]);
|
|
fxz[tid] = d_gp.Sdxdz*(fh[idx_fh2(iF-1,jF,kF-1)] - fh[idx_fh2(iF+1,jF,kF-1)]
|
|
-fh[idx_fh2(iF-1,jF,kF+1)] + fh[idx_fh2(iF+1,jF,kF+1)]);
|
|
fyz[tid] = d_gp.Sdydz*(fh[idx_fh2(iF,jF-1,kF-1)] - fh[idx_fh2(iF,jF+1,kF-1)]
|
|
-fh[idx_fh2(iF,jF-1,kF+1)] + fh[idx_fh2(iF,jF+1,kF+1)]);
|
|
}
|
|
else {
|
|
fxx[tid]=0; fxy[tid]=0; fxz[tid]=0;
|
|
fyy[tid]=0; fyz[tid]=0; fzz[tid]=0;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* ---- Lopsided (upwind advection) kernel ---- */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_lopsided(const double * __restrict__ fh,
|
|
double * __restrict__ f_rhs,
|
|
const double * __restrict__ Sfx,
|
|
const double * __restrict__ Sfy,
|
|
const double * __restrict__ Sfz)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int iminF = d_gp.iminF3, jminF = d_gp.jminF3, kminF = d_gp.kminF3;
|
|
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < d_gp.all;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int i0 = tid % nx;
|
|
int j0 = (tid / nx) % ny;
|
|
int k0 = tid / (nx * ny);
|
|
|
|
if (i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2) continue;
|
|
|
|
int iF = i0 + 1, jF = j0 + 1, kF = k0 + 1;
|
|
double val = 0.0;
|
|
|
|
/* --- x direction --- */
|
|
double sfx = Sfx[tid];
|
|
if (sfx > 0.0) {
|
|
if (i0 <= nx - 4) {
|
|
val += sfx * d_gp.d12dx * (
|
|
-3.0*fh[idx_fh3(iF-1,jF,kF)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF+1,jF,kF)] - 6.0*fh[idx_fh3(iF+2,jF,kF)]
|
|
+ fh[idx_fh3(iF+3,jF,kF)]);
|
|
} else if (i0 <= nx - 3) {
|
|
val += sfx * d_gp.d12dx * (
|
|
fh[idx_fh3(iF-2,jF,kF)] - 8.0*fh[idx_fh3(iF-1,jF,kF)]
|
|
+8.0*fh[idx_fh3(iF+1,jF,kF)] - fh[idx_fh3(iF+2,jF,kF)]);
|
|
} else if (i0 <= nx - 2) {
|
|
val -= sfx * d_gp.d12dx * (
|
|
-3.0*fh[idx_fh3(iF+1,jF,kF)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF-1,jF,kF)] - 6.0*fh[idx_fh3(iF-2,jF,kF)]
|
|
+ fh[idx_fh3(iF-3,jF,kF)]);
|
|
}
|
|
} else if (sfx < 0.0) {
|
|
if ((i0 - 2) >= iminF) {
|
|
val -= sfx * d_gp.d12dx * (
|
|
-3.0*fh[idx_fh3(iF+1,jF,kF)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF-1,jF,kF)] - 6.0*fh[idx_fh3(iF-2,jF,kF)]
|
|
+ fh[idx_fh3(iF-3,jF,kF)]);
|
|
} else if ((i0 - 1) >= iminF) {
|
|
val += sfx * d_gp.d12dx * (
|
|
fh[idx_fh3(iF-2,jF,kF)] - 8.0*fh[idx_fh3(iF-1,jF,kF)]
|
|
+8.0*fh[idx_fh3(iF+1,jF,kF)] - fh[idx_fh3(iF+2,jF,kF)]);
|
|
} else if (i0 >= iminF) {
|
|
val += sfx * d_gp.d12dx * (
|
|
-3.0*fh[idx_fh3(iF-1,jF,kF)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF+1,jF,kF)] - 6.0*fh[idx_fh3(iF+2,jF,kF)]
|
|
+ fh[idx_fh3(iF+3,jF,kF)]);
|
|
}
|
|
}
|
|
|
|
/* --- y direction --- */
|
|
double sfy = Sfy[tid];
|
|
if (sfy > 0.0) {
|
|
if (j0 <= ny - 4) {
|
|
val += sfy * d_gp.d12dy * (
|
|
-3.0*fh[idx_fh3(iF,jF-1,kF)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF,jF+1,kF)] - 6.0*fh[idx_fh3(iF,jF+2,kF)]
|
|
+ fh[idx_fh3(iF,jF+3,kF)]);
|
|
} else if (j0 <= ny - 3) {
|
|
val += sfy * d_gp.d12dy * (
|
|
fh[idx_fh3(iF,jF-2,kF)] - 8.0*fh[idx_fh3(iF,jF-1,kF)]
|
|
+8.0*fh[idx_fh3(iF,jF+1,kF)] - fh[idx_fh3(iF,jF+2,kF)]);
|
|
} else if (j0 <= ny - 2) {
|
|
val -= sfy * d_gp.d12dy * (
|
|
-3.0*fh[idx_fh3(iF,jF+1,kF)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF,jF-1,kF)] - 6.0*fh[idx_fh3(iF,jF-2,kF)]
|
|
+ fh[idx_fh3(iF,jF-3,kF)]);
|
|
}
|
|
} else if (sfy < 0.0) {
|
|
if ((j0 - 2) >= jminF) {
|
|
val -= sfy * d_gp.d12dy * (
|
|
-3.0*fh[idx_fh3(iF,jF+1,kF)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF,jF-1,kF)] - 6.0*fh[idx_fh3(iF,jF-2,kF)]
|
|
+ fh[idx_fh3(iF,jF-3,kF)]);
|
|
} else if ((j0 - 1) >= jminF) {
|
|
val += sfy * d_gp.d12dy * (
|
|
fh[idx_fh3(iF,jF-2,kF)] - 8.0*fh[idx_fh3(iF,jF-1,kF)]
|
|
+8.0*fh[idx_fh3(iF,jF+1,kF)] - fh[idx_fh3(iF,jF+2,kF)]);
|
|
} else if (j0 >= jminF) {
|
|
val += sfy * d_gp.d12dy * (
|
|
-3.0*fh[idx_fh3(iF,jF-1,kF)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF,jF+1,kF)] - 6.0*fh[idx_fh3(iF,jF+2,kF)]
|
|
+ fh[idx_fh3(iF,jF+3,kF)]);
|
|
}
|
|
}
|
|
|
|
/* --- z direction --- */
|
|
double sfz = Sfz[tid];
|
|
if (sfz > 0.0) {
|
|
if (k0 <= nz - 4) {
|
|
val += sfz * d_gp.d12dz * (
|
|
-3.0*fh[idx_fh3(iF,jF,kF-1)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF,jF,kF+1)] - 6.0*fh[idx_fh3(iF,jF,kF+2)]
|
|
+ fh[idx_fh3(iF,jF,kF+3)]);
|
|
} else if (k0 <= nz - 3) {
|
|
val += sfz * d_gp.d12dz * (
|
|
fh[idx_fh3(iF,jF,kF-2)] - 8.0*fh[idx_fh3(iF,jF,kF-1)]
|
|
+8.0*fh[idx_fh3(iF,jF,kF+1)] - fh[idx_fh3(iF,jF,kF+2)]);
|
|
} else if (k0 <= nz - 2) {
|
|
val -= sfz * d_gp.d12dz * (
|
|
-3.0*fh[idx_fh3(iF,jF,kF+1)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF,jF,kF-1)] - 6.0*fh[idx_fh3(iF,jF,kF-2)]
|
|
+ fh[idx_fh3(iF,jF,kF-3)]);
|
|
}
|
|
} else if (sfz < 0.0) {
|
|
if ((k0 - 2) >= kminF) {
|
|
val -= sfz * d_gp.d12dz * (
|
|
-3.0*fh[idx_fh3(iF,jF,kF+1)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF,jF,kF-1)] - 6.0*fh[idx_fh3(iF,jF,kF-2)]
|
|
+ fh[idx_fh3(iF,jF,kF-3)]);
|
|
} else if ((k0 - 1) >= kminF) {
|
|
val += sfz * d_gp.d12dz * (
|
|
fh[idx_fh3(iF,jF,kF-2)] - 8.0*fh[idx_fh3(iF,jF,kF-1)]
|
|
+8.0*fh[idx_fh3(iF,jF,kF+1)] - fh[idx_fh3(iF,jF,kF+2)]);
|
|
} else if (k0 >= kminF) {
|
|
val += sfz * d_gp.d12dz * (
|
|
-3.0*fh[idx_fh3(iF,jF,kF-1)] - 10.0*fh[idx_fh3(iF,jF,kF)]
|
|
+18.0*fh[idx_fh3(iF,jF,kF+1)] - 6.0*fh[idx_fh3(iF,jF,kF+2)]
|
|
+ fh[idx_fh3(iF,jF,kF+3)]);
|
|
}
|
|
}
|
|
|
|
f_rhs[tid] += val;
|
|
}
|
|
}
|
|
|
|
/* ---- KO dissipation kernel (ord=3, 6th-order) ---- */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_kodis(const double * __restrict__ fh,
|
|
double * __restrict__ f_rhs,
|
|
double eps_val)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int iminF = d_gp.iminF3, jminF = d_gp.jminF3, kminF = d_gp.kminF3;
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const double cof = 64.0;
|
|
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < d_gp.all;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int i0 = tid % nx;
|
|
int j0 = (tid / nx) % ny;
|
|
int k0 = tid / (nx * ny);
|
|
int iF = i0 + 1, jF = j0 + 1, kF = k0 + 1;
|
|
|
|
if ((iF-3) >= iminF && (iF+3) <= imaxF &&
|
|
(jF-3) >= jminF && (jF+3) <= jmaxF &&
|
|
(kF-3) >= kminF && (kF+3) <= kmaxF)
|
|
{
|
|
double Dx = (fh[idx_fh3(iF-3,jF,kF)] + fh[idx_fh3(iF+3,jF,kF)])
|
|
- 6.0*(fh[idx_fh3(iF-2,jF,kF)] + fh[idx_fh3(iF+2,jF,kF)])
|
|
+15.0*(fh[idx_fh3(iF-1,jF,kF)] + fh[idx_fh3(iF+1,jF,kF)])
|
|
-20.0* fh[idx_fh3(iF,jF,kF)];
|
|
Dx /= d_gp.dX;
|
|
|
|
double Dy = (fh[idx_fh3(iF,jF-3,kF)] + fh[idx_fh3(iF,jF+3,kF)])
|
|
- 6.0*(fh[idx_fh3(iF,jF-2,kF)] + fh[idx_fh3(iF,jF+2,kF)])
|
|
+15.0*(fh[idx_fh3(iF,jF-1,kF)] + fh[idx_fh3(iF,jF+1,kF)])
|
|
-20.0* fh[idx_fh3(iF,jF,kF)];
|
|
Dy /= d_gp.dY;
|
|
|
|
double Dz = (fh[idx_fh3(iF,jF,kF-3)] + fh[idx_fh3(iF,jF,kF+3)])
|
|
- 6.0*(fh[idx_fh3(iF,jF,kF-2)] + fh[idx_fh3(iF,jF,kF+2)])
|
|
+15.0*(fh[idx_fh3(iF,jF,kF-1)] + fh[idx_fh3(iF,jF,kF+1)])
|
|
-20.0* fh[idx_fh3(iF,jF,kF)];
|
|
Dz /= d_gp.dZ;
|
|
|
|
f_rhs[tid] += (eps_val / cof) * (Dx + Dy + Dz);
|
|
}
|
|
}
|
|
}
|
|
|
|
/* ================================================================== */
|
|
/* Host wrapper helpers */
|
|
/* ================================================================== */
|
|
struct LopsidedKodisTables {
|
|
const double *adv_fields[32];
|
|
const double *ko_fields[32];
|
|
double *rhs_fields[32];
|
|
int soa_signs[3 * 32];
|
|
};
|
|
|
|
struct FDerivTables {
|
|
const double *src_fields[32];
|
|
double *fx_fields[32];
|
|
double *fy_fields[32];
|
|
double *fz_fields[32];
|
|
int soa_signs[3 * 32];
|
|
};
|
|
|
|
struct FDDerivTables {
|
|
const double *src_fields[32];
|
|
double *fxx_fields[32];
|
|
double *fxy_fields[32];
|
|
double *fxz_fields[32];
|
|
double *fyy_fields[32];
|
|
double *fyz_fields[32];
|
|
double *fzz_fields[32];
|
|
int soa_signs[3 * 32];
|
|
};
|
|
|
|
static constexpr int PHASE10_METRIC_FIELD_COUNT = 6;
|
|
|
|
struct Phase10RicciTables {
|
|
const double *src_fields[PHASE10_METRIC_FIELD_COUNT];
|
|
double *dst_fields[PHASE10_METRIC_FIELD_COUNT];
|
|
int soa_signs[3 * PHASE10_METRIC_FIELD_COUNT];
|
|
};
|
|
|
|
struct Rk4FinalizeTables {
|
|
const double *f0_fields[BSSN_STATE_COUNT];
|
|
double *rhs_fields[BSSN_STATE_COUNT];
|
|
double *accum_fields[BSSN_STATE_COUNT];
|
|
};
|
|
|
|
struct PatchBoundaryTables {
|
|
const double *src_fields[32];
|
|
double *dst_fields[BSSN_STATE_COUNT];
|
|
};
|
|
|
|
static const int BLK = 128;
|
|
static inline int grid(size_t n) {
|
|
if (n == 0) return 1;
|
|
size_t g = (n + BLK - 1) / BLK;
|
|
if (g > 2147483647u) g = 2147483647u;
|
|
return (int)g;
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_fderivs_batched(FDerivTables tables, int field_count)
|
|
{
|
|
const int field = blockIdx.y;
|
|
if (field >= field_count) return;
|
|
|
|
const double *src = tables.src_fields[field];
|
|
double *fx = tables.fx_fields[field];
|
|
double *fy = tables.fy_fields[field];
|
|
double *fz = tables.fz_fields[field];
|
|
const int SoA0 = tables.soa_signs[3 * field + 0];
|
|
const int SoA1 = tables.soa_signs[3 * field + 1];
|
|
const int SoA2 = tables.soa_signs[3 * field + 2];
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const int iminF = d_gp.iminF, jminF = d_gp.jminF, kminF = d_gp.kminF;
|
|
|
|
const int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (tid >= d_gp.all) return;
|
|
|
|
const int i0 = tid % nx;
|
|
const int j0 = (tid / nx) % ny;
|
|
const int k0 = tid / (nx * ny);
|
|
|
|
if (i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2) {
|
|
fx[tid] = 0.0;
|
|
fy[tid] = 0.0;
|
|
fz[tid] = 0.0;
|
|
return;
|
|
}
|
|
|
|
const int iF = i0 + 1;
|
|
const int jF = j0 + 1;
|
|
const int kF = k0 + 1;
|
|
|
|
fd_compute_first3(src, iF, jF, kF,
|
|
iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
|
|
SoA0, SoA1, SoA2,
|
|
fx[tid], fy[tid], fz[tid]);
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_fdderivs_batched(FDDerivTables tables, int field_count)
|
|
{
|
|
const int field = blockIdx.y;
|
|
if (field >= field_count) return;
|
|
|
|
const double *src = tables.src_fields[field];
|
|
double *fxx = tables.fxx_fields[field];
|
|
double *fxy = tables.fxy_fields[field];
|
|
double *fxz = tables.fxz_fields[field];
|
|
double *fyy = tables.fyy_fields[field];
|
|
double *fyz = tables.fyz_fields[field];
|
|
double *fzz = tables.fzz_fields[field];
|
|
const int SoA0 = tables.soa_signs[3 * field + 0];
|
|
const int SoA1 = tables.soa_signs[3 * field + 1];
|
|
const int SoA2 = tables.soa_signs[3 * field + 2];
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const int iminF = d_gp.iminF, jminF = d_gp.jminF, kminF = d_gp.kminF;
|
|
|
|
const int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (tid >= d_gp.all) return;
|
|
|
|
const int i0 = tid % nx;
|
|
const int j0 = (tid / nx) % ny;
|
|
const int k0 = tid / (nx * ny);
|
|
|
|
if (i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2) {
|
|
fxx[tid] = 0.0; fxy[tid] = 0.0; fxz[tid] = 0.0;
|
|
fyy[tid] = 0.0; fyz[tid] = 0.0; fzz[tid] = 0.0;
|
|
return;
|
|
}
|
|
|
|
const int iF = i0 + 1;
|
|
const int jF = j0 + 1;
|
|
const int kF = k0 + 1;
|
|
|
|
#if ghost_width != 3
|
|
fd_compute_second6(src, iF, jF, kF,
|
|
iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
|
|
SoA0, SoA1, SoA2,
|
|
fxx[tid], fxy[tid], fxz[tid], fyy[tid], fyz[tid], fzz[tid]);
|
|
#else
|
|
if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
|
|
(jF + 2) <= jmaxF && (jF - 2) >= jminF &&
|
|
(kF + 2) <= kmaxF && (kF - 2) >= kminF)
|
|
{
|
|
const double c = fetch_sym_ord2_direct(src, iF, jF, kF, SoA0, SoA1, SoA2);
|
|
fxx[tid] = d_gp.Fdxdx * (
|
|
-fetch_sym_ord2_direct(src, iF - 2, jF, kF, SoA0, SoA1, SoA2)
|
|
+16.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF, SoA0, SoA1, SoA2));
|
|
fyy[tid] = d_gp.Fdydy * (
|
|
-fetch_sym_ord2_direct(src, iF, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
+16.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF, SoA0, SoA1, SoA2));
|
|
fzz[tid] = d_gp.Fdzdz * (
|
|
-fetch_sym_ord2_direct(src, iF, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
+16.0 * fetch_sym_ord2_direct(src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(src, iF, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF, kF + 2, SoA0, SoA1, SoA2));
|
|
|
|
const double t_jm2 =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF - 2, kF, SoA0, SoA1, SoA2);
|
|
const double t_jm1 =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF - 1, kF, SoA0, SoA1, SoA2);
|
|
const double t_jp1 =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF + 1, kF, SoA0, SoA1, SoA2);
|
|
const double t_jp2 =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF + 2, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF + 2, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF + 2, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF + 2, kF, SoA0, SoA1, SoA2);
|
|
fxy[tid] = d_gp.Fdxdy * (t_jm2 - 8.0 * t_jm1 + 8.0 * t_jp1 - t_jp2);
|
|
|
|
const double t_km2_x =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF - 2, SoA0, SoA1, SoA2);
|
|
const double t_km1_x =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF - 1, SoA0, SoA1, SoA2);
|
|
const double t_kp1_x =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF + 1, SoA0, SoA1, SoA2);
|
|
const double t_kp2_x =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF, kF + 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF + 2, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF + 2, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF + 2, SoA0, SoA1, SoA2);
|
|
fxz[tid] = d_gp.Fdxdz * (t_km2_x - 8.0 * t_km1_x + 8.0 * t_kp1_x - t_kp2_x);
|
|
|
|
const double t_km2_y =
|
|
fetch_sym_ord2_direct(src, iF, jF - 2, kF - 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF - 2, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF - 2, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF - 2, SoA0, SoA1, SoA2);
|
|
const double t_km1_y =
|
|
fetch_sym_ord2_direct(src, iF, jF - 2, kF - 1, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF - 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF - 1, SoA0, SoA1, SoA2);
|
|
const double t_kp1_y =
|
|
fetch_sym_ord2_direct(src, iF, jF - 2, kF + 1, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF + 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF + 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF + 1, SoA0, SoA1, SoA2);
|
|
const double t_kp2_y =
|
|
fetch_sym_ord2_direct(src, iF, jF - 2, kF + 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF + 2, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF + 2, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF + 2, SoA0, SoA1, SoA2);
|
|
fyz[tid] = d_gp.Fdydz * (t_km2_y - 8.0 * t_km1_y + 8.0 * t_kp1_y - t_kp2_y);
|
|
}
|
|
else if ((iF + 1) <= imaxF && (iF - 1) >= iminF &&
|
|
(jF + 1) <= jmaxF && (jF - 1) >= jminF &&
|
|
(kF + 1) <= kmaxF && (kF - 1) >= kminF)
|
|
{
|
|
const double c = fetch_sym_ord2_direct(src, iF, jF, kF, SoA0, SoA1, SoA2);
|
|
fxx[tid] = d_gp.Sdxdx * (
|
|
fetch_sym_ord2_direct(src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(src, iF + 1, jF, kF, SoA0, SoA1, SoA2));
|
|
fyy[tid] = d_gp.Sdydy * (
|
|
fetch_sym_ord2_direct(src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(src, iF, jF + 1, kF, SoA0, SoA1, SoA2));
|
|
fzz[tid] = d_gp.Sdzdz * (
|
|
fetch_sym_ord2_direct(src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(src, iF, jF, kF + 1, SoA0, SoA1, SoA2));
|
|
fxy[tid] = d_gp.Sdxdy * (
|
|
fetch_sym_ord2_direct(src, iF - 1, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 1, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF - 1, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord2_direct(src, iF + 1, jF + 1, kF, SoA0, SoA1, SoA2));
|
|
fxz[tid] = d_gp.Sdxdz * (
|
|
fetch_sym_ord2_direct(src, iF - 1, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 1, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF - 1, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord2_direct(src, iF + 1, jF, kF + 1, SoA0, SoA1, SoA2));
|
|
fyz[tid] = d_gp.Sdydz * (
|
|
fetch_sym_ord2_direct(src, iF, jF - 1, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 1, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF - 1, kF + 1, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord2_direct(src, iF, jF + 1, kF + 1, SoA0, SoA1, SoA2));
|
|
}
|
|
else {
|
|
fxx[tid] = 0.0; fxy[tid] = 0.0; fxz[tid] = 0.0;
|
|
fyy[tid] = 0.0; fyz[tid] = 0.0; fzz[tid] = 0.0;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
static void gpu_fderivs_batch(int field_count,
|
|
double *const *src_fields,
|
|
double *const *fx_fields,
|
|
double *const *fy_fields,
|
|
double *const *fz_fields,
|
|
const int *soa_signs,
|
|
int all);
|
|
static void gpu_fdderivs_batch(int field_count,
|
|
double *const *src_fields,
|
|
double *const *fxx_fields,
|
|
double *const *fxy_fields,
|
|
double *const *fxz_fields,
|
|
double *const *fyy_fields,
|
|
double *const *fyz_fields,
|
|
double *const *fzz_fields,
|
|
const int *soa_signs,
|
|
int all);
|
|
static void gpu_lopsided_kodis_single_batch(double *d_f_adv, double *d_f_ko, double *d_f_rhs,
|
|
double *d_Sfx, double *d_Sfy, double *d_Sfz,
|
|
double SoA0, double SoA1, double SoA2,
|
|
double eps_val, int all);
|
|
|
|
/* symmetry_bd on GPU for ord=2, then launch fderivs kernel */
|
|
static void gpu_fderivs(double *d_f, double *d_fx, double *d_fy, double *d_fz,
|
|
double SoA0, double SoA1, double SoA2, int all)
|
|
{
|
|
#if ghost_width != 3
|
|
double *src_fields[1] = {d_f};
|
|
double *fx_fields[1] = {d_fx};
|
|
double *fy_fields[1] = {d_fy};
|
|
double *fz_fields[1] = {d_fz};
|
|
const int soa_signs[3] = {(int)SoA0, (int)SoA1, (int)SoA2};
|
|
gpu_fderivs_batch(1, src_fields, fx_fields, fy_fields, fz_fields, soa_signs, all);
|
|
#else
|
|
double *fh = g_buf.d_fh2;
|
|
const size_t nx = (size_t)g_buf.prev_nx;
|
|
const size_t ny = (size_t)g_buf.prev_ny;
|
|
const size_t nz = (size_t)g_buf.prev_nz;
|
|
const size_t w_pack = (nx + 2ull) * (ny + 2ull) * (nz + 2ull);
|
|
|
|
kern_symbd_pack_ord2<<<grid(w_pack), BLK>>>(d_f, fh, SoA0, SoA1, SoA2);
|
|
kern_fderivs<<<grid(all), BLK>>>(fh, d_fx, d_fy, d_fz);
|
|
#endif
|
|
}
|
|
|
|
/* symmetry_bd on GPU for ord=2, then launch fdderivs kernel */
|
|
static void gpu_fdderivs(double *d_f,
|
|
double *d_fxx, double *d_fxy, double *d_fxz,
|
|
double *d_fyy, double *d_fyz, double *d_fzz,
|
|
double SoA0, double SoA1, double SoA2, int all)
|
|
{
|
|
#if ghost_width != 3
|
|
double *src_fields[1] = {d_f};
|
|
double *fxx_fields[1] = {d_fxx};
|
|
double *fxy_fields[1] = {d_fxy};
|
|
double *fxz_fields[1] = {d_fxz};
|
|
double *fyy_fields[1] = {d_fyy};
|
|
double *fyz_fields[1] = {d_fyz};
|
|
double *fzz_fields[1] = {d_fzz};
|
|
const int soa_signs[3] = {(int)SoA0, (int)SoA1, (int)SoA2};
|
|
gpu_fdderivs_batch(1, src_fields, fxx_fields, fxy_fields, fxz_fields,
|
|
fyy_fields, fyz_fields, fzz_fields, soa_signs, all);
|
|
#else
|
|
double *fh = g_buf.d_fh2;
|
|
const size_t nx = (size_t)g_buf.prev_nx;
|
|
const size_t ny = (size_t)g_buf.prev_ny;
|
|
const size_t nz = (size_t)g_buf.prev_nz;
|
|
const size_t w_pack = (nx + 2ull) * (ny + 2ull) * (nz + 2ull);
|
|
|
|
kern_symbd_pack_ord2<<<grid(w_pack), BLK>>>(d_f, fh, SoA0, SoA1, SoA2);
|
|
kern_fdderivs<<<grid(all), BLK>>>(fh, d_fxx, d_fxy, d_fxz, d_fyy, d_fyz, d_fzz);
|
|
#endif
|
|
}
|
|
|
|
static void gpu_fderivs_batch(int field_count,
|
|
double *const *src_fields,
|
|
double *const *fx_fields,
|
|
double *const *fy_fields,
|
|
double *const *fz_fields,
|
|
const int *soa_signs,
|
|
int all)
|
|
{
|
|
if (field_count <= 0) return;
|
|
FDerivTables tables = {};
|
|
for (int i = 0; i < field_count; ++i) {
|
|
tables.src_fields[i] = src_fields[i];
|
|
tables.fx_fields[i] = fx_fields[i];
|
|
tables.fy_fields[i] = fy_fields[i];
|
|
tables.fz_fields[i] = fz_fields[i];
|
|
tables.soa_signs[3 * i + 0] = soa_signs[3 * i + 0];
|
|
tables.soa_signs[3 * i + 1] = soa_signs[3 * i + 1];
|
|
tables.soa_signs[3 * i + 2] = soa_signs[3 * i + 2];
|
|
}
|
|
dim3 launch_grid((unsigned int)grid((size_t)all), (unsigned int)field_count);
|
|
kern_fderivs_batched<<<launch_grid, BLK>>>(tables, field_count);
|
|
}
|
|
|
|
static void gpu_fdderivs_batch(int field_count,
|
|
double *const *src_fields,
|
|
double *const *fxx_fields,
|
|
double *const *fxy_fields,
|
|
double *const *fxz_fields,
|
|
double *const *fyy_fields,
|
|
double *const *fyz_fields,
|
|
double *const *fzz_fields,
|
|
const int *soa_signs,
|
|
int all)
|
|
{
|
|
if (field_count <= 0) return;
|
|
FDDerivTables tables = {};
|
|
for (int i = 0; i < field_count; ++i) {
|
|
tables.src_fields[i] = src_fields[i];
|
|
tables.fxx_fields[i] = fxx_fields[i];
|
|
tables.fxy_fields[i] = fxy_fields[i];
|
|
tables.fxz_fields[i] = fxz_fields[i];
|
|
tables.fyy_fields[i] = fyy_fields[i];
|
|
tables.fyz_fields[i] = fyz_fields[i];
|
|
tables.fzz_fields[i] = fzz_fields[i];
|
|
tables.soa_signs[3 * i + 0] = soa_signs[3 * i + 0];
|
|
tables.soa_signs[3 * i + 1] = soa_signs[3 * i + 1];
|
|
tables.soa_signs[3 * i + 2] = soa_signs[3 * i + 2];
|
|
}
|
|
dim3 launch_grid((unsigned int)grid((size_t)all), (unsigned int)field_count);
|
|
kern_fdderivs_batched<<<launch_grid, BLK>>>(tables, field_count);
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase10_ricci_batched(const double * __restrict__ gupxx,
|
|
const double * __restrict__ gupxy,
|
|
const double * __restrict__ gupxz,
|
|
const double * __restrict__ gupyy,
|
|
const double * __restrict__ gupyz,
|
|
const double * __restrict__ gupzz,
|
|
Phase10RicciTables tables)
|
|
{
|
|
const int field = blockIdx.y;
|
|
if (field >= PHASE10_METRIC_FIELD_COUNT) return;
|
|
|
|
const double *src = tables.src_fields[field];
|
|
double *dst = tables.dst_fields[field];
|
|
const int SoA0 = tables.soa_signs[3 * field + 0];
|
|
const int SoA1 = tables.soa_signs[3 * field + 1];
|
|
const int SoA2 = tables.soa_signs[3 * field + 2];
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const int iminF = d_gp.iminF, jminF = d_gp.jminF, kminF = d_gp.kminF;
|
|
const int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (tid >= d_gp.all) return;
|
|
|
|
const int i0 = tid % nx;
|
|
const int j0 = (tid / nx) % ny;
|
|
const int k0 = tid / (nx * ny);
|
|
|
|
double fxx = 0.0, fxy = 0.0, fxz = 0.0;
|
|
double fyy = 0.0, fyz = 0.0, fzz = 0.0;
|
|
|
|
if (!(i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2)) {
|
|
const int iF = i0 + 1;
|
|
const int jF = j0 + 1;
|
|
const int kF = k0 + 1;
|
|
|
|
#if ghost_width != 3
|
|
fd_compute_second6(src, iF, jF, kF,
|
|
iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
|
|
SoA0, SoA1, SoA2,
|
|
fxx, fxy, fxz, fyy, fyz, fzz);
|
|
#else
|
|
if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
|
|
(jF + 2) <= jmaxF && (jF - 2) >= jminF &&
|
|
(kF + 2) <= kmaxF && (kF - 2) >= kminF)
|
|
{
|
|
const double c = fetch_sym_ord2_direct(src, iF, jF, kF, SoA0, SoA1, SoA2);
|
|
fxx = d_gp.Fdxdx * (
|
|
-fetch_sym_ord2_direct(src, iF - 2, jF, kF, SoA0, SoA1, SoA2)
|
|
+16.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF, SoA0, SoA1, SoA2));
|
|
fyy = d_gp.Fdydy * (
|
|
-fetch_sym_ord2_direct(src, iF, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
+16.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF, SoA0, SoA1, SoA2));
|
|
fzz = d_gp.Fdzdz * (
|
|
-fetch_sym_ord2_direct(src, iF, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
+16.0 * fetch_sym_ord2_direct(src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(src, iF, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF, kF + 2, SoA0, SoA1, SoA2));
|
|
|
|
const double t_jm2 =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF - 2, kF, SoA0, SoA1, SoA2);
|
|
const double t_jm1 =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF - 1, kF, SoA0, SoA1, SoA2);
|
|
const double t_jp1 =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF + 1, kF, SoA0, SoA1, SoA2);
|
|
const double t_jp2 =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF + 2, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF + 2, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF + 2, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF + 2, kF, SoA0, SoA1, SoA2);
|
|
fxy = d_gp.Fdxdy * (t_jm2 - 8.0 * t_jm1 + 8.0 * t_jp1 - t_jp2);
|
|
|
|
const double t_km2_x =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF - 2, SoA0, SoA1, SoA2);
|
|
const double t_km1_x =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF - 1, SoA0, SoA1, SoA2);
|
|
const double t_kp1_x =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF + 1, SoA0, SoA1, SoA2);
|
|
const double t_kp2_x =
|
|
fetch_sym_ord2_direct(src, iF - 2, jF, kF + 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF - 1, jF, kF + 2, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF + 1, jF, kF + 2, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 2, jF, kF + 2, SoA0, SoA1, SoA2);
|
|
fxz = d_gp.Fdxdz * (t_km2_x - 8.0 * t_km1_x + 8.0 * t_kp1_x - t_kp2_x);
|
|
|
|
const double t_km2_y =
|
|
fetch_sym_ord2_direct(src, iF, jF - 2, kF - 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF - 2, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF - 2, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF - 2, SoA0, SoA1, SoA2);
|
|
const double t_km1_y =
|
|
fetch_sym_ord2_direct(src, iF, jF - 2, kF - 1, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF - 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF - 1, SoA0, SoA1, SoA2);
|
|
const double t_kp1_y =
|
|
fetch_sym_ord2_direct(src, iF, jF - 2, kF + 1, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF + 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF + 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF + 1, SoA0, SoA1, SoA2);
|
|
const double t_kp2_y =
|
|
fetch_sym_ord2_direct(src, iF, jF - 2, kF + 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord2_direct(src, iF, jF - 1, kF + 2, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord2_direct(src, iF, jF + 1, kF + 2, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 2, kF + 2, SoA0, SoA1, SoA2);
|
|
fyz = d_gp.Fdydz * (t_km2_y - 8.0 * t_km1_y + 8.0 * t_kp1_y - t_kp2_y);
|
|
}
|
|
else if ((iF + 1) <= imaxF && (iF - 1) >= iminF &&
|
|
(jF + 1) <= jmaxF && (jF - 1) >= jminF &&
|
|
(kF + 1) <= kmaxF && (kF - 1) >= kminF)
|
|
{
|
|
const double c = fetch_sym_ord2_direct(src, iF, jF, kF, SoA0, SoA1, SoA2);
|
|
fxx = d_gp.Sdxdx * (
|
|
fetch_sym_ord2_direct(src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(src, iF + 1, jF, kF, SoA0, SoA1, SoA2));
|
|
fyy = d_gp.Sdydy * (
|
|
fetch_sym_ord2_direct(src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(src, iF, jF + 1, kF, SoA0, SoA1, SoA2));
|
|
fzz = d_gp.Sdzdz * (
|
|
fetch_sym_ord2_direct(src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(src, iF, jF, kF + 1, SoA0, SoA1, SoA2));
|
|
fxy = d_gp.Sdxdy * (
|
|
fetch_sym_ord2_direct(src, iF - 1, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 1, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF - 1, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord2_direct(src, iF + 1, jF + 1, kF, SoA0, SoA1, SoA2));
|
|
fxz = d_gp.Sdxdz * (
|
|
fetch_sym_ord2_direct(src, iF - 1, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF + 1, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF - 1, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord2_direct(src, iF + 1, jF, kF + 1, SoA0, SoA1, SoA2));
|
|
fyz = d_gp.Sdydz * (
|
|
fetch_sym_ord2_direct(src, iF, jF - 1, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF + 1, kF - 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord2_direct(src, iF, jF - 1, kF + 1, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord2_direct(src, iF, jF + 1, kF + 1, SoA0, SoA1, SoA2));
|
|
}
|
|
#endif
|
|
}
|
|
|
|
dst[tid] = gupxx[tid] * fxx + gupyy[tid] * fyy + gupzz[tid] * fzz
|
|
+ 2.0 * (gupxy[tid] * fxy + gupxz[tid] * fxz + gupyz[tid] * fyz);
|
|
}
|
|
|
|
static void gpu_phase10_ricci_batch(const double *gupxx,
|
|
const double *gupxy,
|
|
const double *gupxz,
|
|
const double *gupyy,
|
|
const double *gupyz,
|
|
const double *gupzz,
|
|
double *const *src_fields,
|
|
double *const *dst_fields,
|
|
const int *soa_signs,
|
|
int all)
|
|
{
|
|
Phase10RicciTables tables = {};
|
|
for (int i = 0; i < PHASE10_METRIC_FIELD_COUNT; ++i) {
|
|
tables.src_fields[i] = src_fields[i];
|
|
tables.dst_fields[i] = dst_fields[i];
|
|
tables.soa_signs[3 * i + 0] = soa_signs[3 * i + 0];
|
|
tables.soa_signs[3 * i + 1] = soa_signs[3 * i + 1];
|
|
tables.soa_signs[3 * i + 2] = soa_signs[3 * i + 2];
|
|
}
|
|
dim3 launch_grid((unsigned int)grid((size_t)all), (unsigned int)PHASE10_METRIC_FIELD_COUNT);
|
|
kern_phase10_ricci_batched<<<launch_grid, BLK>>>(
|
|
gupxx, gupxy, gupxz, gupyy, gupyz, gupzz, tables);
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase14_lap_chi_derivs(const double * __restrict__ Lap,
|
|
const double * __restrict__ chi,
|
|
double * __restrict__ fxx,
|
|
double * __restrict__ fxy,
|
|
double * __restrict__ fxz,
|
|
double * __restrict__ fyy,
|
|
double * __restrict__ fyz,
|
|
double * __restrict__ fzz,
|
|
double * __restrict__ chix_out,
|
|
double * __restrict__ chiy_out,
|
|
double * __restrict__ chiz_out)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const int iminF = d_gp.iminF, jminF = d_gp.jminF, kminF = d_gp.kminF;
|
|
const int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (tid >= d_gp.all) return;
|
|
|
|
const int i0 = tid % nx;
|
|
const int j0 = (tid / nx) % ny;
|
|
const int k0 = tid / (nx * ny);
|
|
|
|
if (i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2) {
|
|
fxx[tid] = 0.0; fxy[tid] = 0.0; fxz[tid] = 0.0;
|
|
fyy[tid] = 0.0; fyz[tid] = 0.0; fzz[tid] = 0.0;
|
|
chix_out[tid] = 0.0; chiy_out[tid] = 0.0; chiz_out[tid] = 0.0;
|
|
return;
|
|
}
|
|
|
|
const int iF = i0 + 1;
|
|
const int jF = j0 + 1;
|
|
const int kF = k0 + 1;
|
|
|
|
#if ghost_width != 3
|
|
fd_compute_second6(Lap, iF, jF, kF,
|
|
iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
|
|
1, 1, 1,
|
|
fxx[tid], fxy[tid], fxz[tid], fyy[tid], fyz[tid], fzz[tid]);
|
|
fd_compute_first3(chi, iF, jF, kF,
|
|
iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
|
|
1, 1, 1,
|
|
chix_out[tid], chiy_out[tid], chiz_out[tid]);
|
|
#else
|
|
if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
|
|
(jF + 2) <= jmaxF && (jF - 2) >= jminF &&
|
|
(kF + 2) <= kmaxF && (kF - 2) >= kminF)
|
|
{
|
|
const double lap_c = fetch_sym_ord2_direct(Lap, iF, jF, kF, 1, 1, 1);
|
|
fxx[tid] = d_gp.Fdxdx * (
|
|
-fetch_sym_ord2_direct(Lap, iF - 2, jF, kF, 1, 1, 1)
|
|
+16.0 * fetch_sym_ord2_direct(Lap, iF - 1, jF, kF, 1, 1, 1)
|
|
-30.0 * lap_c
|
|
+16.0 * fetch_sym_ord2_direct(Lap, iF + 1, jF, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 2, jF, kF, 1, 1, 1));
|
|
fyy[tid] = d_gp.Fdydy * (
|
|
-fetch_sym_ord2_direct(Lap, iF, jF - 2, kF, 1, 1, 1)
|
|
+16.0 * fetch_sym_ord2_direct(Lap, iF, jF - 1, kF, 1, 1, 1)
|
|
-30.0 * lap_c
|
|
+16.0 * fetch_sym_ord2_direct(Lap, iF, jF + 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF, jF + 2, kF, 1, 1, 1));
|
|
fzz[tid] = d_gp.Fdzdz * (
|
|
-fetch_sym_ord2_direct(Lap, iF, jF, kF - 2, 1, 1, 1)
|
|
+16.0 * fetch_sym_ord2_direct(Lap, iF, jF, kF - 1, 1, 1, 1)
|
|
-30.0 * lap_c
|
|
+16.0 * fetch_sym_ord2_direct(Lap, iF, jF, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF, jF, kF + 2, 1, 1, 1));
|
|
|
|
const double t_jm2 =
|
|
fetch_sym_ord2_direct(Lap, iF - 2, jF - 2, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF - 1, jF - 2, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF + 1, jF - 2, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 2, jF - 2, kF, 1, 1, 1);
|
|
const double t_jm1 =
|
|
fetch_sym_ord2_direct(Lap, iF - 2, jF - 1, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF - 1, jF - 1, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF + 1, jF - 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 2, jF - 1, kF, 1, 1, 1);
|
|
const double t_jp1 =
|
|
fetch_sym_ord2_direct(Lap, iF - 2, jF + 1, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF - 1, jF + 1, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF + 1, jF + 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 2, jF + 1, kF, 1, 1, 1);
|
|
const double t_jp2 =
|
|
fetch_sym_ord2_direct(Lap, iF - 2, jF + 2, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF - 1, jF + 2, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF + 1, jF + 2, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 2, jF + 2, kF, 1, 1, 1);
|
|
fxy[tid] = d_gp.Fdxdy * (t_jm2 - 8.0 * t_jm1 + 8.0 * t_jp1 - t_jp2);
|
|
|
|
const double t_km2_x =
|
|
fetch_sym_ord2_direct(Lap, iF - 2, jF, kF - 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF - 1, jF, kF - 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF + 1, jF, kF - 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 2, jF, kF - 2, 1, 1, 1);
|
|
const double t_km1_x =
|
|
fetch_sym_ord2_direct(Lap, iF - 2, jF, kF - 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF - 1, jF, kF - 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF + 1, jF, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 2, jF, kF - 1, 1, 1, 1);
|
|
const double t_kp1_x =
|
|
fetch_sym_ord2_direct(Lap, iF - 2, jF, kF + 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF - 1, jF, kF + 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF + 1, jF, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 2, jF, kF + 1, 1, 1, 1);
|
|
const double t_kp2_x =
|
|
fetch_sym_ord2_direct(Lap, iF - 2, jF, kF + 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF - 1, jF, kF + 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF + 1, jF, kF + 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 2, jF, kF + 2, 1, 1, 1);
|
|
fxz[tid] = d_gp.Fdxdz * (t_km2_x - 8.0 * t_km1_x + 8.0 * t_kp1_x - t_kp2_x);
|
|
|
|
const double t_km2_y =
|
|
fetch_sym_ord2_direct(Lap, iF, jF - 2, kF - 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF, jF - 1, kF - 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF, jF + 1, kF - 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF, jF + 2, kF - 2, 1, 1, 1);
|
|
const double t_km1_y =
|
|
fetch_sym_ord2_direct(Lap, iF, jF - 2, kF - 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF, jF - 1, kF - 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF, jF + 1, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF, jF + 2, kF - 1, 1, 1, 1);
|
|
const double t_kp1_y =
|
|
fetch_sym_ord2_direct(Lap, iF, jF - 2, kF + 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF, jF - 1, kF + 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF, jF + 1, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF, jF + 2, kF + 1, 1, 1, 1);
|
|
const double t_kp2_y =
|
|
fetch_sym_ord2_direct(Lap, iF, jF - 2, kF + 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(Lap, iF, jF - 1, kF + 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(Lap, iF, jF + 1, kF + 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF, jF + 2, kF + 2, 1, 1, 1);
|
|
fyz[tid] = d_gp.Fdydz * (t_km2_y - 8.0 * t_km1_y + 8.0 * t_kp1_y - t_kp2_y);
|
|
|
|
chix_out[tid] = d_gp.d12dx * (
|
|
fetch_sym_ord2_direct(chi, iF - 2, jF, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF - 1, jF, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF + 1, jF, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF, kF, 1, 1, 1));
|
|
chiy_out[tid] = d_gp.d12dy * (
|
|
fetch_sym_ord2_direct(chi, iF, jF - 2, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF, jF - 1, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF, jF + 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF + 2, kF, 1, 1, 1));
|
|
chiz_out[tid] = d_gp.d12dz * (
|
|
fetch_sym_ord2_direct(chi, iF, jF, kF - 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF, jF, kF - 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF, jF, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF, kF + 2, 1, 1, 1));
|
|
}
|
|
else if ((iF + 1) <= imaxF && (iF - 1) >= iminF &&
|
|
(jF + 1) <= jmaxF && (jF - 1) >= jminF &&
|
|
(kF + 1) <= kmaxF && (kF - 1) >= kminF)
|
|
{
|
|
const double lap_c = fetch_sym_ord2_direct(Lap, iF, jF, kF, 1, 1, 1);
|
|
fxx[tid] = d_gp.Sdxdx * (
|
|
fetch_sym_ord2_direct(Lap, iF - 1, jF, kF, 1, 1, 1)
|
|
- 2.0 * lap_c
|
|
+ fetch_sym_ord2_direct(Lap, iF + 1, jF, kF, 1, 1, 1));
|
|
fyy[tid] = d_gp.Sdydy * (
|
|
fetch_sym_ord2_direct(Lap, iF, jF - 1, kF, 1, 1, 1)
|
|
- 2.0 * lap_c
|
|
+ fetch_sym_ord2_direct(Lap, iF, jF + 1, kF, 1, 1, 1));
|
|
fzz[tid] = d_gp.Sdzdz * (
|
|
fetch_sym_ord2_direct(Lap, iF, jF, kF - 1, 1, 1, 1)
|
|
- 2.0 * lap_c
|
|
+ fetch_sym_ord2_direct(Lap, iF, jF, kF + 1, 1, 1, 1));
|
|
fxy[tid] = d_gp.Sdxdy * (
|
|
fetch_sym_ord2_direct(Lap, iF - 1, jF - 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 1, jF - 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF - 1, jF + 1, kF, 1, 1, 1)
|
|
+ fetch_sym_ord2_direct(Lap, iF + 1, jF + 1, kF, 1, 1, 1));
|
|
fxz[tid] = d_gp.Sdxdz * (
|
|
fetch_sym_ord2_direct(Lap, iF - 1, jF, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF + 1, jF, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF - 1, jF, kF + 1, 1, 1, 1)
|
|
+ fetch_sym_ord2_direct(Lap, iF + 1, jF, kF + 1, 1, 1, 1));
|
|
fyz[tid] = d_gp.Sdydz * (
|
|
fetch_sym_ord2_direct(Lap, iF, jF - 1, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF, jF + 1, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(Lap, iF, jF - 1, kF + 1, 1, 1, 1)
|
|
+ fetch_sym_ord2_direct(Lap, iF, jF + 1, kF + 1, 1, 1, 1));
|
|
chix_out[tid] = d_gp.d2dx * (
|
|
-fetch_sym_ord2_direct(chi, iF - 1, jF, kF, 1, 1, 1)
|
|
+fetch_sym_ord2_direct(chi, iF + 1, jF, kF, 1, 1, 1));
|
|
chiy_out[tid] = d_gp.d2dy * (
|
|
-fetch_sym_ord2_direct(chi, iF, jF - 1, kF, 1, 1, 1)
|
|
+fetch_sym_ord2_direct(chi, iF, jF + 1, kF, 1, 1, 1));
|
|
chiz_out[tid] = d_gp.d2dz * (
|
|
-fetch_sym_ord2_direct(chi, iF, jF, kF - 1, 1, 1, 1)
|
|
+fetch_sym_ord2_direct(chi, iF, jF, kF + 1, 1, 1, 1));
|
|
}
|
|
else {
|
|
fxx[tid] = 0.0; fxy[tid] = 0.0; fxz[tid] = 0.0;
|
|
fyy[tid] = 0.0; fyz[tid] = 0.0; fzz[tid] = 0.0;
|
|
chix_out[tid] = 0.0; chiy_out[tid] = 0.0; chiz_out[tid] = 0.0;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
/* Combined ord=3 advection + KO dissipation.
|
|
* When advection and KO use the same source field, symmetry packing is shared.
|
|
* If they differ (e.g. gxx advection + dxx KO), only KO repacks.
|
|
*/
|
|
static void gpu_lopsided_kodis(double *d_f_adv, double *d_f_ko, double *d_f_rhs,
|
|
double *d_Sfx, double *d_Sfy, double *d_Sfz,
|
|
double SoA0, double SoA1, double SoA2,
|
|
double eps_val, int all)
|
|
{
|
|
#if ghost_width != 3
|
|
gpu_lopsided_kodis_single_batch(d_f_adv, d_f_ko, d_f_rhs,
|
|
d_Sfx, d_Sfy, d_Sfz,
|
|
SoA0, SoA1, SoA2, eps_val, all);
|
|
#else
|
|
double *fh = g_buf.d_fh3;
|
|
const size_t nx = (size_t)g_buf.prev_nx;
|
|
const size_t ny = (size_t)g_buf.prev_ny;
|
|
const size_t nz = (size_t)g_buf.prev_nz;
|
|
const size_t w_pack = (nx + 3ull) * (ny + 3ull) * (nz + 3ull);
|
|
|
|
kern_symbd_pack_ord3<<<grid(w_pack), BLK>>>(d_f_adv, fh, SoA0, SoA1, SoA2);
|
|
kern_lopsided<<<grid(all), BLK>>>(fh, d_f_rhs, d_Sfx, d_Sfy, d_Sfz);
|
|
|
|
if (eps_val > 0.0) {
|
|
if (d_f_ko != d_f_adv) {
|
|
kern_symbd_pack_ord3<<<grid(w_pack), BLK>>>(d_f_ko, fh, SoA0, SoA1, SoA2);
|
|
}
|
|
kern_kodis<<<grid(all), BLK>>>(fh, d_f_rhs, eps_val);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_lopsided_kodis_batched(const double * __restrict__ Sfx,
|
|
const double * __restrict__ Sfy,
|
|
const double * __restrict__ Sfz,
|
|
LopsidedKodisTables tables,
|
|
double eps_val,
|
|
int do_lopsided,
|
|
int do_kodis)
|
|
{
|
|
const int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (tid >= d_gp.all) return;
|
|
|
|
const int field = blockIdx.y;
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int iminF = d_gp.iminF3, jminF = d_gp.jminF3, kminF = d_gp.kminF3;
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const int SoA0 = tables.soa_signs[3 * field + 0];
|
|
const int SoA1 = tables.soa_signs[3 * field + 1];
|
|
const int SoA2 = tables.soa_signs[3 * field + 2];
|
|
const double *adv_src = tables.adv_fields[field];
|
|
const double *ko_src = tables.ko_fields[field];
|
|
double *rhs = tables.rhs_fields[field];
|
|
|
|
const int i0 = tid % nx;
|
|
const int j0 = (tid / nx) % ny;
|
|
const int k0 = tid / (nx * ny);
|
|
const int iF = i0 + 1;
|
|
const int jF = j0 + 1;
|
|
const int kF = k0 + 1;
|
|
|
|
#if ghost_width != 3
|
|
if (do_lopsided && i0 <= nx - 2 && j0 <= ny - 2 && k0 <= nz - 2) {
|
|
const double val =
|
|
fd_lopsided_axis(adv_src, iF, jF, kF, 0, Sfx[tid], iF, iminF, imaxF,
|
|
d_gp.dX, SoA0, SoA1, SoA2)
|
|
+ fd_lopsided_axis(adv_src, iF, jF, kF, 1, Sfy[tid], jF, jminF, jmaxF,
|
|
d_gp.dY, SoA0, SoA1, SoA2)
|
|
+ fd_lopsided_axis(adv_src, iF, jF, kF, 2, Sfz[tid], kF, kminF, kmaxF,
|
|
d_gp.dZ, SoA0, SoA1, SoA2);
|
|
rhs[tid] += val;
|
|
}
|
|
|
|
if (do_kodis) {
|
|
rhs[tid] += fd_ko_term(ko_src, iF, jF, kF,
|
|
iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
|
|
eps_val, SoA0, SoA1, SoA2);
|
|
}
|
|
#else
|
|
if (do_lopsided && i0 <= nx - 2 && j0 <= ny - 2 && k0 <= nz - 2) {
|
|
double val = 0.0;
|
|
|
|
const double sfx = Sfx[tid];
|
|
if (sfx > 0.0) {
|
|
if (i0 <= nx - 4) {
|
|
val += sfx * d_gp.d12dx * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF + 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF + 2, jF, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF + 3, jF, kF, SoA0, SoA1, SoA2));
|
|
} else if (i0 <= nx - 3) {
|
|
val += sfx * d_gp.d12dx * (
|
|
fetch_sym_ord3_direct(adv_src, iF - 2, jF, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord3_direct(adv_src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord3_direct(adv_src, iF + 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord3_direct(adv_src, iF + 2, jF, kF, SoA0, SoA1, SoA2));
|
|
} else {
|
|
val -= sfx * d_gp.d12dx * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF + 1, jF, kF, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF - 2, jF, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF - 3, jF, kF, SoA0, SoA1, SoA2));
|
|
}
|
|
} else if (sfx < 0.0) {
|
|
if ((i0 - 2) >= iminF) {
|
|
val -= sfx * d_gp.d12dx * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF + 1, jF, kF, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF - 2, jF, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF - 3, jF, kF, SoA0, SoA1, SoA2));
|
|
} else if ((i0 - 1) >= iminF) {
|
|
val += sfx * d_gp.d12dx * (
|
|
fetch_sym_ord3_direct(adv_src, iF - 2, jF, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord3_direct(adv_src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord3_direct(adv_src, iF + 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord3_direct(adv_src, iF + 2, jF, kF, SoA0, SoA1, SoA2));
|
|
} else if (i0 >= iminF) {
|
|
val += sfx * d_gp.d12dx * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF + 1, jF, kF, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF + 2, jF, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF + 3, jF, kF, SoA0, SoA1, SoA2));
|
|
}
|
|
}
|
|
|
|
const double sfy = Sfy[tid];
|
|
if (sfy > 0.0) {
|
|
if (j0 <= ny - 4) {
|
|
val += sfy * d_gp.d12dy * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF, jF + 2, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF, jF + 3, kF, SoA0, SoA1, SoA2));
|
|
} else if (j0 <= ny - 3) {
|
|
val += sfy * d_gp.d12dy * (
|
|
fetch_sym_ord3_direct(adv_src, iF, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord3_direct(adv_src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord3_direct(adv_src, iF, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord3_direct(adv_src, iF, jF + 2, kF, SoA0, SoA1, SoA2));
|
|
} else {
|
|
val -= sfy * d_gp.d12dy * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF, jF - 3, kF, SoA0, SoA1, SoA2));
|
|
}
|
|
} else if (sfy < 0.0) {
|
|
if ((j0 - 2) >= jminF) {
|
|
val -= sfy * d_gp.d12dy * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF, jF - 3, kF, SoA0, SoA1, SoA2));
|
|
} else if ((j0 - 1) >= jminF) {
|
|
val += sfy * d_gp.d12dy * (
|
|
fetch_sym_ord3_direct(adv_src, iF, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord3_direct(adv_src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord3_direct(adv_src, iF, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord3_direct(adv_src, iF, jF + 2, kF, SoA0, SoA1, SoA2));
|
|
} else if (j0 >= jminF) {
|
|
val += sfy * d_gp.d12dy * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF, jF + 1, kF, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF, jF + 2, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF, jF + 3, kF, SoA0, SoA1, SoA2));
|
|
}
|
|
}
|
|
|
|
const double sfz = Sfz[tid];
|
|
if (sfz > 0.0) {
|
|
if (k0 <= nz - 4) {
|
|
val += sfz * d_gp.d12dz * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF + 2, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF, jF, kF + 3, SoA0, SoA1, SoA2));
|
|
} else if (k0 <= nz - 3) {
|
|
val += sfz * d_gp.d12dz * (
|
|
fetch_sym_ord3_direct(adv_src, iF, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord3_direct(adv_src, iF, jF, kF + 2, SoA0, SoA1, SoA2));
|
|
} else {
|
|
val -= sfz * d_gp.d12dz * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF, jF, kF - 3, SoA0, SoA1, SoA2));
|
|
}
|
|
} else if (sfz < 0.0) {
|
|
if ((k0 - 2) >= kminF) {
|
|
val -= sfz * d_gp.d12dz * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF, jF, kF - 3, SoA0, SoA1, SoA2));
|
|
} else if ((k0 - 1) >= kminF) {
|
|
val += sfz * d_gp.d12dz * (
|
|
fetch_sym_ord3_direct(adv_src, iF, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
- 8.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
+ 8.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- fetch_sym_ord3_direct(adv_src, iF, jF, kF + 2, SoA0, SoA1, SoA2));
|
|
} else if (k0 >= kminF) {
|
|
val += sfz * d_gp.d12dz * (
|
|
-3.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
-10.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF, SoA0, SoA1, SoA2)
|
|
+18.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF + 1, SoA0, SoA1, SoA2)
|
|
- 6.0 * fetch_sym_ord3_direct(adv_src, iF, jF, kF + 2, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(adv_src, iF, jF, kF + 3, SoA0, SoA1, SoA2));
|
|
}
|
|
}
|
|
|
|
rhs[tid] += val;
|
|
}
|
|
|
|
if (do_kodis && eps_val > 0.0 &&
|
|
(iF - 3) >= iminF && (iF + 3) <= imaxF &&
|
|
(jF - 3) >= jminF && (jF + 3) <= jmaxF &&
|
|
(kF - 3) >= kminF && (kF + 3) <= kmaxF)
|
|
{
|
|
const double cof = 64.0;
|
|
const double Dx =
|
|
(fetch_sym_ord3_direct(ko_src, iF - 3, jF, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(ko_src, iF + 3, jF, kF, SoA0, SoA1, SoA2))
|
|
- 6.0 * (fetch_sym_ord3_direct(ko_src, iF - 2, jF, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(ko_src, iF + 2, jF, kF, SoA0, SoA1, SoA2))
|
|
+ 15.0 * (fetch_sym_ord3_direct(ko_src, iF - 1, jF, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(ko_src, iF + 1, jF, kF, SoA0, SoA1, SoA2))
|
|
- 20.0 * fetch_sym_ord3_direct(ko_src, iF, jF, kF, SoA0, SoA1, SoA2);
|
|
|
|
const double Dy =
|
|
(fetch_sym_ord3_direct(ko_src, iF, jF - 3, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(ko_src, iF, jF + 3, kF, SoA0, SoA1, SoA2))
|
|
- 6.0 * (fetch_sym_ord3_direct(ko_src, iF, jF - 2, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(ko_src, iF, jF + 2, kF, SoA0, SoA1, SoA2))
|
|
+ 15.0 * (fetch_sym_ord3_direct(ko_src, iF, jF - 1, kF, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(ko_src, iF, jF + 1, kF, SoA0, SoA1, SoA2))
|
|
- 20.0 * fetch_sym_ord3_direct(ko_src, iF, jF, kF, SoA0, SoA1, SoA2);
|
|
|
|
const double Dz =
|
|
(fetch_sym_ord3_direct(ko_src, iF, jF, kF - 3, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(ko_src, iF, jF, kF + 3, SoA0, SoA1, SoA2))
|
|
- 6.0 * (fetch_sym_ord3_direct(ko_src, iF, jF, kF - 2, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(ko_src, iF, jF, kF + 2, SoA0, SoA1, SoA2))
|
|
+ 15.0 * (fetch_sym_ord3_direct(ko_src, iF, jF, kF - 1, SoA0, SoA1, SoA2)
|
|
+ fetch_sym_ord3_direct(ko_src, iF, jF, kF + 1, SoA0, SoA1, SoA2))
|
|
- 20.0 * fetch_sym_ord3_direct(ko_src, iF, jF, kF, SoA0, SoA1, SoA2);
|
|
|
|
rhs[tid] += (eps_val / cof) * (Dx / d_gp.dX + Dy / d_gp.dY + Dz / d_gp.dZ);
|
|
}
|
|
#endif
|
|
}
|
|
|
|
static void gpu_lopsided_kodis_single_batch(double *d_f_adv, double *d_f_ko, double *d_f_rhs,
|
|
double *d_Sfx, double *d_Sfy, double *d_Sfz,
|
|
double SoA0, double SoA1, double SoA2,
|
|
double eps_val, int all)
|
|
{
|
|
LopsidedKodisTables tables = {};
|
|
tables.adv_fields[0] = d_f_adv;
|
|
tables.ko_fields[0] = d_f_ko;
|
|
tables.rhs_fields[0] = d_f_rhs;
|
|
tables.soa_signs[0] = (int)SoA0;
|
|
tables.soa_signs[1] = (int)SoA1;
|
|
tables.soa_signs[2] = (int)SoA2;
|
|
|
|
dim3 launch_grid((unsigned int)grid((size_t)all), 1u);
|
|
kern_lopsided_kodis_batched<<<launch_grid, BLK>>>(
|
|
d_Sfx, d_Sfy, d_Sfz, tables, eps_val, 1, eps_val > 0.0 ? 1 : 0);
|
|
}
|
|
|
|
static void gpu_lopsided_kodis_state_batch(double eps_val, int all)
|
|
{
|
|
LopsidedKodisTables tables = {};
|
|
for (int i = 0; i < BSSN_LK_FIELD_COUNT; ++i) {
|
|
tables.adv_fields[i] = g_buf.slot[k_lk_adv_slots[i]];
|
|
tables.ko_fields[i] = g_buf.slot[k_lk_ko_slots[i]];
|
|
tables.rhs_fields[i] = g_buf.slot[k_lk_rhs_slots[i]];
|
|
}
|
|
std::memcpy(tables.soa_signs, k_lk_soa_signs, sizeof(k_lk_soa_signs));
|
|
|
|
dim3 launch_grid((unsigned int)grid((size_t)all), (unsigned int)BSSN_LK_FIELD_COUNT);
|
|
kern_lopsided_kodis_batched<<<launch_grid, BLK>>>(
|
|
g_buf.slot[S_betax], g_buf.slot[S_betay], g_buf.slot[S_betaz], tables, eps_val, 1, 1);
|
|
}
|
|
|
|
__global__ void kern_rk4_finalize(const double * __restrict__ f0,
|
|
double * __restrict__ frhs,
|
|
double * __restrict__ accum,
|
|
double dT,
|
|
int rk4_stage)
|
|
{
|
|
for (int i = blockIdx.x * blockDim.x + threadIdx.x;
|
|
i < d_gp.all;
|
|
i += blockDim.x * gridDim.x)
|
|
{
|
|
const double rhs = frhs[i];
|
|
switch (rk4_stage) {
|
|
case 0:
|
|
accum[i] = rhs;
|
|
frhs[i] = f0[i] + 0.5 * dT * rhs;
|
|
break;
|
|
case 1:
|
|
accum[i] += 2.0 * rhs;
|
|
frhs[i] = f0[i] + 0.5 * dT * rhs;
|
|
break;
|
|
case 2:
|
|
accum[i] += 2.0 * rhs;
|
|
frhs[i] = f0[i] + dT * rhs;
|
|
break;
|
|
default:
|
|
frhs[i] = f0[i] + (dT / 6.0) * (accum[i] + rhs);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_rk4_finalize_batched(Rk4FinalizeTables tables,
|
|
double dT,
|
|
int rk4_stage,
|
|
double chitiny)
|
|
{
|
|
const int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (tid >= d_gp.all) return;
|
|
|
|
const int field = blockIdx.y;
|
|
const double *f0 = tables.f0_fields[field];
|
|
double *frhs = tables.rhs_fields[field];
|
|
double *accum = tables.accum_fields[field];
|
|
|
|
const double rhs = frhs[tid];
|
|
switch (rk4_stage) {
|
|
case 0:
|
|
accum[tid] = rhs;
|
|
frhs[tid] = f0[tid] + 0.5 * dT * rhs;
|
|
break;
|
|
case 1:
|
|
accum[tid] += 2.0 * rhs;
|
|
frhs[tid] = f0[tid] + 0.5 * dT * rhs;
|
|
break;
|
|
case 2:
|
|
accum[tid] += 2.0 * rhs;
|
|
frhs[tid] = f0[tid] + dT * rhs;
|
|
break;
|
|
default:
|
|
frhs[tid] = f0[tid] + (dT / 6.0) * (accum[tid] + rhs);
|
|
break;
|
|
}
|
|
|
|
if (field == 0 && frhs[tid] < chitiny) frhs[tid] = chitiny;
|
|
}
|
|
|
|
static void gpu_rk4_finalize_batch(const StepContext &ctx,
|
|
size_t all,
|
|
double dT,
|
|
int rk4_stage,
|
|
double chitiny)
|
|
{
|
|
Rk4FinalizeTables tables = {};
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
tables.f0_fields[i] = ctx.d_state0[i];
|
|
tables.rhs_fields[i] = g_buf.slot[k_state_rhs_slots[i]];
|
|
tables.accum_fields[i] = ctx.d_accum[i];
|
|
}
|
|
|
|
dim3 launch_grid((unsigned int)grid(all), (unsigned int)BSSN_STATE_COUNT);
|
|
kern_rk4_finalize_batched<<<launch_grid, BLK>>>(tables, dT, rk4_stage, chitiny);
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_copy_patch_boundary_batched(PatchBoundaryTables tables,
|
|
int touch_xmin, int touch_xmax,
|
|
int touch_ymin, int touch_ymax,
|
|
int touch_zmin, int touch_zmax)
|
|
{
|
|
const int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (tid >= d_gp.all) return;
|
|
|
|
const int nx = d_gp.ex[0];
|
|
const int ny = d_gp.ex[1];
|
|
const int i0 = tid % nx;
|
|
const int j0 = (tid / nx) % ny;
|
|
const int k0 = tid / (nx * ny);
|
|
|
|
const bool on_boundary =
|
|
(touch_xmin && i0 == 0) ||
|
|
(touch_xmax && i0 == nx - 1) ||
|
|
(touch_ymin && j0 == 0) ||
|
|
(touch_ymax && j0 == ny - 1) ||
|
|
(touch_zmin && k0 == 0) ||
|
|
(touch_zmax && k0 == d_gp.ex[2] - 1);
|
|
if (!on_boundary) return;
|
|
|
|
const int field = blockIdx.y;
|
|
tables.dst_fields[field][tid] = tables.src_fields[field][tid];
|
|
}
|
|
|
|
static void gpu_copy_patch_boundary_batch(const StepContext &ctx,
|
|
int all,
|
|
int touch_xmin, int touch_xmax,
|
|
int touch_ymin, int touch_ymax,
|
|
int touch_zmin, int touch_zmax)
|
|
{
|
|
if (!(touch_xmin || touch_xmax || touch_ymin || touch_ymax || touch_zmin || touch_zmax))
|
|
return;
|
|
|
|
PatchBoundaryTables tables = {};
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
tables.src_fields[i] = ctx.d_state0[i];
|
|
tables.dst_fields[i] = g_buf.slot[k_state_rhs_slots[i]];
|
|
}
|
|
|
|
dim3 launch_grid((unsigned int)grid((size_t)all), (unsigned int)BSSN_STATE_COUNT);
|
|
kern_copy_patch_boundary_batched<<<launch_grid, BLK>>>(
|
|
tables,
|
|
touch_xmin, touch_xmax,
|
|
touch_ymin, touch_ymax,
|
|
touch_zmin, touch_zmax);
|
|
}
|
|
|
|
__global__ void kern_enforce_ga_cuda(double * __restrict__ dxx,
|
|
double * __restrict__ gxy,
|
|
double * __restrict__ gxz,
|
|
double * __restrict__ dyy,
|
|
double * __restrict__ gyz,
|
|
double * __restrict__ dzz,
|
|
double * __restrict__ Axx,
|
|
double * __restrict__ Axy,
|
|
double * __restrict__ Axz,
|
|
double * __restrict__ Ayy,
|
|
double * __restrict__ Ayz,
|
|
double * __restrict__ Azz)
|
|
{
|
|
constexpr double F1O3 = 1.0 / 3.0;
|
|
constexpr double ONE = 1.0;
|
|
constexpr double TWO = 2.0;
|
|
|
|
for (int i = blockIdx.x * blockDim.x + threadIdx.x;
|
|
i < d_gp.all;
|
|
i += blockDim.x * gridDim.x)
|
|
{
|
|
double lgxx = dxx[i] + ONE;
|
|
double lgyy = dyy[i] + ONE;
|
|
double lgzz = dzz[i] + ONE;
|
|
double lgxy = gxy[i];
|
|
double lgxz = gxz[i];
|
|
double lgyz = gyz[i];
|
|
|
|
double lscale = lgxx * lgyy * lgzz
|
|
+ lgxy * lgyz * lgxz
|
|
+ lgxz * lgxy * lgyz
|
|
- lgxz * lgyy * lgxz
|
|
- lgxy * lgxy * lgzz
|
|
- lgxx * lgyz * lgyz;
|
|
|
|
lscale = ONE / cbrt(lscale);
|
|
|
|
lgxx *= lscale;
|
|
lgxy *= lscale;
|
|
lgxz *= lscale;
|
|
lgyy *= lscale;
|
|
lgyz *= lscale;
|
|
lgzz *= lscale;
|
|
|
|
dxx[i] = lgxx - ONE;
|
|
gxy[i] = lgxy;
|
|
gxz[i] = lgxz;
|
|
dyy[i] = lgyy - ONE;
|
|
gyz[i] = lgyz;
|
|
dzz[i] = lgzz - ONE;
|
|
|
|
const double lgupxx = (lgyy * lgzz - lgyz * lgyz);
|
|
const double lgupxy = - (lgxy * lgzz - lgyz * lgxz);
|
|
const double lgupxz = (lgxy * lgyz - lgyy * lgxz);
|
|
const double lgupyy = (lgxx * lgzz - lgxz * lgxz);
|
|
const double lgupyz = - (lgxx * lgyz - lgxy * lgxz);
|
|
const double lgupzz = (lgxx * lgyy - lgxy * lgxy);
|
|
|
|
const double ltrA = lgupxx * Axx[i] + lgupyy * Ayy[i] + lgupzz * Azz[i]
|
|
+ TWO * (lgupxy * Axy[i] + lgupxz * Axz[i] + lgupyz * Ayz[i]);
|
|
|
|
Axx[i] -= F1O3 * lgxx * ltrA;
|
|
Axy[i] -= F1O3 * lgxy * ltrA;
|
|
Axz[i] -= F1O3 * lgxz * ltrA;
|
|
Ayy[i] -= F1O3 * lgyy * ltrA;
|
|
Ayz[i] -= F1O3 * lgyz * ltrA;
|
|
Azz[i] -= F1O3 * lgzz * ltrA;
|
|
}
|
|
}
|
|
|
|
__global__ void kern_lowerboundset_cuda(double * __restrict__ chi, double tinny)
|
|
{
|
|
for (int i = blockIdx.x * blockDim.x + threadIdx.x;
|
|
i < d_gp.all;
|
|
i += blockDim.x * gridDim.x)
|
|
{
|
|
if (chi[i] < tinny) chi[i] = tinny;
|
|
}
|
|
}
|
|
|
|
enum SommerFace {
|
|
FACE_XMAX = 0,
|
|
FACE_YMAX = 1,
|
|
FACE_ZMAX = 2,
|
|
FACE_XMIN = 3,
|
|
FACE_YMIN = 4,
|
|
FACE_ZMIN = 5
|
|
};
|
|
|
|
__global__ void kern_sommerfeld_face_bam(const double * __restrict__ fh,
|
|
double * __restrict__ f_rhs,
|
|
int face,
|
|
double velocity,
|
|
double x0,
|
|
double y0,
|
|
double z0)
|
|
{
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int imin = (d_gp.Symmetry > 1 && fabs(x0) < d_gp.dX) ? 0 : 1;
|
|
const int jmin = (d_gp.Symmetry > 1 && fabs(y0) < d_gp.dY) ? 0 : 1;
|
|
const int kmin = (d_gp.Symmetry > 0 && fabs(z0) < d_gp.dZ) ? 0 : 1;
|
|
const int imax = nx, jmax = ny, kmax = nz;
|
|
const int plane_count =
|
|
(face == FACE_XMAX || face == FACE_XMIN) ? ny * nz :
|
|
(face == FACE_YMAX || face == FACE_YMIN) ? nx * nz :
|
|
nx * ny;
|
|
|
|
for (int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
tid < plane_count;
|
|
tid += blockDim.x * gridDim.x)
|
|
{
|
|
int i0 = 0, j0 = 0, k0 = 0;
|
|
if (face == FACE_XMAX || face == FACE_XMIN) {
|
|
j0 = tid % ny;
|
|
k0 = tid / ny;
|
|
i0 = (face == FACE_XMAX) ? (nx - 1) : 0;
|
|
} else if (face == FACE_YMAX || face == FACE_YMIN) {
|
|
i0 = tid % nx;
|
|
k0 = tid / nx;
|
|
j0 = (face == FACE_YMAX) ? (ny - 1) : 0;
|
|
} else {
|
|
i0 = tid % nx;
|
|
j0 = tid / nx;
|
|
k0 = (face == FACE_ZMAX) ? (nz - 1) : 0;
|
|
}
|
|
|
|
const int iF = i0 + 1;
|
|
const int jF = j0 + 1;
|
|
const int kF = k0 + 1;
|
|
const int p = idx_ex_d(i0, j0, k0);
|
|
|
|
const double x = x0 + i0 * d_gp.dX;
|
|
const double y = y0 + j0 * d_gp.dY;
|
|
const double z = z0 + k0 * d_gp.dZ;
|
|
const double r = sqrt(x * x + y * y + z * z);
|
|
if (r == 0.0) continue;
|
|
|
|
const double wx = velocity * x / r;
|
|
const double wy = velocity * y / r;
|
|
const double wz = velocity * z / r;
|
|
|
|
double fx = 0.0, fy = 0.0, fz = 0.0;
|
|
if (wx > 0.0) {
|
|
if (iF - 2 >= imin) fx = d_gp.d2dx * (3.0 * fh[idx_fh2(iF, jF, kF)] - 4.0 * fh[idx_fh2(iF - 1, jF, kF)] + fh[idx_fh2(iF - 2, jF, kF)]);
|
|
else if (iF - 1 >= imin) fx = d_gp.d2dx * (-fh[idx_fh2(iF - 1, jF, kF)] + fh[idx_fh2(iF + 1, jF, kF)]);
|
|
else fx = d_gp.d2dx * (-fh[idx_fh2(iF + 2, jF, kF)] + 4.0 * fh[idx_fh2(iF + 1, jF, kF)] - 3.0 * fh[idx_fh2(iF, jF, kF)]);
|
|
} else if (wx < 0.0) {
|
|
if (iF + 2 <= imax) fx = d_gp.d2dx * (-fh[idx_fh2(iF + 2, jF, kF)] + 4.0 * fh[idx_fh2(iF + 1, jF, kF)] - 3.0 * fh[idx_fh2(iF, jF, kF)]);
|
|
else if (iF + 1 <= imax) fx = d_gp.d2dx * (-fh[idx_fh2(iF - 1, jF, kF)] + fh[idx_fh2(iF + 1, jF, kF)]);
|
|
else fx = d_gp.d2dx * (3.0 * fh[idx_fh2(iF, jF, kF)] - 4.0 * fh[idx_fh2(iF - 1, jF, kF)] + fh[idx_fh2(iF - 2, jF, kF)]);
|
|
}
|
|
|
|
if (wy > 0.0) {
|
|
if (jF - 2 >= jmin) fy = d_gp.d2dy * (3.0 * fh[idx_fh2(iF, jF, kF)] - 4.0 * fh[idx_fh2(iF, jF - 1, kF)] + fh[idx_fh2(iF, jF - 2, kF)]);
|
|
else if (jF - 1 >= jmin) fy = d_gp.d2dy * (-fh[idx_fh2(iF, jF - 1, kF)] + fh[idx_fh2(iF, jF + 1, kF)]);
|
|
else fy = d_gp.d2dy * (-fh[idx_fh2(iF, jF + 2, kF)] + 4.0 * fh[idx_fh2(iF, jF + 1, kF)] - 3.0 * fh[idx_fh2(iF, jF, kF)]);
|
|
} else if (wy < 0.0) {
|
|
if (jF + 2 <= jmax) fy = d_gp.d2dy * (-fh[idx_fh2(iF, jF + 2, kF)] + 4.0 * fh[idx_fh2(iF, jF + 1, kF)] - 3.0 * fh[idx_fh2(iF, jF, kF)]);
|
|
else if (jF + 1 <= jmax) fy = d_gp.d2dy * (-fh[idx_fh2(iF, jF - 1, kF)] + fh[idx_fh2(iF, jF + 1, kF)]);
|
|
else fy = d_gp.d2dy * (3.0 * fh[idx_fh2(iF, jF, kF)] - 4.0 * fh[idx_fh2(iF, jF - 1, kF)] + fh[idx_fh2(iF, jF - 2, kF)]);
|
|
}
|
|
|
|
if (wz > 0.0) {
|
|
if (kF - 2 >= kmin) fz = d_gp.d2dz * (3.0 * fh[idx_fh2(iF, jF, kF)] - 4.0 * fh[idx_fh2(iF, jF, kF - 1)] + fh[idx_fh2(iF, jF, kF - 2)]);
|
|
else if (kF - 1 >= kmin) fz = d_gp.d2dz * (-fh[idx_fh2(iF, jF, kF - 1)] + fh[idx_fh2(iF, jF, kF + 1)]);
|
|
else fz = d_gp.d2dz * (-fh[idx_fh2(iF, jF, kF + 2)] + 4.0 * fh[idx_fh2(iF, jF, kF + 1)] - 3.0 * fh[idx_fh2(iF, jF, kF)]);
|
|
} else if (wz < 0.0) {
|
|
if (kF + 2 <= kmax) fz = d_gp.d2dz * (-fh[idx_fh2(iF, jF, kF + 2)] + 4.0 * fh[idx_fh2(iF, jF, kF + 1)] - 3.0 * fh[idx_fh2(iF, jF, kF)]);
|
|
else if (kF + 1 <= kmax) fz = d_gp.d2dz * (-fh[idx_fh2(iF, jF, kF - 1)] + fh[idx_fh2(iF, jF, kF + 1)]);
|
|
else fz = d_gp.d2dz * (3.0 * fh[idx_fh2(iF, jF, kF)] - 4.0 * fh[idx_fh2(iF, jF, kF - 1)] + fh[idx_fh2(iF, jF, kF - 2)]);
|
|
}
|
|
|
|
f_rhs[p] = -velocity * (fx * x + fy * y + fz * z + fh[idx_fh2(iF, jF, kF)]) / r;
|
|
}
|
|
}
|
|
|
|
static void gpu_sommerfeld_routbam(double *d_f0, double *d_f_rhs,
|
|
double velocity,
|
|
double SoA0, double SoA1, double SoA2,
|
|
double *X, double *Y, double *Z,
|
|
const double *bbox,
|
|
int Symmetry)
|
|
{
|
|
if (velocity == 0.0) return;
|
|
const int nx = g_buf.prev_nx;
|
|
const int ny = g_buf.prev_ny;
|
|
const int nz = g_buf.prev_nz;
|
|
const double dX = X[1] - X[0];
|
|
const double dY = Y[1] - Y[0];
|
|
const double dZ = Z[1] - Z[0];
|
|
const bool touch_xmax = fabs(X[nx - 1] - bbox[3]) < dX;
|
|
const bool touch_ymax = fabs(Y[ny - 1] - bbox[4]) < dY;
|
|
const bool touch_zmax = fabs(Z[nz - 1] - bbox[5]) < dZ;
|
|
const bool touch_xmin = fabs(X[0] - bbox[0]) < dX &&
|
|
!(Symmetry == 2 && fabs(bbox[0]) < dX * 0.5);
|
|
const bool touch_ymin = fabs(Y[0] - bbox[1]) < dY &&
|
|
!(Symmetry == 2 && fabs(bbox[1]) < dY * 0.5);
|
|
const bool touch_zmin = fabs(Z[0] - bbox[2]) < dZ &&
|
|
!(Symmetry > 0 && fabs(bbox[2]) < dZ * 0.5);
|
|
|
|
const size_t w_pack = (size_t)(nx + 2) * (ny + 2) * (nz + 2);
|
|
kern_symbd_pack_ord2<<<grid(w_pack), BLK>>>(d_f0, g_buf.d_fh2, SoA0, SoA1, SoA2);
|
|
if (touch_xmax) kern_sommerfeld_face_bam<<<grid((size_t)ny * nz), BLK>>>(g_buf.d_fh2, d_f_rhs, FACE_XMAX, velocity, X[0], Y[0], Z[0]);
|
|
if (touch_ymax) kern_sommerfeld_face_bam<<<grid((size_t)nx * nz), BLK>>>(g_buf.d_fh2, d_f_rhs, FACE_YMAX, velocity, X[0], Y[0], Z[0]);
|
|
if (touch_zmax) kern_sommerfeld_face_bam<<<grid((size_t)nx * ny), BLK>>>(g_buf.d_fh2, d_f_rhs, FACE_ZMAX, velocity, X[0], Y[0], Z[0]);
|
|
if (touch_xmin) kern_sommerfeld_face_bam<<<grid((size_t)ny * nz), BLK>>>(g_buf.d_fh2, d_f_rhs, FACE_XMIN, velocity, X[0], Y[0], Z[0]);
|
|
if (touch_ymin) kern_sommerfeld_face_bam<<<grid((size_t)nx * nz), BLK>>>(g_buf.d_fh2, d_f_rhs, FACE_YMIN, velocity, X[0], Y[0], Z[0]);
|
|
if (touch_zmin) kern_sommerfeld_face_bam<<<grid((size_t)nx * ny), BLK>>>(g_buf.d_fh2, d_f_rhs, FACE_ZMIN, velocity, X[0], Y[0], Z[0]);
|
|
}
|
|
|
|
/* ================================================================== */
|
|
/* C. Point-wise computation kernels */
|
|
/* ================================================================== */
|
|
|
|
/* Phase 1: alpn1, chin1, gxx=dxx+1, gyy=dyy+1, gzz=dzz+1 */
|
|
__global__ void kern_phase1_prep(
|
|
const double* __restrict__ Lap, const double* __restrict__ chi,
|
|
const double* __restrict__ dxx, const double* __restrict__ dyy,
|
|
const double* __restrict__ dzz,
|
|
double* __restrict__ alpn1, double* __restrict__ chin1,
|
|
double* __restrict__ gxx, double* __restrict__ gyy, double* __restrict__ gzz)
|
|
{
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
alpn1[i] = Lap[i] + 1.0;
|
|
chin1[i] = chi[i] + 1.0;
|
|
gxx[i] = dxx[i] + 1.0;
|
|
gyy[i] = dyy[i] + 1.0;
|
|
gzz[i] = dzz[i] + 1.0;
|
|
}
|
|
}
|
|
|
|
/* Phase 2a: chi_rhs, gij_rhs */
|
|
__global__ void kern_phase2_metric_rhs(
|
|
const double* __restrict__ alpn1, const double* __restrict__ chin1,
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
const double* __restrict__ trK,
|
|
const double* __restrict__ Axx, const double* __restrict__ Axy,
|
|
const double* __restrict__ Axz, const double* __restrict__ Ayy,
|
|
const double* __restrict__ Ayz, const double* __restrict__ Azz,
|
|
const double* __restrict__ betaxx, const double* __restrict__ betaxy,
|
|
const double* __restrict__ betaxz, const double* __restrict__ betayx,
|
|
const double* __restrict__ betayy, const double* __restrict__ betayz,
|
|
const double* __restrict__ betazx, const double* __restrict__ betazy,
|
|
const double* __restrict__ betazz,
|
|
double* __restrict__ chi_rhs, double* __restrict__ gxx_rhs,
|
|
double* __restrict__ gyy_rhs, double* __restrict__ gzz_rhs,
|
|
double* __restrict__ gxy_rhs, double* __restrict__ gyz_rhs,
|
|
double* __restrict__ gxz_rhs)
|
|
{
|
|
const double F2o3 = 2.0/3.0, F1o3 = 1.0/3.0, TWO = 2.0;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double db = betaxx[i] + betayy[i] + betazz[i];
|
|
chi_rhs[i] = F2o3 * chin1[i] * (alpn1[i] * trK[i] - db);
|
|
gxx_rhs[i] = -TWO*alpn1[i]*Axx[i] - F2o3*gxx[i]*db
|
|
+ TWO*(gxx[i]*betaxx[i] + gxy[i]*betayx[i] + gxz[i]*betazx[i]);
|
|
gyy_rhs[i] = -TWO*alpn1[i]*Ayy[i] - F2o3*gyy[i]*db
|
|
+ TWO*(gxy[i]*betaxy[i] + gyy[i]*betayy[i] + gyz[i]*betazy[i]);
|
|
gzz_rhs[i] = -TWO*alpn1[i]*Azz[i] - F2o3*gzz[i]*db
|
|
+ TWO*(gxz[i]*betaxz[i] + gyz[i]*betayz[i] + gzz[i]*betazz[i]);
|
|
gxy_rhs[i] = -TWO*alpn1[i]*Axy[i] + F1o3*gxy[i]*db
|
|
+ gxx[i]*betaxy[i] + gxz[i]*betazy[i] + gyy[i]*betayx[i]
|
|
+ gyz[i]*betazx[i] - gxy[i]*betazz[i];
|
|
gyz_rhs[i] = -TWO*alpn1[i]*Ayz[i] + F1o3*gyz[i]*db
|
|
+ gxy[i]*betaxz[i] + gyy[i]*betayz[i] + gxz[i]*betaxy[i]
|
|
+ gzz[i]*betazy[i] - gyz[i]*betaxx[i];
|
|
gxz_rhs[i] = -TWO*alpn1[i]*Axz[i] + F1o3*gxz[i]*db
|
|
+ gxx[i]*betaxz[i] + gxy[i]*betayz[i] + gyz[i]*betayx[i]
|
|
+ gzz[i]*betazx[i] - gxz[i]*betayy[i];
|
|
}
|
|
}
|
|
|
|
/* Phase 2b: metric inverse */
|
|
__global__ void kern_phase2_inverse(
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
double* __restrict__ gupxx, double* __restrict__ gupxy,
|
|
double* __restrict__ gupxz, double* __restrict__ gupyy,
|
|
double* __restrict__ gupyz, double* __restrict__ gupzz)
|
|
{
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double det = gxx[i]*gyy[i]*gzz[i] + gxy[i]*gyz[i]*gxz[i] + gxz[i]*gxy[i]*gyz[i]
|
|
- gxz[i]*gyy[i]*gxz[i] - gxy[i]*gxy[i]*gzz[i] - gxx[i]*gyz[i]*gyz[i];
|
|
double inv = 1.0 / det;
|
|
gupxx[i] = (gyy[i]*gzz[i] - gyz[i]*gyz[i]) * inv;
|
|
gupxy[i] = -(gxy[i]*gzz[i] - gyz[i]*gxz[i]) * inv;
|
|
gupxz[i] = (gxy[i]*gyz[i] - gyy[i]*gxz[i]) * inv;
|
|
gupyy[i] = (gxx[i]*gzz[i] - gxz[i]*gxz[i]) * inv;
|
|
gupyz[i] = -(gxx[i]*gyz[i] - gxy[i]*gxz[i]) * inv;
|
|
gupzz[i] = (gxx[i]*gyy[i] - gxy[i]*gxy[i]) * inv;
|
|
}
|
|
}
|
|
|
|
/* Phase 3: Gamma constraint residuals (co==0 only) */
|
|
__global__ void kern_phase3_gamma_constraint(
|
|
const double* __restrict__ Gamx, const double* __restrict__ Gamy,
|
|
const double* __restrict__ Gamz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ gxxx, const double* __restrict__ gxyx,
|
|
const double* __restrict__ gxzx, const double* __restrict__ gyyx,
|
|
const double* __restrict__ gyzx, const double* __restrict__ gzzx,
|
|
const double* __restrict__ gxxy, const double* __restrict__ gxyy,
|
|
const double* __restrict__ gxzy, const double* __restrict__ gyyy,
|
|
const double* __restrict__ gyzy, const double* __restrict__ gzzy,
|
|
const double* __restrict__ gxxz, const double* __restrict__ gxyz,
|
|
const double* __restrict__ gxzz, const double* __restrict__ gyyz,
|
|
const double* __restrict__ gyzz, const double* __restrict__ gzzz,
|
|
double* __restrict__ Gmx_Res, double* __restrict__ Gmy_Res,
|
|
double* __restrict__ Gmz_Res)
|
|
{
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double uxx=gupxx[i], uxy=gupxy[i], uxz=gupxz[i];
|
|
double uyy=gupyy[i], uyz=gupyz[i], uzz=gupzz[i];
|
|
|
|
Gmx_Res[i] = Gamx[i] - (
|
|
uxx*(uxx*gxxx[i]+uxy*gxyx[i]+uxz*gxzx[i]) +
|
|
uxy*(uxx*gxyx[i]+uxy*gyyx[i]+uxz*gyzx[i]) +
|
|
uxz*(uxx*gxzx[i]+uxy*gyzx[i]+uxz*gzzx[i]) +
|
|
uxx*(uxy*gxxy[i]+uyy*gxyy[i]+uyz*gxzy[i]) +
|
|
uxy*(uxy*gxyy[i]+uyy*gyyy[i]+uyz*gyzy[i]) +
|
|
uxz*(uxy*gxzy[i]+uyy*gyzy[i]+uyz*gzzy[i]) +
|
|
uxx*(uxz*gxxz[i]+uyz*gxyz[i]+uzz*gxzz[i]) +
|
|
uxy*(uxz*gxyz[i]+uyz*gyyz[i]+uzz*gyzz[i]) +
|
|
uxz*(uxz*gxzz[i]+uyz*gyzz[i]+uzz*gzzz[i]));
|
|
|
|
Gmy_Res[i] = Gamy[i] - (
|
|
uxx*(uxy*gxxx[i]+uyy*gxyx[i]+uyz*gxzx[i]) +
|
|
uxy*(uxy*gxyx[i]+uyy*gyyx[i]+uyz*gyzx[i]) +
|
|
uxz*(uxy*gxzx[i]+uyy*gyzx[i]+uyz*gzzx[i]) +
|
|
uxy*(uxy*gxxy[i]+uyy*gxyy[i]+uyz*gxzy[i]) +
|
|
uyy*(uxy*gxyy[i]+uyy*gyyy[i]+uyz*gyzy[i]) +
|
|
uyz*(uxy*gxzy[i]+uyy*gyzy[i]+uyz*gzzy[i]) +
|
|
uxy*(uxz*gxxz[i]+uyz*gxyz[i]+uzz*gxzz[i]) +
|
|
uyy*(uxz*gxyz[i]+uyz*gyyz[i]+uzz*gyzz[i]) +
|
|
uyz*(uxz*gxzz[i]+uyz*gyzz[i]+uzz*gzzz[i]));
|
|
|
|
Gmz_Res[i] = Gamz[i] - (
|
|
uxx*(uxz*gxxx[i]+uyz*gxyx[i]+uzz*gxzx[i]) +
|
|
uxy*(uxz*gxyx[i]+uyz*gyyx[i]+uzz*gyzx[i]) +
|
|
uxz*(uxz*gxzx[i]+uyz*gyzx[i]+uzz*gzzx[i]) +
|
|
uxy*(uxz*gxxy[i]+uyz*gxyy[i]+uzz*gxzy[i]) +
|
|
uyy*(uxz*gxyy[i]+uyz*gyyy[i]+uzz*gyzy[i]) +
|
|
uyz*(uxz*gxzy[i]+uyz*gyzy[i]+uzz*gzzy[i]) +
|
|
uxz*(uxz*gxxz[i]+uyz*gxyz[i]+uzz*gxzz[i]) +
|
|
uyz*(uxz*gxyz[i]+uyz*gyyz[i]+uzz*gyzz[i]) +
|
|
uzz*(uxz*gxzz[i]+uyz*gyzz[i]+uzz*gzzz[i]));
|
|
}
|
|
}
|
|
|
|
/* Phase 4: 18 Christoffel symbols */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase4_christoffel(
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ gxxx, const double* __restrict__ gxyx,
|
|
const double* __restrict__ gxzx, const double* __restrict__ gyyx,
|
|
const double* __restrict__ gyzx, const double* __restrict__ gzzx,
|
|
const double* __restrict__ gxxy, const double* __restrict__ gxyy,
|
|
const double* __restrict__ gxzy, const double* __restrict__ gyyy,
|
|
const double* __restrict__ gyzy, const double* __restrict__ gzzy,
|
|
const double* __restrict__ gxxz, const double* __restrict__ gxyz,
|
|
const double* __restrict__ gxzz, const double* __restrict__ gyyz,
|
|
const double* __restrict__ gyzz, const double* __restrict__ gzzz,
|
|
double* __restrict__ Gxxx, double* __restrict__ Gxxy, double* __restrict__ Gxxz,
|
|
double* __restrict__ Gxyy, double* __restrict__ Gxyz, double* __restrict__ Gxzz,
|
|
double* __restrict__ Gyxx, double* __restrict__ Gyxy, double* __restrict__ Gyxz,
|
|
double* __restrict__ Gyyy, double* __restrict__ Gyyz, double* __restrict__ Gyzz,
|
|
double* __restrict__ Gzxx, double* __restrict__ Gzxy, double* __restrict__ Gzxz,
|
|
double* __restrict__ Gzyy, double* __restrict__ Gzyz, double* __restrict__ Gzzz_o)
|
|
{
|
|
const double H = 0.5, TWO = 2.0;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double uxx=gupxx[i],uxy=gupxy[i],uxz=gupxz[i];
|
|
double uyy=gupyy[i],uyz=gupyz[i],uzz=gupzz[i];
|
|
/* Gamma^x_{xx} */
|
|
Gxxx[i]=H*(uxx*gxxx[i]+uxy*(TWO*gxyx[i]-gxxy[i])+uxz*(TWO*gxzx[i]-gxxz[i]));
|
|
Gyxx[i]=H*(uxy*gxxx[i]+uyy*(TWO*gxyx[i]-gxxy[i])+uyz*(TWO*gxzx[i]-gxxz[i]));
|
|
Gzxx[i]=H*(uxz*gxxx[i]+uyz*(TWO*gxyx[i]-gxxy[i])+uzz*(TWO*gxzx[i]-gxxz[i]));
|
|
/* yy */
|
|
Gxyy[i]=H*(uxx*(TWO*gxyy[i]-gyyx[i])+uxy*gyyy[i]+uxz*(TWO*gyzy[i]-gyyz[i]));
|
|
Gyyy[i]=H*(uxy*(TWO*gxyy[i]-gyyx[i])+uyy*gyyy[i]+uyz*(TWO*gyzy[i]-gyyz[i]));
|
|
Gzyy[i]=H*(uxz*(TWO*gxyy[i]-gyyx[i])+uyz*gyyy[i]+uzz*(TWO*gyzy[i]-gyyz[i]));
|
|
/* zz */
|
|
Gxzz[i]=H*(uxx*(TWO*gxzz[i]-gzzx[i])+uxy*(TWO*gyzz[i]-gzzy[i])+uxz*gzzz[i]);
|
|
Gyzz[i]=H*(uxy*(TWO*gxzz[i]-gzzx[i])+uyy*(TWO*gyzz[i]-gzzy[i])+uyz*gzzz[i]);
|
|
Gzzz_o[i]=H*(uxz*(TWO*gxzz[i]-gzzx[i])+uyz*(TWO*gyzz[i]-gzzy[i])+uzz*gzzz[i]);
|
|
/* xy */
|
|
Gxxy[i]=H*(uxx*gxxy[i]+uxy*gyyx[i]+uxz*(gxzy[i]+gyzx[i]-gxyz[i]));
|
|
Gyxy[i]=H*(uxy*gxxy[i]+uyy*gyyx[i]+uyz*(gxzy[i]+gyzx[i]-gxyz[i]));
|
|
Gzxy[i]=H*(uxz*gxxy[i]+uyz*gyyx[i]+uzz*(gxzy[i]+gyzx[i]-gxyz[i]));
|
|
/* xz */
|
|
Gxxz[i]=H*(uxx*gxxz[i]+uxy*(gxyz[i]+gyzx[i]-gxzy[i])+uxz*gzzx[i]);
|
|
Gyxz[i]=H*(uxy*gxxz[i]+uyy*(gxyz[i]+gyzx[i]-gxzy[i])+uyz*gzzx[i]);
|
|
Gzxz[i]=H*(uxz*gxxz[i]+uyz*(gxyz[i]+gyzx[i]-gxzy[i])+uzz*gzzx[i]);
|
|
/* yz */
|
|
Gxyz[i]=H*(uxx*(gxyz[i]+gxzy[i]-gyzx[i])+uxy*gyyz[i]+uxz*gzzy[i]);
|
|
Gyyz[i]=H*(uxy*(gxyz[i]+gxzy[i]-gyzx[i])+uyy*gyyz[i]+uyz*gzzy[i]);
|
|
Gzyz[i]=H*(uxz*(gxyz[i]+gxzy[i]-gyzx[i])+uyz*gyyz[i]+uzz*gzzy[i]);
|
|
}
|
|
}
|
|
|
|
/* Phase 5: A^ij = gup^ia gup^jb A_ab (stored temporarily in Rxx..Rzz) */
|
|
__global__ void kern_phase5_raise_A(
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ Axx, const double* __restrict__ Axy,
|
|
const double* __restrict__ Axz, const double* __restrict__ Ayy,
|
|
const double* __restrict__ Ayz, const double* __restrict__ Azz,
|
|
double* __restrict__ Rxx, double* __restrict__ Rxy, double* __restrict__ Rxz,
|
|
double* __restrict__ Ryy, double* __restrict__ Ryz, double* __restrict__ Rzz)
|
|
{
|
|
const double TWO = 2.0;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double uxx=gupxx[i],uxy=gupxy[i],uxz=gupxz[i];
|
|
double uyy=gupyy[i],uyz=gupyz[i],uzz=gupzz[i];
|
|
Rxx[i]=uxx*uxx*Axx[i]+uxy*uxy*Ayy[i]+uxz*uxz*Azz[i]
|
|
+TWO*(uxx*uxy*Axy[i]+uxx*uxz*Axz[i]+uxy*uxz*Ayz[i]);
|
|
Ryy[i]=uxy*uxy*Axx[i]+uyy*uyy*Ayy[i]+uyz*uyz*Azz[i]
|
|
+TWO*(uxy*uyy*Axy[i]+uxy*uyz*Axz[i]+uyy*uyz*Ayz[i]);
|
|
Rzz[i]=uxz*uxz*Axx[i]+uyz*uyz*Ayy[i]+uzz*uzz*Azz[i]
|
|
+TWO*(uxz*uyz*Axy[i]+uxz*uzz*Axz[i]+uyz*uzz*Ayz[i]);
|
|
Rxy[i]=uxx*uxy*Axx[i]+uxy*uyy*Ayy[i]+uxz*uyz*Azz[i]
|
|
+(uxx*uyy+uxy*uxy)*Axy[i]+(uxx*uyz+uxz*uxy)*Axz[i]+(uxy*uyz+uxz*uyy)*Ayz[i];
|
|
Rxz[i]=uxx*uxz*Axx[i]+uxy*uyz*Ayy[i]+uxz*uzz*Azz[i]
|
|
+(uxx*uyz+uxy*uxz)*Axy[i]+(uxx*uzz+uxz*uxz)*Axz[i]+(uxy*uzz+uxz*uyz)*Ayz[i];
|
|
Ryz[i]=uxy*uxz*Axx[i]+uyy*uyz*Ayy[i]+uyz*uzz*Azz[i]
|
|
+(uxy*uyz+uyy*uxz)*Axy[i]+(uxy*uzz+uyz*uxz)*Axz[i]+(uyy*uzz+uyz*uyz)*Ayz[i];
|
|
}
|
|
}
|
|
|
|
/* Phase 6: Gamma_rhs part 1 (before fdderivs(beta) and fderivs(Gamma)) */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase6_gamma_rhs_part1(
|
|
const double* __restrict__ Lapx, const double* __restrict__ Lapy,
|
|
const double* __restrict__ Lapz,
|
|
const double* __restrict__ alpn1, const double* __restrict__ chin1,
|
|
const double* __restrict__ chix, const double* __restrict__ chiy,
|
|
const double* __restrict__ chiz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ Kx, const double* __restrict__ Ky,
|
|
const double* __restrict__ Kz,
|
|
const double* __restrict__ Sx, const double* __restrict__ Sy,
|
|
const double* __restrict__ Sz,
|
|
const double* __restrict__ Rxx, const double* __restrict__ Rxy,
|
|
const double* __restrict__ Rxz, const double* __restrict__ Ryy,
|
|
const double* __restrict__ Ryz, const double* __restrict__ Rzz,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
double* __restrict__ Gamx_rhs, double* __restrict__ Gamy_rhs,
|
|
double* __restrict__ Gamz_rhs)
|
|
{
|
|
const double TWO=2.0, F3o2=1.5, F2o3=2.0/3.0, EIGHT=8.0;
|
|
const double PI_V = 3.14159265358979323846;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double uxx=gupxx[i],uxy=gupxy[i],uxz=gupxz[i];
|
|
double uyy=gupyy[i],uyz=gupyz[i],uzz=gupzz[i];
|
|
double lx=Lapx[i],ly=Lapy[i],lz=Lapz[i];
|
|
double a=alpn1[i], c1=chin1[i];
|
|
double cx=chix[i],cy=chiy[i],cz=chiz[i];
|
|
|
|
Gamx_rhs[i] = -TWO*(lx*Rxx[i]+ly*Rxy[i]+lz*Rxz[i])
|
|
+ TWO*a*(
|
|
-F3o2/c1*(cx*Rxx[i]+cy*Rxy[i]+cz*Rxz[i])
|
|
-uxx*(F2o3*Kx[i]+EIGHT*PI_V*Sx[i])
|
|
-uxy*(F2o3*Ky[i]+EIGHT*PI_V*Sy[i])
|
|
-uxz*(F2o3*Kz[i]+EIGHT*PI_V*Sz[i])
|
|
+Gxxx[i]*Rxx[i]+Gxyy[i]*Ryy[i]+Gxzz[i]*Rzz[i]
|
|
+TWO*(Gxxy[i]*Rxy[i]+Gxxz[i]*Rxz[i]+Gxyz[i]*Ryz[i]));
|
|
|
|
Gamy_rhs[i] = -TWO*(lx*Rxy[i]+ly*Ryy[i]+lz*Ryz[i])
|
|
+ TWO*a*(
|
|
-F3o2/c1*(cx*Rxy[i]+cy*Ryy[i]+cz*Ryz[i])
|
|
-uxy*(F2o3*Kx[i]+EIGHT*PI_V*Sx[i])
|
|
-uyy*(F2o3*Ky[i]+EIGHT*PI_V*Sy[i])
|
|
-uyz*(F2o3*Kz[i]+EIGHT*PI_V*Sz[i])
|
|
+Gyxx[i]*Rxx[i]+Gyyy[i]*Ryy[i]+Gyzz[i]*Rzz[i]
|
|
+TWO*(Gyxy[i]*Rxy[i]+Gyxz[i]*Rxz[i]+Gyyz[i]*Ryz[i]));
|
|
|
|
Gamz_rhs[i] = -TWO*(lx*Rxz[i]+ly*Ryz[i]+lz*Rzz[i])
|
|
+ TWO*a*(
|
|
-F3o2/c1*(cx*Rxz[i]+cy*Ryz[i]+cz*Rzz[i])
|
|
-uxz*(F2o3*Kx[i]+EIGHT*PI_V*Sx[i])
|
|
-uyz*(F2o3*Ky[i]+EIGHT*PI_V*Sy[i])
|
|
-uzz*(F2o3*Kz[i]+EIGHT*PI_V*Sz[i])
|
|
+Gzxx[i]*Rxx[i]+Gzyy[i]*Ryy[i]+Gzzz[i]*Rzz[i]
|
|
+TWO*(Gzxy[i]*Rxy[i]+Gzxz[i]*Rxz[i]+Gzyz[i]*Ryz[i]));
|
|
}
|
|
}
|
|
|
|
/* Phase 5+6 fused: raise A^ij in registers, then consume immediately in Gamma_rhs. */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase5_6_gamma_rhs_part1_fused(
|
|
const double* __restrict__ Lapx, const double* __restrict__ Lapy,
|
|
const double* __restrict__ Lapz,
|
|
const double* __restrict__ alpn1, const double* __restrict__ chin1,
|
|
const double* __restrict__ chix, const double* __restrict__ chiy,
|
|
const double* __restrict__ chiz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ Axx, const double* __restrict__ Axy,
|
|
const double* __restrict__ Axz, const double* __restrict__ Ayy,
|
|
const double* __restrict__ Ayz, const double* __restrict__ Azz,
|
|
const double* __restrict__ Kx, const double* __restrict__ Ky,
|
|
const double* __restrict__ Kz,
|
|
const double* __restrict__ Sx, const double* __restrict__ Sy,
|
|
const double* __restrict__ Sz,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
double* __restrict__ Gamx_rhs, double* __restrict__ Gamy_rhs,
|
|
double* __restrict__ Gamz_rhs)
|
|
{
|
|
const double TWO = 2.0, F3o2 = 1.5, F2o3 = 2.0 / 3.0, EIGHT = 8.0;
|
|
const double PI_V = 3.14159265358979323846;
|
|
for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < d_gp.all; i += blockDim.x * gridDim.x) {
|
|
const double uxx = gupxx[i], uxy = gupxy[i], uxz = gupxz[i];
|
|
const double uyy = gupyy[i], uyz = gupyz[i], uzz = gupzz[i];
|
|
const double Axx_v = Axx[i], Axy_v = Axy[i], Axz_v = Axz[i];
|
|
const double Ayy_v = Ayy[i], Ayz_v = Ayz[i], Azz_v = Azz[i];
|
|
|
|
const double Rxx_v = uxx * uxx * Axx_v + uxy * uxy * Ayy_v + uxz * uxz * Azz_v
|
|
+ TWO * (uxx * uxy * Axy_v + uxx * uxz * Axz_v + uxy * uxz * Ayz_v);
|
|
const double Ryy_v = uxy * uxy * Axx_v + uyy * uyy * Ayy_v + uyz * uyz * Azz_v
|
|
+ TWO * (uxy * uyy * Axy_v + uxy * uyz * Axz_v + uyy * uyz * Ayz_v);
|
|
const double Rzz_v = uxz * uxz * Axx_v + uyz * uyz * Ayy_v + uzz * uzz * Azz_v
|
|
+ TWO * (uxz * uyz * Axy_v + uxz * uzz * Axz_v + uyz * uzz * Ayz_v);
|
|
const double Rxy_v = uxx * uxy * Axx_v + uxy * uyy * Ayy_v + uxz * uyz * Azz_v
|
|
+ (uxx * uyy + uxy * uxy) * Axy_v
|
|
+ (uxx * uyz + uxz * uxy) * Axz_v
|
|
+ (uxy * uyz + uxz * uyy) * Ayz_v;
|
|
const double Rxz_v = uxx * uxz * Axx_v + uxy * uyz * Ayy_v + uxz * uzz * Azz_v
|
|
+ (uxx * uyz + uxy * uxz) * Axy_v
|
|
+ (uxx * uzz + uxz * uxz) * Axz_v
|
|
+ (uxy * uzz + uxz * uyz) * Ayz_v;
|
|
const double Ryz_v = uxy * uxz * Axx_v + uyy * uyz * Ayy_v + uyz * uzz * Azz_v
|
|
+ (uxy * uyz + uyy * uxz) * Axy_v
|
|
+ (uxy * uzz + uyz * uxz) * Axz_v
|
|
+ (uyy * uzz + uyz * uyz) * Ayz_v;
|
|
|
|
const double lx = Lapx[i], ly = Lapy[i], lz = Lapz[i];
|
|
const double a = alpn1[i], c1 = chin1[i];
|
|
const double cx = chix[i], cy = chiy[i], cz = chiz[i];
|
|
|
|
Gamx_rhs[i] = -TWO * (lx * Rxx_v + ly * Rxy_v + lz * Rxz_v)
|
|
+ TWO * a * (
|
|
-F3o2 / c1 * (cx * Rxx_v + cy * Rxy_v + cz * Rxz_v)
|
|
-uxx * (F2o3 * Kx[i] + EIGHT * PI_V * Sx[i])
|
|
-uxy * (F2o3 * Ky[i] + EIGHT * PI_V * Sy[i])
|
|
-uxz * (F2o3 * Kz[i] + EIGHT * PI_V * Sz[i])
|
|
+ Gxxx[i] * Rxx_v + Gxyy[i] * Ryy_v + Gxzz[i] * Rzz_v
|
|
+ TWO * (Gxxy[i] * Rxy_v + Gxxz[i] * Rxz_v + Gxyz[i] * Ryz_v));
|
|
|
|
Gamy_rhs[i] = -TWO * (lx * Rxy_v + ly * Ryy_v + lz * Ryz_v)
|
|
+ TWO * a * (
|
|
-F3o2 / c1 * (cx * Rxy_v + cy * Ryy_v + cz * Ryz_v)
|
|
-uxy * (F2o3 * Kx[i] + EIGHT * PI_V * Sx[i])
|
|
-uyy * (F2o3 * Ky[i] + EIGHT * PI_V * Sy[i])
|
|
-uyz * (F2o3 * Kz[i] + EIGHT * PI_V * Sz[i])
|
|
+ Gyxx[i] * Rxx_v + Gyyy[i] * Ryy_v + Gyzz[i] * Rzz_v
|
|
+ TWO * (Gyxy[i] * Rxy_v + Gyxz[i] * Rxz_v + Gyyz[i] * Ryz_v));
|
|
|
|
Gamz_rhs[i] = -TWO * (lx * Rxz_v + ly * Ryz_v + lz * Rzz_v)
|
|
+ TWO * a * (
|
|
-F3o2 / c1 * (cx * Rxz_v + cy * Ryz_v + cz * Rzz_v)
|
|
-uxz * (F2o3 * Kx[i] + EIGHT * PI_V * Sx[i])
|
|
-uyz * (F2o3 * Ky[i] + EIGHT * PI_V * Sy[i])
|
|
-uzz * (F2o3 * Kz[i] + EIGHT * PI_V * Sz[i])
|
|
+ Gzxx[i] * Rxx_v + Gzyy[i] * Ryy_v + Gzzz[i] * Rzz_v
|
|
+ TWO * (Gzxy[i] * Rxy_v + Gzxz[i] * Rxz_v + Gzyz[i] * Ryz_v));
|
|
}
|
|
}
|
|
|
|
/* Phase 8: Gamma_rhs part 2 — after fdderivs(beta) and fderivs(Gamma)
|
|
* Computes: fxx=div(beta_xx), Gamxa, then updates Gamx_rhs etc.
|
|
* Input arrays gxxx..gzzz here hold fdderivs(beta) results,
|
|
* Gamxx..Gamzz hold fderivs(Gamma) results.
|
|
*/
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase8_gamma_rhs_part2(
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
/* fdderivs(betax) -> gxxx,gxyx,gxzx,gyyx,gyzx,gzzx */
|
|
const double* __restrict__ bxx_xx, const double* __restrict__ bxx_xy,
|
|
const double* __restrict__ bxx_xz, const double* __restrict__ bxx_yy,
|
|
const double* __restrict__ bxx_yz, const double* __restrict__ bxx_zz,
|
|
/* fdderivs(betay) -> gxxy,gxyy,gxzy,gyyy,gyzy,gzzy */
|
|
const double* __restrict__ bxy_xx, const double* __restrict__ bxy_xy,
|
|
const double* __restrict__ bxy_xz, const double* __restrict__ bxy_yy,
|
|
const double* __restrict__ bxy_yz, const double* __restrict__ bxy_zz,
|
|
/* fdderivs(betaz) -> gxxz,gxyz,gxzz,gyyz,gyzz,gzzz */
|
|
const double* __restrict__ bxz_xx, const double* __restrict__ bxz_xy,
|
|
const double* __restrict__ bxz_xz, const double* __restrict__ bxz_yy,
|
|
const double* __restrict__ bxz_yz, const double* __restrict__ bxz_zz,
|
|
/* fderivs(Gamx) -> Gamxx,Gamxy,Gamxz */
|
|
const double* __restrict__ Gamxx, const double* __restrict__ Gamxy,
|
|
const double* __restrict__ Gamxz,
|
|
/* fderivs(Gamy) -> Gamyx,Gamyy,Gamyz */
|
|
const double* __restrict__ Gamyx, const double* __restrict__ Gamyy_d,
|
|
const double* __restrict__ Gamyz_d,
|
|
/* fderivs(Gamz) -> Gamzx,Gamzy,Gamzz */
|
|
const double* __restrict__ Gamzx, const double* __restrict__ Gamzy,
|
|
const double* __restrict__ Gamzz_d,
|
|
/* Christoffel symbols */
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
/* betaij first derivs */
|
|
const double* __restrict__ betaxx, const double* __restrict__ betaxy,
|
|
const double* __restrict__ betaxz, const double* __restrict__ betayx,
|
|
const double* __restrict__ betayy, const double* __restrict__ betayz,
|
|
const double* __restrict__ betazx, const double* __restrict__ betazy,
|
|
const double* __restrict__ betazz,
|
|
double* __restrict__ Gamx_rhs, double* __restrict__ Gamy_rhs,
|
|
double* __restrict__ Gamz_rhs,
|
|
double* __restrict__ Gamxa_out, double* __restrict__ Gamya_out,
|
|
double* __restrict__ Gamza_out)
|
|
{
|
|
const double TWO=2.0, F2o3=2.0/3.0, F1o3=1.0/3.0;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double uxx=gupxx[i],uxy=gupxy[i],uxz=gupxz[i];
|
|
double uyy=gupyy[i],uyz=gupyz[i],uzz=gupzz[i];
|
|
/* div(beta_second_derivs) */
|
|
double fxx_v = bxx_xx[i]+bxy_xy[i]+bxz_xz[i];
|
|
double fxy_v = bxx_xy[i]+bxy_yy[i]+bxz_yz[i];
|
|
double fxz_v = bxx_xz[i]+bxy_yz[i]+bxz_zz[i];
|
|
/* Gamma^a contracted */
|
|
double Ga_x = uxx*Gxxx[i]+uyy*Gxyy[i]+uzz*Gxzz[i]
|
|
+TWO*(uxy*Gxxy[i]+uxz*Gxxz[i]+uyz*Gxyz[i]);
|
|
double Ga_y = uxx*Gyxx[i]+uyy*Gyyy[i]+uzz*Gyzz[i]
|
|
+TWO*(uxy*Gyxy[i]+uxz*Gyxz[i]+uyz*Gyyz[i]);
|
|
double Ga_z = uxx*Gzxx[i]+uyy*Gzyy[i]+uzz*Gzzz[i]
|
|
+TWO*(uxy*Gzxy[i]+uxz*Gzxz[i]+uyz*Gzyz[i]);
|
|
Gamxa_out[i]=Ga_x; Gamya_out[i]=Ga_y; Gamza_out[i]=Ga_z;
|
|
double db = betaxx[i] + betayy[i] + betazz[i];
|
|
Gamx_rhs[i] += F2o3*Ga_x*db
|
|
- Ga_x*betaxx[i] - Ga_y*betaxy[i] - Ga_z*betaxz[i]
|
|
+ F1o3*(uxx*fxx_v+uxy*fxy_v+uxz*fxz_v)
|
|
+ uxx*bxx_xx[i]+uyy*bxx_yy[i]+uzz*bxx_zz[i]
|
|
+ TWO*(uxy*bxx_xy[i]+uxz*bxx_xz[i]+uyz*bxx_yz[i]);
|
|
Gamy_rhs[i] += F2o3*Ga_y*db
|
|
- Ga_x*betayx[i] - Ga_y*betayy[i] - Ga_z*betayz[i]
|
|
+ F1o3*(uxy*fxx_v+uyy*fxy_v+uyz*fxz_v)
|
|
+ uxx*bxy_xx[i]+uyy*bxy_yy[i]+uzz*bxy_zz[i]
|
|
+ TWO*(uxy*bxy_xy[i]+uxz*bxy_xz[i]+uyz*bxy_yz[i]);
|
|
Gamz_rhs[i] += F2o3*Ga_z*db
|
|
- Ga_x*betazx[i] - Ga_y*betazy[i] - Ga_z*betazz[i]
|
|
+ F1o3*(uxz*fxx_v+uyz*fxy_v+uzz*fxz_v)
|
|
+ uxx*bxz_xx[i]+uyy*bxz_yy[i]+uzz*bxz_zz[i]
|
|
+ TWO*(uxy*bxz_xy[i]+uxz*bxz_xz[i]+uyz*bxz_yz[i]);
|
|
}
|
|
}
|
|
|
|
/* Phase 9: Christoffel contract — compute g_{ia} Gamma^a_{bc} products
|
|
* Overwrites gxxx..gzzz with lowered Christoffel products needed for Ricci.
|
|
*/
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase9_christoffel_contract(
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
/* output: lowered products g_{ia} Gamma^a_{bc} */
|
|
double* __restrict__ o_gxxx, double* __restrict__ o_gxyx,
|
|
double* __restrict__ o_gxzx, double* __restrict__ o_gyyx,
|
|
double* __restrict__ o_gyzx, double* __restrict__ o_gzzx,
|
|
double* __restrict__ o_gxxy, double* __restrict__ o_gxyy,
|
|
double* __restrict__ o_gxzy, double* __restrict__ o_gyyy,
|
|
double* __restrict__ o_gyzy, double* __restrict__ o_gzzy,
|
|
double* __restrict__ o_gxxz, double* __restrict__ o_gxyz,
|
|
double* __restrict__ o_gxzz, double* __restrict__ o_gyyz,
|
|
double* __restrict__ o_gyzz, double* __restrict__ o_gzzz)
|
|
{
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double g11=gxx[i],g12=gxy[i],g13=gxz[i];
|
|
double g22=gyy[i],g23=gyz[i],g33=gzz[i];
|
|
/* row x: g_{x,a} Gamma^a_{bc} */
|
|
o_gxxx[i]=g11*Gxxx[i]+g12*Gyxx[i]+g13*Gzxx[i];
|
|
o_gxyx[i]=g11*Gxxy[i]+g12*Gyxy[i]+g13*Gzxy[i];
|
|
o_gxzx[i]=g11*Gxxz[i]+g12*Gyxz[i]+g13*Gzxz[i];
|
|
o_gyyx[i]=g11*Gxyy[i]+g12*Gyyy[i]+g13*Gzyy[i];
|
|
o_gyzx[i]=g11*Gxyz[i]+g12*Gyyz[i]+g13*Gzyz[i];
|
|
o_gzzx[i]=g11*Gxzz[i]+g12*Gyzz[i]+g13*Gzzz[i];
|
|
/* row y: g_{y,a} Gamma^a_{bc} */
|
|
o_gxxy[i]=g12*Gxxx[i]+g22*Gyxx[i]+g23*Gzxx[i];
|
|
o_gxyy[i]=g12*Gxxy[i]+g22*Gyxy[i]+g23*Gzxy[i];
|
|
o_gxzy[i]=g12*Gxxz[i]+g22*Gyxz[i]+g23*Gzxz[i];
|
|
o_gyyy[i]=g12*Gxyy[i]+g22*Gyyy[i]+g23*Gzyy[i];
|
|
o_gyzy[i]=g12*Gxyz[i]+g22*Gyyz[i]+g23*Gzyz[i];
|
|
o_gzzy[i]=g12*Gxzz[i]+g22*Gyzz[i]+g23*Gzzz[i];
|
|
/* row z: g_{z,a} Gamma^a_{bc} */
|
|
o_gxxz[i]=g13*Gxxx[i]+g23*Gyxx[i]+g33*Gzxx[i];
|
|
o_gxyz[i]=g13*Gxxy[i]+g23*Gyxy[i]+g33*Gzxy[i];
|
|
o_gxzz[i]=g13*Gxxz[i]+g23*Gyxz[i]+g33*Gzxz[i];
|
|
o_gyyz[i]=g13*Gxyy[i]+g23*Gyyy[i]+g33*Gzyy[i];
|
|
o_gyzz[i]=g13*Gxyz[i]+g23*Gyyz[i]+g33*Gzyz[i];
|
|
o_gzzz[i]=g13*Gxzz[i]+g23*Gyzz[i]+g33*Gzzz[i];
|
|
}
|
|
}
|
|
|
|
/* Phase 8+9 fused: update Gamma rhs, contract Gamma^a, and lower Christoffels in one pass. */
|
|
__global__ __launch_bounds__(128, 2)
|
|
void kern_phase8_9_gamma_rhs_contract_fused(
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ bxx_xx, const double* __restrict__ bxx_xy,
|
|
const double* __restrict__ bxx_xz, const double* __restrict__ bxx_yy,
|
|
const double* __restrict__ bxx_yz, const double* __restrict__ bxx_zz,
|
|
const double* __restrict__ bxy_xx, const double* __restrict__ bxy_xy,
|
|
const double* __restrict__ bxy_xz, const double* __restrict__ bxy_yy,
|
|
const double* __restrict__ bxy_yz, const double* __restrict__ bxy_zz,
|
|
const double* __restrict__ bxz_xx, const double* __restrict__ bxz_xy,
|
|
const double* __restrict__ bxz_xz, const double* __restrict__ bxz_yy,
|
|
const double* __restrict__ bxz_yz, const double* __restrict__ bxz_zz,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
const double* __restrict__ betaxx, const double* __restrict__ betaxy,
|
|
const double* __restrict__ betaxz, const double* __restrict__ betayx,
|
|
const double* __restrict__ betayy, const double* __restrict__ betayz,
|
|
const double* __restrict__ betazx, const double* __restrict__ betazy,
|
|
const double* __restrict__ betazz,
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
double* __restrict__ Gamx_rhs, double* __restrict__ Gamy_rhs,
|
|
double* __restrict__ Gamz_rhs,
|
|
double* __restrict__ Gamxa_out, double* __restrict__ Gamya_out,
|
|
double* __restrict__ Gamza_out,
|
|
double* __restrict__ o_gxxx, double* __restrict__ o_gxyx,
|
|
double* __restrict__ o_gxzx, double* __restrict__ o_gyyx,
|
|
double* __restrict__ o_gyzx, double* __restrict__ o_gzzx,
|
|
double* __restrict__ o_gxxy, double* __restrict__ o_gxyy,
|
|
double* __restrict__ o_gxzy, double* __restrict__ o_gyyy,
|
|
double* __restrict__ o_gyzy, double* __restrict__ o_gzzy,
|
|
double* __restrict__ o_gxxz, double* __restrict__ o_gxyz,
|
|
double* __restrict__ o_gxzz, double* __restrict__ o_gyyz,
|
|
double* __restrict__ o_gyzz, double* __restrict__ o_gzzz)
|
|
{
|
|
const double TWO = 2.0, F2o3 = 2.0 / 3.0, F1o3 = 1.0 / 3.0;
|
|
for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < d_gp.all; i += blockDim.x * gridDim.x) {
|
|
const double uxx = gupxx[i], uxy = gupxy[i], uxz = gupxz[i];
|
|
const double uyy = gupyy[i], uyz = gupyz[i], uzz = gupzz[i];
|
|
|
|
const double Gxxx_v = Gxxx[i], Gxxy_v = Gxxy[i], Gxxz_v = Gxxz[i];
|
|
const double Gxyy_v = Gxyy[i], Gxyz_v = Gxyz[i], Gxzz_v = Gxzz[i];
|
|
const double Gyxx_v = Gyxx[i], Gyxy_v = Gyxy[i], Gyxz_v = Gyxz[i];
|
|
const double Gyyy_v = Gyyy[i], Gyyz_v = Gyyz[i], Gyzz_v = Gyzz[i];
|
|
const double Gzxx_v = Gzxx[i], Gzxy_v = Gzxy[i], Gzxz_v = Gzxz[i];
|
|
const double Gzyy_v = Gzyy[i], Gzyz_v = Gzyz[i], Gzzz_v = Gzzz[i];
|
|
|
|
const double fxx_v = bxx_xx[i] + bxy_xy[i] + bxz_xz[i];
|
|
const double fxy_v = bxx_xy[i] + bxy_yy[i] + bxz_yz[i];
|
|
const double fxz_v = bxx_xz[i] + bxy_yz[i] + bxz_zz[i];
|
|
|
|
const double Ga_x = uxx * Gxxx_v + uyy * Gxyy_v + uzz * Gxzz_v
|
|
+ TWO * (uxy * Gxxy_v + uxz * Gxxz_v + uyz * Gxyz_v);
|
|
const double Ga_y = uxx * Gyxx_v + uyy * Gyyy_v + uzz * Gyzz_v
|
|
+ TWO * (uxy * Gyxy_v + uxz * Gyxz_v + uyz * Gyyz_v);
|
|
const double Ga_z = uxx * Gzxx_v + uyy * Gzyy_v + uzz * Gzzz_v
|
|
+ TWO * (uxy * Gzxy_v + uxz * Gzxz_v + uyz * Gzyz_v);
|
|
|
|
Gamxa_out[i] = Ga_x;
|
|
Gamya_out[i] = Ga_y;
|
|
Gamza_out[i] = Ga_z;
|
|
|
|
const double betaxx_v = betaxx[i], betaxy_v = betaxy[i], betaxz_v = betaxz[i];
|
|
const double betayx_v = betayx[i], betayy_v = betayy[i], betayz_v = betayz[i];
|
|
const double betazx_v = betazx[i], betazy_v = betazy[i], betazz_v = betazz[i];
|
|
const double db = betaxx_v + betayy_v + betazz_v;
|
|
|
|
Gamx_rhs[i] += F2o3 * Ga_x * db
|
|
- Ga_x * betaxx_v - Ga_y * betaxy_v - Ga_z * betaxz_v
|
|
+ F1o3 * (uxx * fxx_v + uxy * fxy_v + uxz * fxz_v)
|
|
+ uxx * bxx_xx[i] + uyy * bxx_yy[i] + uzz * bxx_zz[i]
|
|
+ TWO * (uxy * bxx_xy[i] + uxz * bxx_xz[i] + uyz * bxx_yz[i]);
|
|
Gamy_rhs[i] += F2o3 * Ga_y * db
|
|
- Ga_x * betayx_v - Ga_y * betayy_v - Ga_z * betayz_v
|
|
+ F1o3 * (uxy * fxx_v + uyy * fxy_v + uyz * fxz_v)
|
|
+ uxx * bxy_xx[i] + uyy * bxy_yy[i] + uzz * bxy_zz[i]
|
|
+ TWO * (uxy * bxy_xy[i] + uxz * bxy_xz[i] + uyz * bxy_yz[i]);
|
|
Gamz_rhs[i] += F2o3 * Ga_z * db
|
|
- Ga_x * betazx_v - Ga_y * betazy_v - Ga_z * betazz_v
|
|
+ F1o3 * (uxz * fxx_v + uyz * fxy_v + uzz * fxz_v)
|
|
+ uxx * bxz_xx[i] + uyy * bxz_yy[i] + uzz * bxz_zz[i]
|
|
+ TWO * (uxy * bxz_xy[i] + uxz * bxz_xz[i] + uyz * bxz_yz[i]);
|
|
|
|
const double g11 = gxx[i], g12 = gxy[i], g13 = gxz[i];
|
|
const double g22 = gyy[i], g23 = gyz[i], g33 = gzz[i];
|
|
|
|
o_gxxx[i] = g11 * Gxxx_v + g12 * Gyxx_v + g13 * Gzxx_v;
|
|
o_gxyx[i] = g11 * Gxxy_v + g12 * Gyxy_v + g13 * Gzxy_v;
|
|
o_gxzx[i] = g11 * Gxxz_v + g12 * Gyxz_v + g13 * Gzxz_v;
|
|
o_gyyx[i] = g11 * Gxyy_v + g12 * Gyyy_v + g13 * Gzyy_v;
|
|
o_gyzx[i] = g11 * Gxyz_v + g12 * Gyyz_v + g13 * Gzyz_v;
|
|
o_gzzx[i] = g11 * Gxzz_v + g12 * Gyzz_v + g13 * Gzzz_v;
|
|
|
|
o_gxxy[i] = g12 * Gxxx_v + g22 * Gyxx_v + g23 * Gzxx_v;
|
|
o_gxyy[i] = g12 * Gxxy_v + g22 * Gyxy_v + g23 * Gzxy_v;
|
|
o_gxzy[i] = g12 * Gxxz_v + g22 * Gyxz_v + g23 * Gzxz_v;
|
|
o_gyyy[i] = g12 * Gxyy_v + g22 * Gyyy_v + g23 * Gzyy_v;
|
|
o_gyzy[i] = g12 * Gxyz_v + g22 * Gyyz_v + g23 * Gzyz_v;
|
|
o_gzzy[i] = g12 * Gxzz_v + g22 * Gyzz_v + g23 * Gzzz_v;
|
|
|
|
o_gxxz[i] = g13 * Gxxx_v + g23 * Gyxx_v + g33 * Gzxx_v;
|
|
o_gxyz[i] = g13 * Gxxy_v + g23 * Gyxy_v + g33 * Gzxy_v;
|
|
o_gxzz[i] = g13 * Gxxz_v + g23 * Gyxz_v + g33 * Gzxz_v;
|
|
o_gyyz[i] = g13 * Gxyy_v + g23 * Gyyy_v + g33 * Gzyy_v;
|
|
o_gyzz[i] = g13 * Gxyz_v + g23 * Gyyz_v + g33 * Gzyz_v;
|
|
o_gzzz[i] = g13 * Gxzz_v + g23 * Gyzz_v + g33 * Gzzz_v;
|
|
}
|
|
}
|
|
|
|
/* Phase 10: After fdderivs of a metric component, contract with gup^{ij}
|
|
* R_comp = gup^xx*fxx + gup^yy*fyy + gup^zz*fzz + 2*(gup^xy*fxy + gup^xz*fxz + gup^yz*fyz)
|
|
*/
|
|
__global__ void kern_phase10_ricci_contract(
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ fxx, const double* __restrict__ fxy,
|
|
const double* __restrict__ fxz, const double* __restrict__ fyy,
|
|
const double* __restrict__ fyz, const double* __restrict__ fzz,
|
|
double* __restrict__ R_comp)
|
|
{
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
R_comp[i] = gupxx[i]*fxx[i] + gupyy[i]*fyy[i] + gupzz[i]*fzz[i]
|
|
+ 2.0*(gupxy[i]*fxy[i] + gupxz[i]*fxz[i] + gupyz[i]*fyz[i]);
|
|
}
|
|
}
|
|
|
|
/* Phase 11a: Ricci diagonal assembly (Rxx, Ryy, Rzz) */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase11_ricci_diag(
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ Gamxa, const double* __restrict__ Gamya,
|
|
const double* __restrict__ Gamza,
|
|
const double* __restrict__ Gamxx, const double* __restrict__ Gamxy,
|
|
const double* __restrict__ Gamxz,
|
|
const double* __restrict__ Gamyx, const double* __restrict__ Gamyy_d,
|
|
const double* __restrict__ Gamyz_d,
|
|
const double* __restrict__ Gamzx, const double* __restrict__ Gamzy,
|
|
const double* __restrict__ Gamzz_d,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
/* lowered Christoffel products */
|
|
const double* __restrict__ lxxx, const double* __restrict__ lxyx,
|
|
const double* __restrict__ lxzx, const double* __restrict__ lyyx,
|
|
const double* __restrict__ lyzx, const double* __restrict__ lzzx,
|
|
const double* __restrict__ lxxy, const double* __restrict__ lxyy,
|
|
const double* __restrict__ lxzy, const double* __restrict__ lyyy,
|
|
const double* __restrict__ lyzy, const double* __restrict__ lzzy,
|
|
const double* __restrict__ lxxz, const double* __restrict__ lxyz,
|
|
const double* __restrict__ lxzz, const double* __restrict__ lyyz,
|
|
const double* __restrict__ lyzz, const double* __restrict__ lzzz,
|
|
double* __restrict__ Rxx, double* __restrict__ Ryy, double* __restrict__ Rzz)
|
|
{
|
|
const double H = 0.5, TWO = 2.0;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double uxx=gupxx[i],uxy=gupxy[i],uxz=gupxz[i];
|
|
double uyy=gupyy[i],uyz=gupyz[i],uzz=gupzz[i];
|
|
/* Rxx */
|
|
Rxx[i] = -H*Rxx[i]
|
|
+ gxx[i]*Gamxx[i]+gxy[i]*Gamyx[i]+gxz[i]*Gamzx[i]
|
|
+ Gamxa[i]*lxxx[i]+Gamya[i]*lxyx[i]+Gamza[i]*lxzx[i]
|
|
+ uxx*(TWO*(Gxxx[i]*lxxx[i]+Gyxx[i]*lxyx[i]+Gzxx[i]*lxzx[i])
|
|
+(Gxxx[i]*lxxx[i]+Gyxx[i]*lxxy[i]+Gzxx[i]*lxxz[i]))
|
|
+ uxy*(TWO*(Gxxx[i]*lxyx[i]+Gyxx[i]*lyyx[i]+Gzxx[i]*lyzx[i]
|
|
+Gxxy[i]*lxxx[i]+Gyxy[i]*lxyx[i]+Gzxy[i]*lxzx[i])
|
|
+(Gxxy[i]*lxxx[i]+Gyxy[i]*lxxy[i]+Gzxy[i]*lxxz[i])
|
|
+(Gxxx[i]*lxyx[i]+Gyxx[i]*lxyy[i]+Gzxx[i]*lxyz[i]))
|
|
+ uxz*(TWO*(Gxxx[i]*lxzx[i]+Gyxx[i]*lyzx[i]+Gzxx[i]*lzzx[i]
|
|
+Gxxz[i]*lxxx[i]+Gyxz[i]*lxyx[i]+Gzxz[i]*lxzx[i])
|
|
+(Gxxz[i]*lxxx[i]+Gyxz[i]*lxxy[i]+Gzxz[i]*lxxz[i])
|
|
+(Gxxx[i]*lxzx[i]+Gyxx[i]*lxzy[i]+Gzxx[i]*lxzz[i]))
|
|
+ uyy*(TWO*(Gxxy[i]*lxyx[i]+Gyxy[i]*lyyx[i]+Gzxy[i]*lyzx[i])
|
|
+(Gxxy[i]*lxyx[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxyz[i]))
|
|
+ uyz*(TWO*(Gxxy[i]*lxzx[i]+Gyxy[i]*lyzx[i]+Gzxy[i]*lzzx[i]
|
|
+Gxxz[i]*lxyx[i]+Gyxz[i]*lyyx[i]+Gzxz[i]*lyzx[i])
|
|
+(Gxxz[i]*lxyx[i]+Gyxz[i]*lxyy[i]+Gzxz[i]*lxyz[i])
|
|
+(Gxxy[i]*lxzx[i]+Gyxy[i]*lxzy[i]+Gzxy[i]*lxzz[i]))
|
|
+ uzz*(TWO*(Gxxz[i]*lxzx[i]+Gyxz[i]*lyzx[i]+Gzxz[i]*lzzx[i])
|
|
+(Gxxz[i]*lxzx[i]+Gyxz[i]*lxzy[i]+Gzxz[i]*lxzz[i]));
|
|
|
|
/* Ryy */
|
|
Ryy[i] = -H*Ryy[i]
|
|
+ gxy[i]*Gamxy[i]+gyy[i]*Gamyy_d[i]+gyz[i]*Gamzy[i]
|
|
+ Gamxa[i]*lxyy[i]+Gamya[i]*lyyy[i]+Gamza[i]*lyzy[i]
|
|
+ uxx*(TWO*(Gxxy[i]*lxxy[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxzy[i])
|
|
+(Gxxy[i]*lxyx[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxyz[i]))
|
|
+ uxy*(TWO*(Gxxy[i]*lxyy[i]+Gyxy[i]*lyyy[i]+Gzxy[i]*lyzy[i]
|
|
+Gxyy[i]*lxxy[i]+Gyyy[i]*lxyy[i]+Gzyy[i]*lxzy[i])
|
|
+(Gxyy[i]*lxyx[i]+Gyyy[i]*lxyy[i]+Gzyy[i]*lxyz[i])
|
|
+(Gxxy[i]*lyyx[i]+Gyxy[i]*lyyy[i]+Gzxy[i]*lyyz[i]))
|
|
+ uxz*(TWO*(Gxxy[i]*lxzy[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lzzy[i]
|
|
+Gxyz[i]*lxxy[i]+Gyyz[i]*lxyy[i]+Gzyz[i]*lxzy[i])
|
|
+(Gxyz[i]*lxyx[i]+Gyyz[i]*lxyy[i]+Gzyz[i]*lxyz[i])
|
|
+(Gxxy[i]*lyzx[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lyzz[i]))
|
|
+ uyy*(TWO*(Gxyy[i]*lxyy[i]+Gyyy[i]*lyyy[i]+Gzyy[i]*lyzy[i])
|
|
+(Gxyy[i]*lyyx[i]+Gyyy[i]*lyyy[i]+Gzyy[i]*lyyz[i]))
|
|
+ uyz*(TWO*(Gxyy[i]*lxzy[i]+Gyyy[i]*lyzy[i]+Gzyy[i]*lzzy[i]
|
|
+Gxyz[i]*lxyy[i]+Gyyz[i]*lyyy[i]+Gzyz[i]*lyzy[i])
|
|
+(Gxyz[i]*lyyx[i]+Gyyz[i]*lyyy[i]+Gzyz[i]*lyyz[i])
|
|
+(Gxyy[i]*lyzx[i]+Gyyy[i]*lyzy[i]+Gzyy[i]*lyzz[i]))
|
|
+ uzz*(TWO*(Gxyz[i]*lxzy[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lzzy[i])
|
|
+(Gxyz[i]*lyzx[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lyzz[i]));
|
|
|
|
/* Rzz */
|
|
Rzz[i] = -H*Rzz[i]
|
|
+ gxz[i]*Gamxz[i]+gyz[i]*Gamyz_d[i]+gzz[i]*Gamzz_d[i]
|
|
+ Gamxa[i]*lxzz[i]+Gamya[i]*lyzz[i]+Gamza[i]*lzzz[i]
|
|
+ uxx*(TWO*(Gxxz[i]*lxxz[i]+Gyxz[i]*lxyz[i]+Gzxz[i]*lxzz[i])
|
|
+(Gxxz[i]*lxzx[i]+Gyxz[i]*lxzy[i]+Gzxz[i]*lxzz[i]))
|
|
+ uxy*(TWO*(Gxxz[i]*lxyz[i]+Gyxz[i]*lyyz[i]+Gzxz[i]*lyzz[i]
|
|
+Gxyz[i]*lxxz[i]+Gyyz[i]*lxyz[i]+Gzyz[i]*lxzz[i])
|
|
+(Gxyz[i]*lxzx[i]+Gyyz[i]*lxzy[i]+Gzyz[i]*lxzz[i])
|
|
+(Gxxz[i]*lyzx[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lyzz[i]))
|
|
+ uxz*(TWO*(Gxxz[i]*lxzz[i]+Gyxz[i]*lyzz[i]+Gzxz[i]*lzzz[i]
|
|
+Gxzz[i]*lxxz[i]+Gyzz[i]*lxyz[i]+Gzzz[i]*lxzz[i])
|
|
+(Gxzz[i]*lxzx[i]+Gyzz[i]*lxzy[i]+Gzzz[i]*lxzz[i])
|
|
+(Gxxz[i]*lzzx[i]+Gyxz[i]*lzzy[i]+Gzxz[i]*lzzz[i]))
|
|
+ uyy*(TWO*(Gxyz[i]*lxyz[i]+Gyyz[i]*lyyz[i]+Gzyz[i]*lyzz[i])
|
|
+(Gxyz[i]*lyzx[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lyzz[i]))
|
|
+ uyz*(TWO*(Gxyz[i]*lxzz[i]+Gyyz[i]*lyzz[i]+Gzyz[i]*lzzz[i]
|
|
+Gxzz[i]*lxyz[i]+Gyzz[i]*lyyz[i]+Gzzz[i]*lyzz[i])
|
|
+(Gxzz[i]*lyzx[i]+Gyzz[i]*lyzy[i]+Gzzz[i]*lyzz[i])
|
|
+(Gxyz[i]*lzzx[i]+Gyyz[i]*lzzy[i]+Gzyz[i]*lzzz[i]))
|
|
+ uzz*(TWO*(Gxzz[i]*lxzz[i]+Gyzz[i]*lyzz[i]+Gzzz[i]*lzzz[i])
|
|
+(Gxzz[i]*lzzx[i]+Gyzz[i]*lzzy[i]+Gzzz[i]*lzzz[i]));
|
|
}
|
|
}
|
|
|
|
/* Phase 11b: Ricci off-diagonal assembly (Rxy, Rxz, Ryz) */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase11_ricci_offdiag(
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ Gamxa, const double* __restrict__ Gamya,
|
|
const double* __restrict__ Gamza,
|
|
const double* __restrict__ Gamxx, const double* __restrict__ Gamxy,
|
|
const double* __restrict__ Gamxz,
|
|
const double* __restrict__ Gamyx, const double* __restrict__ Gamyy_d,
|
|
const double* __restrict__ Gamyz_d,
|
|
const double* __restrict__ Gamzx, const double* __restrict__ Gamzy,
|
|
const double* __restrict__ Gamzz_d,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
const double* __restrict__ lxxx, const double* __restrict__ lxyx,
|
|
const double* __restrict__ lxzx, const double* __restrict__ lyyx,
|
|
const double* __restrict__ lyzx, const double* __restrict__ lzzx,
|
|
const double* __restrict__ lxxy, const double* __restrict__ lxyy,
|
|
const double* __restrict__ lxzy, const double* __restrict__ lyyy,
|
|
const double* __restrict__ lyzy, const double* __restrict__ lzzy,
|
|
const double* __restrict__ lxxz, const double* __restrict__ lxyz,
|
|
const double* __restrict__ lxzz, const double* __restrict__ lyyz,
|
|
const double* __restrict__ lyzz, const double* __restrict__ lzzz,
|
|
double* __restrict__ Rxy, double* __restrict__ Rxz, double* __restrict__ Ryz)
|
|
{
|
|
const double H = 0.5, TWO = 2.0;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double uxx=gupxx[i],uxy=gupxy[i],uxz=gupxz[i];
|
|
double uyy=gupyy[i],uyz=gupyz[i],uzz=gupzz[i];
|
|
|
|
/* Rxy */
|
|
Rxy[i] = H*(
|
|
-Rxy[i]
|
|
+gxx[i]*Gamxy[i]+gxy[i]*Gamyy_d[i]+gxz[i]*Gamzy[i]
|
|
+gxy[i]*Gamxx[i]+gyy[i]*Gamyx[i]+gyz[i]*Gamzx[i]
|
|
+Gamxa[i]*lxyx[i]+Gamya[i]*lyyx[i]+Gamza[i]*lyzx[i]
|
|
+Gamxa[i]*lxxy[i]+Gamya[i]*lxyy[i]+Gamza[i]*lxzy[i])
|
|
+uxx*(Gxxx[i]*lxxy[i]+Gyxx[i]*lxyy[i]+Gzxx[i]*lxzy[i]
|
|
+Gxxy[i]*lxxx[i]+Gyxy[i]*lxyx[i]+Gzxy[i]*lxzx[i]
|
|
+Gxxx[i]*lxyx[i]+Gyxx[i]*lxyy[i]+Gzxx[i]*lxyz[i])
|
|
+uxy*(Gxxx[i]*lxyy[i]+Gyxx[i]*lyyy[i]+Gzxx[i]*lyzy[i]
|
|
+Gxxy[i]*lxyx[i]+Gyxy[i]*lyyx[i]+Gzxy[i]*lyzx[i]
|
|
+Gxxy[i]*lxyx[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxyz[i]
|
|
+Gxxy[i]*lxxy[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxzy[i]
|
|
+Gxyy[i]*lxxx[i]+Gyyy[i]*lxyx[i]+Gzyy[i]*lxzx[i]
|
|
+Gxxx[i]*lyyx[i]+Gyxx[i]*lyyy[i]+Gzxx[i]*lyyz[i])
|
|
+uxz*(Gxxx[i]*lxzy[i]+Gyxx[i]*lyzy[i]+Gzxx[i]*lzzy[i]
|
|
+Gxxy[i]*lxzx[i]+Gyxy[i]*lyzx[i]+Gzxy[i]*lzzx[i]
|
|
+Gxxz[i]*lxyx[i]+Gyxz[i]*lxyy[i]+Gzxz[i]*lxyz[i]
|
|
+Gxxz[i]*lxxy[i]+Gyxz[i]*lxyy[i]+Gzxz[i]*lxzy[i]
|
|
+Gxyz[i]*lxxx[i]+Gyyz[i]*lxyx[i]+Gzyz[i]*lxzx[i]
|
|
+Gxxx[i]*lyzx[i]+Gyxx[i]*lyzy[i]+Gzxx[i]*lyzz[i])
|
|
+uyy*(Gxxy[i]*lxyy[i]+Gyxy[i]*lyyy[i]+Gzxy[i]*lyzy[i]
|
|
+Gxyy[i]*lxyx[i]+Gyyy[i]*lyyx[i]+Gzyy[i]*lyzx[i]
|
|
+Gxxy[i]*lyyx[i]+Gyxy[i]*lyyy[i]+Gzxy[i]*lyyz[i])
|
|
+uyz*(Gxxy[i]*lxzy[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lzzy[i]
|
|
+Gxyy[i]*lxzx[i]+Gyyy[i]*lyzx[i]+Gzyy[i]*lzzx[i]
|
|
+Gxxz[i]*lyyx[i]+Gyxz[i]*lyyy[i]+Gzxz[i]*lyyz[i]
|
|
+Gxxz[i]*lxyy[i]+Gyxz[i]*lyyy[i]+Gzxz[i]*lyzy[i]
|
|
+Gxyz[i]*lxyx[i]+Gyyz[i]*lyyx[i]+Gzyz[i]*lyzx[i]
|
|
+Gxxy[i]*lyzx[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lyzz[i])
|
|
+uzz*(Gxxz[i]*lxzy[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lzzy[i]
|
|
+Gxyz[i]*lxzx[i]+Gyyz[i]*lyzx[i]+Gzyz[i]*lzzx[i]
|
|
+Gxxz[i]*lyzx[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lyzz[i]);
|
|
|
|
/* Rxz */
|
|
Rxz[i] = H*(
|
|
-Rxz[i]
|
|
+gxx[i]*Gamxz[i]+gxy[i]*Gamyz_d[i]+gxz[i]*Gamzz_d[i]
|
|
+gxz[i]*Gamxx[i]+gyz[i]*Gamyx[i]+gzz[i]*Gamzx[i]
|
|
+Gamxa[i]*lxzx[i]+Gamya[i]*lyzx[i]+Gamza[i]*lzzx[i]
|
|
+Gamxa[i]*lxxz[i]+Gamya[i]*lxyz[i]+Gamza[i]*lxzz[i])
|
|
+uxx*(Gxxx[i]*lxxz[i]+Gyxx[i]*lxyz[i]+Gzxx[i]*lxzz[i]
|
|
+Gxxz[i]*lxxx[i]+Gyxz[i]*lxyx[i]+Gzxz[i]*lxzx[i]
|
|
+Gxxx[i]*lxzx[i]+Gyxx[i]*lxzy[i]+Gzxx[i]*lxzz[i])
|
|
+uxy*(Gxxx[i]*lxyz[i]+Gyxx[i]*lyyz[i]+Gzxx[i]*lyzz[i]
|
|
+Gxxz[i]*lxyx[i]+Gyxz[i]*lyyx[i]+Gzxz[i]*lyzx[i]
|
|
+Gxxy[i]*lxzx[i]+Gyxy[i]*lxzy[i]+Gzxy[i]*lxzz[i]
|
|
+Gxxy[i]*lxxz[i]+Gyxy[i]*lxyz[i]+Gzxy[i]*lxzz[i]
|
|
+Gxyz[i]*lxxx[i]+Gyyz[i]*lxyx[i]+Gzyz[i]*lxzx[i]
|
|
+Gxxx[i]*lyzx[i]+Gyxx[i]*lyzy[i]+Gzxx[i]*lyzz[i])
|
|
+uxz*(Gxxx[i]*lxzz[i]+Gyxx[i]*lyzz[i]+Gzxx[i]*lzzz[i]
|
|
+Gxxz[i]*lxzx[i]+Gyxz[i]*lyzx[i]+Gzxz[i]*lzzx[i]
|
|
+Gxxz[i]*lxzx[i]+Gyxz[i]*lxzy[i]+Gzxz[i]*lxzz[i]
|
|
+Gxxz[i]*lxxz[i]+Gyxz[i]*lxyz[i]+Gzxz[i]*lxzz[i]
|
|
+Gxzz[i]*lxxx[i]+Gyzz[i]*lxyx[i]+Gzzz[i]*lxzx[i]
|
|
+Gxxx[i]*lzzx[i]+Gyxx[i]*lzzy[i]+Gzxx[i]*lzzz[i])
|
|
+uyy*(Gxxy[i]*lxyz[i]+Gyxy[i]*lyyz[i]+Gzxy[i]*lyzz[i]
|
|
+Gxyz[i]*lxyx[i]+Gyyz[i]*lyyx[i]+Gzyz[i]*lyzx[i]
|
|
+Gxxy[i]*lyzx[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lyzz[i])
|
|
+uyz*(Gxxy[i]*lxzz[i]+Gyxy[i]*lyzz[i]+Gzxy[i]*lzzz[i]
|
|
+Gxyz[i]*lxzx[i]+Gyyz[i]*lyzx[i]+Gzyz[i]*lzzx[i]
|
|
+Gxxz[i]*lyzx[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lyzz[i]
|
|
+Gxxz[i]*lxyz[i]+Gyxz[i]*lyyz[i]+Gzxz[i]*lyzz[i]
|
|
+Gxzz[i]*lxyx[i]+Gyzz[i]*lyyx[i]+Gzzz[i]*lyzx[i]
|
|
+Gxxy[i]*lzzx[i]+Gyxy[i]*lzzy[i]+Gzxy[i]*lzzz[i])
|
|
+uzz*(Gxxz[i]*lxzz[i]+Gyxz[i]*lyzz[i]+Gzxz[i]*lzzz[i]
|
|
+Gxzz[i]*lxzx[i]+Gyzz[i]*lyzx[i]+Gzzz[i]*lzzx[i]
|
|
+Gxxz[i]*lzzx[i]+Gyxz[i]*lzzy[i]+Gzxz[i]*lzzz[i]);
|
|
|
|
/* Ryz */
|
|
Ryz[i] = H*(
|
|
-Ryz[i]
|
|
+gxy[i]*Gamxz[i]+gyy[i]*Gamyz_d[i]+gyz[i]*Gamzz_d[i]
|
|
+gxz[i]*Gamxy[i]+gyz[i]*Gamyy_d[i]+gzz[i]*Gamzy[i]
|
|
+Gamxa[i]*lxzy[i]+Gamya[i]*lyzy[i]+Gamza[i]*lzzy[i]
|
|
+Gamxa[i]*lxyz[i]+Gamya[i]*lyyz[i]+Gamza[i]*lyzz[i])
|
|
+uxx*(Gxxy[i]*lxxz[i]+Gyxy[i]*lxyz[i]+Gzxy[i]*lxzz[i]
|
|
+Gxxz[i]*lxxy[i]+Gyxz[i]*lxyy[i]+Gzxz[i]*lxzy[i]
|
|
+Gxxy[i]*lxzx[i]+Gyxy[i]*lxzy[i]+Gzxy[i]*lxzz[i])
|
|
+uxy*(Gxxy[i]*lxyz[i]+Gyxy[i]*lyyz[i]+Gzxy[i]*lyzz[i]
|
|
+Gxxz[i]*lxyy[i]+Gyxz[i]*lyyy[i]+Gzxz[i]*lyzy[i]
|
|
+Gxyy[i]*lxzx[i]+Gyyy[i]*lxzy[i]+Gzyy[i]*lxzz[i]
|
|
+Gxyy[i]*lxxz[i]+Gyyy[i]*lxyz[i]+Gzyy[i]*lxzz[i]
|
|
+Gxyz[i]*lxxy[i]+Gyyz[i]*lxyy[i]+Gzyz[i]*lxzy[i]
|
|
+Gxxy[i]*lyzx[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lyzz[i])
|
|
+uxz*(Gxxy[i]*lxzz[i]+Gyxy[i]*lyzz[i]+Gzxy[i]*lzzz[i]
|
|
+Gxxz[i]*lxzy[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lzzy[i]
|
|
+Gxyz[i]*lxzx[i]+Gyyz[i]*lxzy[i]+Gzyz[i]*lxzz[i]
|
|
+Gxyz[i]*lxxz[i]+Gyyz[i]*lxyz[i]+Gzyz[i]*lxzz[i]
|
|
+Gxzz[i]*lxxy[i]+Gyzz[i]*lxyy[i]+Gzzz[i]*lxzy[i]
|
|
+Gxxy[i]*lzzx[i]+Gyxy[i]*lzzy[i]+Gzxy[i]*lzzz[i])
|
|
+uyy*(Gxyy[i]*lxyz[i]+Gyyy[i]*lyyz[i]+Gzyy[i]*lyzz[i]
|
|
+Gxyz[i]*lxyy[i]+Gyyz[i]*lyyy[i]+Gzyz[i]*lyzy[i]
|
|
+Gxyy[i]*lyzx[i]+Gyyy[i]*lyzy[i]+Gzyy[i]*lyzz[i])
|
|
+uyz*(Gxyy[i]*lxzz[i]+Gyyy[i]*lyzz[i]+Gzyy[i]*lzzz[i]
|
|
+Gxyz[i]*lxzy[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lzzy[i]
|
|
+Gxyz[i]*lyzx[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lyzz[i]
|
|
+Gxyz[i]*lxyz[i]+Gyyz[i]*lyyz[i]+Gzyz[i]*lyzz[i]
|
|
+Gxzz[i]*lxyy[i]+Gyzz[i]*lyyy[i]+Gzzz[i]*lyzy[i]
|
|
+Gxyy[i]*lzzx[i]+Gyyy[i]*lzzy[i]+Gzyy[i]*lzzz[i])
|
|
+uzz*(Gxyz[i]*lxzz[i]+Gyyz[i]*lyzz[i]+Gzyz[i]*lzzz[i]
|
|
+Gxzz[i]*lxzy[i]+Gyzz[i]*lyzy[i]+Gzzz[i]*lzzy[i]
|
|
+Gxyz[i]*lzzx[i]+Gyyz[i]*lzzy[i]+Gzyz[i]*lzzz[i]);
|
|
}
|
|
}
|
|
|
|
/* Phase 11: fused Ricci assembly (diag + off-diag) */
|
|
__global__ __launch_bounds__(128, 2)
|
|
void kern_phase11_ricci_fused(
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ Gamxa, const double* __restrict__ Gamya,
|
|
const double* __restrict__ Gamza,
|
|
const double* __restrict__ Gamxx, const double* __restrict__ Gamxy,
|
|
const double* __restrict__ Gamxz,
|
|
const double* __restrict__ Gamyx, const double* __restrict__ Gamyy_d,
|
|
const double* __restrict__ Gamyz_d,
|
|
const double* __restrict__ Gamzx, const double* __restrict__ Gamzy,
|
|
const double* __restrict__ Gamzz_d,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
const double* __restrict__ lxxx, const double* __restrict__ lxyx,
|
|
const double* __restrict__ lxzx, const double* __restrict__ lyyx,
|
|
const double* __restrict__ lyzx, const double* __restrict__ lzzx,
|
|
const double* __restrict__ lxxy, const double* __restrict__ lxyy,
|
|
const double* __restrict__ lxzy, const double* __restrict__ lyyy,
|
|
const double* __restrict__ lyzy, const double* __restrict__ lzzy,
|
|
const double* __restrict__ lxxz, const double* __restrict__ lxyz,
|
|
const double* __restrict__ lxzz, const double* __restrict__ lyyz,
|
|
const double* __restrict__ lyzz, const double* __restrict__ lzzz,
|
|
double* __restrict__ Rxx, double* __restrict__ Rxy, double* __restrict__ Rxz,
|
|
double* __restrict__ Ryy, double* __restrict__ Ryz, double* __restrict__ Rzz)
|
|
{
|
|
const double H = 0.5, TWO = 2.0;
|
|
for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < d_gp.all; i += blockDim.x * gridDim.x) {
|
|
double uxx = gupxx[i], uxy = gupxy[i], uxz = gupxz[i];
|
|
double uyy = gupyy[i], uyz = gupyz[i], uzz = gupzz[i];
|
|
|
|
Rxx[i] = -H*Rxx[i]
|
|
+ gxx[i]*Gamxx[i]+gxy[i]*Gamyx[i]+gxz[i]*Gamzx[i]
|
|
+ Gamxa[i]*lxxx[i]+Gamya[i]*lxyx[i]+Gamza[i]*lxzx[i]
|
|
+ uxx*(TWO*(Gxxx[i]*lxxx[i]+Gyxx[i]*lxyx[i]+Gzxx[i]*lxzx[i])
|
|
+(Gxxx[i]*lxxx[i]+Gyxx[i]*lxxy[i]+Gzxx[i]*lxxz[i]))
|
|
+ uxy*(TWO*(Gxxx[i]*lxyx[i]+Gyxx[i]*lyyx[i]+Gzxx[i]*lyzx[i]
|
|
+Gxxy[i]*lxxx[i]+Gyxy[i]*lxyx[i]+Gzxy[i]*lxzx[i])
|
|
+(Gxxy[i]*lxxx[i]+Gyxy[i]*lxxy[i]+Gzxy[i]*lxxz[i])
|
|
+(Gxxx[i]*lxyx[i]+Gyxx[i]*lxyy[i]+Gzxx[i]*lxyz[i]))
|
|
+ uxz*(TWO*(Gxxx[i]*lxzx[i]+Gyxx[i]*lyzx[i]+Gzxx[i]*lzzx[i]
|
|
+Gxxz[i]*lxxx[i]+Gyxz[i]*lxyx[i]+Gzxz[i]*lxzx[i])
|
|
+(Gxxz[i]*lxxx[i]+Gyxz[i]*lxxy[i]+Gzxz[i]*lxxz[i])
|
|
+(Gxxx[i]*lxzx[i]+Gyxx[i]*lxzy[i]+Gzxx[i]*lxzz[i]))
|
|
+ uyy*(TWO*(Gxxy[i]*lxyx[i]+Gyxy[i]*lyyx[i]+Gzxy[i]*lyzx[i])
|
|
+(Gxxy[i]*lxyx[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxyz[i]))
|
|
+ uyz*(TWO*(Gxxy[i]*lxzx[i]+Gyxy[i]*lyzx[i]+Gzxy[i]*lzzx[i]
|
|
+Gxxz[i]*lxyx[i]+Gyxz[i]*lyyx[i]+Gzxz[i]*lyzx[i])
|
|
+(Gxxz[i]*lxyx[i]+Gyxz[i]*lxyy[i]+Gzxz[i]*lxyz[i])
|
|
+(Gxxy[i]*lxzx[i]+Gyxy[i]*lxzy[i]+Gzxy[i]*lxzz[i]))
|
|
+ uzz*(TWO*(Gxxz[i]*lxzx[i]+Gyxz[i]*lyzx[i]+Gzxz[i]*lzzx[i])
|
|
+(Gxxz[i]*lxzx[i]+Gyxz[i]*lxzy[i]+Gzxz[i]*lxzz[i]));
|
|
|
|
Ryy[i] = -H*Ryy[i]
|
|
+ gxy[i]*Gamxy[i]+gyy[i]*Gamyy_d[i]+gyz[i]*Gamzy[i]
|
|
+ Gamxa[i]*lxyy[i]+Gamya[i]*lyyy[i]+Gamza[i]*lyzy[i]
|
|
+ uxx*(TWO*(Gxxy[i]*lxxy[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxzy[i])
|
|
+(Gxxy[i]*lxyx[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxyz[i]))
|
|
+ uxy*(TWO*(Gxxy[i]*lxyy[i]+Gyxy[i]*lyyy[i]+Gzxy[i]*lyzy[i]
|
|
+Gxyy[i]*lxxy[i]+Gyyy[i]*lxyy[i]+Gzyy[i]*lxzy[i])
|
|
+(Gxyy[i]*lxyx[i]+Gyyy[i]*lxyy[i]+Gzyy[i]*lxyz[i])
|
|
+(Gxxy[i]*lyyx[i]+Gyxy[i]*lyyy[i]+Gzxy[i]*lyyz[i]))
|
|
+ uxz*(TWO*(Gxxy[i]*lxzy[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lzzy[i]
|
|
+Gxyz[i]*lxxy[i]+Gyyz[i]*lxyy[i]+Gzyz[i]*lxzy[i])
|
|
+(Gxyz[i]*lxyx[i]+Gyyz[i]*lxyy[i]+Gzyz[i]*lxyz[i])
|
|
+(Gxxy[i]*lyzx[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lyzz[i]))
|
|
+ uyy*(TWO*(Gxyy[i]*lxyy[i]+Gyyy[i]*lyyy[i]+Gzyy[i]*lyzy[i])
|
|
+(Gxyy[i]*lyyx[i]+Gyyy[i]*lyyy[i]+Gzyy[i]*lyyz[i]))
|
|
+ uyz*(TWO*(Gxyy[i]*lxzy[i]+Gyyy[i]*lyzy[i]+Gzyy[i]*lzzy[i]
|
|
+Gxyz[i]*lxyy[i]+Gyyz[i]*lyyy[i]+Gzyz[i]*lyzy[i])
|
|
+(Gxyz[i]*lyyx[i]+Gyyz[i]*lyyy[i]+Gzyz[i]*lyyz[i])
|
|
+(Gxyy[i]*lyzx[i]+Gyyy[i]*lyzy[i]+Gzyy[i]*lyzz[i]))
|
|
+ uzz*(TWO*(Gxyz[i]*lxzy[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lzzy[i])
|
|
+(Gxyz[i]*lyzx[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lyzz[i]));
|
|
|
|
Rzz[i] = -H*Rzz[i]
|
|
+ gxz[i]*Gamxz[i]+gyz[i]*Gamyz_d[i]+gzz[i]*Gamzz_d[i]
|
|
+ Gamxa[i]*lxzz[i]+Gamya[i]*lyzz[i]+Gamza[i]*lzzz[i]
|
|
+ uxx*(TWO*(Gxxz[i]*lxxz[i]+Gyxz[i]*lxyz[i]+Gzxz[i]*lxzz[i])
|
|
+(Gxxz[i]*lxzx[i]+Gyxz[i]*lxzy[i]+Gzxz[i]*lxzz[i]))
|
|
+ uxy*(TWO*(Gxxz[i]*lxyz[i]+Gyxz[i]*lyyz[i]+Gzxz[i]*lyzz[i]
|
|
+Gxyz[i]*lxxz[i]+Gyyz[i]*lxyz[i]+Gzyz[i]*lxzz[i])
|
|
+(Gxyz[i]*lxzx[i]+Gyyz[i]*lxzy[i]+Gzyz[i]*lxzz[i])
|
|
+(Gxxz[i]*lyzx[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lyzz[i]))
|
|
+ uxz*(TWO*(Gxxz[i]*lxzz[i]+Gyxz[i]*lyzz[i]+Gzxz[i]*lzzz[i]
|
|
+Gxzz[i]*lxxz[i]+Gyzz[i]*lxyz[i]+Gzzz[i]*lxzz[i])
|
|
+(Gxzz[i]*lxzx[i]+Gyzz[i]*lxzy[i]+Gzzz[i]*lxzz[i])
|
|
+(Gxxz[i]*lzzx[i]+Gyxz[i]*lzzy[i]+Gzxz[i]*lzzz[i]))
|
|
+ uyy*(TWO*(Gxyz[i]*lxyz[i]+Gyyz[i]*lyyz[i]+Gzyz[i]*lyzz[i])
|
|
+(Gxyz[i]*lyzx[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lyzz[i]))
|
|
+ uyz*(TWO*(Gxyz[i]*lxzz[i]+Gyyz[i]*lyzz[i]+Gzyz[i]*lzzz[i]
|
|
+Gxzz[i]*lxyz[i]+Gyzz[i]*lyyz[i]+Gzzz[i]*lyzz[i])
|
|
+(Gxzz[i]*lyzx[i]+Gyzz[i]*lyzy[i]+Gzzz[i]*lyzz[i])
|
|
+(Gxyz[i]*lzzx[i]+Gyyz[i]*lzzy[i]+Gzyz[i]*lzzz[i]))
|
|
+ uzz*(TWO*(Gxzz[i]*lxzz[i]+Gyzz[i]*lyzz[i]+Gzzz[i]*lzzz[i])
|
|
+(Gxzz[i]*lzzx[i]+Gyzz[i]*lzzy[i]+Gzzz[i]*lzzz[i]));
|
|
|
|
Rxy[i] = H*(
|
|
-Rxy[i]
|
|
+gxx[i]*Gamxy[i]+gxy[i]*Gamyy_d[i]+gxz[i]*Gamzy[i]
|
|
+gxy[i]*Gamxx[i]+gyy[i]*Gamyx[i]+gyz[i]*Gamzx[i]
|
|
+Gamxa[i]*lxyx[i]+Gamya[i]*lyyx[i]+Gamza[i]*lyzx[i]
|
|
+Gamxa[i]*lxxy[i]+Gamya[i]*lxyy[i]+Gamza[i]*lxzy[i])
|
|
+uxx*(Gxxx[i]*lxxy[i]+Gyxx[i]*lxyy[i]+Gzxx[i]*lxzy[i]
|
|
+Gxxy[i]*lxxx[i]+Gyxy[i]*lxyx[i]+Gzxy[i]*lxzx[i]
|
|
+Gxxx[i]*lxyx[i]+Gyxx[i]*lxyy[i]+Gzxx[i]*lxyz[i])
|
|
+uxy*(Gxxx[i]*lxyy[i]+Gyxx[i]*lyyy[i]+Gzxx[i]*lyzy[i]
|
|
+Gxxy[i]*lxyx[i]+Gyxy[i]*lyyx[i]+Gzxy[i]*lyzx[i]
|
|
+Gxxy[i]*lxyx[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxyz[i]
|
|
+Gxxy[i]*lxxy[i]+Gyxy[i]*lxyy[i]+Gzxy[i]*lxzy[i]
|
|
+Gxyy[i]*lxxx[i]+Gyyy[i]*lxyx[i]+Gzyy[i]*lxzx[i]
|
|
+Gxxx[i]*lyyx[i]+Gyxx[i]*lyyy[i]+Gzxx[i]*lyyz[i])
|
|
+uxz*(Gxxx[i]*lxzy[i]+Gyxx[i]*lyzy[i]+Gzxx[i]*lzzy[i]
|
|
+Gxxy[i]*lxzx[i]+Gyxy[i]*lyzx[i]+Gzxy[i]*lzzx[i]
|
|
+Gxxz[i]*lxyx[i]+Gyxz[i]*lxyy[i]+Gzxz[i]*lxyz[i]
|
|
+Gxxz[i]*lxxy[i]+Gyxz[i]*lxyy[i]+Gzxz[i]*lxzy[i]
|
|
+Gxyz[i]*lxxx[i]+Gyyz[i]*lxyx[i]+Gzyz[i]*lxzx[i]
|
|
+Gxxx[i]*lyzx[i]+Gyxx[i]*lyzy[i]+Gzxx[i]*lyzz[i])
|
|
+uyy*(Gxxy[i]*lxyy[i]+Gyxy[i]*lyyy[i]+Gzxy[i]*lyzy[i]
|
|
+Gxyy[i]*lxyx[i]+Gyyy[i]*lyyx[i]+Gzyy[i]*lyzx[i]
|
|
+Gxxy[i]*lyyx[i]+Gyxy[i]*lyyy[i]+Gzxy[i]*lyyz[i])
|
|
+uyz*(Gxxy[i]*lxzy[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lzzy[i]
|
|
+Gxyy[i]*lxzx[i]+Gyyy[i]*lyzx[i]+Gzyy[i]*lzzx[i]
|
|
+Gxxz[i]*lyyx[i]+Gyxz[i]*lyyy[i]+Gzxz[i]*lyyz[i]
|
|
+Gxxz[i]*lxyy[i]+Gyxz[i]*lyyy[i]+Gzxz[i]*lyzy[i]
|
|
+Gxyz[i]*lxyx[i]+Gyyz[i]*lyyx[i]+Gzyz[i]*lyzx[i]
|
|
+Gxxy[i]*lyzx[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lyzz[i])
|
|
+uzz*(Gxxz[i]*lxzy[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lzzy[i]
|
|
+Gxyz[i]*lxzx[i]+Gyyz[i]*lyzx[i]+Gzyz[i]*lzzx[i]
|
|
+Gxxz[i]*lyzx[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lyzz[i]);
|
|
|
|
Rxz[i] = H*(
|
|
-Rxz[i]
|
|
+gxx[i]*Gamxz[i]+gxy[i]*Gamyz_d[i]+gxz[i]*Gamzz_d[i]
|
|
+gxz[i]*Gamxx[i]+gyz[i]*Gamyx[i]+gzz[i]*Gamzx[i]
|
|
+Gamxa[i]*lxzx[i]+Gamya[i]*lyzx[i]+Gamza[i]*lzzx[i]
|
|
+Gamxa[i]*lxxz[i]+Gamya[i]*lxyz[i]+Gamza[i]*lxzz[i])
|
|
+uxx*(Gxxx[i]*lxxz[i]+Gyxx[i]*lxyz[i]+Gzxx[i]*lxzz[i]
|
|
+Gxxz[i]*lxxx[i]+Gyxz[i]*lxyx[i]+Gzxz[i]*lxzx[i]
|
|
+Gxxx[i]*lxzx[i]+Gyxx[i]*lxzy[i]+Gzxx[i]*lxzz[i])
|
|
+uxy*(Gxxx[i]*lxyz[i]+Gyxx[i]*lyyz[i]+Gzxx[i]*lyzz[i]
|
|
+Gxxz[i]*lxyx[i]+Gyxz[i]*lyyx[i]+Gzxz[i]*lyzx[i]
|
|
+Gxxy[i]*lxzx[i]+Gyxy[i]*lxzy[i]+Gzxy[i]*lxzz[i]
|
|
+Gxxy[i]*lxxz[i]+Gyxy[i]*lxyz[i]+Gzxy[i]*lxzz[i]
|
|
+Gxyz[i]*lxxx[i]+Gyyz[i]*lxyx[i]+Gzyz[i]*lxzx[i]
|
|
+Gxxx[i]*lyzx[i]+Gyxx[i]*lyzy[i]+Gzxx[i]*lyzz[i])
|
|
+uxz*(Gxxx[i]*lxzz[i]+Gyxx[i]*lyzz[i]+Gzxx[i]*lzzz[i]
|
|
+Gxxz[i]*lxzx[i]+Gyxz[i]*lyzx[i]+Gzxz[i]*lzzx[i]
|
|
+Gxxz[i]*lxzx[i]+Gyxz[i]*lxzy[i]+Gzxz[i]*lxzz[i]
|
|
+Gxxz[i]*lxxz[i]+Gyxz[i]*lxyz[i]+Gzxz[i]*lxzz[i]
|
|
+Gxzz[i]*lxxx[i]+Gyzz[i]*lxyx[i]+Gzzz[i]*lxzx[i]
|
|
+Gxxx[i]*lzzx[i]+Gyxx[i]*lzzy[i]+Gzxx[i]*lzzz[i])
|
|
+uyy*(Gxxy[i]*lxyz[i]+Gyxy[i]*lyyz[i]+Gzxy[i]*lyzz[i]
|
|
+Gxyz[i]*lxyx[i]+Gyyz[i]*lyyx[i]+Gzyz[i]*lyzx[i]
|
|
+Gxxy[i]*lyzx[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lyzz[i])
|
|
+uyz*(Gxxy[i]*lxzz[i]+Gyxy[i]*lyzz[i]+Gzxy[i]*lzzz[i]
|
|
+Gxyz[i]*lxzx[i]+Gyyz[i]*lyzx[i]+Gzyz[i]*lzzx[i]
|
|
+Gxxz[i]*lyzx[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lyzz[i]
|
|
+Gxxz[i]*lxyz[i]+Gyxz[i]*lyyz[i]+Gzxz[i]*lyzz[i]
|
|
+Gxzz[i]*lxyx[i]+Gyzz[i]*lyyx[i]+Gzzz[i]*lyzx[i]
|
|
+Gxxy[i]*lzzx[i]+Gyxy[i]*lzzy[i]+Gzxy[i]*lzzz[i])
|
|
+uzz*(Gxxz[i]*lxzz[i]+Gyxz[i]*lyzz[i]+Gzxz[i]*lzzz[i]
|
|
+Gxzz[i]*lxzx[i]+Gyzz[i]*lyzx[i]+Gzzz[i]*lzzx[i]
|
|
+Gxxz[i]*lzzx[i]+Gyxz[i]*lzzy[i]+Gzxz[i]*lzzz[i]);
|
|
|
|
Ryz[i] = H*(
|
|
-Ryz[i]
|
|
+gxy[i]*Gamxz[i]+gyy[i]*Gamyz_d[i]+gyz[i]*Gamzz_d[i]
|
|
+gxz[i]*Gamxy[i]+gyz[i]*Gamyy_d[i]+gzz[i]*Gamzy[i]
|
|
+Gamxa[i]*lxzy[i]+Gamya[i]*lyzy[i]+Gamza[i]*lzzy[i]
|
|
+Gamxa[i]*lxyz[i]+Gamya[i]*lyyz[i]+Gamza[i]*lyzz[i])
|
|
+uxx*(Gxxy[i]*lxxz[i]+Gyxy[i]*lxyz[i]+Gzxy[i]*lxzz[i]
|
|
+Gxxz[i]*lxxy[i]+Gyxz[i]*lxyy[i]+Gzxz[i]*lxzy[i]
|
|
+Gxxy[i]*lxzx[i]+Gyxy[i]*lxzy[i]+Gzxy[i]*lxzz[i])
|
|
+uxy*(Gxxy[i]*lxyz[i]+Gyxy[i]*lyyz[i]+Gzxy[i]*lyzz[i]
|
|
+Gxxz[i]*lxyy[i]+Gyxz[i]*lyyy[i]+Gzxz[i]*lyzy[i]
|
|
+Gxyy[i]*lxzx[i]+Gyyy[i]*lxzy[i]+Gzyy[i]*lxzz[i]
|
|
+Gxyy[i]*lxxz[i]+Gyyy[i]*lxyz[i]+Gzyy[i]*lxzz[i]
|
|
+Gxyz[i]*lxxy[i]+Gyyz[i]*lxyy[i]+Gzyz[i]*lxzy[i]
|
|
+Gxxy[i]*lyzx[i]+Gyxy[i]*lyzy[i]+Gzxy[i]*lyzz[i])
|
|
+uxz*(Gxxy[i]*lxzz[i]+Gyxy[i]*lyzz[i]+Gzxy[i]*lzzz[i]
|
|
+Gxxz[i]*lxzy[i]+Gyxz[i]*lyzy[i]+Gzxz[i]*lzzy[i]
|
|
+Gxyz[i]*lxzx[i]+Gyyz[i]*lxzy[i]+Gzyz[i]*lxzz[i]
|
|
+Gxyz[i]*lxxz[i]+Gyyz[i]*lxyz[i]+Gzyz[i]*lxzz[i]
|
|
+Gxzz[i]*lxxy[i]+Gyzz[i]*lxyy[i]+Gzzz[i]*lxzy[i]
|
|
+Gxxy[i]*lzzx[i]+Gyxy[i]*lzzy[i]+Gzxy[i]*lzzz[i])
|
|
+uyy*(Gxyy[i]*lxyz[i]+Gyyy[i]*lyyz[i]+Gzyy[i]*lyzz[i]
|
|
+Gxyz[i]*lxyy[i]+Gyyz[i]*lyyy[i]+Gzyz[i]*lyzy[i]
|
|
+Gxyy[i]*lyzx[i]+Gyyy[i]*lyzy[i]+Gzyy[i]*lyzz[i])
|
|
+uyz*(Gxyy[i]*lxzz[i]+Gyyy[i]*lyzz[i]+Gzyy[i]*lzzz[i]
|
|
+Gxyz[i]*lxzy[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lzzy[i]
|
|
+Gxyz[i]*lyzx[i]+Gyyz[i]*lyzy[i]+Gzyz[i]*lyzz[i]
|
|
+Gxyz[i]*lxyz[i]+Gyyz[i]*lyyz[i]+Gzyz[i]*lyzz[i]
|
|
+Gxzz[i]*lxyy[i]+Gyzz[i]*lyyy[i]+Gzzz[i]*lyzy[i]
|
|
+Gxyy[i]*lzzx[i]+Gyyy[i]*lzzy[i]+Gzyy[i]*lzzz[i])
|
|
+uzz*(Gxyz[i]*lxzz[i]+Gyyz[i]*lyzz[i]+Gzyz[i]*lzzz[i]
|
|
+Gxzz[i]*lxzy[i]+Gyzz[i]*lyzy[i]+Gzzz[i]*lzzy[i]
|
|
+Gxyz[i]*lzzx[i]+Gyyz[i]*lzzy[i]+Gzyz[i]*lzzz[i]);
|
|
}
|
|
}
|
|
|
|
/* Phase 13: chi correction to Ricci tensor
|
|
* After fdderivs(chi), subtract Christoffel*chi_deriv, compute conformal factor f,
|
|
* then add chi contribution to Rxx..Rzz.
|
|
*/
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase12_13_chi_correction_fused(
|
|
const double* __restrict__ chi,
|
|
const double* __restrict__ chin1,
|
|
const double* __restrict__ chix, const double* __restrict__ chiy,
|
|
const double* __restrict__ chiz,
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
double* __restrict__ Rxx, double* __restrict__ Rxy,
|
|
double* __restrict__ Rxz, double* __restrict__ Ryy,
|
|
double* __restrict__ Ryz, double* __restrict__ Rzz)
|
|
{
|
|
const double TWO = 2.0;
|
|
const double F3o2 = 1.5;
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const int iminF = d_gp.iminF, jminF = d_gp.jminF, kminF = d_gp.kminF;
|
|
const int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
|
if (tid >= d_gp.all) return;
|
|
|
|
const int i0 = tid % nx;
|
|
const int j0 = (tid / nx) % ny;
|
|
const int k0 = tid / (nx * ny);
|
|
|
|
double cxx = 0.0, cxy = 0.0, cxz = 0.0;
|
|
double cyy = 0.0, cyz = 0.0, czz = 0.0;
|
|
|
|
if (!(i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2)) {
|
|
const int iF = i0 + 1;
|
|
const int jF = j0 + 1;
|
|
const int kF = k0 + 1;
|
|
|
|
#if ghost_width != 3
|
|
fd_compute_second6(chi, iF, jF, kF,
|
|
iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
|
|
1, 1, 1,
|
|
cxx, cxy, cxz, cyy, cyz, czz);
|
|
#else
|
|
if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
|
|
(jF + 2) <= jmaxF && (jF - 2) >= jminF &&
|
|
(kF + 2) <= kmaxF && (kF - 2) >= kminF)
|
|
{
|
|
const double c = fetch_sym_ord2_direct(chi, iF, jF, kF, 1, 1, 1);
|
|
cxx = d_gp.Fdxdx * (
|
|
-fetch_sym_ord2_direct(chi, iF - 2, jF, kF, 1, 1, 1)
|
|
+16.0 * fetch_sym_ord2_direct(chi, iF - 1, jF, kF, 1, 1, 1)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(chi, iF + 1, jF, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF, kF, 1, 1, 1));
|
|
cyy = d_gp.Fdydy * (
|
|
-fetch_sym_ord2_direct(chi, iF, jF - 2, kF, 1, 1, 1)
|
|
+16.0 * fetch_sym_ord2_direct(chi, iF, jF - 1, kF, 1, 1, 1)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(chi, iF, jF + 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF + 2, kF, 1, 1, 1));
|
|
czz = d_gp.Fdzdz * (
|
|
-fetch_sym_ord2_direct(chi, iF, jF, kF - 2, 1, 1, 1)
|
|
+16.0 * fetch_sym_ord2_direct(chi, iF, jF, kF - 1, 1, 1, 1)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(chi, iF, jF, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF, kF + 2, 1, 1, 1));
|
|
|
|
const double t_jm2 =
|
|
fetch_sym_ord2_direct(chi, iF - 2, jF - 2, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF - 1, jF - 2, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF + 1, jF - 2, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF - 2, kF, 1, 1, 1);
|
|
const double t_jm1 =
|
|
fetch_sym_ord2_direct(chi, iF - 2, jF - 1, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF - 1, jF - 1, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF + 1, jF - 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF - 1, kF, 1, 1, 1);
|
|
const double t_jp1 =
|
|
fetch_sym_ord2_direct(chi, iF - 2, jF + 1, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF - 1, jF + 1, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF + 1, jF + 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF + 1, kF, 1, 1, 1);
|
|
const double t_jp2 =
|
|
fetch_sym_ord2_direct(chi, iF - 2, jF + 2, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF - 1, jF + 2, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF + 1, jF + 2, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF + 2, kF, 1, 1, 1);
|
|
cxy = d_gp.Fdxdy * (t_jm2 - 8.0 * t_jm1 + 8.0 * t_jp1 - t_jp2);
|
|
|
|
const double t_km2_x =
|
|
fetch_sym_ord2_direct(chi, iF - 2, jF, kF - 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF - 1, jF, kF - 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF + 1, jF, kF - 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF, kF - 2, 1, 1, 1);
|
|
const double t_km1_x =
|
|
fetch_sym_ord2_direct(chi, iF - 2, jF, kF - 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF - 1, jF, kF - 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF + 1, jF, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF, kF - 1, 1, 1, 1);
|
|
const double t_kp1_x =
|
|
fetch_sym_ord2_direct(chi, iF - 2, jF, kF + 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF - 1, jF, kF + 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF + 1, jF, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF, kF + 1, 1, 1, 1);
|
|
const double t_kp2_x =
|
|
fetch_sym_ord2_direct(chi, iF - 2, jF, kF + 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF - 1, jF, kF + 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF + 1, jF, kF + 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 2, jF, kF + 2, 1, 1, 1);
|
|
cxz = d_gp.Fdxdz * (t_km2_x - 8.0 * t_km1_x + 8.0 * t_kp1_x - t_kp2_x);
|
|
|
|
const double t_km2_y =
|
|
fetch_sym_ord2_direct(chi, iF, jF - 2, kF - 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF, jF - 1, kF - 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF, jF + 1, kF - 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF + 2, kF - 2, 1, 1, 1);
|
|
const double t_km1_y =
|
|
fetch_sym_ord2_direct(chi, iF, jF - 2, kF - 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF, jF - 1, kF - 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF, jF + 1, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF + 2, kF - 1, 1, 1, 1);
|
|
const double t_kp1_y =
|
|
fetch_sym_ord2_direct(chi, iF, jF - 2, kF + 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF, jF - 1, kF + 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF, jF + 1, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF + 2, kF + 1, 1, 1, 1);
|
|
const double t_kp2_y =
|
|
fetch_sym_ord2_direct(chi, iF, jF - 2, kF + 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(chi, iF, jF - 1, kF + 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(chi, iF, jF + 1, kF + 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF + 2, kF + 2, 1, 1, 1);
|
|
cyz = d_gp.Fdydz * (t_km2_y - 8.0 * t_km1_y + 8.0 * t_kp1_y - t_kp2_y);
|
|
}
|
|
else if ((iF + 1) <= imaxF && (iF - 1) >= iminF &&
|
|
(jF + 1) <= jmaxF && (jF - 1) >= jminF &&
|
|
(kF + 1) <= kmaxF && (kF - 1) >= kminF)
|
|
{
|
|
const double c = fetch_sym_ord2_direct(chi, iF, jF, kF, 1, 1, 1);
|
|
cxx = d_gp.Sdxdx * (
|
|
fetch_sym_ord2_direct(chi, iF - 1, jF, kF, 1, 1, 1)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(chi, iF + 1, jF, kF, 1, 1, 1));
|
|
cyy = d_gp.Sdydy * (
|
|
fetch_sym_ord2_direct(chi, iF, jF - 1, kF, 1, 1, 1)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(chi, iF, jF + 1, kF, 1, 1, 1));
|
|
czz = d_gp.Sdzdz * (
|
|
fetch_sym_ord2_direct(chi, iF, jF, kF - 1, 1, 1, 1)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(chi, iF, jF, kF + 1, 1, 1, 1));
|
|
cxy = d_gp.Sdxdy * (
|
|
fetch_sym_ord2_direct(chi, iF - 1, jF - 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 1, jF - 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF - 1, jF + 1, kF, 1, 1, 1)
|
|
+ fetch_sym_ord2_direct(chi, iF + 1, jF + 1, kF, 1, 1, 1));
|
|
cxz = d_gp.Sdxdz * (
|
|
fetch_sym_ord2_direct(chi, iF - 1, jF, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF + 1, jF, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF - 1, jF, kF + 1, 1, 1, 1)
|
|
+ fetch_sym_ord2_direct(chi, iF + 1, jF, kF + 1, 1, 1, 1));
|
|
cyz = d_gp.Sdydz * (
|
|
fetch_sym_ord2_direct(chi, iF, jF - 1, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF + 1, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(chi, iF, jF - 1, kF + 1, 1, 1, 1)
|
|
+ fetch_sym_ord2_direct(chi, iF, jF + 1, kF + 1, 1, 1, 1));
|
|
}
|
|
#endif
|
|
}
|
|
|
|
const double cx = chix[tid];
|
|
const double cy = chiy[tid];
|
|
const double cz = chiz[tid];
|
|
const double c1 = chin1[tid];
|
|
|
|
cxx -= Gxxx[tid] * cx + Gyxx[tid] * cy + Gzxx[tid] * cz;
|
|
cxy -= Gxxy[tid] * cx + Gyxy[tid] * cy + Gzxy[tid] * cz;
|
|
cxz -= Gxxz[tid] * cx + Gyxz[tid] * cy + Gzxz[tid] * cz;
|
|
cyy -= Gxyy[tid] * cx + Gyyy[tid] * cy + Gzyy[tid] * cz;
|
|
cyz -= Gxyz[tid] * cx + Gyyz[tid] * cy + Gzyz[tid] * cz;
|
|
czz -= Gxzz[tid] * cx + Gyzz[tid] * cy + Gzzz[tid] * cz;
|
|
|
|
const double uxx = gupxx[tid], uxy = gupxy[tid], uxz = gupxz[tid];
|
|
const double uyy = gupyy[tid], uyz = gupyz[tid], uzz = gupzz[tid];
|
|
const double f_val = uxx * (cxx - F3o2 / c1 * cx * cx)
|
|
+ uyy * (cyy - F3o2 / c1 * cy * cy)
|
|
+ uzz * (czz - F3o2 / c1 * cz * cz)
|
|
+ TWO * uxy * (cxy - F3o2 / c1 * cx * cy)
|
|
+ TWO * uxz * (cxz - F3o2 / c1 * cx * cz)
|
|
+ TWO * uyz * (cyz - F3o2 / c1 * cy * cz);
|
|
|
|
const double inv2c = 1.0 / (c1 * TWO);
|
|
Rxx[tid] += (cxx - cx * cx * inv2c + gxx[tid] * f_val) * inv2c;
|
|
Ryy[tid] += (cyy - cy * cy * inv2c + gyy[tid] * f_val) * inv2c;
|
|
Rzz[tid] += (czz - cz * cz * inv2c + gzz[tid] * f_val) * inv2c;
|
|
Rxy[tid] += (cxy - cx * cy * inv2c + gxy[tid] * f_val) * inv2c;
|
|
Rxz[tid] += (cxz - cx * cz * inv2c + gxz[tid] * f_val) * inv2c;
|
|
Ryz[tid] += (cyz - cy * cz * inv2c + gyz[tid] * f_val) * inv2c;
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase13_chi_correction(
|
|
const double* __restrict__ chin1,
|
|
const double* __restrict__ chix, const double* __restrict__ chiy,
|
|
const double* __restrict__ chiz,
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
double* __restrict__ fxx, double* __restrict__ fxy,
|
|
double* __restrict__ fxz, double* __restrict__ fyy,
|
|
double* __restrict__ fyz, double* __restrict__ fzz,
|
|
double* __restrict__ Rxx, double* __restrict__ Rxy,
|
|
double* __restrict__ Rxz, double* __restrict__ Ryy,
|
|
double* __restrict__ Ryz, double* __restrict__ Rzz)
|
|
{
|
|
const double H=0.5, TWO=2.0, F3o2=1.5;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
double cx=chix[i],cy=chiy[i],cz=chiz[i],c1=chin1[i];
|
|
/* subtract Christoffel * chi_deriv */
|
|
fxx[i] -= Gxxx[i]*cx+Gyxx[i]*cy+Gzxx[i]*cz;
|
|
fxy[i] -= Gxxy[i]*cx+Gyxy[i]*cy+Gzxy[i]*cz;
|
|
fxz[i] -= Gxxz[i]*cx+Gyxz[i]*cy+Gzxz[i]*cz;
|
|
fyy[i] -= Gxyy[i]*cx+Gyyy[i]*cy+Gzyy[i]*cz;
|
|
fyz[i] -= Gxyz[i]*cx+Gyyz[i]*cy+Gzyz[i]*cz;
|
|
fzz[i] -= Gxzz[i]*cx+Gyzz[i]*cy+Gzzz[i]*cz;
|
|
|
|
double uxx=gupxx[i],uxy=gupxy[i],uxz=gupxz[i];
|
|
double uyy=gupyy[i],uyz=gupyz[i],uzz=gupzz[i];
|
|
double f_val = uxx*(fxx[i]-F3o2/c1*cx*cx)
|
|
+ uyy*(fyy[i]-F3o2/c1*cy*cy)
|
|
+ uzz*(fzz[i]-F3o2/c1*cz*cz)
|
|
+ TWO*uxy*(fxy[i]-F3o2/c1*cx*cy)
|
|
+ TWO*uxz*(fxz[i]-F3o2/c1*cx*cz)
|
|
+ TWO*uyz*(fyz[i]-F3o2/c1*cy*cz);
|
|
|
|
double inv2c = 1.0/(c1*TWO);
|
|
Rxx[i] += (fxx[i]-cx*cx*inv2c+gxx[i]*f_val)*inv2c;
|
|
Ryy[i] += (fyy[i]-cy*cy*inv2c+gyy[i]*f_val)*inv2c;
|
|
Rzz[i] += (fzz[i]-cz*cz*inv2c+gzz[i]*f_val)*inv2c;
|
|
Rxy[i] += (fxy[i]-cx*cy*inv2c+gxy[i]*f_val)*inv2c;
|
|
Rxz[i] += (fxz[i]-cx*cz*inv2c+gxz[i]*f_val)*inv2c;
|
|
Ryz[i] += (fyz[i]-cy*cz*inv2c+gyz[i]*f_val)*inv2c;
|
|
}
|
|
}
|
|
|
|
/* Phase 15: trK_rhs, Aij_rhs, gauge.
|
|
* Also updates Christoffel with physical chi correction and computes Lap second derivatives on the fly.
|
|
*/
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase15_trK_Aij_gauge(
|
|
const double* __restrict__ alpn1, const double* __restrict__ chin1,
|
|
const double* __restrict__ chix, const double* __restrict__ chiy,
|
|
const double* __restrict__ chiz,
|
|
const double* __restrict__ gxx, const double* __restrict__ gxy,
|
|
const double* __restrict__ gxz, const double* __restrict__ gyy,
|
|
const double* __restrict__ gyz, const double* __restrict__ gzz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ trK,
|
|
const double* __restrict__ Axx, const double* __restrict__ Axy,
|
|
const double* __restrict__ Axz, const double* __restrict__ Ayy,
|
|
const double* __restrict__ Ayz, const double* __restrict__ Azz,
|
|
const double* __restrict__ Lapx, const double* __restrict__ Lapy,
|
|
const double* __restrict__ Lapz,
|
|
const double* __restrict__ betaxx, const double* __restrict__ betaxy,
|
|
const double* __restrict__ betaxz, const double* __restrict__ betayx,
|
|
const double* __restrict__ betayy, const double* __restrict__ betayz,
|
|
const double* __restrict__ betazx, const double* __restrict__ betazy,
|
|
const double* __restrict__ betazz,
|
|
const double* __restrict__ rho,
|
|
const double* __restrict__ Sx_m, const double* __restrict__ Sy_m,
|
|
const double* __restrict__ Sz_m,
|
|
const double* __restrict__ Sxx_m, const double* __restrict__ Sxy_m,
|
|
const double* __restrict__ Sxz_m, const double* __restrict__ Syy_m,
|
|
const double* __restrict__ Syz_m, const double* __restrict__ Szz_m,
|
|
const double* __restrict__ dtSfx, const double* __restrict__ dtSfy,
|
|
const double* __restrict__ dtSfz,
|
|
const double* __restrict__ Rxx, const double* __restrict__ Rxy,
|
|
const double* __restrict__ Rxz, const double* __restrict__ Ryy,
|
|
const double* __restrict__ Ryz, const double* __restrict__ Rzz,
|
|
double* __restrict__ Gxxx, double* __restrict__ Gxxy,
|
|
double* __restrict__ Gxxz, double* __restrict__ Gxyy,
|
|
double* __restrict__ Gxyz_o, double* __restrict__ Gxzz,
|
|
double* __restrict__ Gyxx, double* __restrict__ Gyxy,
|
|
double* __restrict__ Gyxz, double* __restrict__ Gyyy,
|
|
double* __restrict__ Gyyz, double* __restrict__ Gyzz,
|
|
double* __restrict__ Gzxx, double* __restrict__ Gzxy,
|
|
double* __restrict__ Gzxz, double* __restrict__ Gzyy,
|
|
double* __restrict__ Gzyz, double* __restrict__ Gzzz,
|
|
double* __restrict__ dtSfx_rhs, double* __restrict__ dtSfy_rhs,
|
|
double* __restrict__ dtSfz_rhs,
|
|
double* __restrict__ trK_rhs,
|
|
double* __restrict__ Axx_rhs, double* __restrict__ Axy_rhs,
|
|
double* __restrict__ Axz_rhs, double* __restrict__ Ayy_rhs,
|
|
double* __restrict__ Ayz_rhs, double* __restrict__ Azz_rhs,
|
|
double* __restrict__ Lap_rhs,
|
|
double* __restrict__ betax_rhs, double* __restrict__ betay_rhs,
|
|
double* __restrict__ betaz_rhs,
|
|
double* __restrict__ Gamx_rhs, double* __restrict__ Gamy_rhs,
|
|
double* __restrict__ Gamz_rhs,
|
|
double* __restrict__ f_arr, double* __restrict__ S_arr)
|
|
{
|
|
const double TWO=2.0, FOUR=4.0, EIGHT=8.0, H=0.5;
|
|
const double F1o3=1.0/3.0, F2o3=2.0/3.0, F3o2=1.5;
|
|
const double PI_V=3.14159265358979323846;
|
|
const double F16=16.0, F8=8.0;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all; i += blockDim.x*gridDim.x) {
|
|
const int nx = d_gp.ex[0], ny = d_gp.ex[1], nz = d_gp.ex[2];
|
|
const int i0 = i % nx;
|
|
const int j0 = (i / nx) % ny;
|
|
const int k0 = i / (nx * ny);
|
|
const int iF = i0 + 1;
|
|
const int jF = j0 + 1;
|
|
const int kF = k0 + 1;
|
|
const int imaxF = d_gp.imaxF, jmaxF = d_gp.jmaxF, kmaxF = d_gp.kmaxF;
|
|
const int iminF = d_gp.iminF, jminF = d_gp.jminF, kminF = d_gp.kminF;
|
|
double uxx=gupxx[i],uxy=gupxy[i],uxz=gupxz[i];
|
|
double uyy=gupyy[i],uyz=gupyz[i],uzz=gupzz[i];
|
|
double a=alpn1[i], c1=chin1[i];
|
|
double cx=chix[i],cy=chiy[i],cz=chiz[i];
|
|
double lx=Lapx[i],ly=Lapy[i],lz=Lapz[i];
|
|
double fxx_v = 0.0, fxy_v = 0.0, fxz_v = 0.0;
|
|
double fyy_v = 0.0, fyz_v = 0.0, fzz_v = 0.0;
|
|
|
|
if (!(i0 > nx - 2 || j0 > ny - 2 || k0 > nz - 2)) {
|
|
#if ghost_width != 3
|
|
fd_compute_second6(alpn1, iF, jF, kF,
|
|
iminF, jminF, kminF, imaxF, jmaxF, kmaxF,
|
|
1, 1, 1,
|
|
fxx_v, fxy_v, fxz_v, fyy_v, fyz_v, fzz_v);
|
|
#else
|
|
if ((iF + 2) <= imaxF && (iF - 2) >= iminF &&
|
|
(jF + 2) <= jmaxF && (jF - 2) >= jminF &&
|
|
(kF + 2) <= kmaxF && (kF - 2) >= kminF)
|
|
{
|
|
const double c = fetch_sym_ord2_direct(alpn1, iF, jF, kF, 1, 1, 1);
|
|
fxx_v = d_gp.Fdxdx * (
|
|
-fetch_sym_ord2_direct(alpn1, iF - 2, jF, kF, 1, 1, 1)
|
|
+16.0 * fetch_sym_ord2_direct(alpn1, iF - 1, jF, kF, 1, 1, 1)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(alpn1, iF + 1, jF, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 2, jF, kF, 1, 1, 1));
|
|
fyy_v = d_gp.Fdydy * (
|
|
-fetch_sym_ord2_direct(alpn1, iF, jF - 2, kF, 1, 1, 1)
|
|
+16.0 * fetch_sym_ord2_direct(alpn1, iF, jF - 1, kF, 1, 1, 1)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(alpn1, iF, jF + 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF, jF + 2, kF, 1, 1, 1));
|
|
fzz_v = d_gp.Fdzdz * (
|
|
-fetch_sym_ord2_direct(alpn1, iF, jF, kF - 2, 1, 1, 1)
|
|
+16.0 * fetch_sym_ord2_direct(alpn1, iF, jF, kF - 1, 1, 1, 1)
|
|
-30.0 * c
|
|
+16.0 * fetch_sym_ord2_direct(alpn1, iF, jF, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF, jF, kF + 2, 1, 1, 1));
|
|
|
|
const double t_jm2 =
|
|
fetch_sym_ord2_direct(alpn1, iF - 2, jF - 2, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF - 1, jF - 2, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF + 1, jF - 2, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 2, jF - 2, kF, 1, 1, 1);
|
|
const double t_jm1 =
|
|
fetch_sym_ord2_direct(alpn1, iF - 2, jF - 1, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF - 1, jF - 1, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF + 1, jF - 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 2, jF - 1, kF, 1, 1, 1);
|
|
const double t_jp1 =
|
|
fetch_sym_ord2_direct(alpn1, iF - 2, jF + 1, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF - 1, jF + 1, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF + 1, jF + 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 2, jF + 1, kF, 1, 1, 1);
|
|
const double t_jp2 =
|
|
fetch_sym_ord2_direct(alpn1, iF - 2, jF + 2, kF, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF - 1, jF + 2, kF, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF + 1, jF + 2, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 2, jF + 2, kF, 1, 1, 1);
|
|
fxy_v = d_gp.Fdxdy * (t_jm2 - 8.0 * t_jm1 + 8.0 * t_jp1 - t_jp2);
|
|
|
|
const double t_km2_x =
|
|
fetch_sym_ord2_direct(alpn1, iF - 2, jF, kF - 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF - 1, jF, kF - 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF + 1, jF, kF - 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 2, jF, kF - 2, 1, 1, 1);
|
|
const double t_km1_x =
|
|
fetch_sym_ord2_direct(alpn1, iF - 2, jF, kF - 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF - 1, jF, kF - 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF + 1, jF, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 2, jF, kF - 1, 1, 1, 1);
|
|
const double t_kp1_x =
|
|
fetch_sym_ord2_direct(alpn1, iF - 2, jF, kF + 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF - 1, jF, kF + 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF + 1, jF, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 2, jF, kF + 1, 1, 1, 1);
|
|
const double t_kp2_x =
|
|
fetch_sym_ord2_direct(alpn1, iF - 2, jF, kF + 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF - 1, jF, kF + 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF + 1, jF, kF + 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 2, jF, kF + 2, 1, 1, 1);
|
|
fxz_v = d_gp.Fdxdz * (t_km2_x - 8.0 * t_km1_x + 8.0 * t_kp1_x - t_kp2_x);
|
|
|
|
const double t_km2_y =
|
|
fetch_sym_ord2_direct(alpn1, iF, jF - 2, kF - 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF, jF - 1, kF - 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF, jF + 1, kF - 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF, jF + 2, kF - 2, 1, 1, 1);
|
|
const double t_km1_y =
|
|
fetch_sym_ord2_direct(alpn1, iF, jF - 2, kF - 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF, jF - 1, kF - 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF, jF + 1, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF, jF + 2, kF - 1, 1, 1, 1);
|
|
const double t_kp1_y =
|
|
fetch_sym_ord2_direct(alpn1, iF, jF - 2, kF + 1, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF, jF - 1, kF + 1, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF, jF + 1, kF + 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF, jF + 2, kF + 1, 1, 1, 1);
|
|
const double t_kp2_y =
|
|
fetch_sym_ord2_direct(alpn1, iF, jF - 2, kF + 2, 1, 1, 1)
|
|
- 8.0 * fetch_sym_ord2_direct(alpn1, iF, jF - 1, kF + 2, 1, 1, 1)
|
|
+ 8.0 * fetch_sym_ord2_direct(alpn1, iF, jF + 1, kF + 2, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF, jF + 2, kF + 2, 1, 1, 1);
|
|
fyz_v = d_gp.Fdydz * (t_km2_y - 8.0 * t_km1_y + 8.0 * t_kp1_y - t_kp2_y);
|
|
}
|
|
else if ((iF + 1) <= imaxF && (iF - 1) >= iminF &&
|
|
(jF + 1) <= jmaxF && (jF - 1) >= jminF &&
|
|
(kF + 1) <= kmaxF && (kF - 1) >= kminF)
|
|
{
|
|
const double c = fetch_sym_ord2_direct(alpn1, iF, jF, kF, 1, 1, 1);
|
|
fxx_v = d_gp.Sdxdx * (
|
|
fetch_sym_ord2_direct(alpn1, iF - 1, jF, kF, 1, 1, 1)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(alpn1, iF + 1, jF, kF, 1, 1, 1));
|
|
fyy_v = d_gp.Sdydy * (
|
|
fetch_sym_ord2_direct(alpn1, iF, jF - 1, kF, 1, 1, 1)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(alpn1, iF, jF + 1, kF, 1, 1, 1));
|
|
fzz_v = d_gp.Sdzdz * (
|
|
fetch_sym_ord2_direct(alpn1, iF, jF, kF - 1, 1, 1, 1)
|
|
- 2.0 * c
|
|
+ fetch_sym_ord2_direct(alpn1, iF, jF, kF + 1, 1, 1, 1));
|
|
fxy_v = d_gp.Sdxdy * (
|
|
fetch_sym_ord2_direct(alpn1, iF - 1, jF - 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 1, jF - 1, kF, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF - 1, jF + 1, kF, 1, 1, 1)
|
|
+ fetch_sym_ord2_direct(alpn1, iF + 1, jF + 1, kF, 1, 1, 1));
|
|
fxz_v = d_gp.Sdxdz * (
|
|
fetch_sym_ord2_direct(alpn1, iF - 1, jF, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF + 1, jF, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF - 1, jF, kF + 1, 1, 1, 1)
|
|
+ fetch_sym_ord2_direct(alpn1, iF + 1, jF, kF + 1, 1, 1, 1));
|
|
fyz_v = d_gp.Sdydz * (
|
|
fetch_sym_ord2_direct(alpn1, iF, jF - 1, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF, jF + 1, kF - 1, 1, 1, 1)
|
|
- fetch_sym_ord2_direct(alpn1, iF, jF - 1, kF + 1, 1, 1, 1)
|
|
+ fetch_sym_ord2_direct(alpn1, iF, jF + 1, kF + 1, 1, 1, 1));
|
|
}
|
|
#endif
|
|
}
|
|
|
|
/* raised chi/chi */
|
|
double gx=(uxx*cx+uxy*cy+uxz*cz)/c1;
|
|
double gy=(uxy*cx+uyy*cy+uyz*cz)/c1;
|
|
double gz=(uxz*cx+uyz*cy+uzz*cz)/c1;
|
|
|
|
/* Christoffel physical correction */
|
|
Gxxx[i]-=((cx+cx)/c1-gxx[i]*gx)*H;
|
|
Gyxx[i]-=(0.0-gxx[i]*gy)*H;
|
|
Gzxx[i]-=(0.0-gxx[i]*gz)*H;
|
|
Gxyy[i]-=(0.0-gyy[i]*gx)*H;
|
|
Gyyy[i]-=((cy+cy)/c1-gyy[i]*gy)*H;
|
|
Gzyy[i]-=(0.0-gyy[i]*gz)*H;
|
|
Gxzz[i]-=(0.0-gzz[i]*gx)*H;
|
|
Gyzz[i]-=(0.0-gzz[i]*gy)*H;
|
|
Gzzz[i]-=((cz+cz)/c1-gzz[i]*gz)*H;
|
|
Gxxy[i]-=(cy/c1-gxy[i]*gx)*H;
|
|
Gyxy[i]-=(cx/c1-gxy[i]*gy)*H;
|
|
Gzxy[i]-=(0.0-gxy[i]*gz)*H;
|
|
Gxxz[i]-=(cz/c1-gxz[i]*gx)*H;
|
|
Gyxz[i]-=(0.0-gxz[i]*gy)*H;
|
|
Gzxz[i]-=(cx/c1-gxz[i]*gz)*H;
|
|
Gxyz_o[i]-=(0.0-gyz[i]*gx)*H;
|
|
Gyyz[i]-=(cz/c1-gyz[i]*gy)*H;
|
|
Gzyz[i]-=(cy/c1-gyz[i]*gz)*H;
|
|
|
|
/* Lap second-derivative correction: subtract Gamma*Lap_deriv */
|
|
fxx_v -= Gxxx[i]*lx+Gyxx[i]*ly+Gzxx[i]*lz;
|
|
fyy_v -= Gxyy[i]*lx+Gyyy[i]*ly+Gzyy[i]*lz;
|
|
fzz_v -= Gxzz[i]*lx+Gyzz[i]*ly+Gzzz[i]*lz;
|
|
fxy_v -= Gxxy[i]*lx+Gyxy[i]*ly+Gzxy[i]*lz;
|
|
fxz_v -= Gxxz[i]*lx+Gyxz[i]*ly+Gzxz[i]*lz;
|
|
fyz_v -= Gxyz_o[i]*lx+Gyyz[i]*ly+Gzyz[i]*lz;
|
|
|
|
/* D^i D_i alpha */
|
|
double DDA = uxx*fxx_v+uyy*fyy_v+uzz*fzz_v
|
|
+TWO*(uxy*fxy_v+uxz*fxz_v+uyz*fyz_v);
|
|
|
|
/* trace of S_ij (physical) */
|
|
double S_v = c1*(uxx*Sxx_m[i]+uyy*Syy_m[i]+uzz*Szz_m[i]
|
|
+TWO*(uxy*Sxy_m[i]+uxz*Sxz_m[i]+uyz*Syz_m[i]));
|
|
|
|
/* A^ij A_ij */
|
|
double AijAij =
|
|
uxx*(uxx*Axx[i]*Axx[i]+uyy*Axy[i]*Axy[i]+uzz*Axz[i]*Axz[i]
|
|
+TWO*(uxy*Axx[i]*Axy[i]+uxz*Axx[i]*Axz[i]+uyz*Axy[i]*Axz[i]))
|
|
+uyy*(uxx*Axy[i]*Axy[i]+uyy*Ayy[i]*Ayy[i]+uzz*Ayz[i]*Ayz[i]
|
|
+TWO*(uxy*Axy[i]*Ayy[i]+uxz*Axy[i]*Ayz[i]+uyz*Ayy[i]*Ayz[i]))
|
|
+uzz*(uxx*Axz[i]*Axz[i]+uyy*Ayz[i]*Ayz[i]+uzz*Azz[i]*Azz[i]
|
|
+TWO*(uxy*Axz[i]*Ayz[i]+uxz*Axz[i]*Azz[i]+uyz*Ayz[i]*Azz[i]))
|
|
+TWO*(
|
|
uxy*(uxx*Axx[i]*Axy[i]+uyy*Axy[i]*Ayy[i]+uzz*Axz[i]*Ayz[i]
|
|
+uxy*(Axx[i]*Ayy[i]+Axy[i]*Axy[i])
|
|
+uxz*(Axx[i]*Ayz[i]+Axz[i]*Axy[i])
|
|
+uyz*(Axy[i]*Ayz[i]+Axz[i]*Ayy[i]))
|
|
+uxz*(uxx*Axx[i]*Axz[i]+uyy*Axy[i]*Ayz[i]+uzz*Axz[i]*Azz[i]
|
|
+uxy*(Axx[i]*Ayz[i]+Axy[i]*Axz[i])
|
|
+uxz*(Axx[i]*Azz[i]+Axz[i]*Axz[i])
|
|
+uyz*(Axy[i]*Azz[i]+Axz[i]*Ayz[i]))
|
|
+uyz*(uxx*Axy[i]*Axz[i]+uyy*Ayy[i]*Ayz[i]+uzz*Ayz[i]*Azz[i]
|
|
+uxy*(Axy[i]*Ayz[i]+Ayy[i]*Axz[i])
|
|
+uxz*(Axy[i]*Azz[i]+Ayz[i]*Axz[i])
|
|
+uyz*(Ayy[i]*Azz[i]+Ayz[i]*Ayz[i])));
|
|
|
|
double trK_v = trK[i];
|
|
double db = betaxx[i] + betayy[i] + betazz[i];
|
|
|
|
/* trK_rhs step 1: store D^iD_i alpha * chin1 */
|
|
trK_rhs[i] = c1 * DDA;
|
|
|
|
/* f_arr = -(1/3) * (DDA + alpha/chi * (2/3*K^2 - AijAij - 16pi*rho + 8pi*S)) */
|
|
double f_v = F2o3*trK_v*trK_v - AijAij - F16*PI_V*rho[i] + EIGHT*PI_V*S_v;
|
|
f_arr[i] = -F1o3*(uxx*fxx_v+uyy*fyy_v+uzz*fzz_v
|
|
+TWO*(uxy*fxy_v+uxz*fxz_v+uyz*fyz_v)
|
|
+(a/c1)*f_v);
|
|
|
|
/* fij = alpha*(Rij - 8pi*Sij) - D_iD_j alpha */
|
|
double fij_xx=a*(Rxx[i]-EIGHT*PI_V*Sxx_m[i])-fxx_v;
|
|
double fij_xy=a*(Rxy[i]-EIGHT*PI_V*Sxy_m[i])-fxy_v;
|
|
double fij_xz=a*(Rxz[i]-EIGHT*PI_V*Sxz_m[i])-fxz_v;
|
|
double fij_yy=a*(Ryy[i]-EIGHT*PI_V*Syy_m[i])-fyy_v;
|
|
double fij_yz=a*(Ryz[i]-EIGHT*PI_V*Syz_m[i])-fyz_v;
|
|
double fij_zz=a*(Rzz[i]-EIGHT*PI_V*Szz_m[i])-fzz_v;
|
|
|
|
/* Aij_rhs = chi*(fij - gij*f) */
|
|
Axx_rhs[i]=fij_xx-gxx[i]*f_arr[i];
|
|
Ayy_rhs[i]=fij_yy-gyy[i]*f_arr[i];
|
|
Azz_rhs[i]=fij_zz-gzz[i]*f_arr[i];
|
|
Axy_rhs[i]=fij_xy-gxy[i]*f_arr[i];
|
|
Axz_rhs[i]=fij_xz-gxz[i]*f_arr[i];
|
|
Ayz_rhs[i]=fij_yz-gyz[i]*f_arr[i];
|
|
|
|
/* A_il A^l_j */
|
|
double AA_xx=uxx*Axx[i]*Axx[i]+uyy*Axy[i]*Axy[i]+uzz*Axz[i]*Axz[i]
|
|
+TWO*(uxy*Axx[i]*Axy[i]+uxz*Axx[i]*Axz[i]+uyz*Axy[i]*Axz[i]);
|
|
double AA_yy=uxx*Axy[i]*Axy[i]+uyy*Ayy[i]*Ayy[i]+uzz*Ayz[i]*Ayz[i]
|
|
+TWO*(uxy*Axy[i]*Ayy[i]+uxz*Axy[i]*Ayz[i]+uyz*Ayy[i]*Ayz[i]);
|
|
double AA_zz=uxx*Axz[i]*Axz[i]+uyy*Ayz[i]*Ayz[i]+uzz*Azz[i]*Azz[i]
|
|
+TWO*(uxy*Axz[i]*Ayz[i]+uxz*Axz[i]*Azz[i]+uyz*Ayz[i]*Azz[i]);
|
|
double AA_xy=uxx*Axx[i]*Axy[i]+uyy*Axy[i]*Ayy[i]+uzz*Axz[i]*Ayz[i]
|
|
+uxy*(Axx[i]*Ayy[i]+Axy[i]*Axy[i])
|
|
+uxz*(Axx[i]*Ayz[i]+Axz[i]*Axy[i])
|
|
+uyz*(Axy[i]*Ayz[i]+Axz[i]*Ayy[i]);
|
|
double AA_xz=uxx*Axx[i]*Axz[i]+uyy*Axy[i]*Ayz[i]+uzz*Axz[i]*Azz[i]
|
|
+uxy*(Axx[i]*Ayz[i]+Axy[i]*Axz[i])
|
|
+uxz*(Axx[i]*Azz[i]+Axz[i]*Axz[i])
|
|
+uyz*(Axy[i]*Azz[i]+Axz[i]*Ayz[i]);
|
|
double AA_yz=uxx*Axy[i]*Axz[i]+uyy*Ayy[i]*Ayz[i]+uzz*Ayz[i]*Azz[i]
|
|
+uxy*(Axy[i]*Ayz[i]+Ayy[i]*Axz[i])
|
|
+uxz*(Axy[i]*Azz[i]+Ayz[i]*Axz[i])
|
|
+uyz*(Ayy[i]*Azz[i]+Ayz[i]*Ayz[i]);
|
|
|
|
/* trK_rhs final */
|
|
trK_rhs[i] = -trK_rhs[i]
|
|
+ a*(F1o3*trK_v*trK_v
|
|
+uxx*AA_xx+uyy*AA_yy+uzz*AA_zz
|
|
+TWO*(uxy*AA_xy+uxz*AA_xz+uyz*AA_yz)
|
|
+FOUR*PI_V*(rho[i]+S_v));
|
|
|
|
/* Aij_rhs final */
|
|
Axx_rhs[i]=c1*Axx_rhs[i]+a*(trK_v*Axx[i]-TWO*AA_xx)
|
|
+TWO*(Axx[i]*betaxx[i]+Axy[i]*betayx[i]+Axz[i]*betazx[i])-F2o3*Axx[i]*db;
|
|
Ayy_rhs[i]=c1*Ayy_rhs[i]+a*(trK_v*Ayy[i]-TWO*AA_yy)
|
|
+TWO*(Axy[i]*betaxy[i]+Ayy[i]*betayy[i]+Ayz[i]*betazy[i])-F2o3*Ayy[i]*db;
|
|
Azz_rhs[i]=c1*Azz_rhs[i]+a*(trK_v*Azz[i]-TWO*AA_zz)
|
|
+TWO*(Axz[i]*betaxz[i]+Ayz[i]*betayz[i]+Azz[i]*betazz[i])-F2o3*Azz[i]*db;
|
|
Axy_rhs[i]=c1*Axy_rhs[i]+a*(trK_v*Axy[i]-TWO*AA_xy)
|
|
+Axx[i]*betaxy[i]+Axz[i]*betazy[i]+Ayy[i]*betayx[i]
|
|
+Ayz[i]*betazx[i]+F1o3*Axy[i]*db-Axy[i]*betazz[i];
|
|
Ayz_rhs[i]=c1*Ayz_rhs[i]+a*(trK_v*Ayz[i]-TWO*AA_yz)
|
|
+Axy[i]*betaxz[i]+Ayy[i]*betayz[i]+Axz[i]*betaxy[i]
|
|
+Azz[i]*betazy[i]+F1o3*Ayz[i]*db-Ayz[i]*betaxx[i];
|
|
Axz_rhs[i]=c1*Axz_rhs[i]+a*(trK_v*Axz[i]-TWO*AA_xz)
|
|
+Axx[i]*betaxz[i]+Axy[i]*betayz[i]+Ayz[i]*betayx[i]
|
|
+Azz[i]*betazx[i]+F1o3*Axz[i]*db-Axz[i]*betayy[i];
|
|
|
|
/* gauge */
|
|
Lap_rhs[i] = -TWO*a*trK_v;
|
|
betax_rhs[i] = 0.75*dtSfx[i];
|
|
betay_rhs[i] = 0.75*dtSfy[i];
|
|
betaz_rhs[i] = 0.75*dtSfz[i];
|
|
#if (GAUGE == 0)
|
|
dtSfx_rhs[i] = Gamx_rhs[i] - 2.0*dtSfx[i];
|
|
dtSfy_rhs[i] = Gamy_rhs[i] - 2.0*dtSfy[i];
|
|
dtSfz_rhs[i] = Gamz_rhs[i] - 2.0*dtSfz[i];
|
|
#endif
|
|
}
|
|
}
|
|
|
|
/* Phase 18: Hamilton & momentum constraints (co==0 only) */
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_phase18_constraints(
|
|
const double* __restrict__ chin1,
|
|
const double* __restrict__ chix, const double* __restrict__ chiy,
|
|
const double* __restrict__ chiz,
|
|
const double* __restrict__ gupxx, const double* __restrict__ gupxy,
|
|
const double* __restrict__ gupxz, const double* __restrict__ gupyy,
|
|
const double* __restrict__ gupyz, const double* __restrict__ gupzz,
|
|
const double* __restrict__ trK,
|
|
const double* __restrict__ Axx, const double* __restrict__ Axy,
|
|
const double* __restrict__ Axz, const double* __restrict__ Ayy,
|
|
const double* __restrict__ Ayz, const double* __restrict__ Azz,
|
|
const double* __restrict__ Rxx, const double* __restrict__ Rxy,
|
|
const double* __restrict__ Rxz, const double* __restrict__ Ryy,
|
|
const double* __restrict__ Ryz, const double* __restrict__ Rzz,
|
|
const double* __restrict__ rho,
|
|
const double* __restrict__ Sx_m, const double* __restrict__ Sy_m,
|
|
const double* __restrict__ Sz_m,
|
|
const double* __restrict__ Kx, const double* __restrict__ Ky,
|
|
const double* __restrict__ Kz,
|
|
const double* __restrict__ Gxxx, const double* __restrict__ Gxxy,
|
|
const double* __restrict__ Gxxz, const double* __restrict__ Gxyy,
|
|
const double* __restrict__ Gxyz, const double* __restrict__ Gxzz,
|
|
const double* __restrict__ Gyxx, const double* __restrict__ Gyxy,
|
|
const double* __restrict__ Gyxz, const double* __restrict__ Gyyy,
|
|
const double* __restrict__ Gyyz, const double* __restrict__ Gyzz,
|
|
const double* __restrict__ Gzxx, const double* __restrict__ Gzxy,
|
|
const double* __restrict__ Gzxz, const double* __restrict__ Gzyy,
|
|
const double* __restrict__ Gzyz, const double* __restrict__ Gzzz,
|
|
/* dA/dx arrays (fderivs of Aij) */
|
|
const double* __restrict__ dAxx_x, const double* __restrict__ dAxx_y,
|
|
const double* __restrict__ dAxx_z,
|
|
const double* __restrict__ dAxy_x, const double* __restrict__ dAxy_y,
|
|
const double* __restrict__ dAxy_z,
|
|
const double* __restrict__ dAxz_x, const double* __restrict__ dAxz_y,
|
|
const double* __restrict__ dAxz_z,
|
|
const double* __restrict__ dAyy_x, const double* __restrict__ dAyy_y,
|
|
const double* __restrict__ dAyy_z,
|
|
const double* __restrict__ dAyz_x, const double* __restrict__ dAyz_y,
|
|
const double* __restrict__ dAyz_z,
|
|
const double* __restrict__ dAzz_x, const double* __restrict__ dAzz_y,
|
|
const double* __restrict__ dAzz_z,
|
|
double* __restrict__ ham_Res,
|
|
double* __restrict__ movx_Res, double* __restrict__ movy_Res,
|
|
double* __restrict__ movz_Res)
|
|
{
|
|
const double TWO=2.0, F2o3=2.0/3.0, F8=8.0, F16=16.0;
|
|
const double PI_V=3.14159265358979323846;
|
|
for (int i = blockIdx.x*blockDim.x+threadIdx.x; i < d_gp.all;
|
|
i += blockDim.x*gridDim.x)
|
|
{
|
|
double uxx=gupxx[i],uxy=gupxy[i],uxz=gupxz[i];
|
|
double uyy=gupyy[i],uyz=gupyz[i],uzz=gupzz[i];
|
|
double c1=chin1[i];
|
|
|
|
/* Hamiltonian constraint */
|
|
double R_sc = uxx*Rxx[i]+uyy*Ryy[i]+uzz*Rzz[i]
|
|
+TWO*(uxy*Rxy[i]+uxz*Rxz[i]+uyz*Ryz[i]);
|
|
/* AijAij (same as in phase15) */
|
|
double AijAij =
|
|
uxx*(uxx*Axx[i]*Axx[i]+uyy*Axy[i]*Axy[i]+uzz*Axz[i]*Axz[i]
|
|
+TWO*(uxy*Axx[i]*Axy[i]+uxz*Axx[i]*Axz[i]+uyz*Axy[i]*Axz[i]))
|
|
+uyy*(uxx*Axy[i]*Axy[i]+uyy*Ayy[i]*Ayy[i]+uzz*Ayz[i]*Ayz[i]
|
|
+TWO*(uxy*Axy[i]*Ayy[i]+uxz*Axy[i]*Ayz[i]+uyz*Ayy[i]*Ayz[i]))
|
|
+uzz*(uxx*Axz[i]*Axz[i]+uyy*Ayz[i]*Ayz[i]+uzz*Azz[i]*Azz[i]
|
|
+TWO*(uxy*Axz[i]*Ayz[i]+uxz*Axz[i]*Azz[i]+uyz*Ayz[i]*Azz[i]))
|
|
+TWO*(uxy*(uxx*Axx[i]*Axy[i]+uyy*Axy[i]*Ayy[i]+uzz*Axz[i]*Ayz[i]
|
|
+uxy*(Axx[i]*Ayy[i]+Axy[i]*Axy[i])
|
|
+uxz*(Axx[i]*Ayz[i]+Axz[i]*Axy[i])
|
|
+uyz*(Axy[i]*Ayz[i]+Axz[i]*Ayy[i]))
|
|
+uxz*(uxx*Axx[i]*Axz[i]+uyy*Axy[i]*Ayz[i]+uzz*Axz[i]*Azz[i]
|
|
+uxy*(Axx[i]*Ayz[i]+Axy[i]*Axz[i])
|
|
+uxz*(Axx[i]*Azz[i]+Axz[i]*Axz[i])
|
|
+uyz*(Axy[i]*Azz[i]+Axz[i]*Ayz[i]))
|
|
+uyz*(uxx*Axy[i]*Axz[i]+uyy*Ayy[i]*Ayz[i]+uzz*Ayz[i]*Azz[i]
|
|
+uxy*(Axy[i]*Ayz[i]+Ayy[i]*Axz[i])
|
|
+uxz*(Axy[i]*Azz[i]+Ayz[i]*Axz[i])
|
|
+uyz*(Ayy[i]*Azz[i]+Ayz[i]*Ayz[i])));
|
|
|
|
ham_Res[i] = c1*R_sc + F2o3*trK[i]*trK[i] - AijAij - F16*PI_V*rho[i];
|
|
|
|
/* Momentum constraints: need covariant derivative of A */
|
|
double cx=chix[i],cy=chiy[i],cz=chiz[i];
|
|
/* D_j A^j_x etc — subtract Christoffel and chi terms */
|
|
/* gxxx = dAxx_x - 2*Gxxx*Axx - ... - chix*Axx/chin1 etc */
|
|
double mx_xx = dAxx_x[i]-(Gxxx[i]*Axx[i]+Gyxx[i]*Axy[i]+Gzxx[i]*Axz[i]
|
|
+Gxxx[i]*Axx[i]+Gyxx[i]*Axy[i]+Gzxx[i]*Axz[i])-cx*Axx[i]/c1;
|
|
double mx_xy = dAxy_x[i]-(Gxxy[i]*Axx[i]+Gyxy[i]*Axy[i]+Gzxy[i]*Axz[i]
|
|
+Gxxx[i]*Axy[i]+Gyxx[i]*Ayy[i]+Gzxx[i]*Ayz[i])-cx*Axy[i]/c1;
|
|
double mx_xz = dAxz_x[i]-(Gxxz[i]*Axx[i]+Gyxz[i]*Axy[i]+Gzxz[i]*Axz[i]
|
|
+Gxxx[i]*Axz[i]+Gyxx[i]*Ayz[i]+Gzxx[i]*Azz[i])-cx*Axz[i]/c1;
|
|
double mx_yy = dAyy_x[i]-(Gxxy[i]*Axy[i]+Gyxy[i]*Ayy[i]+Gzxy[i]*Ayz[i]
|
|
+Gxxy[i]*Axy[i]+Gyxy[i]*Ayy[i]+Gzxy[i]*Ayz[i])-cx*Ayy[i]/c1;
|
|
double mx_yz = dAyz_x[i]-(Gxxz[i]*Axy[i]+Gyxz[i]*Ayy[i]+Gzxz[i]*Ayz[i]
|
|
+Gxxy[i]*Axz[i]+Gyxy[i]*Ayz[i]+Gzxy[i]*Azz[i])-cx*Ayz[i]/c1;
|
|
double mx_zz = dAzz_x[i]-(Gxxz[i]*Axz[i]+Gyxz[i]*Ayz[i]+Gzxz[i]*Azz[i]
|
|
+Gxxz[i]*Axz[i]+Gyxz[i]*Ayz[i]+Gzxz[i]*Azz[i])-cx*Azz[i]/c1;
|
|
|
|
double my_xx = dAxx_y[i]-(Gxxy[i]*Axx[i]+Gyxy[i]*Axy[i]+Gzxy[i]*Axz[i]
|
|
+Gxxy[i]*Axx[i]+Gyxy[i]*Axy[i]+Gzxy[i]*Axz[i])-cy*Axx[i]/c1;
|
|
double my_xy = dAxy_y[i]-(Gxyy[i]*Axx[i]+Gyyy[i]*Axy[i]+Gzyy[i]*Axz[i]
|
|
+Gxxy[i]*Axy[i]+Gyxy[i]*Ayy[i]+Gzxy[i]*Ayz[i])-cy*Axy[i]/c1;
|
|
double my_xz = dAxz_y[i]-(Gxyz[i]*Axx[i]+Gyyz[i]*Axy[i]+Gzyz[i]*Axz[i]
|
|
+Gxxy[i]*Axz[i]+Gyxy[i]*Ayz[i]+Gzxy[i]*Azz[i])-cy*Axz[i]/c1;
|
|
double my_yy = dAyy_y[i]-(Gxyy[i]*Axy[i]+Gyyy[i]*Ayy[i]+Gzyy[i]*Ayz[i]
|
|
+Gxyy[i]*Axy[i]+Gyyy[i]*Ayy[i]+Gzyy[i]*Ayz[i])-cy*Ayy[i]/c1;
|
|
double my_yz = dAyz_y[i]-(Gxyz[i]*Axy[i]+Gyyz[i]*Ayy[i]+Gzyz[i]*Ayz[i]
|
|
+Gxyy[i]*Axz[i]+Gyyy[i]*Ayz[i]+Gzyy[i]*Azz[i])-cy*Ayz[i]/c1;
|
|
double my_zz = dAzz_y[i]-(Gxyz[i]*Axz[i]+Gyyz[i]*Ayz[i]+Gzyz[i]*Azz[i]
|
|
+Gxyz[i]*Axz[i]+Gyyz[i]*Ayz[i]+Gzyz[i]*Azz[i])-cy*Azz[i]/c1;
|
|
|
|
double mz_xx = dAxx_z[i]-(Gxxz[i]*Axx[i]+Gyxz[i]*Axy[i]+Gzxz[i]*Axz[i]
|
|
+Gxxz[i]*Axx[i]+Gyxz[i]*Axy[i]+Gzxz[i]*Axz[i])-cz*Axx[i]/c1;
|
|
double mz_xy = dAxy_z[i]-(Gxyz[i]*Axx[i]+Gyyz[i]*Axy[i]+Gzyz[i]*Axz[i]
|
|
+Gxxz[i]*Axy[i]+Gyxz[i]*Ayy[i]+Gzxz[i]*Ayz[i])-cz*Axy[i]/c1;
|
|
double mz_xz = dAxz_z[i]-(Gxzz[i]*Axx[i]+Gyzz[i]*Axy[i]+Gzzz[i]*Axz[i]
|
|
+Gxxz[i]*Axz[i]+Gyxz[i]*Ayz[i]+Gzxz[i]*Azz[i])-cz*Axz[i]/c1;
|
|
double mz_yy = dAyy_z[i]-(Gxyz[i]*Axy[i]+Gyyz[i]*Ayy[i]+Gzyz[i]*Ayz[i]
|
|
+Gxyz[i]*Axy[i]+Gyyz[i]*Ayy[i]+Gzyz[i]*Ayz[i])-cz*Ayy[i]/c1;
|
|
double mz_yz = dAyz_z[i]-(Gxzz[i]*Axy[i]+Gyzz[i]*Ayy[i]+Gzzz[i]*Ayz[i]
|
|
+Gxyz[i]*Axz[i]+Gyyz[i]*Ayz[i]+Gzyz[i]*Azz[i])-cz*Ayz[i]/c1;
|
|
double mz_zz = dAzz_z[i]-(Gxzz[i]*Axz[i]+Gyzz[i]*Ayz[i]+Gzzz[i]*Azz[i]
|
|
+Gxzz[i]*Axz[i]+Gyzz[i]*Ayz[i]+Gzzz[i]*Azz[i])-cz*Azz[i]/c1;
|
|
|
|
movx_Res[i] = uxx*mx_xx+uyy*my_xy+uzz*mz_xz
|
|
+uxy*mx_xy+uxz*mx_xz+uyz*my_xz
|
|
+uxy*my_xx+uxz*mz_xx+uyz*mz_xy
|
|
- F2o3*Kx[i] - F8*PI_V*Sx_m[i];
|
|
movy_Res[i] = uxx*mx_xy+uyy*my_yy+uzz*mz_yz
|
|
+uxy*mx_yy+uxz*mx_yz+uyz*my_yz
|
|
+uxy*my_xy+uxz*mz_xy+uyz*mz_yy
|
|
- F2o3*Ky[i] - F8*PI_V*Sy_m[i];
|
|
movz_Res[i] = uxx*mx_xz+uyy*my_yz+uzz*mz_zz
|
|
+uxy*mx_yz+uxz*mx_zz+uyz*my_zz
|
|
+uxy*my_xz+uxz*mz_xz+uyz*mz_yz
|
|
- F2o3*Kz[i] - F8*PI_V*Sz_m[i];
|
|
}
|
|
}
|
|
|
|
static void setup_grid_params(int *ex,
|
|
double *X, double *Y, double *Z,
|
|
int Symmetry, double eps, int co)
|
|
{
|
|
const int nx = ex[0];
|
|
const int ny = ex[1];
|
|
const int nz = ex[2];
|
|
const double dX = X[1] - X[0];
|
|
const double dY = Y[1] - Y[0];
|
|
const double dZ = Z[1] - Z[0];
|
|
const int NO_SYMM = 0;
|
|
const int EQ_SYMM = 1;
|
|
|
|
ensure_gpu_buffers(nx, ny, nz);
|
|
|
|
GridParams gp = {};
|
|
gp.ex[0] = nx;
|
|
gp.ex[1] = ny;
|
|
gp.ex[2] = nz;
|
|
gp.all = nx * ny * nz;
|
|
gp.dX = dX;
|
|
gp.dY = dY;
|
|
gp.dZ = dZ;
|
|
gp.d12dx = 1.0 / (12.0 * dX);
|
|
gp.d12dy = 1.0 / (12.0 * dY);
|
|
gp.d12dz = 1.0 / (12.0 * dZ);
|
|
gp.d2dx = 1.0 / (2.0 * dX);
|
|
gp.d2dy = 1.0 / (2.0 * dY);
|
|
gp.d2dz = 1.0 / (2.0 * dZ);
|
|
gp.Fdxdx = 1.0 / (12.0 * dX * dX);
|
|
gp.Fdydy = 1.0 / (12.0 * dY * dY);
|
|
gp.Fdzdz = 1.0 / (12.0 * dZ * dZ);
|
|
gp.Sdxdx = 1.0 / (dX * dX);
|
|
gp.Sdydy = 1.0 / (dY * dY);
|
|
gp.Sdzdz = 1.0 / (dZ * dZ);
|
|
gp.Fdxdy = 1.0 / (144.0 * dX * dY);
|
|
gp.Fdxdz = 1.0 / (144.0 * dX * dZ);
|
|
gp.Fdydz = 1.0 / (144.0 * dY * dZ);
|
|
gp.Sdxdy = 0.25 / (dX * dY);
|
|
gp.Sdxdz = 0.25 / (dX * dZ);
|
|
gp.Sdydz = 0.25 / (dY * dZ);
|
|
gp.iminF = 1;
|
|
gp.jminF = 1;
|
|
gp.kminF = 1;
|
|
gp.imaxF = nx;
|
|
gp.jmaxF = ny;
|
|
gp.kmaxF = nz;
|
|
if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) gp.kminF = 2 - ghost_width;
|
|
if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) gp.iminF = 2 - ghost_width;
|
|
if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) gp.jminF = 2 - ghost_width;
|
|
gp.iminF3 = 1;
|
|
gp.jminF3 = 1;
|
|
gp.kminF3 = 1;
|
|
if (Symmetry > NO_SYMM && fabs(Z[0]) < dZ) gp.kminF3 = 1 - ghost_width;
|
|
if (Symmetry > EQ_SYMM && fabs(X[0]) < dX) gp.iminF3 = 1 - ghost_width;
|
|
if (Symmetry > EQ_SYMM && fabs(Y[0]) < dY) gp.jminF3 = 1 - ghost_width;
|
|
gp.Symmetry = Symmetry;
|
|
gp.eps = eps;
|
|
gp.co = co;
|
|
gp.fh2_nx = nx + 2;
|
|
gp.fh2_ny = ny + 2;
|
|
gp.fh2_nz = nz + 2;
|
|
gp.fh3_nx = nx + 3;
|
|
gp.fh3_ny = ny + 3;
|
|
gp.fh3_nz = nz + 3;
|
|
upload_grid_params_if_needed(gp);
|
|
}
|
|
|
|
static void compute_patch_boundary_flags(int *ex,
|
|
double *X, double *Y, double *Z,
|
|
const double *bbox,
|
|
int Symmetry,
|
|
int &touch_xmin, int &touch_xmax,
|
|
int &touch_ymin, int &touch_ymax,
|
|
int &touch_zmin, int &touch_zmax)
|
|
{
|
|
const double dX = X[1] - X[0];
|
|
const double dY = Y[1] - Y[0];
|
|
const double dZ = Z[1] - Z[0];
|
|
const int NO_SYMM = 0;
|
|
const int OCTANT = 2;
|
|
|
|
touch_xmax = (std::fabs(X[ex[0] - 1] - bbox[3]) < dX) ? 1 : 0;
|
|
touch_ymax = (std::fabs(Y[ex[1] - 1] - bbox[4]) < dY) ? 1 : 0;
|
|
touch_zmax = (std::fabs(Z[ex[2] - 1] - bbox[5]) < dZ) ? 1 : 0;
|
|
|
|
touch_xmin = (std::fabs(X[0] - bbox[0]) < dX &&
|
|
!(Symmetry == OCTANT && std::fabs(bbox[0]) < dX / 2.0)) ? 1 : 0;
|
|
touch_ymin = (std::fabs(Y[0] - bbox[1]) < dY &&
|
|
!(Symmetry == OCTANT && std::fabs(bbox[1]) < dY / 2.0)) ? 1 : 0;
|
|
touch_zmin = (std::fabs(Z[0] - bbox[2]) < dZ &&
|
|
!(Symmetry > NO_SYMM && std::fabs(bbox[2]) < dZ / 2.0)) ? 1 : 0;
|
|
}
|
|
|
|
static void upload_state_inputs(double **state_host, size_t all)
|
|
{
|
|
const size_t bytes = all * sizeof(double);
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
std::memcpy(g_buf.h_stage + (size_t)i * all, state_host[i], bytes);
|
|
}
|
|
CUDA_CHECK(cudaMemcpy(g_buf.slot[S_chi], g_buf.h_stage,
|
|
(size_t)BSSN_STATE_COUNT * bytes,
|
|
cudaMemcpyHostToDevice));
|
|
}
|
|
|
|
static void upload_matter_cache(StepContext &ctx,
|
|
double **matter_host,
|
|
size_t all)
|
|
{
|
|
const size_t bytes = all * sizeof(double);
|
|
for (int i = 0; i < BSSN_MATTER_COUNT; ++i) {
|
|
std::memcpy(g_buf.h_stage + (size_t)i * all, matter_host[i], bytes);
|
|
}
|
|
CUDA_CHECK(cudaMemcpy(ctx.d_matter_mem, g_buf.h_stage,
|
|
(size_t)BSSN_MATTER_COUNT * bytes,
|
|
cudaMemcpyHostToDevice));
|
|
ctx.matter_ready = true;
|
|
}
|
|
|
|
static void zero_matter_cache(StepContext &ctx, size_t all)
|
|
{
|
|
CUDA_CHECK(cudaMemset(ctx.d_matter_mem, 0,
|
|
(size_t)BSSN_MATTER_COUNT * all * sizeof(double)));
|
|
ctx.matter_ready = true;
|
|
}
|
|
|
|
static void bind_matter_slots(const StepContext &ctx)
|
|
{
|
|
for (int i = 0; i < BSSN_MATTER_COUNT; ++i) {
|
|
g_buf.slot[k_matter_slots[i]] = ctx.d_matter[i];
|
|
}
|
|
}
|
|
|
|
static void bind_state_input_slots(const std::array<double *, BSSN_STATE_COUNT> &state)
|
|
{
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
g_buf.slot[k_state_input_slots[i]] = state[i];
|
|
}
|
|
}
|
|
|
|
static void bind_state_output_slots(const std::array<double *, BSSN_STATE_COUNT> &state)
|
|
{
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
g_buf.slot[k_state_rhs_slots[i]] = state[i];
|
|
}
|
|
}
|
|
|
|
static void launch_rhs_pipeline(int all, double eps, int co)
|
|
{
|
|
const double SYM = 1.0;
|
|
const double ANTI = -1.0;
|
|
|
|
#define D(s) g_buf.slot[s]
|
|
kern_phase1_prep<<<grid(all),BLK>>>(
|
|
D(S_Lap), D(S_chi), D(S_dxx), D(S_dyy), D(S_dzz),
|
|
D(S_alpn1), D(S_chin1), D(S_gxx), D(S_gyy), D(S_gzz));
|
|
|
|
{
|
|
double *src_fields[] = {
|
|
D(S_betax), D(S_betay), D(S_betaz), D(S_chi),
|
|
D(S_dxx), D(S_gxy), D(S_gxz), D(S_dyy),
|
|
D(S_gyz), D(S_dzz), D(S_Lap), D(S_trK)
|
|
};
|
|
double *fx_fields[] = {
|
|
D(S_betaxx), D(S_betayx), D(S_betazx), D(S_chix),
|
|
D(S_gxxx), D(S_gxyx), D(S_gxzx), D(S_gyyx),
|
|
D(S_gyzx), D(S_gzzx), D(S_Lapx), D(S_Kx)
|
|
};
|
|
double *fy_fields[] = {
|
|
D(S_betaxy), D(S_betayy), D(S_betazy), D(S_chiy),
|
|
D(S_gxxy), D(S_gxyy), D(S_gxzy), D(S_gyyy),
|
|
D(S_gyzy), D(S_gzzy), D(S_Lapy), D(S_Ky)
|
|
};
|
|
double *fz_fields[] = {
|
|
D(S_betaxz), D(S_betayz), D(S_betazz), D(S_chiz),
|
|
D(S_gxxz), D(S_gxyz), D(S_gxzz), D(S_gyyz),
|
|
D(S_gyzz), D(S_gzzz), D(S_Lapz), D(S_Kz)
|
|
};
|
|
const int soa_signs[] = {
|
|
(int)ANTI, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)ANTI, (int)SYM,
|
|
(int)SYM, (int)SYM, (int)ANTI,
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)ANTI, (int)ANTI, (int)SYM,
|
|
(int)ANTI, (int)SYM, (int)ANTI,
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)ANTI, (int)ANTI,
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)SYM, (int)SYM
|
|
};
|
|
gpu_fderivs_batch((int)(sizeof(src_fields) / sizeof(src_fields[0])),
|
|
src_fields, fx_fields, fy_fields, fz_fields,
|
|
soa_signs, all);
|
|
}
|
|
|
|
kern_phase2_metric_rhs<<<grid(all),BLK>>>(
|
|
D(S_alpn1), D(S_chin1),
|
|
D(S_gxx), D(S_gxy), D(S_gxz), D(S_gyy), D(S_gyz), D(S_gzz),
|
|
D(S_trK),
|
|
D(S_Axx), D(S_Axy), D(S_Axz), D(S_Ayy), D(S_Ayz), D(S_Azz),
|
|
D(S_betaxx), D(S_betaxy), D(S_betaxz),
|
|
D(S_betayx), D(S_betayy), D(S_betayz),
|
|
D(S_betazx), D(S_betazy), D(S_betazz),
|
|
D(S_chi_rhs), D(S_gxx_rhs), D(S_gyy_rhs), D(S_gzz_rhs),
|
|
D(S_gxy_rhs), D(S_gyz_rhs), D(S_gxz_rhs));
|
|
|
|
kern_phase2_inverse<<<grid(all),BLK>>>(
|
|
D(S_gxx), D(S_gxy), D(S_gxz), D(S_gyy), D(S_gyz), D(S_gzz),
|
|
D(S_gupxx), D(S_gupxy), D(S_gupxz),
|
|
D(S_gupyy), D(S_gupyz), D(S_gupzz));
|
|
|
|
if (co == 0) {
|
|
kern_phase3_gamma_constraint<<<grid(all),BLK>>>(
|
|
D(S_Gamx), D(S_Gamy), D(S_Gamz),
|
|
D(S_gupxx), D(S_gupxy), D(S_gupxz),
|
|
D(S_gupyy), D(S_gupyz), D(S_gupzz),
|
|
D(S_gxxx), D(S_gxyx), D(S_gxzx), D(S_gyyx), D(S_gyzx), D(S_gzzx),
|
|
D(S_gxxy), D(S_gxyy), D(S_gxzy), D(S_gyyy), D(S_gyzy), D(S_gzzy),
|
|
D(S_gxxz), D(S_gxyz), D(S_gxzz), D(S_gyyz), D(S_gyzz), D(S_gzzz),
|
|
D(S_Gmx_Res), D(S_Gmy_Res), D(S_Gmz_Res));
|
|
}
|
|
|
|
kern_phase4_christoffel<<<grid(all),BLK>>>(
|
|
D(S_gupxx), D(S_gupxy), D(S_gupxz),
|
|
D(S_gupyy), D(S_gupyz), D(S_gupzz),
|
|
D(S_gxxx), D(S_gxyx), D(S_gxzx), D(S_gyyx), D(S_gyzx), D(S_gzzx),
|
|
D(S_gxxy), D(S_gxyy), D(S_gxzy), D(S_gyyy), D(S_gyzy), D(S_gzzy),
|
|
D(S_gxxz), D(S_gxyz), D(S_gxzz), D(S_gyyz), D(S_gyzz), D(S_gzzz),
|
|
D(S_Gamxxx), D(S_Gamxxy), D(S_Gamxxz),
|
|
D(S_Gamxyy), D(S_Gamxyz), D(S_Gamxzz),
|
|
D(S_Gamyxx), D(S_Gamyxy), D(S_Gamyxz),
|
|
D(S_Gamyyy), D(S_Gamyyz), D(S_Gamyzz),
|
|
D(S_Gamzxx), D(S_Gamzxy), D(S_Gamzxz),
|
|
D(S_Gamzyy), D(S_Gamzyz), D(S_Gamzzz));
|
|
|
|
kern_phase5_6_gamma_rhs_part1_fused<<<grid(all),BLK>>>(
|
|
D(S_Lapx), D(S_Lapy), D(S_Lapz),
|
|
D(S_alpn1), D(S_chin1),
|
|
D(S_chix), D(S_chiy), D(S_chiz),
|
|
D(S_gupxx), D(S_gupxy), D(S_gupxz),
|
|
D(S_gupyy), D(S_gupyz), D(S_gupzz),
|
|
D(S_Axx), D(S_Axy), D(S_Axz), D(S_Ayy), D(S_Ayz), D(S_Azz),
|
|
D(S_Kx), D(S_Ky), D(S_Kz),
|
|
D(S_Sx), D(S_Sy), D(S_Sz),
|
|
D(S_Gamxxx), D(S_Gamxxy), D(S_Gamxxz),
|
|
D(S_Gamxyy), D(S_Gamxyz), D(S_Gamxzz),
|
|
D(S_Gamyxx), D(S_Gamyxy), D(S_Gamyxz),
|
|
D(S_Gamyyy), D(S_Gamyyz), D(S_Gamyzz),
|
|
D(S_Gamzxx), D(S_Gamzxy), D(S_Gamzxz),
|
|
D(S_Gamzyy), D(S_Gamzyz), D(S_Gamzzz),
|
|
D(S_Gamx_rhs), D(S_Gamy_rhs), D(S_Gamz_rhs));
|
|
|
|
{
|
|
double *src_fields[] = {D(S_betax), D(S_betay), D(S_betaz)};
|
|
double *fxx_fields[] = {D(S_gxxx), D(S_gxxy), D(S_gxxz)};
|
|
double *fxy_fields[] = {D(S_gxyx), D(S_gxyy), D(S_gxyz)};
|
|
double *fxz_fields[] = {D(S_gxzx), D(S_gxzy), D(S_gxzz)};
|
|
double *fyy_fields[] = {D(S_gyyx), D(S_gyyy), D(S_gyyz)};
|
|
double *fyz_fields[] = {D(S_gyzx), D(S_gyzy), D(S_gyzz)};
|
|
double *fzz_fields[] = {D(S_gzzx), D(S_gzzy), D(S_gzzz)};
|
|
const int soa_signs[] = {
|
|
(int)ANTI, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)ANTI, (int)SYM,
|
|
(int)SYM, (int)SYM, (int)ANTI
|
|
};
|
|
gpu_fdderivs_batch((int)(sizeof(src_fields) / sizeof(src_fields[0])),
|
|
src_fields, fxx_fields, fxy_fields, fxz_fields,
|
|
fyy_fields, fyz_fields, fzz_fields,
|
|
soa_signs, all);
|
|
}
|
|
{
|
|
double *src_fields[] = {D(S_Gamx), D(S_Gamy), D(S_Gamz)};
|
|
double *fx_fields[] = {D(S_Gamxx), D(S_Gamyx), D(S_Gamzx)};
|
|
double *fy_fields[] = {D(S_Gamxy), D(S_Gamyy_t), D(S_Gamzy)};
|
|
double *fz_fields[] = {D(S_Gamxz), D(S_Gamyz_t), D(S_Gamzz_t)};
|
|
const int soa_signs[] = {
|
|
(int)ANTI, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)ANTI, (int)SYM,
|
|
(int)SYM, (int)SYM, (int)ANTI
|
|
};
|
|
gpu_fderivs_batch((int)(sizeof(src_fields) / sizeof(src_fields[0])),
|
|
src_fields, fx_fields, fy_fields, fz_fields,
|
|
soa_signs, all);
|
|
}
|
|
|
|
kern_phase8_9_gamma_rhs_contract_fused<<<grid(all),BLK>>>(
|
|
D(S_gupxx), D(S_gupxy), D(S_gupxz),
|
|
D(S_gupyy), D(S_gupyz), D(S_gupzz),
|
|
D(S_gxxx),D(S_gxyx),D(S_gxzx),D(S_gyyx),D(S_gyzx),D(S_gzzx),
|
|
D(S_gxxy),D(S_gxyy),D(S_gxzy),D(S_gyyy),D(S_gyzy),D(S_gzzy),
|
|
D(S_gxxz),D(S_gxyz),D(S_gxzz),D(S_gyyz),D(S_gyzz),D(S_gzzz),
|
|
D(S_Gamxxx),D(S_Gamxxy),D(S_Gamxxz),
|
|
D(S_Gamxyy),D(S_Gamxyz),D(S_Gamxzz),
|
|
D(S_Gamyxx),D(S_Gamyxy),D(S_Gamyxz),
|
|
D(S_Gamyyy),D(S_Gamyyz),D(S_Gamyzz),
|
|
D(S_Gamzxx),D(S_Gamzxy),D(S_Gamzxz),
|
|
D(S_Gamzyy),D(S_Gamzyz),D(S_Gamzzz),
|
|
D(S_betaxx),D(S_betaxy),D(S_betaxz),
|
|
D(S_betayx),D(S_betayy),D(S_betayz),
|
|
D(S_betazx),D(S_betazy),D(S_betazz),
|
|
D(S_gxx),D(S_gxy),D(S_gxz),D(S_gyy),D(S_gyz),D(S_gzz),
|
|
D(S_Gamx_rhs),D(S_Gamy_rhs),D(S_Gamz_rhs),
|
|
D(S_Gamxa),D(S_Gamya),D(S_Gamza),
|
|
D(S_gxxx),D(S_gxyx),D(S_gxzx),D(S_gyyx),D(S_gyzx),D(S_gzzx),
|
|
D(S_gxxy),D(S_gxyy),D(S_gxzy),D(S_gyyy),D(S_gyzy),D(S_gzzy),
|
|
D(S_gxxz),D(S_gxyz),D(S_gxzz),D(S_gyyz),D(S_gyzz),D(S_gzzz));
|
|
|
|
{
|
|
double *src_fields[] = {D(S_dxx), D(S_dyy), D(S_dzz), D(S_gxy), D(S_gxz), D(S_gyz)};
|
|
double *dst_fields[] = {D(S_Rxx), D(S_Ryy), D(S_Rzz), D(S_Rxy), D(S_Rxz), D(S_Ryz)};
|
|
const int soa_signs[] = {
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)ANTI, (int)ANTI, (int)SYM,
|
|
(int)ANTI, (int)SYM, (int)ANTI,
|
|
(int)SYM, (int)ANTI, (int)ANTI
|
|
};
|
|
gpu_phase10_ricci_batch(D(S_gupxx), D(S_gupxy), D(S_gupxz),
|
|
D(S_gupyy), D(S_gupyz), D(S_gupzz),
|
|
src_fields, dst_fields, soa_signs, all);
|
|
}
|
|
|
|
kern_phase11_ricci_fused<<<grid(all),BLK>>>(
|
|
D(S_gxx),D(S_gxy),D(S_gxz),D(S_gyy),D(S_gyz),D(S_gzz),
|
|
D(S_gupxx),D(S_gupxy),D(S_gupxz),D(S_gupyy),D(S_gupyz),D(S_gupzz),
|
|
D(S_Gamxa),D(S_Gamya),D(S_Gamza),
|
|
D(S_Gamxx),D(S_Gamxy),D(S_Gamxz),
|
|
D(S_Gamyx),D(S_Gamyy_t),D(S_Gamyz_t),
|
|
D(S_Gamzx),D(S_Gamzy),D(S_Gamzz_t),
|
|
D(S_Gamxxx),D(S_Gamxxy),D(S_Gamxxz),
|
|
D(S_Gamxyy),D(S_Gamxyz),D(S_Gamxzz),
|
|
D(S_Gamyxx),D(S_Gamyxy),D(S_Gamyxz),
|
|
D(S_Gamyyy),D(S_Gamyyz),D(S_Gamyzz),
|
|
D(S_Gamzxx),D(S_Gamzxy),D(S_Gamzxz),
|
|
D(S_Gamzyy),D(S_Gamzyz),D(S_Gamzzz),
|
|
D(S_gxxx),D(S_gxyx),D(S_gxzx),D(S_gyyx),D(S_gyzx),D(S_gzzx),
|
|
D(S_gxxy),D(S_gxyy),D(S_gxzy),D(S_gyyy),D(S_gyzy),D(S_gzzy),
|
|
D(S_gxxz),D(S_gxyz),D(S_gxzz),D(S_gyyz),D(S_gyzz),D(S_gzzz),
|
|
D(S_Rxx),D(S_Rxy),D(S_Rxz),
|
|
D(S_Ryy),D(S_Ryz),D(S_Rzz));
|
|
|
|
kern_phase12_13_chi_correction_fused<<<grid((size_t)all),BLK>>>(
|
|
D(S_chi), D(S_chin1),
|
|
D(S_chix), D(S_chiy), D(S_chiz),
|
|
D(S_gxx), D(S_gxy), D(S_gxz), D(S_gyy), D(S_gyz), D(S_gzz),
|
|
D(S_gupxx), D(S_gupxy), D(S_gupxz),
|
|
D(S_gupyy), D(S_gupyz), D(S_gupzz),
|
|
D(S_Gamxxx), D(S_Gamxxy), D(S_Gamxxz),
|
|
D(S_Gamxyy), D(S_Gamxyz), D(S_Gamxzz),
|
|
D(S_Gamyxx), D(S_Gamyxy), D(S_Gamyxz),
|
|
D(S_Gamyyy), D(S_Gamyyz), D(S_Gamyzz),
|
|
D(S_Gamzxx), D(S_Gamzxy), D(S_Gamzxz),
|
|
D(S_Gamzyy), D(S_Gamzyz), D(S_Gamzzz),
|
|
D(S_Rxx), D(S_Rxy), D(S_Rxz),
|
|
D(S_Ryy), D(S_Ryz), D(S_Rzz));
|
|
|
|
kern_phase15_trK_Aij_gauge<<<grid(all),BLK>>>(
|
|
D(S_alpn1), D(S_chin1),
|
|
D(S_chix), D(S_chiy), D(S_chiz),
|
|
D(S_gxx), D(S_gxy), D(S_gxz), D(S_gyy), D(S_gyz), D(S_gzz),
|
|
D(S_gupxx), D(S_gupxy), D(S_gupxz),
|
|
D(S_gupyy), D(S_gupyz), D(S_gupzz),
|
|
D(S_trK),
|
|
D(S_Axx), D(S_Axy), D(S_Axz), D(S_Ayy), D(S_Ayz), D(S_Azz),
|
|
D(S_Lapx), D(S_Lapy), D(S_Lapz),
|
|
D(S_betaxx), D(S_betaxy), D(S_betaxz),
|
|
D(S_betayx), D(S_betayy), D(S_betayz),
|
|
D(S_betazx), D(S_betazy), D(S_betazz),
|
|
D(S_rho),
|
|
D(S_Sx), D(S_Sy), D(S_Sz),
|
|
D(S_Sxx), D(S_Sxy), D(S_Sxz), D(S_Syy), D(S_Syz), D(S_Szz),
|
|
D(S_dtSfx), D(S_dtSfy), D(S_dtSfz),
|
|
D(S_Rxx), D(S_Rxy), D(S_Rxz), D(S_Ryy), D(S_Ryz), D(S_Rzz),
|
|
D(S_Gamxxx), D(S_Gamxxy), D(S_Gamxxz),
|
|
D(S_Gamxyy), D(S_Gamxyz), D(S_Gamxzz),
|
|
D(S_Gamyxx), D(S_Gamyxy), D(S_Gamyxz),
|
|
D(S_Gamyyy), D(S_Gamyyz), D(S_Gamyzz),
|
|
D(S_Gamzxx), D(S_Gamzxy), D(S_Gamzxz),
|
|
D(S_Gamzyy), D(S_Gamzyz), D(S_Gamzzz),
|
|
D(S_dtSfx_rhs), D(S_dtSfy_rhs), D(S_dtSfz_rhs),
|
|
D(S_trK_rhs),
|
|
D(S_Axx_rhs), D(S_Axy_rhs), D(S_Axz_rhs),
|
|
D(S_Ayy_rhs), D(S_Ayz_rhs), D(S_Azz_rhs),
|
|
D(S_Lap_rhs),
|
|
D(S_betax_rhs), D(S_betay_rhs), D(S_betaz_rhs),
|
|
D(S_Gamx_rhs), D(S_Gamy_rhs), D(S_Gamz_rhs),
|
|
D(S_f_arr), D(S_S_arr));
|
|
|
|
gpu_lopsided_kodis_state_batch(eps, all);
|
|
|
|
if (co == 0) {
|
|
{
|
|
double *src_fields[] = {D(S_Axx), D(S_Axy), D(S_Axz), D(S_Ayy), D(S_Ayz), D(S_Azz)};
|
|
double *fx_fields[] = {D(S_gxxx), D(S_gxyx), D(S_gxzx), D(S_gyyx), D(S_gyzx), D(S_gzzx)};
|
|
double *fy_fields[] = {D(S_gxxy), D(S_gxyy), D(S_gxzy), D(S_gyyy), D(S_gyzy), D(S_gzzy)};
|
|
double *fz_fields[] = {D(S_gxxz), D(S_gxyz), D(S_gxzz), D(S_gyyz), D(S_gyzz), D(S_gzzz)};
|
|
const int soa_signs[] = {
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)ANTI, (int)ANTI, (int)SYM,
|
|
(int)ANTI, (int)SYM, (int)ANTI,
|
|
(int)SYM, (int)SYM, (int)SYM,
|
|
(int)SYM, (int)ANTI, (int)ANTI,
|
|
(int)SYM, (int)SYM, (int)SYM
|
|
};
|
|
gpu_fderivs_batch((int)(sizeof(src_fields) / sizeof(src_fields[0])),
|
|
src_fields, fx_fields, fy_fields, fz_fields,
|
|
soa_signs, all);
|
|
}
|
|
|
|
kern_phase18_constraints<<<grid(all),BLK>>>(
|
|
D(S_chin1),
|
|
D(S_chix), D(S_chiy), D(S_chiz),
|
|
D(S_gupxx), D(S_gupxy), D(S_gupxz),
|
|
D(S_gupyy), D(S_gupyz), D(S_gupzz),
|
|
D(S_trK),
|
|
D(S_Axx), D(S_Axy), D(S_Axz), D(S_Ayy), D(S_Ayz), D(S_Azz),
|
|
D(S_Rxx), D(S_Rxy), D(S_Rxz), D(S_Ryy), D(S_Ryz), D(S_Rzz),
|
|
D(S_rho), D(S_Sx), D(S_Sy), D(S_Sz),
|
|
D(S_Kx), D(S_Ky), D(S_Kz),
|
|
D(S_Gamxxx), D(S_Gamxxy), D(S_Gamxxz),
|
|
D(S_Gamxyy), D(S_Gamxyz), D(S_Gamxzz),
|
|
D(S_Gamyxx), D(S_Gamyxy), D(S_Gamyxz),
|
|
D(S_Gamyyy), D(S_Gamyyz), D(S_Gamyzz),
|
|
D(S_Gamzxx), D(S_Gamzxy), D(S_Gamzxz),
|
|
D(S_Gamzyy), D(S_Gamzyz), D(S_Gamzzz),
|
|
D(S_gxxx), D(S_gxxy), D(S_gxxz),
|
|
D(S_gxyx), D(S_gxyy), D(S_gxyz),
|
|
D(S_gxzx), D(S_gxzy), D(S_gxzz),
|
|
D(S_gyyx), D(S_gyyy), D(S_gyyz),
|
|
D(S_gyzx), D(S_gyzy), D(S_gyzz),
|
|
D(S_gzzx), D(S_gzzy), D(S_gzzz),
|
|
D(S_ham_Res), D(S_movx_Res), D(S_movy_Res), D(S_movz_Res));
|
|
}
|
|
|
|
#undef D
|
|
}
|
|
|
|
static void download_state_outputs(double **state_host_out, size_t all)
|
|
{
|
|
const size_t bytes = all * sizeof(double);
|
|
CUDA_CHECK(cudaMemcpy(g_buf.h_stage, g_buf.slot[S_chi_rhs],
|
|
(size_t)BSSN_STATE_COUNT * bytes,
|
|
cudaMemcpyDeviceToHost));
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
std::memcpy(state_host_out[i], g_buf.h_stage + (size_t)i * all, bytes);
|
|
}
|
|
}
|
|
|
|
static void download_constraint_outputs(double **constraint_host_out, size_t all)
|
|
{
|
|
const size_t bytes = all * sizeof(double);
|
|
CUDA_CHECK(cudaMemcpy(g_buf.h_stage, g_buf.slot[S_ham_Res],
|
|
(size_t)D2H_CONSTRAINT_SLOT_COUNT * bytes,
|
|
cudaMemcpyDeviceToHost));
|
|
for (int i = 0; i < D2H_CONSTRAINT_SLOT_COUNT; ++i) {
|
|
std::memcpy(constraint_host_out[i], g_buf.h_stage + (size_t)i * all, bytes);
|
|
}
|
|
}
|
|
|
|
__global__ void kern_pack_state_region_batch(const double * __restrict__ src_mem,
|
|
double * __restrict__ dst,
|
|
int nx, int ny,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz,
|
|
int region_all,
|
|
int state_count,
|
|
int all)
|
|
{
|
|
const int state_index = blockIdx.y;
|
|
if (state_index >= state_count) return;
|
|
for (int local = blockIdx.x * blockDim.x + threadIdx.x;
|
|
local < region_all;
|
|
local += blockDim.x * gridDim.x)
|
|
{
|
|
const int ii = local % sx;
|
|
const int jj = (local / sx) % sy;
|
|
const int kk = local / (sx * sy);
|
|
const int src = (i0 + ii) + (j0 + jj) * nx + (k0 + kk) * nx * ny;
|
|
dst[(size_t)state_index * region_all + local] =
|
|
src_mem[(size_t)state_index * all + src];
|
|
}
|
|
}
|
|
|
|
__global__ void kern_unpack_state_region_batch(double * __restrict__ dst_mem,
|
|
const double * __restrict__ src,
|
|
int nx, int ny,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz,
|
|
int region_all,
|
|
int state_count,
|
|
int all)
|
|
{
|
|
const int state_index = blockIdx.y;
|
|
if (state_index >= state_count) return;
|
|
for (int local = blockIdx.x * blockDim.x + threadIdx.x;
|
|
local < region_all;
|
|
local += blockDim.x * gridDim.x)
|
|
{
|
|
const int ii = local % sx;
|
|
const int jj = (local / sx) % sy;
|
|
const int kk = local / (sx * sy);
|
|
const int dst = (i0 + ii) + (j0 + jj) * nx + (k0 + kk) * nx * ny;
|
|
dst_mem[(size_t)state_index * all + dst] =
|
|
src[(size_t)state_index * region_all + local];
|
|
}
|
|
}
|
|
|
|
__device__ __forceinline__ double load_comm_state_cell_sym(const double * __restrict__ src_mem,
|
|
int state_index,
|
|
int x, int y, int z,
|
|
int nx, int ny,
|
|
int all)
|
|
{
|
|
double s = 1.0;
|
|
if (x < 0) {
|
|
x = -x;
|
|
s *= d_comm_state_soa[3 * state_index + 0];
|
|
}
|
|
if (y < 0) {
|
|
y = -y;
|
|
s *= d_comm_state_soa[3 * state_index + 1];
|
|
}
|
|
if (z < 0) {
|
|
z = -z;
|
|
s *= d_comm_state_soa[3 * state_index + 2];
|
|
}
|
|
const int src = x + y * nx + z * nx * ny;
|
|
return s * src_mem[(size_t)state_index * all + src];
|
|
}
|
|
|
|
__global__ void kern_restrict_state_region_batch(const double * __restrict__ src_mem,
|
|
double * __restrict__ dst,
|
|
int nx, int ny,
|
|
int sx, int sy, int sz,
|
|
int fi0, int fj0, int fk0,
|
|
int region_all,
|
|
int state_count,
|
|
int all)
|
|
{
|
|
const int state_index = blockIdx.y;
|
|
if (state_index >= state_count) return;
|
|
#if ghost_width == 5
|
|
const double c1 = 35.0 / 65536.0;
|
|
const double c2 = -405.0 / 65536.0;
|
|
const double c3 = 567.0 / 16384.0;
|
|
const double c4 = -2205.0 / 16384.0;
|
|
const double c5 = 19845.0 / 32768.0;
|
|
const int offs[10] = {-4, -3, -2, -1, 0, 1, 2, 3, 4, 5};
|
|
const double w[10] = {c1, c2, c3, c4, c5, c5, c4, c3, c2, c1};
|
|
const int nst = 10;
|
|
#elif ghost_width == 4
|
|
const double c1 = -5.0 / 2048.0;
|
|
const double c2 = 49.0 / 2048.0;
|
|
const double c3 = -245.0 / 2048.0;
|
|
const double c4 = 1225.0 / 2048.0;
|
|
const int offs[8] = {-3, -2, -1, 0, 1, 2, 3, 4};
|
|
const double w[8] = {c1, c2, c3, c4, c4, c3, c2, c1};
|
|
const int nst = 8;
|
|
#elif ghost_width == 3
|
|
const double c1 = 3.0 / 256.0;
|
|
const double c2 = -25.0 / 256.0;
|
|
const double c3 = 75.0 / 128.0;
|
|
const int offs[6] = {-2, -1, 0, 1, 2, 3};
|
|
const double w[6] = {c1, c2, c3, c3, c2, c1};
|
|
const int nst = 6;
|
|
#else
|
|
const double c1 = -1.0 / 16.0;
|
|
const double c2 = 9.0 / 16.0;
|
|
const int offs[4] = {-1, 0, 1, 2};
|
|
const double w[4] = {c1, c2, c2, c1};
|
|
const int nst = 4;
|
|
#endif
|
|
|
|
for (int local = blockIdx.x * blockDim.x + threadIdx.x;
|
|
local < region_all;
|
|
local += blockDim.x * gridDim.x)
|
|
{
|
|
const int ii = local % sx;
|
|
const int jj = (local / sx) % sy;
|
|
const int kk = local / (sx * sy);
|
|
const int fc_i = fi0 + 2 * ii;
|
|
const int fc_j = fj0 + 2 * jj;
|
|
const int fc_k = fk0 + 2 * kk;
|
|
double sum = 0.0;
|
|
for (int oz = 0; oz < nst; ++oz) {
|
|
const int z = fc_k + offs[oz];
|
|
const double wz = w[oz];
|
|
for (int oy = 0; oy < nst; ++oy) {
|
|
const int y = fc_j + offs[oy];
|
|
const double wyz = wz * w[oy];
|
|
for (int ox = 0; ox < nst; ++ox) {
|
|
const int x = fc_i + offs[ox];
|
|
sum += wyz * w[ox] *
|
|
load_comm_state_cell_sym(src_mem, state_index, x, y, z, nx, ny, all);
|
|
}
|
|
}
|
|
}
|
|
dst[(size_t)state_index * region_all + local] = sum;
|
|
}
|
|
}
|
|
|
|
__global__ void kern_prolong_state_region_batch(const double * __restrict__ src_mem,
|
|
double * __restrict__ dst,
|
|
int nx, int ny,
|
|
int sx, int sy, int sz,
|
|
int ii0, int jj0, int kk0,
|
|
int lbc_i, int lbc_j, int lbc_k,
|
|
int region_all,
|
|
int state_count,
|
|
int all)
|
|
{
|
|
const int state_index = blockIdx.y;
|
|
if (state_index >= state_count) return;
|
|
#if ghost_width == 5
|
|
const double c1 = 13585.0 / 33554432.0;
|
|
const double c2 = -159885.0 / 33554432.0;
|
|
const double c3 = 230945.0 / 8388608.0;
|
|
const double c4 = -969969.0 / 8388608.0;
|
|
const double c5 = 14549535.0 / 16777216.0;
|
|
const double c6 = 4849845.0 / 16777216.0;
|
|
const double c7 = -692835.0 / 8388608.0;
|
|
const double c8 = 188955.0 / 8388608.0;
|
|
const double c9 = -138567.0 / 33554432.0;
|
|
const double c10 = 12155.0 / 33554432.0;
|
|
const int offs[10] = {-4, -3, -2, -1, 0, 1, 2, 3, 4, 5};
|
|
const double wl[10] = {c1, c2, c3, c4, c5, c6, c7, c8, c9, c10};
|
|
const double wr[10] = {c10, c9, c8, c7, c6, c5, c4, c3, c2, c1};
|
|
const int nst = 10;
|
|
#elif ghost_width == 4
|
|
const double c1 = -495.0 / 262144.0;
|
|
const double c2 = 5005.0 / 262144.0;
|
|
const double c3 = -27027.0 / 262144.0;
|
|
const double c4 = 225225.0 / 262144.0;
|
|
const double c5 = 75075.0 / 262144.0;
|
|
const double c6 = -19305.0 / 262144.0;
|
|
const double c7 = 4095.0 / 262144.0;
|
|
const double c8 = -429.0 / 262144.0;
|
|
const int offs[8] = {-3, -2, -1, 0, 1, 2, 3, 4};
|
|
const double wl[8] = {c1, c2, c3, c4, c5, c6, c7, c8};
|
|
const double wr[8] = {c8, c7, c6, c5, c4, c3, c2, c1};
|
|
const int nst = 8;
|
|
#elif ghost_width == 3
|
|
const double c1 = 77.0 / 8192.0;
|
|
const double c2 = -693.0 / 8192.0;
|
|
const double c3 = 3465.0 / 4096.0;
|
|
const double c4 = 1155.0 / 4096.0;
|
|
const double c5 = -495.0 / 8192.0;
|
|
const double c6 = 63.0 / 8192.0;
|
|
const int offs[6] = {-2, -1, 0, 1, 2, 3};
|
|
const double wl[6] = {c1, c2, c3, c4, c5, c6};
|
|
const double wr[6] = {c6, c5, c4, c3, c2, c1};
|
|
const int nst = 6;
|
|
#else
|
|
const double c1 = -7.0 / 128.0;
|
|
const double c2 = 105.0 / 128.0;
|
|
const double c3 = 35.0 / 128.0;
|
|
const double c4 = -5.0 / 128.0;
|
|
const int offs[4] = {-1, 0, 1, 2};
|
|
const double wl[4] = {c1, c2, c3, c4};
|
|
const double wr[4] = {c4, c3, c2, c1};
|
|
const int nst = 4;
|
|
#endif
|
|
|
|
for (int local = blockIdx.x * blockDim.x + threadIdx.x;
|
|
local < region_all;
|
|
local += blockDim.x * gridDim.x)
|
|
{
|
|
const int ii = local % sx;
|
|
const int jj = (local / sx) % sy;
|
|
const int kk = local / (sx * sy);
|
|
const int fine_i = ii0 + ii;
|
|
const int fine_j = jj0 + jj;
|
|
const int fine_k = kk0 + kk;
|
|
const int ci = fine_i / 2 - lbc_i;
|
|
const int cj = fine_j / 2 - lbc_j;
|
|
const int ck = fine_k / 2 - lbc_k;
|
|
const double *wx = ((fine_i / 2) * 2 == fine_i) ? wl : wr;
|
|
const double *wy = ((fine_j / 2) * 2 == fine_j) ? wl : wr;
|
|
const double *wz = ((fine_k / 2) * 2 == fine_k) ? wl : wr;
|
|
double sum = 0.0;
|
|
for (int oz = 0; oz < nst; ++oz) {
|
|
const int z = ck + offs[oz];
|
|
const double wzv = wz[oz];
|
|
for (int oy = 0; oy < nst; ++oy) {
|
|
const int y = cj + offs[oy];
|
|
const double wyz = wzv * wy[oy];
|
|
for (int ox = 0; ox < nst; ++ox) {
|
|
const int x = ci + offs[ox];
|
|
sum += wyz * wx[ox] *
|
|
load_comm_state_cell_sym(src_mem, state_index, x, y, z, nx, ny, all);
|
|
}
|
|
}
|
|
}
|
|
dst[(size_t)state_index * region_all + local] = sum;
|
|
}
|
|
}
|
|
|
|
__global__ void kern_pack_state_subset(const double * __restrict__ src_mem,
|
|
double * __restrict__ dst,
|
|
int subset_count,
|
|
int all)
|
|
{
|
|
const int subset_slot = blockIdx.y;
|
|
if (subset_slot >= subset_count) return;
|
|
const int state_index = d_subset_state_indices[subset_slot];
|
|
for (int src = blockIdx.x * blockDim.x + threadIdx.x;
|
|
src < all;
|
|
src += blockDim.x * gridDim.x)
|
|
{
|
|
dst[(size_t)subset_slot * all + src] =
|
|
src_mem[(size_t)state_index * all + src];
|
|
}
|
|
}
|
|
|
|
__global__ void kern_unpack_state_subset(double * __restrict__ dst_mem,
|
|
const double * __restrict__ src,
|
|
int subset_count,
|
|
int all)
|
|
{
|
|
const int subset_slot = blockIdx.y;
|
|
if (subset_slot >= subset_count) return;
|
|
const int state_index = d_subset_state_indices[subset_slot];
|
|
for (int dst = blockIdx.x * blockDim.x + threadIdx.x;
|
|
dst < all;
|
|
dst += blockDim.x * gridDim.x)
|
|
{
|
|
dst_mem[(size_t)state_index * all + dst] =
|
|
src[(size_t)subset_slot * all + dst];
|
|
}
|
|
}
|
|
|
|
static void copy_state_region_cuda(void *block_tag,
|
|
int state_index,
|
|
double *host_state,
|
|
const int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz,
|
|
cudaMemcpyKind kind)
|
|
{
|
|
if (state_index < 0 || state_index >= BSSN_STATE_COUNT) return;
|
|
if (sx <= 0 || sy <= 0 || sz <= 0) return;
|
|
|
|
const size_t pitch = (size_t)ex[0] * sizeof(double);
|
|
StepContext &ctx = ensure_step_ctx(block_tag, (size_t)ex[0] * ex[1] * ex[2]);
|
|
|
|
cudaMemcpy3DParms p = {};
|
|
p.extent = make_cudaExtent((size_t)sx * sizeof(double), (size_t)sy, (size_t)sz);
|
|
p.srcPos = make_cudaPos((size_t)i0 * sizeof(double), j0, k0);
|
|
p.dstPos = make_cudaPos((size_t)i0 * sizeof(double), j0, k0);
|
|
|
|
if (kind == cudaMemcpyDeviceToHost) {
|
|
p.srcPtr = make_cudaPitchedPtr((void *)ctx.d_state_curr[state_index], pitch, ex[0], ex[1]);
|
|
p.dstPtr = make_cudaPitchedPtr((void *)host_state, pitch, ex[0], ex[1]);
|
|
} else {
|
|
p.srcPtr = make_cudaPitchedPtr((void *)host_state, pitch, ex[0], ex[1]);
|
|
p.dstPtr = make_cudaPitchedPtr((void *)ctx.d_state_curr[state_index], pitch, ex[0], ex[1]);
|
|
}
|
|
CUDA_CHECK(cudaMemcpy3D(&p));
|
|
}
|
|
|
|
static void copy_state_region_packed_cuda(void *block_tag,
|
|
int state_index,
|
|
double *host_buffer,
|
|
const int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz,
|
|
cudaMemcpyKind kind)
|
|
{
|
|
if (state_index < 0 || state_index >= BSSN_STATE_COUNT) return;
|
|
if (sx <= 0 || sy <= 0 || sz <= 0) return;
|
|
|
|
const size_t src_pitch = (size_t)ex[0] * sizeof(double);
|
|
const size_t dst_pitch = (size_t)sx * sizeof(double);
|
|
StepContext &ctx = ensure_step_ctx(block_tag, (size_t)ex[0] * ex[1] * ex[2]);
|
|
|
|
cudaMemcpy3DParms p = {};
|
|
p.extent = make_cudaExtent((size_t)sx * sizeof(double), (size_t)sy, (size_t)sz);
|
|
|
|
if (kind == cudaMemcpyDeviceToHost) {
|
|
p.srcPtr = make_cudaPitchedPtr((void *)ctx.d_state_curr[state_index], src_pitch, ex[0], ex[1]);
|
|
p.srcPos = make_cudaPos((size_t)i0 * sizeof(double), j0, k0);
|
|
p.dstPtr = make_cudaPitchedPtr((void *)host_buffer, dst_pitch, sx, sy);
|
|
p.dstPos = make_cudaPos(0, 0, 0);
|
|
} else {
|
|
p.srcPtr = make_cudaPitchedPtr((void *)host_buffer, dst_pitch, sx, sy);
|
|
p.srcPos = make_cudaPos(0, 0, 0);
|
|
p.dstPtr = make_cudaPitchedPtr((void *)ctx.d_state_curr[state_index], src_pitch, ex[0], ex[1]);
|
|
p.dstPos = make_cudaPos((size_t)i0 * sizeof(double), j0, k0);
|
|
}
|
|
|
|
CUDA_CHECK(cudaMemcpy3D(&p));
|
|
}
|
|
|
|
static void copy_state_region_packed_batch_cuda(void *block_tag,
|
|
int state_count,
|
|
double *host_buffer,
|
|
const int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz,
|
|
cudaMemcpyKind kind)
|
|
{
|
|
if (state_count <= 0 || state_count > BSSN_STATE_COUNT) return;
|
|
if (sx <= 0 || sy <= 0 || sz <= 0) return;
|
|
|
|
StepContext &ctx = ensure_step_ctx(block_tag, (size_t)ex[0] * ex[1] * ex[2]);
|
|
const int region_all = sx * sy * sz;
|
|
const size_t total_doubles = (size_t)state_count * (size_t)region_all;
|
|
double *d_comm = ensure_step_comm_buffer(ctx, total_doubles);
|
|
|
|
if (kind == cudaMemcpyDeviceToHost) {
|
|
dim3 launch_grid((unsigned int)grid((size_t)region_all),
|
|
(unsigned int)state_count);
|
|
kern_pack_state_region_batch<<<launch_grid, BLK>>>(
|
|
ctx.d_state_curr_mem, d_comm, ex[0], ex[1],
|
|
i0, j0, k0, sx, sy, sz, region_all, state_count,
|
|
ex[0] * ex[1] * ex[2]);
|
|
CUDA_CHECK(cudaMemcpy(host_buffer, d_comm,
|
|
total_doubles * sizeof(double),
|
|
cudaMemcpyDeviceToHost));
|
|
} else {
|
|
CUDA_CHECK(cudaMemcpy(d_comm, host_buffer,
|
|
total_doubles * sizeof(double),
|
|
cudaMemcpyHostToDevice));
|
|
dim3 launch_grid((unsigned int)grid((size_t)region_all),
|
|
(unsigned int)state_count);
|
|
kern_unpack_state_region_batch<<<launch_grid, BLK>>>(
|
|
ctx.d_state_curr_mem, d_comm, ex[0], ex[1],
|
|
i0, j0, k0, sx, sy, sz, region_all, state_count,
|
|
ex[0] * ex[1] * ex[2]);
|
|
}
|
|
}
|
|
|
|
static void copy_state_region_packed_batch_device_cuda(void *block_tag,
|
|
int state_count,
|
|
double *device_buffer,
|
|
const int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz,
|
|
int pack_not_unpack)
|
|
{
|
|
if (state_count <= 0 || state_count > BSSN_STATE_COUNT) return;
|
|
if (!device_buffer || sx <= 0 || sy <= 0 || sz <= 0) return;
|
|
|
|
StepContext &ctx = ensure_step_ctx(block_tag, (size_t)ex[0] * ex[1] * ex[2]);
|
|
const int region_all = sx * sy * sz;
|
|
dim3 launch_grid((unsigned int)grid((size_t)region_all),
|
|
(unsigned int)state_count);
|
|
|
|
if (pack_not_unpack) {
|
|
kern_pack_state_region_batch<<<launch_grid, BLK>>>(
|
|
ctx.d_state_curr_mem, device_buffer, ex[0], ex[1],
|
|
i0, j0, k0, sx, sy, sz, region_all, state_count,
|
|
ex[0] * ex[1] * ex[2]);
|
|
} else {
|
|
kern_unpack_state_region_batch<<<launch_grid, BLK>>>(
|
|
ctx.d_state_curr_mem, device_buffer, ex[0], ex[1],
|
|
i0, j0, k0, sx, sy, sz, region_all, state_count,
|
|
ex[0] * ex[1] * ex[2]);
|
|
ctx.state_ready = true;
|
|
}
|
|
}
|
|
|
|
static void download_resident_state(void *block_tag, int *ex, double **state_host_out)
|
|
{
|
|
const size_t all = (size_t)ex[0] * ex[1] * ex[2];
|
|
const size_t bytes = all * sizeof(double);
|
|
StepContext &ctx = ensure_step_ctx(block_tag, all);
|
|
CUDA_CHECK(cudaMemcpy(g_buf.h_stage, ctx.d_state_curr_mem,
|
|
(size_t)BSSN_STATE_COUNT * bytes,
|
|
cudaMemcpyDeviceToHost));
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
std::memcpy(state_host_out[i], g_buf.h_stage + (size_t)i * all, bytes);
|
|
}
|
|
}
|
|
|
|
static void copy_state_subset(void *block_tag,
|
|
int *ex,
|
|
int subset_count,
|
|
const int *state_indices,
|
|
double **state_host,
|
|
cudaMemcpyKind kind)
|
|
{
|
|
if (subset_count <= 0) return;
|
|
const size_t all = (size_t)ex[0] * ex[1] * ex[2];
|
|
const size_t bytes = all * sizeof(double);
|
|
StepContext &ctx = ensure_step_ctx(block_tag, all);
|
|
int active_state_indices[BSSN_STATE_COUNT];
|
|
double *active_state_host[BSSN_STATE_COUNT];
|
|
int active_count = 0;
|
|
|
|
for (int i = 0; i < subset_count; ++i) {
|
|
const int state_index = state_indices[i];
|
|
if (state_index < 0 || state_index >= BSSN_STATE_COUNT) continue;
|
|
if (!state_host[i]) continue;
|
|
active_state_indices[active_count] = state_index;
|
|
active_state_host[active_count] = state_host[i];
|
|
++active_count;
|
|
}
|
|
if (active_count <= 0) return;
|
|
|
|
const size_t total_doubles = (size_t)active_count * all;
|
|
double *d_comm = ensure_step_comm_buffer(ctx, total_doubles);
|
|
double *h_comm = ensure_step_host_comm_buffer(ctx, total_doubles);
|
|
|
|
CUDA_CHECK(cudaMemcpyToSymbol(d_subset_state_indices, active_state_indices,
|
|
(size_t)active_count * sizeof(int),
|
|
0, cudaMemcpyHostToDevice));
|
|
if (kind == cudaMemcpyDeviceToHost) {
|
|
dim3 launch_grid((unsigned int)grid(all), (unsigned int)active_count);
|
|
kern_pack_state_subset<<<launch_grid, BLK>>>(
|
|
ctx.d_state_curr_mem, d_comm, active_count, (int)all);
|
|
CUDA_CHECK(cudaMemcpy(h_comm, d_comm,
|
|
total_doubles * sizeof(double),
|
|
cudaMemcpyDeviceToHost));
|
|
for (int i = 0; i < active_count; ++i) {
|
|
std::memcpy(active_state_host[i],
|
|
h_comm + (size_t)i * all,
|
|
bytes);
|
|
}
|
|
} else {
|
|
for (int i = 0; i < active_count; ++i) {
|
|
std::memcpy(h_comm + (size_t)i * all,
|
|
active_state_host[i],
|
|
bytes);
|
|
}
|
|
CUDA_CHECK(cudaMemcpy(d_comm, h_comm,
|
|
total_doubles * sizeof(double),
|
|
cudaMemcpyHostToDevice));
|
|
dim3 launch_grid((unsigned int)grid(all), (unsigned int)active_count);
|
|
kern_unpack_state_subset<<<launch_grid, BLK>>>(
|
|
ctx.d_state_curr_mem, d_comm, active_count, (int)all);
|
|
}
|
|
}
|
|
|
|
static bool has_resident_state(void *block_tag)
|
|
{
|
|
auto it = g_step_ctx.find(block_tag);
|
|
return it != g_step_ctx.end() && it->second.state_ready;
|
|
}
|
|
|
|
|
|
|
|
#define pow2(x) ((x) * (x))
|
|
#define pow3(x) ((x) * (x) * (x))
|
|
#define pow4(x) ((x) * (x) * (x) * (x))
|
|
|
|
__device__ __forceinline__ void z4c_rhs_point_device(
|
|
double A11,
|
|
double A12,
|
|
double A13,
|
|
double A22,
|
|
double A23,
|
|
double A33,
|
|
double alpha,
|
|
double B1,
|
|
double B2,
|
|
double B3,
|
|
double beta1,
|
|
double beta2,
|
|
double beta3,
|
|
double chi,
|
|
double chiDivFloor,
|
|
double da1,
|
|
double dA111,
|
|
double dA112,
|
|
double dA113,
|
|
double dA122,
|
|
double dA123,
|
|
double dA133,
|
|
double da2,
|
|
double dA211,
|
|
double dA212,
|
|
double dA213,
|
|
double dA222,
|
|
double dA223,
|
|
double dA233,
|
|
double da3,
|
|
double dA311,
|
|
double dA312,
|
|
double dA313,
|
|
double dA322,
|
|
double dA323,
|
|
double dA333,
|
|
double db11,
|
|
double dB11,
|
|
double db12,
|
|
double dB12,
|
|
double db13,
|
|
double dB13,
|
|
double db21,
|
|
double dB21,
|
|
double db22,
|
|
double dB22,
|
|
double db23,
|
|
double dB23,
|
|
double db31,
|
|
double dB31,
|
|
double db32,
|
|
double dB32,
|
|
double db33,
|
|
double dB33,
|
|
double dchi1,
|
|
double dchi2,
|
|
double dchi3,
|
|
double dda11,
|
|
double dda12,
|
|
double dda13,
|
|
double dda22,
|
|
double dda23,
|
|
double dda33,
|
|
double ddb111,
|
|
double ddb112,
|
|
double ddb113,
|
|
double ddb121,
|
|
double ddb122,
|
|
double ddb123,
|
|
double ddb131,
|
|
double ddb132,
|
|
double ddb133,
|
|
double ddb221,
|
|
double ddb222,
|
|
double ddb223,
|
|
double ddb231,
|
|
double ddb232,
|
|
double ddb233,
|
|
double ddb331,
|
|
double ddb332,
|
|
double ddb333,
|
|
double ddchi11,
|
|
double ddchi12,
|
|
double ddchi13,
|
|
double ddchi22,
|
|
double ddchi23,
|
|
double ddchi33,
|
|
double deldelg1111,
|
|
double deldelg1112,
|
|
double deldelg1113,
|
|
double deldelg1122,
|
|
double deldelg1123,
|
|
double deldelg1133,
|
|
double deldelg1211,
|
|
double deldelg1212,
|
|
double deldelg1213,
|
|
double deldelg1222,
|
|
double deldelg1223,
|
|
double deldelg1233,
|
|
double deldelg1311,
|
|
double deldelg1312,
|
|
double deldelg1313,
|
|
double deldelg1322,
|
|
double deldelg1323,
|
|
double deldelg1333,
|
|
double deldelg2211,
|
|
double deldelg2212,
|
|
double deldelg2213,
|
|
double deldelg2222,
|
|
double deldelg2223,
|
|
double deldelg2233,
|
|
double deldelg2311,
|
|
double deldelg2312,
|
|
double deldelg2313,
|
|
double deldelg2322,
|
|
double deldelg2323,
|
|
double deldelg2333,
|
|
double deldelg3311,
|
|
double deldelg3312,
|
|
double deldelg3313,
|
|
double deldelg3322,
|
|
double deldelg3323,
|
|
double deldelg3333,
|
|
double delG11,
|
|
double delg111,
|
|
double delg112,
|
|
double delg113,
|
|
double delG12,
|
|
double delg122,
|
|
double delg123,
|
|
double delG13,
|
|
double delg133,
|
|
double delG21,
|
|
double delg211,
|
|
double delg212,
|
|
double delg213,
|
|
double delG22,
|
|
double delg222,
|
|
double delg223,
|
|
double delG23,
|
|
double delg233,
|
|
double delG31,
|
|
double delg311,
|
|
double delg312,
|
|
double delg313,
|
|
double delG32,
|
|
double delg322,
|
|
double delg323,
|
|
double delG33,
|
|
double delg333,
|
|
double dKhat1,
|
|
double dKhat2,
|
|
double dKhat3,
|
|
double dTheta1,
|
|
double dTheta2,
|
|
double dTheta3,
|
|
double G1,
|
|
double g11,
|
|
double g12,
|
|
double g13,
|
|
double G2,
|
|
double g22,
|
|
double g23,
|
|
double G3,
|
|
double g33,
|
|
double kappa1,
|
|
double kappa2,
|
|
double Khat,
|
|
double &rA11,
|
|
double &rA12,
|
|
double &rA13,
|
|
double &rA22,
|
|
double &rA23,
|
|
double &rA33,
|
|
double &rchi,
|
|
double &rG1,
|
|
double &rg11,
|
|
double &rg12,
|
|
double &rg13,
|
|
double &rG2,
|
|
double &rg22,
|
|
double &rg23,
|
|
double &rG3,
|
|
double &rg33,
|
|
double &rKhat,
|
|
double &rTheta,
|
|
double Theta)
|
|
{
|
|
double AA11;
|
|
double AA12;
|
|
double AA13;
|
|
double AA22;
|
|
double AA23;
|
|
double AA33;
|
|
double Ainv11;
|
|
double Ainv12;
|
|
double Ainv13;
|
|
double Ainv22;
|
|
double Ainv23;
|
|
double Ainv33;
|
|
double cAA;
|
|
double cdda11;
|
|
double cdda12;
|
|
double cdda13;
|
|
double cdda22;
|
|
double cdda23;
|
|
double cdda33;
|
|
double cddf11;
|
|
double cddf12;
|
|
double cddf13;
|
|
double cddf22;
|
|
double cddf23;
|
|
double cddf33;
|
|
double chiguard;
|
|
double chiguarded;
|
|
double chipsipower;
|
|
double ddf11;
|
|
double ddf12;
|
|
double ddf13;
|
|
double ddf22;
|
|
double ddf23;
|
|
double ddf33;
|
|
double detginv;
|
|
double df1;
|
|
double df2;
|
|
double df3;
|
|
double dGd11;
|
|
double dGd12;
|
|
double dGd13;
|
|
double dGd21;
|
|
double dGd22;
|
|
double dGd23;
|
|
double dGd31;
|
|
double dGd32;
|
|
double dGd33;
|
|
double dginv111;
|
|
double dginv112;
|
|
double dginv113;
|
|
double dginv122;
|
|
double dginv123;
|
|
double dginv133;
|
|
double dginv211;
|
|
double dginv212;
|
|
double dginv213;
|
|
double dginv222;
|
|
double dginv223;
|
|
double dginv233;
|
|
double dginv311;
|
|
double dginv312;
|
|
double dginv313;
|
|
double dginv322;
|
|
double dginv323;
|
|
double dginv333;
|
|
double divAinv1;
|
|
double divAinv2;
|
|
double divAinv3;
|
|
double divbeta;
|
|
double dK1;
|
|
double dK2;
|
|
double dK3;
|
|
double dphi1;
|
|
double dphi2;
|
|
double dphi3;
|
|
double dZ11;
|
|
double DZ11;
|
|
double dZ12;
|
|
double DZ12;
|
|
double dZ13;
|
|
double DZ13;
|
|
double dZ21;
|
|
double DZ21;
|
|
double dZ22;
|
|
double DZ22;
|
|
double dZ23;
|
|
double DZ23;
|
|
double dZ31;
|
|
double DZ31;
|
|
double dZ32;
|
|
double DZ32;
|
|
double dZ33;
|
|
double DZ33;
|
|
double dZinv11;
|
|
double DZinv11;
|
|
double dZinv12;
|
|
double DZinv12;
|
|
double dZinv13;
|
|
double DZinv13;
|
|
double dZinv21;
|
|
double DZinv21;
|
|
double dZinv22;
|
|
double DZinv22;
|
|
double dZinv23;
|
|
double DZinv23;
|
|
double dZinv31;
|
|
double DZinv31;
|
|
double dZinv32;
|
|
double DZinv32;
|
|
double dZinv33;
|
|
double DZinv33;
|
|
double DZsym11;
|
|
double DZsym12;
|
|
double DZsym13;
|
|
double DZsym21;
|
|
double DZsym22;
|
|
double DZsym23;
|
|
double DZsym31;
|
|
double DZsym32;
|
|
double DZsym33;
|
|
double f;
|
|
double ff;
|
|
double gamma111;
|
|
double gamma112;
|
|
double gamma113;
|
|
double gamma122;
|
|
double gamma123;
|
|
double gamma133;
|
|
double gamma211;
|
|
double gamma212;
|
|
double gamma213;
|
|
double gamma222;
|
|
double gamma223;
|
|
double gamma233;
|
|
double gamma311;
|
|
double gamma312;
|
|
double gamma313;
|
|
double gamma322;
|
|
double gamma323;
|
|
double gamma333;
|
|
double gammado111;
|
|
double gammado112;
|
|
double gammado113;
|
|
double gammado122;
|
|
double gammado123;
|
|
double gammado133;
|
|
double gammado211;
|
|
double gammado212;
|
|
double gammado213;
|
|
double gammado222;
|
|
double gammado223;
|
|
double gammado233;
|
|
double gammado311;
|
|
double gammado312;
|
|
double gammado313;
|
|
double gammado322;
|
|
double gammado323;
|
|
double gammado333;
|
|
double gammaF111;
|
|
double gammaF112;
|
|
double gammaF113;
|
|
double gammaF121;
|
|
double gammaF122;
|
|
double gammaF123;
|
|
double gammaF131;
|
|
double gammaF132;
|
|
double gammaF133;
|
|
double gammaF211;
|
|
double gammaF212;
|
|
double gammaF213;
|
|
double gammaF221;
|
|
double gammaF222;
|
|
double gammaF223;
|
|
double gammaF231;
|
|
double gammaF232;
|
|
double gammaF233;
|
|
double gammaF311;
|
|
double gammaF312;
|
|
double gammaF313;
|
|
double gammaF321;
|
|
double gammaF322;
|
|
double gammaF323;
|
|
double gammaF331;
|
|
double gammaF332;
|
|
double gammaF333;
|
|
double Gd1;
|
|
double Gd2;
|
|
double Gd3;
|
|
double Gfromg1;
|
|
double Gfromg2;
|
|
double Gfromg3;
|
|
double ginv11;
|
|
double ginv12;
|
|
double ginv13;
|
|
double ginv22;
|
|
double ginv23;
|
|
double ginv33;
|
|
double Hhat;
|
|
double K;
|
|
double lieA11;
|
|
double lieA12;
|
|
double lieA13;
|
|
double lieA22;
|
|
double lieA23;
|
|
double lieA33;
|
|
double liechi;
|
|
double lieg11;
|
|
double lieg12;
|
|
double lieg13;
|
|
double lieg22;
|
|
double lieg23;
|
|
double lieg33;
|
|
double oochipsipower;
|
|
double ootddivbeta1;
|
|
double ootddivbeta2;
|
|
double ootddivbeta3;
|
|
double pseudolieG1;
|
|
double pseudolieG2;
|
|
double pseudolieG3;
|
|
double psim4;
|
|
double R11;
|
|
double R12;
|
|
double R13;
|
|
double R22;
|
|
double R23;
|
|
double R33;
|
|
double Rhat;
|
|
double Rphi11;
|
|
double Rphi12;
|
|
double Rphi13;
|
|
double Rphi22;
|
|
double Rphi23;
|
|
double Rphi33;
|
|
double totdivbeta;
|
|
double trcdda;
|
|
double trcddf;
|
|
double trDZsym;
|
|
double Z1;
|
|
double Z2;
|
|
double Z3;
|
|
double Zinv1;
|
|
double Zinv2;
|
|
double Zinv3;
|
|
|
|
chipsipower =
|
|
-4.;
|
|
|
|
K =
|
|
Khat + 2. * Theta;
|
|
|
|
dK1 =
|
|
dKhat1 + 2. * dTheta1;
|
|
|
|
dK2 =
|
|
dKhat2 + 2. * dTheta2;
|
|
|
|
dK3 =
|
|
dKhat3 + 2. * dTheta3;
|
|
|
|
detginv =
|
|
1 / (2. * g12 * g13 * g23 - g33 * pow2(g12) + g22 * (g11 * g33 - pow2(g13)) -
|
|
g11 * pow2(g23));
|
|
|
|
ginv11 =
|
|
detginv * (g22 * g33 - pow2(g23));
|
|
|
|
ginv12 =
|
|
detginv * (g13 * g23 - g12 * g33);
|
|
|
|
ginv13 =
|
|
detginv * (-(g13 * g22) + g12 * g23);
|
|
|
|
ginv22 =
|
|
detginv * (g11 * g33 - pow2(g13));
|
|
|
|
ginv23 =
|
|
detginv * (g12 * g13 - g11 * g23);
|
|
|
|
ginv33 =
|
|
detginv * (g11 * g22 - pow2(g12));
|
|
|
|
dginv111 =
|
|
-2. * (delg123 * ginv12 * ginv13 + ginv11 * (delg112 * ginv12 + delg113 * ginv13)) -
|
|
delg111 * pow2(ginv11) - delg122 * pow2(ginv12) - delg133 * pow2(ginv13);
|
|
|
|
dginv112 =
|
|
-(ginv11 * (delg111 * ginv12 + delg112 * ginv22 + delg113 * ginv23)) -
|
|
ginv12 * (delg113 * ginv13 + delg122 * ginv22 + delg123 * ginv23) -
|
|
ginv13 * (delg123 * ginv22 + delg133 * ginv23) - delg112 * pow2(ginv12);
|
|
|
|
dginv113 =
|
|
-(ginv11 * (delg111 * ginv13 + delg112 * ginv23 + delg113 * ginv33)) -
|
|
ginv12 * (delg112 * ginv13 + delg122 * ginv23 + delg123 * ginv33) -
|
|
ginv13 * (delg123 * ginv23 + delg133 * ginv33) - delg113 * pow2(ginv13);
|
|
|
|
dginv122 =
|
|
-2. * (delg123 * ginv22 * ginv23 + ginv12 * (delg112 * ginv22 + delg113 * ginv23)) -
|
|
delg111 * pow2(ginv12) - delg122 * pow2(ginv22) - delg133 * pow2(ginv23);
|
|
|
|
dginv123 =
|
|
-(ginv13 * (delg112 * ginv22 + delg113 * ginv23)) - delg133 * ginv23 * ginv33 -
|
|
ginv12 * (delg111 * ginv13 + delg112 * ginv23 + delg113 * ginv33) -
|
|
ginv22 * (delg122 * ginv23 + delg123 * ginv33) - delg123 * pow2(ginv23);
|
|
|
|
dginv133 =
|
|
-2. * (delg123 * ginv23 * ginv33 + ginv13 * (delg112 * ginv23 + delg113 * ginv33)) -
|
|
delg111 * pow2(ginv13) - delg122 * pow2(ginv23) - delg133 * pow2(ginv33);
|
|
|
|
dginv211 =
|
|
-2. * (delg223 * ginv12 * ginv13 + ginv11 * (delg212 * ginv12 + delg213 * ginv13)) -
|
|
delg211 * pow2(ginv11) - delg222 * pow2(ginv12) - delg233 * pow2(ginv13);
|
|
|
|
dginv212 =
|
|
-(ginv11 * (delg211 * ginv12 + delg212 * ginv22 + delg213 * ginv23)) -
|
|
ginv12 * (delg213 * ginv13 + delg222 * ginv22 + delg223 * ginv23) -
|
|
ginv13 * (delg223 * ginv22 + delg233 * ginv23) - delg212 * pow2(ginv12);
|
|
|
|
dginv213 =
|
|
-(ginv11 * (delg211 * ginv13 + delg212 * ginv23 + delg213 * ginv33)) -
|
|
ginv12 * (delg212 * ginv13 + delg222 * ginv23 + delg223 * ginv33) -
|
|
ginv13 * (delg223 * ginv23 + delg233 * ginv33) - delg213 * pow2(ginv13);
|
|
|
|
dginv222 =
|
|
-2. * (delg223 * ginv22 * ginv23 + ginv12 * (delg212 * ginv22 + delg213 * ginv23)) -
|
|
delg211 * pow2(ginv12) - delg222 * pow2(ginv22) - delg233 * pow2(ginv23);
|
|
|
|
dginv223 =
|
|
-(ginv13 * (delg212 * ginv22 + delg213 * ginv23)) - delg233 * ginv23 * ginv33 -
|
|
ginv12 * (delg211 * ginv13 + delg212 * ginv23 + delg213 * ginv33) -
|
|
ginv22 * (delg222 * ginv23 + delg223 * ginv33) - delg223 * pow2(ginv23);
|
|
|
|
dginv233 =
|
|
-2. * (delg223 * ginv23 * ginv33 + ginv13 * (delg212 * ginv23 + delg213 * ginv33)) -
|
|
delg211 * pow2(ginv13) - delg222 * pow2(ginv23) - delg233 * pow2(ginv33);
|
|
|
|
dginv311 =
|
|
-2. * (delg323 * ginv12 * ginv13 + ginv11 * (delg312 * ginv12 + delg313 * ginv13)) -
|
|
delg311 * pow2(ginv11) - delg322 * pow2(ginv12) - delg333 * pow2(ginv13);
|
|
|
|
dginv312 =
|
|
-(ginv11 * (delg311 * ginv12 + delg312 * ginv22 + delg313 * ginv23)) -
|
|
ginv12 * (delg313 * ginv13 + delg322 * ginv22 + delg323 * ginv23) -
|
|
ginv13 * (delg323 * ginv22 + delg333 * ginv23) - delg312 * pow2(ginv12);
|
|
|
|
dginv313 =
|
|
-(ginv11 * (delg311 * ginv13 + delg312 * ginv23 + delg313 * ginv33)) -
|
|
ginv12 * (delg312 * ginv13 + delg322 * ginv23 + delg323 * ginv33) -
|
|
ginv13 * (delg323 * ginv23 + delg333 * ginv33) - delg313 * pow2(ginv13);
|
|
|
|
dginv322 =
|
|
-2. * (delg323 * ginv22 * ginv23 + ginv12 * (delg312 * ginv22 + delg313 * ginv23)) -
|
|
delg311 * pow2(ginv12) - delg322 * pow2(ginv22) - delg333 * pow2(ginv23);
|
|
|
|
dginv323 =
|
|
-(ginv13 * (delg312 * ginv22 + delg313 * ginv23)) - delg333 * ginv23 * ginv33 -
|
|
ginv12 * (delg311 * ginv13 + delg312 * ginv23 + delg313 * ginv33) -
|
|
ginv22 * (delg322 * ginv23 + delg323 * ginv33) - delg323 * pow2(ginv23);
|
|
|
|
dginv333 =
|
|
-2. * (delg323 * ginv23 * ginv33 + ginv13 * (delg312 * ginv23 + delg313 * ginv33)) -
|
|
delg311 * pow2(ginv13) - delg322 * pow2(ginv23) - delg333 * pow2(ginv33);
|
|
|
|
gammado111 =
|
|
0.5 * delg111;
|
|
|
|
gammado112 =
|
|
0.5 * delg211;
|
|
|
|
gammado113 =
|
|
0.5 * delg311;
|
|
|
|
gammado122 =
|
|
-0.5 * delg122 + delg212;
|
|
|
|
gammado123 =
|
|
0.5 * (-delg123 + delg213 + delg312);
|
|
|
|
gammado133 =
|
|
-0.5 * delg133 + delg313;
|
|
|
|
gammado211 =
|
|
delg112 - 0.5 * delg211;
|
|
|
|
gammado212 =
|
|
0.5 * delg122;
|
|
|
|
gammado213 =
|
|
0.5 * (delg123 - delg213 + delg312);
|
|
|
|
gammado222 =
|
|
0.5 * delg222;
|
|
|
|
gammado223 =
|
|
0.5 * delg322;
|
|
|
|
gammado233 =
|
|
-0.5 * delg233 + delg323;
|
|
|
|
gammado311 =
|
|
delg113 - 0.5 * delg311;
|
|
|
|
gammado312 =
|
|
0.5 * (delg123 + delg213 - delg312);
|
|
|
|
gammado313 =
|
|
0.5 * delg133;
|
|
|
|
gammado322 =
|
|
delg223 - 0.5 * delg322;
|
|
|
|
gammado323 =
|
|
0.5 * delg233;
|
|
|
|
gammado333 =
|
|
0.5 * delg333;
|
|
|
|
gamma111 =
|
|
gammado111 * ginv11 + gammado211 * ginv12 + gammado311 * ginv13;
|
|
|
|
gamma112 =
|
|
gammado112 * ginv11 + gammado212 * ginv12 + gammado312 * ginv13;
|
|
|
|
gamma113 =
|
|
gammado113 * ginv11 + gammado213 * ginv12 + gammado313 * ginv13;
|
|
|
|
gamma122 =
|
|
gammado122 * ginv11 + gammado222 * ginv12 + gammado322 * ginv13;
|
|
|
|
gamma123 =
|
|
gammado123 * ginv11 + gammado223 * ginv12 + gammado323 * ginv13;
|
|
|
|
gamma133 =
|
|
gammado133 * ginv11 + gammado233 * ginv12 + gammado333 * ginv13;
|
|
|
|
gamma211 =
|
|
gammado111 * ginv12 + gammado211 * ginv22 + gammado311 * ginv23;
|
|
|
|
gamma212 =
|
|
gammado112 * ginv12 + gammado212 * ginv22 + gammado312 * ginv23;
|
|
|
|
gamma213 =
|
|
gammado113 * ginv12 + gammado213 * ginv22 + gammado313 * ginv23;
|
|
|
|
gamma222 =
|
|
gammado122 * ginv12 + gammado222 * ginv22 + gammado322 * ginv23;
|
|
|
|
gamma223 =
|
|
gammado123 * ginv12 + gammado223 * ginv22 + gammado323 * ginv23;
|
|
|
|
gamma233 =
|
|
gammado133 * ginv12 + gammado233 * ginv22 + gammado333 * ginv23;
|
|
|
|
gamma311 =
|
|
gammado111 * ginv13 + gammado211 * ginv23 + gammado311 * ginv33;
|
|
|
|
gamma312 =
|
|
gammado112 * ginv13 + gammado212 * ginv23 + gammado312 * ginv33;
|
|
|
|
gamma313 =
|
|
gammado113 * ginv13 + gammado213 * ginv23 + gammado313 * ginv33;
|
|
|
|
gamma322 =
|
|
gammado122 * ginv13 + gammado222 * ginv23 + gammado322 * ginv33;
|
|
|
|
gamma323 =
|
|
gammado123 * ginv13 + gammado223 * ginv23 + gammado323 * ginv33;
|
|
|
|
gamma333 =
|
|
gammado133 * ginv13 + gammado233 * ginv23 + gammado333 * ginv33;
|
|
|
|
Gfromg1 =
|
|
gamma111 * ginv11 + gamma122 * ginv22 +
|
|
2. * (gamma112 * ginv12 + gamma113 * ginv13 + gamma123 * ginv23) + gamma133 * ginv33;
|
|
|
|
Gfromg2 =
|
|
gamma211 * ginv11 + gamma222 * ginv22 +
|
|
2. * (gamma212 * ginv12 + gamma213 * ginv13 + gamma223 * ginv23) + gamma233 * ginv33;
|
|
|
|
Gfromg3 =
|
|
gamma311 * ginv11 + gamma322 * ginv22 +
|
|
2. * (gamma312 * ginv12 + gamma313 * ginv13 + gamma323 * ginv23) + gamma333 * ginv33;
|
|
|
|
R11 =
|
|
delG11 * g11 + delG12 * g12 + delG13 * g13 + gammado111 * Gfromg1 +
|
|
gammado112 * Gfromg2 + gammado113 * Gfromg3 +
|
|
(-0.5 * deldelg1111 + 3. * gamma111 * gammado111 +
|
|
2. * (gamma211 * gammado112 + gamma311 * gammado113) +
|
|
gamma211 * gammado211 + gamma311 * gammado311) *
|
|
ginv11 +
|
|
(-deldelg1211 + 3. * (gamma112 * gammado111 + gamma111 * gammado112) +
|
|
2. * (gamma212 * gammado112 + gamma312 * gammado113 +
|
|
gamma211 * gammado122 + gamma311 * gammado123) +
|
|
gamma212 * gammado211 +
|
|
gamma211 * gammado212 + gamma312 * gammado311 + gamma311 * gammado312) *
|
|
ginv12 +
|
|
(-deldelg1311 + 3. * (gamma113 * gammado111 + gamma111 * gammado113) +
|
|
2. * (gamma213 * gammado112 + gamma313 * gammado113 +
|
|
gamma211 * gammado123 + gamma311 * gammado133) +
|
|
gamma213 * gammado211 +
|
|
gamma211 * gammado213 + gamma313 * gammado311 + gamma311 * gammado313) *
|
|
ginv13 +
|
|
(-0.5 * deldelg2211 + 3. * gamma112 * gammado112 +
|
|
2. * (gamma212 * gammado122 + gamma312 * gammado123) +
|
|
gamma212 * gammado212 + gamma312 * gammado312) *
|
|
ginv22 +
|
|
(-deldelg2311 + 3. * (gamma113 * gammado112 + gamma112 * gammado113) +
|
|
2. * (gamma213 * gammado122 + (gamma212 + gamma313) * gammado123 +
|
|
gamma312 * gammado133) +
|
|
gamma213 * gammado212 + gamma212 * gammado213 +
|
|
gamma313 * gammado312 + gamma312 * gammado313) *
|
|
ginv23 +
|
|
(-0.5 * deldelg3311 + 3. * gamma113 * gammado113 +
|
|
2. * (gamma213 * gammado123 + gamma313 * gammado133) + gamma213 * gammado213 +
|
|
gamma313 * gammado313) *
|
|
ginv33;
|
|
|
|
R12 =
|
|
0.5 * (delG21 * g11 + (delG11 + delG22) * g12 + delG23 * g13 + delG12 * g22 +
|
|
delG13 * g23 + (gammado112 + gammado211) * Gfromg1 +
|
|
(gammado122 + gammado212) * Gfromg2 + (gammado123 + gammado213) * Gfromg3) +
|
|
(-0.5 * deldelg1112 + gamma112 * gammado111 +
|
|
(gamma111 + gamma212) * gammado112 + gamma312 * gammado113 +
|
|
gamma111 * gammado211 + 2. * gamma211 * gammado212 +
|
|
gamma311 * (gammado213 + gammado312)) *
|
|
ginv11 +
|
|
(-deldelg1212 + gamma122 * gammado111 +
|
|
(2. * gamma112 + gamma222) * gammado112 + gamma322 * gammado113 +
|
|
(gamma111 + gamma212) * gammado122 + gamma112 * gammado211 +
|
|
(gamma111 + 2. * gamma212) * gammado212 + 2. * gamma211 * gammado222 +
|
|
gamma312 * (gammado123 + gammado213 + gammado312) +
|
|
gamma311 * (gammado223 + gammado322)) *
|
|
ginv12 +
|
|
(-deldelg1312 + gamma123 * gammado111 + (gamma113 + gamma223) * gammado112 +
|
|
(gamma112 + gamma323) * gammado113 + (gamma111 + gamma212) * gammado123 +
|
|
gamma312 * gammado133 + gamma113 * gammado211 +
|
|
(gamma111 + gamma313) * gammado213 +
|
|
2. * (gamma213 * gammado212 + gamma211 * gammado223) +
|
|
gamma313 * gammado312 + gamma311 * (gammado233 + gammado323)) *
|
|
ginv13 +
|
|
(-0.5 * deldelg2212 + gamma122 * gammado112 +
|
|
(gamma112 + gamma222) * gammado122 + gamma322 * gammado123 +
|
|
gamma112 * gammado212 + 2. * gamma212 * gammado222 +
|
|
gamma312 * (gammado223 + gammado322)) *
|
|
ginv22 +
|
|
(-deldelg2312 + gamma123 * gammado112 + gamma122 * gammado113 +
|
|
(gamma113 + gamma223) * gammado122 +
|
|
(gamma112 + gamma222 + gamma323) * gammado123 + gamma322 * gammado133 +
|
|
gamma113 * gammado212 + gamma112 * gammado213 +
|
|
2. * (gamma213 * gammado222 + gamma212 * gammado223) +
|
|
gamma313 * (gammado223 + gammado322) +
|
|
gamma312 * (gammado233 + gammado323)) *
|
|
ginv23 +
|
|
(-0.5 * deldelg3312 + gamma123 * gammado113 +
|
|
(gamma113 + gamma223) * gammado123 + gamma323 * gammado133 +
|
|
gamma113 * gammado213 + 2. * gamma213 * gammado223 +
|
|
gamma313 * (gammado233 + gammado323)) *
|
|
ginv33;
|
|
|
|
R13 =
|
|
0.5 * (delG31 * g11 + delG32 * g12 + (delG11 + delG33) * g13 + delG12 * g23 +
|
|
delG13 * g33 + (gammado113 + gammado311) * Gfromg1 +
|
|
(gammado123 + gammado312) * Gfromg2 + (gammado133 + gammado313) * Gfromg3) +
|
|
(-0.5 * deldelg1113 + gamma113 * gammado111 + gamma213 * gammado112 +
|
|
(gamma111 + gamma313) * gammado113 + gamma111 * gammado311 +
|
|
gamma211 * (gammado213 + gammado312) + 2. * gamma311 * gammado313) *
|
|
ginv11 +
|
|
(-deldelg1213 + gamma123 * gammado111 + (gamma113 + gamma223) * gammado112 +
|
|
(gamma112 + gamma323) * gammado113 + gamma213 * gammado122 +
|
|
(gamma111 + gamma313) * gammado123 + gamma112 * gammado311 +
|
|
gamma111 * gammado312 + gamma212 * (gammado213 + gammado312) +
|
|
gamma211 * (gammado223 + gammado322) +
|
|
2. * (gamma312 * gammado313 + gamma311 * gammado323)) *
|
|
ginv12 +
|
|
(-deldelg1313 + gamma133 * gammado111 + gamma233 * gammado112 +
|
|
(2. * gamma113 + gamma333) * gammado113 +
|
|
(gamma111 + gamma313) * gammado133 + gamma113 * gammado311 +
|
|
gamma213 * (gammado123 + gammado213 + gammado312) +
|
|
(gamma111 + 2. * gamma313) * gammado313 +
|
|
gamma211 * (gammado233 + gammado323) + 2. * gamma311 * gammado333) *
|
|
ginv13 +
|
|
(-0.5 * deldelg2213 + gamma123 * gammado112 + gamma223 * gammado122 +
|
|
(gamma112 + gamma323) * gammado123 + gamma112 * gammado312 +
|
|
gamma212 * (gammado223 + gammado322) + 2. * gamma312 * gammado323) *
|
|
ginv22 +
|
|
(-deldelg2313 + gamma133 * gammado112 + gamma123 * gammado113 +
|
|
gamma233 * gammado122 + (gamma113 + gamma223 + gamma333) * gammado123 +
|
|
(gamma112 + gamma323) * gammado133 + gamma113 * gammado312 +
|
|
gamma112 * gammado313 + gamma213 * (gammado223 + gammado322) +
|
|
gamma212 * (gammado233 + gammado323) +
|
|
2. * (gamma313 * gammado323 + gamma312 * gammado333)) *
|
|
ginv23 +
|
|
(-0.5 * deldelg3313 + gamma133 * gammado113 + gamma233 * gammado123 +
|
|
(gamma113 + gamma333) * gammado133 + gamma113 * gammado313 +
|
|
gamma213 * (gammado233 + gammado323) + 2. * gamma313 * gammado333) *
|
|
ginv33;
|
|
|
|
R22 =
|
|
delG21 * g12 + delG22 * g22 + delG23 * g23 + gammado212 * Gfromg1 +
|
|
gammado222 * Gfromg2 + gammado223 * Gfromg3 +
|
|
(-0.5 * deldelg1122 + gamma112 * (gammado112 + 2. * gammado211) +
|
|
3. * gamma212 * gammado212 + gamma312 * (2. * gammado213 + gammado312)) *
|
|
ginv11 +
|
|
(-deldelg1222 + gamma122 * (gammado112 + 2. * gammado211) +
|
|
gamma112 * (gammado122 + 2. * gammado212) +
|
|
3. * (gamma222 * gammado212 + gamma212 * gammado222) +
|
|
2. * (gamma322 * gammado213 + gamma312 * gammado223) +
|
|
gamma322 * gammado312 + gamma312 * gammado322) *
|
|
ginv12 +
|
|
(-deldelg1322 + gamma123 * (gammado112 + 2. * gammado211) +
|
|
gamma112 * (gammado123 + 2. * gammado213) +
|
|
3. * (gamma223 * gammado212 + gamma212 * gammado223) +
|
|
2. * (gamma323 * gammado213 + gamma312 * gammado233) +
|
|
gamma323 * gammado312 + gamma312 * gammado323) *
|
|
ginv13 +
|
|
(-0.5 * deldelg2222 + gamma122 * (gammado122 + 2. * gammado212) +
|
|
3. * gamma222 * gammado222 + gamma322 * (2. * gammado223 + gammado322)) *
|
|
ginv22 +
|
|
(-deldelg2322 + gamma123 * (gammado122 + 2. * gammado212) +
|
|
gamma122 * (gammado123 + 2. * gammado213) +
|
|
3. * (gamma223 * gammado222 + gamma222 * gammado223) +
|
|
2. * (gamma323 * gammado223 + gamma322 * gammado233) +
|
|
gamma323 * gammado322 + gamma322 * gammado323) *
|
|
ginv23 +
|
|
(-0.5 * deldelg3322 + gamma123 * (gammado123 + 2. * gammado213) +
|
|
3. * gamma223 * gammado223 + gamma323 * (2. * gammado233 + gammado323)) *
|
|
ginv33;
|
|
|
|
R23 =
|
|
0.5 * (delG31 * g12 + delG21 * g13 + delG32 * g22 + (delG22 + delG33) * g23 +
|
|
delG23 * g33 + (gammado213 + gammado312) * Gfromg1 +
|
|
(gammado223 + gammado322) * Gfromg2 + (gammado233 + gammado323) * Gfromg3) +
|
|
(-0.5 * deldelg1123 + gamma113 * gammado211 + gamma213 * gammado212 +
|
|
(gamma212 + gamma313) * gammado213 +
|
|
gamma112 * (gammado113 + gammado311) + gamma212 * gammado312 +
|
|
2. * gamma312 * gammado313) *
|
|
ginv11 +
|
|
(-deldelg1223 + gamma123 * gammado211 + (gamma113 + gamma223) * gammado212 +
|
|
(gamma222 + gamma323) * gammado213 + gamma213 * gammado222 +
|
|
(gamma212 + gamma313) * gammado223 +
|
|
gamma122 * (gammado113 + gammado311) + gamma222 * gammado312 +
|
|
gamma112 * (gammado123 + gammado312) + gamma212 * gammado322 +
|
|
2. * (gamma322 * gammado313 + gamma312 * gammado323)) *
|
|
ginv12 +
|
|
(-deldelg1323 + gamma133 * gammado211 + gamma233 * gammado212 +
|
|
(gamma113 + gamma223 + gamma333) * gammado213 + gamma213 * gammado223 +
|
|
(gamma212 + gamma313) * gammado233 +
|
|
gamma123 * (gammado113 + gammado311) + gamma223 * gammado312 +
|
|
gamma112 * (gammado133 + gammado313) + gamma212 * gammado323 +
|
|
2. * (gamma323 * gammado313 + gamma312 * gammado333)) *
|
|
ginv13 +
|
|
(-0.5 * deldelg2223 + gamma123 * gammado212 + gamma223 * gammado222 +
|
|
(gamma222 + gamma323) * gammado223 +
|
|
gamma122 * (gammado123 + gammado312) + gamma222 * gammado322 +
|
|
2. * gamma322 * gammado323) *
|
|
ginv22 +
|
|
(-deldelg2323 + gamma133 * gammado212 + gamma233 * gammado222 +
|
|
(2. * gamma223 + gamma333) * gammado223 +
|
|
(gamma222 + gamma323) * gammado233 +
|
|
gamma123 * (gammado123 + gammado213 + gammado312) +
|
|
gamma122 * (gammado133 + gammado313) + gamma223 * gammado322 +
|
|
(gamma222 + 2. * gamma323) * gammado323 + 2. * gamma322 * gammado333) *
|
|
ginv23 +
|
|
(-0.5 * deldelg3323 + gamma133 * gammado213 + gamma233 * gammado223 +
|
|
(gamma223 + gamma333) * gammado233 +
|
|
gamma123 * (gammado133 + gammado313) + gamma223 * gammado323 +
|
|
2. * gamma323 * gammado333) *
|
|
ginv33;
|
|
|
|
R33 =
|
|
delG31 * g13 + delG32 * g23 + delG33 * g33 + gammado313 * Gfromg1 +
|
|
gammado323 * Gfromg2 + gammado333 * Gfromg3 +
|
|
(-0.5 * deldelg1133 + gamma113 * (gammado113 + 2. * gammado311) +
|
|
gamma213 * (gammado213 + 2. * gammado312) + 3. * gamma313 * gammado313) *
|
|
ginv11 +
|
|
(-deldelg1233 + gamma123 * (gammado113 + 2. * gammado311) +
|
|
gamma113 * (gammado123 + 2. * gammado312) +
|
|
gamma223 * (gammado213 + 2. * gammado312) +
|
|
gamma213 * (gammado223 + 2. * gammado322) +
|
|
3. * (gamma323 * gammado313 + gamma313 * gammado323)) *
|
|
ginv12 +
|
|
(-deldelg1333 + gamma133 * (gammado113 + 2. * gammado311) +
|
|
gamma233 * (gammado213 + 2. * gammado312) +
|
|
gamma113 * (gammado133 + 2. * gammado313) +
|
|
gamma213 * (gammado233 + 2. * gammado323) +
|
|
3. * (gamma333 * gammado313 + gamma313 * gammado333)) *
|
|
ginv13 +
|
|
(-0.5 * deldelg2233 + gamma123 * (gammado123 + 2. * gammado312) +
|
|
gamma223 * (gammado223 + 2. * gammado322) + 3. * gamma323 * gammado323) *
|
|
ginv22 +
|
|
(-deldelg2333 + gamma133 * (gammado123 + 2. * gammado312) +
|
|
gamma123 * (gammado133 + 2. * gammado313) +
|
|
gamma233 * (gammado223 + 2. * gammado322) +
|
|
gamma223 * (gammado233 + 2. * gammado323) +
|
|
3. * (gamma333 * gammado323 + gamma323 * gammado333)) *
|
|
ginv23 +
|
|
(-0.5 * deldelg3333 + gamma133 * (gammado133 + 2. * gammado313) +
|
|
gamma233 * (gammado233 + 2. * gammado323) + 3. * gamma333 * gammado333) *
|
|
ginv33;
|
|
|
|
chiguard =
|
|
chiDivFloor;
|
|
|
|
chiguarded =
|
|
chi;
|
|
|
|
if (chiguarded < chiguard)
|
|
chiguarded = chiguard;
|
|
|
|
ff =
|
|
chiguarded;
|
|
|
|
oochipsipower =
|
|
1 / chipsipower;
|
|
|
|
f =
|
|
oochipsipower * log(ff);
|
|
|
|
psim4 =
|
|
exp(-4. * f);
|
|
|
|
df1 =
|
|
(dchi1 * oochipsipower) / chiguarded;
|
|
|
|
df2 =
|
|
(dchi2 * oochipsipower) / chiguarded;
|
|
|
|
df3 =
|
|
(dchi3 * oochipsipower) / chiguarded;
|
|
|
|
ddf11 =
|
|
(ddchi11 * oochipsipower) / chiguarded - chipsipower * pow2(df1);
|
|
|
|
ddf12 =
|
|
-(chipsipower * df1 * df2) + (ddchi12 * oochipsipower) / chiguarded;
|
|
|
|
ddf13 =
|
|
-(chipsipower * df1 * df3) + (ddchi13 * oochipsipower) / chiguarded;
|
|
|
|
ddf22 =
|
|
(ddchi22 * oochipsipower) / chiguarded - chipsipower * pow2(df2);
|
|
|
|
ddf23 =
|
|
-(chipsipower * df2 * df3) + (ddchi23 * oochipsipower) / chiguarded;
|
|
|
|
ddf33 =
|
|
(ddchi33 * oochipsipower) / chiguarded - chipsipower * pow2(df3);
|
|
|
|
cddf11 =
|
|
ddf11 - df1 * gamma111 - df2 * gamma211 - df3 * gamma311;
|
|
|
|
cddf12 =
|
|
ddf12 - df1 * gamma112 - df2 * gamma212 - df3 * gamma312;
|
|
|
|
cddf13 =
|
|
ddf13 - df1 * gamma113 - df2 * gamma213 - df3 * gamma313;
|
|
|
|
cddf22 =
|
|
ddf22 - df1 * gamma122 - df2 * gamma222 - df3 * gamma322;
|
|
|
|
cddf23 =
|
|
ddf23 - df1 * gamma123 - df2 * gamma223 - df3 * gamma323;
|
|
|
|
cddf33 =
|
|
ddf33 - df1 * gamma133 - df2 * gamma233 - df3 * gamma333;
|
|
|
|
trcddf =
|
|
cddf11 * ginv11 + cddf22 * ginv22 +
|
|
2. * (cddf12 * ginv12 + cddf13 * ginv13 + cddf23 * ginv23) + cddf33 * ginv33;
|
|
|
|
Rphi11 =
|
|
-2. * (cddf11 + g11 * trcddf) + (4. - 4. * g11 * ginv11) * pow2(df1) -
|
|
g11 * (8. * (df1 * (df2 * ginv12 + df3 * ginv13) + df2 * df3 * ginv23) +
|
|
4. * (ginv22 * pow2(df2) + ginv33 * pow2(df3)));
|
|
|
|
Rphi12 =
|
|
df1 * df2 * (4. - 8. * g12 * ginv12) - 2. * (cddf12 + g12 * trcddf) -
|
|
g12 * (8. * df3 * (df1 * ginv13 + df2 * ginv23) +
|
|
4. * (ginv11 * pow2(df1) + ginv22 * pow2(df2) + ginv33 * pow2(df3)));
|
|
|
|
Rphi13 =
|
|
df1 * (4. * df3 - 8. * df2 * g13 * ginv12) - 2. * (cddf13 + g13 * trcddf) -
|
|
g13 * (8. * df3 * (df1 * ginv13 + df2 * ginv23) +
|
|
4. * (ginv11 * pow2(df1) + ginv22 * pow2(df2) + ginv33 * pow2(df3)));
|
|
|
|
Rphi22 =
|
|
-2. * (cddf22 + g22 * trcddf) + (4. - 4. * g22 * ginv22) * pow2(df2) -
|
|
g22 * (8. * (df1 * (df2 * ginv12 + df3 * ginv13) + df2 * df3 * ginv23) +
|
|
4. * (ginv11 * pow2(df1) + ginv33 * pow2(df3)));
|
|
|
|
Rphi23 =
|
|
df2 * (-8. * df1 * g23 * ginv12 + df3 * (4. - 8. * g23 * ginv23)) -
|
|
2. * (cddf23 + g23 * trcddf) - g23 * (8. * df1 * df3 * ginv13 + 4. * (ginv11 * pow2(df1) + ginv22 * pow2(df2) + ginv33 * pow2(df3)));
|
|
|
|
Rphi33 =
|
|
-2. * (cddf33 + g33 * trcddf) - g33 * (8. * (df1 * (df2 * ginv12 + df3 * ginv13) + df2 * df3 * ginv23) + 4. * (ginv11 * pow2(df1) + ginv22 * pow2(df2))) +
|
|
(4. - 4. * g33 * ginv33) * pow2(df3);
|
|
|
|
cdda11 =
|
|
dda11 - da2 * gamma211 - da3 * gamma311 +
|
|
da1 * (-gamma111 + df1 * (-4. + 2. * g11 * ginv11)) +
|
|
2. * g11 * ((da2 * df1 + da1 * df2) * ginv12 + (da3 * df1 + da1 * df3) * ginv13 + da2 * df2 * ginv22 + (da3 * df2 + da2 * df3) * ginv23 + da3 * df3 * ginv33);
|
|
|
|
cdda12 =
|
|
dda12 - da1 * gamma112 - da2 * gamma212 - da3 * gamma312 +
|
|
2. * (-(da2 * df1) - da1 * df2 + g12 * (da1 * df1 * ginv11 + (da2 * df1 + da1 * df2) * ginv12 + (da3 * df1 + da1 * df3) * ginv13 + da2 * df2 * ginv22 + (da3 * df2 + da2 * df3) * ginv23 + da3 * df3 * ginv33));
|
|
|
|
cdda13 =
|
|
dda13 - da1 * gamma113 - da2 * gamma213 - da3 * gamma313 +
|
|
2. * (-(da3 * df1) - da1 * df3 + g13 * (da1 * df1 * ginv11 + (da2 * df1 + da1 * df2) * ginv12 + (da3 * df1 + da1 * df3) * ginv13 + da2 * df2 * ginv22 + (da3 * df2 + da2 * df3) * ginv23 + da3 * df3 * ginv33));
|
|
|
|
cdda22 =
|
|
dda22 - da1 * gamma122 - da2 * (4. * df2 + gamma222) - da3 * gamma322 +
|
|
2. * g22 * (da1 * df1 * ginv11 + (da2 * df1 + da1 * df2) * ginv12 + (da3 * df1 + da1 * df3) * ginv13 + da2 * df2 * ginv22 + (da3 * df2 + da2 * df3) * ginv23 + da3 * df3 * ginv33);
|
|
|
|
cdda23 =
|
|
dda23 - da1 * gamma123 - da2 * gamma223 - da3 * gamma323 +
|
|
2. * (-(da3 * df2) - da2 * df3 + g23 * (da1 * df1 * ginv11 + (da2 * df1 + da1 * df2) * ginv12 + (da3 * df1 + da1 * df3) * ginv13 + da2 * df2 * ginv22 + (da3 * df2 + da2 * df3) * ginv23 + da3 * df3 * ginv33));
|
|
|
|
cdda33 =
|
|
dda33 - da1 * gamma133 - da2 * gamma233 - da3 * (4. * df3 + gamma333) +
|
|
2. * g33 * (da1 * df1 * ginv11 + (da2 * df1 + da1 * df2) * ginv12 + (da3 * df1 + da1 * df3) * ginv13 + da2 * df2 * ginv22 + (da3 * df2 + da2 * df3) * ginv23 + da3 * df3 * ginv33);
|
|
|
|
trcdda =
|
|
(cdda11 * ginv11 + cdda22 * ginv22 +
|
|
2. * (cdda12 * ginv12 + cdda13 * ginv13 + cdda23 * ginv23) + cdda33 * ginv33) *
|
|
psim4;
|
|
|
|
AA11 =
|
|
2. * (A11 * (A12 * ginv12 + A13 * ginv13) + A12 * A13 * ginv23) + ginv11 * pow2(A11) +
|
|
ginv22 * pow2(A12) + ginv33 * pow2(A13);
|
|
|
|
AA12 =
|
|
(A12 * A13 + A11 * A23) * ginv13 + A12 * (A11 * ginv11 + A22 * ginv22) +
|
|
(A13 * A22 + A12 * A23) * ginv23 + A13 * A23 * ginv33 + ginv12 * (A11 * A22 + pow2(A12));
|
|
|
|
AA13 =
|
|
(A12 * A13 + A11 * A23) * ginv12 + A12 * A23 * ginv22 + (A13 * A23 + A12 * A33) * ginv23 +
|
|
A13 * (A11 * ginv11 + A33 * ginv33) + ginv13 * (A11 * A33 + pow2(A13));
|
|
|
|
AA22 =
|
|
2. * (A12 * (A22 * ginv12 + A23 * ginv13) + A22 * A23 * ginv23) + ginv11 * pow2(A12) +
|
|
ginv22 * pow2(A22) + ginv33 * pow2(A23);
|
|
|
|
AA23 =
|
|
A12 * A13 * ginv11 + (A13 * A22 + A12 * A23) * ginv12 + (A13 * A23 + A12 * A33) * ginv13 +
|
|
A23 * (A22 * ginv22 + A33 * ginv33) + ginv23 * (A22 * A33 + pow2(A23));
|
|
|
|
AA33 =
|
|
2. * (A13 * (A23 * ginv12 + A33 * ginv13) + A23 * A33 * ginv23) + ginv11 * pow2(A13) +
|
|
ginv22 * pow2(A23) + ginv33 * pow2(A33);
|
|
|
|
cAA =
|
|
AA11 * ginv11 + AA22 * ginv22 + 2. * (AA12 * ginv12 + AA13 * ginv13 + AA23 * ginv23) +
|
|
AA33 * ginv33;
|
|
|
|
Ainv11 =
|
|
2. * (A23 * ginv12 * ginv13 + ginv11 * (A12 * ginv12 + A13 * ginv13)) +
|
|
A11 * pow2(ginv11) + A22 * pow2(ginv12) + A33 * pow2(ginv13);
|
|
|
|
Ainv12 =
|
|
ginv11 * (A11 * ginv12 + A12 * ginv22 + A13 * ginv23) +
|
|
ginv12 * (A13 * ginv13 + A22 * ginv22 + A23 * ginv23) +
|
|
ginv13 * (A23 * ginv22 + A33 * ginv23) + A12 * pow2(ginv12);
|
|
|
|
Ainv13 =
|
|
ginv11 * (A11 * ginv13 + A12 * ginv23 + A13 * ginv33) +
|
|
ginv12 * (A12 * ginv13 + A22 * ginv23 + A23 * ginv33) +
|
|
ginv13 * (A23 * ginv23 + A33 * ginv33) + A13 * pow2(ginv13);
|
|
|
|
Ainv22 =
|
|
2. * (A23 * ginv22 * ginv23 + ginv12 * (A12 * ginv22 + A13 * ginv23)) +
|
|
A11 * pow2(ginv12) + A22 * pow2(ginv22) + A33 * pow2(ginv23);
|
|
|
|
Ainv23 =
|
|
ginv13 * (A12 * ginv22 + A13 * ginv23) + A33 * ginv23 * ginv33 +
|
|
ginv12 * (A11 * ginv13 + A12 * ginv23 + A13 * ginv33) +
|
|
ginv22 * (A22 * ginv23 + A23 * ginv33) + A23 * pow2(ginv23);
|
|
|
|
Ainv33 =
|
|
2. * (A23 * ginv23 * ginv33 + ginv13 * (A12 * ginv23 + A13 * ginv33)) +
|
|
A11 * pow2(ginv13) + A22 * pow2(ginv23) + A33 * pow2(ginv33);
|
|
|
|
divAinv1 =
|
|
(-1.5 * (Ainv11 * dchi1 + Ainv12 * dchi2 + Ainv13 * dchi3)) / chiguarded +
|
|
Ainv11 * gamma111 + Ainv22 * gamma122 +
|
|
2. * (Ainv12 * gamma112 + Ainv13 * gamma113 + Ainv23 * gamma123) +
|
|
Ainv33 * gamma133 - (0.66666666666666666667 * dKhat1 + 0.33333333333333333333 * dTheta1) * ginv11 -
|
|
(0.66666666666666666667 * dKhat2 + 0.33333333333333333333 * dTheta2) * ginv12 -
|
|
(0.66666666666666666667 * dKhat3 + 0.33333333333333333333 * dTheta3) * ginv13;
|
|
|
|
divAinv2 =
|
|
(-1.5 * (Ainv12 * dchi1 + Ainv22 * dchi2 + Ainv23 * dchi3)) / chiguarded +
|
|
Ainv11 * gamma211 + Ainv22 * gamma222 +
|
|
2. * (Ainv12 * gamma212 + Ainv13 * gamma213 + Ainv23 * gamma223) +
|
|
Ainv33 * gamma233 - (0.66666666666666666667 * dKhat1 + 0.33333333333333333333 * dTheta1) * ginv12 -
|
|
(0.66666666666666666667 * dKhat2 + 0.33333333333333333333 * dTheta2) * ginv22 -
|
|
(0.66666666666666666667 * dKhat3 + 0.33333333333333333333 * dTheta3) * ginv23;
|
|
|
|
divAinv3 =
|
|
(-1.5 * (Ainv13 * dchi1 + Ainv23 * dchi2 + Ainv33 * dchi3)) / chiguarded +
|
|
Ainv11 * gamma311 + Ainv22 * gamma322 +
|
|
2. * (Ainv12 * gamma312 + Ainv13 * gamma313 + Ainv23 * gamma323) +
|
|
Ainv33 * gamma333 - (0.66666666666666666667 * dKhat1 + 0.33333333333333333333 * dTheta1) * ginv13 -
|
|
(0.66666666666666666667 * dKhat2 + 0.33333333333333333333 * dTheta2) * ginv23 -
|
|
(0.66666666666666666667 * dKhat3 + 0.33333333333333333333 * dTheta3) * ginv33;
|
|
|
|
Rhat =
|
|
psim4 * (ginv11 * (R11 + Rphi11) + ginv22 * (R22 + Rphi22) +
|
|
2. * (ginv12 * (R12 + Rphi12) + ginv13 * (R13 + Rphi13) +
|
|
ginv23 * (R23 + Rphi23)) +
|
|
ginv33 * (R33 + Rphi33));
|
|
|
|
Hhat =
|
|
-cAA + Rhat + 0.66666666666666666667 * pow2(K);
|
|
|
|
divbeta =
|
|
db11 + db22 + db33;
|
|
|
|
totdivbeta =
|
|
0.66666666666666666667 * divbeta;
|
|
|
|
ootddivbeta1 =
|
|
0.33333333333333333333 * (ddb111 + ddb122 + ddb133);
|
|
|
|
ootddivbeta2 =
|
|
0.33333333333333333333 * (ddb121 + ddb222 + ddb233);
|
|
|
|
ootddivbeta3 =
|
|
0.33333333333333333333 * (ddb131 + ddb232 + ddb333);
|
|
|
|
lieg11 =
|
|
2. * (db11 * g11 + db12 * g12 + db13 * g13) - g11 * totdivbeta;
|
|
|
|
lieg12 =
|
|
db21 * g11 + db23 * g13 + db12 * g22 + db13 * g23 + g12 * (db11 + db22 - totdivbeta);
|
|
|
|
lieg13 =
|
|
db31 * g11 + db32 * g12 + db12 * g23 + db13 * g33 + g13 * (db11 + db33 - totdivbeta);
|
|
|
|
lieg22 =
|
|
2. * (db21 * g12 + db22 * g22 + db23 * g23) - g22 * totdivbeta;
|
|
|
|
lieg23 =
|
|
db31 * g12 + db21 * g13 + db32 * g22 + db23 * g33 + g23 * (db22 + db33 - totdivbeta);
|
|
|
|
lieg33 =
|
|
2. * (db31 * g13 + db32 * g23 + db33 * g33) - g33 * totdivbeta;
|
|
|
|
lieA11 =
|
|
2. * (A11 * db11 + A12 * db12 + A13 * db13) - A11 * totdivbeta;
|
|
|
|
lieA12 =
|
|
A22 * db12 + A23 * db13 + A11 * db21 + A13 * db23 + A12 * (db11 + db22 - totdivbeta);
|
|
|
|
lieA13 =
|
|
A23 * db12 + A33 * db13 + A11 * db31 + A12 * db32 + A13 * (db11 + db33 - totdivbeta);
|
|
|
|
lieA22 =
|
|
2. * (A12 * db21 + A22 * db22 + A23 * db23) - A22 * totdivbeta;
|
|
|
|
lieA23 =
|
|
A13 * db21 + A33 * db23 + A12 * db31 + A22 * db32 + A23 * (db22 + db33 - totdivbeta);
|
|
|
|
lieA33 =
|
|
2. * (A13 * db31 + A23 * db32 + A33 * db33) - A33 * totdivbeta;
|
|
|
|
liechi =
|
|
0.16666666666666666667 * chiguarded * chipsipower * divbeta;
|
|
|
|
pseudolieG1 =
|
|
-(db11 * Gfromg1) - db21 * Gfromg2 - db31 * Gfromg3 + ddb221 * ginv22 +
|
|
2. * ddb231 * ginv23 + ddb331 * ginv33 + ginv11 * (ddb111 + ootddivbeta1) +
|
|
ginv12 * (2. * ddb121 + ootddivbeta2) + ginv13 * (2. * ddb131 + ootddivbeta3) +
|
|
Gfromg1 * totdivbeta;
|
|
|
|
pseudolieG2 =
|
|
-(db12 * Gfromg1) - db22 * Gfromg2 - db32 * Gfromg3 + ddb112 * ginv11 +
|
|
2. * ddb132 * ginv13 + ddb332 * ginv33 + ginv12 * (2. * ddb122 + ootddivbeta1) +
|
|
ginv22 * (ddb222 + ootddivbeta2) + ginv23 * (2. * ddb232 + ootddivbeta3) +
|
|
Gfromg2 * totdivbeta;
|
|
|
|
pseudolieG3 =
|
|
-(db13 * Gfromg1) - db23 * Gfromg2 - db33 * Gfromg3 + ddb113 * ginv11 +
|
|
2. * ddb123 * ginv12 + ddb223 * ginv22 + ginv13 * (2. * ddb133 + ootddivbeta1) +
|
|
ginv23 * (2. * ddb233 + ootddivbeta2) + ginv33 * (ddb333 + ootddivbeta3) +
|
|
Gfromg3 * totdivbeta;
|
|
|
|
rg11 =
|
|
-2. * A11 * alpha + lieg11;
|
|
|
|
rg12 =
|
|
-2. * A12 * alpha + lieg12;
|
|
|
|
rg13 =
|
|
-2. * A13 * alpha + lieg13;
|
|
|
|
rg22 =
|
|
-2. * A22 * alpha + lieg22;
|
|
|
|
rg23 =
|
|
-2. * A23 * alpha + lieg23;
|
|
|
|
rg33 =
|
|
-2. * A33 * alpha + lieg33;
|
|
|
|
rA11 =
|
|
lieA11 + alpha * (-2. * AA11 + A11 * K + psim4 * R11 - 0.33333333333333333333 * g11 * Rhat) + psim4 * (-cdda11 + alpha * Rphi11) +
|
|
0.33333333333333333333 * g11 * trcdda;
|
|
|
|
rA12 =
|
|
lieA12 + alpha * (-2. * AA12 + A12 * K + psim4 * R12 - 0.33333333333333333333 * g12 * Rhat) + psim4 * (-cdda12 + alpha * Rphi12) +
|
|
0.33333333333333333333 * g12 * trcdda;
|
|
|
|
rA13 =
|
|
lieA13 + alpha * (-2. * AA13 + A13 * K + psim4 * R13 - 0.33333333333333333333 * g13 * Rhat) + psim4 * (-cdda13 + alpha * Rphi13) +
|
|
0.33333333333333333333 * g13 * trcdda;
|
|
|
|
rA22 =
|
|
lieA22 + alpha * (-2. * AA22 + A22 * K + psim4 * R22 - 0.33333333333333333333 * g22 * Rhat) + psim4 * (-cdda22 + alpha * Rphi22) +
|
|
0.33333333333333333333 * g22 * trcdda;
|
|
|
|
rA23 =
|
|
lieA23 + alpha * (-2. * AA23 + A23 * K + psim4 * R23 - 0.33333333333333333333 * g23 * Rhat) + psim4 * (-cdda23 + alpha * Rphi23) +
|
|
0.33333333333333333333 * g23 * trcdda;
|
|
|
|
rA33 =
|
|
lieA33 + alpha * (-2. * AA33 + A33 * K + psim4 * R33 - 0.33333333333333333333 * g33 * Rhat) + psim4 * (-cdda33 + alpha * Rphi33) +
|
|
0.33333333333333333333 * g33 * trcdda;
|
|
|
|
rG1 =
|
|
-2. * (Ainv11 * da1 + Ainv12 * da2 + Ainv13 * da3) +
|
|
alpha * (2. * divAinv1 + 2. * (-G1 + Gfromg1) * kappa1) + pseudolieG1;
|
|
|
|
rG2 =
|
|
-2. * (Ainv12 * da1 + Ainv22 * da2 + Ainv23 * da3) +
|
|
alpha * (2. * divAinv2 + 2. * (-G2 + Gfromg2) * kappa1) + pseudolieG2;
|
|
|
|
rG3 =
|
|
-2. * (Ainv13 * da1 + Ainv23 * da2 + Ainv33 * da3) +
|
|
alpha * (2. * divAinv3 + 2. * (-G3 + Gfromg3) * kappa1) + pseudolieG3;
|
|
|
|
rKhat =
|
|
-trcdda + alpha * (cAA + kappa1 * (Theta - kappa2 * Theta) +
|
|
0.33333333333333333333 * pow2(K));
|
|
|
|
rchi =
|
|
-0.16666666666666666667 * alpha * chiguarded * chipsipower * K + liechi;
|
|
|
|
rTheta =
|
|
alpha * (0.5 * Hhat - kappa1 * (2. + kappa2) * Theta);
|
|
}
|
|
|
|
|
|
#undef pow4
|
|
#undef pow3
|
|
#undef pow2
|
|
|
|
__device__ __forceinline__ void z4c_contract_gamma_device_host_order(
|
|
const double gxx, const double gxy, const double gxz,
|
|
const double gyy, const double gyz, const double gzz,
|
|
const double gxxx, const double gxyx, const double gxzx,
|
|
const double gyyx, const double gyzx, const double gzzx,
|
|
const double gxxy, const double gxyy, const double gxzy,
|
|
const double gyyy, const double gyzy, const double gzzy,
|
|
const double gxxz, const double gxyz, const double gxzz,
|
|
const double gyyz, const double gyzz, const double gzzz,
|
|
double &Gamxa, double &Gamya, double &Gamza)
|
|
{
|
|
double det = gxx * gyy * gzz + gxy * gyz * gxz + gxz * gxy * gyz -
|
|
gxz * gyy * gxz - gxy * gxy * gzz - gxx * gyz * gyz;
|
|
const double gupxx = (gyy * gzz - gyz * gyz) / det;
|
|
const double gupxy = -(gxy * gzz - gyz * gxz) / det;
|
|
const double gupxz = (gxy * gyz - gyy * gxz) / det;
|
|
const double gupyy = (gxx * gzz - gxz * gxz) / det;
|
|
const double gupyz = -(gxx * gyz - gxy * gxz) / det;
|
|
const double gupzz = (gxx * gyy - gxy * gxy) / det;
|
|
|
|
const double Gamxxx = 0.5 * (gupxx * gxxx + gupxy * (2.0 * gxyx - gxxy) + gupxz * (2.0 * gxzx - gxxz));
|
|
const double Gamyxx = 0.5 * (gupxy * gxxx + gupyy * (2.0 * gxyx - gxxy) + gupyz * (2.0 * gxzx - gxxz));
|
|
const double Gamzxx = 0.5 * (gupxz * gxxx + gupyz * (2.0 * gxyx - gxxy) + gupzz * (2.0 * gxzx - gxxz));
|
|
|
|
const double Gamxyy = 0.5 * (gupxx * (2.0 * gxyy - gyyx) + gupxy * gyyy + gupxz * (2.0 * gyzy - gyyz));
|
|
const double Gamyyy = 0.5 * (gupxy * (2.0 * gxyy - gyyx) + gupyy * gyyy + gupyz * (2.0 * gyzy - gyyz));
|
|
const double Gamzyy = 0.5 * (gupxz * (2.0 * gxyy - gyyx) + gupyz * gyyy + gupzz * (2.0 * gyzy - gyyz));
|
|
|
|
const double Gamxzz = 0.5 * (gupxx * (2.0 * gxzz - gzzx) + gupxy * (2.0 * gyzz - gzzy) + gupxz * gzzz);
|
|
const double Gamyzz = 0.5 * (gupxy * (2.0 * gxzz - gzzx) + gupyy * (2.0 * gyzz - gzzy) + gupyz * gzzz);
|
|
const double Gamzzz = 0.5 * (gupxz * (2.0 * gxzz - gzzx) + gupyz * (2.0 * gyzz - gzzy) + gupzz * gzzz);
|
|
|
|
const double Gamxxy = 0.5 * (gupxx * gxxy + gupxy * gyyx + gupxz * (gxzy + gyzx - gxyz));
|
|
const double Gamyxy = 0.5 * (gupxy * gxxy + gupyy * gyyx + gupyz * (gxzy + gyzx - gxyz));
|
|
const double Gamzxy = 0.5 * (gupxz * gxxy + gupyz * gyyx + gupzz * (gxzy + gyzx - gxyz));
|
|
|
|
const double Gamxxz = 0.5 * (gupxx * gxxz + gupxy * (gxyz + gyzx - gxzy) + gupxz * gzzx);
|
|
const double Gamyxz = 0.5 * (gupxy * gxxz + gupyy * (gxyz + gyzx - gxzy) + gupyz * gzzx);
|
|
const double Gamzxz = 0.5 * (gupxz * gxxz + gupyz * (gxyz + gyzx - gxzy) + gupzz * gzzx);
|
|
|
|
const double Gamxyz = 0.5 * (gupxx * (gxyz + gxzy - gyzx) + gupxy * gyyz + gupxz * gzzy);
|
|
const double Gamyyz = 0.5 * (gupxy * (gxyz + gxzy - gyzx) + gupyy * gyyz + gupyz * gzzy);
|
|
const double Gamzyz = 0.5 * (gupxz * (gxyz + gxzy - gyzx) + gupyz * gyyz + gupzz * gzzy);
|
|
|
|
Gamxa = gupxx * Gamxxx + gupyy * Gamxyy + gupzz * Gamxzz +
|
|
2.0 * (gupxy * Gamxxy + gupxz * Gamxxz + gupyz * Gamxyz);
|
|
Gamya = gupxx * Gamyxx + gupyy * Gamyyy + gupzz * Gamyzz +
|
|
2.0 * (gupxy * Gamyxy + gupxz * Gamyxz + gupyz * Gamyyz);
|
|
Gamza = gupxx * Gamzxx + gupyy * Gamzyy + gupzz * Gamzzz +
|
|
2.0 * (gupxy * Gamzxy + gupxz * Gamzxz + gupyz * Gamzyz);
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 2)
|
|
void kern_z4c_point_rhs(
|
|
const double *Axx, const double *Axy, const double *Axz,
|
|
const double *Ayy, const double *Ayz, const double *Azz,
|
|
const double *alpn1, const double *dtSfx, const double *dtSfy, const double *dtSfz,
|
|
const double *betax, const double *betay, const double *betaz,
|
|
const double *chin1,
|
|
const double *Lapx, const double *Lapy, const double *Lapz,
|
|
const double *Axxx, const double *Axyx, const double *Axzx, const double *Ayyx, const double *Ayzx, const double *Azzx,
|
|
const double *Axxy, const double *Axyy, const double *Axzy, const double *Ayyy, const double *Ayzy, const double *Azzy,
|
|
const double *Axxz, const double *Axyz, const double *Axzz, const double *Ayyz, const double *Ayzz, const double *Azzz,
|
|
const double *betaxx, const double *dBxx, const double *betayx, const double *dByx, const double *betazx, const double *dBzx,
|
|
const double *betaxy, const double *dBxy, const double *betayy, const double *dByy, const double *betazy, const double *dBzy,
|
|
const double *betaxz, const double *dBxz, const double *betayz, const double *dByz, const double *betazz, const double *dBzz,
|
|
const double *chix, const double *chiy, const double *chiz,
|
|
const double *Lapxx, const double *Lapxy, const double *Lapxz, const double *Lapyy, const double *Lapyz, const double *Lapzz,
|
|
const double *sfxxx, const double *sfyxx, const double *sfzxx,
|
|
const double *sfxxy, const double *sfyxy, const double *sfzxy,
|
|
const double *sfxxz, const double *sfyxz, const double *sfzxz,
|
|
const double *sfxyy, const double *sfyyy, const double *sfzyy,
|
|
const double *sfxyz, const double *sfyyz, const double *sfzyz,
|
|
const double *sfxzz, const double *sfyzz, const double *sfzzz,
|
|
const double *chixx, const double *chixy, const double *chixz, const double *chiyy, const double *chiyz, const double *chizz,
|
|
const double *gxxxx, const double *gxyxx, const double *gxzxx, const double *gyyxx, const double *gyzxx, const double *gzzxx,
|
|
const double *gxxxy, const double *gxyxy, const double *gxzxy, const double *gyyxy, const double *gyzxy, const double *gzzxy,
|
|
const double *gxxxz, const double *gxyxz, const double *gxzxz, const double *gyyxz, const double *gyzxz, const double *gzzxz,
|
|
const double *gxxyy, const double *gxyyy, const double *gxzyy, const double *gyyyy, const double *gyzyy, const double *gzzyy,
|
|
const double *gxxyz, const double *gxyyz, const double *gxzyz, const double *gyyyz, const double *gyzyz, const double *gzzyz,
|
|
const double *gxxzz, const double *gxyzz, const double *gxzzz, const double *gyyzz, const double *gyzzz, const double *gzzzz,
|
|
const double *Gamxx, const double *gxxx, const double *gxyx, const double *gxzx,
|
|
const double *Gamyx, const double *gyyx, const double *gyzx,
|
|
const double *Gamzx, const double *gzzx,
|
|
const double *Gamxy, const double *gxxy, const double *gxyy, const double *gxzy,
|
|
const double *Gamyy, const double *gyyy, const double *gyzy,
|
|
const double *Gamzy, const double *gzzy,
|
|
const double *Gamxz, const double *gxxz, const double *gxyz, const double *gxzz,
|
|
const double *Gamyz, const double *gyyz, const double *gyzz,
|
|
const double *Gamzz, const double *gzzz,
|
|
const double *Kx, const double *Ky, const double *Kz,
|
|
const double *TZx, const double *TZy, const double *TZz,
|
|
const double *Gamx, const double *gxx, const double *gxy, const double *gxz,
|
|
const double *Gamy, const double *gyy, const double *gyz,
|
|
const double *Gamz, const double *gzz, const double *trK, const double *TZ,
|
|
double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
|
|
double *chi_rhs,
|
|
double *Gamx_rhs, double *gxx_rhs, double *gxy_rhs, double *gxz_rhs,
|
|
double *Gamy_rhs, double *gyy_rhs, double *gyz_rhs,
|
|
double *Gamz_rhs, double *gzz_rhs, double *trK_rhs, double *TZ_rhs,
|
|
double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
|
|
double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs)
|
|
{
|
|
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < d_gp.all; idx += blockDim.x * gridDim.x) {
|
|
double point_kappa1 = 0.0;
|
|
double kappa2 = 0.0;
|
|
double chi_div_floor = 1.0e-5;
|
|
double rA11 = 0.0, rA12 = 0.0, rA13 = 0.0, rA22 = 0.0, rA23 = 0.0, rA33 = 0.0;
|
|
double rchi = 0.0, rG1 = 0.0, rg11 = 0.0, rg12 = 0.0, rg13 = 0.0;
|
|
double rG2 = 0.0, rg22 = 0.0, rg23 = 0.0, rG3 = 0.0, rg33 = 0.0, rKhat = 0.0, rTheta = 0.0;
|
|
|
|
z4c_rhs_point_device(
|
|
Axx[idx], Axy[idx], Axz[idx], Ayy[idx], Ayz[idx], Azz[idx],
|
|
alpn1[idx], dtSfx[idx], dtSfy[idx], dtSfz[idx],
|
|
betax[idx], betay[idx], betaz[idx],
|
|
chin1[idx], chi_div_floor,
|
|
Lapx[idx],
|
|
Axxx[idx], Axyx[idx], Axzx[idx], Ayyx[idx], Ayzx[idx], Azzx[idx],
|
|
Lapy[idx],
|
|
Axxy[idx], Axyy[idx], Axzy[idx], Ayyy[idx], Ayzy[idx], Azzy[idx],
|
|
Lapz[idx],
|
|
Axxz[idx], Axyz[idx], Axzz[idx], Ayyz[idx], Ayzz[idx], Azzz[idx],
|
|
betaxx[idx], dBxx[idx], betayx[idx], dByx[idx], betazx[idx], dBzx[idx],
|
|
betaxy[idx], dBxy[idx], betayy[idx], dByy[idx], betazy[idx], dBzy[idx],
|
|
betaxz[idx], dBxz[idx], betayz[idx], dByz[idx], betazz[idx], dBzz[idx],
|
|
chix[idx], chiy[idx], chiz[idx],
|
|
Lapxx[idx], Lapxy[idx], Lapxz[idx], Lapyy[idx], Lapyz[idx], Lapzz[idx],
|
|
sfxxx[idx], sfyxx[idx], sfzxx[idx],
|
|
sfxxy[idx], sfyxy[idx], sfzxy[idx],
|
|
sfxxz[idx], sfyxz[idx], sfzxz[idx],
|
|
sfxyy[idx], sfyyy[idx], sfzyy[idx],
|
|
sfxyz[idx], sfyyz[idx], sfzyz[idx],
|
|
sfxzz[idx], sfyzz[idx], sfzzz[idx],
|
|
chixx[idx], chixy[idx], chixz[idx], chiyy[idx], chiyz[idx], chizz[idx],
|
|
gxxxx[idx], gxyxx[idx], gxzxx[idx], gyyxx[idx], gyzxx[idx], gzzxx[idx],
|
|
gxxxy[idx], gxyxy[idx], gxzxy[idx], gyyxy[idx], gyzxy[idx], gzzxy[idx],
|
|
gxxxz[idx], gxyxz[idx], gxzxz[idx], gyyxz[idx], gyzxz[idx], gzzxz[idx],
|
|
gxxyy[idx], gxyyy[idx], gxzyy[idx], gyyyy[idx], gyzyy[idx], gzzyy[idx],
|
|
gxxyz[idx], gxyyz[idx], gxzyz[idx], gyyyz[idx], gyzyz[idx], gzzyz[idx],
|
|
gxxzz[idx], gxyzz[idx], gxzzz[idx], gyyzz[idx], gyzzz[idx], gzzzz[idx],
|
|
Gamxx[idx], gxxx[idx], gxyx[idx], gxzx[idx],
|
|
Gamyx[idx], gyyx[idx], gyzx[idx],
|
|
Gamzx[idx], gzzx[idx],
|
|
Gamxy[idx], gxxy[idx], gxyy[idx], gxzy[idx],
|
|
Gamyy[idx], gyyy[idx], gyzy[idx],
|
|
Gamzy[idx], gzzy[idx],
|
|
Gamxz[idx], gxxz[idx], gxyz[idx], gxzz[idx],
|
|
Gamyz[idx], gyyz[idx], gyzz[idx],
|
|
Gamzz[idx], gzzz[idx],
|
|
Kx[idx], Ky[idx], Kz[idx],
|
|
TZx[idx], TZy[idx], TZz[idx],
|
|
Gamx[idx], gxx[idx], gxy[idx], gxz[idx],
|
|
Gamy[idx], gyy[idx], gyz[idx],
|
|
Gamz[idx], gzz[idx],
|
|
point_kappa1, kappa2,
|
|
trK[idx],
|
|
rA11, rA12, rA13, rA22, rA23, rA33,
|
|
rchi,
|
|
rG1, rg11, rg12, rg13,
|
|
rG2, rg22, rg23,
|
|
rG3, rg33, rKhat, rTheta, TZ[idx]);
|
|
|
|
Axx_rhs[idx] = rA11;
|
|
Axy_rhs[idx] = rA12;
|
|
Axz_rhs[idx] = rA13;
|
|
Ayy_rhs[idx] = rA22;
|
|
Ayz_rhs[idx] = rA23;
|
|
Azz_rhs[idx] = rA33;
|
|
chi_rhs[idx] = rchi;
|
|
Gamx_rhs[idx] = rG1;
|
|
gxx_rhs[idx] = rg11;
|
|
gxy_rhs[idx] = rg12;
|
|
gxz_rhs[idx] = rg13;
|
|
Gamy_rhs[idx] = rG2;
|
|
gyy_rhs[idx] = rg22;
|
|
gyz_rhs[idx] = rg23;
|
|
Gamz_rhs[idx] = rG3;
|
|
gzz_rhs[idx] = rg33;
|
|
trK_rhs[idx] = rKhat;
|
|
TZ_rhs[idx] = rTheta;
|
|
|
|
Lap_rhs[idx] = -2.0 * alpn1[idx] * trK[idx];
|
|
#if (GAUGE == 0)
|
|
betax_rhs[idx] = 0.75 * dtSfx[idx];
|
|
betay_rhs[idx] = 0.75 * dtSfy[idx];
|
|
betaz_rhs[idx] = 0.75 * dtSfz[idx];
|
|
dtSfx_rhs[idx] = Gamx_rhs[idx] - 2.0 * dtSfx[idx];
|
|
dtSfy_rhs[idx] = Gamy_rhs[idx] - 2.0 * dtSfy[idx];
|
|
dtSfz_rhs[idx] = Gamz_rhs[idx] - 2.0 * dtSfz[idx];
|
|
#elif (GAUGE == 1)
|
|
betax_rhs[idx] = Gamx[idx] - 2.0 * betax[idx];
|
|
betay_rhs[idx] = Gamy[idx] - 2.0 * betay[idx];
|
|
betaz_rhs[idx] = Gamz[idx] - 2.0 * betaz[idx];
|
|
dtSfx_rhs[idx] = 0.0;
|
|
dtSfy_rhs[idx] = 0.0;
|
|
dtSfz_rhs[idx] = 0.0;
|
|
#else
|
|
#error "z4c_rhs_cuda.cu currently supports GAUGE == 0 or GAUGE == 1 for Z4C"
|
|
#endif
|
|
}
|
|
}
|
|
|
|
__global__ __launch_bounds__(128, 4)
|
|
void kern_z4c_damping(const double *alpn1,
|
|
const double *gxx, const double *gxy, const double *gxz,
|
|
const double *gyy, const double *gyz, const double *gzz,
|
|
const double *gxxx, const double *gxyx, const double *gxzx,
|
|
const double *gyyx, const double *gyzx, const double *gzzx,
|
|
const double *gxxy, const double *gxyy, const double *gxzy,
|
|
const double *gyyy, const double *gyzy, const double *gzzy,
|
|
const double *gxxz, const double *gxyz, const double *gxzz,
|
|
const double *gyyz, const double *gyzz, const double *gzzz,
|
|
const double *TZ, const double *Gamx, const double *Gamy, const double *Gamz,
|
|
double *TZ_rhs, double *trK_rhs,
|
|
double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs)
|
|
{
|
|
const double kappa1 = 2.0e-2;
|
|
const double kappa2 = 0.0;
|
|
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < d_gp.all; idx += blockDim.x * gridDim.x) {
|
|
double Gamxa = 0.0, Gamya = 0.0, Gamza = 0.0;
|
|
z4c_contract_gamma_device_host_order(
|
|
gxx[idx], gxy[idx], gxz[idx], gyy[idx], gyz[idx], gzz[idx],
|
|
gxxx[idx], gxyx[idx], gxzx[idx], gyyx[idx], gyzx[idx], gzzx[idx],
|
|
gxxy[idx], gxyy[idx], gxzy[idx], gyyy[idx], gyzy[idx], gzzy[idx],
|
|
gxxz[idx], gxyz[idx], gxzz[idx], gyyz[idx], gyzz[idx], gzzz[idx],
|
|
Gamxa, Gamya, Gamza);
|
|
const double a = alpn1[idx];
|
|
const double theta = TZ[idx];
|
|
TZ_rhs[idx] -= a * (2.0 + kappa2) * kappa1 * theta;
|
|
trK_rhs[idx] += a * kappa1 * (1.0 - kappa2) * theta;
|
|
Gamx_rhs[idx] -= 2.0 * a * kappa1 * (Gamx[idx] - Gamxa);
|
|
Gamy_rhs[idx] -= 2.0 * a * kappa1 * (Gamy[idx] - Gamya);
|
|
Gamz_rhs[idx] -= 2.0 * a * kappa1 * (Gamz[idx] - Gamza);
|
|
}
|
|
}
|
|
|
|
static void gpu_z4c_lopsided_kodis_batch(double eps_val, int all,
|
|
int do_lopsided, int do_kodis)
|
|
{
|
|
const int z4c_field_count =
|
|
#if (GAUGE == 0)
|
|
25;
|
|
#else
|
|
22;
|
|
#endif
|
|
const int adv_slots[25] = {
|
|
S_chi, S_trK, S_gxx, S_gxy, S_gxz, S_gyy, S_gyz, S_gzz,
|
|
S_Axx, S_Axy, S_Axz, S_Ayy, S_Ayz, S_Azz,
|
|
S_Gamx, S_Gamy, S_Gamz, S_Lap, S_betax, S_betay, S_betaz,
|
|
#if (GAUGE == 0)
|
|
S_dtSfx, S_dtSfy, S_dtSfz,
|
|
#endif
|
|
S_TZ
|
|
};
|
|
const int ko_slots[25] = {
|
|
S_chi, S_trK, S_gxx, S_gxy, S_gxz, S_gyy, S_gyz, S_gzz,
|
|
S_Axx, S_Axy, S_Axz, S_Ayy, S_Ayz, S_Azz,
|
|
S_Gamx, S_Gamy, S_Gamz, S_Lap, S_betax, S_betay, S_betaz,
|
|
#if (GAUGE == 0)
|
|
S_dtSfx, S_dtSfy, S_dtSfz,
|
|
#endif
|
|
S_TZ
|
|
};
|
|
const int rhs_slots[25] = {
|
|
S_chi_rhs, S_trK_rhs, S_gxx_rhs, S_gxy_rhs, S_gxz_rhs, S_gyy_rhs, S_gyz_rhs, S_gzz_rhs,
|
|
S_Axx_rhs, S_Axy_rhs, S_Axz_rhs, S_Ayy_rhs, S_Ayz_rhs, S_Azz_rhs,
|
|
S_Gamx_rhs, S_Gamy_rhs, S_Gamz_rhs, S_Lap_rhs, S_betax_rhs, S_betay_rhs, S_betaz_rhs,
|
|
#if (GAUGE == 0)
|
|
S_dtSfx_rhs, S_dtSfy_rhs, S_dtSfz_rhs,
|
|
#endif
|
|
S_TZ_rhs
|
|
};
|
|
const int soa_signs[3 * 25] = {
|
|
1, 1, 1,
|
|
1, 1, 1,
|
|
1, 1, 1,
|
|
-1, -1, 1,
|
|
-1, 1, -1,
|
|
1, 1, 1,
|
|
1, -1, -1,
|
|
1, 1, 1,
|
|
1, 1, 1,
|
|
-1, -1, 1,
|
|
-1, 1, -1,
|
|
1, 1, 1,
|
|
1, -1, -1,
|
|
1, 1, 1,
|
|
-1, 1, 1,
|
|
1, -1, 1,
|
|
1, 1, -1,
|
|
1, 1, 1,
|
|
-1, 1, 1,
|
|
1, -1, 1,
|
|
1, 1, -1,
|
|
#if (GAUGE == 0)
|
|
-1, 1, 1,
|
|
1, -1, 1,
|
|
1, 1, -1,
|
|
#endif
|
|
1, 1, 1
|
|
};
|
|
LopsidedKodisTables tables = {};
|
|
for (int i = 0; i < z4c_field_count; ++i) {
|
|
tables.adv_fields[i] = g_buf.slot[adv_slots[i]];
|
|
tables.ko_fields[i] = g_buf.slot[ko_slots[i]];
|
|
tables.rhs_fields[i] = g_buf.slot[rhs_slots[i]];
|
|
tables.soa_signs[3 * i + 0] = soa_signs[3 * i + 0];
|
|
tables.soa_signs[3 * i + 1] = soa_signs[3 * i + 1];
|
|
tables.soa_signs[3 * i + 2] = soa_signs[3 * i + 2];
|
|
}
|
|
dim3 launch_grid((unsigned int)grid((size_t)all), (unsigned int)z4c_field_count);
|
|
kern_lopsided_kodis_batched<<<launch_grid, BLK>>>(
|
|
g_buf.slot[S_betax], g_buf.slot[S_betay], g_buf.slot[S_betaz],
|
|
tables, eps_val, do_lopsided, do_kodis);
|
|
}
|
|
|
|
static void launch_z4c_rhs_pipeline(int all, double eps)
|
|
{
|
|
const int SYM = 1;
|
|
const int ANTI = -1;
|
|
#define D(s) g_buf.slot[s]
|
|
kern_phase1_prep<<<grid(all), BLK>>>(
|
|
D(S_Lap), D(S_chi), D(S_dxx), D(S_dyy), D(S_dzz),
|
|
D(S_alpn1), D(S_chin1), D(S_gxx), D(S_gyy), D(S_gzz));
|
|
|
|
{
|
|
double *src_fields[] = {
|
|
D(S_betax), D(S_betay), D(S_betaz),
|
|
D(S_dtSfx), D(S_dtSfy), D(S_dtSfz),
|
|
D(S_chi), D(S_dxx), D(S_gxy), D(S_gxz), D(S_dyy), D(S_gyz), D(S_dzz),
|
|
D(S_Gamx), D(S_Gamy), D(S_Gamz), D(S_Lap), D(S_trK), D(S_TZ),
|
|
D(S_Axx), D(S_Axy), D(S_Axz), D(S_Ayy), D(S_Ayz), D(S_Azz)
|
|
};
|
|
double *fx_fields[] = {
|
|
D(S_betaxx), D(S_betayx), D(S_betazx),
|
|
D(S_dBxx), D(S_dByx), D(S_dBzx),
|
|
D(S_chix), D(S_gxxx), D(S_gxyx), D(S_gxzx), D(S_gyyx), D(S_gyzx), D(S_gzzx),
|
|
D(S_Gamxx), D(S_Gamyx), D(S_Gamzx), D(S_Lapx), D(S_Kx), D(S_TZx),
|
|
D(S_Axxx), D(S_Axyx), D(S_Axzx), D(S_Ayyx), D(S_Ayzx), D(S_Azzx)
|
|
};
|
|
double *fy_fields[] = {
|
|
D(S_betaxy), D(S_betayy), D(S_betazy),
|
|
D(S_dBxy), D(S_dByy), D(S_dBzy),
|
|
D(S_chiy), D(S_gxxy), D(S_gxyy), D(S_gxzy), D(S_gyyy), D(S_gyzy), D(S_gzzy),
|
|
D(S_Gamxy), D(S_Gamyy_t), D(S_Gamzy), D(S_Lapy), D(S_Ky), D(S_TZy),
|
|
D(S_Axxy), D(S_Axyy), D(S_Axzy), D(S_Ayyy), D(S_Ayzy), D(S_Azzy)
|
|
};
|
|
double *fz_fields[] = {
|
|
D(S_betaxz), D(S_betayz), D(S_betazz),
|
|
D(S_dBxz), D(S_dByz), D(S_dBzz),
|
|
D(S_chiz), D(S_gxxz), D(S_gxyz), D(S_gxzz), D(S_gyyz), D(S_gyzz), D(S_gzzz),
|
|
D(S_Gamxz), D(S_Gamyz_t), D(S_Gamzz_t), D(S_Lapz), D(S_Kz), D(S_TZz),
|
|
D(S_Axxz), D(S_Axyz), D(S_Axzz), D(S_Ayyz), D(S_Ayzz), D(S_Azzz)
|
|
};
|
|
const int soa_signs[] = {
|
|
ANTI, SYM, SYM, SYM, ANTI, SYM, SYM, SYM, ANTI,
|
|
ANTI, SYM, SYM, SYM, ANTI, SYM, SYM, SYM, ANTI,
|
|
SYM, SYM, SYM,
|
|
SYM, SYM, SYM, ANTI, ANTI, SYM, ANTI, SYM, ANTI,
|
|
SYM, SYM, SYM, SYM, ANTI, ANTI, SYM, SYM, SYM,
|
|
ANTI, SYM, SYM, SYM, ANTI, SYM, SYM, SYM, ANTI,
|
|
SYM, SYM, SYM, SYM, SYM, SYM, SYM, SYM, SYM,
|
|
SYM, SYM, SYM, ANTI, ANTI, SYM, ANTI, SYM, ANTI,
|
|
SYM, SYM, SYM, SYM, ANTI, ANTI, SYM, SYM, SYM
|
|
};
|
|
gpu_fderivs_batch((int)(sizeof(src_fields) / sizeof(src_fields[0])),
|
|
src_fields, fx_fields, fy_fields, fz_fields, soa_signs, all);
|
|
}
|
|
|
|
{
|
|
double *src_fields[] = {
|
|
D(S_dxx), D(S_dyy), D(S_dzz), D(S_gxy), D(S_gxz), D(S_gyz),
|
|
D(S_betax), D(S_betay), D(S_betaz), D(S_chi), D(S_Lap)
|
|
};
|
|
double *fxx_fields[] = {
|
|
D(S_gxxxx), D(S_gyyxx), D(S_gzzxx), D(S_gxyxx), D(S_gxzxx), D(S_gyzxx),
|
|
D(S_sfxxx), D(S_sfyxx), D(S_sfzxx), D(S_chixx), D(S_Lapxx)
|
|
};
|
|
double *fxy_fields[] = {
|
|
D(S_gxxxy), D(S_gyyxy), D(S_gzzxy), D(S_gxyxy), D(S_gxzxy), D(S_gyzxy),
|
|
D(S_sfxxy), D(S_sfyxy), D(S_sfzxy), D(S_chixy), D(S_Lapxy)
|
|
};
|
|
double *fxz_fields[] = {
|
|
D(S_gxxxz), D(S_gyyxz), D(S_gzzxz), D(S_gxyxz), D(S_gxzxz), D(S_gyzxz),
|
|
D(S_sfxxz), D(S_sfyxz), D(S_sfzxz), D(S_chixz), D(S_Lapxz)
|
|
};
|
|
double *fyy_fields[] = {
|
|
D(S_gxxyy), D(S_gyyyy), D(S_gzzyy), D(S_gxyyy), D(S_gxzyy), D(S_gyzyy),
|
|
D(S_sfxyy), D(S_sfyyy), D(S_sfzyy), D(S_chiyy), D(S_Lapyy)
|
|
};
|
|
double *fyz_fields[] = {
|
|
D(S_gxxyz), D(S_gyyyz), D(S_gzzyz), D(S_gxyyz), D(S_gxzyz), D(S_gyzyz),
|
|
D(S_sfxyz), D(S_sfyyz), D(S_sfzyz), D(S_chiyz), D(S_Lapyz)
|
|
};
|
|
double *fzz_fields[] = {
|
|
D(S_gxxzz), D(S_gyyzz), D(S_gzzzz), D(S_gxyzz), D(S_gxzzz), D(S_gyzzz),
|
|
D(S_sfxzz), D(S_sfyzz), D(S_sfzzz), D(S_chizz), D(S_Lapzz)
|
|
};
|
|
const int soa_signs[] = {
|
|
SYM, SYM, SYM, SYM, SYM, SYM, SYM, SYM, SYM,
|
|
ANTI, ANTI, SYM, ANTI, SYM, ANTI, SYM, ANTI, ANTI,
|
|
ANTI, SYM, SYM, SYM, ANTI, SYM, SYM, SYM, ANTI,
|
|
SYM, SYM, SYM, SYM, SYM, SYM
|
|
};
|
|
gpu_fdderivs_batch((int)(sizeof(src_fields) / sizeof(src_fields[0])),
|
|
src_fields, fxx_fields, fxy_fields, fxz_fields,
|
|
fyy_fields, fyz_fields, fzz_fields, soa_signs, all);
|
|
}
|
|
|
|
kern_z4c_point_rhs<<<grid(all), BLK>>>(
|
|
D(S_Axx), D(S_Axy), D(S_Axz), D(S_Ayy), D(S_Ayz), D(S_Azz),
|
|
D(S_alpn1), D(S_dtSfx), D(S_dtSfy), D(S_dtSfz),
|
|
D(S_betax), D(S_betay), D(S_betaz), D(S_chin1),
|
|
D(S_Lapx), D(S_Lapy), D(S_Lapz),
|
|
D(S_Axxx), D(S_Axyx), D(S_Axzx), D(S_Ayyx), D(S_Ayzx), D(S_Azzx),
|
|
D(S_Axxy), D(S_Axyy), D(S_Axzy), D(S_Ayyy), D(S_Ayzy), D(S_Azzy),
|
|
D(S_Axxz), D(S_Axyz), D(S_Axzz), D(S_Ayyz), D(S_Ayzz), D(S_Azzz),
|
|
D(S_betaxx), D(S_dBxx), D(S_betayx), D(S_dByx), D(S_betazx), D(S_dBzx),
|
|
D(S_betaxy), D(S_dBxy), D(S_betayy), D(S_dByy), D(S_betazy), D(S_dBzy),
|
|
D(S_betaxz), D(S_dBxz), D(S_betayz), D(S_dByz), D(S_betazz), D(S_dBzz),
|
|
D(S_chix), D(S_chiy), D(S_chiz),
|
|
D(S_Lapxx), D(S_Lapxy), D(S_Lapxz), D(S_Lapyy), D(S_Lapyz), D(S_Lapzz),
|
|
D(S_sfxxx), D(S_sfyxx), D(S_sfzxx), D(S_sfxxy), D(S_sfyxy), D(S_sfzxy),
|
|
D(S_sfxxz), D(S_sfyxz), D(S_sfzxz), D(S_sfxyy), D(S_sfyyy), D(S_sfzyy),
|
|
D(S_sfxyz), D(S_sfyyz), D(S_sfzyz), D(S_sfxzz), D(S_sfyzz), D(S_sfzzz),
|
|
D(S_chixx), D(S_chixy), D(S_chixz), D(S_chiyy), D(S_chiyz), D(S_chizz),
|
|
D(S_gxxxx), D(S_gxyxx), D(S_gxzxx), D(S_gyyxx), D(S_gyzxx), D(S_gzzxx),
|
|
D(S_gxxxy), D(S_gxyxy), D(S_gxzxy), D(S_gyyxy), D(S_gyzxy), D(S_gzzxy),
|
|
D(S_gxxxz), D(S_gxyxz), D(S_gxzxz), D(S_gyyxz), D(S_gyzxz), D(S_gzzxz),
|
|
D(S_gxxyy), D(S_gxyyy), D(S_gxzyy), D(S_gyyyy), D(S_gyzyy), D(S_gzzyy),
|
|
D(S_gxxyz), D(S_gxyyz), D(S_gxzyz), D(S_gyyyz), D(S_gyzyz), D(S_gzzyz),
|
|
D(S_gxxzz), D(S_gxyzz), D(S_gxzzz), D(S_gyyzz), D(S_gyzzz), D(S_gzzzz),
|
|
D(S_Gamxx), D(S_gxxx), D(S_gxyx), D(S_gxzx), D(S_Gamyx), D(S_gyyx), D(S_gyzx), D(S_Gamzx), D(S_gzzx),
|
|
D(S_Gamxy), D(S_gxxy), D(S_gxyy), D(S_gxzy), D(S_Gamyy_t), D(S_gyyy), D(S_gyzy), D(S_Gamzy), D(S_gzzy),
|
|
D(S_Gamxz), D(S_gxxz), D(S_gxyz), D(S_gxzz), D(S_Gamyz_t), D(S_gyyz), D(S_gyzz), D(S_Gamzz_t), D(S_gzzz),
|
|
D(S_Kx), D(S_Ky), D(S_Kz), D(S_TZx), D(S_TZy), D(S_TZz),
|
|
D(S_Gamx), D(S_gxx), D(S_gxy), D(S_gxz), D(S_Gamy), D(S_gyy), D(S_gyz), D(S_Gamz), D(S_gzz), D(S_trK), D(S_TZ),
|
|
D(S_Axx_rhs), D(S_Axy_rhs), D(S_Axz_rhs), D(S_Ayy_rhs), D(S_Ayz_rhs), D(S_Azz_rhs),
|
|
D(S_chi_rhs), D(S_Gamx_rhs), D(S_gxx_rhs), D(S_gxy_rhs), D(S_gxz_rhs),
|
|
D(S_Gamy_rhs), D(S_gyy_rhs), D(S_gyz_rhs), D(S_Gamz_rhs), D(S_gzz_rhs), D(S_trK_rhs), D(S_TZ_rhs),
|
|
D(S_Lap_rhs), D(S_betax_rhs), D(S_betay_rhs), D(S_betaz_rhs), D(S_dtSfx_rhs), D(S_dtSfy_rhs), D(S_dtSfz_rhs));
|
|
|
|
gpu_z4c_lopsided_kodis_batch(0.0, all, 1, 0);
|
|
|
|
kern_z4c_damping<<<grid(all), BLK>>>(
|
|
D(S_alpn1), D(S_gxx), D(S_gxy), D(S_gxz), D(S_gyy), D(S_gyz), D(S_gzz),
|
|
D(S_gxxx), D(S_gxyx), D(S_gxzx), D(S_gyyx), D(S_gyzx), D(S_gzzx),
|
|
D(S_gxxy), D(S_gxyy), D(S_gxzy), D(S_gyyy), D(S_gyzy), D(S_gzzy),
|
|
D(S_gxxz), D(S_gxyz), D(S_gxzz), D(S_gyyz), D(S_gyzz), D(S_gzzz),
|
|
D(S_TZ), D(S_Gamx), D(S_Gamy), D(S_Gamz),
|
|
D(S_TZ_rhs), D(S_trK_rhs), D(S_Gamx_rhs), D(S_Gamy_rhs), D(S_Gamz_rhs));
|
|
|
|
gpu_z4c_lopsided_kodis_batch(eps, all, 0, 1);
|
|
#undef D
|
|
}
|
|
|
|
} // namespace z4c_cuda
|
|
|
|
extern "C" void f_constraint_bssn(int *, double *, double *, double *,
|
|
double *, double *,
|
|
double *, double *, double *, double *, double *, double *,
|
|
double *, double *, double *, double *, double *, double *,
|
|
double *, double *, double *,
|
|
double *, double *, double *, double *, double *, double *, double *, double *, double *, double *,
|
|
double *, double *, double *, double *, double *, double *,
|
|
double *, double *, double *, double *, double *, double *,
|
|
double *, double *, double *, double *, double *, double *,
|
|
double *, double *, double *, double *, double *, double *, double *, double *,
|
|
double *, double *, double *,
|
|
int &);
|
|
|
|
static int compute_rhs_z4c_cuda_host(
|
|
int *ex, double &T, double *X, double *Y, double *Z,
|
|
double *chi_state, double *chi_constraints, double *trK,
|
|
double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
|
|
double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
|
|
double *Gamx, double *Gamy, double *Gamz,
|
|
double *Lap, double *betax, double *betay, double *betaz,
|
|
double *dtSfx, double *dtSfy, double *dtSfz,
|
|
double *TZ,
|
|
double *chi_rhs, double *trK_rhs,
|
|
double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
|
|
double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
|
|
double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
|
|
double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
|
|
double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
|
|
double *TZ_rhs,
|
|
double *rho, double *Sx, double *Sy, double *Sz,
|
|
double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
|
|
double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
|
|
double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
|
|
double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
|
|
double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
|
|
double *Hcon, double *Mxcon, double *Mycon, double *Mzcon, double *Gmxcon, double *Gmycon, double *Gmzcon,
|
|
int &Symmetry, int &Lev, double &eps, int &co)
|
|
{
|
|
(void)T;
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
|
|
const int nx = ex[0], ny = ex[1], nz = ex[2];
|
|
const int all = nx * ny * nz;
|
|
const size_t bytes = (size_t)all * sizeof(double);
|
|
setup_grid_params(ex, X, Y, Z, Symmetry, eps, co);
|
|
|
|
#define D(s) g_buf.slot[s]
|
|
double *h2d_src[] = {
|
|
chi_state, trK, dxx, gxy, gxz, dyy, gyz, dzz,
|
|
Axx, Axy, Axz, Ayy, Ayz, Azz,
|
|
Gamx, Gamy, Gamz,
|
|
Lap, betax, betay, betaz,
|
|
dtSfx, dtSfy, dtSfz, TZ,
|
|
rho, Sx, Sy, Sz,
|
|
Sxx, Sxy, Sxz, Syy, Syz, Szz
|
|
};
|
|
const int h2d_count = (int)(sizeof(h2d_src) / sizeof(h2d_src[0]));
|
|
for (int s = 0; s < h2d_count; ++s) {
|
|
std::memcpy(g_buf.h_stage + (size_t)s * all, h2d_src[s], bytes);
|
|
}
|
|
CUDA_CHECK(cudaMemcpy(D(S_chi), g_buf.h_stage, (size_t)h2d_count * bytes, cudaMemcpyHostToDevice));
|
|
|
|
launch_z4c_rhs_pipeline(all, eps);
|
|
|
|
const int rhs_slots[] = {
|
|
S_chi_rhs, S_trK_rhs,
|
|
S_gxx_rhs, S_gxy_rhs, S_gxz_rhs, S_gyy_rhs, S_gyz_rhs, S_gzz_rhs,
|
|
S_Axx_rhs, S_Axy_rhs, S_Axz_rhs, S_Ayy_rhs, S_Ayz_rhs, S_Azz_rhs,
|
|
S_Gamx_rhs, S_Gamy_rhs, S_Gamz_rhs,
|
|
S_Lap_rhs, S_betax_rhs, S_betay_rhs, S_betaz_rhs,
|
|
S_dtSfx_rhs, S_dtSfy_rhs, S_dtSfz_rhs, S_TZ_rhs
|
|
};
|
|
double *rhs_dst[] = {
|
|
chi_rhs, trK_rhs,
|
|
gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
|
|
Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
|
|
Gamx_rhs, Gamy_rhs, Gamz_rhs,
|
|
Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
|
|
dtSfx_rhs, dtSfy_rhs, dtSfz_rhs, TZ_rhs
|
|
};
|
|
const int rhs_count = (int)(sizeof(rhs_slots) / sizeof(rhs_slots[0]));
|
|
for (int i = 0; i < rhs_count; ++i) {
|
|
CUDA_CHECK(cudaMemcpy(g_buf.h_stage + (size_t)i * all, D(rhs_slots[i]), bytes, cudaMemcpyDeviceToHost));
|
|
}
|
|
for (int i = 0; i < rhs_count; ++i) {
|
|
std::memcpy(rhs_dst[i], g_buf.h_stage + (size_t)i * all, bytes);
|
|
}
|
|
#undef D
|
|
|
|
if (co == 0) {
|
|
#if (ABV == 0)
|
|
f_ricci_gamma(ex, X, Y, Z,
|
|
chi_constraints,
|
|
dxx, gxy, gxz, dyy, gyz, dzz,
|
|
Gamx, Gamy, Gamz,
|
|
Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
|
|
Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
|
|
Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
|
|
Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
|
|
Symmetry);
|
|
#endif
|
|
f_constraint_bssn(ex, X, Y, Z,
|
|
chi_constraints, trK,
|
|
dxx, gxy, gxz, dyy, gyz, dzz,
|
|
Axx, Axy, Axz, Ayy, Ayz, Azz,
|
|
Gamx, Gamy, Gamz,
|
|
Lap, betax, betay, betaz, rho, Sx, Sy, Sz,
|
|
Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
|
|
Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
|
|
Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
|
|
Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
|
|
Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
|
|
Symmetry);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int f_compute_rhs_Z4c(int *ex, double &T,
|
|
double *X, double *Y, double *Z,
|
|
double *chi, double *trK,
|
|
double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
|
|
double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
|
|
double *Gamx, double *Gamy, double *Gamz,
|
|
double *Lap, double *betax, double *betay, double *betaz,
|
|
double *dtSfx, double *dtSfy, double *dtSfz,
|
|
double *TZ,
|
|
double *chi_rhs, double *trK_rhs,
|
|
double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
|
|
double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
|
|
double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
|
|
double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
|
|
double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
|
|
double *TZ_rhs,
|
|
double *rho, double *Sx, double *Sy, double *Sz,
|
|
double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
|
|
double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
|
|
double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
|
|
double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
|
|
double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
|
|
double *Hcon, double *Mxcon, double *Mycon, double *Mzcon, double *Gmxcon, double *Gmycon, double *Gmzcon,
|
|
int &Symmetry, int &Lev, double &eps, int &co)
|
|
{
|
|
return compute_rhs_z4c_cuda_host(
|
|
ex, T, X, Y, Z,
|
|
chi, chi, trK,
|
|
dxx, gxy, gxz, dyy, gyz, dzz,
|
|
Axx, Axy, Axz, Ayy, Ayz, Azz,
|
|
Gamx, Gamy, Gamz,
|
|
Lap, betax, betay, betaz,
|
|
dtSfx, dtSfy, dtSfz,
|
|
TZ,
|
|
chi_rhs, trK_rhs,
|
|
gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
|
|
Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
|
|
Gamx_rhs, Gamy_rhs, Gamz_rhs,
|
|
Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
|
|
dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
|
|
TZ_rhs,
|
|
rho, Sx, Sy, Sz,
|
|
Sxx, Sxy, Sxz, Syy, Syz, Szz,
|
|
Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
|
|
Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
|
|
Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
|
|
Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
|
|
Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
|
|
Symmetry, Lev, eps, co);
|
|
}
|
|
|
|
extern "C" int f_compute_rhs_Z4cnot(int *ex, double &T,
|
|
double *X, double *Y, double *Z,
|
|
double *chi, double *trK,
|
|
double *dxx, double *gxy, double *gxz, double *dyy, double *gyz, double *dzz,
|
|
double *Axx, double *Axy, double *Axz, double *Ayy, double *Ayz, double *Azz,
|
|
double *Gamx, double *Gamy, double *Gamz,
|
|
double *Lap, double *betax, double *betay, double *betaz,
|
|
double *dtSfx, double *dtSfy, double *dtSfz,
|
|
double *TZ,
|
|
double *chi_rhs, double *trK_rhs,
|
|
double *gxx_rhs, double *gxy_rhs, double *gxz_rhs, double *gyy_rhs, double *gyz_rhs, double *gzz_rhs,
|
|
double *Axx_rhs, double *Axy_rhs, double *Axz_rhs, double *Ayy_rhs, double *Ayz_rhs, double *Azz_rhs,
|
|
double *Gamx_rhs, double *Gamy_rhs, double *Gamz_rhs,
|
|
double *Lap_rhs, double *betax_rhs, double *betay_rhs, double *betaz_rhs,
|
|
double *dtSfx_rhs, double *dtSfy_rhs, double *dtSfz_rhs,
|
|
double *TZ_rhs,
|
|
double *rho, double *Sx, double *Sy, double *Sz,
|
|
double *Sxx, double *Sxy, double *Sxz, double *Syy, double *Syz, double *Szz,
|
|
double *Gamxxx, double *Gamxxy, double *Gamxxz, double *Gamxyy, double *Gamxyz, double *Gamxzz,
|
|
double *Gamyxx, double *Gamyxy, double *Gamyxz, double *Gamyyy, double *Gamyyz, double *Gamyzz,
|
|
double *Gamzxx, double *Gamzxy, double *Gamzxz, double *Gamzyy, double *Gamzyz, double *Gamzzz,
|
|
double *Rxx, double *Rxy, double *Rxz, double *Ryy, double *Ryz, double *Rzz,
|
|
double *Hcon, double *Mxcon, double *Mycon, double *Mzcon, double *Gmxcon, double *Gmycon, double *Gmzcon,
|
|
int &Symmetry, int &Lev, double &eps, int &co, double &chitiny)
|
|
{
|
|
const int all = ex[0] * ex[1] * ex[2];
|
|
std::vector<double> chi_clamped(chi, chi + all);
|
|
for (int i = 0; i < all; ++i) {
|
|
if (chi_clamped[i] < chitiny) chi_clamped[i] = chitiny;
|
|
}
|
|
return compute_rhs_z4c_cuda_host(
|
|
ex, T, X, Y, Z,
|
|
chi_clamped.data(), chi, trK,
|
|
dxx, gxy, gxz, dyy, gyz, dzz,
|
|
Axx, Axy, Axz, Ayy, Ayz, Azz,
|
|
Gamx, Gamy, Gamz,
|
|
Lap, betax, betay, betaz,
|
|
dtSfx, dtSfy, dtSfz,
|
|
TZ,
|
|
chi_rhs, trK_rhs,
|
|
gxx_rhs, gxy_rhs, gxz_rhs, gyy_rhs, gyz_rhs, gzz_rhs,
|
|
Axx_rhs, Axy_rhs, Axz_rhs, Ayy_rhs, Ayz_rhs, Azz_rhs,
|
|
Gamx_rhs, Gamy_rhs, Gamz_rhs,
|
|
Lap_rhs, betax_rhs, betay_rhs, betaz_rhs,
|
|
dtSfx_rhs, dtSfy_rhs, dtSfz_rhs,
|
|
TZ_rhs,
|
|
rho, Sx, Sy, Sz,
|
|
Sxx, Sxy, Sxz, Syy, Syz, Szz,
|
|
Gamxxx, Gamxxy, Gamxxz, Gamxyy, Gamxyz, Gamxzz,
|
|
Gamyxx, Gamyxy, Gamyxz, Gamyyy, Gamyyz, Gamyzz,
|
|
Gamzxx, Gamzxy, Gamzxz, Gamzyy, Gamzyz, Gamzzz,
|
|
Rxx, Rxy, Rxz, Ryy, Ryz, Rzz,
|
|
Hcon, Mxcon, Mycon, Mzcon, Gmxcon, Gmycon, Gmzcon,
|
|
Symmetry, Lev, eps, co);
|
|
}
|
|
|
|
extern "C" int z4c_cuda_rk4_substep(void *block_tag,
|
|
int *ex, double *X, double *Y, double *Z,
|
|
double **state_host_in,
|
|
double **state_host_out,
|
|
const double *propspeed,
|
|
const double *soa_flat,
|
|
const double *bbox,
|
|
double &dT,
|
|
double &T,
|
|
int &RK4,
|
|
int &apply_bam_bc,
|
|
int &Symmetry,
|
|
int &Lev,
|
|
double &eps,
|
|
int &co,
|
|
int &keep_resident_state,
|
|
int &apply_enforce_ga,
|
|
double &chitiny)
|
|
{
|
|
(void)T;
|
|
(void)state_host_out;
|
|
|
|
if (RK4 < 0 || RK4 > 3) return 1;
|
|
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
|
|
const bool profile = cuda_profile_enabled();
|
|
const double t_total0 = profile ? cuda_profile_now_ms() : 0.0;
|
|
double state_ms = 0.0;
|
|
double rhs_ms = 0.0;
|
|
double bc_ms = 0.0;
|
|
double finalize_ms = 0.0;
|
|
double output_ms = 0.0;
|
|
|
|
const size_t all = (size_t)ex[0] * ex[1] * ex[2];
|
|
const size_t bytes = all * sizeof(double);
|
|
int touch_xmin = 0, touch_xmax = 0;
|
|
int touch_ymin = 0, touch_ymax = 0;
|
|
int touch_zmin = 0, touch_zmax = 0;
|
|
|
|
setup_grid_params(ex, X, Y, Z, Symmetry, eps, co);
|
|
if (Lev > 0) {
|
|
compute_patch_boundary_flags(ex, X, Y, Z, bbox, Symmetry,
|
|
touch_xmin, touch_xmax,
|
|
touch_ymin, touch_ymax,
|
|
touch_zmin, touch_zmax);
|
|
}
|
|
|
|
StepContext &ctx = ensure_step_ctx(block_tag, all);
|
|
const bool use_resident_state = (keep_resident_state != 0);
|
|
if (use_resident_state) {
|
|
bind_state_input_slots(ctx.d_state_curr);
|
|
bind_state_output_slots(ctx.d_state_next);
|
|
}
|
|
|
|
double t0 = profile ? cuda_profile_now_ms() : 0.0;
|
|
if (!use_resident_state || RK4 == 0 || !ctx.state_ready) {
|
|
upload_state_inputs(state_host_in, all);
|
|
}
|
|
if (apply_enforce_ga) {
|
|
kern_enforce_ga_cuda<<<grid(all), BLK>>>(g_buf.slot[S_dxx], g_buf.slot[S_gxy], g_buf.slot[S_gxz],
|
|
g_buf.slot[S_dyy], g_buf.slot[S_gyz], g_buf.slot[S_dzz],
|
|
g_buf.slot[S_Axx], g_buf.slot[S_Axy], g_buf.slot[S_Axz],
|
|
g_buf.slot[S_Ayy], g_buf.slot[S_Ayz], g_buf.slot[S_Azz]);
|
|
}
|
|
if (RK4 == 0) {
|
|
CUDA_CHECK(cudaMemcpy(ctx.d_state0_mem, g_buf.slot[S_chi],
|
|
(size_t)BSSN_STATE_COUNT * bytes,
|
|
cudaMemcpyDeviceToDevice));
|
|
}
|
|
if (profile) {
|
|
cuda_profile_sync();
|
|
state_ms += cuda_profile_now_ms() - t0;
|
|
}
|
|
|
|
t0 = profile ? cuda_profile_now_ms() : 0.0;
|
|
launch_z4c_rhs_pipeline((int)all, eps);
|
|
if (profile) {
|
|
cuda_profile_sync();
|
|
rhs_ms += cuda_profile_now_ms() - t0;
|
|
}
|
|
|
|
t0 = profile ? cuda_profile_now_ms() : 0.0;
|
|
if (apply_bam_bc) {
|
|
for (int i = 0; i < BSSN_STATE_COUNT; ++i) {
|
|
gpu_sommerfeld_routbam(g_buf.slot[k_state_input_slots[i]],
|
|
g_buf.slot[k_state_rhs_slots[i]],
|
|
propspeed[i],
|
|
soa_flat[3 * i + 0],
|
|
soa_flat[3 * i + 1],
|
|
soa_flat[3 * i + 2],
|
|
X, Y, Z, bbox, Symmetry);
|
|
}
|
|
}
|
|
if (profile) {
|
|
cuda_profile_sync();
|
|
bc_ms += cuda_profile_now_ms() - t0;
|
|
}
|
|
|
|
t0 = profile ? cuda_profile_now_ms() : 0.0;
|
|
gpu_rk4_finalize_batch(ctx, all, dT, RK4, chitiny);
|
|
if (Lev > 0) {
|
|
gpu_copy_patch_boundary_batch(ctx, (int)all,
|
|
touch_xmin, touch_xmax,
|
|
touch_ymin, touch_ymax,
|
|
touch_zmin, touch_zmax);
|
|
}
|
|
if (profile) {
|
|
cuda_profile_sync();
|
|
finalize_ms += cuda_profile_now_ms() - t0;
|
|
}
|
|
|
|
t0 = profile ? cuda_profile_now_ms() : 0.0;
|
|
if (use_resident_state) {
|
|
std::swap(ctx.d_state_curr_mem, ctx.d_state_next_mem);
|
|
ctx.d_state_curr.swap(ctx.d_state_next);
|
|
ctx.state_ready = true;
|
|
} else {
|
|
download_state_outputs(state_host_out, all);
|
|
}
|
|
if (profile) {
|
|
cuda_profile_sync();
|
|
output_ms += cuda_profile_now_ms() - t0;
|
|
CudaProfileStats &stats = cuda_profile_stats();
|
|
stats.calls++;
|
|
stats.total_ms += cuda_profile_now_ms() - t_total0;
|
|
stats.state_ms += state_ms;
|
|
stats.rhs_ms += rhs_ms;
|
|
stats.bc_ms += bc_ms;
|
|
stats.finalize_ms += finalize_ms;
|
|
stats.output_ms += output_ms;
|
|
cuda_profile_maybe_log();
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_download_resident_state(void *block_tag,
|
|
int *ex,
|
|
double **state_host_out)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
download_resident_state(block_tag, ex, state_host_out);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_pack_state_region_to_host_buffer(void *block_tag,
|
|
int state_index,
|
|
double *host_buffer,
|
|
int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
copy_state_region_packed_cuda(block_tag, state_index, host_buffer, ex,
|
|
i0, j0, k0, sx, sy, sz,
|
|
cudaMemcpyDeviceToHost);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_unpack_state_region_from_host_buffer(void *block_tag,
|
|
int state_index,
|
|
double *host_buffer,
|
|
int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
copy_state_region_packed_cuda(block_tag, state_index, host_buffer, ex,
|
|
i0, j0, k0, sx, sy, sz,
|
|
cudaMemcpyHostToDevice);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_pack_state_batch_to_host_buffer(void *block_tag,
|
|
int state_count,
|
|
double *host_buffer,
|
|
int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
copy_state_region_packed_batch_cuda(block_tag, state_count, host_buffer, ex,
|
|
i0, j0, k0, sx, sy, sz,
|
|
cudaMemcpyDeviceToHost);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_unpack_state_batch_from_host_buffer(void *block_tag,
|
|
int state_count,
|
|
double *host_buffer,
|
|
int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
copy_state_region_packed_batch_cuda(block_tag, state_count, host_buffer, ex,
|
|
i0, j0, k0, sx, sy, sz,
|
|
cudaMemcpyHostToDevice);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_pack_state_batch_to_device_buffer(void *block_tag,
|
|
int state_count,
|
|
double *device_buffer,
|
|
int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
copy_state_region_packed_batch_device_cuda(block_tag, state_count, device_buffer, ex,
|
|
i0, j0, k0, sx, sy, sz, 1);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_unpack_state_batch_from_device_buffer(void *block_tag,
|
|
int state_count,
|
|
double *device_buffer,
|
|
int *ex,
|
|
int i0, int j0, int k0,
|
|
int sx, int sy, int sz)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
copy_state_region_packed_batch_device_cuda(block_tag, state_count, device_buffer, ex,
|
|
i0, j0, k0, sx, sy, sz, 0);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_restrict_state_batch_to_device_buffer(void *block_tag,
|
|
int state_count,
|
|
double *device_buffer,
|
|
int *ex,
|
|
int sx, int sy, int sz,
|
|
int fi0, int fj0, int fk0,
|
|
const double *state_soa)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
if (state_count <= 0 || state_count > BSSN_STATE_COUNT) return 1;
|
|
if (!device_buffer || sx <= 0 || sy <= 0 || sz <= 0) return 1;
|
|
StepContext &ctx = ensure_step_ctx(block_tag, (size_t)ex[0] * ex[1] * ex[2]);
|
|
const int region_all = sx * sy * sz;
|
|
upload_comm_state_soa(state_soa, state_count);
|
|
dim3 launch_grid((unsigned int)grid((size_t)region_all),
|
|
(unsigned int)state_count);
|
|
kern_restrict_state_region_batch<<<launch_grid, BLK>>>(
|
|
ctx.d_state_curr_mem, device_buffer,
|
|
ex[0], ex[1], sx, sy, sz,
|
|
fi0, fj0, fk0, region_all, state_count,
|
|
ex[0] * ex[1] * ex[2]);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_prolong_state_batch_to_device_buffer(void *block_tag,
|
|
int state_count,
|
|
double *device_buffer,
|
|
int *ex,
|
|
int sx, int sy, int sz,
|
|
int ii0, int jj0, int kk0,
|
|
int lbc_i, int lbc_j, int lbc_k,
|
|
const double *state_soa)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
if (state_count <= 0 || state_count > BSSN_STATE_COUNT) return 1;
|
|
if (!device_buffer || sx <= 0 || sy <= 0 || sz <= 0) return 1;
|
|
StepContext &ctx = ensure_step_ctx(block_tag, (size_t)ex[0] * ex[1] * ex[2]);
|
|
const int region_all = sx * sy * sz;
|
|
upload_comm_state_soa(state_soa, state_count);
|
|
dim3 launch_grid((unsigned int)grid((size_t)region_all),
|
|
(unsigned int)state_count);
|
|
kern_prolong_state_region_batch<<<launch_grid, BLK>>>(
|
|
ctx.d_state_curr_mem, device_buffer,
|
|
ex[0], ex[1], sx, sy, sz,
|
|
ii0, jj0, kk0, lbc_i, lbc_j, lbc_k,
|
|
region_all, state_count,
|
|
ex[0] * ex[1] * ex[2]);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_download_state_subset(void *block_tag,
|
|
int *ex,
|
|
int subset_count,
|
|
const int *state_indices,
|
|
double **state_host_out)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
copy_state_subset(block_tag, ex, subset_count, state_indices, state_host_out,
|
|
cudaMemcpyDeviceToHost);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_upload_state_subset(void *block_tag,
|
|
int *ex,
|
|
int subset_count,
|
|
const int *state_indices,
|
|
double **state_host_in)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
copy_state_subset(block_tag, ex, subset_count, state_indices, state_host_in,
|
|
cudaMemcpyHostToDevice);
|
|
return 0;
|
|
}
|
|
|
|
extern "C" int z4c_cuda_has_resident_state(void *block_tag)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
return has_resident_state(block_tag) ? 1 : 0;
|
|
}
|
|
|
|
extern "C" void z4c_cuda_release_step_ctx(void *block_tag)
|
|
{
|
|
using namespace z4c_cuda;
|
|
init_gpu_dispatch();
|
|
CUDA_CHECK(cudaSetDevice(g_dispatch.my_device));
|
|
release_step_ctx(block_tag);
|
|
}
|