Merge remote-tracking branch 'origin/tensor_core' into rtl
This commit is contained in:
Hansung Kim
@@ -56,6 +56,7 @@ extern "C" {
|
||||
void dpi_fmax(bool enable, int dst_fmt, int64_t a, int64_t b, int64_t* result, svBitVecVal* fflags);
|
||||
|
||||
void dpi_hmma(bool enable, const svBitVecVal* A_tile, const svBitVecVal* B_tile, const svBitVecVal* C_tile, svBitVecVal* D_tile);
|
||||
void dpi_print_results(int wid, int octet, const svBitVecVal* A_tile, const svBitVecVal* B_tile, const svBitVecVal* C_tile, const svBitVecVal* D_tile);
|
||||
}
|
||||
|
||||
inline uint64_t nan_box(uint32_t value) {
|
||||
@@ -413,4 +414,152 @@ void dpi_hmma(bool enable, const svBitVecVal* A_tile, const svBitVecVal* B_tile,
|
||||
}
|
||||
|
||||
write_float_array(D_tile, &c_D_tile[0][0], M, M);
|
||||
|
||||
// 1 copy per warp
|
||||
float A_tile_full[4][16][8];
|
||||
float B_tile_full[4][8][16];
|
||||
float C_tile_full[4][16][16];
|
||||
float D_tile_full[4][16][16];
|
||||
int steps[4];
|
||||
|
||||
void print_array(float* array, int rows, int cols) {
|
||||
for (int i = 0; i < rows; i += 1) {
|
||||
for (int j = 0; j < cols; j += 1) {
|
||||
std::cout << array[i*cols+j] << " ";
|
||||
}
|
||||
std::cout << "\n";
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
|
||||
void dpi_print_results(int wid, int octet, const svBitVecVal* A_tile, const svBitVecVal* B_tile, const svBitVecVal* C_tile, const svBitVecVal* D_tile) {
|
||||
// std::cout << "A: " << std::endl;
|
||||
fill_float_array(A_tile, &c_A_tile[0][0], M, K);
|
||||
// std::cout << "B: " << std::endl;
|
||||
fill_float_array(B_tile, &c_B_tile[0][0], K, M);
|
||||
// std::cout << "C: " << std::endl;
|
||||
fill_float_array(C_tile, &c_C_tile[0][0], M, M);
|
||||
// for some reason this still holds onto old value? very strange
|
||||
// std::cout << "D: " << std::endl;
|
||||
fill_float_array(D_tile, &c_D_tile[0][0], M, M);
|
||||
|
||||
int octet_row_offset;
|
||||
int octet_col_offset;
|
||||
switch(octet) {
|
||||
case 0:
|
||||
octet_row_offset = 0;
|
||||
octet_col_offset = 0;
|
||||
break;
|
||||
case 1:
|
||||
octet_row_offset = 8;
|
||||
octet_col_offset = 0;
|
||||
break;
|
||||
case 2:
|
||||
octet_row_offset = 0;
|
||||
octet_col_offset = 8;
|
||||
break;
|
||||
case 3:
|
||||
octet_row_offset = 8;
|
||||
octet_col_offset = 8;
|
||||
break;
|
||||
}
|
||||
|
||||
int step_row_offset;
|
||||
int step_col_offset;
|
||||
int step = (steps[wid] % 16) / 4;
|
||||
int set = (steps[wid] / 16);
|
||||
switch(step) {
|
||||
case 0:
|
||||
step_row_offset = 0;
|
||||
step_col_offset = 0;
|
||||
break;
|
||||
case 1:
|
||||
step_row_offset = 2;
|
||||
step_col_offset = 0;
|
||||
break;
|
||||
case 2:
|
||||
step_row_offset = 0;
|
||||
step_col_offset = 4;
|
||||
break;
|
||||
case 3:
|
||||
step_row_offset = 2;
|
||||
step_col_offset = 4;
|
||||
break;
|
||||
}
|
||||
|
||||
if (steps[0] >= 48) {
|
||||
// std::cout << "octet " << octet << " step " << steps[0] << "\n";
|
||||
// print_array(&c_D_tile[0][0], 4, 4);
|
||||
}
|
||||
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+0][octet_col_offset+step_col_offset+0] = c_D_tile[0][0];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+0][octet_col_offset+step_col_offset+1] = c_D_tile[0][1];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+0][octet_col_offset+step_col_offset+2] = c_D_tile[0][2];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+0][octet_col_offset+step_col_offset+3] = c_D_tile[0][3];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+1][octet_col_offset+step_col_offset+0] = c_D_tile[1][0];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+1][octet_col_offset+step_col_offset+1] = c_D_tile[1][1];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+1][octet_col_offset+step_col_offset+2] = c_D_tile[1][2];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+1][octet_col_offset+step_col_offset+3] = c_D_tile[1][3];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+4][octet_col_offset+step_col_offset+0] = c_D_tile[2][0];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+4][octet_col_offset+step_col_offset+1] = c_D_tile[2][1];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+4][octet_col_offset+step_col_offset+2] = c_D_tile[2][2];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+4][octet_col_offset+step_col_offset+3] = c_D_tile[2][3];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+5][octet_col_offset+step_col_offset+0] = c_D_tile[3][0];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+5][octet_col_offset+step_col_offset+1] = c_D_tile[3][1];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+5][octet_col_offset+step_col_offset+2] = c_D_tile[3][2];
|
||||
D_tile_full[wid][octet_row_offset+step_row_offset+5][octet_col_offset+step_col_offset+3] = c_D_tile[3][3];
|
||||
|
||||
if (octet == 0 || octet == 1) {
|
||||
octet_row_offset = octet * 8;
|
||||
if (step == 0) {
|
||||
step_row_offset = 0;
|
||||
}
|
||||
if (step == 1) {
|
||||
step_row_offset = 2;
|
||||
}
|
||||
if (step == 0 || step == 1) {
|
||||
A_tile_full[wid][octet_row_offset+step_row_offset+0][set*2+0] = c_A_tile[0][0];
|
||||
A_tile_full[wid][octet_row_offset+step_row_offset+0][set*2+1] = c_A_tile[0][1];
|
||||
A_tile_full[wid][octet_row_offset+step_row_offset+1][set*2+0] = c_A_tile[1][0];
|
||||
A_tile_full[wid][octet_row_offset+step_row_offset+1][set*2+1] = c_A_tile[1][1];
|
||||
A_tile_full[wid][octet_row_offset+step_row_offset+4][set*2+0] = c_A_tile[2][0];
|
||||
A_tile_full[wid][octet_row_offset+step_row_offset+4][set*2+1] = c_A_tile[2][1];
|
||||
A_tile_full[wid][octet_row_offset+step_row_offset+5][set*2+0] = c_A_tile[3][0];
|
||||
A_tile_full[wid][octet_row_offset+step_row_offset+5][set*2+1] = c_A_tile[3][1];
|
||||
}
|
||||
}
|
||||
|
||||
if (octet == 0 || octet == 2) {
|
||||
octet_col_offset = octet * 4;
|
||||
if (step == 0) {
|
||||
step_col_offset = 0;
|
||||
}
|
||||
else if (step == 2) {
|
||||
step_col_offset = 4;
|
||||
}
|
||||
if (step == 0 || step == 2) {
|
||||
B_tile_full[wid][set*2+0][octet_col_offset+step_col_offset+0] = c_B_tile[0][0];
|
||||
B_tile_full[wid][set*2+0][octet_col_offset+step_col_offset+1] = c_B_tile[0][1];
|
||||
B_tile_full[wid][set*2+0][octet_col_offset+step_col_offset+2] = c_B_tile[0][2];
|
||||
B_tile_full[wid][set*2+0][octet_col_offset+step_col_offset+3] = c_B_tile[0][3];
|
||||
B_tile_full[wid][set*2+1][octet_col_offset+step_col_offset+0] = c_B_tile[1][0];
|
||||
B_tile_full[wid][set*2+1][octet_col_offset+step_col_offset+1] = c_B_tile[1][1];
|
||||
B_tile_full[wid][set*2+1][octet_col_offset+step_col_offset+2] = c_B_tile[1][2];
|
||||
B_tile_full[wid][set*2+1][octet_col_offset+step_col_offset+3] = c_B_tile[1][3];
|
||||
}
|
||||
}
|
||||
|
||||
steps[wid] += 1;
|
||||
if (steps[wid] % 64 == 0) {
|
||||
steps[wid] = 0;
|
||||
std::cout << "warp " << wid << " finished wmma\n";
|
||||
std::cout << "A tile" << "\n";
|
||||
print_array(&A_tile_full[wid][0][0], 16, 8);
|
||||
std::cout << "B tile" << "\n";
|
||||
print_array(&B_tile_full[wid][0][0], 8, 16);
|
||||
// std::cout << "C tile" << "\n";
|
||||
// print_array(&C_tile_full[wid][0][0], 16, 16);
|
||||
std::cout << "D tile" << "\n";
|
||||
print_array(&D_tile_full[wid][0][0], 16, 16);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,5 +45,6 @@ import "DPI-C" function void dpi_fmin(input logic enable, input int dst_fmt, inp
|
||||
import "DPI-C" function void dpi_fmax(input logic enable, input int dst_fmt, input longint a, input longint b, output longint result, output bit[4:0] fflags);
|
||||
|
||||
import "DPI-C" function void dpi_hmma(input logic enable, input bit[3:0][1:0][31:0] A_tile, input bit[1:0][3:0][31:0] B_tile, input bit[3:0][3:0][31:0] C_tile, output bit[3:0][3:0][31:0] D_tile);
|
||||
import "DPI-C" function void dpi_print_results(input int wid, input int octet, input bit[3:0][1:0][31:0] A_tile, input bit[1:0][3:0][31:0] B_tile, input bit[3:0][3:0][31:0] C_tile, input bit[3:0][3:0][31:0] D_tile);
|
||||
|
||||
`endif
|
||||
|
||||
@@ -54,6 +54,7 @@ module VX_commit import VX_gpu_pkg::*; #(
|
||||
wire [`ISSUE_WIDTH-1:0][`NUM_THREADS-1:0] commit_tmask;
|
||||
wire [`ISSUE_WIDTH-1:0] commit_eop;
|
||||
wire [`ISSUE_WIDTH-1:0][`EX_BITS-1:0] commit_sel;
|
||||
`UNUSED_VAR (commit_sel)
|
||||
|
||||
for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin
|
||||
|
||||
@@ -176,14 +177,17 @@ module VX_commit import VX_gpu_pkg::*; #(
|
||||
// relies on 1 cycle delay of arbiter and continuous issuing of tensor instructions,
|
||||
// so probably want to change this at some point
|
||||
// (i.e. pass a "don't count this towards pending instructions" signal down the pipeline)
|
||||
logic [`ISSUE_WIDTH-1:0][4:0] hmma_ctr, hmma_ctr_n;
|
||||
// logic [`ISSUE_WIDTH-1:0][4:0] hmma_ctr, hmma_ctr_n;
|
||||
wire [`ISSUE_WIDTH-1:0] final_hmma;
|
||||
`ifdef EXT_T_ENABLE
|
||||
for (genvar i = 0; i < `ISSUE_WIDTH; ++i) begin
|
||||
assign hmma_ctr_n[i] = (tensor_commit_if[i].valid && tensor_commit_if[i].ready) ? hmma_ctr[i] + 5'b1 : hmma_ctr[i];
|
||||
assign final_hmma[i] = (commit_sel[i] != `EX_BITS'(2) || hmma_ctr == '0);
|
||||
// assign hmma_ctr_n[i] = (tensor_commit_if[i].valid && tensor_commit_if[i].ready) ? hmma_ctr[i] + 5'b1 : hmma_ctr[i];
|
||||
// assign final_hmma[i] = (commit_sel[i] != `EX_BITS'(2) || hmma_ctr == '0);
|
||||
// i suppose this is now a feature and not a bug
|
||||
// if PC is 0, this means it is not final step of a wmma, shouldn't be committed
|
||||
assign final_hmma[i] = (commit_if[i].data.PC != 32'b0);
|
||||
end
|
||||
|
||||
/*
|
||||
always @(posedge clk) begin
|
||||
if (reset) begin
|
||||
hmma_ctr <= '0;
|
||||
@@ -192,6 +196,7 @@ module VX_commit import VX_gpu_pkg::*; #(
|
||||
hmma_ctr <= hmma_ctr_n;
|
||||
end
|
||||
end
|
||||
*/
|
||||
`else
|
||||
assign final_hmma = '1;
|
||||
`endif
|
||||
|
||||
@@ -306,15 +306,14 @@ module VX_operands import VX_gpu_pkg::*; #(
|
||||
cycle <= cycle_n;
|
||||
end
|
||||
|
||||
if (cycle == 32'd25000) begin
|
||||
for (integer k = 0; k < `NUM_REGS * ISSUE_RATIO; ++k) begin
|
||||
$display("warp %0d, thread %0d, register %0d: %0x",
|
||||
i * ISSUE_RATIO + (k / `NUM_REGS),
|
||||
j,
|
||||
k % `NUM_REGS,
|
||||
gpr_ram.ram[k]);
|
||||
end
|
||||
end
|
||||
// if (cycle == 32'd25000) begin
|
||||
// for (integer k = 0; k < `NUM_REGS * ISSUE_RATIO; ++k) begin
|
||||
// integer warp = i * ISSUE_RATIO + (k / `NUM_REGS);
|
||||
// integer thread = j;
|
||||
// integer register = k % `NUM_REGS;
|
||||
// $display("warp %0d, thread %0d, register %0d: %0x", warp, thread, register, gpr_ram.ram[k]);
|
||||
// end
|
||||
// end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@@ -58,7 +58,8 @@ module VX_tensor_core_warp import VX_gpu_pkg::*; #(
|
||||
logic result_valid;
|
||||
logic result_ready;
|
||||
VX_tensor_octet #(
|
||||
|
||||
.ISW(ISW),
|
||||
.OCTET(i)
|
||||
) octet (
|
||||
.clk(clk),
|
||||
.reset(reset),
|
||||
@@ -180,7 +181,8 @@ module VX_tensor_core_warp import VX_gpu_pkg::*; #(
|
||||
endmodule
|
||||
|
||||
module VX_tensor_octet #(
|
||||
|
||||
parameter ISW,
|
||||
parameter OCTET
|
||||
) (
|
||||
input clk,
|
||||
input reset,
|
||||
@@ -285,7 +287,8 @@ module VX_tensor_octet #(
|
||||
|
||||
wire do_hmma = (substep == 1'b1 && operands_valid && operands_ready);
|
||||
VX_tensor_dpu #(
|
||||
|
||||
.ISW(ISW),
|
||||
.OCTET(OCTET)
|
||||
) dpu (
|
||||
.clk(clk),
|
||||
.reset(reset),
|
||||
|
||||
@@ -92,5 +92,5 @@ HMMA_SET3_STEP3_0: begin
|
||||
uop = {NEXT, HMMA_SET3_STEP3_1, `EX_BITS'(`EX_TENSOR), `INST_OP_BITS'(3), `INST_MOD_BITS'(0), 1'b1, 1'b0, 1'b0, 32'b0, 32'b0, `FREG(22), `FREG(6), `FREG(14), `FREG(22)};
|
||||
end
|
||||
HMMA_SET3_STEP3_1: begin
|
||||
uop = {FINISH, HMMA_SET0_STEP0_0, `EX_BITS'(`EX_TENSOR), `INST_OP_BITS'(3), `INST_MOD_BITS'(1), 1'b1, 1'b0, 1'b0, 32'b0, 32'b0, `FREG(23), `FREG(7), `FREG(15), `FREG(23)};
|
||||
uop = {FINISH, HMMA_SET0_STEP0_0, `EX_BITS'(`EX_TENSOR), `INST_OP_BITS'(3), `INST_MOD_BITS'(1), 1'b1, 1'b0, 1'b0, 32'b1, 32'b1, `FREG(23), `FREG(7), `FREG(15), `FREG(23)};
|
||||
end
|
||||
|
||||
@@ -125,16 +125,16 @@ module VX_uop_sequencer import VX_gpu_pkg::*; (
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (uop_start) begin
|
||||
$display("UOP start @ %t", $time);
|
||||
$display("use_uop=%0d, use_uop_1d=%0d, uop_start=%0d, ibuffer_if.valid=%0d, ibuffer_if.ready=%0d", use_uop, use_uop_1d, uop_start, ibuffer_if.valid, ibuffer_if.ready);
|
||||
// $display("UOP start @ %t", $time);
|
||||
// $display("use_uop=%0d, use_uop_1d=%0d, uop_start=%0d, ibuffer_if.valid=%0d, ibuffer_if.ready=%0d", use_uop, use_uop_1d, uop_start, ibuffer_if.valid, ibuffer_if.ready);
|
||||
end
|
||||
|
||||
if (uop_fire) begin
|
||||
$display("UOP fire @ %t", $time);
|
||||
// $display("UOP fire @ %t", $time);
|
||||
end
|
||||
|
||||
if (uop_finish) begin
|
||||
$display("UOP finish @ %t", $time);
|
||||
// $display("UOP finish @ %t", $time);
|
||||
end
|
||||
|
||||
if (reset) begin
|
||||
|
||||
@@ -55,17 +55,21 @@ with open('VX_tensor_ucode.vh', 'w') as f:
|
||||
finish = (next_sequence_num == 0)
|
||||
|
||||
name = "HMMA_SET{}_STEP{}_{}"
|
||||
ucode = "{}, HMMA_SET{}_STEP{}_{}, `EX_BITS'(`EX_TENSOR), `INST_OP_BITS'({}), `INST_MOD_BITS'({}), 1'b1, 1'b0, 1'b0, 32'b0, 32'b0, `FREG({}), `FREG({}), `FREG({}), `FREG({})"
|
||||
ucode = "{}, HMMA_SET{}_STEP{}_{}, `EX_BITS'(`EX_TENSOR), `INST_OP_BITS'({}), `INST_MOD_BITS'({}), 1'b1, 1'b0, 1'b0, 32'b{}, 32'b{}, `FREG({}), `FREG({}), `FREG({}), `FREG({})"
|
||||
|
||||
name = name.format(
|
||||
set_num, step, substep,
|
||||
)
|
||||
|
||||
pc_imm = 1 if finish else 0
|
||||
|
||||
ucode = ucode.format(
|
||||
"FINISH" if finish else "NEXT",
|
||||
next_set_num, next_step, next_substep,
|
||||
step,
|
||||
substep,
|
||||
pc_imm,
|
||||
pc_imm,
|
||||
rs3_rd[(step, substep)],
|
||||
rs1[(set_num, substep)],
|
||||
rs2[(set_num, substep)],
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
`include "VX_fpu_define.vh"
|
||||
|
||||
module VX_tensor_dpu #(
|
||||
|
||||
parameter ISW,
|
||||
parameter OCTET
|
||||
) (
|
||||
input clk,
|
||||
input reset,
|
||||
@@ -22,6 +23,12 @@ module VX_tensor_dpu #(
|
||||
dpi_hmma(valid_in, A_tile, B_tile, C_tile, result_hmma);
|
||||
end
|
||||
|
||||
always @(posedge clk) begin
|
||||
if (~reset && valid_in) begin
|
||||
dpi_print_results(int'(ISW), int'(OCTET), A_tile, B_tile, C_tile, result_hmma);
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
VX_shift_register #(
|
||||
.DATAW (1 + $bits(D_tile)),
|
||||
|
||||
@@ -48,6 +48,7 @@ void vx_wspawn_wait();
|
||||
void vx_spawn_kernel(context_t * ctx, vx_spawn_kernel_cb callback, void * arg);
|
||||
|
||||
void vx_spawn_tasks(int num_tasks, vx_spawn_tasks_cb callback, void * arg);
|
||||
void vx_spawn_tasks_contiguous(int num_tasks, vx_spawn_tasks_cb callback , void * arg);
|
||||
|
||||
void vx_serial(vx_serial_cb callback, void * arg);
|
||||
|
||||
|
||||
@@ -83,6 +83,38 @@ static void __attribute__ ((noinline)) spawn_tasks_rem_stub() {
|
||||
(p_wspawn_args->callback)(task_id, p_wspawn_args->arg);
|
||||
}
|
||||
|
||||
static void __attribute__ ((noinline)) spawn_tasks_contiguous_all_stub() {
|
||||
int NT = vx_num_threads();
|
||||
int NW = vx_num_warps();
|
||||
int cid = vx_core_id();
|
||||
int wid = vx_warp_id();
|
||||
int tid = vx_thread_id();
|
||||
|
||||
wspawn_tasks_args_t* p_wspawn_args = (wspawn_tasks_args_t*)g_wspawn_args[cid];
|
||||
|
||||
int waves = p_wspawn_args->NWs + (wid < p_wspawn_args->RWs);
|
||||
int offset = p_wspawn_args->offset + (NT * wid + tid);
|
||||
|
||||
vx_spawn_tasks_cb callback = p_wspawn_args->callback;
|
||||
void* arg = p_wspawn_args->arg;
|
||||
for (int wave_id = 0; wave_id < waves; ++wave_id) {
|
||||
int task_id = offset + (wave_id * NT * NW);
|
||||
callback(task_id, arg);
|
||||
}
|
||||
}
|
||||
|
||||
static void __attribute__ ((noinline)) spawn_tasks_contiguous_all_cb() {
|
||||
// activate all threads
|
||||
vx_tmc(-1);
|
||||
|
||||
// call stub routine
|
||||
spawn_tasks_contiguous_all_stub();
|
||||
|
||||
// disable warp
|
||||
// deadlock here on warps 1, 2, 3
|
||||
vx_tmc_zero();
|
||||
}
|
||||
|
||||
static void __attribute__ ((noinline)) spawn_tasks_all_cb() {
|
||||
// activate all threads
|
||||
vx_tmc(-1);
|
||||
@@ -94,6 +126,79 @@ static void __attribute__ ((noinline)) spawn_tasks_all_cb() {
|
||||
vx_tmc_zero();
|
||||
}
|
||||
|
||||
void vx_spawn_tasks_contiguous(int num_tasks, vx_spawn_tasks_cb callback , void * arg) {
|
||||
// device specs
|
||||
int NC = vx_num_cores();
|
||||
int NW = vx_num_warps();
|
||||
int NT = vx_num_threads();
|
||||
|
||||
// current core id
|
||||
int core_id = vx_core_id();
|
||||
if (core_id >= NUM_CORES_MAX)
|
||||
return;
|
||||
|
||||
// calculate necessary active cores
|
||||
int WT = NW * NT;
|
||||
int nC = (num_tasks > WT) ? (num_tasks / WT) : 1;
|
||||
int nc = MIN(nC, NC);
|
||||
if (core_id >= nc)
|
||||
return; // terminate extra cores
|
||||
|
||||
// number of tasks per core
|
||||
int tasks_per_core = num_tasks / nc;
|
||||
int tasks_per_core_n1 = tasks_per_core;
|
||||
if (core_id == (nc-1)) {
|
||||
int rem = num_tasks - (nc * tasks_per_core);
|
||||
tasks_per_core_n1 += rem; // last core also executes remaining tasks
|
||||
}
|
||||
|
||||
// number of tasks per warp
|
||||
int TW = tasks_per_core_n1 / NT; // occupied warps
|
||||
int rT = tasks_per_core_n1 - TW * NT; // remaining threads
|
||||
int fW = 1, rW = 0;
|
||||
if (TW >= NW) {
|
||||
fW = TW / NW; // full warps iterations
|
||||
rW = TW - fW * NW; // remaining warps
|
||||
}
|
||||
|
||||
wspawn_tasks_args_t wspawn_args = { callback, arg, core_id * tasks_per_core, fW, rW };
|
||||
g_wspawn_args[core_id] = &wspawn_args;
|
||||
|
||||
if (TW >= 1) {
|
||||
// execute callback on other warps
|
||||
int nw = MIN(TW, NW);
|
||||
vx_wspawn(nw, spawn_tasks_contiguous_all_cb);
|
||||
|
||||
// activate all threads
|
||||
vx_tmc(-1);
|
||||
|
||||
// call stub routine
|
||||
spawn_tasks_contiguous_all_stub();
|
||||
|
||||
// back to single-threaded
|
||||
vx_tmc_one();
|
||||
|
||||
// wait for spawn warps to terminate
|
||||
// deadlock here on warp 0!
|
||||
vx_wspawn_wait();
|
||||
}
|
||||
|
||||
if (rT != 0) {
|
||||
// adjust offset
|
||||
wspawn_args.offset += (tasks_per_core_n1 - rT);
|
||||
|
||||
// activate remaining threads
|
||||
int tmask = (1 << rT) - 1;
|
||||
vx_tmc(tmask);
|
||||
|
||||
// call stub routine
|
||||
spawn_tasks_rem_stub();
|
||||
|
||||
// back to single-threaded
|
||||
vx_tmc_one();
|
||||
}
|
||||
}
|
||||
|
||||
void vx_spawn_tasks(int num_tasks, vx_spawn_tasks_cb callback , void * arg) {
|
||||
// device specs
|
||||
int NC = vx_num_cores();
|
||||
|
||||
@@ -90,7 +90,7 @@ int main()
|
||||
vx_wmma();
|
||||
store_wmma_result();
|
||||
vx_tmc(1);
|
||||
print_wmma_result();
|
||||
// print_wmma_result();
|
||||
|
||||
return 0;
|
||||
}
|
||||
9
tests/regression/sgemm_tcore/Makefile
Normal file
9
tests/regression/sgemm_tcore/Makefile
Normal file
@@ -0,0 +1,9 @@
|
||||
PROJECT = sgemm_tcore
|
||||
|
||||
SRCS = main.cpp common.h
|
||||
|
||||
VX_SRCS = kernel.cpp
|
||||
|
||||
OPTS ?= -n16
|
||||
|
||||
include ../common.mk
|
||||
18
tests/regression/sgemm_tcore/common.h
Normal file
18
tests/regression/sgemm_tcore/common.h
Normal file
@@ -0,0 +1,18 @@
|
||||
#ifndef _COMMON_H_
|
||||
#define _COMMON_H_
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000
|
||||
#define DEV_SMEM_START_ADDR 0xff000000
|
||||
|
||||
typedef struct {
|
||||
uint32_t dim_m;
|
||||
uint32_t dim_n;
|
||||
uint32_t dim_k;
|
||||
uint64_t addr_a;
|
||||
uint64_t addr_b;
|
||||
uint64_t addr_c;
|
||||
} kernel_arg_t;
|
||||
|
||||
#endif
|
||||
285
tests/regression/sgemm_tcore/kernel.cpp
Normal file
285
tests/regression/sgemm_tcore/kernel.cpp
Normal file
@@ -0,0 +1,285 @@
|
||||
#define RISCV_CUSTOM3 0x7B
|
||||
|
||||
#include <stdint.h>
|
||||
#include <vx_intrinsics.h>
|
||||
#include <vx_print.h>
|
||||
#include <vx_spawn.h>
|
||||
#include "common.h"
|
||||
|
||||
#define BM 16
|
||||
#define BN 16
|
||||
#define BK 8
|
||||
|
||||
inline void vx_wmma() {
|
||||
asm volatile (".insn r %0, 0, 0, x0, x0, x0" :: "i"(RISCV_CUSTOM3));
|
||||
}
|
||||
|
||||
void vx_wmma_load(volatile float *smem_A, volatile float *smem_B, int warp_x, int warp_y, int thread_in_warp) {
|
||||
int tid = thread_in_warp;
|
||||
int tg = tid / 4;
|
||||
|
||||
// load A
|
||||
int row = tid % 4;
|
||||
row += (tg * 8) % 16;
|
||||
row += (tg / 4) * 4;
|
||||
|
||||
int smem_A_m = 32;
|
||||
int smem_A_n = 8;
|
||||
int smem_B_m = 8;
|
||||
int smem_B_n = 32;
|
||||
|
||||
int A_offset = (row + BM * warp_y) * smem_A_n;
|
||||
|
||||
asm volatile ("flw f0, %0" :: "m"(smem_A[A_offset + 0]));
|
||||
asm volatile ("flw f1, %0" :: "m"(smem_A[A_offset + 1]));
|
||||
asm volatile ("flw f2, %0" :: "m"(smem_A[A_offset + 2]));
|
||||
asm volatile ("flw f3, %0" :: "m"(smem_A[A_offset + 3]));
|
||||
asm volatile ("flw f4, %0" :: "m"(smem_A[A_offset + 4]));
|
||||
asm volatile ("flw f5, %0" :: "m"(smem_A[A_offset + 5]));
|
||||
asm volatile ("flw f6, %0" :: "m"(smem_A[A_offset + 6]));
|
||||
asm volatile ("flw f7, %0" :: "m"(smem_A[A_offset + 7]));
|
||||
|
||||
// load B
|
||||
int col = tid % 4;
|
||||
col += ((tg % 4) / 2) * 8;
|
||||
col += (tg / 4) * 4;
|
||||
|
||||
asm volatile ("flw f8 , %0" :: "m"(smem_B[(0 * smem_B_n) + warp_x * BN + col]));
|
||||
asm volatile ("flw f9 , %0" :: "m"(smem_B[(1 * smem_B_n) + warp_x * BN + col]));
|
||||
asm volatile ("flw f10, %0" :: "m"(smem_B[(2 * smem_B_n) + warp_x * BN + col]));
|
||||
asm volatile ("flw f11, %0" :: "m"(smem_B[(3 * smem_B_n) + warp_x * BN + col]));
|
||||
asm volatile ("flw f12, %0" :: "m"(smem_B[(4 * smem_B_n) + warp_x * BN + col]));
|
||||
asm volatile ("flw f13, %0" :: "m"(smem_B[(5 * smem_B_n) + warp_x * BN + col]));
|
||||
asm volatile ("flw f14, %0" :: "m"(smem_B[(6 * smem_B_n) + warp_x * BN + col]));
|
||||
asm volatile ("flw f15, %0" :: "m"(smem_B[(7 * smem_B_n) + warp_x * BN + col]));
|
||||
}
|
||||
|
||||
inline void initialize_C() {
|
||||
// initialize C to zeros
|
||||
asm volatile ("fmv.w.x f16, x0");
|
||||
asm volatile ("fmv.w.x f17, x0");
|
||||
asm volatile ("fmv.w.x f18, x0");
|
||||
asm volatile ("fmv.w.x f19, x0");
|
||||
asm volatile ("fmv.w.x f20, x0");
|
||||
asm volatile ("fmv.w.x f21, x0");
|
||||
asm volatile ("fmv.w.x f22, x0");
|
||||
asm volatile ("fmv.w.x f23, x0");
|
||||
}
|
||||
|
||||
inline void write_results(
|
||||
volatile float *local_warp_results,
|
||||
int thread_in_warp,
|
||||
int warp_x,
|
||||
int warp_y,
|
||||
int dim_m,
|
||||
int dim_n,
|
||||
float *C,
|
||||
int threadblock_id_x,
|
||||
int threadblock_id_y
|
||||
) {
|
||||
int tid = thread_in_warp;
|
||||
int tg = tid / 4;
|
||||
|
||||
asm volatile ("fsw f16, %0" :: "m"(local_warp_results[tid*8+0]));
|
||||
asm volatile ("fsw f17, %0" :: "m"(local_warp_results[tid*8+1]));
|
||||
asm volatile ("fsw f18, %0" :: "m"(local_warp_results[tid*8+2]));
|
||||
asm volatile ("fsw f19, %0" :: "m"(local_warp_results[tid*8+3]));
|
||||
asm volatile ("fsw f20, %0" :: "m"(local_warp_results[tid*8+4]));
|
||||
asm volatile ("fsw f21, %0" :: "m"(local_warp_results[tid*8+5]));
|
||||
asm volatile ("fsw f22, %0" :: "m"(local_warp_results[tid*8+6]));
|
||||
asm volatile ("fsw f23, %0" :: "m"(local_warp_results[tid*8+7]));
|
||||
|
||||
/*
|
||||
col = ((threadgroup % 4) // 2) * 8
|
||||
row = (threadgroup * 8) % 16
|
||||
row += (threadgroup // 4) * 4
|
||||
offsets = [(0, 0), (0, 1), (2, 0), (2, 1), (0, 4), (0, 5), (2, 4), (2, 5)]
|
||||
offset = offsets[register-16]
|
||||
row += offset[0]
|
||||
col += offset[1]
|
||||
thread_offsets = [(0, 0), (1, 0), (0, 2), (1, 2)]
|
||||
thread_offset = thread_offsets[thread % 4]
|
||||
row += thread_offset[0]
|
||||
col += thread_offset[1]
|
||||
return (row, col)
|
||||
*/
|
||||
|
||||
int local_col = ((tg % 4) / 2) * 8;
|
||||
int local_row = (tg * 8) % 16;
|
||||
local_row += (tg / 4) * 4;
|
||||
|
||||
// int row_offsets[8] = {0, 0, 2, 2, 0, 0, 2, 2};
|
||||
// int col_offsets[8] = {0, 1, 0, 1, 4, 5, 4, 5};
|
||||
|
||||
// int thread_row_offsets[4] = {0, 1, 0, 1};
|
||||
// int thread_col_offsets[4] = {0, 0, 2, 2};
|
||||
int thread_row_offset = (tid % 4) % 2;
|
||||
int thread_col_offset = ((tid % 4) / 2) * 2;
|
||||
|
||||
float *global_offset_C = C + (threadblock_id_y * BM * 2 + warp_y * BM) * dim_n + threadblock_id_x * BN * 2 + warp_x * BM;
|
||||
for (int i = 0; i < 8; i += 1) {
|
||||
int row_offset = ((i / 2) % 2) * 2;
|
||||
int col_offset = (i / 4) * 4 + i % 2;
|
||||
|
||||
int adjusted_local_row = local_row + thread_row_offset + row_offset;
|
||||
int adjusted_local_col = local_col + thread_col_offset + col_offset;
|
||||
|
||||
float v = local_warp_results[tid*8+i];
|
||||
global_offset_C[adjusted_local_row * dim_n + adjusted_local_col] = v;
|
||||
}
|
||||
}
|
||||
|
||||
void threadblock_barrier(unsigned int tid_in_threadblock, unsigned int barrier_id, unsigned int count) {
|
||||
vx_fence();
|
||||
vx_barrier(barrier_id, count);
|
||||
}
|
||||
|
||||
void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg,
|
||||
const uint32_t tid_in_threadblock,
|
||||
const uint32_t threadblock_dim_x,
|
||||
const uint32_t threadblock_dim_y,
|
||||
const uint32_t threadblock_id_x,
|
||||
const uint32_t threadblock_id_y,
|
||||
const uint32_t threadblock_id,
|
||||
float *sharedmem_per_threadblock) {
|
||||
const float *A = (const float *)arg->addr_a;
|
||||
const float *B = (const float *)arg->addr_b;
|
||||
float *C = (float *)arg->addr_c;
|
||||
|
||||
const uint32_t dim_m = arg->dim_m;
|
||||
const uint32_t dim_n = arg->dim_n;
|
||||
const uint32_t dim_k = arg->dim_k;
|
||||
|
||||
// FIXME: Output block size is assumed to be square, i.e. BM == BN
|
||||
// const uint32_t BM = threadblock_dim_y;
|
||||
// const uint32_t BN = threadblock_dim_y;
|
||||
// const uint32_t BK = threadblock_dim_x;
|
||||
// constexpr uint32_t BM = 8;
|
||||
// constexpr uint32_t BN = 8;
|
||||
// constexpr uint32_t BK = 2;
|
||||
|
||||
const uint32_t warp_in_threadblock = tid_in_threadblock / 32;
|
||||
const uint32_t tid_in_warp = tid_in_threadblock % 32;
|
||||
const uint32_t warp_x = warp_in_threadblock % 2;
|
||||
const uint32_t warp_y = warp_in_threadblock / 2;
|
||||
|
||||
const uint32_t global_a_row = threadblock_dim_y * threadblock_id_y;
|
||||
|
||||
// 32 * 8 block of A, being loaded by 4 warps
|
||||
const uint32_t local_a_row = warp_y * BM + warp_x * (BM / 2) + (tid_in_warp / BK);
|
||||
const uint32_t local_a_col = tid_in_warp % BK;
|
||||
|
||||
// 8 * 32 block of B, being loaded by 4 warps
|
||||
// do a fat coalesced load
|
||||
const uint32_t global_b_col = threadblock_dim_x * threadblock_id_x;
|
||||
const uint32_t local_b_row = warp_in_threadblock;
|
||||
const uint32_t local_b_col = tid_in_warp;
|
||||
|
||||
|
||||
volatile float *local_a = sharedmem_per_threadblock;
|
||||
const size_t local_a_elems = (threadblock_dim_y * BK);
|
||||
volatile float *local_b = sharedmem_per_threadblock + local_a_elems;
|
||||
const size_t local_b_elems = (threadblock_dim_x * BK);
|
||||
volatile float *local_warp_results = local_b + local_b_elems + (warp_in_threadblock * BM * BN);
|
||||
|
||||
// clear out C
|
||||
initialize_C();
|
||||
|
||||
for (uint32_t k = 0; k < dim_k; k += BK) {
|
||||
// Data move from GMEM to SMEM
|
||||
//
|
||||
// Make sure global offset values for A and B are contiguous between
|
||||
// neighboring threads to ensure GMEM coalescing. (not possible for A here, but for B it's doable)
|
||||
|
||||
// coalesced load from global memory -> unit-stride store into shared memory
|
||||
uint32_t global_a_offset =
|
||||
dim_k * (global_a_row + local_a_row) + (k + local_a_col);
|
||||
uint32_t shared_a_offset =
|
||||
BK * local_a_row + local_a_col;
|
||||
|
||||
local_a[shared_a_offset] = A[global_a_offset];
|
||||
|
||||
global_a_offset += dim_k * (BM / 4);
|
||||
shared_a_offset += BK * (BM / 4);
|
||||
|
||||
local_a[shared_a_offset] = A[global_a_offset];
|
||||
|
||||
uint32_t global_b_offset =
|
||||
dim_n * (k + local_b_row) + (global_b_col + local_b_col);
|
||||
uint32_t shared_b_offset =
|
||||
(BN * 2) * (local_b_row) + local_b_col;
|
||||
|
||||
local_b[shared_b_offset] = B[global_b_offset];
|
||||
|
||||
global_b_offset += dim_n * (BK / 2);
|
||||
shared_b_offset += (BN * 2) * (BK / 2);
|
||||
|
||||
local_b[shared_b_offset] = B[global_b_offset];
|
||||
|
||||
// want all 4 warps to reach barrier before moving on (just use barrier 0 for now)
|
||||
threadblock_barrier(tid_in_threadblock, 0, 4);
|
||||
|
||||
// perform wmma
|
||||
vx_wmma_load(local_a, local_b, warp_x, warp_y, tid_in_warp);
|
||||
vx_wmma();
|
||||
|
||||
// same as above
|
||||
threadblock_barrier(tid_in_threadblock, 0, 4);
|
||||
}
|
||||
|
||||
write_results(
|
||||
local_warp_results,
|
||||
tid_in_warp,
|
||||
warp_x,
|
||||
warp_y,
|
||||
dim_m,
|
||||
dim_n,
|
||||
C,
|
||||
threadblock_id_x,
|
||||
threadblock_id_y
|
||||
);
|
||||
}
|
||||
|
||||
void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
|
||||
// @perf: All threads are running these compute whose result is mostly same
|
||||
// across the threadblock
|
||||
const int NT = 32; // vx_num_threads();
|
||||
const int NW = 4; // vx_num_warps();
|
||||
const uint32_t threads_per_threadblock = NT * NW;
|
||||
|
||||
// matches 4 warp capacity
|
||||
const uint32_t threadblock_dim_x = 2 * BN;
|
||||
const uint32_t threadblock_dim_y = 2 * BM;
|
||||
const int threadblock_id = task_id / threads_per_threadblock;
|
||||
const int tid_in_threadblock = task_id % threads_per_threadblock;
|
||||
|
||||
const uint32_t dim_m = arg->dim_m;
|
||||
const uint32_t dim_n = arg->dim_n;
|
||||
const uint32_t dim_n_in_blocks = dim_n / threadblock_dim_x;
|
||||
const int threadblock_id_x = threadblock_id % dim_n_in_blocks;
|
||||
const int threadblock_id_y = threadblock_id / dim_n_in_blocks;
|
||||
|
||||
// "static" shared memory allocation. This would determine threadblock
|
||||
// occupancy of a single cluster
|
||||
// only 1 threadblock running at a time, so this is ok
|
||||
float *sharedmem_per_threadblock =
|
||||
(float *)DEV_SMEM_START_ADDR; // + (2 * BM * BK) + (2 * BN * BK) * threadblock_id;
|
||||
|
||||
thread_block_gemm(arg, tid_in_threadblock, threadblock_dim_x,
|
||||
threadblock_dim_y, threadblock_id_x, threadblock_id_y,
|
||||
threadblock_id, sharedmem_per_threadblock);
|
||||
}
|
||||
|
||||
int main() {
|
||||
kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR;
|
||||
int NT = vx_num_threads();
|
||||
|
||||
// TODO: add support for edge-case (m, n not divisible by 16)
|
||||
const uint32_t grid_size = arg->dim_m * arg->dim_n * NT / (BM * BN);
|
||||
|
||||
// for now, simplifying assumption of just 1 core
|
||||
// vx_spawn_tasks_contiguous first runs warps 1 through NW, then NW+1 through 2*NW, etc.
|
||||
// we can thus treat 1 through NW as a single threadblock for the purposes of the optimization.
|
||||
vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
|
||||
return 0;
|
||||
}
|
||||
270
tests/regression/sgemm_tcore/main.cpp
Normal file
270
tests/regression/sgemm_tcore/main.cpp
Normal file
@@ -0,0 +1,270 @@
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
#include <vortex.h>
|
||||
#include <vector>
|
||||
#include "common.h"
|
||||
|
||||
#define RT_CHECK(_expr) \
|
||||
do { \
|
||||
int _ret = _expr; \
|
||||
if (0 == _ret) \
|
||||
break; \
|
||||
printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \
|
||||
cleanup(); \
|
||||
exit(-1); \
|
||||
} while (false)
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
const char* kernel_file = "kernel.bin";
|
||||
uint32_t count = 0;
|
||||
|
||||
std::vector<float> src_a_data;
|
||||
std::vector<float> src_b_data;
|
||||
std::vector<float> ref_data;
|
||||
|
||||
vx_device_h device = nullptr;
|
||||
std::vector<uint8_t> staging_buf;
|
||||
kernel_arg_t kernel_arg = {};
|
||||
|
||||
static void show_usage() {
|
||||
std::cout << "Vortex Test." << std::endl;
|
||||
std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl;
|
||||
}
|
||||
|
||||
static void parse_args(int argc, char **argv) {
|
||||
int c;
|
||||
while ((c = getopt(argc, argv, "n:k:h?")) != -1) {
|
||||
switch (c) {
|
||||
case 'n':
|
||||
count = atoi(optarg);
|
||||
break;
|
||||
case 'k':
|
||||
kernel_file = optarg;
|
||||
break;
|
||||
case 'h':
|
||||
case '?': {
|
||||
show_usage();
|
||||
exit(0);
|
||||
} break;
|
||||
default:
|
||||
show_usage();
|
||||
exit(-1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void cleanup() {
|
||||
if (device) {
|
||||
vx_mem_free(device, kernel_arg.addr_a);
|
||||
vx_mem_free(device, kernel_arg.addr_b);
|
||||
vx_mem_free(device, kernel_arg.addr_c);
|
||||
vx_dev_close(device);
|
||||
}
|
||||
}
|
||||
|
||||
void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) {
|
||||
src_a_data.resize(dim_m * dim_k);
|
||||
src_b_data.resize(dim_k * dim_n);
|
||||
|
||||
for (uint32_t i = 0; i < src_a_data.size(); ++i) {
|
||||
src_a_data[i] = static_cast<float>(i);
|
||||
std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl;
|
||||
}
|
||||
for (uint32_t i = 0; i < src_b_data.size(); ++i) {
|
||||
src_b_data[i] = static_cast<float>(i);
|
||||
std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) {
|
||||
ref_data.resize(dim_m * dim_n);
|
||||
|
||||
for (uint32_t i = 0; i < dim_m; ++i) {
|
||||
for (uint32_t j = 0; j < dim_n; ++j) {
|
||||
float ref = 0.0f;
|
||||
for (uint32_t k = 0; k < dim_k; ++k) {
|
||||
ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j];
|
||||
}
|
||||
ref_data.at(dim_n * i + j) = ref;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int run_test(const kernel_arg_t& kernel_arg,
|
||||
uint32_t buf_size,
|
||||
uint32_t dim_m, uint32_t dim_n) {
|
||||
// start device
|
||||
std::cout << "start device" << std::endl;
|
||||
RT_CHECK(vx_start(device));
|
||||
|
||||
// wait for completion
|
||||
std::cout << "wait for completion" << std::endl;
|
||||
RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT));
|
||||
|
||||
// download destination buffer
|
||||
std::cout << "download destination buffer" << std::endl;
|
||||
RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size));
|
||||
|
||||
// verify result
|
||||
std::cout << "verify result" << std::endl;
|
||||
{
|
||||
int errors = 0;
|
||||
auto buf_ptr = (float*)staging_buf.data();
|
||||
for (uint32_t i = 0; i < dim_m * dim_n; ++i) {
|
||||
float ref = ref_data.at(i);
|
||||
float cur = buf_ptr[i];
|
||||
if (std::abs((cur - ref) / ref) > 1e-6) {
|
||||
std::cout << "error at result #" << std::dec << i
|
||||
<< std::hex << ": actual=" << cur << ", expected=" << ref << std::endl;
|
||||
++errors;
|
||||
}
|
||||
}
|
||||
if (errors != 0) {
|
||||
std::cout << "Found " << std::dec << errors << " errors!" << std::endl;
|
||||
std::cout << "FAILED!" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
// parse command arguments
|
||||
parse_args(argc, argv);
|
||||
|
||||
if (count == 0) {
|
||||
count = 1;
|
||||
}
|
||||
|
||||
std::srand(50);
|
||||
|
||||
// open device connection
|
||||
std::cout << "open device connection" << std::endl;
|
||||
RT_CHECK(vx_dev_open(&device));
|
||||
|
||||
// FIXME: hardcoded
|
||||
uint32_t dim_m = 64;
|
||||
uint32_t dim_n = 64;
|
||||
uint32_t dim_k = 64;
|
||||
|
||||
generate_source_matrix(dim_m, dim_n, dim_k);
|
||||
generate_reference_matmul(dim_m, dim_n, dim_k);
|
||||
|
||||
uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]);
|
||||
uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]);
|
||||
uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]);
|
||||
|
||||
std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl;
|
||||
|
||||
// upload program
|
||||
std::cout << "upload program" << std::endl;
|
||||
RT_CHECK(vx_upload_kernel_file(device, kernel_file));
|
||||
|
||||
// allocate device memory
|
||||
std::cout << "allocate device memory" << std::endl;
|
||||
RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a));
|
||||
RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b));
|
||||
RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c));
|
||||
|
||||
kernel_arg.dim_m = dim_m;
|
||||
kernel_arg.dim_n = dim_n;
|
||||
kernel_arg.dim_k = dim_k;
|
||||
|
||||
std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl;
|
||||
std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl;
|
||||
std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl;
|
||||
|
||||
// allocate staging buffer
|
||||
{
|
||||
std::cout << "allocate staging buffer" << std::endl;
|
||||
uint32_t staging_buf_size = std::max<uint32_t>(
|
||||
src_a_buf_size,
|
||||
std::max<uint32_t>(
|
||||
src_b_buf_size,
|
||||
std::max<uint32_t>(dst_buf_size, sizeof(kernel_arg_t))));
|
||||
staging_buf.resize(staging_buf_size);
|
||||
}
|
||||
|
||||
// upload kernel argument
|
||||
{
|
||||
std::cout << "upload kernel argument" << std::endl;
|
||||
auto buf_ptr = staging_buf.data();
|
||||
memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t));
|
||||
RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t)));
|
||||
|
||||
std::cout << "uploading argument buffer to device, device mem address="
|
||||
<< std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec
|
||||
<< sizeof(kernel_arg_t) << " bytes\n";
|
||||
std::ofstream file("args.bin", std::ios::binary | std::ios::out);
|
||||
if (!file) {
|
||||
std::cerr << "error: failed to open args.bin for writing\n";
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
file.write(reinterpret_cast<char *>(staging_buf.data()),
|
||||
sizeof(kernel_arg_t));
|
||||
file.close();
|
||||
}
|
||||
|
||||
// upload source buffer
|
||||
{
|
||||
{
|
||||
auto buf_ptr = staging_buf.data();
|
||||
memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float));
|
||||
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(),
|
||||
src_a_buf_size));
|
||||
|
||||
std::cout << "uploading source A matrix to device, device mem address="
|
||||
<< std::hex << kernel_arg.addr_a << ", size=" << std::dec
|
||||
<< src_a_buf_size << " bytes\n";
|
||||
std::ofstream file("input.a.bin", std::ios::binary | std::ios::out);
|
||||
if (!file) {
|
||||
std::cerr << "error: failed to open args.bin for writing\n";
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
file.write(reinterpret_cast<char *>(buf_ptr), src_a_buf_size);
|
||||
file.close();
|
||||
}
|
||||
{
|
||||
auto buf_ptr = staging_buf.data();
|
||||
memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float));
|
||||
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(),
|
||||
src_b_buf_size));
|
||||
|
||||
std::cout << "uploading source B matrix to device, device mem address="
|
||||
<< std::hex << kernel_arg.addr_b << ", size=" << std::dec
|
||||
<< src_b_buf_size << " bytes\n";
|
||||
std::ofstream file("input.b.bin", std::ios::binary | std::ios::out);
|
||||
if (!file) {
|
||||
std::cerr << "error: failed to open args.bin for writing\n";
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
file.write(reinterpret_cast<char *>(buf_ptr), src_b_buf_size);
|
||||
file.close();
|
||||
}
|
||||
}
|
||||
|
||||
// clear destination buffer
|
||||
{
|
||||
std::cout << "clear destination buffer" << std::endl;
|
||||
auto buf_ptr = (int32_t*)staging_buf.data();
|
||||
for (uint32_t i = 0; i < ref_data.size(); ++i) {
|
||||
buf_ptr[i] = 0xdeadbeef;
|
||||
}
|
||||
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size));
|
||||
}
|
||||
|
||||
// run tests
|
||||
std::cout << "run tests" << std::endl;
|
||||
RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n));
|
||||
std::cout << "PASSED!" << std::endl;
|
||||
|
||||
// cleanup
|
||||
std::cout << "cleanup" << std::endl;
|
||||
cleanup();
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user