From 04a64dee7a85568ed286c34afae51a68f93668a5 Mon Sep 17 00:00:00 2001 From: Richard Yan Date: Thu, 24 Oct 2024 17:12:02 -0700 Subject: [PATCH 1/4] fp16 dma kernel --- .../sgemm_gemmini_dma/generate_operands.py | 6 +-- tests/regression/sgemm_gemmini_dma/kernel.cpp | 47 ++++++++++++++----- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/tests/regression/sgemm_gemmini_dma/generate_operands.py b/tests/regression/sgemm_gemmini_dma/generate_operands.py index a0c0e68a..414b07cf 100644 --- a/tests/regression/sgemm_gemmini_dma/generate_operands.py +++ b/tests/regression/sgemm_gemmini_dma/generate_operands.py @@ -21,15 +21,15 @@ matrix_a = generate_fp16_matrix(size) matrix_b = generate_fp16_matrix(size) # Save the operand matrices to binary files -# save_matrix_to_bin("input.a.bin", matrix_a) -# save_matrix_to_bin("input.b.bin", matrix_b) +save_matrix_to_bin("input.a.bin", matrix_a) +save_matrix_to_bin("input.b.bin", matrix_b) # Generate and save the reference matrices for 128x128, 256x256, and 512x512 sizes sizes = [128, 256, 512] for s in sizes: ref_matrix = truncated_matrix_multiplication(matrix_a, matrix_b, s) print(ref_matrix) - # save_matrix_to_bin(f"ref{s}.bin", ref_matrix) + save_matrix_to_bin(f"ref{s}.bin", ref_matrix) print("All files generated successfully.") diff --git a/tests/regression/sgemm_gemmini_dma/kernel.cpp b/tests/regression/sgemm_gemmini_dma/kernel.cpp index 6391e500..85ca285a 100644 --- a/tests/regression/sgemm_gemmini_dma/kernel.cpp +++ b/tests/regression/sgemm_gemmini_dma/kernel.cpp @@ -8,9 +8,9 @@ // fp16 16x16 #define TILE_M 128 -#define TILE_N 128 +#define TILE_N 64 #define TILE_K 128 -#define BOUND_INST 0x800080008ULL +#define BOUND_INST 0x800040008ULL #define NUM_THREADS_IN_CLUSTER 512 // fp32 8x8 @@ -52,12 +52,12 @@ #define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) // #define PRINTF(...) vx_printf(__VA_ARGS__) #define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x)))) -#define POWER +//#define POWER typedef uint16_t smem_elem_t; // typedef float smem_elem_t; -inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { +inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) __attribute__((convergent)) { vx_fence(); vx_barrier(barrier_id, count); } @@ -79,6 +79,22 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, } vx_fence(); + // if (HW_TID() < 128) { + // *((volatile uint32_t *) 0xff000000 + HW_TID()) = HW_TID(); + // for (int i = 0; i < 128; i++) { + // if (HW_TID() == i) { + // volatile uint32_t x = *((volatile uint32_t *) 0xff000000 + HW_TID()); + // if (x != i) { + // PRINTF("%d ", x); + // } + // } + // } + // } + // threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + // if (HW_TID() == 0) { + // PRINTF("\n finished\n"); + // } + // threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); uint32_t marker0, marker1; rd_cycles_force(marker0); @@ -93,24 +109,31 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const uint32_t num_tile_rows_per_tb = num_tiles_m / NUM_CLUSTERS; + if (HW_TID() == 0) gemmini_fence(); + threadblock_barrier(3, NUM_WARPS); + if (HW_TID() == 0) gemmini_fence(); + threadblock_barrier(3, NUM_WARPS); + if (HW_TID() == 0) gemmini_fence(); + threadblock_barrier(3, NUM_WARPS); + if (HW_TID() == 0) gemmini_fence(); + threadblock_barrier(3, NUM_WARPS); + if (HW_TID() == 0) { gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 0); gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 1); // gemmini_extended3_config_ld(repeating_bias ? 0 : (stride_D * sizeof_D), D_scale_factor, low_D, 2); gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); // gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale); - } - for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id; - tile_i < num_tile_rows_per_tb * (threadblock_id + 1); - tile_i += 1) { - for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { - if (HW_TID() == 0) { + for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id; + tile_i < num_tile_rows_per_tb * (threadblock_id + 1); + tile_i += 1) { + for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) { ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, (uint64_t) (A + tile_i * TILE_M * dim_k + tile_k * TILE_K), (uint64_t) (B + tile_k * TILE_K * dim_n + tile_j * TILE_N), k_LOOP_WS_CONFIG_ADDRS_AB) - GEMMINI_CISC_CMD_R((dim_n) << 16 | (dim_k << 8) | 8); + GEMMINI_CISC_CMD_R((dim_n << 20) | (dim_k << 8) | 8); if (tile_k & 1) { GEMMINI_CISC_CMD_I(11); } else { @@ -157,7 +180,7 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, rd_cycles_force(marker1); if (HW_TID() == 0) { #ifdef POWER - PRINTF("%d\n", marker1 - marker0); + // PRINTF("%d\n", marker1 - marker0); #else PRINTF("\ncomplete\n"); PRINTF("total cycles: %d\n", marker1 - marker0); From 8cc0c3bae433ede3e7e8570ef0de2a3a58e7d77d Mon Sep 17 00:00:00 2001 From: Richard Yan Date: Thu, 24 Oct 2024 17:12:29 -0700 Subject: [PATCH 2/4] fp16 no dma kernel --- tests/regression/sgemm_gemmini/kernel.cpp | 154 +++++++++++++--------- 1 file changed, 91 insertions(+), 63 deletions(-) diff --git a/tests/regression/sgemm_gemmini/kernel.cpp b/tests/regression/sgemm_gemmini/kernel.cpp index c609eec1..28f65d7c 100644 --- a/tests/regression/sgemm_gemmini/kernel.cpp +++ b/tests/regression/sgemm_gemmini/kernel.cpp @@ -6,25 +6,53 @@ #include "include/gemmini.h" #include "gemmini_mmio.h" +#define NUM_CLUSTERS 1 +// #define FP32 + +#ifdef FP32 +// fp32 #define TILE_M 64 #define TILE_N 64 #define TILE_K 64 #define TILE_MN 4096 #define TILE_MK 4096 #define TILE_NK 4096 - -#define NUM_CLUSTERS 1 #define NUM_THREADS_IN_CLUSTER 256 -#define SMEM_ADDR_Q0 ((float * const) 0xff000000) -#define SMEM_ADDR_Q1 ((float * const) 0xff004000) -#define SMEM_ADDR_Q2 ((float * const) 0xff008000) -#define SMEM_ADDR_Q3 ((float * const) 0xff00c000) +#define SMEM_ADDR_Q0 ((mem_elem_t * const) 0xff000000) +#define SMEM_ADDR_Q1 ((mem_elem_t * const) 0xff004000) +#define SMEM_ADDR_Q2 ((mem_elem_t * const) 0xff008000) +#define SMEM_ADDR_Q3 ((mem_elem_t * const) 0xff00c000) #define SPAD_ADDR_Q0 0x0 #define SPAD_ADDR_Q1 0x200 #define SPAD_ADDR_Q2 0x400 #define SPAD_ADDR_Q3 0x600 #define SPAD_ADDR_Q4 0x800 +typedef float smem_elem_t; +typedef float mem_elem_t; + +#else +// fp16 +#define TILE_M 128 +#define TILE_N 64 +#define TILE_K 128 +#define TILE_MN 8192 +#define TILE_MK 16384 +#define TILE_NK 8192 +#define NUM_THREADS_IN_CLUSTER 512 + +#define SMEM_ADDR_Q0 ((mem_elem_t * const) 0xff000000) +#define SMEM_ADDR_Q1 ((mem_elem_t * const) 0xff008000) +#define SMEM_ADDR_Q2 ((mem_elem_t * const) 0xff001000) +#define SMEM_ADDR_Q3 ((mem_elem_t * const) 0xff018000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x400 +#define SPAD_ADDR_Q2 0x800 +#define SPAD_ADDR_Q3 0xc00 +#define SPAD_ADDR_Q4 0x1000 +typedef uint16_t smem_elem_t; +typedef uint32_t mem_elem_t; +#endif #define HARDCODE #define REGBLOCK @@ -61,9 +89,9 @@ inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const uint32_t threadblock_id, const uint32_t tid_in_threadblock) { - const float * const A = (const float * const) arg->addr_a; - const float * const B = (const float * const) arg->addr_b; - float * const C = (float * const) arg->addr_c; + const smem_elem_t * const A = (const smem_elem_t * const) arg->addr_a; + const smem_elem_t * const B = (const smem_elem_t * const) arg->addr_b; + smem_elem_t * const C = (smem_elem_t * const) arg->addr_c; if (tid_in_threadblock % NUM_THREADS_IN_CLUSTER == 0) { gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); @@ -123,11 +151,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, tile_i < num_tile_rows_per_tb * (threadblock_id + 1); tile_i += 1) { for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { - float * const smem_c_tile_start = SMEM_ADDR_Q1; + mem_elem_t * const smem_c_tile_start = SMEM_ADDR_Q1; #ifdef OFFLOAD_ACCUMULATE - float * const smem_acc_tile_start = SMEM_ADDR_Q0 + HW_TID(); + mem_elem_t * const smem_acc_tile_start = SMEM_ADDR_Q0 + HW_TID(); #else - float * const smem_acc_tile_start = SMEM_ADDR_Q2 + hw_tid; + mem_elem_t * const smem_acc_tile_start = SMEM_ADDR_Q2 + hw_tid; #endif for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) { @@ -140,19 +168,19 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, // #endif constexpr uint32_t every_iter = j1_stride; - const uint32_t every_2iters_a = i1_stride * dim_k; - const uint32_t runtime_const_a = i0 * dim_k + j1_idx + j0; + const uint32_t every_2iters_a = i1_stride * (dim_k * sizeof(smem_elem_t) / 4); + const uint32_t runtime_const_a = i0 * (dim_k * sizeof(smem_elem_t) / 4) + j1_idx + j0; const uint32_t every_2iters_b = i1_stride * dim_n; const uint32_t runtime_const_b = i0 * dim_n + j1_idx + j0; - const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K + runtime_const_a; - const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N + runtime_const_b; + const mem_elem_t * const dram_a_tile_start = (const mem_elem_t * const) (A + tile_i * TILE_M * dim_k + tile_k * TILE_K + runtime_const_a); + const mem_elem_t * const dram_b_tile_start = (const mem_elem_t * const) (B + tile_k * TILE_K * dim_n + tile_j * TILE_N + runtime_const_b); #ifdef DBUF - float * const smem_a_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q1 : SMEM_ADDR_Q0) + HW_TID(); - float * const smem_b_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q3 : SMEM_ADDR_Q2) + HW_TID(); + mem_elem_t * const smem_a_tile_start = (mem_elem_t * const) (((tile_k & 1) ? SMEM_ADDR_Q1 : SMEM_ADDR_Q0) + HW_TID()); + mem_elem_t * const smem_b_tile_start = (mem_elem_t * const) (((tile_k & 1) ? SMEM_ADDR_Q3 : SMEM_ADDR_Q2) + HW_TID()); #else - float * const smem_a_tile_start = SMEM_ADDR_Q0 + HW_TID(); - float * const smem_b_tile_start = SMEM_ADDR_Q3 + HW_TID(); + mem_elem_t * const smem_a_tile_start = (mem_elem_t * const) (SMEM_ADDR_Q0 + HW_TID()); + mem_elem_t * const smem_b_tile_start = (mem_elem_t * const) (SMEM_ADDR_Q3 + HW_TID()); #endif { @@ -191,10 +219,10 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_b_tile_start[7 * num_threads_in_cluster + hw_tid] = \ dram_b_tile_start[every_iter * 1 + every_2iters_b * 3]; #else - float v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 0]; - float v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 0]; - float v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 1]; - float v3 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 1]; + mem_elem_t v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 0]; + mem_elem_t v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 0]; + mem_elem_t v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 1]; + mem_elem_t v3 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 1]; smem_a_tile_start[0 * num_threads_in_cluster] = v0; smem_a_tile_start[1 * num_threads_in_cluster] = v1; smem_a_tile_start[2 * num_threads_in_cluster] = v2; @@ -236,14 +264,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_a_tile_start[10 * num_threads_in_cluster] = v2; smem_a_tile_start[11 * num_threads_in_cluster] = v3; - v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 4]; - v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 4]; - v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 5]; - v3 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 5]; - smem_b_tile_start[8 * num_threads_in_cluster] = v0; - smem_b_tile_start[9 * num_threads_in_cluster] = v1; - smem_b_tile_start[10 * num_threads_in_cluster] = v2; - smem_b_tile_start[11 * num_threads_in_cluster] = v3; + // v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 4]; + // v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 4]; + // v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 5]; + // v3 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 5]; + // smem_b_tile_start[8 * num_threads_in_cluster] = v0; + // smem_b_tile_start[9 * num_threads_in_cluster] = v1; + // smem_b_tile_start[10 * num_threads_in_cluster] = v2; + // smem_b_tile_start[11 * num_threads_in_cluster] = v3; v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 6]; v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 6]; @@ -254,14 +282,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_a_tile_start[14 * num_threads_in_cluster] = v2; smem_a_tile_start[15 * num_threads_in_cluster] = v3; - v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 6]; - v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 6]; - v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 7]; - v3 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 7]; - smem_b_tile_start[12 * num_threads_in_cluster] = v0; - smem_b_tile_start[13 * num_threads_in_cluster] = v1; - smem_b_tile_start[14 * num_threads_in_cluster] = v2; - smem_b_tile_start[15 * num_threads_in_cluster] = v3; + // v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 6]; + // v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 6]; + // v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 7]; + // v3 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 7]; + // smem_b_tile_start[12 * num_threads_in_cluster] = v0; + // smem_b_tile_start[13 * num_threads_in_cluster] = v1; + // smem_b_tile_start[14 * num_threads_in_cluster] = v2; + // smem_b_tile_start[15 * num_threads_in_cluster] = v3; #endif } #else @@ -440,8 +468,8 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, #ifdef CISC GEMMINI_CISC_CMD_I(9); #else - ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (((uint64_t) TILE_M / DIM) << 32) | - (((uint64_t) TILE_K / DIM) << 16) | ((uint64_t) TILE_N / DIM), k_LOOP_WS_CONFIG_BOUNDS) + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (((uint64_t) TILE_K / DIM) << 32) | + (((uint64_t) TILE_N / DIM) << 16) | ((uint64_t) TILE_M / DIM), k_LOOP_WS_CONFIG_BOUNDS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, 0x278U, k_LOOP_WS) #endif gemmini_fence(); @@ -458,13 +486,13 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, constexpr uint32_t every_iter = j1_stride; const uint32_t every_2iters = i1_stride * dim_n; const uint32_t runtime_const = i0 * dim_n + j1_idx + j0; - float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N + runtime_const; + mem_elem_t * const dram_c_tile_start = (mem_elem_t * const) (C + tile_i * TILE_M * dim_n + tile_j * TILE_N + runtime_const); #ifdef REGBLOCK - float v0 = smem_acc_tile_start[0 * num_threads_in_cluster]; - float v1 = smem_acc_tile_start[1 * num_threads_in_cluster]; - float v2 = smem_acc_tile_start[2 * num_threads_in_cluster]; - float v3 = smem_acc_tile_start[3 * num_threads_in_cluster]; + mem_elem_t v0 = smem_acc_tile_start[0 * num_threads_in_cluster]; + mem_elem_t v1 = smem_acc_tile_start[1 * num_threads_in_cluster]; + mem_elem_t v2 = smem_acc_tile_start[2 * num_threads_in_cluster]; + mem_elem_t v3 = smem_acc_tile_start[3 * num_threads_in_cluster]; #ifdef ACTIVATE uint32_t swish_start, swish_end; rd_cycles_force(swish_start); @@ -498,23 +526,23 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, dram_c_tile_start[every_iter * 0 + every_2iters * 3] = v2; dram_c_tile_start[every_iter * 1 + every_2iters * 3] = v3; - v0 = smem_acc_tile_start[8 * num_threads_in_cluster]; - v1 = smem_acc_tile_start[9 * num_threads_in_cluster]; - v2 = smem_acc_tile_start[10 * num_threads_in_cluster]; - v3 = smem_acc_tile_start[11 * num_threads_in_cluster]; - dram_c_tile_start[every_iter * 0 + every_2iters * 4] = v0; - dram_c_tile_start[every_iter * 1 + every_2iters * 4] = v1; - dram_c_tile_start[every_iter * 0 + every_2iters * 5] = v2; - dram_c_tile_start[every_iter * 1 + every_2iters * 5] = v3; + // v0 = smem_acc_tile_start[8 * num_threads_in_cluster]; + // v1 = smem_acc_tile_start[9 * num_threads_in_cluster]; + // v2 = smem_acc_tile_start[10 * num_threads_in_cluster]; + // v3 = smem_acc_tile_start[11 * num_threads_in_cluster]; + // dram_c_tile_start[every_iter * 0 + every_2iters * 4] = v0; + // dram_c_tile_start[every_iter * 1 + every_2iters * 4] = v1; + // dram_c_tile_start[every_iter * 0 + every_2iters * 5] = v2; + // dram_c_tile_start[every_iter * 1 + every_2iters * 5] = v3; - v0 = smem_acc_tile_start[12 * num_threads_in_cluster]; - v1 = smem_acc_tile_start[13 * num_threads_in_cluster]; - v2 = smem_acc_tile_start[14 * num_threads_in_cluster]; - v3 = smem_acc_tile_start[15 * num_threads_in_cluster]; - dram_c_tile_start[every_iter * 0 + every_2iters * 6] = v0; - dram_c_tile_start[every_iter * 1 + every_2iters * 6] = v1; - dram_c_tile_start[every_iter * 0 + every_2iters * 7] = v2; - dram_c_tile_start[every_iter * 1 + every_2iters * 7] = v3; + // v0 = smem_acc_tile_start[12 * num_threads_in_cluster]; + // v1 = smem_acc_tile_start[13 * num_threads_in_cluster]; + // v2 = smem_acc_tile_start[14 * num_threads_in_cluster]; + // v3 = smem_acc_tile_start[15 * num_threads_in_cluster]; + // dram_c_tile_start[every_iter * 0 + every_2iters * 6] = v0; + // dram_c_tile_start[every_iter * 1 + every_2iters * 6] = v1; + // dram_c_tile_start[every_iter * 0 + every_2iters * 7] = v2; + // dram_c_tile_start[every_iter * 1 + every_2iters * 7] = v3; #else dram_c_tile_start[every_iter * 0 + every_2iters * 0] = \ smem_acc_tile_start[0 * num_threads_in_cluster]; From fb928b5cda18a5970e84d57afda7fb08cc9b30da Mon Sep 17 00:00:00 2001 From: Richard Yan Date: Thu, 24 Oct 2024 17:13:13 -0700 Subject: [PATCH 3/4] new unaligned access kernel, update idle kernel --- tests/regression/idle/.gitignore | 5 + tests/regression/idle/Makefile | 2 +- tests/regression/idle/kernel.cpp | 46 ++++++- tests/regression/unaligned/Makefile | 9 ++ tests/regression/unaligned/common.h | 13 ++ tests/regression/unaligned/kernel.cpp | 123 ++++++++++++++++++ tests/regression/unaligned/kernel.minimal.cpp | 44 +++++++ tests/regression/unaligned/main.cpp | 92 +++++++++++++ tests/regression/unaligned/unaligned | Bin 0 -> 18001 bytes 9 files changed, 328 insertions(+), 6 deletions(-) create mode 100644 tests/regression/idle/.gitignore create mode 100644 tests/regression/unaligned/Makefile create mode 100644 tests/regression/unaligned/common.h create mode 100644 tests/regression/unaligned/kernel.cpp create mode 100644 tests/regression/unaligned/kernel.minimal.cpp create mode 100644 tests/regression/unaligned/main.cpp create mode 100755 tests/regression/unaligned/unaligned diff --git a/tests/regression/idle/.gitignore b/tests/regression/idle/.gitignore new file mode 100644 index 00000000..c24ab099 --- /dev/null +++ b/tests/regression/idle/.gitignore @@ -0,0 +1,5 @@ +*.bin +*.dump +*.elf +idle +.depend diff --git a/tests/regression/idle/Makefile b/tests/regression/idle/Makefile index 3a8ffb18..b5c78ec0 100644 --- a/tests/regression/idle/Makefile +++ b/tests/regression/idle/Makefile @@ -1,4 +1,4 @@ -PROJECT = sgemm_gemmini_dma +PROJECT = idle SRCS = main.cpp common.h diff --git a/tests/regression/idle/kernel.cpp b/tests/regression/idle/kernel.cpp index ccd9bcc5..48c5a1a7 100644 --- a/tests/regression/idle/kernel.cpp +++ b/tests/regression/idle/kernel.cpp @@ -7,7 +7,7 @@ #include "gemmini_mmio.h" #define NUM_CLUSTERS 1 -#define NUM_THREADS_IN_CLUSTER 256 +#define NUM_THREADS_IN_CLUSTER 512 #define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) @@ -22,9 +22,45 @@ void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { // reinterpret_cast(arg->addr_c)[0] = counter; // call barrier in a divergent branch, which will hang the core - if ((vx_thread_id() % NUM_THREADS) == 0) { - vx_barrier(0, NUM_WARPS); - } + asm volatile("li x1, 0xa0a0a0a0"); + asm volatile("li x2, 0xa0a0a0a0"); + asm volatile("li x3, 0xa0a0a0a0"); + asm volatile("li x4, 0xa0a0a0a0"); + asm volatile("li x5, 0xa0a0a0a0"); + asm volatile("li x6, 0xa0a0a0a0"); + asm volatile("li x7, 0xa0a0a0a0"); + asm volatile("li x8, 0xa0a0a0a0"); + asm volatile("li x9, 0xa0a0a0a0"); + asm volatile("li x10, 0xa0a0a0a0"); + asm volatile("li x11, 0xa0a0a0a0"); + asm volatile("li x12, 0xa0a0a0a0"); + asm volatile("li x13, 0xa0a0a0a0"); + asm volatile("li x14, 0xa0a0a0a0"); + asm volatile("li x15, 0xa0a0a0a0"); + asm volatile("li x16, 0xa0a0a0a0"); + asm volatile("li x17, 0xa0a0a0a0"); + asm volatile("li x18, 0xa0a0a0a0"); + asm volatile("li x19, 0xa0a0a0a0"); + asm volatile("li x20, 0xa0a0a0a0"); + asm volatile("li x21, 0xa0a0a0a0"); + asm volatile("li x22, 0xa0a0a0a0"); + asm volatile("li x23, 0xa0a0a0a0"); + asm volatile("li x24, 0xa0a0a0a0"); + asm volatile("li x25, 0xa0a0a0a0"); + asm volatile("li x26, 0xa0a0a0a0"); + asm volatile("li x27, 0xa0a0a0a0"); + asm volatile("li x28, 0xa0a0a0a0"); + asm volatile("li x29, 0xa0a0a0a0"); + asm volatile("li x30, 0xa0a0a0a0"); + asm volatile("li x31, 0xa0a0a0a0"); + asm volatile("csrr a0, 0xcc1"); + asm volatile("beqz a0, bar"); + asm volatile("vx_tmc zero"); + asm volatile("bar:"); + asm volatile("vx_bar zero, a0"); + // if ((vx_thread_id() % NUM_THREADS) == 0) { + // vx_barrier(0, NUM_WARPS); + // } vx_tmc(0); } @@ -35,7 +71,7 @@ int main() { // spawn a single warp in every core const uint32_t grid_size = NUM_THREADS * NUM_CORES; #ifdef RADIANCE - vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); + vx_spawn_tasks_cluster(NUM_THREADS_IN_CLUSTER, (vx_spawn_tasks_cb)kernel_body, arg); #else vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); #endif diff --git a/tests/regression/unaligned/Makefile b/tests/regression/unaligned/Makefile new file mode 100644 index 00000000..a314fc4b --- /dev/null +++ b/tests/regression/unaligned/Makefile @@ -0,0 +1,9 @@ +PROJECT = unaligned + +SRCS = main.cpp common.h + +VX_SRCS = kernel.cpp + +OPTS ?= -n16 + +include ../common.mk diff --git a/tests/regression/unaligned/common.h b/tests/regression/unaligned/common.h new file mode 100644 index 00000000..c16f6f04 --- /dev/null +++ b/tests/regression/unaligned/common.h @@ -0,0 +1,13 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define KERNEL_ARG_DEV_MEM_ADDR 0x9fff0000 +#define DEV_SMEM_START_ADDR 0xff000000 + +typedef struct { + uint32_t placeholder; +} kernel_arg_t; + +#endif diff --git a/tests/regression/unaligned/kernel.cpp b/tests/regression/unaligned/kernel.cpp new file mode 100644 index 00000000..97258d07 --- /dev/null +++ b/tests/regression/unaligned/kernel.cpp @@ -0,0 +1,123 @@ +#include +#include +#include +#include +#include "common.h" + +#define NUM_THREADS_IN_CLUSTER 32 +#define NUM_CLUSTERS 1 + +#define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) +#define rd_cycles(x) rd_cycles_force(x) +#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#define PRINT_BUF ((char *) (0xff020000UL)) +#define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) + +inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) __attribute__((convergent)) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +#define ADDR0 0xff008004UL +#define ADDR1 0xff009004UL +#define ADDR2 0xff00a004UL +#define ADDR3 0xff00b004UL + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) __attribute__((convergent)) { + size_t t = (size_t) (task_id * 4) % 32; + if (t == 0) { + for (int j = 0; j < 0x400; j += 0x100) { + for (int i = 0; i < 8; i++) { + *((volatile uint32_t *) (ADDR0 + j + i * 4)) = 0xbeef; + *((volatile uint32_t *) (ADDR1 + j + i * 4)) = 0xbeef; + } + } + } + threadblock_barrier(0, 1); + // for (int i = 0; i < 8; i++) { + if (HW_TID() % 8 < 5) { + // if (true) { + asm volatile("lower_block:"); + volatile uint32_t a = *((volatile uint32_t *) (ADDR0 + 0x000 + t)); + volatile uint32_t b = *((volatile uint32_t *) (ADDR0 + 0x100 + t)); + volatile uint32_t c = *((volatile uint32_t *) (ADDR0 + 0x200 + t)); + volatile uint32_t d = *((volatile uint32_t *) (ADDR0 + 0x300 + t)); + + volatile uint32_t u = *((volatile uint32_t *) (ADDR1 + 0x000 + t)); + volatile uint32_t v = *((volatile uint32_t *) (ADDR1 + 0x100 + t)); + volatile uint32_t w = *((volatile uint32_t *) (ADDR1 + 0x200 + t)); + volatile uint32_t x = *((volatile uint32_t *) (ADDR1 + 0x300 + t)); + + *((volatile uint32_t *) (ADDR2 + 0x000 + t)) = a; + *((volatile uint32_t *) (ADDR2 + 0x100 + t)) = b; + *((volatile uint32_t *) (ADDR2 + 0x200 + t)) = c; + *((volatile uint32_t *) (ADDR2 + 0x300 + t)) = d; + + *((volatile uint32_t *) (ADDR3 + 0x000 + t)) = u; + *((volatile uint32_t *) (ADDR3 + 0x100 + t)) = v; + *((volatile uint32_t *) (ADDR3 + 0x200 + t)) = w; + *((volatile uint32_t *) (ADDR3 + 0x300 + t)) = x; + } else { + asm volatile("upper_block:"); + volatile uint32_t a = *((volatile uint32_t *) (ADDR1 + 0x000 + t)); + volatile uint32_t b = *((volatile uint32_t *) (ADDR1 + 0x100 + t)); + volatile uint32_t c = *((volatile uint32_t *) (ADDR1 + 0x200 + t)); + volatile uint32_t d = *((volatile uint32_t *) (ADDR1 + 0x300 + t)); + + volatile uint32_t u = *((volatile uint32_t *) (ADDR0 + 0x000 + t)); + volatile uint32_t v = *((volatile uint32_t *) (ADDR0 + 0x100 + t)); + volatile uint32_t w = *((volatile uint32_t *) (ADDR0 + 0x200 + t)); + volatile uint32_t x = *((volatile uint32_t *) (ADDR0 + 0x300 + t)); + + // for (int y = 4; y < 8; y++) { + // if (task_id == y) { + // PRINTF("Task ID: %d, a: %x, b: %x, c: %x, d: %x\n", task_id, a, b, c, d); + // PRINTF("Task ID: %d, u: %x, v: %x, w: %x, x: %x\n", task_id, u, v, w, x); + // } + // } + // threadblock_barrier(1, 1); + + *((volatile uint32_t *) (ADDR3 + 0x000 + t)) = a; + *((volatile uint32_t *) (ADDR3 + 0x100 + t)) = b; + *((volatile uint32_t *) (ADDR3 + 0x200 + t)) = c; + *((volatile uint32_t *) (ADDR3 + 0x300 + t)) = d; + + *((volatile uint32_t *) (ADDR2 + 0x000 + t)) = u; + *((volatile uint32_t *) (ADDR2 + 0x100 + t)) = v; + *((volatile uint32_t *) (ADDR2 + 0x200 + t)) = w; + *((volatile uint32_t *) (ADDR2 + 0x300 + t)) = x; + } + // } + threadblock_barrier(2, 1); + PRINTF("."); + if (task_id == 0) { + bool correct = true; + PRINTF("\n"); + for (int j = 0; j < 0x400; j += 0x100) { + for (int i = 0; i < 8; i++) { + int v2 = *((volatile uint32_t *) (ADDR2 + i * 4 + j)); + if (v2 != 0xbeef) { + correct = false; + PRINTF("mismatch at %x, got %x\n", ADDR2 + i * 4 + j, v2); + } + int v3 = *((volatile uint32_t *) (ADDR3 + i * 4 + j)); + if (v3 != 0xbeef) { + correct = false; + PRINTF("mismatch at %x, got %x\n", ADDR3 + i * 4 + j, v3); + } + } + } + if (correct) { + PRINTF("test passed\n"); + } + } +} + +int main() __attribute__((convergent)) { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + + const uint32_t num_threads_in_cluster = NUM_THREADS_IN_CLUSTER; + const uint32_t grid_size = num_threads_in_cluster * NUM_CLUSTERS; + vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); + return 0; +} diff --git a/tests/regression/unaligned/kernel.minimal.cpp b/tests/regression/unaligned/kernel.minimal.cpp new file mode 100644 index 00000000..1629f6ef --- /dev/null +++ b/tests/regression/unaligned/kernel.minimal.cpp @@ -0,0 +1,44 @@ +#include +#include +#include +#include +#include "common.h" + +#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) + +inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) __attribute__((convergent)) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +#define ADDR0 0xff008004UL +#define ADDR1 0xff009004UL + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { + // size_t t = (size_t) (task_id * 4) % 32; + asm volatile("nop"); + for (int i = 0; i < 8; i++) { + if (i == 0) { + if ((HW_TID() & 0x7) < 2) { + asm volatile("lower_block:"); + volatile uint32_t a = *((volatile uint32_t *) (ADDR0)); + // *((volatile uint32_t *) (ADDR2)) = a; + volatile uint32_t b = a + 1; + } else { + asm volatile("upper_block:"); + volatile uint32_t a = *((volatile uint32_t *) (ADDR1)); + // *((volatile uint32_t *) (ADDR3)) = a; + volatile uint32_t b = a + 1; + } + } + volatile uint32_t a = *((volatile uint32_t *) (ADDR1)); + } + threadblock_barrier(2, 2); +} + +int main() { // __attribute__((convergent)) { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + + vx_spawn_tasks_cluster(64, (vx_spawn_tasks_cb)kernel_body, arg); + return 0; +} diff --git a/tests/regression/unaligned/main.cpp b/tests/regression/unaligned/main.cpp new file mode 100644 index 00000000..a1b1d384 --- /dev/null +++ b/tests/regression/unaligned/main.cpp @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.bin"; +uint32_t count = 0; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + count = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_dev_close(device); + } +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + if (count == 0) { + count = 1; + } + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + return 0; +} diff --git a/tests/regression/unaligned/unaligned b/tests/regression/unaligned/unaligned new file mode 100755 index 0000000000000000000000000000000000000000..7fa0b6c09c38bc4cc36cd4096aeb48b4a3b1d4b0 GIT binary patch literal 18001 zcmeHPe{fXCeP8KxAoJrmFbV<_^T5COVGfC36{IqEAi=2-m>}U$8~k+A9l8srJLT>j zh%^)ED5O@g65PxnZEV^!Gq^L%ICLC~p^4pzoj}IT&~fe1q#obdi7TJuR7nii z&v$pfr*}_xNuA`MPH$)4efRs>@AtdA-`%(S-rLL`knw2+mX~ zOodnRr01rn=AHBcRRn=4rx)`xWU>$r>MXxqk4!eMR(g%CrBvv8k8x#mxyW6l*QE8D zv>sEh_9s*JCz+wEMcZ{!18Ect`74A|Z=cq4(l#y6l=7g^^|}@QZSvc#_54ef9VZPd zSTN=MJ_bGAL|Wq46K>``I=@Qokk$TVO6^TyO+41Sas8Tjcy&CM$PBF>YT3AY<9c5@ z>02w)sa%u?^{E}Z_6nGhO{v`|+k$Q(;$?cB`S`jg|ImGP|G;-|L|x50e>CGown2H4 z3?t}f8Aq<*9pQXd2G+ay@#=hHASuMGY-74UPA zEhpbq0gqL{KU)ERyh6Uesvtj7L4LG?d~F5!zpQ|Np#r|U0`5V+1}_g5~_V z61d?s0)A5k{Kvq{*?AiIE-Pc_?}5)1i-g~=68XN5YW%pyz2Id)zdxKzS&>o%b%(N9sS*EE(%L*PIGBbnmWGHMNilh>e zxOp%Jb)cz8D16jB5{g;Gg(HW}zIZY%lcZHflY^0jK)KRZxNp@ex(7E3^T94_eIyZ% zcl32x&CPw$P|CDYFqQ7;Yj58JY+Y|C9qTibX$$!bbab2DO=f%7MwIlyU0qg7ESWYT z5n11nh*@pT?T3Mf__xVhH1*0-S;hZMR&Eyf`pc_~Jtk{iDYC5{5&ELQ`a zGr(v}Y{E5iYz2lfxA;E#6=Bwjzd+Zk6W4V=Bl(n0L$b&$irqq^-%nV6A@Q|hLd)~~ z$^G|~9smtJKQivu&n?Rx_--v<@2A`*=RfAXehC-n#itC#U)iE4jlDeP5^~Lk&ru@s z_qq+IdUMFyaQpo`Zo_Y7VbBR1uB#+d`97w3hsJ&md><3OfI-lG+Rk*uwJT+6j}1pd z6+_g9b30Swpbh7qLHLjjm(N!OA{@8j_Idje8%|~7aLR^rTT|BMZvZNtZGIF0ojUa;YF8LZN(2dW;Zdf@+{2i~1?`!9pTKdBF% zbpP8LA%Z8ytlGkr;P5}!UzW8gY*+?3U%2ZIkU2{P$ww(ApZkwOp>Rs_G_mA!SBpGN zDEZu_B2N=ZKKFEyrwJsVd!oqG#F5WER^(~I$mjmF$kRlT&waAU(*%*vg^D~)4Efy0 ziabpS`COpL(?pQZH5YlB0P?w|MViiG@HPcg|bCw*3u)H7S_%Yggf~xutO6M!9$WX(R zl599cDa$BjwBd^cJ@4l4TgYt%mFao+-5?ZD|05^|5b~WwFdF54{66I=^K1TT$H>)( zgCjo*4*%!)?(WV}_YX*6bhar;YY;gV8fuZz&rZQwvn+&rC6P@NRs*`$Ybz1QPYRL*}wZ$p^y!uh~IY4hFr}7=nA`j?g$dOJQ$m~dv(u% zrP0=ZAtUqQ4#c0O_#-fg_|u4ATtT^wZg>@<*>Q!R0Zs#Yt_Qpv?@zYAM0G@)*5_s_ zzuNN;p4>udvvS9v;P4OW2R`08+WN}fLIfV@82Le9Z^y`c#UbKlz+IgyZcv}d{?#u~ zuuC^w){^E|w4Kh8eCNp97&-!l`9BN}U#-nu~!YnOI-M=u0LNkv=PyOxRasmfx+CSBUmhDw*1B+;dm@9wQa8GFapZ8+V15 z&Jb$d#~4f{`%|F-tY%=XNLwXSl3&hdXd%crm`owRfx&o0=GPaGgc6xSp;nH3y;#ps zVQ)IrAK7d?wEECyLnR&nlQ51XQ{gm4qnnLrBtCdR%a>h*(plfDEy#r_dVoeKS)MA& zJ+RN*8Q5obckFDxe{VO5*4*a$NEmb`uIq0X3a5$wSD`QlIu3e``0+xaiC(SVEfjh{ z2SJa6?fF@uFb;YibTRNNp!J~RphKWN@4*fz1zrQF`P3d!GvuwgZPxVqQ%ED6_DE#& zg#x`Gi?-=<+2Kw^*W>b|O!95n_Vo(F zEnHsUR7MVjKZ+}gI_xH}&Eu`z;i)Ixzr{5Md?Dcoggt}nQ5!}tMla$z2fGXDJJEKJ z_tDxmPvhrYL60$9=X$YbhNp3x$Gg>2AAnOSzxQ#Sey30v02M*x*W;;gLv%jsPri&0 z4m)3{?eH{y-qr3gM(f%=%O7>Od73^qE$C?(p1#B5Pk34ao~D3j`Bsmy)zi3DW+BVC zA9hZ|j(z=tOjbkH162=HJy7*P)dN)zR6S7j!2cf)@ctq{J%A$c{vrK1E+450XP)*x zQDC0-Hc^CHR*tjznHZqw=X>`Y;iB7*lb@qTFDBlF(PLXt$z(i$rQ zKf`+UrW;Mf6`hWC0U7G_rXPB5q0p#E$sQ+yw3_hieJ@%~(EdRR+C75z^6=iH>$E0| z!1y^`U@muy*4JXi{BKuk#eG;bhoBGnht7uT+>sUp4Ifcrem63(e%2e!nX(B zr*4DM=UjsKp_k%y;?rgAxQqS2RDN2q-YZ~c)j3$Q;OdrxWAR+GX(eRQhcW1 z{$Gk?G~jl#CAcsu7-eyfFv_&6OUw~XWpS@C?3)X99xike9&c>f!pjgg?1+lH#I1t& z<=azTxZBSywIeLwAb9+;r@F91k;ko4e1Rx;-?%Ue@HkT{k5STZM^xNp+t+MQbqO9t zCs7)1>cl&RNkqkHJgh~doYk8v}zKq(-F>m*vUIyfy{qysXpKH6n>a@A9NIRvriTeid zd9ZKj!pd)Bz`rT+sq)h>;gze`V&Jsj|7<|%aK86w-0xR7&%+-fymq1B@t<=F0;lqf z>;B2lDcurhsY%*Pc98ECN4|OZXI|~p@h7U~U(oVgo+R*cetx7vzAI6Lx$wW&ugb!9 z9;+b#wF>xi74R^XAAaETaBFmxzh6QAHQ?k=Xa9M#g8Z2ZIBlUKe;(KM@}eJ5co;bK zPvMtr@n;^=y^z1A{mBW3Wcit!q{pP4MN`UovV#11iF-wp?*BASQ5XYG$y3 zfg4hy^e3zBoYi*RI(b6krLx2=KnE_@&+i0oOcfXFD&RXS*b#I_!q+!Ah?5$f%}wcO z@`#zCwFMjm+0l9b)Zmw~ zg_MXH&I}A3g^G>CL6w+QB5Io3_XKvfo9(;W=zxxiGiGLcP=mp?J;H4J<6VKB9ovvr zq8u&-)L|TU$Nk&4ceUd*O<-$hJ6WAPv*P#5vnDuELZ?X7F_lS-PMMgIaL5X&Lopi< zV@)ZU>WFrQI->3EZENWWPRIRN6bi=@XoCoqf{wlL*_KjM@@&g*e+oz&Q|Dr6wWrJx zC4F3pPT8o#Hl-y%tEQ7?6vuSvG}MECpx3Gnm#!V5nehybenCrVd`HQk~z*vm6l68QJqKo zU!Qata4Nev2LB56f@{fUeZE2o1veRk9a^=4u? zl^P3cD6q9-HGCkXcr=ub3SanW0$C_(rBuq{NGeSWxKd0rCel)oc!(5qY%p#KAB_dV zhp4YVsUzt~pYY-M8CdB)Un(i@WM3qz$Dn8!)|gW^)L^7EIF77B12Gs=>S%r8!$3HI z!L7__{=p%7Zm+?GX;lV(?_|o~Pjs@B%HzC&N z??z1hI)TbaerA21_hY~)OV;Q2Y^MC(2_iC(4SV<`82YBe`usl5)LDL(WBM8B(>EgK z`MsTKkIsyJ%;ndx7)6A9%KH3%&y>GgIsM1=|B}|6>umooJM;~$$JE*39p=STPJR8I!O)cDb*Y@i6XzZJ{GEd7I4f$(_LzRd zp>ODiKc*)f6=pp)`i{EAV{|}u0;_@?n2fA}?`uzQd&NNV9ImJ!# zjMMu(^&) Date: Mon, 28 Oct 2024 13:47:13 -0700 Subject: [PATCH 4/4] update gemmini dma kernel --- tests/regression/sgemm_gemmini_dma/kernel.cpp | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tests/regression/sgemm_gemmini_dma/kernel.cpp b/tests/regression/sgemm_gemmini_dma/kernel.cpp index 4c37a232..2cbd8de5 100644 --- a/tests/regression/sgemm_gemmini_dma/kernel.cpp +++ b/tests/regression/sgemm_gemmini_dma/kernel.cpp @@ -49,6 +49,8 @@ #define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) #define rd_cycles(x) rd_cycles_force(x) #define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#define MARK_BEG() asm volatile ("slti x0, x1, -1047") +#define MARK_END() asm volatile ("slti x0, x1, -499") #define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) // #define PRINTF(...) vx_printf(__VA_ARGS__) #define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x)))) @@ -98,6 +100,7 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, uint32_t marker0, marker1; rd_cycles_force(marker0); + MARK_BEG(); const uint32_t dim_m = arg->dim_m; const uint32_t dim_n = arg->dim_n; @@ -109,15 +112,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const uint32_t num_tile_rows_per_tb = num_tiles_m / NUM_CLUSTERS; - if (HW_TID() == 0) gemmini_fence(); - threadblock_barrier(3, NUM_WARPS); - if (HW_TID() == 0) gemmini_fence(); - threadblock_barrier(3, NUM_WARPS); - if (HW_TID() == 0) gemmini_fence(); - threadblock_barrier(3, NUM_WARPS); - if (HW_TID() == 0) gemmini_fence(); - threadblock_barrier(3, NUM_WARPS); - if (HW_TID() == 0) { gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 0); gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 1); @@ -179,6 +173,7 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, // last thread block complete if (threadblock_id == NUM_CLUSTERS - 1) { threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + MARK_END(); rd_cycles_force(marker1); if (HW_TID() == 0) { #ifdef POWER