diff --git a/kernel/include/gemmini_mmio.h b/kernel/include/gemmini_mmio.h index 072fa8fc..17b18a79 100644 --- a/kernel/include/gemmini_mmio.h +++ b/kernel/include/gemmini_mmio.h @@ -6,8 +6,10 @@ #define SMEM_BASE 0xff000000 #define SMEM_SIZE 0x4000 + #define SMEM_MASK (SMEM_SIZE - 1) -#define SMEM_ADDR_END 0xff008000 +#define SMEM_ADDR_END (SMEM_BASE + SMEM_SIZE) +#define GEMMINI_CTRL (SMEM_BASE + SMEM_SIZE + 0x3000) #define SPAD_BASE 0x0 #define SPAD_ROW_SIZE (DIM * sizeof(elem_t)) @@ -15,14 +17,26 @@ #define SPAD_MASK (SPAD_NUM_ROWS - 1) #define PRINT_BUF ((char *) (SMEM_ADDR_END)) -#define GEMMINI_RS1_ADDR 0xff007010 -#define GEMMINI_RS2_ADDR 0xff007018 -#define GEMMINI_INST_ADDR 0xff007000 -#define GEMMINI_BUSY_ADDR 0xff007020 +#define GEMMINI_RS1_ADDR (GEMMINI_CTRL + 0x10) +#define GEMMINI_RS2_ADDR (GEMMINI_CTRL + 0x18) +#define GEMMINI_INST_ADDR (GEMMINI_CTRL + 0x0) +#define GEMMINI_BUSY_ADDR (GEMMINI_CTRL + 0x20) #define SMEM_TO_SPAD(smem_addr) (SPAD_BASE + ((smem_addr) & SMEM_MASK) / SPAD_ROW_SIZE) #define SPAD_TO_SMEM(spad_addr) (SMEM_BASE + ((spad_addr) & SPAD_MASK) * SPAD_ROW_SIZE) + +// CISC instructions: +// 0, 1, 2: tile-sized matmuls +// 0: k = 0, no accumulation +// 1: k % 2 = 0, buffer regions 0 +// 2: k % 2 = 1, buffer regions 1 +// 8, 9: memory ops +// 8: tile-sized move-in (unused) +// 9: tile-sized move-out +#define GEMMINI_CISC_CMD_I(x) asm("csrwi 0xacc, "#x) +#define GEMMINI_STATUS() ({uint32_t status; asm volatile ("csrr %0, 0xacc" : "=r" (status)); status;}) + // convert normal matrix i,j into tiled smem offset // top_in_tiles = i / DIM // left_in_tiles = j / DIM @@ -33,6 +47,7 @@ // #define fence() { for (int i = 0; i < 10; i++) *((volatile uint32_t *) (0xFFFF0000)) = 0xdeadbeef; } #undef gemmini_fence +//#define gemmini_fence() { while (GEMMINI_STATUS()); } #define gemmini_fence() { while (*((volatile uint32_t *) GEMMINI_BUSY_ADDR)) asm volatile ("nop"); } #undef ROCC_INSTRUCTION_RS1_RS2 diff --git a/tests/kernel/gemmini_mmio/gemmini_mmio.h b/tests/kernel/gemmini_mmio/gemmini_mmio.h deleted file mode 100644 index e2876927..00000000 --- a/tests/kernel/gemmini_mmio/gemmini_mmio.h +++ /dev/null @@ -1,162 +0,0 @@ -#ifndef GEMMINI_MMIO_H -#define GEMMINI_MMIO_H -#ifndef GEMMINI_PARAMS_H - #error INCLUDE GEMMINI.H FIRST -#endif - -#define SMEM_BASE 0xff000000 -#define SMEM_SIZE 0x4000 -#define SMEM_MASK (SMEM_SIZE - 1) -#define SMEM_ADDR_END 0xff008000 - -#define SPAD_BASE 0x0 -#define SPAD_ROW_SIZE (DIM * sizeof(elem_t)) -#define SPAD_NUM_ROWS (SMEM_SIZE / SPAD_ROW_SIZE) -#define SPAD_MASK (SPAD_NUM_ROWS - 1) - -#define PRINT_BUF ((char *) (SMEM_ADDR_END)) -#define GEMMINI_RS1_ADDR 0xff007010 -#define GEMMINI_RS2_ADDR 0xff007018 -#define GEMMINI_INST_ADDR 0xff007000 -#define GEMMINI_BUSY_ADDR 0xff007020 - -#define SMEM_TO_SPAD(smem_addr) (SPAD_BASE + ((smem_addr) & SMEM_MASK) / SPAD_ROW_SIZE) -#define SPAD_TO_SMEM(spad_addr) (SMEM_BASE + ((spad_addr) & SPAD_MASK) * SPAD_ROW_SIZE) - -// convert normal matrix i,j into tiled smem offset -// top_in_tiles = i / DIM -// left_in_tiles = j / DIM -// num_tiles_before_current = top_in_tiles * (J / DIM) + left_in_tiles -// smem_addr = num_tiles_before_current * DIM * DIM + (i % DIM) * DIM + (j % DIM) -#define SMEM_MAT_OFFSET(i, j, J) \ - (((i) / DIM * (J) / DIM + (j) / DIM) * DIM * DIM + ((i) % DIM) * DIM + ((j) % DIM)) - -// #define fence() { for (int i = 0; i < 10; i++) *((volatile uint32_t *) (0xFFFF0000)) = 0xdeadbeef; } -#undef gemmini_fence -#define gemmini_fence() { while (*((volatile uint32_t *) GEMMINI_BUSY_ADDR)) asm volatile ("nop"); } - -#undef ROCC_INSTRUCTION_RS1_RS2 -#define ROCC_INSTRUCTION_RS1_RS2(x, rs1, rs2, funct) { \ - /* printf("function %d\n", funct); */ \ - uint32_t instruction = (0x7B) | (0 << 7) | (3 << 12) | (1 << 15) | (2 << 20) | ((uint32_t) (funct) << 25); \ - *((volatile uint64_t *) GEMMINI_RS1_ADDR) = (volatile uint64_t) (rs1); \ - *((volatile uint64_t *) GEMMINI_RS2_ADDR) = (volatile uint64_t) (rs2); \ - /* *((volatile uint32_t*) GEMMINI_RS2_ADDR) = (uint32_t) ((uint64_t) (rs2) & 0xFFFFFFFFULL); */ \ - /* *((volatile uint32_t*) (GEMMINI_RS2_ADDR + 4)) = (uint32_t) ((uint64_t) (rs2) >> 32); */ \ - /* gemmini_fence(); */ \ - *((volatile uint32_t*) GEMMINI_INST_ADDR) = instruction; \ - /* sprintf((char *) PRINT_BUF, "%llx %llx %d\n", rs1, rs2, funct); */ \ -} - -static void sp_tiled_matmul_full_spad_ws(const uint32_t A_sp_addr_start, const uint32_t B_sp_addr_start, - const uint32_t D_sp_addr_start, const uint32_t C_dst_sp_addr_start, - size_t I, size_t J, size_t K, size_t pad_I, size_t pad_J, size_t pad_K, - bool a_transpose, bool b_transpose, - bool full_C, bool low_D, - bool no_bias, bool repeating_bias, - int act) { - - gemmini_loop_ws_spad(I, J, K, pad_I, pad_J, pad_K, - A_sp_addr_start, B_sp_addr_start + K * J * DIM, NULL, C_dst_sp_addr_start, - a_transpose, b_transpose, - full_C, low_D, false, - act, 0, 0, false); - /* - return; - - - // const uint32_t A_sp_addr_start = 0; - // const uint32_t B_sp_addr_start = BANK_NUM * BANK_ROWS - K * J * DIM; - // const uint32_t D_sp_addr_start = 1 << (ADDR_LEN-1); - const uint32_t C_sp_addr_start = 2 << (ADDR_LEN-2) | (full_C << (ADDR_LEN-3)); - // const int D_blocks = low_D ? (J <= MAX_BLOCK_LEN ? J : MAX_BLOCK_LEN) : - // (J <= MAX_BLOCK_LEN_ACC ? J : MAX_BLOCK_LEN_ACC); - const int C_blocks = 1; //full_C ? 1 : (J <= MAX_BLOCK_LEN ? J : MAX_BLOCK_LEN); - // const size_t sizeof_D = low_D ? sizeof(elem_t) : sizeof(acc_t); - const size_t sizeof_C = full_C ? sizeof(acc_t) : sizeof(elem_t); - gemmini_fence(); - - if (a_transpose || b_transpose || (I < 4)) { - for (size_t k = 0; k < K; k++) { - for (size_t j = 0; j < J; j++) { - for (size_t i = 0; i < I; i++) { - const uint32_t A_sp_addr = a_transpose ? (A_sp_addr_start + (k*I + i)*DIM) : - (A_sp_addr_start + (i*K + k)*DIM); - const uint32_t B_sp_addr = b_transpose ? (B_sp_addr_start + (j*K + k)*DIM) : - (B_sp_addr_start + (k*J + j)*DIM); - const uint32_t C_sp_addr = C_sp_addr_start + (i*J + j)*DIM; - // Compute - uint32_t pre_sp_addr = i == 0 ? B_sp_addr : GARBAGE_ADDR; - uint32_t out_sp_addr = C_sp_addr | ((k == 0 ? 0 : 1) << (ADDR_LEN-2)); - gemmini_extended_preload(pre_sp_addr, out_sp_addr, DIM, DIM, DIM, DIM); - if (i == 0) { // First iteration - gemmini_extended_compute_preloaded(A_sp_addr, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - } else { // All other iterations - gemmini_extended_compute_accumulated(A_sp_addr, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - } - if (k == K - 1) { - // Move-out C (if not normalizing) - // if (((act != LAYERNORM) && (act != SOFTMAX)) && (j == J-1 || j % C_blocks == C_blocks-1)) { - const size_t rounded_j = j; // (j / C_blocks) * C_blocks; - const uint32_t rounded_C_sp_addr = C_sp_addr; // C_sp_addr_start + (i*J + rounded_j)*DIM; - - const uint32_t C_dst_sp_addr = ((uint32_t) C_dst_sp_addr_start) + (i * J + rounded_j) * DIM; // * DIM * sizeof_C; - - // const size_t blocks = rounded_j + C_blocks <= J ? C_blocks : J-rounded_j; - constexpr size_t cols = DIM; // blocks * DIM - (rounded_j + blocks >= J ? pad_J : 0); - constexpr size_t rows = DIM; // DIM - (i == I - 1 ? pad_I : 0); - - gemmini_extended_mvout_spad(C_dst_sp_addr, 1, rounded_C_sp_addr, cols, rows); - // } - } - } - } - } - } else { - for (size_t k = 0; k < K; k++) { - for (size_t j = 0; j < J; j++) { - uint32_t A_sp_addr = A_sp_addr_start + k * DIM; // (i*K + k)*DIM; - const uint32_t B_sp_addr = B_sp_addr_start + (k*J + j)*DIM; - uint32_t C_sp_addr = C_sp_addr_start + j * DIM; // (i*J + j)*DIM; - for (size_t i = 0; i < I; i += 4) { - // Compute - // constexpr uint32_t pre_sp_addr = i == 0 ? B_sp_addr : GARBAGE_ADDR; - const uint32_t out_sp_addr = C_sp_addr | ((k == 0 ? 0 : 1) << (ADDR_LEN-2)); - if (i == 0) { // First iteration - gemmini_extended_preload(B_sp_addr, out_sp_addr, DIM, DIM, DIM, DIM); - gemmini_extended_compute_preloaded(A_sp_addr, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + J * DIM, DIM, DIM, DIM, DIM); - gemmini_extended_compute_accumulated(A_sp_addr + K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + 2 * J * DIM, DIM, DIM, DIM, DIM); - gemmini_extended_compute_accumulated(A_sp_addr + 2 * K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + 3 * J * DIM, DIM, DIM, DIM, DIM); - gemmini_extended_compute_accumulated(A_sp_addr + 3 * K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - } else { // All other iterations - gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr, DIM, DIM, DIM, DIM); - gemmini_extended_compute_accumulated(A_sp_addr, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + J * DIM, DIM, DIM, DIM, DIM); - gemmini_extended_compute_accumulated(A_sp_addr + K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + 2 * J * DIM, DIM, DIM, DIM, DIM); - gemmini_extended_compute_accumulated(A_sp_addr + 2 * K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + 3 * J * DIM, DIM, DIM, DIM, DIM); - gemmini_extended_compute_accumulated(A_sp_addr + 3 * K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM); - } - if (k == K - 1) { - for (int x = 0; x < 3; x++) gemmini_fence(); - gemmini_extended_mvout_spad((uint32_t) C_dst_sp_addr_start + (i * J + j) * DIM, 1, C_sp_addr, DIM, DIM); - gemmini_extended_mvout_spad((uint32_t) C_dst_sp_addr_start + ((i + 1) * J + j) * DIM, 1, C_sp_addr + J * DIM, DIM, DIM); - gemmini_extended_mvout_spad((uint32_t) C_dst_sp_addr_start + ((i + 2) * J + j) * DIM, 1, C_sp_addr + 2 * J * DIM, DIM, DIM); - gemmini_extended_mvout_spad((uint32_t) C_dst_sp_addr_start + ((i + 3) * J + j) * DIM, 1, C_sp_addr + 3 * J * DIM, DIM, DIM); - } - A_sp_addr += 4 * K * DIM; - C_sp_addr += 4 * J * DIM; - } - } - } - } - gemmini_fence(); - */ -} - - -#endif diff --git a/tests/kernel/gemmini_mmio/gemmini_mmio.h b/tests/kernel/gemmini_mmio/gemmini_mmio.h new file mode 120000 index 00000000..5e52dd4f --- /dev/null +++ b/tests/kernel/gemmini_mmio/gemmini_mmio.h @@ -0,0 +1 @@ +../../../kernel/include/gemmini_mmio.h \ No newline at end of file diff --git a/tests/kernel/gemmini_mmio/main.cpp b/tests/kernel/gemmini_mmio/main.cpp index d5be2558..34cbb495 100644 --- a/tests/kernel/gemmini_mmio/main.cpp +++ b/tests/kernel/gemmini_mmio/main.cpp @@ -111,7 +111,7 @@ int main() { sp_tiled_matmul_full_spad_ws(spad_A, spad_B, /*spad_D=*/0, spad_C, /*I=*/I, /*J=*/J, /*K=*/K, /*pad_I=*/0, /*pad_J=*/0, /*pad_K=*/0, /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, - /*no_bias=*/1, /*repeating_bias=*/0, /*act=*/NO_ACTIVATION); + /*acc=*/0, /*act=*/NO_ACTIVATION); rd_cycles(fence_cycles); gemmini_fence(); diff --git a/tests/regression/sgemm_gemmini/kernel.cpp b/tests/regression/sgemm_gemmini/kernel.cpp index f1893f6d..dcd931aa 100644 --- a/tests/regression/sgemm_gemmini/kernel.cpp +++ b/tests/regression/sgemm_gemmini/kernel.cpp @@ -25,11 +25,14 @@ #define SPAD_ADDR_8K 0x100 #define SPAD_ADDR_12K 0x180 -// #define DEBUG_PRINT -// #define EXT_ACCUMULATE +//#define DEBUG_PRINT +//#define EXT_ACCUMULATE #define HARDCODE +#define REGBLOCK #define DBUF -// #define DETAILED_PERF +//#define DETAILED_PERF +//#define ACTIVATE +#define CISC #define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) #ifdef DETAILED_PERF @@ -40,6 +43,7 @@ #define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) #define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) // #define PRINTF(...) vx_printf(__VA_ARGS__) +#define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x)))) inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { vx_fence(); @@ -65,6 +69,9 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, uint32_t marker0, marker1, marker2, marker3, marker4; uint32_t marker5, marker6, marker7, marker8, marker9; + #ifdef ACTIVATE + uint32_t swish_dur = 0; + #endif rd_cycles_force(marker0); const uint32_t dim_m = arg->dim_m; @@ -133,6 +140,41 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, #endif { + #ifndef REGBLOCK + smem_a_tile_start[0 * num_threads_in_cluster + hw_tid] = \ + dram_a_tile_start[every_iter * 0 + every_2iters_a * 0]; + smem_a_tile_start[1 * num_threads_in_cluster + hw_tid] = \ + dram_a_tile_start[every_iter * 1 + every_2iters_a * 0]; + smem_a_tile_start[2 * num_threads_in_cluster + hw_tid] = \ + dram_a_tile_start[every_iter * 0 + every_2iters_a * 1]; + smem_a_tile_start[3 * num_threads_in_cluster + hw_tid] = \ + dram_a_tile_start[every_iter * 1 + every_2iters_a * 1]; + smem_a_tile_start[4 * num_threads_in_cluster + hw_tid] = \ + dram_a_tile_start[every_iter * 0 + every_2iters_a * 2]; + smem_a_tile_start[5 * num_threads_in_cluster + hw_tid] = \ + dram_a_tile_start[every_iter * 1 + every_2iters_a * 2]; + smem_a_tile_start[6 * num_threads_in_cluster + hw_tid] = \ + dram_a_tile_start[every_iter * 0 + every_2iters_a * 3]; + smem_a_tile_start[7 * num_threads_in_cluster + hw_tid] = \ + dram_a_tile_start[every_iter * 1 + every_2iters_a * 3]; + + smem_b_tile_start[0 * num_threads_in_cluster + hw_tid] = \ + dram_b_tile_start[every_iter * 0 + every_2iters_b * 0]; + smem_b_tile_start[1 * num_threads_in_cluster + hw_tid] = \ + dram_b_tile_start[every_iter * 1 + every_2iters_b * 0]; + smem_b_tile_start[2 * num_threads_in_cluster + hw_tid] = \ + dram_b_tile_start[every_iter * 0 + every_2iters_b * 1]; + smem_b_tile_start[3 * num_threads_in_cluster + hw_tid] = \ + dram_b_tile_start[every_iter * 1 + every_2iters_b * 1]; + smem_b_tile_start[4 * num_threads_in_cluster + hw_tid] = \ + dram_b_tile_start[every_iter * 0 + every_2iters_b * 2]; + smem_b_tile_start[5 * num_threads_in_cluster + hw_tid] = \ + dram_b_tile_start[every_iter * 1 + every_2iters_b * 2]; + smem_b_tile_start[6 * num_threads_in_cluster + hw_tid] = \ + dram_b_tile_start[every_iter * 0 + every_2iters_b * 3]; + smem_b_tile_start[7 * num_threads_in_cluster + hw_tid] = \ + dram_b_tile_start[every_iter * 1 + every_2iters_b * 3]; + #else __asm__("load_ab:"); float v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 0]; float v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 0]; @@ -174,64 +216,34 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_b_tile_start[7 * num_threads_in_cluster] = v3; __asm__("end_loadab:"); + #endif } #else - /* smem_a_tile_start[0 * num_threads_in_cluster + hw_tid] = \ - dram_a_tile_start[runtime_const + every_iter * 0 + every_2iters * 0]; - smem_a_tile_start[1 * num_threads_in_cluster + hw_tid] = \ - dram_a_tile_start[runtime_const + every_iter * 1 + every_2iters * 0]; - smem_a_tile_start[2 * num_threads_in_cluster + hw_tid] = \ - dram_a_tile_start[runtime_const + every_iter * 0 + every_2iters * 1]; - smem_a_tile_start[3 * num_threads_in_cluster + hw_tid] = \ - dram_a_tile_start[runtime_const + every_iter * 1 + every_2iters * 1]; - smem_a_tile_start[4 * num_threads_in_cluster + hw_tid] = \ - dram_a_tile_start[runtime_const + every_iter * 0 + every_2iters * 2]; - smem_a_tile_start[5 * num_threads_in_cluster + hw_tid] = \ - dram_a_tile_start[runtime_const + every_iter * 1 + every_2iters * 2]; - smem_a_tile_start[6 * num_threads_in_cluster + hw_tid] = \ - dram_a_tile_start[runtime_const + every_iter * 0 + every_2iters * 3]; - smem_a_tile_start[7 * num_threads_in_cluster + hw_tid] = \ - dram_a_tile_start[runtime_const + every_iter * 1 + every_2iters * 3]; - - smem_b_tile_start[0 * num_threads_in_cluster + hw_tid] = \ - dram_b_tile_start[runtime_const + every_iter * 0 + every_2iters * 0]; - smem_b_tile_start[1 * num_threads_in_cluster + hw_tid] = \ - dram_b_tile_start[runtime_const + every_iter * 1 + every_2iters * 0]; - smem_b_tile_start[2 * num_threads_in_cluster + hw_tid] = \ - dram_b_tile_start[runtime_const + every_iter * 0 + every_2iters * 1]; - smem_b_tile_start[3 * num_threads_in_cluster + hw_tid] = \ - dram_b_tile_start[runtime_const + every_iter * 1 + every_2iters * 1]; - smem_b_tile_start[4 * num_threads_in_cluster + hw_tid] = \ - dram_b_tile_start[runtime_const + every_iter * 0 + every_2iters * 2]; - smem_b_tile_start[5 * num_threads_in_cluster + hw_tid] = \ - dram_b_tile_start[runtime_const + every_iter * 1 + every_2iters * 2]; - smem_b_tile_start[6 * num_threads_in_cluster + hw_tid] = \ - dram_b_tile_start[runtime_const + every_iter * 0 + every_2iters * 3]; - smem_b_tile_start[7 * num_threads_in_cluster + hw_tid] = \ - dram_b_tile_start[runtime_const + every_iter * 1 + every_2iters * 3]; */ + __asm__("loop_load_ab:"); const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K; const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N; float * const smem_a_tile_start = SMEM_ADDR_0K; float * const smem_b_tile_start = SMEM_ADDR_12K; - #pragma GCC unroll 8 // TODO: macro computed - for (uint32_t thread_i = 0, j1 = 0, i1 = 0; + /* for (uint32_t thread_i = 0, j1 = 0, i1 = 0; thread_i < a_elems_per_thread; thread_i += 1, j1 = (j1 + j1_stride) % TILE_K, i1 = (thread_i % i1_iters == 0) ? i1 + i1_stride : i1) { smem_a_tile_start[thread_i * num_threads_in_cluster + hw_tid] = \ dram_a_tile_start[(0 + i0) * dim_k + j1 + j1_idx + j0]; + } */ + #pragma clang loop unroll(disable) + for (int thread_i = 0; thread_i < a_elems_per_thread; thread_i++) { + uint32_t elem_offset = hw_tid + num_threads_in_cluster * thread_i; + smem_a_tile_start[SMEM_MAT_OFFSET(elem_offset / TILE_K, elem_offset % TILE_K, TILE_K)] = \ + dram_a_tile_start[elem_offset / TILE_K * dim_k + elem_offset % TILE_K]; } - // for (int thread_i = 0; thread_i < a_elems_per_thread; thread_i++) { - // uint32_t elem_offset = thread_load_offset + thread_load_stride * thread_i; - // smem_a_tile_start[SMEM_MAT_OFFSET(elem_offset / TILE_K, elem_offset % TILE_K, TILE_K)] = \ - // dram_a_tile_start[elem_offset / TILE_K * dim_k + elem_offset % TILE_K]; - // } - #pragma GCC unroll 8 + __asm__("loop_load_a_end:"); + #pragma clang loop unroll(disable) for (int thread_i = 0; thread_i < b_elems_per_thread; thread_i++) { - uint32_t elem_offset = thread_load_offset + thread_load_stride * thread_i; + uint32_t elem_offset = hw_tid + num_threads_in_cluster * thread_i; smem_b_tile_start[SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N)] = \ dram_b_tile_start[elem_offset / TILE_N * dim_n + elem_offset % TILE_N]; } @@ -274,6 +286,21 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, #ifdef DBUF gemmini_fence(); #endif + #ifdef CISC + #ifndef DBUF + #error MUST ENABLE DBUF + #endif + #ifdef EXT_ACCUMULATE + #error MUST DISABLE EXT ACCUMULATE + #endif + if (tile_k == 0) { + GEMMINI_CISC_CMD_I(0); + } else if (tile_k & 1) { + GEMMINI_CISC_CMD_I(2); + } else { + GEMMINI_CISC_CMD_I(1); + } + #else sp_tiled_matmul_full_spad_ws( #ifdef DBUF (tile_k & 1) ? SPAD_ADDR_4K : SPAD_ADDR_0K, (tile_k & 1) ? SPAD_ADDR_12K : SPAD_ADDR_8K, @@ -284,17 +311,19 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, /*I=*/TILE_M / DIM, /*J=*/TILE_N / DIM, /*K=*/TILE_K / DIM, /*pad_I=*/0, /*pad_J=*/0, /*pad_K=*/0, /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, #ifdef EXT_ACCUMULATE - /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/0x38U); + /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/0x38U) #else - /*acc=*/tile_k != 0, /*act=*/NO_ACTIVATION, /*skips=*/0xB8U); + /*acc=*/tile_k != 0, /*act=*/NO_ACTIVATION, /*skips=*/0xB8U) #endif + + #endif #ifndef DBUF gemmini_fence(); #endif } __asm__("end_gemmini:"); rd_cycles(marker4); - threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + // threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); rd_cycles(marker5); // accumulate C matrix @@ -361,15 +390,15 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, // mvout to scratchpad for activation if (HW_TID() == 0) { __asm__("mvout_spad:"); - #ifdef DBUF - gemmini_fence(); - #endif +// #ifdef DBUF +// gemmini_fence(); +// #endif + #ifdef CISC + GEMMINI_CISC_CMD_I(9); + #else ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (4ULL << 32) | (4ULL << 16) | 4ULL, k_LOOP_WS_CONFIG_BOUNDS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, 0x278U, k_LOOP_WS) - /* #pragma gcc unroll 16 - for (int i = 0; i < TILE_MN / DIM; i += DIM) { - gemmini_mvout_spad(i, 0x80000000ULL + i); // FIXME: C is not necessarily at 0 - } */ + #endif __asm__("mvout_spad_fence:"); gemmini_fence(); } @@ -390,10 +419,21 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const uint32_t runtime_const = i0 * dim_n + j1_idx + j0; float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N + runtime_const; + #ifdef REGBLOCK float v0 = smem_acc_tile_start[0 * num_threads_in_cluster]; float v1 = smem_acc_tile_start[1 * num_threads_in_cluster]; float v2 = smem_acc_tile_start[2 * num_threads_in_cluster]; float v3 = smem_acc_tile_start[3 * num_threads_in_cluster]; + #ifdef ACTIVATE + uint32_t swish_start, swish_end; + rd_cycles_force(swish_start); + v0 = SWISH(1, v0); + v1 = SWISH(1, v1); + v2 = SWISH(1, v2); + v3 = SWISH(1, v3); + rd_cycles_force(swish_end); + swish_dur += swish_end - swish_start; + #endif dram_c_tile_start[every_iter * 0 + every_2iters * 0] = v0; dram_c_tile_start[every_iter * 1 + every_2iters * 0] = v1; dram_c_tile_start[every_iter * 0 + every_2iters * 1] = v2; @@ -403,39 +443,51 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, v1 = smem_acc_tile_start[5 * num_threads_in_cluster]; v2 = smem_acc_tile_start[6 * num_threads_in_cluster]; v3 = smem_acc_tile_start[7 * num_threads_in_cluster]; + #ifdef ACTIVATE + rd_cycles_force(swish_start); + v0 = SWISH(1, v0); + v1 = SWISH(1, v1); + v2 = SWISH(1, v2); + v3 = SWISH(1, v3); + rd_cycles_force(swish_end); + swish_dur += swish_end - swish_start; + #endif dram_c_tile_start[every_iter * 0 + every_2iters * 2] = v0; dram_c_tile_start[every_iter * 1 + every_2iters * 2] = v1; dram_c_tile_start[every_iter * 0 + every_2iters * 3] = v2; dram_c_tile_start[every_iter * 1 + every_2iters * 3] = v3; + #else + dram_c_tile_start[every_iter * 0 + every_2iters * 0] = \ + smem_acc_tile_start[0 * num_threads_in_cluster]; + dram_c_tile_start[every_iter * 1 + every_2iters * 0] = \ + smem_acc_tile_start[1 * num_threads_in_cluster]; + dram_c_tile_start[every_iter * 0 + every_2iters * 1] = \ + smem_acc_tile_start[2 * num_threads_in_cluster]; + dram_c_tile_start[every_iter * 1 + every_2iters * 1] = \ + smem_acc_tile_start[3 * num_threads_in_cluster]; + dram_c_tile_start[every_iter * 0 + every_2iters * 2] = \ + smem_acc_tile_start[4 * num_threads_in_cluster]; + dram_c_tile_start[every_iter * 1 + every_2iters * 2] = \ + smem_acc_tile_start[5 * num_threads_in_cluster]; + dram_c_tile_start[every_iter * 0 + every_2iters * 3] = \ + smem_acc_tile_start[6 * num_threads_in_cluster]; + dram_c_tile_start[every_iter * 1 + every_2iters * 3] = \ + smem_acc_tile_start[7 * num_threads_in_cluster]; + #endif #else - /*dram_c_tile_start[runtime_const + every_iter * 0 + every_2iters * 0] = \ - smem_acc_tile_start[0 * num_threads_in_cluster]; - dram_c_tile_start[runtime_const + every_iter * 1 + every_2iters * 0] = \ - smem_acc_tile_start[1 * num_threads_in_cluster]; - dram_c_tile_start[runtime_const + every_iter * 0 + every_2iters * 1] = \ - smem_acc_tile_start[2 * num_threads_in_cluster]; - dram_c_tile_start[runtime_const + every_iter * 1 + every_2iters * 1] = \ - smem_acc_tile_start[3 * num_threads_in_cluster]; - dram_c_tile_start[runtime_const + every_iter * 0 + every_2iters * 2] = \ - smem_acc_tile_start[4 * num_threads_in_cluster]; - dram_c_tile_start[runtime_const + every_iter * 1 + every_2iters * 2] = \ - smem_acc_tile_start[5 * num_threads_in_cluster]; - dram_c_tile_start[runtime_const + every_iter * 0 + every_2iters * 3] = \ - smem_acc_tile_start[6 * num_threads_in_cluster]; - dram_c_tile_start[runtime_const + every_iter * 1 + every_2iters * 3] = \ - smem_acc_tile_start[7 * num_threads_in_cluster];*/ - #pragma GCC unroll 8 + float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N; + #pragma clang loop unroll(disable) for (int thread_i = 0; thread_i < c_elems_per_thread; thread_i++) { - uint32_t elem_offset = thread_load_offset + thread_load_stride * thread_i; + uint32_t elem_offset = hw_tid + num_threads_in_cluster * thread_i; dram_c_tile_start[elem_offset / TILE_N * dim_n + elem_offset % TILE_N] = \ *(SMEM_ADDR_8K + SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N)); } #endif __asm__("end_mvout_dram:"); - rd_cycles(marker8); + // rd_cycles_force(marker8); } } // last thread block complete @@ -446,6 +498,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, PRINTF("\ncomplete\n"); PRINTF("total cycles: %d\n", marker9 - marker0); } + #ifdef ACTIVATE + if (HW_TID() == 0) { + PRINTF("swish cycles: %d\n", swish_dur); + } + #endif #ifdef DETAILED_PERF vx_tmc(0x81); for (int x = 0; x < num_threads_in_cluster; x += num_threads_in_cluster - 1) {