wu-arch/kernels
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cisc gemmini

Browse Source
This commit is contained in:
Richard Yan
2024-05-08 15:46:20 -07:00
parent 4db59446f3
commit 33066af56e
3 changed files with 46 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
kernel/include/gemmini_mmio.h
View File

@@ -6,8 +6,10 @@
#define SMEM_BASE 0xff000000 #define SMEM_BASE 0xff000000
#define SMEM_SIZE 0x4000 #define SMEM_SIZE 0x4000
#define SMEM_MASK (SMEM_SIZE - 1) #define SMEM_MASK (SMEM_SIZE - 1)
#define SMEM_ADDR_END 0xff008000 #define SMEM_ADDR_END (SMEM_BASE + SMEM_SIZE)
#define GEMMINI_CTRL (SMEM_BASE + SMEM_SIZE + 0x3000)
#define SPAD_BASE 0x0 #define SPAD_BASE 0x0
#define SPAD_ROW_SIZE (DIM * sizeof(elem_t)) #define SPAD_ROW_SIZE (DIM * sizeof(elem_t))
@@ -15,14 +17,26 @@
#define SPAD_MASK (SPAD_NUM_ROWS - 1) #define SPAD_MASK (SPAD_NUM_ROWS - 1)
#define PRINT_BUF ((char *) (SMEM_ADDR_END)) #define PRINT_BUF ((char *) (SMEM_ADDR_END))
#define GEMMINI_RS1_ADDR 0xff007010 #define GEMMINI_RS1_ADDR (GEMMINI_CTRL + 0x10)
#define GEMMINI_RS2_ADDR 0xff007018 #define GEMMINI_RS2_ADDR (GEMMINI_CTRL + 0x18)
#define GEMMINI_INST_ADDR 0xff007000 #define GEMMINI_INST_ADDR (GEMMINI_CTRL + 0x0)
#define GEMMINI_BUSY_ADDR 0xff007020 #define GEMMINI_BUSY_ADDR (GEMMINI_CTRL + 0x20)
#define SMEM_TO_SPAD(smem_addr) (SPAD_BASE + ((smem_addr) & SMEM_MASK) / SPAD_ROW_SIZE) #define SMEM_TO_SPAD(smem_addr) (SPAD_BASE + ((smem_addr) & SMEM_MASK) / SPAD_ROW_SIZE)
#define SPAD_TO_SMEM(spad_addr) (SMEM_BASE + ((spad_addr) & SPAD_MASK) * SPAD_ROW_SIZE) #define SPAD_TO_SMEM(spad_addr) (SMEM_BASE + ((spad_addr) & SPAD_MASK) * SPAD_ROW_SIZE)
// CISC instructions:
// 0, 1, 2: tile-sized matmuls
// 0: k = 0, no accumulation
// 1: k % 2 = 0, buffer regions 0
// 2: k % 2 = 1, buffer regions 1
// 8, 9: memory ops
// 8: tile-sized move-in (unused)
// 9: tile-sized move-out
#define GEMMINI_CISC_CMD_I(x) asm("csrwi 0xacc, "#x)
#define GEMMINI_STATUS() ({uint32_t status; asm volatile ("csrr %0, 0xacc" : "=r" (status)); status;})
// convert normal matrix i,j into tiled smem offset // convert normal matrix i,j into tiled smem offset
// top_in_tiles = i / DIM // top_in_tiles = i / DIM
// left_in_tiles = j / DIM // left_in_tiles = j / DIM
@@ -33,6 +47,7 @@
// #define fence() { for (int i = 0; i < 10; i++) *((volatile uint32_t *) (0xFFFF0000)) = 0xdeadbeef; } // #define fence() { for (int i = 0; i < 10; i++) *((volatile uint32_t *) (0xFFFF0000)) = 0xdeadbeef; }
#undef gemmini_fence #undef gemmini_fence
//#define gemmini_fence() { while (GEMMINI_STATUS()); }
#define gemmini_fence() { while (*((volatile uint32_t *) GEMMINI_BUSY_ADDR)) asm volatile ("nop"); } #define gemmini_fence() { while (*((volatile uint32_t *) GEMMINI_BUSY_ADDR)) asm volatile ("nop"); }
#undef ROCC_INSTRUCTION_RS1_RS2 #undef ROCC_INSTRUCTION_RS1_RS2

162
tests/kernel/gemmini_mmio/gemmini_mmio.h
View File

@@ -1,162 +0,0 @@
#ifndef GEMMINI_MMIO_H
#define GEMMINI_MMIO_H
#ifndef GEMMINI_PARAMS_H
#error INCLUDE GEMMINI.H FIRST
#endif
#define SMEM_BASE 0xff000000
#define SMEM_SIZE 0x4000
#define SMEM_MASK (SMEM_SIZE - 1)
#define SMEM_ADDR_END 0xff008000
#define SPAD_BASE 0x0
#define SPAD_ROW_SIZE (DIM * sizeof(elem_t))
#define SPAD_NUM_ROWS (SMEM_SIZE / SPAD_ROW_SIZE)
#define SPAD_MASK (SPAD_NUM_ROWS - 1)
#define PRINT_BUF ((char *) (SMEM_ADDR_END))
#define GEMMINI_RS1_ADDR 0xff007010
#define GEMMINI_RS2_ADDR 0xff007018
#define GEMMINI_INST_ADDR 0xff007000
#define GEMMINI_BUSY_ADDR 0xff007020
#define SMEM_TO_SPAD(smem_addr) (SPAD_BASE + ((smem_addr) & SMEM_MASK) / SPAD_ROW_SIZE)
#define SPAD_TO_SMEM(spad_addr) (SMEM_BASE + ((spad_addr) & SPAD_MASK) * SPAD_ROW_SIZE)
// convert normal matrix i,j into tiled smem offset
// top_in_tiles = i / DIM
// left_in_tiles = j / DIM
// num_tiles_before_current = top_in_tiles * (J / DIM) + left_in_tiles
// smem_addr = num_tiles_before_current * DIM * DIM + (i % DIM) * DIM + (j % DIM)
#define SMEM_MAT_OFFSET(i, j, J) \
(((i) / DIM * (J) / DIM + (j) / DIM) * DIM * DIM + ((i) % DIM) * DIM + ((j) % DIM))
// #define fence() { for (int i = 0; i < 10; i++) *((volatile uint32_t *) (0xFFFF0000)) = 0xdeadbeef; }
#undef gemmini_fence
#define gemmini_fence() { while (*((volatile uint32_t *) GEMMINI_BUSY_ADDR)) asm volatile ("nop"); }
#undef ROCC_INSTRUCTION_RS1_RS2
#define ROCC_INSTRUCTION_RS1_RS2(x, rs1, rs2, funct) { \
/* printf("function %d\n", funct); */ \
uint32_t instruction = (0x7B) | (0 << 7) | (3 << 12) | (1 << 15) | (2 << 20) | ((uint32_t) (funct) << 25); \
*((volatile uint64_t *) GEMMINI_RS1_ADDR) = (volatile uint64_t) (rs1); \
*((volatile uint64_t *) GEMMINI_RS2_ADDR) = (volatile uint64_t) (rs2); \
/* *((volatile uint32_t*) GEMMINI_RS2_ADDR) = (uint32_t) ((uint64_t) (rs2) & 0xFFFFFFFFULL); */ \
/* *((volatile uint32_t*) (GEMMINI_RS2_ADDR + 4)) = (uint32_t) ((uint64_t) (rs2) >> 32); */ \
/* gemmini_fence(); */ \
*((volatile uint32_t*) GEMMINI_INST_ADDR) = instruction; \
/* sprintf((char *) PRINT_BUF, "%llx %llx %d\n", rs1, rs2, funct); */ \
}
static void sp_tiled_matmul_full_spad_ws(const uint32_t A_sp_addr_start, const uint32_t B_sp_addr_start,
const uint32_t D_sp_addr_start, const uint32_t C_dst_sp_addr_start,
size_t I, size_t J, size_t K, size_t pad_I, size_t pad_J, size_t pad_K,
bool a_transpose, bool b_transpose,
bool full_C, bool low_D,
bool no_bias, bool repeating_bias,
int act) {
gemmini_loop_ws_spad(I, J, K, pad_I, pad_J, pad_K,
A_sp_addr_start, B_sp_addr_start + K * J * DIM, NULL, C_dst_sp_addr_start,
a_transpose, b_transpose,
full_C, low_D, false,
act, 0, 0, false);
/*
return;
// const uint32_t A_sp_addr_start = 0;
// const uint32_t B_sp_addr_start = BANK_NUM * BANK_ROWS - K * J * DIM;
// const uint32_t D_sp_addr_start = 1 << (ADDR_LEN-1);
const uint32_t C_sp_addr_start = 2 << (ADDR_LEN-2) | (full_C << (ADDR_LEN-3));
// const int D_blocks = low_D ? (J <= MAX_BLOCK_LEN ? J : MAX_BLOCK_LEN) :
// (J <= MAX_BLOCK_LEN_ACC ? J : MAX_BLOCK_LEN_ACC);
const int C_blocks = 1; //full_C ? 1 : (J <= MAX_BLOCK_LEN ? J : MAX_BLOCK_LEN);
// const size_t sizeof_D = low_D ? sizeof(elem_t) : sizeof(acc_t);
const size_t sizeof_C = full_C ? sizeof(acc_t) : sizeof(elem_t);
gemmini_fence();
if (a_transpose || b_transpose || (I < 4)) {
for (size_t k = 0; k < K; k++) {
for (size_t j = 0; j < J; j++) {
for (size_t i = 0; i < I; i++) {
const uint32_t A_sp_addr = a_transpose ? (A_sp_addr_start + (k*I + i)*DIM) :
(A_sp_addr_start + (i*K + k)*DIM);
const uint32_t B_sp_addr = b_transpose ? (B_sp_addr_start + (j*K + k)*DIM) :
(B_sp_addr_start + (k*J + j)*DIM);
const uint32_t C_sp_addr = C_sp_addr_start + (i*J + j)*DIM;
// Compute
uint32_t pre_sp_addr = i == 0 ? B_sp_addr : GARBAGE_ADDR;
uint32_t out_sp_addr = C_sp_addr | ((k == 0 ? 0 : 1) << (ADDR_LEN-2));
gemmini_extended_preload(pre_sp_addr, out_sp_addr, DIM, DIM, DIM, DIM);
if (i == 0) { // First iteration
gemmini_extended_compute_preloaded(A_sp_addr, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
} else { // All other iterations
gemmini_extended_compute_accumulated(A_sp_addr, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
}
if (k == K - 1) {
// Move-out C (if not normalizing)
// if (((act != LAYERNORM) && (act != SOFTMAX)) && (j == J-1 || j % C_blocks == C_blocks-1)) {
const size_t rounded_j = j; // (j / C_blocks) * C_blocks;
const uint32_t rounded_C_sp_addr = C_sp_addr; // C_sp_addr_start + (i*J + rounded_j)*DIM;
const uint32_t C_dst_sp_addr = ((uint32_t) C_dst_sp_addr_start) + (i * J + rounded_j) * DIM; // * DIM * sizeof_C;
// const size_t blocks = rounded_j + C_blocks <= J ? C_blocks : J-rounded_j;
constexpr size_t cols = DIM; // blocks * DIM - (rounded_j + blocks >= J ? pad_J : 0);
constexpr size_t rows = DIM; // DIM - (i == I - 1 ? pad_I : 0);
gemmini_extended_mvout_spad(C_dst_sp_addr, 1, rounded_C_sp_addr, cols, rows);
// }
}
}
}
}
} else {
for (size_t k = 0; k < K; k++) {
for (size_t j = 0; j < J; j++) {
uint32_t A_sp_addr = A_sp_addr_start + k * DIM; // (i*K + k)*DIM;
const uint32_t B_sp_addr = B_sp_addr_start + (k*J + j)*DIM;
uint32_t C_sp_addr = C_sp_addr_start + j * DIM; // (i*J + j)*DIM;
for (size_t i = 0; i < I; i += 4) {
// Compute
// constexpr uint32_t pre_sp_addr = i == 0 ? B_sp_addr : GARBAGE_ADDR;
const uint32_t out_sp_addr = C_sp_addr | ((k == 0 ? 0 : 1) << (ADDR_LEN-2));
if (i == 0) { // First iteration
gemmini_extended_preload(B_sp_addr, out_sp_addr, DIM, DIM, DIM, DIM);
gemmini_extended_compute_preloaded(A_sp_addr, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + J * DIM, DIM, DIM, DIM, DIM);
gemmini_extended_compute_accumulated(A_sp_addr + K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + 2 * J * DIM, DIM, DIM, DIM, DIM);
gemmini_extended_compute_accumulated(A_sp_addr + 2 * K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + 3 * J * DIM, DIM, DIM, DIM, DIM);
gemmini_extended_compute_accumulated(A_sp_addr + 3 * K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
} else { // All other iterations
gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr, DIM, DIM, DIM, DIM);
gemmini_extended_compute_accumulated(A_sp_addr, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + J * DIM, DIM, DIM, DIM, DIM);
gemmini_extended_compute_accumulated(A_sp_addr + K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + 2 * J * DIM, DIM, DIM, DIM, DIM);
gemmini_extended_compute_accumulated(A_sp_addr + 2 * K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
gemmini_extended_preload(GARBAGE_ADDR, out_sp_addr + 3 * J * DIM, DIM, DIM, DIM, DIM);
gemmini_extended_compute_accumulated(A_sp_addr + 3 * K * DIM, GARBAGE_ADDR, DIM, DIM, DIM, DIM);
}
if (k == K - 1) {
for (int x = 0; x < 3; x++) gemmini_fence();
gemmini_extended_mvout_spad((uint32_t) C_dst_sp_addr_start + (i * J + j) * DIM, 1, C_sp_addr, DIM, DIM);
gemmini_extended_mvout_spad((uint32_t) C_dst_sp_addr_start + ((i + 1) * J + j) * DIM, 1, C_sp_addr + J * DIM, DIM, DIM);
gemmini_extended_mvout_spad((uint32_t) C_dst_sp_addr_start + ((i + 2) * J + j) * DIM, 1, C_sp_addr + 2 * J * DIM, DIM, DIM);
gemmini_extended_mvout_spad((uint32_t) C_dst_sp_addr_start + ((i + 3) * J + j) * DIM, 1, C_sp_addr + 3 * J * DIM, DIM, DIM);
}
A_sp_addr += 4 * K * DIM;
C_sp_addr += 4 * J * DIM;
}
}
}
}
gemmini_fence();
*/
}
#endif

1
tests/kernel/gemmini_mmio/gemmini_mmio.h Symbolic link
View File

@@ -0,0 +1 @@
../../../kernel/include/gemmini_mmio.h

28
tests/regression/sgemm_gemmini/kernel.cpp
View File

@@ -31,7 +31,8 @@
#define REGBLOCK #define REGBLOCK
#define DBUF #define DBUF
//#define DETAILED_PERF //#define DETAILED_PERF
#define ACTIVATE //#define ACTIVATE
#define CISC
#define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) #define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x))
#ifdef DETAILED_PERF #ifdef DETAILED_PERF
@@ -39,7 +40,7 @@
#else #else
#define rd_cycles(x) #define rd_cycles(x)
#endif #endif
#define HW_TID() ({uint32_t gtid; asm ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) #define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
#define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) #define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__)
// #define PRINTF(...) vx_printf(__VA_ARGS__) // #define PRINTF(...) vx_printf(__VA_ARGS__)
#define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x)))) #define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x))))
@@ -285,6 +286,21 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
#ifdef DBUF #ifdef DBUF
gemmini_fence(); gemmini_fence();
#endif #endif
#ifdef CISC
#ifndef DBUF
#error MUST ENABLE DBUF
#endif
#ifdef EXT_ACCUMULATE
#error MUST DISABLE EXT ACCUMULATE
#endif
if (tile_k == 0) {
GEMMINI_CISC_CMD_I(0);
} else if (tile_k & 1) {
GEMMINI_CISC_CMD_I(2);
} else {
GEMMINI_CISC_CMD_I(1);
}
#else
sp_tiled_matmul_full_spad_ws( sp_tiled_matmul_full_spad_ws(
#ifdef DBUF #ifdef DBUF
(tile_k & 1) ? SPAD_ADDR_4K : SPAD_ADDR_0K, (tile_k & 1) ? SPAD_ADDR_12K : SPAD_ADDR_8K, (tile_k & 1) ? SPAD_ADDR_4K : SPAD_ADDR_0K, (tile_k & 1) ? SPAD_ADDR_12K : SPAD_ADDR_8K,
@@ -299,13 +315,15 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
#else #else
/*acc=*/tile_k != 0, /*act=*/NO_ACTIVATION, /*skips=*/0xB8U) /*acc=*/tile_k != 0, /*act=*/NO_ACTIVATION, /*skips=*/0xB8U)
#endif #endif
#endif
#ifndef DBUF #ifndef DBUF
gemmini_fence(); gemmini_fence();
#endif #endif
} }
__asm__("end_gemmini:"); __asm__("end_gemmini:");
rd_cycles(marker4); rd_cycles(marker4);
threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); // threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
rd_cycles(marker5); rd_cycles(marker5);
// accumulate C matrix // accumulate C matrix
@@ -375,8 +393,12 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
// #ifdef DBUF // #ifdef DBUF
// gemmini_fence(); // gemmini_fence();
// #endif // #endif
#ifdef CISC
GEMMINI_CISC_CMD_I(9);
#else
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (4ULL << 32) | (4ULL << 16) | 4ULL, k_LOOP_WS_CONFIG_BOUNDS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (4ULL << 32) | (4ULL << 16) | 4ULL, k_LOOP_WS_CONFIG_BOUNDS)
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, 0x278U, k_LOOP_WS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, 0x278U, k_LOOP_WS)
#endif
__asm__("mvout_spad_fence:"); __asm__("mvout_spad_fence:");
gemmini_fence(); gemmini_fence();
} }