#ifndef _SGEMM_IMPL_H_ #define _SGEMM_IMPL_H_ #include #include #include "include/gemmini.h" #include "gemmini_mmio.h" // Constraints on parameters: // * Memory: // (BM + BN) * BK * sizeof(T) <= sharedmem size. // BM * BK == BN * BK >= threadblock size >= NT * CORES_PER_CLUSTER // When larger, the kernel runs a sequential loop to read into sharedmem; // but smaller case is not handled. // * Compute: // ( M* N) / (TM*TN) == grid size >= NC*NW*NT // (BM*BN) / (TM*TN) == threadblock size < NT * NW * CORES_PER_CLUSTER // (BM*BN) / (TM*TN) == threadblock size >= NT * CORES_PER_CLUSTER // * Combining BM * BK >= (BM*BN) / (TM*TN) == threadblock yields // BM <= BK*TM*TN #define BM 64 #define BN 64 #define BK 128 #define WM 16 #define WN 8 #define TCM 8 #define TCN 8 #define TCK 16 #define WMITER (WM / TCM) #define WNITER (WN / TCN) #define ELEM_PER_THREAD (WM * WN / NUM_THREADS) // FIXME: NUM_THREADS and NUM_WARPS hardcoded #if ((BM * BN / ELEM_PER_THREAD) > (CORES_PER_CLUSTER * 8 * 8)) #error "threadblock size too big for cluster" #endif // number of loop around the inner 0..TCK..BK loop to simulate perfect-DRAM // scenario #define BK_LOOP 1 // Whether to transpose smem A tile at GMEM->SMEM (produce), or SMEM->RF // (consume). This is because the tensor core expects the A tile to be stored // in column-major order in SMEM, whereas it will be ultimately stored in // row-major in the RF. // // For correctness, only one of either should be 1. E.g., PRODUCE 1 CONSUME 0 // generates the NN kernel where both A and B are stored row-major in GMEM. // To model the case where the A matrix is already stored column-major in GMEM, // set both to 0. #define TRANSPOSE_AT_PRODUCE 0 #define TRANSPOSE_AT_CONSUME 0 // GMEM_COALESCED: When TRANSPOSE_AT_PRODUCE == 1 (i.e. transpose at // GMEM->SMEM), determines whether we do bank-conflict-free accesses for // 1: GMEM loads of A matrix, or // 0: SMEM stores of A matrix. // // Usually, GMEM_COALESCED==1 yields better performance since the memory // behavior of GMEM is more sensitive to bank conflicts. #define GMEM_COALESCED_A 1 #define GEMMINI_DMA 0 #if SMEM_SIZE == 0x4000 #define SMEM_ADDR_Q0 ((float * const) 0xff000000) #define SMEM_ADDR_Q1 ((float * const) 0xff001000) #define SMEM_ADDR_Q2 ((float * const) 0xff002000) #define SMEM_ADDR_Q3 ((float * const) 0xff003000) #define SPAD_ADDR_Q0 0x0 #define SPAD_ADDR_Q1 0x80 #define SPAD_ADDR_Q2 0x100 #define SPAD_ADDR_Q3 0x180 #define BOUND_INST 0x400040004ULL #elif SMEM_SIZE == 0x10000 #define SMEM_ADDR_Q0 ((float * const) 0xff000000) #define SMEM_ADDR_Q1 ((float * const) 0xff004000) #define SMEM_ADDR_Q2 ((float * const) 0xff008000) #define SMEM_ADDR_Q3 ((float * const) 0xff00c000) #define SPAD_ADDR_Q0 0x0 #define SPAD_ADDR_Q1 0x200 #define SPAD_ADDR_Q2 0x400 #define SPAD_ADDR_Q3 0x600 #define BOUND_INST 0x800080008ULL #else #error Unsupported smem size #endif // "fake" fp16 type that only has the correct data width. using float16_t = uint16_t; inline constexpr void map_operand_32lanes(const int tid, int &row, int &col) { const int tg = tid / 4; // A (row major) // Figure 7(a) in paper // row 0~ 3: threadgroups 0 and 2 // row 4~ 7: threadgroups 4 and 6 // row 8~11: threadgroups 1 and 3 // row 12~15: threadgroups 5 and 7 row = tid % 4; row += (tg * 8) % 16; row += (tg / 4) * 4; // B (column major) // NOTE: Matrix B mapping in Figure 7(a) is incorrect; below is the // corrected mapping: // col 0~ 3: threadgroups 0 and 1 // col 4~ 7: threadgroups 4 and 5 // col 8~11: threadgroups 2 and 3 // col 12~15: threadgroups 6 and 7 col = tid % 4; col += ((tg % 4) / 2) * 8; col += (tg / 4) * 4; } inline constexpr void map_operand_8lanes(const int tid, int &row, int &col) { const int tg = tid / 4; // A (row major) // row 0~ 3: threadgroup 0 // row 4~ 7: threadgroup 1 row = tid % 4; row += tg * 4; // B (column major) // col 0~ 3: threadgroup 0 // col 4~ 7: threadgroup 1 col = tid % 4; col += tg * 4; } inline constexpr void map_operand(const int tid, int &row, int &col) { if constexpr (NUM_THREADS == 32) { map_operand_32lanes(tid, row, col); } else if constexpr (NUM_THREADS == 8) { map_operand_8lanes(tid, row, col); } else { // FIXME: not allowed } } inline constexpr void map_c_32lanes(const int tid, int &row, int &col) { const int tg = tid / 4; // C // Figure 7(b), left col = ((tg % 4) / 2) * 8; row = (tg * 8) % 16; row += (tg / 4) * 4; // Figure 7(b), right row += (tid % 4) % 2; col += ((tid % 4) / 2) * 2; } inline constexpr void map_c_8lanes(const int tid, int &row, int &col) { const int tg = tid / 4; // C col = 0; row = tg * 4; // Figure 7(b), right row += (tid % 4) % 2; col += ((tid % 4) / 2) * 2; } inline constexpr void map_c(const int tid, int &row, int &col) { if constexpr (NUM_THREADS == 32) { map_c_32lanes(tid, row, col); } else if constexpr (NUM_THREADS == 8) { map_c_8lanes(tid, row, col); } else { // FIXME: not allowed } } #define RISCV_CUSTOM3 0x7B inline void vx_wmma(const int dest_reg) { if (dest_reg == 0) { asm volatile (".insn r %0, 0, 0, x0, x0, x0" :: "i"(RISCV_CUSTOM3)); } else { asm volatile (".insn r %0, 0, 0, x1, x0, x0" :: "i"(RISCV_CUSTOM3)); } } // `local_k` is assumed to be multiple of TCK template inline void vx_wmma_load_a(volatile const T *smem_A, const int local_k, const int warp_row, const int wm_iter, const int thread_in_warp) { asm volatile ("vx_wmma_load_a_start_%=:" :: ); const int tid = thread_in_warp; const int tg = tid / 4; // @perf: this is duplicately computed in vx_wmma_load_a and vx_wmma_load_b int row = 0; int col = 0; map_operand(tid, row, col); // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do // data movement at the fp32 granularity. Assuming that the matrix is stored // row-major in GMEM, the packed fp16 pairs belong to the same row, // neighboring columns; therefore, it essentially becomes equivalent to // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed // by a factor of two. constexpr int packed_factor = (std::is_same_v ? 2 : 1); constexpr int BK_adjusted = BK / packed_factor; const int local_k_adjusted = local_k / packed_factor; if constexpr (TRANSPOSE_AT_CONSUME) { // A is stored K-major in smem constexpr int smem_A_rows = BM; constexpr int smem_A_cols = BK_adjusted; // int A_offset = (WM * warp_row + TCM * wm_iter + row) * smem_A_cols; // @perf: bank conflicts // f8-f15 stores a single row of A const volatile uint8_t *smem_addr; smem_addr = reinterpret_cast( &reinterpret_cast( smem_A)[(WM * warp_row + TCM * wm_iter + row) * smem_A_cols + local_k /* FIXME: adjust for fp16? */]); // step to the next column // threads read from different rows; bank conflicts asm volatile("flw f0, %0(%1)" ::"i"(0 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f1, %0(%1)" ::"i"(1 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f2, %0(%1)" ::"i"(2 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f3, %0(%1)" ::"i"(3 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f4, %0(%1)" ::"i"(4 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f5, %0(%1)" ::"i"(5 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f6, %0(%1)" ::"i"(6 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f7, %0(%1)" ::"i"(7 * sizeof(float)), "r"(smem_addr)); } else { // A is stored M-major in smem constexpr int smem_AS_rows = BK_adjusted; constexpr int smem_AS_cols = BM; const volatile uint8_t *smem_addr; smem_addr = reinterpret_cast( &reinterpret_cast( smem_A)[((local_k_adjusted + 0) * smem_AS_cols) + (WM * warp_row + TCM * wm_iter) + row]); // f8-f15 stores a single row of A // threads read from different columns; no bank conflicts asm volatile("flw f0, %0(%1)" :: "i"(smem_AS_cols * 0 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f1, %0(%1)" :: "i"(smem_AS_cols * 1 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f2, %0(%1)" :: "i"(smem_AS_cols * 2 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f3, %0(%1)" :: "i"(smem_AS_cols * 3 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f4, %0(%1)" :: "i"(smem_AS_cols * 4 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f5, %0(%1)" :: "i"(smem_AS_cols * 5 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f6, %0(%1)" :: "i"(smem_AS_cols * 6 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f7, %0(%1)" :: "i"(smem_AS_cols * 7 * sizeof(float)), "r"(smem_addr)); } asm volatile ("vx_wmma_load_a_finish_%=:" :: ); } // `local_k` is assumed to be multiple of TCK template inline void vx_wmma_load_b(const volatile T *smem_B, const int local_k, const int warp_col, const int wn_iter, const int thread_in_warp) { asm volatile ("vx_wmma_load_b_start_%=:" :: ); const int tid = thread_in_warp; const int tg = tid / 4; int row = 0; int col = 0; map_operand(tid, row, col); // see comment in vx_wmma_load_a constexpr int packed_factor = (std::is_same_v ? 2 : 1); constexpr int BK_adjusted = BN / packed_factor; constexpr int BN_adjusted = BN / packed_factor; const int local_k_adjusted = local_k / packed_factor; // B is stored N-major in smem constexpr int smem_B_rows = BK_adjusted; constexpr int smem_B_cols = BN; const volatile uint8_t *smem_addr; smem_addr = reinterpret_cast( &reinterpret_cast( smem_B)[((local_k_adjusted + 0) * smem_B_cols) + (WN * warp_col + TCN * wn_iter) + col]); // f8-f15 stores a single column of B // threads read from different columns; no bank conflicts asm volatile("flw f8, %0(%1)" :: "i"(smem_B_cols * 0 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f9, %0(%1)" :: "i"(smem_B_cols * 1 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f10, %0(%1)" :: "i"(smem_B_cols * 2 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f11, %0(%1)" :: "i"(smem_B_cols * 3 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f12, %0(%1)" :: "i"(smem_B_cols * 4 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f13, %0(%1)" :: "i"(smem_B_cols * 5 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f14, %0(%1)" :: "i"(smem_B_cols * 6 * sizeof(float)), "r"(smem_addr)); asm volatile("flw f15, %0(%1)" :: "i"(smem_B_cols * 7 * sizeof(float)), "r"(smem_addr)); asm volatile ("vx_wmma_load_b_finish_%=:" :: ); } inline void initialize_C(const int dest_reg) { // initialize C to zeros if (dest_reg == 0) { asm volatile("fmv.w.x f16, x0"); asm volatile("fmv.w.x f17, x0"); asm volatile("fmv.w.x f18, x0"); asm volatile("fmv.w.x f19, x0"); asm volatile("fmv.w.x f20, x0"); asm volatile("fmv.w.x f21, x0"); asm volatile("fmv.w.x f22, x0"); asm volatile("fmv.w.x f23, x0"); } else { asm volatile("fmv.w.x f24, x0"); asm volatile("fmv.w.x f25, x0"); asm volatile("fmv.w.x f26, x0"); asm volatile("fmv.w.x f27, x0"); asm volatile("fmv.w.x f28, x0"); asm volatile("fmv.w.x f29, x0"); asm volatile("fmv.w.x f30, x0"); asm volatile("fmv.w.x f31, x0"); } } inline void write_results(const int thread_in_warp, const int warp_col, const int warp_row, const int wn_iter, const int wm_iter, const int dim_n, float *C, const int threadblock_id_x, const int threadblock_id_y) { asm volatile ("write_results_start_%=:" :: ); int tid = thread_in_warp; // these are [0, TCM/TCN) int tid_row = 0; int tid_col = 0; map_c(tid, tid_row, tid_col); int local_row = (WM * warp_row + TCM * wm_iter) + tid_row; int local_col = (WN * warp_col + TCN * wn_iter) + tid_col; float *global_offset_C = C + (BM * threadblock_id_y) * dim_n + BN * threadblock_id_x; // @perf: this likely causes a lot of gmem bank conflicts if (wm_iter == 0) { volatile uint8_t *gmem_addr = reinterpret_cast( &global_offset_C[dim_n * (local_row + 0) + (local_col + 0)]); volatile uint8_t *gmem_addr_tmp = gmem_addr + (2 * dim_n) * sizeof(float); asm volatile ("fsw f16, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr)); asm volatile ("fsw f17, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr)); asm volatile ("fsw f18, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr_tmp)); asm volatile ("fsw f19, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr_tmp)); asm volatile ("fsw f20, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr)); asm volatile ("fsw f21, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr)); asm volatile ("fsw f22, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr_tmp)); asm volatile ("fsw f23, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr_tmp)); // asm volatile ("fsw f16, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 0)])); // asm volatile ("fsw f17, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 1)])); // asm volatile ("fsw f18, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 0)])); // asm volatile ("fsw f19, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 1)])); // asm volatile ("fsw f20, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 4)])); // asm volatile ("fsw f21, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 5)])); // asm volatile ("fsw f22, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 4)])); // asm volatile ("fsw f23, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 5)])); } else { volatile uint8_t *gmem_addr = reinterpret_cast( &global_offset_C[dim_n * (local_row + 0) + (local_col + 0)]); volatile uint8_t *gmem_addr_tmp = gmem_addr + (2 * dim_n) * sizeof(float); asm volatile ("fsw f24, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr)); asm volatile ("fsw f25, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr)); asm volatile ("fsw f26, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr_tmp)); asm volatile ("fsw f27, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr_tmp)); asm volatile ("fsw f28, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr)); asm volatile ("fsw f29, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr)); asm volatile ("fsw f30, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr_tmp)); asm volatile ("fsw f31, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr_tmp)); } asm volatile ("write_results_finish_%=:" :: ); } inline void threadblock_barrier(const uint32_t barrier_id, const uint32_t count) { vx_fence(); vx_barrier(barrier_id, count); } // TODO: reduce args by passing leading A/B dimensions template inline void global_dmem_load(const uint32_t dim_m, const uint32_t dim_n, const uint32_t dim_k, const uint32_t k, const T *A, const T *B, volatile T *local_a, volatile T *local_b, const uint32_t tid_in_threadblock, const uint32_t threadblock_id_x, const uint32_t threadblock_id_y) { asm volatile ("global_dmem_load_start_%=:" :: ); // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do // data movement at the fp32 granularity. Assuming that the matrix is stored // row-major in GMEM, the packed fp16 pairs belong to the same row, // neighboring columns; therefore, it essentially becomes equivalent to // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed // by a factor of two. constexpr uint32_t packed_factor = (std::is_same_v ? 2 : 1); constexpr uint32_t BK_adjusted = BK / packed_factor; const uint32_t dim_k_adjusted = dim_k / packed_factor; constexpr uint32_t BN_adjusted = BN / packed_factor; const uint32_t dim_n_adjusted = dim_n / packed_factor; const uint32_t k_adjusted = k / packed_factor; const uint32_t local_a_row = tid_in_threadblock / BK_adjusted; const uint32_t local_a_col = tid_in_threadblock % BK_adjusted; const uint32_t local_as_row = tid_in_threadblock / BM; const uint32_t local_as_col = tid_in_threadblock % BM; const uint32_t local_b_row = tid_in_threadblock / BN_adjusted; const uint32_t local_b_col = tid_in_threadblock % BN_adjusted; // FIXME: need fix for fp16? constexpr uint32_t threads_in_threadblock = (BM * BN) / ELEM_PER_THREAD; // Data move from GMEM to SMEM // // Make sure global offset values for A and B are contiguous between // neighboring threads to ensure GMEM coalescing. // // TODO: Sharedmem swizzling is important here // move A if constexpr (!TRANSPOSE_AT_PRODUCE) { // A is stored M-major in GMEM; // no transpose at GMEM->SMEM movement const uint32_t block_m = threadblock_id_y; const uint32_t global_a_row = k_adjusted + local_as_row; const uint32_t global_a_col = BM * block_m + local_as_col; // number of rows a full TB can read at a time constexpr uint32_t row_stride_as = threads_in_threadblock / BM; const float *global_a = reinterpret_cast(A) + dim_m * global_a_row + global_a_col; volatile float *local_a_tmp = reinterpret_cast(local_a) + BM * local_as_row + local_as_col; #pragma GCC unroll 1 for (uint32_t local_row_offset = 0; local_row_offset < BK_adjusted; local_row_offset += row_stride_as) { // TODO: the code GCC generates for below seems fine atm, but unroll to // assembly to be absolutely sure *local_a_tmp = *global_a; global_a += dim_m * row_stride_as; local_a_tmp += BM * row_stride_as; } } else { if constexpr (!GMEM_COALESCED_A) { // !GMEM_COALESCED_A: threads do uncoalesced read from neighboring row in // GMEM, writes to neighboring cols in SMEM constexpr uint32_t row_stride_as = threads_in_threadblock / BM; const uint32_t global_a_row = BM * threadblock_id_y + local_as_col; const float *global_a = reinterpret_cast(A) + dim_k_adjusted * global_a_row + (k_adjusted + local_as_row); volatile float *local_a_tmp = reinterpret_cast(local_a) + BM * local_as_row + local_as_col; static_assert( row_stride_as * 8 <= BK_adjusted, "manual loop unrolling condition not met; consider increasing BK"); static_assert( (BK_adjusted % (row_stride_as * 8)) == 0, "manual loop unrolling condition not met; BK should be power-of-two"); #pragma GCC unroll 1 for (uint32_t local_row_offset = 0; local_row_offset < BK_adjusted; local_row_offset += row_stride_as * 8) { // @perf: bank conflicts here // const uint32_t global_a_offset = // dim_k_adjusted * (global_a_row) + (k + local_as_row + local_row_offset); // local_a[BM * (local_as_row + local_row_offset) + local_as_col] = // A[global_a_offset]; // *local_a_tmp = *global_a; asm volatile ("flw ft0, (%0)" :: "r"(global_a)); global_a += row_stride_as; asm volatile ("flw ft1, (%0)" :: "r"(global_a)); global_a += row_stride_as; asm volatile ("flw ft2, (%0)" :: "r"(global_a)); global_a += row_stride_as; asm volatile ("flw ft3, (%0)" :: "r"(global_a)); global_a += row_stride_as; asm volatile ("flw ft4, (%0)" :: "r"(global_a)); global_a += row_stride_as; asm volatile ("flw ft5, (%0)" :: "r"(global_a)); global_a += row_stride_as; asm volatile ("flw ft6, (%0)" :: "r"(global_a)); global_a += row_stride_as; asm volatile ("flw ft7, (%0)" :: "r"(global_a)); global_a += row_stride_as; // NOTE: stride is fixed to word size , i.e. sizeof(float) = 4, // regardless of fp16 or fp32. Since Vortex core does not support fp16, // load things at word granularity and reinterpret bits inside the // tensor core. asm volatile ("fsw ft0, %0(%1)" :: "i"(BM * row_stride_as * 0 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft1, %0(%1)" :: "i"(BM * row_stride_as * 1 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft2, %0(%1)" :: "i"(BM * row_stride_as * 2 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft3, %0(%1)" :: "i"(BM * row_stride_as * 3 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft4, %0(%1)" :: "i"(BM * row_stride_as * 4 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft5, %0(%1)" :: "i"(BM * row_stride_as * 5 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft6, %0(%1)" :: "i"(BM * row_stride_as * 6 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft7, %0(%1)" :: "i"(BM * row_stride_as * 7 * sizeof(float)), "r"(local_a_tmp)); local_a_tmp += BM * row_stride_as * 8; } } else { constexpr uint32_t row_stride_a = threads_in_threadblock / BK_adjusted; const uint32_t global_a_row = BM * threadblock_id_y + local_a_row; const float *global_a = reinterpret_cast(A) + dim_k_adjusted * global_a_row + (k_adjusted + local_a_col); // NOTE that SMEM writes are transposed volatile float *local_a_tmp = reinterpret_cast(local_a) + BM * local_a_col + local_a_row; static_assert( row_stride_a * 8 <= BM, "manual loop unrolling condition not met; consider increasing BM"); static_assert( (BM % (row_stride_a * 8)) == 0, "manual loop unrolling condition not met; BM should be power-of-two"); #pragma GCC unroll 1 for (uint32_t local_row_offset = 0; local_row_offset < BM; local_row_offset += row_stride_a * 8) { // const uint32_t global_a_offset = // dim_k_adjusted * (global_a_row + local_row_offset) + (k + local_a_col); // NOTE that SMEM writes are transposed // local_a[BM * (local_a_col) + local_a_row + local_row_offset] = // A[global_a_offset]; asm volatile ("flw ft0, (%0)" :: "r"(global_a)); global_a += dim_k_adjusted * row_stride_a; asm volatile ("flw ft1, (%0)" :: "r"(global_a)); global_a += dim_k_adjusted * row_stride_a; asm volatile ("flw ft2, (%0)" :: "r"(global_a)); global_a += dim_k_adjusted * row_stride_a; asm volatile ("flw ft3, (%0)" :: "r"(global_a)); global_a += dim_k_adjusted * row_stride_a; asm volatile ("flw ft4, (%0)" :: "r"(global_a)); global_a += dim_k_adjusted * row_stride_a; asm volatile ("flw ft5, (%0)" :: "r"(global_a)); global_a += dim_k_adjusted * row_stride_a; asm volatile ("flw ft6, (%0)" :: "r"(global_a)); global_a += dim_k_adjusted * row_stride_a; asm volatile ("flw ft7, (%0)" :: "r"(global_a)); global_a += dim_k_adjusted * row_stride_a; // stride along columns asm volatile ("fsw ft0, %0(%1)" :: "i"(row_stride_a * 0 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft1, %0(%1)" :: "i"(row_stride_a * 1 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft2, %0(%1)" :: "i"(row_stride_a * 2 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft3, %0(%1)" :: "i"(row_stride_a * 3 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft4, %0(%1)" :: "i"(row_stride_a * 4 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft5, %0(%1)" :: "i"(row_stride_a * 5 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft6, %0(%1)" :: "i"(row_stride_a * 6 * sizeof(float)), "r"(local_a_tmp)); asm volatile ("fsw ft7, %0(%1)" :: "i"(row_stride_a * 7 * sizeof(float)), "r"(local_a_tmp)); local_a_tmp += row_stride_a * 8; } } } // end move A // move B constexpr uint32_t row_stride_b = threads_in_threadblock / BN_adjusted; const uint32_t global_b_col = BN_adjusted * threadblock_id_x + local_b_col; // NOTE: not k_adjusted here; k is along the row dimension which is not // compressed for fp16 const float *global_b = reinterpret_cast(B) + dim_n_adjusted * (k + local_b_row) + global_b_col; volatile float *local_b_tmp = reinterpret_cast(local_b) + BN_adjusted * local_b_row + local_b_col; static_assert( row_stride_b * 8 <= BK_adjusted, "manual loop unrolling condition not met; consider increasing BK"); static_assert( (BK_adjusted % (row_stride_b * 8)) == 0, "manual loop unrolling condition not met; BK should be power-of-two"); #pragma GCC unroll 1 for (uint32_t load_offset = 0; load_offset < BK; load_offset += row_stride_b * 8) { // equivalent code: // // *local_b_tmp = *global_b; // global_b += dim_n * row_stride_b; // local_b_tmp += BN * row_stride_b; asm volatile ("flw ft0, (%0)" :: "r"(global_b)); global_b += dim_n_adjusted * row_stride_b; asm volatile ("flw ft1, (%0)" :: "r"(global_b)); global_b += dim_n_adjusted * row_stride_b; asm volatile ("flw ft2, (%0)" :: "r"(global_b)); global_b += dim_n_adjusted * row_stride_b; asm volatile ("flw ft3, (%0)" :: "r"(global_b)); global_b += dim_n_adjusted * row_stride_b; asm volatile ("flw ft4, (%0)" :: "r"(global_b)); global_b += dim_n_adjusted * row_stride_b; asm volatile ("flw ft5, (%0)" :: "r"(global_b)); global_b += dim_n_adjusted * row_stride_b; asm volatile ("flw ft6, (%0)" :: "r"(global_b)); global_b += dim_n_adjusted * row_stride_b; asm volatile ("flw ft7, (%0)" :: "r"(global_b)); global_b += dim_n_adjusted * row_stride_b; asm volatile ("fsw ft0, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); asm volatile ("fsw ft1, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); local_b_tmp += BN_adjusted * row_stride_b * 2; asm volatile ("fsw ft2, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); asm volatile ("fsw ft3, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); local_b_tmp += BN_adjusted * row_stride_b * 2; asm volatile ("fsw ft4, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); asm volatile ("fsw ft5, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); local_b_tmp += BN_adjusted * row_stride_b * 2; asm volatile ("fsw ft6, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); asm volatile ("fsw ft7, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); local_b_tmp += BN_adjusted * row_stride_b * 2; } asm volatile ("global_dmem_load_finish_%=:" :: ); } template inline void thread_block_gemm(const T *A, const T *B, float *C, const uint32_t dim_m, const uint32_t dim_n, const uint32_t dim_k, const uint32_t tid_in_threadblock, const uint32_t threads_per_threadblock, const uint32_t threadblocks_per_cluster, const uint32_t threadblock_id_in_cluster, uint8_t *sharedmem_per_threadblock) { const uint32_t local_a_row = tid_in_threadblock / BK; const uint32_t local_a_col = tid_in_threadblock % BK; const uint32_t local_as_row = tid_in_threadblock / BM; const uint32_t local_as_col = tid_in_threadblock % BM; const uint32_t local_b_row = tid_in_threadblock / BN; const uint32_t local_b_col = tid_in_threadblock % BN; // no double-buffering const uint32_t threads_per_warpgroup = threads_per_threadblock; const uint32_t warp_id_in_warpgroup = tid_in_threadblock / NUM_THREADS; const uint32_t warp_row = warp_id_in_warpgroup / (BN / WN); const uint32_t warp_col = warp_id_in_warpgroup % (BN / WN); const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; const uint32_t warps_per_threadblock_per_core = NUM_WARPS / threads_per_threadblock; volatile T *local_a = reinterpret_cast(sharedmem_per_threadblock); constexpr size_t local_a_elems = (BM * BK); volatile T *local_a_buf = local_a + local_a_elems; volatile T *local_b = local_a_buf + local_a_elems; constexpr size_t local_b_elems = (BK * BN); volatile T *local_b_buf = local_a_buf + local_b_elems; constexpr uint32_t skips = loop_matmul_skips(/*skip_lda=*/0, /*skip_ldb=*/0, /*skip_ldd=*/1, /*skip_ex=*/1, /*skip_stc=*/1); #if (GEMMINI_DMA == 1) if (tid_in_threadblock == 0) { gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, // b_transpose); gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 0); gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 1); gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); gemmini_fence(); } #endif // divide rows (M) by the number of threadblocks const uint32_t dim_m_range = (dim_m / threadblocks_per_cluster); const uint32_t dim_m_start = dim_m_range * threadblock_id_in_cluster; const uint32_t block_m_start = dim_m_start / BM; const uint32_t block_m_end = (dim_m_start + dim_m_range) / BM; #pragma GCC unroll 1 for (uint32_t block_m = block_m_start; block_m < block_m_end; block_m++) { #pragma GCC unroll 1 for (uint32_t block_n = 0; (block_n * BN) < dim_n; block_n++) { // clear out C initialize_C(0); initialize_C(1); if constexpr (GEMMINI_DMA) { // pipeline initiation if (tid_in_threadblock == 0) { // configure dma gmem address to load from // FIXME: block_k is wrong ROCC_INSTRUCTION_RS1_RS2( XCUSTOM_ACC, (uint64_t)(A + block_m * BM * dim_k + /*block_k:*/0 * BK), (uint64_t)(B + /*block_k:*/0 * BK * dim_n + block_n * BN), k_LOOP_WS_CONFIG_ADDRS_AB) // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); gemmini_fence(); GEMMINI_CISC_CMD_I(10); gemmini_fence(); #if 0 // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS // FIXME: block_k is 0 for two times sp_tiled_matmul_full_spad_ws( #if 1 SPAD_ADDR_Q0, SPAD_ADDR_Q1, #else (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q2 : SPAD_ADDR_Q0, (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q3 : SPAD_ADDR_Q1, #endif /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q3, /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, /*pad_J=*/0, /*pad_K=*/0, /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) gemmini_fence(); #endif } threadblock_barrier(threadblock_id_in_cluster, warps_per_threadblock_per_core); } #pragma GCC unroll 1 for (uint32_t block_k = 0; (block_k * BK) < dim_k; block_k++) { // producer code: GMEM->SMEM memory movement // --------------------------------------------------------------------- // // this is either done using DMA or SIMT cores depending on GEMMINI_DMA #if (GEMMINI_DMA == 1) if ((tid_in_threadblock == 0) && ((block_k * BK) != (dim_k - BK))) { // configure dma gmem address to load from // FIXME: block_k is wrong ROCC_INSTRUCTION_RS1_RS2( XCUSTOM_ACC, (uint64_t)(A + block_m * BM * dim_k + (block_k + 1/*runahead*/) * BK), (uint64_t)(B + (block_k + 1/*runahead*/) * BK * dim_n + block_n * BN), k_LOOP_WS_CONFIG_ADDRS_AB) // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); // gemmini_fence(); // block_k is even: opcode 11 (write to local_a_buf) // block_k is odd: opcode 10 (write to local_a) const uint32_t opcode = 11 - (block_k & 1); GEMMINI_CISC_CMD_R(opcode); // // TODO: branch is probably slow // if (block_k & 1) { // GEMMINI_CISC_CMD_I(12); // } else { // block_k == 0 is here // GEMMINI_CISC_CMD_I(13); // } // configure loop iteration bounds // FIXME: shouldn't be necessary // ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, BOUND_INST, // k_LOOP_WS_CONFIG_BOUNDS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, // SPAD_ADDR_Q0, SPAD_ADDR_Q1, k_LOOP_WS_CONFIG_SPAD_AB) // ROCC_INSTRUCTION_RS1_RS2( // XCUSTOM_ACC, // ((uint64_t)(/*a_spad_id:*/ 0) << 18) | // ((uint64_t)(/*b_spad_id:*/ 0) << 16) | // ((uint64_t)(/*act:0*/ 0) << 8) | ((/*low_D:*/ 0) << 2) | // ((/*full_C:*/ 0) << 1) | (/*ex_accumulate:*/ 0), // ((uint64_t)(/*C_spad_addr:*/ A) << 32) | 0x200U | (skips) | // ((/*is_resadd*/ 0) << 2) | ((/*B_transpose:*/ 0) << 1) | // (/*A_transpose:*/ 1), // k_LOOP_WS) // gemmini_fence(); #if 0 uint32_t spad_a_produce; uint32_t spad_b_produce; const uint32_t mask_odd = (block_k & 1) << 31 >> 31; const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; spad_a_produce = ((mask_odd & (SPAD_ADDR_Q0)) | (mask_even & (SPAD_ADDR_Q2))); spad_b_produce = ((mask_odd & (SPAD_ADDR_Q1)) | (mask_even & (SPAD_ADDR_Q3))); // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS // FIXME: block_k is 0 for two times sp_tiled_matmul_full_spad_ws( spad_a_produce, spad_b_produce, /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q1, /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, /*pad_J=*/0, /*pad_K=*/0, /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) #endif } #else global_dmem_load(dim_m, dim_n, dim_k, block_k * BK, A, B, local_a, local_b, tid_in_threadblock, block_n, block_m); threadblock_barrier(threadblock_id_in_cluster, warps_per_threadblock_per_core); #endif // consumer code: SMEM->RF and compute // ---------------------------------------------------------------------- // @perf: this loop spills to stack a lot because of all the flws in const volatile T *local_a_consume; const volatile T *local_b_consume; if constexpr (GEMMINI_DMA) { // local_a_consume = (k_index % 2) ? local_a_buf : local_a; // local_b_consume = (k_index % 2) ? local_b_buf : local_b; // FIXME: swap multiply with bitshifts // const uint32_t mask_odd = (block_k & 1) << 31 >> 31; // const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; // local_a_consume = reinterpret_cast( // (mask_odd & reinterpret_cast(local_a_buf)) | // (mask_even & reinterpret_cast(local_a))); // local_b_consume = reinterpret_cast( // (mask_odd & reinterpret_cast(local_b_buf)) | // (mask_even & reinterpret_cast(local_b))); local_a_consume = local_a + (block_k & 1) * (local_a_elems); local_b_consume = local_b + (block_k & 1) * (local_b_elems); } else { // no double-buffering without DMA local_a_consume = local_a; local_b_consume = local_b; } #pragma GCC unroll 1 for (int i = 0; i < BK_LOOP; i++) { #pragma GCC unroll 4 for (uint32_t local_k = 0; local_k < BK; local_k += TCK) { #pragma GCC unroll 2 for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { // SMEM -> RF vx_wmma_load_b(local_b_consume, local_k, warp_col, wn_iter, tid_in_warp); #pragma GCC unroll 2 for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { // SMEM -> RF vx_wmma_load_a(local_a_consume, local_k, warp_row, wm_iter, tid_in_warp); // perform mma vx_wmma(wm_iter); } } } } if constexpr (GEMMINI_DMA) { // Call gemmini fence at the end of the loop to overlap dma & wmma. // Usually, by this time, dma has finished the copy so that this // becomes a no-op. if (tid_in_threadblock == 0) { gemmini_fence(); } } threadblock_barrier(threadblock_id_in_cluster, warps_per_threadblock_per_core); } if constexpr (write_to_gmem) { #pragma GCC unroll 2 for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { #pragma GCC unroll 2 for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { write_results(tid_in_warp, warp_col, warp_row, wn_iter, wm_iter, dim_n, C, block_n, block_m); } } } } } } #endif