Merge branch 'ae' into ae-flash-ampere

kernels/flash_attention/Makefile

@@ -1,8 +1,8 @@
 PROJECT = flash_attention
 
-# VX_SRCS = kernel.cpp
+VX_SRCS = kernel.cpp
 # VX_SRCS = kernel.gemmini.warpspec.cpp
-VX_SRCS = kernel.gemmini.cpp
+# VX_SRCS = kernel.gemmini.cpp
 VX_INCLUDES = flash_impl.hpp ../sgemm_tcore/sgemm_impl.hpp
 
 OPTS ?= -n16

kernels/flash_attention/flash_impl.hpp

@@ -17,9 +17,9 @@ constexpr uint32_t ROWMAX_SETS = 3;
 // constexpr bool WARP_SPECIALIZED = true;
 // constexpr bool GEMMINI_WARP_SPECIALIZED = false;
 // constexpr bool TENSOR_CORE = true;
-constexpr bool WARP_SPECIALIZED = false;
+constexpr bool WARP_SPECIALIZED = true;
 constexpr bool GEMMINI_WARP_SPECIALIZED = false;
-constexpr bool TENSOR_CORE = false;
+constexpr bool TENSOR_CORE = true;
 
 // temporary safety stop for wrong configs
 static_assert(NUM_CORES == 4);

kernels/sgemm_tcore/kernel.cpp

@@ -95,8 +95,6 @@ void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
   constexpr uint32_t quartile = (128 << 10) >> 2; // 128KB / 4
   static_assert((quartile * 4) == SMEM_SIZE, "wrong quartile constant");
 
-  MARK_BEG();
-
   constexpr uint32_t smem_a_offset = 0;
   constexpr uint32_t smem_a_dbuf_offset = 1 * quartile;
   constexpr uint32_t smem_b_offset =
@@ -121,8 +119,6 @@ void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
                       threadblocks_per_cluster, threadblock_id_in_cluster,
                       sharedmem_per_threadblock);
 
-  MARK_END();
-
   float *gmem_tmp_d0 = reinterpret_cast<float *>(0xd0000000UL);
   float *gmem_tmp_d1 = reinterpret_cast<float *>(0xd1000000UL);
   float *gmem_tmp_d2 = reinterpret_cast<float *>(0xd2000000UL);

kernels/sgemm_tcore/sgemm_impl.hpp

@@ -6,7 +6,7 @@
 #include "include/gemmini.h"
 #include "gemmini_mmio.h"
 
-#define FP_SIZE 16
+#define FP_SIZE 32
 
 // "fake" fp16 type that only has the correct data width.
 using float16_t = uint16_t;
@@ -19,7 +19,7 @@ using float_type = float16_t;
 
 // Generate kernel for the Hopper-style SMEM-decoupled tensor core.  This uses
 // asynchronous HGMMA and HGMMA_WAIT instructions.
-#define TENSOR_HOPPER 1
+#define TENSOR_HOPPER 0
 
 // Constraints on parameters:
 // * Memory:

lib/gemmini/include/gemmini_params.h

@@ -1 +1 @@
-gemmini_params.dim16fp16.h
+gemmini_params.dim8fp32.h

lib/include/gemmini_mmio.h

@@ -12,9 +12,9 @@
 // 64KB
 // #define SMEM_SIZE 0x10000
 // 128KB (FP16 GEMM)
-#define SMEM_SIZE 0x20000
+// #define SMEM_SIZE 0x20000
 // 256KB (FlashAttention)
-// #define SMEM_SIZE 0x40000
+#define SMEM_SIZE 0x40000
 
 #define SMEM_MASK (SMEM_SIZE - 1)
 #define SMEM_ADDR_END (SMEM_BASE + SMEM_SIZE)