diff --git a/kernel/include/gemmini_mmio.h b/kernel/include/gemmini_mmio.h index 17b18a79..9cb67ad4 100644 --- a/kernel/include/gemmini_mmio.h +++ b/kernel/include/gemmini_mmio.h @@ -5,7 +5,8 @@ #endif #define SMEM_BASE 0xff000000 -#define SMEM_SIZE 0x4000 +//#define SMEM_SIZE 0x4000 +#define SMEM_SIZE 0x10000 #define SMEM_MASK (SMEM_SIZE - 1) #define SMEM_ADDR_END (SMEM_BASE + SMEM_SIZE) @@ -31,10 +32,13 @@ // 0: k = 0, no accumulation // 1: k % 2 = 0, buffer regions 0 // 2: k % 2 = 1, buffer regions 1 -// 8, 9: memory ops -// 8: tile-sized move-in (unused) +// 8, 9, 10, 11: memory ops +// 8: tile-sized move-in stride // 9: tile-sized move-out +// 10: tile-sized move-in, buffer regions 0 +// 11: tile-sized move-in, buffer regions 1 #define GEMMINI_CISC_CMD_I(x) asm("csrwi 0xacc, "#x) +#define GEMMINI_CISC_CMD_R(x) asm("csrw 0xacc, %0" :: "r" (x)) #define GEMMINI_STATUS() ({uint32_t status; asm volatile ("csrr %0, 0xacc" : "=r" (status)); status;}) // convert normal matrix i,j into tiled smem offset @@ -62,6 +66,9 @@ /* sprintf((char *) PRINT_BUF, "%llx %llx %d\n", rs1, rs2, funct); */ \ } +#define loop_matmul_skips(skip_lda, skip_ldb, skip_ldd, skip_ex, skip_stc) \ + (((skip_lda) | ((skip_ldb) << 1) | ((skip_ldd) << 2) | ((skip_ex) << 3) | ((skip_stc) << 4)) << 3) + #define sp_tiled_matmul_full_spad_ws(A_sp_addr_start, B_sp_addr_start, D_sp_addr_start, C_dst_sp_addr_start,\ I, J, K, pad_I, pad_J, pad_K, a_transpose, b_transpose, full_C, low_D, acc, act, skips) \ gemmini_loop_ws_spad(I, J, K, pad_I, pad_J, pad_K, A_sp_addr_start, (B_sp_addr_start) + (K) * (J) * DIM, NULL, \ diff --git a/kernel/linker/vx_link32.ld b/kernel/linker/vx_link32.ld index ea5c4e56..e84c0342 100644 --- a/kernel/linker/vx_link32.ld +++ b/kernel/linker/vx_link32.ld @@ -10,8 +10,8 @@ ENTRY(_start) MEMORY { DRAM0 (rwx): ORIGIN = 0x80000000, LENGTH = 512M - DRAM1 (rwx): ORIGIN = 0xa0000000, LENGTH = 32K - DRAM2 (rwx): ORIGIN = 0xa1000000, LENGTH = 32K + DRAM1 (rwx): ORIGIN = 0xa0000000, LENGTH = 16M + DRAM2 (rwx): ORIGIN = 0xa1000000, LENGTH = 16M } SECTIONS @@ -69,6 +69,7 @@ SECTIONS /* .gnu.warning sections are handled specially by elf.em. */ *(.gnu.warning) } + .fini : { KEEP (*(SORT_NONE(.fini))) @@ -128,6 +129,7 @@ SECTIONS KEEP (*(.init_array EXCLUDE_FILE (*crtbegin.o *crtbegin?.o *crtend.o *crtend?.o ) .ctors)) PROVIDE_HIDDEN (__init_array_end = .); } + .fini_array : { PROVIDE_HIDDEN (__fini_array_start = .); @@ -135,6 +137,18 @@ SECTIONS KEEP (*(.fini_array EXCLUDE_FILE (*crtbegin.o *crtbegin?.o *crtend.o *crtend?.o ) .dtors)) PROVIDE_HIDDEN (__fini_array_end = .); } + + .htif_pad : { + . = ALIGN(0x1000); + } + + . = ALIGN(0x1000); + .tohost : { + *(.tohost) + /* . += 0x100; */ + } + + . = ALIGN(0x1000); .ctors : { /* gcc uses crtbegin.o to find the start of @@ -216,6 +230,8 @@ SECTIONS MAX(__DATA_BEGIN__ + 0x800, __BSS_END__ - 0x800)); _end = .; PROVIDE (end = .); . = DATA_SEGMENT_END (.); + + /* Stabs debugging sections. */ .stab 0 : { *(.stab) } .stabstr 0 : { *(.stabstr) } diff --git a/kernel/src/vx_start.S b/kernel/src/vx_start.S index 49e520b6..65dbb9a6 100644 --- a/kernel/src/vx_start.S +++ b/kernel/src/vx_start.S @@ -150,3 +150,4 @@ vx_wspawn_wait: .weak __dso_handle __dso_handle: .long 0 + diff --git a/kernel/tohost.S b/kernel/tohost.S new file mode 100644 index 00000000..cfb4ba25 --- /dev/null +++ b/kernel/tohost.S @@ -0,0 +1,7 @@ +.section ".tohost","aw",@progbits +.align 6 +.globl tohost +tohost: .dword 0 +.align 6 +.globl fromhost +fromhost: .dword 0 diff --git a/tests/regression/bad_apple/.gitignore b/tests/regression/bad_apple/.gitignore new file mode 100644 index 00000000..0322feda --- /dev/null +++ b/tests/regression/bad_apple/.gitignore @@ -0,0 +1,6 @@ +*.bin +*.dump +*.elf +sgemm_wg +.depend +kernel.radiance.elf diff --git a/tests/regression/bad_apple/Makefile b/tests/regression/bad_apple/Makefile new file mode 100644 index 00000000..e83a1a8e --- /dev/null +++ b/tests/regression/bad_apple/Makefile @@ -0,0 +1,9 @@ +PROJECT = bad_apple + +SRCS = main.cpp common.h + +VX_SRCS = kernel.cpp + +OPTS ?= -n16 + +include ../common.mk diff --git a/tests/regression/bad_apple/bad_apple b/tests/regression/bad_apple/bad_apple new file mode 100755 index 00000000..67ade61b Binary files /dev/null and b/tests/regression/bad_apple/bad_apple differ diff --git a/tests/regression/bad_apple/common.h b/tests/regression/bad_apple/common.h new file mode 100644 index 00000000..74941562 --- /dev/null +++ b/tests/regression/bad_apple/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000 +#define DEV_SMEM_START_ADDR 0xff000000 + +typedef struct { + uint32_t dim_m; + uint32_t dim_n; + uint32_t dim_k; + uint64_t addr_a; + uint64_t addr_b; + uint64_t addr_c; +} kernel_arg_t; + +#endif diff --git a/tests/regression/bad_apple/display.py b/tests/regression/bad_apple/display.py new file mode 100644 index 00000000..13b7885b --- /dev/null +++ b/tests/regression/bad_apple/display.py @@ -0,0 +1,103 @@ +import time +import os +import struct +from PIL import Image +from io import BytesIO +import subprocess +import numpy as np +import base64 +import cv2 + +use_fpga = False +buffer_width = 16 +buffer_depth = 1350 +frame_width = 480 +frame_height = 360 +upscale = 3 + +# buffer_width = 16 +# buffer_depth = 0x152 +# frame_width = 240 +# frame_height = 180 +# upscale = 6 + +buffer_size = buffer_depth * buffer_width +truncate = 300 if use_fpga else 0 + +def follow(filename): + frame_count = 0 + with open(filename, "r") as file: + while True: + line = file.readline() + if not line: + time.sleep(0.001) + continue + if truncate and (" 0151 " in line): + frame_count += 1 + if frame_count == truncate: + with open(filename, "w") as f: + f.truncate(0) + frame_count = 0 + yield line + +def process_frame(frame_data): + bits = np.unpackbits(np.frombuffer(frame_data, dtype=np.uint8)) + bits = bits[:frame_width * frame_height] + image_array = bits.reshape((frame_height, frame_width)) + # image_array = np.flipud(np.fliplr(image_array)) + image_array = (image_array * 255).astype(np.uint8) + filtered_image_array = cv2.fastNlMeansDenoising(image_array, None, h=72, templateWindowSize=8, searchWindowSize=8) + # filtered_image_array = cv2.GaussianBlur(image_array, (3, 3), 0) + # filtered_image_array = image_array + + filtered_image_array = np.kron(filtered_image_array, np.ones((upscale, upscale), dtype=np.uint8)) + + image = Image.fromarray(filtered_image_array, mode='L') + + return image + +def display_image(img): + with BytesIO() as output: + # img = img.resize((frame_width * upscale, frame_height * upscale), Image.NEAREST) + img.save(output, format='PNG') + + output.seek(0) + # subprocess.run(["/home/eecs/yrh/.iterm2/imgcat"], input=output.read()) + image_data = output.getvalue() + b64_image_data = base64.b64encode(image_data).decode('utf-8') + + print("\033]", end='') + print(f"1337;File=inline=1", end='') + print(f";size={len(image_data)}", end='') + print(f";name={base64.b64encode('tmp.png'.encode()).decode('utf-8')}", end='') + # print(f";width={frame_width * upscale * 4};height={frame_height * upscale * 4}", end='') + print(f":{b64_image_data}", end='') + print("\a", end='') + print('\n') + +def main(): + if not use_fpga: + filename = "/scratch/yrh/chipyard/sims/vcs/output/chipyard.harness.TestHarness.RadianceClusterConfig/kernel.radiance.out" + else: + filename = "/scratch/yrh/chipyard/sims/firesim/sim/generated-src/xilinx_alveo_u250/xilinx_alveo_u250-firesim-FireSim-FireSimRadianceClusterSynConfig-WithPrintfSynthesis_BaseXilinxAlveoU250Config/synthesized-prints.out0" + # frame_data = {} + frame_data0 = bytearray(buffer_size) + for line in follow(filename): + if not "fb0" in line: + continue + tokens = line.split() + if not len(tokens) == (5 if use_fpga else 3): + continue + offset, data = tokens[3 if use_fpga else 1:] + offset0 = int(offset, 16) + + frame_data0[offset0 * buffer_width : (offset0 + 1) * buffer_width] = bytes.fromhex(data)[::-1] + + if offset0 == buffer_depth - 1: + img = process_frame(frame_data0) + display_image(img) + frame_data0 = bytearray(buffer_size) + +if __name__ == "__main__": + main() + diff --git a/tests/regression/bad_apple/display.py.bak b/tests/regression/bad_apple/display.py.bak new file mode 100644 index 00000000..9beee99d --- /dev/null +++ b/tests/regression/bad_apple/display.py.bak @@ -0,0 +1,83 @@ +import time +import os +import struct +from PIL import Image +from io import BytesIO +import subprocess + +use_fpga = False + +def follow(filename): + with open(filename, "r") as file: + # file.seek(0, os.SEEK_END) # Move to the end of the file + while True: + line = file.readline() + if not line: + time.sleep(0.001) + continue + yield line.strip() + +def process_frame(frame_data): + # Create a byte array from the frame data + byte_array = bytearray() + # for offset, data in frame_data.items(): + # byte_array += struct.pack("> bit_index) & 1 + + return img + +def display_image(img): + with BytesIO() as output: + img = img.resize((120, 90), Image.NEAREST) + img.save(output, format='PNG') + output.seek(0) + subprocess.run(["/home/eecs/yrh/.iterm2/imgcat", "-H", "98%"], input=output.read()) + +def main(): + if not use_fpga: + filename = "/scratch/yrh/chipyard/sims/vcs/output/chipyard.harness.TestHarness.RadianceClusterConfig/kernel.radiance.out" + else: + filename = "/scratch/yrh/chipyard/sims/firesim/sim/generated-src/xilinx_alveo_u250/xilinx_alveo_u250-firesim-FireSim-FireSimRadianceClusterSynConfig-WithPrintfSynthesis_BaseXilinxAlveoU250Config/synthesized-prints.out0" + # frame_data = {} + frame_data0 = [0 for _ in range(80)] + frame_data1 = [0 for _ in range(80)] + for line in follow(filename): + if not "fb0" in line: + continue + tokens = line.split() + if not len(tokens) == 7 if use_fpga else 5: + continue + offset, data = tokens[4 if use_fpga else 2:-1] + offset = int(offset, 16) - 0xff010000 + offset0 = offset + offset1 = offset - 0x200 + + if offset0 >= 0 and offset0 < 320: + frame_data0[offset0 // 4] = int(data, 16) + + if offset1 >= 0 and offset1 < 320: + frame_data1[offset1 // 4] = int(data, 16) + + if offset0 == 0x130 and data == "ff010130": + img = process_frame(frame_data0) + frame_data0 = [0 for _ in range(80)] + display_image(img) + elif offset1 == 0x130 and data == "ff010330": + img = process_frame(frame_data1) + frame_data1 = [0 for _ in range(80)] + display_image(img) + +if __name__ == "__main__": + main() + diff --git a/tests/regression/bad_apple/kernel.cpp b/tests/regression/bad_apple/kernel.cpp new file mode 100644 index 00000000..2917f40f --- /dev/null +++ b/tests/regression/bad_apple/kernel.cpp @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include "common.h" + +#define rd_cycles(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) +#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#define PRINTF(...) sprintf((char *) (0xff010000UL), __VA_ARGS__) + +inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { + vx_tmc(0xff); + const volatile uint32_t *const A = (const volatile uint32_t *const) arg->addr_a; + + // vx_tmc(1); + // for (int i = 0; i < 75; i++) { + // if (task_id == i) { + // PRINTF("%d", task_id); + // } + // } + // threadblock_barrier(2, 5); + + /* + #define mark(x) \ + vx_tmc(0x80); \ + if (task_id == 79) *(((volatile uint32_t *) x)) = x; \ + vx_fence(); \ + vx_barrier(x & 1, 5); \ + vx_tmc(0xff) + + #define write_fb0(value) *(((volatile uint32_t *) 0xff010000UL) + task_id) = (value) + #define write_fb1(value) *(((volatile uint32_t *) 0xff010200UL) + task_id) = (value) + + while (true) { + uint32_t v0, v1, v2, v3; + v0 = A[task_id]; + v1 = A[1 * 75 + task_id]; + v2 = A[2 * 75 + task_id]; + v3 = A[3 * 75 + task_id]; + for (int i = 30; i < 6569; i += 4) { + write_fb1(v0); + v0 = A[(i + 0) * 75 + task_id]; + mark(0xff010130); + + write_fb0(v1); + v1 = A[(i + 1) * 75 + task_id]; + mark(0xff010330); + + write_fb1(v2); + v2 = A[(i + 2) * 75 + task_id]; + mark(0xff010130); + + write_fb0(v3); + v3 = A[(i + 3) * 75 + task_id]; + mark(0xff010330); + } + } + */ + + #define WORKERS 128 + + // #define WORDS 1350 + // #define LINES 338 + // #define ITERS 11 // = 1350 / 128 + + #define WORDS 5400 + #define LINES 1350 + #define T_ITERS 43 + + #define mark_fb0() \ + vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011000UL)) = LINES; \ + vx_fence(); vx_barrier(0, 8); vx_tmc(0xff) + #define mark_fb1() \ + vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011004UL)) = LINES; \ + vx_fence(); vx_barrier(1, 8); vx_tmc(0xff) + #define write_fb0(addr, value) *(((volatile uint32_t *) 0xff018000UL) + addr) = (value) + #define write_fb1(addr, value) *(((volatile uint32_t *) 0xff020000UL) + addr) = (value) + + #define CYCLES_TO_WAIT 240000 + + uint64_t cycles0, cycles1; + cycles0 = 0; + + while (true) { + volatile uint32_t v0, v1; + for (int i = 20; i < 6569; i += 1) { + v0 = A[i * WORDS + task_id]; + v1 = A[i * WORDS + WORKERS + task_id]; + int offset0 = 0 * WORKERS + task_id; + int offset1 = 1 * WORKERS + task_id; + + for (int j = 1; j < T_ITERS; j += 2) { + write_fb0(offset0, v0); + offset0 += 2 * WORKERS; + v0 = A[(i + 0) * WORDS + offset0]; + write_fb0(offset1, v1); + offset1 += 2 * WORKERS; + v1 = A[(i + 0) * WORDS + offset1]; + } + write_fb0(offset0, v0); + write_fb0(offset1, v1); + + /*offset0 += 2 * WORKERS; + v0 = A[(i + 0) * WORDS + offset0]; + write_fb0(offset0, v0);*/ + + if (task_id == 0) { + rd_cycles(cycles1); + while (cycles1 - cycles0 < CYCLES_TO_WAIT) { + rd_cycles(cycles1); + } + cycles0 = cycles1; + } + + threadblock_barrier(0, 8); + mark_fb0(); + } + } +} + +int main() { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + +#ifdef RADIANCE + vx_spawn_tasks_cluster(128, (vx_spawn_tasks_cb)kernel_body, arg); +#else + // NOTE: This kernel assumes contiguous thread scheduling for efficient shared + // memory allocation, and therefore does not work with original vx_spawn_tasks + vx_spawn_tasks_contiguous(8, (vx_spawn_tasks_cb)kernel_body, arg); +#endif + return 0; +} \ No newline at end of file diff --git a/tests/regression/bad_apple/main.cpp b/tests/regression/bad_apple/main.cpp new file mode 100644 index 00000000..54531062 --- /dev/null +++ b/tests/regression/bad_apple/main.cpp @@ -0,0 +1,274 @@ +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.bin"; +uint32_t count = 0; + +std::vector src_a_data; +std::vector src_b_data; +std::vector ref_data; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + count = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(device, kernel_arg.addr_a); + vx_mem_free(device, kernel_arg.addr_b); + vx_mem_free(device, kernel_arg.addr_c); + vx_dev_close(device); + } +} + +void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + src_a_data.resize(dim_m * dim_k); + src_b_data.resize(dim_k * dim_n); + + for (uint32_t i = 0; i < src_a_data.size(); ++i) { + src_a_data[i] = static_cast(i); + std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl; + } + for (uint32_t i = 0; i < src_b_data.size(); ++i) { + src_b_data[i] = static_cast(i); + std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl; + } +} + +void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + ref_data.resize(dim_m * dim_n); + + for (uint32_t i = 0; i < dim_m; ++i) { + for (uint32_t j = 0; j < dim_n; ++j) { + float ref = 0.0f; + for (uint32_t k = 0; k < dim_k; ++k) { + ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j]; + } + ref_data.at(dim_n * i + j) = ref; + } + } +} + +int run_test(const kernel_arg_t& kernel_arg, + uint32_t buf_size, + uint32_t dim_m, uint32_t dim_n) { + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size)); + + // verify result + std::cout << "verify result" << std::endl; + { + int errors = 0; + auto buf_ptr = (float*)staging_buf.data(); + for (uint32_t i = 0; i < dim_m * dim_n; ++i) { + float ref = ref_data.at(i); + float cur = buf_ptr[i]; + if (std::abs((cur - ref) / ref) > 1e-6) { + std::cout << "error at result #" << std::dec << i + << std::hex << ": actual=" << cur << ", expected=" << ref << std::endl; + ++errors; + } + } + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + } + + return 0; +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + if (count == 0) { + count = 1; + } + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + // FIXME: hardcoded + uint32_t dim_m = 64; + uint32_t dim_n = 64; + uint32_t dim_k = 64; + + generate_source_matrix(dim_m, dim_n, dim_k); + generate_reference_matmul(dim_m, dim_n, dim_k); + + uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]); + uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]); + uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]); + + std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl; + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a)); + RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b)); + RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c)); + + kernel_arg.dim_m = dim_m; + kernel_arg.dim_n = dim_n; + kernel_arg.dim_k = dim_k; + + std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl; + std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl; + std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl; + + // allocate staging buffer + { + std::cout << "allocate staging buffer" << std::endl; + uint32_t staging_buf_size = std::max( + src_a_buf_size, + std::max( + src_b_buf_size, + std::max(dst_buf_size, sizeof(kernel_arg_t)))); + staging_buf.resize(staging_buf_size); + } + + // upload kernel argument + { + std::cout << "upload kernel argument" << std::endl; + auto buf_ptr = staging_buf.data(); + kernel_arg.addr_a = (uint64_t) 0x20000; + kernel_arg.addr_b = (uint64_t) 0x28000; + kernel_arg.addr_c = (uint64_t) 0xc0000000ULL; + memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); + + std::cout << "uploading argument buffer to device, device mem address=" + << std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec + << sizeof(kernel_arg_t) << " bytes\n"; + std::ofstream file("args.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(staging_buf.data()), + sizeof(kernel_arg_t)); + file.close(); + + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + } + + // upload source buffer + { + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(), + src_a_buf_size)); + + std::cout << "uploading source A matrix to device, device mem address=" + << std::hex << kernel_arg.addr_a << ", size=" << std::dec + << src_a_buf_size << " bytes\n"; + std::ofstream file("input.a.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_a_buf_size); + file.close(); + } + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(), + src_b_buf_size)); + + std::cout << "uploading source B matrix to device, device mem address=" + << std::hex << kernel_arg.addr_b << ", size=" << std::dec + << src_b_buf_size << " bytes\n"; + std::ofstream file("input.b.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_b_buf_size); + file.close(); + } + } + + // clear destination buffer + { + std::cout << "clear destination buffer" << std::endl; + auto buf_ptr = (int32_t*)staging_buf.data(); + for (uint32_t i = 0; i < ref_data.size(); ++i) { + buf_ptr[i] = 0xdeadbeef; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size)); + } + + // run tests + std::cout << "run tests" << std::endl; + RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n)); + std::cout << "PASSED!" << std::endl; + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + return 0; +} diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 24a871eb..09a57795 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -53,7 +53,8 @@ VX_CFLAGS += -mcmodel=medany -fno-rtti -fno-exceptions -nostartfiles -fdata-sect VX_CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw -I$(GEMMINI_SW_PATH) VX_CFLAGS += -DNDEBUG -DLLVM_VORTEX -VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a +# VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a +VX_LDFLAGS += -Wl,-Bstatic,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a $(VORTEX_KN_PATH)/tohost.S CXXFLAGS += -std=c++17 -Wall -Wextra -pedantic -Wfatal-errors CXXFLAGS += -I$(VORTEX_RT_PATH)/include -I$(VORTEX_KN_PATH)/../hw diff --git a/tests/regression/rickroll/.gitignore b/tests/regression/rickroll/.gitignore new file mode 100644 index 00000000..0322feda --- /dev/null +++ b/tests/regression/rickroll/.gitignore @@ -0,0 +1,6 @@ +*.bin +*.dump +*.elf +sgemm_wg +.depend +kernel.radiance.elf diff --git a/tests/regression/rickroll/Makefile b/tests/regression/rickroll/Makefile new file mode 100644 index 00000000..7717878d --- /dev/null +++ b/tests/regression/rickroll/Makefile @@ -0,0 +1,9 @@ +PROJECT = rickroll + +SRCS = main.cpp common.h + +VX_SRCS = kernel.cpp + +OPTS ?= -n16 + +include ../common.mk diff --git a/tests/regression/rickroll/common.h b/tests/regression/rickroll/common.h new file mode 100644 index 00000000..74941562 --- /dev/null +++ b/tests/regression/rickroll/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000 +#define DEV_SMEM_START_ADDR 0xff000000 + +typedef struct { + uint32_t dim_m; + uint32_t dim_n; + uint32_t dim_k; + uint64_t addr_a; + uint64_t addr_b; + uint64_t addr_c; +} kernel_arg_t; + +#endif diff --git a/tests/regression/rickroll/display_color.py b/tests/regression/rickroll/display_color.py new file mode 100644 index 00000000..aaf6faf7 --- /dev/null +++ b/tests/regression/rickroll/display_color.py @@ -0,0 +1,128 @@ +import time +import os +import struct +from PIL import Image +from io import BytesIO +import subprocess +import numpy as np +import base64 +import cv2 + +use_fpga = False +buffer_width = 16 + +# buffer_depth = 0x152 +# frame_width = 240 +# frame_height = 180 +# upscale = 6 +buffer_depth = 1800 +frame_width = 160 +frame_height = 120 +upscale = 1 + + +buffer_size = int(buffer_depth * buffer_width * 1.5) +truncate = 0 # 300 if use_fpga else 0 + +def follow(filename): + frame_count = 0 + with open(filename, "r") as file: + while True: + current_position = file.tell() + line = file.readline() + if not line: + time.sleep(0.001) + continue + if "fb0" in line and len(line) < (61 if use_fpga else 41): + file.seek(current_position) + time.sleep(0.001) + continue + if truncate and (" 0151 " in line): + frame_count += 1 + if frame_count == truncate: + with open(filename, "w") as f: + f.truncate(0) + frame_count = 0 + yield line + +def process_frame(frame_data): + bits = np.unpackbits(np.frombuffer(frame_data, dtype=np.uint8)) + bits = bits[:frame_width * frame_height] + image_array = bits.reshape((frame_height, frame_width)) + # image_array = np.flipud(np.fliplr(image_array)) + image_array = (image_array * 255).astype(np.uint8) + + raw_array = np.frombuffer(frame_data, dtype=np.uint8) + y_size = frame_width * frame_height + c_size = y_size // 4 + y_array = raw_array[:y_size].reshape((frame_height, frame_width)) + cr_array, cb_array = raw_array[y_size : y_size + c_size].reshape((frame_height // 2, frame_width // 2)), raw_array[y_size + c_size : y_size + 2 * c_size].reshape((frame_height // 2, frame_width // 2)) + + cb_upscaled = cv2.resize(cb_array, (frame_width, frame_height), interpolation=cv2.INTER_LINEAR) + cr_upscaled = cv2.resize(cr_array, (frame_width, frame_height), interpolation=cv2.INTER_LINEAR) + + # Merge the channels back to YCrCb format + ycrcb_frame = cv2.merge((y_array, cb_upscaled, cr_upscaled)) + bgr_frame = cv2.cvtColor(ycrcb_frame, cv2.COLOR_YCrCb2BGR) + is_success, buffer = cv2.imencode(".png", bgr_frame) + io_buf = BytesIO(buffer) + + # filtered_image_array = image_array + + # filtered_image_array = cv2.fastNlMeansDenoising(image_array, None, h=72, templateWindowSize=8, searchWindowSize=8) + # filtered_image_array = cv2.GaussianBlur(image_array, (3, 3), 0) + + # filtered_image_array = cv2.blur(image_array, (2, 2)) + # filtered_image_array = cv2.fastNlMeansDenoising(filtered_image_array, None, h=72, templateWindowSize=4, searchWindowSize=4) + + # filtered_image_array = np.kron(filtered_image_array, np.ones((upscale, upscale), dtype=np.uint8)) + + # image = Image.fromarray(filtered_image_array, mode='L') + + return io_buf + +def display_image(img): + # img = img.resize((frame_width * upscale, frame_height * upscale), Image.NEAREST) + # img.save(output, format='PNG') + output = img + + output.seek(0) + # subprocess.run(["/home/eecs/yrh/.iterm2/imgcat"], input=output.read()) + image_data = output.getvalue() + b64_image_data = base64.b64encode(image_data).decode('utf-8') + + print("\033]", end='') + print(f"1337;File=inline=1", end='') + print(f";size={len(image_data)}", end='') + print(f";name={base64.b64encode('tmp.png'.encode()).decode('utf-8')}", end='') + # print(f";width={frame_width * upscale * 4};height={frame_height * upscale * 4}", end='') + print(f":{b64_image_data}", end='') + print("\a", end='') + print('\n') + +def main(): + if not use_fpga: + filename = "/scratch/yrh/chipyard/sims/vcs/output/chipyard.harness.TestHarness.RadianceClusterConfig/kernel.radiance.out" + else: + filename = "/scratch/yrh/firesim-rundir/sim_slot_0/synthesized-prints.out0" + # frame_data = {} + frame_data0 = bytearray(buffer_size) + for line in follow(filename): + if not "fb0" in line: + continue + tokens = line.split() + if not len(tokens) == (5 if use_fpga else 3): + continue + offset, data = tokens[3 if use_fpga else 1:] + offset0 = int(offset, 16) + + frame_data0[offset0 * buffer_width : (offset0 + 1) * buffer_width] = bytes.fromhex(data)[::-1] + + if offset0 == buffer_depth - 1: + img = process_frame(frame_data0) + display_image(img) + frame_data0 = bytearray(buffer_size) + +if __name__ == "__main__": + main() + diff --git a/tests/regression/rickroll/kernel.cpp b/tests/regression/rickroll/kernel.cpp new file mode 100644 index 00000000..64bc2d37 --- /dev/null +++ b/tests/regression/rickroll/kernel.cpp @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include "common.h" + +#define rd_cycles(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) +#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#define PRINTF(...) sprintf((char *) (0xff010000UL), __VA_ARGS__) + +inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { + vx_tmc(0xff); + const volatile uint32_t *const A = (const volatile uint32_t *const) arg->addr_a; + + #define WORKERS 128 + + // #define WORDS 1350 + // #define LINES 338 + // #define ITERS 11 // = 1350 / 128 + + #define WORDS 7200 + #define LINES 1800 + #define T_ITERS 57 + + #define mark_fb0() \ + vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011000UL)) = LINES; \ + vx_fence(); vx_barrier(0, 8); vx_tmc(0xff) + #define mark_fb1() \ + vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011004UL)) = LINES; \ + vx_fence(); vx_barrier(1, 8); vx_tmc(0xff) + #define write_fb0(addr, value) *(((volatile uint32_t *) 0xff018000UL) + addr) = (value) + #define write_fb1(addr, value) *(((volatile uint32_t *) 0xff020000UL) + addr) = (value) + + #define CYCLES_TO_WAIT 240000 + + uint64_t cycles0, cycles1; + cycles0 = 0; + + while (true) { + volatile uint32_t v0, v1; + for (int i = 0; i < 5301; i += 1) { + v0 = A[i * WORDS + task_id]; + v1 = A[i * WORDS + WORKERS + task_id]; + int offset0 = 0 * WORKERS + task_id; + int offset1 = 1 * WORKERS + task_id; + + for (int j = 1; j < T_ITERS; j += 2) { + write_fb0(offset0, v0); + offset0 += 2 * WORKERS; + v0 = A[(i + 0) * WORDS + offset0]; + write_fb0(offset1, v1); + offset1 += 2 * WORKERS; + v1 = A[(i + 0) * WORDS + offset1]; + } + write_fb0(offset0, v0); + write_fb0(offset1, v1); + + /*offset0 += 2 * WORKERS; + v0 = A[(i + 0) * WORDS + offset0]; + write_fb0(offset0, v0);*/ + + if (task_id == 0) { + rd_cycles(cycles1); + while (cycles1 - cycles0 < CYCLES_TO_WAIT) { + rd_cycles(cycles1); + } + cycles0 = cycles1; + } + + threadblock_barrier(0, 8); + mark_fb0(); + } + } +} + +int main() { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + +#ifdef RADIANCE + vx_spawn_tasks_cluster(128, (vx_spawn_tasks_cb)kernel_body, arg); +#else + // NOTE: This kernel assumes contiguous thread scheduling for efficient shared + // memory allocation, and therefore does not work with original vx_spawn_tasks + vx_spawn_tasks_contiguous(8, (vx_spawn_tasks_cb)kernel_body, arg); +#endif + return 0; +} \ No newline at end of file diff --git a/tests/regression/rickroll/main.cpp b/tests/regression/rickroll/main.cpp new file mode 100644 index 00000000..54531062 --- /dev/null +++ b/tests/regression/rickroll/main.cpp @@ -0,0 +1,274 @@ +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.bin"; +uint32_t count = 0; + +std::vector src_a_data; +std::vector src_b_data; +std::vector ref_data; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + count = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(device, kernel_arg.addr_a); + vx_mem_free(device, kernel_arg.addr_b); + vx_mem_free(device, kernel_arg.addr_c); + vx_dev_close(device); + } +} + +void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + src_a_data.resize(dim_m * dim_k); + src_b_data.resize(dim_k * dim_n); + + for (uint32_t i = 0; i < src_a_data.size(); ++i) { + src_a_data[i] = static_cast(i); + std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl; + } + for (uint32_t i = 0; i < src_b_data.size(); ++i) { + src_b_data[i] = static_cast(i); + std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl; + } +} + +void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + ref_data.resize(dim_m * dim_n); + + for (uint32_t i = 0; i < dim_m; ++i) { + for (uint32_t j = 0; j < dim_n; ++j) { + float ref = 0.0f; + for (uint32_t k = 0; k < dim_k; ++k) { + ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j]; + } + ref_data.at(dim_n * i + j) = ref; + } + } +} + +int run_test(const kernel_arg_t& kernel_arg, + uint32_t buf_size, + uint32_t dim_m, uint32_t dim_n) { + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size)); + + // verify result + std::cout << "verify result" << std::endl; + { + int errors = 0; + auto buf_ptr = (float*)staging_buf.data(); + for (uint32_t i = 0; i < dim_m * dim_n; ++i) { + float ref = ref_data.at(i); + float cur = buf_ptr[i]; + if (std::abs((cur - ref) / ref) > 1e-6) { + std::cout << "error at result #" << std::dec << i + << std::hex << ": actual=" << cur << ", expected=" << ref << std::endl; + ++errors; + } + } + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + } + + return 0; +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + if (count == 0) { + count = 1; + } + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + // FIXME: hardcoded + uint32_t dim_m = 64; + uint32_t dim_n = 64; + uint32_t dim_k = 64; + + generate_source_matrix(dim_m, dim_n, dim_k); + generate_reference_matmul(dim_m, dim_n, dim_k); + + uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]); + uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]); + uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]); + + std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl; + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a)); + RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b)); + RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c)); + + kernel_arg.dim_m = dim_m; + kernel_arg.dim_n = dim_n; + kernel_arg.dim_k = dim_k; + + std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl; + std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl; + std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl; + + // allocate staging buffer + { + std::cout << "allocate staging buffer" << std::endl; + uint32_t staging_buf_size = std::max( + src_a_buf_size, + std::max( + src_b_buf_size, + std::max(dst_buf_size, sizeof(kernel_arg_t)))); + staging_buf.resize(staging_buf_size); + } + + // upload kernel argument + { + std::cout << "upload kernel argument" << std::endl; + auto buf_ptr = staging_buf.data(); + kernel_arg.addr_a = (uint64_t) 0x20000; + kernel_arg.addr_b = (uint64_t) 0x28000; + kernel_arg.addr_c = (uint64_t) 0xc0000000ULL; + memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); + + std::cout << "uploading argument buffer to device, device mem address=" + << std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec + << sizeof(kernel_arg_t) << " bytes\n"; + std::ofstream file("args.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(staging_buf.data()), + sizeof(kernel_arg_t)); + file.close(); + + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + } + + // upload source buffer + { + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(), + src_a_buf_size)); + + std::cout << "uploading source A matrix to device, device mem address=" + << std::hex << kernel_arg.addr_a << ", size=" << std::dec + << src_a_buf_size << " bytes\n"; + std::ofstream file("input.a.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_a_buf_size); + file.close(); + } + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(), + src_b_buf_size)); + + std::cout << "uploading source B matrix to device, device mem address=" + << std::hex << kernel_arg.addr_b << ", size=" << std::dec + << src_b_buf_size << " bytes\n"; + std::ofstream file("input.b.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_b_buf_size); + file.close(); + } + } + + // clear destination buffer + { + std::cout << "clear destination buffer" << std::endl; + auto buf_ptr = (int32_t*)staging_buf.data(); + for (uint32_t i = 0; i < ref_data.size(); ++i) { + buf_ptr[i] = 0xdeadbeef; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size)); + } + + // run tests + std::cout << "run tests" << std::endl; + RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n)); + std::cout << "PASSED!" << std::endl; + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + return 0; +} diff --git a/tests/regression/rickroll/rickroll b/tests/regression/rickroll/rickroll new file mode 100755 index 00000000..67ade61b Binary files /dev/null and b/tests/regression/rickroll/rickroll differ diff --git a/tests/regression/sgemm_gemmini/kernel.cpp b/tests/regression/sgemm_gemmini/kernel.cpp index dcd931aa..221af0a5 100644 --- a/tests/regression/sgemm_gemmini/kernel.cpp +++ b/tests/regression/sgemm_gemmini/kernel.cpp @@ -16,23 +16,26 @@ #define NUM_CLUSTERS 1 #define NUM_THREADS_IN_CLUSTER 128 -#define SMEM_ADDR_0K ((float * const) 0xff000000) -#define SMEM_ADDR_4K ((float * const) 0xff001000) -#define SMEM_ADDR_8K ((float * const) 0xff002000) -#define SMEM_ADDR_12K ((float * const) 0xff003000) -#define SPAD_ADDR_0K 0x0 -#define SPAD_ADDR_4K 0x80 -#define SPAD_ADDR_8K 0x100 -#define SPAD_ADDR_12K 0x180 +#define SMEM_ADDR_Q0 ((float * const) 0xff000000) +#define SMEM_ADDR_Q1 ((float * const) 0xff001000) +#define SMEM_ADDR_Q2 ((float * const) 0xff002000) +#define SMEM_ADDR_Q3 ((float * const) 0xff003000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x80 +#define SPAD_ADDR_Q2 0x100 +#define SPAD_ADDR_Q3 0x180 +#define SPAD_ADDR_Q4 0x200 -//#define DEBUG_PRINT -//#define EXT_ACCUMULATE #define HARDCODE #define REGBLOCK +#define OFFLOAD_ACCUMULATE +#define REMATERIALIZE #define DBUF +//#define CISC + +//#define DEBUG_PRINT //#define DETAILED_PERF //#define ACTIVATE -#define CISC #define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) #ifdef DETAILED_PERF @@ -40,7 +43,11 @@ #else #define rd_cycles(x) #endif -#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#ifdef REMATERIALIZE + #define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#else + #define HW_TID() hw_tid +#endif #define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) // #define PRINTF(...) vx_printf(__VA_ARGS__) #define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x)))) @@ -58,10 +65,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const float * const B = (const float * const) arg->addr_b; float * const C = (float * const) arg->addr_c; - if (HW_TID() == 0) { - gemmini_config_ld(0); + if (tid_in_threadblock % NUM_THREADS_IN_CLUSTER == 0) { gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); - gemmini_config_st(0); + // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, b_transpose); + + // gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale); PRINTF("start\n"); } @@ -100,17 +108,26 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, constexpr uint32_t i1_iters = (DIM * DIM * (TILE_K / DIM)) / num_threads_in_cluster; // num of iters before striding const uint32_t num_tile_rows_per_tb = num_tiles_m / NUM_CLUSTERS; + + if (HW_TID() == 0) { + gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 0); + gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 1); + // gemmini_extended3_config_ld(repeating_bias ? 0 : (stride_D * sizeof_D), D_scale_factor, low_D, 2); + gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); + // gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale); + } + for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id; tile_i < num_tile_rows_per_tb * (threadblock_id + 1); tile_i += 1) { __asm__("i_loop:"); for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { __asm__("j_loop:"); - float * const smem_c_tile_start = SMEM_ADDR_4K; - #ifndef EXT_ACCUMULATE - float * const smem_acc_tile_start = SMEM_ADDR_0K + HW_TID(); + float * const smem_c_tile_start = SMEM_ADDR_Q1; + #ifdef OFFLOAD_ACCUMULATE + float * const smem_acc_tile_start = SMEM_ADDR_Q0 + HW_TID(); #else - float * const smem_acc_tile_start = SMEM_ADDR_8K + hw_tid; + float * const smem_acc_tile_start = SMEM_ADDR_Q2 + hw_tid; #endif __asm__("k_loop:"); @@ -132,11 +149,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K + runtime_const_a; const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N + runtime_const_b; #ifdef DBUF - float * const smem_a_tile_start = ((tile_k & 1) ? SMEM_ADDR_4K : SMEM_ADDR_0K) + HW_TID(); - float * const smem_b_tile_start = ((tile_k & 1) ? SMEM_ADDR_12K : SMEM_ADDR_8K) + HW_TID(); + float * const smem_a_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q1 : SMEM_ADDR_Q0) + HW_TID(); + float * const smem_b_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q3 : SMEM_ADDR_Q2) + HW_TID(); #else - float * const smem_a_tile_start = SMEM_ADDR_0K + HW_TID(); - float * const smem_b_tile_start = SMEM_ADDR_12K + HW_TID(); + float * const smem_a_tile_start = SMEM_ADDR_Q0 + HW_TID(); + float * const smem_b_tile_start = SMEM_ADDR_Q3 + HW_TID(); #endif { @@ -175,7 +192,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_b_tile_start[7 * num_threads_in_cluster + hw_tid] = \ dram_b_tile_start[every_iter * 1 + every_2iters_b * 3]; #else - __asm__("load_ab:"); float v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 0]; float v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 0]; float v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 1]; @@ -185,7 +201,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_a_tile_start[2 * num_threads_in_cluster] = v2; smem_a_tile_start[3 * num_threads_in_cluster] = v3; - __asm__("load_ab1:"); v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 0]; v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 0]; v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 1]; @@ -195,7 +210,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_b_tile_start[2 * num_threads_in_cluster] = v2; smem_b_tile_start[3 * num_threads_in_cluster] = v3; - __asm__("load_ab2:"); v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 2]; v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 2]; v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 3]; @@ -205,7 +219,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_a_tile_start[6 * num_threads_in_cluster] = v2; smem_a_tile_start[7 * num_threads_in_cluster] = v3; - __asm__("load_ab3:"); v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 2]; v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 2]; v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 3]; @@ -214,8 +227,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, smem_b_tile_start[5 * num_threads_in_cluster] = v1; smem_b_tile_start[6 * num_threads_in_cluster] = v2; smem_b_tile_start[7 * num_threads_in_cluster] = v3; - - __asm__("end_loadab:"); #endif } #else @@ -223,8 +234,8 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K; const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N; - float * const smem_a_tile_start = SMEM_ADDR_0K; - float * const smem_b_tile_start = SMEM_ADDR_12K; + float * const smem_a_tile_start = SMEM_ADDR_Q0; + float * const smem_b_tile_start = SMEM_ADDR_Q3; /* for (uint32_t thread_i = 0, j1 = 0, i1 = 0; thread_i < a_elems_per_thread; @@ -281,7 +292,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, // cluster wide barrier to wait for A and B loads to complete threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); rd_cycles(marker3); - __asm__("gemmini:"); if (HW_TID() == 0) { #ifdef DBUF gemmini_fence(); @@ -290,8 +300,8 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, #ifndef DBUF #error MUST ENABLE DBUF #endif - #ifdef EXT_ACCUMULATE - #error MUST DISABLE EXT ACCUMULATE + #ifndef OFFLOAD_ACCUMULATE + #error MUST OFFLOAD ACCUMULATE #endif if (tile_k == 0) { GEMMINI_CISC_CMD_I(0); @@ -303,14 +313,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, #else sp_tiled_matmul_full_spad_ws( #ifdef DBUF - (tile_k & 1) ? SPAD_ADDR_4K : SPAD_ADDR_0K, (tile_k & 1) ? SPAD_ADDR_12K : SPAD_ADDR_8K, + (tile_k & 1) ? SPAD_ADDR_Q1 : SPAD_ADDR_Q0, (tile_k & 1) ? SPAD_ADDR_Q3 : SPAD_ADDR_Q2, #else - SPAD_ADDR_0K, SPAD_ADDR_12K, + SPAD_ADDR_Q0, SPAD_ADDR_Q3, #endif - /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_4K, + /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q1, /*I=*/TILE_M / DIM, /*J=*/TILE_N / DIM, /*K=*/TILE_K / DIM, /*pad_I=*/0, /*pad_J=*/0, /*pad_K=*/0, /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, - #ifdef EXT_ACCUMULATE + #ifndef OFFLOAD_ACCUMULATE /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/0x38U) #else /*acc=*/tile_k != 0, /*act=*/NO_ACTIVATION, /*skips=*/0xB8U) @@ -321,13 +331,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, gemmini_fence(); #endif } - __asm__("end_gemmini:"); rd_cycles(marker4); - // threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + #ifndef DBUF + threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + #endif rd_cycles(marker5); // accumulate C matrix - #ifdef EXT_ACCUMULATE + #ifndef OFFLOAD_ACCUMULATE __asm__("accumulate:"); if (tile_k == 0) { #pragma GCC ivdep @@ -383,7 +394,7 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, } - #ifndef EXT_ACCUMULATE + #ifdef OFFLOAD_ACCUMULATE threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); rd_cycles(marker6); __asm__("mvout_spad_ser:"); @@ -476,13 +487,12 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, #endif #else - float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N; #pragma clang loop unroll(disable) for (int thread_i = 0; thread_i < c_elems_per_thread; thread_i++) { uint32_t elem_offset = hw_tid + num_threads_in_cluster * thread_i; dram_c_tile_start[elem_offset / TILE_N * dim_n + elem_offset % TILE_N] = \ - *(SMEM_ADDR_8K + SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N)); + *(SMEM_ADDR_Q2 + SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N)); } #endif __asm__("end_mvout_dram:"); @@ -513,7 +523,7 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, PRINTF("first barrier: %d\n", marker3 - marker2); PRINTF("gemmini cycles: %d\n", marker4 - marker3); PRINTF("second barrier: %d\n", marker5 - marker4); - #ifdef EXT_ACCUMULATE + #ifndef OFFLOAD_ACCUMULATE PRINTF("accumulation cycles: %d\n", marker6 - marker5); #else PRINTF("smem mvout cycles: %d %d-%d\n", marker7 - marker6, marker7, marker6); diff --git a/tests/regression/sgemm_gemmini_dma/.gitignore b/tests/regression/sgemm_gemmini_dma/.gitignore new file mode 100644 index 00000000..7c35ba59 --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/.gitignore @@ -0,0 +1,5 @@ +*.bin +*.dump +*.elf +sgemm_wg +.depend diff --git a/tests/regression/sgemm_gemmini_dma/Makefile b/tests/regression/sgemm_gemmini_dma/Makefile new file mode 100644 index 00000000..3a8ffb18 --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/Makefile @@ -0,0 +1,9 @@ +PROJECT = sgemm_gemmini_dma + +SRCS = main.cpp common.h + +VX_SRCS = kernel.cpp + +OPTS ?= -n16 + +include ../common.mk diff --git a/tests/regression/sgemm_gemmini_dma/common.h b/tests/regression/sgemm_gemmini_dma/common.h new file mode 100644 index 00000000..74941562 --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000 +#define DEV_SMEM_START_ADDR 0xff000000 + +typedef struct { + uint32_t dim_m; + uint32_t dim_n; + uint32_t dim_k; + uint64_t addr_a; + uint64_t addr_b; + uint64_t addr_c; +} kernel_arg_t; + +#endif diff --git a/tests/regression/sgemm_gemmini_dma/kernel.cpp b/tests/regression/sgemm_gemmini_dma/kernel.cpp new file mode 100644 index 00000000..128c49aa --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/kernel.cpp @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include "common.h" +#include "include/gemmini.h" +#include "gemmini_mmio.h" + +#define TILE_M 64 +#define TILE_N 64 +#define TILE_K 64 +#define SMEM_ADDR_Q0 ((float * const) 0xff000000) +#define SMEM_ADDR_Q1 ((float * const) 0xff004000) +#define SMEM_ADDR_Q2 ((float * const) 0xff008000) +#define SMEM_ADDR_Q3 ((float * const) 0xff00c000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x200 +#define SPAD_ADDR_Q2 0x400 +#define SPAD_ADDR_Q3 0x600 +#define BOUND_INST 0x800080008ULL + +// #define TILE_M 32 +// #define TILE_N 32 +// #define TILE_K 32 +// #define SMEM_ADDR_Q0 ((float * const) 0xff000000) +// #define SMEM_ADDR_Q1 ((float * const) 0xff001000) +// #define SMEM_ADDR_Q2 ((float * const) 0xff002000) +// #define SMEM_ADDR_Q3 ((float * const) 0xff003000) +// #define SPAD_ADDR_Q0 0x0 +// #define SPAD_ADDR_Q1 0x80 +// #define SPAD_ADDR_Q2 0x100 +// #define SPAD_ADDR_Q3 0x180 +// #define BOUND_INST 0x400040004ULL + +#define NUM_CLUSTERS 1 +#define NUM_THREADS_IN_CLUSTER 128 + +#define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x)) +#define rd_cycles(x) rd_cycles_force(x) +#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;}) +#define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__) +// #define PRINTF(...) vx_printf(__VA_ARGS__) +#define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x)))) + +inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg, + const uint32_t threadblock_id, + const uint32_t tid_in_threadblock) { + __asm__("matmul_start:"); + const float * const A = (const float * const) arg->addr_a; + const float * const B = (const float * const) arg->addr_b; + float * const C = (float * const) arg->addr_c; + + if (HW_TID() == 0) { + gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); + // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, b_transpose); + PRINTF("start\n"); + } + + vx_fence(); + + uint32_t marker0, marker1; + rd_cycles_force(marker0); + + const uint32_t dim_m = arg->dim_m; + const uint32_t dim_n = arg->dim_n; + const uint32_t dim_k = arg->dim_k; + const uint32_t num_tiles_m = dim_m / TILE_M; + const uint32_t num_tiles_n = dim_n / TILE_N; + const uint32_t num_tiles_k = dim_k / TILE_K; + constexpr uint32_t num_threads_in_cluster = NUM_THREADS_IN_CLUSTER; + + const uint32_t num_tile_rows_per_tb = num_tiles_m / NUM_CLUSTERS; + + if (HW_TID() == 0) { + gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 0); + gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 1); + // gemmini_extended3_config_ld(repeating_bias ? 0 : (stride_D * sizeof_D), D_scale_factor, low_D, 2); + gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); + // gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale); + } + + for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id; + tile_i < num_tile_rows_per_tb * (threadblock_id + 1); + tile_i += 1) { + for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) { + if (HW_TID() == 0) { + for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) { + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, + (uint64_t) (A + tile_i * TILE_M * dim_k + tile_k * TILE_K), + (uint64_t) (B + tile_k * TILE_K * dim_n + tile_j * TILE_N), k_LOOP_WS_CONFIG_ADDRS_AB) + GEMMINI_CISC_CMD_R((dim_n) << 16 | (dim_k << 8) | 8); + if (tile_k & 1) { + GEMMINI_CISC_CMD_I(11); + } else { + GEMMINI_CISC_CMD_I(10); + } + + if (tile_k == 0) { + gemmini_fence(); + GEMMINI_CISC_CMD_I(0); + } else if (tile_k & 1) { + gemmini_fence(); + GEMMINI_CISC_CMD_I(2); + } else { + gemmini_fence(); + GEMMINI_CISC_CMD_I(1); + } + } + + gemmini_fence(); + gemmini_fence(); + gemmini_fence(); + gemmini_fence(); + // mvout to scratchpad for activation + GEMMINI_CISC_CMD_I(9); + gemmini_fence(); + } + + threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + // activate + + // move out to dram + if (HW_TID() == 0) { + float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N; + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, BOUND_INST, k_LOOP_WS_CONFIG_BOUNDS) + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (uint64_t) dram_c_tile_start, k_LOOP_WS_CONFIG_ADDRS_DC) + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, dim_n, k_LOOP_WS_CONFIG_STRIDES_DC) + ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, loop_matmul_skips(1, 1, 1, 1, 0), k_LOOP_WS) + } + } + } + // last thread block complete + if (threadblock_id == NUM_CLUSTERS - 1) { + threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS); + rd_cycles_force(marker1); + if (HW_TID() == 0) { + PRINTF("\ncomplete\n"); + PRINTF("total cycles: %d\n", marker1 - marker0); + for (int i = 0; i < dim_m; i += 8) { + for (int j = 0; j < dim_n; j += 8) { + PRINTF("%d %d ", (int) (C[i * dim_n + j]), (int) (C[i * dim_n + j + 4])); + } + PRINTF("\n"); + } + } + } + vx_tmc(0); +} + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { + const int threadblock_id = task_id / NUM_THREADS_IN_CLUSTER; + const int tid_in_threadblock = task_id % NUM_THREADS_IN_CLUSTER; + + thread_block_matmul_gemmini(arg, threadblock_id, tid_in_threadblock); +} + +int main() { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + + const uint32_t num_threads_in_cluster = vx_num_threads() * vx_num_warps() * CORES_PER_CLUSTER; + const uint32_t grid_size = num_threads_in_cluster * NUM_CLUSTERS; +#ifdef RADIANCE + vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); +#else + // NOTE: This kernel assumes contiguous thread scheduling for efficient shared + // memory allocation, and therefore does not work with original vx_spawn_tasks + vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); +#endif + return 0; +} \ No newline at end of file diff --git a/tests/regression/sgemm_gemmini_dma/main.cpp b/tests/regression/sgemm_gemmini_dma/main.cpp new file mode 100644 index 00000000..54531062 --- /dev/null +++ b/tests/regression/sgemm_gemmini_dma/main.cpp @@ -0,0 +1,274 @@ +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.bin"; +uint32_t count = 0; + +std::vector src_a_data; +std::vector src_b_data; +std::vector ref_data; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + count = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + vx_mem_free(device, kernel_arg.addr_a); + vx_mem_free(device, kernel_arg.addr_b); + vx_mem_free(device, kernel_arg.addr_c); + vx_dev_close(device); + } +} + +void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + src_a_data.resize(dim_m * dim_k); + src_b_data.resize(dim_k * dim_n); + + for (uint32_t i = 0; i < src_a_data.size(); ++i) { + src_a_data[i] = static_cast(i); + std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl; + } + for (uint32_t i = 0; i < src_b_data.size(); ++i) { + src_b_data[i] = static_cast(i); + std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl; + } +} + +void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) { + ref_data.resize(dim_m * dim_n); + + for (uint32_t i = 0; i < dim_m; ++i) { + for (uint32_t j = 0; j < dim_n; ++j) { + float ref = 0.0f; + for (uint32_t k = 0; k < dim_k; ++k) { + ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j]; + } + ref_data.at(dim_n * i + j) = ref; + } + } +} + +int run_test(const kernel_arg_t& kernel_arg, + uint32_t buf_size, + uint32_t dim_m, uint32_t dim_n) { + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size)); + + // verify result + std::cout << "verify result" << std::endl; + { + int errors = 0; + auto buf_ptr = (float*)staging_buf.data(); + for (uint32_t i = 0; i < dim_m * dim_n; ++i) { + float ref = ref_data.at(i); + float cur = buf_ptr[i]; + if (std::abs((cur - ref) / ref) > 1e-6) { + std::cout << "error at result #" << std::dec << i + << std::hex << ": actual=" << cur << ", expected=" << ref << std::endl; + ++errors; + } + } + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + } + + return 0; +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + if (count == 0) { + count = 1; + } + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + // FIXME: hardcoded + uint32_t dim_m = 64; + uint32_t dim_n = 64; + uint32_t dim_k = 64; + + generate_source_matrix(dim_m, dim_n, dim_k); + generate_reference_matmul(dim_m, dim_n, dim_k); + + uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]); + uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]); + uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]); + + std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl; + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a)); + RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b)); + RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c)); + + kernel_arg.dim_m = dim_m; + kernel_arg.dim_n = dim_n; + kernel_arg.dim_k = dim_k; + + std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl; + std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl; + std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl; + + // allocate staging buffer + { + std::cout << "allocate staging buffer" << std::endl; + uint32_t staging_buf_size = std::max( + src_a_buf_size, + std::max( + src_b_buf_size, + std::max(dst_buf_size, sizeof(kernel_arg_t)))); + staging_buf.resize(staging_buf_size); + } + + // upload kernel argument + { + std::cout << "upload kernel argument" << std::endl; + auto buf_ptr = staging_buf.data(); + kernel_arg.addr_a = (uint64_t) 0x20000; + kernel_arg.addr_b = (uint64_t) 0x28000; + kernel_arg.addr_c = (uint64_t) 0xc0000000ULL; + memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); + + std::cout << "uploading argument buffer to device, device mem address=" + << std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec + << sizeof(kernel_arg_t) << " bytes\n"; + std::ofstream file("args.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(staging_buf.data()), + sizeof(kernel_arg_t)); + file.close(); + + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + } + + // upload source buffer + { + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(), + src_a_buf_size)); + + std::cout << "uploading source A matrix to device, device mem address=" + << std::hex << kernel_arg.addr_a << ", size=" << std::dec + << src_a_buf_size << " bytes\n"; + std::ofstream file("input.a.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_a_buf_size); + file.close(); + } + { + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float)); + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(), + src_b_buf_size)); + + std::cout << "uploading source B matrix to device, device mem address=" + << std::hex << kernel_arg.addr_b << ", size=" << std::dec + << src_b_buf_size << " bytes\n"; + std::ofstream file("input.b.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(buf_ptr), src_b_buf_size); + file.close(); + } + } + + // clear destination buffer + { + std::cout << "clear destination buffer" << std::endl; + auto buf_ptr = (int32_t*)staging_buf.data(); + for (uint32_t i = 0; i < ref_data.size(); ++i) { + buf_ptr[i] = 0xdeadbeef; + } + RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size)); + } + + // run tests + std::cout << "run tests" << std::endl; + RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n)); + std::cout << "PASSED!" << std::endl; + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + return 0; +} diff --git a/tests/regression/sgemm_gemmini_dma/sgemm_gemmini_dma b/tests/regression/sgemm_gemmini_dma/sgemm_gemmini_dma new file mode 100755 index 00000000..67ade61b Binary files /dev/null and b/tests/regression/sgemm_gemmini_dma/sgemm_gemmini_dma differ