wu-arch/kernels
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dma and demo kernels

Browse Source
This commit is contained in:
Richard Yan
2024-06-07 18:11:19 -07:00
parent 33066af56e
commit 7cf59c9480
27 changed files with 1731 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
kernel/include/gemmini_mmio.h
View File

@@ -5,7 +5,8 @@
#endif
#define SMEM_BASE 0xff000000
#define SMEM_SIZE 0x4000
//#define SMEM_SIZE 0x4000
#define SMEM_SIZE 0x10000
#define SMEM_MASK (SMEM_SIZE - 1)
#define SMEM_ADDR_END (SMEM_BASE + SMEM_SIZE)
@@ -31,10 +32,13 @@
// 0: k = 0, no accumulation
// 1: k % 2 = 0, buffer regions 0
// 2: k % 2 = 1, buffer regions 1
// 8, 9: memory ops
// 8: tile-sized move-in (unused)
// 8, 9, 10, 11: memory ops
// 8: tile-sized move-in stride
// 9: tile-sized move-out
// 10: tile-sized move-in, buffer regions 0
// 11: tile-sized move-in, buffer regions 1
#define GEMMINI_CISC_CMD_I(x) asm("csrwi 0xacc, "#x)
#define GEMMINI_CISC_CMD_R(x) asm("csrw 0xacc, %0" :: "r" (x))
#define GEMMINI_STATUS() ({uint32_t status; asm volatile ("csrr %0, 0xacc" : "=r" (status)); status;})
// convert normal matrix i,j into tiled smem offset
@@ -62,6 +66,9 @@
/* sprintf((char *) PRINT_BUF, "%llx %llx %d\n", rs1, rs2, funct); */ \
}
#define loop_matmul_skips(skip_lda, skip_ldb, skip_ldd, skip_ex, skip_stc) \
(((skip_lda) | ((skip_ldb) << 1) | ((skip_ldd) << 2) | ((skip_ex) << 3) | ((skip_stc) << 4)) << 3)
#define sp_tiled_matmul_full_spad_ws(A_sp_addr_start, B_sp_addr_start, D_sp_addr_start, C_dst_sp_addr_start,\
I, J, K, pad_I, pad_J, pad_K, a_transpose, b_transpose, full_C, low_D, acc, act, skips) \
gemmini_loop_ws_spad(I, J, K, pad_I, pad_J, pad_K, A_sp_addr_start, (B_sp_addr_start) + (K) * (J) * DIM, NULL, \

20
kernel/linker/vx_link32.ld
View File

@@ -10,8 +10,8 @@ ENTRY(_start)
MEMORY {
DRAM0 (rwx): ORIGIN = 0x80000000, LENGTH = 512M
DRAM1 (rwx): ORIGIN = 0xa0000000, LENGTH = 32K
DRAM2 (rwx): ORIGIN = 0xa1000000, LENGTH = 32K
DRAM1 (rwx): ORIGIN = 0xa0000000, LENGTH = 16M
DRAM2 (rwx): ORIGIN = 0xa1000000, LENGTH = 16M
}
SECTIONS
@@ -69,6 +69,7 @@ SECTIONS
/* .gnu.warning sections are handled specially by elf.em. */
*(.gnu.warning)
}
.fini :
{
KEEP (*(SORT_NONE(.fini)))
@@ -128,6 +129,7 @@ SECTIONS
KEEP (*(.init_array EXCLUDE_FILE (*crtbegin.o *crtbegin?.o *crtend.o *crtend?.o ) .ctors))
PROVIDE_HIDDEN (__init_array_end = .);
}
.fini_array :
{
PROVIDE_HIDDEN (__fini_array_start = .);
@@ -135,6 +137,18 @@ SECTIONS
KEEP (*(.fini_array EXCLUDE_FILE (*crtbegin.o *crtbegin?.o *crtend.o *crtend?.o ) .dtors))
PROVIDE_HIDDEN (__fini_array_end = .);
}
.htif_pad : {
. = ALIGN(0x1000);
}
. = ALIGN(0x1000);
.tohost : {
*(.tohost)
/* . += 0x100; */
}
. = ALIGN(0x1000);
.ctors :
{
/* gcc uses crtbegin.o to find the start of
@@ -216,6 +230,8 @@ SECTIONS
MAX(__DATA_BEGIN__ + 0x800, __BSS_END__ - 0x800));
_end = .; PROVIDE (end = .);
. = DATA_SEGMENT_END (.);
/* Stabs debugging sections. */
.stab 0 : { *(.stab) }
.stabstr 0 : { *(.stabstr) }

1
kernel/src/vx_start.S
View File

@@ -150,3 +150,4 @@ vx_wspawn_wait:
.weak __dso_handle
__dso_handle:
.long 0

7
kernel/tohost.S Normal file
View File

@@ -0,0 +1,7 @@
.section ".tohost","aw",@progbits
.align 6
.globl tohost
tohost: .dword 0
.align 6
.globl fromhost
fromhost: .dword 0

6
tests/regression/bad_apple/.gitignore vendored Normal file
View File

@@ -0,0 +1,6 @@
*.bin
*.dump
*.elf
sgemm_wg
.depend
kernel.radiance.elf

9
tests/regression/bad_apple/Makefile Normal file
View File

@@ -0,0 +1,9 @@
PROJECT = bad_apple
SRCS = main.cpp common.h
VX_SRCS = kernel.cpp
OPTS ?= -n16
include ../common.mk

BIN
tests/regression/bad_apple/bad_apple Executable file
View File

Binary file not shown.

18
tests/regression/bad_apple/common.h Normal file
View File

@@ -0,0 +1,18 @@
#ifndef _COMMON_H_
#define _COMMON_H_
#include <cstdint>
#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000
#define DEV_SMEM_START_ADDR 0xff000000
typedef struct {
uint32_t dim_m;
uint32_t dim_n;
uint32_t dim_k;
uint64_t addr_a;
uint64_t addr_b;
uint64_t addr_c;
} kernel_arg_t;
#endif

103
tests/regression/bad_apple/display.py Normal file
View File

@@ -0,0 +1,103 @@
import time
import os
import struct
from PIL import Image
from io import BytesIO
import subprocess
import numpy as np
import base64
import cv2
use_fpga = False
buffer_width = 16
buffer_depth = 1350
frame_width = 480
frame_height = 360
upscale = 3
# buffer_width = 16
# buffer_depth = 0x152
# frame_width = 240
# frame_height = 180
# upscale = 6
buffer_size = buffer_depth * buffer_width
truncate = 300 if use_fpga else 0
def follow(filename):
frame_count = 0
with open(filename, "r") as file:
while True:
line = file.readline()
if not line:
time.sleep(0.001)
continue
if truncate and (" 0151 " in line):
frame_count += 1
if frame_count == truncate:
with open(filename, "w") as f:
f.truncate(0)
frame_count = 0
yield line
def process_frame(frame_data):
bits = np.unpackbits(np.frombuffer(frame_data, dtype=np.uint8))
bits = bits[:frame_width * frame_height]
image_array = bits.reshape((frame_height, frame_width))
# image_array = np.flipud(np.fliplr(image_array))
image_array = (image_array * 255).astype(np.uint8)
filtered_image_array = cv2.fastNlMeansDenoising(image_array, None, h=72, templateWindowSize=8, searchWindowSize=8)
# filtered_image_array = cv2.GaussianBlur(image_array, (3, 3), 0)
# filtered_image_array = image_array
filtered_image_array = np.kron(filtered_image_array, np.ones((upscale, upscale), dtype=np.uint8))
image = Image.fromarray(filtered_image_array, mode='L')
return image
def display_image(img):
with BytesIO() as output:
# img = img.resize((frame_width * upscale, frame_height * upscale), Image.NEAREST)
img.save(output, format='PNG')
output.seek(0)
# subprocess.run(["/home/eecs/yrh/.iterm2/imgcat"], input=output.read())
image_data = output.getvalue()
b64_image_data = base64.b64encode(image_data).decode('utf-8')
print("\033]", end='')
print(f"1337;File=inline=1", end='')
print(f";size={len(image_data)}", end='')
print(f";name={base64.b64encode('tmp.png'.encode()).decode('utf-8')}", end='')
# print(f";width={frame_width * upscale * 4};height={frame_height * upscale * 4}", end='')
print(f":{b64_image_data}", end='')
print("\a", end='')
print('\n')
def main():
if not use_fpga:
filename = "/scratch/yrh/chipyard/sims/vcs/output/chipyard.harness.TestHarness.RadianceClusterConfig/kernel.radiance.out"
else:
filename = "/scratch/yrh/chipyard/sims/firesim/sim/generated-src/xilinx_alveo_u250/xilinx_alveo_u250-firesim-FireSim-FireSimRadianceClusterSynConfig-WithPrintfSynthesis_BaseXilinxAlveoU250Config/synthesized-prints.out0"
# frame_data = {}
frame_data0 = bytearray(buffer_size)
for line in follow(filename):
if not "fb0" in line:
continue
tokens = line.split()
if not len(tokens) == (5 if use_fpga else 3):
continue
offset, data = tokens[3 if use_fpga else 1:]
offset0 = int(offset, 16)
frame_data0[offset0 * buffer_width : (offset0 + 1) * buffer_width] = bytes.fromhex(data)[::-1]
if offset0 == buffer_depth - 1:
img = process_frame(frame_data0)
display_image(img)
frame_data0 = bytearray(buffer_size)
if __name__ == "__main__":
main()

83
tests/regression/bad_apple/display.py.bak Normal file
View File

@@ -0,0 +1,83 @@
import time
import os
import struct
from PIL import Image
from io import BytesIO
import subprocess
use_fpga = False
def follow(filename):
with open(filename, "r") as file:
# file.seek(0, os.SEEK_END) # Move to the end of the file
while True:
line = file.readline()
if not line:
time.sleep(0.001)
continue
yield line.strip()
def process_frame(frame_data):
# Create a byte array from the frame data
byte_array = bytearray()
# for offset, data in frame_data.items():
# byte_array += struct.pack("<I", int(data, 16))
for offset in range(75):
byte_array += struct.pack("<I", frame_data[offset])
# Create an image from the byte array
img = Image.new('1', (60, 40)) # 60x40 pixels, 1 bit per pixel
pixels = img.load()
for i in range(40): # 40 rows
for j in range(60): # 60 columns
byte_index = (i * 60 + j) // 8
bit_index = 7 - ((i * 60 + j) % 8)
pixels[j, 39 - i] = (byte_array[byte_index] >> bit_index) & 1
return img
def display_image(img):
with BytesIO() as output:
img = img.resize((120, 90), Image.NEAREST)
img.save(output, format='PNG')
output.seek(0)
subprocess.run(["/home/eecs/yrh/.iterm2/imgcat", "-H", "98%"], input=output.read())
def main():
if not use_fpga:
filename = "/scratch/yrh/chipyard/sims/vcs/output/chipyard.harness.TestHarness.RadianceClusterConfig/kernel.radiance.out"
else:
filename = "/scratch/yrh/chipyard/sims/firesim/sim/generated-src/xilinx_alveo_u250/xilinx_alveo_u250-firesim-FireSim-FireSimRadianceClusterSynConfig-WithPrintfSynthesis_BaseXilinxAlveoU250Config/synthesized-prints.out0"
# frame_data = {}
frame_data0 = [0 for _ in range(80)]
frame_data1 = [0 for _ in range(80)]
for line in follow(filename):
if not "fb0" in line:
continue
tokens = line.split()
if not len(tokens) == 7 if use_fpga else 5:
continue
offset, data = tokens[4 if use_fpga else 2:-1]
offset = int(offset, 16) - 0xff010000
offset0 = offset
offset1 = offset - 0x200
if offset0 >= 0 and offset0 < 320:
frame_data0[offset0 // 4] = int(data, 16)
if offset1 >= 0 and offset1 < 320:
frame_data1[offset1 // 4] = int(data, 16)
if offset0 == 0x130 and data == "ff010130":
img = process_frame(frame_data0)
frame_data0 = [0 for _ in range(80)]
display_image(img)
elif offset1 == 0x130 and data == "ff010330":
img = process_frame(frame_data1)
frame_data1 = [0 for _ in range(80)]
display_image(img)
if __name__ == "__main__":
main()

137
tests/regression/bad_apple/kernel.cpp Normal file
View File

@@ -0,0 +1,137 @@
#include <stdint.h>
#include <vx_intrinsics.h>
#include <vx_print.h>
#include <vx_spawn.h>
#include "common.h"
#define rd_cycles(x) asm volatile ("csrr %0, mcycle" : "=r" (x))
#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
#define PRINTF(...) sprintf((char *) (0xff010000UL), __VA_ARGS__)
inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) {
vx_fence();
vx_barrier(barrier_id, count);
}
void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
vx_tmc(0xff);
const volatile uint32_t *const A = (const volatile uint32_t *const) arg->addr_a;
// vx_tmc(1);
// for (int i = 0; i < 75; i++) {
// if (task_id == i) {
// PRINTF("%d", task_id);
// }
// }
// threadblock_barrier(2, 5);
/*
#define mark(x) \
vx_tmc(0x80); \
if (task_id == 79) *(((volatile uint32_t *) x)) = x; \
vx_fence(); \
vx_barrier(x & 1, 5); \
vx_tmc(0xff)
#define write_fb0(value) *(((volatile uint32_t *) 0xff010000UL) + task_id) = (value)
#define write_fb1(value) *(((volatile uint32_t *) 0xff010200UL) + task_id) = (value)
while (true) {
uint32_t v0, v1, v2, v3;
v0 = A[task_id];
v1 = A[1 * 75 + task_id];
v2 = A[2 * 75 + task_id];
v3 = A[3 * 75 + task_id];
for (int i = 30; i < 6569; i += 4) {
write_fb1(v0);
v0 = A[(i + 0) * 75 + task_id];
mark(0xff010130);
write_fb0(v1);
v1 = A[(i + 1) * 75 + task_id];
mark(0xff010330);
write_fb1(v2);
v2 = A[(i + 2) * 75 + task_id];
mark(0xff010130);
write_fb0(v3);
v3 = A[(i + 3) * 75 + task_id];
mark(0xff010330);
}
}
*/
#define WORKERS 128
// #define WORDS 1350
// #define LINES 338
// #define ITERS 11 // = 1350 / 128
#define WORDS 5400
#define LINES 1350
#define T_ITERS 43
#define mark_fb0() \
vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011000UL)) = LINES; \
vx_fence(); vx_barrier(0, 8); vx_tmc(0xff)
#define mark_fb1() \
vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011004UL)) = LINES; \
vx_fence(); vx_barrier(1, 8); vx_tmc(0xff)
#define write_fb0(addr, value) *(((volatile uint32_t *) 0xff018000UL) + addr) = (value)
#define write_fb1(addr, value) *(((volatile uint32_t *) 0xff020000UL) + addr) = (value)
#define CYCLES_TO_WAIT 240000
uint64_t cycles0, cycles1;
cycles0 = 0;
while (true) {
volatile uint32_t v0, v1;
for (int i = 20; i < 6569; i += 1) {
v0 = A[i * WORDS + task_id];
v1 = A[i * WORDS + WORKERS + task_id];
int offset0 = 0 * WORKERS + task_id;
int offset1 = 1 * WORKERS + task_id;
for (int j = 1; j < T_ITERS; j += 2) {
write_fb0(offset0, v0);
offset0 += 2 * WORKERS;
v0 = A[(i + 0) * WORDS + offset0];
write_fb0(offset1, v1);
offset1 += 2 * WORKERS;
v1 = A[(i + 0) * WORDS + offset1];
}
write_fb0(offset0, v0);
write_fb0(offset1, v1);
/*offset0 += 2 * WORKERS;
v0 = A[(i + 0) * WORDS + offset0];
write_fb0(offset0, v0);*/
if (task_id == 0) {
rd_cycles(cycles1);
while (cycles1 - cycles0 < CYCLES_TO_WAIT) {
rd_cycles(cycles1);
}
cycles0 = cycles1;
}
threadblock_barrier(0, 8);
mark_fb0();
}
}
}
int main() {
kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR;
#ifdef RADIANCE
vx_spawn_tasks_cluster(128, (vx_spawn_tasks_cb)kernel_body, arg);
#else
// NOTE: This kernel assumes contiguous thread scheduling for efficient shared
// memory allocation, and therefore does not work with original vx_spawn_tasks
vx_spawn_tasks_contiguous(8, (vx_spawn_tasks_cb)kernel_body, arg);
#endif
return 0;
}

274
tests/regression/bad_apple/main.cpp Normal file
View File

@@ -0,0 +1,274 @@
#include <iostream>
#include <fstream>
#include <unistd.h>
#include <string.h>
#include <vortex.h>
#include <vector>
#include "common.h"
#define RT_CHECK(_expr) \
do { \
int _ret = _expr; \
if (0 == _ret) \
break; \
printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \
cleanup(); \
exit(-1); \
} while (false)
///////////////////////////////////////////////////////////////////////////////
const char* kernel_file = "kernel.bin";
uint32_t count = 0;
std::vector<float> src_a_data;
std::vector<float> src_b_data;
std::vector<float> ref_data;
vx_device_h device = nullptr;
std::vector<uint8_t> staging_buf;
kernel_arg_t kernel_arg = {};
static void show_usage() {
std::cout << "Vortex Test." << std::endl;
std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl;
}
static void parse_args(int argc, char **argv) {
int c;
while ((c = getopt(argc, argv, "n:k:h?")) != -1) {
switch (c) {
case 'n':
count = atoi(optarg);
break;
case 'k':
kernel_file = optarg;
break;
case 'h':
case '?': {
show_usage();
exit(0);
} break;
default:
show_usage();
exit(-1);
}
}
}
void cleanup() {
if (device) {
vx_mem_free(device, kernel_arg.addr_a);
vx_mem_free(device, kernel_arg.addr_b);
vx_mem_free(device, kernel_arg.addr_c);
vx_dev_close(device);
}
}
void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) {
src_a_data.resize(dim_m * dim_k);
src_b_data.resize(dim_k * dim_n);
for (uint32_t i = 0; i < src_a_data.size(); ++i) {
src_a_data[i] = static_cast<float>(i);
std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl;
}
for (uint32_t i = 0; i < src_b_data.size(); ++i) {
src_b_data[i] = static_cast<float>(i);
std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl;
}
}
void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) {
ref_data.resize(dim_m * dim_n);
for (uint32_t i = 0; i < dim_m; ++i) {
for (uint32_t j = 0; j < dim_n; ++j) {
float ref = 0.0f;
for (uint32_t k = 0; k < dim_k; ++k) {
ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j];
}
ref_data.at(dim_n * i + j) = ref;
}
}
}
int run_test(const kernel_arg_t& kernel_arg,
uint32_t buf_size,
uint32_t dim_m, uint32_t dim_n) {
// start device
std::cout << "start device" << std::endl;
RT_CHECK(vx_start(device));
// wait for completion
std::cout << "wait for completion" << std::endl;
RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT));
// download destination buffer
std::cout << "download destination buffer" << std::endl;
RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size));
// verify result
std::cout << "verify result" << std::endl;
{
int errors = 0;
auto buf_ptr = (float*)staging_buf.data();
for (uint32_t i = 0; i < dim_m * dim_n; ++i) {
float ref = ref_data.at(i);
float cur = buf_ptr[i];
if (std::abs((cur - ref) / ref) > 1e-6) {
std::cout << "error at result #" << std::dec << i
<< std::hex << ": actual=" << cur << ", expected=" << ref << std::endl;
++errors;
}
}
if (errors != 0) {
std::cout << "Found " << std::dec << errors << " errors!" << std::endl;
std::cout << "FAILED!" << std::endl;
return 1;
}
}
return 0;
}
int main(int argc, char *argv[]) {
// parse command arguments
parse_args(argc, argv);
if (count == 0) {
count = 1;
}
std::srand(50);
// open device connection
std::cout << "open device connection" << std::endl;
RT_CHECK(vx_dev_open(&device));
// FIXME: hardcoded
uint32_t dim_m = 64;
uint32_t dim_n = 64;
uint32_t dim_k = 64;
generate_source_matrix(dim_m, dim_n, dim_k);
generate_reference_matmul(dim_m, dim_n, dim_k);
uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]);
uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]);
uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]);
std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl;
// upload program
std::cout << "upload program" << std::endl;
RT_CHECK(vx_upload_kernel_file(device, kernel_file));
// allocate device memory
std::cout << "allocate device memory" << std::endl;
RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a));
RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b));
RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c));
kernel_arg.dim_m = dim_m;
kernel_arg.dim_n = dim_n;
kernel_arg.dim_k = dim_k;
std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl;
std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl;
std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl;
// allocate staging buffer
{
std::cout << "allocate staging buffer" << std::endl;
uint32_t staging_buf_size = std::max<uint32_t>(
src_a_buf_size,
std::max<uint32_t>(
src_b_buf_size,
std::max<uint32_t>(dst_buf_size, sizeof(kernel_arg_t))));
staging_buf.resize(staging_buf_size);
}
// upload kernel argument
{
std::cout << "upload kernel argument" << std::endl;
auto buf_ptr = staging_buf.data();
kernel_arg.addr_a = (uint64_t) 0x20000;
kernel_arg.addr_b = (uint64_t) 0x28000;
kernel_arg.addr_c = (uint64_t) 0xc0000000ULL;
memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t));
std::cout << "uploading argument buffer to device, device mem address="
<< std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec
<< sizeof(kernel_arg_t) << " bytes\n";
std::ofstream file("args.bin", std::ios::binary | std::ios::out);
if (!file) {
std::cerr << "error: failed to open args.bin for writing\n";
exit(EXIT_FAILURE);
}
file.write(reinterpret_cast<char *>(staging_buf.data()),
sizeof(kernel_arg_t));
file.close();
RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t)));
}
// upload source buffer
{
{
auto buf_ptr = staging_buf.data();
memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float));
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(),
src_a_buf_size));
std::cout << "uploading source A matrix to device, device mem address="
<< std::hex << kernel_arg.addr_a << ", size=" << std::dec
<< src_a_buf_size << " bytes\n";
std::ofstream file("input.a.bin", std::ios::binary | std::ios::out);
if (!file) {
std::cerr << "error: failed to open args.bin for writing\n";
exit(EXIT_FAILURE);
}
file.write(reinterpret_cast<char *>(buf_ptr), src_a_buf_size);
file.close();
}
{
auto buf_ptr = staging_buf.data();
memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float));
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(),
src_b_buf_size));
std::cout << "uploading source B matrix to device, device mem address="
<< std::hex << kernel_arg.addr_b << ", size=" << std::dec
<< src_b_buf_size << " bytes\n";
std::ofstream file("input.b.bin", std::ios::binary | std::ios::out);
if (!file) {
std::cerr << "error: failed to open args.bin for writing\n";
exit(EXIT_FAILURE);
}
file.write(reinterpret_cast<char *>(buf_ptr), src_b_buf_size);
file.close();
}
}
// clear destination buffer
{
std::cout << "clear destination buffer" << std::endl;
auto buf_ptr = (int32_t*)staging_buf.data();
for (uint32_t i = 0; i < ref_data.size(); ++i) {
buf_ptr[i] = 0xdeadbeef;
}
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size));
}
// run tests
std::cout << "run tests" << std::endl;
RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n));
std::cout << "PASSED!" << std::endl;
// cleanup
std::cout << "cleanup" << std::endl;
cleanup();
return 0;
}

3
tests/regression/common.mk
View File

@@ -53,7 +53,8 @@ VX_CFLAGS += -mcmodel=medany -fno-rtti -fno-exceptions -nostartfiles -fdata-sect
VX_CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw -I$(GEMMINI_SW_PATH)
VX_CFLAGS += -DNDEBUG -DLLVM_VORTEX
VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a
# VX_LDFLAGS += -Wl,-Bstatic,--gc-sections,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a
VX_LDFLAGS += -Wl,-Bstatic,-T,$(VORTEX_KN_PATH)/linker/vx_link$(XLEN).ld,--defsym=STARTUP_ADDR=$(STARTUP_ADDR) $(VORTEX_KN_PATH)/libvortexrt.a $(VORTEX_KN_PATH)/tohost.S
CXXFLAGS += -std=c++17 -Wall -Wextra -pedantic -Wfatal-errors
CXXFLAGS += -I$(VORTEX_RT_PATH)/include -I$(VORTEX_KN_PATH)/../hw

6
tests/regression/rickroll/.gitignore vendored Normal file
View File

@@ -0,0 +1,6 @@
*.bin
*.dump
*.elf
sgemm_wg
.depend
kernel.radiance.elf

9
tests/regression/rickroll/Makefile Normal file
View File

@@ -0,0 +1,9 @@
PROJECT = rickroll
SRCS = main.cpp common.h
VX_SRCS = kernel.cpp
OPTS ?= -n16
include ../common.mk

18
tests/regression/rickroll/common.h Normal file
View File

@@ -0,0 +1,18 @@
#ifndef _COMMON_H_
#define _COMMON_H_
#include <cstdint>
#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000
#define DEV_SMEM_START_ADDR 0xff000000
typedef struct {
uint32_t dim_m;
uint32_t dim_n;
uint32_t dim_k;
uint64_t addr_a;
uint64_t addr_b;
uint64_t addr_c;
} kernel_arg_t;
#endif

128
tests/regression/rickroll/display_color.py Normal file
View File

@@ -0,0 +1,128 @@
import time
import os
import struct
from PIL import Image
from io import BytesIO
import subprocess
import numpy as np
import base64
import cv2
use_fpga = False
buffer_width = 16
# buffer_depth = 0x152
# frame_width = 240
# frame_height = 180
# upscale = 6
buffer_depth = 1800
frame_width = 160
frame_height = 120
upscale = 1
buffer_size = int(buffer_depth * buffer_width * 1.5)
truncate = 0 # 300 if use_fpga else 0
def follow(filename):
frame_count = 0
with open(filename, "r") as file:
while True:
current_position = file.tell()
line = file.readline()
if not line:
time.sleep(0.001)
continue
if "fb0" in line and len(line) < (61 if use_fpga else 41):
file.seek(current_position)
time.sleep(0.001)
continue
if truncate and (" 0151 " in line):
frame_count += 1
if frame_count == truncate:
with open(filename, "w") as f:
f.truncate(0)
frame_count = 0
yield line
def process_frame(frame_data):
bits = np.unpackbits(np.frombuffer(frame_data, dtype=np.uint8))
bits = bits[:frame_width * frame_height]
image_array = bits.reshape((frame_height, frame_width))
# image_array = np.flipud(np.fliplr(image_array))
image_array = (image_array * 255).astype(np.uint8)
raw_array = np.frombuffer(frame_data, dtype=np.uint8)
y_size = frame_width * frame_height
c_size = y_size // 4
y_array = raw_array[:y_size].reshape((frame_height, frame_width))
cr_array, cb_array = raw_array[y_size : y_size + c_size].reshape((frame_height // 2, frame_width // 2)), raw_array[y_size + c_size : y_size + 2 * c_size].reshape((frame_height // 2, frame_width // 2))
cb_upscaled = cv2.resize(cb_array, (frame_width, frame_height), interpolation=cv2.INTER_LINEAR)
cr_upscaled = cv2.resize(cr_array, (frame_width, frame_height), interpolation=cv2.INTER_LINEAR)
# Merge the channels back to YCrCb format
ycrcb_frame = cv2.merge((y_array, cb_upscaled, cr_upscaled))
bgr_frame = cv2.cvtColor(ycrcb_frame, cv2.COLOR_YCrCb2BGR)
is_success, buffer = cv2.imencode(".png", bgr_frame)
io_buf = BytesIO(buffer)
# filtered_image_array = image_array
# filtered_image_array = cv2.fastNlMeansDenoising(image_array, None, h=72, templateWindowSize=8, searchWindowSize=8)
# filtered_image_array = cv2.GaussianBlur(image_array, (3, 3), 0)
# filtered_image_array = cv2.blur(image_array, (2, 2))
# filtered_image_array = cv2.fastNlMeansDenoising(filtered_image_array, None, h=72, templateWindowSize=4, searchWindowSize=4)
# filtered_image_array = np.kron(filtered_image_array, np.ones((upscale, upscale), dtype=np.uint8))
# image = Image.fromarray(filtered_image_array, mode='L')
return io_buf
def display_image(img):
# img = img.resize((frame_width * upscale, frame_height * upscale), Image.NEAREST)
# img.save(output, format='PNG')
output = img
output.seek(0)
# subprocess.run(["/home/eecs/yrh/.iterm2/imgcat"], input=output.read())
image_data = output.getvalue()
b64_image_data = base64.b64encode(image_data).decode('utf-8')
print("\033]", end='')
print(f"1337;File=inline=1", end='')
print(f";size={len(image_data)}", end='')
print(f";name={base64.b64encode('tmp.png'.encode()).decode('utf-8')}", end='')
# print(f";width={frame_width * upscale * 4};height={frame_height * upscale * 4}", end='')
print(f":{b64_image_data}", end='')
print("\a", end='')
print('\n')
def main():
if not use_fpga:
filename = "/scratch/yrh/chipyard/sims/vcs/output/chipyard.harness.TestHarness.RadianceClusterConfig/kernel.radiance.out"
else:
filename = "/scratch/yrh/firesim-rundir/sim_slot_0/synthesized-prints.out0"
# frame_data = {}
frame_data0 = bytearray(buffer_size)
for line in follow(filename):
if not "fb0" in line:
continue
tokens = line.split()
if not len(tokens) == (5 if use_fpga else 3):
continue
offset, data = tokens[3 if use_fpga else 1:]
offset0 = int(offset, 16)
frame_data0[offset0 * buffer_width : (offset0 + 1) * buffer_width] = bytes.fromhex(data)[::-1]
if offset0 == buffer_depth - 1:
img = process_frame(frame_data0)
display_image(img)
frame_data0 = bytearray(buffer_size)
if __name__ == "__main__":
main()

92
tests/regression/rickroll/kernel.cpp Normal file
View File

@@ -0,0 +1,92 @@
#include <stdint.h>
#include <vx_intrinsics.h>
#include <vx_print.h>
#include <vx_spawn.h>
#include "common.h"
#define rd_cycles(x) asm volatile ("csrr %0, mcycle" : "=r" (x))
#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
#define PRINTF(...) sprintf((char *) (0xff010000UL), __VA_ARGS__)
inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) {
vx_fence();
vx_barrier(barrier_id, count);
}
void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
vx_tmc(0xff);
const volatile uint32_t *const A = (const volatile uint32_t *const) arg->addr_a;
#define WORKERS 128
// #define WORDS 1350
// #define LINES 338
// #define ITERS 11 // = 1350 / 128
#define WORDS 7200
#define LINES 1800
#define T_ITERS 57
#define mark_fb0() \
vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011000UL)) = LINES; \
vx_fence(); vx_barrier(0, 8); vx_tmc(0xff)
#define mark_fb1() \
vx_tmc(0x80); if (task_id == 127) *(((volatile uint32_t *) 0xff011004UL)) = LINES; \
vx_fence(); vx_barrier(1, 8); vx_tmc(0xff)
#define write_fb0(addr, value) *(((volatile uint32_t *) 0xff018000UL) + addr) = (value)
#define write_fb1(addr, value) *(((volatile uint32_t *) 0xff020000UL) + addr) = (value)
#define CYCLES_TO_WAIT 240000
uint64_t cycles0, cycles1;
cycles0 = 0;
while (true) {
volatile uint32_t v0, v1;
for (int i = 0; i < 5301; i += 1) {
v0 = A[i * WORDS + task_id];
v1 = A[i * WORDS + WORKERS + task_id];
int offset0 = 0 * WORKERS + task_id;
int offset1 = 1 * WORKERS + task_id;
for (int j = 1; j < T_ITERS; j += 2) {
write_fb0(offset0, v0);
offset0 += 2 * WORKERS;
v0 = A[(i + 0) * WORDS + offset0];
write_fb0(offset1, v1);
offset1 += 2 * WORKERS;
v1 = A[(i + 0) * WORDS + offset1];
}
write_fb0(offset0, v0);
write_fb0(offset1, v1);
/*offset0 += 2 * WORKERS;
v0 = A[(i + 0) * WORDS + offset0];
write_fb0(offset0, v0);*/
if (task_id == 0) {
rd_cycles(cycles1);
while (cycles1 - cycles0 < CYCLES_TO_WAIT) {
rd_cycles(cycles1);
}
cycles0 = cycles1;
}
threadblock_barrier(0, 8);
mark_fb0();
}
}
}
int main() {
kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR;
#ifdef RADIANCE
vx_spawn_tasks_cluster(128, (vx_spawn_tasks_cb)kernel_body, arg);
#else
// NOTE: This kernel assumes contiguous thread scheduling for efficient shared
// memory allocation, and therefore does not work with original vx_spawn_tasks
vx_spawn_tasks_contiguous(8, (vx_spawn_tasks_cb)kernel_body, arg);
#endif
return 0;
}

274
tests/regression/rickroll/main.cpp Normal file
View File

@@ -0,0 +1,274 @@
#include <iostream>
#include <fstream>
#include <unistd.h>
#include <string.h>
#include <vortex.h>
#include <vector>
#include "common.h"
#define RT_CHECK(_expr) \
do { \
int _ret = _expr; \
if (0 == _ret) \
break; \
printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \
cleanup(); \
exit(-1); \
} while (false)
///////////////////////////////////////////////////////////////////////////////
const char* kernel_file = "kernel.bin";
uint32_t count = 0;
std::vector<float> src_a_data;
std::vector<float> src_b_data;
std::vector<float> ref_data;
vx_device_h device = nullptr;
std::vector<uint8_t> staging_buf;
kernel_arg_t kernel_arg = {};
static void show_usage() {
std::cout << "Vortex Test." << std::endl;
std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl;
}
static void parse_args(int argc, char **argv) {
int c;
while ((c = getopt(argc, argv, "n:k:h?")) != -1) {
switch (c) {
case 'n':
count = atoi(optarg);
break;
case 'k':
kernel_file = optarg;
break;
case 'h':
case '?': {
show_usage();
exit(0);
} break;
default:
show_usage();
exit(-1);
}
}
}
void cleanup() {
if (device) {
vx_mem_free(device, kernel_arg.addr_a);
vx_mem_free(device, kernel_arg.addr_b);
vx_mem_free(device, kernel_arg.addr_c);
vx_dev_close(device);
}
}
void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) {
src_a_data.resize(dim_m * dim_k);
src_b_data.resize(dim_k * dim_n);
for (uint32_t i = 0; i < src_a_data.size(); ++i) {
src_a_data[i] = static_cast<float>(i);
std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl;
}
for (uint32_t i = 0; i < src_b_data.size(); ++i) {
src_b_data[i] = static_cast<float>(i);
std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl;
}
}
void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) {
ref_data.resize(dim_m * dim_n);
for (uint32_t i = 0; i < dim_m; ++i) {
for (uint32_t j = 0; j < dim_n; ++j) {
float ref = 0.0f;
for (uint32_t k = 0; k < dim_k; ++k) {
ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j];
}
ref_data.at(dim_n * i + j) = ref;
}
}
}
int run_test(const kernel_arg_t& kernel_arg,
uint32_t buf_size,
uint32_t dim_m, uint32_t dim_n) {
// start device
std::cout << "start device" << std::endl;
RT_CHECK(vx_start(device));
// wait for completion
std::cout << "wait for completion" << std::endl;
RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT));
// download destination buffer
std::cout << "download destination buffer" << std::endl;
RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size));
// verify result
std::cout << "verify result" << std::endl;
{
int errors = 0;
auto buf_ptr = (float*)staging_buf.data();
for (uint32_t i = 0; i < dim_m * dim_n; ++i) {
float ref = ref_data.at(i);
float cur = buf_ptr[i];
if (std::abs((cur - ref) / ref) > 1e-6) {
std::cout << "error at result #" << std::dec << i
<< std::hex << ": actual=" << cur << ", expected=" << ref << std::endl;
++errors;
}
}
if (errors != 0) {
std::cout << "Found " << std::dec << errors << " errors!" << std::endl;
std::cout << "FAILED!" << std::endl;
return 1;
}
}
return 0;
}
int main(int argc, char *argv[]) {
// parse command arguments
parse_args(argc, argv);
if (count == 0) {
count = 1;
}
std::srand(50);
// open device connection
std::cout << "open device connection" << std::endl;
RT_CHECK(vx_dev_open(&device));
// FIXME: hardcoded
uint32_t dim_m = 64;
uint32_t dim_n = 64;
uint32_t dim_k = 64;
generate_source_matrix(dim_m, dim_n, dim_k);
generate_reference_matmul(dim_m, dim_n, dim_k);
uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]);
uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]);
uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]);
std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl;
// upload program
std::cout << "upload program" << std::endl;
RT_CHECK(vx_upload_kernel_file(device, kernel_file));
// allocate device memory
std::cout << "allocate device memory" << std::endl;
RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a));
RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b));
RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c));
kernel_arg.dim_m = dim_m;
kernel_arg.dim_n = dim_n;
kernel_arg.dim_k = dim_k;
std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl;
std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl;
std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl;
// allocate staging buffer
{
std::cout << "allocate staging buffer" << std::endl;
uint32_t staging_buf_size = std::max<uint32_t>(
src_a_buf_size,
std::max<uint32_t>(
src_b_buf_size,
std::max<uint32_t>(dst_buf_size, sizeof(kernel_arg_t))));
staging_buf.resize(staging_buf_size);
}
// upload kernel argument
{
std::cout << "upload kernel argument" << std::endl;
auto buf_ptr = staging_buf.data();
kernel_arg.addr_a = (uint64_t) 0x20000;
kernel_arg.addr_b = (uint64_t) 0x28000;
kernel_arg.addr_c = (uint64_t) 0xc0000000ULL;
memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t));
std::cout << "uploading argument buffer to device, device mem address="
<< std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec
<< sizeof(kernel_arg_t) << " bytes\n";
std::ofstream file("args.bin", std::ios::binary | std::ios::out);
if (!file) {
std::cerr << "error: failed to open args.bin for writing\n";
exit(EXIT_FAILURE);
}
file.write(reinterpret_cast<char *>(staging_buf.data()),
sizeof(kernel_arg_t));
file.close();
RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t)));
}
// upload source buffer
{
{
auto buf_ptr = staging_buf.data();
memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float));
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(),
src_a_buf_size));
std::cout << "uploading source A matrix to device, device mem address="
<< std::hex << kernel_arg.addr_a << ", size=" << std::dec
<< src_a_buf_size << " bytes\n";
std::ofstream file("input.a.bin", std::ios::binary | std::ios::out);
if (!file) {
std::cerr << "error: failed to open args.bin for writing\n";
exit(EXIT_FAILURE);
}
file.write(reinterpret_cast<char *>(buf_ptr), src_a_buf_size);
file.close();
}
{
auto buf_ptr = staging_buf.data();
memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float));
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(),
src_b_buf_size));
std::cout << "uploading source B matrix to device, device mem address="
<< std::hex << kernel_arg.addr_b << ", size=" << std::dec
<< src_b_buf_size << " bytes\n";
std::ofstream file("input.b.bin", std::ios::binary | std::ios::out);
if (!file) {
std::cerr << "error: failed to open args.bin for writing\n";
exit(EXIT_FAILURE);
}
file.write(reinterpret_cast<char *>(buf_ptr), src_b_buf_size);
file.close();
}
}
// clear destination buffer
{
std::cout << "clear destination buffer" << std::endl;
auto buf_ptr = (int32_t*)staging_buf.data();
for (uint32_t i = 0; i < ref_data.size(); ++i) {
buf_ptr[i] = 0xdeadbeef;
}
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size));
}
// run tests
std::cout << "run tests" << std::endl;
RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n));
std::cout << "PASSED!" << std::endl;
// cleanup
std::cout << "cleanup" << std::endl;
cleanup();
return 0;
}

BIN
tests/regression/rickroll/rickroll Executable file
View File

Binary file not shown.

100
tests/regression/sgemm_gemmini/kernel.cpp
View File

@@ -16,23 +16,26 @@
#define NUM_CLUSTERS 1
#define NUM_THREADS_IN_CLUSTER 128
#define SMEM_ADDR_0K ((float * const) 0xff000000)
#define SMEM_ADDR_4K ((float * const) 0xff001000)
#define SMEM_ADDR_8K ((float * const) 0xff002000)
#define SMEM_ADDR_12K ((float * const) 0xff003000)
#define SPAD_ADDR_0K 0x0
#define SPAD_ADDR_4K 0x80
#define SPAD_ADDR_8K 0x100
#define SPAD_ADDR_12K 0x180
#define SMEM_ADDR_Q0 ((float * const) 0xff000000)
#define SMEM_ADDR_Q1 ((float * const) 0xff001000)
#define SMEM_ADDR_Q2 ((float * const) 0xff002000)
#define SMEM_ADDR_Q3 ((float * const) 0xff003000)
#define SPAD_ADDR_Q0 0x0
#define SPAD_ADDR_Q1 0x80
#define SPAD_ADDR_Q2 0x100
#define SPAD_ADDR_Q3 0x180
#define SPAD_ADDR_Q4 0x200
//#define DEBUG_PRINT
//#define EXT_ACCUMULATE
#define HARDCODE
#define REGBLOCK
#define OFFLOAD_ACCUMULATE
#define REMATERIALIZE
#define DBUF
//#define CISC
//#define DEBUG_PRINT
//#define DETAILED_PERF
//#define ACTIVATE
#define CISC
#define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x))
#ifdef DETAILED_PERF
@@ -40,7 +43,11 @@
#else
#define rd_cycles(x)
#endif
#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
#ifdef REMATERIALIZE
#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
#else
#define HW_TID() hw_tid
#endif
#define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__)
// #define PRINTF(...) vx_printf(__VA_ARGS__)
#define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x))))
@@ -58,10 +65,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
const float * const B = (const float * const) arg->addr_b;
float * const C = (float * const) arg->addr_c;
if (HW_TID() == 0) {
gemmini_config_ld(0);
if (tid_in_threadblock % NUM_THREADS_IN_CLUSTER == 0) {
gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0);
gemmini_config_st(0);
// gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, b_transpose);
// gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale);
PRINTF("start\n");
}
@@ -100,17 +108,26 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
constexpr uint32_t i1_iters = (DIM * DIM * (TILE_K / DIM)) / num_threads_in_cluster; // num of iters before striding
const uint32_t num_tile_rows_per_tb = num_tiles_m / NUM_CLUSTERS;
if (HW_TID() == 0) {
gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 0);
gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 1);
// gemmini_extended3_config_ld(repeating_bias ? 0 : (stride_D * sizeof_D), D_scale_factor, low_D, 2);
gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY);
// gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale);
}
for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id;
tile_i < num_tile_rows_per_tb * (threadblock_id + 1);
tile_i += 1) {
__asm__("i_loop:");
for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) {
__asm__("j_loop:");
float * const smem_c_tile_start = SMEM_ADDR_4K;
#ifndef EXT_ACCUMULATE
float * const smem_acc_tile_start = SMEM_ADDR_0K + HW_TID();
float * const smem_c_tile_start = SMEM_ADDR_Q1;
#ifdef OFFLOAD_ACCUMULATE
float * const smem_acc_tile_start = SMEM_ADDR_Q0 + HW_TID();
#else
float * const smem_acc_tile_start = SMEM_ADDR_8K + hw_tid;
float * const smem_acc_tile_start = SMEM_ADDR_Q2 + hw_tid;
#endif
__asm__("k_loop:");
@@ -132,11 +149,11 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K + runtime_const_a;
const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N + runtime_const_b;
#ifdef DBUF
float * const smem_a_tile_start = ((tile_k & 1) ? SMEM_ADDR_4K : SMEM_ADDR_0K) + HW_TID();
float * const smem_b_tile_start = ((tile_k & 1) ? SMEM_ADDR_12K : SMEM_ADDR_8K) + HW_TID();
float * const smem_a_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q1 : SMEM_ADDR_Q0) + HW_TID();
float * const smem_b_tile_start = ((tile_k & 1) ? SMEM_ADDR_Q3 : SMEM_ADDR_Q2) + HW_TID();
#else
float * const smem_a_tile_start = SMEM_ADDR_0K + HW_TID();
float * const smem_b_tile_start = SMEM_ADDR_12K + HW_TID();
float * const smem_a_tile_start = SMEM_ADDR_Q0 + HW_TID();
float * const smem_b_tile_start = SMEM_ADDR_Q3 + HW_TID();
#endif
{
@@ -175,7 +192,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
smem_b_tile_start[7 * num_threads_in_cluster + hw_tid] = \
dram_b_tile_start[every_iter * 1 + every_2iters_b * 3];
#else
__asm__("load_ab:");
float v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 0];
float v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 0];
float v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 1];
@@ -185,7 +201,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
smem_a_tile_start[2 * num_threads_in_cluster] = v2;
smem_a_tile_start[3 * num_threads_in_cluster] = v3;
__asm__("load_ab1:");
v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 0];
v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 0];
v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 1];
@@ -195,7 +210,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
smem_b_tile_start[2 * num_threads_in_cluster] = v2;
smem_b_tile_start[3 * num_threads_in_cluster] = v3;
__asm__("load_ab2:");
v0 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 2];
v1 = dram_a_tile_start[every_iter * 1 + every_2iters_a * 2];
v2 = dram_a_tile_start[every_iter * 0 + every_2iters_a * 3];
@@ -205,7 +219,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
smem_a_tile_start[6 * num_threads_in_cluster] = v2;
smem_a_tile_start[7 * num_threads_in_cluster] = v3;
__asm__("load_ab3:");
v0 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 2];
v1 = dram_b_tile_start[every_iter * 1 + every_2iters_b * 2];
v2 = dram_b_tile_start[every_iter * 0 + every_2iters_b * 3];
@@ -214,8 +227,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
smem_b_tile_start[5 * num_threads_in_cluster] = v1;
smem_b_tile_start[6 * num_threads_in_cluster] = v2;
smem_b_tile_start[7 * num_threads_in_cluster] = v3;
__asm__("end_loadab:");
#endif
}
#else
@@ -223,8 +234,8 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
const float * const dram_a_tile_start = A + tile_i * TILE_M * dim_k + tile_k * TILE_K;
const float * const dram_b_tile_start = B + tile_k * TILE_K * dim_n + tile_j * TILE_N;
float * const smem_a_tile_start = SMEM_ADDR_0K;
float * const smem_b_tile_start = SMEM_ADDR_12K;
float * const smem_a_tile_start = SMEM_ADDR_Q0;
float * const smem_b_tile_start = SMEM_ADDR_Q3;
/* for (uint32_t thread_i = 0, j1 = 0, i1 = 0;
thread_i < a_elems_per_thread;
@@ -281,7 +292,6 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
// cluster wide barrier to wait for A and B loads to complete
threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
rd_cycles(marker3);
__asm__("gemmini:");
if (HW_TID() == 0) {
#ifdef DBUF
gemmini_fence();
@@ -290,8 +300,8 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
#ifndef DBUF
#error MUST ENABLE DBUF
#endif
#ifdef EXT_ACCUMULATE
#error MUST DISABLE EXT ACCUMULATE
#ifndef OFFLOAD_ACCUMULATE
#error MUST OFFLOAD ACCUMULATE
#endif
if (tile_k == 0) {
GEMMINI_CISC_CMD_I(0);
@@ -303,14 +313,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
#else
sp_tiled_matmul_full_spad_ws(
#ifdef DBUF
(tile_k & 1) ? SPAD_ADDR_4K : SPAD_ADDR_0K, (tile_k & 1) ? SPAD_ADDR_12K : SPAD_ADDR_8K,
(tile_k & 1) ? SPAD_ADDR_Q1 : SPAD_ADDR_Q0, (tile_k & 1) ? SPAD_ADDR_Q3 : SPAD_ADDR_Q2,
#else
SPAD_ADDR_0K, SPAD_ADDR_12K,
SPAD_ADDR_Q0, SPAD_ADDR_Q3,
#endif
/*spad_D=*/0, /*spad_C=*/SPAD_ADDR_4K,
/*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q1,
/*I=*/TILE_M / DIM, /*J=*/TILE_N / DIM, /*K=*/TILE_K / DIM, /*pad_I=*/0, /*pad_J=*/0, /*pad_K=*/0,
/*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0,
#ifdef EXT_ACCUMULATE
#ifndef OFFLOAD_ACCUMULATE
/*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/0x38U)
#else
/*acc=*/tile_k != 0, /*act=*/NO_ACTIVATION, /*skips=*/0xB8U)
@@ -321,13 +331,14 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
gemmini_fence();
#endif
}
__asm__("end_gemmini:");
rd_cycles(marker4);
// threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
#ifndef DBUF
threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
#endif
rd_cycles(marker5);
// accumulate C matrix
#ifdef EXT_ACCUMULATE
#ifndef OFFLOAD_ACCUMULATE
__asm__("accumulate:");
if (tile_k == 0) {
#pragma GCC ivdep
@@ -383,7 +394,7 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
}
#ifndef EXT_ACCUMULATE
#ifdef OFFLOAD_ACCUMULATE
threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
rd_cycles(marker6);
__asm__("mvout_spad_ser:");
@@ -476,13 +487,12 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
#endif
#else
float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N;
#pragma clang loop unroll(disable)
for (int thread_i = 0; thread_i < c_elems_per_thread; thread_i++) {
uint32_t elem_offset = hw_tid + num_threads_in_cluster * thread_i;
dram_c_tile_start[elem_offset / TILE_N * dim_n + elem_offset % TILE_N] = \
*(SMEM_ADDR_8K + SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N));
*(SMEM_ADDR_Q2 + SMEM_MAT_OFFSET(elem_offset / TILE_N, elem_offset % TILE_N, TILE_N));
}
#endif
__asm__("end_mvout_dram:");
@@ -513,7 +523,7 @@ void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
PRINTF("first barrier: %d\n", marker3 - marker2);
PRINTF("gemmini cycles: %d\n", marker4 - marker3);
PRINTF("second barrier: %d\n", marker5 - marker4);
#ifdef EXT_ACCUMULATE
#ifndef OFFLOAD_ACCUMULATE
PRINTF("accumulation cycles: %d\n", marker6 - marker5);
#else
PRINTF("smem mvout cycles: %d %d-%d\n", marker7 - marker6, marker7, marker6);

5
tests/regression/sgemm_gemmini_dma/.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
*.bin
*.dump
*.elf
sgemm_wg
.depend

9
tests/regression/sgemm_gemmini_dma/Makefile Normal file
View File

@@ -0,0 +1,9 @@
PROJECT = sgemm_gemmini_dma
SRCS = main.cpp common.h
VX_SRCS = kernel.cpp
OPTS ?= -n16
include ../common.mk

18
tests/regression/sgemm_gemmini_dma/common.h Normal file
View File

@@ -0,0 +1,18 @@
#ifndef _COMMON_H_
#define _COMMON_H_
#include <cstdint>
#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000
#define DEV_SMEM_START_ADDR 0xff000000
typedef struct {
uint32_t dim_m;
uint32_t dim_n;
uint32_t dim_k;
uint64_t addr_a;
uint64_t addr_b;
uint64_t addr_c;
} kernel_arg_t;
#endif

175
tests/regression/sgemm_gemmini_dma/kernel.cpp Normal file
View File

@@ -0,0 +1,175 @@
#include <stdint.h>
#include <vx_intrinsics.h>
#include <vx_print.h>
#include <vx_spawn.h>
#include "common.h"
#include "include/gemmini.h"
#include "gemmini_mmio.h"
#define TILE_M 64
#define TILE_N 64
#define TILE_K 64
#define SMEM_ADDR_Q0 ((float * const) 0xff000000)
#define SMEM_ADDR_Q1 ((float * const) 0xff004000)
#define SMEM_ADDR_Q2 ((float * const) 0xff008000)
#define SMEM_ADDR_Q3 ((float * const) 0xff00c000)
#define SPAD_ADDR_Q0 0x0
#define SPAD_ADDR_Q1 0x200
#define SPAD_ADDR_Q2 0x400
#define SPAD_ADDR_Q3 0x600
#define BOUND_INST 0x800080008ULL
// #define TILE_M 32
// #define TILE_N 32
// #define TILE_K 32
// #define SMEM_ADDR_Q0 ((float * const) 0xff000000)
// #define SMEM_ADDR_Q1 ((float * const) 0xff001000)
// #define SMEM_ADDR_Q2 ((float * const) 0xff002000)
// #define SMEM_ADDR_Q3 ((float * const) 0xff003000)
// #define SPAD_ADDR_Q0 0x0
// #define SPAD_ADDR_Q1 0x80
// #define SPAD_ADDR_Q2 0x100
// #define SPAD_ADDR_Q3 0x180
// #define BOUND_INST 0x400040004ULL
#define NUM_CLUSTERS 1
#define NUM_THREADS_IN_CLUSTER 128
#define rd_cycles_force(x) asm volatile ("csrr %0, mcycle" : "=r" (x))
#define rd_cycles(x) rd_cycles_force(x)
#define HW_TID() ({uint32_t gtid; asm volatile ("csrr %0, mhartid" : "=r" (gtid)); gtid;})
#define PRINTF(...) sprintf(PRINT_BUF, __VA_ARGS__)
// #define PRINTF(...) vx_printf(__VA_ARGS__)
#define SWISH(beta, x) ((x) / (1 + exp(-(beta) * (x))))
inline void threadblock_barrier(unsigned int barrier_id, unsigned int count) {
vx_fence();
vx_barrier(barrier_id, count);
}
void thread_block_matmul_gemmini(kernel_arg_t *__UNIFORM__ arg,
const uint32_t threadblock_id,
const uint32_t tid_in_threadblock) {
__asm__("matmul_start:");
const float * const A = (const float * const) arg->addr_a;
const float * const B = (const float * const) arg->addr_b;
float * const C = (float * const) arg->addr_c;
if (HW_TID() == 0) {
gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0);
// gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, b_transpose);
PRINTF("start\n");
}
vx_fence();
uint32_t marker0, marker1;
rd_cycles_force(marker0);
const uint32_t dim_m = arg->dim_m;
const uint32_t dim_n = arg->dim_n;
const uint32_t dim_k = arg->dim_k;
const uint32_t num_tiles_m = dim_m / TILE_M;
const uint32_t num_tiles_n = dim_n / TILE_N;
const uint32_t num_tiles_k = dim_k / TILE_K;
constexpr uint32_t num_threads_in_cluster = NUM_THREADS_IN_CLUSTER;
const uint32_t num_tile_rows_per_tb = num_tiles_m / NUM_CLUSTERS;
if (HW_TID() == 0) {
gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 0);
gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, false, 1);
// gemmini_extended3_config_ld(repeating_bias ? 0 : (stride_D * sizeof_D), D_scale_factor, low_D, 2);
gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY);
// gemmini_extended_config_st(stride_C * sizeof_C, act & 3, scale);
}
for (uint32_t tile_i = num_tile_rows_per_tb * threadblock_id;
tile_i < num_tile_rows_per_tb * (threadblock_id + 1);
tile_i += 1) {
for (int tile_j = 0; tile_j < num_tiles_n; tile_j += 1) {
if (HW_TID() == 0) {
for (int tile_k = 0; tile_k < num_tiles_k; tile_k += 1) {
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC,
(uint64_t) (A + tile_i * TILE_M * dim_k + tile_k * TILE_K),
(uint64_t) (B + tile_k * TILE_K * dim_n + tile_j * TILE_N), k_LOOP_WS_CONFIG_ADDRS_AB)
GEMMINI_CISC_CMD_R((dim_n) << 16 | (dim_k << 8) | 8);
if (tile_k & 1) {
GEMMINI_CISC_CMD_I(11);
} else {
GEMMINI_CISC_CMD_I(10);
}
if (tile_k == 0) {
gemmini_fence();
GEMMINI_CISC_CMD_I(0);
} else if (tile_k & 1) {
gemmini_fence();
GEMMINI_CISC_CMD_I(2);
} else {
gemmini_fence();
GEMMINI_CISC_CMD_I(1);
}
}
gemmini_fence();
gemmini_fence();
gemmini_fence();
gemmini_fence();
// mvout to scratchpad for activation
GEMMINI_CISC_CMD_I(9);
gemmini_fence();
}
threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
// activate
// move out to dram
if (HW_TID() == 0) {
float * const dram_c_tile_start = C + tile_i * TILE_M * dim_n + tile_j * TILE_N;
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, BOUND_INST, k_LOOP_WS_CONFIG_BOUNDS)
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, (uint64_t) dram_c_tile_start, k_LOOP_WS_CONFIG_ADDRS_DC)
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, dim_n, k_LOOP_WS_CONFIG_STRIDES_DC)
ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, loop_matmul_skips(1, 1, 1, 1, 0), k_LOOP_WS)
}
}
}
// last thread block complete
if (threadblock_id == NUM_CLUSTERS - 1) {
threadblock_barrier(/*barrier_id=*/0, /*count=*/NUM_WARPS);
rd_cycles_force(marker1);
if (HW_TID() == 0) {
PRINTF("\ncomplete\n");
PRINTF("total cycles: %d\n", marker1 - marker0);
for (int i = 0; i < dim_m; i += 8) {
for (int j = 0; j < dim_n; j += 8) {
PRINTF("%d %d ", (int) (C[i * dim_n + j]), (int) (C[i * dim_n + j + 4]));
}
PRINTF("\n");
}
}
}
vx_tmc(0);
}
void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
const int threadblock_id = task_id / NUM_THREADS_IN_CLUSTER;
const int tid_in_threadblock = task_id % NUM_THREADS_IN_CLUSTER;
thread_block_matmul_gemmini(arg, threadblock_id, tid_in_threadblock);
}
int main() {
kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR;
const uint32_t num_threads_in_cluster = vx_num_threads() * vx_num_warps() * CORES_PER_CLUSTER;
const uint32_t grid_size = num_threads_in_cluster * NUM_CLUSTERS;
#ifdef RADIANCE
vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
#else
// NOTE: This kernel assumes contiguous thread scheduling for efficient shared
// memory allocation, and therefore does not work with original vx_spawn_tasks
vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
#endif
return 0;
}

274
tests/regression/sgemm_gemmini_dma/main.cpp Normal file
View File

@@ -0,0 +1,274 @@
#include <iostream>
#include <fstream>
#include <unistd.h>
#include <string.h>
#include <vortex.h>
#include <vector>
#include "common.h"
#define RT_CHECK(_expr) \
do { \
int _ret = _expr; \
if (0 == _ret) \
break; \
printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \
cleanup(); \
exit(-1); \
} while (false)
///////////////////////////////////////////////////////////////////////////////
const char* kernel_file = "kernel.bin";
uint32_t count = 0;
std::vector<float> src_a_data;
std::vector<float> src_b_data;
std::vector<float> ref_data;
vx_device_h device = nullptr;
std::vector<uint8_t> staging_buf;
kernel_arg_t kernel_arg = {};
static void show_usage() {
std::cout << "Vortex Test." << std::endl;
std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl;
}
static void parse_args(int argc, char **argv) {
int c;
while ((c = getopt(argc, argv, "n:k:h?")) != -1) {
switch (c) {
case 'n':
count = atoi(optarg);
break;
case 'k':
kernel_file = optarg;
break;
case 'h':
case '?': {
show_usage();
exit(0);
} break;
default:
show_usage();
exit(-1);
}
}
}
void cleanup() {
if (device) {
vx_mem_free(device, kernel_arg.addr_a);
vx_mem_free(device, kernel_arg.addr_b);
vx_mem_free(device, kernel_arg.addr_c);
vx_dev_close(device);
}
}
void generate_source_matrix(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) {
src_a_data.resize(dim_m * dim_k);
src_b_data.resize(dim_k * dim_n);
for (uint32_t i = 0; i < src_a_data.size(); ++i) {
src_a_data[i] = static_cast<float>(i);
std::cout << "A: " << i << ": value=" << src_a_data[i] << std::endl;
}
for (uint32_t i = 0; i < src_b_data.size(); ++i) {
src_b_data[i] = static_cast<float>(i);
std::cout << "B: " << i << ": value=" << src_b_data[i] << std::endl;
}
}
void generate_reference_matmul(uint32_t dim_m, uint32_t dim_n, uint32_t dim_k) {
ref_data.resize(dim_m * dim_n);
for (uint32_t i = 0; i < dim_m; ++i) {
for (uint32_t j = 0; j < dim_n; ++j) {
float ref = 0.0f;
for (uint32_t k = 0; k < dim_k; ++k) {
ref += src_a_data[dim_k * i + k] * src_b_data[dim_n * k + j];
}
ref_data.at(dim_n * i + j) = ref;
}
}
}
int run_test(const kernel_arg_t& kernel_arg,
uint32_t buf_size,
uint32_t dim_m, uint32_t dim_n) {
// start device
std::cout << "start device" << std::endl;
RT_CHECK(vx_start(device));
// wait for completion
std::cout << "wait for completion" << std::endl;
RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT));
// download destination buffer
std::cout << "download destination buffer" << std::endl;
RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_c, buf_size));
// verify result
std::cout << "verify result" << std::endl;
{
int errors = 0;
auto buf_ptr = (float*)staging_buf.data();
for (uint32_t i = 0; i < dim_m * dim_n; ++i) {
float ref = ref_data.at(i);
float cur = buf_ptr[i];
if (std::abs((cur - ref) / ref) > 1e-6) {
std::cout << "error at result #" << std::dec << i
<< std::hex << ": actual=" << cur << ", expected=" << ref << std::endl;
++errors;
}
}
if (errors != 0) {
std::cout << "Found " << std::dec << errors << " errors!" << std::endl;
std::cout << "FAILED!" << std::endl;
return 1;
}
}
return 0;
}
int main(int argc, char *argv[]) {
// parse command arguments
parse_args(argc, argv);
if (count == 0) {
count = 1;
}
std::srand(50);
// open device connection
std::cout << "open device connection" << std::endl;
RT_CHECK(vx_dev_open(&device));
// FIXME: hardcoded
uint32_t dim_m = 64;
uint32_t dim_n = 64;
uint32_t dim_k = 64;
generate_source_matrix(dim_m, dim_n, dim_k);
generate_reference_matmul(dim_m, dim_n, dim_k);
uint32_t src_a_buf_size = src_a_data.size() * sizeof(src_a_data[0]);
uint32_t src_b_buf_size = src_b_data.size() * sizeof(src_b_data[0]);
uint32_t dst_buf_size = ref_data.size() * sizeof(src_a_data[0]);
std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl;
// upload program
std::cout << "upload program" << std::endl;
RT_CHECK(vx_upload_kernel_file(device, kernel_file));
// allocate device memory
std::cout << "allocate device memory" << std::endl;
RT_CHECK(vx_mem_alloc(device, src_a_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_a));
RT_CHECK(vx_mem_alloc(device, src_b_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_b));
RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_c));
kernel_arg.dim_m = dim_m;
kernel_arg.dim_n = dim_n;
kernel_arg.dim_k = dim_k;
std::cout << "dev_addr_a=0x" << std::hex << kernel_arg.addr_a << std::endl;
std::cout << "dev_addr_b=0x" << std::hex << kernel_arg.addr_b << std::endl;
std::cout << "dev_addr_c=0x" << std::hex << kernel_arg.addr_c << std::endl;
// allocate staging buffer
{
std::cout << "allocate staging buffer" << std::endl;
uint32_t staging_buf_size = std::max<uint32_t>(
src_a_buf_size,
std::max<uint32_t>(
src_b_buf_size,
std::max<uint32_t>(dst_buf_size, sizeof(kernel_arg_t))));
staging_buf.resize(staging_buf_size);
}
// upload kernel argument
{
std::cout << "upload kernel argument" << std::endl;
auto buf_ptr = staging_buf.data();
kernel_arg.addr_a = (uint64_t) 0x20000;
kernel_arg.addr_b = (uint64_t) 0x28000;
kernel_arg.addr_c = (uint64_t) 0xc0000000ULL;
memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t));
std::cout << "uploading argument buffer to device, device mem address="
<< std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec
<< sizeof(kernel_arg_t) << " bytes\n";
std::ofstream file("args.bin", std::ios::binary | std::ios::out);
if (!file) {
std::cerr << "error: failed to open args.bin for writing\n";
exit(EXIT_FAILURE);
}
file.write(reinterpret_cast<char *>(staging_buf.data()),
sizeof(kernel_arg_t));
file.close();
RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t)));
}
// upload source buffer
{
{
auto buf_ptr = staging_buf.data();
memcpy(buf_ptr, src_a_data.data(), src_a_data.size() * sizeof(float));
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_a, staging_buf.data(),
src_a_buf_size));
std::cout << "uploading source A matrix to device, device mem address="
<< std::hex << kernel_arg.addr_a << ", size=" << std::dec
<< src_a_buf_size << " bytes\n";
std::ofstream file("input.a.bin", std::ios::binary | std::ios::out);
if (!file) {
std::cerr << "error: failed to open args.bin for writing\n";
exit(EXIT_FAILURE);
}
file.write(reinterpret_cast<char *>(buf_ptr), src_a_buf_size);
file.close();
}
{
auto buf_ptr = staging_buf.data();
memcpy(buf_ptr, src_b_data.data(), src_b_data.size() * sizeof(float));
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_b, staging_buf.data(),
src_b_buf_size));
std::cout << "uploading source B matrix to device, device mem address="
<< std::hex << kernel_arg.addr_b << ", size=" << std::dec
<< src_b_buf_size << " bytes\n";
std::ofstream file("input.b.bin", std::ios::binary | std::ios::out);
if (!file) {
std::cerr << "error: failed to open args.bin for writing\n";
exit(EXIT_FAILURE);
}
file.write(reinterpret_cast<char *>(buf_ptr), src_b_buf_size);
file.close();
}
}
// clear destination buffer
{
std::cout << "clear destination buffer" << std::endl;
auto buf_ptr = (int32_t*)staging_buf.data();
for (uint32_t i = 0; i < ref_data.size(); ++i) {
buf_ptr[i] = 0xdeadbeef;
}
RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_c, staging_buf.data(), dst_buf_size));
}
// run tests
std::cout << "run tests" << std::endl;
RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.dim_m, kernel_arg.dim_n));
std::cout << "PASSED!" << std::endl;
// cleanup
std::cout << "cleanup" << std::endl;
cleanup();
return 0;
}

BIN
tests/regression/sgemm_gemmini_dma/sgemm_gemmini_dma Executable file
View File

Binary file not shown.