Merge branch 'kernels' of https://github.com/hansungk/vortex-private into kernels

kernel/src/vx_spawn.c

@@ -258,7 +258,6 @@ void vx_spawn_tasks_cluster(int num_tasks, vx_spawn_tasks_cb callback, void *arg
   // threads, handle this in the last wave amongst other full warps.
   if (rem_threads_in_last_warp != 0 && core_id_in_cluster == 0) {
     // adjust offset
-    // FIXME: consider cluster_id here
     // FIXME: use rem_threads_in_last_warp_this_core
     wspawn_args.offset += (num_tasks_this_cluster - rem_threads_in_last_warp);
 

tests/regression/common.mk

@@ -79,7 +79,7 @@ endif
 endif
 endif
 
-all: $(PROJECT) kernel.bin kernel.dump kernel.radiance.dump
+all: $(PROJECT) kernel.bin kernel.dump kernel.radiance.dump kernel.radiance.$(CONFIG).dump
 
 kernel.dump: kernel.elf
 	$(VX_DP) -D kernel.elf > kernel.dump
@@ -87,6 +87,9 @@ kernel.dump: kernel.elf
 kernel.radiance.dump: kernel.radiance.elf
 	$(VX_DP) -D kernel.radiance.elf > kernel.radiance.dump
 
+kernel.radiance.$(CONFIG).dump: kernel.radiance.$(CONFIG).elf
+	$(VX_DP) -D kernel.radiance.$(CONFIG).elf > kernel.radiance.$(CONFIG).dump
+
 kernel.bin: kernel.elf kernel.radiance.elf
 	$(VX_CP) -O binary kernel.elf kernel.bin
 
@@ -95,13 +98,16 @@ kernel.elf: $(VX_SRCS)
 
 OBJCOPY ?= "riscv32-unknown-elf-objcopy"
 OBJCOPY_FLAGS ?= "LOAD,ALLOC,DATA,CONTENTS"
-kernel.radiance.elf: $(VX_SRCS)
+kernel.radiance.elf: kernel.elf
 	$(VX_CXX) $(VX_CFLAGS) $(VX_SRCS) $(VX_LDFLAGS) -DRADIANCE -o kernel.radiance.elf
 	$(OBJCOPY) --set-section-flags .operand.a=$(OBJCOPY_FLAGS) kernel.radiance.elf
 	$(OBJCOPY) --set-section-flags .operand.b=$(OBJCOPY_FLAGS) kernel.radiance.elf
 	$(OBJCOPY) --update-section .operand.a=input.a.bin kernel.radiance.elf
 	$(OBJCOPY) --update-section .operand.b=input.b.bin kernel.radiance.elf
 
+kernel.radiance.$(CONFIG).elf: kernel.radiance.elf
+	cp $< $@
+
 $(PROJECT): $(SRCS)
 	$(CXX) $(CXXFLAGS) $^ $(LDFLAGS) -o $@
 
@@ -128,7 +134,7 @@ clean:
 	rm -rf $(PROJECT) *.o .depend
 
 clean-all: clean
-	rm -rf kernel.elf kernel.radiance.elf *.dump
+	rm -rf kernel.elf kernel.dump
 
 ifneq ($(MAKECMDGOALS),clean)
     -include .depend

tests/regression/flops/.gitignore

@@ -0,0 +1,5 @@
+*.bin
+*.dump
+*.elf
+flops
+.depend

tests/regression/flops/Makefile

@@ -0,0 +1,9 @@
+PROJECT = flops
+
+SRCS = main.cpp common.h
+
+VX_SRCS = kernel.cpp
+
+OPTS ?= -n16
+
+include ../common.mk

tests/regression/flops/common.h

@@ -0,0 +1,15 @@
+#ifndef _COMMON_H_
+#define _COMMON_H_
+
+#include <cstdint>
+
+#define KERNEL_ARG_DEV_MEM_ADDR 0x7fff0000
+#define DEV_SMEM_START_ADDR 0xff000000
+
+typedef struct {
+  uint32_t size;
+  uint32_t addr_src;
+  uint32_t addr_dst;
+} kernel_arg_t;
+
+#endif

tests/regression/flops/kernel.cpp

@@ -0,0 +1,41 @@
+#include <stdint.h>
+#include <vx_intrinsics.h>
+#include <vx_spawn.h>
+#include "common.h"
+
+void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) {
+  const float *A = (const float *)arg->addr_src;
+  float *C = (float *)arg->addr_dst;
+
+  int incr = A[task_id];
+  float sum = 0.0f;
+  float sum1 = 0.0f;
+  float sum2 = 0.0f;
+  float sum3 = 0.0f;
+  float sum4 = 0.0f;
+  float sum5 = 0.0f;
+#pragma unroll 8
+  for (int i = 0; i < 5000; i++) {
+    sum1 = sum2 + 5.0f;
+    sum2 = sum3 + 5.0f;
+    sum3 = sum4 + 5.0f;
+    sum4 = sum5 + 5.0f;
+    sum5 = sum1 + 5.0f;
+  }
+
+  sum = sum1 + sum2 + sum3 + sum4 + sum5;
+  C[task_id] = static_cast<float>(sum);
+}
+
+int main() {
+  kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR;
+  const uint32_t grid_size = arg->size;
+#ifdef RADIANCE
+  vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
+#else
+  // NOTE: This kernel assumes contiguous thread scheduling for efficient shared
+  // memory allocation, and therefore does not work with original vx_spawn_tasks
+  vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg);
+#endif
+  return 0;
+}

tests/regression/flops/main.cpp

@@ -0,0 +1,252 @@
+#include <iostream>
+#include <fstream>
+#include <unistd.h>
+#include <string.h>
+#include <vortex.h>
+#include <vector>
+#include "common.h"
+
+#define RT_CHECK(_expr)                                         \
+   do {                                                         \
+     int _ret = _expr;                                          \
+     if (0 == _ret)                                             \
+       break;                                                   \
+     printf("Error: '%s' returned %d!\n", #_expr, (int)_ret);   \
+	 cleanup();			                                              \
+     exit(-1);                                                  \
+   } while (false)
+
+///////////////////////////////////////////////////////////////////////////////
+
+const char* kernel_file = "kernel.bin";
+uint32_t count = 0;
+
+std::vector<float> src_data;
+std::vector<float> ref_data;
+
+vx_device_h device = nullptr;
+std::vector<uint8_t> staging_buf;
+kernel_arg_t kernel_arg = {};
+
+static void show_usage() {
+   std::cout << "Vortex Test." << std::endl;
+   std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl;
+}
+
+static void parse_args(int argc, char **argv) {
+  int c;
+  while ((c = getopt(argc, argv, "n:k:h?")) != -1) {
+    switch (c) {
+    case 'n':
+      count = atoi(optarg);
+      break;
+    case 'k':
+      kernel_file = optarg;
+      break;
+    case 'h':
+    case '?': {
+      show_usage();
+      exit(0);
+    } break;
+    default:
+      show_usage();
+      exit(-1);
+    }
+  }
+}
+
+void cleanup() {
+  if (device) {
+    // vx_mem_free(device, kernel_arg.addr_a);
+    // vx_mem_free(device, kernel_arg.addr_b);
+    // vx_mem_free(device, kernel_arg.addr_c);
+    vx_dev_close(device);
+  }
+}
+
+void generate_source_data(size_t size) {
+  src_data.resize(size);
+
+  for (uint32_t i = 0; i < src_data.size(); ++i) {
+    src_data[i] = static_cast<float>(i);
+  }
+}
+
+void generate_reference_data(size_t size) {
+  ref_data.resize(size);
+
+  for (uint32_t i = 0; i < ref_data.size(); ++i) {
+    ref_data[i] = static_cast<float>(i) * 1000.0f;
+  }
+}
+
+int run_test(const kernel_arg_t& kernel_arg,
+             uint32_t buf_size,
+             uint32_t size) {
+  // start device
+  std::cout << "start device" << std::endl;
+  RT_CHECK(vx_start(device));
+
+  // wait for completion
+  std::cout << "wait for completion" << std::endl;
+  RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT));
+
+  // download destination buffer
+  std::cout << "download destination buffer" << std::endl;
+  RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_dst, buf_size));
+
+  std::cout << "downloading result C matrix from device, device mem address="
+            << std::hex << kernel_arg.addr_dst << ", size=" << std::dec
+            << buf_size << " bytes\n";
+  std::ofstream file("output.bin", std::ios::binary | std::ios::out);
+  if (!file) {
+    std::cerr << "error: failed to open output.bin for writing\n";
+    exit(EXIT_FAILURE);
+  }
+  file.write(reinterpret_cast<char *>(staging_buf.data()), buf_size);
+  file.close();
+
+  std::ofstream ref_file("reference.bin", std::ios::binary | std::ios::out);
+  if (!ref_file) {
+    std::cerr << "error: failed to open reference.bin for writing\n";
+    exit(EXIT_FAILURE);
+  }
+  ref_file.write(reinterpret_cast<char *>(ref_data.data()), buf_size);
+  ref_file.close();
+
+  // verify result
+  std::cout << "verify result" << std::endl;
+  {
+    int errors = 0;
+    auto buf_ptr = (float*)staging_buf.data();
+    for (uint32_t i = 0; i < size; ++i) {
+      float ref = ref_data.at(i);
+      float cur = buf_ptr[i];
+      if (std::abs((cur - ref) / ref) > 1e-6) {
+        std::cout << "error at result #" << std::dec << i
+                  << std::hex << ": actual=" << cur << ", expected=" << ref << std::endl;
+        ++errors;
+      }
+    }
+    if (errors != 0) {
+      std::cout << "Found " << std::dec << errors << " errors!" << std::endl;
+      std::cout << "FAILED!" << std::endl;
+      return 1;
+    }
+  }
+
+  return 0;
+}
+
+int main(int argc, char *argv[]) {
+  // parse command arguments
+  parse_args(argc, argv);
+
+  if (count == 0) {
+    count = 1;
+  }
+
+  std::srand(50);
+
+  // open device connection
+  std::cout << "open device connection" << std::endl;
+  RT_CHECK(vx_dev_open(&device));
+
+  size_t size = 64;
+
+  generate_source_data(size);
+  generate_reference_data(size);
+
+  uint32_t src_buf_size = src_data.size() * sizeof(src_data[0]);
+  uint32_t dst_buf_size = ref_data.size() * sizeof(ref_data[0]);
+
+  std::cout << "buffer size: " << dst_buf_size << " bytes" << std::endl;
+
+  // upload program
+  std::cout << "upload program" << std::endl;
+  RT_CHECK(vx_upload_kernel_file(device, kernel_file));
+
+  // allocate device memory
+  std::cout << "allocate device memory" << std::endl;
+  // RT_CHECK(vx_mem_alloc(device, src_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_src));
+  // RT_CHECK(vx_mem_alloc(device, dst_buf_size, VX_MEM_TYPE_GLOBAL, &kernel_arg.addr_dst));
+  kernel_arg.addr_src = 0x20000UL;
+  kernel_arg.addr_dst = 0xc0000000UL;
+  kernel_arg.size = size;
+
+  std::cout << "dev_addr_src=0x" << std::hex << kernel_arg.addr_src << std::endl;
+  std::cout << "dev_addr_dst=0x" << std::hex << kernel_arg.addr_dst << std::endl;
+
+  // allocate staging buffer
+  {
+    std::cout << "allocate staging buffer" << std::endl;
+    uint32_t staging_buf_size = std::max<uint32_t>(
+        src_buf_size,
+        std::max<uint32_t>(
+            src_buf_size,
+            std::max<uint32_t>(dst_buf_size, sizeof(kernel_arg_t))));
+    staging_buf.resize(staging_buf_size);
+  }
+
+  // upload kernel argument
+  {
+    std::cout << "upload kernel argument" << std::endl;
+    auto buf_ptr = staging_buf.data();
+    memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t));
+    RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t)));
+
+    std::cout << "uploading argument buffer to device, device mem address="
+              << std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec
+              << sizeof(kernel_arg_t) << " bytes\n";
+    std::ofstream file("args.bin", std::ios::binary | std::ios::out);
+    if (!file) {
+        std::cerr << "error: failed to open args.bin for writing\n";
+        exit(EXIT_FAILURE);
+    }
+    file.write(reinterpret_cast<char *>(staging_buf.data()),
+               sizeof(kernel_arg_t));
+    file.close();
+  }
+
+  // upload source buffer
+  {
+    {
+        auto buf_ptr = staging_buf.data();
+        memcpy(buf_ptr, src_data.data(), src_data.size() * sizeof(float));
+        RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_src, staging_buf.data(),
+                                src_buf_size));
+
+        std::cout << "uploading source data to device, device mem address="
+                  << std::hex << kernel_arg.addr_src << ", size=" << std::dec
+                  << src_buf_size << " bytes\n";
+        std::ofstream file("input.a.bin", std::ios::binary | std::ios::out);
+        if (!file) {
+        std::cerr << "error: failed to open input.a.bin for writing\n";
+        exit(EXIT_FAILURE);
+        }
+        file.write(reinterpret_cast<char *>(buf_ptr), src_buf_size);
+        file.close();
+    }
+  }
+
+  // clear destination buffer
+  {
+    std::cout << "clear destination buffer" << std::endl;
+    auto buf_ptr = (int32_t*)staging_buf.data();
+    for (uint32_t i = 0; i < ref_data.size(); ++i) {
+      buf_ptr[i] = 0xdeadbeef;
+    }
+    RT_CHECK(vx_copy_to_dev(device, kernel_arg.addr_dst, staging_buf.data(), dst_buf_size));
+  }
+
+  // run tests
+  std::cout << "run tests" << std::endl;
+  RT_CHECK(run_test(kernel_arg, dst_buf_size, kernel_arg.size));
+  std::cout << "PASSED!" << std::endl;
+
+  // cleanup
+  std::cout << "cleanup" << std::endl;
+  cleanup();
+
+  return 0;
+}

tests/regression/sgemm_wg/kernel.cpp

@@ -16,11 +16,11 @@
 //   (BM*BN) / (TM*TN) == threadblock size >= NT * CORES_PER_CLUSTER
 // * Combining BM * BK >= (BM*BN) / (TM*TN) == threadblock yields
 //   BM <= BK*TM*TN
-#define BM 8
+#define BM 32
 #define BN BM
-#define BK 2
+#define BK 8
-#define TM 2
+#define TM 4
-#define TN 2
+#define TN 4
 
 void threadblock_barrier(unsigned int tid_in_threadblock, unsigned int barrier_id, unsigned int count) {
     vx_fence();
@@ -80,14 +80,14 @@ void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg,
     //
     // Make sure global offset values for A and B are contiguous between
     // neighboring threads to ensure GMEM coalescing.
-// #pragma GCC unroll 1
+#pragma GCC unroll 2
     for (uint32_t load_offset = 0; load_offset < BM; load_offset += stride_a) {
       const uint32_t global_a_offset =
           dim_k * (global_a_row + load_offset) + (k + local_a_col);
       local_a[BK * (local_a_row + load_offset) + local_a_col] =
           A[global_a_offset];
     }
-// #pragma GCC unroll 1
+#pragma GCC unroll 2
     for (uint32_t load_offset = 0; load_offset < BK; load_offset += stride_b) {
       const uint32_t global_b_offset =
           dim_n * (k + local_b_row + load_offset) + global_b_col;
@@ -99,7 +99,7 @@ void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg,
                         threadblock_dim_y);
 
     // Compute single tile*tile matmul
-// #pragma GCC unroll 2
+#pragma GCC unroll 4
     for (uint32_t local_k = 0; local_k < BK; local_k++) {
       // First, pump data from SMEM->RF
 #pragma GCC unroll TM

tests/regression/sgemm_wg/main.cpp

@@ -119,6 +119,14 @@ int run_test(const kernel_arg_t& kernel_arg,
   file.write(reinterpret_cast<char *>(staging_buf.data()), buf_size);
   file.close();
 
+  std::ofstream ref_file("reference.c.bin", std::ios::binary | std::ios::out);
+  if (!ref_file) {
+    std::cerr << "error: failed to open reference.c.bin for writing\n";
+    exit(EXIT_FAILURE);
+  }
+  ref_file.write(reinterpret_cast<char *>(ref_data.data()), buf_size);
+  ref_file.close();
+
   // verify result
   std::cout << "verify result" << std::endl;
   {
@@ -158,9 +166,9 @@ int main(int argc, char *argv[]) {
   RT_CHECK(vx_dev_open(&device));
 
   // FIXME: hardcoded
-  uint32_t dim_m = 32;
+  uint32_t dim_m = 128;
-  uint32_t dim_n = 32;
+  uint32_t dim_n = 128;
-  uint32_t dim_k = 32;
+  uint32_t dim_k = 128;
 
   generate_source_matrix(dim_m, dim_n, dim_k);
   generate_reference_matmul(dim_m, dim_n, dim_k);