diff --git a/ci/toolchain_env.sh b/ci/toolchain_env.sh index 3d4e2d41..d0002201 100644 --- a/ci/toolchain_env.sh +++ b/ci/toolchain_env.sh @@ -14,7 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -TOOLDIR=${TOOLDIR:=/opt} +TOOLDIR=${TOOLDIR:=/scratch/hansung/build/vortex-toolchain-prebuilt} +export TOOLDIR export VERILATOR_ROOT=$TOOLDIR/verilator export PATH=$VERILATOR_ROOT/bin:$PATH @@ -25,6 +26,8 @@ export PATH=$SV2V_PATH/bin:$PATH export YOSYS_PATH=$TOOLDIR/yosys export PATH=$YOSYS_PATH/bin:$PATH -export LLVM_VORTEX=$TOOLDIR/llvm-vortex -export POCL_CC_PATH=$TOOLDIR/pocl/compiler -export POCL_RT_PATH=$TOOLDIR/pocl/runtime \ No newline at end of file +# LLVM_POCL seems to be only used in tests/opencl +export LLVM_POCL=/scratch/hansung/build/llvm-vortex2 +export LLVM_VORTEX=/scratch/hansung/build/llvm-vortex2 +export POCL_CC_PATH=/scratch/hansung/build/pocl-vortex2/compiler +export POCL_RT_PATH=/scratch/hansung/build/pocl-vortex2/runtime diff --git a/kernel/linker/vx_link32.ld b/kernel/linker/vx_link32.ld index eae88853..6999eaee 100644 --- a/kernel/linker/vx_link32.ld +++ b/kernel/linker/vx_link32.ld @@ -13,6 +13,7 @@ MEMORY { DRAMARG (rwx): ORIGIN = 0x9fff0000, LENGTH = 8K DRAM1 (rwx): ORIGIN = 0xa0000000, LENGTH = 16M DRAM2 (rwx): ORIGIN = 0xa1000000, LENGTH = 16M + DRAM3 (rwx): ORIGIN = 0xa2000000, LENGTH = 16M } SECTIONS @@ -288,4 +289,8 @@ SECTIONS *(.operand.b) . += 32K; }> DRAM2 + .operand.c : { + *(.operand.c) + . += 32K; + }> DRAM3 } diff --git a/tests/kernel/tensor/Makefile b/tests/kernel/tensor/Makefile index 19cb340e..10548774 100644 --- a/tests/kernel/tensor/Makefile +++ b/tests/kernel/tensor/Makefile @@ -6,3 +6,17 @@ DEPS += b_matrix.h DEPS += c_matrix.h include ../common.mk + +OBJCOPY ?= $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy +OBJCOPY_FLAGS ?= "LOAD,ALLOC,DATA,CONTENTS" +BINFILES := args.bin input.a.bin input.b.bin +$(PROJECT).elf: $(SRCS) $(DEPS) + $(CC) $(CFLAGS) $(SRCS) $(LDFLAGS) -o $(PROJECT).elf + $(OBJCOPY) --set-section-flags .operand.a=$(OBJCOPY_FLAGS) $@ + $(OBJCOPY) --set-section-flags .operand.b=$(OBJCOPY_FLAGS) $@ + $(OBJCOPY) --set-section-flags .operand.c=$(OBJCOPY_FLAGS) $@ + $(OBJCOPY) --set-section-flags .args=$(OBJCOPY_FLAGS) $@ + $(OBJCOPY) --update-section .operand.a=input.a.bin $@ || true + $(OBJCOPY) --update-section .operand.b=input.b.bin $@ || true + $(OBJCOPY) --update-section .operand.c=input.c.bin $@ || true + $(OBJCOPY) --update-section .args=args.bin $@ || true diff --git a/tests/kernel/tensor/check_correctness.py b/tests/kernel/tensor/check_correctness.py index 94f25b2b..aa6147ce 100644 --- a/tests/kernel/tensor/check_correctness.py +++ b/tests/kernel/tensor/check_correctness.py @@ -41,9 +41,13 @@ def C_index(threadgroup, thread, register): def check_sim_output(): file = input("simulator output filename: ") - A_array = np.zeros((16, 8)) - B_array = np.zeros((8, 16)) - C_array = np.zeros((16, 16)) + M = 8 + N = 8 + K = 16 + + A_array = np.zeros((M, K)) + B_array = np.zeros((K, N)) + C_array = np.zeros((M, N)) with open(file) as f: for line in f.readlines(): @@ -87,26 +91,23 @@ def check_sim_output(): if __name__ == "__main__": expected = np.load("abc.npz") - # expected_A = expected['A_array'] - # expected_B = expected['B_array'] - # expected_C = expected['C_array'] - expected_A = expected['A_array'][0:8, 0:8] - expected_B = expected['B_array'][0:8, 0:8] - expected_C = expected['C_array'][0:8, 0:8] + expected_A = expected['A_array'] + expected_B = expected['B_array'] + expected_C = expected['C_array'] expected_C = expected_C + expected_A @ expected_B print('expected A:') print(expected_A) print('expected B:') print(expected_B) print('expected C:') - print(expected_C[0:8, 0:8]) + print(expected_C) expected_C.astype('float32').tofile("c_expected.bin") [got_A, got_B, got_C] = check_sim_output() print('got C:') - print(C_array[0:8, 0:8]) + print(C_array) print('diff C:') - print(expected_C[0:8, 0:8] - C_array[0:8, 0:8]) + print(expected_C - C_array) assert np.allclose(expected_A, got_A) assert np.allclose(expected_B, got_B) assert np.allclose(expected_C, got_C) diff --git a/tests/kernel/tensor/flash_attn.py b/tests/kernel/tensor/flash_attn.py new file mode 100644 index 00000000..dfe92c5f --- /dev/null +++ b/tests/kernel/tensor/flash_attn.py @@ -0,0 +1,158 @@ +import sys +import numpy as np + +def parse_mnk(): + if len(sys.argv) != 4: + print(f"usage: {sys.argv[0]} dimM dimN dimK", file=sys.stderr) + sys.exit(1) + m = int(sys.argv[1]) + n = int(sys.argv[2]) + k = int(sys.argv[3]) + return (m, n, k) + + +# Reorder array in a way that groups two adjacent elements along the column to +# be now adjacent along the row. This way, when the resulting fp16 array is +# read in column-major order with 32-bit granularity, the fp16 elements will be +# read in the same order as regular fp32 elements in column-major. +# +# For example: +# [[1 2] +# [3 4] +# [5 6] +# [7 8]] +# becomes +# [[1 3 2 4] +# [5 7 6 8]] +def pack_fp16_by_column(array): + rows = array.shape[0] + cols = array.shape[1] + + T = array.transpose([1, 0]) + T_packed = T.reshape([cols, -1, 2]) + result = T_packed.transpose([1, 0, 2]) + return result + + +# Do the same as pack_fp16_by_column, but for every two elements along the row. +def pack_fp16_by_row(array): + rows = array.shape[0] + cols = array.shape[1] + + result = array.reshape([rows, -1, 2]) + return result + + +if __name__ == "__main__": + seqlen, _, headdim = parse_mnk() + + rand = True + if not rand: + A_array = np.arange(seqlen * headdim).reshape([seqlen, headdim]) + B_array = np.arange(headdim * seqlen).reshape([headdim, seqlen]) + C_array = np.arange(seqlen * seqlen).reshape([seqlen, headdim]) + else: + np.random.seed(0) + A_array = np.random.rand(seqlen, headdim) - 0.5 + B_array = np.random.rand(headdim, seqlen) - 0.5 + C_array = np.random.rand(seqlen, headdim) - 0.5 + # C_array = np.zeros([M, N]) + + fp16 = False + if fp16: + A_packed = pack_fp16_by_row(A_array) + AT_packed = A_packed.transpose([1, 0, 2]) + AT_array = AT_packed.reshape([-1, seqlen * 2]) + AT_array.astype('float16').tofile("input.a.col.bin") + # print('AT:') + # print(AT_array) + B_packed = pack_fp16_by_column(B_array) + B_array = B_packed.reshape([-1, headdim * 2]) + B_array.astype('float16').tofile("input.b.row.bin") + # print('B:') + # print(B_array) + else: + A_array.astype('float32').tofile("input.a.row.bin") + AT_array = A_array.transpose([1, 0]) + AT_array.astype('float32').tofile("input.a.col.bin") + B_array.astype('float32').tofile("input.b.bin") + C_array.astype('float32').tofile("input.c.bin") + # print('AT:') + # print(AT_array) + # print('B:') + # print(B_array) + + assert((seqlen % 64) == 0) + + Br = 64 + Bc = Br + + rowmax = np.zeros([Br]) + rowsum = np.zeros([Br]) + O = np.zeros([Br, headdim]) + + def exp2(x): + return (x**2) / 2.0 + x + 1.0 + + full_S = A_array @ B_array + full_S_T = full_S.transpose([1, 0]) + full_S.astype('float32').tofile("full_S.bin") + + col_to_save = 0 + + for col in range(0, seqlen, Bc): + print(f"tile iteration {col}~{col + Bc} ======================================") + + # FIXME: only work with the first 64 rows of Q for now + Q_tile = A_array[0:64, :] + K_tile = B_array[:, col:col+Bc] + + S = Q_tile @ K_tile + if col == col_to_save: + print('S_expected:') + print(S) + S.astype('float32').tofile("S_expected.bin") + + # generate rowmax result in online softmax + rowmax_this = np.max(S, axis=1) + rowmax_prev = rowmax.copy() + rowmax = np.maximum(rowmax, rowmax_this) + if col == col_to_save: + rowmax.astype('float32').tofile("rowmax.bin") + + # subtrace rowmax from each row by broadcasting + # (placeholder for exp) + x = S - rowmax[:, np.newaxis] + P = exp2(x) + # for i in range(3, 4): + # P += (x**i) / np.math.factorial(i) + # P = np.exp(exp) + # print('P error:') + # print(P / np.exp(x)) + if col == col_to_save: + print('P_expected:') + print(P) + P.astype('float32').tofile("P_expected.bin") + + rowsum_this = np.sum(P, axis=1) + x = rowmax_prev - rowmax_this + rowsum = exp2(x) * rowsum + rowsum_this + if col == col_to_save: + rowsum.astype('float32').tofile("rowsum.bin") + + x = rowmax_prev - rowmax + O = O / (exp2(x)[:, np.newaxis]) + if col == col_to_save: + print('O_before_PV:') + print(O) + O.astype('float32').tofile("O_before_PV.bin") + + V = C_array[col:col+Bc, :] + if col == col_to_save: + V.astype('float32').tofile("V_expected.bin") + # O = P.transpose([1, 0]) @ V + O = O + P @ V + if col == col_to_save: + print('O_after_PV:') + print(O) + O.astype('float32').tofile("O_after_PV.bin") diff --git a/tests/kernel/tensor/generate_matrix.py b/tests/kernel/tensor/generate_matrix.py index 35ad7d73..8626ba43 100644 --- a/tests/kernel/tensor/generate_matrix.py +++ b/tests/kernel/tensor/generate_matrix.py @@ -1,32 +1,108 @@ +import sys import numpy as np -A_array = np.random.rand(16, 8) -B_array = np.random.rand(8, 16) -C_array = np.random.rand(16, 16) -# A_array = np.zeros((16, 8)) -# B_array = np.zeros((8, 16)) -# A_array[0,:] = 1.0 -# B_array[:,4] = 1.0 -# C_array = np.zeros((16, 16)) -# for i in range(16): -# for j in range(16): -# C_array[i,j] = i * 16 + j -with open('a_matrix.h', 'w') as f: - for i in range(A_array.shape[0]): - for j in range(A_array.shape[1]): - f.write(f'{A_array[i,j]}f, ') - f.write('\n') +def parse_mnk(): + if len(sys.argv) != 4: + print(f"usage: {sys.argv[0]} dimM dimN dimK", file=sys.stderr) + sys.exit(1) + m = int(sys.argv[1]) + n = int(sys.argv[2]) + k = int(sys.argv[3]) + return (m, n, k) -with open('b_matrix.h', 'w') as f: - for i in range(B_array.shape[0]): - for j in range(B_array.shape[1]): - f.write(f'{B_array[i,j]}f, ') - f.write('\n') -with open('c_matrix.h', 'w') as f: - for i in range(C_array.shape[0]): - for j in range(C_array.shape[1]): - f.write(f'{C_array[i,j]}f, ') - f.write('\n') +# Reorder array in a way that groups two adjacent elements along the column to +# be now adjacent along the row. This way, when the resulting fp16 array is +# read in column-major order with 32-bit granularity, the fp16 elements will be +# read in the same order as regular fp32 elements in column-major. +# +# For example: +# [[1 2] +# [3 4] +# [5 6] +# [7 8]] +# becomes +# [[1 3 2 4] +# [5 7 6 8]] +def pack_fp16_by_column(array): + rows = array.shape[0] + cols = array.shape[1] + + T = array.transpose([1, 0]) + T_packed = T.reshape([cols, -1, 2]) + result = T_packed.transpose([1, 0, 2]) + return result + + +# Do the same as pack_fp16_by_column, but for every two elements along the row. +def pack_fp16_by_row(array): + rows = array.shape[0] + cols = array.shape[1] + + result = array.reshape([rows, -1, 2]) + return result + + +if __name__ == "__main__": + M, N, K = parse_mnk() + + rand = False + if not rand: + A_array = np.arange(M * K).reshape([M, K]) + B_array = np.arange(K * N).reshape([K, N]) + # C_array = np.arange(M * N).reshape([M, N]) + C_array = np.zeros([M, N]) + else: + np.random.seed(0) + A_array = np.random.rand(M, K) - 0.5 + B_array = np.random.rand(K, N) - 0.5 + C_array = np.random.rand(N, K) - 0.5 + # C_array = np.zeros([M, N]) + + with open('a_matrix.h', 'w') as f: + for i in range(A_array.shape[0]): + for j in range(A_array.shape[1]): + f.write(f'{A_array[i,j]:f}f, ') + f.write('\n') + with open('b_matrix.h', 'w') as f: + for i in range(B_array.shape[0]): + for j in range(B_array.shape[1]): + f.write(f'{B_array[i,j]:f}f, ') + f.write('\n') + with open('c_matrix.h', 'w') as f: + for i in range(C_array.shape[0]): + for j in range(C_array.shape[1]): + f.write(f'{C_array[i,j]:f}f, ') + f.write('\n') + + np.savez("abc", A_array=A_array, B_array=B_array, C_array=C_array) + + fp16 = False + if fp16: + A_packed = pack_fp16_by_row(A_array) + AT_packed = A_packed.transpose([1, 0, 2]) + AT_array = AT_packed.reshape([-1, M * 2]) + AT_array.astype('float16').tofile("input.a.col.bin") + print('AT:') + print(AT_array) + B_packed = pack_fp16_by_column(B_array) + B_array = B_packed.reshape([-1, N * 2]) + B_array.astype('float16').tofile("input.b.row.bin") + print('B:') + print(B_array) + else: + A_array.astype('float32').tofile("input.a.row.bin") + AT_array = A_array.transpose([1, 0]) + AT_array.astype('float32').tofile("input.a.col.bin") + B_array.astype('float32').tofile("input.b.bin") + C_array.astype('float32').tofile("input.c.bin") + print('AT:') + print(AT_array) + print('B:') + print(B_array) + + D_expected = A_array @ B_array + D_expected.astype('float32').tofile("d_expected.bin") + print('D_expected:') + print(D_expected) -np.savez("abc", A_array=A_array, B_array=B_array, C_array=C_array) \ No newline at end of file diff --git a/tests/kernel/tensor/main.cpp b/tests/kernel/tensor/main.cpp index d1deb67c..c373507a 100644 --- a/tests/kernel/tensor/main.cpp +++ b/tests/kernel/tensor/main.cpp @@ -4,9 +4,15 @@ #include #include #include -#include "test_data.h" constexpr int DIM_M = 8; +constexpr int DIM_N = 8; +constexpr int DIM_K = 8; + +// #include "test_data.h" +const float *A = reinterpret_cast(0xa0000000UL); +const float *B = reinterpret_cast(0xa1000000UL); +const float *C = reinterpret_cast(0xa2000000UL); // single "substep" wmma instruction // use accum buffer 0 (f16-f23) @@ -97,48 +103,64 @@ void vx_wmma_load() { map_operand_8lanes(tid, row, col); // load A - // each operand element is read twice by two threadgroups (Sec. III-B); - // i.e. 8 regs * 32 lanes = 256 fp32 elements = 2 * (16 * 8) elements - asm volatile("flw f0, %0" ::"m"(A[row][0])); - asm volatile("flw f1, %0" ::"m"(A[row][1])); - asm volatile("flw f2, %0" ::"m"(A[row][2])); - asm volatile("flw f3, %0" ::"m"(A[row][3])); - asm volatile("flw f4, %0" ::"m"(A[row][4])); - asm volatile("flw f5, %0" ::"m"(A[row][5])); - asm volatile("flw f6, %0" ::"m"(A[row][6])); - asm volatile("flw f7, %0" ::"m"(A[row][7])); + // A is stored K-major in the memory, + // loaded K-major into the RF. + // + // For 32 lanes config, each operand element is read twice by two + // threadgroups (Sec. III-B); i.e. 8 regs * 32 lanes = 256 fp32 elements = 2 + // * (16 * 8) elements + asm volatile("flw f0, %0" ::"m"(A[DIM_K * row + 0])); + asm volatile("flw f1, %0" ::"m"(A[DIM_K * row + 1])); + asm volatile("flw f2, %0" ::"m"(A[DIM_K * row + 2])); + asm volatile("flw f3, %0" ::"m"(A[DIM_K * row + 3])); + asm volatile("flw f4, %0" ::"m"(A[DIM_K * row + 4])); + asm volatile("flw f5, %0" ::"m"(A[DIM_K * row + 5])); + asm volatile("flw f6, %0" ::"m"(A[DIM_K * row + 6])); + asm volatile("flw f7, %0" ::"m"(A[DIM_K * row + 7])); // load B - asm volatile("flw f8 , %0" ::"m"(B[0][col])); - asm volatile("flw f9 , %0" ::"m"(B[1][col])); - asm volatile("flw f10, %0" ::"m"(B[2][col])); - asm volatile("flw f11, %0" ::"m"(B[3][col])); - asm volatile("flw f12, %0" ::"m"(B[4][col])); - asm volatile("flw f13, %0" ::"m"(B[5][col])); - asm volatile("flw f14, %0" ::"m"(B[6][col])); - asm volatile("flw f15, %0" ::"m"(B[7][col])); + // B is stored N-major in the memory, + // loaded K-major into the RF. + asm volatile("flw f8 , %0" ::"m"(B[DIM_N * 0 + col])); + asm volatile("flw f9 , %0" ::"m"(B[DIM_N * 1 + col])); + asm volatile("flw f10, %0" ::"m"(B[DIM_N * 2 + col])); + asm volatile("flw f11, %0" ::"m"(B[DIM_N * 3 + col])); + asm volatile("flw f12, %0" ::"m"(B[DIM_N * 4 + col])); + asm volatile("flw f13, %0" ::"m"(B[DIM_N * 5 + col])); + asm volatile("flw f14, %0" ::"m"(B[DIM_N * 6 + col])); + asm volatile("flw f15, %0" ::"m"(B[DIM_N * 7 + col])); + // B is stored K-major in the memory, + // loaded K-major into the RF. + // asm volatile("flw f8 , %0" ::"m"(B[DIM_K * row + 0])); + // asm volatile("flw f9 , %0" ::"m"(B[DIM_K * row + 1])); + // asm volatile("flw f10, %0" ::"m"(B[DIM_K * row + 2])); + // asm volatile("flw f11, %0" ::"m"(B[DIM_K * row + 3])); + // asm volatile("flw f12, %0" ::"m"(B[DIM_K * row + 4])); + // asm volatile("flw f13, %0" ::"m"(B[DIM_K * row + 5])); + // asm volatile("flw f14, %0" ::"m"(B[DIM_K * row + 6])); + // asm volatile("flw f15, %0" ::"m"(B[DIM_K * row + 7])); map_c_8lanes(tid, row, col); // load C // accum buffer 0 - asm volatile("flw f16, %0" ::"m"(C[row + 0][col + 0])); - asm volatile("flw f17, %0" ::"m"(C[row + 0][col + 1])); - asm volatile("flw f18, %0" ::"m"(C[row + 2][col + 0])); - asm volatile("flw f19, %0" ::"m"(C[row + 2][col + 1])); - asm volatile("flw f20, %0" ::"m"(C[row + 0][col + 4])); - asm volatile("flw f21, %0" ::"m"(C[row + 0][col + 5])); - asm volatile("flw f22, %0" ::"m"(C[row + 2][col + 4])); - asm volatile("flw f23, %0" ::"m"(C[row + 2][col + 5])); + asm volatile("flw f16, %0" ::"m"(C[DIM_N * (row + 0) + col + 0])); + asm volatile("flw f17, %0" ::"m"(C[DIM_N * (row + 0) + col + 1])); + asm volatile("flw f18, %0" ::"m"(C[DIM_N * (row + 2) + col + 0])); + asm volatile("flw f19, %0" ::"m"(C[DIM_N * (row + 2) + col + 1])); + asm volatile("flw f20, %0" ::"m"(C[DIM_N * (row + 0) + col + 4])); + asm volatile("flw f21, %0" ::"m"(C[DIM_N * (row + 0) + col + 5])); + asm volatile("flw f22, %0" ::"m"(C[DIM_N * (row + 2) + col + 4])); + asm volatile("flw f23, %0" ::"m"(C[DIM_N * (row + 2) + col + 5])); // accum buffer 1 - asm volatile("flw f24, %0" ::"m"(C[row + 0][col + 0])); - asm volatile("flw f25, %0" ::"m"(C[row + 0][col + 1])); - asm volatile("flw f26, %0" ::"m"(C[row + 2][col + 0])); - asm volatile("flw f27, %0" ::"m"(C[row + 2][col + 1])); - asm volatile("flw f28, %0" ::"m"(C[row + 0][col + 4])); - asm volatile("flw f29, %0" ::"m"(C[row + 0][col + 5])); - asm volatile("flw f30, %0" ::"m"(C[row + 2][col + 4])); - asm volatile("flw f31, %0" ::"m"(C[row + 2][col + 5])); + asm volatile("flw f24, %0" ::"m"(C[DIM_N * (row + 0) + col + 0])); + asm volatile("flw f25, %0" ::"m"(C[DIM_N * (row + 0) + col + 1])); + asm volatile("flw f26, %0" ::"m"(C[DIM_N * (row + 2) + col + 0])); + asm volatile("flw f27, %0" ::"m"(C[DIM_N * (row + 2) + col + 1])); + asm volatile("flw f28, %0" ::"m"(C[DIM_N * (row + 0) + col + 4])); + asm volatile("flw f29, %0" ::"m"(C[DIM_N * (row + 0) + col + 5])); + asm volatile("flw f30, %0" ::"m"(C[DIM_N * (row + 2) + col + 4])); + asm volatile("flw f31, %0" ::"m"(C[DIM_N * (row + 2) + col + 5])); } // hardcoded device address for result @@ -155,24 +177,24 @@ void store_wmma_result() { map_c_8lanes(tid, row, col); // store C - float *const results_wid = results + (DIM_M * DIM_M * wid); + float *const results_wid = results + (DIM_M * DIM_N * wid); // uncomment to have two accum buffers in rf - // asm volatile("fsw f16, %0" ::"m"(results_wid[DIM_M * (row + 0) + (col + 0)])); - // asm volatile("fsw f17, %0" ::"m"(results_wid[DIM_M * (row + 0) + (col + 1)])); - // asm volatile("fsw f18, %0" ::"m"(results_wid[DIM_M * (row + 2) + (col + 0)])); - // asm volatile("fsw f19, %0" ::"m"(results_wid[DIM_M * (row + 2) + (col + 1)])); - // asm volatile("fsw f20, %0" ::"m"(results_wid[DIM_M * (row + 0) + (col + 4)])); - // asm volatile("fsw f21, %0" ::"m"(results_wid[DIM_M * (row + 0) + (col + 5)])); - // asm volatile("fsw f22, %0" ::"m"(results_wid[DIM_M * (row + 2) + (col + 4)])); - // asm volatile("fsw f23, %0" ::"m"(results_wid[DIM_M * (row + 2) + (col + 5)])); - asm volatile("fsw f24, %0" ::"m"(results_wid[DIM_M * (row + 0) + (col + 0)])); - asm volatile("fsw f25, %0" ::"m"(results_wid[DIM_M * (row + 0) + (col + 1)])); - asm volatile("fsw f26, %0" ::"m"(results_wid[DIM_M * (row + 2) + (col + 0)])); - asm volatile("fsw f27, %0" ::"m"(results_wid[DIM_M * (row + 2) + (col + 1)])); - asm volatile("fsw f28, %0" ::"m"(results_wid[DIM_M * (row + 0) + (col + 4)])); - asm volatile("fsw f29, %0" ::"m"(results_wid[DIM_M * (row + 0) + (col + 5)])); - asm volatile("fsw f30, %0" ::"m"(results_wid[DIM_M * (row + 2) + (col + 4)])); - asm volatile("fsw f31, %0" ::"m"(results_wid[DIM_M * (row + 2) + (col + 5)])); + // asm volatile("fsw f16, %0" ::"m"(results_wid[DIM_N * (row + 0) + (col + 0)])); + // asm volatile("fsw f17, %0" ::"m"(results_wid[DIM_N * (row + 0) + (col + 1)])); + // asm volatile("fsw f18, %0" ::"m"(results_wid[DIM_N * (row + 2) + (col + 0)])); + // asm volatile("fsw f19, %0" ::"m"(results_wid[DIM_N * (row + 2) + (col + 1)])); + // asm volatile("fsw f20, %0" ::"m"(results_wid[DIM_N * (row + 0) + (col + 4)])); + // asm volatile("fsw f21, %0" ::"m"(results_wid[DIM_N * (row + 0) + (col + 5)])); + // asm volatile("fsw f22, %0" ::"m"(results_wid[DIM_N * (row + 2) + (col + 4)])); + // asm volatile("fsw f23, %0" ::"m"(results_wid[DIM_N * (row + 2) + (col + 5)])); + asm volatile("fsw f24, %0" ::"m"(results_wid[DIM_N * (row + 0) + (col + 0)])); + asm volatile("fsw f25, %0" ::"m"(results_wid[DIM_N * (row + 0) + (col + 1)])); + asm volatile("fsw f26, %0" ::"m"(results_wid[DIM_N * (row + 2) + (col + 0)])); + asm volatile("fsw f27, %0" ::"m"(results_wid[DIM_N * (row + 2) + (col + 1)])); + asm volatile("fsw f28, %0" ::"m"(results_wid[DIM_N * (row + 0) + (col + 4)])); + asm volatile("fsw f29, %0" ::"m"(results_wid[DIM_N * (row + 0) + (col + 5)])); + asm volatile("fsw f30, %0" ::"m"(results_wid[DIM_N * (row + 2) + (col + 4)])); + asm volatile("fsw f31, %0" ::"m"(results_wid[DIM_N * (row + 2) + (col + 5)])); } void print_wmma_result() { @@ -211,9 +233,8 @@ int main() { const int num_warps = vx_num_warps(); // vx_wspawn(num_warps, wmma); - vx_wspawn(1, wmma); wmma(); - vx_wspawn_wait(); + // vx_wspawn_wait(); return 0; } diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 6c55d435..50efc499 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -49,7 +49,10 @@ VX_CP = $(LLVM_VORTEX)/bin/llvm-objcopy #VX_CP = $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy VX_CFLAGS += -v -O3 -std=c++17 -VX_CFLAGS += -mcmodel=medany -fno-rtti -fno-exceptions -nostartfiles -fdata-sections -ffunction-sections -mllvm -inline-threshold=8192 +VX_CFLAGS += -mcmodel=medany -fno-rtti -fno-exceptions -nostartfiles -fdata-sections -ffunction-sections +# comment out below for regression/basic, which uses GCC that doesn't +# understand these flags +VX_CFLAGS += -mllvm -inline-threshold=262144 VX_CFLAGS += -I$(VORTEX_KN_PATH)/include -I$(VORTEX_KN_PATH)/../hw -I$(GEMMINI_SW_PATH) VX_CFLAGS += -DNDEBUG -DLLVM_VORTEX @@ -104,23 +107,28 @@ kernel.bin: kernel.elf kernel.radiance.elf OBJCOPY ?= $(RISCV_TOOLCHAIN_PATH)/bin/$(RISCV_PREFIX)-objcopy OBJCOPY_FLAGS ?= "LOAD,ALLOC,DATA,CONTENTS" -kernel.elf: $(VX_SRCS) - $(VX_CXX) $(VX_CFLAGS) $(VX_SRCS) $(VX_LDFLAGS) -o $@ +BINFILES := args.bin input.a.bin input.b.bin input.c.bin +kernel.elf: $(VX_SRCS) $(VX_INCLUDES) $(BINFILES) + $(VX_CXX) $(VX_CFLAGS) -o $@ $(VX_SRCS) $(VX_LDFLAGS) $(OBJCOPY) --set-section-flags .operand.a=$(OBJCOPY_FLAGS) $@ $(OBJCOPY) --set-section-flags .operand.b=$(OBJCOPY_FLAGS) $@ + $(OBJCOPY) --set-section-flags .operand.c=$(OBJCOPY_FLAGS) $@ $(OBJCOPY) --set-section-flags .args=$(OBJCOPY_FLAGS) $@ - $(OBJCOPY) --update-section .operand.a=input.a.bin $@ - $(OBJCOPY) --update-section .operand.b=input.b.bin $@ - $(OBJCOPY) --update-section .args=args.bin $@ + $(OBJCOPY) --update-section .operand.a=input.a.bin $@ || true + $(OBJCOPY) --update-section .operand.b=input.b.bin $@ || true + $(OBJCOPY) --update-section .operand.c=input.c.bin $@ || true + $(OBJCOPY) --update-section .args=args.bin $@ || true -kernel.radiance.elf: $(VX_SRCS) +kernel.radiance.elf: $(VX_SRCS) $(VX_INCLUDES) $(BINFILES) $(VX_CXX) $(VX_CFLAGS) $(VX_SRCS) $(VX_LDFLAGS) -DRADIANCE -o $@ $(OBJCOPY) --set-section-flags .operand.a=$(OBJCOPY_FLAGS) $@ $(OBJCOPY) --set-section-flags .operand.b=$(OBJCOPY_FLAGS) $@ + $(OBJCOPY) --set-section-flags .operand.c=$(OBJCOPY_FLAGS) $@ $(OBJCOPY) --set-section-flags .args=$(OBJCOPY_FLAGS) $@ - $(OBJCOPY) --update-section .operand.a=input.a.bin $@ - $(OBJCOPY) --update-section .operand.b=input.b.bin $@ - $(OBJCOPY) --update-section .args=args.bin $@ + $(OBJCOPY) --update-section .operand.a=input.a.bin $@ || true + $(OBJCOPY) --update-section .operand.b=input.b.bin $@ || true + $(OBJCOPY) --update-section .operand.c=input.c.bin $@ || true + $(OBJCOPY) --update-section .args=args.bin $@ || true ifneq ($(CONFIG),) kernel$(CONFIGEXT).elf: kernel.elf diff --git a/tests/regression/flash_attention/.gitignore b/tests/regression/flash_attention/.gitignore new file mode 100644 index 00000000..84cd4dff --- /dev/null +++ b/tests/regression/flash_attention/.gitignore @@ -0,0 +1 @@ +flash_attention diff --git a/tests/regression/flash_attention/Makefile b/tests/regression/flash_attention/Makefile new file mode 100644 index 00000000..4f49f927 --- /dev/null +++ b/tests/regression/flash_attention/Makefile @@ -0,0 +1,12 @@ +PROJECT = flash_attention + +SRCS = main.cpp common.h + +VX_SRCS = kernel.cpp +VX_INCLUDES = ../sgemm_tcore/sgemm_impl.hpp + +OPTS ?= -n16 + +VX_CFLAGS += -I../sgemm_tcore + +include ../common.mk diff --git a/tests/regression/flash_attention/common.h b/tests/regression/flash_attention/common.h new file mode 100644 index 00000000..9c09726f --- /dev/null +++ b/tests/regression/flash_attention/common.h @@ -0,0 +1,18 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define KERNEL_ARG_DEV_MEM_ADDR 0x9fff0000 +#define DEV_SMEM_START_ADDR 0xff000000 + +typedef struct { + uint32_t dim_seqlen; + uint32_t dim_headdim; + uint64_t addr_q; + uint64_t addr_k; + uint64_t addr_v; + uint64_t addr_o; +} kernel_arg_t; + +#endif diff --git a/tests/regression/flash_attention/half.hpp b/tests/regression/flash_attention/half.hpp new file mode 100644 index 00000000..debe094f --- /dev/null +++ b/tests/regression/flash_attention/half.hpp @@ -0,0 +1,4018 @@ +// half - IEEE 754-based half-precision floating-point library. +// +// Copyright (c) 2012-2019 Christian Rau +// Copyright (c) 2020 0xBYTESHIFT +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, +// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +/// \file +/// Main header file for half-precision functionality. + +#pragma once + +#define HALF_TWOS_COMPLEMENT_INT 1 + +// any error throwing C++ exceptions? +#if defined(HALF_ERRHANDLING_THROW_INVALID) || defined(HALF_ERRHANDLING_THROW_DIVBYZERO) || defined(HALF_ERRHANDLING_THROW_OVERFLOW) || defined(HALF_ERRHANDLING_THROW_UNDERFLOW) || defined(HALF_ERRHANDLING_THROW_INEXACT) +#define HALF_ERRHANDLING_THROWS 1 +#endif + +// any error handling enabled? +#define HALF_ERRHANDLING (HALF_ERRHANDLING_FLAGS||HALF_ERRHANDLING_ERRNO||HALF_ERRHANDLING_FENV||HALF_ERRHANDLING_THROWS) + +#if HALF_ERRHANDLING + #define HALF_UNUSED_NOERR(name) name +#else + #define HALF_UNUSED_NOERR(name) +#endif + +// support constexpr +#if HALF_ERRHANDLING + #define constexpr_NOERR +#else + #define constexpr_NOERR constexpr +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HALF_ERRHANDLING_ERRNO + #include +#endif +#include +#include + +#ifndef HALF_ENABLE_F16C_INTRINSICS + /// Enable F16C intruction set intrinsics. + /// Defining this to 1 enables the use of [F16C compiler intrinsics](https://en.wikipedia.org/wiki/F16C) for converting between + /// half-precision and single-precision values which may result in improved performance. This will not perform additional checks + /// for support of the F16C instruction set, so an appropriate target platform is required when enabling this feature. + /// + /// Unless predefined it will be enabled automatically when the `__F16C__` symbol is defined, which some compilers do on supporting platforms. + #define HALF_ENABLE_F16C_INTRINSICS __F16C__ +#endif + +#if HALF_ENABLE_F16C_INTRINSICS + #include +#endif + +#ifndef HALF_ERRHANDLING_OVERFLOW_TO_INEXACT +/// Raise INEXACT exception on overflow. +/// Defining this to 1 (default) causes overflow errors to automatically raise inexact exceptions in addition. +/// These will be raised after any possible handling of the underflow exception. +#define HALF_ERRHANDLING_OVERFLOW_TO_INEXACT 1 +#endif + +#ifndef HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT +/// Raise INEXACT exception on underflow. +/// Defining this to 1 (default) causes underflow errors to automatically raise inexact exceptions in addition. +/// These will be raised after any possible handling of the underflow exception. +/// +/// **Note:** This will actually cause underflow (and the accompanying inexact) exceptions to be raised *only* when the result +/// is inexact, while if disabled bare underflow errors will be raised for *any* (possibly exact) subnormal result. +#define HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT 1 +#endif + +/// Default rounding mode. +/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and more precise types +/// (unless using half_cast() and specifying the rounding mode directly) as well as in arithmetic operations and mathematical +/// functions. It can be redefined (before including half.hpp) to one of the standard rounding modes using their respective +/// constants or the equivalent values of +/// [std::float_round_style](https://en.cppreference.com/w/cpp/types/numeric_limits/float_round_style): +/// +/// `std::float_round_style` | value | rounding +/// ---------------------------------|-------|------------------------- +/// `std::round_indeterminate` | -1 | fastest +/// `std::round_toward_zero` | 0 | toward zero +/// `std::round_to_nearest` | 1 | to nearest (default) +/// `std::round_toward_infinity` | 2 | toward positive infinity +/// `std::round_toward_neg_infinity` | 3 | toward negative infinity +/// +/// By default this is set to `1` (`std::round_to_nearest`), which rounds results to the nearest representable value. It can even +/// be set to [std::numeric_limits::round_style](https://en.cppreference.com/w/cpp/types/numeric_limits/round_style) to synchronize +/// the rounding mode with that of the built-in single-precision implementation (which is likely `std::round_to_nearest`, though). +#ifndef HALF_ROUND_STYLE + #define HALF_ROUND_STYLE 1 // = std::round_to_nearest +#endif + +/// Value signaling overflow. +/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to a positive value signaling the overflow of an +/// operation, in particular it just evaluates to positive infinity. +/// +/// **See also:** Documentation for [HUGE_VAL](https://en.cppreference.com/w/cpp/numeric/math/HUGE_VAL) +#define HUGE_VALH std::numeric_limits::infinity() + +/// Fast half-precision fma function. +/// This symbol is defined if the fma() function generally executes as fast as, or faster than, a separate +/// half-precision multiplication followed by an addition, which is always the case. +/// +/// **See also:** Documentation for [FP_FAST_FMA](https://en.cppreference.com/w/cpp/numeric/math/fma) +#define FP_FAST_FMAH 1 + +/// Half rounding mode. +/// In correspondence with `FLT_ROUNDS` from `` this symbol expands to the rounding mode used for +/// half-precision operations. It is an alias for [HALF_ROUND_STYLE](\ref HALF_ROUND_STYLE). +/// +/// **See also:** Documentation for [FLT_ROUNDS](https://en.cppreference.com/w/cpp/types/climits/FLT_ROUNDS) +#define HLF_ROUNDS HALF_ROUND_STYLE + +#ifndef FP_ILOGB0 + #define FP_ILOGB0 INT_MIN +#endif +#ifndef FP_ILOGBNAN + #define FP_ILOGBNAN INT_MAX +#endif +#ifndef FP_SUBNORMAL + #define FP_SUBNORMAL 0 +#endif +#ifndef FP_ZERO + #define FP_ZERO 1 +#endif +#ifndef FP_NAN + #define FP_NAN 2 +#endif +#ifndef FP_INFINITE + #define FP_INFINITE 3 +#endif +#ifndef FP_NORMAL + #define FP_NORMAL 4 +#endif + +#if !defined(FE_ALL_EXCEPT) + #define FE_INVALID 0x10 + #define FE_DIVBYZERO 0x08 + #define FE_OVERFLOW 0x04 + #define FE_UNDERFLOW 0x02 + #define FE_INEXACT 0x01 + #define FE_ALL_EXCEPT (FE_INVALID|FE_DIVBYZERO|FE_OVERFLOW|FE_UNDERFLOW|FE_INEXACT) +#endif + + +/// Main namespace for half-precision functionality. +/// This namespace contains all the functionality provided by the library. +namespace half_float { + class half; + + /// Library-defined half-precision literals. + /// Import this namespace to enable half-precision floating-point literals: + /// ~~~~{.cpp} + /// using namespace half_float::literal; + /// half_float::half = 4.2_h; + /// ~~~~ + namespace literal { + half operator "" _h(long double); + } + + /// \internal + /// \brief Implementation details. + namespace detail { + /// Conditional type. + template struct conditional : std::conditional {}; + + /// Helper for tag dispatching. + template struct bool_type : std::integral_constant {}; + using std::true_type; + using std::false_type; + + /// Type traits for floating-point types. + template struct is_float : std::is_floating_point {}; + + /// Type traits for floating-point bits. + template struct bits { using type = unsigned char; }; + template struct bits : bits {}; + template struct bits : bits {}; + template struct bits : bits {}; + + /// Unsigned integer of (at least) 16 bits width. + using uint16 = std::uint_least16_t; + + /// Fastest unsigned integer of (at least) 32 bits width. + using uint32 = std::uint_fast32_t; + + /// Fastest signed integer of (at least) 32 bits width. + using int32 = std::int_fast32_t; + + /// Unsigned integer of (at least) 32 bits width. + template<> struct bits { using type = std::uint_least32_t; }; + + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits { using type = std::uint_least64_t; }; + template using bits_t = typename bits::type; + + #ifdef HALF_ARITHMETIC_TYPE + /// Type to use for arithmetic computations and mathematic functions internally. + typedef HALF_ARITHMETIC_TYPE internal_t; + #endif + + /// Tag type for binary construction. + struct binary_t {}; + + /// Tag for binary construction. + constexpr binary_t binary = binary_t(); + + /// \name Implementation defined classification and arithmetic + /// \{ + + /// Check for infinity. + /// \tparam T argument type (builtin floating-point type) + /// \param arg value to query + /// \retval true if infinity + /// \retval false else + template bool builtin_isinf(T arg) { return std::isinf(arg); } + + /// Check for NaN. + /// \tparam T argument type (builtin floating-point type) + /// \param arg value to query + /// \retval true if not a number + /// \retval false else + template bool builtin_isnan(T arg) { return std::isnan(arg); } + + /// Check sign. + /// \tparam T argument type (builtin floating-point type) + /// \param arg value to query + /// \retval true if signbit set + /// \retval false else + template bool builtin_signbit(T arg) { return std::signbit(arg); } + + /// Platform-independent sign mask. + /// \param arg integer value in two's complement + /// \retval -1 if \a arg negative + /// \retval 0 if \a arg positive + inline uint32 sign_mask(uint32 arg) { + static const int N = std::numeric_limits::digits - 1; + #if HALF_TWOS_COMPLEMENT_INT + return static_cast(arg) >> N; + #else + return -((arg>>N)&1); + #endif + } + + /// Platform-independent arithmetic right shift. + /// \param arg integer value in two's complement + /// \param i shift amount (at most 31) + /// \return \a arg right shifted for \a i bits with possible sign extension + inline uint32 arithmetic_shift(uint32 arg, int i) { + #if HALF_TWOS_COMPLEMENT_INT + return static_cast(arg) >> i; + #else + return static_cast(arg)/(static_cast(1)<>(std::numeric_limits::digits-1))&1); + #endif + } + + /// \} + /// \name Error handling + /// \{ + + /// Internal exception flags. + /// \return reference to global exception flags + inline int& errflags() { thread_local int flags = 0; return flags; } + + /// Raise floating-point exception. + /// \param flags exceptions to raise + /// \param cond condition to raise exceptions for + inline void raise(int HALF_UNUSED_NOERR(flags), bool HALF_UNUSED_NOERR(cond) = true) { + #if HALF_ERRHANDLING + if(!cond) + return; + #if HALF_ERRHANDLING_FLAGS + errflags() |= flags; + #endif + #if HALF_ERRHANDLING_ERRNO + if(flags & FE_INVALID) + errno = EDOM; + else if(flags & (FE_DIVBYZERO|FE_OVERFLOW|FE_UNDERFLOW)) + errno = ERANGE; + #endif + #if HALF_ERRHANDLING_FENV + std::feraiseexcept(flags); + #endif + #ifdef HALF_ERRHANDLING_THROW_INVALID + if(flags & FE_INVALID) + throw std::domain_error(HALF_ERRHANDLING_THROW_INVALID); + #endif + #ifdef HALF_ERRHANDLING_THROW_DIVBYZERO + if(flags & FE_DIVBYZERO) + throw std::domain_error(HALF_ERRHANDLING_THROW_DIVBYZERO); + #endif + #ifdef HALF_ERRHANDLING_THROW_OVERFLOW + if(flags & FE_OVERFLOW) + throw std::overflow_error(HALF_ERRHANDLING_THROW_OVERFLOW); + #endif + #ifdef HALF_ERRHANDLING_THROW_UNDERFLOW + if(flags & FE_UNDERFLOW) + throw std::underflow_error(HALF_ERRHANDLING_THROW_UNDERFLOW); + #endif + #ifdef HALF_ERRHANDLING_THROW_INEXACT + if(flags & FE_INEXACT) + throw std::range_error(HALF_ERRHANDLING_THROW_INEXACT); + #endif + #if HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT + if((flags & FE_UNDERFLOW) && !(flags & FE_INEXACT)) + raise(FE_INEXACT); + #endif + #if HALF_ERRHANDLING_OVERFLOW_TO_INEXACT + if((flags & FE_OVERFLOW) && !(flags & FE_INEXACT)) + raise(FE_INEXACT); + #endif + #endif + } + + /// Check and signal for any NaN. + /// \param x first half-precision value to check + /// \param y second half-precision value to check + /// \retval true if either \a x or \a y is NaN + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline constexpr_NOERR bool compsignal(unsigned int x, unsigned int y) { + #if HALF_ERRHANDLING + raise(FE_INVALID, (x&0x7FFF)>0x7C00 || (y&0x7FFF)>0x7C00); + #endif + return (x&0x7FFF) > 0x7C00 || (y&0x7FFF) > 0x7C00; + } + + /// Signal and silence signaling NaN. + /// \param nan half-precision NaN value + /// \return quiet NaN + /// \exception FE_INVALID if \a nan is signaling NaN + inline constexpr_NOERR unsigned int signal(unsigned int nan) { + #if HALF_ERRHANDLING + raise(FE_INVALID, !(nan&0x200)); + #endif + return nan | 0x200; + } + + /// Signal and silence signaling NaNs. + /// \param x first half-precision value to check + /// \param y second half-precision value to check + /// \return quiet NaN + /// \exception FE_INVALID if \a x or \a y is signaling NaN + inline constexpr_NOERR unsigned int signal(unsigned int x, unsigned int y) { + #if HALF_ERRHANDLING + raise(FE_INVALID, ((x&0x7FFF)>0x7C00 && !(x&0x200)) || ((y&0x7FFF)>0x7C00 && !(y&0x200))); + #endif + return ((x&0x7FFF)>0x7C00) ? (x|0x200) : (y|0x200); + } + + /// Signal and silence signaling NaNs. + /// \param x first half-precision value to check + /// \param y second half-precision value to check + /// \param z third half-precision value to check + /// \return quiet NaN + /// \exception FE_INVALID if \a x, \a y or \a z is signaling NaN + inline constexpr_NOERR unsigned int signal(unsigned int x, unsigned int y, unsigned int z) { + #if HALF_ERRHANDLING + raise(FE_INVALID, ((x&0x7FFF)>0x7C00 && !(x&0x200)) || ((y&0x7FFF)>0x7C00 && !(y&0x200)) || ((z&0x7FFF)>0x7C00 && !(z&0x200))); + #endif + return ((x&0x7FFF)>0x7C00) ? (x|0x200) : ((y&0x7FFF)>0x7C00) ? (y|0x200) : (z|0x200); + } + + /// Select value or signaling NaN. + /// \param x preferred half-precision value + /// \param y ignored half-precision value except for signaling NaN + /// \return \a y if signaling NaN, \a x otherwise + /// \exception FE_INVALID if \a y is signaling NaN + inline constexpr_NOERR unsigned int select(unsigned int x, unsigned int HALF_UNUSED_NOERR(y)) { + #if HALF_ERRHANDLING + return (((y&0x7FFF)>0x7C00) && !(y&0x200)) ? signal(y) : x; + #else + return x; + #endif + } + + /// Raise domain error and return NaN. + /// return quiet NaN + /// \exception FE_INVALID + inline constexpr_NOERR unsigned int invalid() { + #if HALF_ERRHANDLING + raise(FE_INVALID); + #endif + return 0x7FFF; + } + + /// Raise pole error and return infinity. + /// \param sign half-precision value with sign bit only + /// \return half-precision infinity with sign of \a sign + /// \exception FE_DIVBYZERO + inline constexpr_NOERR unsigned int pole(unsigned int sign = 0) { + #if HALF_ERRHANDLING + raise(FE_DIVBYZERO); + #endif + return sign | 0x7C00; + } + + /// Check value for underflow. + /// \param arg non-zero half-precision value to check + /// \return \a arg + /// \exception FE_UNDERFLOW if arg is subnormal + inline constexpr_NOERR unsigned int check_underflow(unsigned int arg) { + #if HALF_ERRHANDLING && !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT + raise(FE_UNDERFLOW, !(arg&0x7C00)); + #endif + return arg; + } + + /// \} + /// \name Conversion and rounding + /// \{ + + /// Half-precision overflow. + /// \tparam R rounding mode to use + /// \param sign half-precision value with sign bit only + /// \return rounded overflowing half-precision value + /// \exception FE_OVERFLOW + template constexpr_NOERR unsigned int overflow(unsigned int sign = 0) { + #if HALF_ERRHANDLING + raise(FE_OVERFLOW); + #endif + return (R==std::round_toward_infinity) ? (sign+0x7C00-(sign>>15)) : + (R==std::round_toward_neg_infinity) ? (sign+0x7BFF+(sign>>15)) : + (R==std::round_toward_zero) ? (sign|0x7BFF) : + (sign|0x7C00); + } + + /// Half-precision underflow. + /// \tparam R rounding mode to use + /// \param sign half-precision value with sign bit only + /// \return rounded underflowing half-precision value + /// \exception FE_UNDERFLOW + template constexpr_NOERR unsigned int underflow(unsigned int sign = 0) { + #if HALF_ERRHANDLING + raise(FE_UNDERFLOW); + #endif + return (R==std::round_toward_infinity) ? (sign+1-(sign>>15)) : + (R==std::round_toward_neg_infinity) ? (sign+(sign>>15)) : + sign; + } + + /// Round half-precision number. + /// \tparam R rounding mode to use + /// \tparam I `true` to always raise INEXACT exception, `false` to raise only for rounded results + /// \param value finite half-precision number to round + /// \param g guard bit (most significant discarded bit) + /// \param s sticky bit (or of all but the most significant discarded bits) + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded or \a I is `true` + template constexpr_NOERR unsigned int rounded(unsigned int value, int g, int s) { + #if HALF_ERRHANDLING + value += (R==std::round_to_nearest) ? (g&(s|value)) : + (R==std::round_toward_infinity) ? (~(value>>15)&(g|s)) : + (R==std::round_toward_neg_infinity) ? ((value>>15)&(g|s)) : 0; + if((value&0x7C00) == 0x7C00) + raise(FE_OVERFLOW); + else if(value & 0x7C00) + raise(FE_INEXACT, I || (g|s)!=0); + else + raise(FE_UNDERFLOW, !(HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT) || I || (g|s)!=0); + return value; + #else + return (R==std::round_to_nearest) ? (value+(g&(s|value))) : + (R==std::round_toward_infinity) ? (value+(~(value>>15)&(g|s))) : + (R==std::round_toward_neg_infinity) ? (value+((value>>15)&(g|s))) : + value; + #endif + } + + /// Round half-precision number to nearest integer value. + /// \tparam R rounding mode to use + /// \tparam E `true` for round to even, `false` for round away from zero + /// \tparam I `true` to raise INEXACT exception (if inexact), `false` to never raise it + /// \param value half-precision value to round + /// \return half-precision bits for nearest integral value + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded and \a I is `true` + template unsigned int integral(unsigned int value) { + unsigned int abs = value & 0x7FFF; + if(abs < 0x3C00) { + raise(FE_INEXACT, I); + return ((R==std::round_to_nearest) ? (0x3C00&-static_cast(abs>=(0x3800+E))) : + (R==std::round_toward_infinity) ? (0x3C00&-(~(value>>15)&(abs!=0))) : + (R==std::round_toward_neg_infinity) ? (0x3C00&-static_cast(value>0x8000)) : + 0) | (value&0x8000); + } + if(abs >= 0x6400) + return (abs>0x7C00) ? signal(value) : value; + unsigned int exp = 25 - (abs>>10), mask = (1<>exp)&E)) : + (R==std::round_toward_infinity) ? (mask&((value>>15)-1)) : + (R==std::round_toward_neg_infinity) ? (mask&-(value>>15)) : + 0) + value) & ~mask; + } + + /// Convert fixed point to half-precision floating-point. + /// \tparam R rounding mode to use + /// \tparam F number of fractional bits (at least 11) + /// \tparam S `true` for signed, `false` for unsigned + /// \tparam N `true` for additional normalization step, `false` if already normalized to 1.F + /// \tparam I `true` to always raise INEXACT exception, `false` to raise only for rounded results + /// \param m mantissa in Q1.F fixed point format + /// \param exp exponent + /// \param sign half-precision value with sign bit only + /// \param s sticky bit (or of all but the most significant already discarded bits) + /// \return value converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded or \a I is `true` + template unsigned int fixed2half(uint32 m, int exp = 14, unsigned int sign = 0, int s = 0) { + if(S) { + uint32 msign = sign_mask(m); + m = (m^msign) - msign; + sign = msign & 0x8000; + } + if(N) + for(; m<(static_cast(1)<(sign+(m>>(F-10-exp)), (m>>(F-11-exp))&1, s|((m&((static_cast(1)<<(F-11-exp))-1))!=0)); + return rounded(sign+(exp<<10)+(m>>(F-10)), (m>>(F-11))&1, s|((m&((static_cast(1)<<(F-11))-1))!=0)); + } + + /// Convert IEEE single-precision to half-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \tparam R rounding mode to use + /// \param value single-precision value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half_impl(float value, true_type) { + #if HALF_ENABLE_F16C_INTRINSICS + return _mm_cvtsi128_si32(_mm_cvtps_ph(_mm_set_ss(value), + (R==std::round_to_nearest) ? _MM_FROUND_TO_NEAREST_INT : + (R==std::round_toward_zero) ? _MM_FROUND_TO_ZERO : + (R==std::round_toward_infinity) ? _MM_FROUND_TO_POS_INF : + (R==std::round_toward_neg_infinity) ? _MM_FROUND_TO_NEG_INF : + _MM_FROUND_CUR_DIRECTION)); + #else + bits_t fbits; + std::memcpy(&fbits, &value, sizeof(float)); + #if 1 + unsigned int sign = (fbits>>16) & 0x8000; + fbits &= 0x7FFFFFFF; + if(fbits >= 0x7F800000) + return sign | 0x7C00 | ((fbits>0x7F800000) ? (0x200|((fbits>>13)&0x3FF)) : 0); + if(fbits >= 0x47800000) + return overflow(sign); + if(fbits >= 0x38800000) + return rounded(sign|(((fbits>>23)-112)<<10)|((fbits>>13)&0x3FF), (fbits>>12)&1, (fbits&0xFFF)!=0); + if(fbits >= 0x33000000) + { + int i = 125 - (fbits>>23); + fbits = (fbits&0x7FFFFF) | 0x800000; + return rounded(sign|(fbits>>(i+1)), (fbits>>i)&1, (fbits&((static_cast(1)<(sign); + return sign; + #else + static const uint16 base_table[512] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, + 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, + 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7C00, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, + 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, + 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFC00 }; + static const unsigned char shift_table[256] = { + 24, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 }; + int sexp = fbits >> 23, exp = sexp & 0xFF, i = shift_table[exp]; + fbits &= 0x7FFFFF; + uint32 m = (fbits|((exp!=0)<<23)) & -static_cast(exp!=0xFF); + return rounded(base_table[sexp]+(fbits>>i), (m>>(i-1))&1, (((static_cast(1)<<(i-1))-1)&m)!=0); + #endif + #endif + } + + /// Convert IEEE double-precision to half-precision. + /// \tparam R rounding mode to use + /// \param value double-precision value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half_impl(double value, true_type) { + #if HALF_ENABLE_F16C_INTRINSICS + if(R == std::round_indeterminate) + return _mm_cvtsi128_si32(_mm_cvtps_ph(_mm_cvtpd_ps(_mm_set_sd(value)), _MM_FROUND_CUR_DIRECTION)); + #endif + bits_t dbits; + std::memcpy(&dbits, &value, sizeof(double)); + uint32 hi = dbits >> 32, lo = dbits & 0xFFFFFFFF; + unsigned int sign = (hi>>16) & 0x8000; + hi &= 0x7FFFFFFF; + if(hi >= 0x7FF00000) + return sign | 0x7C00 | ((dbits&0xFFFFFFFFFFFFF) ? (0x200|((hi>>10)&0x3FF)) : 0); + if(hi >= 0x40F00000) + return overflow(sign); + if(hi >= 0x3F100000) + return rounded(sign|(((hi>>20)-1008)<<10)|((hi>>10)&0x3FF), (hi>>9)&1, ((hi&0x1FF)|lo)!=0); + if(hi >= 0x3E600000) { + int i = 1018 - (hi>>20); + hi = (hi&0xFFFFF) | 0x100000; + return rounded(sign|(hi>>(i+1)), (hi>>i)&1, ((hi&((static_cast(1)<(sign); + return sign; + } + + /// Convert non-IEEE floating-point to half-precision. + /// \tparam R rounding mode to use + /// \tparam T source type (builtin floating-point type) + /// \param value floating-point value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half_impl(T value, ...) { + unsigned int hbits = static_cast(builtin_signbit(value)) << 15; + if(value == T()) + return hbits; + if(builtin_isnan(value)) + return hbits | 0x7FFF; + if(builtin_isinf(value)) + return hbits | 0x7C00; + int exp; + std::frexp(value, &exp); + if(exp > 16) + return overflow(hbits); + if(exp < -13) + value = std::ldexp(value, 25); + else { + value = std::ldexp(value, 12-exp); + hbits |= ((exp+13)<<10); + } + T ival, frac = std::modf(value, &ival); + int m = std::abs(static_cast(ival)); + return rounded(hbits+(m>>1), m&1, frac!=T()); + } + + /// Convert floating-point to half-precision. + /// \tparam R rounding mode to use + /// \tparam T source type (builtin floating-point type) + /// \param value floating-point value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half(T value) { + return float2half_impl(value, bool_type::is_iec559&&sizeof(bits_t)==sizeof(T)>()); + } + template unsigned int float2half(T value) { + return float2half_impl<(std::float_round_style)(HALF_ROUND_STYLE)>(value, bool_type::is_iec559&&sizeof(bits_t)==sizeof(T)>()); + } + + /// Convert integer to half-precision floating-point. + /// \tparam R rounding mode to use + /// \tparam T type to convert (builtin integer type) + /// \param value integral value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int int2half(T value) { + unsigned int bits = static_cast(value<0) << 15; + if(!value) + return bits; + if(bits) + value = -value; + if(value > 0xFFFF) + return overflow(bits); + unsigned int m = static_cast(value), exp = 24; + for(; m<0x400; m<<=1,--exp) ; + for(; m>0x7FF; m>>=1,++exp) ; + bits |= (exp<<10) + m; + return (exp>24) ? rounded(bits, (value>>(exp-25))&1, (((1<<(exp-25))-1)&value)!=0) : bits; + } + + /// Convert half-precision to IEEE single-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \param value half-precision value to convert + /// \return single-precision value + inline float half2float_impl(unsigned int value, float, true_type) { + #if HALF_ENABLE_F16C_INTRINSICS + return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(value))); + #else + #if 0 + bits_t fbits = static_cast>(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) + { + fbits |= 0x38000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,fbits-=0x800000) ; + fbits += static_cast>(abs) << 13; + } + #else + static const bits_t mantissa_table[2048] = { + 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000, + 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000, + 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, + 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000, + 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, + 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, + 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, + 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000, + 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, + 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, + 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000, + 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, + 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, + 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, + 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, + 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, + 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, + 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, + 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000, + 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, + 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, + 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, + 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000, + 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, + 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, + 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000, + 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, + 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, + 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, + 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, + 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, + 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000, + 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, + 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000, + 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, + 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, + 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000, + 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000, + 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, + 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, + 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000, + 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, + 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, + 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, + 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, + 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000, + 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000, + 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, + 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000, + 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, + 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, + 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, + 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000, + 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, + 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, + 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000, + 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, + 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000, + 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, + 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, + 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000, + 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000, + 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, + 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, + 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, + 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, + 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, + 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000, + 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, + 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, + 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000, + 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, + 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, + 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, + 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, + 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, + 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000, + 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, + 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, + 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, + 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, + 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, + 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000, + 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, + 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, + 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000, + 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, + 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, + 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, + 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, + 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, + 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000, + 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, + 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, + 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, + 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, + 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, + 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000, + 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, + 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, + 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000, + 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, + 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, + 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, + 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, + 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, + 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000, + 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, + 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, + 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, + 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, + 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, + 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000, + 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, + 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, + 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000, + 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, + 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, + 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, + 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, + 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, + 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000, + 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, + 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, + 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, + 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, + 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, + 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 }; + static const bits_t exponent_table[64] = { + 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000, + 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000, + 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, + 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 }; + static const unsigned short offset_table[64] = { + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 }; + bits_t fbits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10]; + #endif + float out; + std::memcpy(&out, &fbits, sizeof(float)); + return out; + #endif + } + + /// Convert half-precision to IEEE double-precision. + /// \param value half-precision value to convert + /// \return double-precision value + inline double half2float_impl(unsigned int value, double, true_type) { + #if HALF_ENABLE_F16C_INTRINSICS + return _mm_cvtsd_f64(_mm_cvtps_pd(_mm_cvtph_ps(_mm_cvtsi32_si128(value)))); + #else + uint32 hi = static_cast(value&0x8000) << 16; + unsigned int abs = value & 0x7FFF; + if(abs) { + hi |= 0x3F000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,hi-=0x100000) ; + hi += static_cast(abs) << 10; + } + bits_t dbits = static_cast>(hi) << 32; + double out; + std::memcpy(&out, &dbits, sizeof(double)); + return out; + #endif + } + + /// Convert half-precision to non-IEEE floating-point. + /// \tparam T type to convert to (builtin integer type) + /// \param value half-precision value to convert + /// \return floating-point value + template T half2float_impl(unsigned int value, T, ...) { + T out; + unsigned int abs = value & 0x7FFF; + if(abs > 0x7C00) + out = (std::numeric_limits::has_signaling_NaN && !(abs&0x200)) ? std::numeric_limits::signaling_NaN() : + std::numeric_limits::has_quiet_NaN ? std::numeric_limits::quiet_NaN() : T(); + else if(abs == 0x7C00) + out = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() : std::numeric_limits::max(); + else if(abs > 0x3FF) + out = std::ldexp(static_cast((abs&0x3FF)|0x400), (abs>>10)-25); + else + out = std::ldexp(static_cast(abs), -24); + return (value&0x8000) ? -out : out; + } + + /// Convert half-precision to floating-point. + /// \tparam T type to convert to (builtin integer type) + /// \param value half-precision value to convert + /// \return floating-point value + template T half2float(unsigned int value) { + return half2float_impl(value, T(), bool_type::is_iec559&&sizeof(bits_t)==sizeof(T)>()); + } + + /// Convert half-precision floating-point to integer. + /// \tparam R rounding mode to use + /// \tparam E `true` for round to even, `false` for round away from zero + /// \tparam I `true` to raise INEXACT exception (if inexact), `false` to never raise it + /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) + /// \param value half-precision value to convert + /// \return rounded integer value + /// \exception FE_INVALID if value is not representable in type \a T + /// \exception FE_INEXACT if value had to be rounded and \a I is `true` + template T half2int(unsigned int value) { + unsigned int abs = value & 0x7FFF; + if(abs >= 0x7C00) { + raise(FE_INVALID); + return (value&0x8000) ? std::numeric_limits::min() : std::numeric_limits::max(); + } + if(abs < 0x3800) { + raise(FE_INEXACT, I); + return (R==std::round_toward_infinity) ? T(~(value>>15)&(abs!=0)) : + (R==std::round_toward_neg_infinity) ? -T(value>0x8000) : + T(); + } + int exp = 25 - (abs>>10); + unsigned int m = (value&0x3FF) | 0x400; + int32 i = static_cast((exp<=0) ? (m<<-exp) : ((m+( + (R==std::round_to_nearest) ? ((1<<(exp-1))-(~(m>>exp)&E)) : + (R==std::round_toward_infinity) ? (((1<>15)-1)) : + (R==std::round_toward_neg_infinity) ? (((1<>15)) : 0))>>exp)); + if((!std::numeric_limits::is_signed && (value&0x8000)) || (std::numeric_limits::digits<16 && + ((value&0x8000) ? (-i::min()) : (i>std::numeric_limits::max())))) + raise(FE_INVALID); + else if(I && exp > 0 && (m&((1<((value&0x8000) ? -i : i); + } + + /// \} + /// \name Mathematics + /// \{ + + /// upper part of 64-bit multiplication. + /// \tparam R rounding mode to use + /// \param x first factor + /// \param y second factor + /// \return upper 32 bit of \a x * \a y + template uint32 mulhi(uint32 x, uint32 y) { + uint32 xy = (x>>16) * (y&0xFFFF), yx = (x&0xFFFF) * (y>>16), c = (xy&0xFFFF) + (yx&0xFFFF) + (((x&0xFFFF)*(y&0xFFFF))>>16); + return (x>>16)*(y>>16) + (xy>>16) + (yx>>16) + (c>>16) + + ((R==std::round_to_nearest) ? ((c>>15)&1) : (R==std::round_toward_infinity) ? ((c&0xFFFF)!=0) : 0); + } + + /// 64-bit multiplication. + /// \param x first factor + /// \param y second factor + /// \return upper 32 bit of \a x * \a y rounded to nearest + inline uint32 multiply64(uint32 x, uint32 y) { + return static_cast((static_cast(x)*static_cast(y)+0x80000000)>>32); + } + + /// 64-bit division. + /// \param x upper 32 bit of dividend + /// \param y divisor + /// \param s variable to store sticky bit for rounding + /// \return (\a x << 32) / \a y + inline uint32 divide64(uint32 x, uint32 y, int &s) { + unsigned long long xx = static_cast(x) << 32; + return s = (xx%y!=0), static_cast(xx/y); + } + + /// Half precision positive modulus. + /// \tparam Q `true` to compute full quotient, `false` else + /// \tparam R `true` to compute signed remainder, `false` for positive remainder + /// \param x first operand as positive finite half-precision value + /// \param y second operand as positive finite half-precision value + /// \param quo adress to store quotient at, `nullptr` if \a Q `false` + /// \return modulus of \a x / \a y + template unsigned int mod(unsigned int x, unsigned int y, int *quo = NULL) { + unsigned int q = 0; + if(x > y) { + int absx = x, absy = y, expx = 0, expy = 0; + for(; absx<0x400; absx<<=1,--expx) ; + for(; absy<0x400; absy<<=1,--expy) ; + expx += absx >> 10; + expy += absy >> 10; + int mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400; + for(int d=expx-expy; d; --d) { + if(!Q && mx == my) + return 0; + if(mx >= my) { + mx -= my; + q += Q; + } + mx <<= 1; + q <<= static_cast(Q); + } + if(!Q && mx == my) + return 0; + if(mx >= my) { + mx -= my; + ++q; + } + if(Q) { + q &= (1<<(std::numeric_limits::digits-1)) - 1; + if(!mx) + return *quo = q, 0; + } + for(; mx<0x400; mx<<=1,--expy) ; + x = (expy>0) ? ((expy<<10)|(mx&0x3FF)) : (mx>>(1-expy)); + } + if(R) { + unsigned int a, b; + if(y < 0x800) { + a = (x<0x400) ? (x<<1) : (x+0x400); + b = y; + } else { + a = x; + b = y - 0x400; + } + if(a > b || (a == b && (q&1))) { + int exp = (y>>10) + (y<=0x3FF), d = exp - (x>>10) - (x<=0x3FF); + int m = (((y&0x3FF)|((y>0x3FF)<<10))<<1) - (((x&0x3FF)|((x>0x3FF)<<10))<<(1-d)); + for(; m<0x800 && exp>1; m<<=1,--exp) ; + x = 0x8000 + ((exp-1)<<10) + (m>>1); + q += Q; + } + } + if(Q) + *quo = q; + return x; + } + + /// Fixed point square root. + /// \tparam F number of fractional bits + /// \param r radicand in Q1.F fixed point format + /// \param exp exponent + /// \return square root as Q1.F/2 + template uint32 sqrt(uint32 &r, int &exp) { + int i = exp & 1; + r <<= i; + exp = (exp-i) / 2; + uint32 m = 0; + for(uint32 bit=static_cast(1)<>=2) { + if(r < m+bit) + m >>= 1; + else { + r -= m + bit; + m = (m>>1) + bit; + } + } + return m; + } + + /// Fixed point binary exponential. + /// This uses the BKM algorithm in E-mode. + /// \param m exponent in [0,1) as Q0.31 + /// \param n number of iterations (at most 32) + /// \return 2 ^ \a m as Q1.31 + inline uint32 exp2(uint32 m, unsigned int n = 32) { + static const uint32 logs[] = { + 0x80000000, 0x4AE00D1D, 0x2934F098, 0x15C01A3A, 0x0B31FB7D, 0x05AEB4DD, 0x02DCF2D1, 0x016FE50B, + 0x00B84E23, 0x005C3E10, 0x002E24CA, 0x001713D6, 0x000B8A47, 0x0005C53B, 0x0002E2A3, 0x00017153, + 0x0000B8AA, 0x00005C55, 0x00002E2B, 0x00001715, 0x00000B8B, 0x000005C5, 0x000002E3, 0x00000171, + 0x000000B9, 0x0000005C, 0x0000002E, 0x00000017, 0x0000000C, 0x00000006, 0x00000003, 0x00000001 }; + if(!m) + return 0x80000000; + uint32 mx = 0x80000000, my = 0; + for(unsigned int i=1; i> i; + } + } + return mx; + } + + /// Fixed point binary logarithm. + /// This uses the BKM algorithm in L-mode. + /// \param m mantissa in [1,2) as Q1.30 + /// \param n number of iterations (at most 32) + /// \return log2(\a m) as Q0.31 + inline uint32 log2(uint32 m, unsigned int n = 32) { + static const uint32 logs[] = { + 0x80000000, 0x4AE00D1D, 0x2934F098, 0x15C01A3A, 0x0B31FB7D, 0x05AEB4DD, 0x02DCF2D1, 0x016FE50B, + 0x00B84E23, 0x005C3E10, 0x002E24CA, 0x001713D6, 0x000B8A47, 0x0005C53B, 0x0002E2A3, 0x00017153, + 0x0000B8AA, 0x00005C55, 0x00002E2B, 0x00001715, 0x00000B8B, 0x000005C5, 0x000002E3, 0x00000171, + 0x000000B9, 0x0000005C, 0x0000002E, 0x00000017, 0x0000000C, 0x00000006, 0x00000003, 0x00000001 }; + if(m == 0x40000000) + return 0; + uint32 mx = 0x40000000, my = 0; + for(unsigned int i=1; i>i); + if(mz <= m) { + mx = mz; + my += logs[i]; + } + } + return my; + } + + /// Fixed point sine and cosine. + /// This uses the CORDIC algorithm in rotation mode. + /// \param mz angle in [-pi/2,pi/2] as Q1.30 + /// \param n number of iterations (at most 31) + /// \return sine and cosine of \a mz as Q1.30 + inline std::pair sincos(uint32 mz, unsigned int n = 31) { + static const uint32 angles[] = { + 0x3243F6A9, 0x1DAC6705, 0x0FADBAFD, 0x07F56EA7, 0x03FEAB77, 0x01FFD55C, 0x00FFFAAB, 0x007FFF55, + 0x003FFFEB, 0x001FFFFD, 0x00100000, 0x00080000, 0x00040000, 0x00020000, 0x00010000, 0x00008000, + 0x00004000, 0x00002000, 0x00001000, 0x00000800, 0x00000400, 0x00000200, 0x00000100, 0x00000080, + 0x00000040, 0x00000020, 0x00000010, 0x00000008, 0x00000004, 0x00000002, 0x00000001 }; + uint32 mx = 0x26DD3B6A, my = 0; + for(unsigned int i=0; i0x3FF)<<10); + int exp = (abs>>10) + (abs<=0x3FF) - 15; + if(abs < 0x3A48) + return k = 0, m << (exp+20); + unsigned long long y = m * 0xA2F9836E4E442, mask = (1ULL<<(62-exp)) - 1, yi = (y+(mask>>1)) & ~mask, f = y - yi; + uint32 sign = -static_cast(f>>63); + k = static_cast(yi>>(62-exp)); + return (multiply64(static_cast((sign ? -f : f)>>(31-exp)), 0xC90FDAA2)^sign) - sign; + } + + /// Get arguments for atan2 function. + /// \param abs half-precision floating-point value + /// \return \a abs and sqrt(1 - \a abs^2) as Q0.30 + inline std::pair atan2_args(unsigned int abs) { + int exp = -15; + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + uint32 my = ((abs&0x3FF)|0x400) << 5, r = my * my; + int rexp = 2 * exp; + r = 0x40000000 - ((rexp>-31) ? ((r>>-rexp)|((r&((static_cast(1)<<-rexp)-1))!=0)) : 1); + for(rexp=0; r<0x40000000; r<<=1,--rexp) ; + uint32 mx = sqrt<30>(r, rexp); + int d = exp - rexp; + if(d < 0) + return std::make_pair((d<-14) ? ((my>>(-d-14))+((my>>(-d-15))&1)) : (my<<(14+d)), (mx<<14)+(r<<13)/mx); + if(d > 0) + return std::make_pair(my<<14, (d>14) ? ((mx>>(d-14))+((mx>>(d-15))&1)) : ((d==14) ? mx : ((mx<<(14-d))+(r<<(13-d))/mx))); + return std::make_pair(my<<13, (mx<<13)+(r<<12)/mx); + } + + /// Get exponentials for hyperbolic computation + /// \param abs half-precision floating-point value + /// \param exp variable to take unbiased exponent of larger result + /// \param n number of BKM iterations (at most 32) + /// \return exp(abs) and exp(-\a abs) as Q1.31 with same exponent + inline std::pair hyperbolic_args(unsigned int abs, int &exp, unsigned int n = 32) { + uint32 mx = detail::multiply64(static_cast((abs&0x3FF)+((abs>0x3FF)<<10))<<21, 0xB8AA3B29), my; + int e = (abs>>10) + (abs<=0x3FF); + if(e < 14) { + exp = 0; + mx >>= 14 - e; + } else { + exp = mx >> (45-e); + mx = (mx<<(e-14)) & 0x7FFFFFFF; + } + mx = exp2(mx, n); + int d = exp << 1, s; + if(mx > 0x80000000) { + my = divide64(0x80000000, mx, s); + my |= s; + ++d; + } else + my = mx; + return std::make_pair(mx, (d<31) ? ((my>>d)|((my&((static_cast(1)< unsigned int exp2_post(uint32 m, int exp, bool esign, unsigned int sign = 0) { + int s = 0; + if(esign) { + if(m > 0x80000000) { + m = divide64(0x80000000, m, s); + ++exp; + } + if(exp > 25) + return underflow(sign); + else if(exp == 25) + return rounded(sign, 1, (m&0x7FFFFFFF)!=0); + exp = -exp; + } else if(exp > 15) + return overflow(sign); + return fixed2half(m, exp+14, sign, s); + } + + /// Postprocessing for binary logarithm. + /// \tparam R rounding mode to use + /// \tparam L logarithm for base transformation as Q1.31 + /// \param m fractional part of logarithm as Q0.31 + /// \param ilog signed integer part of logarithm + /// \param exp biased exponent of result + /// \param sign sign bit of result + /// \return value base-transformed and converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int log2_post(uint32 m, int ilog, int exp, unsigned int sign = 0) { + uint32 msign = sign_mask(ilog); + m = (((static_cast(ilog)<<27)+(m>>4))^msign) - msign; + if(!m) + return 0; + for(; m<0x80000000; m<<=1,--exp) ; + int i = m >= L, s; + exp += i; + m >>= 1 + i; + sign ^= msign & 0x8000; + if(exp < -11) + return underflow(sign); + m = divide64(m, L, s); + return fixed2half(m, exp, sign, 1); + } + + /// Hypotenuse square root and postprocessing. + /// \tparam R rounding mode to use + /// \param r mantissa as Q2.30 + /// \param exp unbiased exponent + /// \return square root converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int hypot_post(uint32 r, int exp) { + int i = r >> 31; + if((exp+=i) > 46) + return overflow(); + if(exp < -34) + return underflow(); + r = (r>>i) | (r&i); + uint32 m = sqrt<30>(r, exp+=15); + return fixed2half(m, exp-1, 0, r!=0); + } + + /// Division and postprocessing for tangents. + /// \tparam R rounding mode to use + /// \param my dividend as Q1.31 + /// \param mx divisor as Q1.31 + /// \param exp biased exponent of result + /// \param sign sign bit of result + /// \return quotient converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int tangent_post(uint32 my, uint32 mx, int exp, unsigned int sign = 0) { + int i = my >= mx, s; + exp += i; + if(exp > 29) + return overflow(sign); + if(exp < -11) + return underflow(sign); + uint32 m = divide64(my>>(i+1), mx, s); + return fixed2half(m, exp, sign, s); + } + + /// Area function and postprocessing. + /// This computes the value directly in Q2.30 using the representation `asinh|acosh(x) = log(x+sqrt(x^2+|-1))`. + /// \tparam R rounding mode to use + /// \tparam S `true` for asinh, `false` for acosh + /// \param arg half-precision argument + /// \return asinh|acosh(\a arg) converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int area(unsigned int arg) { + int abs = arg & 0x7FFF, expx = (abs>>10) + (abs<=0x3FF) - 15, expy = -15, ilog, i; + uint32 mx = static_cast((abs&0x3FF)|((abs>0x3FF)<<10)) << 20, my, r; + for(; abs<0x400; abs<<=1,--expy) ; + expy += abs >> 10; + r = ((abs&0x3FF)|0x400) << 5; + r *= r; + i = r >> 31; + expy = 2*expy + i; + r >>= i; + if(S) { + if(expy < 0) { + r = 0x40000000 + ((expy>-30) ? ((r>>-expy)|((r&((static_cast(1)<<-expy)-1))!=0)) : 1); + expy = 0; + } else { + r += 0x40000000 >> expy; + i = r >> 31; + r = (r>>i) | (r&i); + expy += i; + } + } else { + r -= 0x40000000 >> expy; + for(; r<0x40000000; r<<=1,--expy) ; + } + my = sqrt<30>(r, expy); + my = (my<<15) + (r<<14)/my; + if(S) { + mx >>= expy - expx; + ilog = expy; + } else { + my >>= expx - expy; + ilog = expx; + } + my += mx; + i = my >> 31; + static const int G = S && (R==std::round_to_nearest); + return log2_post(log2(my>>i, 26+S+G)+(G<<3), ilog+i, 17, arg&(static_cast(S)<<15)); + } + + /// Class for 1.31 unsigned floating-point computation + struct f31 { + /// Constructor. + /// \param mant mantissa as 1.31 + /// \param e exponent + constexpr f31(uint32 mant, int e) : m(mant), exp(e) {} + + /// Constructor. + /// \param abs unsigned half-precision value + f31(unsigned int abs) : exp(-15) { + for(; abs<0x400; abs<<=1,--exp) ; + m = static_cast((abs&0x3FF)|0x400) << 21; + exp += (abs>>10); + } + + /// Addition operator. + /// \param a first operand + /// \param b second operand + /// \return \a a + \a b + friend f31 operator+(f31 a, f31 b) { + if(b.exp > a.exp) + std::swap(a, b); + int d = a.exp - b.exp; + uint32 m = a.m + ((d<32) ? (b.m>>d) : 0); + int i = (m&0xFFFFFFFF) < a.m; + return f31(((m+i)>>i)|0x80000000, a.exp+i); + } + + /// Subtraction operator. + /// \param a first operand + /// \param b second operand + /// \return \a a - \a b + friend f31 operator-(f31 a, f31 b) { + int d = a.exp - b.exp, exp = a.exp; + uint32 m = a.m - ((d<32) ? (b.m>>d) : 0); + if(!m) + return f31(0, -32); + for(; m<0x80000000; m<<=1,--exp) ; + return f31(m, exp); + } + + /// Multiplication operator. + /// \param a first operand + /// \param b second operand + /// \return \a a * \a b + friend f31 operator*(f31 a, f31 b) { + uint32 m = multiply64(a.m, b.m); + int i = m >> 31; + return f31(m<<(1-i), a.exp + b.exp + i); + } + + /// Division operator. + /// \param a first operand + /// \param b second operand + /// \return \a a / \a b + friend f31 operator/(f31 a, f31 b) { + int i = a.m >= b.m, s; + uint32 m = divide64((a.m+i)>>i, b.m, s); + return f31(m, a.exp - b.exp + i - 1); + } + + uint32 m; ///< mantissa as 1.31. + int exp; ///< exponent. + }; + + /// Error function and postprocessing. + /// This computes the value directly in Q1.31 using the approximations given + /// [here](https://en.wikipedia.org/wiki/Error_function#Approximation_with_elementary_functions). + /// \tparam R rounding mode to use + /// \tparam C `true` for comlementary error function, `false` else + /// \param arg half-precision function argument + /// \return approximated value of error function in half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int erf(unsigned int arg) { + unsigned int abs = arg & 0x7FFF, sign = arg & 0x8000; + f31 x(abs), x2 = x * x * f31(0xB8AA3B29, 0), t = f31(0x80000000, 0) / (f31(0x80000000, 0)+f31(0xA7BA054A, -2)*x), t2 = t * t; + f31 e = ((f31(0x87DC2213, 0)*t2+f31(0xB5F0E2AE, 0))*t2+f31(0x82790637, -2)-(f31(0xBA00E2B8, 0)*t2+f31(0x91A98E62, -2))*t) * t / + ((x2.exp<0) ? f31(exp2((x2.exp>-32) ? (x2.m>>-x2.exp) : 0, 30), 0) : f31(exp2((x2.m<>(31-x2.exp))); + return (!C || sign) ? fixed2half(0x80000000-(e.m>>(C-e.exp)), 14+C, sign&(C-1U)) : + (e.exp<-25) ? underflow() : fixed2half(e.m>>1, e.exp+14, 0, e.m&1); + } + + /// Gamma function and postprocessing. + /// This approximates the value of either the gamma function or its logarithm directly in Q1.31. + /// \tparam R rounding mode to use + /// \tparam L `true` for lograithm of gamma function, `false` for gamma function + /// \param arg half-precision floating-point value + /// \return lgamma/tgamma(\a arg) in half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if \a arg is not a positive integer + template unsigned int gamma(unsigned int arg) { +/* static const double p[] ={ 2.50662827563479526904, 225.525584619175212544, -268.295973841304927459, 80.9030806934622512966, -5.00757863970517583837, 0.0114684895434781459556 }; + double t = arg + 4.65, s = p[0]; + for(unsigned int i=0; i<5; ++i) + s += p[i+1] / (arg+i); + return std::log(s) + (arg-0.5)*std::log(t) - t; +*/ static const f31 pi(0xC90FDAA2, 1), lbe(0xB8AA3B29, 0); + unsigned int abs = arg & 0x7FFF, sign = arg & 0x8000; + bool bsign = sign != 0; + f31 z(abs), x = sign ? (z+f31(0x80000000, 0)) : z, t = x + f31(0x94CCCCCD, 2), s = + f31(0xA06C9901, 1) + f31(0xBBE654E2, -7)/(x+f31(0x80000000, 2)) + f31(0xA1CE6098, 6)/(x+f31(0x80000000, 1)) + + f31(0xE1868CB7, 7)/x - f31(0x8625E279, 8)/(x+f31(0x80000000, 0)) - f31(0xA03E158F, 2)/(x+f31(0xC0000000, 1)); + int i = (s.exp>=2) + (s.exp>=4) + (s.exp>=8) + (s.exp>=16); + s = f31((static_cast(s.exp)<<(31-i))+(log2(s.m>>1, 28)>>i), i) / lbe; + if(x.exp != -1 || x.m != 0x80000000) { + i = (t.exp>=2) + (t.exp>=4) + (t.exp>=8); + f31 l = f31((static_cast(t.exp)<<(31-i))+(log2(t.m>>1, 30)>>i), i) / lbe; + s = (x.exp<-1) ? (s-(f31(0x80000000, -1)-x)*l) : (s+(x-f31(0x80000000, -1))*l); + } + s = x.exp ? (s-t) : (t-s); + if(bsign) { + if(z.exp >= 0) { + sign &= (L|((z.m>>(31-z.exp))&1)) - 1; + for(z=f31((z.m<<(1+z.exp))&0xFFFFFFFF, -1); z.m<0x80000000; z.m<<=1,--z.exp) ; + } + if(z.exp == -1) + z = f31(0x80000000, 0) - z; + if(z.exp < -1) { + z = z * pi; + z.m = sincos(z.m>>(1-z.exp), 30).first; + for(z.exp=1; z.m<0x80000000; z.m<<=1,--z.exp) ; + } + else + z = f31(0x80000000, 0); + } if(L) { + if(bsign) { + f31 l(0x92868247, 0); + if(z.exp < 0) { + uint32 m = log2((z.m+1)>>1, 27); + z = f31(-((static_cast(z.exp)<<26)+(m>>5)), 5); + for(; z.m<0x80000000; z.m<<=1,--z.exp) ; + l = l + z / lbe; + } + sign = static_cast(x.exp&&(l.exp(x.exp==0) << 15; + if(s.exp < -24) + return underflow(sign); + if(s.exp > 15) + return overflow(sign); + } + } else { + s = s * lbe; + uint32 m; + if(s.exp < 0) { + m = s.m >> -s.exp; + s.exp = 0; + } else { + m = (s.m<>(31-s.exp)); + } + s.m = exp2(m, 27); + if(!x.exp) + s = f31(0x80000000, 0) / s; + if(bsign) { + if(z.exp < 0) + s = s * z; + s = pi / s; + if(s.exp < -24) + return underflow(sign); + } else if(z.exp > 0 && !(z.m&((1<<(31-z.exp))-1))) + return ((s.exp+14)<<10) + (s.m>>21); + if(s.exp > 15) + return overflow(sign); + } + return fixed2half(s.m, s.exp+14, sign); + } + /// \} + + template struct half_caster; + } + + /// Half-precision floating-point type. + /// This class implements an IEEE-conformant half-precision floating-point type with the usual arithmetic + /// operators and conversions. It is implicitly convertible to single-precision floating-point, which makes artihmetic + /// expressions and functions with mixed-type operands to be of the most precise operand type. + /// + /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and + /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which + /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the + /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of + /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most + /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit + /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if + /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on + /// nearly any reasonable platform. + /// + /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable + /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation. + class half { + public: + /// \name Construction and assignment + /// \{ + + /// Default constructor. + /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics + /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics. + constexpr half() noexcept : data_() {} + + /// Conversion constructor. + /// \param rhs float to convert + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + //explicit half(float rhs) : data_(static_cast(detail::float2half(rhs))) {} + + /// Conversion constructor. + /// \param rhs float to convert + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template + half(T rhs) : data_(static_cast(detail::float2half(static_cast(rhs)))) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + operator float() const { return detail::half2float(data_); } + + /// Assignment operator. + /// \param rhs single-precision value to copy from + /// \return reference to this half + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + half& operator=(const float &rhs) { data_ = static_cast(detail::float2half(rhs)); return *this; } + + template + half& operator=(const T &rhs) { return *this = static_cast(rhs); } + + /// \} + /// \name Arithmetic updates + /// \{ + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to add + /// \return reference to this half + /// \exception FE_... according to operator+(half,half) + half& operator+=(half rhs) { return *this = *this + rhs; } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to subtract + /// \return reference to this half + /// \exception FE_... according to operator-(half,half) + half& operator-=(half rhs) { return *this = *this - rhs; } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to multiply with + /// \return reference to this half + /// \exception FE_... according to operator*(half,half) + half& operator*=(half rhs) { return *this = *this * rhs; } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to divide by + /// \return reference to this half + /// \exception FE_... according to operator/(half,half) + half& operator/=(half rhs) { return *this = *this / rhs; } + + /* + /// Arithmetic assignment. + /// \param rhs single-precision value to add + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator+=(float rhs) { return *this = *this + rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to subtract + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator-=(float rhs) { return *this = *this - rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to multiply with + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator*=(float rhs) { return *this = *this * rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to divide by + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator/=(float rhs) { return *this = *this / rhs; } + */ + + /// \} + /// \name Increment and decrement + /// \{ + + /// Prefix increment. + /// \return incremented half value + /// \exception FE_... according to operator+(half,half) + half& operator++() { return *this = *this + half(detail::binary, 0x3C00); } + + /// Prefix decrement. + /// \return decremented half value + /// \exception FE_... according to operator-(half,half) + half& operator--() { return *this = *this + half(detail::binary, 0xBC00); } + + /// Postfix increment. + /// \return non-incremented half value + /// \exception FE_... according to operator+(half,half) + half operator++(int) { half out(*this); ++*this; return out; } + + /// Postfix decrement. + /// \return non-decremented half value + /// \exception FE_... according to operator-(half,half) + half operator--(int) { half out(*this); --*this; return out; } + /// \} + detail::uint16 get_data()const{ return data_; } + + private: + /// Rounding mode to use + static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE); + + /// Constructor. + /// \param bits binary representation to set half to + constexpr half(detail::binary_t, unsigned int bits) noexcept : data_(static_cast(bits)) {} + + /// Internal binary representation + detail::uint16 data_; + + friend constexpr_NOERR bool operator==(half, half); + template friend constexpr_NOERR bool operator==(half, T); + template friend constexpr_NOERR bool operator==(T, half); + friend constexpr_NOERR bool operator!=(half, half); + template friend constexpr_NOERR bool operator!=(half, T); + template friend constexpr_NOERR bool operator!=(T, half); + friend constexpr_NOERR bool operator<(half, half); + template friend constexpr_NOERR bool operator<(half, T); + template friend constexpr_NOERR bool operator<(T, half); + friend constexpr_NOERR bool operator>(half, half); + template friend constexpr_NOERR bool operator>(half, T); + template friend constexpr_NOERR bool operator>(T, half); + friend constexpr_NOERR bool operator<=(half, half); + template friend constexpr_NOERR bool operator<=(half, T); + template friend constexpr_NOERR bool operator<=(T, half); + friend constexpr_NOERR bool operator>=(half, half); + template friend constexpr_NOERR bool operator>=(half, T); + template friend constexpr_NOERR bool operator>=(T, half); + friend constexpr half operator+(half); + friend constexpr half operator-(half); + friend half operator+(half, half); + template friend half operator+(half, T); + template friend half operator+(T, half); + friend half operator-(half, half); + template friend half operator-(half, T); + template friend half operator-(T, half); + friend half operator*(half, half); + template friend half operator*(half, T); + template friend half operator*(T, half); + friend half operator/(half, half); + template friend half operator/(half, T); + template friend half operator/(T, half); + template friend std::basic_ostream& operator<<(std::basic_ostream&, half); + template friend std::basic_istream& operator>>(std::basic_istream&, half&); + friend constexpr half fabs(half); + friend half fmod(half, half); + friend half remainder(half, half); + friend half remquo(half, half, int*); + friend half fma(half, half, half); + friend constexpr_NOERR half fmax(half, half); + friend constexpr_NOERR half fmin(half, half); + friend half fdim(half, half); + friend half nanh(const char*); + friend half exp(half); + friend half exp2(half); + friend half expm1(half); + friend half log(half); + friend half log10(half); + friend half log2(half); + friend half log1p(half); + friend half sqrt(half); + friend half cbrt(half); + friend half hypot(half, half); + friend half hypot(half, half, half); + friend half pow(half, half); + friend void sincos(half, half*, half*); + friend half sin(half); + friend half cos(half); + friend half tan(half); + friend half asin(half); + friend half acos(half); + friend half atan(half); + friend half atan2(half, half); + friend half sinh(half); + friend half cosh(half); + friend half tanh(half); + friend half asinh(half); + friend half acosh(half); + friend half atanh(half); + friend half erf(half); + friend half erfc(half); + friend half lgamma(half); + friend half tgamma(half); + friend half ceil(half); + friend half floor(half); + friend half trunc(half); + friend half round(half); + friend long lround(half); + friend half rint(half); + friend long lrint(half); + friend half nearbyint(half); + friend long long llround(half); + friend long long llrint(half); + friend half frexp(half, int*); + friend half scalbln(half, long); + friend half modf(half, half*); + friend int ilogb(half); + friend half logb(half); + friend half nextafter(half, half); + friend half nexttoward(half, long double); + friend constexpr half copysign(half, half); + friend constexpr int fpclassify(half); + friend constexpr bool isfinite(half); + friend constexpr bool isinf(half); + friend constexpr bool isnan(half); + friend constexpr bool isnormal(half); + friend constexpr bool signbit(half); + friend constexpr bool isgreater(half, half); + friend constexpr bool isgreaterequal(half, half); + friend constexpr bool isless(half, half); + friend constexpr bool islessequal(half, half); + friend constexpr bool islessgreater(half, half); + template friend struct detail::half_caster; + friend class std::numeric_limits; + friend struct std::hash; + friend half literal::operator "" _h(long double); + }; + + namespace literal { + /// Half literal. + /// While this returns a properly rounded half-precision value, half literals can unfortunately not be constant + /// expressions due to rather involved conversions. So don't expect this to be a literal literal without involving + /// conversion operations at runtime. It is a convenience feature, not a performance optimization. + /// \param value literal value + /// \return half with of given value (possibly rounded) + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator "" _h(long double value) { return half(detail::binary, detail::float2half(value)); } + } + + namespace detail { + /// Helper class for half casts. + /// This class template has to be specialized for all valid cast arguments to define an appropriate static + /// `cast` member function and a corresponding `type` member denoting its return type. + /// \tparam T destination type + /// \tparam U source type + /// \tparam R rounding mode to use + template struct half_caster {}; + template struct half_caster { + static_assert(std::is_arithmetic::value, "half_cast from non-arithmetic type unsupported"); + static half cast(U arg) { return cast_impl(arg, is_float()); }; + private: + static half cast_impl(U arg, true_type) { return half(binary, float2half(arg)); } + static half cast_impl(U arg, false_type) { return half(binary, int2half(arg)); } + }; + template struct half_caster { + static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); + static T cast(half arg) { return cast_impl(arg, is_float()); } + private: + static T cast_impl(half arg, true_type) { return half2float(arg.data_); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } + }; + template struct half_caster { + static half cast(half arg) { return arg; } + }; + } +} + +/// Extensions to the C++ standard library. +namespace std { + /// Numeric limits for half-precision floats. + /// **See also:** Documentation for [std::numeric_limits](https://en.cppreference.com/w/cpp/types/numeric_limits) + template<> class numeric_limits { + public: + /// Is template specialization. + static constexpr bool is_specialized = true; + + /// Supports signed values. + static constexpr bool is_signed = true; + + /// Is not an integer type. + static constexpr bool is_integer = false; + + /// Is not exact. + static constexpr bool is_exact = false; + + /// Doesn't provide modulo arithmetic. + static constexpr bool is_modulo = false; + + /// Has a finite set of values. + static constexpr bool is_bounded = true; + + /// IEEE conformant. + static constexpr bool is_iec559 = true; + + /// Supports infinity. + static constexpr bool has_infinity = true; + + /// Supports quiet NaNs. + static constexpr bool has_quiet_NaN = true; + + /// Supports signaling NaNs. + static constexpr bool has_signaling_NaN = true; + + /// Supports subnormal values. + static constexpr float_denorm_style has_denorm = denorm_present; + + /// Supports no denormalization detection. + static constexpr bool has_denorm_loss = false; + + #if HALF_ERRHANDLING_THROWS + static constexpr bool traps = true; + #else + /// Traps only if [HALF_ERRHANDLING_THROW_...](\ref HALF_ERRHANDLING_THROW_INVALID) is acitvated. + static constexpr bool traps = false; + #endif + + /// Does not support no pre-rounding underflow detection. + static constexpr bool tinyness_before = false; + + /// Rounding mode. + static constexpr float_round_style round_style = half_float::half::round_style; + + /// Significant digits. + static constexpr int digits = 11; + + /// Significant decimal digits. + static constexpr int digits10 = 3; + + /// Required decimal digits to represent all possible values. + static constexpr int max_digits10 = 5; + + /// Number base. + static constexpr int radix = 2; + + /// One more than smallest exponent. + static constexpr int min_exponent = -13; + + /// Smallest normalized representable power of 10. + static constexpr int min_exponent10 = -4; + + /// One more than largest exponent + static constexpr int max_exponent = 16; + + /// Largest finitely representable power of 10. + static constexpr int max_exponent10 = 4; + + /// Smallest positive normal value. + static constexpr half_float::half min() noexcept { return half_float::half(half_float::detail::binary, 0x0400); } + + /// Smallest finite value. + static constexpr half_float::half lowest() noexcept { return half_float::half(half_float::detail::binary, 0xFBFF); } + + /// Largest finite value. + static constexpr half_float::half max() noexcept { return half_float::half(half_float::detail::binary, 0x7BFF); } + + /// Difference between 1 and next representable value. + static constexpr half_float::half epsilon() noexcept { return half_float::half(half_float::detail::binary, 0x1400); } + + /// Maximum rounding error in ULP (units in the last place). + static constexpr half_float::half round_error() noexcept + { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); } + + /// Positive infinity. + static constexpr half_float::half infinity() noexcept { return half_float::half(half_float::detail::binary, 0x7C00); } + + /// Quiet NaN. + static constexpr half_float::half quiet_NaN() noexcept { return half_float::half(half_float::detail::binary, 0x7FFF); } + + /// Signaling NaN. + static constexpr half_float::half signaling_NaN() noexcept { return half_float::half(half_float::detail::binary, 0x7DFF); } + + /// Smallest positive subnormal value. + static constexpr half_float::half denorm_min() noexcept { return half_float::half(half_float::detail::binary, 0x0001); } + }; + + /// Hash function for half-precision floats. + /// **See also:** Documentation for [std::hash](https://en.cppreference.com/w/cpp/utility/hash) + template<> struct hash { + /// Type of function argument. + typedef half_float::half argument_type; + + /// Function return type. + typedef size_t result_type; + + /// Compute hash function. + /// \param arg half to hash + /// \return hash value + result_type operator()(argument_type arg) const { return hash()(arg.data_&-static_cast(arg.data_!=0x8000)); } + }; +} + +namespace half_float { + /// \anchor compop + /// \name Comparison operators + /// \{ + + /// Comparison for equality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline constexpr_NOERR bool operator==(half x, half y) { + return !detail::compsignal(x.data_, y.data_) && (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)); + } + template + inline constexpr_NOERR bool operator==(half x, T y) { return x == static_cast(y); } + template + inline constexpr_NOERR bool operator==(T x, half y) { return static_cast(x) == y; } + + /// Comparison for inequality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline constexpr_NOERR bool operator!=(half x, half y) { + return detail::compsignal(x.data_, y.data_) || (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)); + } + template + inline constexpr_NOERR bool operator!=(half x, T y) { return x != static_cast(y); } + template + inline constexpr_NOERR bool operator!=(T x, half y) { return static_cast(x) != y; } + + /// Comparison for less than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline constexpr_NOERR bool operator<(half x, half y) { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) < ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + template + inline constexpr_NOERR bool operator<(half x, T y) { return x < static_cast(y); } + template + inline constexpr_NOERR bool operator<(T x, half y) { return static_cast(x) < y; } + + /// Comparison for greater than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline constexpr_NOERR bool operator>(half x, half y) { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) > ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + template + inline constexpr_NOERR bool operator>(half x, T y) { return x > static_cast(y); } + template + inline constexpr_NOERR bool operator>(T x, half y) { return static_cast(x) > y; } + + /// Comparison for less equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline constexpr_NOERR bool operator<=(half x, half y) { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) <= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + template + inline constexpr_NOERR bool operator<=(half x, T y) { return x <= static_cast(y); } + template + inline constexpr_NOERR bool operator<=(T x, half y) { return static_cast(x) <= y; } + + /// Comparison for greater equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline constexpr_NOERR bool operator>=(half x, half y) { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) >= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + template + inline constexpr_NOERR bool operator>=(half x, T y) { return x >= static_cast(y); } + template + inline constexpr_NOERR bool operator>=(T x, half y) { return static_cast(x) >= y; } + + /// \} + /// \anchor arithmetics + /// \name Arithmetic operators + /// \{ + + /// Identity. + /// \param arg operand + /// \return unchanged operand + inline constexpr half operator+(half arg) { return arg; } + + /// Negation. + /// \param arg operand + /// \return negated operand + inline constexpr half operator-(half arg) { return half(detail::binary, arg.data_^0x8000); } + + /// Addition. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return sum of half expressions + /// \exception FE_INVALID if \a x and \a y are infinities with different signs or signaling NaNs + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator+(half x, half y) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)+detail::half2float(y.data_))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF; + bool sub = ((x.data_^y.data_)&0x8000) != 0; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : (absy!=0x7C00) ? x.data_ : + (sub && absx==0x7C00) ? detail::invalid() : y.data_); + if(!absx) + return absy ? y : half(detail::binary, (half::round_style==std::round_toward_neg_infinity) ? (x.data_|y.data_) : (x.data_&y.data_)); + if(!absy) + return x; + unsigned int sign = ((sub && absy>absx) ? y.data_ : x.data_) & 0x8000; + if(absy > absx) + std::swap(absx, absy); + int exp = (absx>>10) + (absx<=0x3FF), d = exp - (absy>>10) - (absy<=0x3FF), mx = ((absx&0x3FF)|((absx>0x3FF)<<10)) << 3, my; + if(d < 13) { + my = ((absy&0x3FF)|((absy>0x3FF)<<10)) << 3; + my = (my>>d) | ((my&((1<(half::round_style==std::round_toward_neg_infinity)<<15); + for(; mx<0x2000 && exp>1; mx<<=1,--exp) ; + } else { + mx += my; + int i = mx >> 14; + if((exp+=i) > 30) + return half(detail::binary, detail::overflow(sign)); + mx = (mx>>i) | (mx&i); + } + return half(detail::binary, detail::rounded(sign+((exp-1)<<10)+(mx>>3), (mx>>2)&1, (mx&0x3)!=0)); + #endif + } + template + inline half operator+(half x, T y) { return x + static_cast(y); } + template + inline half operator+(T x, half y) { return static_cast(x) + y; } + + /// Subtraction. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return difference of half expressions + /// \exception FE_INVALID if \a x and \a y are infinities with equal signs or signaling NaNs + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator-(half x, half y) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)-detail::half2float(y.data_))); + #else + return x + (-y); + #endif + } + template + inline half operator-(half x, T y) { return x - static_cast(y); } + template + inline half operator-(T x, half y) { return static_cast(x) - y; } + + /// Multiplication. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return product of half expressions + /// \exception FE_INVALID if multiplying 0 with infinity or if \a x or \a y is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator*(half x, half y) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)*detail::half2float(y.data_))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = -16; + unsigned int sign = (x.data_^y.data_) & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + ((absx==0x7C00 && !absy)||(absy==0x7C00 && !absx)) ? detail::invalid() : (sign|0x7C00)); + if(!absx || !absy) + return half(detail::binary, sign); + for(; absx<0x400; absx<<=1,--exp) ; + for(; absy<0x400; absy<<=1,--exp) ; + detail::uint32 m = static_cast((absx&0x3FF)|0x400) * static_cast((absy&0x3FF)|0x400); + int i = m >> 21, s = m & i; + exp += (absx>>10) + (absy>>10) + i; + if(exp > 29) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -11) + return half(detail::binary, detail::underflow(sign)); + return half(detail::binary, detail::fixed2half(m>>i, exp, sign, s)); + #endif + } + template + inline half operator*(half x, T y) { return x * static_cast(y); } + template + inline half operator*(T x, half y) { return static_cast(x) * y; } + + /// Division. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return quotient of half expressions + /// \exception FE_INVALID if dividing 0s or infinities with each other or if \a x or \a y is signaling NaN + /// \exception FE_DIVBYZERO if dividing finite value by 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator/(half x, half y) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)/detail::half2float(y.data_))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = 14; + unsigned int sign = (x.data_^y.data_) & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==absy) ? detail::invalid() : (sign|((absx==0x7C00) ? 0x7C00 : 0))); + if(!absx) + return half(detail::binary, absy ? sign : detail::invalid()); + if(!absy) + return half(detail::binary, detail::pole(sign)); + for(; absx<0x400; absx<<=1,--exp) ; + for(; absy<0x400; absy<<=1,++exp) ; + detail::uint32 mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400; + int i = mx < my; + exp += (absx>>10) - (absy>>10) - i; + if(exp > 29) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -11) + return half(detail::binary, detail::underflow(sign)); + mx <<= 12 + i; + my <<= 1; + return half(detail::binary, detail::fixed2half(mx/my, exp, sign, mx%my!=0)); + #endif + } + template + inline half operator/(half x, T y) { return x / static_cast(y); } + template + inline half operator/(T x, half y) { return static_cast(x) / y; } + + /// \} + /// \anchor streaming + /// \name Input and output + /// \{ + + /// Output operator. + /// This uses the built-in functionality for streaming out floating-point numbers. + /// \param out output stream to write into + /// \param arg half expression to write + /// \return reference to output stream + template std::basic_ostream& operator<<(std::basic_ostream &out, half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return out << detail::half2float(arg.data_); + #else + return out << detail::half2float(arg.data_); + #endif + } + + /// Input operator. + /// This uses the built-in functionality for streaming in floating-point numbers, specifically double precision floating + /// point numbers (unless overridden with [HALF_ARITHMETIC_TYPE](\ref HALF_ARITHMETIC_TYPE)). So the input string is first + /// rounded to double precision using the underlying platform's current floating-point rounding mode before being rounded + /// to half-precision using the library's half-precision rounding mode. + /// \param in input stream to read from + /// \param arg half to read into + /// \return reference to input stream + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template std::basic_istream& operator>>(std::basic_istream &in, half &arg) { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t f; + #else + double f; + #endif + if(in >> f) + arg.data_ = detail::float2half(f); + return in; + } + + /// \} + /// \anchor basic + /// \name Basic mathematical operations + /// \{ + + /// Absolute value. + /// **See also:** Documentation for [std::fabs](https://en.cppreference.com/w/cpp/numeric/math/fabs). + /// \param arg operand + /// \return absolute value of \a arg + inline constexpr half fabs(half arg) { return half(detail::binary, arg.data_&0x7FFF); } + + /// Absolute value. + /// **See also:** Documentation for [std::abs](https://en.cppreference.com/w/cpp/numeric/math/fabs). + /// \param arg operand + /// \return absolute value of \a arg + inline constexpr half abs(half arg) { return fabs(arg); } + + /// Remainder of division. + /// **See also:** Documentation for [std::fmod](https://en.cppreference.com/w/cpp/numeric/math/fmod). + /// \param x first operand + /// \param y second operand + /// \return remainder of floating-point division. + /// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN + inline half fmod(half x, half y) { + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, sign = x.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==0x7C00) ? detail::invalid() : x.data_); + if(!absy) + return half(detail::binary, detail::invalid()); + if(!absx) + return x; + if(absx == absy) + return half(detail::binary, sign); + return half(detail::binary, sign|detail::mod(absx, absy)); + } + + /// Remainder of division. + /// **See also:** Documentation for [std::remainder](https://en.cppreference.com/w/cpp/numeric/math/remainder). + /// \param x first operand + /// \param y second operand + /// \return remainder of floating-point division. + /// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN + inline half remainder(half x, half y) { + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, sign = x.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==0x7C00) ? detail::invalid() : x.data_); + if(!absy) + return half(detail::binary, detail::invalid()); + if(absx == absy) + return half(detail::binary, sign); + return half(detail::binary, sign^detail::mod(absx, absy)); + } + + /// Remainder of division. + /// **See also:** Documentation for [std::remquo](https://en.cppreference.com/w/cpp/numeric/math/remquo). + /// \param x first operand + /// \param y second operand + /// \param quo address to store some bits of quotient at + /// \return remainder of floating-point division. + /// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN + inline half remquo(half x, half y, int *quo) { + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, value = x.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==0x7C00) ? detail::invalid() : (*quo = 0, x.data_)); + if(!absy) + return half(detail::binary, detail::invalid()); + bool qsign = ((value^y.data_)&0x8000) != 0; + int q = 1; + if(absx != absy) + value ^= detail::mod(absx, absy, &q); + return *quo = qsign ? -q : q, half(detail::binary, value); + } + + /// Fused multiply add. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::fma](https://en.cppreference.com/w/cpp/numeric/math/fma). + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return ( \a x * \a y ) + \a z rounded as one operation. + /// \exception FE_INVALID according to operator*() and operator+() unless any argument is a quiet NaN and no argument is a signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding the final addition + inline half fma(half x, half y, half z) { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t fx = detail::half2float(x.data_), fy = detail::half2float(y.data_), fz = detail::half2float(z.data_); + #if FP_FAST_FMA + return half(detail::binary, detail::float2half(std::fma(fx, fy, fz))); + #else + return half(detail::binary, detail::float2half(fx*fy+fz)); + #endif + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, absz = z.data_ & 0x7FFF, exp = -15; + unsigned int sign = (x.data_^y.data_) & 0x8000; + bool sub = ((sign^z.data_)&0x8000) != 0; + if(absx >= 0x7C00 || absy >= 0x7C00 || absz >= 0x7C00) + return (absx>0x7C00 || absy>0x7C00 || absz>0x7C00) ? half(detail::binary, detail::signal(x.data_, y.data_, z.data_)) : + (absx==0x7C00) ? half(detail::binary, (!absy || (sub && absz==0x7C00)) ? detail::invalid() : (sign|0x7C00)) : + (absy==0x7C00) ? half(detail::binary, (!absx || (sub && absz==0x7C00)) ? detail::invalid() : (sign|0x7C00)) : z; + if(!absx || !absy) + return absz ? z : half(detail::binary, (half::round_style==std::round_toward_neg_infinity) ? (z.data_|sign) : (z.data_&sign)); + for(; absx<0x400; absx<<=1,--exp) ; + for(; absy<0x400; absy<<=1,--exp) ; + detail::uint32 m = static_cast((absx&0x3FF)|0x400) * static_cast((absy&0x3FF)|0x400); + int i = m >> 21; + exp += (absx>>10) + (absy>>10) + i; + m <<= 3 - i; + if(absz) { + int expz = 0; + for(; absz<0x400; absz<<=1,--expz) ; + expz += absz >> 10; + detail::uint32 mz = static_cast((absz&0x3FF)|0x400) << 13; + if(expz > exp || (expz == exp && mz > m)) { + std::swap(m, mz); + std::swap(exp, expz); + if(sub) + sign = z.data_ & 0x8000; + } + int d = exp - expz; + mz = (d<23) ? ((mz>>d)|((mz&((static_cast(1)<(half::round_style==std::round_toward_neg_infinity)<<15); + for(; m<0x800000; m<<=1,--exp) ; + } else { + m += mz; + i = m >> 24; + m = (m>>i) | (m&i); + exp += i; + } + } + if(exp > 30) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -10) + return half(detail::binary, detail::underflow(sign)); + return half(detail::binary, detail::fixed2half(m, exp-1, sign)); + #endif + } + + /// Maximum of half expressions. + /// **See also:** Documentation for [std::fmax](https://en.cppreference.com/w/cpp/numeric/math/fmax). + /// \param x first operand + /// \param y second operand + /// \return maximum of operands, ignoring quiet NaNs + /// \exception FE_INVALID if \a x or \a y is signaling NaN + inline constexpr_NOERR half fmax(half x, half y) { + return half(detail::binary, (!isnan(y) && (isnan(x) || (x.data_^(0x8000|(0x8000-(x.data_>>15)))) < + (y.data_^(0x8000|(0x8000-(y.data_>>15)))))) ? detail::select(y.data_, x.data_) : detail::select(x.data_, y.data_)); + } + + /// Minimum of half expressions. + /// **See also:** Documentation for [std::fmin](https://en.cppreference.com/w/cpp/numeric/math/fmin). + /// \param x first operand + /// \param y second operand + /// \return minimum of operands, ignoring quiet NaNs + /// \exception FE_INVALID if \a x or \a y is signaling NaN + inline constexpr_NOERR half fmin(half x, half y) { + return half(detail::binary, (!isnan(y) && (isnan(x) || (x.data_^(0x8000|(0x8000-(x.data_>>15)))) > + (y.data_^(0x8000|(0x8000-(y.data_>>15)))))) ? detail::select(y.data_, x.data_) : detail::select(x.data_, y.data_)); + } + + /// Positive difference. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::fdim](https://en.cppreference.com/w/cpp/numeric/math/fdim). + /// \param x first operand + /// \param y second operand + /// \return \a x - \a y or 0 if difference negative + /// \exception FE_... according to operator-(half,half) + inline half fdim(half x, half y) { + if(isnan(x) || isnan(y)) + return half(detail::binary, detail::signal(x.data_, y.data_)); + return (x.data_^(0x8000|(0x8000-(x.data_>>15)))) <= (y.data_^(0x8000|(0x8000-(y.data_>>15)))) ? half(detail::binary, 0) : (x-y); + } + + /// Get NaN value. + /// **See also:** Documentation for [std::nan](https://en.cppreference.com/w/cpp/numeric/math/nan). + /// \param arg string code + /// \return quiet NaN + inline half nanh(const char *arg) { + unsigned int value = 0x7FFF; + while(*arg) + value ^= static_cast(*arg++) & 0xFF; + return half(detail::binary, value); + } + + /// \} + /// \anchor exponential + /// \name Exponential functions + /// \{ + + /// Exponential function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::exp](https://en.cppreference.com/w/cpp/numeric/math/exp). + /// \param arg function argument + /// \return e raised to \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half exp(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::exp(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? (0x7C00&((arg.data_>>15)-1U)) : detail::signal(arg.data_)); + if(abs >= 0x4C80) + return half(detail::binary, (arg.data_&0x8000) ? detail::underflow() : detail::overflow()); + detail::uint32 m = detail::multiply64(static_cast((abs&0x3FF)+((abs>0x3FF)<<10))<<21, 0xB8AA3B29); + int e = (abs>>10) + (abs<=0x3FF), exp; + if(e < 14) { + exp = 0; + m >>= 14 - e; + } else { + exp = m >> (45-e); + m = (m<<(e-14)) & 0x7FFFFFFF; + } + return half(detail::binary, detail::exp2_post(detail::exp2(m, 26), exp, (arg.data_&0x8000)!=0)); + #endif + } + + /// Binary exponential. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::exp2](https://en.cppreference.com/w/cpp/numeric/math/exp2). + /// \param arg function argument + /// \return 2 raised to \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half exp2(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::exp2(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? (0x7C00&((arg.data_>>15)-1U)) : detail::signal(arg.data_)); + if(abs >= 0x4E40) + return half(detail::binary, (arg.data_&0x8000) ? detail::underflow() : detail::overflow()); + int e = (abs>>10) + (abs<=0x3FF), exp = (abs&0x3FF) + ((abs>0x3FF)<<10); + detail::uint32 m = detail::exp2((static_cast(exp)<<(6+e))&0x7FFFFFFF, 28); + exp >>= 25 - e; + if(m == 0x80000000) { + if(arg.data_&0x8000) + exp = -exp; + else if(exp > 15) + return half(detail::binary, detail::overflow()); + return half(detail::binary, detail::fixed2half(m, exp+14)); + } + return half(detail::binary, detail::exp2_post(m, exp, (arg.data_&0x8000)!=0)); + #endif + } + + /// Exponential minus one. + /// This function may be 1 ULP off the correctly rounded exact result in <0.05% of inputs for `std::round_to_nearest` + /// and in <1% of inputs for any other rounding mode. + /// + /// **See also:** Documentation for [std::expm1](https://en.cppreference.com/w/cpp/numeric/math/expm1). + /// \param arg function argument + /// \return e raised to \a arg and subtracted by 1 + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half expm1(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::expm1(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? (0x7C00+(sign>>1)) : detail::signal(arg.data_)); + if(abs >= 0x4A00) + return half(detail::binary, (arg.data_&0x8000) ? detail::rounded(0xBBFF, 1, 1) : detail::overflow()); + detail::uint32 m = detail::multiply64(static_cast((abs&0x3FF)+((abs>0x3FF)<<10))<<21, 0xB8AA3B29); + int e = (abs>>10) + (abs<=0x3FF), exp; + if(e < 14) { + exp = 0; + m >>= 14 - e; + } else { + exp = m >> (45-e); + m = (m<<(e-14)) & 0x7FFFFFFF; + } + m = detail::exp2(m); + if(sign) { + int s = 0; + if(m > 0x80000000) { + ++exp; + m = detail::divide64(0x80000000, m, s); + } + m = 0x80000000 - ((m>>exp)|((m&((static_cast(1)<>exp) : 1; + for(exp+=14; m<0x80000000 && exp; m<<=1,--exp) ; + if(exp > 29) + return half(detail::binary, detail::overflow()); + return half(detail::binary, detail::rounded(sign+(exp<<10)+(m>>21), (m>>20)&1, (m&0xFFFFF)!=0)); + #endif + } + + /// Natural logarithm. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::log](https://en.cppreference.com/w/cpp/numeric/math/log). + /// \param arg function argument + /// \return logarithm of \a arg to base e + /// \exception FE_INVALID for signaling NaN or negative argument + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::log(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(arg.data_ & 0x8000) + return half(detail::binary, (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs >= 0x7C00) + return (abs==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + return half(detail::binary, detail::log2_post( + detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 27)+8, exp, 17)); + #endif + } + + /// Common logarithm. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::log10](https://en.cppreference.com/w/cpp/numeric/math/log10). + /// \param arg function argument + /// \return logarithm of \a arg to base 10 + /// \exception FE_INVALID for signaling NaN or negative argument + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log10(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::log10(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(arg.data_ & 0x8000) + return half(detail::binary, (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs >= 0x7C00) + return (abs==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + switch(abs) { + case 0x4900: return half(detail::binary, 0x3C00); + case 0x5640: return half(detail::binary, 0x4000); + case 0x63D0: return half(detail::binary, 0x4200); + case 0x70E2: return half(detail::binary, 0x4400); + } + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + return half(detail::binary, detail::log2_post( + detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 27)+8, exp, 16)); + #endif + } + + /// Binary logarithm. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::log2](https://en.cppreference.com/w/cpp/numeric/math/log2). + /// \param arg function argument + /// \return logarithm of \a arg to base 2 + /// \exception FE_INVALID for signaling NaN or negative argument + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log2(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::log2(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15, s = 0; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(arg.data_ & 0x8000) + return half(detail::binary, (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs >= 0x7C00) + return (abs==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + if(abs == 0x3C00) + return half(detail::binary, 0); + for(; abs<0x400; abs<<=1,--exp) ; + exp += (abs>>10); + if(!(abs&0x3FF)) { + unsigned int value = static_cast(exp<0) << 15, m = std::abs(exp) << 6; + for(exp=18; m<0x400; m<<=1,--exp) ; + return half(detail::binary, value+(exp<<10)+m); + } + detail::uint32 ilog = exp, sign = detail::sign_mask(ilog), m = + (((ilog<<27)+(detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 28)>>4))^sign) - sign; + if(!m) + return half(detail::binary, 0); + for(exp=14; m<0x8000000 && exp; m<<=1,--exp) ; + for(; m>0xFFFFFFF; m>>=1,++exp) + s |= m & 1; + return half(detail::binary, detail::fixed2half(m, exp, sign&0x8000, s)); + #endif + } + + /// Natural logarithm plus one. + /// This function may be 1 ULP off the correctly rounded exact result in <0.05% of inputs for `std::round_to_nearest` + /// and in ~1% of inputs for any other rounding mode. + /// + /// **See also:** Documentation for [std::log1p](https://en.cppreference.com/w/cpp/numeric/math/log1p). + /// \param arg function argument + /// \return logarithm of \a arg plus 1 to base e + /// \exception FE_INVALID for signaling NaN or argument <-1 + /// \exception FE_DIVBYZERO for -1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log1p(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::log1p(detail::half2float(arg.data_)))); + #else + if(arg.data_ >= 0xBC00) + return half(detail::binary, (arg.data_==0xBC00) ? detail::pole(0x8000) : (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + detail::uint32 m = static_cast((abs&0x3FF)|0x400) << 20; + if(arg.data_ & 0x8000) { + m = 0x40000000 - (m>>-exp); + for(exp=0; m<0x40000000; m<<=1,--exp) ; + } else { + if(exp < 0) { + m = 0x40000000 + (m>>-exp); + exp = 0; + } else { + m += 0x40000000 >> exp; + int i = m >> 31; + m >>= i; + exp += i; + } + } + return half(detail::binary, detail::log2_post(detail::log2(m), exp, 17)); + #endif + } + + /// \} + /// \anchor power + /// \name Power functions + /// \{ + + /// Square root. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::sqrt](https://en.cppreference.com/w/cpp/numeric/math/sqrt). + /// \param arg function argument + /// \return square root of \a arg + /// \exception FE_INVALID for signaling NaN and negative arguments + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half sqrt(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::sqrt(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = 15; + if(!abs || arg.data_ >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (arg.data_>0x8000) ? detail::invalid() : arg.data_); + for(; abs<0x400; abs<<=1,--exp) ; + detail::uint32 r = static_cast((abs&0x3FF)|0x400) << 10, m = detail::sqrt<20>(r, exp+=abs>>10); + return half(detail::binary, detail::rounded((exp<<10)+(m&0x3FF), r>m, r!=0)); + #endif + } + + /// Cubic root. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::cbrt](https://en.cppreference.com/w/cpp/numeric/math/cbrt). + /// \param arg function argument + /// \return cubic root of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half cbrt(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::cbrt(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs || abs == 0x3C00 || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1, --exp); + detail::uint32 ilog = exp + (abs>>10), sign = detail::sign_mask(ilog), f, m = + (((ilog<<27)+(detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 24)>>4))^sign) - sign; + for(exp=2; m<0x80000000; m<<=1,--exp) ; + m = detail::multiply64(m, 0xAAAAAAAB); + int i = m >> 31, s; + exp += i; + m <<= 1 - i; + if(exp < 0) { + f = m >> -exp; + exp = 0; + } else { + f = (m<> (31-exp); + } + m = detail::exp2(f, (half::round_style==std::round_to_nearest) ? 29 : 26); + if(sign) { + if(m > 0x80000000) { + m = detail::divide64(0x80000000, m, s); + ++exp; + } + exp = -exp; + } + return half(detail::binary, (half::round_style==std::round_to_nearest) ? + detail::fixed2half(m, exp+14, arg.data_&0x8000) : + detail::fixed2half((m+0x80)>>8, exp+14, arg.data_&0x8000)); + #endif + } + + /// Hypotenuse function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::hypot](https://en.cppreference.com/w/cpp/numeric/math/hypot). + /// \param x first argument + /// \param y second argument + /// \return square root of sum of squares without internal over- or underflows + /// \exception FE_INVALID if \a x or \a y is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding of the final square root + inline half hypot(half x, half y) { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t fx = detail::half2float(x.data_), fy = detail::half2float(y.data_); + return half(detail::binary, detail::float2half(std::hypot(fx, fy))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, expx = 0, expy = 0; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx==0x7C00) ? detail::select(0x7C00, y.data_) : + (absy==0x7C00) ? detail::select(0x7C00, x.data_) : detail::signal(x.data_, y.data_)); + if(!absx) + return half(detail::binary, absy ? detail::check_underflow(absy) : 0); + if(!absy) + return half(detail::binary, detail::check_underflow(absx)); + if(absy > absx) + std::swap(absx, absy); + for(; absx<0x400; absx<<=1,--expx) ; + for(; absy<0x400; absy<<=1,--expy) ; + detail::uint32 mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400; + mx *= mx; + my *= my; + int ix = mx >> 21, iy = my >> 21; + expx = 2*(expx+(absx>>10)) - 15 + ix; + expy = 2*(expy+(absy>>10)) - 15 + iy; + mx <<= 10 - ix; + my <<= 10 - iy; + int d = expx - expy; + my = (d<30) ? ((my>>d)|((my&((static_cast(1)<(mx+my, expx)); + #endif + } + + /// Hypotenuse function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::hypot](https://en.cppreference.com/w/cpp/numeric/math/hypot). + /// \param x first argument + /// \param y second argument + /// \param z third argument + /// \return square root of sum of squares without internal over- or underflows + /// \exception FE_INVALID if \a x, \a y or \a z is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding of the final square root + inline half hypot(half x, half y, half z) { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t fx = detail::half2float(x.data_), fy = detail::half2float(y.data_), fz = detail::half2float(z.data_); + return half(detail::binary, detail::float2half(std::sqrt(fx*fx+fy*fy+fz*fz))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, absz = z.data_ & 0x7FFF, expx = 0, expy = 0, expz = 0; + if(!absx) + return hypot(y, z); + if(!absy) + return hypot(x, z); + if(!absz) + return hypot(x, y); + if(absx >= 0x7C00 || absy >= 0x7C00 || absz >= 0x7C00) + return half(detail::binary, (absx==0x7C00) ? detail::select(0x7C00, detail::select(y.data_, z.data_)) : + (absy==0x7C00) ? detail::select(0x7C00, detail::select(x.data_, z.data_)) : + (absz==0x7C00) ? detail::select(0x7C00, detail::select(x.data_, y.data_)) : + detail::signal(x.data_, y.data_, z.data_)); + if(absz > absy) + std::swap(absy, absz); + if(absy > absx) + std::swap(absx, absy); + if(absz > absy) + std::swap(absy, absz); + for(; absx<0x400; absx<<=1,--expx) ; + for(; absy<0x400; absy<<=1,--expy) ; + for(; absz<0x400; absz<<=1,--expz) ; + detail::uint32 mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400, mz = (absz&0x3FF) | 0x400; + mx *= mx; + my *= my; + mz *= mz; + int ix = mx >> 21, iy = my >> 21, iz = mz >> 21; + expx = 2*(expx+(absx>>10)) - 15 + ix; + expy = 2*(expy+(absy>>10)) - 15 + iy; + expz = 2*(expz+(absz>>10)) - 15 + iz; + mx <<= 10 - ix; + my <<= 10 - iy; + mz <<= 10 - iz; + int d = expy - expz; + mz = (d<30) ? ((mz>>d)|((mz&((static_cast(1)<>1) | (my&1); + if(++expy > expx) { + std::swap(mx, my); + std::swap(expx, expy); + } + } + d = expx - expy; + my = (d<30) ? ((my>>d)|((my&((static_cast(1)<(mx+my, expx)); + #endif + } + + /// Power function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in ~0.00025% of inputs. + /// + /// **See also:** Documentation for [std::pow](https://en.cppreference.com/w/cpp/numeric/math/pow). + /// \param x base + /// \param y exponent + /// \return \a x raised to \a y + /// \exception FE_INVALID if \a x or \a y is signaling NaN or if \a x is finite an negative and \a y is finite and not integral + /// \exception FE_DIVBYZERO if \a x is 0 and \a y is negative + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half pow(half x, half y) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::pow(detail::half2float(x.data_), detail::half2float(y.data_)))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = -15; + if(!absy || x.data_ == 0x3C00) + return half(detail::binary, detail::select(0x3C00, (x.data_==0x3C00) ? y.data_ : x.data_)); + bool is_int = absy >= 0x6400 || (absy>=0x3C00 && !(absy&((1<<(25-(absy>>10)))-1))); + unsigned int sign = x.data_ & (static_cast((absy<0x6800)&&is_int&&((absy>>(25-(absy>>10)))&1))<<15); + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absy==0x7C00) ? ((absx==0x3C00) ? 0x3C00 : (!absx && y.data_==0xFC00) ? detail::pole() : + (0x7C00&-((y.data_>>15)^(absx>0x3C00)))) : (sign|(0x7C00&((y.data_>>15)-1U)))); + if(!absx) + return half(detail::binary, (y.data_&0x8000) ? detail::pole(sign) : sign); + if((x.data_&0x8000) && !is_int) + return half(detail::binary, detail::invalid()); + if(x.data_ == 0xBC00) + return half(detail::binary, sign|0x3C00); + if(y.data_ == 0x3800) + return sqrt(x); + if(y.data_ == 0x3C00) + return half(detail::binary, detail::check_underflow(x.data_)); + if(y.data_ == 0x4000) + return x * x; + for(; absx<0x400; absx<<=1,--exp) ; + detail::uint32 ilog = exp + (absx>>10), msign = detail::sign_mask(ilog), f, m = + (((ilog<<27)+((detail::log2(static_cast((absx&0x3FF)|0x400)<<20)+8)>>4))^msign) - msign; + for(exp=-11; m<0x80000000; m<<=1,--exp) ; + for(; absy<0x400; absy<<=1,--exp) ; + m = detail::multiply64(m, static_cast((absy&0x3FF)|0x400)<<21); + int i = m >> 31; + exp += (absy>>10) + i; + m <<= 1 - i; + if(exp < 0) { + f = m >> -exp; + exp = 0; + } else { + f = (m<> (31-exp); + } + return half(detail::binary, detail::exp2_post(detail::exp2(f), exp, ((msign&1)^(y.data_>>15))!=0, sign)); + #endif + } + + /// \} + /// \anchor trigonometric + /// \name Trigonometric functions + /// \{ + + /// Compute sine and cosine simultaneously. + /// This returns the same results as sin() and cos() but is faster than calling each function individually. + /// + /// This function is exact to rounding for all rounding modes. + /// \param arg function argument + /// \param sin variable to take sine of \a arg + /// \param cos variable to take cosine of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline void sincos(half arg, half *sin, half *cos) { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t f = detail::half2float(arg.data_); + *sin = half(detail::binary, detail::float2half(std::sin(f))); + *cos = half(detail::binary, detail::float2half(std::cos(f))); + #else + int abs = arg.data_ & 0x7FFF, sign = arg.data_ >> 15, k; + if(abs >= 0x7C00) + *sin = *cos = half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + else if(!abs) { + *sin = arg; + *cos = half(detail::binary, 0x3C00); + } else if(abs < 0x2500) { + *sin = half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x3BFF, 1, 1)); + } else { + if(half::round_style != std::round_to_nearest) { + switch(abs) { + case 0x48B7: + *sin = half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x1D07, 1, 1)); + *cos = half(detail::binary, detail::rounded(0xBBFF, 1, 1)); + return; + case 0x598C: + *sin = half(detail::binary, detail::rounded((arg.data_&0x8000)|0x3BFF, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x80FC, 1, 1)); + return; + case 0x6A64: + *sin = half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x3BFE, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x27FF, 1, 1)); + return; + case 0x6D8C: + *sin = half(detail::binary, detail::rounded((arg.data_&0x8000)|0x0FE6, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x3BFF, 1, 1)); + return; + } + } + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); + switch(k & 3) { + case 1: sc = std::make_pair(sc.second, -sc.first); break; + case 2: sc = std::make_pair(-sc.first, -sc.second); break; + case 3: sc = std::make_pair(-sc.second, sc.first); break; + } + *sin = half(detail::binary, detail::fixed2half((sc.first^-static_cast(sign))+sign)); + *cos = half(detail::binary, detail::fixed2half(sc.second)); + } + #endif + } + + /// Sine function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::sin](https://en.cppreference.com/w/cpp/numeric/math/sin). + /// \param arg function argument + /// \return sine value of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half sin(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::sin(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, k; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2900) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + if(half::round_style != std::round_to_nearest) + switch(abs) { + case 0x48B7: return half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x1D07, 1, 1)); + case 0x6A64: return half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x3BFE, 1, 1)); + case 0x6D8C: return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x0FE6, 1, 1)); + } + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); + detail::uint32 sign = -static_cast(((k>>1)&1)^(arg.data_>>15)); + return half(detail::binary, detail::fixed2half((((k&1) ? sc.second : sc.first)^sign) - sign)); + #endif + } + + /// Cosine function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::cos](https://en.cppreference.com/w/cpp/numeric/math/cos). + /// \param arg function argument + /// \return cosine value of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half cos(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::cos(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, k; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2500) + return half(detail::binary, detail::rounded(0x3BFF, 1, 1)); + if(half::round_style != std::round_to_nearest && abs == 0x598C) + return half(detail::binary, detail::rounded(0x80FC, 1, 1)); + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); + detail::uint32 sign = -static_cast(((k>>1)^k)&1); + return half(detail::binary, detail::fixed2half((((k&1) ? sc.first : sc.second)^sign) - sign)); + #endif + } + + /// Tangent function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::tan](https://en.cppreference.com/w/cpp/numeric/math/tan). + /// \param arg function argument + /// \return tangent value of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half tan(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::tan(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = 13, k; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2700) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + if(half::round_style != std::round_to_nearest) + switch(abs) { + case 0x658C: return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x07E6, 1, 1)); + case 0x7330: return half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x4B62, 1, 1)); + } + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 30); + if(k & 1) + sc = std::make_pair(-sc.second, sc.first); + detail::uint32 signy = detail::sign_mask(sc.first), signx = detail::sign_mask(sc.second); + detail::uint32 my = (sc.first^signy) - signy, mx = (sc.second^signx) - signx; + for(; my<0x80000000; my<<=1,--exp) ; + for(; mx<0x80000000; mx<<=1,++exp) ; + return half(detail::binary, detail::tangent_post(my, mx, exp, (signy^signx^arg.data_)&0x8000)); + #endif + } + + /// Arc sine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::asin](https://en.cppreference.com/w/cpp/numeric/math/asin). + /// \param arg function argument + /// \return arc sine value of \a arg + /// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half asin(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::asin(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(!abs) + return arg; + if(abs >= 0x3C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (abs>0x3C00) ? detail::invalid() : + detail::rounded(sign|0x3E48, 0, 1)); + if(abs < 0x2900) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + if(half::round_style != std::round_to_nearest && (abs == 0x2B44 || abs == 0x2DC3)) + return half(detail::binary, detail::rounded(arg.data_+1, 1, 1)); + std::pair sc = detail::atan2_args(abs); + detail::uint32 m = detail::atan2(sc.first, sc.second, (half::round_style==std::round_to_nearest) ? 27 : 26); + return half(detail::binary, detail::fixed2half(m, 14, sign)); + #endif + } + + /// Arc cosine function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::acos](https://en.cppreference.com/w/cpp/numeric/math/acos). + /// \param arg function argument + /// \return arc cosine value of \a arg + /// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half acos(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::acos(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ >> 15; + if(!abs) + return half(detail::binary, detail::rounded(0x3E48, 0, 1)); + if(abs >= 0x3C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (abs>0x3C00) ? detail::invalid() : + sign ? detail::rounded(0x4248, 0, 1) : 0); + std::pair cs = detail::atan2_args(abs); + detail::uint32 m = detail::atan2(cs.second, cs.first, 28); + return half(detail::binary, detail::fixed2half(sign ? (0xC90FDAA2-m) : m, 15, 0, sign)); + #endif + } + + /// Arc tangent function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::atan](https://en.cppreference.com/w/cpp/numeric/math/atan). + /// \param arg function argument + /// \return arc tangent value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half atan(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::atan(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::rounded(sign|0x3E48, 0, 1) : detail::signal(arg.data_)); + if(abs <= 0x2700) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + int exp = (abs>>10) + (abs<=0x3FF); + detail::uint32 my = (abs&0x3FF) | ((abs>0x3FF)<<10); + detail::uint32 m = (exp>15) ? detail::atan2(my<<19, 0x20000000>>(exp-15), (half::round_style==std::round_to_nearest) ? 26 : 24) : + detail::atan2(my<<(exp+4), 0x20000000, (half::round_style==std::round_to_nearest) ? 30 : 28); + return half(detail::binary, detail::fixed2half(m, 14, sign)); + #endif + } + + /// Arc tangent function. + /// This function may be 1 ULP off the correctly rounded exact result in ~0.005% of inputs for `std::round_to_nearest`, + /// in ~0.1% of inputs for `std::round_toward_zero` and in ~0.02% of inputs for any other rounding mode. + /// + /// **See also:** Documentation for [std::atan2](https://en.cppreference.com/w/cpp/numeric/math/atan2). + /// \param y numerator + /// \param x denominator + /// \return arc tangent value + /// \exception FE_INVALID if \a x or \a y is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half atan2(half y, half x) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::atan2(detail::half2float(y.data_), detail::half2float(x.data_)))); + #else + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, signx = x.data_ >> 15, signy = y.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) { + if(absx > 0x7C00 || absy > 0x7C00) + return half(detail::binary, detail::signal(x.data_, y.data_)); + if(absy == 0x7C00) + return half(detail::binary, (absx<0x7C00) ? detail::rounded(signy|0x3E48, 0, 1) : + signx ? detail::rounded(signy|0x40B6, 0, 1) : + detail::rounded(signy|0x3A48, 0, 1)); + return (x.data_==0x7C00) ? half(detail::binary, signy) : half(detail::binary, detail::rounded(signy|0x4248, 0, 1)); + } + if(!absy) + return signx ? half(detail::binary, detail::rounded(signy|0x4248, 0, 1)) : y; + if(!absx) + return half(detail::binary, detail::rounded(signy|0x3E48, 0, 1)); + int d = (absy>>10) + (absy<=0x3FF) - (absx>>10) - (absx<=0x3FF); + if(d > (signx ? 18 : 12)) + return half(detail::binary, detail::rounded(signy|0x3E48, 0, 1)); + if(signx && d < -11) + return half(detail::binary, detail::rounded(signy|0x4248, 0, 1)); + if(!signx && d < ((half::round_style==std::round_toward_zero) ? -15 : -9)) { + for(; absy<0x400; absy<<=1,--d) ; + detail::uint32 mx = ((absx<<1)&0x7FF) | 0x800, my = ((absy<<1)&0x7FF) | 0x800; + int i = my < mx; + d -= i; + if(d < -25) + return half(detail::binary, detail::underflow(signy)); + my <<= 11 + i; + return half(detail::binary, detail::fixed2half(my/mx, d+14, signy, my%mx!=0)); + } + detail::uint32 m = detail::atan2( ((absy&0x3FF)|((absy>0x3FF)<<10))<<(19+((d<0) ? d : (d>0) ? 0 : -1)), + ((absx&0x3FF)|((absx>0x3FF)<<10))<<(19-((d>0) ? d : (d<0) ? 0 : 1))); + return half(detail::binary, detail::fixed2half(signx ? (0xC90FDAA2-m) : m, 15, signy, signx)); + #endif + } + + /// \} + /// \anchor hyperbolic + /// \name Hyperbolic functions + /// \{ + + /// Hyperbolic sine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::sinh](https://en.cppreference.com/w/cpp/numeric/math/sinh). + /// \param arg function argument + /// \return hyperbolic sine value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half sinh(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::sinh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp; + if(!abs || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + if(abs <= 0x2900) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + std::pair mm = detail::hyperbolic_args(abs, exp, (half::round_style==std::round_to_nearest) ? 29 : 27); + detail::uint32 m = mm.first - mm.second; + for(exp+=13; m<0x80000000 && exp; m<<=1,--exp) ; + unsigned int sign = arg.data_ & 0x8000; + if(exp > 29) + return half(detail::binary, detail::overflow(sign)); + return half(detail::binary, detail::fixed2half(m, exp, sign)); + #endif + } + + /// Hyperbolic cosine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::cosh](https://en.cppreference.com/w/cpp/numeric/math/cosh). + /// \param arg function argument + /// \return hyperbolic cosine value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half cosh(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::cosh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : 0x7C00); + std::pair mm = detail::hyperbolic_args(abs, exp, (half::round_style==std::round_to_nearest) ? 23 : 26); + detail::uint32 m = mm.first + mm.second, i = (~m&0xFFFFFFFF) >> 31; + m = (m>>i) | (m&i) | 0x80000000; + if((exp+=13+i) > 29) + return half(detail::binary, detail::overflow()); + return half(detail::binary, detail::fixed2half(m, exp)); + #endif + } + + /// Hyperbolic tangent. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::tanh](https://en.cppreference.com/w/cpp/numeric/math/tanh). + /// \param arg function argument + /// \return hyperbolic tangent value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half tanh(half arg) { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::tanh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (arg.data_-0x4000)); + if(abs >= 0x4500) + return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x3BFF, 1, 1)); + if(abs < 0x2700) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + if(half::round_style != std::round_to_nearest && abs == 0x2D3F) + return half(detail::binary, detail::rounded(arg.data_-3, 0, 1)); + std::pair mm = detail::hyperbolic_args(abs, exp, 27); + detail::uint32 my = mm.first - mm.second - (half::round_style!=std::round_to_nearest), mx = mm.first + mm.second, i = (~mx&0xFFFFFFFF) >> 31; + for(exp=13; my<0x80000000; my<<=1,--exp) ; + mx = (mx>>i) | 0x80000000; + return half(detail::binary, detail::tangent_post(my, mx, exp-i, arg.data_&0x8000)); + #endif + } + + /// Hyperbolic area sine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::asinh](https://en.cppreference.com/w/cpp/numeric/math/asinh). + /// \param arg function argument + /// \return area sine value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half asinh(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::asinh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if(!abs || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + if(abs <= 0x2900) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + if(half::round_style != std::round_to_nearest) + switch(abs) + { + case 0x32D4: return half(detail::binary, detail::rounded(arg.data_-13, 1, 1)); + case 0x3B5B: return half(detail::binary, detail::rounded(arg.data_-197, 1, 1)); + } + return half(detail::binary, detail::area(arg.data_)); + #endif + } + + /// Hyperbolic area cosine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::acosh](https://en.cppreference.com/w/cpp/numeric/math/acosh). + /// \param arg function argument + /// \return area cosine value of \a arg + /// \exception FE_INVALID for signaling NaN or arguments <1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half acosh(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::acosh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if((arg.data_&0x8000) || abs < 0x3C00) + return half(detail::binary, (abs<=0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs == 0x3C00) + return half(detail::binary, 0); + if(arg.data_ >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + return half(detail::binary, detail::area(arg.data_)); + #endif + } + + /// Hyperbolic area tangent. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::atanh](https://en.cppreference.com/w/cpp/numeric/math/atanh). + /// \param arg function argument + /// \return area tangent value of \a arg + /// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 + /// \exception FE_DIVBYZERO for +/-1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half atanh(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::atanh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = 0; + if(!abs) + return arg; + if(abs >= 0x3C00) + return half(detail::binary, (abs==0x3C00) ? detail::pole(arg.data_&0x8000) : (abs<=0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2700) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + detail::uint32 m = static_cast((abs&0x3FF)|((abs>0x3FF)<<10)) << ((abs>>10)+(abs<=0x3FF)+6), my = 0x80000000 + m, mx = 0x80000000 - m; + for(; mx<0x80000000; mx<<=1,++exp) ; + int i = my >= mx, s; + return half(detail::binary, detail::log2_post(detail::log2( + (detail::divide64(my>>i, mx, s)+1)>>1, 27)+0x10, exp+i-1, 16, arg.data_&0x8000)); + #endif + } + + /// \} + /// \anchor special + /// \name Error and gamma functions + /// \{ + + /// Error function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.5% of inputs. + /// + /// **See also:** Documentation for [std::erf](https://en.cppreference.com/w/cpp/numeric/math/erf). + /// \param arg function argument + /// \return error function value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half erf(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::erf(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF; + if(!abs || abs >= 0x7C00) + return (abs>=0x7C00) ? half(detail::binary, (abs==0x7C00) ? (arg.data_-0x4000) : detail::signal(arg.data_)) : arg; + if(abs >= 0x4200) + return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x3BFF, 1, 1)); + return half(detail::binary, detail::erf(arg.data_)); + #endif + } + + /// Complementary error function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.5% of inputs. + /// + /// **See also:** Documentation for [std::erfc](https://en.cppreference.com/w/cpp/numeric/math/erfc). + /// \param arg function argument + /// \return 1 minus error function value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half erfc(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::erfc(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(abs >= 0x7C00) + return (abs>=0x7C00) ? half(detail::binary, (abs==0x7C00) ? (sign>>1) : detail::signal(arg.data_)) : arg; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x4400) + return half(detail::binary, detail::rounded((sign>>1)-(sign>>15), sign>>15, 1)); + return half(detail::binary, detail::erf(arg.data_)); + #endif + } + + /// Natural logarithm of gamma function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in ~0.025% of inputs. + /// + /// **See also:** Documentation for [std::lgamma](https://en.cppreference.com/w/cpp/numeric/math/lgamma). + /// \param arg function argument + /// \return natural logarith of gamma function for \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_DIVBYZERO for 0 or negative integer arguments + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half lgamma(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::lgamma(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? 0x7C00 : detail::signal(arg.data_)); + if(!abs || arg.data_ >= 0xE400 || (arg.data_ >= 0xBC00 && !(abs&((1<<(25-(abs>>10)))-1)))) + return half(detail::binary, detail::pole()); + if(arg.data_ == 0x3C00 || arg.data_ == 0x4000) + return half(detail::binary, 0); + return half(detail::binary, detail::gamma(arg.data_)); + #endif + } + + /// Gamma function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.25% of inputs. + /// + /// **See also:** Documentation for [std::tgamma](https://en.cppreference.com/w/cpp/numeric/math/tgamma). + /// \param arg function argument + /// \return gamma function value of \a arg + /// \exception FE_INVALID for signaling NaN, negative infinity or negative integer arguments + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half tgamma(half arg) { + #if defined(HALF_ARITHMETIC_TYPE) + return half(detail::binary, detail::float2half(std::tgamma(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF; + if(!abs) + return half(detail::binary, detail::pole(arg.data_)); + if(abs >= 0x7C00) + return (arg.data_==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + if(arg.data_ >= 0xE400 || (arg.data_ >= 0xBC00 && !(abs&((1<<(25-(abs>>10)))-1)))) + return half(detail::binary, detail::invalid()); + if(arg.data_ >= 0xCA80) + return half(detail::binary, detail::underflow((1-((abs>>(25-(abs>>10)))&1))<<15)); + if(arg.data_ <= 0x100 || (arg.data_ >= 0x4900 && arg.data_ < 0x8000)) + return half(detail::binary, detail::overflow()); + if(arg.data_ == 0x3C00) + return arg; + return half(detail::binary, detail::gamma(arg.data_)); + #endif + } + + /// \} + /// \anchor rounding + /// \name Rounding + /// \{ + + /// Nearest integer not less than half value. + /// **See also:** Documentation for [std::ceil](https://en.cppreference.com/w/cpp/numeric/math/ceil). + /// \param arg half to round + /// \return nearest integer not less than \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half ceil(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer not greater than half value. + /// **See also:** Documentation for [std::floor](https://en.cppreference.com/w/cpp/numeric/math/floor). + /// \param arg half to round + /// \return nearest integer not greater than \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half floor(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer not greater in magnitude than half value. + /// **See also:** Documentation for [std::trunc](https://en.cppreference.com/w/cpp/numeric/math/trunc). + /// \param arg half to round + /// \return nearest integer not greater in magnitude than \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half trunc(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer. + /// **See also:** Documentation for [std::round](https://en.cppreference.com/w/cpp/numeric/math/round). + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half round(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer. + /// **See also:** Documentation for [std::lround](https://en.cppreference.com/w/cpp/numeric/math/round). + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases + /// \exception FE_INVALID if value is not representable as `long` + inline long lround(half arg) { return detail::half2int(arg.data_); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::rint](https://en.cppreference.com/w/cpp/numeric/math/rint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half rint(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::lrint](https://en.cppreference.com/w/cpp/numeric/math/rint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID if value is not representable as `long` + /// \exception FE_INEXACT if value had to be rounded + inline long lrint(half arg) { return detail::half2int(arg.data_); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::nearbyint](https://en.cppreference.com/w/cpp/numeric/math/nearbyint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID for signaling NaN + inline half nearbyint(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + /// Nearest integer. + /// **See also:** Documentation for [std::llround](https://en.cppreference.com/w/cpp/numeric/math/round). + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases + /// \exception FE_INVALID if value is not representable as `long long` + inline long long llround(half arg) { return detail::half2int(arg.data_); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::llrint](https://en.cppreference.com/w/cpp/numeric/math/rint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID if value is not representable as `long long` + /// \exception FE_INEXACT if value had to be rounded + inline long long llrint(half arg) { return detail::half2int(arg.data_); } + + /// \} + /// \anchor float + /// \name Floating point manipulation + /// \{ + + /// Decompress floating-point number. + /// **See also:** Documentation for [std::frexp](https://en.cppreference.com/w/cpp/numeric/math/frexp). + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return significant in range [0.5, 1) + /// \exception FE_INVALID for signaling NaN + inline half frexp(half arg, int *exp) { + *exp = 0; + unsigned int abs = arg.data_ & 0x7FFF; + if(abs >= 0x7C00 || !abs) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1,--*exp) ; + *exp += (abs>>10) - 14; + return half(detail::binary, (arg.data_&0x8000)|0x3800|(abs&0x3FF)); + } + + /// Multiply by power of two. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::scalbln](https://en.cppreference.com/w/cpp/numeric/math/scalbn). + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half scalbln(half arg, long exp) { + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(abs >= 0x7C00 || !abs) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + if(exp > 30) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -10) + return half(detail::binary, detail::underflow(sign)); + else if(exp > 0) + return half(detail::binary, sign|(exp<<10)|(abs&0x3FF)); + unsigned int m = (abs&0x3FF) | 0x400; + return half(detail::binary, detail::rounded(sign|(m>>(1-exp)), (m>>-exp)&1, (m&((1<<-exp)-1))!=0)); + } + + /// Multiply by power of two. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::scalbn](https://en.cppreference.com/w/cpp/numeric/math/scalbn). + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half scalbn(half arg, int exp) { return scalbln(arg, exp); } + + /// Multiply by power of two. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::ldexp](https://en.cppreference.com/w/cpp/numeric/math/ldexp). + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half ldexp(half arg, int exp) { return scalbln(arg, exp); } + + /// Extract integer and fractional parts. + /// **See also:** Documentation for [std::modf](https://en.cppreference.com/w/cpp/numeric/math/modf). + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part + /// \exception FE_INVALID for signaling NaN + inline half modf(half arg, half *iptr) { + unsigned int abs = arg.data_ & 0x7FFF; + if(abs > 0x7C00) { + arg = half(detail::binary, detail::signal(arg.data_)); + return *iptr = arg, arg; + } + if(abs >= 0x6400) + return *iptr = arg, half(detail::binary, arg.data_&0x8000); + if(abs < 0x3C00) + return iptr->data_ = arg.data_ & 0x8000, arg; + unsigned int exp = abs >> 10, mask = (1<<(25-exp)) - 1, m = arg.data_ & mask; + iptr->data_ = arg.data_ & ~mask; + if(!m) + return half(detail::binary, arg.data_&0x8000); + for(; m<0x400; m<<=1,--exp) ; + return half(detail::binary, (arg.data_&0x8000)|(exp<<10)|(m&0x3FF)); + } + + /// Extract exponent. + /// **See also:** Documentation for [std::ilogb](https://en.cppreference.com/w/cpp/numeric/math/ilogb). + /// \param arg number to query + /// \return floating-point exponent + /// \retval FP_ILOGB0 for zero + /// \retval FP_ILOGBNAN for NaN + /// \retval INT_MAX for infinity + /// \exception FE_INVALID for 0 or infinite values + inline int ilogb(half arg) { + int abs = arg.data_ & 0x7FFF, exp; + if(!abs || abs >= 0x7C00) { + detail::raise(FE_INVALID); + return !abs ? FP_ILOGB0 : (abs==0x7C00) ? INT_MAX : FP_ILOGBNAN; + } + for(exp=(abs>>10)-15; abs<0x200; abs<<=1,--exp) ; + return exp; + } + + /// Extract exponent. + /// **See also:** Documentation for [std::logb](https://en.cppreference.com/w/cpp/numeric/math/logb). + /// \param arg number to query + /// \return floating-point exponent + /// \exception FE_INVALID for signaling NaN + /// \exception FE_DIVBYZERO for 0 + inline half logb(half arg) { + int abs = arg.data_ & 0x7FFF, exp; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? 0x7C00 : detail::signal(arg.data_)); + for(exp=(abs>>10)-15; abs<0x200; abs<<=1,--exp) ; + unsigned int value = static_cast(exp<0) << 15; + if(exp) { + unsigned int m = std::abs(exp) << 6; + for(exp=18; m<0x400; m<<=1,--exp) ; + value |= (exp<<10) + m; + } + return half(detail::binary, value); + } + + /// Next representable value. + /// **See also:** Documentation for [std::nextafter](https://en.cppreference.com/w/cpp/numeric/math/nextafter). + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW for infinite result from finite argument + /// \exception FE_UNDERFLOW for subnormal result + inline half nextafter(half from, half to) { + int fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; + if(fabs > 0x7C00 || tabs > 0x7C00) + return half(detail::binary, detail::signal(from.data_, to.data_)); + if(from.data_ == to.data_ || !(fabs|tabs)) + return to; + if(!fabs) { + detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT); + return half(detail::binary, (to.data_&0x8000)+1); + } + unsigned int out = from.data_ + (((from.data_>>15)^static_cast( + (from.data_^(0x8000|(0x8000-(from.data_>>15))))<(to.data_^(0x8000|(0x8000-(to.data_>>15))))))<<1) - 1; + detail::raise(FE_OVERFLOW, fabs<0x7C00 && (out&0x7C00)==0x7C00); + detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT && (out&0x7C00)<0x400); + return half(detail::binary, out); + } + + /// Next representable value. + /// **See also:** Documentation for [std::nexttoward](https://en.cppreference.com/w/cpp/numeric/math/nexttoward). + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW for infinite result from finite argument + /// \exception FE_UNDERFLOW for subnormal result + inline half nexttoward(half from, long double to) { + int fabs = from.data_ & 0x7FFF; + if(fabs > 0x7C00) + return half(detail::binary, detail::signal(from.data_)); + long double lfrom = static_cast(from); + if(detail::builtin_isnan(to) || lfrom == to) + return half(static_cast(to)); + if(!fabs) { + detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT); + return half(detail::binary, (static_cast(detail::builtin_signbit(to))<<15)+1); + } + unsigned int out = from.data_ + (((from.data_>>15)^static_cast(lfrom 0x7C00; } + + /// Check if normal number. + /// **See also:** Documentation for [std::isnormal](https://en.cppreference.com/w/cpp/numeric/math/isnormal). + /// \param arg number to check + /// \retval true if normal number + /// \retval false if either subnormal, zero, infinity or NaN + inline constexpr bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); } + + /// Check sign. + /// **See also:** Documentation for [std::signbit](https://en.cppreference.com/w/cpp/numeric/math/signbit). + /// \param arg number to check + /// \retval true for negative number + /// \retval false for positive number + inline constexpr bool signbit(half arg) { return (arg.data_&0x8000) != 0; } + + /// \} + /// \anchor compfunc + /// \name Comparison + /// \{ + + /// Quiet comparison for greater than. + /// **See also:** Documentation for [std::isgreater](https://en.cppreference.com/w/cpp/numeric/math/isgreater). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else + inline constexpr bool isgreater(half x, half y) { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) > ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comparison for greater equal. + /// **See also:** Documentation for [std::isgreaterequal](https://en.cppreference.com/w/cpp/numeric/math/isgreaterequal). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else + inline constexpr bool isgreaterequal(half x, half y) { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) >= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comparison for less than. + /// **See also:** Documentation for [std::isless](https://en.cppreference.com/w/cpp/numeric/math/isless). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else + inline constexpr bool isless(half x, half y) { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) < ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comparison for less equal. + /// **See also:** Documentation for [std::islessequal](https://en.cppreference.com/w/cpp/numeric/math/islessequal). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else + inline constexpr bool islessequal(half x, half y) { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) <= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comarison for less or greater. + /// **See also:** Documentation for [std::islessgreater](https://en.cppreference.com/w/cpp/numeric/math/islessgreater). + /// \param x first operand + /// \param y second operand + /// \retval true if either less or greater + /// \retval false else + inline constexpr bool islessgreater(half x, half y) { + return x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF) && !isnan(x) && !isnan(y); + } + + /// Quiet check if unordered. + /// **See also:** Documentation for [std::isunordered](https://en.cppreference.com/w/cpp/numeric/math/isunordered). + /// \param x first operand + /// \param y second operand + /// \retval true if unordered (one or two NaN operands) + /// \retval false else + inline constexpr bool isunordered(half x, half y) { return isnan(x) || isnan(y); } + + /// \} + /// \anchor casting + /// \name Casting + /// \{ + + /// Cast to or from half-precision floating-point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the default rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s returns the argument unmodified. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + /// \exception FE_INVALID if \a T is integer type and result is not representable as \a T + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template T half_cast(U arg) { return detail::half_caster::cast(arg); } + + /// Cast to or from half-precision floating-point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the specified rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s returns the argument unmodified. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam R rounding mode to use. + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + /// \exception FE_INVALID if \a T is integer type and result is not representable as \a T + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template T half_cast(U arg) { return detail::half_caster::cast(arg); } + /// \} + + /// \} + /// \anchor errors + /// \name Error handling + /// \{ + + /// Clear exception flags. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::feclearexcept](https://en.cppreference.com/w/cpp/numeric/fenv/feclearexcept). + /// \param excepts OR of exceptions to clear + /// \retval 0 all selected flags cleared successfully + inline int feclearexcept(int excepts) { detail::errflags() &= ~excepts; return 0; } + + /// Test exception flags. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::fetestexcept](https://en.cppreference.com/w/cpp/numeric/fenv/fetestexcept). + /// \param excepts OR of exceptions to test + /// \return OR of selected exceptions if raised + inline int fetestexcept(int excepts) { return detail::errflags() & excepts; } + + /// Raise exception flags. + /// This raises the specified floating point exceptions and also invokes any additional automatic exception handling as + /// configured with the [HALF_ERRHANDLIG_...](\ref HALF_ERRHANDLING_ERRNO) preprocessor symbols. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::feraiseexcept](https://en.cppreference.com/w/cpp/numeric/fenv/feraiseexcept). + /// \param excepts OR of exceptions to raise + /// \retval 0 all selected exceptions raised successfully + inline int feraiseexcept(int excepts) { detail::errflags() |= excepts; detail::raise(excepts); return 0; } + + /// Save exception flags. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::fegetexceptflag](https://en.cppreference.com/w/cpp/numeric/fenv/feexceptflag). + /// \param flagp adress to store flag state at + /// \param excepts OR of flags to save + /// \retval 0 for success + inline int fegetexceptflag(int *flagp, int excepts) { *flagp = detail::errflags() & excepts; return 0; } + + /// Restore exception flags. + /// This only copies the specified exception state (including unset flags) without incurring any additional exception handling. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::fesetexceptflag](https://en.cppreference.com/w/cpp/numeric/fenv/feexceptflag). + /// \param flagp adress to take flag state from + /// \param excepts OR of flags to restore + /// \retval 0 for success + inline int fesetexceptflag(const int *flagp, int excepts) { detail::errflags() = (detail::errflags()|(*flagp&excepts)) & (*flagp|~excepts); return 0; } + + /// Throw C++ exceptions based on set exception flags. + /// This function manually throws a corresponding C++ exception if one of the specified flags is set, + /// no matter if automatic throwing (via [HALF_ERRHANDLING_THROW_...](\ref HALF_ERRHANDLING_THROW_INVALID)) is enabled or not. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// \param excepts OR of exceptions to test + /// \param msg error message to use for exception description + /// \throw std::domain_error if `FE_INVALID` or `FE_DIVBYZERO` is selected and set + /// \throw std::overflow_error if `FE_OVERFLOW` is selected and set + /// \throw std::underflow_error if `FE_UNDERFLOW` is selected and set + /// \throw std::range_error if `FE_INEXACT` is selected and set + inline void fethrowexcept(int excepts, const char *msg = "") { + excepts &= detail::errflags(); + if(excepts & (FE_INVALID|FE_DIVBYZERO)) + throw std::domain_error(msg); + if(excepts & FE_OVERFLOW) + throw std::overflow_error(msg); + if(excepts & FE_UNDERFLOW) + throw std::underflow_error(msg); + if(excepts & FE_INEXACT) + throw std::range_error(msg); + } + /// \} +} + + +#undef HALF_UNUSED_NOERR +#undef constexpr_NOERR +#undef HALF_TWOS_COMPLEMENT_INT +#ifdef HALF_POP_WARNINGS + #pragma warning(pop) + #undef HALF_POP_WARNINGS +#endif \ No newline at end of file diff --git a/tests/regression/flash_attention/kernel.cpp b/tests/regression/flash_attention/kernel.cpp new file mode 100644 index 00000000..430817eb --- /dev/null +++ b/tests/regression/flash_attention/kernel.cpp @@ -0,0 +1,950 @@ +#include +#include +#include +#include +#include +#include "common.h" +#include "sgemm_impl.hpp" +#include "include/gemmini.h" +#include "gemmini_mmio.h" + +#define B_ROW BM +#define B_COL BN +// FIXME +#define HEADDIM B_COL + +constexpr uint32_t ROWMAX_SETS = 3; +constexpr bool DEBUG = true; +constexpr bool WARP_SPECIALIZED = true; + +constexpr uint32_t DEV_FAKE_SMEM_START_ADDR = 0xf0000000; + +// temporary safety stop for wrong configs +static_assert(NUM_CORES == 4); +static_assert(NUM_THREADS == 8); +static_assert(NUM_WARPS == 8); + +inline void thread_block_init_sharedmem(const uint32_t tid_in_threadblock, + const uint32_t threads_per_threadblock, + float *smem_O, float *smem_rowmax, + float *smem_rowsum, + float *smem_O_row_scale) { + asm volatile("threadblock_init_sharedmem_start_%=:" ::); + + const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; + const uint32_t warp_id = tid_in_threadblock / NUM_THREADS; + const uint32_t warps_in_threadblock = threads_per_threadblock / NUM_THREADS; + + static_assert((B_ROW % NUM_THREADS) == 0, + "B_ROW must be a multiple of NUM_THREADS"); + static_assert(B_ROW < (NUM_THREADS * CORES_PER_CLUSTER * + (NUM_WARPS / (WARP_SPECIALIZED ? 2 : 1))), + "not enough warps to initialize rowmax/rowsum"); + + // each thread initializes one element in rowmax/rowsum + // multiple warps participate for the whole vector + constexpr uint32_t needed_warps = B_ROW / NUM_THREADS; + if (warp_id < needed_warps /* more warps in HW than needed? */) { + uint32_t offset = NUM_THREADS * warp_id + tid_in_warp; +#pragma GCC unroll + for (int i = 0; i < ROWMAX_SETS; i++) { + smem_rowmax[offset + i * ROWMAX_SETS] = FLT_MIN; + } + smem_rowsum[offset] = 0.0f; + smem_O_row_scale[offset] = 0.0f; + } + + // each warp clears out a row of smem_O + // FIXME: dedup this pattern +#pragma GCC unroll 1 + for (int row_offset = 0; row_offset < B_COL; + row_offset += warps_in_threadblock) { + const uint32_t row = row_offset + warp_id; + uint32_t thread_offset = HEADDIM * row + tid_in_warp; + constexpr uint32_t per_row_iter = HEADDIM / NUM_THREADS; + const float one = 0.0f; +#pragma GCC unroll + for (int i = 0; i < per_row_iter; i++) { + smem_O[thread_offset] = 0.0f; + thread_offset += NUM_THREADS; + } + } + + asm volatile("threadblock_init_sharedmem_finish_%=:" ::); +} + +inline void thread_block_copy_rowmax(const float *src, float *dest, + const uint32_t tid_in_threadblock, + const uint32_t threads_per_threadblock, + const uint32_t threadblock_id_in_cluster) { + asm volatile("threadblock_copy_rowmax_start_%=:" ::); + + const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; + const uint32_t warp_id = tid_in_threadblock / NUM_THREADS; + const uint32_t warps_in_threadblock = threads_per_threadblock / NUM_THREADS; + const uint32_t warps_per_threadblock_per_core = + warps_in_threadblock / CORES_PER_CLUSTER; + + // each thread copies one element in rowmax + // multiple warps participate for the whole vector + constexpr uint32_t num_warps = B_ROW / NUM_THREADS; + if (warp_id < num_warps) { + uint32_t offset = NUM_THREADS * warp_id + tid_in_warp; + dest[offset] = src[offset]; + } + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + + asm volatile("threadblock_copy_rowmax_finish_%=:" ::); +} + +inline void thread_block_copy_tile(const float *src, float *dest, + const uint32_t tid_in_threadblock, + const uint32_t threads_per_threadblock, + const uint32_t threadblock_id_in_cluster) { + asm volatile("threadblock_copy_tile_start_%=:" ::); + + const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; + const uint32_t warp_id = tid_in_threadblock / NUM_THREADS; + const uint32_t warps_in_threadblock = threads_per_threadblock / NUM_THREADS; + const uint32_t warps_per_threadblock_per_core = + warps_in_threadblock / CORES_PER_CLUSTER; + + // FIXME: dedup this pattern +#pragma GCC unroll 1 + for (int row_offset = 0; row_offset < B_ROW; + row_offset += warps_in_threadblock) { + const uint32_t row = row_offset + warp_id; + const uint32_t first_thread_offset = B_COL * row; + + constexpr uint32_t per_row_iter = B_COL / NUM_THREADS; + uint32_t thread_offset = first_thread_offset + tid_in_warp; +#pragma GCC unroll + for (int i = 0; i < per_row_iter; i++) { + dest[thread_offset] = src[thread_offset]; + thread_offset += NUM_THREADS; + } + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + } + + asm volatile("threadblock_copy_tile_finish_%=:" ::); +} + +template +inline float exponential_taylor_term(const float x) { + asm volatile("exponential_taylor_term_start_%=:" ::); + + float res = 1.0f; + + if constexpr (order == 1) { + res = x; + } else if constexpr (order == 2) { + res = x * x; + res /= 2.0f; + } else if constexpr (order == 3) { + res = x * x * x; + res /= 6.0f; + } + + asm volatile("exponential_taylor_term_end_%=:" ::); + return res; +} + +__attribute__((always_inline)) inline void thread_block_online_softmax( + const float *smem_S, float *smem_P, const uint32_t tid_in_threadblock, + const uint32_t threads_per_threadblock, + const uint32_t threadblock_id_in_cluster, float *smem_scratchpad, + float *smem_rowmax, float *smem_rowsum, float *smem_O_row_scale) { + asm volatile("thread_block_online_softmax_start_%=:" ::); + + const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; + const uint32_t warp_id = tid_in_threadblock / NUM_THREADS; + const uint32_t warps_in_threadblock = threads_per_threadblock / NUM_THREADS; + const uint32_t warps_per_threadblock_per_core = + warps_in_threadblock / CORES_PER_CLUSTER; + + // float ft[8]; + // asm volatile("fmv.s %0, f16" : "=f"(ft[0])); + // asm volatile("fmv.s %0, f17" : "=f"(ft[1])); + // asm volatile("fmv.s %0, f18" : "=f"(ft[2])); + // asm volatile("fmv.s %0, f19" : "=f"(ft[3])); + // asm volatile("fmv.s %0, f20" : "=f"(ft[4])); + // asm volatile("fmv.s %0, f21" : "=f"(ft[5])); + // asm volatile("fmv.s %0, f22" : "=f"(ft[6])); + // asm volatile("fmv.s %0, f23" : "=f"(ft[7])); + + float *smem_rowmax_this = smem_rowmax + B_ROW; + +#pragma GCC unroll 1 + for (int row_offset = 0; row_offset < B_ROW; + row_offset += warps_in_threadblock) { + const uint32_t row = row_offset + warp_id; + const uint32_t first_thread_offset = B_COL * row; + + // rowmax + // + // two-level tree reduction: reduce each row into NUM_THREADS intermediate + // maxes, then reduce it down to one row max + // one warp handles one row in tile + + constexpr uint32_t per_row_iter = B_COL / NUM_THREADS; + uint32_t thread_offset = first_thread_offset + tid_in_warp; + // FIXME: threadblock_id needs to be in here too + float *warp_smem = smem_scratchpad + (warp_id * NUM_THREADS); + +// #define DUMB_ROWMAX +#ifdef DUMB_ROWMAX + // FIXME remove + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + + // no tree reduction; a single thread in a warp does serialized max across + // the entire row + if (tid_in_warp == 0) { + float rowmax = smem_S[first_thread_offset]; +#pragma GCC unroll 16 + for (int i = 0; i < B_COL; i++) { + asm volatile("fmax.s %0, %1, %2" + : "=f"(rowmax) + : "f"(rowmax), "f"(smem_S[first_thread_offset + i])); + } + smem_rowmax_this[row] = rowmax; + + // update previous rowmax + // i.e. mi_new = max(mi, mij) + float prev_rowmax = smem_rowmax[row]; + // stage prev rowmax in scratchpad for warp-wide broadcast + warp_smem[0] = prev_rowmax; + asm volatile("fmax.s %0, %1, %2" + : "=f"(rowmax) + : "f"(rowmax), "f"(prev_rowmax)); + smem_rowmax[row] = rowmax; + } + +#else + static_assert((B_COL % NUM_THREADS) == 0, + "B_COL must be a multiple of NUM_THREADS"); + float per_thread_max = FLT_MIN; +#pragma GCC unroll + for (int i = 0; i < per_row_iter; i++) { + const float next = smem_S[thread_offset]; + asm volatile("fmax.s %0, %1, %2" + : "=f"(per_thread_max) + : "f"(per_thread_max), "f"(next)); + thread_offset += NUM_THREADS; + } + // stage per-thread max value in smem + warp_smem[tid_in_warp] = per_thread_max; + + // sync writes to warp_smem + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + +// #define PARALLEL_ROWMAX +#ifndef PARALLEL_ROWMAX + // elect 0-th thread to reduce all other thread's values in the warp + if (tid_in_warp == 0) { + float rowmax = per_thread_max; + for (int i = 1; i < NUM_THREADS; i++) { + float other = warp_smem[i]; + asm volatile("fmax.s %0, %1, %2" + : "=f"(rowmax) + : "f"(rowmax), "f"(other)); + } + smem_rowmax_this[row] = rowmax; + + // update previous rowmax + // i.e. mi_new = max(mi, mij) + float prev_rowmax = smem_rowmax[row]; + // stage prev rowmax in scratchpad for warp-wide broadcast + warp_smem[0] = prev_rowmax; + asm volatile("fmax.s %0, %1, %2" + : "=f"(rowmax) + : "f"(rowmax), "f"(prev_rowmax)); + smem_rowmax[row] = rowmax; + } +#else + if (warp_id < warps_in_threadblock / NUM_THREADS) { + const uint32_t row = row_offset + NUM_THREADS * warp_id + tid_in_warp; + float *const thread_smem = smem_scratchpad + (tid_in_warp * NUM_THREADS); + float rowmax = FLT_MIN; +#pragma GCC unroll + for (int i = 0; i < NUM_THREADS; i++) { + const float f = thread_smem[i]; + asm volatile("fmax.s %0, %1, %2" : "=f"(rowmax) : "f"(rowmax), "f"(f)); + } + smem_rowmax_this[row] = rowmax; + + // update previous rowmax + // i.e. mi_new = max(mi, mij) + float prev_rowmax = smem_rowmax[row]; + // stage prev rowmax in scratchpad for warp-wide broadcast + thread_smem[0] = prev_rowmax; + asm volatile("fmax.s %0, %1, %2" + : "=f"(rowmax) + : "f"(rowmax), "f"(prev_rowmax)); + smem_rowmax[row] = rowmax; + } +#endif // PARALLEL_ROWMAX +#endif // DUMB_ROWMAX + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + + // broadcast prev rowmax to all threads in the warp + // NOTE: memory consistency is a little sketchy here + const float rowmax_prev = warp_smem[0]; + const float rowmax_this = smem_rowmax_this[row]; + + // exponential + // + // B_ROW / (B_ROW * B_COL / (exp_elem * threads_per_threadblock)) + // const uint32_t row_stride = + // (exp_elem_per_thread * threads_per_threadblock) / B_COL; + + // broadcast updated rowmax to all threads in the warp + const float rowmax_new = smem_rowmax[row]; + + asm volatile("flashattn_exp_p_start_%=:" ::); + + thread_offset = first_thread_offset + tid_in_warp; +#pragma GCC unroll + for (int i = 0; i < per_row_iter; i++) { + float f0 = smem_S[thread_offset]; + + f0 -= rowmax_new; + + // 2nd-order Taylor approximation + float exp = 1.0f; + exp += exponential_taylor_term<1>(f0); + exp += exponential_taylor_term<2>(f0); + + // Store S transposed to the shared memory + + smem_P[thread_offset] = exp; + + thread_offset += NUM_THREADS; + } + + asm volatile("flashattn_exp_p_end_%=:" ::); + + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + + // rowsum + // + // two-level tree reduction, similar to rowmax + + asm volatile("flashattn_rowsum_start_%=:" ::); + + float per_thread_sum = 0.0f; + + thread_offset = first_thread_offset + tid_in_warp; +#pragma GCC unroll + for (int i = 0; i < per_row_iter; i++) { + per_thread_sum += smem_P[thread_offset]; + thread_offset += NUM_THREADS; + } + // stage per-thread sum value in smem + // FIXME: threadblock_id needs to be in here too + warp_smem = smem_scratchpad + (warp_id * NUM_THREADS); + warp_smem[tid_in_warp] = per_thread_sum; + + // sync writes to warp_smem + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + + // 0-th thread collects all other thread's values in the warp + if (tid_in_warp == 0) { + float rowsum = per_thread_sum; + for (int iter = 1; iter < NUM_THREADS; iter++) { + float other = warp_smem[iter]; + rowsum += other; + } + + const float mi_prev = rowmax_prev; + const float mi_this = rowmax_this; + + const float x = mi_prev - mi_this; + // 2nd-order Taylor approximation + float exp = 1.0f; + exp += exponential_taylor_term<1>(x); + exp += exponential_taylor_term<2>(x); + + // update rowsum + const float rowsum_prev = smem_rowsum[row]; + float rowsum_new = exp * rowsum_prev + rowsum; + + smem_rowsum[row] = rowsum_new; + } + + asm volatile("flashattn_rowsum_end_%=:" ::); + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + + // compute Oi rescale factor + // FIXME: parallelize this across threads + // + asm volatile("flashattn_rescale_factor_start_%=:" ::); + + thread_offset = first_thread_offset + tid_in_warp; +#pragma GCC unroll + for (int i = 0; i < per_row_iter; i++) { + const float mi_prev = rowmax_prev; + const float mi_new = rowmax_new; + + const float x = mi_prev - mi_new; + // 2nd-order Taylor approximation + float exp = 1.0f; + exp += exponential_taylor_term<1>(x); + exp += exponential_taylor_term<2>(x); + + // @perf: div vs. expansion on e(-x)? + smem_O_row_scale[row] = 1.0f / exp; + + thread_offset += NUM_THREADS; + } + + asm volatile("flashattn_rescale_factor_end_%=:" ::); + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + } + + asm volatile("thread_block_online_softmax_finish_%=:" ::); +} + +__attribute__((always_inline)) inline void thread_block_O_rescale( + const float *smem_O_in, float *smem_O_out, const float *smem_O_row_scale, + const uint32_t tid_in_threadblock, const uint32_t threads_per_threadblock, + const uint32_t threadblock_id_in_cluster) { + asm volatile("thread_block_O_rescale_start_%=:" ::); + + const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; + const uint32_t warp_id = tid_in_threadblock / NUM_THREADS; + const uint32_t warps_in_threadblock = threads_per_threadblock / NUM_THREADS; + const uint32_t warps_per_threadblock_per_core = + warps_in_threadblock / CORES_PER_CLUSTER; + +#pragma GCC unroll 1 + for (int row_offset = 0; row_offset < B_ROW; + row_offset += warps_in_threadblock) { + const uint32_t row = row_offset + warp_id; + const uint32_t first_thread_offset = B_COL * row; + constexpr uint32_t per_row_iter = B_COL / NUM_THREADS; + uint32_t thread_offset = first_thread_offset + tid_in_warp; + + // Oi rescale + // +#pragma GCC unroll + for (int i = 0; i < per_row_iter; i++) { + const float o = smem_O_in[thread_offset]; + const float scale = smem_O_row_scale[row]; + smem_O_out[thread_offset] = (o * scale); + + thread_offset += NUM_THREADS; + } + } + + asm volatile("thread_block_O_rescale_finish_%=:" ::); +} + +void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { + // @perf: All threads are running these compute whose result is mostly same + // across the threadblock + +#ifdef RADIANCE + constexpr uint32_t cores_per_cluster = CORES_PER_CLUSTER; +#else + constexpr uint32_t cores_per_cluster = 1; +#endif + + // FIXME: headdim not considered + constexpr uint32_t threads_per_threadblock_theoretical = + (B_ROW * B_COL) / (ELEM_PER_THREAD); + constexpr uint32_t hw_threads_per_cluster = + CORES_PER_CLUSTER * NUM_THREADS * NUM_WARPS; + // cap maximum threadblock size to # of HW threads in cluster, to prevent + // multiple "wave" invocations which slows down the kernel + constexpr uint32_t threads_per_threadblock = + (threads_per_threadblock_theoretical > hw_threads_per_cluster) + ? hw_threads_per_cluster + : threads_per_threadblock_theoretical; + constexpr uint32_t threadblocks_per_cluster = + hw_threads_per_cluster / threads_per_threadblock; + constexpr uint32_t warps_per_threadblock_per_core = + NUM_WARPS / threadblocks_per_cluster; + + const uint32_t threadblock_id = task_id / threads_per_threadblock; + const uint32_t threadblock_id_in_cluster = + threadblock_id % threadblocks_per_cluster; + const uint32_t tid_in_threadblock = task_id % threads_per_threadblock; + const uint32_t warp_id = tid_in_threadblock / NUM_THREADS; + constexpr uint32_t warps_in_threadblock = + threads_per_threadblock / NUM_THREADS; + + // warpgroup context + constexpr uint32_t threads_per_warpgroup = + threads_per_threadblock / (WARP_SPECIALIZED ? 2 : 1); + constexpr uint32_t warpgroups_per_cluster = + threadblocks_per_cluster * (WARP_SPECIALIZED ? 2 : 1); + const uint32_t warps_per_warpgroup_per_core = + NUM_WARPS / warpgroups_per_cluster; + const uint32_t warpgroup_id = task_id / threads_per_warpgroup; + const uint32_t warpgroup_id_in_cluster = + warpgroup_id % warpgroups_per_cluster; + const uint32_t tid_in_warpgroup = tid_in_threadblock % threads_per_warpgroup; + + // FIXME do proper software pipelining + // if (WARP_SPECIALIZED && warpgroup_id_in_cluster != 1) { + // return; + // } + + const uint32_t dim_seqlen = arg->dim_seqlen; + const uint32_t dim_headdim = arg->dim_headdim; + + // get global memory addresses from kernel arguments + const float *gmem_Q = reinterpret_cast(arg->addr_q); + const float *gmem_K = reinterpret_cast(arg->addr_k); + const float *gmem_V = reinterpret_cast(arg->addr_v); + float *gmem_O = reinterpret_cast(arg->addr_o); + + float *gmem_tmp_d0 = reinterpret_cast(0xd0000000UL); + float *gmem_tmp_d1 = reinterpret_cast(0xd1000000UL); + float *gmem_tmp_d2 = reinterpret_cast(0xd2000000UL); + float *gmem_tmp_d3 = reinterpret_cast(0xd3000000UL); + float *gmem_tmp_d4 = reinterpret_cast(0xd4000000UL); + float *gmem_tmp_d5 = reinterpret_cast(0xd5000000UL); + float *gmem_tmp_d6 = reinterpret_cast(0xd6000000UL); + float *gmem_tmp_d7 = reinterpret_cast(0xd7000000UL); + float *gmem_tmp_e0 = reinterpret_cast(0xe0000000UL); + float *gmem_tmp_e1 = reinterpret_cast(0xe1000000UL); + float *gmem_tmp_e2 = reinterpret_cast(0xe2000000UL); + float *gmem_tmp_e3 = reinterpret_cast(0xe3000000UL); + + // static shared memory allocation + constexpr uint32_t smem_Q_size = B_ROW * HEADDIM; + constexpr uint32_t smem_K_size = B_COL * HEADDIM; + constexpr uint32_t smem_QK_size = B_ROW * B_COL; + constexpr uint32_t smem_V_size = B_COL * HEADDIM; + constexpr uint32_t smem_O_size = B_COL * HEADDIM; + uint8_t *smem_per_threadblock = reinterpret_cast( + DEV_SMEM_START_ADDR + + sizeof(float_type) * + (smem_QK_size + smem_V_size + smem_O_size) * + threadblock_id_in_cluster); + float *smem_cursor = reinterpret_cast(DEV_FAKE_SMEM_START_ADDR); + float *smem_Q0 = smem_cursor; + smem_cursor += smem_Q_size; + float *smem_Q1 = smem_cursor; + smem_cursor += smem_Q_size; + float *smem_K0 = smem_cursor; + smem_cursor += smem_K_size; + float *smem_K1 = smem_cursor; + smem_cursor += smem_K_size; + float *smem_V0 = smem_cursor; + smem_cursor += smem_V_size; + float *smem_V1 = smem_cursor; + smem_cursor += smem_V_size; + float *smem_S0 = smem_cursor; + smem_cursor += smem_QK_size; + float *smem_S1 = smem_cursor; + smem_cursor += smem_QK_size; + float *smem_P0 = smem_S0; // in-place update + float *smem_P1 = smem_S1; // in-place update + float *smem_O0 = smem_cursor; + smem_cursor += smem_O_size; + float *smem_O1 = smem_cursor; + smem_cursor += smem_O_size; + + // allocate rowmax/rowsum storage at the end of the sharedmem address space + constexpr uint32_t smem_rowmax_size = B_ROW * ROWMAX_SETS; + constexpr uint32_t smem_rowsum_size = B_ROW; + constexpr uint32_t smem_O_row_scale_size = B_ROW; + smem_cursor = reinterpret_cast(SMEM_ADDR_END); + + smem_cursor -= smem_rowmax_size; + float *smem_rowmax_0 = smem_cursor; + smem_cursor -= smem_rowmax_size; + float *smem_rowmax_1 = smem_cursor; + smem_cursor -= smem_rowsum_size; + float *smem_rowsum_0 = smem_cursor; + smem_cursor -= smem_rowsum_size; + float *smem_rowsum_1 = smem_cursor; + smem_cursor -= smem_O_row_scale_size; + float *smem_O_row_scale_0 = smem_cursor; + smem_cursor -= smem_O_row_scale_size; + float *smem_O_row_scale_1 = smem_cursor; + + // sharedmem "scratchpad" area to put temporary data, e.g. for tree reduction + // in rowsum + // NOTE: out-of bounds is not checked + // TODO: reduce this from B_ROW to NUM_WARPS + constexpr uint32_t smem_scratchpad_size = + threads_per_warpgroup * 2 /*arbitrary slack*/; + smem_cursor -= smem_scratchpad_size; + float *smem_scratchpad_0 = smem_cursor; + smem_cursor -= smem_scratchpad_size; + float *smem_scratchpad_1 = smem_cursor; + + // select the correct buffer by warpgroup + float *smem_Q = (warpgroup_id % 2) ? smem_Q1 : smem_Q0; + float *smem_K = (warpgroup_id % 2) ? smem_K1 : smem_K0; + float *smem_V = (warpgroup_id % 2) ? smem_V1 : smem_V0; + float *smem_S = (warpgroup_id % 2) ? smem_S1 : smem_S0; + float *smem_O = (warpgroup_id % 2) ? smem_O1 : smem_O0; + float *smem_P = smem_S; + float *smem_O_row_scale = + (warpgroup_id % 2) ? smem_O_row_scale_1 : smem_O_row_scale_0; + float *smem_rowmax = (warpgroup_id % 2) ? smem_rowmax_1 : smem_rowmax_0; + float *smem_rowsum = (warpgroup_id % 2) ? smem_rowsum_1 : smem_rowsum_0; + float *smem_scratchpad = + (warpgroup_id % 2) ? smem_scratchpad_1 : smem_scratchpad_0; + + // initialize rowmax/rowsum values in sharedmem + thread_block_init_sharedmem(tid_in_warpgroup, threads_per_warpgroup, smem_O, + smem_rowmax, smem_rowsum, smem_O_row_scale); + + constexpr uint32_t global_barrier_id = NUM_WARPS - 1; // arbitrary + + // delay warpgroup 0 by 1 iteration to do ping-pong scheduling + if (warpgroup_id == 1) { + threadblock_barrier(global_barrier_id, warps_per_threadblock_per_core); + } + + // read Q and K into SMEM before the loop starts + // + static_assert(B_ROW == B_COL, "currently only supports square tiles"); + + // load Q; this stays in SMEM for the entire loop + load_tile_to_smem( + dim_seqlen, warpgroup_id, 0 /* dim_k == headdim */, gmem_Q, smem_Q, + tid_in_warpgroup); + + // load K + load_tile_to_smem( + dim_seqlen, /*tile_k=*/0, 0 /* dim_k == headdim */, gmem_K, smem_K, + tid_in_warpgroup); + + // protect write to SMEM + threadblock_barrier(warpgroup_id_in_cluster, warps_per_warpgroup_per_core); + + asm volatile ("tile_loop_start_%=:" :: ); + + // "inner loop" along the columns of K^T + const uint32_t k_tiles = (dim_seqlen / B_COL); + for (uint32_t tile_k = 0; tile_k < k_tiles; tile_k++) { + // float *smem_P_produce = (tile_k % 2) ? smem_P0 : smem_P1; + // float *smem_P_consume = (tile_k % 2) ? smem_P1 : smem_P0; + // float *smem_V_produce = (tile_k % 2) ? smem_V0 : smem_V1; + // float *smem_V_consume = (tile_k % 2) ? smem_V1 : smem_V0; + // float *smem_O_row_scale_produce = + // (tile_k % 2) ? smem_O_row_scale_0 : smem_O_row_scale_1; + // float *smem_O_row_scale_consume = + // (tile_k % 2) ? smem_O_row_scale_1 : smem_O_row_scale_0; + + constexpr bool skip_gemm_qk = false; + if constexpr (!skip_gemm_qk) { + // GEMM I: S = Q*K + // + // FIXME: deduplicate this between GEMM II + if constexpr (!WARP_SPECIALIZED) { + // clear out accumulators before GEMM + initialize_accum_regs<0>(); + initialize_accum_regs<1>(); + + thread_block_gemm_single_tile< + float, MemLayout::MN_major, MemLayout::MN_major, B_ROW, B_COL, + HEADDIM, /*leading_dim_a=*/0, /*leading_dim_b=*/0, + /*load_accum=*/false, + /*write_to_smem=*/true>( + smem_Q, smem_K, nullptr /*ignore accum*/, smem_S, tid_in_warpgroup, + threads_per_warpgroup, warpgroups_per_cluster, + warpgroup_id_in_cluster); + } else { + // when warp-specialized, there's only enough warps to do 64x32 tile + // size so we need to do 2 GEMM calls + static_assert(B_ROW / 2 == 32, + "tile size assumption for warp-specialization not met"); + + // assumes smem_Q is K-major + // FIXME: fix this to MN-major + float *smem_Q_half0 = smem_Q; + float *smem_Q_half1 = smem_Q + (B_ROW / 2); // MN-major + // float *smem_Q_half1 = smem_Q + (B_ROW / 2) * HEADDIM; // K-major + float *smem_S_half0 = smem_S; + float *smem_S_half1 = smem_S + (B_ROW / 2) * B_COL; + + // clear out accumulators before GEMM + initialize_accum_regs<0>(); + initialize_accum_regs<1>(); + + // split by rows into 2 chunks + thread_block_gemm_single_tile< + float, MemLayout::MN_major, MemLayout::MN_major, B_ROW / 2, + B_COL, HEADDIM, /*leading_dim_a=*/B_ROW, /*leading_dim_b=*/0, + /*load_accum=*/false, + /*write_to_smem=*/true>( + smem_Q_half0, smem_K, nullptr /*ignore accum*/, smem_S_half0, + tid_in_warpgroup, threads_per_warpgroup, warpgroups_per_cluster, + warpgroup_id_in_cluster); + + initialize_accum_regs<0>(); + initialize_accum_regs<1>(); + + thread_block_gemm_single_tile< + float, MemLayout::MN_major, MemLayout::MN_major, B_ROW / 2, + B_COL, HEADDIM, /*leading_dim_a=*/B_ROW, /*leading_dim_b=*/0, + /*load_accum=*/false, + /*write_to_smem=*/true>( + smem_Q_half1, smem_K, nullptr /*ignore accum*/, smem_S_half1, + tid_in_warpgroup, threads_per_warpgroup, warpgroups_per_cluster, + warpgroup_id_in_cluster); + } + } else { + // load Q*K + load_tile_to_smem( + dim_seqlen, warpgroup_id /* parallelize across rows */, tile_k, + gmem_Q /*contains S*/, smem_S, tid_in_warpgroup); + } + + // protect write to SMEM (smem_S) before softmax + threadblock_barrier(warpgroup_id_in_cluster, warps_per_warpgroup_per_core); + + if constexpr (DEBUG) { + if (warpgroup_id == 0) { + if (tile_k == 0) { + thread_block_copy_tile(smem_S, gmem_tmp_d0, + tid_in_warpgroup, threads_per_warpgroup, + warpgroup_id_in_cluster); + } else if (tile_k == 1) { + thread_block_copy_tile(smem_S, gmem_tmp_d1, + tid_in_warpgroup, threads_per_warpgroup, + warpgroup_id_in_cluster); + } + + threadblock_barrier(warpgroup_id_in_cluster, + warps_per_warpgroup_per_core); + } + } + + // inter-warpgroup barrier before online softmax + threadblock_barrier(global_barrier_id, warps_per_threadblock_per_core); + + // Online softmax + // + thread_block_online_softmax(smem_S, smem_P, tid_in_warpgroup, + threads_per_warpgroup, warpgroup_id_in_cluster, + smem_scratchpad, smem_rowmax, smem_rowsum, + smem_O_row_scale); + + // data movement for K and V + // + // Q stays in SMEM for the entire loop + // + // load K for the next iteration + load_tile_to_smem( + dim_seqlen, tile_k + 1, 0 /* dim_k == headdim */, gmem_K, smem_K, + tid_in_warpgroup); + + // load V for the current iteration + // V dimension is [seqlen, headdim], stored N(headdim)-major + load_tile_to_smem( + HEADDIM, 0 /* full N-dimension */, tile_k, gmem_V, smem_V, + tid_in_warpgroup); + + // protect write to SMEM + threadblock_barrier(warpgroup_id_in_cluster, warps_per_warpgroup_per_core); + + if constexpr (DEBUG) { + if (warpgroup_id == 0) { + if (tile_k == 0) { + thread_block_copy_rowmax(smem_rowmax, gmem_tmp_e0, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + thread_block_copy_rowmax(smem_rowsum, gmem_tmp_e2, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + } else if (tile_k == 1) { + thread_block_copy_rowmax(smem_rowmax, gmem_tmp_e1, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + thread_block_copy_rowmax(smem_rowsum, gmem_tmp_e3, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + } + + threadblock_barrier(warpgroup_id_in_cluster, + warps_per_warpgroup_per_core); + } + } + + // inter-warpgroup barrier before GEMM II + threadblock_barrier(global_barrier_id, warps_per_threadblock_per_core); + + // GEMM II: O = O + P*V + + // Oi rescale + thread_block_O_rescale(smem_O, smem_O /*in-place*/, + smem_O_row_scale, tid_in_warpgroup, + threads_per_warpgroup, warpgroup_id_in_cluster); + + // rescale-to-PV-GEMM barrier + threadblock_barrier(warpgroup_id_in_cluster, warps_per_warpgroup_per_core); + + if constexpr (DEBUG) { + if (warpgroup_id == 0) { + // O before PV + if (tile_k == 0) { + thread_block_copy_tile(smem_P, gmem_tmp_d2, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + thread_block_copy_tile(smem_O, gmem_tmp_d4, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + } else if (tile_k == 1) { + thread_block_copy_tile(smem_P, gmem_tmp_d3, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + thread_block_copy_tile(smem_O, gmem_tmp_d5, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + } + + threadblock_barrier(warpgroup_id_in_cluster, + warps_per_warpgroup_per_core); + } + } + + if constexpr (!WARP_SPECIALIZED) { + // clear out accumulators before GEMM + initialize_accum_regs<0>(); + initialize_accum_regs<1>(); + + thread_block_gemm_single_tile( + smem_P, smem_V, smem_O /*load accum*/, smem_O, tid_in_warpgroup, + threads_per_warpgroup, warpgroups_per_cluster, + warpgroup_id_in_cluster); + + // FIXME: wrong but fast + // thread_block_gemm_single_tile( + // smem_P, smem_V, smem_O /*load accum*/, smem_O, + // tid_in_warpgroup, threads_per_warpgroup, + // warpgroups_per_cluster, warpgroup_id_in_cluster); + } else { + // when warp-specialized, there's only enough warps to do 64x32 tile + // size so we need to do 2 GEMM calls + static_assert(B_ROW / 2 == 32, + "tile size assumption for warp-specialization not met"); + + // assumes smem_P is K-major + float *smem_P_half0 = smem_P; + float *smem_P_half1 = smem_P + (B_ROW / 2) * B_COL; + float *smem_O_half0 = smem_O; + float *smem_O_half1 = smem_O + (B_ROW / 2) * HEADDIM; + + // clear out accumulators before GEMM + initialize_accum_regs<0>(); + initialize_accum_regs<1>(); + + // split by rows into 2 chunks + thread_block_gemm_single_tile< + float, MemLayout::K_major, MemLayout::MN_major, B_ROW / 2, HEADDIM, + B_COL, /*leading_dim_a=*/0, /*leading_dim_b=*/0, + /*load_accum=*/true, + /*write_to_smem=*/true>( + smem_P_half0, smem_V, smem_O_half0 /*load accum*/, smem_O_half0, + tid_in_warpgroup, threads_per_warpgroup, warpgroups_per_cluster, + warpgroup_id_in_cluster); + + initialize_accum_regs<0>(); + initialize_accum_regs<1>(); + + thread_block_gemm_single_tile< + float, MemLayout::K_major, MemLayout::MN_major, B_ROW / 2, HEADDIM, + B_COL, /*leading_dim_a=*/0, /*leading_dim_b=*/0, + /*load_accum=*/true, + /*write_to_smem=*/true>( + smem_P_half1, smem_V, smem_O_half1 /*load accum*/, smem_O_half1, + tid_in_warpgroup, threads_per_warpgroup, warpgroups_per_cluster, + warpgroup_id_in_cluster); + } + + threadblock_barrier(warpgroup_id_in_cluster, warps_per_warpgroup_per_core); + + if constexpr (DEBUG) { + if (warpgroup_id == 0) { + // O after PV + if (tile_k == 0) { + thread_block_copy_tile(smem_O, gmem_tmp_d6, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + } else if (tile_k == 1) { + thread_block_copy_tile(smem_O, gmem_tmp_d7, tid_in_warpgroup, + threads_per_warpgroup, + warpgroup_id_in_cluster); + } + + threadblock_barrier(warpgroup_id_in_cluster, + warps_per_warpgroup_per_core); + } + } + + tile_iter_end: + // synchronize progress of two warpgroups + // threadblock_barrier(threadblock_id_in_cluster, + // warps_per_threadblock_per_core); + // threadblock_barrier(3, // FIXME + // NUM_WARPS); + } + + asm volatile ("tile_loop_finish_%=:" :: ); + + // wait for warpgroup 1 to finish, which called the global barrier before + // entering the loop + if (warpgroup_id == 0) { + threadblock_barrier(global_barrier_id, warps_per_threadblock_per_core); + } +} + +int main() { + kernel_arg_t *arg = (kernel_arg_t *)KERNEL_ARG_DEV_MEM_ADDR; + + // FIXME:: use actuall seqlen/headdim + const uint32_t problem_size = (B_ROW * B_COL) / (ELEM_PER_THREAD); + const uint32_t hw_threads_per_cluster = + CORES_PER_CLUSTER * vx_num_threads() * vx_num_warps(); + // prevent launching more threads than the necessary problem size + // TODO: this does not take into account multiple clusters + const uint32_t grid_size = (problem_size > hw_threads_per_cluster) + ? hw_threads_per_cluster + : problem_size; + +#ifdef RADIANCE + vx_spawn_tasks_cluster(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); +#else + // NOTE: This kernel assumes contiguous thread scheduling for efficient shared + // memory allocation, and therefore does not work with original vx_spawn_tasks + vx_spawn_tasks_contiguous(grid_size, (vx_spawn_tasks_cb)kernel_body, arg); +#endif + return 0; +} diff --git a/tests/regression/flash_attention/main.cpp b/tests/regression/flash_attention/main.cpp new file mode 100644 index 00000000..b1b8d522 --- /dev/null +++ b/tests/regression/flash_attention/main.cpp @@ -0,0 +1,166 @@ +#include +#include +#include +#include +#include +#include +#include +#include "common.h" +#include "half.hpp" + +using half_float::half; +using half_float::half_cast; + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.bin"; +uint32_t count = 0; + +std::vector ref_data; + +vx_device_h device = nullptr; +std::vector staging_buf; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:h?")) != -1) { + switch (c) { + case 'n': + count = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + case '?': { + show_usage(); + exit(0); + } break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + // vx_mem_free(device, kernel_arg.addr_a); + // vx_mem_free(device, kernel_arg.addr_b); + // vx_mem_free(device, kernel_arg.addr_c); + vx_dev_close(device); + } +} + +int run_test(const kernel_arg_t& kernel_arg, + uint32_t buf_size) { + // start device + std::cout << "start device" << std::endl; + RT_CHECK(vx_start(device)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_ready_wait(device, VX_MAX_TIMEOUT)); + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + RT_CHECK(vx_copy_from_dev(device, staging_buf.data(), kernel_arg.addr_o, buf_size)); + + return 0; +} + +int main(int argc, char *argv[]) { + // parse command arguments + parse_args(argc, argv); + + if (count == 0) { + count = 1; + } + + std::srand(50); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_dev_open(&device)); + + uint32_t dim_seqlen = 128; + uint32_t dim_headdim = 64; + + using float_type = half; + + uint32_t dst_buf_size = + dim_seqlen * dim_headdim * sizeof(ref_data[0]); + + // upload program + std::cout << "upload program" << std::endl; + RT_CHECK(vx_upload_kernel_file(device, kernel_file)); + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + kernel_arg.addr_q = 0xa0000000; + kernel_arg.addr_k = 0xa1000000; + kernel_arg.addr_v = 0xa2000000; + kernel_arg.addr_o = 0xc0000000; + + kernel_arg.dim_seqlen = dim_seqlen; + kernel_arg.dim_headdim = dim_headdim; + + std::cout << "dev_addr_q=0x" << std::hex << kernel_arg.addr_q << std::endl; + std::cout << "dev_addr_k=0x" << std::hex << kernel_arg.addr_k << std::endl; + std::cout << "dev_addr_v=0x" << std::hex << kernel_arg.addr_v << std::endl; + std::cout << "dev_addr_o=0x" << std::hex << kernel_arg.addr_o << std::endl; + + // allocate staging buffer + { + std::cout << "allocate staging buffer" << std::endl; + uint32_t staging_buf_size = sizeof(kernel_arg_t); + staging_buf.resize(staging_buf_size); + } + + // upload kernel argument + { + std::cout << "upload kernel argument" << std::endl; + auto buf_ptr = staging_buf.data(); + memcpy(buf_ptr, &kernel_arg, sizeof(kernel_arg_t)); + RT_CHECK(vx_copy_to_dev(device, KERNEL_ARG_DEV_MEM_ADDR, staging_buf.data(), sizeof(kernel_arg_t))); + + std::cout << "uploading argument buffer to device, device mem address=" + << std::hex << KERNEL_ARG_DEV_MEM_ADDR << ", size=" << std::dec + << sizeof(kernel_arg_t) << " bytes\n"; + std::ofstream file("args.bin", std::ios::binary | std::ios::out); + if (!file) { + std::cerr << "error: failed to open args.bin for writing\n"; + exit(EXIT_FAILURE); + } + file.write(reinterpret_cast(staging_buf.data()), + sizeof(kernel_arg_t)); + file.close(); + } + + // run tests + std::cout << "run tests" << std::endl; + RT_CHECK(run_test(kernel_arg, dst_buf_size)); + std::cout << "PASSED!" << std::endl; + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + return 0; +} diff --git a/tests/regression/sgemm_tcore/Makefile b/tests/regression/sgemm_tcore/Makefile index deb2c1ca..30655c95 100644 --- a/tests/regression/sgemm_tcore/Makefile +++ b/tests/regression/sgemm_tcore/Makefile @@ -3,6 +3,7 @@ PROJECT = sgemm_tcore SRCS = main.cpp common.h VX_SRCS = kernel.cpp +VX_INCLUDES = sgemm_impl.hpp OPTS ?= -n16 diff --git a/tests/regression/sgemm_tcore/kernel.cpp b/tests/regression/sgemm_tcore/kernel.cpp index 8f658d67..59fd7194 100644 --- a/tests/regression/sgemm_tcore/kernel.cpp +++ b/tests/regression/sgemm_tcore/kernel.cpp @@ -3,538 +3,44 @@ #include #include #include "common.h" -#include "util.hpp" +#include "sgemm_impl.hpp" #include "include/gemmini.h" #include "gemmini_mmio.h" -#define GEMMINI_DMA 0 -#if SMEM_SIZE == 0x4000 -#define SMEM_ADDR_Q0 ((float * const) 0xff000000) -#define SMEM_ADDR_Q1 ((float * const) 0xff001000) -#define SMEM_ADDR_Q2 ((float * const) 0xff002000) -#define SMEM_ADDR_Q3 ((float * const) 0xff003000) -#define SPAD_ADDR_Q0 0x0 -#define SPAD_ADDR_Q1 0x80 -#define SPAD_ADDR_Q2 0x100 -#define SPAD_ADDR_Q3 0x180 -#define BOUND_INST 0x400040004ULL -#elif SMEM_SIZE == 0x10000 -#define SMEM_ADDR_Q0 ((float * const) 0xff000000) -#define SMEM_ADDR_Q1 ((float * const) 0xff004000) -#define SMEM_ADDR_Q2 ((float * const) 0xff008000) -#define SMEM_ADDR_Q3 ((float * const) 0xff00c000) -#define SPAD_ADDR_Q0 0x0 -#define SPAD_ADDR_Q1 0x200 -#define SPAD_ADDR_Q2 0x400 -#define SPAD_ADDR_Q3 0x600 -#define BOUND_INST 0x800080008ULL -#else -#error Unsupported smem size -#endif +constexpr bool DEBUG = true; -// FIXME: NUM_THREADS and NUM_WARPS hardcoded -#if ((BM * BN / ELEM_PER_THREAD) > (CORES_PER_CLUSTER * 8 * 8)) -#error "threadblock size too big for cluster" -#endif +template +inline void thread_block_copy_tile(const float *src, float *dest, + const uint32_t tid_in_threadblock, + const uint32_t threads_per_threadblock, + const uint32_t threadblock_id_in_cluster) { + asm volatile("threadblock_copy_tile_start_%=:" ::); -template -inline void global_dmem_load(const uint32_t dim_n, const uint32_t dim_k, - const uint32_t k, const T *A, const T *B, - volatile T *local_a, volatile T *local_b, - const uint32_t tid_in_threadblock, - const uint32_t threadblock_id_x, - const uint32_t threadblock_id_y) { - // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do - // data movement at the fp32 granularity. Assuming that the matrix is stored - // row-major in GMEM, the packed fp16 pairs belong to the same row, - // neighboring columns; therefore, it essentially becomes equivalent to - // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed - // by a factor of two. - constexpr uint32_t packed_factor = (std::is_same_v ? 2 : 1); - constexpr uint32_t BK_adjusted = BK / packed_factor; - const uint32_t dim_k_adjusted = dim_k / packed_factor; - constexpr uint32_t BN_adjusted = BN / packed_factor; - const uint32_t dim_n_adjusted = dim_n / packed_factor; - const uint32_t k_adjusted = k / packed_factor; - - const uint32_t local_a_row = tid_in_threadblock / BK_adjusted; - const uint32_t local_a_col = tid_in_threadblock % BK_adjusted; - const uint32_t local_as_row = tid_in_threadblock / BM; - const uint32_t local_as_col = tid_in_threadblock % BM; - const uint32_t local_b_row = tid_in_threadblock / BN_adjusted; - const uint32_t local_b_col = tid_in_threadblock % BN_adjusted; - - // FIXME: need fix for fp16? - constexpr uint32_t threads_in_threadblock = (BM * BN) / ELEM_PER_THREAD; - - // Data move from GMEM to SMEM - // - // Make sure global offset values for A and B are contiguous between - // neighboring threads to ensure GMEM coalescing. - // - // TODO: Sharedmem swizzling is important here - - // move A - if constexpr (!TRANSPOSE_AT_PRODUCE) { - // No transpose at GMEM->SMEM movement - // FIXME: !TRANSPOSE_AS code is old - - const uint32_t global_a_row = BM * threadblock_id_y + local_a_row; - // number of rows a full TB can read at a time - // this is equivalent to threadblock_dim_y (assuming threadblock_dim_x == - // BK) - constexpr uint32_t row_stride_a = threads_in_threadblock / BK_adjusted; - const float *global_a = reinterpret_cast(A) + - dim_k_adjusted * global_a_row + - (k_adjusted + local_a_col); - volatile float *local_a_tmp = reinterpret_cast(local_a) + - BK_adjusted * local_a_row + local_a_col; - -#pragma GCC unroll 1 - for (uint32_t local_row_offset = 0; local_row_offset < BM; - local_row_offset += row_stride_a) { - *local_a_tmp = *global_a; - - // move to the next "row-chunk", when threadblock is smaller than BM*BK - global_a += dim_k_adjusted * row_stride_a; - local_a_tmp += BK_adjusted * row_stride_a; - } - } else { - if constexpr (!GMEM_COALESCED_A) { - // !GMEM_COALESCED_A: threads do uncoalesced read from neighboring row in - // GMEM, writes to neighboring cols in SMEM - constexpr uint32_t row_stride_as = threads_in_threadblock / BM; - const uint32_t global_a_row = BM * threadblock_id_y + local_as_col; - const float *global_a = - reinterpret_cast(A) + dim_k_adjusted * global_a_row + (k_adjusted + local_as_row); - volatile float *local_a_tmp = - reinterpret_cast(local_a) + BM * local_as_row + local_as_col; - - static_assert( - row_stride_as * 8 <= BK_adjusted, - "manual loop unrolling condition not met; consider increasing BK"); - static_assert( - (BK_adjusted % (row_stride_as * 8)) == 0, - "manual loop unrolling condition not met; BK should be power-of-two"); - -#pragma GCC unroll 1 - for (uint32_t local_row_offset = 0; local_row_offset < BK_adjusted; - local_row_offset += row_stride_as * 8) { - // @perf: bank conflicts here - // const uint32_t global_a_offset = - // dim_k_adjusted * (global_a_row) + (k + local_as_row + local_row_offset); - // FIXME experimenting with global coalescing - // const uint32_t global_a_offset = - // dim_k_adjusted * (global_a_row + local_row_offset) + (k + local_as_col); - // local_a[BM * (local_as_row + local_row_offset) + local_as_col] = - // A[global_a_offset]; - - // *local_a_tmp = *global_a; - asm volatile ("flw ft0, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft1, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft2, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft3, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft4, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft5, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft6, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - asm volatile ("flw ft7, (%0)" :: "r"(global_a)); - global_a += row_stride_as; - - // NOTE: stride is fixed to word size , i.e. sizeof(float) = 4, - // regardless of fp16 or fp32. Since Vortex core does not support fp16, - // load things at word granularity and reinterpret bits inside the - // tensor core. - asm volatile ("fsw ft0, %0(%1)" :: "i"(BM * row_stride_as * 0 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft1, %0(%1)" :: "i"(BM * row_stride_as * 1 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft2, %0(%1)" :: "i"(BM * row_stride_as * 2 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft3, %0(%1)" :: "i"(BM * row_stride_as * 3 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft4, %0(%1)" :: "i"(BM * row_stride_as * 4 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft5, %0(%1)" :: "i"(BM * row_stride_as * 5 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft6, %0(%1)" :: "i"(BM * row_stride_as * 6 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft7, %0(%1)" :: "i"(BM * row_stride_as * 7 * sizeof(float)), "r"(local_a_tmp)); - local_a_tmp += BM * row_stride_as * 8; - } - } else { - constexpr uint32_t row_stride_a = threads_in_threadblock / BK_adjusted; - const uint32_t global_a_row = BM * threadblock_id_y + local_a_row; - const float *global_a = reinterpret_cast(A) + - dim_k_adjusted * global_a_row + - (k_adjusted + local_a_col); - // NOTE that SMEM writes are transposed - volatile float *local_a_tmp = - reinterpret_cast(local_a) + BM * local_a_col + - local_a_row; - - static_assert( - row_stride_a * 8 <= BM, - "manual loop unrolling condition not met; consider increasing BM"); - static_assert( - (BM % (row_stride_a * 8)) == 0, - "manual loop unrolling condition not met; BM should be power-of-two"); - -#pragma GCC unroll 1 - for (uint32_t local_row_offset = 0; local_row_offset < BM; - local_row_offset += row_stride_a * 8) { - // const uint32_t global_a_offset = - // dim_k_adjusted * (global_a_row + local_row_offset) + (k + local_a_col); - // NOTE that SMEM writes are transposed - // local_a[BM * (local_a_col) + local_a_row + local_row_offset] = - // A[global_a_offset]; - - asm volatile ("flw ft0, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft1, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft2, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft3, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft4, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft5, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft6, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - asm volatile ("flw ft7, (%0)" :: "r"(global_a)); - global_a += dim_k_adjusted * row_stride_a; - - // stride along columns - asm volatile ("fsw ft0, %0(%1)" :: "i"(row_stride_a * 0 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft1, %0(%1)" :: "i"(row_stride_a * 1 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft2, %0(%1)" :: "i"(row_stride_a * 2 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft3, %0(%1)" :: "i"(row_stride_a * 3 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft4, %0(%1)" :: "i"(row_stride_a * 4 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft5, %0(%1)" :: "i"(row_stride_a * 5 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft6, %0(%1)" :: "i"(row_stride_a * 6 * sizeof(float)), "r"(local_a_tmp)); - asm volatile ("fsw ft7, %0(%1)" :: "i"(row_stride_a * 7 * sizeof(float)), "r"(local_a_tmp)); - local_a_tmp += row_stride_a * 8; - } - } - } // end move A - - // move B - constexpr uint32_t row_stride_b = threads_in_threadblock / BN_adjusted; - const uint32_t global_b_col = BN_adjusted * threadblock_id_x + local_b_col; - // NOTE: not k_adjusted here; k is along the row dimension which is not - // compressed for fp16 - const float *global_b = reinterpret_cast(B) + - dim_n_adjusted * (k + local_b_row) + global_b_col; - volatile float *local_b_tmp = reinterpret_cast(local_b) + - BN_adjusted * local_b_row + local_b_col; - - static_assert( - row_stride_b * 8 <= BK_adjusted, - "manual loop unrolling condition not met; consider increasing BK"); - static_assert( - (BK_adjusted % (row_stride_b * 8)) == 0, - "manual loop unrolling condition not met; BK should be power-of-two"); - -#pragma GCC unroll 1 - for (uint32_t load_offset = 0; load_offset < BK; - load_offset += row_stride_b * 8) { - // equivalent code: - // - // *local_b_tmp = *global_b; - // global_b += dim_n * row_stride_b; - // local_b_tmp += BN * row_stride_b; - - asm volatile ("flw ft0, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft1, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft2, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft3, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft4, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft5, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft6, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - asm volatile ("flw ft7, (%0)" :: "r"(global_b)); - global_b += dim_n_adjusted * row_stride_b; - - asm volatile ("fsw ft0, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft1, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); - local_b_tmp += BN_adjusted * row_stride_b * 2; - asm volatile ("fsw ft2, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft3, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); - local_b_tmp += BN_adjusted * row_stride_b * 2; - asm volatile ("fsw ft4, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft5, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); - local_b_tmp += BN_adjusted * row_stride_b * 2; - asm volatile ("fsw ft6, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 0 * sizeof(float)), "r"(local_b_tmp)); - asm volatile ("fsw ft7, %0(%1)" :: "i"(BN_adjusted * row_stride_b * 1 * sizeof(float)), "r"(local_b_tmp)); - local_b_tmp += BN_adjusted * row_stride_b * 2; - } -} - -template -inline void thread_block_gemm(kernel_arg_t *__UNIFORM__ arg, - const uint32_t tid_in_threadblock, - const uint32_t threads_per_threadblock, - const uint32_t threadblock_dim_y, - /*const uint32_t threadblock_id_x, - const uint32_t threadblock_id_y,*/ - const uint32_t threadblocks_per_cluster, - const uint32_t threadblock_id_in_cluster, - uint8_t *sharedmem_per_threadblock) { - const T *A = (const T *)arg->addr_a; - const T *B = (const T *)arg->addr_b; - float *C = (float *)arg->addr_c; - - const uint32_t dim_m = arg->dim_m; - const uint32_t dim_n = arg->dim_n; - const uint32_t dim_k = arg->dim_k; - - const uint32_t local_a_row = tid_in_threadblock / BK; - const uint32_t local_a_col = tid_in_threadblock % BK; - const uint32_t local_as_row = tid_in_threadblock / BM; - const uint32_t local_as_col = tid_in_threadblock % BM; - const uint32_t local_b_row = tid_in_threadblock / BN; - const uint32_t local_b_col = tid_in_threadblock % BN; - - // no double-buffering - const uint32_t threads_per_warpgroup = threads_per_threadblock; - const uint32_t warp_id_in_warpgroup = tid_in_threadblock / NUM_THREADS; - const uint32_t warp_row = warp_id_in_warpgroup / (BN / WN); - const uint32_t warp_col = warp_id_in_warpgroup % (BN / WN); const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; - - volatile T *local_a = reinterpret_cast(sharedmem_per_threadblock); - constexpr size_t local_a_elems = (BM * BK); - volatile T *local_a_buf = local_a + local_a_elems; - - volatile T *local_b = local_a_buf + local_a_elems; - constexpr size_t local_b_elems = (BK * BN); - volatile T *local_b_buf = local_a_buf + local_b_elems; - - constexpr uint32_t skips = - loop_matmul_skips(/*skip_lda=*/0, /*skip_ldb=*/0, /*skip_ldd=*/1, - /*skip_ex=*/1, /*skip_stc=*/1); - -#if (GEMMINI_DMA == 1) - if (tid_in_threadblock == 0) { - gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); - // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, - // b_transpose); - - gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, - false, 0); - gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, - false, 1); - gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); - - gemmini_fence(); - } -#endif - - // divide rows (M) by the number of threadblocks - const uint32_t dim_m_range = (dim_m / threadblocks_per_cluster); - const uint32_t dim_m_start = dim_m_range * threadblock_id_in_cluster; - const uint32_t block_m_start = dim_m_start / BM; - const uint32_t block_m_end = (dim_m_start + dim_m_range) / BM; + const uint32_t warp_id = tid_in_threadblock / NUM_THREADS; + const uint32_t warps_in_threadblock = threads_per_threadblock / NUM_THREADS; + const uint32_t warps_per_threadblock_per_core = + warps_in_threadblock / CORES_PER_CLUSTER; #pragma GCC unroll 1 - for (uint32_t block_m = block_m_start; block_m < block_m_end; block_m++) { -#pragma GCC unroll 1 - for (uint32_t block_n = 0; (block_n * BN) < dim_n; block_n++) { - // clear out C - initialize_C(0); - initialize_C(1); + for (int row_offset = 0; row_offset < tile_dim_row; + row_offset += warps_in_threadblock) { + const uint32_t row = row_offset + warp_id; + const uint32_t first_thread_offset = tile_dim_col * row; - if constexpr (GEMMINI_DMA) { - // pipeline initiation - if (tid_in_threadblock == 0) { - // configure dma gmem address to load from - // FIXME: block_k is wrong - ROCC_INSTRUCTION_RS1_RS2( - XCUSTOM_ACC, - (uint64_t)(A + block_m * BM * dim_k + /*block_k:*/0 * BK), - (uint64_t)(B + /*block_k:*/0 * BK * dim_n + block_n * BN), - k_LOOP_WS_CONFIG_ADDRS_AB) - // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB - GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); - gemmini_fence(); - - GEMMINI_CISC_CMD_I(10); - gemmini_fence(); - -#if 0 - // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS - // FIXME: block_k is 0 for two times - sp_tiled_matmul_full_spad_ws( -#if 1 - SPAD_ADDR_Q0, SPAD_ADDR_Q1, -#else - (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q2 : SPAD_ADDR_Q0, - (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q3 : SPAD_ADDR_Q1, -#endif - /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q3, - /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, - /*pad_J=*/0, /*pad_K=*/0, - /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, - /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) - gemmini_fence(); -#endif - } - - threadblock_barrier(threadblock_id_in_cluster, threadblock_dim_y); - } - -#pragma GCC unroll 1 - for (uint32_t block_k = 0; (block_k * BK) < (dim_k); block_k++) { - - // producer code: GMEM->SMEM memory movement - // --------------------------------------------------------------------- - // - // this is either done using DMA or SIMT cores depending on GEMMINI_DMA - -#if (GEMMINI_DMA == 1) - if ((tid_in_threadblock == 0) && ((block_k * BK) != (dim_k - BK))) { - // configure dma gmem address to load from - // FIXME: block_k is wrong - ROCC_INSTRUCTION_RS1_RS2( - XCUSTOM_ACC, - (uint64_t)(A + block_m * BM * dim_k + (block_k + 1/*runahead*/) * BK), - (uint64_t)(B + (block_k + 1/*runahead*/) * BK * dim_n + block_n * BN), - k_LOOP_WS_CONFIG_ADDRS_AB) - // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB - GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); - // gemmini_fence(); - - // block_k is even: opcode 11 (write to local_a_buf) - // block_k is odd: opcode 10 (write to local_a) - const uint32_t opcode = 11 - (block_k & 1); - GEMMINI_CISC_CMD_R(opcode); - // // TODO: branch is probably slow - // if (block_k & 1) { - // GEMMINI_CISC_CMD_I(12); - // } else { // block_k == 0 is here - // GEMMINI_CISC_CMD_I(13); - // } - - // configure loop iteration bounds - // FIXME: shouldn't be necessary - // ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, BOUND_INST, - // k_LOOP_WS_CONFIG_BOUNDS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, - // SPAD_ADDR_Q0, SPAD_ADDR_Q1, k_LOOP_WS_CONFIG_SPAD_AB) - // ROCC_INSTRUCTION_RS1_RS2( - // XCUSTOM_ACC, - // ((uint64_t)(/*a_spad_id:*/ 0) << 18) | - // ((uint64_t)(/*b_spad_id:*/ 0) << 16) | - // ((uint64_t)(/*act:0*/ 0) << 8) | ((/*low_D:*/ 0) << 2) | - // ((/*full_C:*/ 0) << 1) | (/*ex_accumulate:*/ 0), - // ((uint64_t)(/*C_spad_addr:*/ A) << 32) | 0x200U | (skips) | - // ((/*is_resadd*/ 0) << 2) | ((/*B_transpose:*/ 0) << 1) | - // (/*A_transpose:*/ 1), - // k_LOOP_WS) - // gemmini_fence(); - -#if 0 - uint32_t spad_a_produce; - uint32_t spad_b_produce; - const uint32_t mask_odd = (block_k & 1) << 31 >> 31; - const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; - spad_a_produce = - ((mask_odd & (SPAD_ADDR_Q0)) | (mask_even & (SPAD_ADDR_Q2))); - spad_b_produce = - ((mask_odd & (SPAD_ADDR_Q1)) | (mask_even & (SPAD_ADDR_Q3))); - // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS - // FIXME: block_k is 0 for two times - sp_tiled_matmul_full_spad_ws( - spad_a_produce, - spad_b_produce, - /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q1, - /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, - /*pad_J=*/0, /*pad_K=*/0, - /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, - /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) -#endif - } -#else - global_dmem_load(dim_n, dim_k, block_k * BK, A, B, local_a, local_b, - tid_in_threadblock, block_n, block_m); - - threadblock_barrier(threadblock_id_in_cluster, threadblock_dim_y); -#endif - - // consumer code: SMEM->RF and compute - // ---------------------------------------------------------------------- - // @perf: this loop spills to stack a lot because of all the flws in - const volatile T *local_a_consume; - const volatile T *local_b_consume; - if constexpr (GEMMINI_DMA) { - // local_a_consume = (k_index % 2) ? local_a_buf : local_a; - // local_b_consume = (k_index % 2) ? local_b_buf : local_b; - // FIXME: swap multiply with bitshifts - // const uint32_t mask_odd = (block_k & 1) << 31 >> 31; - // const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; - // local_a_consume = reinterpret_cast( - // (mask_odd & reinterpret_cast(local_a_buf)) | - // (mask_even & reinterpret_cast(local_a))); - // local_b_consume = reinterpret_cast( - // (mask_odd & reinterpret_cast(local_b_buf)) | - // (mask_even & reinterpret_cast(local_b))); - local_a_consume = local_a + (block_k & 1) * (local_a_elems); - local_b_consume = local_b + (block_k & 1) * (local_b_elems); - } else { - // no double-buffering without DMA - local_a_consume = local_a; - local_b_consume = local_b; - } - -#pragma GCC unroll 1 - for (int i = 0; i < BK_LOOP; i++) { -#pragma GCC unroll 4 - for (uint32_t local_k = 0; local_k < BK; local_k += TCK) { -#pragma GCC unroll 2 - for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { - // SMEM -> RF - vx_wmma_load_b(local_b_consume, local_k, warp_col, wn_iter, - tid_in_warp); -#pragma GCC unroll 2 - for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { - // SMEM -> RF - vx_wmma_load_a(local_a_consume, local_k, warp_row, wm_iter, - tid_in_warp); - // perform mma - vx_wmma(wm_iter); - } - } - } - } - - if constexpr (GEMMINI_DMA) { - // Call gemmini fence at the end of the loop to overlap dma & wmma. - // Hopefully by this time, dma would have finished so that this is a - // no-op - if (tid_in_threadblock == 0) { - gemmini_fence(); - } - } - - threadblock_barrier(threadblock_id_in_cluster, threadblock_dim_y); - } - -#pragma GCC unroll 2 - for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { -#pragma GCC unroll 2 - for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { - write_results(tid_in_warp, warp_col, warp_row, wn_iter, wm_iter, - dim_n, C, block_n, block_m); - } - } + constexpr uint32_t per_row_iter = tile_dim_col / NUM_THREADS; + uint32_t thread_offset = first_thread_offset + tid_in_warp; +#pragma GCC unroll + for (int i = 0; i < per_row_iter; i++) { + dest[thread_offset] = src[thread_offset]; + thread_offset += NUM_THREADS; } + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); } + + asm volatile("threadblock_copy_tile_finish_%=:" ::); } void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { @@ -547,18 +53,21 @@ void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { constexpr uint32_t cores_per_cluster = 1; #endif - uint32_t threads_per_threadblock = (BM * BN) / (ELEM_PER_THREAD); - const uint32_t hw_threads_per_cluster = - cores_per_cluster * vx_num_threads() * vx_num_warps(); + constexpr uint32_t threads_per_threadblock_theoretical = + (BM * BN) / (ELEM_PER_THREAD); + constexpr uint32_t hw_threads_per_cluster = + CORES_PER_CLUSTER * NUM_THREADS * NUM_WARPS; // cap maximum threadblock size to # of HW threads in cluster, to prevent // multiple "wave" invocations which slows down the kernel - if (threads_per_threadblock > hw_threads_per_cluster) { - threads_per_threadblock = hw_threads_per_cluster; - } - const uint32_t threadblocks_per_cluster = + constexpr uint32_t threads_per_threadblock = + (threads_per_threadblock_theoretical > hw_threads_per_cluster) + ? hw_threads_per_cluster + : threads_per_threadblock_theoretical; + constexpr uint32_t threadblocks_per_cluster = hw_threads_per_cluster / threads_per_threadblock; + constexpr uint32_t warps_per_threadblock_per_core = + NUM_WARPS / threadblocks_per_cluster; - const uint32_t threadblock_dim_y = vx_num_warps() / threadblocks_per_cluster; const int threadblock_id = task_id / threads_per_threadblock; const int threadblock_id_in_cluster = threadblock_id % threadblocks_per_cluster; @@ -572,20 +81,40 @@ void kernel_body(int task_id, kernel_arg_t *__UNIFORM__ arg) { const uint32_t problem_size = (dim_m * dim_n) / (ELEM_PER_THREAD); const uint32_t num_threadblocks = problem_size / threads_per_threadblock; - using float_type = float; - // "static" shared memory allocation. This would determine threadblock // occupancy of a single cluster uint8_t *sharedmem_per_threadblock = reinterpret_cast( - DEV_SMEM_START_ADDR + sizeof(float_type) * 2 /*overkill for non-dma*/ * - (2 * BM * BK) * threadblock_id_in_cluster); + DEV_SMEM_START_ADDR + + sizeof(float_type) * 2 * (2 * BM * BK) * threadblock_id_in_cluster); - thread_block_gemm( - arg, tid_in_threadblock, threads_per_threadblock, threadblock_dim_y, - /*threadblock_id_x, threadblock_id_y,*/ - threadblocks_per_cluster, - // threadblock_id, - threadblock_id_in_cluster, sharedmem_per_threadblock); + thread_block_gemm( + (const float_type *)arg->addr_a, (const float_type *)arg->addr_b, + (float *)arg->addr_c, arg->dim_m, arg->dim_n, arg->dim_k, + tid_in_threadblock, threadblocks_per_cluster, threadblock_id_in_cluster, + sharedmem_per_threadblock); + + float *gmem_tmp_d0 = reinterpret_cast(0xd0000000UL); + float *gmem_tmp_d1 = reinterpret_cast(0xd1000000UL); + + const float *smem_A = reinterpret_cast(sharedmem_per_threadblock); + const float *smem_B = smem_A + 2 * BM * BK; + + if constexpr (DEBUG) { + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + + thread_block_copy_tile(smem_A, gmem_tmp_d0, tid_in_threadblock, + threads_per_threadblock, + threadblock_id_in_cluster); + thread_block_copy_tile(smem_B, gmem_tmp_d1, tid_in_threadblock, + threads_per_threadblock, + threadblock_id_in_cluster); + } } int main() { diff --git a/tests/regression/sgemm_tcore/main.cpp b/tests/regression/sgemm_tcore/main.cpp index 34bb2c4d..8e4e6061 100644 --- a/tests/regression/sgemm_tcore/main.cpp +++ b/tests/regression/sgemm_tcore/main.cpp @@ -173,7 +173,8 @@ int main(int argc, char *argv[]) { uint32_t dim_n = 64; uint32_t dim_k = 64; - using float_type = float; + using float_type = half; + generate_source_matrix(dim_m, dim_n, dim_k); generate_reference_matmul(dim_m, dim_n, dim_k); diff --git a/tests/regression/sgemm_tcore/sgemm_impl.hpp b/tests/regression/sgemm_tcore/sgemm_impl.hpp new file mode 100644 index 00000000..6674edd0 --- /dev/null +++ b/tests/regression/sgemm_tcore/sgemm_impl.hpp @@ -0,0 +1,1064 @@ +#ifndef _SGEMM_IMPL_H_ +#define _SGEMM_IMPL_H_ + +#include +#include +#include "include/gemmini.h" +#include "gemmini_mmio.h" + +#define FP_SIZE 32 + +// "fake" fp16 type that only has the correct data width. +using float16_t = uint16_t; + +#if (FP_SIZE == 32) +using float_type = float; +#elif (FP_SIZE == 16) +using float_type = float16_t; +#endif + +// Constraints on parameters: +// * Memory: +// (BM + BN) * BK * sizeof(T) <= sharedmem size. +// BM * BK == BN * BK >= threadblock size >= NT * CORES_PER_CLUSTER +// When larger, the kernel runs a sequential loop to read into sharedmem; +// but smaller case is not handled. +// * Compute: +// ( M* N) / (TM*TN) == grid size >= NC*NW*NT +// (BM*BN) / (TM*TN) == threadblock size < NT * NW * CORES_PER_CLUSTER +// (BM*BN) / (TM*TN) == threadblock size >= NT * CORES_PER_CLUSTER +// * Combining BM * BK >= (BM*BN) / (TM*TN) == threadblock yields +// BM <= BK*TM*TN +#define BM 64 +#define BN 64 +#if (FP_SIZE == 32) +#define BK 64 +#elif (FP_SIZE == 16) +#define BK 128 +#else +#error "unsupported FP_SIZE" +#endif +#define WM 16 +#define WN 8 +#define TCM 8 +#define TCN 8 +#if (FP_SIZE == 32) +#define TCK 8 +#elif (FP_SIZE == 16) +#define TCK 16 +#else +#error "unsupported FP_SIZE" +#endif +#define WMITER (WM / TCM) +#define WNITER (WN / TCN) +#define ELEM_PER_THREAD (WM * WN / NUM_THREADS) +// FIXME: NUM_THREADS and NUM_WARPS hardcoded +#if ((BM * BN / ELEM_PER_THREAD) > (CORES_PER_CLUSTER * 8 * 8)) +#error "threadblock size too big for cluster" +#endif + +// number of loop around the inner 0..TCK..BK loop to simulate perfect-DRAM +// scenario +#define BK_LOOP 1 +// Whether to transpose smem A tile at GMEM->SMEM (produce), or SMEM->RF +// (consume). This is because the tensor core expects the A tile to be stored +// in column-major order in SMEM, whereas it will be ultimately stored in +// row-major in the RF. +// +// For correctness, only one of either should be 1. E.g., PRODUCE 1 CONSUME 0 +// generates the NN kernel where both A and B are stored row-major in GMEM. +// To model the case where the A matrix is already stored column-major in GMEM, +// set both to 0. +#define TRANSPOSE_AT_PRODUCE 0 +#define TRANSPOSE_AT_CONSUME 0 + +#define GEMMINI_DMA 1 +#define GEMMINI_DMA_MN_MAJOR 1 +#if SMEM_SIZE == 0x4000 +#define SMEM_ADDR_Q0 ((float * const) 0xff000000) +#define SMEM_ADDR_Q1 ((float * const) 0xff001000) +#define SMEM_ADDR_Q2 ((float * const) 0xff002000) +#define SMEM_ADDR_Q3 ((float * const) 0xff003000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x80 +#define SPAD_ADDR_Q2 0x100 +#define SPAD_ADDR_Q3 0x180 +#define BOUND_INST 0x400040004ULL +#elif SMEM_SIZE >= 0x10000 +#define SMEM_ADDR_Q0 ((float * const) 0xff000000) +#define SMEM_ADDR_Q1 ((float * const) 0xff004000) +#define SMEM_ADDR_Q2 ((float * const) 0xff008000) +#define SMEM_ADDR_Q3 ((float * const) 0xff00c000) +#define SPAD_ADDR_Q0 0x0 +#define SPAD_ADDR_Q1 0x200 +#define SPAD_ADDR_Q2 0x400 +#define SPAD_ADDR_Q3 0x600 +#define BOUND_INST 0x800080008ULL +#else +#error Unsupported smem size +#endif + +enum class MemLayout { + MN_major, + K_major, +}; + +inline constexpr void map_operand_32lanes(const int tid, int &row, int &col) { + const int tg = tid / 4; + + // A (row major) + // Figure 7(a) in paper + // row 0~ 3: threadgroups 0 and 2 + // row 4~ 7: threadgroups 4 and 6 + // row 8~11: threadgroups 1 and 3 + // row 12~15: threadgroups 5 and 7 + row = tid % 4; + row += (tg * 8) % 16; + row += (tg / 4) * 4; + + // B (column major) + // NOTE: Matrix B mapping in Figure 7(a) is incorrect; below is the + // corrected mapping: + // col 0~ 3: threadgroups 0 and 1 + // col 4~ 7: threadgroups 4 and 5 + // col 8~11: threadgroups 2 and 3 + // col 12~15: threadgroups 6 and 7 + col = tid % 4; + col += ((tg % 4) / 2) * 8; + col += (tg / 4) * 4; +} + +inline constexpr void map_operand_8lanes(const int tid, int &row, int &col) { + const int tg = tid / 4; + + // A (row major) + // row 0~ 3: threadgroup 0 + // row 4~ 7: threadgroup 1 + row = tid % 4; + row += tg * 4; + + // B (column major) + // col 0~ 3: threadgroup 0 + // col 4~ 7: threadgroup 1 + col = tid % 4; + col += tg * 4; +} + +inline constexpr void map_operand(const int tid, int &row, int &col) { + if constexpr (NUM_THREADS == 32) { + map_operand_32lanes(tid, row, col); + } else if constexpr (NUM_THREADS == 8) { + map_operand_8lanes(tid, row, col); + } else { + // FIXME: not allowed + } +} + +inline constexpr void map_c_32lanes(const int tid, int &row, int &col) { + const int tg = tid / 4; + + // C + // Figure 7(b), left + col = ((tg % 4) / 2) * 8; + row = (tg * 8) % 16; + row += (tg / 4) * 4; + + // Figure 7(b), right + row += (tid % 4) % 2; + col += ((tid % 4) / 2) * 2; +} + +inline constexpr void map_c_8lanes(const int tid, int &row, int &col) { + const int tg = tid / 4; + + // C + col = 0; + row = tg * 4; + + // Figure 7(b), right + row += (tid % 4) % 2; + col += ((tid % 4) / 2) * 2; +} + +inline constexpr void map_c(const int tid, int &row, int &col) { + if constexpr (NUM_THREADS == 32) { + map_c_32lanes(tid, row, col); + } else if constexpr (NUM_THREADS == 8) { + map_c_8lanes(tid, row, col); + } else { + // FIXME: not allowed + } +} + +#define RISCV_CUSTOM3 0x7B + +inline void vx_wmma(const int dest_reg) { + if (dest_reg == 0) { + asm volatile (".insn r %0, 0, 0, x0, x0, x0" :: "i"(RISCV_CUSTOM3)); + } else { + asm volatile (".insn r %0, 0, 0, x1, x0, x0" :: "i"(RISCV_CUSTOM3)); + } +} + +// Remap logical row/col coordinate of a matrix element to a memory index that +// follows the 2-level block-row-major layout that Gemmini DMA uses +template +inline constexpr std::pair +remap_to_gemmini_dma_layout(const uint32_t logical_row, + const uint32_t logical_col) { + static_assert(DIM == 8, + "GEMMINI_DMA layout remapping code only written for DIM == 8"); + + if constexpr (use_dma) { + constexpr int dim_blocks_in_row = (dim_col / DIM); + const uint32_t row = + (logical_row / dim_blocks_in_row) * DIM + (logical_col / DIM); + const uint32_t col = + (logical_row % dim_blocks_in_row) * DIM + (logical_col % DIM); + return {row, col}; + } else { + // pass-through + return {logical_row, logical_col}; + } +} + +// `local_k` is assumed to be multiple of TCK +template +inline void wmma_load_a(volatile const T *smem_A, const int local_k, + const int warp_row, const int wm_iter, + const int thread_in_warp) { + asm volatile ("wmma_load_a_start_%=:" :: ); + + const int tid = thread_in_warp; + const int tg = tid / 4; + + // @perf: this is duplicately computed in wmma_load_a and wmma_load_b + int row = 0; + int col = 0; + map_operand(tid, row, col); + + // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do + // data movement at the fp32 granularity. Assuming that the matrix is stored + // row-major in GMEM, the packed fp16 pairs belong to the same row, + // neighboring columns; therefore, it essentially becomes equivalent to + // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed + // by a factor of two. + constexpr int packed_factor = (std::is_same_v ? 2 : 1); + const int local_k_adjusted = local_k / packed_factor; + + static_assert(!GEMMINI_DMA || (layout == MemLayout::K_major) || + GEMMINI_DMA_MN_MAJOR, + "GEMMINI_DMA only supported for K-major A tile"); + static_assert((layout != MemLayout::K_major) || (FP_SIZE == 32), + "fp16 is not really tested for K-major A layout"); + + if constexpr (layout == MemLayout::K_major) { + constexpr int smem_A_cols = leading_dim; + + // f8-f15 stores a single row of A + const uint32_t smem_logical_row = WM * warp_row + TCM * wm_iter + row; + const uint32_t smem_logical_col = + local_k_adjusted + 0; /* FIXME: fp16 adjust necessary? */ + // if using Gemmini DMA, remap logical row/col to Gemmini's 2-level + // block-row-major layout + const auto [smem_row, smem_col] = + remap_to_gemmini_dma_layout(smem_logical_row, + smem_logical_col); + + const volatile uint8_t *smem_addr; + smem_addr = reinterpret_cast( + &reinterpret_cast( + smem_A)[smem_A_cols * smem_row + smem_col]); + // step to the next column + // @perf: bank conflicts; threads read from different rows + // below is correct for GEMMINI_DMA; smem_col is always a multiple of 8, + // and the next 7 elements in the row are guaranteed to be consecutive in + // the memory + asm volatile("flw f0, %0(%1)" ::"i"(0 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f1, %0(%1)" ::"i"(1 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f2, %0(%1)" ::"i"(2 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f3, %0(%1)" ::"i"(3 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f4, %0(%1)" ::"i"(4 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f5, %0(%1)" ::"i"(5 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f6, %0(%1)" ::"i"(6 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f7, %0(%1)" ::"i"(7 * sizeof(float)), "r"(smem_addr)); + } else if constexpr (layout == MemLayout::MN_major) { + constexpr int smem_AS_cols = leading_dim; + + const volatile uint8_t *smem_addr; + smem_addr = reinterpret_cast( + &reinterpret_cast( + smem_A)[((local_k_adjusted + 0) * smem_AS_cols) + + (WM * warp_row + TCM * wm_iter) + row]); + // f8-f15 stores a single row of A + // threads read from different columns; no bank conflicts + asm volatile("flw f0, %0(%1)" :: "i"(smem_AS_cols * 0 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f1, %0(%1)" :: "i"(smem_AS_cols * 1 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f2, %0(%1)" :: "i"(smem_AS_cols * 2 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f3, %0(%1)" :: "i"(smem_AS_cols * 3 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f4, %0(%1)" :: "i"(smem_AS_cols * 4 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f5, %0(%1)" :: "i"(smem_AS_cols * 5 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f6, %0(%1)" :: "i"(smem_AS_cols * 6 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f7, %0(%1)" :: "i"(smem_AS_cols * 7 * sizeof(float)), "r"(smem_addr)); + } else { + static_assert(layout == + MemLayout::K_major /* fake cond that is always false */, + "unsupported memory layout"); + } + + asm volatile ("wmma_load_a_finish_%=:" :: ); +} + +// Convenience wrapper for wmma_load_a if tile layout is packed, i.e. +// leading_dim == col. +template +inline void wmma_load_a(volatile const T *smem_A, const int local_k, + const int warp_row, const int wm_iter, + const int thread_in_warp) { + // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do + // data movement at the fp32 granularity. Assuming that the matrix is stored + // row-major in GMEM, the packed fp16 pairs belong to the same row, + // neighboring columns; therefore, it essentially becomes equivalent to + // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed + // by a factor of two. + constexpr int packed_factor = (std::is_same_v ? 2 : 1); + constexpr int tile_dim_k_adjusted = tile_dim_k / packed_factor; + constexpr int leading_dim = (layout == MemLayout::K_major) + ? tile_dim_k_adjusted + : tile_dim_m; + + wmma_load_a(smem_A, local_k, warp_row, wm_iter, + thread_in_warp); +} + +// `local_k` is assumed to be multiple of TCK +template +inline void wmma_load_b(const volatile T *smem_B, const int local_k, + const int warp_col, const int wn_iter, + const int thread_in_warp) { + asm volatile ("wmma_load_b_start_%=:" :: ); + + static_assert(layout == MemLayout::MN_major, + "only N-major layout for the B tile is supported"); + + const int tid = thread_in_warp; + const int tg = tid / 4; + + int row = 0; + int col = 0; + map_operand(tid, row, col); + + // see comment in wmma_load_a + constexpr int packed_factor = (std::is_same_v ? 2 : 1); + constexpr int tile_dim_k_adjusted = tile_dim_k / packed_factor; + const int local_k_adjusted = local_k / packed_factor; + + // B is stored N-major in smem + constexpr int smem_B_cols = tile_dim_n; + + const uint32_t smem_logical_row = local_k_adjusted + 0; + const uint32_t smem_logical_col = (WN * warp_col + TCN * wn_iter) + col; + // if using Gemmini DMA, remap logical row/col to Gemmini's 2-level + // block-row-major layout + const auto [smem_row, smem_col] = + remap_to_gemmini_dma_layout(smem_logical_row, + smem_logical_col); + + const volatile uint8_t *smem_addr; + smem_addr = reinterpret_cast( + &reinterpret_cast( + smem_B)[smem_B_cols * smem_row + smem_col]); + // f8-f15 stores a single column of B + // threads read from different columns; no bank conflicts + if constexpr (GEMMINI_DMA) { + // for GEMMINI_DMA, moving rows for the next 7 elements in the same column + // is the same as moving DIM elements forward in the memory because of the + // block-row-major layout + asm volatile("flw f8, %0(%1)" :: "i"(DIM * 0 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f9, %0(%1)" :: "i"(DIM * 1 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f10, %0(%1)" :: "i"(DIM * 2 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f11, %0(%1)" :: "i"(DIM * 3 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f12, %0(%1)" :: "i"(DIM * 4 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f13, %0(%1)" :: "i"(DIM * 5 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f14, %0(%1)" :: "i"(DIM * 6 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f15, %0(%1)" :: "i"(DIM * 7 * sizeof(float)), "r"(smem_addr)); + } else { + asm volatile("flw f8, %0(%1)" :: "i"(smem_B_cols * 0 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f9, %0(%1)" :: "i"(smem_B_cols * 1 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f10, %0(%1)" :: "i"(smem_B_cols * 2 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f11, %0(%1)" :: "i"(smem_B_cols * 3 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f12, %0(%1)" :: "i"(smem_B_cols * 4 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f13, %0(%1)" :: "i"(smem_B_cols * 5 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f14, %0(%1)" :: "i"(smem_B_cols * 6 * sizeof(float)), "r"(smem_addr)); + asm volatile("flw f15, %0(%1)" :: "i"(smem_B_cols * 7 * sizeof(float)), "r"(smem_addr)); + } + + asm volatile ("wmma_load_b_finish_%=:" :: ); +} + +// Initialize the accumulator registers to zero before starting FMA operations +// with the tensor cores. +template inline void initialize_accum_regs() { + if constexpr (accum_reg_set == 0) { + asm volatile("fmv.w.x f16, x0"); + asm volatile("fmv.w.x f17, x0"); + asm volatile("fmv.w.x f18, x0"); + asm volatile("fmv.w.x f19, x0"); + asm volatile("fmv.w.x f20, x0"); + asm volatile("fmv.w.x f21, x0"); + asm volatile("fmv.w.x f22, x0"); + asm volatile("fmv.w.x f23, x0"); + } else { + asm volatile("fmv.w.x f24, x0"); + asm volatile("fmv.w.x f25, x0"); + asm volatile("fmv.w.x f26, x0"); + asm volatile("fmv.w.x f27, x0"); + asm volatile("fmv.w.x f28, x0"); + asm volatile("fmv.w.x f29, x0"); + asm volatile("fmv.w.x f30, x0"); + asm volatile("fmv.w.x f31, x0"); + } +} + +// `C` is expected to be in N-major layout. +__attribute__((always_inline)) inline void +wmma_load_accum(const int thread_in_warp, const int warp_col, + const int warp_row, const int wn_iter, const int wm_iter, + const int dim_n, const float *C) { + asm volatile("wmma_load_accum_start_%=:" ::); + + const int tid = thread_in_warp; + + // these are [0, TCM/TCN) + int tid_row = 0; + int tid_col = 0; + map_c(tid, tid_row, tid_col); + + int local_row = (WM * warp_row + TCM * wm_iter) + tid_row; + int local_col = (WN * warp_col + TCN * wn_iter) + tid_col; + + // @copypaste from wmma_store + // @perf: this likely causes a lot of gmem bank conflicts + if (wm_iter == 0) { + const uint8_t *addr = reinterpret_cast( + &C[dim_n * (local_row + 0) + (local_col + 0)]); + const uint8_t *addr_tworow = addr + (2 * dim_n) * sizeof(float); + asm volatile("flw f16, %0(%1)" ::"i"(0 * sizeof(float)), "r"(addr)); + asm volatile("flw f17, %0(%1)" ::"i"(1 * sizeof(float)), "r"(addr)); + asm volatile("flw f18, %0(%1)" ::"i"(0 * sizeof(float)), "r"(addr_tworow)); + asm volatile("flw f19, %0(%1)" ::"i"(1 * sizeof(float)), "r"(addr_tworow)); + asm volatile("flw f20, %0(%1)" ::"i"(4 * sizeof(float)), "r"(addr)); + asm volatile("flw f21, %0(%1)" ::"i"(5 * sizeof(float)), "r"(addr)); + asm volatile("flw f22, %0(%1)" ::"i"(4 * sizeof(float)), "r"(addr_tworow)); + asm volatile("flw f23, %0(%1)" ::"i"(5 * sizeof(float)), "r"(addr_tworow)); + } else { + const uint8_t *addr = reinterpret_cast( + &C[dim_n * (local_row + 0) + (local_col + 0)]); + const uint8_t *addr_tworow = addr + (2 * dim_n) * sizeof(float); + asm volatile("flw f24, %0(%1)" ::"i"(0 * sizeof(float)), "r"(addr)); + asm volatile("flw f25, %0(%1)" ::"i"(1 * sizeof(float)), "r"(addr)); + asm volatile("flw f26, %0(%1)" ::"i"(0 * sizeof(float)), "r"(addr_tworow)); + asm volatile("flw f27, %0(%1)" ::"i"(1 * sizeof(float)), "r"(addr_tworow)); + asm volatile("flw f28, %0(%1)" ::"i"(4 * sizeof(float)), "r"(addr)); + asm volatile("flw f29, %0(%1)" ::"i"(5 * sizeof(float)), "r"(addr)); + asm volatile("flw f30, %0(%1)" ::"i"(4 * sizeof(float)), "r"(addr_tworow)); + asm volatile("flw f31, %0(%1)" ::"i"(5 * sizeof(float)), "r"(addr_tworow)); + } + + asm volatile("wmma_load_accum_finish_%=:" ::); +} + +// Write out the matrix data stored in RF to memory +__attribute__((always_inline)) inline void +wmma_store(const int thread_in_warp, const int warp_col, const int warp_row, + const int wn_iter, const int wm_iter, const int dim_n, + float *write_addr) { + asm volatile ("wmma_store_start_%=:" :: ); + + const int tid = thread_in_warp; + + // these are [0, TCM/TCN) + int tid_row = 0; + int tid_col = 0; + map_c(tid, tid_row, tid_col); + + int local_row = (WM * warp_row + TCM * wm_iter) + tid_row; + int local_col = (WN * warp_col + TCN * wn_iter) + tid_col; + + // @perf: this likely causes a lot of gmem bank conflicts + if (wm_iter == 0) { + volatile uint8_t *addr = reinterpret_cast( + &write_addr[dim_n * (local_row + 0) + (local_col + 0)]); + volatile uint8_t *addr_tworow = addr + (2 * dim_n) * sizeof(float); + asm volatile("fsw f16, %0(%1)" ::"i"(0 * sizeof(float)), "r"(addr)); + asm volatile("fsw f17, %0(%1)" ::"i"(1 * sizeof(float)), "r"(addr)); + asm volatile("fsw f18, %0(%1)" ::"i"(0 * sizeof(float)), "r"(addr_tworow)); + asm volatile("fsw f19, %0(%1)" ::"i"(1 * sizeof(float)), "r"(addr_tworow)); + asm volatile("fsw f20, %0(%1)" ::"i"(4 * sizeof(float)), "r"(addr)); + asm volatile("fsw f21, %0(%1)" ::"i"(5 * sizeof(float)), "r"(addr)); + asm volatile("fsw f22, %0(%1)" ::"i"(4 * sizeof(float)), "r"(addr_tworow)); + asm volatile("fsw f23, %0(%1)" ::"i"(5 * sizeof(float)), "r"(addr_tworow)); + } else { + volatile uint8_t *addr = reinterpret_cast( + &write_addr[dim_n * (local_row + 0) + (local_col + 0)]); + volatile uint8_t *addr_tworow = addr + (2 * dim_n) * sizeof(float); + asm volatile("fsw f24, %0(%1)" ::"i"(0 * sizeof(float)), "r"(addr)); + asm volatile("fsw f25, %0(%1)" ::"i"(1 * sizeof(float)), "r"(addr)); + asm volatile("fsw f26, %0(%1)" ::"i"(0 * sizeof(float)), "r"(addr_tworow)); + asm volatile("fsw f27, %0(%1)" ::"i"(1 * sizeof(float)), "r"(addr_tworow)); + asm volatile("fsw f28, %0(%1)" ::"i"(4 * sizeof(float)), "r"(addr)); + asm volatile("fsw f29, %0(%1)" ::"i"(5 * sizeof(float)), "r"(addr)); + asm volatile("fsw f30, %0(%1)" ::"i"(4 * sizeof(float)), "r"(addr_tworow)); + asm volatile("fsw f31, %0(%1)" ::"i"(5 * sizeof(float)), "r"(addr_tworow)); + } + + asm volatile ("wmma_store_finish_%=:" :: ); +} + +inline void threadblock_barrier(const uint32_t barrier_id, const uint32_t count) { + vx_fence(); + vx_barrier(barrier_id, count); +} + +// Move a single matrix tile from global memory (GMEM) to shared memory (SMEM). +// `dim_major`: major dimension of the matrix in GMEM, e.g. if K-major, K; or +// MN-major, M/N. +// +// Note that there's not a single way to specify a layout of the matrix. +// Identifying a matrix to be K-major and specifying the mn_index of a tile, +// is equivalent to identifying it as MN-major and specifying the k_index +// (provided `dim_major` is set accordingly). +template +__attribute__((always_inline)) inline void +load_tile_to_smem(const uint32_t dim_major, const uint32_t mn_index, + const uint32_t k_index, const T *global_addr, + volatile T *local_addr, const uint32_t tid_in_threadblock) { + asm volatile("load_tile_to_smem_start_%=:" ::); + + // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do + // data movement at the fp32 granularity. The tensor core hardware assumes + // the fp16 elements are contiguously stored along the K-dimension; + // therefore, this essentially becomes equivalent to a fp32 GEMM where the + // K-dimension is shrinked by the factor of two. + constexpr uint32_t packed_factor = (std::is_same_v ? 2 : 1); + + constexpr uint32_t tile_dim_k_packed = tile_dim_k / packed_factor; + constexpr uint32_t gmem_dim_row = + (gmem_layout == MemLayout::K_major) ? tile_dim_mn : tile_dim_k_packed; + constexpr uint32_t gmem_dim_col = + (gmem_layout == MemLayout::K_major) ? tile_dim_k_packed : tile_dim_mn; + constexpr uint32_t smem_dim_col = + (smem_layout == MemLayout::K_major) ? tile_dim_k_packed : tile_dim_mn; + + const uint32_t dim_major_ = + (gmem_layout == MemLayout::K_major) ? dim_major / packed_factor : dim_major; + + // threads in the threadblock always do contiguous accesses in the gmem + const uint32_t local_row_gmem = tid_in_threadblock / gmem_dim_col; + const uint32_t local_col_gmem = tid_in_threadblock % gmem_dim_col; + + constexpr bool transposed_write = (gmem_layout != smem_layout); + // if transposed, threads write to smem in reversed col/row + const uint32_t local_row_smem = + transposed_write ? local_col_gmem : local_row_gmem; + const uint32_t local_col_smem = + transposed_write ? local_row_gmem : local_col_gmem; + + const uint32_t global_row_mn_major = tile_dim_k_packed * k_index + local_row_gmem; + const uint32_t global_col_mn_major = gmem_dim_col * mn_index + local_col_gmem; + const uint32_t global_row_k_major = gmem_dim_row * mn_index + local_row_gmem; + const uint32_t global_col_k_major = tile_dim_k_packed * k_index + local_col_gmem; + const uint32_t global_row = (gmem_layout == MemLayout::K_major) + ? global_row_k_major + : global_row_mn_major; + const uint32_t global_col = (gmem_layout == MemLayout::K_major) + ? global_col_k_major + : global_col_mn_major; + + const float *global = reinterpret_cast(global_addr) + + dim_major_ * global_row + global_col; + volatile float *local = reinterpret_cast(local_addr) + + smem_dim_col * local_row_smem + local_col_smem; + + constexpr uint32_t row_stride = threads_per_threadblock / gmem_dim_col; + + static_assert(row_stride * 8 <= gmem_dim_row, + "manual loop unrolling condition not met; tile row dimension " + "is too shallow"); + static_assert((gmem_dim_row % (row_stride * 8)) == 0, + "manual loop unrolling condition not met; tile row dimension " + "should be power-of-two"); + +#pragma GCC unroll 1 + // loop-unrolled flw/fsw to increase reuse distance and IPC + for (uint32_t load_offset = 0; load_offset < gmem_dim_row; + load_offset += row_stride * 8) { + // equivalent code: + // + // *local = *global; + // global += dim_major * row_stride; + // local += BN * row_stride; + + // read same-column elements into fp registers + asm volatile("flw ft0, (%0)" ::"r"(global)); + global += dim_major_ * row_stride; + asm volatile("flw ft1, (%0)" ::"r"(global)); + global += dim_major_ * row_stride; + asm volatile("flw ft2, (%0)" ::"r"(global)); + global += dim_major_ * row_stride; + asm volatile("flw ft3, (%0)" ::"r"(global)); + global += dim_major_ * row_stride; + asm volatile("flw ft4, (%0)" ::"r"(global)); + global += dim_major_ * row_stride; + asm volatile("flw ft5, (%0)" ::"r"(global)); + global += dim_major_ * row_stride; + asm volatile("flw ft6, (%0)" ::"r"(global)); + global += dim_major_ * row_stride; + asm volatile("flw ft7, (%0)" ::"r"(global)); + global += dim_major_ * row_stride; + + // need to branch because address offset constant in the inline assembly + // cannot be larger than a certain limit + if constexpr (!transposed_write) { + asm volatile("fsw ft0, %0(%1)" ::"i"(smem_dim_col * row_stride * 0 * + sizeof(float)), + "r"(local)); + asm volatile("fsw ft1, %0(%1)" ::"i"(smem_dim_col * row_stride * 1 * + sizeof(float)), + "r"(local)); + local += smem_dim_col * row_stride * 2; + asm volatile("fsw ft2, %0(%1)" ::"i"(smem_dim_col * row_stride * 0 * + sizeof(float)), + "r"(local)); + asm volatile("fsw ft3, %0(%1)" ::"i"(smem_dim_col * row_stride * 1 * + sizeof(float)), + "r"(local)); + local += smem_dim_col * row_stride * 2; + asm volatile("fsw ft4, %0(%1)" ::"i"(smem_dim_col * row_stride * 0 * + sizeof(float)), + "r"(local)); + asm volatile("fsw ft5, %0(%1)" ::"i"(smem_dim_col * row_stride * 1 * + sizeof(float)), + "r"(local)); + local += smem_dim_col * row_stride * 2; + asm volatile("fsw ft6, %0(%1)" ::"i"(smem_dim_col * row_stride * 0 * + sizeof(float)), + "r"(local)); + asm volatile("fsw ft7, %0(%1)" ::"i"(smem_dim_col * row_stride * 1 * + sizeof(float)), + "r"(local)); + local += smem_dim_col * row_stride * 2; + } else { + // currently, tensor core hardware only supports MN-major SMEM tile + // layout for correct results + static_assert(gmem_layout == MemLayout::K_major); + static_assert(smem_layout == MemLayout::MN_major); + + asm volatile("fsw ft0, %0(%1)" ::"i"(row_stride * 0 * sizeof(float)), + "r"(local)); + asm volatile("fsw ft1, %0(%1)" ::"i"(row_stride * 1 * sizeof(float)), + "r"(local)); + asm volatile("fsw ft2, %0(%1)" ::"i"(row_stride * 2 * sizeof(float)), + "r"(local)); + asm volatile("fsw ft3, %0(%1)" ::"i"(row_stride * 3 * sizeof(float)), + "r"(local)); + asm volatile("fsw ft4, %0(%1)" ::"i"(row_stride * 4 * sizeof(float)), + "r"(local)); + asm volatile("fsw ft5, %0(%1)" ::"i"(row_stride * 5 * sizeof(float)), + "r"(local)); + asm volatile("fsw ft6, %0(%1)" ::"i"(row_stride * 6 * sizeof(float)), + "r"(local)); + asm volatile("fsw ft7, %0(%1)" ::"i"(row_stride * 7 * sizeof(float)), + "r"(local)); + local += row_stride * 8; + } + } + + asm volatile("load_tile_to_smem_finish_new_%=:" ::); +} + +// Do a single tile*tile matrix multiplication using the matrix data stored in +// SMEM. Useful in fused kernels where GEMMs are done at a per-tile scope. +template +__attribute__((always_inline)) inline void thread_block_gemm_single_tile( + const T *local_a, const T *local_b, const T *local_c, T *result_addr, + const uint32_t tid_in_threadblock, const uint32_t threads_per_threadblock, + const uint32_t threadblocks_per_cluster, + const uint32_t threadblock_id_in_cluster) { + // no double-buffering + // FIXME: duplicated from thread_block_gemm + const uint32_t threads_per_warpgroup = threads_per_threadblock; + const uint32_t warp_id_in_warpgroup = tid_in_threadblock / NUM_THREADS; + const uint32_t warp_row = warp_id_in_warpgroup / (tile_dim_n / WN); + const uint32_t warp_col = warp_id_in_warpgroup % (tile_dim_n / WN); + const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; + const uint32_t warps_per_threadblock_per_core = + NUM_WARPS / threadblocks_per_cluster; + + // TODO: it would be useful if this bit is split out into a function, so that + // preloading accumulation tile can be used for full GEMMs at the start of + // the K-loop. + if constexpr (load_accum) { +#pragma GCC unroll + for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { +#pragma GCC unroll + for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { + wmma_load_accum(tid_in_warp, warp_col, warp_row, wn_iter, wm_iter, + tile_dim_n, local_c); + } + } + } + +#pragma GCC unroll 1 + for (int i = 0; i < BK_LOOP; i++) { +#pragma GCC unroll 4 + for (uint32_t local_k = 0; local_k < tile_dim_k; local_k += TCK) { +#pragma GCC unroll 2 + for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { + // SMEM -> RF + static_assert(leading_dim_b == 0, + "leading_dim for wmma_load_b is not implemented yet"); + wmma_load_b( + local_b, local_k, warp_col, wn_iter, tid_in_warp); +#pragma GCC unroll 2 + for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { + // SMEM -> RF + if constexpr (leading_dim_a == 0) { + wmma_load_a( + local_a, local_k, warp_row, wm_iter, tid_in_warp); + } else { + wmma_load_a(local_a, local_k, warp_row, + wm_iter, tid_in_warp); + } + // perform mma + vx_wmma(wm_iter); + } + } + } + } + + if constexpr (GEMMINI_DMA) { + // Call gemmini fence at the end of the loop to overlap dma & wmma. + // Usually, by this time, dma has finished the copy so that this + // becomes a no-op. + if (tid_in_threadblock == 0) { + gemmini_fence(); + } + } + + if constexpr (write_to_mem) { + // need to protect smem reads in the earlier step from writes in below, + // especially when the destination address overlaps with the source address + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + +#pragma GCC unroll + for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { +#pragma GCC unroll + for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { + wmma_store(tid_in_warp, warp_col, warp_row, wn_iter, wm_iter, tile_dim_n, + result_addr); + } + } + } +} + +template < + typename T, uint32_t threads_per_threadblock, bool write_to_gmem = true, + // by default, A/B tiles are placed at the start of the smem + uint32_t smem_a_offset = 0, // byte offset of A tile in shared + // memory + uint32_t smem_a_dbuf_offset = 0, // byte offset of A + // double-buffer tile in shared + // memory + uint32_t smem_b_offset = sizeof(float) * BM * BK, // byte offset of B tile + // in shared memory + uint32_t smem_b_dbuf_offset = sizeof(float) * BM * + BK // byte offset of B double-buffer + // tile in shared memory + > +inline void thread_block_gemm(const T *A, const T *B, float *C, + const uint32_t dim_m, const uint32_t dim_n, + const uint32_t dim_k, + const uint32_t tid_in_threadblock, + const uint32_t threadblocks_per_cluster, + const uint32_t threadblock_id_in_cluster, + uint8_t *sharedmem_per_threadblock) { + const uint32_t threads_per_warpgroup = threads_per_threadblock; + const uint32_t warp_id_in_warpgroup = tid_in_threadblock / NUM_THREADS; + const uint32_t warp_row = warp_id_in_warpgroup / (BN / WN); + const uint32_t warp_col = warp_id_in_warpgroup % (BN / WN); + const uint32_t tid_in_warp = tid_in_threadblock % NUM_THREADS; + const uint32_t warps_per_threadblock_per_core = + NUM_WARPS / threadblocks_per_cluster; + + T *local_a = reinterpret_cast(sharedmem_per_threadblock + smem_a_offset); + T *local_a_buf = + reinterpret_cast(sharedmem_per_threadblock + smem_a_dbuf_offset); + T *local_b = reinterpret_cast(sharedmem_per_threadblock + smem_b_offset); + T *local_b_buf = + reinterpret_cast(sharedmem_per_threadblock + smem_b_dbuf_offset); + + constexpr uint32_t skips = + loop_matmul_skips(/*skip_lda=*/0, /*skip_ldb=*/0, /*skip_ldd=*/1, + /*skip_ex=*/1, /*skip_stc=*/1); + +#if (GEMMINI_DMA == 1) + if (tid_in_threadblock == 0) { + gemmini_extended_config_ex(WEIGHT_STATIONARY, 0, 0, 1, 0, 0); + // gemmini_extended_config_ex(dataflow, act & 3, 0, 1, a_transpose, + // b_transpose); + + gemmini_extended3_config_ld(dim_k * sizeof(elem_t), MVIN_SCALE_IDENTITY, + false, 0); + gemmini_extended3_config_ld(dim_n * sizeof(elem_t), MVIN_SCALE_IDENTITY, + false, 1); + gemmini_extended_config_st(dim_n * sizeof(elem_t), 0, MVIN_SCALE_IDENTITY); + + gemmini_fence(); + } +#endif + + // divide rows (M) by the number of threadblocks + const uint32_t dim_m_range = (dim_m / threadblocks_per_cluster); + const uint32_t dim_m_start = dim_m_range * threadblock_id_in_cluster; + const uint32_t block_m_start = dim_m_start / BM; + const uint32_t block_m_end = (dim_m_start + dim_m_range) / BM; + +#pragma GCC unroll 1 + for (uint32_t block_m = block_m_start; block_m < block_m_end; block_m++) { +#pragma GCC unroll 1 + for (uint32_t block_n = 0; (block_n * BN) < dim_n; block_n++) { + // clear out accumulators + initialize_accum_regs<0>(); + initialize_accum_regs<1>(); + + if constexpr (GEMMINI_DMA) { + // pipeline initiation + if (tid_in_threadblock == 0) { + // configure dma gmem address to load from + // FIXME: block_k is wrong + ROCC_INSTRUCTION_RS1_RS2( + XCUSTOM_ACC, + (uint64_t)(A + block_m * BM * dim_k + /*block_k:*/0 * BK), + (uint64_t)(B + /*block_k:*/0 * BK * dim_n + block_n * BN), + k_LOOP_WS_CONFIG_ADDRS_AB) + // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB + GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); + gemmini_fence(); + + GEMMINI_CISC_CMD_I(10); + gemmini_fence(); + +#if 0 + // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS + // FIXME: block_k is 0 for two times + sp_tiled_matmul_full_spad_ws( +#if 1 + SPAD_ADDR_Q0, SPAD_ADDR_Q1, +#else + (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q2 : SPAD_ADDR_Q0, + (/*block_k:*/ 0 & 1) ? SPAD_ADDR_Q3 : SPAD_ADDR_Q1, +#endif + /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q3, + /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, + /*pad_J=*/0, /*pad_K=*/0, + /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, + /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) + gemmini_fence(); +#endif + } + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + } + +#pragma GCC unroll 1 + for (uint32_t block_k = 0; (block_k * BK) < dim_k; block_k++) { + + // producer code: GMEM->SMEM memory movement + // --------------------------------------------------------------------- + // + // this is either done using DMA or SIMT cores depending on GEMMINI_DMA + +#if (GEMMINI_DMA == 1) + if ((tid_in_threadblock == 0) && ((block_k * BK) != (dim_k - BK))) { + // configure dma gmem address to load from + // FIXME: block_k is wrong + ROCC_INSTRUCTION_RS1_RS2( + XCUSTOM_ACC, + (uint64_t)(A + block_m * BM * dim_k + (block_k + 1/*runahead*/) * BK), + (uint64_t)(B + (block_k + 1/*runahead*/) * BK * dim_n + block_n * BN), + k_LOOP_WS_CONFIG_ADDRS_AB) + // GEMMINI_CISC(8) does k_LOOP_WS_CONFIG_STRIDES_AB + GEMMINI_CISC_CMD_R((dim_n << 16) | (dim_k << 8) | 8); + // gemmini_fence(); + + // block_k is even: opcode 11 (write to local_a_buf) + // block_k is odd: opcode 10 (write to local_a) + const uint32_t opcode = 11 - (block_k & 1); + GEMMINI_CISC_CMD_R(opcode); + // // TODO: branch is probably slow + // if (block_k & 1) { + // GEMMINI_CISC_CMD_I(12); + // } else { // block_k == 0 is here + // GEMMINI_CISC_CMD_I(13); + // } + + // configure loop iteration bounds + // FIXME: shouldn't be necessary + // ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, 0, BOUND_INST, + // k_LOOP_WS_CONFIG_BOUNDS) ROCC_INSTRUCTION_RS1_RS2(XCUSTOM_ACC, + // SPAD_ADDR_Q0, SPAD_ADDR_Q1, k_LOOP_WS_CONFIG_SPAD_AB) + // ROCC_INSTRUCTION_RS1_RS2( + // XCUSTOM_ACC, + // ((uint64_t)(/*a_spad_id:*/ 0) << 18) | + // ((uint64_t)(/*b_spad_id:*/ 0) << 16) | + // ((uint64_t)(/*act:0*/ 0) << 8) | ((/*low_D:*/ 0) << 2) | + // ((/*full_C:*/ 0) << 1) | (/*ex_accumulate:*/ 0), + // ((uint64_t)(/*C_spad_addr:*/ A) << 32) | 0x200U | (skips) | + // ((/*is_resadd*/ 0) << 2) | ((/*B_transpose:*/ 0) << 1) | + // (/*A_transpose:*/ 1), + // k_LOOP_WS) + // gemmini_fence(); + +#if 0 + uint32_t spad_a_produce; + uint32_t spad_b_produce; + const uint32_t mask_odd = (block_k & 1) << 31 >> 31; + const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; + spad_a_produce = + ((mask_odd & (SPAD_ADDR_Q0)) | (mask_even & (SPAD_ADDR_Q2))); + spad_b_produce = + ((mask_odd & (SPAD_ADDR_Q1)) | (mask_even & (SPAD_ADDR_Q3))); + // sp_tiled_matmul_full_spad_ws includes CONFIG_BOUNDS + // FIXME: block_k is 0 for two times + sp_tiled_matmul_full_spad_ws( + spad_a_produce, + spad_b_produce, + /*spad_D=*/0, /*spad_C=*/SPAD_ADDR_Q1, + /*I=*/BM / DIM, /*J=*/BN / DIM, /*K=*/BK / DIM, /*pad_I=*/0, + /*pad_J=*/0, /*pad_K=*/0, + /*a_transpose=*/0, /*b_transpose=*/0, /*full_C=*/0, /*low_D=*/0, + /*acc=*/0, /*act=*/NO_ACTIVATION, /*skips=*/skips) +#endif + } +#else + // move A + if constexpr (!TRANSPOSE_AT_PRODUCE) { + load_tile_to_smem( + dim_m, block_m, block_k, A, local_a, tid_in_threadblock); + } else { + load_tile_to_smem( + dim_k, block_m, block_k, A, local_a, tid_in_threadblock); + } + + // move B + load_tile_to_smem(dim_n, block_n, block_k, B, + local_b, tid_in_threadblock); + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); +#endif + + // consumer code: SMEM->RF and compute + // ---------------------------------------------------------------------- + // @perf: this loop spills to stack a lot because of all the flws in + const T *local_a_consume; + const T *local_b_consume; + if constexpr (GEMMINI_DMA) { + // local_a_consume = (k_index % 2) ? local_a_buf : local_a; + // local_b_consume = (k_index % 2) ? local_b_buf : local_b; + // FIXME: swap multiply with bitshifts + // const uint32_t mask_odd = (block_k & 1) << 31 >> 31; + // const uint32_t mask_even = ((block_k & 1) ^ 1) << 31 >> 31; + // local_a_consume = reinterpret_cast( + // (mask_odd & reinterpret_cast(local_a_buf)) | + // (mask_even & reinterpret_cast(local_a))); + // local_b_consume = reinterpret_cast( + // (mask_odd & reinterpret_cast(local_b_buf)) | + // (mask_even & reinterpret_cast(local_b))); + local_a_consume = local_a + (block_k & 1) * (BM * BK); + local_b_consume = local_b + (block_k & 1) * (BK * BN); + } else { + // no double-buffering without DMA + local_a_consume = local_a; + local_b_consume = local_b; + } + + constexpr MemLayout layout_a = + TRANSPOSE_AT_CONSUME ? MemLayout::K_major : MemLayout::MN_major; + thread_block_gemm_single_tile( + local_a_consume, local_b_consume, + static_cast(nullptr) /*ignore accum*/, + static_cast(nullptr) /*ignore result*/, tid_in_threadblock, + threads_per_threadblock, threadblocks_per_cluster, + threadblock_id_in_cluster); + + if constexpr (GEMMINI_DMA) { + // Call gemmini fence at the end of the loop to overlap dma & wmma. + // Usually, by this time, dma has finished the copy so that this + // becomes a no-op. + if (tid_in_threadblock == 0) { + gemmini_fence(); + } + } + + threadblock_barrier(threadblock_id_in_cluster, + warps_per_threadblock_per_core); + } + + if constexpr (write_to_gmem) { +#pragma GCC unroll + for (int wm_iter = 0; wm_iter < WMITER; wm_iter++) { +#pragma GCC unroll + for (int wn_iter = 0; wn_iter < WNITER; wn_iter++) { + float *global_offset_C = C + (BM * block_m) * dim_n + BN * block_n; + wmma_store(tid_in_warp, warp_col, warp_row, wn_iter, wm_iter, dim_n, + global_offset_C); + } + } + } + } + } +} + +#endif diff --git a/tests/regression/sgemm_tcore/util.hpp b/tests/regression/sgemm_tcore/util.hpp deleted file mode 100644 index 7950bddc..00000000 --- a/tests/regression/sgemm_tcore/util.hpp +++ /dev/null @@ -1,343 +0,0 @@ -#ifndef _UTIL_H_ -#define _UTIL_H_ - -#include -#include -#include "include/gemmini.h" -#include "gemmini_mmio.h" - -// Constraints on parameters: -// * Memory: -// (BM + BN) * BK * sizeof(T) <= sharedmem size. -// BM * BK == BN * BK >= threadblock size >= NT * CORES_PER_CLUSTER -// When larger, the kernel runs a sequential loop to read into sharedmem; -// but smaller case is not handled. -// * Compute: -// ( M* N) / (TM*TN) == grid size >= NC*NW*NT -// (BM*BN) / (TM*TN) == threadblock size < NT * NW * CORES_PER_CLUSTER -// (BM*BN) / (TM*TN) == threadblock size >= NT * CORES_PER_CLUSTER -// * Combining BM * BK >= (BM*BN) / (TM*TN) == threadblock yields -// BM <= BK*TM*TN -#define BM 64 -#define BN 64 -#define BK 64 -#define WM 16 -#define WN 8 -#define TCM 8 -#define TCN 8 -#define TCK 8 -#define WMITER (WM / TCM) -#define WNITER (WN / TCN) -#define ELEM_PER_THREAD (WMITER * WNITER * (TCM * TCN) / NUM_THREADS) - -// number of loop around the inner 0..TCK..BK loop to simulate perfect-DRAM -// scenario -#define BK_LOOP 1 -// Whether to transpose smem A tile at GMEM->SMEM (produce), or SMEM->RF -// (consume). This is because the tensor core expects the A tile to be stored -// in column-major order in SMEM, whereas it will be ultimately stored in -// row-major in the RF. -// -// For correctness, only one of either should be 1. E.g., PRODUCE 1 CONSUME 0 -// generates the NN kernel where both A and B are stored row-major in GMEM. -// To model the case where the A matrix is already stored transposed in GMEM -// ("TN" kernel), set both to 0. -#define TRANSPOSE_AT_PRODUCE 1 -#define TRANSPOSE_AT_CONSUME 0 -// GMEM_COALESCED: When TRANSPOSE_AT_PRODUCE == 1 (i.e. transpose at -// GMEM->SMEM), determines whether we do bank-conflict-free accesses for -// 1: GMEM loads of A matrix, or -// 0: SMEM stores of A matrix. -// -// Usually, GMEM_COALESCED==1 yields better performance since the memory -// behavior of GMEM is more sensitive to bank conflicts. -#define GMEM_COALESCED_A 1 - -// "fake" fp16 type that only has the correct data width. -using float16_t = uint16_t; - -inline constexpr void map_operand_32lanes(const int tid, int &row, int &col) { - const int tg = tid / 4; - - // A (row major) - // Figure 7(a) in paper - // row 0~ 3: threadgroups 0 and 2 - // row 4~ 7: threadgroups 4 and 6 - // row 8~11: threadgroups 1 and 3 - // row 12~15: threadgroups 5 and 7 - row = tid % 4; - row += (tg * 8) % 16; - row += (tg / 4) * 4; - - // B (column major) - // NOTE: Matrix B mapping in Figure 7(a) is incorrect; below is the - // corrected mapping: - // col 0~ 3: threadgroups 0 and 1 - // col 4~ 7: threadgroups 4 and 5 - // col 8~11: threadgroups 2 and 3 - // col 12~15: threadgroups 6 and 7 - col = tid % 4; - col += ((tg % 4) / 2) * 8; - col += (tg / 4) * 4; -} - -inline constexpr void map_operand_8lanes(const int tid, int &row, int &col) { - const int tg = tid / 4; - - // A (row major) - // row 0~ 3: threadgroup 0 - // row 4~ 7: threadgroup 1 - row = tid % 4; - row += tg * 4; - - // B (column major) - // col 0~ 3: threadgroup 0 - // col 4~ 7: threadgroup 1 - col = tid % 4; - col += tg * 4; -} - -inline constexpr void map_operand(const int tid, int &row, int &col) { - if constexpr (NUM_THREADS == 32) { - map_operand_32lanes(tid, row, col); - } else if constexpr (NUM_THREADS == 8) { - map_operand_8lanes(tid, row, col); - } else { - // FIXME: not allowed - } -} - -inline constexpr void map_c_32lanes(const int tid, int &row, int &col) { - const int tg = tid / 4; - - // C - // Figure 7(b), left - col = ((tg % 4) / 2) * 8; - row = (tg * 8) % 16; - row += (tg / 4) * 4; - - // Figure 7(b), right - row += (tid % 4) % 2; - col += ((tid % 4) / 2) * 2; -} - -inline constexpr void map_c_8lanes(const int tid, int &row, int &col) { - const int tg = tid / 4; - - // C - col = 0; - row = tg * 4; - - // Figure 7(b), right - row += (tid % 4) % 2; - col += ((tid % 4) / 2) * 2; -} - -inline constexpr void map_c(const int tid, int &row, int &col) { - if constexpr (NUM_THREADS == 32) { - map_c_32lanes(tid, row, col); - } else if constexpr (NUM_THREADS == 8) { - map_c_8lanes(tid, row, col); - } else { - // FIXME: not allowed - } -} - -#define RISCV_CUSTOM3 0x7B - -inline void vx_wmma(const int dest_reg) { - if (dest_reg == 0) { - asm volatile (".insn r %0, 0, 0, x0, x0, x0" :: "i"(RISCV_CUSTOM3)); - } else { - asm volatile (".insn r %0, 0, 0, x1, x0, x0" :: "i"(RISCV_CUSTOM3)); - } -} - -// `local_k` is assumed to be multiple of TCK -template -inline void vx_wmma_load_a(volatile const T *smem_A, const int local_k, - const int warp_row, const int wm_iter, const int thread_in_warp) { - const int tid = thread_in_warp; - const int tg = tid / 4; - - // @perf: this is duplicately computed in vx_wmma_load_a and vx_wmma_load_b - int row = 0; - int col = 0; - map_operand(tid, row, col); - - // In fp16 mode, bit-pack two fp16 elements into each fp32 element, and do - // data movement at the fp32 granularity. Assuming that the matrix is stored - // row-major in GMEM, the packed fp16 pairs belong to the same row, - // neighboring columns; therefore, it essentially becomes equivalent to - // moving a fp32 matrix whose column dimensions (dim_k/BK/k) are compressed - // by a factor of two. - constexpr uint32_t packed_factor = (std::is_same_v ? 2 : 1); - constexpr uint32_t BK_adjusted = BK / packed_factor; - - constexpr int smem_A_rows = BM; - constexpr int smem_A_cols = BK_adjusted; - constexpr int smem_AS_rows = BK_adjusted; - constexpr int smem_AS_cols = BM; - - if constexpr (TRANSPOSE_AT_CONSUME) { - // int A_offset = (WM * warp_row + TCM * wm_iter + row) * smem_A_cols; - - // @perf: bank conflicts - // f8-f15 stores a single row of A - const volatile uint8_t *smem_addr; - smem_addr = reinterpret_cast( - &reinterpret_cast( - smem_A)[(WM * warp_row + TCM * wm_iter + row) * smem_A_cols + - local_k]); - // step to the next column - // threads read from different rows; bank conflicts - asm volatile("flw f0, %0(%1)" ::"i"(0 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f1, %0(%1)" ::"i"(1 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f2, %0(%1)" ::"i"(2 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f3, %0(%1)" ::"i"(3 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f4, %0(%1)" ::"i"(4 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f5, %0(%1)" ::"i"(5 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f6, %0(%1)" ::"i"(6 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f7, %0(%1)" ::"i"(7 * sizeof(float)), "r"(smem_addr)); - } else { - // read smem A tile as-is; bank-conflict-free AS load - // smem A tile is stored column-major - // f8-f15 stores a single row of A - const volatile uint8_t *smem_addr; - smem_addr = reinterpret_cast( - &reinterpret_cast( - smem_A)[((local_k + 0) * smem_AS_cols) + - (WM * warp_row + TCM * wm_iter) + row]); - // step to the next row - // threads read from different columns; no bank conflicts - asm volatile("flw f0, %0(%1)" :: "i"(smem_AS_cols * 0 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f1, %0(%1)" :: "i"(smem_AS_cols * 1 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f2, %0(%1)" :: "i"(smem_AS_cols * 2 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f3, %0(%1)" :: "i"(smem_AS_cols * 3 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f4, %0(%1)" :: "i"(smem_AS_cols * 4 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f5, %0(%1)" :: "i"(smem_AS_cols * 5 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f6, %0(%1)" :: "i"(smem_AS_cols * 6 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f7, %0(%1)" :: "i"(smem_AS_cols * 7 * sizeof(float)), "r"(smem_addr)); - } -} - -// `local_k` is assumed to be multiple of TCK -template -inline void vx_wmma_load_b(const volatile T *smem_B, const int local_k, - const int warp_col, const int wn_iter, - const int thread_in_warp) { - const int tid = thread_in_warp; - const int tg = tid / 4; - - int row = 0; - int col = 0; - map_operand(tid, row, col); - - // see comment in vx_wmma_load_a - constexpr uint32_t packed_factor = (std::is_same_v ? 2 : 1); - constexpr uint32_t BN_adjusted = BN / packed_factor; - - constexpr int smem_B_rows = BK; - constexpr int smem_B_cols = BN_adjusted; - - // f8-f15 stores a single column of B - const volatile uint8_t *smem_addr; - smem_addr = reinterpret_cast( - &reinterpret_cast( - smem_B)[((local_k + 0) * smem_B_cols) + - (WN * warp_col + TCN * wn_iter) + col]); - // step to the next row - // threads read from different columns; no bank conflicts - asm volatile("flw f8, %0(%1)" :: "i"(smem_B_cols * 0 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f9, %0(%1)" :: "i"(smem_B_cols * 1 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f10, %0(%1)" :: "i"(smem_B_cols * 2 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f11, %0(%1)" :: "i"(smem_B_cols * 3 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f12, %0(%1)" :: "i"(smem_B_cols * 4 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f13, %0(%1)" :: "i"(smem_B_cols * 5 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f14, %0(%1)" :: "i"(smem_B_cols * 6 * sizeof(float)), "r"(smem_addr)); - asm volatile("flw f15, %0(%1)" :: "i"(smem_B_cols * 7 * sizeof(float)), "r"(smem_addr)); -} - -inline void initialize_C(const int dest_reg) { - // initialize C to zeros - if (dest_reg == 0) { - asm volatile("fmv.w.x f16, x0"); - asm volatile("fmv.w.x f17, x0"); - asm volatile("fmv.w.x f18, x0"); - asm volatile("fmv.w.x f19, x0"); - asm volatile("fmv.w.x f20, x0"); - asm volatile("fmv.w.x f21, x0"); - asm volatile("fmv.w.x f22, x0"); - asm volatile("fmv.w.x f23, x0"); - } else { - asm volatile("fmv.w.x f24, x0"); - asm volatile("fmv.w.x f25, x0"); - asm volatile("fmv.w.x f26, x0"); - asm volatile("fmv.w.x f27, x0"); - asm volatile("fmv.w.x f28, x0"); - asm volatile("fmv.w.x f29, x0"); - asm volatile("fmv.w.x f30, x0"); - asm volatile("fmv.w.x f31, x0"); - } -} - -inline void write_results(const int thread_in_warp, const int warp_col, - const int warp_row, const int wn_iter, - const int wm_iter, const int dim_n, - float *C, const int threadblock_id_x, - const int threadblock_id_y) { - int tid = thread_in_warp; - - // these are [0, TCM/TCN) - int tid_row = 0; - int tid_col = 0; - map_c(tid, tid_row, tid_col); - - int local_row = (WM * warp_row + TCM * wm_iter) + tid_row; - int local_col = (WN * warp_col + TCN * wn_iter) + tid_col; - - float *global_offset_C = - C + (BM * threadblock_id_y) * dim_n + BN * threadblock_id_x; - - // @perf: this likely causes a lot of gmem bank conflicts - if (wm_iter == 0) { - volatile uint8_t *gmem_addr = reinterpret_cast( - &global_offset_C[dim_n * (local_row + 0) + (local_col + 0)]); - volatile uint8_t *gmem_addr_tmp = gmem_addr + (2 * dim_n) * sizeof(float); - asm volatile ("fsw f16, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f17, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f18, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f19, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f20, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f21, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f22, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f23, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr_tmp)); - // asm volatile ("fsw f16, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 0)])); - // asm volatile ("fsw f17, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 1)])); - // asm volatile ("fsw f18, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 0)])); - // asm volatile ("fsw f19, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 1)])); - // asm volatile ("fsw f20, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 4)])); - // asm volatile ("fsw f21, %0" :: "m"(global_offset_C[dim_n * (local_row + 0) + (local_col + 5)])); - // asm volatile ("fsw f22, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 4)])); - // asm volatile ("fsw f23, %0" :: "m"(global_offset_C[dim_n * (local_row + 2) + (local_col + 5)])); - } else { - volatile uint8_t *gmem_addr = reinterpret_cast( - &global_offset_C[dim_n * (local_row + 0) + (local_col + 0)]); - volatile uint8_t *gmem_addr_tmp = gmem_addr + (2 * dim_n) * sizeof(float); - asm volatile ("fsw f24, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f25, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f26, %0(%1)" :: "i"(0 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f27, %0(%1)" :: "i"(1 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f28, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f29, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr)); - asm volatile ("fsw f30, %0(%1)" :: "i"(4 * sizeof(float)), "r"(gmem_addr_tmp)); - asm volatile ("fsw f31, %0(%1)" :: "i"(5 * sizeof(float)), "r"(gmem_addr_tmp)); - } -} - -inline void threadblock_barrier(const uint32_t barrier_id, const uint32_t count) { - vx_fence(); - vx_barrier(barrier_id, count); -} - -#endif