From a6ac4acf404668e941d9009f64ccc9c97b6fb2a6 Mon Sep 17 00:00:00 2001 From: Masamichi Takagi Date: Tue, 19 Jun 2018 17:52:38 +0900 Subject: [PATCH] rusage: Fix initialization of rusage->num_processors Refs: #1064 Change-Id: I4c04127a766b9c71f726113b8b7d6416ff971bff --- kernel/mem.c | 7 +- test/rusage/Makefile | 22 +++- test/rusage/README | 5 +- test/rusage/run.sh | 30 ++++- test/rusage/rusage104.patch | 47 ++++++++ test/rusage/rusage104_lin.c | 38 +++++++ test/rusage/rusage104_mck.c | 221 ++++++++++++++++++++++++++++++++++++ test/rusage/util.h | 44 +++++++ 8 files changed, 402 insertions(+), 12 deletions(-) create mode 100644 test/rusage/rusage104.patch create mode 100644 test/rusage/rusage104_lin.c create mode 100644 test/rusage/rusage104_mck.c create mode 100644 test/rusage/util.h diff --git a/kernel/mem.c b/kernel/mem.c index e799ed86..27595637 100644 --- a/kernel/mem.c +++ b/kernel/mem.c @@ -1746,11 +1746,16 @@ static void rusage_init() { int npages; unsigned long phys; + const struct ihk_mc_cpu_info *cpu_info = ihk_mc_get_cpu_info(); + + if (!cpu_info) { + panic("rusage_init: PANIC: ihk_mc_get_cpu_info returned NULL"); + } npages = (sizeof(struct rusage_global) + PAGE_SIZE -1) >> PAGE_SHIFT; rusage = ihk_mc_alloc_pages(npages, IHK_MC_AP_CRITICAL); memset(rusage, 0, npages * PAGE_SIZE); - rusage->num_processors = num_processors; + rusage->num_processors = cpu_info->ncpus; rusage->num_numa_nodes = ihk_mc_get_nr_numa_nodes(); rusage->ns_per_tsc = ihk_mc_get_ns_per_tsc(); phys = virt_to_phys(rusage); diff --git a/test/rusage/Makefile b/test/rusage/Makefile index 93fe4728..b10e3926 100644 --- a/test/rusage/Makefile +++ b/test/rusage/Makefile @@ -1,18 +1,18 @@ .SUFFIXES: # Clear suffixes -.SUFFIXES: .c +.SUFFIXES: .c CC = gcc CPPFLAGS = -I$(HOME)/project/os/install/include CCFLAGS = -g -LDFLAGS = -L$(HOME)/project/os/install/lib -lihk -Wl,-rpath -Wl,$(HOME)/project/os/install/lib -EXES = -SRCS = +LDFLAGS = -L$(HOME)/project/os/install/lib -lihk -Wl,-rpath -Wl,$(HOME)/project/os/install/lib -lpthread +EXES = +SRCS = OBJS = $(SRCS:.c=.o) CPPFLAGSMCK = -I$(HOME)/usr/include CCFLAGSMCK = -g -O0 -LDFLAGSMCK = -static +LDFLAGSMCK = -static -lpthread SRCSMCK = $(shell ls rusage*.c) EXESMCK = $(SRCSMCK:.c=) OBJSMCK = $(SRCSMCK:.c=.o) @@ -163,5 +163,17 @@ rusage103: rusage103.o rusage103.o: rusage103.c $(CC) $(CCFLAGS) $(CPPFLAGS) -c $< +%_mck: %_mck.o + $(CC) -o $@ $^ $(LDFLAGSMCK) + +%_mck.o:: %_mck.c + $(CC) $(CCFLAGSMCK) $(CPPFLAGSMCK) -c $< + +%_lin.o:: %_lin.c + $(CC) $(CCFLAGS) $(CPPFLAGS) -c $< + +%_lin: %_lin.o + $(CC) -o $@ $^ $(LDFLAGS) + clean: rm -f core $(EXES) $(OBJS) $(EXESMCK) $(OBJSMCK) diff --git a/test/rusage/README b/test/rusage/README index 55f15daf..15ba045f 100644 --- a/test/rusage/README +++ b/test/rusage/README @@ -102,15 +102,16 @@ rusage021: Run npb bt-mz.S.4 2-ppn x 2-tpn x 2-node (wallaby{14,15}) [OK] 2-ppn x 1-tpn x 2-node (polaris,kochab) [OK] -rusage100: Test ihk_os_getrusage() +rusage100: Test ihk_os_getrusage() anon mmap,num_threads=1 [OK] rusage101: Test ihk_os_getrusage() anon mmap,num_threads=2 [OK] -rusage102: Test ihk_os_getrusage() +rusage102: Test ihk_os_getrusage() file map,num_threads=1 [OK] rusage103: Test ihk_os_getrusage() anon mmap@numa#1 [OK] +rusage104: Test ihk_os_getrusage(), user time per CPU diff --git a/test/rusage/run.sh b/test/rusage/run.sh index 8f5d7035..e54f4ee1 100755 --- a/test/rusage/run.sh +++ b/test/rusage/run.sh @@ -17,13 +17,18 @@ case ${testname} in printf "*** Enable debug messages in rusage.h, memory.c, fileobj.c, shmobj.c, process.c by defining DEBUG macro, e.g. #define RUSAGE_DEBUG and then recompile IHK/McKernel.\n" printf "*** Install xpmem by git-clone https://github.com/hjelmn/xpmem.\n" ;; - rusage10?) + rusage100 | rusage101 | rusage102 | rusage103) printf "*** Refer to rusage100.patch to add syscall #900 by editing syscall_list.h and syscall.c and recompile IHK/McKernel.\n" ;; + rusage104) + printf "*** Apply rusage104.patch to enable syscall #900" + printf "which reports rusage values.\n" + ;; *) printf "*** Enable debug messages in rusage.h, memory.c, fileobj.c, shmobj.c, process.c by defining DEBUG macro, e.g. #define RUSAGE_DEBUG and then recompile IHK/McKernel.\n" ;; esac + read -p "*** Hit return when ready!" key case ${testname} in @@ -45,6 +50,11 @@ case ${testname} in bn=npb/NPB3.3.1-MZ/NPB3.3-MZ-MPI/bin/bt-mz.S.4 perl -e 'print "polaris:2\nkochab:2\n"' > ./hostfile ;; + rusage104) + bn=${testname} + make clean > /dev/null 2> /dev/null + make ${bn}_mck ${bn}_lin + ;; *) bn=${testname} make clean > /dev/null 2> /dev/null @@ -141,8 +151,11 @@ case ${testname} in rusage103) bootopt="-m 256M@1" ;; + rusage104) + bootopt="-c 1,2,3 -m 256M" + ;; *) - echo Unknown test case + echo Unknown test case exit 255 esac @@ -199,6 +212,12 @@ else echo "================================================" >> ./${testname}.log sudo ${install}/sbin/ihkosctl 0 kmsg >> ./${testname}.log ;; + rusage104) + ${install}/bin/mcexec ${mcexecopt} ./${bn}_mck + ${install}/bin/mcexec ${mcexecopt} ./${bn}_lin + sudo ${install}/sbin/ihkosctl 0 kmsg > ./${testname}.log + grep user ./${testname}.log + ;; *) ${install}/bin/mcexec ${mcexecopt} ./${bn} ${testopt} sudo ${install}/sbin/ihkosctl 0 kmsg > ./${testname}.log @@ -206,10 +225,13 @@ else fi case ${testname} in - rusage10?) + rusage100 | rusage101 | rusage102 | rusage103) printf "*** Check the ihk_os_getrusage() result (the first part of ${testname}.log) matches with the syscall #900 result (the second part) \n" ;; - + rusage104) + printf "*** It behaves as expected when there's no [NG] and " + printf "\"All tests finished\" is shown\n" + ;; *) printf "*** cat ${testname}.log (kmsg) > ./match.pl to confirm there's no stray add/sub.\n" printf "*** Look ${testname}.log (kmsg) to confirm memory_stat_*[*] returned to zero when the last thread exits.\n" diff --git a/test/rusage/rusage104.patch b/test/rusage/rusage104.patch new file mode 100644 index 00000000..7930af69 --- /dev/null +++ b/test/rusage/rusage104.patch @@ -0,0 +1,47 @@ +diff --git arch/x86_64/kernel/include/syscall_list.h arch/x86_64/kernel/include/syscall_list.h +index 48b1ea0..3717bb2 100644 +--- arch/x86_64/kernel/include/syscall_list.h ++++ arch/x86_64/kernel/include/syscall_list.h +@@ -168,4 +168,6 @@ SYSCALL_HANDLED(802, linux_mlock) + SYSCALL_HANDLED(803, suspend_threads) + SYSCALL_HANDLED(804, resume_threads) + SYSCALL_HANDLED(811, linux_spawn) ++ ++SYSCALL_HANDLED(900, dump_rusage) + /**** End of File ****/ +diff --git kernel/syscall.c kernel/syscall.c +index 8dc0a0e..9969a4b 100644 +--- kernel/syscall.c ++++ kernel/syscall.c +@@ -9477,3 +9477,31 @@ long syscall(int num, ihk_mc_user_context_t *ctx) + + return l; + } ++ ++SYSCALL_DECLARE(dump_rusage) ++{ ++ /* rusage debug */ ++ int i; ++ for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) { ++ kprintf("memory_stat_rss[%d]=%ld\n", i, rusage->memory_stat_rss[i]); ++ } ++ for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) { ++ kprintf("memory_stat_mapped_file[%d]=%ld\n", i, rusage->memory_stat_mapped_file[i]); ++ } ++ kprintf("memory_max_usage=%ld\n", rusage->memory_max_usage); ++ kprintf("memory_kmem_usage=%ld\n", rusage->memory_kmem_usage); ++ kprintf("memory_kmem_max_usage=%ld\n", rusage->memory_kmem_max_usage); ++ for (i = 0; i < rusage->num_numa_nodes; i++) { ++ kprintf("memory_numa_stat[%d]=%ld\n", i, rusage->memory_numa_stat[i]); ++ } ++ kprintf("ns_per_tsc=%ld\n", rusage->ns_per_tsc); ++ for (i = 0; i < rusage->num_processors; i++) { ++ kprintf("cpu[%d].user_tsc=%ld\n", i, rusage->cpu[i].user_tsc); ++ kprintf("cpu[%d].system_tsc=%ld\n", i, rusage->cpu[i].system_tsc); ++ } ++ ++ kprintf("num_threads=%d\n", rusage->num_threads); ++ kprintf("max_num_threads=%d\n", rusage->max_num_threads); ++ ++ return 0; ++} diff --git a/test/rusage/rusage104_lin.c b/test/rusage/rusage104_lin.c new file mode 100644 index 00000000..192c36a8 --- /dev/null +++ b/test/rusage/rusage104_lin.c @@ -0,0 +1,38 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include "ihklib.h" +#include "mckernel/ihklib_rusage.h" +#include "util.h" + +#define DELAY0 (100UL * 1000 * 1000) +#define DELAY1 (200UL * 1000 * 1000) +#define SCALE 1.5 +#define WITHIN_RANGE(x, y, s) (x >= y && x <= y * s) + +int main(int argc, char **argv) +{ + int ret = 0; + struct mckernel_rusage rusage; + + if ((ret = ihk_os_getrusage(0, &rusage, sizeof(rusage)))) { + fprintf(stderr, "%s: ihk_os_getrusage failed\n", __func__); + ret = -EINVAL; + goto out; + } + + OKNG(WITHIN_RANGE(rusage.cpuacct_usage_percpu[1], DELAY0, SCALE), + "cpu 0: user time: expected: %ld nsec, reported: %ld nsec\n", + DELAY0, rusage.cpuacct_usage_percpu[1]); + OKNG(WITHIN_RANGE(rusage.cpuacct_usage_percpu[2], DELAY1, SCALE), + "cpu 1: user time: expected: %ld nsec, reported: %ld nsec\n", + DELAY1, rusage.cpuacct_usage_percpu[2]); + + printf("All tests finished\n"); + + out: + return ret; +} + diff --git a/test/rusage/rusage104_mck.c b/test/rusage/rusage104_mck.c new file mode 100644 index 00000000..a68f7d97 --- /dev/null +++ b/test/rusage/rusage104_mck.c @@ -0,0 +1,221 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define DELAY0 (100UL * 1000 * 1000) +#define DELAY1 (200UL * 1000 * 1000) + +struct thr_arg { + unsigned long delay; +}; + +struct thr_arg thr_arg[2] = { { .delay = DELAY0 }, { .delay = DELAY1 } }; +pthread_t thr[2]; + +#define DIFFNSEC(end, start) ((end.tv_sec - start.tv_sec) * 1000000000UL + \ + (end.tv_nsec - start.tv_nsec)) +#define TIMER_KIND CLOCK_MONOTONIC_RAW /* CLOCK_THREAD_CPUTIME_ID */ + +static int print_cpu_last_executed_on(void) +{ + int ret = 0; + char fn[256]; + char *result; + pid_t tid = syscall(SYS_gettid); + int fd; + int offset; + int amount = 0; + char *list; + char *token; + int i; + int cpu; + + sprintf(fn, "/proc/%d/task/%d/stat", getpid(), (int)tid); + fd = open(fn, O_RDONLY); + if (fd == -1) { + printf("open() failed\n"); + goto fn_fail; + } + + result = malloc(65536); + if (result == NULL) { + printf("malloc() failed"); + goto fn_fail; + } + + offset = 0; + while (1) { + amount = read(fd, result + offset, 65536); + // printf("amount=%d\n", amount); + if (amount == -1) { + printf("read() failed"); + goto fn_fail; + } + if (amount == 0) { + goto eof; + } + offset += amount; + } + eof:; + //printf("result:%s\n", result); + + list = result; + for (i = 0; i < 39; i++) { + token = strsep(&list, " "); + } + + cpu = sched_getcpu(); + if (cpu == -1) { + printf("getcpu() failed\n"); + goto fn_fail; + } + + printf("[INFO] stat-cpu=%02d,sched_getcpu=%02d,tid=%d\n", + token ? atoi(token) : -1, cpu, tid); + fn_exit: + free(result); + return ret; + fn_fail: + ret = -1; + goto fn_exit; +} + +static inline void asm_loop(unsigned long n) +{ + int j; + + for (j = 0; j < (n); j++) { + asm volatile( + "movq $0, %%rcx\n\t" + "1:\t" + "addq $1, %%rcx\n\t" + "cmpq $99, %%rcx\n\t" + "jle 1b\n\t" + : + : + : "rcx", "cc"); + } +} + +double nspw; /* nsec per work */ +unsigned long nsec; + +void fwq_init(void) +{ + struct timespec start, end; + int i; + + clock_gettime(TIMER_KIND, &start); +#define N_INIT 10000000 + asm_loop(N_INIT); + clock_gettime(TIMER_KIND, &end); + nsec = DIFFNSEC(end, start); + nspw = nsec / (double)N_INIT; +} + +#if 1 +void fwq(long delay_nsec) +{ + if (delay_nsec < 0) { + return; + } + asm_loop(delay_nsec / nspw); +} +#else +/* For machines with large core-to-core performance variation (e.g. OFP) */ +void fwq(long delay_nsec) +{ + struct timespec start, end; + + if (delay_nsec < 0) { + return; + } + clock_gettime(TIMER_KIND, &start); + + while (1) { + clock_gettime(TIMER_KIND, &end); + if (DIFFNSEC(end, start) >= delay_nsec) { + break; + } + asm_loop(2); /* ~150 ns per iteration on FOP */ + } +} +#endif + +void *util_thread(void *_arg) +{ + struct thr_arg *arg = (struct thr_arg *)_arg; + + print_cpu_last_executed_on(); + fwq(arg->delay); + pthread_exit(NULL); +} + +int main(int argc, char **argv) +{ + int i; + int ret = 0; + cpu_set_t cpuset; + pthread_attr_t attr[2]; + + fwq_init(); + + /* Migrate to cpu#0 */ + CPU_ZERO(&cpuset); + CPU_SET(0, &cpuset); + sched_setaffinity(0, sizeof(cpu_set_t), &cpuset); + print_cpu_last_executed_on(); + + for (i = 0; i < 2; i++) { + CPU_ZERO(&cpuset); + CPU_SET(i + 1, &cpuset); + + if ((ret = pthread_attr_init(&attr[i]))) { + printf("%s: ERROR: pthread_attr_init failed (%d)\n", + __func__, ret); + ret = -EINVAL; + goto out; + } + + if ((ret = pthread_attr_setaffinity_np(&attr[i], + sizeof(cpu_set_t), &cpuset))) { + printf("%s: ERROR: pthread_attr_setaffinity_np " + "failed (%d)\n", __func__, ret); + ret = -EINVAL; + goto out; + } + + if ((ret = pthread_create(&thr[i], &attr[i], + util_thread, &thr_arg[i]))) { + fprintf(stderr, "ERROR: pthread_create failed (%d)\n", + ret); + ret = -EINVAL; + goto out; + } + } + + for (i = 0; i < 2; i++) { + pthread_join(thr[i], NULL); + } + + if ((ret = syscall(900))) { + fprintf(stderr, "%s: syscall failed\n", __func__); + ret = -EINVAL; + goto out; + } + out: + return ret; +} + diff --git a/test/rusage/util.h b/test/rusage/util.h new file mode 100644 index 00000000..f33a1868 --- /dev/null +++ b/test/rusage/util.h @@ -0,0 +1,44 @@ +#ifndef __UTIL_H_INCLUDED__ +#define __UTIL_H_INCLUDED__ + +#define DEBUG + +#ifdef DEBUG +#define dprintf(...) do { \ + char msg[1024]; \ + sprintf(msg, __VA_ARGS__); \ + fprintf(stderr, "%s,%s", __func__, msg); \ +} while (0) +#else +#define dprintf(...) do { } while (0) +#endif + +#define eprintf(...) do { \ + char msg[1024]; \ + sprintf(msg, __VA_ARGS__); \ + fprintf(stderr, "%s,%s", __func__, msg); \ +} while (0) + +#define CHKANDJUMP(cond, err, ...) do { \ + if (cond) { \ + eprintf(__VA_ARGS__); \ + ret = err; \ + goto fn_fail; \ + } \ +} while (0) + + +#define OKNG(cond, ...) do { \ + if (cond) { \ + printf("[ OK ] "); \ + printf(__VA_ARGS__); \ + } else { \ + printf("[ NG ] "); \ + printf(__VA_ARGS__); \ + ret = -EINVAL; \ + goto out; \ + } \ +} while (0) + +#endif +