rusage: Fix initialization of rusage->num_processors

Refs: #1064
Change-Id: I4c04127a766b9c71f726113b8b7d6416ff971bff

kernel/mem.c

@@ -1746,11 +1746,16 @@ static void rusage_init()
 {
 	int npages;
 	unsigned long phys;
+	const struct ihk_mc_cpu_info *cpu_info = ihk_mc_get_cpu_info();
+
+	if (!cpu_info) {
+		panic("rusage_init: PANIC: ihk_mc_get_cpu_info returned NULL");
+	}
 
 	npages = (sizeof(struct rusage_global) + PAGE_SIZE -1) >> PAGE_SHIFT;
 	rusage = ihk_mc_alloc_pages(npages, IHK_MC_AP_CRITICAL);
 	memset(rusage, 0, npages * PAGE_SIZE);
-	rusage->num_processors = num_processors;
+	rusage->num_processors = cpu_info->ncpus;
 	rusage->num_numa_nodes = ihk_mc_get_nr_numa_nodes();
 	rusage->ns_per_tsc = ihk_mc_get_ns_per_tsc();
 	phys = virt_to_phys(rusage);

test/rusage/Makefile

@@ -1,18 +1,18 @@
 .SUFFIXES:	# Clear suffixes
-.SUFFIXES:	.c 
+.SUFFIXES:	.c
 
 CC = gcc
 
 CPPFLAGS = -I$(HOME)/project/os/install/include
 CCFLAGS = -g
-LDFLAGS = -L$(HOME)/project/os/install/lib -lihk -Wl,-rpath -Wl,$(HOME)/project/os/install/lib
+LDFLAGS = -L$(HOME)/project/os/install/lib -lihk -Wl,-rpath -Wl,$(HOME)/project/os/install/lib -lpthread
-EXES = 
+EXES =
-SRCS = 
+SRCS =
 OBJS = $(SRCS:.c=.o)
 
 CPPFLAGSMCK = -I$(HOME)/usr/include
 CCFLAGSMCK = -g -O0
-LDFLAGSMCK = -static 
+LDFLAGSMCK = -static -lpthread
 SRCSMCK = $(shell ls rusage*.c)
 EXESMCK = $(SRCSMCK:.c=)
 OBJSMCK = $(SRCSMCK:.c=.o)
@@ -163,5 +163,17 @@ rusage103: rusage103.o
 rusage103.o: rusage103.c
 	$(CC) $(CCFLAGS) $(CPPFLAGS) -c $<
 
+%_mck: %_mck.o
+	$(CC) -o $@ $^ $(LDFLAGSMCK)
+
+%_mck.o:: %_mck.c
+	$(CC) $(CCFLAGSMCK) $(CPPFLAGSMCK) -c $<
+
+%_lin.o:: %_lin.c
+	$(CC) $(CCFLAGS) $(CPPFLAGS) -c $<
+
+%_lin: %_lin.o
+	$(CC) -o $@ $^ $(LDFLAGS)
+
 clean:
 	rm -f core $(EXES) $(OBJS) $(EXESMCK) $(OBJSMCK)

test/rusage/README

@@ -102,15 +102,16 @@ rusage021: Run npb bt-mz.S.4
 2-ppn x 2-tpn x 2-node (wallaby{14,15}) [OK]
 2-ppn x 1-tpn x 2-node (polaris,kochab) [OK]
 
-rusage100: Test ihk_os_getrusage() 
+rusage100: Test ihk_os_getrusage()
 anon mmap,num_threads=1 [OK]
 
 rusage101: Test ihk_os_getrusage()
 anon mmap,num_threads=2 [OK]
 
-rusage102: Test ihk_os_getrusage()  
+rusage102: Test ihk_os_getrusage()
 file map,num_threads=1 [OK]
 
 rusage103: Test ihk_os_getrusage()
 anon mmap@numa#1 [OK]
 
+rusage104: Test ihk_os_getrusage(), user time per CPU

test/rusage/run.sh

@@ -17,13 +17,18 @@ case ${testname} in
 	printf "*** Enable debug messages in rusage.h, memory.c, fileobj.c, shmobj.c, process.c by defining DEBUG macro, e.g. #define RUSAGE_DEBUG and then recompile IHK/McKernel.\n"
 	printf "*** Install xpmem by git-clone https://github.com/hjelmn/xpmem.\n"
 	;;
-    rusage10?)
+    rusage100 | rusage101 | rusage102 | rusage103)
 	printf "*** Refer to rusage100.patch to add syscall #900 by editing syscall_list.h and syscall.c and recompile IHK/McKernel.\n"
 	;;
+    rusage104)
+	printf "*** Apply rusage104.patch to enable syscall #900"
+	printf "which reports rusage values.\n"
+	;;
     *)
 	printf "*** Enable debug messages in rusage.h, memory.c, fileobj.c, shmobj.c, process.c by defining DEBUG macro, e.g. #define RUSAGE_DEBUG and then recompile IHK/McKernel.\n"
 	;;
 esac
+
 read -p "*** Hit return when ready!" key
 
 case ${testname} in
@@ -45,6 +50,11 @@ case ${testname} in
 	bn=npb/NPB3.3.1-MZ/NPB3.3-MZ-MPI/bin/bt-mz.S.4
 	perl -e 'print "polaris:2\nkochab:2\n"' > ./hostfile
 	;;
+    rusage104)
+	bn=${testname}
+	make clean > /dev/null 2> /dev/null
+	make ${bn}_mck ${bn}_lin
+	;;
     *)
 	bn=${testname}
 	make clean > /dev/null 2> /dev/null
@@ -141,8 +151,11 @@ case ${testname} in
     rusage103)
 	bootopt="-m 256M@1"
 	;;
+    rusage104)
+	bootopt="-c 1,2,3 -m 256M"
+	;;
     *)
-	echo Unknown test case 
+	echo Unknown test case
 	exit 255
 esac
 
@@ -199,6 +212,12 @@ else
 	    echo "================================================" >> ./${testname}.log
 	    sudo ${install}/sbin/ihkosctl 0 kmsg >> ./${testname}.log
 	    ;;
+	rusage104)
+	    ${install}/bin/mcexec ${mcexecopt} ./${bn}_mck
+	    ${install}/bin/mcexec ${mcexecopt} ./${bn}_lin
+	    sudo ${install}/sbin/ihkosctl 0 kmsg > ./${testname}.log
+	    grep user ./${testname}.log
+	    ;;
 	*)
 	    ${install}/bin/mcexec ${mcexecopt} ./${bn} ${testopt}
 	    sudo ${install}/sbin/ihkosctl 0 kmsg > ./${testname}.log
@@ -206,10 +225,13 @@ else
 fi
 
 case ${testname} in
-    rusage10?)
+    rusage100 | rusage101 | rusage102 | rusage103)
 	printf "*** Check the ihk_os_getrusage() result (the first part of ${testname}.log) matches with the syscall #900 result (the second part) \n"
 	;;
-
+    rusage104)
+	printf "*** It behaves as expected when there's no [NG] and "
+	printf "\"All tests finished\" is shown\n"
+	;;
     *)
 	printf "*** cat ${testname}.log (kmsg) > ./match.pl to confirm there's no stray add/sub.\n"
 	printf "*** Look ${testname}.log (kmsg) to confirm memory_stat_*[*] returned to zero when the last thread exits.\n"

test/rusage/rusage104.patch

@@ -0,0 +1,47 @@
+diff --git arch/x86_64/kernel/include/syscall_list.h arch/x86_64/kernel/include/syscall_list.h
+index 48b1ea0..3717bb2 100644
+--- arch/x86_64/kernel/include/syscall_list.h
++++ arch/x86_64/kernel/include/syscall_list.h
+@@ -168,4 +168,6 @@ SYSCALL_HANDLED(802, linux_mlock)
+ SYSCALL_HANDLED(803, suspend_threads)
+ SYSCALL_HANDLED(804, resume_threads)
+ SYSCALL_HANDLED(811, linux_spawn)
++
++SYSCALL_HANDLED(900, dump_rusage)
+ /**** End of File ****/
+diff --git kernel/syscall.c kernel/syscall.c
+index 8dc0a0e..9969a4b 100644
+--- kernel/syscall.c
++++ kernel/syscall.c
+@@ -9477,3 +9477,31 @@ long syscall(int num, ihk_mc_user_context_t *ctx)
+ 
+ 	return l;
+ }
++
++SYSCALL_DECLARE(dump_rusage)
++{
++	/* rusage debug */
++	int i;
++	for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) {
++		kprintf("memory_stat_rss[%d]=%ld\n", i, rusage->memory_stat_rss[i]);
++	}
++	for(i = 0; i < IHK_MAX_NUM_PGSIZES; i++) {
++		kprintf("memory_stat_mapped_file[%d]=%ld\n", i, rusage->memory_stat_mapped_file[i]);
++	}
++	kprintf("memory_max_usage=%ld\n", rusage->memory_max_usage);
++	kprintf("memory_kmem_usage=%ld\n", rusage->memory_kmem_usage);
++	kprintf("memory_kmem_max_usage=%ld\n", rusage->memory_kmem_max_usage);
++	for (i = 0; i < rusage->num_numa_nodes; i++) {
++		kprintf("memory_numa_stat[%d]=%ld\n", i, rusage->memory_numa_stat[i]);
++	}
++	kprintf("ns_per_tsc=%ld\n", rusage->ns_per_tsc);
++	for (i = 0; i < rusage->num_processors; i++) {
++		kprintf("cpu[%d].user_tsc=%ld\n", i, rusage->cpu[i].user_tsc);
++		kprintf("cpu[%d].system_tsc=%ld\n", i, rusage->cpu[i].system_tsc);
++	}
++
++	kprintf("num_threads=%d\n", rusage->num_threads);
++	kprintf("max_num_threads=%d\n", rusage->max_num_threads);
++
++	return 0;
++}

test/rusage/rusage104_lin.c

@@ -0,0 +1,38 @@
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include "ihklib.h"
+#include "mckernel/ihklib_rusage.h"
+#include "util.h"
+
+#define DELAY0 (100UL * 1000 * 1000)
+#define DELAY1 (200UL * 1000 * 1000)
+#define SCALE 1.5
+#define WITHIN_RANGE(x, y, s) (x >= y && x <= y * s)
+
+int main(int argc, char **argv)
+{
+	int ret = 0;
+	struct mckernel_rusage rusage;
+
+	if ((ret = ihk_os_getrusage(0, &rusage, sizeof(rusage)))) {
+		fprintf(stderr, "%s: ihk_os_getrusage failed\n", __func__);
+		ret = -EINVAL;
+		goto out;
+	}
+
+	OKNG(WITHIN_RANGE(rusage.cpuacct_usage_percpu[1], DELAY0, SCALE),
+	     "cpu 0: user time: expected: %ld nsec, reported: %ld nsec\n",
+	     DELAY0, rusage.cpuacct_usage_percpu[1]);
+	OKNG(WITHIN_RANGE(rusage.cpuacct_usage_percpu[2], DELAY1, SCALE),
+	     "cpu 1: user time: expected: %ld nsec, reported: %ld nsec\n",
+	     DELAY1, rusage.cpuacct_usage_percpu[2]);
+
+	printf("All tests finished\n");
+
+ out:
+	return ret;
+}
+

test/rusage/rusage104_mck.c

@@ -0,0 +1,221 @@
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <string.h>
+#include <pthread.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+
+#define DELAY0 (100UL * 1000 * 1000)
+#define DELAY1 (200UL * 1000 * 1000)
+
+struct thr_arg {
+	unsigned long delay;
+};
+
+struct thr_arg thr_arg[2] = { { .delay = DELAY0 }, { .delay = DELAY1 } };
+pthread_t thr[2];
+
+#define DIFFNSEC(end, start) ((end.tv_sec - start.tv_sec) * 1000000000UL + \
+		(end.tv_nsec - start.tv_nsec))
+#define TIMER_KIND CLOCK_MONOTONIC_RAW /* CLOCK_THREAD_CPUTIME_ID */
+
+static int print_cpu_last_executed_on(void)
+{
+	int ret = 0;
+	char fn[256];
+	char *result;
+	pid_t tid = syscall(SYS_gettid);
+	int fd;
+	int offset;
+	int amount = 0;
+	char *list;
+	char *token;
+	int i;
+	int cpu;
+
+	sprintf(fn, "/proc/%d/task/%d/stat", getpid(), (int)tid);
+	fd = open(fn, O_RDONLY);
+	if (fd == -1) {
+		printf("open() failed\n");
+		goto fn_fail;
+	}
+
+	result = malloc(65536);
+	if (result == NULL) {
+		printf("malloc() failed");
+		goto fn_fail;
+	}
+
+	offset = 0;
+	while (1) {
+		amount = read(fd, result + offset, 65536);
+		//		printf("amount=%d\n", amount);
+		if (amount == -1) {
+			printf("read() failed");
+			goto fn_fail;
+		}
+		if (amount == 0) {
+			goto eof;
+		}
+		offset += amount;
+	}
+ eof:;
+	//printf("result:%s\n", result);
+
+	list = result;
+	for (i = 0; i < 39; i++) {
+		token = strsep(&list, " ");
+	}
+
+	cpu = sched_getcpu();
+	if (cpu == -1) {
+		printf("getcpu() failed\n");
+		goto fn_fail;
+	}
+
+	printf("[INFO] stat-cpu=%02d,sched_getcpu=%02d,tid=%d\n",
+		token ? atoi(token) : -1, cpu, tid);
+ fn_exit:
+	free(result);
+	return ret;
+ fn_fail:
+	ret = -1;
+	goto fn_exit;
+}
+
+static inline void asm_loop(unsigned long n)
+{
+	int j;
+
+	for (j = 0; j < (n); j++) {
+		asm volatile(
+			     "movq $0, %%rcx\n\t"
+			     "1:\t"
+			     "addq $1, %%rcx\n\t"
+			     "cmpq $99, %%rcx\n\t"
+			     "jle 1b\n\t"
+			     :
+			     :
+			     : "rcx", "cc");
+	}
+}
+
+double nspw; /* nsec per work */
+unsigned long nsec;
+
+void fwq_init(void)
+{
+	struct timespec start, end;
+	int i;
+
+	clock_gettime(TIMER_KIND, &start);
+#define N_INIT 10000000
+	asm_loop(N_INIT);
+	clock_gettime(TIMER_KIND, &end);
+	nsec = DIFFNSEC(end, start);
+	nspw = nsec / (double)N_INIT;
+}
+
+#if 1
+void fwq(long delay_nsec)
+{
+	if (delay_nsec < 0) {
+		return;
+	}
+	asm_loop(delay_nsec / nspw);
+}
+#else
+/* For machines with large core-to-core performance variation (e.g. OFP) */
+void fwq(long delay_nsec)
+{
+	struct timespec start, end;
+
+	if (delay_nsec < 0) {
+		return;
+	}
+	clock_gettime(TIMER_KIND, &start);
+
+	while (1) {
+		clock_gettime(TIMER_KIND, &end);
+		if (DIFFNSEC(end, start) >= delay_nsec) {
+			break;
+		}
+		asm_loop(2); /* ~150 ns per iteration on FOP */
+	}
+}
+#endif
+
+void *util_thread(void *_arg)
+{
+	struct thr_arg *arg = (struct thr_arg *)_arg;
+
+	print_cpu_last_executed_on();
+	fwq(arg->delay);
+	pthread_exit(NULL);
+}
+
+int main(int argc, char **argv)
+{
+	int i;
+	int ret = 0;
+	cpu_set_t cpuset;
+	pthread_attr_t attr[2];
+
+	fwq_init();
+
+	/* Migrate to cpu#0 */
+	CPU_ZERO(&cpuset);
+	CPU_SET(0, &cpuset);
+	sched_setaffinity(0, sizeof(cpu_set_t), &cpuset);
+	print_cpu_last_executed_on();
+
+	for (i = 0; i < 2; i++) {
+		CPU_ZERO(&cpuset);
+		CPU_SET(i + 1, &cpuset);
+
+		if ((ret = pthread_attr_init(&attr[i]))) {
+			printf("%s: ERROR: pthread_attr_init failed (%d)\n",
+				__func__, ret);
+			ret = -EINVAL;
+			goto out;
+		}
+
+		if ((ret = pthread_attr_setaffinity_np(&attr[i],
+			sizeof(cpu_set_t), &cpuset))) {
+			printf("%s: ERROR: pthread_attr_setaffinity_np "
+				"failed (%d)\n", __func__, ret);
+			ret = -EINVAL;
+			goto out;
+		}
+
+		if ((ret = pthread_create(&thr[i], &attr[i],
+			util_thread, &thr_arg[i]))) {
+			fprintf(stderr, "ERROR: pthread_create failed (%d)\n",
+				ret);
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
+	for (i = 0; i < 2; i++) {
+		pthread_join(thr[i], NULL);
+	}
+
+	if ((ret = syscall(900))) {
+		fprintf(stderr, "%s: syscall failed\n", __func__);
+		ret = -EINVAL;
+		goto out;
+	}
+ out:
+	return ret;
+}
+

test/rusage/util.h

@@ -0,0 +1,44 @@
+#ifndef __UTIL_H_INCLUDED__
+#define __UTIL_H_INCLUDED__
+
+#define DEBUG
+
+#ifdef DEBUG
+#define dprintf(...) do {			 \
+	char msg[1024];			 \
+	sprintf(msg, __VA_ARGS__);		 \
+	fprintf(stderr, "%s,%s", __func__, msg); \
+} while (0)
+#else
+#define dprintf(...) do {  } while (0)
+#endif
+
+#define eprintf(...) do {			 \
+	char msg[1024];			 \
+	sprintf(msg, __VA_ARGS__);		 \
+	fprintf(stderr, "%s,%s", __func__, msg); \
+} while (0)
+
+#define CHKANDJUMP(cond, err, ...) do { \
+	if (cond) {			\
+		eprintf(__VA_ARGS__);   \
+		ret = err;		\
+		goto fn_fail;		\
+	}				\
+} while (0)
+
+
+#define OKNG(cond, ...) do {				\
+	if (cond) {					\
+		printf("[ OK ] ");			\
+		printf(__VA_ARGS__);			\
+	} else {					\
+		printf("[ NG ] ");			\
+		printf(__VA_ARGS__);			\
+		ret = -EINVAL;				\
+		goto out;				\
+	}						\
+} while (0)
+
+#endif
+