From 2ba3ec8a4c065c5e035dacfce5cc63644edba0ee Mon Sep 17 00:00:00 2001 From: Masamichi Takagi Date: Mon, 26 Nov 2018 10:20:57 +0900 Subject: [PATCH] mcstat: Fix memory related stats Refs: #1237 Change-Id: I0574cd71fe3b07aeda3ef981bd82d04ce5862f4f --- tools/mcstat/Makefile.in | 4 +- tools/mcstat/mcstat.c | 128 +++++++++++++++++++++++++++++---------- 2 files changed, 97 insertions(+), 35 deletions(-) diff --git a/tools/mcstat/Makefile.in b/tools/mcstat/Makefile.in index b834af06..948f6358 100644 --- a/tools/mcstat/Makefile.in +++ b/tools/mcstat/Makefile.in @@ -11,8 +11,8 @@ KDIR ?= @KDIR@ ARCH=@ARCH@ VPATH=@abs_srcdir@ IHKDIR ?= $(VPATH)/../../../ihk/linux/include/ -CFLAGS=-Wall -O -I. -I${IHKDIR} -I@abs_srcdir@/../../executer/include -LDFLAGS=@LDFLAGS@ +CFLAGS=-Wall -O -I. -I${IHKDIR} -I@abs_srcdir@/../../executer/include -I@abs_builddir@/../../../ihk/linux/include/ihk +LDFLAGS=-L@abs_builddir@/../../../ihk/linux/user -lihk -Wl,-rpath -Wl,$(prefix)/lib @LDFLAGS@ TARGET=mcstat .PHONY: all clean install diff --git a/tools/mcstat/mcstat.c b/tools/mcstat/mcstat.c index e8c1d00f..3fe7c833 100644 --- a/tools/mcstat/mcstat.c +++ b/tools/mcstat/mcstat.c @@ -16,8 +16,8 @@ #include #include +#include #include -#include // mcctrl_ioctl_getrusage_desc is defined here #undef IHK_MAX_NUM_NUMA_NODES #include // mckernel_rusage is defined here @@ -28,7 +28,22 @@ #define CONV_UNIT(d) (((float)(d))/scale) #define UPDATE_COUNTER(c) (c = (c + 1) % 10) -struct mckernel_rusage rbuf; +struct my_rusage { + struct mckernel_rusage rusage; + + /* Initial amount posted to allocator. Note that the amount + * used before the initialization is not included. + */ + unsigned long memory_total; + + /* Current of sum of kernel and user */ + unsigned long memory_cur_usage; + + /* Max of sum of kernel and user */ + unsigned long memory_max_usage; +}; + +struct my_rusage rbuf; static void mcstatistics(int idx, int once, int delay, int count); static int mcstatus(int idx, int delay, int count); @@ -114,22 +129,67 @@ statistics_header(char *unit) * the device, and cannot be rebooted by others. */ static int -mygetrusage(int idx, struct mckernel_rusage *rbp) +mygetrusage(int idx, struct my_rusage *rbp) { - int fd, rc; - struct mcctrl_ioctl_getrusage_desc rusage; + int rc; + int num_numa_nodes; + int i; + unsigned long *memtotal = NULL; - if ((fd = devopen(idx)) < 0) { - return -1; - } - rusage.rusage = rbp; - rusage.size_rusage = sizeof(struct mckernel_rusage); - memset(rbp, 0, sizeof(struct mckernel_rusage)); - if ((rc = ioctl(fd, IHK_OS_GETRUSAGE, &rusage)) < 0) { - perror("ioctl"); exit(-1); - } - close(fd); - return 0; + rc = ihk_os_getrusage(idx, &rbp->rusage, + sizeof(struct mckernel_rusage)); + if (rc) { + printf("%s: error: ihk_os_getrusage: %s\n", + __func__, strerror(-rc)); + goto out; + } + + num_numa_nodes = ihk_os_get_num_numa_nodes(idx); + if (num_numa_nodes <= 0) { + printf("%s: error: ihk_os_get_num_numa_nodes: %d\n", + __func__, num_numa_nodes); + rc = num_numa_nodes < 0 ? : -EINVAL; + goto out; + } + + /* Calculate total by taking a sum over NUMA nodes */ + + memtotal = calloc(num_numa_nodes, sizeof(unsigned long)); + if (!memtotal) { + printf("%s: error: assigining memory\n", + __func__); + rc = -ENOMEM; + goto out; + } + + rc = ihk_os_query_total_mem(idx, memtotal, num_numa_nodes); + if (rc) { + printf("%s: error: ihk_os_query_total_mem: %s\n", + __func__, strerror(-rc)); + goto out; + } + + rbp->memory_total = 0; + for (i = 0; i < num_numa_nodes; i++) { + rbp->memory_total += memtotal[i]; + } + + /* Calculate current by taking a sum over NUMA nodes */ + + rbp->memory_cur_usage = rbp->rusage.memory_kmem_usage; + for (i = 0; i < num_numa_nodes; i++) { + rbp->memory_cur_usage += rbp->rusage.memory_numa_stat[i]; + } + + /* Calculate max by taking a sum of kernel and user */ + + rbp->memory_max_usage = rbp->rusage.memory_kmem_max_usage + + rbp->rusage.memory_max_usage; + + rc = 0; +out: + free(memtotal); + return rc; } static void @@ -143,19 +203,20 @@ mcstatistics(int idx, int once, int delay, int count) printf("Device has not been created.\n"); exit(-1); } - if (rbuf.memory_max_usage < MiB100) { - scale = MiB; unit = "MB"; - } else { - scale = GiB; unit = "GB"; - } + if (rbuf.rusage.memory_max_usage < MiB100) { + scale = MiB; unit = "MB"; + } else { + scale = GiB; unit = "GB"; + } statistics_header(unit); for (;;) { + printf("%9.3f%9.3f%9.3f %9ld%9ld %7d %3d\n", + CONV_UNIT(rbuf.memory_total), + CONV_UNIT(rbuf.memory_cur_usage), CONV_UNIT(rbuf.memory_max_usage), - CONV_UNIT(rbuf.memory_kmem_usage), - CONV_UNIT(rbuf.memory_kmem_max_usage), - rbuf.cpuacct_stat_system, rbuf.cpuacct_stat_user, - rbuf.num_threads, rbuf.max_num_threads); + rbuf.rusage.cpuacct_stat_system, rbuf.rusage.cpuacct_stat_user, + rbuf.rusage.num_threads, rbuf.rusage.max_num_threads); if (count > 0 && --count == 0) break; sleep(delay); if (mygetrusage(idx, &rbuf) < 0) { @@ -173,11 +234,12 @@ mcstatistics(int idx, int once, int delay, int count) rusage->cpuacct_stat_system = st / 10000000; rusage->cpuacct_stat_user = ut / 10000000; rusage->cpuacct_usage = ut; - printf("cpuacct_usage = %x\n", rbuf.cpuacct_usage); + printf("cpuacct_usage = %x\n", rbuf.rusage.cpuacct_usage); */ - for (i = 0; i < rbuf.max_num_threads; i++) { - printf("cpuacct_usage_percpu[%d] = %ld\n", i, rbuf.cpuacct_usage_percpu[i]); - } + for (i = 0; i < rbuf.rusage.max_num_threads; i++) { + printf("cpuacct_usage_percpu[%d] = %ld\n", + i, rbuf.rusage.cpuacct_usage_percpu[i]); + } } /* ihk_os_status enum is defined in ihk/linux/include/ihk/status.h */ @@ -279,10 +341,10 @@ mcosusage(int idx, int once, int delay, int count) unsigned char show = 0; struct ihk_os_cpu_monitor mon[MAX_CPUS]; - if (mygetrusage(idx, &rbuf) < 0) { - printf("Device has not been created.\n"); - } - ncpus = rbuf.max_num_threads; + if (mygetrusage(idx, &rbuf) < 0) { + printf("Device has not been created.\n"); + } + ncpus = rbuf.rusage.max_num_threads; osusage_header(); for(;;) { if ((fd = devopen(idx)) < 0) {