From bacfb0c2b9dbede246ca593af1058434f87387c5 Mon Sep 17 00:00:00 2001 From: Balazs Gerofi Date: Thu, 7 Jun 2018 08:49:03 +0900 Subject: [PATCH] OFP: mpimcexec wrapper around mpirun for OFP users --- Makefile.in | 2 + arch/x86_64/tools/mpimcexec.in | 112 +++++++++++++++++++++++++++++++++ configure | 3 +- configure.ac | 1 + 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100755 arch/x86_64/tools/mpimcexec.in diff --git a/Makefile.in b/Makefile.in index 45923b74..57362715 100755 --- a/Makefile.in +++ b/Makefile.in @@ -1,5 +1,6 @@ TARGET = @TARGET@ SBINDIR = @SBINDIR@ +BINDIR = @BINDIR@ INCDIR = @INCDIR@ ETCDIR = @ETCDIR@ MANDIR = @MANDIR@ @@ -47,6 +48,7 @@ install: mkdir -p -m 755 $(SBINDIR); \ install -m 755 arch/x86_64/tools/mcreboot-smp-x86.sh $(SBINDIR)/mcreboot.sh; \ install -m 755 arch/x86_64/tools/mcstop+release-smp-x86.sh $(SBINDIR)/mcstop+release.sh; \ + install -m 755 arch/x86_64/tools/mpimcexec $(BINDIR)/mpimcexec; \ install -m 755 arch/x86_64/tools/mcoverlay-destroy-smp-x86.sh $(SBINDIR)/mcoverlay-destroy.sh; \ install -m 755 arch/x86_64/tools/mcoverlay-create-smp-x86.sh $(SBINDIR)/mcoverlay-create.sh; \ install -m 755 arch/x86_64/tools/eclair-dump-backtrace.exp $(SBINDIR)/eclair-dump-backtrace.exp;\ diff --git a/arch/x86_64/tools/mpimcexec.in b/arch/x86_64/tools/mpimcexec.in new file mode 100755 index 00000000..abef6694 --- /dev/null +++ b/arch/x86_64/tools/mpimcexec.in @@ -0,0 +1,112 @@ +#!/bin/bash +# +# OFP McKernel MPI wrapper script +# author: Balazs Gerofi +# Copyright (C) 2018 RIKEN R-CCS +# + +prefix="@prefix@" +BINDIR="${prefix}/bin" + +if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then + echo "You need at least bash-4.0 to run this script." >&2 + exit 1 +fi + +RANKS="" +NODES="" +PPN="" +MPI_ENV="" +COMMAND="" +NUMA="" +HOSTFILE="" + +help_exit() { + echo "" + echo "Spawn an McKernel MPI job on Oakforest-PACS." + echo "usage: `basename $0` -ppn ranks_per_node [--nodes nodes] [-n ranks] [--env additional_environment]... command" + echo "" + echo " -ppn | --ppn | --ranks-per-node Number of MPI ranks per node (required)" + echo " -n | --n | --ranks Total number of MPI ranks in the job" + echo " --nodes Number of nodes to be used" + echo " --env | -env Pass an additional environment variable" + echo " -m | --numa Preferred NUMA node(s)" + echo " -h | --hostfile Host file for MPI" + echo " --help Show help message" + exit 1 +} + +# Parse options +while true; do + case $1 in + -ppn | --ppn | --ranks-per-node ) + PPN=$2 + shift 2 + ;; + -n | --n | --ranks ) + RANKS=$2 + shift 2 + ;; + -m | --numa ) + NUMA="-m $2" + shift 2 + ;; + --nodes ) + NODES=$2 + shift 2 + ;; + --env | -env ) + if [ -z "`echo $2 | grep I_MPI_PIN`" ]; then + MPI_ENV=`echo "${MPI_ENV} -env $2" | xargs` + fi + shift 2 + ;; + -h | --hostfile ) + HOSTFILE="-hostfile $2" + shift 2 + ;; + --help ) + help_exit + ;; + * ) + COMMAND=$@ + break + ;; + esac +done + +if [ -z ${PPN} ]; then + echo "error: please specify the number of ranks per node" + help_exit +fi + +# Unless explicitly specified, use Fujitsu inherited value +if [ -z ${NODES} ]; then + NODES=${PJM_VNODES} +fi + +if [ -z ${RANKS} ] && [ -z ${NODES} ]; then + echo "error: please specify the total number of ranks or the number of nodes" + help_exit +fi + +# Calculate total job size if not specified +if [ -z ${RANKS} ]; then + let RANKS=(${PPN}*${NODES}) +fi + +# Support direct SSH when not executed from Fujitsu job system +if [ -z ${PJM_VNODES} ]; then + HOSTFILE="-launcher-exec ssh ${HOSTFILE}" +fi + +export I_MPI_PIN=off +export PSM2_RCVTHREAD=0 +export HFI_NO_CPUAFFINITY=1 +export I_MPI_COLL_INTRANODE_SHM_THRESHOLD=4194304 +export PSM2_MQ_RNDV_HFI_WINDOW=4194304 +export PSM2_MQ_EAGER_SDMA_SZ=65536 +export PSM2_MQ_RNDV_HFI_THRESH=200000 + +mpirun ${HOSTFILE} -n ${RANKS} -ppn ${PPN} ${MPI_ENV} ${BINDIR}/mcexec -n ${PPN} ${NUMA} --enable-hfi1 --mpol-threshold=1M --stack-premap=4M,4G --extend-heap-by=8M --disable-sched-yield --mpol-shm-premap ${COMMAND} + diff --git a/configure b/configure index 44057115..36aced87 100755 --- a/configure +++ b/configure @@ -5060,7 +5060,7 @@ ac_config_headers="$ac_config_headers config.h" # POSTK_DEBUG_ARCH_DEP_37 # AC_CONFIG_FILES arch dependfiles separate -ac_config_files="$ac_config_files Makefile executer/user/Makefile executer/user/mcexec.1:executer/user/mcexec.1in executer/user/vmcore2mckdump executer/user/arch/$ARCH/Makefile executer/user/arch/x86_64/Makefile executer/kernel/mcctrl/Makefile executer/kernel/mcctrl/arch/$ARCH/Makefile executer/kernel/mcoverlayfs/Makefile executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile executer/kernel/mcoverlayfs/linux-4.0.9/Makefile executer/kernel/mcoverlayfs/linux-4.6.7/Makefile executer/include/qlmpilib.h kernel/Makefile kernel/Makefile.build kernel/include/swapfmt.h arch/x86_64/tools/mcreboot-attached-mic.sh arch/x86_64/tools/mcshutdown-attached-mic.sh arch/x86_64/tools/mcreboot-builtin-x86.sh arch/x86_64/tools/mcreboot-smp-x86.sh arch/x86_64/tools/mcstop+release-smp-x86.sh arch/x86_64/tools/mcoverlay-destroy-smp-x86.sh arch/x86_64/tools/mcoverlay-create-smp-x86.sh arch/x86_64/tools/eclair-dump-backtrace.exp arch/x86_64/tools/mcshutdown-builtin-x86.sh arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in arch/x86_64/tools/irqbalance_mck.service arch/x86_64/tools/irqbalance_mck.in tools/mcstat/Makefile" +ac_config_files="$ac_config_files Makefile executer/user/Makefile executer/user/mcexec.1:executer/user/mcexec.1in executer/user/vmcore2mckdump executer/user/arch/$ARCH/Makefile executer/user/arch/x86_64/Makefile executer/kernel/mcctrl/Makefile executer/kernel/mcctrl/arch/$ARCH/Makefile executer/kernel/mcoverlayfs/Makefile executer/kernel/mcoverlayfs/linux-3.10.0-327.36.1.el7/Makefile executer/kernel/mcoverlayfs/linux-4.0.9/Makefile executer/kernel/mcoverlayfs/linux-4.6.7/Makefile executer/include/qlmpilib.h kernel/Makefile kernel/Makefile.build kernel/include/swapfmt.h arch/x86_64/tools/mcreboot-attached-mic.sh arch/x86_64/tools/mcshutdown-attached-mic.sh arch/x86_64/tools/mcreboot-builtin-x86.sh arch/x86_64/tools/mcreboot-smp-x86.sh arch/x86_64/tools/mcstop+release-smp-x86.sh arch/x86_64/tools/mcoverlay-destroy-smp-x86.sh arch/x86_64/tools/mcoverlay-create-smp-x86.sh arch/x86_64/tools/eclair-dump-backtrace.exp arch/x86_64/tools/mcshutdown-builtin-x86.sh arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in arch/x86_64/tools/mpimcexec arch/x86_64/tools/irqbalance_mck.service arch/x86_64/tools/irqbalance_mck.in tools/mcstat/Makefile" if test "$TARGET" = "smp-x86"; then @@ -5795,6 +5795,7 @@ do "arch/x86_64/tools/eclair-dump-backtrace.exp") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/eclair-dump-backtrace.exp" ;; "arch/x86_64/tools/mcshutdown-builtin-x86.sh") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/mcshutdown-builtin-x86.sh" ;; "arch/x86_64/tools/mcreboot.1") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in" ;; + "arch/x86_64/tools/mpimcexec") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/mpimcexec" ;; "arch/x86_64/tools/irqbalance_mck.service") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/irqbalance_mck.service" ;; "arch/x86_64/tools/irqbalance_mck.in") CONFIG_FILES="$CONFIG_FILES arch/x86_64/tools/irqbalance_mck.in" ;; "tools/mcstat/Makefile") CONFIG_FILES="$CONFIG_FILES tools/mcstat/Makefile" ;; diff --git a/configure.ac b/configure.ac index 3dcbd72c..7e6d5078 100644 --- a/configure.ac +++ b/configure.ac @@ -568,6 +568,7 @@ AC_CONFIG_FILES([ arch/x86_64/tools/eclair-dump-backtrace.exp arch/x86_64/tools/mcshutdown-builtin-x86.sh arch/x86_64/tools/mcreboot.1:arch/x86_64/tools/mcreboot.1in + arch/x86_64/tools/mpimcexec arch/x86_64/tools/irqbalance_mck.service arch/x86_64/tools/irqbalance_mck.in tools/mcstat/Makefile