OFP: mpimcexec wrapper around mpirun for OFP users

arch/x86_64/tools/mpimcexec.in

@@ -0,0 +1,112 @@
+#!/bin/bash
+#
+# OFP McKernel MPI wrapper script
+# author: Balazs Gerofi <bgerofi@riken.jp>
+#      Copyright (C) 2018  RIKEN R-CCS
+#
+
+prefix="@prefix@"
+BINDIR="${prefix}/bin"
+
+if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then
+	echo "You need at least bash-4.0 to run this script." >&2
+	exit 1
+fi
+
+RANKS=""
+NODES=""
+PPN=""
+MPI_ENV=""
+COMMAND=""
+NUMA=""
+HOSTFILE=""
+
+help_exit() {
+	echo ""
+	echo "Spawn an McKernel MPI job on Oakforest-PACS."
+	echo "usage: `basename $0` -ppn ranks_per_node [--nodes nodes] [-n ranks] [--env additional_environment]... command"
+	echo ""
+	echo "   -ppn | --ppn | --ranks-per-node         Number of MPI ranks per node (required)"
+	echo "   -n | --n | --ranks                      Total number of MPI ranks in the job"
+	echo "   --nodes                                 Number of nodes to be used"
+	echo "   --env | -env                            Pass an additional environment variable"
+	echo "   -m | --numa                             Preferred NUMA node(s)"
+	echo "   -h | --hostfile                         Host file for MPI"
+	echo "   --help                                  Show help message"
+	exit 1
+}
+
+# Parse options
+while true; do
+	case $1 in
+		-ppn | --ppn | --ranks-per-node )
+			PPN=$2
+			shift 2
+			;;
+		-n | --n | --ranks )
+			RANKS=$2
+			shift 2
+			;;
+		-m | --numa )
+			NUMA="-m $2"
+			shift 2
+			;;
+		--nodes )
+			NODES=$2
+			shift 2
+			;;
+		--env | -env )
+			if [ -z "`echo $2 | grep I_MPI_PIN`" ]; then
+				MPI_ENV=`echo "${MPI_ENV} -env $2" | xargs`
+			fi
+			shift 2
+			;;
+		-h | --hostfile )
+			HOSTFILE="-hostfile $2"
+			shift 2
+			;;
+		--help )
+			help_exit
+			;;
+		* )
+			COMMAND=$@
+			break
+			;;
+	esac
+done
+
+if [ -z ${PPN} ]; then
+	echo "error: please specify the number of ranks per node"
+	help_exit
+fi
+
+# Unless explicitly specified, use Fujitsu inherited value
+if [ -z ${NODES} ]; then
+	NODES=${PJM_VNODES}
+fi
+
+if [ -z ${RANKS} ] && [ -z ${NODES} ]; then
+	echo "error: please specify the total number of ranks or the number of nodes"
+	help_exit
+fi
+
+# Calculate total job size if not specified
+if [ -z ${RANKS} ]; then
+	let RANKS=(${PPN}*${NODES})
+fi
+
+# Support direct SSH when not executed from Fujitsu job system
+if [ -z ${PJM_VNODES} ]; then
+	HOSTFILE="-launcher-exec ssh ${HOSTFILE}"
+fi
+
+export I_MPI_PIN=off
+export PSM2_RCVTHREAD=0
+export HFI_NO_CPUAFFINITY=1
+export I_MPI_COLL_INTRANODE_SHM_THRESHOLD=4194304
+export PSM2_MQ_RNDV_HFI_WINDOW=4194304
+export PSM2_MQ_EAGER_SDMA_SZ=65536
+export PSM2_MQ_RNDV_HFI_THRESH=200000
+
+mpirun ${HOSTFILE} -n ${RANKS} -ppn ${PPN} ${MPI_ENV} ${BINDIR}/mcexec -n ${PPN} ${NUMA} --enable-hfi1 --mpol-threshold=1M --stack-premap=4M,4G --extend-heap-by=8M --disable-sched-yield --mpol-shm-premap ${COMMAND}
+