#!/bin/bash
# mcstop+release-smp-x86.sh.in COPYRIGHT FUJITSU LIMITED 2018

# IHK SMP-x86 example McKernel unload script.
# author: Balazs Gerofi <bgerofi@riken.jp>
#      Copyright (C) 2015  RIKEN AICS
# 
# This is an example script for destroying McKernel and releasing IHK resources
# Note that the script does no output anything unless an error occurs.

prefix="@prefix@"
BINDIR="@BINDIR@"
SBINDIR="@SBINDIR@"
ETCDIR=@ETCDIR@
KMODDIR="@KMODDIR@"
KERNDIR="@KERNDIR@"

mem=""
cpus=""
kill_in_use=""
dont_unload="no"

RMMOD_PATH=/sbin/rmmod
while getopts r:kR OPT
do
	case ${OPT} in
	r)
		RMMOD_PATH=${OPTARG}
		;;
	k)
		kill_in_use=1
		;;
	R)
		dont_unload="yes"
		;;
	\?) exit 1
		;;
	esac
done
if [[ "$#" -ge "$OPTIND" ]]; then
	shift $((OPTIND - 1))
	echo "Extra arguments: $@" >&2
	exit 1
fi

if [ ! -f ${RMMOD_PATH} ]; then
	echo "error: ${RMMOD_PATH} doesn't exist, please specify by -r <path_to_rmmod>"
	exit 1
fi

# No SMP module? Exit.
if ! grep ihk_smp_@ARCH@ /proc/modules &>/dev/null; then exit 0; fi

if [[ -f /run/systemd/system/irqbalance.service.d/mckernel.conf ]]; then
	sudo systemctl stop irqbalance 2>/dev/null
fi

# Destroy all LWK instances
if ls /dev/mcos* 1>/dev/null 2>&1; then
	for i in /dev/mcos*; do
		ind=`echo $i|cut -c10-`;

		# Are processes still using it?
		PROCS=$(lsof -t $i 2>/dev/null)
		if [[ -n "$PROCS" ]]; then
			if ((kill_in_use)); then
				kill -9 $PROCS
			else
				echo "Proesses still using $i: $PROCS" >&2
				exit 1
			fi
		fi

		# Retry when conflicting with ihkmond
		nretry=0
		until sudo ${SBINDIR}/ihkconfig 0 destroy $ind || [ $nretry -ge 4 ]; do
		    sleep 0.25
		    nretry=$[ $nretry + 1 ]
		done
		if [ $nretry -eq 4 ]; then
		    echo "error: destroying LWK instance $ind failed" >&2
		    exit 1
		fi
	done
fi

# Allow ihkmond to flush kmsg buffer
sleep 2.0

# Query IHK-SMP resources and release them
if ! sudo ${SBINDIR}/ihkconfig 0 query cpu > /dev/null; then
	echo "error: querying cpus" >&2
	exit 1
fi

cpus=`sudo ${SBINDIR}/ihkconfig 0 query cpu`
if [ "${cpus}" != "" ]; then
	if ! sudo ${SBINDIR}/ihkconfig 0 release cpu $cpus > /dev/null; then
		echo "warning: failed to release CPUs" >&2
	fi
fi

#if ! ${SBINDIR}/ihkconfig 0 query mem > /dev/null; then
#	echo "error: querying memory" >&2
#	exit 1
#fi
#
#mem=`${SBINDIR}/ihkconfig 0 query mem`
#if [ "${mem}" != "" ]; then
#	if ! ${SBINDIR}/ihkconfig 0 release mem $mem > /dev/null; then
#		echo "error: releasing memory" >&2
#		exit 1
#	fi
#fi

# Release all memory
if ! sudo ${SBINDIR}/ihkconfig 0 release mem "all" > /dev/null; then
	echo "error: releasing memory" >&2
	exit 1
fi

if [ "${dont_unload}" == "yes" ]; then
	exit 0
fi

# Remove delegator if loaded
if grep mcctrl /proc/modules &>/dev/null; then
	if ! sudo ${RMMOD_PATH} mcctrl 2>/dev/null; then
		echo "error: removing mcctrl" >&2
		exit 1
	fi
fi

# Remove SMP module
if grep ihk_smp_@ARCH@ /proc/modules &>/dev/null; then
	if ! sudo ${RMMOD_PATH} ihk_smp_@ARCH@ 2>/dev/null; then
		echo "error: removing ihk_smp_@ARCH@" >&2
		exit 1
	fi
fi

# Remove core module
if grep -E 'ihk\s' /proc/modules &>/dev/null; then
	if ! sudo ${RMMOD_PATH} ihk 2>/dev/null; then
		echo "error: removing ihk" >&2
		exit 1
	fi
fi

# Stop ihkmond
pid=`pidof ihkmond`
if [ "${pid}" != "" ]; then
    sudo kill -9 ${pid} > /dev/null 2> /dev/null
fi

# Start irqbalance with the original settings
if [ -f /run/systemd/system/irqbalance.service.d/mckernel.conf ]; then
	for f in /run/sysconfig/irqbalance_mck_affinities/proc/irq/*/smp_affinity; do
		cat "$f" | sudo tee "${f#/run/sysconfig/irqbalance_mck_affinities}" >/dev/null 2>&1
	done
	sudo rm -rf /run/systemd/system/irqbalance.service.d/mckernel.conf \
		/run/sysconfig/irqbalance_mck /run/sysconfig/irqbalance_mck_affinities
	sudo rmdir /run/systemd/system/irqbalance.service.d 2>/dev/null
	sudo systemctl daemon-reload
	sudo systemctl restart irqbalance.service
fi

# Set back default swappiness
echo 60 | sudo tee /proc/sys/vm/swappiness 1>/dev/null