#!/bin/bash # IHK SMP-x86 example boot script. # author: Balazs Gerofi # Copyright (C) 2014 RIKEN AICS # # This is an example script for loading IHK, configuring a partition and # booting McKernel on it. # The script reserves half of the CPU cores and 512MB of RAM from NUMA node 0 # when IHK is loaded for the first time, otherwise it destroys the current # McKernel instance and reboots it using the same set of resources as it used # previously. # Note that the script does not output anything unless an error occurs. prefix="@prefix@" BINDIR="${prefix}/bin" SBINDIR="${prefix}/sbin" KMODDIR="${prefix}/kmod" KERNDIR="${prefix}/@TARGET@/kernel" ENABLE_MCOVERLAYFS="@ENABLE_MCOVERLAYFS@" mem="512M@0" cpus="" INTERVAL=1 LOGMODE=0 while getopts :i:k:c:m: OPT do case ${OPT} in i) INTERVAL=${OPTARG} expr "${INTERVAL}" + 1 > /dev/null 2>&1 if [ $? -ge 2 ] then echo "invalid -i value" exit 1 fi if [ ${INTERVAL} -le 0 ] then echo "invalid -i value" exit 1 fi ;; k) LOGMODE=${OPTARG} expr "${LOGMODE}" + 1 > /dev/null 2>&1 if [ $? -ge 2 ] then echo "invalid -k value" exit 1 fi if [ ${LOGMODE} -lt 0 -o ${LOGMODE} -gt 2 ] then echo "invalid -k value" exit 1 fi ;; c) cpus=${OPTARG} ;; m) mem=${OPTARG} ;; *) echo "invalid option -${OPT}" exit 1 esac done ihk_ikc_irq_core=0 release=`uname -r` major=`echo ${release} | sed -e 's/^\([0-9]*\).*/\1/'` minor=`echo ${release} | sed -e 's/^[0-9]*.\([0-9]*\).*/\1/'` patch=`echo ${release} | sed -e 's/^[0-9]*.[0-9]*.\([0-9]*\).*/\1/'` linux_version_code=`expr \( ${major} \* 65536 \) + \( ${minor} \* 256 \) + ${patch}` rhel_release=`echo ${release} | sed -e 's/^[0-9]*.[0-9]*.[0-9]*-\([0-9]*\).*/\1/'` if [ "${release}" == "${rhel_release}" ]; then rhel_release=""; fi if [ "${ENABLE_MCOVERLAYFS}" == "yes" ]; then enable_mcoverlay=`if ( [ ${linux_version_code} -ge 262144 ] && [ ${linux_version_code} -lt 262400 ] ); then echo "yes"; else echo "no"; fi` else enable_mcoverlay=no fi if [ "$cpus" == "" ]; then # Get the number of CPUs on NUMA node 0 nr_cpus=`lscpu --parse | awk -F"," '{if ($4 == 0) print $4}' | wc -l` # Use the second half of the cores let nr_cpus="$nr_cpus / 2" cpus=`lscpu --parse | awk -F"," '{if ($4 == 0) print $1}' | tail -n $nr_cpus | xargs echo -n | sed 's/ /,/g'` if [ "$cpus" == "" ]; then echo "error: no available CPUs on NUMA node 0?"; exit; fi fi # Remove mcoverlay if loaded if [ "$enable_mcoverlay" == "yes" ]; then if [ "`lsmod | grep mcoverlay`" != "" ]; then if [ "`cat /proc/mounts | grep /tmp/mcos/mcos0_sys`" != "" ]; then umount -l /tmp/mcos/mcos0_sys; fi if [ "`cat /proc/mounts | grep /tmp/mcos/mcos0_proc`" != "" ]; then umount -l /tmp/mcos/mcos0_proc; fi if [ "`cat /proc/mounts | grep /tmp/mcos/linux_proc`" != "" ]; then umount -l /tmp/mcos/linux_proc; fi if [ "`cat /proc/mounts | grep /tmp/mcos`" != "" ]; then umount -l /tmp/mcos; fi if [ -e /tmp/mcos ]; then rm -rf /tmp/mcos; fi if ! rmmod mcoverlay; then echo "error: removing mcoverlay"; exit; fi fi fi # Load IHK if not loaded if [ "`lsmod | grep ihk`" == "" ]; then if ! insmod ${KMODDIR}/ihk.ko; then echo "error: loading ihk"; exit; fi; fi # Load IHK-SMP if not loaded and reserve CPUs and memory if [ "`lsmod | grep ihk_smp_x86`" == "" ]; then ihk_irq="" for i in `seq 64 255`; do if [ ! -d /proc/irq/$i ] && [ "`cat /proc/interrupts | grep ":" | awk '{print $1}' | grep -o '[0-9]*' | grep -e '^$i$'`" == "" ]; then ihk_irq=$i break fi done if [ "$ihk_irq" == "" ]; then echo "error: no IRQ available"; exit; fi if ! insmod ${KMODDIR}/ihk-smp-x86.ko ihk_start_irq=$ihk_irq ihk_ikc_irq_core=$ihk_ikc_irq_core; then echo "error: loading ihk-smp-x86"; exit; fi; if ! ${SBINDIR}/ihkconfig 0 reserve cpu ${cpus}; then echo "error: reserving CPUs"; exit; fi if ! ${SBINDIR}/ihkconfig 0 reserve mem ${mem}; then echo "error: reserving memory"; exit; fi # If loaded, but no resources allocated, get CPUs and memory else if ! ${SBINDIR}/ihkconfig 0 query cpu > /dev/null; then echo "error: querying cpus"; exit; fi cpus_allocated=`${SBINDIR}/ihkosctl 0 query cpu` if [ "$cpus_allocated" == "" ]; then if ! ${SBINDIR}/ihkconfig 0 reserve cpu ${cpus}; then echo "error: reserving CPUs"; exit; fi fi if ! ${SBINDIR}/ihkosctl 0 query mem > /dev/null; then echo "error: querying memory"; exit; fi mem_allocated=`${SBINDIR}/ihkosctl 0 query mem` if [ "$mem_allocated" == "" ]; then if ! ${SBINDIR}/ihkconfig 0 reserve mem ${mem}; then echo "error: reserving memory"; exit; fi fi fi # Load mcctrl if not loaded if [ "`lsmod | grep mcctrl`" == "" ]; then if ! insmod ${KMODDIR}/mcctrl.ko; then echo "error: inserting mcctrl.ko"; exit; fi fi # Check for existing OS instance and destroy if [ -c /dev/mcos0 ]; then # Query CPU cores and memory of OS instance so that the same values are used as previously if ! ${SBINDIR}/ihkosctl 0 query cpu > /dev/null; then echo "error: querying cpus"; exit; fi cpus=`${SBINDIR}/ihkosctl 0 query cpu` if ! ${SBINDIR}/ihkosctl 0 query mem > /dev/null; then echo "error: querying memory"; exit; fi mem=`${SBINDIR}/ihkosctl 0 query mem` if ! ${SBINDIR}/ihkconfig 0 destroy 0; then echo "warning: destroy failed"; fi else # Otherwise query IHK-SMP for resources if ! ${SBINDIR}/ihkconfig 0 query cpu > /dev/null; then echo "error: querying cpus"; exit; fi cpus=`${SBINDIR}/ihkconfig 0 query cpu` if ! ${SBINDIR}/ihkconfig 0 query mem > /dev/null; then echo "error: querying memory"; exit; fi mem=`${SBINDIR}/ihkconfig 0 query mem` fi if ! ${SBINDIR}/ihkconfig 0 create; then echo "error: create"; exit; fi if ! ${SBINDIR}/ihkosctl 0 assign cpu ${cpus}; then echo "error: assign CPUs"; exit; fi if ! ${SBINDIR}/ihkosctl 0 assign mem ${mem}; then echo "error: assign memory"; exit; fi if ! ${SBINDIR}/ihkosctl 0 load ${KERNDIR}/mckernel.img; then echo "error: loading kernel image"; exit; fi if ! ${SBINDIR}/ihkosctl 0 kargs "hidos ksyslogd=${LOGMODE}"; then echo "error: setting kernel arguments"; exit; fi if ! ${SBINDIR}/ihkosctl 0 boot; then echo "error: booting"; exit; fi if ! chown `logname` /dev/mcd* /dev/mcos*; then echo "error: chowning device files"; exit; fi if [ "$enable_mcoverlay" == "yes" ]; then if [ ! -e /tmp/mcos ]; then mkdir -p /tmp/mcos; fi if ! mount -t tmpfs tmpfs /tmp/mcos; then echo "error: mount /tmp/mcos"; exit; fi if [ ! -e /tmp/mcos/linux_proc ]; then mkdir -p /tmp/mcos/linux_proc; fi if ! mount --bind /proc /tmp/mcos/linux_proc; then echo "error: mount /tmp/mcos/linux_proc"; exit; fi if ! insmod ${KMODDIR}/mcoverlay.ko; then echo "error: inserting mcoverlay.ko"; exit; fi while [ ! -e /proc/mcos0 ] do sleep 1 done if [ ! -e /tmp/mcos/mcos0_proc ]; then mkdir -p /tmp/mcos/mcos0_proc; fi if [ ! -e /tmp/mcos/mcos0_proc_upper ]; then mkdir -p /tmp/mcos/mcos0_proc_upper; fi if [ ! -e /tmp/mcos/mcos0_proc_work ]; then mkdir -p /tmp/mcos/mcos0_proc_work; fi if ! mount -t mcoverlay mcoverlay -o lowerdir=/proc/mcos0:/proc,upperdir=/tmp/mcos/mcos0_proc_upper,workdir=/tmp/mcos/mcos0_proc_work,nocopyupw,nofscheck /tmp/mcos/mcos0_proc; then echo "error: mount /tmp/mcos/mcos0_proc"; exit; fi mount --make-rprivate /proc while [ ! -e /sys/devices/virtual/mcos/mcos0/sys ] do sleep 1 done if [ ! -e /tmp/mcos/mcos0_sys ]; then mkdir -p /tmp/mcos/mcos0_sys; fi if [ ! -e /tmp/mcos/mcos0_sys_upper ]; then mkdir -p /tmp/mcos/mcos0_sys_upper; fi if [ ! -e /tmp/mcos/mcos0_sys_work ]; then mkdir -p /tmp/mcos/mcos0_sys_work; fi if ! mount -t mcoverlay mcoverlay -o lowerdir=/sys/devices/virtual/mcos/mcos0/sys:/sys,upperdir=/tmp/mcos/mcos0_sys_upper,workdir=/tmp/mcos/mcos0_sys_work,nocopyupw,nofscheck /tmp/mcos/mcos0_sys; then echo "error: mount /tmp/mcos/mcos0_sys"; exit; fi mount --make-rprivate /sys for cpuid in `find /sys/devices/system/cpu/* -maxdepth 0 -name "cpu[0123456789]*" -printf "%f "`; do if [ ! -e "/sys/devices/virtual/mcos/mcos0/sys/devices/system/cpu/$cpuid" ]; then rm -rf /tmp/mcos/mcos0_sys/devices/system/cpu/$cpuid fi done for cpuid in `find /sys/bus/cpu/devices/* -maxdepth 0 -name "cpu[0123456789]*" -printf "%f "`; do if [ ! -e "/sys/devices/virtual/mcos/mcos0/sys/bus/cpu/devices/$cpuid" ]; then rm -rf /tmp/mcos/mcos0_sys/bus/cpu/devices/$cpuid fi done fi if [ ${LOGMODE} -ne 0 ] then SBINDIR=${SBINDIR} ${SBINDIR}/mcklogd -i ${INTERVAL} fi