reproductible builds: remove most install paths in c code

In order to speed up test bot work it would be helpful to check for
identical build outputs and skip tests if required.

This removes most use of the install path in c code:
 - ql_mpi uses /proc/self/exe and looks for talker/server in same
directory as itself
 - mcexec looks for libihk.so in /proc/self/maps and use that path for
LD_PRELOAD prefix path
 - rootfsdir is not used right now but until a better fix happens just
hardcode it, someone who wants to change it can set it through cmake

There is one last occurence of the install directory, MCEXEC_PATH in
mcctrl's binfmt code, for which the build system will just overwrite it
to a constant string at build time instead of trying to remove it too
hard. It would be possible to pass it as a kernel parameter or look for
mcexec in PATH but this is too much work for now.

Change-Id: I5d1352bc5748a1ea10dcae4be630f30a07609296

CMakeLists.txt

@@ -104,11 +104,10 @@ list(GET LINUX_VERSION 3 LINUX_VERSION_RELEASE)
 math(EXPR LINUX_VERSION_CODE "${LINUX_VERSION_MAJOR} * 65536 + ${LINUX_VERSION_MINOR} * 256 + ${LINUX_VERSION_PATCH}")
 
 # compat with various install paths
-set(MCKERNEL_LIBDIR ${CMAKE_INSTALL_FULL_LIBDIR})
 set(BINDIR ${CMAKE_INSTALL_FULL_BINDIR})
 set(SBINDIR ${CMAKE_INSTALL_FULL_SBINDIR})
 set(ETCDIR ${CMAKE_INSTALL_FULL_SYSCONFDIR})
-set(ROOTFSDIR "${CMAKE_INSTALL_PREFIX}/rootfs")
+set(ROOTFSDIR "/rootfs")
 if (CMAKE_INSTALL_PREFIX STREQUAL "/usr")
 	set(KMODDIR "/lib/modules/${UNAME_R}/extra/mckernel")
 	set(MCKERNELDIR "${CMAKE_INSTALL_FULL_DATADIR}/mckernel/${BUILD_TARGET}")

config.h.in

@@ -30,15 +30,6 @@
 /* Path of bind-mount source directory */
 #cmakedefine ROOTFSDIR "${ROOTFSDIR}"
 
-/* Path of install directory for libraries */
-#cmakedefine MCKERNEL_LIBDIR "${MCKERNEL_LIBDIR}"
-
-/* Path of install directory for binary */
-#cmakedefine BINDIR "${BINDIR}"
-
-/* Path of install directory for system binary */
-#cmakedefine SBINDIR "${SBINDIR}"
-
 /* for non-RHEL kernels */
 #ifndef RHEL_RELEASE_VERSION
 #define RHEL_RELEASE_VERSION(a,b) (((a) << 8) + (b))

executer/kernel/mcctrl/CMakeLists.txt

@@ -4,6 +4,8 @@ if(ARCH STREQUAL "x86_64")
 	set(ARCH_C_FLAGS "-mno-red-zone -mcmodel=kernel")
 endif()
 
+set(MCEXEC_PATH "${CMAKE_INSTALL_FULL_BINDIR}/mcexec" CACHE STRING "mcexec path for binfmt")
+
 kmod(mcctrl
 	C_FLAGS
 		-I${IHK_FULL_SOURCE_DIR}/linux/include
@@ -16,7 +18,7 @@ kmod(mcctrl
 		-I${CMAKE_CURRENT_SOURCE_DIR}/arch/${ARCH}/include
 		-I${PROJECT_BINARY_DIR}
 		-I${PROJECT_SOURCE_DIR}/kernel/include
-		-DMCEXEC_PATH=\\"${CMAKE_INSTALL_FULL_BINDIR}/mcexec\\"
+		-DMCEXEC_PATH=\\"${MCEXEC_PATH}\\"
 		${ARCH_C_FLAGS}
 	SOURCES
 		driver.c control.c ikc.c syscall.c procfs.c binfmt_mcexec.c

executer/user/mcexec.c

@@ -1975,7 +1975,7 @@ opendev()
 }
 
 #define LD_PRELOAD_PREPARE(name) do { \
-		sprintf(elembuf, "%s%s/" name, nelem > 0 ? ":" : "", MCKERNEL_LIBDIR); \
+		sprintf(elembuf, "%s%s/" name, nelem > 0 ? ":" : "", libdir); \
 	} while (0)
 
 #define LD_PRELOAD_APPEND do {	\
@@ -1988,6 +1988,57 @@ opendev()
 		nelem++; \
 	} while (0)
 
+static ssize_t find_libdir(char *libdir, size_t len)
+{
+	FILE *filep;
+	ssize_t rc;
+	size_t linelen = 0;
+	char *line = NULL;
+	char *ihklib, *slash;
+
+	filep = fopen("/proc/self/maps", "r");
+	if (!filep) {
+		rc = -errno;
+		fprintf(stderr, "could not open /proc/self/maps: %d\n", -rc);
+		return rc;
+	}
+
+	while ((rc = getline(&line, &linelen, filep)) > 0) {
+		ihklib = strstr(line, "libihk.so");
+		if (ihklib)
+			break;
+	}
+	if (!ihklib) {
+		fprintf(stderr, "mcexec does not have libihk.so loaded?\n");
+		rc = -ENOENT;
+		goto out;
+	}
+
+	slash = strchr(line, '/');
+	if (!slash) {
+		rc = -EINVAL;
+		goto out;
+	}
+
+	/* leave / iff root of filesystem */
+	if (slash + 1 != ihklib)
+		ihklib--;
+
+	*ihklib = 0;
+
+	rc = snprintf(libdir, len, "%s", slash);
+
+	if (rc > len) {
+		rc = -ERANGE;
+		goto out;
+	}
+
+out:
+	fclose(filep);
+	free(line);
+	return rc;
+}
+
 static void ld_preload_init()
 {
 	char envbuf[PATH_MAX];
@@ -1995,6 +2046,12 @@ static void ld_preload_init()
 	size_t remainder = PATH_MAX;
 	int nelem = 0;
 	char elembuf[PATH_MAX];
+	char libdir[PATH_MAX];
+
+	if (find_libdir(libdir, sizeof(libdir)) < 0) {
+		fprintf(stderr, "warning: did not set LD_PRELOAD\n");
+		return;
+	}
 
 	memset(envbuf, 0, PATH_MAX);
 

executer/user/ql_mpiexec_start.c

@@ -12,6 +12,7 @@
 #include <sys/stat.h>
 #include <sys/mman.h>
 #include <sys/wait.h>
+#include <limits.h>
 #include <fcntl.h>
 #include <errno.h>
 #include <pthread.h>
@@ -20,12 +21,11 @@
 #include "../include/qlmpi.h"
 #include "../include/md5.h"
 
-#define MCEXEC "mcexec"
 #define QL_PIPE_PATH "/tmp/"
 #define QL_PIPE_IN_EXTENTION ".in"
 #define QL_PIPE_OUT_EXTENTION ".out"
-#define QL_SERVER_EXECUTION SBINDIR "/ql_server"
-#define QL_TALKER_EXECUTION SBINDIR "/ql_talker"
+char *bindir;
+char *mcexec_path;
 
 extern char **environ;
 
@@ -279,7 +279,7 @@ term_server()
 {
 	char buf[1024];
 
-	sprintf(buf,"ssh %s %s %c %s %s %s", target_host, QL_TALKER_EXECUTION,
+	sprintf(buf,"ssh %s %s/ql_talker %c %s %s %s", target_host, bindir,
 	        QL_RET_FINAL, "-n", ql_name ,ql_sock_file);
 	system(buf);
 }
@@ -702,6 +702,28 @@ int main(int argc, char *argv[])
 	int f_flg = 0;
 #endif
 
+	rc = readlink("/proc/self/exe", bindir, PATH_MAX);
+	if (rc >= PATH_MAX) {
+		fprintf(stderr, "/proc/self/exe path too long\n");
+		exit(1);
+	}
+	bindir[rc] = 0;
+
+	ptr = strrchr(bindir, '/');
+	if (!ptr) {
+		fprintf(stderr, "invalid executable path: %s\n", bindir);
+		exit(1);
+	} else if (ptr == bindir) {
+		bindir[1] = 0;
+	} else {
+		ptr = 0;
+	}
+
+	if (snprintf(mcexec_path, PATH_MAX, "%s/mcexec", bindir) >= PATH_MAX) {
+		fprintf(stderr, "mcexec path to long\n");
+		exit(1);
+	}
+
 	for (a = environ, n = 0; *a; a++, n++);
 	for (a = argv; *a; a++) {
 		if (!strcmp(*a, "-genv") ||
@@ -898,7 +920,7 @@ int main(int argc, char *argv[])
 #endif
 
 #ifndef QL_MPIEXEC_FINALIZE
-	sprintf(tmp, "ssh %s ""%s %s %s""", target_host, QL_SERVER_EXECUTION ,ql_sock_path ,ql_file);
+	sprintf(tmp, "ssh %s ""%s/ql_server %s %s""", target_host, bindir ,ql_sock_path ,ql_file);
 
 #ifdef QL_DEBUG
 	printf(" system   %s\n", tmp);
@@ -965,7 +987,7 @@ int main(int argc, char *argv[])
 			for (a = mpi_opt_top, b = args + 1; a != usr_opt_top;
 			     a++)
 				*(b++) = *a;
-			*(b++) = BINDIR "/mcexec";
+			*(b++) = mcexec_path;
 			for (; *a; a++)
 				*(b++) = *a;
 			*b = NULL;
@@ -1029,21 +1051,21 @@ int main(int argc, char *argv[])
 		exit(1);
 
 #ifdef QL_MPIEXEC_FINALIZE
-	sprintf(tmp,"ssh %s %s %c %s %s %s",
-	        target_host, QL_TALKER_EXECUTION, QL_RET_RESUME, "-n", ql_name , ql_sock_file);
+	sprintf(tmp,"ssh %s %s/ql_talker %c %s %s %s",
+	        target_host, bindir, QL_RET_RESUME, "-n", ql_name , ql_sock_file);
 	rc = system(tmp);
 	write(wfd, "F", 1);
 #else
 	if (f_flg == 1) {
-		sprintf(tmp,"ssh %s %s %c %c %s %s",
-		        target_host, QL_TALKER_EXECUTION, QL_COM_CONN,
+		sprintf(tmp,"ssh %s %s/ql_talker %c %c %s %s",
+		        target_host, bindir, QL_COM_CONN,
 		        QL_EXEC_END, ql_name ,ql_sock_file);
 		rc = system(tmp);
 		/* send N and recv E */
 	}
 	else{
-		sprintf(tmp,"ssh %s %s %c %c %s %s",
-		        target_host, QL_TALKER_EXECUTION, QL_RET_RESUME,
+		sprintf(tmp,"ssh %s %s/ql_talker %c %c %s %s",
+		        target_host, bindir, QL_RET_RESUME,
 		        QL_EXEC_END, ql_name , ql_sock_file);
 		rc = system(tmp);
 		/* send R and recv E */