From 3e3ccf377c828b0139bd015e60e50d6418f12d1f Mon Sep 17 00:00:00 2001 From: Dominique Martinet Date: Fri, 21 Sep 2018 12:30:52 +0900 Subject: [PATCH] compiler.h: add READ_ONCE/WRITE_ONCE macro These macros are needed to make sure the compiler does not optimize away atomic constructs such as "while (!READ_ONCE(foo))" loops that do not modify foo within the loop Also move the barrier() define where it belongs while we are here, it is needed for READ_ONCE/WRITE_ONCE and including ihk/cpu.h here causes include loops Change-Id: Ia533a849ed674719ccbc0495be47d22a3c47b8f8 --- arch/arm64/kernel/include/arch-lock.h | 1 + arch/x86_64/kernel/include/arch-lock.h | 1 + ihk | 2 +- kernel/include/lwk/compiler-gcc.h | 7 +-- kernel/include/lwk/compiler.h | 65 ++++++++++++++++++++++++-- lib/include/ihk/cpu.h | 2 - 6 files changed, 65 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kernel/include/arch-lock.h b/arch/arm64/kernel/include/arch-lock.h index e53f2532..cd1a7691 100644 --- a/arch/arm64/kernel/include/arch-lock.h +++ b/arch/arm64/kernel/include/arch-lock.h @@ -7,6 +7,7 @@ #include #include #include "affinity.h" +#include //#define DEBUG_SPINLOCK //#define DEBUG_MCS_RWLOCK diff --git a/arch/x86_64/kernel/include/arch-lock.h b/arch/x86_64/kernel/include/arch-lock.h index a9366bfa..da4bdbfe 100644 --- a/arch/x86_64/kernel/include/arch-lock.h +++ b/arch/x86_64/kernel/include/arch-lock.h @@ -6,6 +6,7 @@ #include #include +#include //#define DEBUG_SPINLOCK //#define DEBUG_MCS_RWLOCK diff --git a/ihk b/ihk index 00634a82..d9c74adf 160000 --- a/ihk +++ b/ihk @@ -1 +1 @@ -Subproject commit 00634a823ff0d3debfa381da5f1ac2f994fae388 +Subproject commit d9c74adf3f3037b5e1c0d9f40dd2e18e4fa70165 diff --git a/kernel/include/lwk/compiler-gcc.h b/kernel/include/lwk/compiler-gcc.h index 7dddaa75..c0695a9f 100644 --- a/kernel/include/lwk/compiler-gcc.h +++ b/kernel/include/lwk/compiler-gcc.h @@ -12,11 +12,8 @@ /* Optimization barrier */ /* The "volatile" is due to gcc bugs */ -/* XXX: barrier is also defined in lib/include/ihk/cpu.h, - * it would be cleaner to restore this here at some point, but we have - * quite a few C files not including either this or kernel's compiler.h - * #define barrier() __asm__ __volatile__("": : :"memory") - */ +#define barrier() __asm__ __volatile__("": : :"memory") + /* * This version is i.e. to prevent dead stores elimination on @ptr * where gcc and llvm may behave differently when otherwise using diff --git a/kernel/include/lwk/compiler.h b/kernel/include/lwk/compiler.h index df62035b..ee987f2c 100644 --- a/kernel/include/lwk/compiler.h +++ b/kernel/include/lwk/compiler.h @@ -3,6 +3,8 @@ #ifndef __ASSEMBLY__ +#include + #ifdef __CHECKER__ # define __user __attribute__((noderef, address_space(1))) # define __kernel __attribute__((address_space(0))) @@ -175,11 +177,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, # define unlikely(x) __builtin_expect(!!(x), 0) #endif -/* Optimization barrier */ -#ifndef barrier -# define barrier() __memory_barrier() -#endif - #ifndef barrier_data # define barrier_data(ptr) barrier() #endif @@ -490,4 +487,62 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, (_________p1); \ }) +extern void *memcpy(void *dest, const void *src, size_t n); + +static __always_inline void __read_once_size(const volatile void *p, void *res, int size) +{ + switch (size) { + case 1: *(unsigned char *)res = *(volatile unsigned char *)p; break; + case 2: *(unsigned short *)res = *(volatile unsigned short *)p; break; + case 4: *(unsigned int *)res = *(volatile unsigned int *)p; break; + case 8: *(unsigned long long *)res = *(volatile unsigned long long *)p; break; + default: + barrier(); + memcpy((void *)res, (const void *)p, size); + barrier(); + } +} + +static __always_inline void __write_once_size(volatile void *p, void *res, int size) +{ + switch (size) { + case 1: *(volatile unsigned char *)p = *(unsigned char *)res; break; + case 2: *(volatile unsigned short *)p = *(unsigned short *)res; break; + case 4: *(volatile unsigned int *)p = *(unsigned int *)res; break; + case 8: *(volatile unsigned long long *)p = *(unsigned long long *)res; break; + default: + barrier(); + memcpy((void *)p, (const void *)res, size); + barrier(); + } +} + +/* + * Prevent the compiler from merging or refetching reads or writes. The + * compiler is also forbidden from reordering successive instances of + * READ_ONCE, WRITE_ONCE and ACCESS_ONCE (see below), but only when the + * compiler is aware of some particular ordering. One way to make the + * compiler aware of ordering is to put the two invocations of READ_ONCE, + * WRITE_ONCE or ACCESS_ONCE() in different C statements. + * + * In contrast to ACCESS_ONCE these two macros will also work on aggregate + * data types like structs or unions. If the size of the accessed data + * type exceeds the word size of the machine (e.g., 32 bits or 64 bits) + * READ_ONCE() and WRITE_ONCE() will fall back to memcpy and print a + * compile-time warning. + * + * Their two major use cases are: (1) Mediating communication between + * process-level code and irq/NMI handlers, all running on the same CPU, + * and (2) Ensuring that the compiler does not fold, spindle, or otherwise + * mutilate accesses that either do not require ordering or that interact + * with an explicit memory barrier or atomic instruction that provides the + * required ordering. + */ + +#define READ_ONCE(x) \ + ({ union { typeof(x) __val; char __c[1]; } __u; __read_once_size(&(x), __u.__c, sizeof(x)); __u.__val; }) + +#define WRITE_ONCE(x, val) \ + ({ typeof(x) __val = (val); __write_once_size(&(x), &__val, sizeof(__val)); __val; }) + #endif /* __LWK_COMPILER_H */ diff --git a/lib/include/ihk/cpu.h b/lib/include/ihk/cpu.h index b6b5a922..c76e866f 100644 --- a/lib/include/ihk/cpu.h +++ b/lib/include/ihk/cpu.h @@ -25,8 +25,6 @@ void cpu_safe_halt(void); void cpu_restore_interrupt(unsigned long); void cpu_pause(void); -#define barrier() arch_barrier() - unsigned long cpu_disable_interrupt_save(void); struct ihk_mc_interrupt_handler {