From 10c09aa10e0fce21959844f5768d054774e303d1 Mon Sep 17 00:00:00 2001 From: Balazs Gerofi Date: Sun, 6 Sep 2020 11:51:52 +0900 Subject: [PATCH] MM: generic lockless kmalloc and page cache Change-Id: I71ad498fdd10136d9c72ffe2b16b9122d1bc9673 --- kernel/include/cls.h | 9 +++- kernel/include/kmalloc.h | 94 ++++++++++++++++++++++++++++++++++++++++ lib/include/ihk/mm.h | 89 +++++++++++++++++++++++++++++++++++++ 3 files changed, 191 insertions(+), 1 deletion(-) diff --git a/kernel/include/cls.h b/kernel/include/cls.h index ec0c3ab5..6df5837a 100644 --- a/kernel/include/cls.h +++ b/kernel/include/cls.h @@ -20,10 +20,17 @@ * CPU Local Storage (cls) */ +struct kmalloc_cache_header { + struct kmalloc_cache_header *next; +}; + struct kmalloc_header { unsigned int front_magic; int cpu_id; - struct list_head list; + union { + struct list_head list; + struct kmalloc_cache_header *cache; + }; int size; /* The size of this chunk without the header */ unsigned int end_magic; /* 32 bytes */ diff --git a/kernel/include/kmalloc.h b/kernel/include/kmalloc.h index 9b89395c..a389ee7c 100644 --- a/kernel/include/kmalloc.h +++ b/kernel/include/kmalloc.h @@ -36,4 +36,98 @@ int memcheckall(void); int freecheck(int runcount); void kmalloc_consolidate_free_list(void); +#ifndef unlikely +#define unlikely(x) __builtin_expect(!!(x), 0) +#endif + +/* + * Generic lockless kmalloc cache. + */ +static inline void kmalloc_cache_free(void *elem) +{ + struct kmalloc_cache_header *current = NULL; + struct kmalloc_cache_header *new = + (struct kmalloc_cache_header *)elem; + struct kmalloc_header *header; + register struct kmalloc_cache_header *cache; + + if (unlikely(!elem)) + return; + + /* Get cache pointer from kmalloc header */ + header = (struct kmalloc_header *)((void *)elem - + sizeof(struct kmalloc_header)); + if (unlikely(!header->cache)) { + kprintf("%s: WARNING: no cache for 0x%lx\n", + __func__, elem); + return; + } + + cache = header->cache; + +retry: + current = cache->next; + new->next = current; + + if (!__sync_bool_compare_and_swap(&cache->next, current, new)) { + goto retry; + } +} + +static inline void kmalloc_cache_prealloc(struct kmalloc_cache_header *cache, + size_t size, int nr_elem) +{ + struct kmalloc_cache_header *elem; + int i; + + if (unlikely(cache->next)) + return; + + for (i = 0; i < nr_elem; ++i) { + struct kmalloc_header *header; + + elem = (struct kmalloc_cache_header *) + kmalloc(size, IHK_MC_AP_NOWAIT); + + if (!elem) { + kprintf("%s: ERROR: allocating cache element\n", __func__); + continue; + } + + /* Store cache pointer in kmalloc_header */ + header = (struct kmalloc_header *)((void *)elem - + sizeof(struct kmalloc_header)); + header->cache = cache; + + kmalloc_cache_free(elem); + } +} + +static inline void *kmalloc_cache_alloc(struct kmalloc_cache_header *cache, + size_t size) +{ + register struct kmalloc_cache_header *first, *next; + +retry: + next = NULL; + first = cache->next; + + if (first) { + next = first->next; + + if (!__sync_bool_compare_and_swap(&cache->next, + first, next)) { + goto retry; + } + } + else { + kprintf("%s: calling pre-alloc for 0x%lx...\n", __func__, cache); + + kmalloc_cache_prealloc(cache, size, 384); + goto retry; + } + + return (void *)first; +} + #endif diff --git a/lib/include/ihk/mm.h b/lib/include/ihk/mm.h index aadb4997..47e79db0 100644 --- a/lib/include/ihk/mm.h +++ b/lib/include/ihk/mm.h @@ -20,6 +20,7 @@ #include #include #include +#include struct memobj; struct process_vm; @@ -269,4 +270,92 @@ void ihk_mc_query_mem_free_page(void *dump_page_set); int ihk_mc_chk_page_address(pte_t mem_addr); int ihk_mc_get_mem_user_page(void *arg0, page_table_t pt, pte_t *ptep, void *pgaddr, int pgshift); +#ifndef unlikely +#define unlikely(x) __builtin_expect(!!(x), 0) +#endif + +extern int zero_at_free; + +/* + * Generic lockless page cache. + * TODO: Store nr of pages in header and double-check at alloc time.. + */ +struct ihk_mc_page_cache_header; + +struct ihk_mc_page_cache_header { + struct ihk_mc_page_cache_header *next; +}; + + +static inline void ihk_mc_page_cache_free( + struct ihk_mc_page_cache_header *cache, void *page) +{ + struct ihk_mc_page_cache_header *current = NULL; + struct ihk_mc_page_cache_header *new = + (struct ihk_mc_page_cache_header *)page; + + if (unlikely(!page)) + return; + +retry: + current = cache->next; + new->next = current; + + if (!__sync_bool_compare_and_swap(&cache->next, current, new)) { + goto retry; + } +} + +static inline void ihk_mc_page_cache_prealloc( + struct ihk_mc_page_cache_header *cache, + int nr_pages, + int nr_elem) +{ + int i; + + if (unlikely(cache->next)) + return; + + for (i = 0; i < nr_elem; ++i) { + void *pages; + + pages = ihk_mc_alloc_pages(nr_pages, IHK_MC_AP_NOWAIT); + + if (!pages) { + kprintf("%s: ERROR: allocating pages..\n", __func__); + continue; + } + + ihk_mc_page_cache_free(cache, pages); + } +} + +static inline void *ihk_mc_page_cache_alloc( + struct ihk_mc_page_cache_header *cache, + int nr_pages) +{ + register struct ihk_mc_page_cache_header *first, *next; + +retry: + next = NULL; + first = cache->next; + + if (first) { + next = first->next; + + if (!__sync_bool_compare_and_swap(&cache->next, + first, next)) { + goto retry; + } + } + else { + kprintf("%s: calling pre-alloc for 0x%lx...\n", __func__, cache); + + ihk_mc_page_cache_prealloc(cache, nr_pages, 256); + goto retry; + } + + return (void *)first; +} + #endif