kmalloc cache: embed cache pointer into kmalloc_header

Conflicts:
	kernel/mem.c

kernel/include/cls.h

@@ -19,10 +19,17 @@
  * CPU Local Storage (cls)
  */
 
+struct kmalloc_cache_header {
+	struct kmalloc_cache_header *next;
+};
+
 struct kmalloc_header {
 	unsigned int front_magic;
 	int cpu_id;
-	struct list_head list;
+	union {
+		struct list_head list;
+		struct kmalloc_cache_header *cache;
+	};
 	int size; /* The size of this chunk without the header */
 	unsigned int end_magic;
 	/* 32 bytes */

kernel/include/kmalloc.h

@@ -19,15 +19,6 @@
 void panic(const char *);
 int kprintf(const char *format, ...);
 
-struct kmalloc_cache_header {
-	struct kmalloc_cache_header *next;
-};
-
-void *kmalloc_cache_alloc(struct kmalloc_cache_header *cache,
-		size_t size);
-void kmalloc_cache_free(struct kmalloc_cache_header *cache,
-		void *elem);
-
 #define kmalloc(size, flag) ({\
 void *r = _kmalloc(size, flag, __FILE__, __LINE__);\
 if(r == NULL){\
@@ -47,4 +38,99 @@ int memcheckall();
 int freecheck(int runcount);
 void kmalloc_consolidate_free_list(void);
 
+#ifndef unlikely
+#define unlikely(x)	__builtin_expect(!!(x), 0)
+#endif
+
+/*
+ * Generic lockless kmalloc cache.
+ */
+static inline void kmalloc_cache_free(void *elem)
+{
+	struct kmalloc_cache_header *current = NULL;
+	struct kmalloc_cache_header *new =
+		(struct kmalloc_cache_header *)elem;
+	struct kmalloc_header *header;
+	register struct kmalloc_cache_header *cache;
+
+	if (unlikely(!elem))
+		return;
+
+	/* Get cache pointer from kmalloc header */
+	header = (struct kmalloc_header *)((void *)elem -
+				sizeof(struct kmalloc_header));
+	if (unlikely(!header->cache)) {
+		kprintf("%s: WARNING: no cache for 0x%lx\n",
+			__FUNCTION__, elem);
+		return;
+	}
+
+	cache = header->cache;
+
+retry:
+	current = cache->next;
+	new->next = current;
+
+	if (!__sync_bool_compare_and_swap(&cache->next, current, new)) {
+		goto retry;
+	}
+}
+
+static inline void kmalloc_cache_prealloc(struct kmalloc_cache_header *cache,
+		size_t size)
+{
+	struct kmalloc_cache_header *elem;
+	int i;
+
+	if (unlikely(cache->next))
+		return;
+
+	kprintf("%s: pre-allocating for 0x%lx...\n",
+			__FUNCTION__, cache);
+
+	for (i = 0; i < 256; ++i) {
+		struct kmalloc_header *header;
+
+		elem = (struct kmalloc_cache_header *)
+			kmalloc(size, IHK_MC_AP_NOWAIT);
+
+		if (!elem) {
+			kprintf("%s: ERROR: allocating cache element\n", __FUNCTION__);
+			continue;
+		}
+
+		/* Store cache pointer in kmalloc_header */
+		header = (struct kmalloc_header *)((void *)elem -
+				sizeof(struct kmalloc_header));
+		header->cache = cache;
+
+		kmalloc_cache_free(elem);
+	}
+}
+
+static inline void *kmalloc_cache_alloc(struct kmalloc_cache_header *cache,
+		size_t size)
+{
+	register struct kmalloc_cache_header *first, *next;
+
+retry:
+	next = NULL;
+	first = cache->next;
+
+	if (first) {
+		next = first->next;
+
+		if (!__sync_bool_compare_and_swap(&cache->next,
+					first, next)) {
+			goto retry;
+		}
+	}
+	else {
+		kmalloc_cache_prealloc(cache, size);
+		goto retry;
+	}
+
+	return (void *)first;
+}
+
 #endif

kernel/mem.c

@@ -2224,6 +2224,7 @@ split_and_return:
 		}
 
 		list_del(&chunk->list);
+		ZERO_LIST_HEAD(&chunk->list);
 		cpu_restore_interrupt(kmalloc_irq_flags);
 		return ((void *)chunk + sizeof(struct kmalloc_header));
 	}

lib/include/list.h

@@ -41,6 +41,12 @@ static inline void INIT_LIST_HEAD(struct list_head *list)
 	list->prev = list;
 }
 
+static inline void ZERO_LIST_HEAD(struct list_head *list)
+{
+	list->next = 0;
+	list->prev = 0;
+}
+
 /*
  * Insert a new entry between two known consecutive entries.
  *