BSS/data: demand paging for non-file section and respect user requested NUMA allocation policy

kernel/host.c

@@ -91,11 +91,15 @@ int prepare_process_ranges_args_envs(struct thread *thread,
 	struct address_space *as = vm->address_space;
 	long aout_base;
 	int error;
+	struct vm_range *range;
+	unsigned long ap_flags;
+	enum ihk_mc_pt_attribute ptattr;
 	
 	n = p->num_sections;
 
 	aout_base = (pn->reloc)? vm->region.map_end: 0;
 	for (i = 0; i < n; i++) {
+		ap_flags = 0;
 		if (pn->sections[i].interp && (interp_nbase == (uintptr_t)-1)) {
 			interp_obase = pn->sections[i].vaddr;
 			interp_obase -= (interp_obase % pn->interp_align);
@@ -116,48 +120,49 @@ int prepare_process_ranges_args_envs(struct thread *thread,
 		s = (pn->sections[i].vaddr) & PAGE_MASK;
 		e = (pn->sections[i].vaddr + pn->sections[i].len
 				+ PAGE_SIZE - 1) & PAGE_MASK;
-		range_npages = (e - s) >> PAGE_SHIFT;
+		range_npages = ((pn->sections[i].vaddr - s) +
+			pn->sections[i].filesz + PAGE_SIZE - 1) >> PAGE_SHIFT;
 		flags = VR_NONE;
 		flags |= PROT_TO_VR_FLAG(pn->sections[i].prot);
 		flags |= VRFLAG_PROT_TO_MAXPROT(flags);
+		flags |= VR_DEMAND_PAGING;
 
-		if ((up_v = ihk_mc_alloc_pages(range_npages, IHK_MC_AP_NOWAIT)) 
-				== NULL) {
-			kprintf("ERROR: alloc pages for ELF section %i\n", i);
-			goto err;
-		} 
-		
-		up = virt_to_phys(up_v);
-		if (add_process_memory_range(vm, s, e, up, flags, NULL, 0,
-					PAGE_SHIFT, NULL) != 0) {
-			ihk_mc_free_pages(up_v, range_npages);
+		/* Non-TEXT sections that are large respect user allocation policy */
+		if (i >= 1 && pn->sections[i].len >= AP_USER_THRESHOLD) {
+			dkprintf("%s: section: %d size: %d pages -> IHK_MC_AP_USER\n",
+					__FUNCTION__, i, range_npages);
+			ap_flags = IHK_MC_AP_USER;
+			flags |= VR_AP_USER;
+		}
+
+		if (add_process_memory_range(vm, s, e, NOPHYS, flags, NULL, 0,
+					pn->sections[i].len > LARGE_PAGE_SIZE ?
+					LARGE_PAGE_SHIFT : PAGE_SHIFT,
+					&range) != 0) {
 			kprintf("ERROR: adding memory range for ELF section %i\n", i);
 			goto err;
 		}
 
-		{
-			void *_virt = (void *)s;
-			unsigned long _phys;
-			if (ihk_mc_pt_virt_to_phys(as->page_table, 
-						_virt, &_phys)) {
-				kprintf("ERROR: no mapping for 0x%lX\n", _virt);
-			}
-			for (_virt = (void *)s + PAGE_SIZE; 
-					(unsigned long)_virt < e; _virt += PAGE_SIZE) {
-				unsigned long __phys;
-				if (ihk_mc_pt_virt_to_phys(as->page_table, 
-							_virt, &__phys)) {
-					kprintf("ERROR: no mapping for 0x%lX\n", _virt);
-					panic("mapping");
-				}
-				if (__phys != _phys + PAGE_SIZE) {
-					kprintf("0x%lX + PAGE_SIZE is not physically contigous, from 0x%lX to 0x%lX\n", _virt - PAGE_SIZE, _phys, __phys);
-					panic("mondai");
-				}
+		if ((up_v = ihk_mc_alloc_pages(range_npages,
+						IHK_MC_AP_NOWAIT | ap_flags)) == NULL) {
+			kprintf("ERROR: alloc pages for ELF section %i\n", i);
+			goto err;
+		}
 
-				_phys = __phys;
-			}
-			dkprintf("0x%lX -> 0x%lX is physically contigous\n", s, e);
+		up = virt_to_phys(up_v);
+
+		ptattr = arch_vrflag_to_ptattr(range->flag, PF_POPULATE, NULL);
+		error = ihk_mc_pt_set_range(vm->address_space->page_table, vm,
+				(void *)range->start,
+				(void *)range->start + (range_npages * PAGE_SIZE),
+				up, ptattr,
+				range->pgshift);
+
+		if (error) {
+			kprintf("%s: ihk_mc_pt_set_range failed. %d\n",
+					__FUNCTION__, error);
+			ihk_mc_free_pages(up_v, range_npages);
+			goto err;
 		}
 
 		p->sections[i].remote_pa = up;

kernel/process.c

@@ -1910,7 +1910,7 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
 	unsigned long minsz;
 	unsigned long at_rand;
 	struct process *proc = thread->proc;
-	unsigned long __flag;
+	unsigned long ap_flag;
 
 	/* Create stack range */
 	end = STACK_TOP(&thread->vm->region) & LARGE_PAGE_MASK;
@@ -1925,13 +1925,14 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
 	}
 	start = (end - size) & LARGE_PAGE_MASK;
 
-	/* Put large stacks in user requested region */
-	__flag = (size >= (2 * 1024 * 1024)) ? IHK_MC_AP_USER : 0;
+	/* Apply user allocation policy to stacks */
+	/* TODO: make threshold kernel or mcexec argument */
+	ap_flag = (size >= AP_USER_THRESHOLD) ? IHK_MC_AP_USER : 0;
 	dkprintf("%s: size: %lu %s\n", __FUNCTION__, size,
-			__flag ? "(IHK_MC_AP_USER)" : "");
+			ap_flag ? "(IHK_MC_AP_USER)" : "");
 
 	stack = ihk_mc_alloc_aligned_pages(minsz >> PAGE_SHIFT,
-				LARGE_PAGE_P2ALIGN, IHK_MC_AP_NOWAIT | __flag);
+				LARGE_PAGE_P2ALIGN, IHK_MC_AP_NOWAIT | ap_flag);
 
 	if (!stack) {
 		kprintf("%s: error: couldn't allocate initial stack\n",
@@ -1942,7 +1943,7 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
 	memset(stack, 0, minsz);
 
 	vrflag = VR_STACK | VR_DEMAND_PAGING;
-	vrflag |= (__flag ? VR_AP_USER : 0);
+	vrflag |= (ap_flag ? VR_AP_USER : 0);
 	vrflag |= PROT_TO_VR_FLAG(pn->stack_prot);
 	vrflag |= VR_MAXPROT_READ | VR_MAXPROT_WRITE | VR_MAXPROT_EXEC;
 #define	NOPHYS	((uintptr_t)-1)

kernel/syscall.c

@@ -1429,14 +1429,16 @@ do_mmap(const intptr_t addr0, const size_t len0, const int prot,
 		npages = len >> PAGE_SHIFT;
 		/* Small allocations mostly benefit from closest RAM,
 		 * otherwise follow user requested policy */
-		unsigned long __flag = (len >= 2097152) ? IHK_MC_AP_USER : 0;
+		unsigned long ap_flag =
+			(len >= AP_USER_THRESHOLD || flags & MAP_STACK) ?
+			IHK_MC_AP_USER : 0;
 
-		if (__flag) {
+		if (ap_flag) {
 			vrflags |= VR_AP_USER;
 		}
 
 		p = ihk_mc_alloc_aligned_pages(npages, p2align,
-				IHK_MC_AP_NOWAIT | __flag);
+				IHK_MC_AP_NOWAIT | ap_flag);
 		if (p == NULL) {
 			dkprintf("%s: warning: failed to allocate %d contiguous pages "
 					" (bytes: %lu, pgshift: %d), enabling demand paging\n",

lib/include/ihk/mm.h

@@ -54,6 +54,8 @@ typedef unsigned long ihk_mc_ap_flag;
 #define IHK_MC_AP_BANDWIDTH       0x010000
 #define IHK_MC_AP_LATENCY         0x020000
 
+#define AP_USER_THRESHOLD         (2097152)
+
 enum ihk_mc_pt_prepare_flag {
 	IHK_MC_PT_FIRST_LEVEL,
 	IHK_MC_PT_LAST_LEVEL,