brk(): more forceful heap extension

kernel/host.c

@@ -209,7 +209,9 @@ int prepare_process_ranges_args_envs(struct thread *thread,
 		pn->at_entry += aout_base;
 	}
 
-	vm->region.brk_start = vm->region.brk_end =
+	vm->region.brk_start =
+	vm->region.brk_end =
+	vm->region.brk_end_allocated =
 		(vm->region.data_end + LARGE_PAGE_SIZE - 1) & LARGE_PAGE_MASK;
 
 	/* Map, copy and update args and envs */

kernel/include/process.h

@@ -384,7 +384,7 @@ struct vm_regions {
 	unsigned long vm_start, vm_end;
 	unsigned long text_start, text_end;
 	unsigned long data_start, data_end;
-	unsigned long brk_start, brk_end;
+	unsigned long brk_start, brk_end, brk_end_allocated;
 	unsigned long map_start, map_end;
 	unsigned long stack_start, stack_end;
 	unsigned long user_start, user_end;
@@ -767,8 +767,8 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
                         int argc, char **argv, 
                         int envc, char **env);
 unsigned long extend_process_region(struct process_vm *vm,
-                                    unsigned long start, unsigned long end,
-                                    unsigned long address, unsigned long flag);
+		unsigned long end_allocated,
+		unsigned long address, unsigned long flag);
 extern enum ihk_mc_pt_attribute arch_vrflag_to_ptattr(unsigned long flag, uint64_t fault, pte_t *ptep);
 enum ihk_mc_pt_attribute common_vrflag_to_ptattr(unsigned long flag, uint64_t fault, pte_t *ptep);
 

kernel/process.c

@@ -2057,51 +2057,43 @@ int init_process_stack(struct thread *thread, struct program_load_desc *pn,
 
 
 unsigned long extend_process_region(struct process_vm *vm,
-                                    unsigned long start, unsigned long end,
-                                    unsigned long address, unsigned long flag)
+		unsigned long end_allocated,
+		unsigned long address, unsigned long flag)
 {
-	unsigned long aligned_end, aligned_new_end;
+	unsigned long new_end_allocated;
 	void *p;
 	int rc;
 
-	if (!address || address < start || address >= USER_END) {
-		return end;
-	}
-
-	aligned_end = ((end + LARGE_PAGE_SIZE - 1) & LARGE_PAGE_MASK);
-
-	if (aligned_end >= address) {
-		return address;
-	}
-
-	aligned_new_end = (address + LARGE_PAGE_SIZE - 1) & LARGE_PAGE_MASK;
+	new_end_allocated = (address + (8 * LARGE_PAGE_SIZE) - 1) & LARGE_PAGE_MASK;
 
 	if (flag & VR_DEMAND_PAGING) {
 		p = 0;
 	}
 	else {
-		p = ihk_mc_alloc_aligned_pages((aligned_new_end - aligned_end) >> PAGE_SHIFT,
+		p = ihk_mc_alloc_aligned_pages(
+				(new_end_allocated - end_allocated) >> PAGE_SHIFT,
 				LARGE_PAGE_P2ALIGN, IHK_MC_AP_NOWAIT |
 				(!(vm->proc->mpol_flags & MPOL_NO_HEAP) ? IHK_MC_AP_USER : 0));
 
 		if (!p) {
-			return end;
+			return end_allocated;
 		}
 	}
 
-	if ((rc = add_process_memory_range(vm, aligned_end, aligned_new_end,
+	if ((rc = add_process_memory_range(vm, end_allocated, new_end_allocated,
 					(p == 0 ? 0 : virt_to_phys(p)), flag, NULL, 0,
 					LARGE_PAGE_SHIFT, NULL)) != 0) {
-		ihk_mc_free_pages(p, (aligned_new_end - aligned_end) >> PAGE_SHIFT);
-		return end;
+		ihk_mc_free_pages(p, (new_end_allocated - end_allocated) >> PAGE_SHIFT);
+		return end_allocated;
 	}
+
 #ifdef ENABLE_RUSAGE
 {
 	int processor_id;
 	long curr;
 	processor_id = ihk_mc_get_processor_id();
-	rusage_rss[processor_id] += ((aligned_new_end - aligned_end) >> PAGE_SHIFT) * PAGE_SIZE;
-	curr = ihk_atomic_add_long_return (((aligned_new_end - aligned_end) >> PAGE_SHIFT) * PAGE_SIZE, &rusage_rss_current);
+	rusage_rss[processor_id] += ((new_end_allocated - end_allocated) >> PAGE_SHIFT) * PAGE_SIZE;
+	curr = ihk_atomic_add_long_return (((new_end_allocated - end_allocated) >> PAGE_SHIFT) * PAGE_SIZE, &rusage_rss_current);
 	if (rusage_rss_max < curr) {
 		atomic_cmpxchg8(&rusage_rss_max, rusage_rss_max, curr);
 	}
@@ -2110,7 +2102,8 @@ unsigned long extend_process_region(struct process_vm *vm,
 	}
 }
 #endif
-	return address;
+
+	return new_end_allocated;
 }
 
 // Original version retained because dcfa (src/mccmd/client/ibmic/main.c) calls this

kernel/syscall.c

@@ -1565,27 +1565,36 @@ SYSCALL_DECLARE(brk)
 	flush_nfo_tlb();
 
 	/* brk change fail, including glibc trick brk(0) to obtain current brk */
-	if(address < region->brk_start) {
+	if (address < region->brk_start) {
 		r = region->brk_end;
 		goto out;
 	}
 
 	/* brk change fail, because we don't shrink memory region  */
-	if(address < region->brk_end) {
+	if (address < region->brk_end) {
 		r = region->brk_end;
 		goto out;
 	}
 
-	/* try to extend memory region */
+	/* If already allocated, just expand and return */
+	if (address < region->brk_end_allocated) {
+		region->brk_end = address;
+		r = region->brk_end;
+		goto out;
+	}
+
+	/* Try to extend memory region */
 	vrflag = VR_PROT_READ | VR_PROT_WRITE;
 	vrflag |= VRFLAG_PROT_TO_MAXPROT(vrflag);
 	ihk_mc_spinlock_lock_noirq(&cpu_local_var(current)->vm->memory_range_lock);
-	region->brk_end = extend_process_region(cpu_local_var(current)->vm,
-			region->brk_start, region->brk_end, address, vrflag);
+	region->brk_end_allocated =
+		extend_process_region(cpu_local_var(current)->vm,
+				region->brk_end_allocated, address, vrflag);
 	ihk_mc_spinlock_unlock_noirq(&cpu_local_var(current)->vm->memory_range_lock);
 	dkprintf("SC(%d)[sys_brk] brk_end set to %lx\n",
 			ihk_mc_get_processor_id(), region->brk_end);
 
+	region->brk_end = address;
 	r = region->brk_end;
 
 out: