gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MM: straight mapping

Browse Source 
Change-Id: I70871f8c382fb00aa719ed501cc5de436d916d7f
This commit is contained in:
Balazs Gerofi
2020-05-27 13:54:04 +09:00
committed by Masamichi Takagi
parent 100bbe6231
commit 201f5ce500
13 changed files with 385 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
arch/arm64/kernel/memory.c
View File

@ -2344,6 +2344,14 @@ static int clear_range(struct page_table *pt, struct process_vm *vm,
if (memobj && ((memobj->flags & MF_PREMAP))) {
args.free_physical = 0;
}
if (vm->proc->straight_va &&
(void *)start == vm->proc->straight_va &&
(void *)end == (vm->proc->straight_va +
vm->proc->straight_len)) {
args.free_physical = 0;
}
args.memobj = memobj;
args.vm = vm;

8
arch/x86_64/kernel/memory.c
View File

@ -1651,6 +1651,14 @@ static int clear_range(struct page_table *pt, struct process_vm *vm,
if (memobj && ((memobj->flags & MF_PREMAP))) {
args.free_physical = 0;
}
if (vm->proc->straight_va &&
(void *)start == vm->proc->straight_va &&
(void *)end == (vm->proc->straight_va +
vm->proc->straight_len)) {
args.free_physical = 0;
}
args.memobj = memobj;
args.vm = vm;

2
executer/include/uprotocol.h
View File

@ -150,6 +150,8 @@ struct program_load_desc {
int thp_disable;
int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */
int uti_use_last_cpu; /* Work-around not to share CPU with OpenMP thread */
int straight_map;
size_t straight_map_threshold;
int nr_processes;
int process_rank;
__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];

10
executer/kernel/mcctrl/syscall.c
View File

@ -655,6 +655,9 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
goto put_and_out;
}
// Force regular page size
pgsize = PAGE_SIZE;
rva = (unsigned long)addr & ~(pgsize - 1);
rpa = rpa & ~(pgsize - 1);
@ -666,7 +669,8 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
/* LWK may hold large page based mappings that align rva outside
* Linux' VMA, make sure we don't try to map to those pages */
if (rva + (pix * PAGE_SIZE) < vma->vm_start) {
if (rva + (pix * PAGE_SIZE) < vma->vm_start ||
rva + (pix * PAGE_SIZE) > vma->vm_end) {
continue;
}
@ -677,11 +681,11 @@ static int rus_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
if (error) {
pr_err("%s: error inserting mapping for 0x%#lx "
"(req: TID: %d, syscall: %lu) error: %d,"
" vm_start: 0x%lx, vm_end: 0x%lx\n",
" vm_start: 0x%lx, vm_end: 0x%lx, pgsize: %lu, ind: %lu\n",
__func__,
(unsigned long)addr, packet.fault_tid,
rsysnum, error,
vma->vm_start, vma->vm_end);
vma->vm_start, vma->vm_end, pgsize, pix);
}
}
else

25
executer/user/mcexec.c
View File

@ -187,6 +187,8 @@ static int mpol_no_stack = 0;
static int mpol_no_bss = 0;
static int mpol_shm_premap = 0;
static int no_bind_ikc_map = 0;
static int straight_map = 0;
static unsigned long straight_map_threshold = (1024*1024);
static unsigned long mpol_threshold = 0;
static unsigned long heap_extension = -1;
static int profile = 0;
@ -1674,6 +1676,18 @@ static struct option mcexec_options[] = {
.flag = NULL,
.val = 'M',
},
{
.name = "enable-straight-map",
.has_arg = no_argument,
.flag = &straight_map,
.val = 1,
},
{
.name = "straight-map-threshold",
.has_arg = required_argument,
.flag = NULL,
.val = 'S',
},
{
.name = "disable-sched-yield",
.has_arg = no_argument,
@ -2095,10 +2109,10 @@ int main(int argc, char **argv)
/* Parse options ("+" denotes stop at the first non-option) */
#ifdef ADD_ENVS_OPTION
while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:u:",
while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:e:s:m:u:S:",
mcexec_options, NULL)) != -1) {
#else /* ADD_ENVS_OPTION */
while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:u:",
while ((opt = getopt_long(argc, argv, "+c:n:t:M:h:s:m:u:S:",
mcexec_options, NULL)) != -1) {
#endif /* ADD_ENVS_OPTION */
switch (opt) {
@ -2140,6 +2154,10 @@ int main(int argc, char **argv)
heap_extension = atobytes(optarg);
break;
case 'S':
straight_map_threshold = atobytes(optarg);
break;
#ifdef ADD_ENVS_OPTION
case 'e':
add_env_list(&extra_env, optarg);
@ -2678,6 +2696,9 @@ int main(int argc, char **argv)
desc->uti_use_last_cpu = uti_use_last_cpu;
desc->thp_disable = get_thp_disable();
desc->straight_map = straight_map;
desc->straight_map_threshold = straight_map_threshold;
/* user_start and user_end are set by this call */
if (ioctl(fd, MCEXEC_UP_PREPARE_IMAGE, (unsigned long)desc) != 0) {
perror("prepare");

3
kernel/host.c
View File

@ -546,6 +546,9 @@ static int process_msg_prepare_process(unsigned long rphys)
proc->uti_thread_rank = pn->uti_thread_rank;
proc->uti_use_last_cpu = pn->uti_use_last_cpu;
proc->straight_map = pn->straight_map;
proc->straight_map_threshold = pn->straight_map_threshold;
#ifdef PROFILE_ENABLE
proc->profile = pn->profile;
thread->profile = pn->profile;

9
kernel/include/process.h
View File

@ -390,6 +390,7 @@ struct vm_range {
struct rb_node vm_rb_node;
unsigned long start, end;
unsigned long flag;
unsigned long straight_start;
struct memobj *memobj;
off_t objoff;
int pgshift; /* page size. 0 means THP */
@ -563,6 +564,9 @@ struct process {
int clone_count;
int thp_disable;
int straight_map;
size_t straight_map_threshold;
// perf_event
int perf_status;
#define PP_NONE 0
@ -578,6 +582,11 @@ struct process {
#endif // PROFILE_ENABLE
int nr_processes; /* For partitioned execution */
int process_rank; /* Rank in partition */
void *straight_va;
size_t straight_len;
unsigned long straight_pa;
int coredump_barrier_count, coredump_barrier_count2;
mcs_rwlock_lock_t coredump_lock; // lock for coredump
};

2
kernel/include/profile.h
View File

@ -40,6 +40,8 @@ enum profile_event_type {
PROFILE_remote_page_fault,
PROFILE_mpol_alloc_missed,
PROFILE_mmap_anon_contig_phys,
PROFILE_mmap_anon_straight,
PROFILE_mmap_anon_not_straight,
PROFILE_mmap_anon_no_contig_phys,
PROFILE_mmap_regular_file,
PROFILE_mmap_device_file,

2
kernel/include/syscall.h
View File

@ -217,6 +217,8 @@ struct program_load_desc {
int thp_disable;
int uti_thread_rank; /* N-th clone() spawns a thread on Linux CPU */
int uti_use_last_cpu; /* Work-around not to share CPU with OpenMP thread */
int straight_map;
size_t straight_map_threshold;
int nr_processes;
int process_rank;
__cpu_set_unit cpu_set[PLD_CPU_SET_SIZE];

81
kernel/process.c
View File

@ -945,6 +945,11 @@ int split_process_memory_range(struct process_vm *vm, struct vm_range *range,
}
newrange->start = addr;
newrange->straight_start = 0;
if (range->straight_start) {
newrange->straight_start =
range->straight_start + (addr - range->start);
}
newrange->end = range->end;
newrange->flag = range->flag;
newrange->pgshift = range->pgshift;
@ -1045,6 +1050,11 @@ static int free_process_memory_range(struct process_vm *vm,
start = range->start;
end = range->end;
/* No regular page table manipulation for straight mappings */
if (range->straight_start || ((void *)start == vm->proc->straight_va))
goto straight_out;
if (!(range->flag & (VR_REMOTE | VR_IO_NOCACHE | VR_RESERVED))) {
neighbor = previous_process_memory_range(vm, range);
pgsize = -1;
@ -1126,11 +1136,37 @@ static int free_process_memory_range(struct process_vm *vm,
memobj_unref(range->memobj);
}
straight_out:
rb_erase(&range->vm_rb_node, &vm->vm_range_tree);
for (i = 0; i < VM_RANGE_CACHE_SIZE; ++i) {
if (vm->range_cache[i] == range)
vm->range_cache[i] = NULL;
}
/* For straight ranges just free physical memory */
if (range->straight_start) {
ihk_mc_free_pages(phys_to_virt(vm->proc->straight_pa +
(range->straight_start - (unsigned long)vm->proc->straight_va)),
(range->end - range->start) >> PAGE_SHIFT);
dkprintf("%s: straight range 0x%lx @ straight 0x%lx physical memory freed\n",
__FUNCTION__, range->start, range->straight_start);
}
/* For the main straight mapping, free page tables */
else if (range->start == (unsigned long)vm->proc->straight_va &&
range->end == ((unsigned long)vm->proc->straight_va +
vm->proc->straight_len)) {
ihk_mc_spinlock_lock_noirq(&vm->page_table_lock);
error = ihk_mc_pt_clear_range(vm->address_space->page_table, vm,
(void *)start, (void *)end);
ihk_mc_spinlock_unlock_noirq(&vm->page_table_lock);
dkprintf("%s: straight mapping 0x%lx unmapped\n",
__FUNCTION__, vm->proc->straight_va);
vm->proc->straight_va = 0;
vm->proc->straight_len = 0;
}
kfree(range);
dkprintf("free_process_memory_range(%p,%lx-%lx): 0\n",
@ -1148,6 +1184,50 @@ int remove_process_memory_range(struct process_vm *vm,
dkprintf("remove_process_memory_range(%p,%lx,%lx)\n",
vm, start, end);
/*
* Convert to real virtual address for straight ranges,
* but not for the main straight mapping
*/
if (vm->proc->straight_va &&
start >= (unsigned long)vm->proc->straight_va &&
end <= ((unsigned long)vm->proc->straight_va +
vm->proc->straight_len) &&
!(start == (unsigned long)vm->proc->straight_va &&
end == ((unsigned long)vm->proc->straight_va +
vm->proc->straight_len))) {
struct vm_range *range_iter;
struct vm_range *range = NULL;
unsigned long len = end - start;
range_iter = lookup_process_memory_range(vm, 0, -1);
while (range_iter) {
if (range_iter->straight_start &&
start >= range_iter->straight_start &&
start < (range_iter->straight_start +
(range_iter->end - range_iter->start))) {
range = range_iter;
break;
}
range_iter = next_process_memory_range(vm, range_iter);
}
if (!range) {
kprintf("%s: WARNING: no straight mapping range found for 0x%lx\n",
__FUNCTION__, start);
return 0;
}
dkprintf("%s: straight range converted from 0x%lx:%lu -> 0x%lx:%lu\n",
__FUNCTION__,
start, len,
range->start + (start - range->straight_start), len);
start = range->start + (start - range->straight_start);
end = start + len;
}
next = lookup_process_memory_range(vm, start, end);
while ((range = next) && range->start < end) {
next = next_process_memory_range(vm, range);
@ -1345,6 +1425,7 @@ int add_process_memory_range(struct process_vm *vm,
range->objoff = offset;
range->pgshift = pgshift;
range->private_data = NULL;
range->straight_start = 0;
rc = 0;
if (phys == NOPHYS) {

2
kernel/profile.c
View File

@ -63,6 +63,8 @@ char *profile_event_names[] =
"remote_page_fault",
"mpol_alloc_missed",
"mmap_anon_contig_phys",
"|-------mmap_straight",
"|---mmap_not_straight",
"mmap_anon_no_contig_phys",
"mmap_regular_file",
"mmap_device_file",

214
kernel/syscall.c
View File

@ -1582,10 +1582,20 @@ int do_munmap(void *addr, size_t len, int holding_memory_range_lock)
{
int error;
int ro_freed;
struct thread *thread = cpu_local_var(current);
begin_free_pages_pending();
error = remove_process_memory_range(cpu_local_var(current)->vm,
(intptr_t)addr, (intptr_t)addr+len, &ro_freed);
/* No host involvement for straight mapping ranges */
if (thread->proc->straight_va &&
addr >= thread->proc->straight_va &&
(addr + len) <=
(thread->proc->straight_va + thread->proc->straight_len)) {
goto out;
}
if (error || !ro_freed) {
clear_host_pte((uintptr_t)addr, len, holding_memory_range_lock);
}
@ -1596,6 +1606,8 @@ int do_munmap(void *addr, size_t len, int holding_memory_range_lock)
/* through */
}
}
out:
finish_free_pages_pending();
dkprintf("%s: 0x%lx:%lu, error: %ld\n",
@ -1667,6 +1679,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
void *p = NULL;
int vrflags;
uintptr_t phys;
intptr_t straight_phys;
struct memobj *memobj = NULL;
int maxprot;
int denied;
@ -1708,6 +1721,124 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
flush_nfo_tlb();
/* Initialize straight large memory mapping */
if (proc->straight_map && !proc->straight_va) {
unsigned long straight_pa_start = 0xFFFFFFFFFFFFFFFF;
unsigned long straight_pa_end = 0;
int i;
int p2align = PAGE_P2ALIGN;
size_t psize = PAGE_SIZE;
unsigned long vrflags;
enum ihk_mc_pt_attribute ptattr;
struct vm_range *range;
vrflags = PROT_TO_VR_FLAG(PROT_READ | PROT_WRITE);
vrflags |= VRFLAG_PROT_TO_MAXPROT(vrflags);
vrflags |= VR_DEMAND_PAGING;
for (i = 0; i < ihk_mc_get_nr_memory_chunks(); ++i) {
unsigned long start, end;
ihk_mc_get_memory_chunk(i, &start, &end, NULL);
if (straight_pa_start > start) {
straight_pa_start = start;
}
if (straight_pa_end < end) {
straight_pa_end = end;
}
}
kprintf("%s: straight_pa_start: 0x%lx, straight_pa_end: 0x%lx\n",
__FUNCTION__, straight_pa_start, straight_pa_end);
error = arch_get_smaller_page_size(NULL,
straight_pa_end - straight_pa_start,
&psize, &p2align);
if (error) {
kprintf("%s: arch_get_smaller_page_size failed: %d\n",
__FUNCTION__, error);
goto straight_out;
}
//psize = PTL2_SIZE;
//p2align = PTL2_SHIFT - PTL1_SHIFT;
// Force 512G page
//psize = (1UL << 39);
//p2align = 39 - PAGE_SHIFT;
// Force 512MB page
psize = (1UL << 29);
p2align = 29 - PAGE_SHIFT;
kprintf("%s: using page shift: %d, psize: %lu\n",
__FUNCTION__, p2align + PAGE_SHIFT, psize);
straight_pa_start &= ~(psize - 1);
straight_pa_end = (straight_pa_end + psize - 1) & ~(psize - 1);
kprintf("%s: aligned straight_pa_start: 0x%lx, straight_pa_end: 0x%lx\n",
__FUNCTION__, straight_pa_start, straight_pa_end);
proc->straight_len = straight_pa_end - straight_pa_start;
error = search_free_space(proc->straight_len,
PAGE_SHIFT + p2align, (uintptr_t *)&proc->straight_va);
if (error) {
kprintf("%s: search_free_space() failed: %d\n",
__FUNCTION__, error);
proc->straight_va = 0;
goto straight_out;
}
dkprintf("%s: straight_va: 0x%lx to be used\n",
__FUNCTION__, proc->straight_va);
if (add_process_memory_range(proc->vm, (unsigned long)proc->straight_va,
(unsigned long)proc->straight_va + proc->straight_len,
NOPHYS, vrflags, NULL, 0,
PAGE_SHIFT + p2align, &range) != 0) {
kprintf("%s: error: adding straight memory range \n",
__FUNCTION__);
proc->straight_va = 0;
goto straight_out;
}
kprintf("%s: straight_va: 0x%lx, range->pgshift: %d, range OK\n",
__FUNCTION__, proc->straight_va, range->pgshift);
ptattr = arch_vrflag_to_ptattr(range->flag, PF_POPULATE, NULL);
error = ihk_mc_pt_set_range(proc->vm->address_space->page_table,
proc->vm,
(void *)range->start,
(void *)range->end,
straight_pa_start, ptattr,
range->pgshift,
range, 0);
if (error) {
kprintf("%s: ihk_mc_pt_set_range() failed: %d\n",
__FUNCTION__, error);
proc->straight_va = 0;
goto straight_out;
}
//ihk_mc_pt_print_pte(proc->vm->address_space->page_table, range->start);
region->map_end = (unsigned long)proc->straight_va + proc->straight_len;
proc->straight_pa = straight_pa_start;
kprintf("%s: straight mapping: 0x%lx:%lu @ 0x%lx, "
"psize: %lu, straight_map_threshold: %lu\n",
__FUNCTION__,
proc->straight_va,
proc->straight_len,
proc->straight_pa,
psize,
proc->straight_map_threshold);
}
straight_out:
if (flags & MAP_HUGETLB) {
pgshift = (flags >> MAP_HUGE_SHIFT) & 0x3F;
if (!pgshift) {
@ -1735,6 +1866,15 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
if ((flags & MAP_FIXED) && proc->straight_va &&
((void *)addr >= proc->straight_va) &&
((void *)addr + len) <= (proc->straight_va + proc->straight_len)) {
kprintf("%s: can't map MAP_FIXED into straight mapping\n",
__FUNCTION__);
error = -EINVAL;
goto out;
}
if (flags & MAP_FIXED) {
/* clear specified address range */
error = do_munmap((void *)addr, len, 1/* holding memory_range_lock */);
@ -1791,6 +1931,7 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
}
phys = 0;
straight_phys = 0;
off = 0;
maxprot = PROT_READ | PROT_WRITE | PROT_EXEC;
if (!(flags & MAP_ANONYMOUS)) {
@ -1978,6 +2119,31 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
}
vrflags |= VRFLAG_PROT_TO_MAXPROT(PROT_TO_VR_FLAG(maxprot));
/*
* Large anonymous non-fix allocations are in straight mapping,
* pretend demand paging to avoid filling in PTEs
*/
if ((flags & MAP_ANONYMOUS) && proc->straight_map &&
!(flags & MAP_FIXED) && phys) {
if (len >= proc->straight_map_threshold) {
dkprintf("%s: range 0x%lx:%lu will be straight, addding VR_DEMAND\n",
__FUNCTION__, addr, len);
vrflags |= VR_DEMAND_PAGING;
straight_phys = phys;
phys = 0;
#ifdef PROFILE_ENABLE
profile_event_add(PROFILE_mmap_anon_straight, len);
#endif // PROFILE_ENABLE
}
else {
#ifdef PROFILE_ENABLE
if (cpu_local_var(current)->profile)
kprintf("%s: contiguous but not straight? len: %lu\n", __func__, len);
profile_event_add(PROFILE_mmap_anon_not_straight, len);
#endif // PROFILE_ENABLE
}
}
error = add_process_memory_range(thread->vm, addr, addr+len, phys,
vrflags, memobj, off, pgshift, &range);
if (error) {
@ -1988,6 +2154,19 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
goto out;
}
/* Update straight mapping start address */
if (straight_phys) {
extern int zero_at_free;
range->straight_start =
(unsigned long)proc->straight_va +
(straight_phys - proc->straight_pa);
dkprintf("%s: range 0x%lx:%lu is straight starting at 0x%lx\n",
__FUNCTION__, addr, len, range->straight_start);
if (!zero_at_free) {
memset((void *)phys_to_virt(straight_phys), 0, len);
}
}
/* Determine pre-populated size */
populate_len = memobj ? min(len, memobj->size) : len;
@ -2042,12 +2221,13 @@ do_mmap(const uintptr_t addr0, const size_t len0, const int prot,
ro_vma_mapped = 0;
out:
if (ro_vma_mapped) {
if (ro_vma_mapped && !range->straight_start) {
(void)set_host_vma(addr, len, PROT_READ | PROT_WRITE | PROT_EXEC, 1/* holding memory_range_lock */);
}
ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
if (!error && populated_mapping && !((vrflags & VR_PROT_MASK) == VR_PROT_NONE)) {
if (!error && populated_mapping &&
!((vrflags & VR_PROT_MASK) == VR_PROT_NONE) && !range->straight_start) {
error = populate_process_memory(thread->vm,
(void *)addr, populate_len);
@ -2087,7 +2267,9 @@ out:
addr, len, addr0, len0, prot, flags,
fd, off0, error, addr);
return (!error)? addr: error;
return !error ?
(range->straight_start ? range->straight_start : addr) :
error;
}
SYSCALL_DECLARE(munmap)
@ -2167,6 +2349,16 @@ SYSCALL_DECLARE(mprotect)
return 0;
}
if (thread->proc->straight_va &&
((void *)start >= thread->proc->straight_va) &&
(void *)end <= (thread->proc->straight_va +
thread->proc->straight_len)) {
kprintf("%s: ignored for straight mapping 0x%lx\n",
__FUNCTION__, start);
error = 0;
goto out_straight;
}
flush_nfo_tlb();
ihk_rwspinlock_write_lock_noirq(&thread->vm->memory_range_lock);
@ -2260,6 +2452,8 @@ out:
}
}
ihk_rwspinlock_write_unlock_noirq(&thread->vm->memory_range_lock);
out_straight:
dkprintf("[%d]sys_mprotect(%lx,%lx,%x): %d\n",
ihk_mc_get_processor_id(), start, len0, prot, error);
return error;
@ -8730,6 +8924,15 @@ SYSCALL_DECLARE(mremap)
dkprintf("sys_mremap(%#lx,%#lx,%#lx,%#x,%#lx)\n",
oldaddr, oldsize0, newsize0, flags, newaddr);
if (vm->proc->straight_va &&
(void *)oldaddr >= vm->proc->straight_va &&
(void *)oldaddr < vm->proc->straight_va + vm->proc->straight_len) {
kprintf("%s: reject for straight range 0x%lx\n",
__FUNCTION__, oldaddr);
return -EINVAL;
}
ihk_rwspinlock_write_lock_noirq(&vm->memory_range_lock);
/* check arguments */
@ -9133,6 +9336,11 @@ SYSCALL_DECLARE(mbind)
__FUNCTION__,
addr, len, mode, nodemask, flags);
/* No bind support for straight mapped processes */
if (cpu_local_var(current)->proc->straight_va) {
return 0;
}
/* Validate arguments */
if (addr & ~PAGE_MASK) {
return -EINVAL;

40
kernel/xpmem.c
View File

@ -1919,21 +1919,35 @@ static int xpmem_remap_pte(
goto out;
}
seg_pte = ihk_mc_pt_lookup_pte(seg_tg->vm->address_space->page_table,
(void *)seg_vaddr, seg_vmr->pgshift, &seg_pgaddr, &seg_pgsize,
&seg_p2align);
if (!seg_pte) {
ret = -EFAULT;
ekprintf("%s: ERROR: ihk_mc_pt_lookup_pte() failed\n",
__FUNCTION__);
goto out;
if (seg_tg->vm->proc->straight_va &&
seg_vaddr >= (unsigned long)seg_tg->vm->proc->straight_va &&
seg_vaddr < ((unsigned long)seg_tg->vm->proc->straight_va +
seg_tg->vm->proc->straight_len)) {
seg_phys = (((unsigned long)seg_vaddr & PAGE_MASK) -
(unsigned long)seg_tg->vm->proc->straight_va) +
seg_tg->vm->proc->straight_pa;
dkprintf("%s: 0x%lx in PID %d is straight -> phys: 0x%lx\n",
__func__, (unsigned long)seg_vaddr & PAGE_MASK,
seg_tg->tgid, seg_phys);
}
XPMEM_DEBUG("seg_pte=0x%016lx, seg_pgaddr=0x%p, seg_pgsize=%lu, "
"seg_p2align=%d",
*seg_pte, seg_pgaddr, seg_pgsize, seg_p2align);
else {
seg_phys = pte_get_phys(seg_pte);
XPMEM_DEBUG("seg_phys=0x%lx", seg_phys);
seg_pte = ihk_mc_pt_lookup_pte(seg_tg->vm->address_space->page_table,
(void *)seg_vaddr, seg_vmr->pgshift, &seg_pgaddr, &seg_pgsize,
&seg_p2align);
if (!seg_pte) {
ret = -EFAULT;
ekprintf("%s: ERROR: ihk_mc_pt_lookup_pte() failed\n",
__FUNCTION__);
goto out;
}
XPMEM_DEBUG("seg_pte=0x%016lx, seg_pgaddr=0x%p, seg_pgsize=%lu, "
"seg_p2align=%d",
*seg_pte, seg_pgaddr, seg_pgsize, seg_p2align);
seg_phys = pte_get_phys(seg_pte);
XPMEM_DEBUG("seg_phys=0x%lx", seg_phys);
}
att_pte = ihk_mc_pt_lookup_pte(vm->address_space->page_table,
(void *)vaddr, vmr->pgshift, &att_pgaddr, &att_pgsize,