gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
8f4f68b87713a4abeb195d58eab9adcc74884db6
mckernel/arch/x86/kernel/memory.c

2436 lines
55 KiB
C
Raw Blame History

/**
* \file memory.c
* License details are found in the file LICENSE.
* \brief
* Acquire physical pages and manipulate page table entries.
* \author Taku Shimosawa <shimosawa@is.s.u-tokyo.ac.jp> \par
* Copyright (C) 2011 - 2012 Taku Shimosawa
* \author Gou Nakamura <go.nakamura.yw@hitachi-solutions.com> \par
* Copyright (C) 2015 RIKEN AICS
*/
/*
* HISTORY
*/
#include <ihk/cpu.h>
#include <ihk/debug.h>
#include <ihk/mm.h>
#include <types.h>
#include <memory.h>
#include <string.h>
#include <errno.h>
#include <list.h>
#include <process.h>
#include <page.h>
#include <cls.h>
#include <kmalloc.h>
#define dkprintf(...) do { if (0) kprintf(__VA_ARGS__); } while (0)
#define ekprintf(...) kprintf(__VA_ARGS__)
static char *last_page;
extern char _head[], _end[];
extern unsigned long x86_kernel_phys_base;
/* Arch specific early allocation routine */
void *early_alloc_pages(int nr_pages)
{
void *p;
if (!last_page) {
last_page = (char *)(((unsigned long)_end + PAGE_SIZE - 1)
& PAGE_MASK);
/* Convert the virtual address from text's to straight maps */
last_page = phys_to_virt(virt_to_phys(last_page));
} else if (last_page == (void *)-1) {
panic("Early allocator is already finalized. Do not use it.\n");
}
p = last_page;
last_page += (nr_pages * PAGE_SIZE);
return p;
}
void early_alloc_invalidate(void)
{
last_page = (void *)-1;
}
void *ihk_mc_allocate(int size, int flag)
{
if (!cpu_local_var(kmalloc_initialized)) {
kprintf("%s: error, kmalloc not yet initialized\n", __FUNCTION__);
return NULL;
}
return kmalloc(size, IHK_MC_AP_NOWAIT);
}
void ihk_mc_free(void *p)
{
if (!cpu_local_var(kmalloc_initialized)) {
kprintf("%s: error, kmalloc not yet initialized\n", __FUNCTION__);
return;
}
kfree(p);
}
void *get_last_early_heap(void)
{
return last_page;
}
void flush_tlb(void)
{
unsigned long cr3;
asm volatile("movq %%cr3, %0; movq %0, %%cr3" : "=r"(cr3) : : "memory");
}
void flush_tlb_single(unsigned long addr)
{
asm volatile("invlpg (%0)" :: "r" (addr) : "memory");
}
struct page_table {
pte_t entry[PT_ENTRIES];
};
static struct page_table *init_pt;
static ihk_spinlock_t init_pt_lock;
static int use_1gb_page = 0;
static void check_available_page_size(void)
{
uint32_t edx;
asm ("cpuid" : "=d" (edx) : "a" (0x80000001) : "%rbx", "%rcx");
use_1gb_page = (edx & (1 << 26))? 1: 0;
kprintf("use_1gb_page: %d\n", use_1gb_page);
return;
}
static unsigned long setup_l2(struct page_table *pt,
unsigned long page_head, unsigned long start,
unsigned long end)
{
int i;
unsigned long phys;
for (i = 0; i < PT_ENTRIES; i++) {
phys = page_head + ((unsigned long)i << PTL2_SHIFT);
if (phys + PTL2_SIZE <= start || phys >= end) {
pt->entry[i] = 0;
continue;
}
pt->entry[i] = phys | PFL2_KERN_ATTR | PFL2_SIZE;
}
return virt_to_phys(pt);
}
static unsigned long setup_l3(struct page_table *pt,
unsigned long page_head, unsigned long start,
unsigned long end)
{
int i;
unsigned long phys, pt_phys;
for (i = 0; i < PT_ENTRIES; i++) {
phys = page_head + ((unsigned long)i << PTL3_SHIFT);
if (phys + PTL3_SIZE <= start || phys >= end) {
pt->entry[i] = 0;
continue;
}
pt_phys = setup_l2(ihk_mc_alloc_pages(1, IHK_MC_AP_CRITICAL), phys, start, end);
pt->entry[i] = pt_phys | PFL3_PDIR_ATTR;
}
return virt_to_phys(pt);
}
static void init_normal_area(struct page_table *pt)
{
unsigned long map_start, map_end, phys, pt_phys;
int ident_index, virt_index;
map_start = ihk_mc_get_memory_address(IHK_MC_GMA_MAP_START, 0);
map_end = ihk_mc_get_memory_address(IHK_MC_GMA_MAP_END, 0);
kprintf("map_start = %lx, map_end = %lx\n", map_start, map_end);
ident_index = map_start >> PTL4_SHIFT;
virt_index = (MAP_ST_START >> PTL4_SHIFT) & (PT_ENTRIES - 1);
memset(pt, 0, sizeof(struct page_table));
for (phys = (map_start & ~(PTL4_SIZE - 1)); phys < map_end;
phys += PTL4_SIZE) {
pt_phys = setup_l3(ihk_mc_alloc_pages(1, IHK_MC_AP_CRITICAL), phys,
map_start, map_end);
pt->entry[ident_index++] = pt_phys | PFL4_PDIR_ATTR;
pt->entry[virt_index++] = pt_phys | PFL4_PDIR_ATTR;
}
}
static struct page_table *__alloc_new_pt(enum ihk_mc_ap_flag ap_flag)
{
struct page_table *newpt = ihk_mc_alloc_pages(1, ap_flag);
if(newpt)
memset(newpt, 0, sizeof(struct page_table));
return newpt;
}
/*
* XXX: Confusingly, L4 and L3 automatically add PRESENT,
* but L2 and L1 do not!
*/
enum ihk_mc_pt_attribute attr_mask
= 0
| PTATTR_FILEOFF
| PTATTR_WRITABLE
| PTATTR_USER
| PTATTR_ACTIVE
| 0;
#define ATTR_MASK attr_mask
void enable_ptattr_no_execute(void)
{
attr_mask |= PTATTR_NO_EXECUTE;
return;
}
#if 0
static unsigned long attr_to_l4attr(enum ihk_mc_pt_attribute attr)
{
return (attr & ATTR_MASK) | PFL4_PRESENT;
}
#endif
static unsigned long attr_to_l3attr(enum ihk_mc_pt_attribute attr)
{
unsigned long r = (attr & (ATTR_MASK | PTATTR_LARGEPAGE));
if ((attr & PTATTR_UNCACHABLE) && (attr & PTATTR_LARGEPAGE)) {
return r | PFL3_PCD | PFL3_PWT;
}
return r;
}
static unsigned long attr_to_l2attr(enum ihk_mc_pt_attribute attr)
{
unsigned long r = (attr & (ATTR_MASK | PTATTR_LARGEPAGE));
if ((attr & PTATTR_UNCACHABLE) && (attr & PTATTR_LARGEPAGE)) {
return r | PFL2_PCD | PFL2_PWT;
}
return r;
}
static unsigned long attr_to_l1attr(enum ihk_mc_pt_attribute attr)
{
if (attr & PTATTR_UNCACHABLE) {
return (attr & ATTR_MASK) | PFL1_PCD | PFL1_PWT;
}
else if (attr & PTATTR_WRITE_COMBINED) {
return (attr & ATTR_MASK) | PFL1_PWT;
}
else {
return (attr & ATTR_MASK);
}
}
#define GET_VIRT_INDICES(virt, l4i, l3i, l2i, l1i) \
l4i = ((virt) >> PTL4_SHIFT) & (PT_ENTRIES - 1); \
l3i = ((virt) >> PTL3_SHIFT) & (PT_ENTRIES - 1); \
l2i = ((virt) >> PTL2_SHIFT) & (PT_ENTRIES - 1); \
l1i = ((virt) >> PTL1_SHIFT) & (PT_ENTRIES - 1)
#define GET_INDICES_VIRT(l4i, l3i, l2i, l1i) \
( ((uint64_t)(l4i) << PTL4_SHIFT) \
| ((uint64_t)(l3i) << PTL3_SHIFT) \
| ((uint64_t)(l2i) << PTL2_SHIFT) \
| ((uint64_t)(l1i) << PTL1_SHIFT) \
)
void set_pte(pte_t *ppte, unsigned long phys, enum ihk_mc_pt_attribute attr)
{
if (attr & PTATTR_LARGEPAGE) {
*ppte = phys | attr_to_l2attr(attr) | PFL2_SIZE;
}
else {
*ppte = phys | attr_to_l1attr(attr);
}
}
#if 0
/*
* get_pte()
*
* Descripton: walks the page tables (creates tables if not existing)
* and returns a pointer to the PTE corresponding to the
* virtual address.
*/
pte_t *get_pte(struct page_table *pt, void *virt, enum ihk_mc_pt_attribute attr, enum ihk_mc_ap_flag ap_flag)
{
int l4idx, l3idx, l2idx, l1idx;
unsigned long v = (unsigned long)virt;
struct page_table *newpt;
if (!pt) {
pt = init_pt;
}
GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx);
/* TODO: more detailed attribute check */
if (pt->entry[l4idx] & PFL4_PRESENT) {
pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK);
} else {
if((newpt = __alloc_new_pt(ap_flag)) == NULL)
return NULL;
pt->entry[l4idx] = virt_to_phys(newpt) | attr_to_l4attr(attr);
pt = newpt;
}
if (pt->entry[l3idx] & PFL3_PRESENT) {
pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK);
} else {
if((newpt = __alloc_new_pt(ap_flag)) == NULL)
return NULL;
pt->entry[l3idx] = virt_to_phys(newpt) | attr_to_l3attr(attr);
pt = newpt;
}
/* PTATTR_LARGEPAGE */
if (attr & PTATTR_LARGEPAGE) {
return &(pt->entry[l2idx]);
}
/* Requested regular page, but large is allocated? */
if (pt->entry[l2idx] & PFL2_SIZE) {
return NULL;
}
if (pt->entry[l2idx] & PFL2_PRESENT) {
pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK);
} else {
if((newpt = __alloc_new_pt(ap_flag)) == NULL)
return NULL;
pt->entry[l2idx] = virt_to_phys(newpt) | attr_to_l2attr(attr)
| PFL2_PRESENT;
pt = newpt;
}
return &(pt->entry[l1idx]);
}
#endif
static int __set_pt_page(struct page_table *pt, void *virt, unsigned long phys,
enum ihk_mc_pt_attribute attr)
{
int l4idx, l3idx, l2idx, l1idx;
unsigned long v = (unsigned long)virt;
struct page_table *newpt;
enum ihk_mc_ap_flag ap_flag;
int in_kernel =
(((unsigned long long)virt) >= 0xffff000000000000ULL);
unsigned long init_pt_lock_flags;
int ret = -ENOMEM;
init_pt_lock_flags = 0; /* for avoidance of warning */
if (in_kernel) {
init_pt_lock_flags = ihk_mc_spinlock_lock(&init_pt_lock);
}
ap_flag = (attr & PTATTR_FOR_USER) ?
IHK_MC_AP_NOWAIT: IHK_MC_AP_CRITICAL;
if (!pt) {
pt = init_pt;
}
if (attr & PTATTR_LARGEPAGE) {
phys &= LARGE_PAGE_MASK;
} else {
phys &= PAGE_MASK;
}
GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx);
/* TODO: more detailed attribute check */
if (pt->entry[l4idx] & PFL4_PRESENT) {
pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK);
} else {
if((newpt = __alloc_new_pt(ap_flag)) == NULL)
goto out;
pt->entry[l4idx] = virt_to_phys(newpt) | PFL4_PDIR_ATTR;
pt = newpt;
}
if (pt->entry[l3idx] & PFL3_PRESENT) {
pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK);
} else {
if((newpt = __alloc_new_pt(ap_flag)) == NULL)
goto out;
pt->entry[l3idx] = virt_to_phys(newpt) | PFL3_PDIR_ATTR;
pt = newpt;
}
if (attr & PTATTR_LARGEPAGE) {
if (pt->entry[l2idx] & PFL2_PRESENT) {
if ((pt->entry[l2idx] & PAGE_MASK) != phys) {
goto out;
} else {
ret = 0;
goto out;
}
} else {
pt->entry[l2idx] = phys | attr_to_l2attr(attr)
| PFL2_SIZE;
ret = 0;
goto out;
}
}
if (pt->entry[l2idx] & PFL2_PRESENT) {
pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK);
} else {
if((newpt = __alloc_new_pt(ap_flag)) == NULL)
goto out;
pt->entry[l2idx] = virt_to_phys(newpt) | PFL2_PDIR_ATTR;
pt = newpt;
}
if (pt->entry[l1idx] & PFL1_PRESENT) {
if ((pt->entry[l1idx] & PT_PHYSMASK) != phys) {
kprintf("EBUSY: page table for 0x%lX is already set\n", virt);
ret = -EBUSY;
goto out;
} else {
ret = 0;
goto out;
}
}
pt->entry[l1idx] = phys | attr_to_l1attr(attr);
ret = 0;
out:
if (in_kernel) {
ihk_mc_spinlock_unlock(&init_pt_lock, init_pt_lock_flags);
}
return ret;
}
static int __clear_pt_page(struct page_table *pt, void *virt, int largepage)
{
int l4idx, l3idx, l2idx, l1idx;
unsigned long v = (unsigned long)virt;
if (!pt) {
pt = init_pt;
}
if (largepage) {
v &= LARGE_PAGE_MASK;
} else {
v &= PAGE_MASK;
}
GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx);
if (!(pt->entry[l4idx] & PFL4_PRESENT)) {
return -EINVAL;
}
pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK);
if (!(pt->entry[l3idx] & PFL3_PRESENT)) {
return -EINVAL;
}
pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK);
if (largepage) {
if (!(pt->entry[l2idx] & PFL2_PRESENT)) {
return -EINVAL;
} else {
pt->entry[l2idx] = 0;
return 0;
}
}
if (!(pt->entry[l2idx] & PFL2_PRESENT)) {
return -EINVAL;
}
pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK);
pt->entry[l1idx] = 0;
return 0;
}
uint64_t ihk_mc_pt_virt_to_pagemap(struct page_table *pt, unsigned long virt)
{
int error;
unsigned long phys;
uint64_t pagemap;
error = ihk_mc_pt_virt_to_phys(pt, (void *)virt, &phys);
if (error) {
return 0;
}
pagemap = PM_PFRAME(phys >> PAGE_SHIFT);
pagemap |= PM_PSHIFT(PAGE_SHIFT) | PM_PRESENT;
return pagemap;
}
int ihk_mc_pt_virt_to_phys(struct page_table *pt,
const void *virt, unsigned long *phys)
{
int l4idx, l3idx, l2idx, l1idx;
unsigned long v = (unsigned long)virt;
if (!pt) {
pt = init_pt;
}
GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx);
if (!(pt->entry[l4idx] & PFL4_PRESENT)) {
return -EFAULT;
}
pt = phys_to_virt(pte_get_phys(&pt->entry[l4idx]));
if (!(pt->entry[l3idx] & PFL3_PRESENT)) {
return -EFAULT;
}
if ((pt->entry[l3idx] & PFL3_SIZE)) {
*phys = pte_get_phys(&pt->entry[l3idx])
| (v & (PTL3_SIZE - 1));
return 0;
}
pt = phys_to_virt(pte_get_phys(&pt->entry[l3idx]));
if (!(pt->entry[l2idx] & PFL2_PRESENT)) {
return -EFAULT;
}
if ((pt->entry[l2idx] & PFL2_SIZE)) {
*phys = pte_get_phys(&pt->entry[l2idx])
| (v & (PTL2_SIZE - 1));
return 0;
}
pt = phys_to_virt(pte_get_phys(&pt->entry[l2idx]));
if (!(pt->entry[l1idx] & PFL1_PRESENT)) {
return -EFAULT;
}
*phys = pte_get_phys(&pt->entry[l1idx]) | (v & (PTL1_SIZE - 1));
return 0;
}
int ihk_mc_pt_print_pte(struct page_table *pt, void *virt)
{
int l4idx, l3idx, l2idx, l1idx;
unsigned long v = (unsigned long)virt;
if (!pt) {
pt = init_pt;
}
GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx);
if (!(pt->entry[l4idx] & PFL4_PRESENT)) {
__kprintf("0x%lX l4idx not present! \n", (unsigned long)virt);
__kprintf("l4 entry: 0x%lX\n", pt->entry[l4idx]);
return -EFAULT;
}
pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK);
__kprintf("l3 table: 0x%lX l3idx: %d \n", virt_to_phys(pt), l3idx);
if (!(pt->entry[l3idx] & PFL3_PRESENT)) {
__kprintf("0x%lX l3idx not present! \n", (unsigned long)virt);
__kprintf("l3 entry: 0x%lX\n", pt->entry[l3idx]);
return -EFAULT;
}
pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK);
__kprintf("l2 table: 0x%lX l2idx: %d \n", virt_to_phys(pt), l2idx);
if (!(pt->entry[l2idx] & PFL2_PRESENT)) {
__kprintf("0x%lX l2idx not present! \n", (unsigned long)virt);
__kprintf("l2 entry: 0x%lX\n", pt->entry[l2idx]);
return -EFAULT;
}
if ((pt->entry[l2idx] & PFL2_SIZE)) {
return 0;
}
pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK);
__kprintf("l1 table: 0x%lX l1idx: %d \n", virt_to_phys(pt), l1idx);
if (!(pt->entry[l1idx] & PFL1_PRESENT)) {
__kprintf("0x%lX l1idx not present! \n", (unsigned long)virt);
__kprintf("l1 entry: 0x%lX\n", pt->entry[l1idx]);
return -EFAULT;
}
__kprintf("l1 entry: 0x%lX\n", pt->entry[l1idx]);
return 0;
}
int set_pt_large_page(struct page_table *pt, void *virt, unsigned long phys,
enum ihk_mc_pt_attribute attr)
{
return __set_pt_page(pt, virt, phys, attr | PTATTR_LARGEPAGE
| PTATTR_ACTIVE);
}
int ihk_mc_pt_set_large_page(page_table_t pt, void *virt,
unsigned long phys, enum ihk_mc_pt_attribute attr)
{
return __set_pt_page(pt, virt, phys, attr | PTATTR_LARGEPAGE
| PTATTR_ACTIVE);
}
int ihk_mc_pt_set_page(page_table_t pt, void *virt,
unsigned long phys, enum ihk_mc_pt_attribute attr)
{
return __set_pt_page(pt, virt, phys, attr | PTATTR_ACTIVE);
}
int ihk_mc_pt_prepare_map(page_table_t p, void *virt, unsigned long size,
enum ihk_mc_pt_prepare_flag flag)
{
int l4idx, l4e, ret = 0;
unsigned long v = (unsigned long)virt;
struct page_table *pt = p, *newpt;
unsigned long l;
enum ihk_mc_pt_attribute attr = PTATTR_WRITABLE;
if (!pt) {
pt = init_pt;
}
l4idx = ((v) >> PTL4_SHIFT) & (PT_ENTRIES - 1);
if (flag == IHK_MC_PT_FIRST_LEVEL) {
l4e = ((v + size) >> PTL4_SHIFT) & (PT_ENTRIES - 1);
for (; l4idx <= l4e; l4idx++) {
if (pt->entry[l4idx] & PFL4_PRESENT) {
return 0;
} else {
newpt = __alloc_new_pt(IHK_MC_AP_CRITICAL);
if (!newpt) {
ret = -ENOMEM;
} else {
pt->entry[l4idx] = virt_to_phys(newpt)
| PFL4_PDIR_ATTR;
}
}
}
} else {
/* Call without ACTIVE flag */
l = v + size;
for (; v < l; v += PAGE_SIZE) {
if ((ret = __set_pt_page(pt, (void *)v, 0, attr))) {
break;
}
}
}
return ret;
}
struct page_table *ihk_mc_pt_create(enum ihk_mc_ap_flag ap_flag)
{
struct page_table *pt = ihk_mc_alloc_pages(1, ap_flag);
if(pt == NULL)
return NULL;
memset(pt->entry, 0, PAGE_SIZE);
/* Copy the kernel space */
memcpy(pt->entry + PT_ENTRIES / 2, init_pt->entry + PT_ENTRIES / 2,
sizeof(pt->entry[0]) * PT_ENTRIES / 2);
return pt;
}
static void destroy_page_table(int level, struct page_table *pt)
{
int ix;
unsigned long entry;
struct page_table *lower;
if ((level < 1) || (4 < level)) {
panic("destroy_page_table: level is out of range");
}
if (pt == NULL) {
panic("destroy_page_table: pt is NULL");
}
if (level > 1) {
for (ix = 0; ix < PT_ENTRIES; ++ix) {
entry = pt->entry[ix];
if (!(entry & PF_PRESENT)) {
/* entry is not valid */
continue;
}
if (entry & PF_SIZE) {
/* not a page table */
continue;
}
lower = (struct page_table *)phys_to_virt(entry & PT_PHYSMASK);
destroy_page_table(level-1, lower);
}
}
ihk_mc_free_pages(pt, 1);
return;
}
void ihk_mc_pt_destroy(struct page_table *pt)
{
const int level = 4; /* PML4 */
/* clear shared entry */
memset(pt->entry + PT_ENTRIES / 2, 0, sizeof(pt->entry[0]) * PT_ENTRIES / 2);
destroy_page_table(level, pt);
return;
}
int ihk_mc_pt_clear_page(page_table_t pt, void *virt)
{
return __clear_pt_page(pt, virt, 0);
}
int ihk_mc_pt_clear_large_page(page_table_t pt, void *virt)
{
return __clear_pt_page(pt, virt, 1);
}
typedef int walk_pte_fn_t(void *args, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end);
static int walk_pte_l1(struct page_table *pt, uint64_t base, uint64_t start,
uint64_t end, walk_pte_fn_t *funcp, void *args)
{
int six;
int eix;
int ret;
int i;
int error;
uint64_t off;
six = (start <= base)? 0: ((start - base) >> PTL1_SHIFT);
eix = ((end == 0) || ((base + PTL2_SIZE) <= end))? PT_ENTRIES
: (((end - base) + (PTL1_SIZE - 1)) >> PTL1_SHIFT);
ret = -ENOENT;
for (i = six; i < eix; ++i) {
off = i * PTL1_SIZE;
error = (*funcp)(args, &pt->entry[i], base+off, start, end);
if (!error) {
ret = 0;
}
else if (error != -ENOENT) {
ret = error;
break;
}
}
return ret;
}
static int walk_pte_l2(struct page_table *pt, uint64_t base, uint64_t start,
uint64_t end, walk_pte_fn_t *funcp, void *args)
{
int six;
int eix;
int ret;
int i;
int error;
uint64_t off;
six = (start <= base)? 0: ((start - base) >> PTL2_SHIFT);
eix = ((end == 0) || ((base + PTL3_SIZE) <= end))? PT_ENTRIES
: (((end - base) + (PTL2_SIZE - 1)) >> PTL2_SHIFT);
ret = -ENOENT;
for (i = six; i < eix; ++i) {
off = i * PTL2_SIZE;
error = (*funcp)(args, &pt->entry[i], base+off, start, end);
if (!error) {
ret = 0;
}
else if (error != -ENOENT) {
ret = error;
break;
}
}
return ret;
}
static int walk_pte_l3(struct page_table *pt, uint64_t base, uint64_t start,
uint64_t end, walk_pte_fn_t *funcp, void *args)
{
int six;
int eix;
int ret;
int i;
int error;
uint64_t off;
six = (start <= base)? 0: ((start - base) >> PTL3_SHIFT);
eix = ((end == 0) || ((base + PTL4_SIZE) <= end))? PT_ENTRIES
: (((end - base) + (PTL3_SIZE - 1)) >> PTL3_SHIFT);
ret = -ENOENT;
for (i = six; i < eix; ++i) {
off = i * PTL3_SIZE;
error = (*funcp)(args, &pt->entry[i], base+off, start, end);
if (!error) {
ret = 0;
}
else if (error != -ENOENT) {
ret = error;
break;
}
}
return ret;
}
static int walk_pte_l4(struct page_table *pt, uint64_t base, uint64_t start,
uint64_t end, walk_pte_fn_t *funcp, void *args)
{
int six;
int eix;
int ret;
int i;
int error;
uint64_t off;
six = (start <= base)? 0: ((start - base) >> PTL4_SHIFT);
eix = (end == 0)? PT_ENTRIES
:(((end - base) + (PTL4_SIZE - 1)) >> PTL4_SHIFT);
ret = -ENOENT;
for (i = six; i < eix; ++i) {
off = i * PTL4_SIZE;
error = (*funcp)(args, &pt->entry[i], base+off, start, end);
if (!error) {
ret = 0;
}
else if (error != -ENOENT) {
ret = error;
break;
}
}
return ret;
}
static int split_large_page(pte_t *ptep, size_t pgsize)
{
struct page_table *pt;
uintptr_t phys_base;
int i;
uintptr_t phys;
struct page *page;
pte_t pte;
if ((pgsize != PTL3_SIZE) && (pgsize != PTL2_SIZE)) {
ekprintf("split_large_page:invalid pgsize %#lx\n", pgsize);
return -EINVAL;
}
pt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (pt == NULL) {
ekprintf("split_large_page:__alloc_new_pt failed\n");
return -ENOMEM;
}
pte = *ptep;
if (pgsize == PTL2_SIZE) {
/* break down to basic page size */
pte &= ~PFL2_SIZE;
}
if (pte_is_fileoff(ptep, pgsize)) {
phys_base = NOPHYS;
}
else {
phys_base = pte_get_phys(ptep);
}
for (i = 0; i < PT_ENTRIES; ++i) {
if (phys_base != NOPHYS) {
phys = phys_base + (i * pgsize / PT_ENTRIES);
page = phys_to_page(phys);
if (page) {
page_map(page);
}
}
pt->entry[i] = pte;
pte += pgsize / PT_ENTRIES;
}
*ptep = (virt_to_phys(pt) & PT_PHYSMASK) | PFL2_PDIR_ATTR;
/* Do not do this check for large pages as they don't come from the zeroobj
* and are not actually mapped.
* TODO: clean up zeroobj as we don't really need it, anonymous mappings
* should be allocated for real */
if (pgsize != PTL2_SIZE) {
if (phys_base != NOPHYS) {
page = phys_to_page(phys_base);
if (pgsize != PTL2_SIZE && page && page_unmap(page)) {
kprintf("split_large_page:page_unmap:%p\n", page);
panic("split_large_page:page_unmap\n");
}
}
}
return 0;
}
struct visit_pte_args {
page_table_t pt;
enum visit_pte_flag flags;
int pgshift;
pte_visitor_t *funcp;
void *arg;
};
static int visit_pte_l1(void *arg0, pte_t *ptep, uintptr_t base,
uintptr_t start, uintptr_t end)
{
struct visit_pte_args *args = arg0;
if ((*ptep == PTE_NULL) && (args->flags & VPTEF_SKIP_NULL)) {
return 0;
}
return (*args->funcp)(args->arg, args->pt, ptep, (void *)base,
PTL1_SHIFT);
}
static int visit_pte_l2(void *arg0, pte_t *ptep, uintptr_t base,
uintptr_t start, uintptr_t end)
{
int error;
struct visit_pte_args *args = arg0;
struct page_table *pt;
if ((*ptep == PTE_NULL) && (args->flags & VPTEF_SKIP_NULL)) {
return 0;
}
if (((*ptep == PTE_NULL) || (*ptep & PFL2_SIZE))
&& (start <= base)
&& (((base + PTL2_SIZE) <= end)
|| (end == 0))
&& (!args->pgshift || (args->pgshift == PTL2_SHIFT))) {
error = (*args->funcp)(args->arg, args->pt, ptep,
(void *)base, PTL2_SHIFT);
if (error != -E2BIG) {
return error;
}
}
if (*ptep & PFL2_SIZE) {
ekprintf("visit_pte_l2:split large page\n");
return -ENOMEM;
}
if (*ptep == PTE_NULL) {
pt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (!pt) {
return -ENOMEM;
}
*ptep = virt_to_phys(pt) | PFL2_PDIR_ATTR;
}
else {
pt = phys_to_virt(*ptep & PT_PHYSMASK);
}
error = walk_pte_l1(pt, base, start, end, &visit_pte_l1, arg0);
return error;
}
static int visit_pte_l3(void *arg0, pte_t *ptep, uintptr_t base,
uintptr_t start, uintptr_t end)
{
int error;
struct visit_pte_args *args = arg0;
struct page_table *pt;
if ((*ptep == PTE_NULL) && (args->flags & VPTEF_SKIP_NULL)) {
return 0;
}
if (((*ptep == PTE_NULL) || (*ptep & PFL3_SIZE))
&& (start <= base)
&& (((base + PTL3_SIZE) <= end)
|| (end == 0))
&& (!args->pgshift || (args->pgshift == PTL3_SHIFT))
&& use_1gb_page) {
error = (*args->funcp)(args->arg, args->pt, ptep,
(void *)base, PTL3_SHIFT);
if (error != -E2BIG) {
return error;
}
}
if (*ptep & PFL3_SIZE) {
ekprintf("visit_pte_l3:split large page\n");
return -ENOMEM;
}
if (*ptep == PTE_NULL) {
pt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (!pt) {
return -ENOMEM;
}
*ptep = virt_to_phys(pt) | PFL3_PDIR_ATTR;
}
else {
pt = phys_to_virt(*ptep & PT_PHYSMASK);
}
error = walk_pte_l2(pt, base, start, end, &visit_pte_l2, arg0);
return error;
}
static int visit_pte_l4(void *arg0, pte_t *ptep, uintptr_t base,
uintptr_t start, uintptr_t end)
{
int error;
struct visit_pte_args *args = arg0;
struct page_table *pt;
if ((*ptep == PTE_NULL) && (args->flags & VPTEF_SKIP_NULL)) {
return 0;
}
if (*ptep == PTE_NULL) {
pt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (!pt) {
return -ENOMEM;
}
*ptep = virt_to_phys(pt) | PFL4_PDIR_ATTR;
}
else {
pt = phys_to_virt(*ptep & PT_PHYSMASK);
}
error = walk_pte_l3(pt, base, start, end, &visit_pte_l3, arg0);
return error;
}
int visit_pte_range(page_table_t pt, void *start0, void *end0, int pgshift,
enum visit_pte_flag flags, pte_visitor_t *funcp, void *arg)
{
const uintptr_t start = (uintptr_t)start0;
const uintptr_t end = (uintptr_t)end0;
struct visit_pte_args args;
args.pt = pt;
args.flags = flags;
args.funcp = funcp;
args.arg = arg;
args.pgshift = pgshift;
return walk_pte_l4(pt, 0, start, end, &visit_pte_l4, &args);
}
struct clear_range_args {
int free_physical;
uint8_t padding[4];
struct memobj *memobj;
struct process_vm *vm;
};
static int clear_range_l1(void *args0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct clear_range_args *args = args0;
uint64_t phys;
struct page *page;
pte_t old;
if (*ptep == PTE_NULL) {
return -ENOENT;
}
old = xchg(ptep, PTE_NULL);
remote_flush_tlb_cpumask(args->vm, base, ihk_mc_get_processor_id());
page = NULL;
if (!pte_is_fileoff(&old, PTL1_SIZE)) {
phys = pte_get_phys(&old);
page = phys_to_page(phys);
}
if (page && page_is_in_memobj(page) && (old & PFL1_DIRTY)) {
memobj_flush_page(args->memobj, phys, PTL1_SIZE);
}
if (!(old & PFL1_FILEOFF) && args->free_physical) {
if (page && page_unmap(page)) {
ihk_mc_free_pages(phys_to_virt(phys), 1);
dkprintf("%s: freeing regular page at 0x%lx\n", __FUNCTION__, base);
}
args->vm->currss -= PTL1_SIZE;
}
return 0;
}
static int clear_range_l2(void *args0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct clear_range_args *args = args0;
uint64_t phys;
struct page_table *pt;
int error;
struct page *page;
pte_t old;
if (*ptep == PTE_NULL) {
return -ENOENT;
}
if ((*ptep & PFL2_SIZE)
&& ((base < start) || (end < (base + PTL2_SIZE)))) {
error = -EINVAL;
ekprintf("clear_range_l2(%p,%p,%lx,%lx,%lx):"
"split page. %d\n",
args0, ptep, base, start, end, error);
return error;
}
if (*ptep & PFL2_SIZE) {
old = xchg(ptep, PTE_NULL);
remote_flush_tlb_cpumask(args->vm, base,
ihk_mc_get_processor_id());
page = NULL;
if (!pte_is_fileoff(&old, PTL2_SIZE)) {
phys = pte_get_phys(&old);
page = phys_to_page(phys);
}
if (page && page_is_in_memobj(page) && (old & PFL2_DIRTY)) {
memobj_flush_page(args->memobj, phys, PTL2_SIZE);
}
if (!(old & PFL2_FILEOFF) && args->free_physical) {
if (page && page_unmap(page)) {
ihk_mc_free_pages(phys_to_virt(phys), PTL2_SIZE/PTL1_SIZE);
dkprintf("%s: freeing large page at 0x%lx\n", __FUNCTION__, base);
}
args->vm->currss -= PTL2_SIZE;
}
return 0;
}
pt = phys_to_virt(*ptep & PT_PHYSMASK);
error = walk_pte_l1(pt, base, start, end, &clear_range_l1, args0);
if (error && (error != -ENOENT)) {
return error;
}
if ((start <= base) && ((base + PTL2_SIZE) <= end)) {
*ptep = PTE_NULL;
remote_flush_tlb_cpumask(args->vm, base,
ihk_mc_get_processor_id());
ihk_mc_free_pages(pt, 1);
}
return 0;
}
static int clear_range_l3(void *args0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct clear_range_args *args = args0;
int error;
uint64_t phys;
pte_t old;
struct page *page;
struct page_table *pt;
if (*ptep == PTE_NULL) {
return -ENOENT;
}
if ((*ptep & PFL3_SIZE)
&& ((base < start) || (end < (base + PTL3_SIZE)))) {
error = -EINVAL;
ekprintf("clear_range_l3(%p,%p,%lx,%lx,%lx):"
"split page. %d\n",
args0, ptep, base, start, end, error);
return error;
}
if (*ptep & PFL3_SIZE) {
old = xchg(ptep, PTE_NULL);
remote_flush_tlb_cpumask(args->vm, base,
ihk_mc_get_processor_id());
page = NULL;
if (!pte_is_fileoff(&old, PTL3_SIZE)) {
phys = pte_get_phys(&old);
page = phys_to_page(phys);
}
if (page && page_is_in_memobj(page) && (old & PFL3_DIRTY)) {
memobj_flush_page(args->memobj, phys, PTL3_SIZE);
}
if (!(old & PFL3_FILEOFF) && args->free_physical) {
if (page && page_unmap(page)) {
ihk_mc_free_pages(phys_to_virt(phys), PTL3_SIZE/PTL1_SIZE);
}
args->vm->currss -= PTL3_SIZE;
}
return 0;
}
pt = phys_to_virt(*ptep & PT_PHYSMASK);
error = walk_pte_l2(pt, base, start, end, &clear_range_l2, args0);
if (error && (error != -ENOENT)) {
return error;
}
if (use_1gb_page && (start <= base) && ((base + PTL3_SIZE) <= end)) {
*ptep = PTE_NULL;
remote_flush_tlb_cpumask(args->vm, base,
ihk_mc_get_processor_id());
ihk_mc_free_pages(pt, 1);
}
return 0;
}
static int clear_range_l4(void *args0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct page_table *pt;
if (*ptep == PTE_NULL) {
return -ENOENT;
}
pt = phys_to_virt(*ptep & PT_PHYSMASK);
return walk_pte_l3(pt, base, start, end, &clear_range_l3, args0);
}
static int clear_range(struct page_table *pt, struct process_vm *vm,
uintptr_t start, uintptr_t end, int free_physical,
struct memobj *memobj)
{
int error;
struct clear_range_args args;
if ((start < vm->region.user_start)
|| (vm->region.user_end < end)
|| (end <= start)) {
ekprintf("clear_range(%p,%p,%p,%x):"
"invalid start and/or end.\n",
pt, start, end, free_physical);
return -EINVAL;
}
args.free_physical = free_physical;
args.memobj = memobj;
args.vm = vm;
error = walk_pte_l4(pt, 0, start, end, &clear_range_l4, &args);
return error;
}
int ihk_mc_pt_clear_range(page_table_t pt, struct process_vm *vm,
void *start, void *end)
{
#define KEEP_PHYSICAL 0
return clear_range(pt, vm, (uintptr_t)start, (uintptr_t)end,
KEEP_PHYSICAL, NULL);
}
int ihk_mc_pt_free_range(page_table_t pt, struct process_vm *vm,
void *start, void *end, struct memobj *memobj)
{
#define FREE_PHYSICAL 1
return clear_range(pt, vm, (uintptr_t)start, (uintptr_t)end,
FREE_PHYSICAL, memobj);
}
struct change_attr_args {
pte_t clrpte;
pte_t setpte;
};
static int change_attr_range_l1(void *arg0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct change_attr_args *args = arg0;
if ((*ptep == PTE_NULL) || (*ptep & PFL1_FILEOFF)) {
return -ENOENT;
}
*ptep = (*ptep & ~args->clrpte) | args->setpte;
return 0;
}
static int change_attr_range_l2(void *arg0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct change_attr_args *args = arg0;
int error;
struct page_table *pt;
if ((*ptep == PTE_NULL) || (*ptep & PFL2_FILEOFF)) {
return -ENOENT;
}
if ((*ptep & PFL2_SIZE)
&& ((base < start) || (end < (base + PTL2_SIZE)))) {
error = -EINVAL;
ekprintf("change_attr_range_l2(%p,%p,%lx,%lx,%lx):"
"split page. %d\n",
arg0, ptep, base, start, end, error);
return error;
}
if (*ptep & PFL2_SIZE) {
if (!(*ptep & PFL2_FILEOFF)) {
*ptep = (*ptep & ~args->clrpte) | args->setpte;
}
return 0;
}
pt = phys_to_virt(*ptep & PT_PHYSMASK);
return walk_pte_l1(pt, base, start, end, &change_attr_range_l1, arg0);
}
static int change_attr_range_l3(void *arg0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct change_attr_args *args = arg0;
int error;
struct page_table *pt;
if ((*ptep == PTE_NULL) || (*ptep & PFL3_FILEOFF)) {
return -ENOENT;
}
if ((*ptep & PFL3_SIZE)
&& ((base < start) || (end < (base + PTL3_SIZE)))) {
error = -EINVAL;
ekprintf("change_attr_range_l3(%p,%p,%lx,%lx,%lx):"
"split page. %d\n",
arg0, ptep, base, start, end, error);
return error;
}
if (*ptep & PFL3_SIZE) {
if (!(*ptep & PFL3_FILEOFF)) {
*ptep = (*ptep & ~args->clrpte) | args->setpte;
}
return 0;
}
pt = phys_to_virt(*ptep & PT_PHYSMASK);
return walk_pte_l2(pt, base, start, end, &change_attr_range_l2, arg0);
}
static int change_attr_range_l4(void *arg0, pte_t *ptep, uint64_t base,
uint64_t start, uint64_t end)
{
struct page_table *pt;
if (*ptep == PTE_NULL) {
return -ENOENT;
}
pt = phys_to_virt(*ptep & PT_PHYSMASK);
return walk_pte_l3(pt, base, start, end, &change_attr_range_l3, arg0);
}
int ihk_mc_pt_change_attr_range(page_table_t pt, void *start0, void *end0,
enum ihk_mc_pt_attribute clrattr,
enum ihk_mc_pt_attribute setattr)
{
const intptr_t start = (intptr_t)start0;
const intptr_t end = (intptr_t)end0;
struct change_attr_args args;
args.clrpte = attr_to_l1attr(clrattr);
args.setpte = attr_to_l1attr(setattr);
return walk_pte_l4(pt, 0, start, end, &change_attr_range_l4, &args);
}
static pte_t *lookup_pte(struct page_table *pt, uintptr_t virt, int pgshift,
uintptr_t *basep, size_t *sizep, int *p2alignp)
{
int l4idx, l3idx, l2idx, l1idx;
pte_t *ptep;
uintptr_t base;
size_t size;
int p2align;
GET_VIRT_INDICES(virt, l4idx, l3idx, l2idx, l1idx);
ptep = NULL;
if (!pgshift) {
pgshift = (use_1gb_page)? PTL3_SHIFT: PTL2_SHIFT;
}
if (pt->entry[l4idx] == PTE_NULL) {
if (pgshift > PTL3_SHIFT) {
pgshift = PTL3_SHIFT;
}
goto out;
}
pt = phys_to_virt(pte_get_phys(&pt->entry[l4idx]));
if ((pt->entry[l3idx] == PTE_NULL)
|| (pt->entry[l3idx] & PFL3_SIZE)) {
if (pgshift >= PTL3_SHIFT) {
ptep = &pt->entry[l3idx];
pgshift = PTL3_SHIFT;
}
goto out;
}
pt = phys_to_virt(pte_get_phys(&pt->entry[l3idx]));
if ((pt->entry[l2idx] == PTE_NULL)
|| (pt->entry[l2idx] & PFL2_SIZE)) {
if (pgshift >= PTL2_SHIFT) {
ptep = &pt->entry[l2idx];
pgshift = PTL2_SHIFT;
}
goto out;
}
pt = phys_to_virt(pte_get_phys(&pt->entry[l2idx]));
ptep = &pt->entry[l1idx];
pgshift = PTL1_SHIFT;
out:
size = (size_t)1 << pgshift;
base = virt & ~(size - 1);
p2align = pgshift - PAGE_SHIFT;
if (basep) *basep = base;
if (sizep) *sizep = size;
if (p2alignp) *p2alignp = p2align;
return ptep;
}
pte_t *ihk_mc_pt_lookup_pte(page_table_t pt, void *virt, int pgshift,
void **basep, size_t *sizep, int *p2alignp)
{
pte_t *ptep;
uintptr_t base;
size_t size;
int p2align;
dkprintf("ihk_mc_pt_lookup_pte(%p,%p,%d)\n", pt, virt, pgshift);
ptep = lookup_pte(pt, (uintptr_t)virt, pgshift, &base, &size, &p2align);
if (basep) *basep = (void *)base;
if (sizep) *sizep = size;
if (p2alignp) *p2alignp = p2align;
dkprintf("ihk_mc_pt_lookup_pte(%p,%p,%d): %p %lx %lx %d\n",
pt, virt, pgshift, ptep, base, size, p2align);
return ptep;
}
struct set_range_args {
page_table_t pt;
uintptr_t phys;
enum ihk_mc_pt_attribute attr;
int pgshift;
uintptr_t diff;
struct process_vm *vm;
};
int set_range_l1(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
uintptr_t end)
{
struct set_range_args *args = args0;
int error;
uintptr_t phys;
dkprintf("set_range_l1(%lx,%lx,%lx)\n", base, start, end);
if (*ptep != PTE_NULL) {
error = -EBUSY;
ekprintf("set_range_l1(%lx,%lx,%lx):page exists. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, args->vm, start, base, KEEP_PHYSICAL, NULL);
goto out;
}
phys = args->phys + (base - start);
*ptep = phys | attr_to_l1attr(args->attr);
error = 0;
out:
dkprintf("set_range_l1(%lx,%lx,%lx): %d %lx\n",
base, start, end, error, *ptep);
return error;
}
int set_range_l2(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
uintptr_t end)
{
struct set_range_args *args = args0;
int error;
struct page_table *pt;
uintptr_t phys;
struct page_table *newpt = NULL;
pte_t pte;
dkprintf("set_range_l2(%lx,%lx,%lx)\n", base, start, end);
retry:
if (*ptep == PTE_NULL) {
if ((start <= base) && ((base + PTL2_SIZE) <= end)
&& ((args->diff & (PTL2_SIZE - 1)) == 0)
&& (!args->pgshift
|| (args->pgshift == PTL2_SHIFT))) {
phys = args->phys + (base - start);
*ptep = phys | attr_to_l2attr(
args->attr|PTATTR_LARGEPAGE);
error = 0;
dkprintf("set_range_l2(%lx,%lx,%lx):"
"2MiB page. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
if (!newpt) {
newpt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (newpt == NULL) {
error = -ENOMEM;
ekprintf("set_range_l2(%lx,%lx,%lx):"
"__alloc_new_pt failed. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, args->vm, start, base,
KEEP_PHYSICAL, NULL);
goto out;
}
}
pte = virt_to_phys(newpt) | PFL2_PDIR_ATTR;
pte = atomic_cmpxchg8(ptep, PTE_NULL, pte);
if (pte != PTE_NULL) {
/* failed to set PDTe */
goto retry;
}
pt = newpt;
newpt = NULL;
}
else if (*ptep & PFL2_SIZE) {
error = -EBUSY;
ekprintf("set_range_l2(%lx,%lx,%lx):"
"page exists. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, args->vm, start, base, KEEP_PHYSICAL, NULL);
goto out;
}
else {
pt = phys_to_virt(*ptep & PT_PHYSMASK);
}
error = walk_pte_l1(pt, base, start, end, &set_range_l1, args0);
if (error) {
ekprintf("set_range_l2(%lx,%lx,%lx):"
"walk_pte_l1 failed. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
error = 0;
out:
if (newpt) {
ihk_mc_free_pages(newpt, 1);
}
dkprintf("set_range_l2(%lx,%lx,%lx): %d %lx\n",
base, start, end, error, *ptep);
return error;
}
int set_range_l3(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
uintptr_t end)
{
struct page_table *newpt = NULL;
pte_t pte;
struct page_table *pt;
int error;
struct set_range_args *args = args0;
uintptr_t phys;
dkprintf("set_range_l3(%lx,%lx,%lx)\n", base, start, end);
retry:
if (*ptep == PTE_NULL) {
if ((start <= base) && ((base + PTL3_SIZE) <= end)
&& ((args->diff & (PTL3_SIZE - 1)) == 0)
&& (!args->pgshift
|| (args->pgshift == PTL3_SHIFT))
&& use_1gb_page) {
phys = args->phys + (base - start);
*ptep = phys | attr_to_l3attr(
args->attr|PTATTR_LARGEPAGE);
error = 0;
dkprintf("set_range_l3(%lx,%lx,%lx):"
"1GiB page. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
if (!newpt) {
newpt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (newpt == NULL) {
error = -ENOMEM;
ekprintf("set_range_l3(%lx,%lx,%lx):"
"__alloc_new_pt failed. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, args->vm, start,
base, KEEP_PHYSICAL, NULL);
goto out;
}
}
pte = virt_to_phys(newpt) | PFL3_PDIR_ATTR;
pte = atomic_cmpxchg8(ptep, PTE_NULL, pte);
if (pte != PTE_NULL) {
/* failed to set PDPTe */
goto retry;
}
pt = newpt;
newpt = NULL;
}
else if (*ptep & PFL3_SIZE) {
error = -EBUSY;
ekprintf("set_range_l3(%lx,%lx,%lx):"
"page exists. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, args->vm, start, base,
KEEP_PHYSICAL, NULL);
goto out;
}
else {
pt = phys_to_virt(*ptep & PT_PHYSMASK);
}
error = walk_pte_l2(pt, base, start, end, &set_range_l2, args0);
if (error) {
ekprintf("set_range_l3(%lx,%lx,%lx):"
"walk_pte_l2 failed. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
error = 0;
out:
if (newpt) {
ihk_mc_free_pages(newpt, 1);
}
dkprintf("set_range_l3(%lx,%lx,%lx): %d\n",
base, start, end, error, *ptep);
return error;
}
int set_range_l4(void *args0, pte_t *ptep, uintptr_t base, uintptr_t start,
uintptr_t end)
{
struct set_range_args *args = args0;
struct page_table *newpt = NULL;
pte_t pte;
struct page_table *pt;
int error;
dkprintf("set_range_l4(%lx,%lx,%lx)\n", base, start, end);
retry:
if (*ptep == PTE_NULL) {
if (!newpt) {
newpt = __alloc_new_pt(IHK_MC_AP_NOWAIT);
if (newpt == NULL) {
error = -ENOMEM;
ekprintf("set_range_l4(%lx,%lx,%lx):"
"__alloc_new_pt failed. %d %lx\n",
base, start, end, error, *ptep);
(void)clear_range(args->pt, args->vm, start,
base, KEEP_PHYSICAL, NULL);
goto out;
}
}
pte = virt_to_phys(newpt) | PFL4_PDIR_ATTR;
pte = atomic_cmpxchg8(ptep, PTE_NULL, pte);
if (pte != PTE_NULL) {
/* failed to set PML4e */
goto retry;
}
pt = newpt;
newpt = NULL;
}
else {
pt = phys_to_virt(*ptep & PT_PHYSMASK);
}
error = walk_pte_l3(pt, base, start, end, &set_range_l3, args0);
if (error) {
ekprintf("set_range_l4(%lx,%lx,%lx):"
"walk_pte_l3 failed. %d %lx\n",
base, start, end, error, *ptep);
goto out;
}
error = 0;
out:
if (newpt) {
ihk_mc_free_pages(newpt, 1);
}
dkprintf("set_range_l4(%lx,%lx,%lx): %d %lx\n",
base, start, end, error, *ptep);
return error;
}
int ihk_mc_pt_set_range(page_table_t pt, struct process_vm *vm, void *start,
void *end, uintptr_t phys, enum ihk_mc_pt_attribute attr,
int pgshift)
{
int error;
struct set_range_args args;
dkprintf("ihk_mc_pt_set_range(%p,%p,%p,%lx,%x)\n",
pt, start, end, phys, attr);
args.pt = pt;
args.phys = phys;
args.attr = attr;
args.diff = (uintptr_t)start ^ phys;
args.vm = vm;
args.pgshift = pgshift;
error = walk_pte_l4(pt, 0, (uintptr_t)start, (uintptr_t)end,
&set_range_l4, &args);
if (error) {
ekprintf("ihk_mc_pt_set_range(%p,%p,%p,%lx,%x):"
"walk_pte_l4 failed. %d\n",
pt, start, end, phys, attr, error);
goto out;
}
error = 0;
out:
dkprintf("ihk_mc_pt_set_range(%p,%p,%p,%lx,%x): %d\n",
pt, start, end, phys, attr, error);
return error;
}
int ihk_mc_pt_set_pte(page_table_t pt, pte_t *ptep, size_t pgsize,
uintptr_t phys, enum ihk_mc_pt_attribute attr)
{
int error;
dkprintf("ihk_mc_pt_set_pte(%p,%p,%lx,%lx,%x)\n",
pt, ptep, pgsize, phys, attr);
if (pgsize == PTL1_SIZE) {
*ptep = phys | attr_to_l1attr(attr);
}
else if (pgsize == PTL2_SIZE) {
*ptep = phys | attr_to_l2attr(attr | PTATTR_LARGEPAGE);
}
else if ((pgsize == PTL3_SIZE) && (use_1gb_page)) {
*ptep = phys | attr_to_l3attr(attr | PTATTR_LARGEPAGE);
}
else {
error = -EINVAL;
ekprintf("ihk_mc_pt_set_pte(%p,%p,%lx,%lx,%x):"
"page size. %d %lx\n",
pt, ptep, pgsize, phys, attr, error, *ptep);
panic("ihk_mc_pt_set_pte:page size");
goto out;
}
error = 0;
out:
dkprintf("ihk_mc_pt_set_pte(%p,%p,%lx,%lx,%x): %d %lx\n",
pt, ptep, pgsize, phys, attr, error, *ptep);
return error;
}
int ihk_mc_pt_split(page_table_t pt, struct process_vm *vm, void *addr)
{
int error;
pte_t *ptep;
void *pgaddr;
size_t pgsize;
intptr_t phys;
struct page *page;
retry:
ptep = ihk_mc_pt_lookup_pte(pt, addr, 0, &pgaddr, &pgsize, NULL);
if (ptep && !pte_is_null(ptep) && (pgaddr != addr)) {
page = NULL;
if (!pte_is_fileoff(ptep, pgsize)) {
phys = pte_get_phys(ptep);
page = phys_to_page(phys);
}
if (page && (page_is_in_memobj(page)
|| page_is_multi_mapped(page))) {
error = -EINVAL;
kprintf("ihk_mc_pt_split:NYI:page break down\n");
goto out;
}
error = split_large_page(ptep, pgsize);
if (error) {
kprintf("ihk_mc_pt_split:split_large_page failed. %d\n", error);
goto out;
}
remote_flush_tlb_cpumask(vm, (intptr_t)pgaddr,
ihk_mc_get_processor_id());
goto retry;
}
error = 0;
out:
return error;
} /* ihk_mc_pt_split() */
int arch_get_smaller_page_size(void *args, size_t cursize, size_t *newsizep,
int *p2alignp)
{
size_t newsize;
int p2align;
int error;
if (0) {
/* dummy */
panic("not reached");
}
else if ((cursize > PTL3_SIZE) && use_1gb_page) {
/* 1GiB */
newsize = PTL3_SIZE;
p2align = PTL3_SHIFT - PTL1_SHIFT;
}
else if (cursize > PTL2_SIZE) {
/* 2MiB */
newsize = PTL2_SIZE;
p2align = PTL2_SHIFT - PTL1_SHIFT;
}
else if (cursize > PTL1_SIZE) {
/* 4KiB : basic page size */
newsize = PTL1_SIZE;
p2align = PTL1_SHIFT - PTL1_SHIFT;
}
else {
error = -ENOMEM;
newsize = 0;
p2align = -1;
goto out;
}
error = 0;
if (newsizep) *newsizep = newsize;
if (p2alignp) *p2alignp = p2align;
out:
dkprintf("arch_get_smaller_page_size(%p,%lx): %d %lx %d\n",
args, cursize, error, newsize, p2align);
return error;
}
enum ihk_mc_pt_attribute arch_vrflag_to_ptattr(unsigned long flag, uint64_t fault, pte_t *ptep)
{
enum ihk_mc_pt_attribute attr;
attr = common_vrflag_to_ptattr(flag, fault, ptep);
if ((fault & PF_PROT)
|| ((fault & (PF_POPULATE | PF_PATCH))
&& (flag & VR_PRIVATE))) {
attr |= PTATTR_DIRTY;
}
return attr;
}
struct move_args {
uintptr_t src;
uintptr_t dest;
struct process_vm *vm;
};
static int move_one_page(void *arg0, page_table_t pt, pte_t *ptep,
void *pgaddr, int pgshift)
{
int error;
struct move_args *args = arg0;
const size_t pgsize = (size_t)1 << pgshift;
uintptr_t dest;
pte_t apte;
uintptr_t phys;
enum ihk_mc_pt_attribute attr;
dkprintf("move_one_page(%p,%p,%p %#lx,%p,%d)\n",
arg0, pt, ptep, *ptep, pgaddr, pgshift);
if (pte_is_fileoff(ptep, pgsize)) {
error = -ENOTSUPP;
kprintf("move_one_page(%p,%p,%p %#lx,%p,%d):fileoff. %d\n",
arg0, pt, ptep, *ptep, pgaddr, pgshift, error);
goto out;
}
dest = args->dest + ((uintptr_t)pgaddr - args->src);
apte = PTE_NULL;
pte_xchg(ptep, &apte);
phys = apte & PT_PHYSMASK;
attr = apte & ~PT_PHYSMASK;
error = ihk_mc_pt_set_range(pt, args->vm, (void *)dest,
(void *)(dest + pgsize), phys, attr, pgshift);
if (error) {
kprintf("move_one_page(%p,%p,%p %#lx,%p,%d):"
"set failed. %d\n",
arg0, pt, ptep, *ptep, pgaddr, pgshift, error);
goto out;
}
error = 0;
out:
dkprintf("move_one_page(%p,%p,%p %#lx,%p,%d):%d\n",
arg0, pt, ptep, *ptep, pgaddr, pgshift, error);
return error;
}
int move_pte_range(page_table_t pt, struct process_vm *vm,
void *src, void *dest, size_t size)
{
int error;
struct move_args args;
dkprintf("move_pte_range(%p,%p,%p,%#lx)\n", pt, src, dest, size);
args.src = (uintptr_t)src;
args.dest = (uintptr_t)dest;
args.vm = vm;
error = visit_pte_range(pt, src, src+size, 0, VPTEF_SKIP_NULL,
&move_one_page, &args);
flush_tlb(); /* XXX: TLB flush */
if (error) {
goto out;
}
error = 0;
out:
dkprintf("move_pte_range(%p,%p,%p,%#lx):%d\n",
pt, src, dest, size, error);
return error;
}
void load_page_table(struct page_table *pt)
{
unsigned long pt_addr;
if (!pt) {
pt = init_pt;
}
pt_addr = virt_to_phys(pt);
asm volatile ("movq %0, %%cr3" : : "r"(pt_addr) : "memory");
}
void ihk_mc_load_page_table(struct page_table *pt)
{
load_page_table(pt);
}
struct page_table *get_init_page_table(void)
{
return init_pt;
}
static unsigned long fixed_virt;
static void init_fixed_area(struct page_table *pt)
{
fixed_virt = MAP_FIXED_START;
return;
}
void init_text_area(struct page_table *pt)
{
unsigned long __end, phys, virt;
int i, nlpages;
__end = ((unsigned long)_end + LARGE_PAGE_SIZE * 2 - 1)
& LARGE_PAGE_MASK;
nlpages = (__end - MAP_KERNEL_START) >> LARGE_PAGE_SHIFT;
kprintf("TEXT: # of large pages = %d\n", nlpages);
kprintf("TEXT: Base address = %lx\n", x86_kernel_phys_base);
phys = x86_kernel_phys_base;
virt = MAP_KERNEL_START;
for (i = 0; i < nlpages; i++) {
set_pt_large_page(pt, (void *)virt, phys, PTATTR_WRITABLE);
virt += LARGE_PAGE_SIZE;
phys += LARGE_PAGE_SIZE;
}
}
void *map_fixed_area(unsigned long phys, unsigned long size, int uncachable)
{
unsigned long poffset, paligned;
int i, npages;
void *v = (void *)fixed_virt;
enum ihk_mc_pt_attribute attr;
poffset = phys & (PAGE_SIZE - 1);
paligned = phys & PAGE_MASK;
npages = (poffset + size + PAGE_SIZE - 1) >> PAGE_SHIFT;
attr = PTATTR_WRITABLE | PTATTR_ACTIVE;
#if 0 /* In the case of LAPIC MMIO, something will happen */
attr |= PTATTR_NO_EXECUTE;
#endif
if (uncachable) {
attr |= PTATTR_UNCACHABLE;
}
kprintf("map_fixed: %lx => %p (%d pages)\n", paligned, v, npages);
for (i = 0; i < npages; i++) {
if(__set_pt_page(init_pt, (void *)fixed_virt, paligned, attr)){
return NULL;
}
fixed_virt += PAGE_SIZE;
paligned += PAGE_SIZE;
}
flush_tlb();
return (char *)v + poffset;
}
void init_low_area(struct page_table *pt)
{
set_pt_large_page(pt, 0, 0, PTATTR_NO_EXECUTE|PTATTR_WRITABLE);
}
static void init_vsyscall_area(struct page_table *pt)
{
extern char vsyscall_page[];
int error;
#define VSYSCALL_ADDR ((void *)(0xffffffffff600000))
error = __set_pt_page(pt, VSYSCALL_ADDR,
virt_to_phys(vsyscall_page), PTATTR_ACTIVE|PTATTR_USER);
if (error) {
panic("init_vsyscall_area:__set_pt_page failed");
}
return;
}
void init_page_table(void)
{
check_available_page_size();
init_pt = ihk_mc_alloc_pages(1, IHK_MC_AP_CRITICAL);
ihk_mc_spinlock_init(&init_pt_lock);
memset(init_pt, 0, sizeof(PAGE_SIZE));
/* Normal memory area */
init_normal_area(init_pt);
init_fixed_area(init_pt);
init_low_area(init_pt);
init_text_area(init_pt);
init_vsyscall_area(init_pt);
load_page_table(init_pt);
kprintf("Page table is now at %p\n", init_pt);
}
extern void __reserve_arch_pages(unsigned long, unsigned long,
void (*)(struct ihk_page_allocator_desc *,
unsigned long, unsigned long, int));
void ihk_mc_reserve_arch_pages(struct ihk_page_allocator_desc *pa_allocator,
unsigned long start, unsigned long end,
void (*cb)(struct ihk_page_allocator_desc *,
unsigned long, unsigned long, int))
{
/* Reserve Text + temporal heap */
cb(pa_allocator, virt_to_phys(_head), virt_to_phys(get_last_early_heap()), 0);
/* Reserve trampoline area to boot the second ap */
cb(pa_allocator, ap_trampoline, ap_trampoline + AP_TRAMPOLINE_SIZE, 0);
/* Reserve the null page */
cb(pa_allocator, 0, PAGE_SIZE, 0);
/*
* Micro-arch specific
* TODO: this does nothing in SMP mode, update it for KNC if necessary
*/
__reserve_arch_pages(start, end, cb);
}
unsigned long virt_to_phys(void *v)
{
unsigned long va = (unsigned long)v;
if (va >= MAP_KERNEL_START) {
return va - MAP_KERNEL_START + x86_kernel_phys_base;
} else {
return va - MAP_ST_START;
}
}
void *phys_to_virt(unsigned long p)
{
return (void *)(p + MAP_ST_START);
}
int copy_from_user(void *dst, const void *src, size_t siz)
{
struct process_vm *vm = cpu_local_var(current)->vm;
return read_process_vm(vm, dst, src, siz);
}
int strlen_user(const char *s)
{
struct process_vm *vm = cpu_local_var(current)->vm;
struct vm_range *range;
unsigned long pgstart;
int maxlen;
const char *head = s;
maxlen = 4096 - (((unsigned long)s) & 0x0000000000000fffUL);
pgstart = ((unsigned long)s) & 0xfffffffffffff000UL;
if(!pgstart || pgstart >= MAP_KERNEL_START)
return -EFAULT;
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
for(;;){
range = lookup_process_memory_range(vm, pgstart, pgstart+1);
if(range == NULL){
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
return -EFAULT;
}
if((range->flag & VR_PROT_MASK) == VR_PROT_NONE){
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
return -EFAULT;
}
while(*s && maxlen > 0){
s++;
maxlen--;
}
if(!*s)
break;
maxlen = 4096;
pgstart += 4096;
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
return s - head;
}
int strcpy_from_user(char *dst, const char *src)
{
struct process_vm *vm = cpu_local_var(current)->vm;
struct vm_range *range;
unsigned long pgstart;
int maxlen;
int err = 0;
maxlen = 4096 - (((unsigned long)src) & 0x0000000000000fffUL);
pgstart = ((unsigned long)src) & 0xfffffffffffff000UL;
if(!pgstart || pgstart >= MAP_KERNEL_START)
return -EFAULT;
ihk_mc_spinlock_lock_noirq(&vm->memory_range_lock);
for(;;){
range = lookup_process_memory_range(vm, pgstart, pgstart + 1);
if(range == NULL){
err = -EFAULT;
break;
}
if((range->flag & VR_PROT_MASK) == VR_PROT_NONE){
err = -EFAULT;
break;
}
while(*src && maxlen > 0){
*(dst++) = *(src++);
maxlen--;
}
if(!*src){
*dst = '\0';
break;
}
maxlen = 4096;
pgstart += 4096;
}
ihk_mc_spinlock_unlock_noirq(&vm->memory_range_lock);
return err;
}
long getlong_user(const long *p)
{
int error;
long l;
error = copy_from_user(&l, p, sizeof(l));
if (error) {
return error;
}
return l;
}
int getint_user(const int *p)
{
int error;
int i;
error = copy_from_user(&i, p, sizeof(i));
if (error) {
return error;
}
return i;
}
int read_process_vm(struct process_vm *vm, void *kdst, const void *usrc, size_t siz)
{
const uintptr_t ustart = (uintptr_t)usrc;
const uintptr_t uend = ustart + siz;
uint64_t reason;
uintptr_t addr;
int error;
const void *from;
void *to;
size_t remain;
size_t cpsize;
unsigned long pa;
void *va;
if ((ustart < vm->region.user_start)
|| (vm->region.user_end <= ustart)
|| ((vm->region.user_end - ustart) < siz)) {
kprintf("%s: error: out of user range\n", __FUNCTION__);
return -EFAULT;
}
reason = PF_USER; /* page not present */
for (addr = ustart & PAGE_MASK; addr < uend; addr += PAGE_SIZE) {
if (!addr)
return -EINVAL;
error = page_fault_process_vm(vm, (void *)addr, reason);
if (error) {
kprintf("%s: error: PF for %p failed\n", __FUNCTION__, addr);
return error;
}
}
from = usrc;
to = kdst;
remain = siz;
while (remain > 0) {
cpsize = PAGE_SIZE - ((uintptr_t)from & (PAGE_SIZE - 1));
if (cpsize > remain) {
cpsize = remain;
}
error = ihk_mc_pt_virt_to_phys(vm->address_space->page_table, from, &pa);
if (error) {
kprintf("%s: error: resolving physical address or %p\n", __FUNCTION__, from);
return error;
}
if (pa < ihk_mc_get_memory_address(IHK_MC_GMA_MAP_START, 0) ||
pa >= ihk_mc_get_memory_address(IHK_MC_GMA_MAP_END, 0)) {
dkprintf("%s: pa is outside of LWK memory, to: %p, pa: %p,"
"cpsize: %d\n", __FUNCTION__, to, pa, cpsize);
va = ihk_mc_map_virtual(pa, 1, PTATTR_ACTIVE);
memcpy(to, va, cpsize);
ihk_mc_unmap_virtual(va, 1, 1);
}
else {
va = phys_to_virt(pa);
memcpy(to, va, cpsize);
}
from += cpsize;
to += cpsize;
remain -= cpsize;
}
return 0;
} /* read_process_vm() */
int copy_to_user(void *dst, const void *src, size_t siz)
{
struct process_vm *vm = cpu_local_var(current)->vm;
return write_process_vm(vm, dst, src, siz);
}
int setlong_user(long *dst, long data)
{
return copy_to_user(dst, &data, sizeof(data));
}
int setint_user(int *dst, int data)
{
return copy_to_user(dst, &data, sizeof(data));
}
int write_process_vm(struct process_vm *vm, void *udst, const void *ksrc, size_t siz)
{
const uintptr_t ustart = (uintptr_t)udst;
const uintptr_t uend = ustart + siz;
uint64_t reason;
uintptr_t addr;
int error;
const void *from;
void *to;
size_t remain;
size_t cpsize;
unsigned long pa;
void *va;
if ((ustart < vm->region.user_start)
|| (vm->region.user_end <= ustart)
|| ((vm->region.user_end - ustart) < siz)) {
return -EFAULT;
}
reason = PF_POPULATE | PF_WRITE | PF_USER;
for (addr = ustart & PAGE_MASK; addr < uend; addr += PAGE_SIZE) {
error = page_fault_process_vm(vm, (void *)addr, reason);
if (error) {
return error;
}
}
from = ksrc;
to = udst;
remain = siz;
while (remain > 0) {
cpsize = PAGE_SIZE - ((uintptr_t)to & (PAGE_SIZE - 1));
if (cpsize > remain) {
cpsize = remain;
}
error = ihk_mc_pt_virt_to_phys(vm->address_space->page_table, to, &pa);
if (error) {
return error;
}
va = phys_to_virt(pa);
memcpy(va, from, cpsize);
from += cpsize;
to += cpsize;
remain -= cpsize;
}
return 0;
} /* write_process_vm() */
int patch_process_vm(struct process_vm *vm, void *udst, const void *ksrc, size_t siz)
{
const uintptr_t ustart = (uintptr_t)udst;
const uintptr_t uend = ustart + siz;
uint64_t reason;
uintptr_t addr;
int error;
const void *from;
void *to;
size_t remain;
size_t cpsize;
unsigned long pa;
void *va;
kprintf("patch_process_vm(%p,%p,%p,%lx)\n", vm, udst, ksrc, siz);
if ((ustart < vm->region.user_start)
|| (vm->region.user_end <= ustart)
|| ((vm->region.user_end - ustart) < siz)) {
kprintf("patch_process_vm(%p,%p,%p,%lx):not in user\n", vm, udst, ksrc, siz);
return -EFAULT;
}
reason = PF_PATCH | PF_WRITE | PF_USER;
for (addr = ustart & PAGE_MASK; addr < uend; addr += PAGE_SIZE) {
error = page_fault_process_vm(vm, (void *)addr, reason);
if (error) {
kprintf("patch_process_vm(%p,%p,%p,%lx):pf(%lx):%d\n", vm, udst, ksrc, siz, addr, error);
return error;
}
}
from = ksrc;
to = udst;
remain = siz;
while (remain > 0) {
cpsize = PAGE_SIZE - ((uintptr_t)to & (PAGE_SIZE - 1));
if (cpsize > remain) {
cpsize = remain;
}
error = ihk_mc_pt_virt_to_phys(vm->address_space->page_table, to, &pa);
if (error) {
kprintf("patch_process_vm(%p,%p,%p,%lx):v2p(%p):%d\n", vm, udst, ksrc, siz, to, error);
return error;
}
if (pa < ihk_mc_get_memory_address(IHK_MC_GMA_MAP_START, 0) ||
pa >= ihk_mc_get_memory_address(IHK_MC_GMA_MAP_END, 0)) {
dkprintf("%s: pa is outside of LWK memory, from: %p,"
"pa: %p, cpsize: %d\n", __FUNCTION__, from, pa, cpsize);
va = ihk_mc_map_virtual(pa, 1, PTATTR_ACTIVE);
memcpy(va, from, cpsize);
ihk_mc_unmap_virtual(va, 1, 1);
}
else {
va = phys_to_virt(pa);
memcpy(va, from, cpsize);
}
from += cpsize;
to += cpsize;
remain -= cpsize;
}
kprintf("patch_process_vm(%p,%p,%p,%lx):%d\n", vm, udst, ksrc, siz, 0);
return 0;
} /* patch_process_vm() */
Reference in New Issue View Git Blame Copy Permalink