/* arch-memory.h COPYRIGHT FUJITSU LIMITED 2015-2018 */
#ifndef __HEADER_ARM64_COMMON_ARCH_MEMORY_H
#define __HEADER_ARM64_COMMON_ARCH_MEMORY_H

#include <const.h>
#include <errno.h>

#ifndef __ASSEMBLY__
#include <list.h>
#include <page.h>
void panic(const char *);
#endif  /*__ASSEMBLY__*/

#define _SZ4KB  (1UL<<12)
#define _SZ16KB (1UL<<14)
#define _SZ64KB (1UL<<16)

#ifdef CONFIG_ARM64_64K_PAGES
# define GRANULE_SIZE	_SZ64KB
# define BLOCK_SHIFT	PAGE_SHIFT
# define BLOCK_SIZE	PAGE_SIZE
# define TABLE_SHIFT	PMD_SHIFT
#else
# define GRANULE_SIZE	_SZ4KB
# define BLOCK_SHIFT	SECTION_SHIFT
# define BLOCK_SIZE	SECTION_SIZE
# define TABLE_SHIFT	PUD_SHIFT
#endif

#define VA_BITS		CONFIG_ARM64_VA_BITS

/*
 * Address define
 */
/* early alloc area address */
/* START:_end, SIZE:512 pages */
#define MAP_EARLY_ALLOC_SHIFT	9
#define MAP_EARLY_ALLOC_SIZE	(UL(1) << (PAGE_SHIFT + MAP_EARLY_ALLOC_SHIFT))

#ifndef __ASSEMBLY__
# define ALIGN_UP(x, align)     ALIGN_DOWN((x) + (align) - 1, align)
# define ALIGN_DOWN(x, align)   ((x) & ~((align) - 1))
extern char _end[];
# define MAP_EARLY_ALLOC	(ALIGN_UP((unsigned long)_end, BLOCK_SIZE))
# define MAP_EARLY_ALLOC_END	(MAP_EARLY_ALLOC + MAP_EARLY_ALLOC_SIZE)
#endif /* !__ASSEMBLY__ */

/* bootparam area address */
/* START:early alloc area end, SIZE:2MiB */
#define MAP_BOOT_PARAM_SHIFT	21
#define MAP_BOOT_PARAM_SIZE	(UL(1) << MAP_BOOT_PARAM_SHIFT)

#ifndef __ASSEMBLY__
# define MAP_BOOT_PARAM		(ALIGN_UP(MAP_EARLY_ALLOC_END, BLOCK_SIZE))
# define MAP_BOOT_PARAM_END	(MAP_BOOT_PARAM + MAP_BOOT_PARAM_SIZE)
#endif /* !__ASSEMBLY__ */

#if (VA_BITS == 39 && GRANULE_SIZE == _SZ4KB)
#
# define LD_TASK_UNMAPPED_BASE	UL(0x0000000400000000)
# define TASK_UNMAPPED_BASE	UL(0x0000000800000000)
# define USER_END		UL(0x0000002000000000)
# define MAP_VMAP_START		UL(0xffffffbdc0000000)
# define MAP_VMAP_SIZE		UL(0x0000000100000000)
# define MAP_FIXED_START	UL(0xffffffbffbdfd000)
# define MAP_ST_START		UL(0xffffffc000000000)
# define MAP_KERNEL_START	UL(0xffffffffff800000)
#
#elif (VA_BITS == 42 && GRANULE_SIZE == _SZ64KB)
#
# define LD_TASK_UNMAPPED_BASE	UL(0x0000002000000000)
# define TASK_UNMAPPED_BASE	UL(0x0000004000000000)
# define USER_END		UL(0x0000010000000000)
# define MAP_VMAP_START		UL(0xfffffdfee0000000)
# define MAP_VMAP_SIZE		UL(0x0000000100000000)
# define MAP_FIXED_START	UL(0xfffffdfffbdd0000)
# define MAP_ST_START		UL(0xfffffe0000000000)
# define MAP_KERNEL_START	UL(0xffffffffe0000000)
#
#elif (VA_BITS == 48 && GRANULE_SIZE == _SZ4KB)
#
# define LD_TASK_UNMAPPED_BASE	UL(0x0000080000000000)
# define TASK_UNMAPPED_BASE	UL(0x0000100000000000)
# define USER_END		UL(0x0000400000000000)
# define MAP_VMAP_START		UL(0xffff7bffc0000000)
# define MAP_VMAP_SIZE		UL(0x0000000100000000)
# define MAP_FIXED_START	UL(0xffff7ffffbdfd000)
# define MAP_ST_START		UL(0xffff800000000000)
# define MAP_KERNEL_START	UL(0xffffffffff800000)
#
#elif (VA_BITS == 48 && GRANULE_SIZE == _SZ64KB)
#
# define LD_TASK_UNMAPPED_BASE	UL(0x0000080000000000)
# define TASK_UNMAPPED_BASE	UL(0x0000100000000000)
# define USER_END		UL(0x0000400000000000)
# define MAP_VMAP_START		UL(0xffff780000000000)
# define MAP_VMAP_SIZE		UL(0x0000000100000000)
# define MAP_FIXED_START	UL(0xffff7ffffbdd0000)
# define MAP_ST_START		UL(0xffff800000000000)
# define MAP_KERNEL_START	UL(0xffffffffe0000000)
#
#else
# error address space is not defined.
#endif

#define MAP_ST_SIZE		(MAP_KERNEL_START - MAP_ST_START)
#define STACK_TOP(region)	((region)->user_end)

/*
 * pagetable define
 */
#if GRANULE_SIZE == _SZ4KB
# define __PTL4_SHIFT  39
# define __PTL3_SHIFT  30
# define __PTL2_SHIFT  21
# define __PTL1_SHIFT  12
# define PTL4_INDEX_MASK ((UL(1) << 9) - 1)
# define PTL3_INDEX_MASK PTL4_INDEX_MASK
# define PTL2_INDEX_MASK PTL3_INDEX_MASK
# define PTL1_INDEX_MASK PTL2_INDEX_MASK
# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 4)
# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 4)
# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 4)
#elif GRANULE_SIZE == _SZ16KB
# define __PTL4_SHIFT  47
# define __PTL3_SHIFT  36
# define __PTL2_SHIFT  25
# define __PTL1_SHIFT  14
# define PTL4_INDEX_MASK ((UL(1) << 1) - 1)
# define PTL3_INDEX_MASK ((UL(1) << 11) - 1)
# define PTL2_INDEX_MASK PTL3_INDEX_MASK
# define PTL1_INDEX_MASK PTL2_INDEX_MASK
# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0)
# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5)
# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 7)
#elif GRANULE_SIZE == _SZ64KB
# define __PTL4_SHIFT  55
# define __PTL3_SHIFT  42
# define __PTL2_SHIFT  29
# define __PTL1_SHIFT  16
# define PTL4_INDEX_MASK 0
# define PTL3_INDEX_MASK ((UL(1) << 6) - 1)
# define PTL2_INDEX_MASK ((UL(1) << 13) - 1)
# define PTL1_INDEX_MASK PTL2_INDEX_MASK
# define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0)
# define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0)
# define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5)
# define __PTL1_CONT_SHIFT (__PTL1_SHIFT + 5)
#else
# error granule size error.
#endif

#ifndef __ASSEMBLY__
extern int first_level_block_support;
#endif /* __ASSEMBLY__ */

# define __PTL4_SIZE  (UL(1) << __PTL4_SHIFT)
# define __PTL3_SIZE  (UL(1) << __PTL3_SHIFT)
# define __PTL2_SIZE  (UL(1) << __PTL2_SHIFT)
# define __PTL1_SIZE  (UL(1) << __PTL1_SHIFT)
# define __PTL4_MASK  (~(__PTL4_SIZE - 1))
# define __PTL3_MASK  (~(__PTL3_SIZE - 1))
# define __PTL2_MASK  (~(__PTL2_SIZE - 1))
# define __PTL1_MASK  (~(__PTL1_SIZE - 1))

# define __PTL4_CONT_SIZE  (UL(1) << __PTL4_CONT_SHIFT)
# define __PTL3_CONT_SIZE  (UL(1) << __PTL3_CONT_SHIFT)
# define __PTL2_CONT_SIZE  (UL(1) << __PTL2_CONT_SHIFT)
# define __PTL1_CONT_SIZE  (UL(1) << __PTL1_CONT_SHIFT)
# define __PTL4_CONT_MASK  (~(__PTL4_CONT_SIZE - 1))
# define __PTL3_CONT_MASK  (~(__PTL3_CONT_SIZE - 1))
# define __PTL2_CONT_MASK  (~(__PTL2_CONT_SIZE - 1))
# define __PTL1_CONT_MASK  (~(__PTL1_CONT_SIZE - 1))
# define __PTL4_CONT_COUNT  (UL(1) << (__PTL4_CONT_SHIFT - __PTL4_SHIFT))
# define __PTL3_CONT_COUNT  (UL(1) << (__PTL3_CONT_SHIFT - __PTL3_SHIFT))
# define __PTL2_CONT_COUNT  (UL(1) << (__PTL2_CONT_SHIFT - __PTL2_SHIFT))
# define __PTL1_CONT_COUNT  (UL(1) << (__PTL1_CONT_SHIFT - __PTL1_SHIFT))

/* calculate entries */
#if (CONFIG_ARM64_PGTABLE_LEVELS > 3) && (VA_BITS > __PTL4_SHIFT)
# define __PTL4_ENTRIES  (UL(1) << (VA_BITS - __PTL4_SHIFT))
# define __PTL3_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
# define __PTL2_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
# define __PTL1_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
#elif (CONFIG_ARM64_PGTABLE_LEVELS > 2) && (VA_BITS > __PTL3_SHIFT)
# define __PTL4_ENTRIES  1
# define __PTL3_ENTRIES  (UL(1) << (VA_BITS - __PTL3_SHIFT))
# define __PTL2_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
# define __PTL1_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
#elif (CONFIG_ARM64_PGTABLE_LEVELS > 1) && (VA_BITS > __PTL2_SHIFT)
# define __PTL4_ENTRIES  1
# define __PTL3_ENTRIES  1
# define __PTL2_ENTRIES  (UL(1) << (VA_BITS - __PTL2_SHIFT))
# define __PTL1_ENTRIES  (UL(1) << (__PTL1_SHIFT - 3))
#elif VA_BITS > __PTL1_SHIFT
# define __PTL4_ENTRIES  1
# define __PTL3_ENTRIES  1
# define __PTL2_ENTRIES  1
# define __PTL1_ENTRIES  (UL(1) << (VA_BITS - __PTL1_SHIFT))
#else
# define __PTL4_ENTRIES  1
# define __PTL3_ENTRIES  1
# define __PTL2_ENTRIES  1
# define __PTL1_ENTRIES  1
#endif

#ifndef __ASSEMBLY__
static const unsigned int  PTL4_SHIFT   = __PTL4_SHIFT;
static const unsigned int  PTL3_SHIFT   = __PTL3_SHIFT;
static const unsigned int  PTL2_SHIFT   = __PTL2_SHIFT;
static const unsigned int  PTL1_SHIFT   = __PTL1_SHIFT;
static const unsigned long PTL4_SIZE    = __PTL4_SIZE; 
static const unsigned long PTL3_SIZE    = __PTL3_SIZE; 
static const unsigned long PTL2_SIZE    = __PTL2_SIZE; 
static const unsigned long PTL1_SIZE    = __PTL1_SIZE; 
static const unsigned long PTL4_MASK    = __PTL4_MASK;
static const unsigned long PTL3_MASK    = __PTL3_MASK;
static const unsigned long PTL2_MASK    = __PTL2_MASK;
static const unsigned long PTL1_MASK    = __PTL1_MASK;
static const unsigned int  PTL4_ENTRIES = __PTL4_ENTRIES;
static const unsigned int  PTL3_ENTRIES = __PTL3_ENTRIES;
static const unsigned int  PTL2_ENTRIES = __PTL2_ENTRIES;
static const unsigned int  PTL1_ENTRIES = __PTL1_ENTRIES;
static const unsigned int  PTL4_CONT_SHIFT = __PTL4_CONT_SHIFT;
static const unsigned int  PTL3_CONT_SHIFT = __PTL3_CONT_SHIFT;
static const unsigned int  PTL2_CONT_SHIFT = __PTL2_CONT_SHIFT;
static const unsigned int  PTL1_CONT_SHIFT = __PTL1_CONT_SHIFT;
static const unsigned long PTL4_CONT_SIZE  = __PTL4_CONT_SIZE;
static const unsigned long PTL3_CONT_SIZE  = __PTL3_CONT_SIZE;
static const unsigned long PTL2_CONT_SIZE  = __PTL2_CONT_SIZE;
static const unsigned long PTL1_CONT_SIZE  = __PTL1_CONT_SIZE;
static const unsigned long PTL4_CONT_MASK  = __PTL4_CONT_MASK;
static const unsigned long PTL3_CONT_MASK  = __PTL3_CONT_MASK;
static const unsigned long PTL2_CONT_MASK  = __PTL2_CONT_MASK;
static const unsigned long PTL1_CONT_MASK  = __PTL1_CONT_MASK;
static const unsigned int  PTL4_CONT_COUNT = __PTL4_CONT_COUNT;
static const unsigned int  PTL3_CONT_COUNT = __PTL3_CONT_COUNT;
static const unsigned int  PTL2_CONT_COUNT = __PTL2_CONT_COUNT;
static const unsigned int  PTL1_CONT_COUNT = __PTL1_CONT_COUNT;
#else
# define PTL4_SHIFT   __PTL4_SHIFT
# define PTL3_SHIFT   __PTL3_SHIFT
# define PTL2_SHIFT   __PTL2_SHIFT
# define PTL1_SHIFT   __PTL1_SHIFT
# define PTL4_SIZE    __PTL4_SIZE
# define PTL3_SIZE    __PTL3_SIZE
# define PTL2_SIZE    __PTL2_SIZE
# define PTL1_SIZE    __PTL1_SIZE
# define PTL4_MASK    __PTL4_MASK
# define PTL3_MASK    __PTL3_MASK
# define PTL2_MASK    __PTL2_MASK
# define PTL1_MASK    __PTL1_MASK
# define PTL4_ENTRIES __PTL4_ENTRIES
# define PTL3_ENTRIES __PTL3_ENTRIES
# define PTL2_ENTRIES __PTL2_ENTRIES
# define PTL1_ENTRIES __PTL1_ENTRIES
# define PTL4_CONT_SHIFT __PTL4_CONT_SHIFT
# define PTL3_CONT_SHIFT __PTL3_CONT_SHIFT
# define PTL2_CONT_SHIFT __PTL2_CONT_SHIFT
# define PTL1_CONT_SHIFT __PTL1_CONT_SHIFT
# define PTL4_CONT_SIZE __PTL4_CONT_SIZE
# define PTL3_CONT_SIZE __PTL3_CONT_SIZE
# define PTL2_CONT_SIZE __PTL2_CONT_SIZE
# define PTL1_CONT_SIZE __PTL1_CONT_SIZE
# define PTL4_CONT_MASK __PTL4_CONT_MASK
# define PTL3_CONT_MASK __PTL3_CONT_MASK
# define PTL2_CONT_MASK __PTL2_CONT_MASK
# define PTL1_CONT_MASK __PTL1_CONT_MASK
# define PTL4_CONT_COUNT __PTL4_CONT_COUNT
# define PTL3_CONT_COUNT __PTL3_CONT_COUNT
# define PTL2_CONT_COUNT __PTL2_CONT_COUNT
# define PTL1_CONT_COUNT __PTL1_CONT_COUNT
#endif/*__ASSEMBLY__*/

#define __page_size(pgshift)        (UL(1) << (pgshift))
#define __page_mask(pgsize)         (~((pgsize) - 1))
#define __page_offset(addr, size)   ((unsigned long)(addr) & ((size) - 1))
#define __page_align(addr, size)    ((unsigned long)(addr) & ~((size) - 1))
#define __page_align_up(addr, size) __page_align((unsigned long)(addr) + (size) - 1, size)

/*
 * nornal page
 */
#define PAGE_SHIFT          __PTL1_SHIFT
#define PAGE_SIZE           __page_size(PAGE_SHIFT)
#define PAGE_MASK           __page_mask(PAGE_SIZE)
#define PAGE_P2ALIGN        0
#define page_offset(addr)   __page_offset(addr, PAGE_SIZE)
#define page_align(addr)    __page_align(addr, PAGE_SIZE)
#define page_align_up(addr) __page_align_up(addr, PAGE_SIZE)

/*
 * large page
 */
#define LARGE_PAGE_SHIFT          __PTL2_SHIFT
#define LARGE_PAGE_SIZE           __page_size(LARGE_PAGE_SHIFT)
#define LARGE_PAGE_MASK           __page_mask(LARGE_PAGE_SIZE)
#define LARGE_PAGE_P2ALIGN        (LARGE_PAGE_SHIFT - PAGE_SHIFT)
#define large_page_offset(addr)   __page_offset(addr, LARGE_PAGE_SIZE)
#define large_page_align(addr)    __page_align(addr, LARGE_PAGE_SIZE)
#define large_page_align_up(addr) __page_align_up(addr, LARGE_PAGE_SIZE)

/*
 *
 */
#define TTBR_ASID_SHIFT  48
#define TTBR_ASID_MASK   (0xFFFFUL << TTBR_ASID_SHIFT)
#define TTBR_BADDR_MASK  (~TTBR_ASID_MASK)

#include "pgtable-hwdef.h"

#define KERNEL_PHYS_OFFSET

#define PT_PHYSMASK PHYS_MASK
/* We allow user programs to access all the memory (D_Block, D_Page) */
#define PFL_KERN_BLK_ATTR		PROT_SECT_NORMAL_EXEC
#define PFL_KERN_PAGE_ATTR		PAGE_KERNEL_EXEC
/* for the page table entry that points another page table (D_Table) */
#define PFL_PDIR_TBL_ATTR		PMD_TYPE_TABLE

#ifdef CONFIG_ARM64_64K_PAGES
# define SWAPPER_PGTABLE_LEVELS	(CONFIG_ARM64_PGTABLE_LEVELS)
#else
# define SWAPPER_PGTABLE_LEVELS	(CONFIG_ARM64_PGTABLE_LEVELS - 1)
#endif
#define SWAPPER_DIR_SIZE	(SWAPPER_PGTABLE_LEVELS * PAGE_SIZE)
#define IDMAP_DIR_SIZE		(3 * PAGE_SIZE)

/* [Page level Write Throgh] ページキャッシュ方式  0:ライトバック 1:ライトスルー */
#define PFL1_PWT		0 //< DEBUG_ARCH_DEP, devobj.cの直接参照を関数化 (is_pte_pwd)
/* [Page level Cache Disable] ページキャッシュ 0:有効 1:無効 */
#define PFL1_PCD		0 //< DEBUG_ARCH_DEP, devobj.cの直接参照を関数化 (is_pte_pcd)

#define	PTE_NULL		(0)

#define PTE_FILEOFF		PTE_SPECIAL

#ifdef CONFIG_ARM64_64K_PAGES
# define USER_STACK_PREPAGE_SIZE	PAGE_SIZE
# define USER_STACK_PAGE_MASK		PAGE_MASK
# define USER_STACK_PAGE_P2ALIGN	PAGE_P2ALIGN
# define USER_STACK_PAGE_SHIFT		PAGE_SHIFT
#else
# define USER_STACK_PREPAGE_SIZE	LARGE_PAGE_SIZE
# define USER_STACK_PAGE_MASK		LARGE_PAGE_MASK
# define USER_STACK_PAGE_P2ALIGN	LARGE_PAGE_P2ALIGN
# define USER_STACK_PAGE_SHIFT		LARGE_PAGE_SHIFT
#endif

#define PT_ENTRIES		(PAGE_SIZE >> 3)

#ifndef __ASSEMBLY__

#include <ihk/types.h>

typedef unsigned long pte_t;

/*
 * pagemap kernel ABI bits
 */
#define PM_ENTRY_BYTES      sizeof(uint64_t)
#define PM_STATUS_BITS      3
#define PM_STATUS_OFFSET    (64 - PM_STATUS_BITS)
#define PM_STATUS_MASK      (((1LL << PM_STATUS_BITS) - 1) << PM_STATUS_OFFSET)
#define PM_STATUS(nr)       (((nr) << PM_STATUS_OFFSET) & PM_STATUS_MASK)
#define PM_PSHIFT_BITS      6
#define PM_PSHIFT_OFFSET    (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
#define PM_PSHIFT_MASK      (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
#define PM_PSHIFT(x)        (((uint64_t) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
#define PM_PFRAME_MASK      ((1LL << PM_PSHIFT_OFFSET) - 1)
#define PM_PFRAME(x)        ((x) & PM_PFRAME_MASK)

#define PM_PRESENT          PM_STATUS(4LL)
#define PM_SWAP             PM_STATUS(2LL)


/* For easy conversion, it is better to be the same as architecture's ones */
enum ihk_mc_pt_attribute {
	/* ページが物理メモリにロードされているか */
	PTATTR_ACTIVE     = PTE_VALID,
	/* Read/Writeフラグ */
	PTATTR_WRITABLE   = PTE_RDONLY,	//共通定義と意味が反転するので注意
	/* ユーザ/特権フラグ */
	PTATTR_USER       = PTE_USER | PTE_NG,
	/* ページの変更を示す */
	PTATTR_DIRTY      = PTE_DIRTY,
	/* ラージページを示す */
	PTATTR_LARGEPAGE  = PMD_TABLE_BIT, //共通定義と意味が反転するので注意
	/* remap_file_page フラグ */
	PTATTR_FILEOFF    = PTE_FILEOFF,
	/* 実行不可フラグ */
	PTATTR_NO_EXECUTE = PTE_UXN,
	/* キャッシュ無し */
	PTATTR_UNCACHABLE = PTE_ATTRINDX(1),
	/* ユーザ空間向けを示す */
	PTATTR_FOR_USER   = UL(1) << (PHYS_MASK_SHIFT - 1),
	/* WriteCombine */
	PTATTR_WRITE_COMBINED = PTE_ATTRINDX(2),
	/* converted flag */
	ARCH_PTATTR_FLIPPED = PTE_PROT_NONE,
};
extern enum ihk_mc_pt_attribute attr_mask;

static inline int pfn_is_write_combined(uintptr_t pfn)
{
	return ((pfn & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_NC));
}

//共通部と意味がするビット定義
#define attr_flip_bits (PTATTR_WRITABLE | PTATTR_LARGEPAGE)

static inline int pgsize_to_tbllv(size_t pgsize);
static inline int pte_is_type_page(const pte_t *ptep, size_t pgsize)
{
	int ret = 0; //default D_TABLE
	int level = pgsize_to_tbllv(pgsize);

	switch (level) {
	case 4:
	case 3:
	case 2:
		// check D_BLOCK
		ret = ((*ptep & PMD_TYPE_MASK) == PMD_TYPE_SECT);
		break;
	case 1:
		// check D_PAGE
		ret = ((*ptep & PTE_TYPE_MASK) == PTE_TYPE_PAGE);
		break;
	}
	return ret;
}

static inline int pte_is_null(pte_t *ptep)
{
	return (*ptep == PTE_NULL);
}

static inline int pte_is_present(pte_t *ptep)
{
	return !!(*ptep & PMD_SECT_VALID);
}

static inline int pte_is_writable(pte_t *ptep)
{
	extern int kprintf(const char *format, ...);
	kprintf("ERROR: %s is not implemented. \n", __func__);
	return 0;
}

static inline int pte_is_dirty(pte_t *ptep, size_t pgsize)
{
	int ret = 0;
	int do_check = pte_is_type_page(ptep, pgsize);
	if (do_check) {
		ret = !!(*ptep & PTE_DIRTY);
	}	
	return ret;
}

static inline int pte_is_fileoff(pte_t *ptep, size_t pgsize)
{
	int ret = 0;
	int do_check = pte_is_type_page(ptep, pgsize);
	if (do_check) {
		ret = !!(*ptep & PTE_FILEOFF);
	}

	return ret;
}

static inline void pte_update_phys(pte_t *ptep, unsigned long phys)
{
	*ptep = (*ptep & ~PT_PHYSMASK) | (phys & PT_PHYSMASK);
}

static inline uintptr_t pte_get_phys(pte_t *ptep)
{
	return (uintptr_t)(*ptep & PT_PHYSMASK);
}

static inline off_t pte_get_off(pte_t *ptep, size_t pgsize)
{
	return (off_t)(*ptep & PHYS_MASK);
}

static inline enum ihk_mc_pt_attribute pte_get_attr(pte_t *ptep, size_t pgsize)
{
	enum ihk_mc_pt_attribute attr;

	attr = *ptep & attr_mask;
	attr ^= attr_flip_bits;
	if ((*ptep & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_DEVICE_nGnRE)) {
		attr |= PTATTR_UNCACHABLE;
	} else if ((*ptep & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_NC)) {
		attr |= PTATTR_WRITE_COMBINED;
	}
	if (((pgsize == PTL2_SIZE) || (pgsize == PTL3_SIZE))
	    && ((*ptep & PMD_TYPE_MASK) == PMD_TYPE_SECT)) {
		attr |= PTATTR_LARGEPAGE;
	}

	return attr;
}

static inline void pte_make_null(pte_t *ptep, size_t pgsize)
{
	*ptep = PTE_NULL;
}

static inline void pte_make_fileoff(off_t off,
		enum ihk_mc_pt_attribute ptattr, size_t pgsize, pte_t *ptep)
{
	if (((PTL4_SIZE == pgsize || PTL4_CONT_SIZE == pgsize)
				&& CONFIG_ARM64_PGTABLE_LEVELS > 3) ||
	    ((PTL3_SIZE == pgsize || PTL3_CONT_SIZE == pgsize)
				&& CONFIG_ARM64_PGTABLE_LEVELS > 2) ||
	     (PTL2_SIZE == pgsize || PTL2_CONT_SIZE == pgsize) ||
	     (PTL1_SIZE == pgsize || PTL1_CONT_SIZE == pgsize)) {
		*ptep = PTE_FILEOFF | off | PTE_TYPE_PAGE;
	}
}

#if 0	/* XXX: workaround. cannot use panic() here */
static inline void pte_xchg(pte_t *ptep, pte_t *valp)
{
	*valp = xchg(ptep, *valp);
}
#else
#define	pte_xchg(p,vp)	do { *(vp) = xchg((p), *(vp)); } while (0)
#endif

static inline void pte_clear_dirty(pte_t *ptep, size_t pgsize)
{
	int do_clear = pte_is_type_page(ptep, pgsize);
	if (do_clear) {
		*ptep = *ptep & ~PTE_DIRTY;
	}
}

static inline void pte_set_dirty(pte_t *ptep, size_t pgsize)
{
	int do_set = pte_is_type_page(ptep, pgsize);
	if (do_set) {
		*ptep |= PTE_DIRTY;
	}
}

static inline int pte_is_contiguous(const pte_t *ptep)
{
	return !!(*ptep & PTE_CONT);
}

static inline int pgsize_is_contiguous(size_t pgsize)
{
	int ret = 0;

	if ((pgsize == PTL4_CONT_SIZE && CONFIG_ARM64_PGTABLE_LEVELS > 3) ||
	    (pgsize == PTL3_CONT_SIZE && CONFIG_ARM64_PGTABLE_LEVELS > 2) ||
	    (pgsize == PTL2_CONT_SIZE) ||
	    (pgsize == PTL1_CONT_SIZE)) {
		ret = 1;
	}
	return ret;
}

static inline int pgsize_to_tbllv(size_t pgsize)
{
	int level = -EINVAL;

	if ((pgsize == PTL4_CONT_SIZE || pgsize == PTL4_SIZE)
	    && (CONFIG_ARM64_PGTABLE_LEVELS > 3)) {
		level = 4;
	} else if ((pgsize == PTL3_CONT_SIZE || pgsize == PTL3_SIZE)
		   && (CONFIG_ARM64_PGTABLE_LEVELS > 2)) {
		level = 3;
	} else if (pgsize == PTL2_CONT_SIZE || pgsize == PTL2_SIZE) {
		level = 2;
	} else if (pgsize == PTL1_CONT_SIZE || pgsize == PTL1_SIZE) {
		level = 1;
	}
	return level;
}

static inline size_t tbllv_to_pgsize(int level)
{
	size_t pgsize = 0;

	switch (level) {
	case 4:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
			pgsize = PTL4_SIZE;
		} else {
			panic("page table level 4 is invalid.");
		}
		break;
	case 3:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
			pgsize = PTL3_SIZE;
		} else {
			panic("page table level 3 is invalid.");
		}
		break;
	case 2:
		pgsize = PTL2_SIZE;
		break;
	case 1:
		pgsize = PTL1_SIZE;
		break;
	default:
		panic("page table level is invalid.");
	}
	return pgsize;
}

static inline size_t tbllv_to_contpgsize(int level)
{
	size_t pgsize = 0;

	switch (level) {
	case 4:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
			pgsize = PTL4_CONT_SIZE;
		} else {
			panic("page table level 4 is invalid.");
		}
		break;
	case 3:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
			pgsize = PTL3_CONT_SIZE;
		} else {
			panic("page table level 3 is invalid.");
		}
		break;
	case 2:
		pgsize = PTL2_CONT_SIZE;
		break;
	case 1:
		pgsize = PTL1_CONT_SIZE;
		break;
	default:
		panic("page table level is invalid.");
	}
	return pgsize;
}

static inline int tbllv_to_contpgshift(int level)
{
	int ret = 0;

	switch (level) {
	case 4:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
			ret = PTL4_CONT_SHIFT;
		} else {
			panic("page table level 4 is invalid.");
		}

		break;
	case 3:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
			ret = PTL3_CONT_SHIFT;
		} else {
			panic("page table level 3 is invalid.");
		}

		break;
	case 2:
		ret = PTL2_CONT_SHIFT;
		break;
	case 1:
		ret = PTL1_CONT_SHIFT;
		break;
	default:
		panic("page table level is invalid.");
	}
	return ret;
}

static inline pte_t *get_contiguous_head(pte_t *__ptep, size_t __pgsize)
{
	unsigned long align;
	int shift = 0;

	switch (pgsize_to_tbllv(__pgsize)) {
	case 4:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
			shift = PTL4_CONT_SHIFT - PTL4_SHIFT;
		} else {
			panic("page table level 4 is invalid.");
		}
		break;
	case 3:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
			shift = PTL3_CONT_SHIFT - PTL3_SHIFT;
		} else {
			panic("page table level 3 is invalid.");
		}
		break;
	case 2:
		shift = PTL2_CONT_SHIFT - PTL2_SHIFT;
		break;
	case 1:
		shift = PTL1_CONT_SHIFT - PTL1_SHIFT;
		break;
	default:
		panic("page table level is invalid.");
	}
	align = sizeof(*__ptep) << shift;
	return  (pte_t *)__page_align(__ptep, align);
}

static inline pte_t *get_contiguous_tail(pte_t *__ptep, size_t __pgsize)
{
	unsigned long align;
	int shift = 0;

	switch (pgsize_to_tbllv(__pgsize)) {
	case 4:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 3) {
			shift = PTL4_CONT_SHIFT - PTL4_SHIFT;
		} else {
			panic("page table level 4 is invalid.");
		}
		break;
	case 3:
		if (CONFIG_ARM64_PGTABLE_LEVELS > 2) {
			shift = PTL3_CONT_SHIFT - PTL3_SHIFT;
		} else {
			panic("page table level 3 is invalid.");
		}
		break;
	case 2:
		shift = PTL2_CONT_SHIFT - PTL2_SHIFT;
		break;
	case 1:
		shift = PTL1_CONT_SHIFT - PTL1_SHIFT;
		break;
	default:
		panic("page table level is invalid.");
	}
	align = sizeof(*__ptep) << shift;
	return  (pte_t *)__page_align_up(__ptep + 1, align) - 1;
}

static inline int split_contiguous_pages(pte_t *ptep, size_t pgsize)
{
	int ret;
	pte_t *head = get_contiguous_head(ptep, pgsize);
	pte_t *tail = get_contiguous_tail(ptep, pgsize);
	pte_t *ptr;

	uintptr_t phys;
	struct page *page;

	phys = pte_get_phys(head);
	page = phys_to_page(phys);
	if (page && (page_is_in_memobj(page)
		     || page_is_multi_mapped(page))) {
		ret = -EINVAL;
		goto out;
	}

	for (ptr = head; ptr <= tail; ptr++) {
		*ptr &= ~PTE_CONT;
	}

	ret = 0;
out:
	return ret;
}

static inline int page_is_contiguous_head(pte_t *ptep, size_t pgsize)
{
	pte_t *ptr = get_contiguous_head(ptep, pgsize);

	return (ptr == ptep);
}

static inline int page_is_contiguous_tail(pte_t *ptep, size_t pgsize)
{
	pte_t *ptr = get_contiguous_tail(ptep, pgsize);

	return (ptr == ptep);
}

/* Return true if PTE doesn't belong to a contiguous PTE group or PTE
 * is the head of a contiguous PTE group
 */
static inline int pte_is_head(pte_t *ptep, pte_t *old, size_t cont_size)
{
	if (!pte_is_contiguous(old))
		return 1;
	return page_is_contiguous_head(ptep, cont_size);
}

struct page_table;
void arch_adjust_allocate_page_size(struct page_table *pt,
				    uintptr_t fault_addr,
				    pte_t *ptep,
				    void **pgaddrp,
				    size_t *pgsizep);
void set_pte(pte_t *ppte, unsigned long phys, enum ihk_mc_pt_attribute attr);
pte_t *get_pte(struct page_table *pt, void *virt, enum ihk_mc_pt_attribute attr);

struct page_table *get_init_page_table(void);
void *early_alloc_pages(int nr_pages);
void *get_last_early_heap(void);
void flush_tlb(void);
void flush_tlb_single(unsigned long addr);

void *map_fixed_area(unsigned long phys, unsigned long size, int uncachable);

void set_address_space_id(struct page_table *pt, int asid);
int get_address_space_id(const struct page_table *pt);

typedef pte_t translation_table_t;
void set_translation_table(struct page_table *pt, translation_table_t* tt);
translation_table_t* get_translation_table(const struct page_table *pt);
translation_table_t* get_translation_table_as_paddr(const struct page_table *pt);

extern unsigned long ap_trampoline;
//#define AP_TRAMPOLINE       0x10000
#define AP_TRAMPOLINE_SIZE  0x2000

/* Local is cachable */
#define IHK_IKC_QUEUE_PT_ATTR (PTATTR_NO_EXECUTE | PTATTR_WRITABLE)

#endif /* !__ASSEMBLY__ */

#endif /* !__HEADER_ARM64_COMMON_ARCH_MEMORY_H */
