From 96b6d773a9e11d4d1254680a23dfa760f46933f0 Mon Sep 17 00:00:00 2001 From: "TOIDA,Suguru" Date: Fri, 30 Nov 2018 09:18:52 +0900 Subject: [PATCH] ARMv8.2-LPA support Change-Id: I12a6eac55af2e7f6a643e4e04ed59a85769f4063 --- arch/arm64/kernel/include/arch-memory.h | 7 +++--- arch/arm64/kernel/include/arch/mman.h | 11 +++------ arch/arm64/kernel/include/arch/shm.h | 11 +++------ arch/arm64/kernel/include/sysreg.h | 8 +++++++ arch/arm64/kernel/memory.c | 31 +++++++++++++++---------- arch/arm64/kernel/syscall.c | 16 ++++++++----- 6 files changed, 47 insertions(+), 37 deletions(-) diff --git a/arch/arm64/kernel/include/arch-memory.h b/arch/arm64/kernel/include/arch-memory.h index 47bf7ffa..2180684d 100644 --- a/arch/arm64/kernel/include/arch-memory.h +++ b/arch/arm64/kernel/include/arch-memory.h @@ -111,7 +111,6 @@ void panic(const char *); # define PTL3_INDEX_MASK PTL4_INDEX_MASK # define PTL2_INDEX_MASK PTL3_INDEX_MASK # define PTL1_INDEX_MASK PTL2_INDEX_MASK -# define FIRST_LEVEL_BLOCK_SUPPORT 1 # define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0) # define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 4) # define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 4) @@ -125,7 +124,6 @@ void panic(const char *); # define PTL3_INDEX_MASK ((UL(1) << 11) - 1) # define PTL2_INDEX_MASK PTL3_INDEX_MASK # define PTL1_INDEX_MASK PTL2_INDEX_MASK -# define FIRST_LEVEL_BLOCK_SUPPORT 0 # define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0) # define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0) # define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5) @@ -139,7 +137,6 @@ void panic(const char *); # define PTL3_INDEX_MASK ((UL(1) << 6) - 1) # define PTL2_INDEX_MASK ((UL(1) << 13) - 1) # define PTL1_INDEX_MASK PTL2_INDEX_MASK -# define FIRST_LEVEL_BLOCK_SUPPORT 0 # define __PTL4_CONT_SHIFT (__PTL4_SHIFT + 0) # define __PTL3_CONT_SHIFT (__PTL3_SHIFT + 0) # define __PTL2_CONT_SHIFT (__PTL2_SHIFT + 5) @@ -148,6 +145,10 @@ void panic(const char *); # error granule size error. #endif +#ifndef __ASSEMBLY__ +extern int first_level_block_support; +#endif /* __ASSEMBLY__ */ + # define __PTL4_SIZE (UL(1) << __PTL4_SHIFT) # define __PTL3_SIZE (UL(1) << __PTL3_SHIFT) # define __PTL2_SIZE (UL(1) << __PTL2_SHIFT) diff --git a/arch/arm64/kernel/include/arch/mman.h b/arch/arm64/kernel/include/arch/mman.h index 300ddbee..546c90ab 100644 --- a/arch/arm64/kernel/include/arch/mman.h +++ b/arch/arm64/kernel/include/arch/mman.h @@ -21,14 +21,9 @@ /* Bits [26:31] are reserved, see mman-common.h for MAP_HUGETLB usage */ #define MAP_HUGE_SHIFT 26 -#if FIRST_LEVEL_BLOCK_SUPPORT -# define MAP_HUGE_FIRST_BLOCK (__PTL3_SHIFT << MAP_HUGE_SHIFT) -# define MAP_HUGE_FIRST_CONT_BLOCK (__PTL3_CONT_SHIFT << MAP_HUGE_SHIFT) -#else -# define MAP_HUGE_FIRST_BLOCK -1 /* not supported */ -# define MAP_HUGE_FIRST_CONT_BLOCK -1 /* not supported */ -#endif -#define MAP_HUGE_SECOND_BLOCK (__PTL2_SHIFT << MAP_HUGE_SHIFT) +#define MAP_HUGE_FIRST_BLOCK (__PTL3_SHIFT << MAP_HUGE_SHIFT) +#define MAP_HUGE_FIRST_CONT_BLOCK (__PTL3_CONT_SHIFT << MAP_HUGE_SHIFT) +#define MAP_HUGE_SECOND_BLOCK (__PTL2_SHIFT << MAP_HUGE_SHIFT) #define MAP_HUGE_SECOND_CONT_BLOCK (__PTL2_CONT_SHIFT << MAP_HUGE_SHIFT) #define MAP_HUGE_THIRD_CONT_BLOCK (__PTL1_CONT_SHIFT << MAP_HUGE_SHIFT) diff --git a/arch/arm64/kernel/include/arch/shm.h b/arch/arm64/kernel/include/arch/shm.h index 8c9be4a6..414b4275 100644 --- a/arch/arm64/kernel/include/arch/shm.h +++ b/arch/arm64/kernel/include/arch/shm.h @@ -6,14 +6,9 @@ /* shmflg */ #define SHM_HUGE_SHIFT 26 -#if FIRST_LEVEL_BLOCK_SUPPORT -# define SHM_HUGE_FIRST_BLOCK (__PTL3_SHIFT << SHM_HUGE_SHIFT) -# define SHM_HUGE_FIRST_CONT_BLOCK (__PTL3_CONT_SHIFT << SHM_HUGE_SHIFT) -#else -# define SHM_HUGE_FIRST_BLOCK -1 /* not supported */ -# define SHM_HUGE_FIRST_CONT_BLOCK -1 /* not supported */ -#endif -#define SHM_HUGE_SECOND_BLOCK (__PTL2_SHIFT << SHM_HUGE_SHIFT) +#define SHM_HUGE_FIRST_BLOCK (__PTL3_SHIFT << SHM_HUGE_SHIFT) +#define SHM_HUGE_FIRST_CONT_BLOCK (__PTL3_CONT_SHIFT << SHM_HUGE_SHIFT) +#define SHM_HUGE_SECOND_BLOCK (__PTL2_SHIFT << SHM_HUGE_SHIFT) #define SHM_HUGE_SECOND_CONT_BLOCK (__PTL2_CONT_SHIFT << SHM_HUGE_SHIFT) #define SHM_HUGE_THIRD_CONT_BLOCK (__PTL1_CONT_SHIFT << SHM_HUGE_SHIFT) diff --git a/arch/arm64/kernel/include/sysreg.h b/arch/arm64/kernel/include/sysreg.h index 44f7c638..abfdbef1 100644 --- a/arch/arm64/kernel/include/sysreg.h +++ b/arch/arm64/kernel/include/sysreg.h @@ -178,6 +178,14 @@ #define ID_AA64MMFR0_TGRAN64_SUPPORTED 0x0 #define ID_AA64MMFR0_TGRAN16_NI 0x0 #define ID_AA64MMFR0_TGRAN16_SUPPORTED 0x1 +#define ID_AA64MMFR0_PARANGE_48 0x5 +#define ID_AA64MMFR0_PARANGE_52 0x6 + +#ifdef CONFIG_ARM64_PA_BITS_52 +#define ID_AA64MMFR0_PARANGE_MAX ID_AA64MMFR0_PARANGE_52 +#else +#define ID_AA64MMFR0_PARANGE_MAX ID_AA64MMFR0_PARANGE_48 +#endif /* id_aa64mmfr1 */ #define ID_AA64MMFR1_PAN_SHIFT 20 diff --git a/arch/arm64/kernel/memory.c b/arch/arm64/kernel/memory.c index 7e18acca..4cefc9f2 100644 --- a/arch/arm64/kernel/memory.c +++ b/arch/arm64/kernel/memory.c @@ -1631,21 +1631,15 @@ static int split_large_page(pte_t *ptep, size_t pgsize) unsigned long under_pgsize; // ラージページ判定 - switch (pgsize) - { -#if FIRST_LEVEL_BLOCK_SUPPORT - case __PTL3_SIZE: + if (first_level_block_support && pgsize == PTL3_SIZE) { table_level = 3; entries = PTL3_ENTRIES; under_pgsize = PTL2_SIZE; - break; -#endif - case __PTL2_SIZE: + } else if (pgsize == PTL2_SIZE) { table_level = 2; entries = PTL2_ENTRIES; under_pgsize = PTL1_SIZE; - break; - default: + } else { ekprintf("split_large_page:invalid pgsize %#lx\n", pgsize); return -EINVAL; } @@ -2200,7 +2194,7 @@ static pte_t *lookup_pte(translation_table_t* tt, uintptr_t virt, int pgshift, ptep = NULL; if (!pgshift) { - if (FIRST_LEVEL_BLOCK_SUPPORT) { + if (first_level_block_support) { pgshift = PTL3_CONT_SHIFT; } else { pgshift = PTL2_CONT_SHIFT; @@ -2413,7 +2407,7 @@ retry: if (ptl_null(ptep, level) || (args->overwrite && ptl_type_page(ptep, level))) { pte_t pte; uintptr_t phys; - if (level == 2 || (level == 3 && FIRST_LEVEL_BLOCK_SUPPORT)) { + if (level == 2 || (level == 3 && first_level_block_support)) { if ((start <= base) && ((base + tbl.pgsize) <= end) && ((args->diff & (tbl.pgsize - 1)) == 0) && (!args->pgshift @@ -2579,7 +2573,7 @@ int ihk_mc_pt_set_pte(page_table_t pt, pte_t *ptep, size_t pgsize, pte = phys | attr_to_l2attr(attr | PTATTR_LARGEPAGE); ptl2_set(ptep, pte); } - else if (pgsize == PTL3_SIZE && FIRST_LEVEL_BLOCK_SUPPORT) { + else if (pgsize == PTL3_SIZE && first_level_block_support) { if (phys & (PTL3_SIZE - 1)) { kprintf("%s: error: phys needs to be PTL3_SIZE aligned\n", __FUNCTION__); error = -1; @@ -2928,10 +2922,23 @@ void init_low_area(struct page_table *pt) set_pt_large_page(pt, 0, 0, PTATTR_NO_EXECUTE|PTATTR_WRITABLE); } +int first_level_block_support; void init_page_table(void) { + uint64_t parange; + ihk_mc_spinlock_init(&init_pt_lock); + if (PAGE_SIZE == _SZ4KB) { + first_level_block_support = 1; + } else if (PAGE_SIZE == _SZ16KB) { + first_level_block_support = 0; + } else { + parange = read_sysreg(id_aa64mmfr0_el1) & 7; + first_level_block_support = + (parange >= ID_AA64MMFR0_PARANGE_52); + } + /* Normal memory area */ init_normal_area(init_pt); init_fixed_area(init_pt); diff --git a/arch/arm64/kernel/syscall.c b/arch/arm64/kernel/syscall.c index 4cf45aee..d72e07b3 100644 --- a/arch/arm64/kernel/syscall.c +++ b/arch/arm64/kernel/syscall.c @@ -1750,10 +1750,12 @@ SYSCALL_DECLARE(mmap) if (hugeshift == 0) { /* default hugepage size */ flags |= MAP_HUGE_SECOND_BLOCK; - } else if (hugeshift == MAP_HUGE_SECOND_BLOCK || - hugeshift == MAP_HUGE_FIRST_BLOCK || + } else if ((first_level_block_support && + hugeshift == MAP_HUGE_FIRST_BLOCK) || + (first_level_block_support && + hugeshift == MAP_HUGE_FIRST_CONT_BLOCK) || + hugeshift == MAP_HUGE_SECOND_BLOCK || hugeshift == MAP_HUGE_SECOND_CONT_BLOCK || - hugeshift == MAP_HUGE_FIRST_CONT_BLOCK || hugeshift == MAP_HUGE_THIRD_CONT_BLOCK) { /*nop*/ } else { @@ -1825,10 +1827,12 @@ SYSCALL_DECLARE(shmget) if (hugeshift == 0) { /* default hugepage size */ shmflg |= SHM_HUGE_SECOND_BLOCK; - } else if (hugeshift == SHM_HUGE_SECOND_BLOCK || - hugeshift == SHM_HUGE_FIRST_BLOCK || + } else if ((first_level_block_support && + hugeshift == SHM_HUGE_FIRST_BLOCK) || + (first_level_block_support && + hugeshift == SHM_HUGE_FIRST_CONT_BLOCK) || + hugeshift == SHM_HUGE_SECOND_BLOCK || hugeshift == SHM_HUGE_SECOND_CONT_BLOCK || - hugeshift == SHM_HUGE_FIRST_CONT_BLOCK || hugeshift == SHM_HUGE_THIRD_CONT_BLOCK) { /*nop*/ } else {