From 2900ce20f7a14260fdd85d7b7aab4a07c77a2d8e Mon Sep 17 00:00:00 2001 From: Balazs Gerofi Date: Mon, 16 Oct 2017 18:15:55 +0900 Subject: [PATCH] HFI1: hfi1_unmap_device_addresses() at process terminate time --- arch/x86_64/kernel/memory.c | 20 +++++ kernel/include/process.h | 5 +- kernel/process.c | 1 + kernel/syscall.c | 6 ++ kernel/user_exp_rcv.c | 3 +- kernel/user_sdma.c | 145 +++++++++++++++++++++++++++++++----- 6 files changed, 158 insertions(+), 22 deletions(-) diff --git a/arch/x86_64/kernel/memory.c b/arch/x86_64/kernel/memory.c index 876b6bda..2835463a 100644 --- a/arch/x86_64/kernel/memory.c +++ b/arch/x86_64/kernel/memory.c @@ -731,6 +731,26 @@ static void destroy_page_table(int level, struct page_table *pt) return; } +void ihk_mc_pt_destroy_pgd_subtree(struct page_table *pt, void *virt) +{ + int l4idx, l3idx, l2idx, l1idx; + unsigned long v = (unsigned long)virt; + struct page_table *lower; + + GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx); + + if (!(pt->entry[l4idx] & PF_PRESENT)) + return; + + lower = (struct page_table *) + phys_to_virt(pt->entry[l4idx] & PT_PHYSMASK); + destroy_page_table(3, lower); + + pt->entry[l4idx] = 0; + dkprintf("%s: virt: 0x%lx, l4idx: %d subtree destroyed\n", + __FUNCTION__, virt, l4idx); +} + void ihk_mc_pt_destroy(struct page_table *pt) { const int level = 4; /* PML4 */ diff --git a/kernel/include/process.h b/kernel/include/process.h index 5b8d4679..89b39da8 100644 --- a/kernel/include/process.h +++ b/kernel/include/process.h @@ -575,12 +575,15 @@ struct process { int nr_processes; /* For partitioned execution */ int process_rank; /* Rank in partition */ - void *fd_priv_table[265]; + void *fd_priv_table[256]; /* HFI1 specific */ void *hfi1_kregbase; void *hfi1_piobase; void *hfi1_rcvarray_wc; + size_t hfi1_rcvarray_wc_len; void *hfi1_cq_comps; + size_t hfi1_cq_comps_len; + ihk_spinlock_t hfi1_lock; }; /* diff --git a/kernel/process.c b/kernel/process.c index 7c9afaf4..48687774 100644 --- a/kernel/process.c +++ b/kernel/process.c @@ -148,6 +148,7 @@ init_process(struct process *proc, struct process *parent) mcs_rwlock_init(&proc->threads_lock); mcs_rwlock_init(&proc->children_lock); ihk_mc_spinlock_init(&proc->mckfd_lock); + ihk_mc_spinlock_init(&proc->hfi1_lock); waitq_init(&proc->waitpid_q); ihk_atomic_set(&proc->refcount, 2); proc->monitoring_event = NULL; diff --git a/kernel/syscall.c b/kernel/syscall.c index 3313f7d2..9c6fd336 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -1078,6 +1078,12 @@ void terminate(int rc, int sig) mcs_rwlock_writer_unlock(&proc->threads_lock, &lock); vm = proc->vm; + + { + extern int hfi1_unmap_device_addresses(struct process *proc); + hfi1_unmap_device_addresses(proc); + } + free_all_process_memory_range(vm); if (proc->saved_cmdline) { diff --git a/kernel/user_exp_rcv.c b/kernel/user_exp_rcv.c index 63f5caa5..0ae9ea41 100644 --- a/kernel/user_exp_rcv.c +++ b/kernel/user_exp_rcv.c @@ -232,13 +232,12 @@ int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd, struct hfi1_tid_info *tinf /* Technically should never be needed (because mapped previously * on update), but this call is no-op if addresses have been set * previously - */ if (hfi1_map_device_addresses(fd) < 0) { kprintf("%s: Could not map hfi1 device addresses\n", __FUNCTION__); return -EINVAL; } - + */ dkprintf("Clear called, cnt %d\n", tinfo->tidcnt); for (tididx = 0; tididx < tinfo->tidcnt; tididx++) { diff --git a/kernel/user_sdma.c b/kernel/user_sdma.c index adc28f03..3dbadb9f 100644 --- a/kernel/user_sdma.c +++ b/kernel/user_sdma.c @@ -51,6 +51,14 @@ #include #include +//#define DEBUG_PRINT_HFI1_SDMA + +#ifdef DEBUG_PRINT_HFI1_SDMA +#define dkprintf(...) kprintf(__VA_ARGS__) +#else +#define dkprintf(...) do { if(0) kprintf(__VA_ARGS__); } while (0) +#endif + #ifdef __HFI1_ORIG__ #include @@ -564,12 +572,16 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) #endif unsigned long phys; unsigned long len; + unsigned long irqstate; + int ret = 0; struct process *proc = cpu_local_var(current)->proc; struct process_vm *vm = cpu_local_var(current)->vm; struct hfi1_user_sdma_comp_q *cq = fd->cq; struct hfi1_devdata *dd = fd->dd; + irqstate = ihk_mc_spinlock_lock(&proc->hfi1_lock); + /* * Map device addresses if not mapped or mapping changed. */ @@ -592,7 +604,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) virt, phys, attr) < 0) { kprintf("%s: ERROR: failed to map kregbase: 0x%lx -> 0x%lx\n", __FUNCTION__, virt, phys); - return -1; + ret = -1; + goto unlock_out; } ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table, @@ -600,7 +613,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) if (!ptep && !pte_is_present(ptep)) { kprintf("%s: ERROR: no mapping in McKernel for kregbase: 0x%lx?\n", __FUNCTION__, virt); - return -1; + ret = -1; + goto unlock_out; } lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(), @@ -608,17 +622,19 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) if (!lptep && !pte_is_present(lptep)) { kprintf("%s: ERROR: no mapping in Linux for kregbase: 0x%lx?\n", __FUNCTION__, virt); - return -1; + ret = -1; + goto unlock_out; } *ptep = *lptep; } - kprintf("%s: hfi1_kregbase: 0x%lx - 0x%lx -> 0x%lx:%lu\n", + dkprintf("%s: hfi1_kregbase: 0x%lx - 0x%lx -> 0x%lx:%lu\n", __FUNCTION__, hfi1_kregbase, hfi1_kregbase + TXE_PIO_SEND, (phys - TXE_PIO_SEND), TXE_PIO_SEND); + //ihk_mc_pt_print_pte(vm->address_space->page_table, hfi1_kregbase); proc->hfi1_kregbase = hfi1_kregbase; } @@ -635,7 +651,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) virt, phys, attr) < 0) { kprintf("%s: ERROR: failed to map piobase: 0x%lx -> 0x%lx\n", __FUNCTION__, virt, phys); - return -1; + ret = -1; + goto unlock_out; } ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table, @@ -643,7 +660,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) if (!ptep && !pte_is_present(ptep)) { kprintf("%s: ERROR: no mapping in McKernel for piobase: 0x%lx?\n", __FUNCTION__, virt); - return -1; + ret = -1; + goto unlock_out; } lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(), @@ -651,13 +669,14 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) if (!lptep && !pte_is_present(lptep)) { kprintf("%s: ERROR: no mapping in Linux for piobase: 0x%lx?\n", __FUNCTION__, virt); - return -1; + ret = -1; + goto unlock_out; } *ptep = *lptep; } - kprintf("%s: hfi1_piobase: 0x%lx - 0x%lx -> 0x%lx:%lu\n", + dkprintf("%s: hfi1_piobase: 0x%lx - 0x%lx -> 0x%lx:%lu\n", __FUNCTION__, hfi1_piobase, hfi1_piobase + TXE_PIO_SIZE, @@ -679,7 +698,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) virt, phys, attr) < 0) { kprintf("%s: ERROR: failed to map rcvarray_wc: 0x%lx -> 0x%lx\n", __FUNCTION__, virt, phys); - return -1; + ret = -1; + goto unlock_out; } ptep = ihk_mc_pt_lookup_pte(vm->address_space->page_table, @@ -687,7 +707,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) if (!ptep && !pte_is_present(ptep)) { kprintf("%s: ERROR: no mapping in McKernel for rcvarray: 0x%lx?\n", __FUNCTION__, virt); - return -1; + ret = -1; + goto unlock_out; } lptep = ihk_mc_pt_lookup_pte(ihk_mc_get_linux_kernel_pgt(), @@ -695,13 +716,14 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) if (!lptep && !pte_is_present(lptep)) { kprintf("%s: ERROR: no mapping in Linux for rcvarray: 0x%lx?\n", __FUNCTION__, virt); - return -1; + ret = -1; + goto unlock_out; } *ptep = *lptep; } - kprintf("%s: hfi1_rcvarray_wc: 0x%lx - 0x%lx -> 0x%lx:%lu\n", + dkprintf("%s: hfi1_rcvarray_wc: 0x%lx - 0x%lx -> 0x%lx:%lu\n", __FUNCTION__, hfi1_rcvarray_wc, hfi1_rcvarray_wc + dd->chip_rcv_array_count * 8, @@ -709,6 +731,7 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) dd->chip_rcv_array_count * 8); proc->hfi1_rcvarray_wc = hfi1_rcvarray_wc; + proc->hfi1_rcvarray_wc_len = dd->chip_rcv_array_count * 8; } /* @@ -727,7 +750,8 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) if (!lptep && !pte_is_present(lptep)) { kprintf("%s: ERROR: no mapping in Linux for cq: 0x%lx?\n", __FUNCTION__, virt); - return -1; + ret = -1; + goto unlock_out; } phys = pte_get_phys(lptep); @@ -750,20 +774,103 @@ int hfi1_map_device_addresses(struct hfi1_filedata *fd) if (!ptep) { kprintf("%s: ERROR: no PTE in McKernel for cq: 0x%lx?\n", __FUNCTION__, virt); - return -1; + ret = -1; + goto unlock_out; } *ptep = *lptep; - kprintf("%s: cq: 0x%lx -> 0x%lx mapped\n", - __FUNCTION__, - virt, phys); } + dkprintf("%s: hfi1_cq_comps: 0x%lx - 0x%lx mapped\n", + __FUNCTION__, + cq->comps, len); + proc->hfi1_cq_comps = cq->comps; + proc->hfi1_cq_comps_len = len; } - /* TODO: unmap these at close() time */ - return 0; +unlock_out: + ihk_mc_spinlock_unlock(&proc->hfi1_lock, irqstate); + + return ret; +} + + +int hfi1_unmap_device_addresses(struct process *proc) +{ + unsigned long irqstate; + int ret = 0; + + struct process_vm *vm = proc->vm; + extern void ihk_mc_pt_destroy_pgd_subtree(struct page_table *pt, + void *virt); + + irqstate = ihk_mc_spinlock_lock(&proc->hfi1_lock); + + /* + * Unmap device addresses if mapped. + */ + if (proc->hfi1_kregbase) { + + ihk_mc_pt_destroy_pgd_subtree(vm->address_space->page_table, + proc->hfi1_kregbase); +/* + ihk_mc_pt_clear_kernel_range(vm->address_space->page_table, vm, + proc->hfi1_kregbase, proc->hfi1_kregbase + TXE_PIO_SEND); + + kprintf("%s: hfi1_kregbase unmapped\n", + __FUNCTION__); +*/ + proc->hfi1_kregbase = 0; + } + + if (proc->hfi1_piobase) { + + ihk_mc_pt_destroy_pgd_subtree(vm->address_space->page_table, + proc->hfi1_piobase); +/* + ihk_mc_pt_clear_kernel_range(vm->address_space->page_table, vm, + proc->hfi1_piobase, proc->hfi1_piobase + TXE_PIO_SIZE); + + kprintf("%s: hfi1_piobase unmapped\n", + __FUNCTION__); +*/ + proc->hfi1_piobase = 0; + } + + if (proc->hfi1_rcvarray_wc) { + + ihk_mc_pt_destroy_pgd_subtree(vm->address_space->page_table, + proc->hfi1_rcvarray_wc); +/* + ihk_mc_pt_clear_kernel_range(vm->address_space->page_table, vm, + proc->hfi1_rcvarray_wc, + proc->hfi1_rcvarray_wc + proc->hfi1_rcvarray_wc_len); + + kprintf("%s: hfi1_rcvarray_wc unmapped\n", + __FUNCTION__); +*/ + proc->hfi1_rcvarray_wc = 0; + } + + if (proc->hfi1_cq_comps) { + + ihk_mc_pt_destroy_pgd_subtree(vm->address_space->page_table, + proc->hfi1_cq_comps); +/* + ihk_mc_pt_clear_kernel_range(vm->address_space->page_table, vm, + proc->hfi1_cq_comps, + proc->hfi1_cq_comps + proc->hfi1_cq_comps_len); + + kprintf("%s: hfi1_cq_comps unmapped\n", + __FUNCTION__); +*/ + proc->hfi1_cq_comps = 0; + } + + ihk_mc_spinlock_unlock(&proc->hfi1_lock, irqstate); + + return ret; } struct kmalloc_cache_header tids_cache = {NULL};