gh0s7/mckernel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tofu: manage stag ranges in VM range split and misc cleanup

Browse Source 
Conflicts:
	kernel/process.c

Change-Id: I480850fe93a7963a5bd4d1687fb1e5c43f58057f
This commit is contained in:
Balazs Gerofi
2021-01-12 20:36:12 +09:00
committed by Masamichi Takagi
parent 507b937509
commit bf926f234a
3 changed files with 127 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
kernel/include/tofu/generate_headers.sh
View File

@ -6,10 +6,25 @@ CURRENT_DIR=`pwd`
cd ${SCRIPT_DIR}
DWARF_TOOL=~/src/mckernel-apollo+a64fx/mckernel/tools/dwarf-extract-struct/dwarf-extract-struct
DWARF_TOOL=${SCRIPT_DIR}/../../../tools/dwarf-extract-struct/dwarf-extract-struct
if [ ! -x ${DWARF_TOOL} ]; then
echo "error: couldn't find DWARF extractor executable (${DWARF_TOOL}), have you compiled it?"
cd -
exit 1
fi
echo "Looking for Tofu driver debug symbols..."
if [ "`find /lib/modules/ -name "tof_module.tar.gz" | xargs -r ls -t | head -n 1 | wc -l`" == "0" ]; then
echo "error: couldn't find Tofu modules with debug symbols"
cd -
exit 1
fi
MODULE_TAR_GZ=`find /lib/modules/ -name "tof_module.tar.gz" | xargs ls -t | head -n 1`
echo "Using Tofu driver debug symbols: ${MODULE_TAR_GZ}"
KMODULE=tof_utofu.ko
if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2>&1 > /dev/null; then
if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then
echo "error: uncompressing kernel module with debug symbols"
cd -
exit 1
@ -22,7 +37,7 @@ ${DWARF_TOOL} ${KMODULE} tof_utofu_bg common tni bgid bch | sed "s/struct FILL_I
rm ${KMODULE}
KMODULE=tof_core.ko
if ! tar zxvf /lib/modules/`uname -r`+debug/extra/tof_module.tar.gz ${KMODULE} 2>&1 > /dev/null; then
if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then
echo "error: uncompressing kernel module with debug symbols"
cd -
exit 1
@ -33,4 +48,4 @@ ${DWARF_TOOL} ${KMODULE} tof_core_bg lock reg irq subnet gpid sighandler | sed "
rm ${KMODULE}
#cat tofu_generated*.h
cd -
cd - > /dev/null

14
kernel/process.c
View File

@ -965,8 +965,18 @@ int split_process_memory_range(struct process_vm *vm, struct vm_range *range,
newrange->private_data = range->private_data;
#ifdef ENABLE_TOFU
/* TODO: figure out which entries to put on which list! */
INIT_LIST_HEAD(&newrange->tofu_stag_list);
{
extern int tofu_stag_split_vm_range_on_addr(struct process_vm *vm,
struct vm_range *range_low, struct vm_range *range_high,
uintptr_t addr);
int moved =
tofu_stag_split_vm_range_on_addr(vm, range, newrange, addr);
if (moved > 0) {
kprintf("%s: moved %d stag ranges\n", __func__, moved);
}
}
#endif
if (range->memobj) {
@ -1182,7 +1192,7 @@ straight_out:
entries = tofu_stag_range_remove_overlapping(vm, range);
if (entries > 0) {
kprintf("%s: removed %d Tofu stag entries for range 0x%lx:%lu\n",
dkprintf("%s: removed %d Tofu stag entries for range 0x%lx:%lu\n",
__func__,
entries,
range->start,

109
kernel/tofu/tof_utofu_main.c
View File

@ -160,6 +160,83 @@ int tofu_stag_range_remove_overlapping(struct process_vm *vm,
return entries;
}
void tofu_stag_range_remove_by_addr(struct process_vm *vm,
uintptr_t addr, size_t len)
{
struct tofu_stag_range *tsr, *next;
int hash;
ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
for (hash = 0; hash < TOFU_STAG_HASH_SIZE; ++hash) {
list_for_each_entry_safe(tsr, next,
&vm->tofu_stag_hash[hash], hash) {
if (tsr->start >= addr && tsr->end <= (addr + len)) {
linux_spin_lock(&tsr->ucq->trans.mru_lock);
tof_utofu_free_stag(tsr->ucq, tsr->stag);
linux_spin_unlock(&tsr->ucq->trans.mru_lock);
kprintf("%s: removed stag %d in %p:%lu\n",
__func__, tsr->stag, addr, len);
__tofu_stag_range_remove(vm, tsr);
}
{
uintptr_t max_start, min_end;
max_start = addr > tsr->start ? addr : tsr->start;
min_end = (addr + len) < tsr->end ? (addr + len) : tsr->end;
if ((tsr->start != 0 || vm->proc->status == PS_EXITED) &&
(max_start < min_end)) {
linux_spin_lock(&tsr->ucq->trans.mru_lock);
tof_utofu_free_stag(tsr->ucq, tsr->stag);
linux_spin_unlock(&tsr->ucq->trans.mru_lock);
kprintf("%s: removed stag %p:%lu (overlaps with range %p:%lu)\n",
__func__, tsr->start, (tsr->end - tsr->start), addr, len);
__tofu_stag_range_remove(vm, tsr);
}
}
}
}
ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
}
int tofu_stag_split_vm_range_on_addr(struct process_vm *vm,
struct vm_range *range_low, struct vm_range *range_high,
uintptr_t addr)
{
struct tofu_stag_range *tsr, *next;
int moved = 0;
ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
list_for_each_entry_safe(tsr, next,
&range_low->tofu_stag_list, list) {
if (tsr->start >= addr) {
list_del(&tsr->list);
list_add_tail(&tsr->list, &range_high->tofu_stag_list);
++moved;
kprintf("%s: stag: %d @ %p:%lu moved to high range..\n",
__func__,
tsr->stag,
tsr->start,
(unsigned long)(tsr->end - tsr->start));
}
if (tsr->start < addr && tsr->end > addr) {
kprintf("%s: WARNING: VM range split in middle of stag range..\n", __func__);
}
}
ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
return moved;
}
#define TOF_UTOFU_VERSION TOF_UAPI_VERSION
@ -1463,7 +1540,8 @@ static int tof_utofu_free_stag(struct tof_utofu_cq *ucq, int stag){
#endif // PROFILE_ENABLE
kref_put(&ucq->trans.mru[stag].mbpt->kref, tof_utofu_mbpt_release);
ucq->trans.mru[stag].mbpt = NULL;
dkprintf("%s: stag: %d deallocated\n", __func__, stag);
dkprintf("%s: TNI: %d, CQ: %d, STAG: %d deallocated\n",
__func__, ucq->tni, ucq->cqid, stag);
#ifdef PROFILE_ENABLE
profile_event_add(PROFILE_tofu_stag_free_stag_dealloc, rdtsc() - ts_rolling);
profile_event_add(PROFILE_tofu_stag_free_stag, rdtsc() - ts);
@ -1486,7 +1564,7 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon
return -EFAULT;
}
//tof_log_if("[IN] tni=%d cqid=%d num=%u stags=%p\n", ucq->tni, ucq->cqid, req.num, req.stags);
dkprintf("%: [IN] tni=%d cqid=%d num=%u stags=%p\n",
dkprintf("%s: [IN] tni=%d cqid=%d num=%u stags=%p\n",
__func__, ucq->tni, ucq->cqid, req.num, req.stags);
if(req.num > 1024 || req.stags == NULL){
@ -1559,10 +1637,8 @@ static int tof_utofu_ioctl_free_stags(struct tof_utofu_device *dev, unsigned lon
void tof_utofu_release_cq(void *pde_data)
{
struct tof_utofu_cq *ucq;
//int stag;
struct tof_utofu_device *dev;
unsigned long irqflags;
struct process_vm *vm = cpu_local_var(current)->vm;
int do_free = 1;
dev = (struct tof_utofu_device *)pde_data;
@ -1574,15 +1650,10 @@ void tof_utofu_release_cq(void *pde_data)
do_free = 0;
}
#if 0
for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) {
linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags);
tof_utofu_free_stag(ucq, stag);
linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags);
#endif
{
int i;
struct tofu_stag_range *tsr, *next;
struct process_vm *vm = cpu_local_var(current)->vm;
ihk_mc_spinlock_lock_noirq(&vm->tofu_stag_lock);
for (i = 0; i < TOFU_STAG_HASH_SIZE; ++i) {
@ -1610,6 +1681,17 @@ void tof_utofu_release_cq(void *pde_data)
ihk_mc_spinlock_unlock_noirq(&vm->tofu_stag_lock);
}
/* Loop through as well just to make sure everything is cleaned up */
if (do_free) {
int stag;
for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) {
linux_spin_lock_irqsave(&ucq->trans.mru_lock, irqflags);
tof_utofu_free_stag(ucq, stag);
linux_spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags);
}
}
dkprintf("%s: UCQ (pde: %p) TNI %d, CQ %d\n",
__func__, pde_data, ucq->tni, ucq->cqid);
}
@ -2356,9 +2438,10 @@ void tof_utofu_finalize(void)
list_for_each_entry_safe(tsr, next,
&vm->tofu_stag_hash[i], hash) {
dkprintf("%s: WARNING: stray stag %d for TNI %d CQ %d?\n",
__func__, tsr->stag, tsr->ucq->tni, tsr->ucq->cqid);
dkprintf("%s: WARNING: stray stag %d (%p:%lu) for TNI %d CQ %d?\n",
__func__, tsr->stag,
tsr->start, tsr->end - tsr->start,
tsr->ucq->tni, tsr->ucq->cqid);
}
}
kprintf("%s: STAG processing done\n", __func__);