From 507b937509732733ddeb7052e91ea9ca1f8a2bf4 Mon Sep 17 00:00:00 2001 From: Balazs Gerofi Date: Fri, 8 Jan 2021 21:34:11 +0900 Subject: [PATCH] Tofu: mcctrl side MMU notifier and CQ/BCH cleanup Conflicts: executer/kernel/mcctrl/arch/arm64/archdeps.c executer/kernel/mcctrl/syscall.c Change-Id: Ided8172331a5469c6ced68fa98a42302812efe71 --- executer/kernel/mcctrl/arch/arm64/archdeps.c | 472 +++++++++- .../arm64/include/tofu/generate_headers.sh | 41 + .../mcctrl/arch/arm64/include/tofu/tof_icc.h | 831 ++++++++++++++++++ .../mcctrl/arch/arm64/include/tofu/tof_uapi.h | 345 ++++++++ .../arm64/include/tofu/tof_utofu_bg_bch.h | 6 + .../arm64/include/tofu/tof_utofu_cq_trans.h | 6 + .../tofu/tofu_generated-tof_utofu_bg.h | 21 + .../tofu/tofu_generated-tof_utofu_cq.h | 37 + .../tofu/tofu_generated-tof_utofu_device.h | 17 + .../tofu/tofu_generated-tof_utofu_mbpt.h | 33 + executer/kernel/mcctrl/control.c | 13 +- executer/kernel/mcctrl/mcctrl.h | 9 +- executer/kernel/mcctrl/syscall.c | 143 ++- 13 files changed, 1912 insertions(+), 62 deletions(-) create mode 100755 executer/kernel/mcctrl/arch/arm64/include/tofu/generate_headers.sh create mode 100644 executer/kernel/mcctrl/arch/arm64/include/tofu/tof_icc.h create mode 100644 executer/kernel/mcctrl/arch/arm64/include/tofu/tof_uapi.h create mode 100644 executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_bg_bch.h create mode 100644 executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_cq_trans.h create mode 100644 executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_bg.h create mode 100644 executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_cq.h create mode 100644 executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_device.h create mode 100644 executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_mbpt.h diff --git a/executer/kernel/mcctrl/arch/arm64/archdeps.c b/executer/kernel/mcctrl/arch/arm64/archdeps.c index 9472e4ea..8856527c 100644 --- a/executer/kernel/mcctrl/arch/arm64/archdeps.c +++ b/executer/kernel/mcctrl/arch/arm64/archdeps.c @@ -8,6 +8,10 @@ #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) */ #include #include +#include +#include +#include +#include #include #include "config.h" #include "../../mcctrl.h" @@ -36,6 +40,29 @@ int (*mcctrl_tof_utofu_release_cq)(struct inode *inode, struct file_operations *mcctrl_tof_utofu_procfs_ops_bch; int (*mcctrl_tof_utofu_release_bch)(struct inode *inode, struct file *filp); +int (*mcctrl_tof_core_cq_cacheflush)(int tni, int cqid); +int (*mcctrl_tof_core_disable_bch)(int tni, int bgid); +int (*mcctrl_tof_core_unset_bg)(int tni, int bgid); +typedef void (*tof_core_signal_handler)(int, int, uint64_t, uint64_t); +void (*mcctrl_tof_core_register_signal_bg)(int tni, int bgid, + tof_core_signal_handler handler); +struct tof_utofu_bg; +struct tof_utofu_bg *mcctrl_tof_utofu_bg; + + +/* Tofu MMU notifier */ +struct mmu_notifier_ops *mcctrl_tof_utofu_mn_ops; +struct mmu_notifier_ops __mcctrl_tof_utofu_mn_ops; +static void (*mcctrl_tof_utofu_mn_invalidate_range_end)( + struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, + unsigned long end); +void __mcctrl_tof_utofu_mn_invalidate_range_end( + struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, + unsigned long end); #endif int arch_symbols_init(void) @@ -74,6 +101,49 @@ int arch_symbols_init(void) (void *)kallsyms_lookup_name("tof_utofu_release_bch"); if (WARN_ON(!mcctrl_tof_utofu_release_bch)) return -EFAULT; + + mcctrl_tof_core_cq_cacheflush = + (void *)kallsyms_lookup_name("tof_core_cq_cacheflush"); + if (WARN_ON(!mcctrl_tof_core_cq_cacheflush)) + return -EFAULT; + + mcctrl_tof_core_disable_bch = + (void *)kallsyms_lookup_name("tof_core_disable_bch"); + if (WARN_ON(!mcctrl_tof_core_disable_bch)) + return -EFAULT; + + mcctrl_tof_core_unset_bg = + (void *)kallsyms_lookup_name("tof_core_unset_bg"); + if (WARN_ON(!mcctrl_tof_core_unset_bg)) + return -EFAULT; + + mcctrl_tof_core_register_signal_bg = + (void *)kallsyms_lookup_name("tof_core_register_signal_bg"); + if (WARN_ON(!mcctrl_tof_core_register_signal_bg)) + return -EFAULT; + + mcctrl_tof_utofu_bg = + (void *)kallsyms_lookup_name("tof_utofu_bg"); + if (WARN_ON(!mcctrl_tof_utofu_bg)) + return -EFAULT; + + mcctrl_tof_utofu_mn_ops = + (void *)kallsyms_lookup_name("tof_utofu_mn_ops"); + if (WARN_ON(!mcctrl_tof_utofu_mn_ops)) + return -EFAULT; + /* + * Copy original content and update redirected function, + * CQ will be pointed to this structure after init ioctl() + */ + memcpy(&__mcctrl_tof_utofu_mn_ops, mcctrl_tof_utofu_mn_ops, + sizeof(*mcctrl_tof_utofu_mn_ops)); + __mcctrl_tof_utofu_mn_ops.invalidate_range = + __mcctrl_tof_utofu_mn_invalidate_range_end; + + mcctrl_tof_utofu_mn_invalidate_range_end = + (void *)kallsyms_lookup_name("tof_utofu_mn_invalidate_range_end"); + if (WARN_ON(!mcctrl_tof_utofu_mn_invalidate_range_end)) + return -EFAULT; #endif return 0; @@ -474,6 +544,7 @@ void mcctrl_tofu_hijack_release_handlers(void) __mcctrl_tof_utofu_release_cq; mcctrl_tof_utofu_procfs_ops_bch->release = __mcctrl_tof_utofu_release_bch; + wmb(); } void mcctrl_tofu_restore_release_handlers(void) @@ -482,8 +553,327 @@ void mcctrl_tofu_restore_release_handlers(void) mcctrl_tof_utofu_release_cq; mcctrl_tof_utofu_procfs_ops_bch->release = mcctrl_tof_utofu_release_bch; + wmb(); } +/* + * Tofu cleanup functions + */ +#include +#include +#include +#include +#include +#include +#include + +#define TOF_UTOFU_VERSION TOF_UAPI_VERSION +#define TOF_UTOFU_NUM_STAG_NTYPES 3 +#define TOF_UTOFU_NUM_STAG_BITS(size) ((size) + 13) +#define TOF_UTOFU_NUM_STAG(size) ((uint64_t)1 << TOF_UTOFU_NUM_STAG_BITS(size)) +#define TOF_UTOFU_STAG_TRANS_BITS 3 +#define TOF_UTOFU_STAG_TRANS_SIZE ((uint64_t)1 << TOF_UTOFU_STAG_TRANS_BITS) +#define TOF_UTOFU_STAG_TRANS_TABLE_LEN(size) (TOF_UTOFU_NUM_STAG(size) * TOF_UTOFU_STAG_TRANS_SIZE) +#define TOF_UTOFU_STEERING_TABLE_LEN(size) (TOF_UTOFU_NUM_STAG(size) * TOF_ICC_STEERING_SIZE) +#define TOF_UTOFU_MB_TABLE_LEN(size) (TOF_UTOFU_NUM_STAG(size) * TOF_ICC_MB_SIZE) +#define TOF_UTOFU_STAG_MEM_LEN(size) (TOF_UTOFU_STEERING_TABLE_LEN(size) * 4) +#define TOF_UTOFU_SPECIAL_STAG 4096 + +#define TOF_UTOFU_ICC_COMMON_REGISTER (tof_icc_reg_pa + 0x0B000000) +#define TOF_UTOFU_REG_START tof_icc_reg_pa +#define TOF_UTOFU_REG_END (TOF_UTOFU_ICC_COMMON_REGISTER + 0x000FFFFF) + +#define TOF_UTOFU_SET_SUBNET_TNI 0 /* This number is kernel TNIs number in setting subnet */ +#define TOF_UTOFU_KCQ 11 +#define TOF_UTOFU_LINKDOWN_PORT_MASK 0x000003FF + +#define TOF_UTOFU_ALLOC_STAG_LPG 0x2 +#define TOF_UTOFU_BLANK_MBVA (-1) + +#define TOF_UTOFU_MRU_EMPTY (-1) + +struct tof_utofu_trans_list { + int16_t prev; + int16_t next; + uint8_t pgszbits; + struct tof_utofu_mbpt *mbpt; +}; + + +/* + * Bit 30 marks a kref as McKernel internal. + * This can be used to distinguish krefs from Linux and + * it also ensures that a non deallocated kref will not + * crash the Linux allocator. + */ +#define MCKERNEL_KREF_MARK (1U << 30) +static inline unsigned int mcctrl_kref_is_mckernel(const struct kref *kref) +{ + return (refcount_read(&kref->refcount) & (MCKERNEL_KREF_MARK)); +} + +/** + * kref_put - decrement refcount for object. + * @kref: object. + * @release: pointer to the function that will clean up the object when the + * last reference to the object is released. + * This pointer is required, and it is not acceptable to pass kfree + * in as this function. If the caller does pass kfree to this + * function, you will be publicly mocked mercilessly by the kref + * maintainer, and anyone else who happens to notice it. You have + * been warned. + * + * Decrement the refcount, and if 0, call release(). + * Return 1 if the object was removed, otherwise return 0. Beware, if this + * function returns 0, you still can not count on the kref from remaining in + * memory. Only use the return value if you want to see if the kref is now + * gone, not present. + */ +static inline int mcctrl_kref_put(struct kref *kref, void (*release)(struct kref *kref)) +{ + if (atomic_dec_return(&kref->refcount.refs) == MCKERNEL_KREF_MARK) { + release(kref); + return 1; + } + return 0; +} + +static int tof_utofu_cq_cacheflush(struct tof_utofu_cq *ucq){ + return mcctrl_tof_core_cq_cacheflush(ucq->tni, ucq->cqid); +} + +static void tof_utofu_trans_mru_delete(struct tof_utofu_cq *ucq, int stag){ + struct tof_utofu_trans_list *mru = ucq->trans.mru; + int prev = mru[stag].prev; + int next = mru[stag].next; + if(prev == TOF_UTOFU_MRU_EMPTY || next == TOF_UTOFU_MRU_EMPTY){ /* already deleted */ + return; + } + if(prev == stag){ /* a single entry */ + ucq->trans.mruhead = TOF_UTOFU_MRU_EMPTY; + }else{ + if(ucq->trans.mruhead == stag){ + ucq->trans.mruhead = next; + } + mru[prev].next = next; + mru[next].prev = prev; + } + mru[stag].prev = TOF_UTOFU_MRU_EMPTY; + mru[stag].next = TOF_UTOFU_MRU_EMPTY; +} + +static void tof_utofu_trans_disable(struct tof_utofu_cq *ucq, int stag){ + struct tof_trans_table *table = ucq->trans.table; + atomic64_set((atomic64_t *)&table[stag], 0); + tof_utofu_trans_mru_delete(ucq, stag); +} + +/* McKernel scatterlist is simply a contiguous buffer. */ +struct scatterlist { + void *pages; + unsigned int offset; + unsigned int length; + unsigned long dma_address; + unsigned int dma_length; +}; + +static uintptr_t tof_utofu_disable_mbpt(struct tof_utofu_mbpt *mbpt, int idx){ + int i0, i1; + struct tof_icc_mbpt_entry *ent; + uintptr_t ipa; + i0 = idx / (PAGE_SIZE / TOF_ICC_MBPT_SIZE); + i1 = idx - i0 * (PAGE_SIZE / TOF_ICC_MBPT_SIZE); + //ent = sg_virt(&mbpt->sg[i0]); + ent = mbpt->sg->pages + (i0 * PAGE_SIZE); + if(!ent[i1].enable){ + return 0; + } + ent[i1].enable = 0; + ipa = (uint64_t)ent[i1].ipa << 12; + ent[i1].ipa = 0; + return ipa; +} + +static void tof_utofu_free_mbpt(struct tof_utofu_cq *ucq, struct tof_utofu_mbpt *mbpt){ + int i; + + for(i = 0; i < mbpt->nsgents * PAGE_SIZE / sizeof(struct tof_icc_mbpt_entry); i++){ + uintptr_t iova; + iova = tof_utofu_disable_mbpt(mbpt, i); +#if 0 + /* + * NOTE: Not performed for McKernel managed stags. + */ + if(iova){ + tof_smmu_release_ipa_cq(ucq->tni, ucq->cqid, iova, mbpt->pgsz); + } +#endif + } + +#if 0 + /* + * NOTE: Everyhing below has been allocated in McKernel, do nothing here!! + * This leaks memory in McKernel, but it doesn't crash Linux. + * Memory will be released once McKernel is unbooted. + */ + tof_smmu_iova_unmap_sg(ucq->tni, ucq->cqid, mbpt->sg, mbpt->nsgents); + + for(i = 0; i < mbpt->nsgents; i++){ + tof_util_free_pages((unsigned long)sg_virt(&mbpt->sg[i]), 0); + } + tof_util_free(mbpt->sg); + tof_util_free(mbpt); +#endif +} + +static void tof_utofu_mbpt_release(struct kref *kref) +{ + struct tof_utofu_mbpt *mbpt = container_of(kref, struct tof_utofu_mbpt, kref); + //atomic64_inc((atomic64_t *)&kref_free_count); + tof_utofu_free_mbpt(mbpt->ucq, mbpt); +} + +static int tof_utofu_free_stag(struct tof_utofu_cq *ucq, int stag){ + if(stag < 0 || stag >= TOF_UTOFU_NUM_STAG(ucq->num_stag) || + ucq->steering == NULL){ + return -EINVAL; + } + if(!(ucq->steering[stag].enable)){ + return -ENOENT; + } + if (!mcctrl_kref_is_mckernel(&ucq->trans.mru[stag].mbpt->kref)) { + printk("%s: stag: %d is not an McKernel kref\n", __func__, stag); + return -EINVAL; + } + ucq->steering[stag].enable = 0; + ucq->mb[stag].enable = 0; + tof_utofu_trans_disable(ucq, stag); + dma_wmb(); + tof_utofu_cq_cacheflush(ucq); + mcctrl_kref_put(&ucq->trans.mru[stag].mbpt->kref, tof_utofu_mbpt_release); + ucq->trans.mru[stag].mbpt = NULL; + dprintk("%s: TNI: %d, CQ: %d: stag %d deallocated\n", + __func__, ucq->tni, ucq->cqid, stag); + return 0; +} + +void mcctrl_mckernel_tof_utofu_release_cq(void *pde_data) +{ + struct tof_utofu_cq *ucq; + struct tof_utofu_device *dev; + unsigned long irqflags; + int stag; + + dev = (struct tof_utofu_device *)pde_data; + ucq = container_of(dev, struct tof_utofu_cq, common); + + if (!ucq->common.enabled) { + return; + } + + dprintk("%s: UCQ (PDE: 0x%lx) TNI %d CQ %d\n", + __func__, (unsigned long)pde_data, ucq->tni, ucq->cqid); + + /* + * Only release stags here, actual cleanup is still performed + * in the Tofu driver + */ + for (stag = 0; stag < TOF_UTOFU_NUM_STAG(ucq->num_stag); stag++) { + spin_lock_irqsave(&ucq->trans.mru_lock, irqflags); + tof_utofu_free_stag(ucq, stag); + spin_unlock_irqrestore(&ucq->trans.mru_lock, irqflags); + } +} + +static inline void tof_core_unregister_signal_bg(int tni, int bgid) +{ + return mcctrl_tof_core_register_signal_bg(tni, bgid, NULL); +} + +static struct tof_utofu_bg *tof_utofu_bg_get(int tni, int bgid){ + if((unsigned int)tni >= TOF_ICC_NTNIS || + (unsigned int)bgid >= TOF_ICC_NBGS){ + return NULL; + } + //return &tof_utofu_bg[tni][bgid]; + + // Convert [][] notion into pointer aritmethic + return mcctrl_tof_utofu_bg + (tni * TOF_ICC_NBGS) + bgid; +} + +static int __tof_utofu_unset_bg(struct tof_utofu_bg *ubg){ + if(ubg->common.enabled){ + mcctrl_tof_core_unset_bg(ubg->tni, ubg->bgid); + ubg->common.enabled = false; + tof_core_unregister_signal_bg(ubg->tni, ubg->bgid); + } + return 0; +} + +static int mcctrl_tof_utofu_disable_bch(struct tof_utofu_bg *ubg){ + int ret; + int tni, bgid; + + if(!ubg->bch.enabled){ + return -EPERM; + } + + ret = mcctrl_tof_core_disable_bch(ubg->tni, ubg->bgid); + if(ret < 0){ + return ret; + } + + for(tni = 0; tni < TOF_ICC_NTNIS; tni++){ + uint64_t mask = ubg->bch.bgmask[tni]; + for(bgid = 0; bgid < TOF_ICC_NBGS; bgid++){ + if((mask >> bgid) & 1){ + ret = __tof_utofu_unset_bg(tof_utofu_bg_get(tni, bgid)); + if(ret < 0){ + /* OK? */ + //BUG(); + return ret; + } + } + } + } + + /* Not performed in McKernel handler */ + //tof_smmu_release_ipa_bg(ubg->tni, ubg->bgid, ubg->bch.iova, TOF_ICC_BCH_DMA_ALIGN); + //put_page(ubg->bch.page); + ubg->bch.enabled = false; + smp_mb(); + dprintk("%s: tni=%d bgid=%d\n", __func__, ubg->tni, ubg->bgid); + return 0; +} + +void mcctrl_mckernel_tof_utofu_release_bch(void *pde_data) +{ + struct tof_utofu_bg *ubg; + struct tof_utofu_device *dev = (struct tof_utofu_device *)pde_data; + + ubg = container_of(dev, struct tof_utofu_bg, common); + //tof_log_if("tni=%d bgid=%d\n", ubg->tni, ubg->bgid); + dprintk("%s: tni=%d bgid=%d\n", __func__, ubg->tni, ubg->bgid); + mcctrl_tof_utofu_disable_bch(ubg); +} + +void mcctrl_tofu_cleanup_file(struct mcctrl_file_to_pidfd *f2pfd) +{ + /* Figure out whether CQ or BCH */ + if (strstr(f2pfd->tofu_dev_path, "cq")) { + dprintk("%s: PID: %d, fd: %d (%s) -> release CQ\n", + __func__, f2pfd->pid, f2pfd->fd, f2pfd->tofu_dev_path); + mcctrl_mckernel_tof_utofu_release_cq(f2pfd->pde_data); + } + + else if (strstr(f2pfd->tofu_dev_path, "bch")) { + dprintk("%s: PID: %d, fd: %d (%s) -> release BCH\n", + __func__, f2pfd->pid, f2pfd->fd, f2pfd->tofu_dev_path); + mcctrl_mckernel_tof_utofu_release_bch(f2pfd->pde_data); + } +} + + int __mcctrl_tof_utofu_release_handler(struct inode *inode, struct file *filp, int (*__release_func)(struct inode *inode, struct file *filp)) { @@ -528,28 +918,22 @@ int __mcctrl_tof_utofu_release_handler(struct inode *inode, struct file *filp, ret = mcctrl_ikc_send_wait(f2pfd->os, ppd->ikc_target_cpu, &isp, -20, NULL, NULL, 0); if (ret != 0) { - dprintk("%s: WARNING: failed to send IKC msg: %d\n", - __func__, ret); + pr_err("%s: WARNING: IKC req for PID: %d, fd: %d failed\n", + __func__, f2pfd->pid, f2pfd->fd); } + /* Disable any remaining STAGs/BCH in mcctrl anyway */ + mcctrl_tofu_cleanup_file(f2pfd); + mcctrl_file_to_pidfd_hash_remove(filp, f2pfd->os, current->group_leader, f2pfd->fd); mcctrl_put_per_proc_data(ppd); - /* Do not call into Linux driver if timed out in SIGKILL.. */ - if (ret == -ETIME && __fatal_signal_pending(current)) { - pr_err("%s: WARNING: failed to send IKC msg in SIGKILL: %d\n", - __func__, ret); - goto out_no_release; - } out: dprintk("%s: current PID: %d, comm: %s -> calling release\n", __func__, task_tgid_vnr(current), current->comm); return __release_func(inode, filp); - -out_no_release: - return ret; } int __mcctrl_tof_utofu_release_cq(struct inode *inode, struct file *filp) @@ -563,4 +947,70 @@ int __mcctrl_tof_utofu_release_bch(struct inode *inode, struct file *filp) return __mcctrl_tof_utofu_release_handler(inode, filp, mcctrl_tof_utofu_release_bch); } + +/* + * Tofu MMU notifier functions + */ +void __mcctrl_tof_utofu_mn_invalidate_range_end( + struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, + unsigned long end) +{ + char tmpname[TASK_COMM_LEN]; + + /* Not an offloaded syscall? */ + if (current->mm != mm) { + goto out_call_real; + } + + /* Not mcexec? Just in case.. */ + get_task_comm(tmpname, current); + if (strncmp(tmpname, "mcexec", TASK_COMM_LEN)) { + goto out_call_real; + } + + /* This is only called for Tofu enabled mcexec processes */ + dprintk("%s: skipping tof_utofu_mn_invalidate_range_end() " + "for mcexec PID %d\n", + __func__, task_tgid_vnr(current)); + return; + +out_call_real: + return mcctrl_tof_utofu_mn_invalidate_range_end(mn, mm, start, end); +} + +int __mcctrl_tof_utofu_ioctl_init_cq(struct tof_utofu_device *dev, + unsigned long arg) { + struct tof_utofu_cq *ucq; + + ucq = container_of(dev, struct tof_utofu_cq, common); + if (!ucq->common.enabled) { + return -EINVAL; + } + + dprintk("%s: Tofu TNI %d CQ %d (PDE: 0x%lx) MMU notifier to be hijacked\n", + __func__, ucq->tni, ucq->cqid, (unsigned long)dev); + /* Override the MMU notifier */ + ucq->mn.ops = &__mcctrl_tof_utofu_mn_ops; + + return 0; +} + +long __mcctrl_tof_utofu_unlocked_ioctl_cq(void *pde_data, unsigned int cmd, + unsigned long arg) { + struct tof_utofu_device *dev = (struct tof_utofu_device *)pde_data; + int ret; + + switch (cmd) { + /* We only care about init, where we hijack the MMU notifier */ + case TOF_IOCTL_INIT_CQ: + ret = __mcctrl_tof_utofu_ioctl_init_cq(dev, arg); + break; + default: + ret = 0; + } + + return ret; +} #endif diff --git a/executer/kernel/mcctrl/arch/arm64/include/tofu/generate_headers.sh b/executer/kernel/mcctrl/arch/arm64/include/tofu/generate_headers.sh new file mode 100755 index 00000000..7e094e0b --- /dev/null +++ b/executer/kernel/mcctrl/arch/arm64/include/tofu/generate_headers.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +SCRIPT="`readlink -f ${BASH_SOURCE[0]:-}`" +SCRIPT_DIR=$(dirname ${SCRIPT}) +CURRENT_DIR=`pwd` + +cd ${SCRIPT_DIR} + +DWARF_TOOL=${SCRIPT_DIR}/../../../../../../../tools/dwarf-extract-struct/dwarf-extract-struct +if [ ! -x ${DWARF_TOOL} ]; then + echo "error: couldn't find DWARF extractor executable (${DWARF_TOOL}), have you compiled it?" + cd - + exit 1 +fi + +echo "Looking for Tofu driver debug symbols..." +if [ "`find /lib/modules/ -name "tof_module.tar.gz" | xargs -r ls -t | head -n 1 | wc -l`" == "0" ]; then + echo "error: couldn't find Tofu modules with debug symbols" + cd - + exit 1 +fi + +MODULE_TAR_GZ=`find /lib/modules/ -name "tof_module.tar.gz" | xargs ls -t | head -n 1` +echo "Using Tofu driver debug symbols: ${MODULE_TAR_GZ}" + +KMODULE=tof_utofu.ko +if ! tar zxvf ${MODULE_TAR_GZ} ${KMODULE} 2>&1 > /dev/null; then + echo "error: uncompressing kernel module with debug symbols" + cd - + exit 1 +fi + +${DWARF_TOOL} ${KMODULE} tof_utofu_device enabled subnet gpid > tofu_generated-tof_utofu_device.h +${DWARF_TOOL} ${KMODULE} tof_utofu_cq common tni cqid mn trans steering mb num_stag | sed "s/struct FILL_IN_MANUALLY trans;/#include \"tof_utofu_cq_trans.h\"/g" > tofu_generated-tof_utofu_cq.h +${DWARF_TOOL} ${KMODULE} tof_utofu_mbpt ucq iova sg nsgents mbptstart pgsz kref > tofu_generated-tof_utofu_mbpt.h +${DWARF_TOOL} ${KMODULE} tof_utofu_bg common tni bgid bch | sed "s/struct FILL_IN_MANUALLY bch;/#include \"tof_utofu_bg_bch.h\"/g" > tofu_generated-tof_utofu_bg.h +rm ${KMODULE} + + +#cat tofu_generated*.h +cd - > /dev/null diff --git a/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_icc.h b/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_icc.h new file mode 100644 index 00000000..9b887d5b --- /dev/null +++ b/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_icc.h @@ -0,0 +1,831 @@ +#ifndef _TOF_ICC_H_ +#define _TOF_ICC_H_ + +#include +#ifdef __KERNEL__ +#include +#else +#include +typedef uint64_t phys_addr_t; +#endif + +/* constants related to the Tofu Interconnect D */ + +#define TOF_ICC_NTNIS 6 +#define TOF_ICC_NCQS 12 +#define TOF_ICC_NBGS 48 +#define TOF_ICC_NBCHS 16 +#define TOF_ICC_NPORTS 10 +#define TOF_ICC_NVMSIDS 16 + +#define TOF_ICC_RH_LEN 8 +#define TOF_ICC_ECRC_LEN 4 +#define TOF_ICC_FRAME_ALIGN 32 +#define TOF_ICC_TLP_LEN(len) (((len) + 1) * TOF_ICC_FRAME_ALIGN) +#define TOF_ICC_TLP_PAYLOAD_MAX (TOF_ICC_TLP_LEN(61) - TOF_ICC_ECRC_LEN) +#define TOF_ICC_FRAME_LEN(len) (TOF_ICC_RH_LEN + TOF_ICC_TLP_LEN(len)) +#define TOF_ICC_FRAME_LEN_MIN TOF_ICC_FRAME_LEN(2) +#define TOF_ICC_FRAME_LEN_MAX TOF_ICC_FRAME_LEN(61) +#define TOF_ICC_FRAME_BUF_SIZE_BITS 11 +#define TOF_ICC_FRAME_BUF_SIZE (1 << TOF_ICC_FRAME_BUF_SIZE_BITS) +#define TOF_ICC_FRAME_BUF_ALIGN_BITS 8 +#define TOF_ICC_FRAME_BUF_ALIGN (1 << TOF_ICC_FRAME_BUF_ALIGN_BITS) +#define TOF_ICC_PB_SIZE_BITS 11 +#define TOF_ICC_PB_SIZE (1 << TOF_ICC_PB_SIZE_BITS) +#define TOF_ICC_PB_ALIGN_BITS 11 +#define TOF_ICC_PB_ALIGN (1 << TOF_ICC_PB_ALIGN_BITS) + +#define TOF_ICC_ST_ALIGN_BITS 8 +#define TOF_ICC_ST_ALIGN (1 << TOF_ICC_ST_ALIGN_BITS) + +#define TOF_ICC_MBT_ALIGN_BITS 8 +#define TOF_ICC_MBT_ALIGN (1 << TOF_ICC_MBT_ALIGN_BITS) + +#define TOF_ICC_MBPT_ALIGN_BITS 8 +#define TOF_ICC_MBPT_ALIGN (1 << TOF_ICC_MBPT_ALIGN_BITS) + +#define TOF_ICC_BG_BSEQ_SIZE_BITS 24 +#define TOF_ICC_BG_BSEQ_SIZE (1 << TOF_ICC_BG_BSEQ_SIZE_BITS) + +#define TOF_ICC_BCH_DMA_ALIGN_BITS 8 +#define TOF_ICC_BCH_DMA_ALIGN (1 << TOF_ICC_BCH_DMA_ALIGN_BITS) + +/* this is a CPU-specific constant, but referred in the ICC spec. */ +#define TOF_ICC_CACHE_LINE_SIZE_BITS 8 +#define TOF_ICC_CACHE_LINE_SIZE (1 << TOF_ICC_CACHE_LINE_SIZE_BITS) + +#define TOF_ICC_TOQ_DESC_SIZE_BITS 5 +#define TOF_ICC_TOQ_DESC_SIZE (1 << TOF_ICC_TOQ_DESC_SIZE_BITS) +#define TOF_ICC_TCQ_DESC_SIZE_BITS 3 +#define TOF_ICC_TCQ_DESC_SIZE (1 << TOF_ICC_TCQ_DESC_SIZE_BITS) +#define TOF_ICC_TCQ_NLINE_BITS (TOF_ICC_CACHE_LINE_SIZE_BITS - TOF_ICC_TCQ_DESC_SIZE_BITS) +#define TOF_ICC_MRQ_DESC_SIZE_BITS 5 +#define TOF_ICC_MRQ_DESC_SIZE (1 << TOF_ICC_MRQ_DESC_SIZE_BITS) +#define TOF_ICC_PBQ_DESC_SIZE_BITS 3 +#define TOF_ICC_PBQ_DESC_SIZE (1 << TOF_ICC_PBQ_DESC_SIZE_BITS) +#define TOF_ICC_PRQ_DESC_SIZE_BITS 3 +#define TOF_ICC_PRQ_DESC_SIZE (1 << TOF_ICC_PRQ_DESC_SIZE_BITS) +#define TOF_ICC_PRQ_NLINE_BITS (TOF_ICC_CACHE_LINE_SIZE_BITS - TOF_ICC_PBQ_DESC_SIZE_BITS) + +#define TOF_ICC_TOQ_SIZE_NTYPES 6 +#define TOF_ICC_TOQ_SIZE_BITS(size) ((size) * 2 + 11) +#define TOF_ICC_TOQ_SIZE(size) (1 << TOF_ICC_TOQ_SIZE_BITS(size)) +#define TOF_ICC_TOQ_LEN(size) (TOF_ICC_TOQ_SIZE(size) * TOF_ICC_TOQ_DESC_SIZE) +#define TOF_ICC_TCQ_LEN(size) (TOF_ICC_TOQ_SIZE(size) * TOF_ICC_TCQ_DESC_SIZE) + +#define TOF_ICC_MRQ_SIZE_NTYPES 6 +#define TOF_ICC_MRQ_SIZE_BITS(size) ((size) * 2 + 11) +#define TOF_ICC_MRQ_SIZE(size) (1 << TOF_ICC_MRQ_SIZE_BITS(size)) +#define TOF_ICC_MRQ_LEN(size) (TOF_ICC_MRQ_SIZE(size) * TOF_ICC_MRQ_DESC_SIZE) + +#define TOF_ICC_PBQ_SIZE_NTYPES 6 +#define TOF_ICC_PBQ_SIZE_BITS(size) ((size) * 2 + 11) +#define TOF_ICC_PBQ_SIZE(size) (1 << TOF_ICC_PBQ_SIZE_BITS(size)) +#define TOF_ICC_PBQ_LEN(size) (TOF_ICC_PBQ_SIZE(size) * TOF_ICC_PBQ_DESC_SIZE) + +#define TOF_ICC_PRQ_SIZE_NTYPES 6 +#define TOF_ICC_PRQ_SIZE_BITS(size) ((size) * 2 + 11) +#define TOF_ICC_PRQ_SIZE(size) (1 << TOF_ICC_PRQ_SIZE_BITS(size)) +#define TOF_ICC_PRQ_LEN(size) (TOF_ICC_PRQ_SIZE(size) * TOF_ICC_PRQ_DESC_SIZE) + +#define TOF_ICC_STEERING_TABLE_ALIGN_BITS 8 +#define TOF_ICC_STEERING_TABLE_ALIGN (1 << TOF_ICC_STEERING_TABLE_ALIGN_BITS) +#define TOF_ICC_STEERING_SIZE_BITS 4 +#define TOF_ICC_STEERING_SIZE (1 << TOF_ICC_STEERING_SIZE_BITS) + +#define TOF_ICC_MB_TABLE_ALIGN_BITS 8 +#define TOF_ICC_MB_TABLE_ALIGN (1 << TOF_ICC_MB_TABLE_ALIGN_BITS) +#define TOF_ICC_MB_SIZE_BITS 4 +#define TOF_ICC_MB_SIZE (1 << TOF_ICC_MB_SIZE_BITS) +#define TOF_ICC_MB_PS_ENCODE(bits) ((bits) % 9 == 3 ? (bits) / 9 - 1 : (bits) / 13 + 3) + +#define TOF_ICC_MBPT_ALIGN_BITS 8 +#define TOF_ICC_MBPT_ALIGN (1 << TOF_ICC_MBPT_ALIGN_BITS) +#define TOF_ICC_MBPT_SIZE_BITS 3 +#define TOF_ICC_MBPT_SIZE (1 << TOF_ICC_MBPT_SIZE_BITS) + +#define TOF_ICC_X_BITS 5 +#define TOF_ICC_Y_BITS 5 +#define TOF_ICC_Z_BITS 5 +#define TOF_ICC_A_BITS 1 +#define TOF_ICC_B_BITS 2 +#define TOF_ICC_C_BITS 1 +#define TOF_ICC_MAX_X_SIZE (1 << TOF_ICC_X_BITS) +#define TOF_ICC_MAX_Y_SIZE (1 << TOF_ICC_Y_BITS) +#define TOF_ICC_MAX_Z_SIZE (1 << TOF_ICC_Z_BITS) +#define TOF_ICC_A_SIZE 2 +#define TOF_ICC_B_SIZE 3 +#define TOF_ICC_C_SIZE 2 +#define TOF_ICC_X_MASK ((1 << TOF_ICC_X_BITS) - 1) +#define TOF_ICC_Y_MASK ((1 << TOF_ICC_Y_BITS) - 1) +#define TOF_ICC_Z_MASK ((1 << TOF_ICC_Z_BITS) - 1) +#define TOF_ICC_A_MASK ((1 << TOF_ICC_A_BITS) - 1) +#define TOF_ICC_B_MASK ((1 << TOF_ICC_B_BITS) - 1) +#define TOF_ICC_C_MASK ((1 << TOF_ICC_C_BITS) - 1) +#define TOF_ICC_ABC_SIZE (TOF_ICC_A_SIZE * TOF_ICC_B_SIZE * TOF_ICC_C_SIZE) + +static inline int tof_icc_get_framelen(int len){ + len = TOF_ICC_RH_LEN + round_up(len + TOF_ICC_ECRC_LEN, TOF_ICC_FRAME_ALIGN); + if(len < TOF_ICC_FRAME_LEN_MIN){ + len = TOF_ICC_FRAME_LEN_MIN; + } + return len; +} + +/** Descriptors **/ +/** commands and rcodes **/ +enum { + TOF_ICC_TOQ_NOP, + TOF_ICC_TOQ_PUT, + TOF_ICC_TOQ_WRITE_PIGGYBACK_BUFFER, + TOF_ICC_TOQ_PUT_PIGGYBACK, + TOF_ICC_TOQ_GET, + TOF_ICC_TOQ_GETL, + TOF_ICC_TOQ_ATOMIC_READ_MODIFY_WRITE = 0xe, + TOF_ICC_TOQ_TRANSMIT_RAW_PACKET1 = 0x10, + TOF_ICC_TOQ_TRANSMIT_RAW_PACKET2, + TOF_ICC_TOQ_TRANSMIT_SYSTEM_PACKET1, + TOF_ICC_TOQ_TRANSMIT_SYSTEM_PACKET2, + + TOF_ICC_TOQ_NCOMMANDS, +}; + +enum { + TOF_ICC_MRQ_ATOMIC_READ_MODIFY_WRITE_HALFWAY_NOTICE = 0x1, + TOF_ICC_MRQ_ATOMIC_READ_MODIFY_WRITE_NOTICE, + TOF_ICC_MRQ_ATOMIC_READ_MODIFY_WRITE_REMOTE_ERROR, + TOF_ICC_MRQ_PUT_HALFWAY_NOTICE, + TOF_ICC_MRQ_PUT_LAST_HALFWAY_NOTICE, + TOF_ICC_MRQ_GET_HALFWAY_NOTICE, + TOF_ICC_MRQ_GET_LAST_HALFWAY_NOTICE, + TOF_ICC_MRQ_PUT_NOTICE, + TOF_ICC_MRQ_PUT_LAST_NOTICE, + TOF_ICC_MRQ_GET_NOTICE, + TOF_ICC_MRQ_GET_LAST_NOTICE, + TOF_ICC_MRQ_PUT_REMOTE_ERROR, + TOF_ICC_MRQ_PUT_LAST_REMOTE_ERROR, + TOF_ICC_MRQ_GET_REMOTE_ERROR, + TOF_ICC_MRQ_GET_LAST_REMOTE_ERROR, + + TOF_ICC_MRQ_NCOMMANDS, +}; + +enum { + TOF_ICC_PRQ_UNKNOWN_TLP, + TOF_ICC_PRQ_SYSTEM_TLP, + TOF_ICC_PRQ_ADDRESS_RANGE_EXCEPTION = 0x6, + TOF_ICC_PRQ_CQ_EXCEPTION = 0x8, + TOF_ICC_PRQ_ILLEGAL_TLP_FLAGS, + TOF_ICC_PRQ_ILLEGAL_TLP_LENGTH, + TOF_ICC_PRQ_CQ_ERROR = 0xc, +}; + +/** structures **/ +struct tof_icc_steering_entry { + uint64_t res1:6; + uint64_t readonly:1; + uint64_t enable:1; + uint64_t mbva:32; + uint64_t res2:8; + uint64_t mbid:16; + uint64_t length; /* for optimization */ +}; + +struct tof_icc_mb_entry { + uint64_t ps:3; + uint64_t res1:4; + uint64_t enable:1; + uint64_t ipa:32; + uint64_t res2:24; + uint64_t npage; /* for optimization */ +}; + +struct tof_icc_mbpt_entry { + uint64_t res1:7; + uint64_t enable:1; + uint64_t res2:4; + uint64_t ipa:28; + uint64_t res3:24; +}; + +struct tof_icc_cq_stag_offset { + uint64_t offset:40; + uint64_t stag:18; + uint64_t cqid:6; +}; + +struct tof_icc_toq_common_header1 { + uint8_t interrupt:1; + uint8_t res1:4; + uint8_t source_type:2; + uint8_t flip:1; + uint8_t command; + union { + uint8_t mtu; + struct { + uint8_t res:4; + uint8_t op:4; + } armw; + } mtuop; + uint8_t sps:4; + uint8_t pa:1; + uint8_t pb:2; + uint8_t pc:1; + uint8_t rx; + uint8_t ry; + uint8_t rz; + uint8_t ra:1; + uint8_t rb:2; + uint8_t rc:1; + uint8_t res3:1; + uint8_t ri:3; +}; + +struct tof_icc_toq_common_header2 { + uint8_t gap; + uint8_t s:1; + uint8_t r:1; + uint8_t q:1; + uint8_t p:1; + uint8_t res1:1; + uint8_t j:1; + uint8_t res2:2; + uint16_t edata; + union{ + struct { + uint32_t length:24; + uint32_t res:8; + } normal; + struct { + uint32_t length:6; + uint32_t res:26; + } piggyback; + } len; +}; + +struct tof_icc_toq_descriptor { + struct tof_icc_toq_common_header1 head1; + uint64_t res[3]; +}; + +struct tof_icc_toq_nop { + struct tof_icc_toq_common_header1 head1; + uint64_t res[3]; +}; + +struct tof_icc_toq_put { + struct tof_icc_toq_common_header1 head1; + struct tof_icc_toq_common_header2 head2; + struct tof_icc_cq_stag_offset remote; + struct tof_icc_cq_stag_offset local; +}; + +struct tof_icc_toq_write_piggyback_buffer { + struct tof_icc_toq_common_header1 head1; + uint64_t data[3]; +}; + +struct tof_icc_toq_put_piggyback { + struct tof_icc_toq_common_header1 head1; + struct tof_icc_toq_common_header2 head2; + struct tof_icc_cq_stag_offset remote; + uint64_t data; +}; + +struct tof_icc_toq_get { + struct tof_icc_toq_common_header1 head1; + struct tof_icc_toq_common_header2 head2; + struct tof_icc_cq_stag_offset remote; + struct tof_icc_cq_stag_offset local; +}; + +struct tof_icc_toq_atomic_read_modify_write { + struct tof_icc_toq_common_header1 head1; + struct tof_icc_toq_common_header2 head2; + struct tof_icc_cq_stag_offset remote; + uint64_t data; +}; + +struct tof_icc_toq_transmit_raw_packet1 { + struct tof_icc_toq_common_header1 head1; + uint8_t gap; + uint8_t res4[3]; + uint32_t length:12; + uint32_t res5:20; + uint64_t res6; + uint64_t pa:48; /* for optimization */ + uint64_t res7:16; +}; + +struct tof_icc_toq_transmit_raw_packet2 { + uint8_t interrupt:1; + uint8_t res1:4; + uint8_t source_type:2; + uint8_t flip:1; + uint8_t command; + uint8_t res2:7; + uint8_t e:1; + uint8_t res3[4]; + uint8_t port:5; + uint8_t res4:1; + uint8_t vc:2; + uint8_t gap; + uint8_t res5[3]; + uint32_t length:12; + uint32_t res6:20; + uint64_t res7; + uint64_t pa:48; /* for optimization */ + uint64_t res8:16; +}; + +struct tof_icc_toq_transmit_system_packet { + struct tof_icc_toq_common_header1 head1; /* rx, ry, rz should be rdx, rdy, rdz */ + uint8_t gap; + uint8_t res4[3]; + uint32_t length:12; + uint32_t res5:20; + uint64_t res6; + uint64_t pa:48; /* for optimization */ + uint64_t res7:16; +}; + +struct tof_icc_tcq_descriptor { + uint8_t res1:5; + uint8_t counter_unmatch:1; + uint8_t res2:1; + uint8_t flip:1; + uint8_t rcode; + uint8_t res3[2]; + union{ + struct { + uint32_t length:24; + uint32_t res:8; + } normal; + struct { + uint32_t length:6; + uint32_t res:26; + } piggyback; + } len; +}; + +struct tof_icc_mrq_common_header1 { + uint8_t res1:7; + uint8_t flip:1; + uint8_t id; + uint8_t rcode; + uint8_t res2:4; + uint8_t pa:1; + uint8_t pb:2; + uint8_t pc:1; + uint8_t x; + uint8_t y; + uint8_t z; + uint8_t a:1; + uint8_t b:2; + uint8_t c:1; + uint8_t res3:1; + uint8_t i:3; +}; + +struct tof_icc_mrq_common_header2 { + uint8_t res1; + uint8_t res2:4; + uint8_t initial:1; + uint8_t res3:3; + uint16_t edata; + union { + struct { + uint32_t length:11; + uint32_t res:21; + } normal; + struct { + uint32_t op:4; + uint32_t res:28; + } armw; + } lenop; +}; + +struct tof_icc_mrq_atomic_read_modify_write_halfway_notice { + struct tof_icc_mrq_common_header1 head1; + struct tof_icc_mrq_common_header2 head2; + struct tof_icc_cq_stag_offset local; + struct tof_icc_cq_stag_offset remote; +}; + +struct tof_icc_mrq_descriptor { + struct tof_icc_mrq_common_header1 head1; + struct tof_icc_mrq_common_header2 head2; + struct tof_icc_cq_stag_offset cso1; + struct tof_icc_cq_stag_offset cso2; +}; + +struct tof_icc_pbq_descriptor { + uint64_t res1:7; + uint64_t f:1; + uint64_t res2:3; + uint64_t pa:29; + uint64_t res3:24; +}; + +struct tof_icc_prq_descriptor { + uint64_t rcode:7; + uint64_t f:1; + uint64_t res1:3; + uint64_t pa:29; + uint64_t res2:8; + uint64_t w:1; + uint64_t res3:5; + uint64_t l:1; + uint64_t e:1; + uint64_t res4:8; +}; + + +/** Registers **/ +/* useful packed structures */ +struct tof_icc_reg_subnet { + uint64_t lz:6; + uint64_t sz:6; + uint64_t nz:6; + uint64_t ly:6; + uint64_t sy:6; + uint64_t ny:6; + uint64_t lx:6; + uint64_t sx:6; + uint64_t nx:6; + uint64_t res:10; +}; + +struct tof_icc_reg_bg_address { + uint32_t bgid:6; + uint32_t tni:3; + uint32_t c:1; + uint32_t b:2; + uint32_t a:1; + uint32_t z:5; + uint32_t y:5; + uint32_t x:5; + uint32_t pc:1; + uint32_t pb:2; + uint32_t pa:1; +}; + +/* relative offset of interrupt controller registers */ +#define TOF_ICC_IRQREG_IRR 0x0 +#define TOF_ICC_IRQREG_IMR 0x8 +#define TOF_ICC_IRQREG_IRC 0x10 +#define TOF_ICC_IRQREG_IMC 0x18 +#define TOF_ICC_IRQREG_ICL 0x20 + +/* TOFU REGISTERS */ +#define tof_icc_reg_pa 0x40000000 + +/* CQ */ +#define TOF_ICC_REG_CQ_PA(tni, cqid) (tof_icc_reg_pa + 0 + (tni) * 0x1000000 + (cqid) * 0x10000) +#define TOF_ICC_REG_CQ_TOQ_DIRECT_DESCRIPTOR 0x0 +#define TOF_ICC_REG_CQ_TOQ_FETCH_START 0x40 +#define TOF_ICC_REG_CQ_MRQ_FULL_POINTER 0x48 +#define TOF_ICC_REG_CQ_TOQ_PIGGYBACK_BUFFER0 0x50 +#define TOF_ICC_REG_CQ_TOQ_PIGGYBACK_BUFFER1 0x58 +#define TOF_ICC_REG_CQ_TOQ_PIGGYBACK_BUFFER2 0x60 +#define TOF_ICC_REG_CQ_TCQ_NUM_NOTICE 0x68 +#define TOF_ICC_REG_CQ_MRQ_NUM_NOTICE 0x70 +#define TOF_ICC_REG_CQ_TX_PAYLOAD_BYTE 0x78 +#define TOF_ICC_REG_CQ_RX_PAYLOAD_BYTE 0x80 +#define TOF_ICC_REG_CQ_DUMP_START 0x0 +#define TOF_ICC_REG_CQ_DUMP_END 0x88 + +/* BCH */ +#define TOF_ICC_REG_BCH_PA(tni, bgid) (tof_icc_reg_pa + 0x0000e00000 + (tni) * 0x1000000 + (bgid) * 0x10000) +#define TOF_ICC_REG_BCH_IDATA 0x800 +#define TOF_ICC_REG_BCH_READY 0x840 +#define TOF_ICC_REG_BCH_READY_STATE BIT(63) +#define TOF_ICC_REG_BCH_IGNORED_SIGNAL_COUNT 0x848 +#define TOF_ICC_REG_BCH_DUMP_START 0x800 +#define TOF_ICC_REG_BCH_DUMP_END 0x850 + +/* CQS */ +#define TOF_ICC_REG_CQS_PA(tni, cqid) (tof_icc_reg_pa + 0x0000400000 + (tni) * 0x1000000 + (cqid) * 0x10000) +#define TOF_ICC_REG_CQS_STATUS 0x0 +#define TOF_ICC_REG_CQS_STATUS_DESCRIPTOR_PROCESS_STOP BIT(63) +#define TOF_ICC_REG_CQS_STATUS_DESCRIPTOR_FETCH_STOP BIT(62) +#define TOF_ICC_REG_CQS_STATUS_BLANK_ENTRY_FLIP_BIT BIT(61) +#define TOF_ICC_REG_CQS_STATUS_CACHE_FLUSH_BUSY BIT(60) +#define TOF_ICC_REG_CQS_STATUS_CQ_ENABLE BIT(59) +#define TOF_ICC_REG_CQS_STATUS_SESSION_DEAD BIT(58) +#define TOF_ICC_REG_CQS_STATUS_SESSION_OFFSET_OVERFLOW BIT(57) +#define TOF_ICC_REG_CQS_STATUS_SESSION_OFFSET GENMASK(56, 32) +#define TOF_ICC_REG_CQS_STATUS_NEXT_DESCRIPTOR_OFFSET GENMASK(29, 5) +#define TOF_ICC_REG_CQS_ENABLE 0x8 +#define TOF_ICC_REG_CQS_CACHE_FLUSH 0x10 +#define TOF_ICC_REG_CQS_FETCH_STOP 0x18 +#define TOF_ICC_REG_CQS_MODE 0x20 +#define TOF_ICC_REG_CQS_MODE_SYSTEM BIT(63) +#define TOF_ICC_REG_CQS_MODE_TRP2_ENABLE BIT(62) +#define TOF_ICC_REG_CQS_MODE_TRP1_ENABLE BIT(61) +#define TOF_ICC_REG_CQS_MODE_SESSION BIT(60) +#define TOF_ICC_REG_CQS_MODE_SUBNET_NX GENMASK(53, 48) +#define TOF_ICC_REG_CQS_MODE_SUBNET_SX GENMASK(47, 42) +#define TOF_ICC_REG_CQS_MODE_SUBNET_LX GENMASK(41, 36) +#define TOF_ICC_REG_CQS_MODE_SUBNET_NY GENMASK(35, 30) +#define TOF_ICC_REG_CQS_MODE_SUBNET_SY GENMASK(29, 24) +#define TOF_ICC_REG_CQS_MODE_SUBNET_LY GENMASK(23, 18) +#define TOF_ICC_REG_CQS_MODE_SUBNET_NZ GENMASK(17, 12) +#define TOF_ICC_REG_CQS_MODE_SUBNET_SZ GENMASK(11, 6) +#define TOF_ICC_REG_CQS_MODE_SUBNET_LZ GENMASK(5, 0) +#define TOF_ICC_REG_CQS_GPID 0x28 +#define TOF_ICC_REG_CQS_TOQ_IPA 0x30 +#define TOF_ICC_REG_CQS_TOQ_SIZE 0x38 +#define TOF_ICC_REG_CQS_TCQ_IPA 0x40 +#define TOF_ICC_REG_CQS_TCQ_IPA_CACHE_INJECTION BIT(63) +#define TOF_ICC_REG_CQS_MRQ_IPA 0x48 +#define TOF_ICC_REG_CQS_MRQ_IPA_CACHE_INJECTION BIT(63) +#define TOF_ICC_REG_CQS_MRQ_SIZE 0x50 +#define TOF_ICC_REG_CQS_MRQ_MASK 0x58 +#define TOF_ICC_REG_CQS_TCQ_DESCRIPTOR_COALESCING_TIMER 0x60 +#define TOF_ICC_REG_CQS_MRQ_DESCRIPTOR_COALESCING_TIMER 0x68 +#define TOF_ICC_REG_CQS_MRQ_INTERRUPT_COALESCING_TIMER 0x70 +#define TOF_ICC_REG_CQS_MRQ_INTERRUPT_COALESCING_COUNT 0x78 +#define TOF_ICC_REG_CQS_TOQ_DIRECT_SOURCE_COUNT 0x80 +#define TOF_ICC_REG_CQS_TOQ_DIRECT_DESCRIPTOR_COUNT 0x88 +#define TOF_ICC_REG_CQS_MEMORY_BLOCK_TABLE_ENABLE 0x90 +#define TOF_ICC_REG_CQS_MEMORY_BLOCK_TABLE_IPA 0x98 +#define TOF_ICC_REG_CQS_MEMORY_BLOCK_TABLE_SIZE 0xa0 +#define TOF_ICC_REG_CQS_STEERING_TABLE_ENABLE 0xa8 +#define TOF_ICC_REG_CQS_STEERING_TABLE_IPA 0xb0 +#define TOF_ICC_REG_CQS_STEERING_TABLE_SIZE 0xb8 +#define TOF_ICC_REG_CQS_MRQ_INTERRUPT_MASK 0xc0 +#define TOF_ICC_REG_CQS_IRR 0xc8 +#define TOF_ICC_REG_CQS_IMR 0xd0 +#define TOF_ICC_REG_CQS_IRC 0xd8 +#define TOF_ICC_REG_CQS_IMC 0xe0 +#define TOF_ICC_REG_CQS_ICL 0xe8 +#define TOF_ICC_REG_CQS_DUMP_START 0x0 +#define TOF_ICC_REG_CQS_DUMP_END 0xf0 + +/* BGS */ +#define TOF_ICC_REG_BGS_PA(tni, bgid) (tof_icc_reg_pa + 0x0000800000 + (tni) * 0x1000000 + (bgid) * 0x10000) +#define TOF_ICC_REG_BGS_ENABLE 0x0 +#define TOF_ICC_REG_BGS_IRR 0x8 +#define TOF_ICC_REG_BGS_IMR 0x10 +#define TOF_ICC_REG_BGS_IRC 0x18 +#define TOF_ICC_REG_BGS_IMC 0x20 +#define TOF_ICC_REG_BGS_ICL 0x28 +#define TOF_ICC_REG_BGS_STATE 0x30 +#define TOF_ICC_REG_BGS_STATE_ENABLE BIT(0) +#define TOF_ICC_REG_BGS_EXCEPTION_INFO_GPID_UNMATCH 0x38 +#define TOF_ICC_REG_BGS_EXCEPTION_INFO_GPID_UNMATCH_BG_ADDRESS GENMASK(27, 0) +#define TOF_ICC_REG_BGS_EXCEPTION_INFO_ADDRESS_UNMATCH 0x40 +#define TOF_ICC_REG_BGS_EXCEPTION_INFO_ADDRESS_UNMATCH_BG_ADDRESS GENMASK(27, 0) +#define TOF_ICC_REG_BGS_SIGNAL_A 0x48 +#define TOF_ICC_REG_BGS_SIGNAL_A_SIG_RECV BIT(63) +#define TOF_ICC_REG_BGS_SIGNAL_A_TLP_RECV BIT(62) +#define TOF_ICC_REG_BGS_SIGNAL_A_SIG_SEND BIT(61) +#define TOF_ICC_REG_BGS_SIGNAL_A_OP_TYPE GENMASK(3, 0) +#define TOF_ICC_REG_BGS_SIGNAL_B 0x50 +#define TOF_ICC_REG_BGS_SIGNAL_B_SIG_RECV BIT(63) +#define TOF_ICC_REG_BGS_SIGNAL_B_TLP_RECV BIT(62) +#define TOF_ICC_REG_BGS_SIGNAL_B_SIG_SEND BIT(61) +#define TOF_ICC_REG_BGS_SIGNAL_B_OP_TYPE GENMASK(3, 0) +#define TOF_ICC_REG_BGS_SIGNAL_MASK 0x58 +#define TOF_ICC_REG_BGS_SIGNAL_MASK_SIG_RECV BIT(63) +#define TOF_ICC_REG_BGS_SIGNAL_MASK_TLP_RECV BIT(62) +#define TOF_ICC_REG_BGS_SIGNAL_MASK_SIG_SEND BIT(61) +#define TOF_ICC_REG_BGS_SIGNAL_MASK_TLP_SEND BIT(60) +#define TOF_ICC_REG_BGS_LOCAL_LINK 0x60 +#define TOF_ICC_REG_BGS_LOCAL_LINK_BGID_RECV GENMASK(37, 32) +#define TOF_ICC_REG_BGS_LOCAL_LINK_BGID_SEND GENMASK(5, 0) +#define TOF_ICC_REG_BGS_REMOTE_LINK 0x68 +#define TOF_ICC_REG_BGS_REMOTE_LINK_BG_ADDRESS_RECV GENMASK(59, 32) +#define TOF_ICC_REG_BGS_REMOTE_LINK_BG_ADDRESS_SEND GENMASK(31, 0) +#define TOF_ICC_REG_BGS_SUBNET_SIZE 0x70 +#define TOF_ICC_REG_BGS_GPID_BSEQ 0x78 +#define TOF_ICC_REG_BGS_DATA_A0 0x108 +#define TOF_ICC_REG_BGS_DATA_AE 0x178 +#define TOF_ICC_REG_BGS_DATA_B0 0x188 +#define TOF_ICC_REG_BGS_DATA_BE 0x1f8 +#define TOF_ICC_REG_BGS_BCH_MASK 0x800 +#define TOF_ICC_REG_BGS_BCH_MASK_MASK BIT(63) +#define TOF_ICC_REG_BGS_BCH_MASK_STATUS 0x808 +#define TOF_ICC_REG_BGS_BCH_MASK_STATUS_RUN BIT(63) +#define TOF_ICC_REG_BGS_BCH_NOTICE_IPA 0x810 +#define TOF_ICC_REG_BGS_DUMP_START 0x0 +#define TOF_ICC_REG_BGS_DUMP_END 0x818 + +/* TNI */ +#define TOF_ICC_REG_TNI_PA(tni) (tof_icc_reg_pa + 0x0000c00000 + (tni) * 0x1000000) +#define TOF_ICC_REG_TNI_IRR 0x8 +#define TOF_ICC_REG_TNI_IMR 0x10 +#define TOF_ICC_REG_TNI_IRC 0x18 +#define TOF_ICC_REG_TNI_IMC 0x20 +#define TOF_ICC_REG_TNI_ICL 0x28 +#define TOF_ICC_REG_TNI_STATE 0x30 +#define TOF_ICC_REG_TNI_STATE_MASK GENMASK(1, 0) +#define TOF_ICC_REG_TNI_STATE_DISABLE 0 +#define TOF_ICC_REG_TNI_STATE_NORMAL 2 +#define TOF_ICC_REG_TNI_STATE_ERROR 3 +#define TOF_ICC_REG_TNI_ENABLE 0x38 +#define TOF_ICC_REG_TNI_CQ_PRESENT 0x40 +#define TOF_ICC_REG_TNI_EXCEPTION_INFO_INACTIVE_BG 0x48 +#define TOF_ICC_REG_TNI_EXCEPTION_INFO_INACTIVE_BG_DEST_BG GENMASK(37, 32) +#define TOF_ICC_REG_TNI_EXCEPTION_INFO_INACTIVE_BG_SOURCE_BG_ADDRESS GENMASK(27, 0) +#define TOF_ICC_REG_TNI_PRQ_FULL_POINTER 0x100 +#define TOF_ICC_REG_TNI_PBQ_PA 0x108 +#define TOF_ICC_REG_TNI_PBQ_SIZE 0x110 +#define TOF_ICC_REG_TNI_PRQ_PA 0x118 +#define TOF_ICC_REG_TNI_PRQ_PA_CACHE_INJECTION BIT(63) +#define TOF_ICC_REG_TNI_PRQ_SIZE 0x120 +#define TOF_ICC_REG_TNI_PRQ_MASK 0x128 +#define TOF_ICC_REG_TNI_PRQ_ENTRY_COALESCING_TIMER 0x130 +#define TOF_ICC_REG_TNI_PRQ_INTERRUPT_COALESCING_TIMER 0x138 +#define TOF_ICC_REG_TNI_PRQ_INTERRUPT_COALESCING_COUNT 0x140 +#define TOF_ICC_REG_TNI_SEND_COUNT 0x148 +#define TOF_ICC_REG_TNI_NO_SEND_COUNT 0x150 +#define TOF_ICC_REG_TNI_BLOCK_SEND_COUNT 0x158 +#define TOF_ICC_REG_TNI_RECEIVE_COUNT 0x160 +#define TOF_ICC_REG_TNI_NO_RECEIVE_COUNT 0x168 +#define TOF_ICC_REG_TNI_NUM_SEND_TLP 0x170 +#define TOF_ICC_REG_TNI_BYTE_SEND_TLP 0x178 +#define TOF_ICC_REG_TNI_NUM_SEND_SYSTEM_TLP 0x180 +#define TOF_ICC_REG_TNI_NUM_RECEIVE_TLP 0x188 +#define TOF_ICC_REG_TNI_BYTE_RECEIVE_TLP 0x190 +#define TOF_ICC_REG_TNI_NUM_RECEIVE_NULLIFIED_TLP 0x198 +#define TOF_ICC_REG_TNI_RX_NUM_UNKNOWN_TLP 0x1a0 +#define TOF_ICC_REG_TNI_RX_NUM_SYSTEM_TLP 0x1a8 +#define TOF_ICC_REG_TNI_RX_NUM_EXCEPTION_TLP 0x1b0 +#define TOF_ICC_REG_TNI_RX_NUM_DISCARD_UNKNOWN_TLP 0x1b8 +#define TOF_ICC_REG_TNI_RX_NUM_DISCARD_SYSTEM_TLP 0x1c0 +#define TOF_ICC_REG_TNI_RX_NUM_DISCARD_EXCEPTION_TLP 0x1c8 +#define TOF_ICC_REG_TNI_DUMP_START 0x8 +#define TOF_ICC_REG_TNI_DUMP_END 0x1d0 + +/* Port */ +#define TOF_ICC_REG_PORT_PA(port) (tof_icc_reg_pa + 0x0006000000 + (port) * 0x1000) +#define TOF_ICC_REG_PORT_TX_VC0_ZERO_CREDIT_COUNT 0x0 +#define TOF_ICC_REG_PORT_TX_VC1_ZERO_CREDIT_COUNT 0x8 +#define TOF_ICC_REG_PORT_TX_VC2_ZERO_CREDIT_COUNT 0x10 +#define TOF_ICC_REG_PORT_TX_VC3_ZERO_CREDIT_COUNT 0x18 +#define TOF_ICC_REG_PORT_FREE_RUN_COUNT 0x80 +#define TOF_ICC_REG_PORT_NUM_SEND_DLLP 0xc0 +#define TOF_ICC_REG_PORT_NUM_SEND_TLP 0xc8 +#define TOF_ICC_REG_PORT_BYTE_SEND_TLP 0xd0 +#define TOF_ICC_REG_PORT_NUM_SEND_SYSTEM_TLP 0xd8 +#define TOF_ICC_REG_PORT_NUM_SEND_NULLIFIED_TLP 0xe0 +#define TOF_ICC_REG_PORT_NUM_TX_DISCARD_SYSTEM_TLP 0xe8 +#define TOF_ICC_REG_PORT_NUM_TX_DISCARD_NORMAL_TLP 0xf0 +#define TOF_ICC_REG_PORT_NUM_TX_FILTERED_NORMAL_TLP 0xf8 +#define TOF_ICC_REG_PORT_NUM_VIRTUAL_CUT_THROUGH_TLP 0x100 +#define TOF_ICC_REG_PORT_NUM_GENERATE_NULLIFIED_TLP 0x108 +#define TOF_ICC_REG_PORT_NUM_RECEIVE_DLLP 0x110 +#define TOF_ICC_REG_PORT_NUM_RECEIVE_TLP 0x118 +#define TOF_ICC_REG_PORT_BYTE_RECEIVE_TLP 0x120 +#define TOF_ICC_REG_PORT_NUM_RECEIVE_SYSTEM_TLP 0x128 +#define TOF_ICC_REG_PORT_NUM_RECEIVE_NULLIFIED_TLP 0x130 +#define TOF_ICC_REG_PORT_NUM_RX_DISCARD_SYSTEM_TLP 0x138 +#define TOF_ICC_REG_PORT_NUM_RX_DISCARD_NORMAL_TLP 0x140 +#define TOF_ICC_REG_PORT_NUM_RX_FILTERED_NORMAL_TLP 0x158 +#define TOF_ICC_REG_PORT_NUM_RX_DISCARD_NULLIFIED_TLP 0x160 +#define TOF_ICC_REG_PORT_FRAME_LCRC_ERROR_COUNT 0x170 +#define TOF_ICC_REG_PORT_TX_RETRY_BUFFER_CE_COUNT 0x180 +#define TOF_ICC_REG_PORT_RX_VC_BUFFER_CE_COUNT 0x188 +#define TOF_ICC_REG_PORT_XB_CE_COUNT 0x190 +#define TOF_ICC_REG_PORT_ACK_NACK_TIME_OUT_COUNT 0x198 +#define TOF_ICC_REG_PORT_SLICE0_FCS_ERROR_COUNT 0x1a0 +#define TOF_ICC_REG_PORT_SLICE1_FCS_ERROR_COUNT 0x1a8 +#define TOF_ICC_REG_PORT_DUMP_START 0x0 +#define TOF_ICC_REG_PORT_DUMP_END 0x1b0 + +/* XB */ +#define TOF_ICC_REG_XB_PA (tof_icc_reg_pa + 0x000600f000) +#define TOF_ICC_REG_XB_STQ_ENABLE 0x0 +#define TOF_ICC_REG_XB_STQ_UPDATE_INTERVAL 0x8 +#define TOF_ICC_REG_XB_STQ_PA 0x10 +#define TOF_ICC_REG_XB_STQ_SIZE 0x18 +#define TOF_ICC_REG_XB_STQ_NEXT_OFFSET 0x20 +#define TOF_ICC_REG_XB_DUMP_START 0x0 +#define TOF_ICC_REG_XB_DUMP_END 0x28 + +#define TOF_ICC_XB_TC_DATA_CYCLE_COUNT(tni) ((tni) * 0x10 + 0x0) +#define TOF_ICC_XB_TC_WAIT_CYCLE_COUNT(tni) ((tni) * 0x10 + 0x8) +#define TOF_ICC_XB_TD_DATA_CYCLE_COUNT(tnr) ((tnr) * 0x10 + 0x60) +#define TOF_ICC_XB_TD_WAIT_CYCLE_COUNT(tnr) ((tnr) * 0x10 + 0x68) + +/* Tofu */ +#define TOF_ICC_REG_TOFU_PA (tof_icc_reg_pa + 0x0007000000) +#define TOF_ICC_REG_TOFU_NODE_ADDRESS 0x0 +#define TOF_ICC_REG_TOFU_NODE_ADDRESS_X GENMASK(22, 18) +#define TOF_ICC_REG_TOFU_NODE_ADDRESS_Y GENMASK(17, 13) +#define TOF_ICC_REG_TOFU_NODE_ADDRESS_Z GENMASK(12, 8) +#define TOF_ICC_REG_TOFU_NODE_ADDRESS_A BIT(7) +#define TOF_ICC_REG_TOFU_NODE_ADDRESS_B GENMASK(6, 5) +#define TOF_ICC_REG_TOFU_NODE_ADDRESS_C BIT(4) +#define TOF_ICC_REG_TOFU_PORT_SETTING 0x8 +#define TOF_ICC_REG_TOFU_TD_TLP_FILTER(tnr) ((tnr) * 0x10 + 0x10) +#define TOF_ICC_REG_TOFU_TD_SETTINGS(tnr) ((tnr) * 0x10 + 0x18) +#define TOF_ICC_REG_TOFU_TNR_MSI_BASE 0xc0 +#define TOF_ICC_REG_TOFU_TNR_IRR 0xc8 +#define TOF_ICC_REG_TOFU_TNR_IMR 0xd0 +#define TOF_ICC_REG_TOFU_TNR_IRC 0xd8 +#define TOF_ICC_REG_TOFU_TNR_IMC 0xe0 +#define TOF_ICC_REG_TOFU_TNR_ICL 0xe8 +#define TOF_ICC_REG_TOFU_TNI_VMS(tni, vmsid) ((tni) * 0x100 + (vmsid) * 0x8 + 0x100) +#define TOF_ICC_REG_TOFU_TNI_VMS_CQ00(tni) ((tni) * 0x100 + 0x180) +#define TOF_ICC_REG_TOFU_TNI_VMS_BG00(tni) ((tni) * 0x100 + 0x1a0) +#define TOF_ICC_REG_TOFU_TNI_VMS_BG16(tni) ((tni) * 0x100 + 0x1a8) +#define TOF_ICC_REG_TOFU_TNI_VMS_BG32(tni) ((tni) * 0x100 + 0x1b0) +#define TOF_ICC_REG_TOFU_TNI_MSI_BASE(tni) ((tni) * 0x100 + 0x1c0) +#define TOF_ICC_REG_TOFU_DUMP_START 0x0 +#define TOF_ICC_REG_TOFU_DUMP_END 0x6c8 + +/** Interrupts **/ +#define TOF_ICC_IRQ_CQS_TOQ_READ_EXCEPTION BIT(0) +#define TOF_ICC_IRQ_CQS_TOQ_DIRECT_DESCRIPTOR_EXCEPTION BIT(1) +#define TOF_ICC_IRQ_CQS_TOQ_MARKED_UE BIT(2) +#define TOF_ICC_IRQ_CQS_TCQ_WRITE_EXCEPTION BIT(3) +#define TOF_ICC_IRQ_CQS_TOQ_SOURCE_TYPE_EXCEPTION BIT(4) +#define TOF_ICC_IRQ_CQS_TCQ_WRITE_ACKNOWLEDGE BIT(5) +#define TOF_ICC_IRQ_CQS_MRQ_WRITE_ACKNOWLEDGE BIT(7) +#define TOF_ICC_IRQ_CQS_MRQ_WRITE_EXCEPTION BIT(8) +#define TOF_ICC_IRQ_CQS_MRQ_OVERFLOW BIT(9) +#define TOF_ICC_IRQ_CQS_STEERING_READ_EXCEPTION BIT(36) +#define TOF_ICC_IRQ_CQS_MB_READ_EXCEPTION BIT(38) +#define TOF_ICC_IRQ_CQS_PAYLOAD_READ_EXCEPTION BIT(39) +#define TOF_ICC_IRQ_CQS_PAYLOAD_WRITE_EXCEPTION BIT(40) +/* Just for convinience of irr value, no exists CQS CACHEFLUSH_TIMEOUT interrupt */ +#define TOF_ICC_DUMMY_IRQ_CQS_CACHEFLUSH_TIMEOUT BIT(63) + +#define TOF_ICC_IRQ_BGS_NODE_ADDRESS_UNMATCH BIT(0) +#define TOF_ICC_IRQ_BGS_BG_RECV_ADDRESS_EXCEPTION BIT(1) +#define TOF_ICC_IRQ_BGS_BG_SEND_ADDRESS_EXCEPTION BIT(2) +#define TOF_ICC_IRQ_BGS_GPID_UNMATCH BIT(3) +#define TOF_ICC_IRQ_BGS_BSEQ_UNMATCH BIT(4) +#define TOF_ICC_IRQ_BGS_SIGNAL_STATE_ERROR BIT(5) +#define TOF_ICC_IRQ_BGS_SYNCHRONIZATION_ACKNOWLEDGE BIT(24) +#define TOF_ICC_IRQ_BGS_ERROR_SYNCHRONIZATION_ACKNOWLEDGE BIT(25) +#define TOF_ICC_IRQ_BGS_DMA_COMPLETION_EXCEPTION BIT(26) + +#define TOF_ICC_IRQ_TNI_PBQ_READ_EXCEPTION BIT(0) +#define TOF_ICC_IRQ_TNI_PBQ_MARKED_UE BIT(1) +#define TOF_ICC_IRQ_TNI_PBQ_UNDERFLOW BIT(2) +#define TOF_ICC_IRQ_TNI_PRQ_PACKET_DISCARD BIT(3) +#define TOF_ICC_IRQ_TNI_PRQ_WRITE_ACKNOWLEDGE BIT(4) +#define TOF_ICC_IRQ_TNI_PRQ_WRITE_EXCEPTION BIT(5) +#define TOF_ICC_IRQ_TNI_PRQ_OVERFLOW BIT(6) +#define TOF_ICC_IRQ_TNI_INACTIVE_BG BIT(16) +#define TOF_ICC_IRQ_TNI_STAGE2_TRANSLATION_FAULT BIT(32) + +#define TOF_ICC_IRQ_TNR_TNR0_RX_FILTER_OUT BIT(0) +#define TOF_ICC_IRQ_TNR_TNR0_TX_FILTER_OUT BIT(1) +#define TOF_ICC_IRQ_TNR_TNR0_PORT_ERROR BIT(2) +#define TOF_ICC_IRQ_TNR_TNR0_DATELINE_ERROR BIT(3) +#define TOF_ICC_IRQ_TNR_TNR0_ROUTING_ERROR BIT(4) +#define TOF_ICC_IRQ_TNR_TNR1_RX_FILTER_OUT BIT(6) +#define TOF_ICC_IRQ_TNR_TNR1_TX_FILTER_OUT BIT(7) +#define TOF_ICC_IRQ_TNR_TNR1_PORT_ERROR BIT(8) +#define TOF_ICC_IRQ_TNR_TNR1_DATELINE_ERROR BIT(9) +#define TOF_ICC_IRQ_TNR_TNR1_ROUTING_ERROR BIT(10) +#define TOF_ICC_IRQ_TNR_TNR2_RX_FILTER_OUT BIT(12) +#define TOF_ICC_IRQ_TNR_TNR2_TX_FILTER_OUT BIT(13) +#define TOF_ICC_IRQ_TNR_TNR2_PORT_ERROR BIT(14) +#define TOF_ICC_IRQ_TNR_TNR2_DATELINE_ERROR BIT(15) +#define TOF_ICC_IRQ_TNR_TNR2_ROUTING_ERROR BIT(16) +#define TOF_ICC_IRQ_TNR_TNR3_RX_FILTER_OUT BIT(18) +#define TOF_ICC_IRQ_TNR_TNR3_TX_FILTER_OUT BIT(19) +#define TOF_ICC_IRQ_TNR_TNR3_PORT_ERROR BIT(20) +#define TOF_ICC_IRQ_TNR_TNR3_DATELINE_ERROR BIT(21) +#define TOF_ICC_IRQ_TNR_TNR3_ROUTING_ERROR BIT(22) +#define TOF_ICC_IRQ_TNR_TNR4_RX_FILTER_OUT BIT(24) +#define TOF_ICC_IRQ_TNR_TNR4_TX_FILTER_OUT BIT(25) +#define TOF_ICC_IRQ_TNR_TNR4_PORT_ERROR BIT(26) +#define TOF_ICC_IRQ_TNR_TNR4_DATELINE_ERROR BIT(27) +#define TOF_ICC_IRQ_TNR_TNR4_ROUTING_ERROR BIT(28) +#define TOF_ICC_IRQ_TNR_TNR5_RX_FILTER_OUT BIT(30) +#define TOF_ICC_IRQ_TNR_TNR5_TX_FILTER_OUT BIT(31) +#define TOF_ICC_IRQ_TNR_TNR5_PORT_ERROR BIT(32) +#define TOF_ICC_IRQ_TNR_TNR5_DATELINE_ERROR BIT(33) +#define TOF_ICC_IRQ_TNR_TNR5_ROUTING_ERROR BIT(34) +#define TOF_ICC_IRQ_TNR_TNR6_RX_FILTER_OUT BIT(36) +#define TOF_ICC_IRQ_TNR_TNR6_TX_FILTER_OUT BIT(37) +#define TOF_ICC_IRQ_TNR_TNR6_PORT_ERROR BIT(38) +#define TOF_ICC_IRQ_TNR_TNR6_DATELINE_ERROR BIT(39) +#define TOF_ICC_IRQ_TNR_TNR6_ROUTING_ERROR BIT(40) +#define TOF_ICC_IRQ_TNR_TNR7_RX_FILTER_OUT BIT(42) +#define TOF_ICC_IRQ_TNR_TNR7_TX_FILTER_OUT BIT(43) +#define TOF_ICC_IRQ_TNR_TNR7_PORT_ERROR BIT(44) +#define TOF_ICC_IRQ_TNR_TNR7_DATELINE_ERROR BIT(45) +#define TOF_ICC_IRQ_TNR_TNR7_ROUTING_ERROR BIT(46) +#define TOF_ICC_IRQ_TNR_TNR8_RX_FILTER_OUT BIT(48) +#define TOF_ICC_IRQ_TNR_TNR8_TX_FILTER_OUT BIT(49) +#define TOF_ICC_IRQ_TNR_TNR8_PORT_ERROR BIT(50) +#define TOF_ICC_IRQ_TNR_TNR8_DATELINE_ERROR BIT(51) +#define TOF_ICC_IRQ_TNR_TNR8_ROUTING_ERROR BIT(52) +#define TOF_ICC_IRQ_TNR_TNR9_RX_FILTER_OUT BIT(54) +#define TOF_ICC_IRQ_TNR_TNR9_TX_FILTER_OUT BIT(55) +#define TOF_ICC_IRQ_TNR_TNR9_PORT_ERROR BIT(56) +#define TOF_ICC_IRQ_TNR_TNR9_DATELINE_ERROR BIT(57) +#define TOF_ICC_IRQ_TNR_TNR9_ROUTING_ERROR BIT(58) + +#endif + +/* vim: set noet ts=8 sw=8 sts=0 tw=0 : */ + diff --git a/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_uapi.h b/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_uapi.h new file mode 100644 index 00000000..1931fbf8 --- /dev/null +++ b/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_uapi.h @@ -0,0 +1,345 @@ +#ifndef _TOF_UAPI_H_ +#define _TOF_UAPI_H_ + +#ifdef __KERNEL__ +#include +#else +#include +#include +#include +#include +#endif + +enum tof_sig_errno_cq { + TOF_TOQ_DIRECT_DESCRIPTOR_EXCEPTION, + TOF_TOQ_SOURCE_TYPE_EXCEPTION, + TOF_MRQ_OVERFLOW, + TOF_CQS_CACHEFLUSH_TIMEOUT, +}; + +enum tof_sig_errno_bg { + TOF_NODE_ADDRESS_UNMATCH, + TOF_BSEQ_UNMATCH, + TOF_SIGNAL_STATE_ERROR, + TOF_ERROR_SYNCHRONIZATION_ACKNOWLEDGE, +}; + +#define TOF_UAPI_VERSION 0x2a00 + +struct tof_init_cq { + uint16_t version; + uint8_t session_mode; + uint8_t toq_size; + uint8_t mrq_size; + uint8_t num_stag; + uint8_t tcq_cinj; + uint8_t mrq_cinj; + void *toq_mem; + void *tcq_mem; + void *mrq_mem; +}; + +struct tof_alloc_stag { + uint32_t flags; + int stag; + uint64_t offset; + void *va; + uint64_t len; +}; + +struct tof_free_stags { + uint16_t num; + int *stags; +}; + +struct tof_addr { + uint8_t pa; + uint8_t pb; + uint8_t pc; + uint8_t x; + uint8_t y; + uint8_t z; + uint8_t a; + uint8_t b; + uint8_t c; +}; + +struct tof_set_bg { + int tni; + int gate; + int source_lgate; + struct tof_addr source_raddr; + int source_rtni; + int source_rgate; + int dest_lgate; + struct tof_addr dest_raddr; + int dest_rtni; + int dest_rgate; +}; + +struct tof_enable_bch { + void *addr; + int bseq; + int num; + struct tof_set_bg *bgs; +}; + +struct tof_set_subnet { + int res0; + int res1; + uint8_t nx; + uint8_t sx; + uint8_t lx; + uint8_t ny; + uint8_t sy; + uint8_t ly; + uint8_t nz; + uint8_t sz; + uint8_t lz; +}; + +struct tof_reg_user { + uid_t uid; + uint32_t gpid; + struct tof_set_subnet subnet; + uint64_t *cqmask; + uint64_t *bgmask; +}; + +struct tof_notify_linkdown { + int num; + struct { + uint8_t x; + uint8_t y; + uint8_t z; + uint8_t a; + uint8_t b; + uint8_t c; + uint16_t ports; + } *items; +}; + +struct tof_get_port_stat { + int port_no; + uint64_t mask; + uint64_t pa[31]; +}; + +struct tof_get_cq_stat { + int tni; + int cqid; + uint64_t txbyte; + uint64_t rxbyte; +}; + +struct tof_load_register { + uint64_t pa; + uint64_t len; + void *buf; +}; + +struct tof_load_resource { + uint64_t rsc_id; + uint64_t offset; + uint64_t len; + void *buf; +}; + +union tof_trans_table_bitfield { + struct { + uint64_t start:36; + uint64_t len:27; + uint64_t ps_code:1; + } bits; + uint64_t atomic; +}; + +struct tof_trans_table { + union tof_trans_table_bitfield steering; + union tof_trans_table_bitfield mbpt; +}; + +void tof_utofu_set_linkdown_callback(void (*callback)(int, const void *)); +void tof_utofu_unset_linkdown_callback(void); + +#define TOF_MMAP_CQ_REGISTER 0 +#ifdef __KERNEL__ +#define TOF_MMAP_CQ_TRANSTABLE (PAGE_SIZE) +#else +#define TOF_MMAP_CQ_TRANSTABLE (sysconf(_SC_PAGESIZE)) +#endif +#define TOF_MMAP_BCH_REGISTER 0 +#define TOF_MMAP_XB_STQ 0 + +#define TOF_ST_RDWR 0x0 +#define TOF_ST_RDONLY 0x1 +#define TOF_ST_LPG 0x2 + +#define TOF_STAG_TRANS_PS_CODE_64KB 0 +#define TOF_STAG_TRANS_PS_CODE_2MB 1 + +#define TOF_IOC_MAGIC 'd' +#define TOF_IOCTL_INIT_CQ _IOWR(TOF_IOC_MAGIC, 0, long) +#define TOF_IOCTL_ALLOC_STAG _IOWR(TOF_IOC_MAGIC, 1, long) +#define TOF_IOCTL_FREE_STAGS _IOWR(TOF_IOC_MAGIC, 2, long) +#define TOF_IOCTL_ENABLE_BCH _IOWR(TOF_IOC_MAGIC, 3, long) +#define TOF_IOCTL_DISABLE_BCH _IOWR(TOF_IOC_MAGIC, 4, long) +#define TOF_IOCTL_SET_RT_SIGNAL _IOWR(TOF_IOC_MAGIC, 5, long) +#define TOF_IOCTL_SET_SUBNET _IOWR(TOF_IOC_MAGIC, 6, long) +#define TOF_IOCTL_REG_USER _IOWR(TOF_IOC_MAGIC, 7, long) +#define TOF_IOCTL_NOTIFY_LINKDOWN _IOWR(TOF_IOC_MAGIC, 8, long) +#define TOF_IOCTL_GET_PORT_STAT _IOWR(TOF_IOC_MAGIC, 9, long) +#define TOF_IOCTL_GET_CQ_STAT _IOWR(TOF_IOC_MAGIC, 10, long) +#define TOF_IOCTL_LOAD_REGISTER _IOWR(TOF_IOC_MAGIC, 11, long) +#define TOF_IOCTL_LOAD_RESOURCE _IOWR(TOF_IOC_MAGIC, 12, long) + +#ifndef __KERNEL__ +#define TOF_INIT_CQ TOF_IOCTL_INIT_CQ +#define TOF_ALLOC_STAG TOF_IOCTL_ALLOC_STAG +#define TOF_FREE_STAGS TOF_IOCTL_FREE_STAGS +#define TOF_ENABLE_BCH TOF_IOCTL_ENABLE_BCH +#define TOF_DISABLE_BCH TOF_IOCTL_DISABLE_BCH +#define TOF_SET_RT_SIGNAL TOF_IOCTL_SET_RT_SIGNAL +#define TOF_SET_SUBNET TOF_IOCTL_SET_SUBNET +#define TOF_REG_USER TOF_IOCTL_REG_USER +#define TOF_NOTIFY_LINKDOWN TOF_IOCTL_NOTIFY_LINKDOWN +#define TOF_GET_PORT_STAT TOF_IOCTL_GET_PORT_STAT +#define TOF_GET_CQ_STAT TOF_IOCTL_GET_CQ_STAT +#define TOF_LOAD_REGISTER TOF_IOCTL_LOAD_REGISTER +#define TOF_LOAD_RESOURCE TOF_IOCTL_LOAD_RESOURCE +#endif + +enum { + /* TOQ (0 - 71) */ + TOF_RSC_TNI0_TOQ0 = 0, TOF_RSC_TNI0_TOQ1, TOF_RSC_TNI0_TOQ2, TOF_RSC_TNI0_TOQ3, + TOF_RSC_TNI0_TOQ4, TOF_RSC_TNI0_TOQ5, TOF_RSC_TNI0_TOQ6, TOF_RSC_TNI0_TOQ7, + TOF_RSC_TNI0_TOQ8, TOF_RSC_TNI0_TOQ9, TOF_RSC_TNI0_TOQ10, TOF_RSC_TNI0_TOQ11, + TOF_RSC_TNI1_TOQ0, TOF_RSC_TNI1_TOQ1, TOF_RSC_TNI1_TOQ2, TOF_RSC_TNI1_TOQ3, + TOF_RSC_TNI1_TOQ4, TOF_RSC_TNI1_TOQ5, TOF_RSC_TNI1_TOQ6, TOF_RSC_TNI1_TOQ7, + TOF_RSC_TNI1_TOQ8, TOF_RSC_TNI1_TOQ9, TOF_RSC_TNI1_TOQ10, TOF_RSC_TNI1_TOQ11, + TOF_RSC_TNI2_TOQ0, TOF_RSC_TNI2_TOQ1, TOF_RSC_TNI2_TOQ2, TOF_RSC_TNI2_TOQ3, + TOF_RSC_TNI2_TOQ4, TOF_RSC_TNI2_TOQ5, TOF_RSC_TNI2_TOQ6, TOF_RSC_TNI2_TOQ7, + TOF_RSC_TNI2_TOQ8, TOF_RSC_TNI2_TOQ9, TOF_RSC_TNI2_TOQ10, TOF_RSC_TNI2_TOQ11, + TOF_RSC_TNI3_TOQ0, TOF_RSC_TNI3_TOQ1, TOF_RSC_TNI3_TOQ2, TOF_RSC_TNI3_TOQ3, + TOF_RSC_TNI3_TOQ4, TOF_RSC_TNI3_TOQ5, TOF_RSC_TNI3_TOQ6, TOF_RSC_TNI3_TOQ7, + TOF_RSC_TNI3_TOQ8, TOF_RSC_TNI3_TOQ9, TOF_RSC_TNI3_TOQ10, TOF_RSC_TNI3_TOQ11, + TOF_RSC_TNI4_TOQ0, TOF_RSC_TNI4_TOQ1, TOF_RSC_TNI4_TOQ2, TOF_RSC_TNI4_TOQ3, + TOF_RSC_TNI4_TOQ4, TOF_RSC_TNI4_TOQ5, TOF_RSC_TNI4_TOQ6, TOF_RSC_TNI4_TOQ7, + TOF_RSC_TNI4_TOQ8, TOF_RSC_TNI4_TOQ9, TOF_RSC_TNI4_TOQ10, TOF_RSC_TNI4_TOQ11, + TOF_RSC_TNI5_TOQ0, TOF_RSC_TNI5_TOQ1, TOF_RSC_TNI5_TOQ2, TOF_RSC_TNI5_TOQ3, + TOF_RSC_TNI5_TOQ4, TOF_RSC_TNI5_TOQ5, TOF_RSC_TNI5_TOQ6, TOF_RSC_TNI5_TOQ7, + TOF_RSC_TNI5_TOQ8, TOF_RSC_TNI5_TOQ9, TOF_RSC_TNI5_TOQ10, TOF_RSC_TNI5_TOQ11, + + /* TOQ (72 - 143) */ + TOF_RSC_TNI0_TCQ0, TOF_RSC_TNI0_TCQ1, TOF_RSC_TNI0_TCQ2, TOF_RSC_TNI0_TCQ3, + TOF_RSC_TNI0_TCQ4, TOF_RSC_TNI0_TCQ5, TOF_RSC_TNI0_TCQ6, TOF_RSC_TNI0_TCQ7, + TOF_RSC_TNI0_TCQ8, TOF_RSC_TNI0_TCQ9, TOF_RSC_TNI0_TCQ10, TOF_RSC_TNI0_TCQ11, + TOF_RSC_TNI1_TCQ0, TOF_RSC_TNI1_TCQ1, TOF_RSC_TNI1_TCQ2, TOF_RSC_TNI1_TCQ3, + TOF_RSC_TNI1_TCQ4, TOF_RSC_TNI1_TCQ5, TOF_RSC_TNI1_TCQ6, TOF_RSC_TNI1_TCQ7, + TOF_RSC_TNI1_TCQ8, TOF_RSC_TNI1_TCQ9, TOF_RSC_TNI1_TCQ10, TOF_RSC_TNI1_TCQ11, + TOF_RSC_TNI2_TCQ0, TOF_RSC_TNI2_TCQ1, TOF_RSC_TNI2_TCQ2, TOF_RSC_TNI2_TCQ3, + TOF_RSC_TNI2_TCQ4, TOF_RSC_TNI2_TCQ5, TOF_RSC_TNI2_TCQ6, TOF_RSC_TNI2_TCQ7, + TOF_RSC_TNI2_TCQ8, TOF_RSC_TNI2_TCQ9, TOF_RSC_TNI2_TCQ10, TOF_RSC_TNI2_TCQ11, + TOF_RSC_TNI3_TCQ0, TOF_RSC_TNI3_TCQ1, TOF_RSC_TNI3_TCQ2, TOF_RSC_TNI3_TCQ3, + TOF_RSC_TNI3_TCQ4, TOF_RSC_TNI3_TCQ5, TOF_RSC_TNI3_TCQ6, TOF_RSC_TNI3_TCQ7, + TOF_RSC_TNI3_TCQ8, TOF_RSC_TNI3_TCQ9, TOF_RSC_TNI3_TCQ10, TOF_RSC_TNI3_TCQ11, + TOF_RSC_TNI4_TCQ0, TOF_RSC_TNI4_TCQ1, TOF_RSC_TNI4_TCQ2, TOF_RSC_TNI4_TCQ3, + TOF_RSC_TNI4_TCQ4, TOF_RSC_TNI4_TCQ5, TOF_RSC_TNI4_TCQ6, TOF_RSC_TNI4_TCQ7, + TOF_RSC_TNI4_TCQ8, TOF_RSC_TNI4_TCQ9, TOF_RSC_TNI4_TCQ10, TOF_RSC_TNI4_TCQ11, + TOF_RSC_TNI5_TCQ0, TOF_RSC_TNI5_TCQ1, TOF_RSC_TNI5_TCQ2, TOF_RSC_TNI5_TCQ3, + TOF_RSC_TNI5_TCQ4, TOF_RSC_TNI5_TCQ5, TOF_RSC_TNI5_TCQ6, TOF_RSC_TNI5_TCQ7, + TOF_RSC_TNI5_TCQ8, TOF_RSC_TNI5_TCQ9, TOF_RSC_TNI5_TCQ10, TOF_RSC_TNI5_TCQ11, + + /* MRQ (144 - 215) */ + TOF_RSC_TNI0_MRQ0, TOF_RSC_TNI0_MRQ1, TOF_RSC_TNI0_MRQ2, TOF_RSC_TNI0_MRQ3, + TOF_RSC_TNI0_MRQ4, TOF_RSC_TNI0_MRQ5, TOF_RSC_TNI0_MRQ6, TOF_RSC_TNI0_MRQ7, + TOF_RSC_TNI0_MRQ8, TOF_RSC_TNI0_MRQ9, TOF_RSC_TNI0_MRQ10, TOF_RSC_TNI0_MRQ11, + TOF_RSC_TNI1_MRQ0, TOF_RSC_TNI1_MRQ1, TOF_RSC_TNI1_MRQ2, TOF_RSC_TNI1_MRQ3, + TOF_RSC_TNI1_MRQ4, TOF_RSC_TNI1_MRQ5, TOF_RSC_TNI1_MRQ6, TOF_RSC_TNI1_MRQ7, + TOF_RSC_TNI1_MRQ8, TOF_RSC_TNI1_MRQ9, TOF_RSC_TNI1_MRQ10, TOF_RSC_TNI1_MRQ11, + TOF_RSC_TNI2_MRQ0, TOF_RSC_TNI2_MRQ1, TOF_RSC_TNI2_MRQ2, TOF_RSC_TNI2_MRQ3, + TOF_RSC_TNI2_MRQ4, TOF_RSC_TNI2_MRQ5, TOF_RSC_TNI2_MRQ6, TOF_RSC_TNI2_MRQ7, + TOF_RSC_TNI2_MRQ8, TOF_RSC_TNI2_MRQ9, TOF_RSC_TNI2_MRQ10, TOF_RSC_TNI2_MRQ11, + TOF_RSC_TNI3_MRQ0, TOF_RSC_TNI3_MRQ1, TOF_RSC_TNI3_MRQ2, TOF_RSC_TNI3_MRQ3, + TOF_RSC_TNI3_MRQ4, TOF_RSC_TNI3_MRQ5, TOF_RSC_TNI3_MRQ6, TOF_RSC_TNI3_MRQ7, + TOF_RSC_TNI3_MRQ8, TOF_RSC_TNI3_MRQ9, TOF_RSC_TNI3_MRQ10, TOF_RSC_TNI3_MRQ11, + TOF_RSC_TNI4_MRQ0, TOF_RSC_TNI4_MRQ1, TOF_RSC_TNI4_MRQ2, TOF_RSC_TNI4_MRQ3, + TOF_RSC_TNI4_MRQ4, TOF_RSC_TNI4_MRQ5, TOF_RSC_TNI4_MRQ6, TOF_RSC_TNI4_MRQ7, + TOF_RSC_TNI4_MRQ8, TOF_RSC_TNI4_MRQ9, TOF_RSC_TNI4_MRQ10, TOF_RSC_TNI4_MRQ11, + TOF_RSC_TNI5_MRQ0, TOF_RSC_TNI5_MRQ1, TOF_RSC_TNI5_MRQ2, TOF_RSC_TNI5_MRQ3, + TOF_RSC_TNI5_MRQ4, TOF_RSC_TNI5_MRQ5, TOF_RSC_TNI5_MRQ6, TOF_RSC_TNI5_MRQ7, + TOF_RSC_TNI5_MRQ8, TOF_RSC_TNI5_MRQ9, TOF_RSC_TNI5_MRQ10, TOF_RSC_TNI5_MRQ11, + + /* PBQ (216 - 221) */ + TOF_RSC_TNI0_PBQ, TOF_RSC_TNI1_PBQ, TOF_RSC_TNI2_PBQ, TOF_RSC_TNI3_PBQ, + TOF_RSC_TNI4_PBQ, TOF_RSC_TNI5_PBQ, + + /* PRQ (222 - 227) */ + TOF_RSC_TNI0_PRQ, TOF_RSC_TNI1_PRQ, TOF_RSC_TNI2_PRQ, TOF_RSC_TNI3_PRQ, + TOF_RSC_TNI4_PRQ, TOF_RSC_TNI5_PRQ, + + /* STEERINGTABLE (228 - 299) */ + TOF_RSC_TNI0_STEERINGTABLE0, TOF_RSC_TNI0_STEERINGTABLE1, TOF_RSC_TNI0_STEERINGTABLE2, + TOF_RSC_TNI0_STEERINGTABLE3, TOF_RSC_TNI0_STEERINGTABLE4, TOF_RSC_TNI0_STEERINGTABLE5, + TOF_RSC_TNI0_STEERINGTABLE6, TOF_RSC_TNI0_STEERINGTABLE7, TOF_RSC_TNI0_STEERINGTABLE8, + TOF_RSC_TNI0_STEERINGTABLE9, TOF_RSC_TNI0_STEERINGTABLE10, TOF_RSC_TNI0_STEERINGTABLE11, + TOF_RSC_TNI1_STEERINGTABLE0, TOF_RSC_TNI1_STEERINGTABLE1, TOF_RSC_TNI1_STEERINGTABLE2, + TOF_RSC_TNI1_STEERINGTABLE3, TOF_RSC_TNI1_STEERINGTABLE4, TOF_RSC_TNI1_STEERINGTABLE5, + TOF_RSC_TNI1_STEERINGTABLE6, TOF_RSC_TNI1_STEERINGTABLE7, TOF_RSC_TNI1_STEERINGTABLE8, + TOF_RSC_TNI1_STEERINGTABLE9, TOF_RSC_TNI1_STEERINGTABLE10, TOF_RSC_TNI1_STEERINGTABLE11, + TOF_RSC_TNI2_STEERINGTABLE0, TOF_RSC_TNI2_STEERINGTABLE1, TOF_RSC_TNI2_STEERINGTABLE2, + TOF_RSC_TNI2_STEERINGTABLE3, TOF_RSC_TNI2_STEERINGTABLE4, TOF_RSC_TNI2_STEERINGTABLE5, + TOF_RSC_TNI2_STEERINGTABLE6, TOF_RSC_TNI2_STEERINGTABLE7, TOF_RSC_TNI2_STEERINGTABLE8, + TOF_RSC_TNI2_STEERINGTABLE9, TOF_RSC_TNI2_STEERINGTABLE10, TOF_RSC_TNI2_STEERINGTABLE11, + TOF_RSC_TNI3_STEERINGTABLE0, TOF_RSC_TNI3_STEERINGTABLE1, TOF_RSC_TNI3_STEERINGTABLE2, + TOF_RSC_TNI3_STEERINGTABLE3, TOF_RSC_TNI3_STEERINGTABLE4, TOF_RSC_TNI3_STEERINGTABLE5, + TOF_RSC_TNI3_STEERINGTABLE6, TOF_RSC_TNI3_STEERINGTABLE7, TOF_RSC_TNI3_STEERINGTABLE8, + TOF_RSC_TNI3_STEERINGTABLE9, TOF_RSC_TNI3_STEERINGTABLE10, TOF_RSC_TNI3_STEERINGTABLE11, + TOF_RSC_TNI4_STEERINGTABLE0, TOF_RSC_TNI4_STEERINGTABLE1, TOF_RSC_TNI4_STEERINGTABLE2, + TOF_RSC_TNI4_STEERINGTABLE3, TOF_RSC_TNI4_STEERINGTABLE4, TOF_RSC_TNI4_STEERINGTABLE5, + TOF_RSC_TNI4_STEERINGTABLE6, TOF_RSC_TNI4_STEERINGTABLE7, TOF_RSC_TNI4_STEERINGTABLE8, + TOF_RSC_TNI4_STEERINGTABLE9, TOF_RSC_TNI4_STEERINGTABLE10, TOF_RSC_TNI4_STEERINGTABLE11, + TOF_RSC_TNI5_STEERINGTABLE3, TOF_RSC_TNI5_STEERINGTABLE4, TOF_RSC_TNI5_STEERINGTABLE5, + TOF_RSC_TNI5_STEERINGTABLE6, TOF_RSC_TNI5_STEERINGTABLE7, TOF_RSC_TNI5_STEERINGTABLE8, + TOF_RSC_TNI5_STEERINGTABLE9, TOF_RSC_TNI5_STEERINGTABLE10, TOF_RSC_TNI5_STEERINGTABLE11, + + /* MBTABLE (300 - 371) */ + TOF_RSC_TNI0_MBTABLE0, TOF_RSC_TNI0_MBTABLE1, TOF_RSC_TNI0_MBTABLE2, + TOF_RSC_TNI0_MBTABLE3, TOF_RSC_TNI0_MBTABLE4, TOF_RSC_TNI0_MBTABLE5, + TOF_RSC_TNI0_MBTABLE6, TOF_RSC_TNI0_MBTABLE7, TOF_RSC_TNI0_MBTABLE8, + TOF_RSC_TNI0_MBTABLE9, TOF_RSC_TNI0_MBTABLE10, TOF_RSC_TNI0_MBTABLE11, + TOF_RSC_TNI1_MBTABLE0, TOF_RSC_TNI1_MBTABLE1, TOF_RSC_TNI1_MBTABLE2, + TOF_RSC_TNI1_MBTABLE3, TOF_RSC_TNI1_MBTABLE4, TOF_RSC_TNI1_MBTABLE5, + TOF_RSC_TNI1_MBTABLE6, TOF_RSC_TNI1_MBTABLE7, TOF_RSC_TNI1_MBTABLE8, + TOF_RSC_TNI1_MBTABLE9, TOF_RSC_TNI1_MBTABLE10, TOF_RSC_TNI1_MBTABLE11, + TOF_RSC_TNI2_MBTABLE0, TOF_RSC_TNI2_MBTABLE1, TOF_RSC_TNI2_MBTABLE2, + TOF_RSC_TNI2_MBTABLE3, TOF_RSC_TNI2_MBTABLE4, TOF_RSC_TNI2_MBTABLE5, + TOF_RSC_TNI2_MBTABLE6, TOF_RSC_TNI2_MBTABLE7, TOF_RSC_TNI2_MBTABLE8, + TOF_RSC_TNI2_MBTABLE9, TOF_RSC_TNI2_MBTABLE10, TOF_RSC_TNI2_MBTABLE11, + TOF_RSC_TNI3_MBTABLE0, TOF_RSC_TNI3_MBTABLE1, TOF_RSC_TNI3_MBTABLE2, + TOF_RSC_TNI3_MBTABLE3, TOF_RSC_TNI3_MBTABLE4, TOF_RSC_TNI3_MBTABLE5, + TOF_RSC_TNI3_MBTABLE6, TOF_RSC_TNI3_MBTABLE7, TOF_RSC_TNI3_MBTABLE8, + TOF_RSC_TNI3_MBTABLE9, TOF_RSC_TNI3_MBTABLE10, TOF_RSC_TNI3_MBTABLE11, + TOF_RSC_TNI4_MBTABLE0, TOF_RSC_TNI4_MBTABLE1, TOF_RSC_TNI4_MBTABLE2, + TOF_RSC_TNI4_MBTABLE3, TOF_RSC_TNI4_MBTABLE4, TOF_RSC_TNI4_MBTABLE5, + TOF_RSC_TNI4_MBTABLE6, TOF_RSC_TNI4_MBTABLE7, TOF_RSC_TNI4_MBTABLE8, + TOF_RSC_TNI4_MBTABLE9, TOF_RSC_TNI4_MBTABLE10, TOF_RSC_TNI4_MBTABLE11, + TOF_RSC_TNI5_MBTABLE0, TOF_RSC_TNI5_MBTABLE1, TOF_RSC_TNI5_MBTABLE2, + TOF_RSC_TNI5_MBTABLE3, TOF_RSC_TNI5_MBTABLE4, TOF_RSC_TNI5_MBTABLE5, + TOF_RSC_TNI5_MBTABLE6, TOF_RSC_TNI5_MBTABLE7, TOF_RSC_TNI5_MBTABLE8, + TOF_RSC_TNI5_MBTABLE9, TOF_RSC_TNI5_MBTABLE10, TOF_RSC_TNI5_MBTABLE11, + + TOF_RSC_NUM /* 372 */ +}; +#define TOF_RSC_TOQ(TNI, CQID) (TOF_RSC_TNI0_TOQ0 + (TNI * 12) + CQID) +#define TOF_RSC_TCQ(TNI, CQID) (TOF_RSC_TNI0_TCQ0 + (TNI * 12) + CQID) +#define TOF_RSC_MRQ(TNI, CQID) (TOF_RSC_TNI0_MRQ0 + (TNI * 12) + CQID) +#define TOF_RSC_PBQ(TNI) (TOF_RSC_TNI0_PBQ + TNI) +#define TOF_RSC_PRQ(TNI) (TOF_RSC_TNI0_PRQ + TNI) +#define TOF_RSC_STT(TNI, CQID) (TOF_RSC_TNI0_STEERINGTABLE0 + (TNI * 12) + CQID) +#define TOF_RSC_MBT(TNI, CQID) (TOF_RSC_TNI0_MBTABLE0 + (TNI * 12) + CQID) + +#endif + +/* vim: set noet ts=8 sw=8 sts=0 tw=0 : */ + diff --git a/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_bg_bch.h b/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_bg_bch.h new file mode 100644 index 00000000..c0d1ea4b --- /dev/null +++ b/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_bg_bch.h @@ -0,0 +1,6 @@ + struct { + bool enabled; + uint64_t bgmask[TOF_ICC_NTNIS]; + uintptr_t iova; + void *kaddr; + } bch; diff --git a/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_cq_trans.h b/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_cq_trans.h new file mode 100644 index 00000000..8c7c7656 --- /dev/null +++ b/executer/kernel/mcctrl/arch/arm64/include/tofu/tof_utofu_cq_trans.h @@ -0,0 +1,6 @@ + struct { + struct tof_utofu_trans_list *mru; + struct tof_trans_table *table; + int mruhead; + ihk_spinlock_t mru_lock; + } trans; diff --git a/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_bg.h b/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_bg.h new file mode 100644 index 00000000..0bba3714 --- /dev/null +++ b/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_bg.h @@ -0,0 +1,21 @@ +struct tof_utofu_bg { + union { + char whole_struct[160]; + struct { + char padding0[0]; + struct tof_utofu_device common; + }; + struct { + char padding1[80]; + uint8_t tni; + }; + struct { + char padding2[81]; + uint8_t bgid; + }; + struct { + char padding3[88]; + #include "tof_utofu_bg_bch.h" + }; + }; +}; diff --git a/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_cq.h b/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_cq.h new file mode 100644 index 00000000..bc1728af --- /dev/null +++ b/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_cq.h @@ -0,0 +1,37 @@ +struct tof_utofu_cq { + union { + char whole_struct[384]; + struct { + char padding0[0]; + struct tof_utofu_device common; + }; + struct { + char padding1[80]; + uint8_t tni; + }; + struct { + char padding2[81]; + uint8_t cqid; + }; + struct { + char padding3[104]; + #include "tof_utofu_cq_trans.h" + }; + struct { + char padding4[128]; + struct tof_icc_steering_entry *steering; + }; + struct { + char padding5[136]; + struct tof_icc_mb_entry *mb; + }; + struct { + char padding6[186]; + uint8_t num_stag; + }; + struct { + char padding7[336]; + struct mmu_notifier mn; + }; + }; +}; diff --git a/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_device.h b/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_device.h new file mode 100644 index 00000000..24244739 --- /dev/null +++ b/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_device.h @@ -0,0 +1,17 @@ +struct tof_utofu_device { + union { + char whole_struct[80]; + struct { + char padding0[0]; + bool enabled; + }; + struct { + char padding1[12]; + uint32_t gpid; + }; + struct { + char padding2[24]; + uint64_t subnet; + }; + }; +}; diff --git a/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_mbpt.h b/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_mbpt.h new file mode 100644 index 00000000..81ab721c --- /dev/null +++ b/executer/kernel/mcctrl/arch/arm64/include/tofu/tofu_generated-tof_utofu_mbpt.h @@ -0,0 +1,33 @@ +struct tof_utofu_mbpt { + union { + char whole_struct[56]; + struct { + char padding0[0]; + struct kref kref; + }; + struct { + char padding1[8]; + struct tof_utofu_cq *ucq; + }; + struct { + char padding2[16]; + uintptr_t iova; + }; + struct { + char padding3[24]; + struct scatterlist *sg; + }; + struct { + char padding4[32]; + size_t nsgents; + }; + struct { + char padding5[40]; + uintptr_t mbptstart; + }; + struct { + char padding6[48]; + size_t pgsz; + }; + }; +}; diff --git a/executer/kernel/mcctrl/control.c b/executer/kernel/mcctrl/control.c index 18df5e32..c6d220ba 100644 --- a/executer/kernel/mcctrl/control.c +++ b/executer/kernel/mcctrl/control.c @@ -230,6 +230,9 @@ static long mcexec_prepare_image(ihk_os_t os, dprintk("%s: pid %d, rpgtable: 0x%lx added\n", __FUNCTION__, ppd->pid, ppd->rpgtable); +#ifdef ENABLE_TOFU + ppd->enable_tofu = pdesc->enable_tofu; +#endif ret = 0; @@ -1258,7 +1261,7 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd) process is gone and the application should be terminated. */ packet = (struct ikc_scd_packet *)ptd->data; dprintk("%s: calling __return_syscall (hash),target pid=%d,tid=%d\n", __FUNCTION__, ppd->pid, packet->req.rtid); - __return_syscall(ppd->ud->os, packet, -ERESTARTSYS, + __return_syscall(ppd->ud->os, ppd, packet, -ERESTARTSYS, packet->req.rtid); ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet); @@ -1282,7 +1285,7 @@ void mcctrl_put_per_proc_data(struct mcctrl_per_proc_data *ppd) /* We use ERESTARTSYS to tell the LWK that the proxy * process is gone and the application should be terminated */ - __return_syscall(ppd->ud->os, packet, -ERESTARTSYS, + __return_syscall(ppd->ud->os, ppd, packet, -ERESTARTSYS, packet->req.rtid); ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet); } @@ -1323,7 +1326,7 @@ int mcexec_syscall(struct mcctrl_usrdata *ud, struct ikc_scd_packet *packet) /* We use ERESTARTSYS to tell the LWK that the proxy * process is gone and the application should be terminated */ - __return_syscall(ud->os, packet, -ERESTARTSYS, + __return_syscall(ud->os, NULL, packet, -ERESTARTSYS, packet->req.rtid); ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet); @@ -1766,7 +1769,7 @@ long mcexec_ret_syscall(ihk_os_t os, struct syscall_ret_desc *__user arg) ihk_device_unmap_memory(ihk_os_to_dev(os), phys, ret.size); } - __return_syscall(os, packet, ret.ret, task_pid_vnr(current)); + __return_syscall(os, ppd, packet, ret.ret, task_pid_vnr(current)); error = 0; out: @@ -2794,7 +2797,7 @@ static long mcexec_terminate_thread_unsafe(ihk_os_t os, int pid, int tid, long c __FUNCTION__, tid); goto no_ptd; } - __return_syscall(usrdata->os, packet, code, tid); + __return_syscall(usrdata->os, ppd, packet, code, tid); ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet); /* Drop reference for this function */ diff --git a/executer/kernel/mcctrl/mcctrl.h b/executer/kernel/mcctrl/mcctrl.h index e9c5c7fa..f0f320c8 100644 --- a/executer/kernel/mcctrl/mcctrl.h +++ b/executer/kernel/mcctrl/mcctrl.h @@ -263,6 +263,7 @@ struct mcctrl_per_proc_data { struct list_head devobj_pager_list; struct semaphore devobj_pager_lock; + int enable_tofu; }; struct sysfsm_req { @@ -464,7 +465,8 @@ struct mcctrl_per_thread_data *mcctrl_get_per_thread_data(struct mcctrl_per_proc struct task_struct *task); int mcctrl_clear_pte_range(uintptr_t start, uintptr_t len); -void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet, +void __return_syscall(ihk_os_t os, struct mcctrl_per_proc_data *ppd, + struct ikc_scd_packet *packet, long ret, int stid); int clear_pte_range(uintptr_t start, uintptr_t len); @@ -577,10 +579,13 @@ struct mcctrl_file_to_pidfd { int pid; int fd; struct list_head hash; + char tofu_dev_path[128]; + void *pde_data; }; int mcctrl_file_to_pidfd_hash_insert(struct file *filp, - ihk_os_t os, int pid, struct task_struct *group_leader, int fd); + ihk_os_t os, int pid, struct task_struct *group_leader, int fd, + char *path, void *pde_data); struct mcctrl_file_to_pidfd *mcctrl_file_to_pidfd_hash_lookup( struct file *filp, struct task_struct *group_leader); int mcctrl_file_to_pidfd_hash_remove(struct file *filp, diff --git a/executer/kernel/mcctrl/syscall.c b/executer/kernel/mcctrl/syscall.c index b7e974f7..57eac652 100644 --- a/executer/kernel/mcctrl/syscall.c +++ b/executer/kernel/mcctrl/syscall.c @@ -1864,7 +1864,8 @@ void mcctrl_file_to_pidfd_hash_init(void) } int mcctrl_file_to_pidfd_hash_insert(struct file *filp, - ihk_os_t os, int pid, struct task_struct *group_leader, int fd) + ihk_os_t os, int pid, struct task_struct *group_leader, int fd, + char *path, void *pde_data) { unsigned long irqflags; struct mcctrl_file_to_pidfd *file2pidfd_iter; @@ -1882,6 +1883,9 @@ int mcctrl_file_to_pidfd_hash_insert(struct file *filp, file2pidfd->pid = pid; file2pidfd->group_leader = group_leader; file2pidfd->fd = fd; + /* Only copy the name under /proc/tofu/dev/ */ + strncpy(file2pidfd->tofu_dev_path, path + 15, 128); + file2pidfd->pde_data = pde_data; spin_lock_irqsave(&mcctrl_file_to_pidfd_hash_lock, irqflags); list_for_each_entry(file2pidfd_iter, @@ -1980,7 +1984,8 @@ unlock_out: } #endif -void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet, +void __return_syscall(ihk_os_t os, struct mcctrl_per_proc_data *ppd, + struct ikc_scd_packet *packet, long ret, int stid) { unsigned long phys; @@ -2011,54 +2016,104 @@ void __return_syscall(ihk_os_t os, struct ikc_scd_packet *packet, res->stid = stid; #ifdef ENABLE_TOFU - /* Record PDE_DATA after open() calls for Tofu driver */ - if (packet->req.number == __NR_openat && ret > 1) { + /* Tofu enabled process? */ + if (ppd && ppd->enable_tofu) { char *pathbuf, *fullpath; - struct fd f; - int fd; - fd = ret; - f = fdget(fd); + /* Record PDE_DATA after open() calls for Tofu driver */ + if (packet->req.number == __NR_openat && ret > 1) { + struct fd f; + int fd; - if (!f.file) { - goto out_notify; + fd = ret; + f = fdget(fd); + + if (!f.file) { + goto out_notify; + } + + pathbuf = (char *)__get_free_page(GFP_ATOMIC); + if (!pathbuf) { + goto out_fdput_open; + } + + fullpath = d_path(&f.file->f_path, pathbuf, PAGE_SIZE); + if (IS_ERR(fullpath)) { + goto out_free_open; + } + + if (!strncmp("/proc/tofu/dev/", fullpath, 15)) { + res->pde_data = PDE_DATA(file_inode(f.file)); + dprintk("%s: fd: %d, path: %s, PDE_DATA: 0x%lx\n", + __func__, + fd, + fullpath, + (unsigned long)res->pde_data); + dprintk("%s: pgd_index: %ld, pmd_index: %ld, pte_index: %ld\n", + __func__, + pgd_index((unsigned long)res->pde_data), + pmd_index((unsigned long)res->pde_data), + pte_index((unsigned long)res->pde_data)); + dprintk("MAX_USER_VA_BITS: %d, PGDIR_SHIFT: %d\n", + MAX_USER_VA_BITS, PGDIR_SHIFT); + mcctrl_file_to_pidfd_hash_insert(f.file, os, + task_tgid_vnr(current), + current->group_leader, fd, + fullpath, res->pde_data); + } + +out_free_open: + free_page((unsigned long)pathbuf); +out_fdput_open: + fdput(f); } - pathbuf = kmalloc(PATH_MAX, GFP_ATOMIC); - if (!pathbuf) { - goto out_fdput; - } + /* Ioctl on Tofu CQ? */ + else if (packet->req.number == __NR_ioctl && + packet->req.args[0] > 0 && ret == 0) { + struct fd f; + int fd; + int tni, cq; + long __ret; - fullpath = d_path(&f.file->f_path, pathbuf, PATH_MAX); - if (IS_ERR(fullpath)) { - goto out_free; - } + fd = packet->req.args[0]; + f = fdget(fd); - if (!strncmp("/proc/tofu/dev/", fullpath, 15)) { - res->pde_data = PDE_DATA(file_inode(f.file)); - dprintk("%s: fd: %d, path: %s, PDE_DATA: 0x%lx\n", - __func__, - fd, - fullpath, - (unsigned long)res->pde_data); - dprintk("%s: pgd_index: %ld, pmd_index: %ld, pte_index: %ld\n", - __func__, - pgd_index((unsigned long)res->pde_data), - pmd_index((unsigned long)res->pde_data), - pte_index((unsigned long)res->pde_data)); -#ifdef CONFIG_ARM64 - dprintk("CONFIG_ARM64_VA_BITS: %d, PGDIR_SHIFT: %d\n", - CONFIG_ARM64_VA_BITS, PGDIR_SHIFT); -#endif - mcctrl_file_to_pidfd_hash_insert(f.file, os, - task_tgid_vnr(current), - current->group_leader, fd); - } + if (!f.file) { + goto out_notify; + } -out_free: - kfree(pathbuf); -out_fdput: - fdput(f); + pathbuf = (char *)__get_free_page(GFP_ATOMIC); + if (!pathbuf) { + goto out_fdput_ioctl; + } + + fullpath = d_path(&f.file->f_path, pathbuf, PAGE_SIZE); + if (IS_ERR(fullpath)) { + goto out_free_ioctl; + } + + /* Looking for /proc/tofu/dev/tniXcqY pattern */ + __ret = sscanf(fullpath, "/proc/tofu/dev/tni%dcq%d", &tni, &cq); + if (__ret == 2) { + extern long __mcctrl_tof_utofu_unlocked_ioctl_cq(void *pde_data, + unsigned int cmd, unsigned long arg); + + dprintk("%s: ioctl(): fd: %d, path: %s\n", + __func__, + fd, + fullpath); + + __ret = __mcctrl_tof_utofu_unlocked_ioctl_cq( + PDE_DATA(file_inode(f.file)), + packet->req.args[1], packet->req.args[2]); + } + +out_free_ioctl: + free_page((unsigned long)pathbuf); +out_fdput_ioctl: + fdput(f); + } } out_notify: @@ -2365,7 +2420,7 @@ int __do_in_kernel_irq_syscall(ihk_os_t os, struct ikc_scd_packet *packet) if (ret == -ENOSYS) return -ENOSYS; - __return_syscall(os, packet, ret, 0); + __return_syscall(os, NULL, packet, ret, 0); return 0; } @@ -2590,7 +2645,7 @@ sched_setparam_out: break; } - __return_syscall(os, packet, ret, 0); + __return_syscall(os, NULL, packet, ret, 0); out_no_syscall_return: ihk_ikc_release_packet((struct ihk_ikc_free_packet *)packet);