From 64e2639adc7b4f63cad18f47c16489487c965a33 Mon Sep 17 00:00:00 2001 From: Aram Santogidis Date: Tue, 1 Aug 2017 16:17:04 +0900 Subject: [PATCH] * The relevant files have been modified in order to compile with McKernel. Conflicts: kernel/Makefile.build.in --- arch/x86_64/kernel/cpu.c | 7 + kernel/Makefile.build.in | 1 + kernel/file_ops.c | 27 +++- kernel/include/hfi1/common.h | 4 + kernel/include/hfi1/file_ops.h | 9 ++ kernel/include/hfi1/hfi.h | 65 +++++++- kernel/include/hfi1/hfi1_user.h | 19 ++- kernel/include/hfi1/ihk_hfi1_common.h | 218 ++++++++++++++++++++++++++ kernel/include/hfi1/iowait.h | 39 +++++ kernel/include/hfi1/sdma.h | 54 ++++++- kernel/include/hfi1/sdma_txreq.h | 4 +- kernel/include/hfi1/user_exp_rcv.h | 6 +- kernel/include/hfi1/user_sdma.h | 31 ++++ kernel/sdma.c | 80 ++++++++-- kernel/syscall.c | 13 +- kernel/user_sdma.c | 114 +++++++++++--- lib/include/ihk/cpu.h | 1 + 17 files changed, 637 insertions(+), 55 deletions(-) create mode 100644 kernel/include/hfi1/file_ops.h create mode 100644 kernel/include/hfi1/ihk_hfi1_common.h diff --git a/arch/x86_64/kernel/cpu.c b/arch/x86_64/kernel/cpu.c index f7464d6e..b1e5f2a4 100644 --- a/arch/x86_64/kernel/cpu.c +++ b/arch/x86_64/kernel/cpu.c @@ -1225,6 +1225,13 @@ void cpu_pause(void) asm volatile("pause" ::: "memory"); } +/* From: kernel-xppsl_1.5.2/arch/x86/include/asm/processor.h */ +/* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */ +void cpu_relax(void) +{ + asm volatile("rep; nop" ::: "memory"); +} + /*@ @ assigns \nothing; @ ensures \interrupt_disabled > 0; diff --git a/kernel/Makefile.build.in b/kernel/Makefile.build.in index 77bf0238..faf699a6 100644 --- a/kernel/Makefile.build.in +++ b/kernel/Makefile.build.in @@ -8,6 +8,7 @@ OBJS += process.o copy.o waitq.o futex.o timer.o plist.o fileobj.o shmobj.o OBJS += zeroobj.o procfs.o devobj.o sysfs.o xpmem.o profile.o freeze.o OBJS += rbtree.o OBJS += pager.o +OBJS += file_ops.o user_sdma.o sdma.o # POSTK_DEBUG_ARCH_DEP_18 coredump arch separation. DEPSRCS=$(wildcard $(SRC)/*.c) diff --git a/kernel/file_ops.c b/kernel/file_ops.c index f16cc995..a04a27bc 100644 --- a/kernel/file_ops.c +++ b/kernel/file_ops.c @@ -44,6 +44,15 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ + +#include +#include +#include +#include +#include + +#ifdef __HFI1_ORIG__ + #include #include #include @@ -404,15 +413,23 @@ static long hfi1_file_ioctl(struct file *fp, unsigned int cmd, return ret; } +#endif /* __HFI1_ORIG__ */ +#ifdef __HFI1_ORIG__ static ssize_t hfi1_aio_write(struct kiocb *kiocb, const struct iovec *iovec, unsigned long dim, loff_t offset) { struct hfi1_filedata *fd = kiocb->ki_filp->private_data; +#else +ssize_t hfi1_aio_write(void *private_data, const struct iovec *iovec, unsigned long dim) +{ + struct hfi1_filedata *fd = private_data; +#endif /* __HFI1_ORIG__ */ struct hfi1_user_sdma_pkt_q *pq = fd->pq; struct hfi1_user_sdma_comp_q *cq = fd->cq; int done = 0, reqs = 0; + hfi1_cdbg(AIOWRITE, "+"); if (!cq || !pq) return -EIO; @@ -429,9 +446,15 @@ static ssize_t hfi1_aio_write(struct kiocb *kiocb, const struct iovec *iovec, int ret; unsigned long count = 0; +#ifdef __HFI1_ORIG__ ret = hfi1_user_sdma_process_request( kiocb->ki_filp, (struct iovec *)(iovec + done), dim, &count); +#else + ret = hfi1_user_sdma_process_request( + private_data, (struct iovec *)(iovec + done), + dim, &count); +#endif /* __HFI1_ORIG__ */ if (ret) { reqs = ret; break; @@ -440,9 +463,10 @@ static ssize_t hfi1_aio_write(struct kiocb *kiocb, const struct iovec *iovec, done += count; reqs++; } - + hfi1_cdbg(AIOWRITE, "-"); return reqs; } +#ifdef __HFI1_ORIG__ static int hfi1_file_mmap(struct file *fp, struct vm_area_struct *vma) { @@ -1556,3 +1580,4 @@ void hfi1_device_remove(struct hfi1_devdata *dd) user_remove(dd); hfi1_diag_remove(dd); } +#endif /* __HFI1_ORIG__ */ \ No newline at end of file diff --git a/kernel/include/hfi1/common.h b/kernel/include/hfi1/common.h index 2d22d906..fc867f7c 100644 --- a/kernel/include/hfi1/common.h +++ b/kernel/include/hfi1/common.h @@ -48,7 +48,11 @@ #ifndef _COMMON_H #define _COMMON_H +#ifdef __HFI1_ORIG__ #include "update/hfi1_user.h" +#else +#include +#endif /* __HFI1_ORIG__ */ /* * This file contains defines, structures, etc. that are used diff --git a/kernel/include/hfi1/file_ops.h b/kernel/include/hfi1/file_ops.h new file mode 100644 index 00000000..46189533 --- /dev/null +++ b/kernel/include/hfi1/file_ops.h @@ -0,0 +1,9 @@ +#ifndef _HFI1_FILE_OPS_H_ +#define _HFI1_FILE_OPS_H_ + +#include +#include + +ssize_t hfi1_aio_write(void *private_data, const struct iovec *iovec, unsigned long dim); + +#endif \ No newline at end of file diff --git a/kernel/include/hfi1/hfi.h b/kernel/include/hfi1/hfi.h index 66cbd778..c56b2fcb 100644 --- a/kernel/include/hfi1/hfi.h +++ b/kernel/include/hfi1/hfi.h @@ -47,6 +47,10 @@ * */ +#include + +#ifdef __HFI1_ORIG__ + #include #include #include @@ -92,7 +96,10 @@ #define DROP_PACKET_OFF 0 #define DROP_PACKET_ON 1 +#endif /* __HFI1_ORIG__ */ + extern unsigned long hfi1_cap_mask; + #define HFI1_CAP_KGET_MASK(mask, cap) ((mask) & HFI1_CAP_##cap) #define HFI1_CAP_UGET_MASK(mask, cap) \ (((mask) >> HFI1_CAP_USER_SHIFT) & HFI1_CAP_##cap) @@ -125,6 +132,8 @@ extern unsigned long hfi1_cap_mask; #define NUM_SEND_CTXT_ERR_STATUS_COUNTERS 5 #define NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS 24 +#ifdef __HFI1_ORIG__ + /* * per driver stats, either not device nor port-specific, or * summed over all of the devices and ports. @@ -194,7 +203,7 @@ struct tid_queue { u32 enqueue; /* count of tid enqueues */ u32 dequeue; /* count of tid dequeues */ }; - + struct hfi1_ctxtdata { /* shadow the ctxt's RcvCtrl register */ u64 rcvctrl; @@ -358,6 +367,15 @@ struct hfi1_ctxtdata { int (*do_interrupt)(struct hfi1_ctxtdata *rcd, int threaded); }; +#endif /* __HFI1_ORIG__ */ + +#ifndef __HFI1_ORIG__ +struct hfi1_ctxtdata { + unsigned ctxt; +}; +#endif /* __HFI1_ORIG__ */ + +#ifdef __HFI1_ORIG__ /* * Represents a single packet at a high level. Put commonly computed things in * here so we do not have to keep doing them over and over. The rule of thumb is @@ -584,6 +602,7 @@ struct vl_arb_cache { struct ib_vl_weight_elem table[VL_ARB_TABLE_SIZE]; }; + /* * The structure below encapsulates data relevant to a physical IB Port. * Current chips support only one such port, but the separation @@ -788,7 +807,15 @@ struct hfi1_pportdata { /* Does this port need to prescan for FECNs */ bool cc_prescan; }; +#endif /* __HFI1_ORIG__ */ +#ifndef __HFI1_ORIG__ +struct hfi1_pportdata { + +}; +#endif /* __HFI1_ORIG__ */ + +#ifdef __HFI1_ORIG__ typedef int (*rhf_rcv_function_ptr)(struct hfi1_packet *packet); typedef void (*opcode_handler)(struct hfi1_packet *packet); @@ -857,6 +884,7 @@ struct sdma_vl_map; #define SERIAL_MAX 16 /* length of the serial number */ typedef int (*send_routine)(struct rvt_qp *, struct hfi1_pkt_state *, u64); + struct hfi1_devdata { struct hfi1_ibdev verbs_dev; /* must be first */ struct list_head list; @@ -1169,7 +1197,7 @@ struct hfi1_devdata { /* hfi1_pportdata, points to array of (physical) port-specific * data structs, indexed by pidx (0..n-1) */ - struct hfi1_pportdata *pport; + struct hfi1_pportdata *pport; //used /* receive context data */ struct hfi1_ctxtdata **rcd; u64 __percpu *int_counter; @@ -1221,6 +1249,20 @@ struct hfi1_devdata { struct kobject kobj; }; +#endif /* __HFI1_ORIG__ */ + +#ifndef __HFI1_ORIG__ +struct hfi1_devdata { + struct list_head list; + /* pointers to related structs for this device */ + /* pci access data structure */ + struct pci_dev *pcidev; + dma_addr_t sdma_pad_phys; + /* array of engines sized by num_sdma */ + struct sdma_engine *per_sdma; + struct hfi1_pportdata *pport; +}; +#endif /* __HFI1_ORIG__ */ /* 8051 firmware version helper */ #define dc8051_ver(a, b) ((a) << 8 | (b)) @@ -1230,6 +1272,8 @@ struct hfi1_devdata { #define PT_EAGER 1 #define PT_INVALID 2 +#ifdef __HFI1_ORIG__ + struct tid_rb_node; struct mmu_rb_node; struct mmu_rb_handler; @@ -1254,6 +1298,19 @@ struct hfi1_filedata { spinlock_t invalid_lock; struct mm_struct *mm; }; +#endif /* __HFI1_ORIG__ */ + +#ifndef __HFI1_ORIG__ +/* Private data for file operations */ +struct hfi1_filedata { + struct hfi1_ctxtdata *uctxt; + unsigned subctxt; + struct hfi1_user_sdma_comp_q *cq; + struct hfi1_user_sdma_pkt_q *pq; +}; +#endif /* __HFI1_ORIG__ */ + +#ifdef __HFI1_ORIG__ extern struct list_head hfi1_dev_list; extern spinlock_t hfi1_devs_lock; @@ -2039,10 +2096,12 @@ void hfi1_format_hwerrors(u64 hwerrs, const struct hfi1_hwerror_msgs *hwerrmsgs, size_t nhwerrmsgs, char *msg, size_t lmsg); +#endif /* __HFI1_ORIG__ */ #define USER_OPCODE_CHECK_VAL 0xC0 #define USER_OPCODE_CHECK_MASK 0xC0 #define OPCODE_CHECK_VAL_DISABLED 0x0 #define OPCODE_CHECK_MASK_DISABLED 0x0 +#ifdef __HFI1_ORIG__ static inline void hfi1_reset_cpu_counters(struct hfi1_devdata *dd) { @@ -2149,4 +2208,6 @@ __print_symbolic(opcode, \ ib_opcode_name(UD_SEND_ONLY), \ ib_opcode_name(UD_SEND_ONLY_WITH_IMMEDIATE), \ ib_opcode_name(CNP)) + +#endif /* __HFI1_ORIG__ */ #endif /* _HFI1_KERNEL_H */ diff --git a/kernel/include/hfi1/hfi1_user.h b/kernel/include/hfi1/hfi1_user.h index 4705969a..d4f87d3b 100644 --- a/kernel/include/hfi1/hfi1_user.h +++ b/kernel/include/hfi1/hfi1_user.h @@ -56,8 +56,12 @@ #ifndef _LINUX__HFI1_USER_H #define _LINUX__HFI1_USER_H +#ifdef __HFI1_ORIG__ + #include +#endif /* __HFI1_ORIG__ */ + /* * This version number is given to the driver by the user code during * initialization in the spu_userversion field of hfi1_user_info, so @@ -211,6 +215,8 @@ struct hfi1_cmd; #define HFI1_POLL_TYPE_ANYRCV 0x0 #define HFI1_POLL_TYPE_URGENT 0x1 +#ifdef __HFI1_ORIG__ + /* * This structure is passed to the driver to tell it where * user code buffers are, sizes, etc. The offsets and sizes of the @@ -264,6 +270,7 @@ struct hfi1_tid_info { /* length of transfer buffer programmed by this request */ __u32 length; }; +#endif /* __HFI1_ORIG__ */ enum hfi1_sdma_comp_state { FREE = 0, @@ -280,6 +287,8 @@ struct hfi1_sdma_comp_entry { __u32 errcode; }; +#ifdef __HFI1_ORIG__ + /* * Device status and notifications from driver to user-space. */ @@ -353,6 +362,7 @@ struct hfi1_base_info { __u64 subctxt_rcvegrbuf; __u64 subctxt_rcvhdrbuf; }; +#endif /* __HFI1_ORIG__ */ enum sdma_req_opcode { EXPECTED = 0, @@ -391,7 +401,7 @@ struct sdma_req_info { * in charge of managing its own ring. */ __u16 comp_idx; -} __packed; +} __attribute__((packed)); /* * SW KDETH header. @@ -402,7 +412,7 @@ struct hfi1_kdeth_header { __le16 jkey; __le16 hcrc; __le32 swdata[7]; -} __packed; +} __attribute__((packed)); /* * Structure describing the headers that User space uses. The @@ -413,8 +423,9 @@ struct hfi1_pkt_header { __be16 lrh[4]; __be32 bth[3]; struct hfi1_kdeth_header kdeth; -} __packed; +} __attribute__((packed)); +#ifdef __HFI1_ORIG__ /* * The list of usermode accessible registers. @@ -435,5 +446,5 @@ enum hfi1_ureg { /* (RW) Receive TID flow table */ ur_rcvtidflowtable = 256 }; - +#endif /* __HFI1_ORIG__ */ #endif /* _LINIUX__HFI1_USER_H */ diff --git a/kernel/include/hfi1/ihk_hfi1_common.h b/kernel/include/hfi1/ihk_hfi1_common.h new file mode 100644 index 00000000..e42b32db --- /dev/null +++ b/kernel/include/hfi1/ihk_hfi1_common.h @@ -0,0 +1,218 @@ +#ifndef _IHK_HFI1_COMMON_H_ +#define _IHK_HFI1_COMMON_H_ + +#include +#include +#include +#include +#include +#include + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) +#endif + +/* From: mckernel/kernel/include/xpmem_private.h */ +#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK) +#define min(x, y) ({ \ + __typeof__(x) _min1 = (x); \ + __typeof__(y) _min2 = (y); \ + (void) (&_min1 == &_min2); \ + _min1 < _min2 ? _min1 : _min2;}) + + +#define BIT_ULL(nr) (1ULL << (nr)) + +/* Disable debug macros */ +#define hfi1_cdbg(...) do {} while(0) +#define SDMA_DBG(...) do {} while(0) +#define WARN_ON(...) do {} while(0) +#define WARN_ON_ONCE WARN_ON // use the local definition +#define trace_hfi1_ahg_allocate(...) do {} while(0) +#define trace_hfi1_ahg_deallocate(...) do {} while(0) + +/* Byte swapping */ +#define be32_to_cpu(x) __builtin_bswap32(x) +#define be16_to_cpu(x) __builtin_bswap16(x) +#define le32_to_cpu(x) x +#define le16_to_cpu(x) x +#define cpu_to_le16(x) x +#define cpu_to_le32(x) x +#define cpu_to_le64(x) x +#define __cpu_to_le64(x) x +#define __le64_to_cpu(x) x +#define __le32_to_cpu(x) x +#define __le16_to_cpu(x) x +//TODO: double-check +#define cpu_to_be16(x) __builtin_bswap16(x) +#define cpu_to_be32(x) __builtin_bswap32(x) + +/* Compiler */ +#ifndef likely +# define likely(x) __builtin_expect(!!(x), 1) +#endif + +#ifndef unlikely +# define unlikely(x) __builtin_expect(!!(x), 0) +#endif + +/* From: kernel-xppsl_1.5.2/include/linux/compiler.h */ +#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x)) + +/* Atomic ops */ +#define atomic_inc ihk_atomic_inc +#define atomic_dec ihk_atomic_dec +#define atomic_read ihk_atomic_read +#define atomic_add ihk_atomic_add +#define atomic_t ihk_atomic_t + +/* TODO***********************************/ +#define spin_lock_irqsave(lock, flags) do {} while(0) +#define spin_unlock_irqsave(lock, flags) do {} while(0) +#define spin_unlock_irqrestore(lock, flags) do {} while(0) +typedef ihk_spinlock_t spinlock_t; +#define ____cacheline_aligned_in_smp +#define __iomem +#define spin_lock(...) do {} while(0) +#define spin_unlock(...) do {} while(0) +#define smp_wmb() barrier() +#define smp_rmb() barrier() +/***********************************************/ + +/* TODO: Figure the corresponding flag for McKernel-kmalloc()*/ +#define __GFP_ZERO 0 +#define GFP_KERNEL 0 + +/* kernel-xppsl_1.5.2/include/linux/seqlock.h */ +/***********************************************/ +typedef struct seqcount { + unsigned sequence; +} seqcount_t; + +typedef struct { + struct seqcount seqcount; + spinlock_t lock; +} seqlock_t; + +static inline unsigned raw_seqcount_begin(const seqcount_t *s) +{ + unsigned ret = ACCESS_ONCE(s->sequence); + smp_rmb(); + return ret & ~1; +} +/***********************************************/ + +/* Misc */ +/* From: kernel-xppsl_1.5.2/include/linux/kernel.h */ +#define min_t(type, x, y) ({ \ + type __min1 = (x); \ + type __min2 = (y); \ + __min1 < __min2 ? __min1: __min2; }) + +#define SIZE_MAX (~(size_t)0) +#define MAX_TID_PAIR_ENTRIES 1024 /* max receive expected pairs */ +#define PIO_BLOCK_SIZE 64 /* bytes */ +/* From: chip.c/h */ +//num_vls = HFI1_MAX_VLS_SUPPORTED; +//num_vls = dd->chip_sdma_engines; +#define HFI1_MAX_VLS_SUPPORTED 8 + + +/* integer typedefs */ +typedef __signed__ char __s8; +typedef unsigned char __u8; + +typedef __signed__ short __s16; +typedef unsigned short __u16; + +typedef __signed__ int __s32; +typedef unsigned int __u32; + +typedef __signed__ long __s64; +typedef unsigned long __u64; + +typedef __u64 u64; +typedef __s64 s64; + +typedef __u32 u32; +typedef __s32 s32; + +typedef __u16 u16; +typedef __s16 s16; + +typedef __u8 u8; +typedef __s8 s8; + +typedef __u16 __le16; +typedef __u16 __be16; +typedef __u32 __le32; +typedef __u32 __be32; +typedef __u64 __le64; +typedef __u64 __be64; + +typedef unsigned int uint; + +/* TODO: There should be a header file that I can include */ +typedef _Bool bool; +#define false 0 +#define true !false + +/* TODO: double check this typedef */ +typedef u64 dma_addr_t; + +/* From: kernel-xppsl_1.5.2/include/linux/types.h */ +typedef unsigned gfp_t; + +/* kernel-xppsl_1.5.2/include/asm-generic/io.h */ +#ifndef __raw_writeq +static inline void __raw_writeq(u64 b, volatile void __iomem *addr) +{ + *(volatile u64 __force *) addr = b; +} +#endif +#define writeq(b, addr) __raw_writeq(__cpu_to_le64(b), addr) + + +/* TODO: I'm not sure if this definition is correct */ +#define LOCK_PREFIX "lock; " + +/* From: kernel-xppsl_1.5.2/arch/x86/include/asm/bitops.h */ +#define BITOP_ADDR(x) "+m" (*(volatile long *) (x)) +#define LINUX_ADDR BITOP_ADDR(addr) + +/* From: kernel-xppsl_1.5.2/arch/x86/include/asm/bitops.h */ +static inline int test_and_set_bit(int nr, volatile unsigned long *addr) +{ + int oldbit; + + asm volatile(LOCK_PREFIX "bts %2,%1\n\t" + "sbb %0,%0" : "=r" (oldbit), LINUX_ADDR : "Ir" (nr) : "memory"); + + return oldbit; +} + +/* From: kernel-xppsl_1.5.2/arch/x86/include/asm/atomic.h */ +static inline int atomic_dec_and_test(atomic_t *v) +{ + unsigned char c; + + asm volatile(LOCK_PREFIX "decl %0; sete %1" + : "+m" (v->counter), "=qm" (c) + : : "memory"); + return c != 0; +} + +/* From: kernel-xppsl_1.5.2/include/linux/slab.h */ +static inline void *kmalloc_array(size_t n, size_t size, gfp_t flags) +{ + if (size != 0 && n > SIZE_MAX / size) + return NULL; + return __kmalloc(n * size, flags); +} + +static inline void *kcalloc(size_t n, size_t size, gfp_t flags) +{ + return kmalloc(n * size, flags | __GFP_ZERO); +} + +#endif diff --git a/kernel/include/hfi1/iowait.h b/kernel/include/hfi1/iowait.h index 829d9368..7857a3cf 100644 --- a/kernel/include/hfi1/iowait.h +++ b/kernel/include/hfi1/iowait.h @@ -47,6 +47,7 @@ * */ +#ifdef __HFI1_ORIG__ #include #include #include @@ -58,6 +59,7 @@ * @work: pointer to work structure */ typedef void (*restart_t)(struct work_struct *work); +#endif /* __HFI1_ORIG__ */ #define IOWAIT_PENDING_IB 0x0 #define IOWAIT_PENDING_TID 0x1 @@ -88,7 +90,11 @@ struct sdma_engine; */ struct iowait; struct iowait_work { +#ifdef __HFI1_ORIG__ struct work_struct iowork; +#else +//TODO: +#endif /* __HFI1_ORIG__ */ struct list_head tx_head; struct iowait *iow; }; @@ -140,9 +146,13 @@ struct iowait { unsigned seq); void (*wakeup)(struct iowait *wait, int reason); void (*sdma_drained)(struct iowait *wait); +#ifdef __HFI1_ORIG__ seqlock_t *lock; wait_queue_head_t wait_dma; wait_queue_head_t wait_pio; +#else +//TODO: +#endif /* __HFI1_ORIG__ */ atomic_t sdma_busy; atomic_t pio_busy; u32 count; @@ -154,6 +164,8 @@ struct iowait { #define SDMA_AVAIL_REASON 0 +#ifdef __HFI1_ORIG__ + void iowait_set_flag(struct iowait *wait, u32 flag); bool iowait_flag_set(struct iowait *wait, u32 flag); void iowait_clear_flag(struct iowait *wait, u32 flag); @@ -182,6 +194,7 @@ static inline bool iowait_schedule( struct workqueue_struct *wq, int cpu) { + hfi1_cdbg(AIOWRITE, "."); return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_IB_SE].iowork); } @@ -196,6 +209,7 @@ static inline bool iowait_tid_schedule( struct workqueue_struct *wq, int cpu) { + hfi1_cdbg(AIOWRITE, "."); return !!queue_work_on(cpu, wq, &wait->wait[IOWAIT_TID_SE].iowork); } @@ -208,6 +222,7 @@ static inline bool iowait_tid_schedule( */ static inline void iowait_sdma_drain(struct iowait *wait) { + hfi1_cdbg(AIOWRITE, "."); wait_event(wait->wait_dma, !atomic_read(&wait->sdma_busy)); } @@ -219,6 +234,7 @@ static inline void iowait_sdma_drain(struct iowait *wait) */ static inline int iowait_sdma_pending(struct iowait *wait) { + hfi1_cdbg(AIOWRITE, "."); return atomic_read(&wait->sdma_busy); } @@ -228,17 +244,21 @@ static inline int iowait_sdma_pending(struct iowait *wait) */ static inline void iowait_sdma_inc(struct iowait *wait) { + hfi1_cdbg(AIOWRITE, "."); atomic_inc(&wait->sdma_busy); } +#endif /** * iowait_sdma_add - add count to pending * @wait: iowait_work structure */ static inline void iowait_sdma_add(struct iowait *wait, int count) { + hfi1_cdbg(AIOWRITE, "."); atomic_add(count, &wait->sdma_busy); } +#ifdef __HFI1_ORIG__ /** * iowait_pio_drain() - wait for pios to drain @@ -250,6 +270,7 @@ static inline void iowait_sdma_add(struct iowait *wait, int count) */ static inline void iowait_pio_drain(struct iowait *wait) { + hfi1_cdbg(AIOWRITE, "."); wait_event_timeout(wait->wait_pio, !atomic_read(&wait->pio_busy), HZ); @@ -263,6 +284,7 @@ static inline void iowait_pio_drain(struct iowait *wait) */ static inline int iowait_pio_pending(struct iowait *w) { + hfi1_cdbg(AIOWRITE, "."); return atomic_read(&w->pio_busy); } @@ -274,6 +296,7 @@ static inline int iowait_pio_pending(struct iowait *w) */ static inline void iowait_drain_wakeup(struct iowait *w) { + hfi1_cdbg(AIOWRITE, "."); wake_up(&w->wait_dma); wake_up(&w->wait_pio); if (w->sdma_drained) @@ -286,6 +309,7 @@ static inline void iowait_drain_wakeup(struct iowait *w) */ static inline void iowait_pio_inc(struct iowait *wait) { + hfi1_cdbg(AIOWRITE, "."); atomic_inc(&wait->pio_busy); } @@ -295,6 +319,7 @@ static inline void iowait_pio_inc(struct iowait *wait) */ static inline int iowait_pio_dec(struct iowait *wait) { + hfi1_cdbg(AIOWRITE, "."); if (!wait) return 0; return atomic_dec_and_test(&wait->pio_busy); @@ -306,6 +331,7 @@ static inline int iowait_pio_dec(struct iowait *wait) */ static inline int iowait_sdma_dec(struct iowait *wait) { + hfi1_cdbg(AIOWRITE, "."); if (!wait) return 0; return atomic_dec_and_test(&wait->sdma_busy); @@ -319,6 +345,7 @@ static inline struct sdma_txreq *iowait_get_txhead(struct iowait_work *wait) { struct sdma_txreq *tx = NULL; + hfi1_cdbg(AIOWRITE, "."); if (!list_empty(&wait->tx_head)) { tx = list_first_entry( &wait->tx_head, @@ -333,6 +360,7 @@ static inline u16 iowait_get_desc(struct iowait_work *w) { u16 num_desc = 0; struct sdma_txreq *tx = NULL; + hfi1_cdbg(AIOWRITE, "."); if (!list_empty(&w->tx_head)) { tx = list_first_entry( @@ -348,6 +376,7 @@ static inline u32 iowait_get_all_desc(struct iowait *w) { u32 num_desc = 0; + hfi1_cdbg(AIOWRITE, "."); num_desc = iowait_get_desc(&w->wait[IOWAIT_IB_SE]); num_desc += iowait_get_desc(&w->wait[IOWAIT_TID_SE]); return num_desc; @@ -359,9 +388,11 @@ static inline u32 iowait_get_all_desc(struct iowait *w) */ static inline bool iowait_packet_queued(struct iowait_work *w) { + hfi1_cdbg(AIOWRITE, "."); return !list_empty(&w->tx_head); } +#endif /* __HFI1_ORIG__ */ /** * inc_wait_count - increment wait counts * @w: the log work struct @@ -369,11 +400,13 @@ static inline bool iowait_packet_queued(struct iowait_work *w) */ static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n) { + hfi1_cdbg(AIOWRITE, "."); if (!w) return; w->iow->tx_count++; w->iow->count += n; } +#ifdef __HFI1_ORIG__ /** * iowait_get_tid_work - return iowait_work for tid SE @@ -381,15 +414,18 @@ static inline void iowait_inc_wait_count(struct iowait_work *w, u16 n) */ static inline struct iowait_work *iowait_get_tid_work(struct iowait *w) { + hfi1_cdbg(AIOWRITE, "."); return &w->wait[IOWAIT_TID_SE]; } +#endif /* __HFI1_ORIG__ */ /** * iowait_get_ib_work - return iowait_work for ib SE * @w: the iowait struct */ static inline struct iowait_work *iowait_get_ib_work(struct iowait *w) { + hfi1_cdbg(AIOWRITE, "."); return &w->wait[IOWAIT_IB_SE]; } @@ -399,12 +435,15 @@ static inline struct iowait_work *iowait_get_ib_work(struct iowait *w) */ static inline struct iowait *iowait_ioww_to_iow(struct iowait_work *w) { + hfi1_cdbg(AIOWRITE, "."); if (likely(w)) return w->iow; return NULL; } +#ifdef __HFI1_ORIG__ void iowait_cancel_work(struct iowait *w); int iowait_set_work_flag(struct iowait_work *w); +#endif /* __HFI1_ORIG__ */ #endif diff --git a/kernel/include/hfi1/sdma.h b/kernel/include/hfi1/sdma.h index 7a9201e1..bb1f8d59 100644 --- a/kernel/include/hfi1/sdma.h +++ b/kernel/include/hfi1/sdma.h @@ -47,6 +47,11 @@ * */ +#include +#include + +#ifdef __HFI1_ORIG__ + #include #include #include @@ -57,6 +62,11 @@ #include "verbs.h" #include "sdma_txreq.h" +#define hfi1_cdbg(which, fmt, ...) \ + __hfi1_trace_##which(__func__, fmt, ##__VA_ARGS__) +extern void __hfi1_trace_AIOWRITE(const char *func, char *fmt, ...); +#endif /* __HFI1_ORIG__ */ + /* Hardware limit */ #define MAX_DESC 64 /* Hardware limit for SDMA packet size */ @@ -192,6 +202,7 @@ struct sdma_set_state_action { unsigned go_s99_running_totrue:1; }; +#ifdef __HFI1_ORIG__ struct sdma_state { struct kref kref; struct completion comp; @@ -203,6 +214,11 @@ struct sdma_state { unsigned previous_op; enum sdma_events last_event; }; +#else +struct sdma_state { + enum sdma_states current_state; +}; +#endif /* __HFI1_ORIG__ */ /** * DOC: sdma exported routines @@ -394,6 +410,7 @@ struct sdma_engine { /* private: */ struct list_head dmawait; +#ifdef __HFI1_ORIG__ /* CONFIG SDMA for now, just blindly duplicate */ /* private: */ struct tasklet_struct sdma_hw_clean_up_task @@ -409,14 +426,20 @@ struct sdma_engine { u32 progress_check_head; /* private: */ struct work_struct flush_worker; +#endif /* __HFI1_ORIG__ */ /* protect flush list */ spinlock_t flushlist_lock; /* private: */ struct list_head flushlist; +#ifdef __HFI1_ORIG__ struct cpumask cpu_mask; struct kobject kobj; +#endif /* __HFI1_ORIG__ */ }; + +#ifdef __HFI1_ORIG__ + int sdma_init(struct hfi1_devdata *dd, u8 port); void sdma_start(struct hfi1_devdata *dd); void sdma_exit(struct hfi1_devdata *dd); @@ -441,6 +464,7 @@ static inline int sdma_empty(struct sdma_engine *sde) return sde->descq_tail == sde->descq_head; } +#endif /* __HFI1_ORIG__ */ static inline u16 sdma_descq_freecnt(struct sdma_engine *sde) { return sde->descq_cnt - @@ -478,9 +502,11 @@ static inline int sdma_running(struct sdma_engine *engine) unsigned long flags; int ret; + hfi1_cdbg(AIOWRITE, "+"); spin_lock_irqsave(&engine->tail_lock, flags); ret = __sdma_running(engine); spin_unlock_irqrestore(&engine->tail_lock, flags); + hfi1_cdbg(AIOWRITE, "-"); return ret; } @@ -619,6 +645,7 @@ static inline int sdma_txinit( { return sdma_txinit_ahg(tx, flags, tlen, 0, 0, NULL, 0, cb); } +#ifdef __HFI1_ORIG__ /* helpers - don't use */ static inline int sdma_mapping_type(struct sdma_desc *d) @@ -639,6 +666,7 @@ static inline dma_addr_t sdma_mapping_addr(struct sdma_desc *d) >> SDMA_DESC0_PHY_ADDR_SHIFT; } +#endif /* __HFI1_ORIG__ */ static inline void make_tx_sdma_desc( struct sdma_txreq *tx, int type, @@ -666,7 +694,6 @@ static inline void make_tx_sdma_desc( int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx, int type, void *kvaddr, struct page *page, unsigned long offset, u16 len); -int _pad_sdma_tx_descs(struct hfi1_devdata *, struct sdma_txreq *); void __sdma_txclean(struct hfi1_devdata *, struct sdma_txreq *); static inline void sdma_txclean(struct hfi1_devdata *dd, struct sdma_txreq *tx) @@ -674,6 +701,8 @@ static inline void sdma_txclean(struct hfi1_devdata *dd, struct sdma_txreq *tx) if (tx->num_desc) __sdma_txclean(dd, tx); } +#ifdef __HFI1_ORIG__ +int _pad_sdma_tx_descs(struct hfi1_devdata *, struct sdma_txreq *); /* helpers used by public routines */ static inline void _sdma_close_tx(struct hfi1_devdata *dd, @@ -689,6 +718,7 @@ static inline void _sdma_close_tx(struct hfi1_devdata *dd, SDMA_DESC1_INT_REQ_FLAG); } +#endif /* __HFI1_ORIG__ */ static inline int _sdma_txadd_daddr( struct hfi1_devdata *dd, int type, @@ -707,11 +737,13 @@ static inline int _sdma_txadd_daddr( /* special cases for last */ if (!tx->tlen) { if (tx->packet_len & (sizeof(u32) - 1)) { - rval = _pad_sdma_tx_descs(dd, tx); + //TODO: _pad_sdma_tx_descs + //rval = _pad_sdma_tx_descs(dd, tx); if (rval) return rval; } else { - _sdma_close_tx(dd, tx); + //TODO: _sdma_close_tx + //_sdma_close_tx(dd, tx); } } tx->num_desc++; @@ -743,7 +775,7 @@ static inline int sdma_txadd_page( { dma_addr_t addr; int rval; - + hfi1_cdbg(AIOWRITE, "+"); if ((unlikely(tx->num_desc == tx->desc_limit))) { rval = ext_coal_sdma_tx_descs(dd, tx, SDMA_MAP_PAGE, NULL, page, offset, len); @@ -751,6 +783,7 @@ static inline int sdma_txadd_page( return rval; } +#ifdef __HFI1_ORIG__ addr = dma_map_page( &dd->pcidev->dev, page, @@ -762,7 +795,11 @@ static inline int sdma_txadd_page( __sdma_txclean(dd, tx); return -ENOSPC; } +#else + //TODO: dma_map_page +#endif /* __HFI1_ORIG__ */ + hfi1_cdbg(AIOWRITE, "-"); return _sdma_txadd_daddr( dd, SDMA_MAP_PAGE, tx, addr, len); } @@ -833,6 +870,7 @@ static inline int sdma_txadd_kvaddr( return rval; } +#ifdef __HFI1_ORIG__ addr = dma_map_single( &dd->pcidev->dev, kvaddr, @@ -843,6 +881,9 @@ static inline int sdma_txadd_kvaddr( __sdma_txclean(dd, tx); return -ENOSPC; } +#else +//TODO: dma_map_single +#endif /* __HFI1_ORIG__ */ return _sdma_txadd_daddr( dd, SDMA_MAP_SINGLE, tx, addr, len); @@ -885,6 +926,7 @@ static inline u32 sdma_build_ahg_descriptor( ((data & SDMA_AHG_VALUE_MASK) << SDMA_AHG_VALUE_SHIFT)); } +#ifdef __HFI1_ORIG__ /** * sdma_progress - use seq number of detect head progress @@ -1061,6 +1103,7 @@ struct sdma_engine *sdma_select_engine_sc( u32 selector, u8 sc5); +#endif /* __HFI1_ORIG__ */ struct sdma_engine *sdma_select_engine_vl( struct hfi1_devdata *dd, u32 selector, @@ -1068,6 +1111,8 @@ struct sdma_engine *sdma_select_engine_vl( struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd, u32 selector, u8 vl); +#ifdef __HFI1_ORIG__ + ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf); ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf, size_t count); @@ -1095,4 +1140,5 @@ extern uint mod_num_sdma; void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid); +#endif /* __HFI1_ORIG__ */ #endif diff --git a/kernel/include/hfi1/sdma_txreq.h b/kernel/include/hfi1/sdma_txreq.h index 24bb6f1b..1a28fca5 100644 --- a/kernel/include/hfi1/sdma_txreq.h +++ b/kernel/include/hfi1/sdma_txreq.h @@ -48,6 +48,8 @@ #ifndef HFI1_SDMA_TXREQ_H #define HFI1_SDMA_TXREQ_H +#include + /* increased for AHG */ #define NUM_DESC 6 @@ -105,7 +107,7 @@ struct sdma_txreq { /* private: */ struct iowait *wait; /* private: */ - callback_t complete; + callback_t complete; #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER u64 sn; #endif diff --git a/kernel/include/hfi1/user_exp_rcv.h b/kernel/include/hfi1/user_exp_rcv.h index 6758c6d7..7ebc0bc0 100644 --- a/kernel/include/hfi1/user_exp_rcv.h +++ b/kernel/include/hfi1/user_exp_rcv.h @@ -46,8 +46,9 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ - +#ifdef __HFI1_ORIG__ #include "hfi.h" +#endif /* __HFI1_ORIG__ */ #define EXP_TID_TIDLEN_MASK 0x7FFULL #define EXP_TID_TIDLEN_SHIFT 0 @@ -70,6 +71,7 @@ (tid) |= EXP_TID_SET(field, (value)); \ } while (0) +#ifdef __HFI1_ORIG__ struct tid_group { struct list_head list; unsigned base; @@ -154,4 +156,6 @@ int hfi1_user_exp_rcv_setup(struct file *, struct hfi1_tid_info *); int hfi1_user_exp_rcv_clear(struct file *, struct hfi1_tid_info *); int hfi1_user_exp_rcv_invalid(struct file *, struct hfi1_tid_info *); +#endif /* __HFI1_ORIG__ */ + #endif /* _HFI1_USER_EXP_RCV_H */ diff --git a/kernel/include/hfi1/user_sdma.h b/kernel/include/hfi1/user_sdma.h index 1b04c7cc..65c3a995 100644 --- a/kernel/include/hfi1/user_sdma.h +++ b/kernel/include/hfi1/user_sdma.h @@ -1,3 +1,7 @@ + +#ifndef _HFI1_USER_SDMA_H +#define _HFI1_USER_SDMA_H + /* * Copyright(c) 2015, 2016 Intel Corporation. * @@ -44,6 +48,17 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef __HFI1_ORIG__ + #include #include @@ -53,6 +68,7 @@ extern uint extended_psn; +#endif /* __HFI1_ORIG__ */ /* * Define fields in the KDETH header so we can update the header * template. @@ -105,16 +121,24 @@ struct hfi1_user_sdma_pkt_q { atomic_t n_reqs; u16 reqidx; struct hfi1_devdata *dd; +#ifdef __HFI1_ORIG__ struct kmem_cache *txreq_cache; +#else + void *txreq_cache; //unused +#endif /* __HFI1_ORIG__ */ struct user_sdma_request *reqs; unsigned long *req_in_use; struct iowait busy; unsigned state; +#ifdef __HFI1_ORIG__ wait_queue_head_t wait; unsigned long unpinned; struct mmu_rb_handler *handler; atomic_t n_locked; struct mm_struct *mm; +#else + //TODO: +#endif /* __HFI1_ORIG__ */ }; struct hfi1_user_sdma_comp_q { @@ -122,7 +146,14 @@ struct hfi1_user_sdma_comp_q { struct hfi1_sdma_comp_entry *comps; }; +int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec, + unsigned long dim, unsigned long *count); +#ifdef __HFI1_ORIG__ + int hfi1_user_sdma_alloc_queues(struct hfi1_ctxtdata *, struct file *); int hfi1_user_sdma_free_queues(struct hfi1_filedata *); int hfi1_user_sdma_process_request(struct file *, struct iovec *, unsigned long, unsigned long *); + +#endif /* __HFI1_ORIG__ */ +#endif /* _HFI1_SDMA_H */ \ No newline at end of file diff --git a/kernel/sdma.c b/kernel/sdma.c index 22d2fb2e..e66dcc46 100644 --- a/kernel/sdma.c +++ b/kernel/sdma.c @@ -45,6 +45,15 @@ * */ +#include +#include +#include +#include + +unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT; + +#ifdef __HFI1_ORIG__ + #include #include #include @@ -62,11 +71,15 @@ #include "iowait.h" #include "trace.h" +#endif /* __HFI1_ORIG__ */ + /* must be a power of 2 >= 64 <= 32768 */ #define SDMA_DESCQ_CNT 2048 #define SDMA_DESC_INTR 64 #define INVALID_TAIL 0xffff +#ifdef __HFI1_ORIG__ + static uint sdma_descq_cnt = SDMA_DESCQ_CNT; module_param(sdma_descq_cnt, uint, S_IRUGO); MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries"); @@ -226,7 +239,9 @@ static const struct sdma_set_state_action sdma_action_table[] = { }, }; +#endif /* __HFI1_ORIG__ */ #define SDMA_TAIL_UPDATE_THRESH 0x1F +#ifdef __HFI1_ORIG__ /* declare all statics here rather than keep sorting */ static void sdma_complete(struct kref *); @@ -368,7 +383,7 @@ static inline void complete_tx(struct sdma_engine *sde, /* protect against complete modifying */ struct iowait *wait = tx->wait; callback_t complete = tx->complete; - + hfi1_cdbg(AIOWRITE, "+"); #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER trace_hfi1_sdma_out_sn(sde, tx->sn); if (WARN_ON_ONCE(sde->head_sn != tx->sn)) @@ -381,6 +396,7 @@ static inline void complete_tx(struct sdma_engine *sde, (*complete)(tx, res); if (iowait_sdma_dec(wait)) iowait_drain_wakeup(wait); + hfi1_cdbg(AIOWRITE, "-"); } /* @@ -773,11 +789,16 @@ struct sdma_engine *sdma_select_engine_vl( struct sdma_map_elem *e; struct sdma_engine *rval; + hfi1_cdbg(AIOWRITE, "+"); /* NOTE This should only happen if SC->VL changed after the initial * checks on the QP/AH * Default will return engine 0 below */ +#ifdef __HFI1_ORIG__ if (vl >= num_vls) { +#else + if (vl >= HFI1_MAX_VLS_SUPPORTED) { +#endif /* __HFI1_ORIG__ */ rval = NULL; goto done; } @@ -795,6 +816,7 @@ struct sdma_engine *sdma_select_engine_vl( done: rval = !rval ? &dd->per_sdma[0] : rval; trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx); + hfi1_cdbg(AIOWRITE, "-"); return rval; } @@ -864,6 +886,7 @@ struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd, const struct cpumask *current_mask = tsk_cpus_allowed(current); unsigned long cpu_id; + hfi1_cdbg(AIOWRITE, "+"); /* * To ensure that always the same sdma engine(s) will be * selected make sure the process is pinned to this CPU only. @@ -1658,6 +1681,7 @@ static inline void sdma_unmap_desc( break; } } +#endif /* __HFI1_ORIG__ */ /* * return the mode as indicated by the first @@ -1689,13 +1713,15 @@ void __sdma_txclean( if (tx->num_desc) { u8 skip = 0, mode = ahg_mode(tx); + /* TODO: enable sdma_unmap_desc */ /* unmap first */ - sdma_unmap_desc(dd, &tx->descp[0]); + //sdma_unmap_desc(dd, &tx->descp[0]); /* determine number of AHG descriptors to skip */ if (mode > SDMA_AHG_APPLY_UPDATE1) skip = mode >> 1; - for (i = 1 + skip; i < tx->num_desc; i++) - sdma_unmap_desc(dd, &tx->descp[i]); + /* TODO: enable sdma_unmap_desc */ + // for (i = 1 + skip; i < tx->num_desc; i++) + // sdma_unmap_desc(dd, &tx->descp[i]); tx->num_desc = 0; } kfree(tx->coalesce_buf); @@ -1706,6 +1732,7 @@ void __sdma_txclean( kfree(tx->descp); } } +#ifdef __HFI1_ORIG__ static inline u16 sdma_gethead(struct sdma_engine *sde) { @@ -1824,6 +1851,7 @@ static void sdma_make_progress(struct sdma_engine *sde, u64 status) u16 hwhead, swhead; int idle_check_done = 0; + hfi1_cdbg(AIOWRITE, "+"); hwhead = sdma_gethead(sde); /* The reason for some of the complexity of this code is that @@ -1875,6 +1903,7 @@ retry: sde->last_status = status; if (progress) sdma_desc_avail(sde, sdma_descq_freecnt(sde)); + hfi1_cdbg(AIOWRITE, "-"); } /* @@ -1888,6 +1917,7 @@ retry: */ void sdma_engine_interrupt(struct sdma_engine *sde, u64 status) { + hfi1_cdbg(AIOWRITE, "+"); trace_hfi1_sdma_engine_interrupt(sde, status); write_seqlock(&sde->head_lock); sdma_set_desc_cnt(sde, sdma_desct_intr); @@ -1899,6 +1929,7 @@ void sdma_engine_interrupt(struct sdma_engine *sde, u64 status) sde->sdma_int_cnt++; sdma_make_progress(sde, status); write_sequnlock(&sde->head_lock); + hfi1_cdbg(AIOWRITE, "-"); } /** @@ -2000,12 +2031,15 @@ static void sdma_setlengen(struct sdma_engine *sde) (4ULL << SD(LEN_GEN_GENERATION_SHIFT))); } +#endif /* __HFI1_ORIG__ */ static inline void sdma_update_tail(struct sdma_engine *sde, u16 tail) { + hfi1_cdbg(AIOWRITE, "."); /* Commit writes to memory and advance the tail on the chip */ smp_wmb(); /* see get_txhead() */ writeq(tail, sde->tail_csr); } +#ifdef __HFI1_ORIG__ /* * This is called when changing to state s10_hw_start_up_halt_wait as @@ -2270,6 +2304,7 @@ void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *sde) } } +#endif /* __HFI1_ORIG__ */ /* * add the generation number into * the qw1 and return @@ -2306,12 +2341,12 @@ static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx) u16 tail; struct sdma_desc *descp = tx->descp; u8 skip = 0, mode = ahg_mode(tx); - + hfi1_cdbg(AIOWRITE, "+"); tail = sde->descq_tail & sde->sdma_mask; sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1])); - trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1], - tail, &sde->descq[tail]); + // trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1], + // tail, &sde->descq[tail]); tail = ++sde->descq_tail & sde->sdma_mask; descp++; if (mode > SDMA_AHG_APPLY_UPDATE1) @@ -2329,18 +2364,19 @@ static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx) qw1 = add_gen(sde, descp->qw[1]); } sde->descq[tail].qw[1] = cpu_to_le64(qw1); - trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1, - tail, &sde->descq[tail]); + // trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1, + // tail, &sde->descq[tail]); tail = ++sde->descq_tail & sde->sdma_mask; } tx->next_descq_idx = tail; #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER tx->sn = sde->tail_sn++; - trace_hfi1_sdma_in_sn(sde, tx->sn); + // trace_hfi1_sdma_in_sn(sde, tx->sn); WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]); #endif sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx; sde->desc_avail -= tx->num_desc; + hfi1_cdbg(AIOWRITE, "-"); return tail; } @@ -2354,6 +2390,7 @@ static int sdma_check_progress( { int ret; + hfi1_cdbg(AIOWRITE, "+"); sde->desc_avail = sdma_descq_freecnt(sde); if (tx->num_desc <= sde->desc_avail) return -EAGAIN; @@ -2369,8 +2406,10 @@ static int sdma_check_progress( } else { ret = -EBUSY; } + hfi1_cdbg(AIOWRITE, "-"); return ret; } +#ifdef __HFI1_ORIG__ /** * sdma_send_txreq() - submit a tx req to ring @@ -2394,6 +2433,7 @@ int sdma_send_txreq(struct sdma_engine *sde, u16 tail; unsigned long flags; + hfi1_cdbg(AIOWRITE, "+"); /* user should have supplied entire packet */ if (unlikely(tx->tlen)) return -EINVAL; @@ -2410,6 +2450,7 @@ retry: sdma_update_tail(sde, tail); unlock: spin_unlock_irqrestore(&sde->tail_lock, flags); + hfi1_cdbg(AIOWRITE, "-"); return ret; unlock_noconn: if (wait) @@ -2436,6 +2477,7 @@ nodesc: goto unlock; } +#endif /* __HFI1_ORIG__ */ /** * sdma_send_txlist() - submit a list of tx req to ring * @sde: sdma engine to use @@ -2473,6 +2515,7 @@ int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait, u16 tail = INVALID_TAIL; u32 submit_count = 0, flush_count = 0, total_count; + hfi1_cdbg(AIOWRITE, "+"); spin_lock_irqsave(&sde->tail_lock, flags); retry: list_for_each_entry_safe(tx, tx_next, tx_list, list) { @@ -2502,6 +2545,7 @@ update_tail: sdma_update_tail(sde, tail); spin_unlock_irqrestore(&sde->tail_lock, flags); *count_out = total_count; + hfi1_cdbg(AIOWRITE, "-"); return ret; unlock_noconn: spin_lock(&sde->flushlist_lock); @@ -2511,14 +2555,15 @@ unlock_noconn: tx->next_descq_idx = 0; #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER tx->sn = sde->tail_sn++; - trace_hfi1_sdma_in_sn(sde, tx->sn); + // trace_hfi1_sdma_in_sn(sde, tx->sn); #endif list_add_tail(&tx->list, &sde->flushlist); flush_count++; iowait_inc_wait_count(wait, tx->num_desc); } spin_unlock(&sde->flushlist_lock); - schedule_work(&sde->flush_worker); + // TODO: schedule_work + //schedule_work(&sde->flush_worker); ret = -ECOMM; goto update_tail; nodesc: @@ -2530,6 +2575,7 @@ nodesc: sde->descq_full_count++; goto update_tail; } +#ifdef __HFI1_ORIG__ static void sdma_process_event(struct sdma_engine *sde, enum sdma_events event) { @@ -3083,6 +3129,7 @@ enomem: return -ENOMEM; } +#endif /* __HFI1_ORIG__ */ /* * ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors * @@ -3103,6 +3150,8 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx, int type, void *kvaddr, struct page *page, unsigned long offset, u16 len) { +//TODO: ext_coal_sdma_tx_descs +#ifdef __HFI1_ORIG__ int pad_len, rval; dma_addr_t addr; @@ -3162,9 +3211,10 @@ int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx, return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx, addr, tx->tlen); } - +#endif /* __HFI1_ORIG__ */ return 1; } +#ifdef __HFI1_ORIG__ /* Update sdes when the lmc changes */ void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid) @@ -3209,6 +3259,7 @@ int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) return rval; } +#endif /* __HFI1_ORIG__ */ /* * Add ahg to the sdma_txreq * @@ -3316,6 +3367,7 @@ void sdma_ahg_free(struct sdma_engine *sde, int ahg_index) return; clear_bit(ahg_index, &sde->ahg_bits); } +#ifdef __HFI1_ORIG__ /* * SPC freeze handling for SDMA engines. Called when the driver knows @@ -3410,3 +3462,5 @@ void _sdma_engine_progress_schedule( CCE_INT_FORCE + (8 * (IS_SDMA_START / 64)), sde->progress_mask); } + +#endif /* __HFI1_ORIG__ */ \ No newline at end of file diff --git a/kernel/syscall.c b/kernel/syscall.c index ca8b1cf2..5ea4ed28 100644 --- a/kernel/syscall.c +++ b/kernel/syscall.c @@ -67,7 +67,7 @@ #include #include -//#include +#include #define SYSCALL_BY_IKC @@ -481,7 +481,7 @@ long do_syscall(struct syscall_request *req, int cpu, int pid) #endif // PROFILE_ENABLE if (req->number == __NR_open && rc > 0) { - if (res.private_data && !strncmp(req->args[0], "/dev/hfi", 8)) { + if (res.private_data && !strncmp((const char *)req->args[0], "/dev/hfi", 8)) { thread->proc->fd_priv_table[rc] = res.private_data; kprintf("%s: PID: %d, open fd: %d, filename: %s, private_data: 0x%lx\n", __FUNCTION__, thread->proc->pid, rc, req->args[0], res.private_data); @@ -3093,13 +3093,10 @@ SYSCALL_DECLARE(writev) { struct process *proc = cpu_local_var(current)->proc; int fd = ihk_mc_syscall_arg0(ctx); + struct iovec *iovec = (struct iovec *)ihk_mc_syscall_arg1(ctx); int iovcnt = ihk_mc_syscall_arg2(ctx); - if (fd < 256) { - //struct hfi1_filedata *hf = (struct hfi1_filedata *)proc->fd_priv_table[fd]; - kprintf("%s: fd[%d], 0x%lx, iovcnt[%d]\n", __FUNCTION__, fd, proc->fd_priv_table[fd], iovcnt); - } else { - kprintf("%s: fd[%d] > 256\n", __FUNCTION__, fd); - } + void *private_data = proc->fd_priv_table[fd]; + if (!private_data) hfi1_aio_write(private_data, iovec, iovcnt); return syscall_generic_forwarding(__NR_writev, ctx); } diff --git a/kernel/user_sdma.c b/kernel/user_sdma.c index d626dddf..734e39a2 100644 --- a/kernel/user_sdma.c +++ b/kernel/user_sdma.c @@ -44,6 +44,13 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ + +#include +#include +#include + +#ifdef __HFI1_ORIG__ + #include #include #include @@ -69,9 +76,11 @@ #include "trace.h" #include "mmu_rb.h" -static uint hfi1_sdma_comp_ring_size = 128; + module_param_named(sdma_comp_size, hfi1_sdma_comp_ring_size, uint, S_IRUGO); MODULE_PARM_DESC(sdma_comp_size, "Size of User SDMA completion ring. Default: 128"); +#endif /* __HFI1_ORIG__ */ +static uint hfi1_sdma_comp_ring_size = 128; /* The maximum number of Data io vectors per message/request */ #define MAX_VECTORS_PER_REQ 8 @@ -133,8 +142,11 @@ static unsigned initial_pkt_count = 8; #define SDMA_IOWAIT_TIMEOUT 1000 /* in milliseconds */ +#ifdef __HFI1_ORIG__ + struct sdma_mmu_node; +#endif /* __HFI1_ORIG__ */ struct user_sdma_iovec { struct list_head list; struct iovec iov; @@ -149,6 +161,7 @@ struct user_sdma_iovec { u64 offset; struct sdma_mmu_node *node; }; +#ifdef __HFI1_ORIG__ struct sdma_mmu_node { struct mmu_rb_node rb; @@ -164,6 +177,7 @@ struct evict_data { u32 target; /* target count to evict */ }; +#endif /* __HFI1_ORIG__ */ struct user_sdma_request { struct sdma_req_info info; struct hfi1_user_sdma_pkt_q *pq; @@ -237,6 +251,7 @@ struct user_sdma_txreq { unsigned busycount; u64 seqnum; }; +#ifdef __HFI1_ORIG__ #define SDMA_DBG(req, fmt, ...) \ hfi1_cdbg(SDMA, "[%u:%u:%u:%u] " fmt, (req)->pq->dd->unit, \ @@ -246,24 +261,28 @@ struct user_sdma_txreq { hfi1_cdbg(SDMA, "[%u:%u:%u] " fmt, (pq)->dd->unit, (pq)->ctxt, \ (pq)->subctxt, ##__VA_ARGS__) +#endif /* __HFI1_ORIG__ */ + static int user_sdma_send_pkts(struct user_sdma_request *, unsigned); -static int num_user_pages(const struct iovec *); -static void user_sdma_txreq_cb(struct sdma_txreq *, int); static inline void pq_update(struct hfi1_user_sdma_pkt_q *); -static void user_sdma_free_request(struct user_sdma_request *, bool); -static int pin_vector_pages(struct user_sdma_request *, - struct user_sdma_iovec *); -static void unpin_vector_pages(struct mm_struct *, struct page **, unsigned, - unsigned); static int check_header_template(struct user_sdma_request *, struct hfi1_pkt_header *, u32, u32); static int set_txreq_header(struct user_sdma_request *, struct user_sdma_txreq *, u32); static int set_txreq_header_ahg(struct user_sdma_request *, struct user_sdma_txreq *, u32); +static void user_sdma_free_request(struct user_sdma_request *, bool); static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *, struct hfi1_user_sdma_comp_q *, u16, enum hfi1_sdma_comp_state, int); +static void user_sdma_txreq_cb(struct sdma_txreq *, int); + +#ifdef __HFI1_ORIG__ +static int num_user_pages(const struct iovec *); +static int pin_vector_pages(struct user_sdma_request *, + struct user_sdma_iovec *); +static void unpin_vector_pages(struct mm_struct *, struct page **, unsigned, + unsigned); static inline u32 set_pkt_bth_psn(__be32, u8, u32); static inline u32 get_lrh_len(struct hfi1_pkt_header, u32 len); @@ -476,6 +495,8 @@ int hfi1_user_sdma_free_queues(struct hfi1_filedata *fd) return 0; } +#endif /* __HFI1_ORIG__ */ + static u8 dlid_to_selector(u16 dlid) { static u8 mapping[256]; @@ -497,11 +518,19 @@ static u8 dlid_to_selector(u16 dlid) return mapping[hash]; } +#ifdef __HFI1_ORIG__ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec, unsigned long dim, unsigned long *count) { int ret = 0, i; struct hfi1_filedata *fd = fp->private_data; +#else +int hfi1_user_sdma_process_request(void *private_data, struct iovec *iovec, + unsigned long dim, unsigned long *count) +{ + int ret = 0, i; + struct hfi1_filedata *fd = private_data; +#endif /* __HFI1_ORIG__ */ struct hfi1_ctxtdata *uctxt = fd->uctxt; struct hfi1_user_sdma_pkt_q *pq = fd->pq; struct hfi1_user_sdma_comp_q *cq = fd->cq; @@ -515,6 +544,7 @@ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec, u16 dlid; u32 selector; + hfi1_cdbg(AIOWRITE, "+"); if (iovec[idx].iov_len < sizeof(info) + sizeof(req->hdr)) { hfi1_cdbg( SDMA, @@ -530,8 +560,8 @@ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec, return -EFAULT; } - trace_hfi1_sdma_user_reqinfo(dd, uctxt->ctxt, fd->subctxt, - (u16 *)&info); + // trace_hfi1_sdma_user_reqinfo(dd, uctxt->ctxt, fd->subctxt, + // (u16 *)&info); if (info.comp_idx >= hfi1_sdma_comp_ring_size) { hfi1_cdbg(SDMA, @@ -619,6 +649,9 @@ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec, ret = -EINVAL; goto free_req; } + +// TODO: Enable this validation and checking +#ifdef __HFI1_ORIG__ /* * Validate the vl. Do not trust packets from user space blindly. * VL comes from PBC, SC comes from LRH, and the VL needs to @@ -640,6 +673,7 @@ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec, ret = -EINVAL; goto free_req; } +#endif /* __HFI1_ORIG__ */ /* * Also should check the BTH.lnh. If it says the next header is GRH then @@ -667,11 +701,16 @@ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec, for (i = 0; i < req->data_iovs; i++) { INIT_LIST_HEAD(&req->iovs[i].list); memcpy(&req->iovs[i].iov, iovec + idx++, sizeof(struct iovec)); +#ifdef __HFI1_ORIG__ + hfi1_cdbg(AIOWRITE, "+pin_vector_pages"); + // TODO: pin_vector_pages ret = pin_vector_pages(req, &req->iovs[i]); + hfi1_cdbg(AIOWRITE, "-pin_vector_pages"); if (ret) { req->status = ret; goto free_req; } +#endif /* __HFI1_ORIG__ */ req->data_len += req->iovs[i].iov.iov_len; } SDMA_DBG(req, "total data length %u", req->data_len); @@ -719,7 +758,8 @@ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec, dlid = be16_to_cpu(req->hdr.lrh[1]); selector = dlid_to_selector(dlid); selector += uctxt->ctxt + fd->subctxt; - req->sde = sdma_select_user_engine(dd, selector, vl); + /* TODO: check the rcu stuff */ + //req->sde = sdma_select_user_engine(dd, selector, vl); if (!req->sde || !sdma_running(req->sde)) { ret = -ECOMM; @@ -772,20 +812,28 @@ int hfi1_user_sdma_process_request(struct file *fp, struct iovec *iovec, goto free_req; return ret; } +#ifdef __HFI1_ORIG__ + hfi1_cdbg(AIOWRITE, "+wait_event_interruptible_timeout"); wait_event_interruptible_timeout( pq->busy.wait_dma, (pq->state == SDMA_PKT_Q_ACTIVE), msecs_to_jiffies( SDMA_IOWAIT_TIMEOUT)); + hfi1_cdbg(AIOWRITE, "-wait_event_interruptible_timeout"); +#else + while (pq->state != SDMA_PKT_Q_ACTIVE) cpu_pause(); +#endif /* __HFI1_ORIG__ */ } } *count += idx; + hfi1_cdbg(AIOWRITE, "-"); return 0; free_req: user_sdma_free_request(req, true); if (req_queued) pq_update(pq); set_comp_state(pq, cq, info.comp_idx, ERROR, req->status); + hfi1_cdbg(AIOWRITE, "-"); return ret; } @@ -862,6 +910,7 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts) struct hfi1_user_sdma_pkt_q *pq = NULL; struct user_sdma_iovec *iovec = NULL; + hfi1_cdbg(AIOWRITE, "+"); if (!req->pq) return -EINVAL; @@ -899,7 +948,11 @@ static int user_sdma_send_pkts(struct user_sdma_request *req, unsigned maxpkts) return -EFAULT; } +#ifdef __HFI1_ORIG__ tx = kmem_cache_alloc(pq->txreq_cache, GFP_KERNEL); +#else + tx = kmalloc(sizeof(struct user_sdma_txreq), GFP_KERNEL | __GFP_ZERO); +#endif /* __HFI1_ORIG__ */ if (!tx) return -ENOMEM; @@ -1087,14 +1140,21 @@ dosend: if (test_bit(SDMA_REQ_HAVE_AHG, &req->flags)) sdma_ahg_free(req->sde, req->ahg_idx); } + hfi1_cdbg(AIOWRITE, "-"); return ret; free_txreq: sdma_txclean(pq->dd, &tx->txreq); free_tx: +#ifdef __HFI1_ORIG__ kmem_cache_free(pq->txreq_cache, tx); + hfi1_cdbg(AIOWRITE, "-"); +#else + kfree(tx); +#endif /* __HFI1_ORIG__ */ return ret; } +#ifdef __HFI1_ORIG__ /* * How many pages in this iovec element? @@ -1212,6 +1272,7 @@ static void unpin_vector_pages(struct mm_struct *mm, struct page **pages, kfree(pages); } +#endif /* __HFI1_ORIG__ */ static int check_header_template(struct user_sdma_request *req, struct hfi1_pkt_header *hdr, u32 lrhlen, u32 datalen) @@ -1388,8 +1449,8 @@ static int set_txreq_header(struct user_sdma_request *req, req->omfactor != KDETH_OM_SMALL); } done: - trace_hfi1_sdma_user_header(pq->dd, pq->ctxt, pq->subctxt, - req->info.comp_idx, hdr, tidval); + // trace_hfi1_sdma_user_header(pq->dd, pq->ctxt, pq->subctxt, + // req->info.comp_idx, hdr, tidval); return sdma_txadd_kvaddr(pq->dd, &tx->txreq, hdr, sizeof(*hdr)); } @@ -1475,9 +1536,9 @@ static int set_txreq_header_ahg(struct user_sdma_request *req, AHG_HEADER_SET(req->ahg, diff, 7, 16, 14, val); } - trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt, - req->info.comp_idx, req->sde->this_idx, - req->ahg_idx, req->ahg, diff, tidval); + // trace_hfi1_sdma_user_header_ahg(pq->dd, pq->ctxt, pq->subctxt, + // req->info.comp_idx, req->sde->this_idx, + // req->ahg_idx, req->ahg, diff, tidval); return diff; } @@ -1510,7 +1571,8 @@ static void user_sdma_txreq_cb(struct sdma_txreq *txreq, int status) } req->seqcomp = tx->seqnum; - kmem_cache_free(pq->txreq_cache, tx); + //TODO: kmem_cache_free + //kmem_cache_free(pq->txreq_cache, tx); tx = NULL; idx = req->info.comp_idx; @@ -1538,12 +1600,14 @@ static inline void pq_update(struct hfi1_user_sdma_pkt_q *pq) { if (atomic_dec_and_test(&pq->n_reqs)) { xchg(&pq->state, SDMA_PKT_Q_INACTIVE); - wake_up(&pq->wait); + //TODO: wake_up + //wake_up(&pq->wait); } } static void user_sdma_free_request(struct user_sdma_request *req, bool unpin) { + hfi1_cdbg(AIOWRITE, "+"); if (!list_empty(&req->txps)) { struct sdma_txreq *t, *p; @@ -1552,7 +1616,9 @@ static void user_sdma_free_request(struct user_sdma_request *req, bool unpin) container_of(t, struct user_sdma_txreq, txreq); list_del_init(&t->list); sdma_txclean(req->pq->dd, t); +#ifdef __HFI1_ORIG__ kmem_cache_free(req->pq->txreq_cache, tx); +#endif /* __HFI1_ORIG__ */ } } if (req->data_iovs) { @@ -1564,17 +1630,20 @@ static void user_sdma_free_request(struct user_sdma_request *req, bool unpin) if (!node) continue; +//TODO: +#ifdef __HFI1_ORIG__ if (unpin) hfi1_mmu_rb_remove(req->pq->handler, &node->rb); else atomic_dec(&node->refcount); +#endif /* __HFI1_ORIG__ */ } } kfree(req->tids); clear_bit(req->info.comp_idx, req->pq->req_in_use); + hfi1_cdbg(AIOWRITE, "-"); } - static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq, struct hfi1_user_sdma_comp_q *cq, u16 idx, enum hfi1_sdma_comp_state state, @@ -1585,9 +1654,10 @@ static inline void set_comp_state(struct hfi1_user_sdma_pkt_q *pq, cq->comps[idx].status = state; if (state == ERROR) cq->comps[idx].errcode = -ret; - trace_hfi1_sdma_user_completion(pq->dd, pq->ctxt, pq->subctxt, - idx, state, ret); + // trace_hfi1_sdma_user_completion(pq->dd, pq->ctxt, pq->subctxt, + // idx, state, ret); } +#ifdef __HFI1_ORIG__ static bool sdma_rb_filter(struct mmu_rb_node *node, unsigned long addr, unsigned long len) @@ -1651,3 +1721,5 @@ static int sdma_rb_invalidate(void *arg, struct mmu_rb_node *mnode) return 1; return 0; } + +#endif /* __HFI1_ORIG__ */ \ No newline at end of file diff --git a/lib/include/ihk/cpu.h b/lib/include/ihk/cpu.h index 1d51c5d4..c0c8fbd5 100644 --- a/lib/include/ihk/cpu.h +++ b/lib/include/ihk/cpu.h @@ -24,6 +24,7 @@ void cpu_halt(void); void cpu_safe_halt(void); void cpu_restore_interrupt(unsigned long); void cpu_pause(void); +void cpu_relax(void); #define barrier() arch_barrier()