diff --git a/elfboot/Makefile b/arch/x86/elfboot/Makefile similarity index 100% rename from elfboot/Makefile rename to arch/x86/elfboot/Makefile diff --git a/elfboot/elfboot.c b/arch/x86/elfboot/elfboot.c similarity index 100% rename from elfboot/elfboot.c rename to arch/x86/elfboot/elfboot.c diff --git a/elfboot/head.S b/arch/x86/elfboot/head.S similarity index 100% rename from elfboot/head.S rename to arch/x86/elfboot/head.S diff --git a/elfboot/raw.lds b/arch/x86/elfboot/raw.lds similarity index 100% rename from elfboot/raw.lds rename to arch/x86/elfboot/raw.lds diff --git a/elfboot/test.h b/arch/x86/elfboot/test.h similarity index 100% rename from elfboot/test.h rename to arch/x86/elfboot/test.h diff --git a/elfboot/test_main.c b/arch/x86/elfboot/test_main.c similarity index 100% rename from elfboot/test_main.c rename to arch/x86/elfboot/test_main.c diff --git a/kboot/Makefile b/arch/x86/kboot/Makefile similarity index 100% rename from kboot/Makefile rename to arch/x86/kboot/Makefile diff --git a/kboot/data.S b/arch/x86/kboot/data.S similarity index 100% rename from kboot/data.S rename to arch/x86/kboot/data.S diff --git a/kboot/kernel.lds.S b/arch/x86/kboot/kernel.lds.S similarity index 100% rename from kboot/kernel.lds.S rename to arch/x86/kboot/kernel.lds.S diff --git a/kboot/main.c b/arch/x86/kboot/main.c similarity index 100% rename from kboot/main.c rename to arch/x86/kboot/main.c diff --git a/arch/x86/kernel/context.S b/arch/x86/kernel/context.S new file mode 100644 index 00000000..9cb37e9f --- /dev/null +++ b/arch/x86/kernel/context.S @@ -0,0 +1,38 @@ +#define X86_CPU_LOCAL_OFFSET_TSS 128 +#define X86_TSS_OFFSET_SP0 4 +#define X86_CPU_LOCAL_OFFSET_SP0 \ + (X86_CPU_LOCAL_OFFSET_TSS + X86_TSS_OFFSET_SP0) + +.text +.globl aal_mc_switch_context +aal_mc_switch_context: + pushfq + popq %rax + testq %rdi, %rdi + jz 1f /* skip saving if "old_ctx" is null. */ + movq %rbp, 8(%rdi) + movq %rbx, 16(%rdi) + movq %rax, 72(%rdi) /* rflags */ + movq %rsi, 24(%rdi) + movq %rdi, 32(%rdi) + movq %r12, 40(%rdi) + movq %r13, 48(%rdi) + movq %r14, 56(%rdi) + movq %r15, 64(%rdi) + movq %rsp, 0(%rdi) +1: + movq 0(%rsi), %rsp + movq 80(%rsi), %rbp + movq %rbp, %gs:(X86_CPU_LOCAL_OFFSET_SP0) + movq 64(%rsi), %r15 + movq 56(%rsi), %r14 + movq 48(%rsi), %r13 + movq 40(%rsi), %r12 + movq 32(%rsi), %rdi + movq 16(%rsi), %rbx + movq 72(%rsi), %rax + pushq %rax + popfq + movq 8(%rsi), %rbp + movq 24(%rsi), %rsi + retq diff --git a/arch/x86/kernel/cpu.c b/arch/x86/kernel/cpu.c new file mode 100644 index 00000000..45785c23 --- /dev/null +++ b/arch/x86/kernel/cpu.c @@ -0,0 +1,665 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LAPIC_ID 0x020 +#define LAPIC_TIMER 0x320 +#define LAPIC_TIMER_INITIAL 0x380 +#define LAPIC_TIMER_CURRENT 0x390 +#define LAPIC_TIMER_DIVIDE 0x3e0 +#define LAPIC_SPURIOUS 0x0f0 +#define LAPIC_EOI 0x0b0 +#define LAPIC_ICR0 0x300 +#define LAPIC_ICR2 0x310 +#define LAPIC_ESR 0x280 + +#define APIC_INT_LEVELTRIG 0x08000 +#define APIC_INT_ASSERT 0x04000 +#define APIC_ICR_BUSY 0x01000 +#define APIC_DEST_PHYSICAL 0x00000 +#define APIC_DM_FIXED 0x00000 +#define APIC_DM_NMI 0x00400 +#define APIC_DM_INIT 0x00500 +#define APIC_DM_STARTUP 0x00600 + + +#define DEBUG_PRINT_CPU + +#ifdef DEBUG_PRINT_CPU +#define dkprintf kprintf +#else +#define dkprintf(...) +#endif + + +struct x86_cpu_local_variables *get_x86_this_cpu_local(void); +void *get_x86_this_cpu_kstack(void); +void init_processors_local(int max_id); +void assign_processor_id(void); +void arch_delay(int); +void x86_set_warm_reset(void); +void x86_init_perfctr(void); + +extern int kprintf(const char *format, ...); + +static struct idt_entry{ + uint32_t desc[4]; +} idt[256] __attribute__((aligned(16))); + +static struct x86_desc_ptr idt_desc, gdt_desc; + +static uint64_t gdt[] __attribute__((aligned(16))) = { + 0, /* 0 */ + 0, /* 8 */ + 0, /* 16 */ + 0, /* 24 */ + 0x00af9b000000ffff, /* 32 : KERNEL_CS */ + 0x00cf93000000ffff, /* 40 : KERNEL_DS */ + 0x00affb000000ffff, /* 48 : USER_CS */ + 0x00aff3000000ffff, /* 56 : USER_DS */ + 0x0000890000000067, /* 64 : TSS */ + 0, /* (72: TSS) */ +}; + +struct tss64 tss __attribute__((aligned(16))); + +static void set_idt_entry(int idx, unsigned long addr) +{ + idt[idx].desc[0] = (addr & 0xffff) | (KERNEL_CS << 16); + idt[idx].desc[1] = (addr & 0xffff0000) | 0x8e00; + idt[idx].desc[2] = (addr >> 32); + idt[idx].desc[3] = 0; +} + +static void set_idt_entry_trap_gate(int idx, unsigned long addr) +{ + idt[idx].desc[0] = (addr & 0xffff) | (KERNEL_CS << 16); + idt[idx].desc[1] = (addr & 0xffff0000) | 0xef00; + idt[idx].desc[2] = (addr >> 32); + idt[idx].desc[3] = 0; +} + +extern uint64_t generic_common_handlers[]; + +void reload_idt(void) +{ + asm volatile("lidt %0" : : "m"(idt_desc) : "memory"); +} + +static struct list_head handlers[256 - 32]; +extern char page_fault[], general_protection_exception[]; + +static void init_idt(void) +{ + int i; + + idt_desc.size = sizeof(idt) - 1; + idt_desc.address = (unsigned long)idt; + + for (i = 0; i < 256; i++) { + if (i >= 32) { + INIT_LIST_HEAD(&handlers[i - 32]); + } + set_idt_entry(i, generic_common_handlers[i]); + } + + set_idt_entry(13, (unsigned long)general_protection_exception); + set_idt_entry(14, (unsigned long)page_fault); + + reload_idt(); +} + +void init_fpu(void) +{ + unsigned long reg; + + asm volatile("movq %%cr0, %0" : "=r"(reg)); + /* Unset EM and TS flag. */ + reg &= ~((1 << 2) | (1 << 3)); + /* Set MP flag */ + reg |= 1 << 1; + asm volatile("movq %0, %%cr0" : : "r"(reg)); + +#ifdef ENABLE_SSE + asm volatile("movq %%cr4, %0" : "=r"(reg)); + /* Set OSFXSR flag. */ + reg |= (1 << 9); + asm volatile("movq %0, %%cr4" : : "r"(reg)); +#endif + + asm volatile("finit"); +} + +void reload_gdt(struct x86_desc_ptr *gdt_ptr) +{ + asm volatile("pushq %1\n" + "leaq 1f(%%rip), %%rbx\n" + "pushq %%rbx\n" + "lgdt %0\n" + "lretq\n" + "1:\n" : : + "m" (*gdt_ptr), + "i" (KERNEL_CS) : "rbx"); + asm volatile("movl %0, %%ds" : : "r"(KERNEL_DS)); + asm volatile("movl %0, %%ss" : : "r"(KERNEL_DS)); + /* And, set TSS */ + asm volatile("ltr %0" : : "r"((short)GLOBAL_TSS) : "memory"); +} + +void init_gdt(void) +{ + register unsigned long stack_pointer asm("rsp"); + unsigned long tss_addr = (unsigned long)&tss; + + memset(&tss, 0, sizeof(tss)); + tss.rsp0 = stack_pointer; + + /* 0x89 = Present (8) | Type = 9 (TSS) */ + gdt[GLOBAL_TSS_ENTRY] = (sizeof(tss) - 1) + | ((tss_addr & 0xffffff) << 16) + | (0x89UL << 40) | ((tss_addr & 0xff000000) << 32); + gdt[GLOBAL_TSS_ENTRY + 1] = (tss_addr >> 32); + + gdt_desc.size = sizeof(gdt) - 1; + gdt_desc.address = (unsigned long)gdt; + + /* Load the new GDT, and set up CS, DS and SS. */ + reload_gdt(&gdt_desc); +} + +static void *lapic_vp; +void lapic_write(int reg, unsigned int value) +{ + *(volatile unsigned int *)((char *)lapic_vp + reg) = value; +} + +unsigned int lapic_read(int reg) +{ + return *(volatile unsigned int *)((char *)lapic_vp + reg); +} + +void lapic_icr_write(unsigned int h, unsigned int l) +{ + lapic_write(LAPIC_ICR2, (unsigned int)h); + lapic_write(LAPIC_ICR0, l); +} + +void init_lapic(void) +{ + unsigned long baseaddr; + + /* Enable Local APIC */ + baseaddr = rdmsr(MSR_IA32_APIC_BASE); + if (!lapic_vp) { + lapic_vp = map_fixed_area(baseaddr & PAGE_MASK, PAGE_SIZE, 1); + } + baseaddr |= 0x800; + wrmsr(MSR_IA32_APIC_BASE, baseaddr); + + lapic_write(LAPIC_SPURIOUS, 0x1ff); +} + +void lapic_ack(void) +{ + lapic_write(LAPIC_EOI, 0); +} + +static void set_kstack(unsigned long ptr) +{ + struct x86_cpu_local_variables *v; + + v = get_x86_this_cpu_local(); + v->kernel_stack = ptr; + v->tss.rsp0 = ptr; +} + +static void init_smp_processor(void) +{ + struct x86_cpu_local_variables *v; + unsigned long tss_addr; + + v = get_x86_this_cpu_local(); + tss_addr = (unsigned long)&v->tss; + + v->apic_id = lapic_read(LAPIC_ID) >> LAPIC_ID_SHIFT; + + memcpy(v->gdt, gdt, sizeof(v->gdt)); + + memset(&v->tss, 0, sizeof(v->tss)); + + v->gdt[GLOBAL_TSS_ENTRY] = (sizeof(v->tss) - 1) + | ((tss_addr & 0xffffff) << 16) + | (0x89UL << 40) | ((tss_addr & 0xff000000) << 32); + v->gdt[GLOBAL_TSS_ENTRY + 1] = (tss_addr >> 32); + + v->gdt_ptr.size = sizeof(v->gdt) - 1; + v->gdt_ptr.address = (unsigned long)v->gdt; + + /* Load the new GDT, and set up CS, DS and SS. */ + reload_gdt(&v->gdt_ptr); + + set_kstack((unsigned long)get_x86_this_cpu_kstack()); +} + +static char *trampoline_va, *first_page_va; + +void aal_mc_init_ap(void) +{ + struct aal_mc_cpu_info *cpu_info = aal_mc_get_cpu_info(); + + trampoline_va = map_fixed_area(AP_TRAMPOLINE, AP_TRAMPOLINE_SIZE, + 0); + first_page_va = map_fixed_area(0, PAGE_SIZE, 0); + + kprintf("# of cpus : %d\n", cpu_info->ncpus); + init_processors_local(cpu_info->ncpus); + + /* Do initialization for THIS cpu (BSP) */ + assign_processor_id(); + + init_smp_processor(); +} + +extern void init_page_table(void); + +extern char x86_syscall[]; +long (*__x86_syscall_handler)(int, aal_mc_user_context_t *); + +void init_syscall(void) +{ + unsigned long r; + + r = rdmsr(MSR_EFER); + r |= 1; /* SYSCALL Enable */ + wrmsr(MSR_EFER, r); + + r = (((unsigned long)KERNEL_CS) << 32) + | (((unsigned long)USER_CS) << 48); + wrmsr(MSR_STAR, r); + + wrmsr(MSR_LSTAR, (unsigned long)x86_syscall); +} + +void init_cpu(void) +{ + init_fpu(); + init_lapic(); + init_syscall(); + x86_init_perfctr(); +} + +void setup_x86(void) +{ + cpu_disable_interrupt(); + + init_idt(); + + init_gdt(); + + init_page_table(); + + init_cpu(); + + kprintf("setup_x86 done.\n"); +} + +static volatile int cpu_boot_status; + +void call_ap_func(void (*next_func)(void)) +{ + cpu_boot_status = 1; + next_func(); +} + +void setup_x86_ap(void (*next_func)(void)) +{ + unsigned long rsp; + cpu_disable_interrupt(); + + assign_processor_id(); + + init_smp_processor(); + + reload_idt(); + + init_cpu(); + + rsp = (unsigned long)get_x86_this_cpu_kstack(); + + asm volatile("movq %0, %%rdi\n" + "movq %1, %%rsp\n" + "call *%2" : : "r"(next_func), "r"(rsp), "r"(call_ap_func) + : "rdi"); + while(1); +} + +void arch_show_interrupt_context(const void *reg); + +void handle_interrupt(int vector, struct x86_regs *regs) +{ + struct aal_mc_interrupt_handler *h; + + dkprintf("CPU[%d] got interrupt, vector: %d, RIP: 0x%lX\n", + aal_mc_get_processor_id(), vector, regs->rip); + + if (vector < 0 || vector > 255) { + panic("Invalid interrupt vector."); + } else if (vector < 32) { + if (vector == 8 || + (vector >= 10 && vector <= 15) || vector == 17) { + kprintf("Exception %d, rflags: 0x%lX CS: 0x%lX, RIP: 0x%lX\n", + vector, regs->rflags, regs->cs, regs->rip); + } else { + kprintf("Exception %d, rflags: 0x%lX CS: 0x%lX, RIP: 0x%lX\n", + vector, regs->rflags, regs->cs, regs->rip); + } + arch_show_interrupt_context(regs); + panic("Unhandled exception"); + } else { + list_for_each_entry(h, &handlers[vector - 32], list) { + if (h->func) { + h->func(h->priv); + } + } + } + + lapic_ack(); +} + +void gpe_handler(struct x86_regs *regs) +{ + kprintf("General protection fault (err: %lx, %lx:%lx)\n", + regs->error, regs->cs, regs->rip); + arch_show_interrupt_context(regs); + panic("GPF"); +} + +void x86_issue_ipi(unsigned int apicid, unsigned int low) +{ + lapic_icr_write(apicid << LAPIC_ICR_ID_SHIFT, low); +} + +static void outb(uint8_t v, uint16_t port) +{ + asm volatile("outb %0, %1" : : "a" (v), "d" (port)); +} + +static void set_warm_reset_vector(unsigned long ip) +{ + /* Write CMOS */ + x86_set_warm_reset(); + + /* Set vector */ + *(unsigned short *)(first_page_va + 0x469) = (ip >> 4); + *(unsigned short *)(first_page_va + 0x467) = ip & 0xf; +} + +static void wait_icr_idle(void) +{ + while (lapic_read(LAPIC_ICR0) & APIC_ICR_BUSY) { + cpu_pause(); + } +} + +static void __x86_wakeup(int apicid, unsigned long ip) +{ + int retry = 3; + + set_warm_reset_vector(ip); + + /* Clear the error */ + lapic_write(LAPIC_ESR, 0); + lapic_read(LAPIC_ESR); + + /* INIT */ + x86_issue_ipi(apicid, + APIC_INT_LEVELTRIG | APIC_INT_ASSERT | APIC_DM_INIT); + wait_icr_idle(); + + x86_issue_ipi(apicid, + APIC_INT_LEVELTRIG | APIC_DM_INIT); + wait_icr_idle(); + + while (retry--) { + lapic_read(LAPIC_ESR); + x86_issue_ipi(apicid, APIC_DM_STARTUP | (ip >> 12)); + wait_icr_idle(); + + arch_delay(200); + + if (cpu_boot_status) + break; + } +} + +/** AAL Functions **/ + +void cpu_halt(void) +{ + asm volatile("hlt"); +} + +void cpu_enable_interrupt(void) +{ + asm volatile("sti"); +} + +void cpu_disable_interrupt(void) +{ + asm volatile("cli"); +} + +void cpu_restore_interrupt(unsigned long flags) +{ + asm volatile("push %0; popf" : : "g"(flags) : "memory", "cc"); +} + +void cpu_pause(void) +{ + asm volatile("pause"); +} + +unsigned long cpu_disable_interrupt_save(void) +{ + unsigned long flags; + + asm volatile("pushf; pop %0; cli" : "=r"(flags) : : "memory", "cc"); + + return flags; +} + +int aal_mc_register_interrupt_handler(int vector, + struct aal_mc_interrupt_handler *h) +{ + if (vector < 32 || vector > 255) { + return -EINVAL; + } + + list_add_tail(&h->list, &handlers[vector - 32]); + + return 0; +} +int aal_mc_unregister_interrupt_handler(int vector, + struct aal_mc_interrupt_handler *h) +{ + list_del(&h->list); + + return 0; +} + +extern unsigned long __page_fault_handler_address; + +void aal_mc_set_page_fault_handler(void (*h)(unsigned long, void *)) +{ + __page_fault_handler_address = (unsigned long)h; +} + +extern char trampoline_code_data[], trampoline_code_data_end[]; +struct page_table *get_init_page_table(void); +unsigned long get_transit_page_table(void); + +void aal_mc_boot_cpu(int cpuid, unsigned long pc) +{ + unsigned long *p; + + p = (unsigned long *)trampoline_va; + + memcpy(p, trampoline_code_data, + trampoline_code_data_end - trampoline_code_data); + + p[1] = (unsigned long)virt_to_phys(get_init_page_table()); + p[2] = (unsigned long)setup_x86_ap; + p[3] = pc; + p[6] = (unsigned long)get_transit_page_table(); + if (!p[6]) { + p[6] = p[1]; + } + + cpu_boot_status = 0; + + __x86_wakeup(cpuid, AP_TRAMPOLINE); + + /* XXX: Time out */ + while (!cpu_boot_status) { + cpu_pause(); + } +} + +void aal_mc_init_context(aal_mc_kernel_context_t *new_ctx, + void *stack_pointer, void (*next_function)(void)) +{ + unsigned long *sp; + + if (!stack_pointer) { + stack_pointer = get_x86_this_cpu_kstack(); + } + + sp = stack_pointer; + memset(new_ctx, 0, sizeof(aal_mc_kernel_context_t)); + + /* Set the return address */ + new_ctx->rsp = (unsigned long)(sp - 1); + sp[-1] = (unsigned long)next_function; +} + +extern char enter_user_mode[]; + +void aal_mc_init_user_process(aal_mc_kernel_context_t *ctx, + aal_mc_user_context_t **puctx, + void *stack_pointer, unsigned long new_pc, + unsigned long user_sp) +{ + char *sp; + aal_mc_user_context_t *uctx; + + sp = stack_pointer; + sp -= sizeof(aal_mc_user_context_t); + uctx = (aal_mc_user_context_t *)sp; + + *puctx = uctx; + + memset(uctx, 0, sizeof(aal_mc_user_context_t)); + uctx->cs = USER_CS; + uctx->rip = new_pc; + uctx->ss = USER_DS; + uctx->rsp = user_sp; + uctx->rflags = RFLAGS_IF; + + aal_mc_init_context(ctx, sp, (void (*)(void))enter_user_mode); + ctx->rsp0 = (unsigned long)stack_pointer; +} + +void aal_mc_modify_user_context(aal_mc_user_context_t *uctx, + enum aal_mc_user_context_regtype reg, + unsigned long value) +{ + if (reg == AAL_UCR_STACK_POINTER) { + uctx->rsp = value; + } else if (reg == AAL_UCR_PROGRAM_COUNTER) { + uctx->rip = value; + } +} + +void aal_mc_print_user_context(aal_mc_user_context_t *uctx) +{ + kprintf("CS:RIP = %04lx:%16lx\n", uctx->cs, uctx->rip); + kprintf("%16lx %16lx %16lx %16lx\n%16lx %16lx %16lx\n", + uctx->rax, uctx->rbx, uctx->rcx, uctx->rdx, + uctx->rsi, uctx->rdi, uctx->rsp); +} + +void aal_mc_set_syscall_handler(long (*handler)(int, aal_mc_user_context_t *)) +{ + __x86_syscall_handler = handler; +} + +void aal_mc_delay_us(int us) +{ + arch_delay(us); +} + +void arch_show_interrupt_context(const void *reg) +{ + const struct x86_regs *regs = reg; + int irqflags; + + irqflags = kprintf_lock(); + + __kprintf("CS:EIP = %4lX:%16lX\n", regs->cs, regs->rip); + __kprintf(" RAX RBX RCX RDX\n"); + __kprintf("%16lX %16lX %16lX %16lX\n", + regs->rax, regs->rbx, regs->rcx, regs->rdx); + __kprintf(" RSI RDI RSP\n"); + __kprintf("%16lX %16lX %16lX\n", + regs->rsi, regs->rdi, regs->rsp); + __kprintf(" R8 R9 R10 R11\n"); + __kprintf("%16lX %16lX %16lX %16lX\n", + regs->r8, regs->r9, regs->r10, regs->r11); + __kprintf(" CS SS \n"); + __kprintf("%16lX %16lX\n", + regs->cs, regs->ss); + + kprintf_unlock(irqflags); +} + +int aal_mc_arch_set_special_register(enum aal_asr_type type, + unsigned long value) +{ + /* GS modification is not permitted */ + switch (type) { + case AAL_ASR_X86_FS: + wrmsr(MSR_FS_BASE, value); + return 0; + default: + return -EINVAL; + } +} + +int aal_mc_arch_get_special_register(enum aal_asr_type type, + unsigned long *value) +{ + /* GS modification is not permitted */ + switch (type) { + case AAL_ASR_X86_FS: + *value = rdmsr(MSR_FS_BASE); + return 0; + default: + return -EINVAL; + } +} + +int aal_mc_interrupt_cpu(int cpu, int vector) +{ + kprintf("[%d] aal_mc_interrupt_cpu: %d\n", aal_mc_get_processor_id(), cpu); + + wait_icr_idle(); + x86_issue_ipi(cpu, vector); + return 0; +} diff --git a/arch/x86/kernel/include/arch-lock.h b/arch/x86/kernel/include/arch-lock.h new file mode 100644 index 00000000..7b811f16 --- /dev/null +++ b/arch/x86/kernel/include/arch-lock.h @@ -0,0 +1,84 @@ +/* + * Excerpted from Linux 3.0: arch/x86/include/asm/spinlock.h + */ +#ifndef __HEADER_X86_COMMON_ARCH_LOCK +#define __HEADER_X86_COMMON_ARCH_LOCK + +#include + +//#define DEBUG_SPINLOCK + +#ifdef DEBUG_SPINLOCK +int __kprintf(const char *format, ...); +#endif + +typedef int aal_spinlock_t; + +#define AAL_STATIC_SPINLOCK_FUNCS + +static void aal_mc_spinlock_init(aal_spinlock_t *lock) +{ + *lock = 0; +} +#define SPIN_LOCK_UNLOCKED 0 + +static unsigned long aal_mc_spinlock_lock(aal_spinlock_t *lock) +{ + int inc = 0x00010000; + int tmp; + unsigned long flags; + + flags = cpu_disable_interrupt_save(); + +#if 0 + asm volatile("lock ; xaddl %0, %1\n" + "movzwl %w0, %2\n\t" + "shrl $16, %0\n\t" + "1:\t" + "cmpl %0, %2\n\t" + "je 2f\n\t" + "rep ; nop\n\t" + "movzwl %1, %2\n\t" + "jmp 1b\n" + "2:" + : "+Q" (inc), "+m" (*lock), "=r" (tmp) : : "memory", "cc"); +#endif + +#ifdef DEBUG_SPINLOCK + __kprintf("[%d] trying to grab lock: 0x%lX\n", + aal_mc_get_processor_id(), lock); +#endif + asm volatile("lock; xaddl %0, %1\n" + "movzwl %w0, %2\n\t" + "shrl $16, %0\n\t" + "1:\t" + "cmpl %0, %2\n\t" + "je 2f\n\t" + "rep ; nop\n\t" + "movzwl %1, %2\n\t" + /* don't need lfence here, because loads are in-order */ + "jmp 1b\n" + "2:" + : "+r" (inc), "+m" (*lock), "=&r" (tmp) + : + : "memory", "cc"); + +#ifdef DEBUG_SPINLOCK + __kprintf("[%d] holding lock: 0x%lX\n", aal_mc_get_processor_id(), lock); +#endif + + return flags; +} + +static void aal_mc_spinlock_unlock(aal_spinlock_t *lock, unsigned long flags) +{ + asm volatile ("lock incw %0" : "+m"(*lock) : : "memory", "cc"); + + cpu_restore_interrupt(flags); +#ifdef DEBUG_SPINLOCK + __kprintf("[%d] released lock: 0x%lX\n", aal_mc_get_processor_id(), lock); +#endif +} + +#endif + diff --git a/arch/x86/kernel/include/arch-memory.h b/arch/x86/kernel/include/arch-memory.h new file mode 100644 index 00000000..afd4612c --- /dev/null +++ b/arch/x86/kernel/include/arch-memory.h @@ -0,0 +1,108 @@ +#ifndef __HEADER_X86_COMMON_ARCH_MEMORY_H +#define __HEADER_X86_COMMON_ARCH_MEMORY_H + +#define KERNEL_CS_ENTRY 4 +#define KERNEL_DS_ENTRY 5 +#define USER_CS_ENTRY 6 +#define USER_DS_ENTRY 7 +#define GLOBAL_TSS_ENTRY 8 + +#define KERNEL_CS (KERNEL_CS_ENTRY * 8) +#define KERNEL_DS (KERNEL_DS_ENTRY * 8) +#define USER_CS (USER_CS_ENTRY * 8 + 3) +#define USER_DS (USER_DS_ENTRY * 8 + 3) +#define GLOBAL_TSS (GLOBAL_TSS_ENTRY * 8) + +#define PAGE_SHIFT 12 +#define PAGE_SIZE (1UL << PAGE_SHIFT) +#define PAGE_MASK (~((unsigned long)PAGE_SIZE - 1)) + +#define LARGE_PAGE_SHIFT 21 +#define LARGE_PAGE_SIZE (1UL << LARGE_PAGE_SHIFT) +#define LARGE_PAGE_MASK (~((unsigned long)LARGE_PAGE_SIZE - 1)) + +#define USER_END 0x0000800000000000UL +#define MAP_ST_START 0xffff800000000000UL +#define MAP_VMAP_START 0xfffff00000000000UL +#define MAP_FIXED_START 0xffffffff70000000UL +#define MAP_KERNEL_START 0xffffffff80000000UL + +#define MAP_VMAP_SIZE 0x0000000100000000UL + +#define KERNEL_PHYS_OFFSET MAP_ST_START + +#define PTL4_SHIFT 39 +#define PTL4_SIZE (1UL << PTL4_SHIFT) +#define PTL3_SHIFT 30 +#define PTL3_SIZE (1UL << PTL3_SHIFT) +#define PTL2_SHIFT 21 +#define PTL2_SIZE (1UL << PTL2_SHIFT) +#define PTL1_SHIFT 12 +#define PTL1_SIZE (1UL << PTL1_SHIFT) + +#define PT_ENTRIES 512 + +#define PFL4_PRESENT 0x01 +#define PFL4_WRITABLE 0x02 +#define PFL4_USER 0x04 + +#define PFL3_PRESENT 0x01 +#define PFL3_WRITABLE 0x02 +#define PFL3_USER 0x04 +#define PFL3_ACCESSED 0x20 +#define PFL3_DIRTY 0x40 +#define PFL3_SIZE 0x80 /* Used in 1G page */ +#define PFL3_GLOBAL 0x100 + +#define PFL2_PRESENT 0x01 +#define PFL2_WRITABLE 0x02 +#define PFL2_USER 0x04 +#define PFL2_ACCESSED 0x20 +#define PFL2_DIRTY 0x40 +#define PFL2_SIZE 0x80 /* Used in 2M page */ +#define PFL2_GLOBAL 0x100 +#define PFL2_PWT 0x08 +#define PFL2_PCD 0x10 + +#define PFL1_PRESENT 0x01 +#define PFL1_WRITABLE 0x02 +#define PFL1_USER 0x04 +#define PFL1_ACCESSED 0x20 +#define PFL1_DIRTY 0x40 +#define PFL1_PWT 0x08 +#define PFL1_PCD 0x10 + +/* We allow user programs to access all the memory */ +#define PFL4_KERN_ATTR (PFL4_PRESENT | PFL4_WRITABLE) +#define PFL3_KERN_ATTR (PFL3_PRESENT | PFL3_WRITABLE) +#define PFL2_KERN_ATTR (PFL2_PRESENT | PFL2_WRITABLE) +#define PFL1_KERN_ATTR (PFL1_PRESENT | PFL1_WRITABLE) + +/* For easy conversion, it is better to be the same as architecture's ones */ +enum aal_mc_pt_attribute { + PTATTR_ACTIVE = 0x01, + PTATTR_WRITABLE = 0x02, + PTATTR_USER = 0x04, + PTATTR_LARGEPAGE = 0x80, + PTATTR_UNCACHABLE = 0x10000, +}; + +typedef unsigned long pte_t; + +struct page_table; +void set_pte(pte_t *ppte, unsigned long phys, int attr); +pte_t *get_pte(struct page_table *pt, void *virt, int attr); + +void *early_alloc_page(void); +void *get_last_early_heap(void); +void flush_tlb(void); +void flush_tlb_single(unsigned long addr); + +void *map_fixed_area(unsigned long phys, unsigned long size, int uncachable); + +#define AP_TRAMPOLINE 0x10000 +#define AP_TRAMPOLINE_SIZE 0x4000 + +/* Local is cachable */ +#define AAL_IKC_QUEUE_PT_ATTR (PTATTR_WRITABLE | PTATTR_UNCACHABLE) +#endif diff --git a/arch/x86/kernel/include/bitops.h b/arch/x86/kernel/include/bitops.h new file mode 100644 index 00000000..a5ee1009 --- /dev/null +++ b/arch/x86/kernel/include/bitops.h @@ -0,0 +1,15 @@ +#ifndef HEADER_X86_COMMON_BITOPS_H +#define HEADER_X86_COMMON_BITOPS_H + +static inline int fls(int x) +{ + int r; + asm("bsrl %1,%0\n\t" + "jnz 1f\n\t" + "movl $-1,%0\n" + "1:" : "=r" (r) : "rm" (x)); + + return r + 1; +} + +#endif diff --git a/arch/x86/kernel/include/cpulocal.h b/arch/x86/kernel/include/cpulocal.h new file mode 100644 index 00000000..dc4e6696 --- /dev/null +++ b/arch/x86/kernel/include/cpulocal.h @@ -0,0 +1,39 @@ +#ifndef HEADER_X86_COMMON_CPULOCAL_H +#define HEADER_X86_COMMON_CPULOCAL_H + +#include +#include + +/* + * CPU Local Page + * 0 - : struct x86_cpu_local_varibles + * - 4096 : kernel stack + */ + +#define X86_CPU_LOCAL_OFFSET_TSS 128 +#define X86_CPU_LOCAL_OFFSET_KSTACK 16 +#define X86_CPU_LOCAL_OFFSET_USTACK 24 + +struct x86_cpu_local_variables { +/* 0 */ + unsigned long processor_id; + + unsigned long apic_id; +/* 16 */ + unsigned long kernel_stack; + unsigned long user_stack; + +/* 32 */ + struct x86_desc_ptr gdt_ptr; + unsigned short pad[3]; +/* 48 */ + uint64_t gdt[10]; +/* 128 */ + struct tss64 tss; + +} __attribute__((packed)); + +struct x86_cpu_local_variables *get_x86_cpu_local_variable(int id); + + +#endif diff --git a/arch/x86/kernel/include/ihk/atomic.h b/arch/x86/kernel/include/ihk/atomic.h new file mode 100644 index 00000000..9d8b7490 --- /dev/null +++ b/arch/x86/kernel/include/ihk/atomic.h @@ -0,0 +1,86 @@ +#ifndef HEADER_X86_COMMON_AAL_ATOMIC_H +#define HEADER_X86_COMMON_AAL_ATOMIC_H + +typedef struct { + int counter; +} aal_atomic_t; + +#define AAL_ATOMIC_INIT(i) { (i) } + + +static inline int aal_atomic_read(const aal_atomic_t *v) +{ + return (*(volatile int *)&(v)->counter); +} + +static inline void aal_atomic_set(aal_atomic_t *v, int i) +{ + v->counter = i; +} + +static inline void aal_atomic_add(int i, aal_atomic_t *v) +{ + asm volatile("lock addl %1,%0" + : "+m" (v->counter) + : "ir" (i)); +} + +static inline void aal_atomic_sub(int i, aal_atomic_t *v) +{ + asm volatile("lock subl %1,%0" + : "+m" (v->counter) + : "ir" (i)); +} + +static inline void aal_atomic_inc(aal_atomic_t *v) +{ + asm volatile("lock incl %0" + : "+m" (v->counter)); +} + +static inline void aal_atomic_dec(aal_atomic_t *v) +{ + asm volatile("lock decl %0" + : "+m" (v->counter)); +} + +static inline int aal_atomic_dec_and_test(aal_atomic_t *v) +{ + unsigned char c; + + asm volatile("lock decl %0; sete %1" + : "+m" (v->counter), "=qm" (c) + : : "memory"); + return c != 0; +} + +static inline int aal_atomic_inc_and_test(aal_atomic_t *v) +{ + unsigned char c; + + asm volatile("lock incl %0; sete %1" + : "+m" (v->counter), "=qm" (c) + : : "memory"); + return c != 0; +} + +static inline int aal_atomic_add_return(int i, aal_atomic_t *v) +{ + int __i; + + __i = i; + asm volatile("lock xaddl %0, %1" + : "+r" (i), "+m" (v->counter) + : : "memory"); + return i + __i; +} + +static inline int aal_atomic_sub_return(int i, aal_atomic_t *v) +{ + return aal_atomic_add_return(-i, v); +} + +#define aal_atomic_inc_return(v) (aal_atomic_add_return(1, v)) +#define aal_atomic_dec_return(v) (aal_atomic_sub_return(1, v)) + +#endif diff --git a/arch/x86/kernel/include/ihk/context.h b/arch/x86/kernel/include/ihk/context.h new file mode 100644 index 00000000..5ca9e3eb --- /dev/null +++ b/arch/x86/kernel/include/ihk/context.h @@ -0,0 +1,27 @@ +#ifndef __HEADER_X86_COMMON_CONTEXT_H +#define __HEADER_X86_COMMON_CONTEXT_H + +#include + +struct x86_kregs { + unsigned long rsp, rbp, rbx, rsi, rdi, r12, r13, r14, r15, rflags; + unsigned long rsp0; +}; + +typedef struct x86_kregs aal_mc_kernel_context_t; +/* XXX: User context should contain floating point registers */ +typedef struct x86_regs aal_mc_user_context_t; + +#define aal_mc_syscall_arg0(uc) (uc)->rdi +#define aal_mc_syscall_arg1(uc) (uc)->rsi +#define aal_mc_syscall_arg2(uc) (uc)->rdx +#define aal_mc_syscall_arg3(uc) (uc)->r10 +#define aal_mc_syscall_arg4(uc) (uc)->r8 +#define aal_mc_syscall_arg5(uc) (uc)->r9 + +#define aal_mc_syscall_ret(uc) (uc)->rax + +#define aal_mc_syscall_pc(uc) (uc)->rip +#define aal_mc_syscall_sp(uc) (uc)->rsp + +#endif diff --git a/arch/x86/kernel/include/ihk/ikc.h b/arch/x86/kernel/include/ihk/ikc.h new file mode 100644 index 00000000..24e2bd40 --- /dev/null +++ b/arch/x86/kernel/include/ihk/ikc.h @@ -0,0 +1,11 @@ +#ifndef HEADER_X86_COMMON_AAL_IKC_H +#define HEADER_X86_COMMON_AAL_IKC_H + +#include + +/* manycore side */ +int aal_mc_ikc_init_first(struct aal_ikc_channel_desc *, + aal_ikc_ph_t handler); + +#endif + diff --git a/arch/x86/kernel/include/ihk/types.h b/arch/x86/kernel/include/ihk/types.h new file mode 100644 index 00000000..412b50e9 --- /dev/null +++ b/arch/x86/kernel/include/ihk/types.h @@ -0,0 +1,20 @@ +#ifndef X86_COMMON_TYPES_H +#define X86_COMMON_TYPES_H + +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; +typedef char int8_t; +typedef short int16_t; +typedef int int32_t; +typedef long long int64_t; + +typedef long long ptrdiff_t; +typedef unsigned long long size_t; +typedef long long ssize_t; + +#define NULL ((void *)0) + +#endif + diff --git a/arch/x86/kernel/include/registers.h b/arch/x86/kernel/include/registers.h new file mode 100644 index 00000000..38d40c6b --- /dev/null +++ b/arch/x86/kernel/include/registers.h @@ -0,0 +1,148 @@ +#ifndef __HEADER_X86_COMMON_REGISTERS_H +#define __HEADER_X86_COMMON_REGISTERS_H + +#include + +#define RFLAGS_IF (1 << 9) + +#define MSR_EFER 0xc0000080 +#define MSR_STAR 0xc0000081 +#define MSR_LSTAR 0xc0000082 +#define MSR_FMASK 0xc0000084 +#define MSR_FS_BASE 0xc0000100 +#define MSR_GS_BASE 0xc0000101 + +#define MSR_IA32_APIC_BASE 0x000000001b + +#define CVAL(event, mask) \ + ((((event) & 0xf00) << 24) | ((mask) << 8) | ((event) & 0xff)) +#define CVAL2(event, mask, inv, count) \ + ((((event) & 0xf00) << 24) | ((mask) << 8) | ((event) & 0xff) | \ + ((inv & 1) << 23) | ((count & 0xff) << 24)) + +/* AMD */ +#define MSR_PERF_CTL_0 0xc0010000 +#define MSR_PERF_CTR_0 0xc0010004 + +static void wrmsr(unsigned int idx, unsigned long value){ + unsigned int high, low; + + high = value >> 32; + low = value & 0xffffffffU; + + asm volatile("wrmsr" : : "c" (idx), "a" (low), "d" (high) : "memory"); +} + +static unsigned long rdpmc(unsigned int counter) +{ + unsigned int high, low; + + asm volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (counter)); + + return (unsigned long)high << 32 | low; +} + +static unsigned long rdmsr(unsigned int index) +{ + unsigned int high, low; + + asm volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (index)); + + return (unsigned long)high << 32 | low; +} + +static unsigned long rdtsc(void) +{ + unsigned int high, low; + + asm volatile("rdtsc" : "=a" (low), "=d" (high)); + + return (unsigned long)high << 32 | low; +} + +static void set_perfctl(int counter, int event, int mask) +{ + unsigned long value; + + value = ((unsigned long)(event & 0x700) << 32) + | (event & 0xff) | ((mask & 0xff) << 8) | (1 << 18) + | (1 << 17); + + wrmsr(MSR_PERF_CTL_0 + counter, value); +} + +static void start_perfctr(int counter) +{ + unsigned long value; + + value = rdmsr(MSR_PERF_CTL_0 + counter); + value |= (1 << 22); + wrmsr(MSR_PERF_CTL_0 + counter, value); +} +static void stop_perfctr(int counter) +{ + unsigned long value; + + value = rdmsr(MSR_PERF_CTL_0 + counter); + value &= ~(1 << 22); + wrmsr(MSR_PERF_CTL_0 + counter, value); +} + +static void clear_perfctl(int counter) +{ + wrmsr(MSR_PERF_CTL_0 + counter, 0); +} + +static void set_perfctr(int counter, unsigned long value) +{ + wrmsr(MSR_PERF_CTR_0 + counter, value); +} + +static unsigned long read_perfctr(int counter) +{ + return rdpmc(counter); +} + +#define aal_mc_mb() asm volatile("mfence" : : : "memory"); + +struct x86_desc_ptr { + uint16_t size; + uint64_t address; +} __attribute__((packed)); + +struct tss64 { + unsigned int reserved0; + unsigned long rsp0; + unsigned long rsp1; + unsigned long rsp2; + unsigned int reserved1, reserved2; + unsigned long ist[7]; + unsigned int reserved3, reserved4; + unsigned short reserved5; + unsigned short iomap_address; +} __attribute__((packed)); + +struct x86_regs { + unsigned long r11, r10, r9, r8; + unsigned long rdi, rsi, rdx, rcx, rbx, rax; + unsigned long error, rip, cs, rflags, rsp, ss; +}; + +/* + * Page fault error code bits: + * + * bit 0 == 0: no page found 1: protection fault + * bit 1 == 0: read access 1: write access + * bit 2 == 0: kernel-mode access 1: user-mode access + * bit 3 == 1: use of reserved bit detected + * bit 4 == 1: fault was an instruction fetch + */ +enum x86_pf_error_code { + PF_PROT = 1 << 0, + PF_WRITE = 1 << 1, + PF_USER = 1 << 2, + PF_RSVD = 1 << 3, + PF_INSTR = 1 << 4, +}; + +#endif diff --git a/arch/x86/kernel/interrupt.S b/arch/x86/kernel/interrupt.S new file mode 100644 index 00000000..88b2d3b4 --- /dev/null +++ b/arch/x86/kernel/interrupt.S @@ -0,0 +1,128 @@ +#define X86_CPU_LOCAL_OFFSET_TSS 128 +#define X86_TSS_OFFSET_SP0 4 +#define X86_CPU_LOCAL_OFFSET_SP0 \ + (X86_CPU_LOCAL_OFFSET_TSS + X86_TSS_OFFSET_SP0) +#define X86_CPU_LOCAL_OFFSET_KSTACK 16 +#define X86_CPU_LOCAL_OFFSET_USTACK 24 +#define KERNEL_CS 32 +#define KERNEL_DS 40 +#define USER_CS (48 + 3) +#define USER_DS (56 + 3) + +#define PUSH_ALL_REGS \ + pushq %rax; \ + pushq %rbx; \ + pushq %rcx; \ + pushq %rdx; \ + pushq %rsi; \ + pushq %rdi; \ + pushq %r8; \ + pushq %r9; \ + pushq %r10; \ + pushq %r11; +#define POP_ALL_REGS \ + popq %r11; \ + popq %r10; \ + popq %r9; \ + popq %r8; \ + popq %rdi; \ + popq %rsi; \ + popq %rdx; \ + popq %rcx; \ + popq %rbx; \ + popq %rax + +.data +.globl generic_common_handlers +generic_common_handlers: +.text +vector=0 +.rept 256 +1: + cld + pushq $vector + jmp common_interrupt +.previous + .quad 1b +.text +vector=vector+1 +.endr + +common_interrupt: + PUSH_ALL_REGS + movq 80(%rsp), %rdi + movq %rsp, %rsi + call handle_interrupt /* Enter C code */ + POP_ALL_REGS + addq $8, %rsp + iretq + +.globl __page_fault_handler_address +__page_fault_handler_address: + .quad 0 + +.globl page_fault +page_fault: + cld + PUSH_ALL_REGS + movq %cr2, %rdi + movq %rsp, %rsi + movq %rbp, %rdx + movq __page_fault_handler_address(%rip), %rax + andq %rax, %rax + jz 1f + call *%rax + POP_ALL_REGS + addq $8, %rsp + iretq +1: + jmp 1b + +.globl general_protection_exception +general_protection_exception: + cld + PUSH_ALL_REGS + movq %rsp, %rdi + call gpe_handler + POP_ALL_REGS + addq $8, %rsp + iretq + +.globl x86_syscall +x86_syscall: + cld + movq %rsp, %gs:24 + movq %gs:(X86_CPU_LOCAL_OFFSET_SP0), %rsp + + pushq $(USER_DS) + pushq $0 + pushq %r11 + pushq $(USER_CS) + pushq %rcx + pushq $0 + movq %gs:24, %rcx + movq %rcx, 32(%rsp) + PUSH_ALL_REGS + movq 72(%rsp), %rdi + movw %ss, %ax + movw %ax, %ds + movq %rsp, %rsi + callq *__x86_syscall_handler(%rip) +1: + movq %rax, 72(%rsp) + POP_ALL_REGS +#ifdef USE_SYSRET + movq 8(%rsp), %rcx + movq 24(%rsp), %r11 + movq 32(%rsp), %rsp + sysretq +#else + addq $8, %rsp + iretq +#endif + +.globl enter_user_mode +enter_user_mode: + POP_ALL_REGS + addq $8, %rsp + iretq diff --git a/arch/x86/kernel/local.c b/arch/x86/kernel/local.c new file mode 100644 index 00000000..0f811a84 --- /dev/null +++ b/arch/x86/kernel/local.c @@ -0,0 +1,85 @@ +#include +#include +#include +#include +#include +#include +#include + +struct x86_cpu_local_variables *locals; + +void init_processors_local(int max_id) +{ + /* Is contiguous allocating adequate?? */ + locals = aal_mc_alloc_pages(max_id, 0); + memset(locals, 0, PAGE_SIZE * max_id); + + kprintf("locals = %p\n", locals); +} + +struct x86_cpu_local_variables *get_x86_cpu_local_variable(int id) +{ + return (struct x86_cpu_local_variables *) + ((char *)locals + (id << PAGE_SHIFT)); +} + +static void *get_x86_cpu_local_kstack(int id) +{ + return ((char *)locals + ((id + 1) << PAGE_SHIFT)); +} + +struct x86_cpu_local_variables *get_x86_this_cpu_local(void) +{ + int id = aal_mc_get_processor_id(); + + return get_x86_cpu_local_variable(id); +} + +void *get_x86_this_cpu_kstack(void) +{ + int id = aal_mc_get_processor_id(); + + return get_x86_cpu_local_kstack(id); +} + +static void set_fs_base(void *address) +{ + wrmsr(MSR_FS_BASE, (unsigned long)address); +} + +static void set_gs_base(void *address) +{ + wrmsr(MSR_GS_BASE, (unsigned long)address); +} + +static aal_atomic_t last_processor_id = AAL_ATOMIC_INIT(-1); + +void assign_processor_id(void) +{ + int id; + struct x86_cpu_local_variables *v; + + id = aal_atomic_inc_return(&last_processor_id); + + v = get_x86_cpu_local_variable(id); + set_gs_base(v); + + v->processor_id = id; +} + +/** AAL **/ +int aal_mc_get_processor_id(void) +{ + int id; + + asm volatile("movl %%gs:0, %0" : "=r"(id)); + + return id; +} + +int aal_mc_get_hardware_processor_id(void) +{ + struct x86_cpu_local_variables *v = get_x86_this_cpu_local(); + + return v->apic_id; +} diff --git a/arch/x86/kernel/lock.c b/arch/x86/kernel/lock.c new file mode 100644 index 00000000..43cfa0fc --- /dev/null +++ b/arch/x86/kernel/lock.c @@ -0,0 +1,35 @@ +#include + +#if 0 + +void aal_mc_spinlock_init(aal_spinlock_t *lock) +{ + *lock = 0; +} + +void aal_mc_spinlock_lock(aal_spinlock_t *lock, unsigned long *flags) +{ + int inc = 0x00010000; + int tmp; + + cpu_disable_interrupt_save(flags); + asm volatile("lock ; xaddl %0, %1\n" + "movzwl %w0, %2\n\t" + "shrl $16, %0\n\t" + "1:\t" + "cmpl %0, %2\n\t" + "je 2f\n\t" + "rep ; nop\n\t" + "movzwl %1, %2\n\t" + "jmp 1b\n" + "2:" + : "+Q" (inc), "+m" (*lock), "=r" (tmp) : : "memory", "cc"); +} + +void aal_mc_spinlock_unlock(aal_spinlock_t *lock, unsigned long *flags) +{ + asm volatile ("lock incw %0" : "+m"(*lock) : : "memory", "cc"); + cpu_restore_interrupt(*flags); +} + +#endif diff --git a/arch/x86/kernel/memory.c b/arch/x86/kernel/memory.c new file mode 100644 index 00000000..bae2b717 --- /dev/null +++ b/arch/x86/kernel/memory.c @@ -0,0 +1,703 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +static char *last_page; +extern char _head[], _end[]; + +struct aal_mc_pa_ops *pa_ops; + +extern unsigned long x86_kernel_phys_base; + +void *early_alloc_page(void) +{ + void *p; + + if (!last_page) { + last_page = (char *)(((unsigned long)_end + PAGE_SIZE - 1) + & PAGE_MASK); + /* Convert the virtual address from text's to straight maps */ + last_page = phys_to_virt(virt_to_phys(last_page)); + } else if (last_page == (void *)-1) { + panic("Early allocator is already finalized. Do not use it.\n"); + } + p = last_page; + last_page += PAGE_SIZE; + + return p; +} + +void *arch_alloc_page(enum aal_mc_ap_flag flag) +{ + if (pa_ops) + return pa_ops->alloc_page(1, flag); + else + return early_alloc_page(); +} +void arch_free_page(void *ptr) +{ + if (pa_ops) + pa_ops->free_page(ptr, 1); +} + +void *aal_mc_alloc_pages(int npages, enum aal_mc_ap_flag flag) +{ + if (pa_ops) + return pa_ops->alloc_page(npages, flag); + else + return NULL; +} + +void aal_mc_free_pages(void *p, int npages) +{ + if (pa_ops) + pa_ops->free_page(p, npages); +} + +void *aal_mc_allocate(int size, enum aal_mc_ap_flag flag) +{ + if (pa_ops && pa_ops->alloc) + return pa_ops->alloc(size, flag); + else + return aal_mc_alloc_pages(1, flag); +} + +void aal_mc_free(void *p) +{ + if (pa_ops && pa_ops->free) + return pa_ops->free(p); + else + return aal_mc_free_pages(p, 1); +} + +void *get_last_early_heap(void) +{ + return last_page; +} + +void flush_tlb(void) +{ + unsigned long cr3; + + asm volatile("movq %%cr3, %0; movq %0, %%cr3" : "=r"(cr3) : : "memory"); +} + +void flush_tlb_single(unsigned long addr) +{ + asm volatile("invlpg (%0)" :: "r" (addr) : "memory"); +} + +struct page_table { + pte_t entry[PT_ENTRIES]; +}; + +static struct page_table *init_pt; + +static unsigned long setup_l2(struct page_table *pt, + unsigned long page_head, unsigned long start, + unsigned long end) +{ + int i; + unsigned long phys; + + for (i = 0; i < PT_ENTRIES; i++) { + phys = page_head + ((unsigned long)i << PTL2_SHIFT); + + if (phys + PTL2_SIZE <= start || phys >= end) { + pt->entry[i] = 0; + continue; + } + + pt->entry[i] = phys | PFL2_KERN_ATTR | PFL2_SIZE; + } + + return virt_to_phys(pt); +} + +static unsigned long setup_l3(struct page_table *pt, + unsigned long page_head, unsigned long start, + unsigned long end) +{ + int i; + unsigned long phys, pt_phys; + + for (i = 0; i < PT_ENTRIES; i++) { + phys = page_head + ((unsigned long)i << PTL3_SHIFT); + + if (phys + PTL3_SIZE <= start || phys >= end) { + pt->entry[i] = 0; + continue; + } + pt_phys = setup_l2(arch_alloc_page(0), phys, start, end); + + pt->entry[i] = pt_phys | PFL3_KERN_ATTR; + } + + return virt_to_phys(pt); +} + +static void init_normal_area(struct page_table *pt) +{ + unsigned long map_start, map_end, phys, pt_phys; + int ident_index, virt_index; + + map_start = aal_mc_get_memory_address(AAL_MC_GMA_MAP_START, 0); + map_end = aal_mc_get_memory_address(AAL_MC_GMA_MAP_END, 0); + + kprintf("map_start = %lx, map_end = %lx\n", map_start, map_end); + ident_index = map_start >> PTL4_SHIFT; + virt_index = (MAP_ST_START >> PTL4_SHIFT) & (PT_ENTRIES - 1); + + memset(pt, 0, sizeof(struct page_table)); + + for (phys = (map_start & ~(PTL4_SIZE - 1)); phys < map_end; + phys += PTL4_SIZE) { + pt_phys = setup_l3(arch_alloc_page(0), phys, + map_start, map_end); + + pt->entry[ident_index++] = pt_phys | PFL4_KERN_ATTR; + pt->entry[virt_index++] = pt_phys | PFL4_KERN_ATTR; + } +} + +static struct page_table *__alloc_new_pt(void) +{ + struct page_table *newpt = arch_alloc_page(0); + + memset(newpt, 0, sizeof(struct page_table)); + + return newpt; +} + +/* + * XXX: Confusingly, L4 and L3 automatically add PRESENT, + * but L2 and L1 do not! + */ + +#define ATTR_MASK (PTATTR_WRITABLE | PTATTR_USER | PTATTR_ACTIVE) +static unsigned long attr_to_l4attr(enum aal_mc_pt_attribute attr) +{ + return (attr & ATTR_MASK) | PFL4_PRESENT; +} +static unsigned long attr_to_l3attr(enum aal_mc_pt_attribute attr) +{ + return (attr & ATTR_MASK) | PFL3_PRESENT; +} +static unsigned long attr_to_l2attr(enum aal_mc_pt_attribute attr) +{ + unsigned long r = (attr & (ATTR_MASK | PTATTR_LARGEPAGE)); + + if ((attr & PTATTR_UNCACHABLE) && (attr & PTATTR_LARGEPAGE)) { + return r | PFL2_PCD | PFL2_PWT; + } + return r; +} +static unsigned long attr_to_l1attr(enum aal_mc_pt_attribute attr) +{ + if (attr & PTATTR_UNCACHABLE) { + return (attr & ATTR_MASK) | PFL1_PWT | PFL1_PWT; + } else { + return (attr & ATTR_MASK); + } +} + +#define GET_VIRT_INDICES(virt, l4i, l3i, l2i, l1i) \ + l4i = ((virt) >> PTL4_SHIFT) & (PT_ENTRIES - 1); \ + l3i = ((virt) >> PTL3_SHIFT) & (PT_ENTRIES - 1); \ + l2i = ((virt) >> PTL2_SHIFT) & (PT_ENTRIES - 1); \ + l1i = ((virt) >> PTL1_SHIFT) & (PT_ENTRIES - 1) + + +void set_pte(pte_t *ppte, unsigned long phys, int attr) +{ + if (attr & PTATTR_LARGEPAGE) { + *ppte = phys | attr_to_l2attr(attr) | PFL2_SIZE; + } + else { + *ppte = phys | attr_to_l1attr(attr); + } +} + + +/* + * get_pte() + * + * Descripton: walks the page tables (creates tables if not existing) + * and returns a pointer to the PTE corresponding to the + * virtual address. + */ +pte_t *get_pte(struct page_table *pt, void *virt, int attr) +{ + int l4idx, l3idx, l2idx, l1idx; + unsigned long v = (unsigned long)virt; + struct page_table *newpt; + + if (!pt) { + pt = init_pt; + } + + GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx); + + /* TODO: more detailed attribute check */ + if (pt->entry[l4idx] & PFL4_PRESENT) { + pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK); + } else { + newpt = __alloc_new_pt(); + pt->entry[l4idx] = virt_to_phys(newpt) | attr_to_l4attr(attr); + pt = newpt; + } + + if (pt->entry[l3idx] & PFL3_PRESENT) { + pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK); + } else { + newpt = __alloc_new_pt(); + pt->entry[l3idx] = virt_to_phys(newpt) | attr_to_l3attr(attr); + pt = newpt; + } + + /* TODO: PTATTR_LARGEPAGE + if (attr & PTATTR_LARGEPAGE) { + if (pt->entry[l2idx] & PFL2_PRESENT) { + if ((pt->entry[l2idx] & PAGE_MASK) != phys) { + return -EBUSY; + } else { + return 0; + } + } else { + pt->entry[l2idx] = phys | attr_to_l2attr(attr) + | PFL2_SIZE; + return 0; + } + } + */ + + if (pt->entry[l2idx] & PFL2_PRESENT) { + pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK); + } else { + newpt = __alloc_new_pt(); + pt->entry[l2idx] = virt_to_phys(newpt) | attr_to_l2attr(attr) + | PFL2_PRESENT; + pt = newpt; + } + + return &(pt->entry[l1idx]); +} + +static int __set_pt_page(struct page_table *pt, void *virt, unsigned long phys, + int attr) +{ + int l4idx, l3idx, l2idx, l1idx; + unsigned long v = (unsigned long)virt; + struct page_table *newpt; + + if (!pt) { + pt = init_pt; + } + if (attr & PTATTR_LARGEPAGE) { + phys &= LARGE_PAGE_MASK; + } else { + phys &= PAGE_MASK; + } + + GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx); + + /* TODO: more detailed attribute check */ + if (pt->entry[l4idx] & PFL4_PRESENT) { + pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK); + } else { + newpt = __alloc_new_pt(); + pt->entry[l4idx] = virt_to_phys(newpt) | attr_to_l4attr(attr); + pt = newpt; + } + + if (pt->entry[l3idx] & PFL3_PRESENT) { + pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK); + } else { + newpt = __alloc_new_pt(); + pt->entry[l3idx] = virt_to_phys(newpt) | attr_to_l3attr(attr); + pt = newpt; + } + + if (attr & PTATTR_LARGEPAGE) { + if (pt->entry[l2idx] & PFL2_PRESENT) { + if ((pt->entry[l2idx] & PAGE_MASK) != phys) { + return -EBUSY; + } else { + return 0; + } + } else { + pt->entry[l2idx] = phys | attr_to_l2attr(attr) + | PFL2_SIZE; + return 0; + } + } + + if (pt->entry[l2idx] & PFL2_PRESENT) { + pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK); + } else { + newpt = __alloc_new_pt(); + pt->entry[l2idx] = virt_to_phys(newpt) | attr_to_l2attr(attr) + | PFL2_PRESENT; + pt = newpt; + } + + if (pt->entry[l1idx] & PFL1_PRESENT) { + if ((pt->entry[l1idx] & PAGE_MASK) != phys) { + return -EBUSY; + } else { + return 0; + } + } + pt->entry[l1idx] = phys | attr_to_l1attr(attr); + return 0; +} + +static int __clear_pt_page(struct page_table *pt, void *virt, int largepage) +{ + int l4idx, l3idx, l2idx, l1idx; + unsigned long v = (unsigned long)virt; + + if (!pt) { + pt = init_pt; + } + if (largepage) { + v &= LARGE_PAGE_MASK; + } else { + v &= PAGE_MASK; + } + + GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx); + + if (!(pt->entry[l4idx] & PFL4_PRESENT)) { + return -EINVAL; + } + pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK); + + if (!(pt->entry[l3idx] & PFL3_PRESENT)) { + return -EINVAL; + } + pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK); + + if (largepage) { + if (!(pt->entry[l2idx] & PFL2_PRESENT)) { + return -EINVAL; + } else { + pt->entry[l2idx] = 0; + return 0; + } + } + pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK); + + pt->entry[l1idx] = 0; + + return 0; +} + +int aal_mc_pt_virt_to_phys(struct page_table *pt, + void *virt, unsigned long *phys) +{ + int l4idx, l3idx, l2idx, l1idx; + unsigned long v = (unsigned long)virt; + + if (!pt) { + pt = init_pt; + } + + GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx); + + if (!(pt->entry[l4idx] & PFL4_PRESENT)) { + return -EFAULT; + } + pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK); + + if (!(pt->entry[l3idx] & PFL3_PRESENT)) { + return -EFAULT; + } + pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK); + + if (!(pt->entry[l2idx] & PFL2_PRESENT)) { + return -EFAULT; + } + if ((pt->entry[l2idx] & PFL2_SIZE)) { + *phys = (pt->entry[l2idx] & LARGE_PAGE_MASK) | + (v & (LARGE_PAGE_SIZE - 1)); + return 0; + } + pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK); + + if (!(pt->entry[l1idx] & PFL1_PRESENT)) { + return -EFAULT; + } + + *phys = (pt->entry[l1idx] & PAGE_MASK) | (v & (PAGE_SIZE - 1)); + return 0; +} + +int aal_mc_pt_print_pte(struct page_table *pt, void *virt) +{ + int l4idx, l3idx, l2idx, l1idx; + unsigned long v = (unsigned long)virt; + + if (!pt) { + pt = init_pt; + } + + GET_VIRT_INDICES(v, l4idx, l3idx, l2idx, l1idx); + + if (!(pt->entry[l4idx] & PFL4_PRESENT)) { + __kprintf("0x%lX l4idx not present! \n", (unsigned long)virt); + return -EFAULT; + } + pt = phys_to_virt(pt->entry[l4idx] & PAGE_MASK); + + __kprintf("l3 table: 0x%lX l3idx: %d \n", virt_to_phys(pt), l3idx); + if (!(pt->entry[l3idx] & PFL3_PRESENT)) { + __kprintf("0x%lX l3idx not present! \n", (unsigned long)virt); + return -EFAULT; + } + pt = phys_to_virt(pt->entry[l3idx] & PAGE_MASK); + + __kprintf("l2 table: 0x%lX l2idx: %d \n", virt_to_phys(pt), l2idx); + if (!(pt->entry[l2idx] & PFL2_PRESENT)) { + __kprintf("0x%lX l2idx not present! \n", (unsigned long)virt); + return -EFAULT; + } + if ((pt->entry[l2idx] & PFL2_SIZE)) { + return 0; + } + pt = phys_to_virt(pt->entry[l2idx] & PAGE_MASK); + + __kprintf("l1 table: 0x%lX l1idx: %d \n", virt_to_phys(pt), l1idx); + if (!(pt->entry[l1idx] & PFL1_PRESENT)) { + __kprintf("0x%lX PTE (l1) not present! entry: 0x%lX\n", + (unsigned long)virt, pt->entry[l1idx]); + return -EFAULT; + } + + return 0; +} + +int set_pt_large_page(struct page_table *pt, void *virt, unsigned long phys, + enum aal_mc_pt_attribute attr) +{ + return __set_pt_page(pt, virt, phys, attr | PTATTR_LARGEPAGE + | PTATTR_ACTIVE); +} + +int aal_mc_pt_set_large_page(page_table_t pt, void *virt, + unsigned long phys, enum aal_mc_pt_attribute attr) +{ + return __set_pt_page(pt, virt, phys, attr | PTATTR_LARGEPAGE + | PTATTR_ACTIVE); +} + +int aal_mc_pt_set_page(page_table_t pt, void *virt, + unsigned long phys, enum aal_mc_pt_attribute attr) +{ + return __set_pt_page(pt, virt, phys, attr | PTATTR_ACTIVE); +} + +int aal_mc_pt_prepare_map(page_table_t p, void *virt, unsigned long size, + enum aal_mc_pt_prepare_flag flag) +{ + int l4idx, l4e, ret = 0; + unsigned long v = (unsigned long)virt; + struct page_table *pt = p, *newpt; + unsigned long l; + enum aal_mc_pt_attribute attr = PTATTR_WRITABLE; + + if (!pt) { + pt = init_pt; + } + + l4idx = ((v) >> PTL4_SHIFT) & (PT_ENTRIES - 1); + + if (flag == AAL_MC_PT_FIRST_LEVEL) { + l4e = ((v + size) >> PTL4_SHIFT) & (PT_ENTRIES - 1); + + for (; l4idx <= l4e; l4idx++) { + if (pt->entry[l4idx] & PFL4_PRESENT) { + return 0; + } else { + newpt = __alloc_new_pt(); + if (!newpt) { + ret = -ENOMEM; + } else { + pt->entry[l4idx] = virt_to_phys(newpt) + | attr_to_l4attr(attr); + } + } + } + } else { + /* Call without ACTIVE flag */ + l = v + size; + for (; v < l; v += PAGE_SIZE) { + if ((ret = __set_pt_page(pt, (void *)v, 0, attr))) { + break; + } + } + } + return ret; +} + +struct page_table *aal_mc_pt_create(void) +{ + struct page_table *pt = aal_mc_alloc_pages(1, 0); + + memset(pt->entry, 0, PAGE_SIZE); + /* Copy the kernel space */ + memcpy(pt->entry + PT_ENTRIES / 2, init_pt->entry + PT_ENTRIES / 2, + sizeof(pt->entry[0]) * PT_ENTRIES / 2); + + return pt; +} + +int aal_mc_pt_clear_page(page_table_t pt, void *virt) +{ + return __clear_pt_page(pt, virt, 0); +} + +void load_page_table(struct page_table *pt) +{ + unsigned long pt_addr; + + if (!pt) { + pt = init_pt; + } + + pt_addr = virt_to_phys(pt); + + asm volatile ("movq %0, %%cr3" : : "r"(pt_addr) : "memory"); +} + +void aal_mc_load_page_table(struct page_table *pt) +{ + load_page_table(pt); +} + +struct page_table *get_init_page_table(void) +{ + return init_pt; +} + +static unsigned long fixed_virt; +static void init_fixed_area(struct page_table *pt) +{ + fixed_virt = MAP_FIXED_START; + + return; +} + +void init_text_area(struct page_table *pt) +{ + unsigned long __end, phys, virt; + int i, nlpages; + + __end = ((unsigned long)_end + LARGE_PAGE_SIZE * 2 - 1) + & LARGE_PAGE_MASK; + nlpages = (__end - MAP_KERNEL_START) >> LARGE_PAGE_SHIFT; + + kprintf("TEXT: # of large pages = %d\n", nlpages); + kprintf("TEXT: Base address = %lx\n", x86_kernel_phys_base); + + phys = x86_kernel_phys_base; + virt = MAP_KERNEL_START; + for (i = 0; i < nlpages; i++) { + set_pt_large_page(pt, (void *)virt, phys, PTATTR_WRITABLE); + + virt += LARGE_PAGE_SIZE; + phys += LARGE_PAGE_SIZE; + } +} + +void *map_fixed_area(unsigned long phys, unsigned long size, int uncachable) +{ + unsigned long poffset, paligned; + int i, npages; + int flag = PTATTR_WRITABLE | PTATTR_ACTIVE; + void *v = (void *)fixed_virt; + + poffset = phys & (PAGE_SIZE - 1); + paligned = phys & PAGE_MASK; + npages = (poffset + size + PAGE_SIZE - 1) >> PAGE_SHIFT; + + if (uncachable) { + flag |= PTATTR_UNCACHABLE; + } + + kprintf("map_fixed: %lx => %p (%d pages)\n", paligned, v, npages); + + for (i = 0; i < npages; i++) { + __set_pt_page(init_pt, (void *)fixed_virt, paligned, flag); + + fixed_virt += PAGE_SIZE; + paligned += PAGE_SIZE; + } + + flush_tlb(); + + return (char *)v + poffset; +} + +void init_low_area(struct page_table *pt) +{ + set_pt_large_page(pt, 0, 0, PTATTR_WRITABLE); +} + +void init_page_table(void) +{ + init_pt = arch_alloc_page(0); + + memset(init_pt, 0, sizeof(PAGE_SIZE)); + + /* Normal memory area */ + init_normal_area(init_pt); + init_fixed_area(init_pt); + init_low_area(init_pt); + init_text_area(init_pt); + + load_page_table(init_pt); + kprintf("Page table is now at %p\n", init_pt); +} + +extern void __reserve_arch_pages(unsigned long, unsigned long, + void (*)(unsigned long, unsigned long, int)); + +void aal_mc_reserve_arch_pages(unsigned long start, unsigned long end, + void (*cb)(unsigned long, unsigned long, int)) +{ + /* Reserve Text + temporal heap */ + cb(virt_to_phys(_head), virt_to_phys(get_last_early_heap()), 0); + /* Reserve trampoline area to boot the second ap */ + cb(AP_TRAMPOLINE, AP_TRAMPOLINE + AP_TRAMPOLINE_SIZE, 0); + /* Reserve the null page */ + cb(0, PAGE_SIZE, 0); + /* Micro-arch specific */ + __reserve_arch_pages(start, end, cb); +} + +void aal_mc_set_page_allocator(struct aal_mc_pa_ops *ops) +{ + last_page = NULL; + pa_ops = ops; +} + +unsigned long virt_to_phys(void *v) +{ + unsigned long va = (unsigned long)v; + + if (va >= MAP_KERNEL_START) { + return va - MAP_KERNEL_START + x86_kernel_phys_base; + } else { + return va - MAP_ST_START; + } +} +void *phys_to_virt(unsigned long p) +{ + return (void *)(p + MAP_ST_START); +} diff --git a/arch/x86/kernel/mikc.c b/arch/x86/kernel/mikc.c new file mode 100644 index 00000000..64e25305 --- /dev/null +++ b/arch/x86/kernel/mikc.c @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include + +extern void arch_set_mikc_queue(void *r, void *w); +aal_ikc_ph_t arch_master_channel_packet_handler; + +int aal_mc_ikc_init_first_local(struct aal_ikc_channel_desc *channel, + aal_ikc_ph_t packet_handler) +{ + struct aal_ikc_queue_head *rq, *wq; + + aal_ikc_system_init(NULL); + + memset(channel, 0, sizeof(struct aal_ikc_channel_desc)); + + /* Place both sides in this side */ + rq = arch_alloc_page(0); + wq = arch_alloc_page(0); + + aal_ikc_init_queue(rq, 0, 0, PAGE_SIZE, MASTER_IKCQ_PKTSIZE); + aal_ikc_init_queue(wq, 0, 0, PAGE_SIZE, MASTER_IKCQ_PKTSIZE); + + arch_master_channel_packet_handler = packet_handler; + + aal_ikc_init_desc(channel, IKC_OS_HOST, 0, rq, wq, + aal_ikc_master_channel_packet_handler); + aal_ikc_enable_channel(channel); + + /* Set boot parameter */ + arch_set_mikc_queue(rq, wq); + + return 0; +} diff --git a/arch/x86/kernel/perfctr.c b/arch/x86/kernel/perfctr.c new file mode 100644 index 00000000..3a617ecb --- /dev/null +++ b/arch/x86/kernel/perfctr.c @@ -0,0 +1,129 @@ +#include +#include +#include +#include +#include + +extern unsigned int *x86_march_perfmap; + +#define X86_CR4_PCE 0x00000100 + +void x86_init_perfctr(void) +{ + unsigned long reg; + + /* Allow PMC to be read from user space */ + asm volatile("movq %%cr4, %0" : "=r"(reg)); + reg |= X86_CR4_PCE; + asm volatile("movq %0, %%cr4" : : "r"(reg)); +} + +static int set_perfctr_x86_direct(int counter, int mode, unsigned int value) +{ + if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) { + return -EINVAL; + } + + if (mode & PERFCTR_USER_MODE) { + value |= 1 << 16; + } + if (mode & PERFCTR_KERNEL_MODE) { + value |= 1 << 17; + } + // wrmsr(MSR_PERF_GLOBAL_CTRL, 0); + + value |= (1 << 22) | (1 << 18); /* EN */ + + wrmsr(MSR_IA32_PERFEVTSEL0 + counter, value); + + kprintf("wrmsr: %d <= %x\n", MSR_PERF_GLOBAL_CTRL, 0); + kprintf("wrmsr: %d <= %x\n", MSR_IA32_PERFEVTSEL0 + counter, value); + return 0; +} + +static int set_perfctr_x86(int counter, int event, int mask, int inv, int count, + int mode) +{ + return set_perfctr_x86_direct(counter, mode, + CVAL2(event, mask, inv, count)); +} + +int aal_mc_perfctr_init(int counter, enum aal_perfctr_type type, int mode) +{ + if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) { + return -EINVAL; + } + if (type < 0 || type >= PERFCTR_MAX_TYPE) { + return -EINVAL; + } + if (!x86_march_perfmap[type]) { + return -EINVAL; + } + + return set_perfctr_x86_direct(counter, mode, x86_march_perfmap[type]); +} + +#ifdef HAVE_MARCH_PERFCTR_START +extern void x86_march_perfctr_start(unsigned long counter_mask); +#endif + +int aal_mc_perfctr_start(unsigned long counter_mask) +{ + unsigned int value = 0; + +#ifdef HAVE_MARCH_PERFCTR_START + x86_march_perfctr_start(counter_mask); +#endif + counter_mask &= ((1 << X86_IA32_NUM_PERF_COUNTERS) - 1); + value = rdmsr(MSR_PERF_GLOBAL_CTRL); + value |= counter_mask; + wrmsr(MSR_PERF_GLOBAL_CTRL, value); + + return 0; +} + +int aal_mc_perfctr_stop(unsigned long counter_mask) +{ + unsigned int value; + + counter_mask &= ((1 << X86_IA32_NUM_PERF_COUNTERS) - 1); + value = rdmsr(MSR_PERF_GLOBAL_CTRL); + value &= ~counter_mask; + wrmsr(MSR_PERF_GLOBAL_CTRL, value); + + return 0; +} + +int aal_mc_perfctr_reset(int counter) +{ + if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) { + return -EINVAL; + } + + wrmsr(MSR_IA32_PMC0 + counter, 0); + + return 0; +} + +int aal_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value) +{ + int i, j; + + for (i = 0, j = 0; i < X86_IA32_NUM_PERF_COUNTERS && counter_mask; + i++, counter_mask >>= 1) { + if (counter_mask & 1) { + value[j++] = rdpmc(i); + } + } + return 0; +} + +unsigned long aal_mc_perfctr_read(int counter) +{ + if (counter < 0 || counter >= X86_IA32_NUM_PERF_COUNTERS) { + return -EINVAL; + } + + return rdpmc(counter); +} + diff --git a/arch/x86/kernel/trampoline.S b/arch/x86/kernel/trampoline.S new file mode 100644 index 00000000..a9f7350f --- /dev/null +++ b/arch/x86/kernel/trampoline.S @@ -0,0 +1,150 @@ +#define BOOT_CS 0x10 +#define BOOT_DS 0x18 +#define BOOT_CS64 0x20 + +#define MSR_EFER 0xc0000080 +#define EFER_LME (1 << 8) + +.section .rodata, "a", @progbits +.code16 + +.globl trampoline_code_data +base = . +trampoline_code_data: + jmp cpu_start_body + + .org 8 +header_pgtbl: + .quad 0 /* page table address */ +func_address: + .quad 0 /* load address */ +arg: + .quad 0 /* next address */ +stack_ptr: + .quad 0 /* initial stack */ +debug: + .quad 0 /* debug area */ +transit_pgtbl: + .quad 0 /* 32->64 bit table address */ + +cpu_start_body: + cli + wbinvd + + movw %cs, %ax + movw %ax, %ds + movw %ax, %es + movw %ax, %ss + + xorl %ebx, %ebx + movw %cs, %bx + shll $4, %ebx + + movw $0x29, debug - base + + /* Adjust GDT ptr to the 32-bit physical address */ + addl %ebx, boot_gdtptr + 2 - base + addl %ebx, 2f - base + addl %ebx, start_64_vec - base + + lgdtl boot_gdtptr - base + lidtl boot_idtptr - base + + jmp 1f +1: + + movl %cr0, %edx + orb $1, %dl + movl %edx, %cr0 + + ljmpl *(2f - base) +2: .long protect_start - base + .word BOOT_CS + +.balign 8 +.code32 +protect_start: + movl $(BOOT_DS), %eax + movl %eax, %ds + movl %eax, %ss + + /* Enable PAE */ + movl %cr4, %eax + orl $0x20, %eax + movl %eax, %cr4 + + leal (stack_end - base)(%ebx), %esp + + /* Load a page table */ + movl (transit_pgtbl - base)(%ebx), %eax + movl %eax, %cr3 + +1: + + /* Enable Long Mode */ + movl $MSR_EFER, %ecx + movl $EFER_LME, %eax + xorl %edx, %edx + wrmsr + + /* Enable Paging */ + movl %cr0, %edx + orl $0x80000000, %edx + movl %edx, %cr0 + + ljmp *(start_64_vec - base)(%ebx) + +.code64 +.balign 8 +start_64: + /* Okay, we are completely in the long mode ! */ + /* So, use the real page table! */ + movq (header_pgtbl - base)(%ebx), %rax + movq %rax, %cr3 + + movq (func_address - base)(%ebx), %rcx + cmpq $0, %rcx + /* If Loading IP is zero, just enter the infinite loop */ + jz 3f + + movq (stack_ptr - base)(%ebx), %rax + cmpq $0, %rax + jz 1f + movq %rax, %rsp +1: + /* Now, we prepare the parameters */ + movq (arg - base)(%ebx), %rdi + jmp *%rcx + +3: + cli + hlt + jmp 3b + +boot_idtptr: + .short 0 + .long 0 + +boot_gdtptr: + .short boot_gdt32_end - boot_gdt32 + .long boot_gdt32 - base + .align 4 +boot_gdt32: + .quad 0 + .quad 0 + .quad 0x00cf9b000000ffff + .quad 0x00cf93000000ffff + .quad 0x00af9b000000ffff + .quad 0x0000890000000067 +boot_gdt32_end: + +start_64_vec: + .long start_64 - base + .word BOOT_CS64, 0 + +stack: + .org 0x1000 +stack_end: +.globl trampoline_code_data_end +trampoline_code_data_end: + diff --git a/linux/include/uprotocol.h b/executer/include/uprotocol.h similarity index 100% rename from linux/include/uprotocol.h rename to executer/include/uprotocol.h diff --git a/linux/mod_mcctrl/Makefile b/executer/kernel/Makefile similarity index 100% rename from linux/mod_mcctrl/Makefile rename to executer/kernel/Makefile diff --git a/linux/mod_mcctrl/control.c b/executer/kernel/control.c similarity index 100% rename from linux/mod_mcctrl/control.c rename to executer/kernel/control.c diff --git a/linux/mod_mcctrl/driver.c b/executer/kernel/driver.c similarity index 100% rename from linux/mod_mcctrl/driver.c rename to executer/kernel/driver.c diff --git a/linux/mod_mcctrl/ikc.c b/executer/kernel/ikc.c similarity index 100% rename from linux/mod_mcctrl/ikc.c rename to executer/kernel/ikc.c diff --git a/linux/mod_mcctrl/mcctrl.h b/executer/kernel/mcctrl.h similarity index 100% rename from linux/mod_mcctrl/mcctrl.h rename to executer/kernel/mcctrl.h diff --git a/linux/mod_mcctrl/syscall.c b/executer/kernel/syscall.c similarity index 100% rename from linux/mod_mcctrl/syscall.c rename to executer/kernel/syscall.c diff --git a/linux/executer/Makefile b/executer/user/Makefile similarity index 100% rename from linux/executer/Makefile rename to executer/user/Makefile diff --git a/linux/executer/mcexec.c b/executer/user/mcexec.c similarity index 100% rename from linux/executer/mcexec.c rename to executer/user/mcexec.c diff --git a/kernel/knf.lds b/kernel/config/attached-mic.lds similarity index 100% rename from kernel/knf.lds rename to kernel/config/attached-mic.lds diff --git a/kernel/mee.lds b/kernel/config/builtin-x86.lds similarity index 100% rename from kernel/mee.lds rename to kernel/config/builtin-x86.lds diff --git a/kernel/configs/config.knf b/kernel/config/config.attached-mic similarity index 100% rename from kernel/configs/config.knf rename to kernel/config/config.attached-mic diff --git a/kernel/configs/config.mee b/kernel/config/config.builtin-x86 similarity index 100% rename from kernel/configs/config.mee rename to kernel/config/config.builtin-x86 diff --git a/kernel/scripts/mkimage.knf b/kernel/script/mkimage.attached-mic similarity index 100% rename from kernel/scripts/mkimage.knf rename to kernel/script/mkimage.attached-mic diff --git a/kernel/scripts/mkimage.mee b/kernel/script/mkimage.builtin-x86 similarity index 100% rename from kernel/scripts/mkimage.mee rename to kernel/script/mkimage.builtin-x86 diff --git a/lib/abort.c b/lib/abort.c new file mode 100644 index 00000000..59a9ab81 --- /dev/null +++ b/lib/abort.c @@ -0,0 +1,21 @@ +#include +#include + +void panic(const char *msg) +{ + cpu_disable_interrupt(); + + kprintf(msg); + + while (1) { + cpu_halt(); + } +} + +extern void arch_show_interrupt_context(const void*); + +void aal_mc_debug_show_interrupt_context(const void *reg) +{ + arch_show_interrupt_context(reg); +} + diff --git a/lib/include/ctype.h b/lib/include/ctype.h new file mode 100644 index 00000000..afa36392 --- /dev/null +++ b/lib/include/ctype.h @@ -0,0 +1,54 @@ +#ifndef _LINUX_CTYPE_H +#define _LINUX_CTYPE_H + +/* + * NOTE! This ctype does not handle EOF like the standard C + * library is required to. + */ + +#define _U 0x01 /* upper */ +#define _L 0x02 /* lower */ +#define _D 0x04 /* digit */ +#define _C 0x08 /* cntrl */ +#define _P 0x10 /* punct */ +#define _S 0x20 /* white space (space/lf/tab) */ +#define _X 0x40 /* hex digit */ +#define _SP 0x80 /* hard space (0x20) */ + +extern unsigned char _ctype[]; + +#define __ismask(x) (_ctype[(int)(unsigned char)(x)]) + +#define isalnum(c) ((__ismask(c)&(_U|_L|_D)) != 0) +#define isalpha(c) ((__ismask(c)&(_U|_L)) != 0) +#define iscntrl(c) ((__ismask(c)&(_C)) != 0) +#define isdigit(c) ((__ismask(c)&(_D)) != 0) +#define isgraph(c) ((__ismask(c)&(_P|_U|_L|_D)) != 0) +#define islower(c) ((__ismask(c)&(_L)) != 0) +#define isprint(c) ((__ismask(c)&(_P|_U|_L|_D|_SP)) != 0) +#define ispunct(c) ((__ismask(c)&(_P)) != 0) +#define isspace(c) ((__ismask(c)&(_S)) != 0) +#define isupper(c) ((__ismask(c)&(_U)) != 0) +#define isxdigit(c) ((__ismask(c)&(_D|_X)) != 0) + +#define isascii(c) (((unsigned char)(c))<=0x7f) +#define toascii(c) (((unsigned char)(c))&0x7f) + +static inline unsigned char __tolower(unsigned char c) +{ + if (isupper(c)) + c -= 'A'-'a'; + return c; +} + +static inline unsigned char __toupper(unsigned char c) +{ + if (islower(c)) + c -= 'a'-'A'; + return c; +} + +#define tolower(c) __tolower(c) +#define toupper(c) __toupper(c) + +#endif diff --git a/lib/include/errno.h b/lib/include/errno.h new file mode 100644 index 00000000..ef7ddb96 --- /dev/null +++ b/lib/include/errno.h @@ -0,0 +1,169 @@ +#ifndef _ASM_GENERIC_ERRNO_BASE_H +#define _ASM_GENERIC_ERRNO_BASE_H + +#define EPERM 1 /* Operation not permitted */ +#define ENOENT 2 /* No such file or directory */ +#define ESRCH 3 /* No such process */ +#define EINTR 4 /* Interrupted system call */ +#define EIO 5 /* I/O error */ +#define ENXIO 6 /* No such device or address */ +#define E2BIG 7 /* Argument list too long */ +#define ENOEXEC 8 /* Exec format error */ +#define EBADF 9 /* Bad file number */ +#define ECHILD 10 /* No child processes */ +#define EAGAIN 11 /* Try again */ +#define ENOMEM 12 /* Out of memory */ +#define EACCES 13 /* Permission denied */ +#define EFAULT 14 /* Bad address */ +#define ENOTBLK 15 /* Block device required */ +#define EBUSY 16 /* Device or resource busy */ +#define EEXIST 17 /* File exists */ +#define EXDEV 18 /* Cross-device link */ +#define ENODEV 19 /* No such device */ +#define ENOTDIR 20 /* Not a directory */ +#define EISDIR 21 /* Is a directory */ +#define EINVAL 22 /* Invalid argument */ +#define ENFILE 23 /* File table overflow */ +#define EMFILE 24 /* Too many open files */ +#define ENOTTY 25 /* Not a typewriter */ +#define ETXTBSY 26 /* Text file busy */ +#define EFBIG 27 /* File too large */ +#define ENOSPC 28 /* No space left on device */ +#define ESPIPE 29 /* Illegal seek */ +#define EROFS 30 /* Read-only file system */ +#define EMLINK 31 /* Too many links */ +#define EPIPE 32 /* Broken pipe */ +#define EDOM 33 /* Math argument out of domain of func */ +#define ERANGE 34 /* Math result not representable */ + +#define EDEADLK 35 /* Resource deadlock would occur */ +#define ENAMETOOLONG 36 /* File name too long */ +#define ENOLCK 37 /* No record locks available */ +#define ENOSYS 38 /* Function not implemented */ +#define ENOTEMPTY 39 /* Directory not empty */ +#define ELOOP 40 /* Too many symbolic links encountered */ +#define EWOULDBLOCK EAGAIN /* Operation would block */ +#define ENOMSG 42 /* No message of desired type */ +#define EIDRM 43 /* Identifier removed */ +#define ECHRNG 44 /* Channel number out of range */ +#define EL2NSYNC 45 /* Level 2 not synchronized */ +#define EL3HLT 46 /* Level 3 halted */ +#define EL3RST 47 /* Level 3 reset */ +#define ELNRNG 48 /* Link number out of range */ +#define EUNATCH 49 /* Protocol driver not attached */ +#define ENOCSI 50 /* No CSI structure available */ +#define EL2HLT 51 /* Level 2 halted */ +#define EBADE 52 /* Invalid exchange */ +#define EBADR 53 /* Invalid request descriptor */ +#define EXFULL 54 /* Exchange full */ +#define ENOANO 55 /* No anode */ +#define EBADRQC 56 /* Invalid request code */ +#define EBADSLT 57 /* Invalid slot */ + +#define EDEADLOCK EDEADLK + +#define EBFONT 59 /* Bad font file format */ +#define ENOSTR 60 /* Device not a stream */ +#define ENODATA 61 /* No data available */ +#define ETIME 62 /* Timer expired */ +#define ENOSR 63 /* Out of streams resources */ +#define ENONET 64 /* Machine is not on the network */ +#define ENOPKG 65 /* Package not installed */ +#define EREMOTE 66 /* Object is remote */ +#define ENOLINK 67 /* Link has been severed */ +#define EADV 68 /* Advertise error */ +#define ESRMNT 69 /* Srmount error */ +#define ECOMM 70 /* Communication error on send */ +#define EPROTO 71 /* Protocol error */ +#define EMULTIHOP 72 /* Multihop attempted */ +#define EDOTDOT 73 /* RFS specific error */ +#define EBADMSG 74 /* Not a data message */ +#define EOVERFLOW 75 /* Value too large for defined data type */ +#define ENOTUNIQ 76 /* Name not unique on network */ +#define EBADFD 77 /* File descriptor in bad state */ +#define EREMCHG 78 /* Remote address changed */ +#define ELIBACC 79 /* Can not access a needed shared library */ +#define ELIBBAD 80 /* Accessing a corrupted shared library */ +#define ELIBSCN 81 /* .lib section in a.out corrupted */ +#define ELIBMAX 82 /* Attempting to link in too many shared libraries */ +#define ELIBEXEC 83 /* Cannot exec a shared library directly */ +#define EILSEQ 84 /* Illegal byte sequence */ +#define ERESTART 85 /* Interrupted system call should be restarted */ +#define ESTRPIPE 86 /* Streams pipe error */ +#define EUSERS 87 /* Too many users */ +#define ENOTSOCK 88 /* Socket operation on non-socket */ +#define EDESTADDRREQ 89 /* Destination address required */ +#define EMSGSIZE 90 /* Message too long */ +#define EPROTOTYPE 91 /* Protocol wrong type for socket */ +#define ENOPROTOOPT 92 /* Protocol not available */ +#define EPROTONOSUPPORT 93 /* Protocol not supported */ +#define ESOCKTNOSUPPORT 94 /* Socket type not supported */ +#define EOPNOTSUPP 95 /* Operation not supported on transport endpoint */ +#define EPFNOSUPPORT 96 /* Protocol family not supported */ +#define EAFNOSUPPORT 97 /* Address family not supported by protocol */ +#define EADDRINUSE 98 /* Address already in use */ +#define EADDRNOTAVAIL 99 /* Cannot assign requested address */ +#define ENETDOWN 100 /* Network is down */ +#define ENETUNREACH 101 /* Network is unreachable */ +#define ENETRESET 102 /* Network dropped connection because of reset */ +#define ECONNABORTED 103 /* Software caused connection abort */ +#define ECONNRESET 104 /* Connection reset by peer */ +#define ENOBUFS 105 /* No buffer space available */ +#define EISCONN 106 /* Transport endpoint is already connected */ +#define ENOTCONN 107 /* Transport endpoint is not connected */ +#define ESHUTDOWN 108 /* Cannot send after transport endpoint shutdown */ +#define ETOOMANYREFS 109 /* Too many references: cannot splice */ +#define ETIMEDOUT 110 /* Connection timed out */ +#define ECONNREFUSED 111 /* Connection refused */ +#define EHOSTDOWN 112 /* Host is down */ +#define EHOSTUNREACH 113 /* No route to host */ +#define EALREADY 114 /* Operation already in progress */ +#define EINPROGRESS 115 /* Operation now in progress */ +#define ESTALE 116 /* Stale NFS file handle */ +#define EUCLEAN 117 /* Structure needs cleaning */ +#define ENOTNAM 118 /* Not a XENIX named type file */ +#define ENAVAIL 119 /* No XENIX semaphores available */ +#define EISNAM 120 /* Is a named type file */ +#define EREMOTEIO 121 /* Remote I/O error */ +#define EDQUOT 122 /* Quota exceeded */ + +#define ENOMEDIUM 123 /* No medium found */ +#define EMEDIUMTYPE 124 /* Wrong medium type */ +#define ECANCELED 125 /* Operation Canceled */ +#define ENOKEY 126 /* Required key not available */ +#define EKEYEXPIRED 127 /* Key has expired */ +#define EKEYREVOKED 128 /* Key has been revoked */ +#define EKEYREJECTED 129 /* Key was rejected by service */ + +/* for robust mutexes */ +#define EOWNERDEAD 130 /* Owner died */ +#define ENOTRECOVERABLE 131 /* State not recoverable */ + +#define ERFKILL 132 /* Operation not possible due to RF-kill */ + + +#ifdef __KERNEL__ + +/* Should never be seen by user programs */ +#define ERESTARTSYS 512 +#define ERESTARTNOINTR 513 +#define ERESTARTNOHAND 514 /* restart if no handler.. */ +#define ENOIOCTLCMD 515 /* No ioctl command */ +#define ERESTART_RESTARTBLOCK 516 /* restart by calling sys_restart_syscall */ + +/* Defined for the NFSv3 protocol */ +#define EBADHANDLE 521 /* Illegal NFS file handle */ +#define ENOTSYNC 522 /* Update synchronization mismatch */ +#define EBADCOOKIE 523 /* Cookie is stale */ +#define ENOTSUPP 524 /* Operation is not supported */ +#define ETOOSMALL 525 /* Buffer or request is too small */ +#define ESERVERFAULT 526 /* An untranslatable error occurred */ +#define EBADTYPE 527 /* Type not supported by server */ +#define EJUKEBOX 528 /* Request initiated, but will not complete before timeout */ +#define EIOCBQUEUED 529 /* iocb queued, will get completion event */ +#define EIOCBRETRY 530 /* iocb queued, will trigger a retry */ + +#endif + + +#endif diff --git a/lib/include/ihk/cpu.h b/lib/include/ihk/cpu.h new file mode 100644 index 00000000..7949f247 --- /dev/null +++ b/lib/include/ihk/cpu.h @@ -0,0 +1,81 @@ +#ifndef AAL_CPU_H +#define AAL_CPU_H + +#include +#include + +void cpu_enable_interrupt(void); +void cpu_disable_interrupt(void); +void cpu_halt(void); +void cpu_restore_interrupt(unsigned long); +void cpu_pause(void); + +#define barrier() asm volatile("" : : : "memory") + +unsigned long cpu_disable_interrupt_save(void); + +struct aal_mc_interrupt_handler { + struct list_head list; + void (*func)(void *); + void *priv; +}; +int aal_mc_register_interrupt_handler(int vector, + struct aal_mc_interrupt_handler *h); +int aal_mc_unregister_interrupt_handler(int vector, + struct aal_mc_interrupt_handler *h); + +enum aal_mc_gv_type { + AAL_GV_IKC = 1, +}; + +int aal_mc_get_vector(enum aal_mc_gv_type type); +int aal_mc_interrupt_host(int cpu, int vector); + +struct aal_mc_cpu_info { + int ncpus; + int *hw_ids; + int *nodes; +}; + +struct aal_mc_cpu_info *aal_mc_get_cpu_info(void); +void aal_mc_boot_cpu(int cpuid, unsigned long pc); +int aal_mc_get_processor_id(void); +int aal_mc_get_hardware_processor_id(void); + +void aal_mc_delay_us(int us); +void aal_mc_set_syscall_handler(long (*handler)(int, aal_mc_user_context_t *)); + +void aal_mc_init_ap(void); + +void aal_mc_init_context(aal_mc_kernel_context_t *new_ctx, + void *stack_pointer, + void (*next_function)(void)); +void aal_mc_switch_context(aal_mc_kernel_context_t *old_ctx, + aal_mc_kernel_context_t *new_ctx); +int aal_mc_interrupt_cpu(int cpu, int vector); + +void aal_mc_init_user_process(aal_mc_kernel_context_t *ctx, + aal_mc_user_context_t **puctx, + void *stack_pointer, unsigned long user_pc, + unsigned long user_sp); + +enum aal_mc_user_context_regtype { + AAL_UCR_STACK_POINTER = 1, + AAL_UCR_PROGRAM_COUNTER = 2, +}; + +void aal_mc_modify_user_context(aal_mc_user_context_t *uctx, + enum aal_mc_user_context_regtype reg, + unsigned long value); + +void aal_mc_debug_show_interrupt_context(const void *reg); + +enum aal_asr_type { + AAL_ASR_X86_FS, + AAL_ASR_X86_GS, +}; + +int aal_mc_arch_set_special_register(enum aal_asr_type, unsigned long value); +int aal_mc_arch_get_special_register(enum aal_asr_type, unsigned long *value); + +#endif diff --git a/lib/include/ihk/debug.h b/lib/include/ihk/debug.h new file mode 100644 index 00000000..bbc7ddc3 --- /dev/null +++ b/lib/include/ihk/debug.h @@ -0,0 +1,19 @@ +#ifndef AAL_DEBUG_H +#define AAL_DEBUG_H + +#include + +struct aal_kmsg_buf { + int tail; + int len; + char str[AAL_KMSG_SIZE - sizeof(int) * 2]; +}; + +extern int kprintf(const char *format, ...); +extern int kprintf_lock(); +extern void kprintf_unlock(int irqflags); +extern int __kprintf(const char *format, ...); + +extern void panic(const char *msg); + +#endif diff --git a/lib/include/ihk/dma.h b/lib/include/ihk/dma.h new file mode 100644 index 00000000..2b1ca130 --- /dev/null +++ b/lib/include/ihk/dma.h @@ -0,0 +1,21 @@ +#ifndef __HEADER_GENERIC_INCLUDE_DMA_H +#define __HEADER_GENERIC_INCLUDE_DMA_H + +#include + +struct aal_dma_request { + aal_os_t src_os; + unsigned long src_phys; + aal_os_t dest_os; + unsigned long dest_phys; + unsigned long size; + + void (*callback)(void *); + void *priv; + aal_os_t notify_os; + unsigned long *notify; +}; + +int aal_mc_dma_request(int channel, struct aal_dma_request *req); + +#endif diff --git a/lib/include/ihk/lock.h b/lib/include/ihk/lock.h new file mode 100644 index 00000000..2462aa0e --- /dev/null +++ b/lib/include/ihk/lock.h @@ -0,0 +1,13 @@ +#ifndef __HEADER_GENERIC_AAL_LOCK +#define __HEADER_GENERIC_AAL_LOCK + +#include + +#ifndef AAL_STATIC_SPINLOCK_FUNCS +void aal_mc_spinlock_init(aal_spinlock_t *); +void aal_mc_spinlock_lock(aal_spinlock_t *, unsigned long *); +void aal_mc_spinlock_unlock(aal_spinlock_t *, unsigned long *); +#endif + +#endif + diff --git a/lib/include/ihk/mm.h b/lib/include/ihk/mm.h new file mode 100644 index 00000000..c9276eb6 --- /dev/null +++ b/lib/include/ihk/mm.h @@ -0,0 +1,99 @@ +#ifndef __HEADER_GENERIC_AAL_MM_H +#define __HEADER_GENERIC_AAL_MM_H + +#include + +enum aal_mc_gma_type { + AAL_MC_GMA_MAP_START, + AAL_MC_GMA_MAP_END, + AAL_MC_GMA_AVAIL_START, + AAL_MC_GMA_AVAIL_END, + AAL_MC_GMA_HEAP_START, + AAL_MC_NR_RESERVED_AREAS, + AAL_MC_RESERVED_AREA_START, + AAL_MC_RESERVED_AREA_END, +}; + +enum aal_mc_ma_type { + AAL_MC_MA_AVAILABLE, + AAL_MC_MA_RESERVED, + AAL_MC_MA_SPECIAL, +}; + +enum aal_mc_ap_flag { + AAL_MC_AP_FLAG, +}; + +enum aal_mc_pt_prepare_flag { + AAL_MC_PT_FIRST_LEVEL, + AAL_MC_PT_LAST_LEVEL, +}; + +struct aal_mc_memory_area { + unsigned long start; + unsigned long size; + enum aal_mc_ma_type type; +}; + +struct aal_mc_memory_node { + int node; + int nareas; + struct aal_mc_memory_area *areas; +}; + +unsigned long aal_mc_get_memory_address(enum aal_mc_gma_type, int); + +void aal_mc_reserve_arch_pages(unsigned long start, unsigned long end, + void (*cb)(unsigned long, unsigned long, int)); + +struct aal_mc_pa_ops { + void *(*alloc_page)(int, enum aal_mc_ap_flag); + void (*free_page)(void *, int); + + void *(*alloc)(int, enum aal_mc_ap_flag); + void (*free)(void *); +}; + +void aal_mc_set_page_allocator(struct aal_mc_pa_ops *); +void aal_mc_set_page_fault_handler(void (*h)(unsigned long, void *, unsigned long)); + +unsigned long aal_mc_map_memory(void *os, unsigned long phys, + unsigned long size); +void aal_mc_unmap_memory(void *os, unsigned long phys, unsigned long size); + +void *aal_mc_map_virtual(unsigned long phys, int npages, + enum aal_mc_pt_attribute attr); +void aal_mc_unmap_virtual(void *va, int npages, int free_physical); + +extern void *sbox_base; +extern unsigned int free_bitmap_micpa; +void aal_mc_map_micpa(unsigned long host_pa, unsigned long* mic_pa); +int aal_mc_free_micpa(unsigned long mic_pa); +void aal_mc_clean_micpa(void); + +void *aal_mc_alloc_pages(int npages, enum aal_mc_ap_flag flag); +void aal_mc_free_pages(void *p, int npages); +void *aal_mc_allocate(int size, enum aal_mc_ap_flag flag); +void aal_mc_free(void *p); + +void *arch_alloc_page(enum aal_mc_ap_flag flag); +void arch_free_page(void *ptr); + +typedef void *page_table_t; + +int aal_mc_pt_set_page(page_table_t pt, void *virt, unsigned long phys, + enum aal_mc_pt_attribute attr); +int aal_mc_pt_set_large_page(page_table_t pt, void *virt, + unsigned long phys, enum aal_mc_pt_attribute attr); +int aal_mc_pt_change_page(page_table_t pt, void *virt, + enum aal_mc_pt_attribute); +int aal_mc_pt_clear_page(page_table_t pt, void *virt); +int aal_mc_pt_prepare_map(page_table_t pt, void *virt, unsigned long size, + enum aal_mc_pt_prepare_flag); + +struct page_table *aal_mc_pt_create(void); +void aal_mc_load_page_table(struct page_table *pt); +int aal_mc_pt_virt_to_phys(struct page_table *pt, + void *virt, unsigned long *phys); + +#endif diff --git a/lib/include/ihk/page_alloc.h b/lib/include/ihk/page_alloc.h new file mode 100644 index 00000000..9f5fd0c3 --- /dev/null +++ b/lib/include/ihk/page_alloc.h @@ -0,0 +1,28 @@ +#ifndef __HEADER_GENERIC_AAL_PAGE_ALLOC +#define __HEADER_GENERIC_AAL_PAGE_ALLOC + +struct aal_page_allocator_desc { + unsigned long start; + unsigned int last; + unsigned int count; + unsigned int flag; + unsigned int shift; + aal_spinlock_t lock; + unsigned int pad; + + unsigned long map[0]; +}; + +unsigned long aal_pagealloc_count(void *__desc); +void *__aal_pagealloc_init(unsigned long start, unsigned long size, + unsigned long unit, void *initial, + unsigned long *pdescsize); +void *aal_pagealloc_init(unsigned long start, unsigned long size, + unsigned long unit); +void aal_pagealloc_destroy(void *__desc); +unsigned long aal_pagealloc_alloc(void *__desc, int npages); +void aal_pagealloc_reserve(void *desc, unsigned long start, unsigned long end); +void aal_pagealloc_free(void *__desc, unsigned long address, int npages); +unsigned long aal_pagealloc_count(void *__desc); + +#endif diff --git a/lib/include/ihk/perfctr.h b/lib/include/ihk/perfctr.h new file mode 100644 index 00000000..883682a0 --- /dev/null +++ b/lib/include/ihk/perfctr.h @@ -0,0 +1,41 @@ +#ifndef HEADER_GENERIC_AAL_PERFCTR_H +#define HEADER_GENERIC_AAL_PERFCTR_H + +#define PERFCTR_USER_MODE 0x01 +#define PERFCTR_KERNEL_MODE 0x02 + +enum aal_perfctr_type { + APT_TYPE_DATA_PAGE_WALK, + APT_TYPE_DATA_READ_MISS, + APT_TYPE_DATA_WRITE_MISS, + APT_TYPE_BANK_CONFLICTS, + APT_TYPE_CODE_CACHE_MISS, + APT_TYPE_INSTRUCTIONS_EXECUTED, + APT_TYPE_INSTRUCTIONS_EXECUTED_V_PIPE, + + APT_TYPE_L2_READ_MISS, + APT_TYPE_L2_CODE_READ_MISS_CACHE_FILL, + APT_TYPE_L2_DATA_READ_MISS_CACHE_FILL, + APT_TYPE_L2_CODE_READ_MISS_MEM_FILL, + APT_TYPE_L2_DATA_READ_MISS_MEM_FILL, + + APT_TYPE_L1D_REQUEST, + APT_TYPE_L1I_REQUEST, + APT_TYPE_L1_MISS, + APT_TYPE_LLC_MISS, + APT_TYPE_DTLB_MISS, + APT_TYPE_ITLB_MISS, + APT_TYPE_STALL, + APT_TYPE_CYCLE, + PERFCTR_MAX_TYPE, +}; + +int aal_mc_perfctr_init(int counter, enum aal_perfctr_type type, int mode); +int aal_mc_perfctr_start(unsigned long counter_mask); +int aal_mc_perfctr_stop(unsigned long counter_mask); +int aal_mc_perfctr_reset(int counter); +int aal_mc_perfctr_read_mask(unsigned long counter_mask, unsigned long *value); +unsigned long aal_mc_perfctr_read(int counter); + +#endif + diff --git a/lib/include/limits.h b/lib/include/limits.h new file mode 100644 index 00000000..f335b541 --- /dev/null +++ b/lib/include/limits.h @@ -0,0 +1,7 @@ +#ifndef __HEADER_LIMITS +#define __HEADER_LIMITS + +#define INT_MAX 0x7fffffff +#define INT_MIN -0x80000000 + +#endif diff --git a/lib/include/list.h b/lib/include/list.h new file mode 100644 index 00000000..39b832d0 --- /dev/null +++ b/lib/include/list.h @@ -0,0 +1,581 @@ +#ifndef _LINUX_LIST_H +#define _LINUX_LIST_H + +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +/** + * container_of - cast a member of a structure out to the containing structure + * @ptr: the pointer to the member. + * @type: the type of the container struct this is embedded in. + * @member: the name of the member within the struct. + * + */ +#define container_of(ptr, type, member) ({ \ + const typeof( ((type *)0)->member ) *__mptr = (ptr); \ + (type *)( (char *)__mptr - offsetof(type,member) );}) + +struct list_head { + struct list_head *next, *prev; +}; + +#define LIST_POISON1 ((void *)0x00100129) +#define LIST_POISON2 ((void *)0x00200229) + +/* + * Simple doubly linked list implementation. + * + * Some of the internal functions ("__xxx") are useful when + * manipulating whole lists rather than single entries, as + * sometimes we already know the next/prev entries and we can + * generate better code by using them directly rather than + * using the generic single-entry routines. + */ + +#define LIST_HEAD_INIT(name) { &(name), &(name) } + +#define LIST_HEAD(name) \ + struct list_head name = LIST_HEAD_INIT(name) + +static inline void INIT_LIST_HEAD(struct list_head *list) +{ + list->next = list; + list->prev = list; +} + +/* + * Insert a new entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +#ifndef CONFIG_DEBUG_LIST +static inline void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next) +{ + next->prev = new; + new->next = next; + new->prev = prev; + prev->next = new; +} +#else +extern void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next); +#endif + +/** + * list_add - add a new entry + * @new: new entry to be added + * @head: list head to add it after + * + * Insert a new entry after the specified head. + * This is good for implementing stacks. + */ +static inline void list_add(struct list_head *new, struct list_head *head) +{ + __list_add(new, head, head->next); +} + + +/** + * list_add_tail - add a new entry + * @new: new entry to be added + * @head: list head to add it before + * + * Insert a new entry before the specified head. + * This is useful for implementing queues. + */ +static inline void list_add_tail(struct list_head *new, struct list_head *head) +{ + __list_add(new, head->prev, head); +} + +/* + * Delete a list entry by making the prev/next entries + * point to each other. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_del(struct list_head * prev, struct list_head * next) +{ + next->prev = prev; + prev->next = next; +} + +/** + * list_del - deletes entry from list. + * @entry: the element to delete from the list. + * Note: list_empty() on entry does not return true after this, the entry is + * in an undefined state. + */ +#ifndef CONFIG_DEBUG_LIST +static inline void __list_del_entry(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); +} + +static inline void list_del(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); + entry->next = LIST_POISON1; + entry->prev = LIST_POISON2; +} +#else +extern void __list_del_entry(struct list_head *entry); +extern void list_del(struct list_head *entry); +#endif + +/** + * list_replace - replace old entry by new one + * @old : the element to be replaced + * @new : the new element to insert + * + * If @old was empty, it will be overwritten. + */ +static inline void list_replace(struct list_head *old, + struct list_head *new) +{ + new->next = old->next; + new->next->prev = new; + new->prev = old->prev; + new->prev->next = new; +} + +static inline void list_replace_init(struct list_head *old, + struct list_head *new) +{ + list_replace(old, new); + INIT_LIST_HEAD(old); +} + +/** + * list_del_init - deletes entry from list and reinitialize it. + * @entry: the element to delete from the list. + */ +static inline void list_del_init(struct list_head *entry) +{ + __list_del_entry(entry); + INIT_LIST_HEAD(entry); +} + +/** + * list_move - delete from one list and add as another's head + * @list: the entry to move + * @head: the head that will precede our entry + */ +static inline void list_move(struct list_head *list, struct list_head *head) +{ + __list_del_entry(list); + list_add(list, head); +} + +/** + * list_move_tail - delete from one list and add as another's tail + * @list: the entry to move + * @head: the head that will follow our entry + */ +static inline void list_move_tail(struct list_head *list, + struct list_head *head) +{ + __list_del_entry(list); + list_add_tail(list, head); +} + +/** + * list_is_last - tests whether @list is the last entry in list @head + * @list: the entry to test + * @head: the head of the list + */ +static inline int list_is_last(const struct list_head *list, + const struct list_head *head) +{ + return list->next == head; +} + +/** + * list_empty - tests whether a list is empty + * @head: the list to test. + */ +static inline int list_empty(const struct list_head *head) +{ + return head->next == head; +} + +/** + * list_empty_careful - tests whether a list is empty and not being modified + * @head: the list to test + * + * Description: + * tests whether a list is empty _and_ checks that no other CPU might be + * in the process of modifying either member (next or prev) + * + * NOTE: using list_empty_careful() without synchronization + * can only be safe if the only activity that can happen + * to the list entry is list_del_init(). Eg. it cannot be used + * if another CPU could re-list_add() it. + */ +static inline int list_empty_careful(const struct list_head *head) +{ + struct list_head *next = head->next; + return (next == head) && (next == head->prev); +} + +/** + * list_rotate_left - rotate the list to the left + * @head: the head of the list + */ +static inline void list_rotate_left(struct list_head *head) +{ + struct list_head *first; + + if (!list_empty(head)) { + first = head->next; + list_move_tail(first, head); + } +} + +/** + * list_is_singular - tests whether a list has just one entry. + * @head: the list to test. + */ +static inline int list_is_singular(const struct list_head *head) +{ + return !list_empty(head) && (head->next == head->prev); +} + +static inline void __list_cut_position(struct list_head *list, + struct list_head *head, struct list_head *entry) +{ + struct list_head *new_first = entry->next; + list->next = head->next; + list->next->prev = list; + list->prev = entry; + entry->next = list; + head->next = new_first; + new_first->prev = head; +} + +/** + * list_cut_position - cut a list into two + * @list: a new list to add all removed entries + * @head: a list with entries + * @entry: an entry within head, could be the head itself + * and if so we won't cut the list + * + * This helper moves the initial part of @head, up to and + * including @entry, from @head to @list. You should + * pass on @entry an element you know is on @head. @list + * should be an empty list or a list you do not care about + * losing its data. + * + */ +static inline void list_cut_position(struct list_head *list, + struct list_head *head, struct list_head *entry) +{ + if (list_empty(head)) + return; + if (list_is_singular(head) && + (head->next != entry && head != entry)) + return; + if (entry == head) + INIT_LIST_HEAD(list); + else + __list_cut_position(list, head, entry); +} + +static inline void __list_splice(const struct list_head *list, + struct list_head *prev, + struct list_head *next) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + + first->prev = prev; + prev->next = first; + + last->next = next; + next->prev = last; +} + +/** + * list_splice - join two lists, this is designed for stacks + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice(const struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) + __list_splice(list, head, head->next); +} + +/** + * list_splice_tail - join two lists, each list being a queue + * @list: the new list to add. + * @head: the place to add it in the first list. + */ +static inline void list_splice_tail(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) + __list_splice(list, head->prev, head); +} + +/** + * list_splice_init - join two lists and reinitialise the emptied list. + * @list: the new list to add. + * @head: the place to add it in the first list. + * + * The list at @list is reinitialised + */ +static inline void list_splice_init(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) { + __list_splice(list, head, head->next); + INIT_LIST_HEAD(list); + } +} + +/** + * list_splice_tail_init - join two lists and reinitialise the emptied list + * @list: the new list to add. + * @head: the place to add it in the first list. + * + * Each of the lists is a queue. + * The list at @list is reinitialised + */ +static inline void list_splice_tail_init(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) { + __list_splice(list, head->prev, head); + INIT_LIST_HEAD(list); + } +} + +/** + * list_entry - get the struct for this entry + * @ptr: the &struct list_head pointer. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_struct within the struct. + */ +#define list_entry(ptr, type, member) \ + container_of(ptr, type, member) + +/** + * list_first_entry - get the first element from a list + * @ptr: the list head to take the element from. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_struct within the struct. + * + * Note, that list is expected to be not empty. + */ +#define list_first_entry(ptr, type, member) \ + list_entry((ptr)->next, type, member) + +/** + * list_for_each - iterate over a list + * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + */ +#define list_for_each(pos, head) \ + for (pos = (head)->next; pos != (head); pos = pos->next) + +/** + * __list_for_each - iterate over a list + * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + * + * This variant doesn't differ from list_for_each() any more. + * We don't do prefetching in either case. + */ +#define __list_for_each(pos, head) \ + for (pos = (head)->next; pos != (head); pos = pos->next) + +/** + * list_for_each_prev - iterate over a list backwards + * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + */ +#define list_for_each_prev(pos, head) \ + for (pos = (head)->prev; pos != (head); pos = pos->prev) + +/** + * list_for_each_safe - iterate over a list safe against removal of list entry + * @pos: the &struct list_head to use as a loop cursor. + * @n: another &struct list_head to use as temporary storage + * @head: the head for your list. + */ +#define list_for_each_safe(pos, n, head) \ + for (pos = (head)->next, n = pos->next; pos != (head); \ + pos = n, n = pos->next) + +/** + * list_for_each_prev_safe - iterate over a list backwards safe against removal of list entry + * @pos: the &struct list_head to use as a loop cursor. + * @n: another &struct list_head to use as temporary storage + * @head: the head for your list. + */ +#define list_for_each_prev_safe(pos, n, head) \ + for (pos = (head)->prev, n = pos->prev; \ + pos != (head); \ + pos = n, n = pos->prev) + +/** + * list_for_each_entry - iterate over list of given type + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry(pos, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_reverse - iterate backwards over list of given type. + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry_reverse(pos, head, member) \ + for (pos = list_entry((head)->prev, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.prev, typeof(*pos), member)) + +/** + * list_prepare_entry - prepare a pos entry for use in list_for_each_entry_continue() + * @pos: the type * to use as a start point + * @head: the head of the list + * @member: the name of the list_struct within the struct. + * + * Prepares a pos entry for use as a start point in list_for_each_entry_continue(). + */ +#define list_prepare_entry(pos, head, member) \ + ((pos) ? : list_entry(head, typeof(*pos), member)) + +/** + * list_for_each_entry_continue - continue iteration over list of given type + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Continue to iterate over list of given type, continuing after + * the current position. + */ +#define list_for_each_entry_continue(pos, head, member) \ + for (pos = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_continue_reverse - iterate backwards from the given point + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Start to iterate over list of given type backwards, continuing after + * the current position. + */ +#define list_for_each_entry_continue_reverse(pos, head, member) \ + for (pos = list_entry(pos->member.prev, typeof(*pos), member); \ + &pos->member != (head); \ + pos = list_entry(pos->member.prev, typeof(*pos), member)) + +/** + * list_for_each_entry_from - iterate over list of given type from the current point + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate over list of given type, continuing from current position. + */ +#define list_for_each_entry_from(pos, head, member) \ + for (; &pos->member != (head); \ + pos = list_entry(pos->member.next, typeof(*pos), member)) + +/** + * list_for_each_entry_safe - iterate over list of given type safe against removal of list entry + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + */ +#define list_for_each_entry_safe(pos, n, head, member) \ + for (pos = list_entry((head)->next, typeof(*pos), member), \ + n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +/** + * list_for_each_entry_safe_continue - continue list iteration safe against removal + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate over list of given type, continuing after current point, + * safe against removal of list entry. + */ +#define list_for_each_entry_safe_continue(pos, n, head, member) \ + for (pos = list_entry(pos->member.next, typeof(*pos), member), \ + n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +/** + * list_for_each_entry_safe_from - iterate over list from current point safe against removal + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate over list of given type from current point, safe against + * removal of list entry. + */ +#define list_for_each_entry_safe_from(pos, n, head, member) \ + for (n = list_entry(pos->member.next, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.next, typeof(*n), member)) + +/** + * list_for_each_entry_safe_reverse - iterate backwards over list safe against removal + * @pos: the type * to use as a loop cursor. + * @n: another type * to use as temporary storage + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * Iterate backwards over list of given type, safe against removal + * of list entry. + */ +#define list_for_each_entry_safe_reverse(pos, n, head, member) \ + for (pos = list_entry((head)->prev, typeof(*pos), member), \ + n = list_entry(pos->member.prev, typeof(*pos), member); \ + &pos->member != (head); \ + pos = n, n = list_entry(n->member.prev, typeof(*n), member)) + +/** + * list_safe_reset_next - reset a stale list_for_each_entry_safe loop + * @pos: the loop cursor used in the list_for_each_entry_safe loop + * @n: temporary storage used in list_for_each_entry_safe + * @member: the name of the list_struct within the struct. + * + * list_safe_reset_next is not safe to use in general if the list may be + * modified concurrently (eg. the lock is dropped in the loop body). An + * exception to this is if the cursor element (pos) is pinned in the list, + * and list_safe_reset_next is called after re-taking the lock and before + * completing the current iteration of the loop body. + */ +#define list_safe_reset_next(pos, n, member) \ + n = list_entry(pos->member.next, typeof(*pos), member) + +/* + * Double linked lists with a single pointer list head. + * Mostly useful for hash tables where the two pointer list head is + * too wasteful. + * You lose the ability to access the tail in O(1). + */ + +#endif diff --git a/lib/include/memory.h b/lib/include/memory.h new file mode 100644 index 00000000..b1b24504 --- /dev/null +++ b/lib/include/memory.h @@ -0,0 +1,23 @@ +#ifndef __HEADER_GENERIC_MEMORY_H +#define __HEADER_GENERIC_MEMORY_H + +#include + +#ifndef KERNEL_PHYS_OFFSET +#define KERNEL_PHYS_OFFSET 0 + +static unsigned long virt_to_phys(void *v) +{ + return (unsigned long)v - KERNEL_PHYS_OFFSET; +} +static void *phys_to_virt(unsigned long p) +{ + return (void *)(p + KERNEL_PHYS_OFFSET); +} +#endif + +unsigned long virt_to_phys(void *v); +void *phys_to_virt(unsigned long p); + +#endif + diff --git a/lib/include/string.h b/lib/include/string.h new file mode 100644 index 00000000..d0aadb5e --- /dev/null +++ b/lib/include/string.h @@ -0,0 +1,20 @@ +#ifndef __STRING_H +#define __STRING_H + +#include + +size_t strlen(const char *p); +size_t strnlen(const char *p, size_t maxlen); +char *strcpy(char *dest, const char *src); +char *strncpy(char *dest, const char *src, size_t maxlen); +int strcmp(const char *s1, const char *s2); +int strncmp(const char *s1, const char *s2, size_t n); +char *strstr(const char *haystack, const char *needle); +void *memcpy(void *dest, const void *src, size_t n); +void *memcpy_long(void *dest, const void *src, size_t n); +int memcmp(const void *s1, const void *s2, size_t n); +void *memset(void *s, int n, size_t l); + +unsigned long strtol(const char *cp, char **endp, unsigned int base); + +#endif diff --git a/lib/include/types.h b/lib/include/types.h new file mode 100644 index 00000000..3690dd8c --- /dev/null +++ b/lib/include/types.h @@ -0,0 +1,7 @@ +#ifndef TYPES_H +#define TYPES_H + +#include + +#endif + diff --git a/lib/page_alloc.c b/lib/page_alloc.c new file mode 100644 index 00000000..7a5358ce --- /dev/null +++ b/lib/page_alloc.c @@ -0,0 +1,253 @@ +/* + * AAL - Generic page allocator (manycore version) + * (C) Copyright 2011 Taku Shimosawa. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +void *allocate_pages(int npages, enum aal_mc_ap_flag flag); +void free_pages(void *, int npages); + +#define MAP_INDEX(n) ((n) >> 6) +#define MAP_BIT(n) ((n) & 0x3f) +#define ADDRESS(desc, index, bit) \ + ((desc)->start + (((index) * 64 + (bit)) << ((desc)->shift))) + +void *__aal_pagealloc_init(unsigned long start, unsigned long size, + unsigned long unit, void *initial, + unsigned long *pdescsize) +{ + /* Unit must be power of 2, and size and start must be unit-aligned */ + struct aal_page_allocator_desc *desc; + int i, page_shift, descsize, mapsize, mapaligned; + int flag = 0; + + if (!unit) { + return NULL; + } + page_shift = fls(unit) - 1; + + /* round up to 64-bit */ + mapsize = (size >> page_shift); + mapaligned = ((mapsize + 63) >> 6) << 3; + descsize = sizeof(*desc) + mapaligned; + + descsize = (descsize + PAGE_SIZE - 1) >> PAGE_SHIFT; + + if (initial) { + desc = initial; + *pdescsize = descsize; + } else { + desc = (void *)allocate_pages(descsize, 0); + } + flag = descsize; + memset(desc, 0, descsize * PAGE_SIZE); + + if (!desc) { + kprintf("AAL: failed to allocate page-allocator-desc "\ + "(%lx, %lx, %lx)\n", start, size, unit); + return NULL; + } + + desc->start = start; + desc->last = 0; + desc->count = mapaligned >> 3; + desc->shift = page_shift; + desc->flag = flag; + + kprintf("Page allocator: %lx - %lx (%d)\n", start, start + size, + page_shift); + + aal_mc_spinlock_init(&desc->lock); + + /* Reserve align padding area */ + for (i = mapsize; i < mapaligned * 8; i++) { + desc->map[MAP_INDEX(i)] |= (1UL << MAP_BIT(i)); + } + + return desc; +} + +void *aal_pagealloc_init(unsigned long start, unsigned long size, + unsigned long unit) +{ + return __aal_pagealloc_init(start, size, unit, NULL, NULL); +} + +void aal_pagealloc_destroy(void *__desc) +{ + struct aal_page_allocator_desc *desc = __desc; + + free_pages(desc, desc->flag); +} + +static unsigned long __aal_pagealloc_large(struct aal_page_allocator_desc *desc, + int nblocks) +{ + unsigned long flags; + unsigned int i, j, mi; + + flags = aal_mc_spinlock_lock(&desc->lock); + for (i = 0, mi = desc->last; i < desc->count; i++, mi++) { + if (mi >= desc->count) { + mi = 0; + } + if (mi + nblocks >= desc->count) { + continue; + } + for (j = mi; j < mi + nblocks; j++) { + if (desc->map[j]) { + break; + } + } + if (j == mi + nblocks) { + for (j = mi; j < mi + nblocks; j++) { + desc->map[j] = (unsigned long)-1; + } + aal_mc_spinlock_unlock(&desc->lock, flags); + return ADDRESS(desc, mi, 0); + } + } + aal_mc_spinlock_unlock(&desc->lock, flags); + + return 0; +} + +unsigned long aal_pagealloc_alloc(void *__desc, int npages) +{ + struct aal_page_allocator_desc *desc = __desc; + unsigned int i, mi; + int j; + unsigned long v, mask, flags; + + /* If requested page is more than the half of the element, + * we allocate the whole element (ulong) */ + if (npages >= 32) { + return __aal_pagealloc_large(desc, (npages + 63) >> 6); + } + + mask = (1UL << npages) - 1; + + flags = aal_mc_spinlock_lock(&desc->lock); + for (i = 0, mi = desc->last; i < desc->count; i++, mi++) { + if (mi >= desc->count) { + mi = 0; + } + + v = desc->map[mi]; + if (v == (unsigned long)-1) + continue; + + for (j = 0; j <= 64 - npages; j++) { + if (!(v & (mask << j))) { /* free */ + desc->map[mi] |= (mask << j); + + aal_mc_spinlock_unlock(&desc->lock, flags); + return ADDRESS(desc, mi, j); + } + } + } + aal_mc_spinlock_unlock(&desc->lock, flags); + + /* We use null pointer for failure */ + return 0; +} + +void aal_pagealloc_reserve(void *__desc, unsigned long start, unsigned long end) +{ + int i, n; + struct aal_page_allocator_desc *desc = __desc; + unsigned long flags; + + n = (end + (1 << desc->shift) - 1 - desc->start) >> desc->shift; + i = ((start - desc->start) >> desc->shift); + if (i < 0 || n < 0) { + return; + } + + flags = aal_mc_spinlock_lock(&desc->lock); + for (; i < n; i++) { + if (!(i & 63) && i + 63 < n) { + desc->map[MAP_INDEX(i)] = (unsigned long)-1L; + i += 63; + } else { + desc->map[MAP_INDEX(i)] |= (1UL << MAP_BIT(i)); + } + } + aal_mc_spinlock_unlock(&desc->lock, flags); +} + +void aal_pagealloc_free(void *__desc, unsigned long address, int npages) +{ + struct aal_page_allocator_desc *desc = __desc; + int i; + unsigned mi; + unsigned long flags; + + /* XXX: Parameter check */ + if (npages >= 32) { + npages = (npages + 63) & ~63; + } + flags = aal_mc_spinlock_lock(&desc->lock); + mi = (address - desc->start) >> desc->shift; + for (i = 0; i < npages; i++, mi++) { + desc->map[MAP_INDEX(mi)] &= ~(1UL << MAP_BIT(mi)); + } + aal_mc_spinlock_unlock(&desc->lock, flags); +} + +unsigned long aal_pagealloc_count(void *__desc) +{ + struct aal_page_allocator_desc *desc = __desc; + unsigned long i, j, n = 0; + unsigned long flags; + + flags = aal_mc_spinlock_lock(&desc->lock); + /* XXX: Very silly counting */ + for (i = 0; i < desc->count; i++) { + for (j = 0; j < 64; j++) { + if (!(desc->map[i] & (1UL << j))) { + n++; + } + } + } + aal_mc_spinlock_unlock(&desc->lock, flags); + + return n; +} + +void __aal_pagealloc_zero_free_pages(void *__desc) +{ + struct aal_page_allocator_desc *desc = __desc; + unsigned int mi; + int j; + unsigned long v, flags; + +kprintf("zeroing free memory... "); + + flags = aal_mc_spinlock_lock(&desc->lock); + for (mi = 0; mi < desc->count; mi++) { + + v = desc->map[mi]; + if (v == (unsigned long)-1) + continue; + + for (j = 0; j < 64; j++) { + if (!(v & ((unsigned long)1 << j))) { /* free */ + + memset(phys_to_virt(ADDRESS(desc, mi, j)), 0, PAGE_SIZE); + } + } + } + aal_mc_spinlock_unlock(&desc->lock, flags); + +kprintf("\nzeroing done\n"); +} + + diff --git a/lib/string.c b/lib/string.c new file mode 100644 index 00000000..9123f30a --- /dev/null +++ b/lib/string.c @@ -0,0 +1,158 @@ +#include + +size_t strlen(const char *p) +{ + const char *head = p; + + while(*p){ + p++; + } + + return p - head; +} + +size_t strnlen(const char *p, size_t maxlen) +{ + const char *head = p; + + while(*p && maxlen > 0){ + p++; + maxlen--; + } + + return p - head; +} + +char *strcpy(char *dest, const char *src) +{ + char *head = dest; + + while((*(dest++) = *(src++))); + + return head; +} + +char *strncpy(char *dest, const char *src, size_t maxlen) +{ + char *head = dest; + ssize_t len = maxlen; + + while((*(dest++) = *(src++)) && --len); + if(len > 0){ + while(len--){ + *(dest++) = '\0'; + } + } + + return head; +} + +int strcmp(const char *s1, const char *s2) +{ + while(*s1 && *s1 == *s2){ + s1++; + s2++; + } + + return *s1 - *s2; +} + +int strncmp(const char *s1, const char *s2, size_t n) +{ + while(*s1 && *s1 == *s2 && n > 1){ + s1++; + s2++; + n--; + } + return *s1 - *s2; +} + +char *strstr(const char *haystack, const char *needle) +{ + int len = strlen(needle); + + while(*haystack){ + if(!strncmp(haystack, needle, len)){ + return (char *)haystack; + } + haystack++; + } + return NULL; +} + +void *memcpy(void *dest, const void *src, size_t n) +{ + const char *p1 = src; + char *p2 = dest; + + while(n > 0){ + *p2 = *p1; + p1++; + p2++; + n--; + } + + return dest; +} + +void *memcpy_long(void *dest, const void *src, size_t n) +{ + const unsigned long *p1 = src; + unsigned long *p2 = dest; + + n /= sizeof(unsigned long); + while (n > 0) { + *(p2++) = *(p1++); + n--; + } + + return dest; +} + +void *memset(void *s, int c, size_t n) +{ + char *s_aligned = (void *)(((unsigned long)s + 7) & ~7); + char *e_aligned = (void *)(((unsigned long)s + n) & ~7); + char *e = ((char *)s + n); + char *p; + unsigned long *l; +#define C ((unsigned long)(c & 0xff)) + unsigned long pat = C | C << 8 | C << 16 | C << 24 | C << 32 | + C << 40 | C << 48 | C << 56; +#undef C + + if(s_aligned < e_aligned){ + p = s; + while(p < s_aligned){ + *(p++) = (char)c; + } + l = (unsigned long *)s_aligned; + while((char *)l < e_aligned){ + *(l++) = pat; + } + p = e_aligned; + while(p < e){ + *(p++) = (char)c; + } + }else{ + p = s; + while(p < e){ + *(p++) = (char)c; + } + } + + return s; +} + +int memcmp(const void *s1, const void *s2, size_t n) +{ + const char *p1 = s1; + const char *p2 = s2; + + while(*p1 == *p2 && n > 1){ + p1++; + p2++; + n--; + } + return *p1 - *p2; +} diff --git a/lib/vsprintf.c b/lib/vsprintf.c new file mode 100644 index 00000000..7846226a --- /dev/null +++ b/lib/vsprintf.c @@ -0,0 +1,1440 @@ +/* + * linux/lib/vsprintf.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +/* vsprintf.c -- Lars Wirzenius & Linus Torvalds. */ +/* + * Wirzenius wrote this portably, Torvalds fucked it up :-) + */ + +/* + * Fri Jul 13 2001 Crutcher Dunnavant + * - changed to provide snprintf and vsnprintf functions + * So Feb 1 16:51:32 CET 2004 Juergen Quade + * - scnprintf and vscnprintf + */ + +#include +#include +#include +#include +#include +#include + +unsigned char _ctype[] = { + _C,_C,_C,_C,_C,_C,_C,_C, /* 0-7 */ + _C,_C|_S,_C|_S,_C|_S,_C|_S,_C|_S,_C,_C, /* 8-15 */ + _C,_C,_C,_C,_C,_C,_C,_C, /* 16-23 */ + _C,_C,_C,_C,_C,_C,_C,_C, /* 24-31 */ + _S|_SP,_P,_P,_P,_P,_P,_P,_P, /* 32-39 */ + _P,_P,_P,_P,_P,_P,_P,_P, /* 40-47 */ + _D,_D,_D,_D,_D,_D,_D,_D, /* 48-55 */ + _D,_D,_P,_P,_P,_P,_P,_P, /* 56-63 */ + _P,_U|_X,_U|_X,_U|_X,_U|_X,_U|_X,_U|_X,_U, /* 64-71 */ + _U,_U,_U,_U,_U,_U,_U,_U, /* 72-79 */ + _U,_U,_U,_U,_U,_U,_U,_U, /* 80-87 */ + _U,_U,_U,_P,_P,_P,_P,_P, /* 88-95 */ + _P,_L|_X,_L|_X,_L|_X,_L|_X,_L|_X,_L|_X,_L, /* 96-103 */ + _L,_L,_L,_L,_L,_L,_L,_L, /* 104-111 */ + _L,_L,_L,_L,_L,_L,_L,_L, /* 112-119 */ + _L,_L,_L,_P,_P,_P,_P,_C, /* 120-127 */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 128-143 */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* 144-159 */ + _S|_SP,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P, /* 160-175 */ + _P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P,_P, /* 176-191 */ + _U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U,_U, /* 192-207 */ + _U,_U,_U,_U,_U,_U,_U,_P,_U,_U,_U,_U,_U,_U,_U,_L, /* 208-223 */ + _L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L,_L, /* 224-239 */ + _L,_L,_L,_L,_L,_L,_L,_P,_L,_L,_L,_L,_L,_L,_L,_L}; /* 240-255 */ + + +#define do_div(n,base) ({ \ + uint32_t __base = (base); \ + uint32_t __rem; \ + __rem = ((uint64_t)(n)) % __base; \ + (n) = ((uint64_t)(n)) / __base; \ + __rem; \ + }) + +#define EXPORT_SYMBOL(x) +#define unlikely(c) c +#define WARN_ON(c) + +/* Works only for digits and letters, but small and fast */ +#define TOLOWER(x) ((x) | 0x20) + +static unsigned int simple_guess_base(const char *cp) +{ + if (cp[0] == '0') { + if (TOLOWER(cp[1]) == 'x' && isxdigit(cp[2])) + return 16; + else + return 8; + } else { + return 10; + } +} + +/** + * simple_strtoul - convert a string to an unsigned long + * @cp: The start of the string + * @endp: A pointer to the end of the parsed string will be placed here + * @base: The number base to use + */ +unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base) +{ + unsigned long result = 0; + + if (!base) + base = simple_guess_base(cp); + + if (base == 16 && cp[0] == '0' && TOLOWER(cp[1]) == 'x') + cp += 2; + + while (isxdigit(*cp)) { + unsigned int value; + + value = isdigit(*cp) ? *cp - '0' : TOLOWER(*cp) - 'a' + 10; + if (value >= base) + break; + result = result * base + value; + cp++; + } + + if (endp) + *endp = (char *)cp; + return result; +} +EXPORT_SYMBOL(simple_strtoul); + +/** + * simple_strtol - convert a string to a signed long + * @cp: The start of the string + * @endp: A pointer to the end of the parsed string will be placed here + * @base: The number base to use + */ +long simple_strtol(const char *cp, char **endp, unsigned int base) +{ + if(*cp == '-') + return -simple_strtoul(cp + 1, endp, base); + return simple_strtoul(cp, endp, base); +} +EXPORT_SYMBOL(simple_strtol); + +/** + * simple_strtoull - convert a string to an unsigned long long + * @cp: The start of the string + * @endp: A pointer to the end of the parsed string will be placed here + * @base: The number base to use + */ +unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base) +{ + unsigned long long result = 0; + + if (!base) + base = simple_guess_base(cp); + + if (base == 16 && cp[0] == '0' && TOLOWER(cp[1]) == 'x') + cp += 2; + + while (isxdigit(*cp)) { + unsigned int value; + + value = isdigit(*cp) ? *cp - '0' : TOLOWER(*cp) - 'a' + 10; + if (value >= base) + break; + result = result * base + value; + cp++; + } + + if (endp) + *endp = (char *)cp; + return result; +} +EXPORT_SYMBOL(simple_strtoull); + +/** + * simple_strtoll - convert a string to a signed long long + * @cp: The start of the string + * @endp: A pointer to the end of the parsed string will be placed here + * @base: The number base to use + */ +long long simple_strtoll(const char *cp, char **endp, unsigned int base) +{ + if(*cp=='-') + return -simple_strtoull(cp + 1, endp, base); + return simple_strtoull(cp, endp, base); +} + +/** + * strict_strtoul - convert a string to an unsigned long strictly + * @cp: The string to be converted + * @base: The number base to use + * @res: The converted result value + * + * strict_strtoul converts a string to an unsigned long only if the + * string is really an unsigned long string, any string containing + * any invalid char at the tail will be rejected and -EINVAL is returned, + * only a newline char at the tail is acceptible because people generally + * change a module parameter in the following way: + * + * echo 1024 > /sys/module/e1000/parameters/copybreak + * + * echo will append a newline to the tail. + * + * It returns 0 if conversion is successful and *res is set to the converted + * value, otherwise it returns -EINVAL and *res is set to 0. + * + * simple_strtoul just ignores the successive invalid characters and + * return the converted value of prefix part of the string. + */ +int strict_strtoul(const char *cp, unsigned int base, unsigned long *res) +{ + char *tail; + unsigned long val; + size_t len; + + *res = 0; + len = strlen(cp); + if (len == 0) + return -EINVAL; + + val = simple_strtoul(cp, &tail, base); + if (tail == cp) + return -EINVAL; + if ((*tail == '\0') || + ((len == (size_t)(tail - cp) + 1) && (*tail == '\n'))) { + *res = val; + return 0; + } + + return -EINVAL; +} +EXPORT_SYMBOL(strict_strtoul); + +/** + * strict_strtol - convert a string to a long strictly + * @cp: The string to be converted + * @base: The number base to use + * @res: The converted result value + * + * strict_strtol is similiar to strict_strtoul, but it allows the first + * character of a string is '-'. + * + * It returns 0 if conversion is successful and *res is set to the converted + * value, otherwise it returns -EINVAL and *res is set to 0. + */ +int strict_strtol(const char *cp, unsigned int base, long *res) +{ + int ret; + if (*cp == '-') { + ret = strict_strtoul(cp + 1, base, (unsigned long *)res); + if (!ret) + *res = -(*res); + } else { + ret = strict_strtoul(cp, base, (unsigned long *)res); + } + + return ret; +} +EXPORT_SYMBOL(strict_strtol); + +/** + * strict_strtoull - convert a string to an unsigned long long strictly + * @cp: The string to be converted + * @base: The number base to use + * @res: The converted result value + * + * strict_strtoull converts a string to an unsigned long long only if the + * string is really an unsigned long long string, any string containing + * any invalid char at the tail will be rejected and -EINVAL is returned, + * only a newline char at the tail is acceptible because people generally + * change a module parameter in the following way: + * + * echo 1024 > /sys/module/e1000/parameters/copybreak + * + * echo will append a newline to the tail of the string. + * + * It returns 0 if conversion is successful and *res is set to the converted + * value, otherwise it returns -EINVAL and *res is set to 0. + * + * simple_strtoull just ignores the successive invalid characters and + * return the converted value of prefix part of the string. + */ +int strict_strtoull(const char *cp, unsigned int base, unsigned long long *res) +{ + char *tail; + unsigned long long val; + size_t len; + + *res = 0; + len = strlen(cp); + if (len == 0) + return -EINVAL; + + val = simple_strtoull(cp, &tail, base); + if (tail == cp) + return -EINVAL; + if ((*tail == '\0') || + ((len == (size_t)(tail - cp) + 1) && (*tail == '\n'))) { + *res = val; + return 0; + } + + return -EINVAL; +} +EXPORT_SYMBOL(strict_strtoull); + +/** + * strict_strtoll - convert a string to a long long strictly + * @cp: The string to be converted + * @base: The number base to use + * @res: The converted result value + * + * strict_strtoll is similiar to strict_strtoull, but it allows the first + * character of a string is '-'. + * + * It returns 0 if conversion is successful and *res is set to the converted + * value, otherwise it returns -EINVAL and *res is set to 0. + */ +int strict_strtoll(const char *cp, unsigned int base, long long *res) +{ + int ret; + if (*cp == '-') { + ret = strict_strtoull(cp + 1, base, (unsigned long long *)res); + if (!ret) + *res = -(*res); + } else { + ret = strict_strtoull(cp, base, (unsigned long long *)res); + } + + return ret; +} +EXPORT_SYMBOL(strict_strtoll); + +static int skip_atoi(const char **s) +{ + int i=0; + + while (isdigit(**s)) + i = i*10 + *((*s)++) - '0'; + return i; +} + +/* Decimal conversion is by far the most typical, and is used + * for /proc and /sys data. This directly impacts e.g. top performance + * with many processes running. We optimize it for speed + * using code from + * http://www.cs.uiowa.edu/~jones/bcd/decimal.html + * (with permission from the author, Douglas W. Jones). */ + +/* Formats correctly any integer in [0,99999]. + * Outputs from one to five digits depending on input. + * On i386 gcc 4.1.2 -O2: ~250 bytes of code. */ +static char* put_dec_trunc(char *buf, unsigned q) +{ + unsigned d3, d2, d1, d0; + d1 = (q>>4) & 0xf; + d2 = (q>>8) & 0xf; + d3 = (q>>12); + + d0 = 6*(d3 + d2 + d1) + (q & 0xf); + q = (d0 * 0xcd) >> 11; + d0 = d0 - 10*q; + *buf++ = d0 + '0'; /* least significant digit */ + d1 = q + 9*d3 + 5*d2 + d1; + if (d1 != 0) { + q = (d1 * 0xcd) >> 11; + d1 = d1 - 10*q; + *buf++ = d1 + '0'; /* next digit */ + + d2 = q + 2*d2; + if ((d2 != 0) || (d3 != 0)) { + q = (d2 * 0xd) >> 7; + d2 = d2 - 10*q; + *buf++ = d2 + '0'; /* next digit */ + + d3 = q + 4*d3; + if (d3 != 0) { + q = (d3 * 0xcd) >> 11; + d3 = d3 - 10*q; + *buf++ = d3 + '0'; /* next digit */ + if (q != 0) + *buf++ = q + '0'; /* most sign. digit */ + } + } + } + return buf; +} +/* Same with if's removed. Always emits five digits */ +static char* put_dec_full(char *buf, unsigned q) +{ + /* BTW, if q is in [0,9999], 8-bit ints will be enough, */ + /* but anyway, gcc produces better code with full-sized ints */ + unsigned d3, d2, d1, d0; + d1 = (q>>4) & 0xf; + d2 = (q>>8) & 0xf; + d3 = (q>>12); + + /* Possible ways to approx. divide by 10 */ + /* gcc -O2 replaces multiply with shifts and adds */ + // (x * 0xcd) >> 11: 11001101 - shorter code than * 0x67 (on i386) + // (x * 0x67) >> 10: 1100111 + // (x * 0x34) >> 9: 110100 - same + // (x * 0x1a) >> 8: 11010 - same + // (x * 0x0d) >> 7: 1101 - same, shortest code (on i386) + + d0 = 6*(d3 + d2 + d1) + (q & 0xf); + q = (d0 * 0xcd) >> 11; + d0 = d0 - 10*q; + *buf++ = d0 + '0'; + d1 = q + 9*d3 + 5*d2 + d1; + q = (d1 * 0xcd) >> 11; + d1 = d1 - 10*q; + *buf++ = d1 + '0'; + + d2 = q + 2*d2; + q = (d2 * 0xd) >> 7; + d2 = d2 - 10*q; + *buf++ = d2 + '0'; + + d3 = q + 4*d3; + q = (d3 * 0xcd) >> 11; /* - shorter code */ + /* q = (d3 * 0x67) >> 10; - would also work */ + d3 = d3 - 10*q; + *buf++ = d3 + '0'; + *buf++ = q + '0'; + return buf; +} +/* No inlining helps gcc to use registers better */ +static char* __attribute__((noinline)) + put_dec(char *buf, unsigned long long num) +{ + while (1) { + unsigned rem; + if (num < 100000) + return put_dec_trunc(buf, num); + rem = do_div(num, 100000); + buf = put_dec_full(buf, rem); + } +} + +#define ZEROPAD 1 /* pad with zero */ +#define SIGN 2 /* unsigned/signed long */ +#define PLUS 4 /* show plus */ +#define SPACE 8 /* space if plus */ +#define LEFT 16 /* left justified */ +#define SMALL 32 /* Must be 32 == 0x20 */ +#define SPECIAL 64 /* 0x */ + +enum format_type { + FORMAT_TYPE_NONE, /* Just a string part */ + FORMAT_TYPE_WIDTH, + FORMAT_TYPE_PRECISION, + FORMAT_TYPE_CHAR, + FORMAT_TYPE_STR, + FORMAT_TYPE_PTR, + FORMAT_TYPE_PERCENT_CHAR, + FORMAT_TYPE_INVALID, + FORMAT_TYPE_LONG_LONG, + FORMAT_TYPE_ULONG, + FORMAT_TYPE_LONG, + FORMAT_TYPE_UBYTE, + FORMAT_TYPE_BYTE, + FORMAT_TYPE_USHORT, + FORMAT_TYPE_SHORT, + FORMAT_TYPE_UINT, + FORMAT_TYPE_INT, + FORMAT_TYPE_NRCHARS, + FORMAT_TYPE_SIZE_T, + FORMAT_TYPE_PTRDIFF, + FORMAT_TYPE_FLOAT, +}; + +struct printf_spec { + enum format_type type; + int flags; /* flags to number() */ + int field_width; /* width of output field */ + int base; + int precision; /* # of digits/chars */ + int qualifier; +}; + +static char *number(char *buf, char *end, unsigned long long num, + struct printf_spec spec) +{ + /* we are called with base 8, 10 or 16, only, thus don't need "G..." */ + static const char digits[16] = "0123456789ABCDEF"; /* "GHIJKLMNOPQRSTUVWXYZ"; */ + + char tmp[66]; + char sign; + char locase; + int need_pfx = ((spec.flags & SPECIAL) && spec.base != 10); + int i; + + /* locase = 0 or 0x20. ORing digits or letters with 'locase' + * produces same digits or (maybe lowercased) letters */ + locase = (spec.flags & SMALL); + if (spec.flags & LEFT) + spec.flags &= ~ZEROPAD; + sign = 0; + if (spec.flags & SIGN) { + if ((signed long long) num < 0) { + sign = '-'; + num = - (signed long long) num; + spec.field_width--; + } else if (spec.flags & PLUS) { + sign = '+'; + spec.field_width--; + } else if (spec.flags & SPACE) { + sign = ' '; + spec.field_width--; + } + } + if (need_pfx) { + spec.field_width--; + if (spec.base == 16) + spec.field_width--; + } + + /* generate full string in tmp[], in reverse order */ + i = 0; + if (num == 0) + tmp[i++] = '0'; + /* Generic code, for any base: + else do { + tmp[i++] = (digits[do_div(num,base)] | locase); + } while (num != 0); + */ + else if (spec.base != 10) { /* 8 or 16 */ + int mask = spec.base - 1; + int shift = 3; + if (spec.base == 16) shift = 4; + do { + tmp[i++] = (digits[((unsigned char)num) & mask] | locase); + num >>= shift; + } while (num); + } else { /* base 10 */ + i = put_dec(tmp, num) - tmp; + } + + /* printing 100 using %2d gives "100", not "00" */ + if (i > spec.precision) + spec.precision = i; + /* leading space padding */ + spec.field_width -= spec.precision; + if (!(spec.flags & (ZEROPAD+LEFT))) { + while(--spec.field_width >= 0) { + if (buf < end) + *buf = ' '; + ++buf; + } + } + /* sign */ + if (sign) { + if (buf < end) + *buf = sign; + ++buf; + } + /* "0x" / "0" prefix */ + if (need_pfx) { + if (buf < end) + *buf = '0'; + ++buf; + if (spec.base == 16) { + if (buf < end) + *buf = ('X' | locase); + ++buf; + } + } + /* zero or space padding */ + if (!(spec.flags & LEFT)) { + char c = (spec.flags & ZEROPAD) ? '0' : ' '; + while (--spec.field_width >= 0) { + if (buf < end) + *buf = c; + ++buf; + } + } + /* hmm even more zero padding? */ + while (i <= --spec.precision) { + if (buf < end) + *buf = '0'; + ++buf; + } + /* actual digits of result */ + while (--i >= 0) { + if (buf < end) + *buf = tmp[i]; + ++buf; + } + /* trailing space padding */ + while (--spec.field_width >= 0) { + if (buf < end) + *buf = ' '; + ++buf; + } + return buf; +} + +static char *string(char *buf, char *end, char *s, struct printf_spec spec) +{ + int len, i; + + if ((unsigned long)s < 4096) + s = ""; + + len = strnlen(s, spec.precision); + + if (!(spec.flags & LEFT)) { + while (len < spec.field_width--) { + if (buf < end) + *buf = ' '; + ++buf; + } + } + for (i = 0; i < len; ++i) { + if (buf < end) + *buf = *s; + ++buf; ++s; + } + while (len < spec.field_width--) { + if (buf < end) + *buf = ' '; + ++buf; + } + return buf; +} + +static char *symbol_string(char *buf, char *end, void *ptr, + struct printf_spec spec, char ext) +{ + unsigned long value = (unsigned long) ptr; +#ifdef CONFIG_KALLSYMS + char sym[KSYM_SYMBOL_LEN]; + if (ext != 'f') + sprint_symbol(sym, value); + else + kallsyms_lookup(value, NULL, NULL, NULL, sym); + return string(buf, end, sym, spec); +#else + spec.field_width = 2*sizeof(void *); + spec.flags |= SPECIAL | SMALL | ZEROPAD; + spec.base = 16; + return number(buf, end, value, spec); +#endif +} + + +/* + * Show a '%p' thing. A kernel extension is that the '%p' is followed + * by an extra set of alphanumeric characters that are extended format + * specifiers. + * + * Right now we handle: + * + * - 'F' For symbolic function descriptor pointers with offset + * - 'f' For simple symbolic function names without offset + * - 'S' For symbolic direct pointers + * - 'R' For a struct resource pointer, it prints the range of + * addresses (not the name nor the flags) + * - 'M' For a 6-byte MAC address, it prints the address in the + * usual colon-separated hex notation + * - 'I' [46] for IPv4/IPv6 addresses printed in the usual way (dot-separated + * decimal for v4 and colon separated network-order 16 bit hex for v6) + * - 'i' [46] for 'raw' IPv4/IPv6 addresses, IPv6 omits the colons, IPv4 is + * currently the same + * + * Note: The difference between 'S' and 'F' is that on ia64 and ppc64 + * function pointers are really function descriptors, which contain a + * pointer to the real address. + */ +static char *pointer(const char *fmt, char *buf, char *end, void *ptr, + struct printf_spec spec) +{ + if (!ptr) + return string(buf, end, "(null)", spec); + + switch (*fmt) { + case 'S': + return symbol_string(buf, end, ptr, spec, *fmt); + } + spec.flags |= SMALL; + if (spec.field_width == -1) { + spec.field_width = 2*sizeof(void *); + spec.flags |= ZEROPAD; + } + spec.base = 16; + + return number(buf, end, (unsigned long) ptr, spec); +} + +/* + * Helper function to decode printf style format. + * Each call decode a token from the format and return the + * number of characters read (or likely the delta where it wants + * to go on the next call). + * The decoded token is returned through the parameters + * + * 'h', 'l', or 'L' for integer fields + * 'z' support added 23/7/1999 S.H. + * 'z' changed to 'Z' --davidm 1/25/99 + * 't' added for ptrdiff_t + * + * @fmt: the format string + * @type of the token returned + * @flags: various flags such as +, -, # tokens.. + * @field_width: overwritten width + * @base: base of the number (octal, hex, ...) + * @precision: precision of a number + * @qualifier: qualifier of a number (long, size_t, ...) + */ +static int format_decode(const char *fmt, struct printf_spec *spec) +{ + const char *start = fmt; + + /* we finished early by reading the field width */ + if (spec->type == FORMAT_TYPE_WIDTH) { + if (spec->field_width < 0) { + spec->field_width = -spec->field_width; + spec->flags |= LEFT; + } + spec->type = FORMAT_TYPE_NONE; + goto precision; + } + + /* we finished early by reading the precision */ + if (spec->type == FORMAT_TYPE_PRECISION) { + if (spec->precision < 0) + spec->precision = 0; + + spec->type = FORMAT_TYPE_NONE; + goto qualifier; + } + + /* By default */ + spec->type = FORMAT_TYPE_NONE; + + for (; *fmt ; ++fmt) { + if (*fmt == '%') + break; + } + + /* Return the current non-format string */ + if (fmt != start || !*fmt) + return fmt - start; + + /* Process flags */ + spec->flags = 0; + + while (1) { /* this also skips first '%' */ + char found = 1; + + ++fmt; + + switch (*fmt) { + case '-': spec->flags |= LEFT; break; + case '+': spec->flags |= PLUS; break; + case ' ': spec->flags |= SPACE; break; + case '#': spec->flags |= SPECIAL; break; + case '0': spec->flags |= ZEROPAD; break; + default: found = 0; + } + + if (!found) + break; + } + + /* get field width */ + spec->field_width = -1; + + if (isdigit(*fmt)) + spec->field_width = skip_atoi(&fmt); + else if (*fmt == '*') { + /* it's the next argument */ + spec->type = FORMAT_TYPE_WIDTH; + return ++fmt - start; + } + +precision: + /* get the precision */ + spec->precision = -1; + if (*fmt == '.') { + ++fmt; + if (isdigit(*fmt)) { + spec->precision = skip_atoi(&fmt); + if (spec->precision < 0) + spec->precision = 0; + } else if (*fmt == '*') { + /* it's the next argument */ + spec->type = FORMAT_TYPE_PRECISION; + return ++fmt - start; + } + } + +qualifier: + /* get the conversion qualifier */ + spec->qualifier = -1; + if (*fmt == 'h' || *fmt == 'l' || *fmt == 'L' || + *fmt == 'Z' || *fmt == 'z' || *fmt == 't') { + spec->qualifier = *fmt++; + if (unlikely(spec->qualifier == *fmt)) { + if (spec->qualifier == 'l') { + spec->qualifier = 'L'; + ++fmt; + } else if (spec->qualifier == 'h') { + spec->qualifier = 'H'; + ++fmt; + } + } + } + + /* default base */ + spec->base = 10; + switch (*fmt) { + case 'c': + spec->type = FORMAT_TYPE_CHAR; + return ++fmt - start; + + case 's': + spec->type = FORMAT_TYPE_STR; + return ++fmt - start; + + case 'f': + spec->base = 16; + spec->type = FORMAT_TYPE_FLOAT; + return ++fmt - start; + + case 'p': + spec->type = FORMAT_TYPE_PTR; + return fmt - start; + /* skip alnum */ + + case 'n': + spec->type = FORMAT_TYPE_NRCHARS; + return ++fmt - start; + + case '%': + spec->type = FORMAT_TYPE_PERCENT_CHAR; + return ++fmt - start; + + /* integer number formats - set up the flags and "break" */ + case 'o': + spec->base = 8; + break; + + case 'x': + spec->flags |= SMALL; + + case 'X': + spec->base = 16; + break; + + case 'd': + case 'i': + spec->flags |= SIGN; + case 'u': + break; + + default: + spec->type = FORMAT_TYPE_INVALID; + return fmt - start; + } + + if (spec->qualifier == 'L') + spec->type = FORMAT_TYPE_LONG_LONG; + else if (spec->qualifier == 'l') { + if (spec->flags & SIGN) + spec->type = FORMAT_TYPE_LONG; + else + spec->type = FORMAT_TYPE_ULONG; + } else if (spec->qualifier == 'Z' || spec->qualifier == 'z') { + spec->type = FORMAT_TYPE_SIZE_T; + } else if (spec->qualifier == 't') { + spec->type = FORMAT_TYPE_PTRDIFF; + } else if (spec->qualifier == 'H') { + if (spec->flags & SIGN) + spec->type = FORMAT_TYPE_BYTE; + else + spec->type = FORMAT_TYPE_UBYTE; + } else if (spec->qualifier == 'h') { + if (spec->flags & SIGN) + spec->type = FORMAT_TYPE_SHORT; + else + spec->type = FORMAT_TYPE_USHORT; + } else { + if (spec->flags & SIGN) + spec->type = FORMAT_TYPE_INT; + else + spec->type = FORMAT_TYPE_UINT; + } + + return ++fmt - start; +} + +/** + * vsnprintf - Format a string and place it in a buffer + * @buf: The buffer to place the result into + * @size: The size of the buffer, including the trailing null space + * @fmt: The format string to use + * @args: Arguments for the format string + * + * This function follows C99 vsnprintf, but has some extensions: + * %pS output the name of a text symbol + * %pF output the name of a function pointer with its offset + * %pf output the name of a function pointer without its offset + * %pR output the address range in a struct resource + * + * The return value is the number of characters which would + * be generated for the given input, excluding the trailing + * '\0', as per ISO C99. If you want to have the exact + * number of characters written into @buf as return value + * (not including the trailing '\0'), use vscnprintf(). If the + * return is greater than or equal to @size, the resulting + * string is truncated. + * + * Call this function if you are already dealing with a va_list. + * You probably want snprintf() instead. + */ +int vsnprintf(char *buf, size_t size, const char *fmt, va_list args) +{ + unsigned long long num; + char *str, *end, c; + int read; + struct printf_spec spec = {0}; + + /* Reject out-of-range values early. Large positive sizes are + used for unknown buffer sizes. */ + if (unlikely((int) size < 0)) { + /* There can be only one.. */ + return 0; + } + + str = buf; + end = buf + size; + + /* Make sure end is always >= buf */ + if (end < buf) { + end = ((void *)-1); + size = end - buf; + } + + while (*fmt) { + const char *old_fmt = fmt; + + read = format_decode(fmt, &spec); + + fmt += read; + + switch (spec.type) { + case FORMAT_TYPE_NONE: { + int copy = read; + if (str < end) { + if (copy > end - str) + copy = end - str; + memcpy(str, old_fmt, copy); + } + str += read; + break; + } + + case FORMAT_TYPE_WIDTH: + spec.field_width = va_arg(args, int); + break; + + case FORMAT_TYPE_PRECISION: + spec.precision = va_arg(args, int); + break; + + case FORMAT_TYPE_CHAR: + if (!(spec.flags & LEFT)) { + while (--spec.field_width > 0) { + if (str < end) + *str = ' '; + ++str; + + } + } + c = (unsigned char) va_arg(args, int); + if (str < end) + *str = c; + ++str; + while (--spec.field_width > 0) { + if (str < end) + *str = ' '; + ++str; + } + break; + + case FORMAT_TYPE_STR: + str = string(str, end, va_arg(args, char *), spec); + break; + + case FORMAT_TYPE_PTR: + str = pointer(fmt+1, str, end, va_arg(args, void *), + spec); + while (isalnum(*fmt)) + fmt++; + break; + + case FORMAT_TYPE_PERCENT_CHAR: + if (str < end) + *str = '%'; + ++str; + break; + + case FORMAT_TYPE_INVALID: + if (str < end) + *str = '%'; + ++str; + break; + + case FORMAT_TYPE_NRCHARS: { + int qualifier = spec.qualifier; + + if (qualifier == 'l') { + long *ip = va_arg(args, long *); + *ip = (str - buf); + } else if (qualifier == 'Z' || + qualifier == 'z') { + size_t *ip = va_arg(args, size_t *); + *ip = (str - buf); + } else { + int *ip = va_arg(args, int *); + *ip = (str - buf); + } + break; + } + + default: + switch (spec.type) { + case FORMAT_TYPE_LONG_LONG: + num = va_arg(args, long long); + break; + case FORMAT_TYPE_ULONG: + num = va_arg(args, unsigned long); + break; + case FORMAT_TYPE_LONG: + num = va_arg(args, long); + break; + case FORMAT_TYPE_SIZE_T: + num = va_arg(args, size_t); + break; + case FORMAT_TYPE_PTRDIFF: + num = va_arg(args, ptrdiff_t); + break; + case FORMAT_TYPE_UBYTE: + num = (unsigned char) va_arg(args, int); + break; + case FORMAT_TYPE_BYTE: + num = (signed char) va_arg(args, int); + break; + case FORMAT_TYPE_USHORT: + num = (unsigned short) va_arg(args, int); + break; + case FORMAT_TYPE_SHORT: + num = (short) va_arg(args, int); + break; + case FORMAT_TYPE_INT: + num = (int) va_arg(args, int); + break; +/* + case FORMAT_TYPE_FLOAT: + d = va_arg(args, double); + pc = (char *)&d; + num = *(unsigned long *)pc; + spec.type = FORMAT_TYPE_LONG; + break; +*/ + default: + num = va_arg(args, unsigned int); + } + + str = number(str, end, num, spec); + } + } + + if (size > 0) { + if (str < end) + *str = '\0'; + else + end[-1] = '\0'; + } + + /* the trailing null byte doesn't count towards the total */ + return str-buf; + +} +EXPORT_SYMBOL(vsnprintf); + +/** + * vscnprintf - Format a string and place it in a buffer + * @buf: The buffer to place the result into + * @size: The size of the buffer, including the trailing null space + * @fmt: The format string to use + * @args: Arguments for the format string + * + * The return value is the number of characters which have been written into + * the @buf not including the trailing '\0'. If @size is <= 0 the function + * returns 0. + * + * Call this function if you are already dealing with a va_list. + * You probably want scnprintf() instead. + * + * See the vsnprintf() documentation for format string extensions over C99. + */ +int vscnprintf(char *buf, size_t size, const char *fmt, va_list args) +{ + int i; + + i=vsnprintf(buf,size,fmt,args); + return (i >= size) ? (size - 1) : i; +} +EXPORT_SYMBOL(vscnprintf); + +/** + * snprintf - Format a string and place it in a buffer + * @buf: The buffer to place the result into + * @size: The size of the buffer, including the trailing null space + * @fmt: The format string to use + * @...: Arguments for the format string + * + * The return value is the number of characters which would be + * generated for the given input, excluding the trailing null, + * as per ISO C99. If the return is greater than or equal to + * @size, the resulting string is truncated. + * + * See the vsnprintf() documentation for format string extensions over C99. + */ +int snprintf(char * buf, size_t size, const char *fmt, ...) +{ + va_list args; + int i; + + va_start(args, fmt); + i=vsnprintf(buf,size,fmt,args); + va_end(args); + return i; +} +EXPORT_SYMBOL(snprintf); + +/** + * scnprintf - Format a string and place it in a buffer + * @buf: The buffer to place the result into + * @size: The size of the buffer, including the trailing null space + * @fmt: The format string to use + * @...: Arguments for the format string + * + * The return value is the number of characters written into @buf not including + * the trailing '\0'. If @size is <= 0 the function returns 0. + */ + +int scnprintf(char * buf, size_t size, const char *fmt, ...) +{ + va_list args; + int i; + + va_start(args, fmt); + i = vsnprintf(buf, size, fmt, args); + va_end(args); + return (i >= size) ? (size - 1) : i; +} +EXPORT_SYMBOL(scnprintf); + +/** + * vsprintf - Format a string and place it in a buffer + * @buf: The buffer to place the result into + * @fmt: The format string to use + * @args: Arguments for the format string + * + * The function returns the number of characters written + * into @buf. Use vsnprintf() or vscnprintf() in order to avoid + * buffer overflows. + * + * Call this function if you are already dealing with a va_list. + * You probably want sprintf() instead. + * + * See the vsnprintf() documentation for format string extensions over C99. + */ +int vsprintf(char *buf, const char *fmt, va_list args) +{ + return vsnprintf(buf, INT_MAX, fmt, args); +} +EXPORT_SYMBOL(vsprintf); + +/** + * sprintf - Format a string and place it in a buffer + * @buf: The buffer to place the result into + * @fmt: The format string to use + * @...: Arguments for the format string + * + * The function returns the number of characters written + * into @buf. Use snprintf() or scnprintf() in order to avoid + * buffer overflows. + * + * See the vsnprintf() documentation for format string extensions over C99. + */ +int sprintf(char * buf, const char *fmt, ...) +{ + va_list args; + int i; + + va_start(args, fmt); + i=vsnprintf(buf, INT_MAX, fmt, args); + va_end(args); + return i; +} +EXPORT_SYMBOL(sprintf); + +/** + * vsscanf - Unformat a buffer into a list of arguments + * @buf: input buffer + * @fmt: format of buffer + * @args: arguments + */ +int vsscanf(const char * buf, const char * fmt, va_list args) +{ + const char *str = buf; + char *next; + char digit; + int num = 0; + int qualifier; + int base; + int field_width; + int is_sign = 0; + + while(*fmt && *str) { + /* skip any white space in format */ + /* white space in format matchs any amount of + * white space, including none, in the input. + */ + if (isspace(*fmt)) { + while (isspace(*fmt)) + ++fmt; + while (isspace(*str)) + ++str; + } + + /* anything that is not a conversion must match exactly */ + if (*fmt != '%' && *fmt) { + if (*fmt++ != *str++) + break; + continue; + } + + if (!*fmt) + break; + ++fmt; + + /* skip this conversion. + * advance both strings to next white space + */ + if (*fmt == '*') { + while (!isspace(*fmt) && *fmt) + fmt++; + while (!isspace(*str) && *str) + str++; + continue; + } + + /* get field width */ + field_width = -1; + if (isdigit(*fmt)) + field_width = skip_atoi(&fmt); + + /* get conversion qualifier */ + qualifier = -1; + if (*fmt == 'h' || *fmt == 'l' || *fmt == 'L' || + *fmt == 'Z' || *fmt == 'z') { + qualifier = *fmt++; + if (unlikely(qualifier == *fmt)) { + if (qualifier == 'h') { + qualifier = 'H'; + fmt++; + } else if (qualifier == 'l') { + qualifier = 'L'; + fmt++; + } + } + } + base = 10; + is_sign = 0; + + if (!*fmt || !*str) + break; + + switch(*fmt++) { + case 'c': + { + char *s = (char *) va_arg(args,char*); + if (field_width == -1) + field_width = 1; + do { + *s++ = *str++; + } while (--field_width > 0 && *str); + num++; + } + continue; + case 's': + { + char *s = (char *) va_arg(args, char *); + if(field_width == -1) + field_width = INT_MAX; + /* first, skip leading white space in buffer */ + while (isspace(*str)) + str++; + + /* now copy until next white space */ + while (*str && !isspace(*str) && field_width--) { + *s++ = *str++; + } + *s = '\0'; + num++; + } + continue; + case 'n': + /* return number of characters read so far */ + { + int *i = (int *)va_arg(args,int*); + *i = str - buf; + } + continue; + case 'o': + base = 8; + break; + case 'x': + case 'X': + base = 16; + break; + case 'i': + base = 0; + case 'd': + is_sign = 1; + case 'u': + break; + case '%': + /* looking for '%' in str */ + if (*str++ != '%') + return num; + continue; + default: + /* invalid format; stop here */ + return num; + } + + /* have some sort of integer conversion. + * first, skip white space in buffer. + */ + while (isspace(*str)) + str++; + + digit = *str; + if (is_sign && digit == '-') + digit = *(str + 1); + + if (!digit + || (base == 16 && !isxdigit(digit)) + || (base == 10 && !isdigit(digit)) + || (base == 8 && (!isdigit(digit) || digit > '7')) + || (base == 0 && !isdigit(digit))) + break; + + switch(qualifier) { + case 'H': /* that's 'hh' in format */ + if (is_sign) { + signed char *s = (signed char *) va_arg(args,signed char *); + *s = (signed char) simple_strtol(str,&next,base); + } else { + unsigned char *s = (unsigned char *) va_arg(args, unsigned char *); + *s = (unsigned char) simple_strtoul(str, &next, base); + } + break; + case 'h': + if (is_sign) { + short *s = (short *) va_arg(args,short *); + *s = (short) simple_strtol(str,&next,base); + } else { + unsigned short *s = (unsigned short *) va_arg(args, unsigned short *); + *s = (unsigned short) simple_strtoul(str, &next, base); + } + break; + case 'l': + if (is_sign) { + long *l = (long *) va_arg(args,long *); + *l = simple_strtol(str,&next,base); + } else { + unsigned long *l = (unsigned long*) va_arg(args,unsigned long*); + *l = simple_strtoul(str,&next,base); + } + break; + case 'L': + if (is_sign) { + long long *l = (long long*) va_arg(args,long long *); + *l = simple_strtoll(str,&next,base); + } else { + unsigned long long *l = (unsigned long long*) va_arg(args,unsigned long long*); + *l = simple_strtoull(str,&next,base); + } + break; + case 'Z': + case 'z': + { + size_t *s = (size_t*) va_arg(args,size_t*); + *s = (size_t) simple_strtoul(str,&next,base); + } + break; + default: + if (is_sign) { + int *i = (int *) va_arg(args, int*); + *i = (int) simple_strtol(str,&next,base); + } else { + unsigned int *i = (unsigned int*) va_arg(args, unsigned int*); + *i = (unsigned int) simple_strtoul(str,&next,base); + } + break; + } + num++; + + if (!next) + break; + str = next; + } + + /* + * Now we've come all the way through so either the input string or the + * format ended. In the former case, there can be a %n at the current + * position in the format that needs to be filled. + */ + if (*fmt == '%' && *(fmt + 1) == 'n') { + int *p = (int *)va_arg(args, int *); + *p = str - buf; + } + + return num; +} +EXPORT_SYMBOL(vsscanf); + +/** + * sscanf - Unformat a buffer into a list of arguments + * @buf: input buffer + * @fmt: formatting of buffer + * @...: resulting arguments + */ +int sscanf(const char * buf, const char * fmt, ...) +{ + va_list args; + int i; + + va_start(args,fmt); + i = vsscanf(buf,fmt,args); + va_end(args); + return i; +} +EXPORT_SYMBOL(sscanf); + +unsigned long strtol(const char *cp, char **endp, unsigned int base) +{ + return simple_strtol(cp, endp, base); +}