/* * Copyright (c) 2017 Intel Corporation * * SPDX-License-Identifier: Apache-2.0 */ #include #include #include #include #include /* Exports */ GTEXT(_x86_syscall_entry_stub) GTEXT(_x86_userspace_enter) GTEXT(z_arch_user_string_nlen) GTEXT(z_arch_user_string_nlen_fault_start) GTEXT(z_arch_user_string_nlen_fault_end) GTEXT(z_arch_user_string_nlen_fixup) /* Imports */ GDATA(_k_syscall_table) #ifdef CONFIG_X86_KPTI /* Switch from the shadow to the kernel page table, switch to the interrupted * thread's kernel stack, and copy all context from the trampoline stack. * * Assumes all registers are callee-saved since this gets called from other * ASM code. Assumes a particular stack layout which is correct for * _exception_enter and _interrupt_enter when invoked with a call instruction: * * 28 SS * 24 ES * 20 EFLAGS * 16 CS * 12 EIP * 8 isr_param or exc code * 4 isr or exc handler * 0 return address */ SECTION_FUNC(TEXT, z_x86_trampoline_to_kernel) /* Check interrupted code segment to see if we came from ring 3 * and hence on the trampoline stack */ testb $3, 16(%esp) /* Offset of CS */ jz 1f /* Stash these regs as we need to use them */ pushl %esi pushl %edi /* Switch to kernel page table */ movl $z_x86_kernel_pdpt, %esi movl %esi, %cr3 /* Save old trampoline stack pointer in %edi */ movl %esp, %edi /* %esp = _kernel->current->stack_info.start * * This is the lowest address of the user mode stack, and higest * address of the kernel stack, they are adjacent. * We want to transplant context here. */ movl $_kernel, %esi movl _kernel_offset_to_current(%esi), %esi movl _thread_offset_to_stack_start(%esi), %esp /* Transplant stack context and restore ESI/EDI. Taking care to zero * or put uninteresting values where we stashed ESI/EDI since the * trampoline page is insecure and there might a context switch * on the way out instead of returning to the original thread * immediately. */ pushl 36(%edi) /* SS */ pushl 32(%edi) /* ESP */ pushl 28(%edi) /* EFLAGS */ pushl 24(%edi) /* CS */ pushl 20(%edi) /* EIP */ pushl 16(%edi) /* error code or isr parameter */ pushl 12(%edi) /* exception/irq handler */ pushl 8(%edi) /* return address */ movl 4(%edi), %esi /* restore ESI */ movl $0, 4(%edi) /* Zero old esi storage area */ xchgl %edi, (%edi) /* Exchange old edi to restore it and put old sp in the storage area */ /* Trampoline stack should have nothing sensitive in it at this point */ 1: ret /* Copy interrupt return stack context to the trampoline stack, switch back * to the user page table, and only then 'iret'. We jump to this instead * of calling 'iret' if KPTI is turned on. * * Stack layout is expected to be as follows: * * 16 SS * 12 ESP * 8 EFLAGS * 4 CS * 0 EIP * * This function is conditionally macroed to KPTI_IRET/KPTI_IRET_USER */ SECTION_FUNC(TEXT, z_x86_trampoline_to_user) /* Check interrupted code segment to see if we came from ring 3 * and hence on the trampoline stack */ testb $3, 4(%esp) /* Offset of CS */ jz 1f /* Otherwise, fall through ... */ SECTION_FUNC(TEXT, z_x86_trampoline_to_user_always) /* Stash EDI, need a free register */ pushl %edi /* Store old stack pointer and switch to trampoline stack */ movl %esp, %edi movl $z_trampoline_stack_end, %esp /* Lock IRQs until we get out, we don't want anyone else using the * trampoline stack */ cli /* Copy context */ pushl 20(%edi) /* SS */ pushl 16(%edi) /* ESP */ pushl 12(%edi) /* EFLAGS */ pushl 8(%edi) /* CS */ pushl 4(%edi) /* EIP */ xchgl %edi, (%edi) /* Exchange old edi to restore it and put trampoline stack address in its old storage area */ /* Switch to user page table */ pushl %eax movl $z_x86_user_pdpt, %eax movl %eax, %cr3 popl %eax movl $0, -4(%esp) /* Delete stashed EAX data */ /* Trampoline stack should have nothing sensitive in it at this point */ 1: iret #endif /* CONFIG_X86_KPTI */ /* Landing site for syscall SW IRQ. Marshal arguments and call C function for * further processing. We're on the kernel stack for the invoking thread, * unless KPTI is enabled, in which case we're on the trampoline stack and * need to get off it before enabling interrupts. */ SECTION_FUNC(TEXT, _x86_syscall_entry_stub) #ifdef CONFIG_X86_KPTI /* Stash these regs as we need to use them */ pushl %esi pushl %edi /* Switch to kernel page table */ movl $z_x86_kernel_pdpt, %esi movl %esi, %cr3 /* Save old trampoline stack pointer in %edi */ movl %esp, %edi /* %esp = _kernel->current->stack_info.start * * This is the lowest address of the user mode stack, and higest * address of the kernel stack, they are adjacent. * We want to transplant context here. */ movl $_kernel, %esi movl _kernel_offset_to_current(%esi), %esi movl _thread_offset_to_stack_start(%esi), %esp /* Transplant context according to layout above. Variant of logic * in x86_trampoline_to_kernel */ pushl 24(%edi) /* SS */ pushl 20(%edi) /* ESP */ pushl 16(%edi) /* EFLAGS */ pushl 12(%edi) /* CS */ pushl 8(%edi) /* EIP */ movl 4(%edi), %esi /* restore ESI */ movl $0, 4(%edi) /* Zero old esi storage area */ xchgl %edi, (%edi) /* Exchange old edi to restore it and put old sp in the storage area */ /* Trampoline stack should have nothing sensitive in it at this point */ #endif /* CONFIG_X86_KPTI */ sti /* re-enable interrupts */ cld /* clear direction flag, restored on 'iret' */ /* call_id is in ESI. bounds-check it, must be less than * K_SYSCALL_LIMIT */ cmp $K_SYSCALL_LIMIT, %esi jae _bad_syscall _id_ok: #ifdef CONFIG_BOUNDS_CHECK_BYPASS_MITIGATION /* Prevent speculation with bogus system call IDs */ lfence #endif /* Marshal arguments per calling convention to match what is expected * for _k_syscall_handler_t functions */ push %esp /* ssf */ push %ebp /* arg6 */ push %edi /* arg5 */ push %ebx /* arg4 */ #ifndef CONFIG_X86_IAMCU push %ecx /* arg3 */ push %edx /* arg2 */ push %eax /* arg1 */ #endif /* from the call ID in ESI, load EBX with the actual function pointer * to call by looking it up in the system call dispatch table */ xor %edi, %edi mov _k_syscall_table(%edi, %esi, 4), %ebx /* Run the handler, which is some entry in _k_syscall_table */ INDIRECT_CALL(%ebx) /* EAX now contains return value. Pop or xor everything else to prevent * information leak from kernel mode. */ #ifndef CONFIG_X86_IAMCU pop %edx /* old arg1 value, discard it */ pop %edx pop %ecx #endif pop %ebx pop %edi #ifndef CONFIG_X86_IAMCU /* Discard ssf and arg6 */ add $8, %esp #else pop %ecx /* Clean ECX and get arg6 off the stack */ pop %edx /* Clean EDX and get ssf off the stack */ #endif KPTI_IRET_USER _bad_syscall: /* ESI had a bogus syscall value in it, replace with the bad syscall * handler's ID, and put the bad ID as its first argument. This * clobbers ESI but the bad syscall handler never returns * anyway, it's going to generate a kernel oops */ mov %esi, %eax mov $K_SYSCALL_BAD, %esi jmp _id_ok /* * size_t z_arch_user_string_nlen(const char *s, size_t maxsize, int *err_arg) */ SECTION_FUNC(TEXT, z_arch_user_string_nlen) push %ebp mov %esp, %ebp /* error value, set to -1 initially. This location is -4(%ebp) */ push $-1 /* Do the strlen operation, based on disassembly of minimal libc */ xor %eax, %eax /* EAX = 0, length count */ mov 0x8(%ebp), %edx /* EDX base of string */ /* This code might page fault */ strlen_loop: z_arch_user_string_nlen_fault_start: cmpb $0x0, (%edx, %eax, 1) /* *(EDX + EAX) == 0? Could fault. */ z_arch_user_string_nlen_fault_end: je strlen_done cmp 0xc(%ebp), %eax /* Max length reached? */ je strlen_done inc %eax /* EAX++ and loop again */ jmp strlen_loop strlen_done: /* Set error value to 0 since we succeeded */ movl $0, -4(%ebp) z_arch_user_string_nlen_fixup: /* Write error value to err pointer parameter */ movl 0x10(%ebp), %ecx pop %edx movl %edx, (%ecx) pop %ebp ret /* FUNC_NORETURN void _x86_userspace_enter(k_thread_entry_t user_entry, * void *p1, void *p2, void *p3, * u32_t stack_end, * u32_t stack_start) * * A one-way trip to userspace. */ SECTION_FUNC(TEXT, _x86_userspace_enter) pop %esi /* Discard return address on stack */ /* Fetch parameters on the stack */ #ifndef CONFIG_X86_IAMCU pop %eax /* user_entry */ pop %edx /* p1 */ pop %ecx /* p2 */ #endif pop %esi /* p3 */ pop %ebx /* stack_end (high address) */ pop %edi /* stack_start (low address) */ /* Move to the kernel stack for this thread, so we can erase the * user stack. The kernel stack is the page immediately before * the user stack. * * For security reasons, we must erase the entire user stack. * We don't know what previous contexts it was used and do not * want to leak any information. */ mov %edi, %esp /* Stash some registers we are going to need to erase the user * stack. */ push %ecx push %edi push %eax /* Compute size of user stack in 4-byte chunks and put in ECX */ mov %ebx, %ecx sub %edi, %ecx shr $2, %ecx /* Divide by 4 */ #ifdef CONFIG_INIT_STACKS mov $0xAAAAAAAA, %eax #else xor %eax, %eax #endif /* Copy 4 bytes of memory at a time, starting at ES:EDI, with whatever * is in EAX. Repeat this ECX times. Stack sizes are always at least * 4-byte aligned. */ cld rep stosl /* Restore registers */ pop %eax pop %edi pop %ecx /* Now set stack pointer to the base of the user stack. Now that this * is set we won't need EBX any more. */ mov %ebx, %esp /* Set segment registers (except CS and SS which are done in * a special way by 'iret' below) */ mov $USER_DATA_SEG, %bx mov %bx, %ds mov %bx, %es /* Push arguments to z_thread_entry() */ push %esi /* p3 */ #ifndef CONFIG_X86_IAMCU push %ecx /* p2 */ push %edx /* p1 */ push %eax /* user_entry */ #endif /* NULL return address */ push $0 /* Save stack pointer at this position, this is where it will be * when we land in z_thread_entry() */ mov %esp, %edi /* Inter-privilege 'iret' pops all of these. Need to fake an interrupt * return to enter user mode as far calls cannot change privilege * level */ push $USER_DATA_SEG /* SS */ push %edi /* ESP */ pushfl /* EFLAGS */ push $USER_CODE_SEG /* CS */ push $z_thread_entry /* EIP */ #ifdef CONFIG_EXECUTION_BENCHMARKING /* Save the eax and edx registers before reading the time stamp * once done pop the values. */ push %eax push %edx rdtsc mov %eax,__end_drop_to_usermode_time mov %edx,__end_drop_to_usermode_time+4 pop %edx pop %eax #endif /* We will land in z_thread_entry() in user mode after this */ KPTI_IRET_USER