zephyr/arch/x86/core/userspace.S

409 lines
11 KiB
ArmAsm

/*
* Copyright (c) 2017 Intel Corporation
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <kernel_structs.h>
#include <arch/x86/asm.h>
#include <arch/cpu.h>
#include <offsets_short.h>
#include <syscall.h>
/* Exports */
GTEXT(_x86_syscall_entry_stub)
GTEXT(_x86_userspace_enter)
GTEXT(z_arch_user_string_nlen)
GTEXT(z_arch_user_string_nlen_fault_start)
GTEXT(z_arch_user_string_nlen_fault_end)
GTEXT(z_arch_user_string_nlen_fixup)
/* Imports */
GDATA(_k_syscall_table)
#ifdef CONFIG_X86_KPTI
/* Switch from the shadow to the kernel page table, switch to the interrupted
* thread's kernel stack, and copy all context from the trampoline stack.
*
* Assumes all registers are callee-saved since this gets called from other
* ASM code. Assumes a particular stack layout which is correct for
* _exception_enter and _interrupt_enter when invoked with a call instruction:
*
* 28 SS
* 24 ES
* 20 EFLAGS
* 16 CS
* 12 EIP
* 8 isr_param or exc code
* 4 isr or exc handler
* 0 return address
*/
SECTION_FUNC(TEXT, z_x86_trampoline_to_kernel)
/* Check interrupted code segment to see if we came from ring 3
* and hence on the trampoline stack
*/
testb $3, 16(%esp) /* Offset of CS */
jz 1f
/* Stash these regs as we need to use them */
pushl %esi
pushl %edi
/* Switch to kernel page table */
movl $z_x86_kernel_pdpt, %esi
movl %esi, %cr3
/* Save old trampoline stack pointer in %edi */
movl %esp, %edi
/* %esp = _kernel->current->stack_info.start
*
* This is the lowest address of the user mode stack, and higest
* address of the kernel stack, they are adjacent.
* We want to transplant context here.
*/
movl $_kernel, %esi
movl _kernel_offset_to_current(%esi), %esi
movl _thread_offset_to_stack_start(%esi), %esp
/* Transplant stack context and restore ESI/EDI. Taking care to zero
* or put uninteresting values where we stashed ESI/EDI since the
* trampoline page is insecure and there might a context switch
* on the way out instead of returning to the original thread
* immediately.
*/
pushl 36(%edi) /* SS */
pushl 32(%edi) /* ESP */
pushl 28(%edi) /* EFLAGS */
pushl 24(%edi) /* CS */
pushl 20(%edi) /* EIP */
pushl 16(%edi) /* error code or isr parameter */
pushl 12(%edi) /* exception/irq handler */
pushl 8(%edi) /* return address */
movl 4(%edi), %esi /* restore ESI */
movl $0, 4(%edi) /* Zero old esi storage area */
xchgl %edi, (%edi) /* Exchange old edi to restore it and put
old sp in the storage area */
/* Trampoline stack should have nothing sensitive in it at this point */
1:
ret
/* Copy interrupt return stack context to the trampoline stack, switch back
* to the user page table, and only then 'iret'. We jump to this instead
* of calling 'iret' if KPTI is turned on.
*
* Stack layout is expected to be as follows:
*
* 16 SS
* 12 ESP
* 8 EFLAGS
* 4 CS
* 0 EIP
*
* This function is conditionally macroed to KPTI_IRET/KPTI_IRET_USER
*/
SECTION_FUNC(TEXT, z_x86_trampoline_to_user)
/* Check interrupted code segment to see if we came from ring 3
* and hence on the trampoline stack
*/
testb $3, 4(%esp) /* Offset of CS */
jz 1f
/* Otherwise, fall through ... */
SECTION_FUNC(TEXT, z_x86_trampoline_to_user_always)
/* Stash EDI, need a free register */
pushl %edi
/* Store old stack pointer and switch to trampoline stack */
movl %esp, %edi
movl $z_trampoline_stack_end, %esp
/* Lock IRQs until we get out, we don't want anyone else using the
* trampoline stack
*/
cli
/* Copy context */
pushl 20(%edi) /* SS */
pushl 16(%edi) /* ESP */
pushl 12(%edi) /* EFLAGS */
pushl 8(%edi) /* CS */
pushl 4(%edi) /* EIP */
xchgl %edi, (%edi) /* Exchange old edi to restore it and put
trampoline stack address in its old storage
area */
/* Switch to user page table */
pushl %eax
movl $z_x86_user_pdpt, %eax
movl %eax, %cr3
popl %eax
movl $0, -4(%esp) /* Delete stashed EAX data */
/* Trampoline stack should have nothing sensitive in it at this point */
1:
iret
#endif /* CONFIG_X86_KPTI */
/* Landing site for syscall SW IRQ. Marshal arguments and call C function for
* further processing. We're on the kernel stack for the invoking thread,
* unless KPTI is enabled, in which case we're on the trampoline stack and
* need to get off it before enabling interrupts.
*/
SECTION_FUNC(TEXT, _x86_syscall_entry_stub)
#ifdef CONFIG_X86_KPTI
/* Stash these regs as we need to use them */
pushl %esi
pushl %edi
/* Switch to kernel page table */
movl $z_x86_kernel_pdpt, %esi
movl %esi, %cr3
/* Save old trampoline stack pointer in %edi */
movl %esp, %edi
/* %esp = _kernel->current->stack_info.start
*
* This is the lowest address of the user mode stack, and higest
* address of the kernel stack, they are adjacent.
* We want to transplant context here.
*/
movl $_kernel, %esi
movl _kernel_offset_to_current(%esi), %esi
movl _thread_offset_to_stack_start(%esi), %esp
/* Transplant context according to layout above. Variant of logic
* in x86_trampoline_to_kernel */
pushl 24(%edi) /* SS */
pushl 20(%edi) /* ESP */
pushl 16(%edi) /* EFLAGS */
pushl 12(%edi) /* CS */
pushl 8(%edi) /* EIP */
movl 4(%edi), %esi /* restore ESI */
movl $0, 4(%edi) /* Zero old esi storage area */
xchgl %edi, (%edi) /* Exchange old edi to restore it and put
old sp in the storage area */
/* Trampoline stack should have nothing sensitive in it at this point */
#endif /* CONFIG_X86_KPTI */
sti /* re-enable interrupts */
cld /* clear direction flag, restored on 'iret' */
/* call_id is in ESI. bounds-check it, must be less than
* K_SYSCALL_LIMIT
*/
cmp $K_SYSCALL_LIMIT, %esi
jae _bad_syscall
_id_ok:
#ifdef CONFIG_BOUNDS_CHECK_BYPASS_MITIGATION
/* Prevent speculation with bogus system call IDs */
lfence
#endif
/* Marshal arguments per calling convention to match what is expected
* for _k_syscall_handler_t functions
*/
push %esp /* ssf */
push %ebp /* arg6 */
push %edi /* arg5 */
push %ebx /* arg4 */
#ifndef CONFIG_X86_IAMCU
push %ecx /* arg3 */
push %edx /* arg2 */
push %eax /* arg1 */
#endif
/* from the call ID in ESI, load EBX with the actual function pointer
* to call by looking it up in the system call dispatch table
*/
xor %edi, %edi
mov _k_syscall_table(%edi, %esi, 4), %ebx
/* Run the handler, which is some entry in _k_syscall_table */
INDIRECT_CALL(%ebx)
/* EAX now contains return value. Pop or xor everything else to prevent
* information leak from kernel mode.
*/
#ifndef CONFIG_X86_IAMCU
pop %edx /* old arg1 value, discard it */
pop %edx
pop %ecx
#endif
pop %ebx
pop %edi
#ifndef CONFIG_X86_IAMCU
/* Discard ssf and arg6 */
add $8, %esp
#else
pop %ecx /* Clean ECX and get arg6 off the stack */
pop %edx /* Clean EDX and get ssf off the stack */
#endif
KPTI_IRET_USER
_bad_syscall:
/* ESI had a bogus syscall value in it, replace with the bad syscall
* handler's ID, and put the bad ID as its first argument. This
* clobbers ESI but the bad syscall handler never returns
* anyway, it's going to generate a kernel oops
*/
mov %esi, %eax
mov $K_SYSCALL_BAD, %esi
jmp _id_ok
/*
* size_t z_arch_user_string_nlen(const char *s, size_t maxsize, int *err_arg)
*/
SECTION_FUNC(TEXT, z_arch_user_string_nlen)
push %ebp
mov %esp, %ebp
/* error value, set to -1 initially. This location is -4(%ebp) */
push $-1
/* Do the strlen operation, based on disassembly of minimal libc */
xor %eax, %eax /* EAX = 0, length count */
mov 0x8(%ebp), %edx /* EDX base of string */
/* This code might page fault */
strlen_loop:
z_arch_user_string_nlen_fault_start:
cmpb $0x0, (%edx, %eax, 1) /* *(EDX + EAX) == 0? Could fault. */
z_arch_user_string_nlen_fault_end:
je strlen_done
cmp 0xc(%ebp), %eax /* Max length reached? */
je strlen_done
inc %eax /* EAX++ and loop again */
jmp strlen_loop
strlen_done:
/* Set error value to 0 since we succeeded */
movl $0, -4(%ebp)
z_arch_user_string_nlen_fixup:
/* Write error value to err pointer parameter */
movl 0x10(%ebp), %ecx
pop %edx
movl %edx, (%ecx)
pop %ebp
ret
/* FUNC_NORETURN void _x86_userspace_enter(k_thread_entry_t user_entry,
* void *p1, void *p2, void *p3,
* u32_t stack_end,
* u32_t stack_start)
*
* A one-way trip to userspace.
*/
SECTION_FUNC(TEXT, _x86_userspace_enter)
pop %esi /* Discard return address on stack */
/* Fetch parameters on the stack */
#ifndef CONFIG_X86_IAMCU
pop %eax /* user_entry */
pop %edx /* p1 */
pop %ecx /* p2 */
#endif
pop %esi /* p3 */
pop %ebx /* stack_end (high address) */
pop %edi /* stack_start (low address) */
/* Move to the kernel stack for this thread, so we can erase the
* user stack. The kernel stack is the page immediately before
* the user stack.
*
* For security reasons, we must erase the entire user stack.
* We don't know what previous contexts it was used and do not
* want to leak any information.
*/
mov %edi, %esp
/* Stash some registers we are going to need to erase the user
* stack.
*/
push %ecx
push %edi
push %eax
/* Compute size of user stack in 4-byte chunks and put in ECX */
mov %ebx, %ecx
sub %edi, %ecx
shr $2, %ecx /* Divide by 4 */
#ifdef CONFIG_INIT_STACKS
mov $0xAAAAAAAA, %eax
#else
xor %eax, %eax
#endif
/* Copy 4 bytes of memory at a time, starting at ES:EDI, with whatever
* is in EAX. Repeat this ECX times. Stack sizes are always at least
* 4-byte aligned.
*/
cld
rep stosl
/* Restore registers */
pop %eax
pop %edi
pop %ecx
/* Now set stack pointer to the base of the user stack. Now that this
* is set we won't need EBX any more.
*/
mov %ebx, %esp
/* Set segment registers (except CS and SS which are done in
* a special way by 'iret' below)
*/
mov $USER_DATA_SEG, %bx
mov %bx, %ds
mov %bx, %es
/* Push arguments to z_thread_entry() */
push %esi /* p3 */
#ifndef CONFIG_X86_IAMCU
push %ecx /* p2 */
push %edx /* p1 */
push %eax /* user_entry */
#endif
/* NULL return address */
push $0
/* Save stack pointer at this position, this is where it will be
* when we land in z_thread_entry()
*/
mov %esp, %edi
/* Inter-privilege 'iret' pops all of these. Need to fake an interrupt
* return to enter user mode as far calls cannot change privilege
* level
*/
push $USER_DATA_SEG /* SS */
push %edi /* ESP */
pushfl /* EFLAGS */
push $USER_CODE_SEG /* CS */
push $z_thread_entry /* EIP */
#ifdef CONFIG_EXECUTION_BENCHMARKING
/* Save the eax and edx registers before reading the time stamp
* once done pop the values.
*/
push %eax
push %edx
rdtsc
mov %eax,__end_drop_to_usermode_time
mov %edx,__end_drop_to_usermode_time+4
pop %edx
pop %eax
#endif
/* We will land in z_thread_entry() in user mode after this */
KPTI_IRET_USER