470 lines
14 KiB
ArmAsm
470 lines
14 KiB
ArmAsm
/*
|
|
* Copyright (c) 2010-2015 Wind River Systems, Inc.
|
|
*
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
/**
|
|
* @file
|
|
* @brief Kernel swapper code for IA-32
|
|
*
|
|
* This module implements the __swap() routine for the IA-32 architecture.
|
|
*
|
|
* Note that the file arch/x86/include/swapstk.h defines
|
|
* a representation of the save stack frame generated by __swap() in order
|
|
* to generate offsets (in the form of absolute symbols) for consumption by
|
|
* host tools. Please update swapstk.h if changing the structure of the
|
|
* save frame on the stack.
|
|
*/
|
|
|
|
#include <kernel_structs.h>
|
|
#include <arch/x86/asm.h>
|
|
#include <offsets_short.h>
|
|
|
|
/* exports (internal APIs) */
|
|
|
|
GTEXT(__swap)
|
|
GTEXT(_x86_thread_entry_wrapper)
|
|
GTEXT(_x86_user_thread_entry_wrapper)
|
|
|
|
/* externs */
|
|
#ifdef CONFIG_X86_USERSPACE
|
|
GTEXT(_x86_swap_update_page_tables)
|
|
#endif
|
|
GDATA(_k_neg_eagain)
|
|
|
|
/**
|
|
*
|
|
* @brief Initiate a cooperative context switch
|
|
*
|
|
* The __swap() routine is invoked by various kernel services to effect
|
|
* a cooperative context switch. Prior to invoking __swap(), the
|
|
* caller disables interrupts (via irq_lock) and the return 'key'
|
|
* is passed as a parameter to __swap(). The 'key' actually represents
|
|
* the EFLAGS register prior to disabling interrupts via a 'cli' instruction.
|
|
*
|
|
* Given that __swap() is called to effect a cooperative context switch, only
|
|
* the non-volatile integer registers need to be saved in the TCS of the
|
|
* outgoing thread. The restoration of the integer registers of the incoming
|
|
* thread depends on whether that thread was preemptively context switched out.
|
|
* The _INT_ACTIVE and _EXC_ACTIVE bits in the k_thread->thread_state field
|
|
* will signify that the thread was preemptively context switched out, and thus
|
|
* both the volatile and non-volatile integer registers need to be restored.
|
|
*
|
|
* The non-volatile registers need to be scrubbed to ensure they contain no
|
|
* sensitive information that could compromise system security. This is to
|
|
* make sure that information will not be leaked from one application to
|
|
* another via these volatile registers.
|
|
*
|
|
* Here, the integer registers (EAX, ECX, EDX) have been scrubbed. Any changes
|
|
* to this routine that alter the values of these registers MUST be reviewed
|
|
* for potential security impacts.
|
|
*
|
|
* Floating point registers are handled using a lazy save/restore
|
|
* mechanism since it's expected relatively few threads will be created
|
|
* with the K_FP_REGS or K_SSE_REGS option bits. The kernel data structure
|
|
* maintains a 'current_fp' field to keep track of the thread that "owns"
|
|
* the floating point registers. Floating point registers consist of
|
|
* ST0->ST7 (x87 FPU and MMX registers) and XMM0 -> XMM7.
|
|
*
|
|
* All floating point registers are considered 'volatile' thus they will
|
|
* only be saved/restored when a preemptive context switch occurs.
|
|
*
|
|
* Floating point registers are currently NOT scrubbed, and are subject to
|
|
* potential security leaks.
|
|
*
|
|
* @return -EAGAIN, or a return value set by a call to
|
|
* _set_thread_return_value()
|
|
*
|
|
* C function prototype:
|
|
*
|
|
* unsigned int __swap (unsigned int eflags);
|
|
*
|
|
*/
|
|
|
|
.macro read_tsc var_name
|
|
push %eax
|
|
push %edx
|
|
rdtsc
|
|
mov %eax,\var_name
|
|
mov %edx,\var_name+4
|
|
pop %edx
|
|
pop %eax
|
|
.endm
|
|
SECTION_FUNC(TEXT, __swap)
|
|
#ifdef CONFIG_EXECUTION_BENCHMARKING
|
|
/* Save the eax and edx registers before reading the time stamp
|
|
* once done pop the values.
|
|
*/
|
|
push %eax
|
|
push %edx
|
|
rdtsc
|
|
mov %eax,__start_swap_time
|
|
mov %edx,__start_swap_time+4
|
|
pop %edx
|
|
pop %eax
|
|
#endif
|
|
#ifdef CONFIG_X86_IAMCU
|
|
/* save EFLAGS on stack right before return address, just as SYSV would
|
|
* have done
|
|
*/
|
|
pushl 0(%esp)
|
|
movl %eax, 4(%esp)
|
|
#endif
|
|
/*
|
|
* Push all non-volatile registers onto the stack; do not copy
|
|
* any of these registers into the k_thread. Only the 'esp' register
|
|
* after all the pushes have been performed) will be stored in the
|
|
* k_thread.
|
|
*/
|
|
|
|
pushl %edi
|
|
|
|
movl $_kernel, %edi
|
|
|
|
pushl %esi
|
|
pushl %ebx
|
|
pushl %ebp
|
|
|
|
/*
|
|
* Carve space for the return value. Setting it to a default of
|
|
* -EAGAIN eliminates the need for the timeout code to set it.
|
|
* If another value is ever needed, it can be modified with
|
|
* _set_thread_return_value().
|
|
*/
|
|
|
|
pushl _k_neg_eagain
|
|
|
|
|
|
/* save esp into k_thread structure */
|
|
|
|
movl _kernel_offset_to_current(%edi), %edx
|
|
movl %esp, _thread_offset_to_esp(%edx)
|
|
|
|
#ifdef CONFIG_KERNEL_EVENT_LOGGER_CONTEXT_SWITCH
|
|
/* Register the context switch */
|
|
push %edx
|
|
call _sys_k_event_logger_context_switch
|
|
pop %edx
|
|
#endif
|
|
movl _kernel_offset_to_ready_q_cache(%edi), %eax
|
|
|
|
/*
|
|
* At this point, the %eax register contains the 'k_thread *' of the
|
|
* thread to be swapped in, and %edi still contains &_kernel. %edx
|
|
* has the pointer to the outgoing thread.
|
|
*/
|
|
#ifdef CONFIG_X86_USERSPACE
|
|
|
|
#ifdef CONFIG_X86_IAMCU
|
|
push %eax
|
|
#else
|
|
push %edx
|
|
push %eax
|
|
#endif
|
|
call _x86_swap_update_page_tables
|
|
#ifdef CONFIG_X86_IAMCU
|
|
pop %eax
|
|
#else
|
|
pop %eax
|
|
pop %edx
|
|
#endif
|
|
#endif
|
|
|
|
#ifdef CONFIG_FP_SHARING
|
|
/*
|
|
* Clear the CR0[TS] bit (in the event the current thread
|
|
* doesn't have floating point enabled) to prevent the "device not
|
|
* available" exception when executing the subsequent fxsave/fnsave
|
|
* and/or fxrstor/frstor instructions.
|
|
*
|
|
* Indeed, it's possible that none of the aforementioned instructions
|
|
* need to be executed, for example, the incoming thread doesn't
|
|
* utilize floating point operations. However, the code responsible
|
|
* for setting the CR0[TS] bit appropriately for the incoming thread
|
|
* (just after the 'restoreContext_NoFloatSwap' label) will leverage
|
|
* the fact that the following 'clts' was performed already.
|
|
*/
|
|
|
|
clts
|
|
|
|
|
|
/*
|
|
* Determine whether the incoming thread utilizes floating point registers
|
|
* _and_ whether the thread was context switched out preemptively.
|
|
*/
|
|
|
|
testb $_FP_USER_MASK, _thread_offset_to_user_options(%eax)
|
|
je restoreContext_NoFloatSwap
|
|
|
|
|
|
/*
|
|
* The incoming thread uses floating point registers:
|
|
* Was it the last thread to use floating point registers?
|
|
* If so, there there is no need to restore the floating point context.
|
|
*/
|
|
|
|
movl _kernel_offset_to_current_fp(%edi), %ebx
|
|
cmpl %ebx, %eax
|
|
je restoreContext_NoFloatSwap
|
|
|
|
|
|
/*
|
|
* The incoming thread uses floating point registers and it was _not_
|
|
* the last thread to use those registers:
|
|
* Check whether the current FP context actually needs to be saved
|
|
* before swapping in the context of the incoming thread.
|
|
*/
|
|
|
|
testl %ebx, %ebx
|
|
jz restoreContext_NoFloatSave
|
|
|
|
|
|
/*
|
|
* The incoming thread uses floating point registers and it was _not_
|
|
* the last thread to use those registers _and_ the current FP context
|
|
* needs to be saved.
|
|
*
|
|
* Given that the ST[0] -> ST[7] and XMM0 -> XMM7 registers are all
|
|
* 'volatile', only save the registers if the "current FP context"
|
|
* was preemptively context switched.
|
|
*/
|
|
|
|
testb $_INT_OR_EXC_MASK, _thread_offset_to_thread_state(%ebx)
|
|
je restoreContext_NoFloatSave
|
|
|
|
|
|
#ifdef CONFIG_SSE
|
|
testb $K_SSE_REGS, _thread_offset_to_user_options(%ebx)
|
|
je x87FloatSave
|
|
|
|
/*
|
|
* 'fxsave' does NOT perform an implicit 'fninit', therefore issue an
|
|
* 'fninit' to ensure a "clean" FPU state for the incoming thread
|
|
* (for the case when the fxrstor is not executed).
|
|
*/
|
|
|
|
fxsave _thread_offset_to_preempFloatReg(%ebx)
|
|
fninit
|
|
jmp floatSaveDone
|
|
|
|
x87FloatSave:
|
|
#endif /* CONFIG_SSE */
|
|
|
|
/* 'fnsave' performs an implicit 'fninit' after saving state! */
|
|
|
|
fnsave _thread_offset_to_preempFloatReg(%ebx)
|
|
|
|
/* fall through to 'floatSaveDone' */
|
|
|
|
floatSaveDone:
|
|
restoreContext_NoFloatSave:
|
|
|
|
/*********************************************************
|
|
* Restore floating point context of the incoming thread.
|
|
*********************************************************/
|
|
|
|
/*
|
|
* Again, given that the ST[0] -> ST[7] and XMM0 -> XMM7 registers are
|
|
* all 'volatile', only restore the registers if the incoming thread
|
|
* was previously preemptively context switched out.
|
|
*/
|
|
|
|
testb $_INT_OR_EXC_MASK, _thread_offset_to_thread_state(%eax)
|
|
je restoreContext_NoFloatRestore
|
|
|
|
#ifdef CONFIG_SSE
|
|
testb $K_SSE_REGS, _thread_offset_to_user_options(%eax)
|
|
je x87FloatRestore
|
|
|
|
fxrstor _thread_offset_to_preempFloatReg(%eax)
|
|
jmp floatRestoreDone
|
|
|
|
x87FloatRestore:
|
|
|
|
#endif /* CONFIG_SSE */
|
|
|
|
frstor _thread_offset_to_preempFloatReg(%eax)
|
|
|
|
/* fall through to 'floatRestoreDone' */
|
|
|
|
floatRestoreDone:
|
|
restoreContext_NoFloatRestore:
|
|
|
|
/* record that the incoming thread "owns" the floating point registers */
|
|
|
|
movl %eax, _kernel_offset_to_current_fp(%edi)
|
|
|
|
|
|
/*
|
|
* Branch point when none of the floating point registers need to be
|
|
* swapped because: a) the incoming thread does not use them OR
|
|
* b) the incoming thread is the last thread that used those registers.
|
|
*/
|
|
|
|
restoreContext_NoFloatSwap:
|
|
|
|
/*
|
|
* Leave CR0[TS] clear if incoming thread utilizes the floating point
|
|
* registers
|
|
*/
|
|
|
|
testb $_FP_USER_MASK, _thread_offset_to_user_options(%eax)
|
|
jne CROHandlingDone
|
|
|
|
/*
|
|
* The incoming thread does NOT currently utilize the floating point
|
|
* registers, so set CR0[TS] to ensure the "device not available"
|
|
* exception occurs on the first attempt to access a x87 FPU, MMX,
|
|
* or XMM register.
|
|
*/
|
|
|
|
movl %cr0, %edx
|
|
orl $0x8, %edx
|
|
movl %edx, %cr0
|
|
|
|
CROHandlingDone:
|
|
|
|
#endif /* CONFIG_FP_SHARING */
|
|
|
|
/* update _kernel.current to reflect incoming thread */
|
|
|
|
movl %eax, _kernel_offset_to_current(%edi)
|
|
|
|
/* recover thread stack pointer from k_thread */
|
|
|
|
movl _thread_offset_to_esp(%eax), %esp
|
|
|
|
|
|
/* load return value from a possible _set_thread_return_value() */
|
|
|
|
popl %eax
|
|
|
|
/* pop the non-volatile registers from the stack */
|
|
|
|
popl %ebp
|
|
popl %ebx
|
|
popl %esi
|
|
popl %edi
|
|
|
|
/*
|
|
* %eax may contain one of these values:
|
|
*
|
|
* - the return value for __swap() that was set up by a call to
|
|
* _set_thread_return_value()
|
|
* - -EINVAL
|
|
*/
|
|
|
|
/* Utilize the 'eflags' parameter to __swap() */
|
|
|
|
pushl 4(%esp)
|
|
#ifdef CONFIG_INT_LATENCY_BENCHMARK
|
|
testl $0x200, (%esp)
|
|
jz skipIntLatencyStop
|
|
|
|
/* save %eax since it used as the return value for __swap */
|
|
pushl %eax
|
|
/* interrupts are being reenabled, stop accumulating time */
|
|
call _int_latency_stop
|
|
/* restore __swap's %eax */
|
|
popl %eax
|
|
|
|
skipIntLatencyStop:
|
|
#endif
|
|
popfl
|
|
#if CONFIG_X86_IAMCU
|
|
/* Remember that eflags we stuck into the stack before the return
|
|
* address? need to get it out of there since the calling convention
|
|
* will not do that for us.
|
|
*/
|
|
popl %edx
|
|
movl %edx, (%esp)
|
|
#endif
|
|
|
|
#ifdef CONFIG_EXECUTION_BENCHMARKING
|
|
/* Save the eax and edx registers before reading the time stamp
|
|
* once done pop the values.
|
|
*/
|
|
cmp $0x1,__read_swap_end_time_value
|
|
jne time_read_not_needed
|
|
movw $0x2,__read_swap_end_time_value
|
|
read_tsc __common_var_swap_end_time
|
|
time_read_not_needed:
|
|
#endif
|
|
ret
|
|
|
|
#if defined(CONFIG_X86_IAMCU)
|
|
/**
|
|
*
|
|
* @brief Adjust stack/parameters before invoking thread entry function
|
|
*
|
|
* This function adjusts the initial stack frame created by _new_thread() such
|
|
* that the GDB stack frame unwinders recognize it as the outermost frame in
|
|
* the thread's stack. For targets that use the IAMCU calling convention, the
|
|
* first three arguments are popped into eax, edx, and ecx. The function then
|
|
* jumps to _thread_entry().
|
|
*
|
|
* GDB normally stops unwinding a stack when it detects that it has
|
|
* reached a function called main(). Kernel threads, however, do not have
|
|
* a main() function, and there does not appear to be a simple way of stopping
|
|
* the unwinding of the stack.
|
|
*
|
|
* SYS V Systems:
|
|
*
|
|
* Given the initial thread created by _new_thread(), GDB expects to find a
|
|
* return address on the stack immediately above the thread entry routine
|
|
* _thread_entry, in the location occupied by the initial EFLAGS.
|
|
* GDB attempts to examine the memory at this return address, which typically
|
|
* results in an invalid access to page 0 of memory.
|
|
*
|
|
* This function overwrites the initial EFLAGS with zero. When GDB subsequently
|
|
* attempts to examine memory at address zero, the PeekPoke driver detects
|
|
* an invalid access to address zero and returns an error, which causes the
|
|
* GDB stack unwinder to stop somewhat gracefully.
|
|
*
|
|
* The initial EFLAGS cannot be overwritten until after _Swap() has swapped in
|
|
* the new thread for the first time. This routine is called by _Swap() the
|
|
* first time that the new thread is swapped in, and it jumps to
|
|
* _thread_entry after it has done its work.
|
|
*
|
|
* IAMCU Systems:
|
|
*
|
|
* There is no EFLAGS on the stack when we get here. _thread_entry() takes
|
|
* four arguments, and we need to pop off the first three into the
|
|
* appropriate registers. Instead of using the 'call' instruction, we push
|
|
* a NULL return address onto the stack and jump into _thread_entry,
|
|
* ensuring the stack won't be unwound further. Placing some kind of return
|
|
* address on the stack is mandatory so this isn't conditionally compiled.
|
|
*
|
|
* __________________
|
|
* | param3 | <------ Top of the stack
|
|
* |__________________|
|
|
* | param2 | Stack Grows Down
|
|
* |__________________| |
|
|
* | param1 | V
|
|
* |__________________|
|
|
* | pEntry | <---- ESP when invoked by _Swap() on IAMCU
|
|
* |__________________|
|
|
* | initial EFLAGS | <---- ESP when invoked by _Swap() on Sys V
|
|
* |__________________| (Zeroed by this routine on Sys V)
|
|
*
|
|
* The address of the thread entry function needs to be in %edi when this is
|
|
* invoked. It will either be _thread_entry, or if userspace is enabled,
|
|
* _arch_drop_to_user_mode if this is a user thread.
|
|
*
|
|
* @return this routine does NOT return.
|
|
*/
|
|
|
|
SECTION_FUNC(TEXT, _x86_thread_entry_wrapper)
|
|
#ifdef CONFIG_X86_IAMCU
|
|
/* IAMCU calling convention has first 3 arguments supplied in
|
|
* registers not the stack
|
|
*/
|
|
pop %eax
|
|
pop %edx
|
|
pop %ecx
|
|
push $0 /* Null return address */
|
|
#endif
|
|
jmp *%edi
|
|
#endif
|