2018-01-26 07:24:15 +08:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2018 Intel Corporation
|
|
|
|
*
|
|
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
|
|
*/
|
2018-09-14 06:06:35 +08:00
|
|
|
#ifndef ZEPHYR_KERNEL_INCLUDE_KSWAP_H_
|
|
|
|
#define ZEPHYR_KERNEL_INCLUDE_KSWAP_H_
|
2018-01-26 07:24:15 +08:00
|
|
|
|
|
|
|
#include <ksched.h>
|
|
|
|
#include <kernel_arch_func.h>
|
|
|
|
|
|
|
|
#ifdef CONFIG_STACK_SENTINEL
|
|
|
|
extern void _check_stack_sentinel(void);
|
|
|
|
#else
|
|
|
|
#define _check_stack_sentinel() /**/
|
|
|
|
#endif
|
|
|
|
|
2018-02-10 06:34:05 +08:00
|
|
|
|
kernel: Rework SMP irq_lock() compatibility layer
This was wrong in two ways, one subtle and one awful.
The subtle problem was that the IRQ lock isn't actually globally
recursive, it gets reset when you context switch (i.e. a _Swap()
implicitly releases and reacquires it). So the recursive count I was
keeping needs to be per-thread or else we risk deadlock any time we
swap away from a thread holding the lock.
And because part of my brain apparently knew this, there was an
"optimization" in the code that tested the current count vs. zero
outside the lock, on the argument that if it was non-zero we must
already hold the lock. Which would be true of a per-thread counter,
but NOT a global one: the other CPU may be holding that lock, and this
test will tell you *you* do. The upshot is that a recursive
irq_lock() would almost always SUCCEED INCORRECTLY when there was lock
contention. That this didn't break more things is amazing to me.
The rework is actually simpler than the original, thankfully. Though
there are some further subtleties:
* The lock state implied by irq_lock() allows the lock to be
implicitly released on context switch (i.e. you can _Swap() with the
lock held at a recursion level higher than 1, which needs to allow
other processes to run). So return paths into threads from _Swap()
and interrupt/exception exit need to check and restore the global
lock state, spinning as needed.
* The idle loop design specifies a k_cpu_idle() function that is on
common architectures expected to enable interrupts (for obvious
reasons), but there is no place to put non-arch code to wire it into
the global lock accounting. So on SMP, even CPU0 needs to use the
"dumb" spinning idle loop.
Finally this patch contains a simple bugfix too, found by inspection:
the interrupt return code used when CONFIG_SWITCH is enabled wasn't
correctly setting the active flag on the threads, opening up the
potential for a race that might result in a thread being scheduled on
two CPUs simultaneously.
Signed-off-by: Andy Ross <andrew.j.ross@intel.com>
2018-04-13 03:50:05 +08:00
|
|
|
/* In SMP, the irq_lock() is a spinlock which is implicitly released
|
|
|
|
* and reacquired on context switch to preserve the existing
|
|
|
|
* semantics. This means that whenever we are about to return to a
|
|
|
|
* thread (via either _Swap() or interrupt/exception return!) we need
|
|
|
|
* to restore the lock state to whatever the thread's counter
|
|
|
|
* expects.
|
|
|
|
*/
|
|
|
|
void _smp_reacquire_global_lock(struct k_thread *thread);
|
|
|
|
void _smp_release_global_lock(struct k_thread *thread);
|
|
|
|
|
2018-01-26 07:24:15 +08:00
|
|
|
/* context switching and scheduling-related routines */
|
|
|
|
#ifdef CONFIG_USE_SWITCH
|
|
|
|
|
|
|
|
/* New style context switching. _arch_switch() is a lower level
|
|
|
|
* primitive that doesn't know about the scheduler or return value.
|
|
|
|
* Needed for SMP, where the scheduler requires spinlocking that we
|
|
|
|
* don't want to have to do in per-architecture assembly.
|
|
|
|
*/
|
2018-09-12 09:33:37 +08:00
|
|
|
static inline int _Swap(unsigned int key)
|
2018-01-26 07:24:15 +08:00
|
|
|
{
|
|
|
|
struct k_thread *new_thread, *old_thread;
|
2018-05-15 05:46:27 +08:00
|
|
|
int ret = 0;
|
2018-01-26 07:24:15 +08:00
|
|
|
|
2018-07-23 16:19:26 +08:00
|
|
|
#ifdef CONFIG_EXECUTION_BENCHMARKING
|
|
|
|
extern void read_timer_start_of_swap(void);
|
|
|
|
read_timer_start_of_swap();
|
|
|
|
#endif
|
|
|
|
|
2018-01-26 08:39:35 +08:00
|
|
|
old_thread = _current;
|
2018-01-26 07:24:15 +08:00
|
|
|
|
|
|
|
_check_stack_sentinel();
|
|
|
|
|
2018-07-04 21:03:03 +08:00
|
|
|
#ifdef CONFIG_TRACING
|
|
|
|
sys_trace_thread_switched_out();
|
|
|
|
#endif
|
|
|
|
|
2018-01-26 07:24:15 +08:00
|
|
|
new_thread = _get_next_ready_thread();
|
|
|
|
|
2018-05-15 05:46:27 +08:00
|
|
|
if (new_thread != old_thread) {
|
|
|
|
old_thread->swap_retval = -EAGAIN;
|
2018-01-26 07:24:15 +08:00
|
|
|
|
2018-01-30 06:55:20 +08:00
|
|
|
#ifdef CONFIG_SMP
|
2018-05-31 02:23:02 +08:00
|
|
|
_current_cpu->swap_ok = 0;
|
|
|
|
|
2018-05-15 05:46:27 +08:00
|
|
|
new_thread->base.cpu = _arch_curr_cpu()->id;
|
kernel: Rework SMP irq_lock() compatibility layer
This was wrong in two ways, one subtle and one awful.
The subtle problem was that the IRQ lock isn't actually globally
recursive, it gets reset when you context switch (i.e. a _Swap()
implicitly releases and reacquires it). So the recursive count I was
keeping needs to be per-thread or else we risk deadlock any time we
swap away from a thread holding the lock.
And because part of my brain apparently knew this, there was an
"optimization" in the code that tested the current count vs. zero
outside the lock, on the argument that if it was non-zero we must
already hold the lock. Which would be true of a per-thread counter,
but NOT a global one: the other CPU may be holding that lock, and this
test will tell you *you* do. The upshot is that a recursive
irq_lock() would almost always SUCCEED INCORRECTLY when there was lock
contention. That this didn't break more things is amazing to me.
The rework is actually simpler than the original, thankfully. Though
there are some further subtleties:
* The lock state implied by irq_lock() allows the lock to be
implicitly released on context switch (i.e. you can _Swap() with the
lock held at a recursion level higher than 1, which needs to allow
other processes to run). So return paths into threads from _Swap()
and interrupt/exception exit need to check and restore the global
lock state, spinning as needed.
* The idle loop design specifies a k_cpu_idle() function that is on
common architectures expected to enable interrupts (for obvious
reasons), but there is no place to put non-arch code to wire it into
the global lock accounting. So on SMP, even CPU0 needs to use the
"dumb" spinning idle loop.
Finally this patch contains a simple bugfix too, found by inspection:
the interrupt return code used when CONFIG_SWITCH is enabled wasn't
correctly setting the active flag on the threads, opening up the
potential for a race that might result in a thread being scheduled on
two CPUs simultaneously.
Signed-off-by: Andy Ross <andrew.j.ross@intel.com>
2018-04-13 03:50:05 +08:00
|
|
|
|
2018-05-15 05:46:27 +08:00
|
|
|
_smp_release_global_lock(new_thread);
|
2018-01-30 06:55:20 +08:00
|
|
|
#endif
|
|
|
|
|
2018-05-15 05:46:27 +08:00
|
|
|
_current = new_thread;
|
|
|
|
_arch_switch(new_thread->switch_handle,
|
|
|
|
&old_thread->switch_handle);
|
2018-01-26 07:24:15 +08:00
|
|
|
|
2018-05-15 05:46:27 +08:00
|
|
|
ret = _current->swap_retval;
|
|
|
|
}
|
2018-01-26 07:24:15 +08:00
|
|
|
|
2018-07-04 21:03:03 +08:00
|
|
|
#ifdef CONFIG_TRACING
|
|
|
|
sys_trace_thread_switched_in();
|
|
|
|
#endif
|
|
|
|
|
2018-01-26 07:24:15 +08:00
|
|
|
irq_unlock(key);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
#else /* !CONFIG_USE_SWITCH */
|
|
|
|
|
2018-09-12 09:33:37 +08:00
|
|
|
extern int __swap(unsigned int key);
|
2018-01-26 07:24:15 +08:00
|
|
|
|
2018-09-12 09:33:37 +08:00
|
|
|
static inline int _Swap(unsigned int key)
|
2018-01-26 07:24:15 +08:00
|
|
|
{
|
2018-09-12 09:33:37 +08:00
|
|
|
int ret;
|
2018-01-26 07:24:15 +08:00
|
|
|
_check_stack_sentinel();
|
|
|
|
|
2018-10-26 22:54:16 +08:00
|
|
|
#ifndef CONFIG_ARM
|
2018-07-04 21:03:03 +08:00
|
|
|
#ifdef CONFIG_TRACING
|
|
|
|
sys_trace_thread_switched_out();
|
2018-10-26 22:54:16 +08:00
|
|
|
#endif
|
2018-07-04 21:03:03 +08:00
|
|
|
#endif
|
|
|
|
ret = __swap(key);
|
2018-10-26 22:54:16 +08:00
|
|
|
#ifndef CONFIG_ARM
|
2018-07-04 21:03:03 +08:00
|
|
|
#ifdef CONFIG_TRACING
|
|
|
|
sys_trace_thread_switched_in();
|
2018-10-26 22:54:16 +08:00
|
|
|
#endif
|
2018-07-04 21:03:03 +08:00
|
|
|
#endif
|
|
|
|
|
|
|
|
return ret;
|
2018-01-26 07:24:15 +08:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2018-09-14 06:06:35 +08:00
|
|
|
#endif /* ZEPHYR_KERNEL_INCLUDE_KSWAP_H_ */
|