zephyr/arch/xtensa/core/xtensa-asm2.c

/*
 * Copyright (c) 2017, Intel Corporation
 *
 * SPDX-License-Identifier: Apache-2.0
 */
#include <string.h>
#include <xtensa-asm2.h>
#include <zephyr/kernel.h>
#include <ksched.h>
#include <zephyr/kernel_structs.h>
#include <kernel_internal.h>
#include <kswap.h>
#include <_soc_inthandlers.h>
#include <zephyr/toolchain.h>
#include <zephyr/logging/log.h>
#include <offsets.h>
#include <zsr.h>

LOG_MODULE_DECLARE(os, CONFIG_KERNEL_LOG_LEVEL);

extern char xtensa_arch_except_epc[];

void *xtensa_init_stack(struct k_thread *thread, int *stack_top,
			void (*entry)(void *, void *, void *),
			void *arg1, void *arg2, void *arg3)
{
	void *ret;
	_xtensa_irq_stack_frame_a11_t *frame;

	/* Not-a-cpu ID Ensures that the first time this is run, the
	 * stack will be invalidated.  That covers the edge case of
	 * restarting a thread on a stack that had previously been run
	 * on one CPU, but then initialized on this one, and
	 * potentially run THERE and not HERE.
	 */
	thread->arch.last_cpu = -1;

	/* We cheat and shave 16 bytes off, the top four words are the
	 * A0-A3 spill area for the caller of the entry function,
	 * which doesn't exist.  It will never be touched, so we
	 * arrange to enter the function with a CALLINC of 1 and a
	 * stack pointer 16 bytes above the top, so its ENTRY at the
	 * start will decrement the stack pointer by 16.
	 */
	const int bsasz = sizeof(*frame) - 16;

	frame = (void *)(((char *) stack_top) - bsasz);

	(void)memset(frame, 0, bsasz);

	frame->bsa.pc = (uintptr_t)z_thread_entry;
	frame->bsa.ps = PS_WOE | PS_UM | PS_CALLINC(1);

#if XCHAL_HAVE_THREADPTR && defined(CONFIG_THREAD_LOCAL_STORAGE)
	frame->bsa.threadptr = thread->tls;
#endif

	/* Arguments to z_thread_entry().  Remember these start at A6,
	 * which will be rotated into A2 by the ENTRY instruction that
	 * begins the C function.  And A4-A7 and A8-A11 are optional
	 * quads that live below the BSA!
	 */
	frame->a7 = (uintptr_t)arg1;  /* a7 */
	frame->a6 = (uintptr_t)entry; /* a6 */
	frame->a5 = 0;                /* a5 */
	frame->a4 = 0;                /* a4 */

	frame->a11 = 0;                /* a11 */
	frame->a10 = 0;                /* a10 */
	frame->a9  = (uintptr_t)arg3;  /* a9 */
	frame->a8  = (uintptr_t)arg2;  /* a8 */

	/* Finally push the BSA pointer and return the stack pointer
	 * as the handle
	 */
	frame->ptr_to_bsa = (void *)&frame->bsa;
	ret = &frame->ptr_to_bsa;

	return ret;
}

void arch_new_thread(struct k_thread *thread, k_thread_stack_t *stack,
		     char *stack_ptr, k_thread_entry_t entry,
		     void *p1, void *p2, void *p3)
{
	thread->switch_handle = xtensa_init_stack(thread,
						  (int *)stack_ptr, entry,
						  p1, p2, p3);
#ifdef CONFIG_KERNEL_COHERENCE
	__ASSERT((((size_t)stack) % XCHAL_DCACHE_LINESIZE) == 0, "");
	__ASSERT((((size_t)stack_ptr) % XCHAL_DCACHE_LINESIZE) == 0, "");
	sys_cache_data_flush_and_invd_range(stack, (char *)stack_ptr - (char *)stack);
#endif
}

void z_irq_spurious(const void *arg)
{
	int irqs, ie;

	ARG_UNUSED(arg);

	__asm__ volatile("rsr.interrupt %0" : "=r"(irqs));
	__asm__ volatile("rsr.intenable %0" : "=r"(ie));
	LOG_ERR(" ** Spurious INTERRUPT(s) %p, INTENABLE = %p",
		(void *)irqs, (void *)ie);
	z_xtensa_fatal_error(K_ERR_SPURIOUS_IRQ, NULL);
}

void z_xtensa_dump_stack(const z_arch_esf_t *stack)
{
	_xtensa_irq_stack_frame_raw_t *frame = (void *)stack;
	_xtensa_irq_bsa_t *bsa = frame->ptr_to_bsa;
	uintptr_t num_high_regs;
	int reg_blks_remaining;

	/* Calculate number of high registers. */
	num_high_regs = (uint8_t *)bsa - (uint8_t *)frame + sizeof(void *);
	num_high_regs /= sizeof(uintptr_t);

	/* And high registers are always comes in 4 in a block. */
	reg_blks_remaining = (int)num_high_regs / 4;

	LOG_ERR(" **  A0 %p  SP %p  A2 %p  A3 %p",
		(void *)bsa->a0,
		((char *)bsa + sizeof(*bsa)),
		(void *)bsa->a2, (void *)bsa->a3);

	if (reg_blks_remaining > 0) {
		reg_blks_remaining--;

		LOG_ERR(" **  A4 %p  A5 %p  A6 %p  A7 %p",
			(void *)frame->blks[reg_blks_remaining].r0,
			(void *)frame->blks[reg_blks_remaining].r1,
			(void *)frame->blks[reg_blks_remaining].r2,
			(void *)frame->blks[reg_blks_remaining].r3);
	}

	if (reg_blks_remaining > 0) {
		reg_blks_remaining--;

		LOG_ERR(" **  A8 %p  A9 %p A10 %p A11 %p",
			(void *)frame->blks[reg_blks_remaining].r0,
			(void *)frame->blks[reg_blks_remaining].r1,
			(void *)frame->blks[reg_blks_remaining].r2,
			(void *)frame->blks[reg_blks_remaining].r3);
	}

	if (reg_blks_remaining > 0) {
		reg_blks_remaining--;

		LOG_ERR(" ** A12 %p A13 %p A14 %p A15 %p",
			(void *)frame->blks[reg_blks_remaining].r0,
			(void *)frame->blks[reg_blks_remaining].r1,
			(void *)frame->blks[reg_blks_remaining].r2,
			(void *)frame->blks[reg_blks_remaining].r3);
	}

#if XCHAL_HAVE_LOOPS
	LOG_ERR(" ** LBEG %p LEND %p LCOUNT %p",
		(void *)bsa->lbeg,
		(void *)bsa->lend,
		(void *)bsa->lcount);
#endif

	LOG_ERR(" ** SAR %p", (void *)bsa->sar);

#ifdef CONFIG_XTENSA_MMU
	uint32_t vaddrstatus, vaddr0, vaddr1;

	__asm__ volatile("rsr.vaddrstatus %0" : "=r"(vaddrstatus));
	__asm__ volatile("rsr.vaddr0 %0" : "=r"(vaddr0));
	__asm__ volatile("rsr.vaddr1 %0" : "=r"(vaddr1));

	LOG_ERR(" ** VADDRSTATUS %p VADDR0 %p VADDR1 %p",
		(void *)vaddrstatus, (void *)vaddr0, (void *)vaddr1);
#endif /* CONFIG_XTENSA_MMU */
}

static inline unsigned int get_bits(int offset, int num_bits, unsigned int val)
{
	int mask;

	mask = BIT(num_bits) - 1;
	val = val >> offset;
	return val & mask;
}

static ALWAYS_INLINE void usage_stop(void)
{
#ifdef CONFIG_SCHED_THREAD_USAGE
	z_sched_usage_stop();
#endif
}

#ifdef CONFIG_MULTITHREADING
void *z_arch_get_next_switch_handle(struct k_thread *interrupted)
{
	return _current_cpu->nested <= 1 ?
		z_get_next_switch_handle(interrupted) : interrupted;
}
#else
void *z_arch_get_next_switch_handle(struct k_thread *interrupted)
{
	return interrupted;
}
#endif /* CONFIG_MULTITHREADING */

static inline void *return_to(void *interrupted)
{
	return z_arch_get_next_switch_handle(interrupted);
}

/* The wrapper code lives here instead of in the python script that
 * generates _xtensa_handle_one_int*().  Seems cleaner, still kind of
 * ugly.
 *
 * This may be unused depending on number of interrupt levels
 * supported by the SoC.
 */
#define DEF_INT_C_HANDLER(l)				\
__unused void *xtensa_int##l##_c(void *interrupted_stack)	\
{							   \
	uint32_t irqs, intenable, m;			   \
	usage_stop();					   \
	__asm__ volatile("rsr.interrupt %0" : "=r"(irqs)); \
	__asm__ volatile("rsr.intenable %0" : "=r"(intenable)); \
	irqs &= intenable;					\
	while ((m = _xtensa_handle_one_int##l(irqs))) {		\
		irqs ^= m;					\
		__asm__ volatile("wsr.intclear %0" : : "r"(m)); \
	}							\
	return return_to(interrupted_stack);		\
}

#if XCHAL_NMILEVEL >= 2
DEF_INT_C_HANDLER(2)
#endif

#if XCHAL_NMILEVEL >= 3
DEF_INT_C_HANDLER(3)
#endif

#if XCHAL_NMILEVEL >= 4
DEF_INT_C_HANDLER(4)
#endif

#if XCHAL_NMILEVEL >= 5
DEF_INT_C_HANDLER(5)
#endif

#if XCHAL_NMILEVEL >= 6
DEF_INT_C_HANDLER(6)
#endif

#if XCHAL_NMILEVEL >= 7
DEF_INT_C_HANDLER(7)
#endif

static inline DEF_INT_C_HANDLER(1)

/* C handler for level 1 exceptions/interrupts.  Hooked from the
 * DEF_EXCINT 1 vector declaration in assembly code.  This one looks
 * different because exceptions and interrupts land at the same
 * vector; other interrupt levels have their own vectors.
 */
void *xtensa_excint1_c(int *interrupted_stack)
{
	int cause, vaddr;
	_xtensa_irq_bsa_t *bsa = (void *)*(int **)interrupted_stack;
	bool is_fatal_error = false;
	uint32_t ps;
	void *pc;

	__asm__ volatile("rsr.exccause %0" : "=r"(cause));

#ifdef CONFIG_XTENSA_MMU
	/* TLB miss exception comes through level 1 interrupt also.
	 * We need to preserve execution context after we have handled
	 * the TLB miss, so we cannot unconditionally unmask interrupts.
	 * For other cause, we can unmask interrupts so this would act
	 * the same as if there is no MMU.
	 */
	switch (cause) {
	case EXCCAUSE_ITLB_MISS:
		/* Instruction TLB miss */
		__fallthrough;
	case EXCCAUSE_DTLB_MISS:
		/* Data TLB miss */

		/* Do not unmask interrupt while handling TLB misses. */
		break;
	default:
		/* For others, we can unmask interrupts. */
		bsa->ps &= ~PS_INTLEVEL_MASK;
		break;
	}
#endif /* CONFIG_XTENSA_MMU */

	switch (cause) {
	case EXCCAUSE_LEVEL1_INTERRUPT:
		return xtensa_int1_c(interrupted_stack);
	case EXCCAUSE_SYSCALL:
		/* Just report it to the console for now */
		LOG_ERR(" ** SYSCALL PS %p PC %p",
			(void *)bsa->ps, (void *)bsa->pc);
		z_xtensa_dump_stack(interrupted_stack);

		/* Xtensa exceptions don't automatically advance PC,
		 * have to skip the SYSCALL instruction manually or
		 * else it will just loop forever
		 */
		bsa->pc += 3;
		break;
#ifdef CONFIG_XTENSA_MMU
	case EXCCAUSE_ITLB_MISS:
		/* Instruction TLB miss */
		__fallthrough;
	case EXCCAUSE_DTLB_MISS:
		/* Data TLB miss */

		/**
		 * The way it works is, when we try to access an address
		 * that is not mapped, we will have a miss. The HW then
		 * will try to get the correspondent memory in the page
		 * table. As the page table is not mapped in memory we will
		 * have a second miss, which will trigger an exception.
		 * In the exception (here) what we do is to exploit this
		 * hardware capability just trying to load the page table
		 * (not mapped address), which will cause a miss, but then
		 * the hardware will automatically map it again from
		 * the page table. This time it will work since the page
		 * necessary to map the page table itself are wired map.
		 */
		__asm__ volatile("wsr a0, " ZSR_EXTRA0_STR "\n\t"
				 "rsr.ptevaddr a0\n\t"
				 "l32i a0, a0, 0\n\t"
				 "rsr a0, " ZSR_EXTRA0_STR "\n\t"
				 "rsync"
				 : : : "a0", "memory");

		/* Since we are dealing with TLB misses, we will probably not
		 * want to switch to another thread.
		 */
		return interrupted_stack;
#endif /* CONFIG_XTENSA_MMU */
	default:
		ps = bsa->ps;
		pc = (void *)bsa->pc;

		__asm__ volatile("rsr.excvaddr %0" : "=r"(vaddr));

		/* Default for exception */
		int reason = K_ERR_CPU_EXCEPTION;

		/* We need to distinguish between an ill in xtensa_arch_except,
		 * e.g for k_panic, and any other ill. For exceptions caused by
		 * xtensa_arch_except calls, we also need to pass the reason_p
		 * to z_xtensa_fatal_error. Since the ARCH_EXCEPT frame is in the
		 * BSA, the first arg reason_p is stored at the A2 offset.
		 * We assign EXCCAUSE the unused, reserved code 63; this may be
		 * problematic if the app or new boards also decide to repurpose
		 * this code.
		 */
		if ((pc ==  (void *) &xtensa_arch_except_epc) && (cause == 0)) {
			cause = 63;
			__asm__ volatile("wsr.exccause %0" : : "r"(cause));
			reason = bsa->a2;
		}

		LOG_ERR(" ** FATAL EXCEPTION");
		LOG_ERR(" ** CPU %d EXCCAUSE %d (%s)",
			arch_curr_cpu()->id, cause,
			z_xtensa_exccause(cause));
		LOG_ERR(" **  PC %p VADDR %p",
			pc, (void *)vaddr);
		LOG_ERR(" **  PS %p", (void *)bsa->ps);
		LOG_ERR(" **    (INTLEVEL:%d EXCM: %d UM:%d RING:%d WOE:%d OWB:%d CALLINC:%d)",
			get_bits(0, 4, ps), get_bits(4, 1, ps),
			get_bits(5, 1, ps), get_bits(6, 2, ps),
			get_bits(18, 1, ps),
			get_bits(8, 4, ps), get_bits(16, 2, ps));

		/* FIXME: legacy xtensa port reported "HW" exception
		 * for all unhandled exceptions, which seems incorrect
		 * as these are software errors.  Should clean this
		 * up.
		 */
		z_xtensa_fatal_error(reason,
				     (void *)interrupted_stack);
		break;
	}


	switch (cause) {
	case EXCCAUSE_SYSCALL:
	case EXCCAUSE_LEVEL1_INTERRUPT:
	case EXCCAUSE_ALLOCA:
	case EXCCAUSE_ITLB_MISS:
	case EXCCAUSE_DTLB_MISS:
		is_fatal_error = false;
		break;
	default:
		is_fatal_error = true;
		break;
	}

	if (is_fatal_error) {
		uint32_t ignore;

		/* We are going to manipulate _current_cpu->nested manually.
		 * Since the error is fatal, for recoverable errors, code
		 * execution must not return back to the current thread as
		 * it is being terminated (via above z_xtensa_fatal_error()).
		 * So we need to prevent more interrupts coming in which
		 * will affect the nested value as we are going outside of
		 * normal interrupt handling procedure.
		 *
		 * Setting nested to 1 has two effects:
		 * 1. Force return_to() to choose a new thread.
		 *    Since the current thread is being terminated, it will
		 *    not be chosen again.
		 * 2. When context switches to the newly chosen thread,
		 *    nested must be zero for normal code execution,
		 *    as that is not in interrupt context at all.
		 *    After returning from this function, the rest of
		 *    interrupt handling code will decrement nested,
		 *    resulting it being zero before switching to another
		 *    thread.
		 */
		__asm__ volatile("rsil %0, %1"
				: "=r" (ignore) : "i"(XCHAL_NMILEVEL));

		_current_cpu->nested = 1;
	}

	return return_to(interrupted_stack);
}

#if defined(CONFIG_GDBSTUB)
void *xtensa_debugint_c(int *interrupted_stack)
{
	extern void z_gdb_isr(z_arch_esf_t *esf);

	z_gdb_isr((void *)interrupted_stack);

	return return_to(interrupted_stack);
}
#endif

int z_xtensa_irq_is_enabled(unsigned int irq)
{
	uint32_t ie;

	__asm__ volatile("rsr.intenable %0" : "=r"(ie));

	return (ie & (1 << irq)) != 0U;
}

#ifdef CONFIG_XTENSA_MORE_SPIN_RELAX_NOPS
/* Some compilers might "optimize out" (i.e. remove) continuous NOPs.
 * So force no optimization to avoid that.
 */
__no_optimization
void arch_spin_relax(void)
{
#define NOP1(_, __) __asm__ volatile("nop.n;");
	LISTIFY(CONFIG_XTENSA_NUM_SPIN_RELAX_NOPS, NOP1, (;))
#undef NOP1
}
#endif /* CONFIG_XTENSA_MORE_SPIN_RELAX_NOPS */