Verified Commit 7e6dbad0 authored by Felix Kopp's avatar Felix Kopp
Browse files

sched: complete rework of context switching

The old strategy was to only do context switching
from within the PendSV handler.  This worked fine
until now because all syscalls were handled either
atomically or just returned -EAGAIN if the
resource was locked or busy.  However, with the
introduction of I/O wait, we need to be able to
sleep directly from within the kernel by moving
the context switching completely into the kernel.
parent 60f1ebea
......@@ -22,12 +22,14 @@ target_sources(ardix_arch PRIVATE
atom_get_put.S
atom.c
atomic.c
do_switch.S
entry.c
handle_fault.c
handle_fault.S
handle_pend_sv.S
handle_reset.c
handle_svc.S
leave.S
mutex.S
sched.c
serial.c
......
/* See the end of this file for copyright, license, and warranty information. */
#pragma once
.include "asm.S"
/**
* @brief Perform a syscall.
*
* This is called by the syscall exception handler. It is responsible for
* finishing the context switch, obtaining the syscall number and arguments,
* and invoking the respective system call. If the return value of this
* function is nonzero, the scheduler is invoked after the call and before
* returning to userspace.
*
* @param sp current stack pointer
* @returns Whether rescheduling is required
*/
int arch_enter(void *sp);
.text
/* void _do_switch(struct context *old, struct context *new); */
func_begin _do_switch
/* ldm/stm can't use sp in the reglist, we need to store it individually */
stmia r0!, {r4-r11}
str sp, [r0]
str lr, [r0, #4] /* this becomes pc when we are switched back to */
ldmia r1!, {r4-r11}
ldr sp, [r1]
ldr pc, [r1, #4] /* this is the lr that we stored */
func_end _do_switch
/*
* This file is part of Ardix.
......
/* See the end of this file for copyright, license, and warranty information. */
#include <arch-generic/entry.h>
#include <arch/hardware.h>
#include <ardix/types.h>
......@@ -17,13 +16,12 @@
extern uint16_t __syscall_return_point;
#endif
int arch_enter(void *sp)
void arch_enter(struct exc_context *context)
{
struct reg_snapshot *regs = sp;
enum syscall sc_num = arch_syscall_num(regs);
int (*handler)(sysarg_t arg1, sysarg_t arg2, sysarg_t arg3,
sysarg_t arg4, sysarg_t arg5, sysarg_t arg6);
int sc_ret;
enum syscall number = sc_num(context);
long (*handler)(sysarg_t arg1, sysarg_t arg2, sysarg_t arg3,
sysarg_t arg4, sysarg_t arg5, sysarg_t arg6);
long sc_ret;
# ifdef CONFIG_CHECK_SYSCALL_SOURCE
/*
......@@ -32,31 +30,28 @@ int arch_enter(void *sp)
* the instructions are always 2-byte aligned. Additionally, the PC
* points to the instruction *after* the SVC, not SVC itself.
*/
if (((uintptr_t)regs->hw.pc & 0xfffffffe) != (uintptr_t)&__syscall_return_point) {
arch_syscall_set_rval(regs, -EACCES);
if (((uintptr_t)regs->sp->pc & 0xfffffffe) != (uintptr_t)&__syscall_return_point) {
sc_set_rval(regs, -EACCES);
return;
}
# endif
if (sc_num > NSYSCALLS) {
arch_syscall_set_rval(regs, -ENOSYS);
return 0;
if (number > NSYSCALLS) {
sc_set_rval(context, -ENOSYS);
return;
}
handler = sys_table[sc_num];
handler = sys_table[number];
if (handler == NULL) {
arch_syscall_set_rval(regs, -ENOSYS);
return 0;
sc_set_rval(context, -ENOSYS);
return;
}
/* TODO: not every syscall uses the max amount of parameters (duh) */
sc_ret = handler(arch_syscall_arg1(regs), arch_syscall_arg2(regs), arch_syscall_arg3(regs),
arch_syscall_arg4(regs), arch_syscall_arg5(regs), arch_syscall_arg6(regs));
sc_ret = handler(sc_arg1(context), sc_arg2(context), sc_arg3(context),
sc_arg4(context), sc_arg5(context), sc_arg6(context));
arch_syscall_set_rval(regs, sc_ret);
int ret = need_resched;
need_resched = 0;
return ret;
sc_set_rval(context, sc_ret);
}
/*
......
......@@ -4,35 +4,35 @@
.text
/* __naked __noreturn void arch_handle_fault(struct reg_snapshot *regs, int irqnum); */
.extern arch_handle_fault
/* __naked __noreturn void handle_fault(struct exc_context *context, int irqnum); */
.extern handle_fault
func_begin handle_hard_fault
push {r4-r11,lr}
prepare_entry
mov r0, sp
mov r1, #-13 /* IRQNO_HARD_FAULT */
b arch_handle_fault
b handle_fault
func_end handle_hard_fault
func_begin handle_mm_fault
push {r4-r11,lr}
prepare_entry
mov r0, sp
mov r1, #-12 /* IRQNO_MM_FAULT */
b arch_handle_fault
b handle_fault
func_end handle_mm_fault
func_begin handle_bus_fault
push {r4-r11,lr}
prepare_entry
mov r0, sp
mov r1, #-11 /* IRQNO_BUS_FAULT */
b arch_handle_fault
b handle_fault
func_end handle_bus_fault
func_begin handle_usage_fault
push {r4-r11,lr}
prepare_entry
mov r0, sp
mov r1, #-10 /* IRQNO_USAGE_FAULT */
b arch_handle_fault
b handle_fault
func_end handle_usage_fault
/*
......
......@@ -7,32 +7,37 @@
#include <toolchain.h>
static void uart_write_sync(const char *s)
{
char c;
while ((c = *s++) != '\0') {
mom_are_we_there_yet(UART->UART_SR & UART_SR_TXRDY);
UART->UART_THR = c;
}
}
/** Setup UART to manual byte-by-byte control */
static inline void uart_emergency_setup(void)
{
UART->UART_IDR = 0xffffffff;
mom_are_we_there_yet(UART->UART_SR & UART_SR_TXRDY);
UART->UART_PTCR = UART_PTCR_RXTDIS | UART_PTCR_TXTDIS;
UART->UART_CR = UART_CR_RXDIS | UART_CR_RSTRX
| UART_CR_TXDIS | UART_CR_RSTTX;
UART->UART_IDR = 0xffffffff;
UART->UART_CR = UART_CR_TXEN;
UART->UART_CR = UART_CR_RXEN | UART_CR_TXEN;
}
static void uart_write_sync(const char *s)
{
char c;
while ((c = *s++) != '\0') {
mom_are_we_there_yet(UART->UART_SR & UART_SR_TXRDY);
UART->UART_THR = c;
}
UART->UART_THR = '\0';
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wswitch"
static inline void print_err_msg(enum irqno irqno)
{
uart_write_sync("\n\n########## SERIOUS BRUH MOMENT! ##########\n");
switch (irqno) {
case IRQNO_HARD_FAULT:
uart_write_sync("Hard");
......@@ -48,11 +53,11 @@ static inline void print_err_msg(enum irqno irqno)
break;
}
uart_write_sync(" Fault encountered, system halted.\n\n");
uart_write_sync(" Fault encountered, cannot continue\nRegister dump:\n");
}
#pragma GCC diagnostic pop /* -Wswitch */
static void reg_to_str(char *dest, uint32_t val)
static void reg_to_str(char *dest, word_t val)
{
for (int i = 28; i >= 0; i -= 4) {
uint8_t digit = (val >> i) & 0x0f;
......@@ -66,91 +71,56 @@ static void reg_to_str(char *dest, uint32_t val)
}
}
static void print_regs(struct reg_snapshot *regs)
static void print_reg(const char *name, word_t val)
{
static char reg_line[] = "r0 = 0x????????\n";
char *reg_val = &reg_line[8]; /* first question mark */
reg_to_str(reg_val, regs->hw.r0);
uart_write_sync(reg_line);
reg_line[1] = '1';
reg_to_str(reg_val, regs->hw.r1);
uart_write_sync(reg_line);
reg_line[1] = '2';
reg_to_str(reg_val, regs->hw.r2);
uart_write_sync(reg_line);
reg_line[1] = '3';
reg_to_str(reg_val, regs->hw.r3);
uart_write_sync(reg_line);
reg_line[1] = '4';
reg_to_str(reg_val, regs->sw.r4);
uart_write_sync(reg_line);
reg_line[1] = '5';
reg_to_str(reg_val, regs->sw.r5);
uart_write_sync(reg_line);
reg_line[1] = '6';
reg_to_str(reg_val, regs->sw.r6);
uart_write_sync(reg_line);
reg_line[1] = '7';
reg_to_str(reg_val, regs->sw.r7);
uart_write_sync(reg_line);
reg_line[1] = '8';
reg_to_str(reg_val, regs->sw.r8);
uart_write_sync(reg_line);
reg_line[1] = '9';
reg_to_str(reg_val, regs->sw.r9);
uart_write_sync(reg_line);
reg_line[1] = '1';
reg_line[2] = '0';
reg_to_str(reg_val, regs->sw.r10);
uart_write_sync(reg_line);
reg_line[2] = '1';
reg_to_str(reg_val, regs->sw.r11);
uart_write_sync(reg_line);
reg_line[2] = '2';
reg_to_str(reg_val, regs->hw.r12);
uart_write_sync(reg_line);
reg_line[0] = 's';
reg_line[1] = 'p';
reg_line[2] = ' ';
reg_to_str(reg_val, (uint32_t)(regs + 1)); /* where SP was before reg save */
uart_write_sync(reg_line);
reg_line[0] = 'l';
reg_line[1] = 'r';
reg_to_str(reg_val, (uint32_t)regs->hw.lr);
uart_write_sync(reg_line);
reg_line[0] = 'p';
reg_line[1] = 'c';
reg_to_str(reg_val, (uint32_t)regs->hw.pc);
uart_write_sync(reg_line);
/* static saves stack space, which might be limited */
static char line[] = "???? = 0x????????\n";
char c;
char *name_pos = line;
while ((c = *name++) != '\0')
*name_pos++ = c;
while (name_pos < &line[4])
*name_pos++ = ' ';
reg_to_str(&line[9], val);
uart_write_sync(line);
}
static void print_regs(struct exc_context *context)
{
print_reg("R0", context->sp->r0);
print_reg("R1", context->sp->r1);
print_reg("R2", context->sp->r2);
print_reg("R3", context->sp->r3);
print_reg("R4", context->r4);
print_reg("R5", context->r5);
print_reg("R6", context->r6);
print_reg("R7", context->r7);
print_reg("R8", context->r8);
print_reg("R9", context->r9);
print_reg("R10", context->r10);
print_reg("R11", context->r11);
print_reg("R12", context->sp->r12);
print_reg("SP", *(word_t *)&context->sp);
print_reg("LR", *(word_t *)&context->sp->lr);
print_reg("PC", *(word_t *)&context->sp->pc);
print_reg("xPSR", context->sp->psr);
}
#include <arch/debug.h>
__naked __noreturn void arch_handle_fault(struct reg_snapshot *regs, enum irqno irqno)
__naked __noreturn void handle_fault(struct exc_context *context, enum irqno irqno)
{
uart_emergency_setup();
print_err_msg(irqno);
print_regs(regs);
print_regs(context);
if (SCB->HFSR & SCB_HFSR_FORCED_Msk)
print_reg("CFSR", SCB->CFSR);
uart_write_sync("\nSystem halted, goodbye\n\n");
/* give developers a chance to inspect the system */
__breakpoint;
/* but never leave this function */
while (1);
}
......
......@@ -4,42 +4,15 @@
.text
/* void *sched_switch(void *curr_sp); */
.extern sched_switch
/* void schedule(void); */
.extern schedule
/* void handle_pend_sv(void); */
func_begin handle_pend_sv
/*
* Some registers have already been saved by hardware at this point,
* we only need to take care of r4-r11 and lr (the latter of which is
* required because lr is overwritten when entering the irq).
* The stuff we push onto the stack manually looks about like this:
*
* <<< stack grow direction (decreasing addresses) <<<
* r4 r5 r6 r7 r8 r9 r10 r11 lr
*/
push {r4-r11,lr}
/*
* Now that our stack is completely saved, we can proceed to call the
* Kernel's scheduler. This updates `_current_process` to the process
* we want to execute next.
*/
/* TODO: Implement banked stack pointer */
mov r0, sp
bl sched_switch /* sp = sched_switch(sp); */
mov sp, r0
/*
* The new stack pointer contains the state of the new process, so we
* load it into our registers using the same procedure as above,
* just in reverse order.
*/
pop {r4-r11,lr}
clrex
prepare_entry
bl schedule
prepare_leave
bx lr
......
......@@ -5,9 +5,9 @@
#include <config.h>
#ifdef DEBUG
#define __breakpoint __asm__ volatile("\tbkpt\n" ::: )
# define __breakpoint __asm__ volatile("bkpt")
#else
#define __breakpoint
# define __breakpoint
#endif
/*
......
......@@ -3,17 +3,20 @@
#pragma once
#include <stdint.h>
#include <toolchain.h>
typedef uint32_t word_t;
typedef uint32_t sysarg_t;
/** Current system frequency in Hertz. */
extern volatile uint32_t SystemCoreClock;
/**
* All registers that are automatically saved by hardware routines when entering
* an IRQ, in the correct order.
* @brief Hardware context save upon entering kernel space.
* This is stored on the stack of the thread that ran before exception entry,
* i.e. PSP if user space and MSP if kernel space.
*/
struct reg_hw_snapshot {
struct hw_context {
word_t r0;
word_t r1;
word_t r2;
......@@ -24,11 +27,12 @@ struct reg_hw_snapshot {
word_t psr;
};
struct reg_snapshot {
word_t r0;
word_t r1;
word_t r2;
word_t r3;
/**
* @brief Software context save from an exception handler upon entering kernel space.
* This is always stored on the main stack.
* The `prepare_entry` macro in `arch/include/asm.S` creates this snapshot.
*/
struct exc_context {
word_t r4;
word_t r5;
word_t r6;
......@@ -37,34 +41,92 @@ struct reg_snapshot {
word_t r9;
word_t r10;
word_t r11;
/*
* ATTENTION: the following registers might actually be stored on the
* other stack; don't access them unless you know exactly
* what you're doing
* Old stack pointer used before exception entry.
* Bit 2 in lr defines which stack was used.
*/
struct hw_context *sp;
void *lr;
};
/**
* @brief Used for in-kernel context switching.
* This is where `do_switch()` stores the register values.
*/
struct context {
word_t r4;
word_t r5;
word_t r6;
word_t r7;
word_t r8;
word_t r9;
word_t r10;
word_t r11;
void *sp;
void *pc;
};
word_t _r12;
void *_lr; /* alias r14 */
void *_pc; /* alias r15 */
word_t _psr;
/**
* @brief Task Control Block.
* This is a low level structure used by `do_switch()` to do the actual context
* switching,
*/
struct tcb {
struct context context;
struct hw_context *hw_context;
};
#define arch_syscall_num(reg_snap) ((reg_snap)->r7)
#define arch_syscall_arg1(reg_snap) ((reg_snap)->r0)
#define arch_syscall_arg2(reg_snap) ((reg_snap)->r1)
#define arch_syscall_arg3(reg_snap) ((reg_snap)->r2)
#define arch_syscall_arg4(reg_snap) ((reg_snap)->r3)
#define arch_syscall_arg5(reg_snap) ((reg_snap)->r4)
#define arch_syscall_arg6(reg_snap) ((reg_snap)->r5)
__always_inline sysarg_t sc_num(const struct exc_context *ctx)
{
return ctx->r7;
}
__always_inline sysarg_t sc_arg1(const struct exc_context *ctx)
{
return ctx->sp->r0;
}
#define arch_syscall_set_rval(reg_snap, val) ((reg_snap)->r0 = (word_t)(val));
__always_inline sysarg_t sc_arg2(const struct exc_context *ctx)
{
return ctx->sp->r1;
}
__always_inline sysarg_t sc_arg3(const struct exc_context *ctx)
{
return ctx->sp->r2;
}
__always_inline sysarg_t sc_arg4(const struct exc_context *ctx)
{
return ctx->sp->r3;
}
__always_inline sysarg_t sc_arg5(const struct exc_context *ctx)
{
return ctx->r4;
}
__always_inline sysarg_t sc_arg6(const struct exc_context *ctx)
{
return ctx->r5;
}
__always_inline void sc_set_rval(struct exc_context *ctx, long rval)
{
/* raw cast */
*(long *)&ctx->sp->r0 = rval;
}
#ifdef ARDIX_ARCH
# define __SAM3X8E__
# define DONT_USE_CMSIS_INIT
# define __PROGRAM_START
# include <sam3x8e.h>
# undef __PROGRAM_START
# undef DONT_USE_CMSIS_INIT
# undef __SAM3X8E__
#endif /* ARDIX_ARCH */
/*
......
/* See the end of this file for copyright, license, and warranty information. */
.include "asm.S"
.text
/* void _leave(void); */
func_begin _leave
pop {r4-r12,lr}
cpsie i
msr psp, r12
bx lr
func_end _leave
/*
* This file is part of Ardix.
* Copyright (c) 2020, 2021 Felix Kopp <owo@fef.moe>.
*
* Ardix is non-violent software: you may only use, redistribute,
* and/or modify it under the terms of the CNPLv6+ as found in
* the LICENSE file in the source code root directory or at
* <https://git.pixie.town/thufie/CNPL>.
*
* Ardix comes with ABSOLUTELY NO WARRANTY, to the extent
* permitted by applicable law. See the CNPLv6+ for details.
*/
/* See the end of this file for copyright, license, and warranty information. */
#include <arch-generic/do_switch.h>
#include <arch-generic/sched.h>
#include <arch/hardware.h>
#include <arch/interrupt.h>
#include <arch/linker.h>
#include <ardix/atomic.h>
#include <ardix/kevent.h>