/* $NetBSD: cpuswitch.S,v 1.41 2023/03/01 08:17:24 riastradh Exp $ */

/*-
 * Copyright (c) 2014, 2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Matt Thomas of 3am Software Foundry.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <aarch64/asm.h>
#include <aarch64/locore.h>

#include "assym.h"

#include "opt_cpuoptions.h"
#include "opt_ddb.h"
#include "opt_kasan.h"

RCSID("$NetBSD: cpuswitch.S,v 1.41 2023/03/01 08:17:24 riastradh Exp $")

	ARMV8_DEFINE_OPTIONS

/*
 * At IPL_SCHED:
 *	x0 = oldlwp
 *	x1 = newlwp
 *	x2 = returning
 * returns x0-x2 unchanged
 */
ENTRY_NP(cpu_switchto)
	/*
	 * Store the callee saved register on the stack.
	 */
	sub	sp, sp, #TF_SIZE		/* make switchframe */
	stp	x19, x20, [sp, #TF_X19]
	stp	x21, x22, [sp, #TF_X21]
	stp	x23, x24, [sp, #TF_X23]
	stp	x25, x26, [sp, #TF_X25]
	stp	x27, x28, [sp, #TF_X27]
	stp	x29, x30, [sp, #TF_X29]
#ifdef DDB
	/* mark as switchframe for backtrace */
	.if TF_SP + 8 == TF_PC
	stp	xzr, lr, [sp, #TF_SP]
	.else
	str	xzr, [sp, #TF_SP]
	str	lr, [sp, #TF_PC]
	.endif
#endif

	/*
	 * Save the current stack pointer and the CPACR and save them in
	 * old lwp md area.
	 */
	mov	x4, sp
	mrs	x5, cpacr_el1
	str	x5, [x0, #L_MD_CPACR]
	ldr	x6, [x0, #L_PCB]	/* x6 = lwp_getpcb(oldlwp) */
	str	x4, [x6, #PCB_TF]

	/* We are done with the old lwp */
	ldr	x6, [x1, #L_PCB]	/* x6 = lwp_getpcb(newlwp) */
	ldr	x4, [x6, #PCB_TF]	/* get trapframe ptr (aka SP) */
#ifdef DDB
	str	xzr, [x6, #PCB_TF]	/* clear l->l_addr->pcb_tf */
#endif
	ldr	x5, [x1, #L_MD_CPACR]	/* get cpacr_el1 */

	DISABLE_INTERRUPT
	mov	sp, x4			/* restore stack pointer */
	msr	cpacr_el1, x5		/* restore cpacr_el1 */
	isb

#ifdef ARMV83_PAC
	/* Switch the PAC key. */
	adrl	x4, _C_LABEL(aarch64_pac_enabled)
	ldr	w4, [x4]
	cbz	w4, 1f
	ldp	x5, x6, [x1, #L_MD_IA_KERN]
	msr	APIAKeyLo_EL1, x5
	msr	APIAKeyHi_EL1, x6

	/* Other keys only need to be updated when switching to user process */
	ldr	w5, [x1, #L_FLAG]
	and	w5, w5, #LW_SYSTEM	/* (lwp->l_flag & LW_SYSTEM) ? */
	cbnz	w5, 1f

	ldp	x5, x6, [x1, #L_MD_IB_USER]
	msr	APIBKeyLo_EL1, x5
	msr	APIBKeyHi_EL1, x6
	ldp	x5, x6, [x1, #L_MD_DA_USER]
	msr	APDAKeyLo_EL1, x5
	msr	APDAKeyHi_EL1, x6
	ldp	x5, x6, [x1, #L_MD_DB_USER]
	msr	APDBKeyLo_EL1, x5
	msr	APDBKeyHi_EL1, x6
	ldp	x5, x6, [x1, #L_MD_GA_USER]
	msr	APGAKeyLo_EL1, x5
	msr	APGAKeyHi_EL1, x6
1:
	isb
#endif

	msr	tpidr_el1, x1		/* switch curlwp to new lwp */
	ldr	x3, [x1, #L_CPU]

	/*
	 * Issue barriers to coordinate mutex_exit on this CPU with
	 * mutex_vector_enter on another CPU.
	 *
	 * 1. Any prior mutex_exit by oldlwp must be visible to other
	 *    CPUs before we set ci_curlwp := newlwp on this one,
	 *    requiring a store-before-store barrier.
	 *
	 * 2. ci_curlwp := newlwp must be visible on all other CPUs
	 *    before any subsequent mutex_exit by newlwp can even test
	 *    whether there might be waiters, requiring a
	 *    store-before-load barrier.
	 *
	 * See kern_mutex.c for details -- this is necessary for
	 * adaptive mutexes to detect whether the lwp is on the CPU in
	 * order to safely block without requiring atomic r/m/w in
	 * mutex_exit.
	 */
	dmb	ishst			/* store-before-store */
	str	x1, [x3, #CI_CURLWP]	/* switch curlwp to new lwp */
	dmb	ish			/* store-before-load */

	ENABLE_INTERRUPT

	/*
	 * Restore callee save registers.
	 */
	ldp	x19, x20, [sp, #TF_X19]
	ldp	x21, x22, [sp, #TF_X21]
	ldp	x23, x24, [sp, #TF_X23]
	ldp	x25, x26, [sp, #TF_X25]
	ldp	x27, x28, [sp, #TF_X27]
	ldp	x29, lr, [sp, #TF_X29]
	add	sp, sp, #TF_SIZE	/* unwind switchframe */

	ret
END(cpu_switchto)

/*
 * void
 * cpu_switchto_softint(struct lwp *softlwp, int ipl)
 * {
 *	build a switchframe on kernel stack.
 *	craft TF_X30 to have softint_cleanup.
 *	pinned_lwp = curlwp
 *	switch to softlwp context.
 *	call softint_dispatch(pinned_lwp, ipl);
 *	switch back to pinned_lwp context.
 *	unwind switchframe made on kernel stack.
 *	return to caller this time.
 * }
 */
ENTRY_NP(cpu_switchto_softint)
#ifdef DDB
	mov	x7, sp			/* keep original sp for backtrace */
#endif
	stp	x19, x20, [sp, #-16]!	/* save */
	sub	sp, sp, #TF_SIZE	/* make switchframe */
	adr	x2, softint_cleanup	/* return address for cpu_switchto() */
	mrs	x19, daif		/* x19 := original interrupt mask */
	stp	x19, lr, [sp, #TF_X19]	/* x20 := original lr */
	stp	x21, x22, [sp, #TF_X21]
	stp	x23, x24, [sp, #TF_X23]
	stp	x25, x26, [sp, #TF_X25]
	stp	x27, x28, [sp, #TF_X27]
	stp	x29, x2, [sp, #TF_X29]	/* tf->tf_lr = softint_cleanup; */

	mrs	x19, tpidr_el1		/* x19 := curlwp */
	mov	x4, sp

	mrs	x5, cpacr_el1
	ldr	x6, [x19, #L_PCB]	/* x6 = lwp_getpcb(curlwp) */
	str	x4, [x6, #PCB_TF]
	str	x5, [x19, #L_MD_CPACR]

#ifdef KASAN
	/* clear the new stack */
	stp	x0, x1, [sp, #-16]!
#ifdef DDB
	stp	x7, lr, [sp, #-16]!	/* original sp and lr */
#endif
	bl	_C_LABEL(kasan_softint)
#ifdef DDB
	ldp	x7, lr, [sp], #16
#endif
	ldp	x0, x1, [sp], #16
#endif

	ldr	x4, [x0, #L_MD_UTF]

	DISABLE_INTERRUPT
	ldr	x20, [x19, #L_CPU]	/* x20 := curlwp->l_cpu */

	/* onto new stack */
	sub	sp, x4, #TF_SIZE	/* new sp := softlwp->l_md_utf - 1 */
	msr	tpidr_el1, x0		/* curlwp = softlwp; */
	dmb	ishst			/* for mutex_enter; see cpu_switchto */
	str	x0, [x20, #CI_CURLWP]	/* curcpu()->ci_curlwp = softlwp; */
	/*
	 * No need for barrier after ci->ci_curlwp = softlwp -- when we
	 * enter a softint lwp, it can't be holding any mutexes, so it
	 * can't release any until after it has acquired them, so we
	 * need not participate in the protocol with mutex_vector_enter
	 * barriers here.
	 */

	mov	x5, #CPACR_FPEN_NONE
	msr	cpacr_el1, x5		/* cpacr_el1 = CPACR_FPEN_NONE */
	isb

#ifdef ARMV83_PAC
	/* Switch the PAC key. */
	adrl	x4, _C_LABEL(aarch64_pac_enabled)
	ldr	w4, [x4]
	cbz	w4, 1f
	ldp	x5, x6, [x0, #L_MD_IA_KERN]
	msr	APIAKeyLo_EL1, x5
	msr	APIAKeyHi_EL1, x6
	isb
1:
#endif
	ENABLE_INTERRUPT

#ifdef DDB
	/* for db_trace.c:db_sp_trace() */
	stp	x7, lr, [sp, #-16]!	/* push original sp,lr for backtrace info */
#endif

	/* softint_dispatch(pinned_lwp, ipl) */
	mov	x0, x19			/* x0 := pinned_lwp */
	bl	_C_LABEL(softint_dispatch)

#ifdef DDB
	add	sp, sp, #16		/* pop backtrace info */
#endif

	ldr	x6, [x19, #L_PCB]	/* x6 = lwp_getpcb(curlwp) */
	ldr	x4, [x6, #PCB_TF]	/* x4 := pinned_lwp->l_addr->pcb_tf */
#ifdef DDB
	str	xzr, [x6, #PCB_TF]	/* clear l->l_addr->pcb_tf */
#endif
	ldr	x5, [x19, #L_MD_CPACR]	/* x5 := pinned_lwp->l_md_cpacr */

	DISABLE_INTERRUPT
	msr	tpidr_el1, x19		/* curlwp = pinned_lwp */
	ldr	x3, [x19, #L_CPU]	/* x3 = curlwp->l_cpu */
	dmb	ishst			/* for mutex_enter; see cpu_switchto */
	str	x19, [x3, #CI_CURLWP]	/* curlwp->l_cpu->ci_curlwp := x19 */
	dmb	ish			/* for mutex_enter; see cpu_switchto */

	mov	sp, x4			/* restore pinned_lwp sp */
	msr	cpacr_el1, x5		/* restore pinned_lwp cpacr */
	isb

#ifdef ARMV83_PAC
	/* Restore the PAC key. */
	adrl	x4, _C_LABEL(aarch64_pac_enabled)
	ldr	w4, [x4]
	cbz	w4, 1f
	ldp	x5, x6, [x19, #L_MD_IA_KERN]
	msr	APIAKeyLo_EL1, x5
	msr	APIAKeyHi_EL1, x6
	isb
1:
#endif

	ldp	x19, lr, [sp, #TF_X19]	/* restore pinned_lwp lr */
	msr	daif, x19		/* restore interrupt mask */

	add	sp, sp, #TF_SIZE	/* unwind switchframe */
	ldp	x19, x20, [sp], #16	/* restore */
	ret
END(cpu_switchto_softint)

/*
 * void
 * softint_cleanup(struct lwp *softlwp)
 * {
 *	cpu_switchto() bottom half arranges to start this when softlwp.
 *	kernel thread is to yield CPU for the pinned_lwp in the above.
 *	curcpu()->ci_mtx_count += 1;
 *	this returns as if cpu_switchto_softint finished normally.
 * }
 */
ENTRY_NP(softint_cleanup)
	mov	lr, x20			/* restore original lr */

	mrs	x20, tpidr_el1		/* curlwp */
	ldr	x3, [x20, #L_CPU]	/* curcpu */
	ldr	w2, [x3, #CI_MTX_COUNT]	/* ->ci_mtx_count */
	add	w2, w2, #1
	str	w2, [x3, #CI_MTX_COUNT]

	msr	daif, x19		/* restore interrupt mask */
	ldp	x19, x20, [sp], #16	/* restore */
	ret
END(softint_cleanup)

/*
 * Called at IPL_SCHED:
 *	x0 = old lwp (from cpu_switchto)
 *	x1 = new lwp (from cpu_switchto)
 *	x27 = func
 *	x28 = arg
 */
ENTRY_NP(lwp_trampoline)
	stp	x0, x1, [sp, #-16]!	/* save x0, x1 */
	bl	_C_LABEL(lwp_startup)
	ldp	x0, x1, [sp], #16	/* restore x0, x1 */

#ifdef ARMV83_PAC
	/* Generate a new PAC key. */
	adrl	x4, _C_LABEL(aarch64_pac_enabled)
	ldr	w4, [x4]
	cbz	w4, 1f
	mov	x26, x1

	bl	_C_LABEL(cprng_strong64)
	mov	x25, x0
	bl	_C_LABEL(cprng_strong64)
	stp	x25, x0, [x26, #L_MD_IA_KERN]
	msr	APIAKeyLo_EL1, x25
	msr	APIAKeyHi_EL1, x0
	isb
1:
#endif

	/*
	 * When the x27 function returns, it will jump to el0_trap_exit.
	 */
	adr	lr, el0_trap_exit	/* tail call via lr */
	mov	x0, x28			/* mov arg into place */
	mov	x16, x27		/* use x16 as jump register, for BTI */
	br	x16			/* call function with arg */
END(lwp_trampoline)


/*
 * int cpu_set_onfault(struct faultbuf *fb)
 */
ENTRY_NP(cpu_set_onfault)
	mrs	x2, tpidr_el1		/* x2 = curlwp */
	str	x0, [x2, #L_MD_ONFAULT] /* l_md.md_onfault = fb */

	stp	x19, x20, [x0, #(FB_X19 * 8)]
	stp	x21, x22, [x0, #(FB_X21 * 8)]
	stp	x23, x24, [x0, #(FB_X23 * 8)]
	stp	x25, x26, [x0, #(FB_X25 * 8)]
	stp	x27, x28, [x0, #(FB_X27 * 8)]
	stp	x29, x30, [x0, #(FB_X29 * 8)]
	mov	x1, sp
	str	x1, [x0, #(FB_SP * 8)]
	mov	x0, #0
	ret
END(cpu_set_onfault)

/*
 * setjmp(9)
 * int setjmp(label_t *label);
 * void longjmp(label_t *label);
 */
ENTRY_NP(setjmp)
	stp	x19, x20, [x0, #(LBL_X19 * 8)]
	stp	x21, x22, [x0, #(LBL_X21 * 8)]
	stp	x23, x24, [x0, #(LBL_X23 * 8)]
	stp	x25, x26, [x0, #(LBL_X25 * 8)]
	stp	x27, x28, [x0, #(LBL_X27 * 8)]
	stp	x29, x30, [x0, #(LBL_X29 * 8)]
	mov	x1, sp
	str	x1, [x0, #(LBL_SP * 8)]
	mov	x0, #0
	ret
END(setjmp)

ENTRY_NP(longjmp)
	ldp	x19, x20, [x0, #(LBL_X19 * 8)]
	ldp	x21, x22, [x0, #(LBL_X21 * 8)]
	ldp	x23, x24, [x0, #(LBL_X23 * 8)]
	ldp	x25, x26, [x0, #(LBL_X25 * 8)]
	ldp	x27, x28, [x0, #(LBL_X27 * 8)]
	ldp	x29, x30, [x0, #(LBL_X29 * 8)]
	ldr	x1, [x0, #(LBL_SP * 8)]
	mov	sp, x1
	mov	x0, #1
	ret
END(longjmp)

ENTRY_NP(load_fpregs)
	ldp	q0, q1, [x0, #FPREG_Q0]
	ldp	q2, q3, [x0, #FPREG_Q2]
	ldp	q4, q5, [x0, #FPREG_Q4]
	ldp	q6, q7, [x0, #FPREG_Q6]
	ldp	q8, q9, [x0, #FPREG_Q8]
	ldp	q10, q11, [x0, #FPREG_Q10]
	ldp	q12, q13, [x0, #FPREG_Q12]
	ldp	q14, q15, [x0, #FPREG_Q14]
	ldp	q16, q17, [x0, #FPREG_Q16]
	ldp	q18, q19, [x0, #FPREG_Q18]
	ldp	q20, q21, [x0, #FPREG_Q20]
	ldp	q22, q23, [x0, #FPREG_Q22]
	ldp	q24, q25, [x0, #FPREG_Q24]
	ldp	q26, q27, [x0, #FPREG_Q26]
	ldp	q28, q29, [x0, #FPREG_Q28]
	ldp	q30, q31, [x0, #FPREG_Q30]
	ldr	w8, [x0, #FPREG_FPCR]
	ldr	w9, [x0, #FPREG_FPSR]
	msr	fpcr, x8
	msr	fpsr, x9
	ret
END(load_fpregs)

ENTRY_NP(save_fpregs)
	stp	q0, q1, [x0, #FPREG_Q0]
	stp	q2, q3, [x0, #FPREG_Q2]
	stp	q4, q5, [x0, #FPREG_Q4]
	stp	q6, q7, [x0, #FPREG_Q6]
	stp	q8, q9, [x0, #FPREG_Q8]
	stp	q10, q11, [x0, #FPREG_Q10]
	stp	q12, q13, [x0, #FPREG_Q12]
	stp	q14, q15, [x0, #FPREG_Q14]
	stp	q16, q17, [x0, #FPREG_Q16]
	stp	q18, q19, [x0, #FPREG_Q18]
	stp	q20, q21, [x0, #FPREG_Q20]
	stp	q22, q23, [x0, #FPREG_Q22]
	stp	q24, q25, [x0, #FPREG_Q24]
	stp	q26, q27, [x0, #FPREG_Q26]
	stp	q28, q29, [x0, #FPREG_Q28]
	stp	q30, q31, [x0, #FPREG_Q30]
	mrs	x8, fpcr
	mrs	x9, fpsr
	str	w8, [x0, #FPREG_FPCR]
	str	w9, [x0, #FPREG_FPSR]
	ret
END(save_fpregs)