/*	$NetBSD: lock_stubs.S,v 1.29 2022/05/19 06:41:45 skrll Exp $	*/

/*-
 * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Andrew Doran and Nick Hudson.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "opt_multiprocessor.h"
#include "opt_lockdebug.h"

#define __MUTEX_PRIVATE

#include <machine/asm.h>
#include <machine/mutex.h>
#include <machine/reg.h>
#include <machine/cpu.h>

#include "assym.h"

/*
 * uintptr_t _lock_cas(volatile uintptr_t *ptr, uintptr_t old, uintptr_t new);
 *
 * Perform an atomic compare-and-swap operation.
 *
 * On single CPU systems this can use a restartable sequence as there
 * we don't need the overhead of interlocking.
 */


#ifndef LOCKDEBUG

	.global mutex_enter_crit_start
	.global mutex_enter_crit_end

	.import mutex_vector_enter, code
	.import mutex_vector_exit, code
	.import mutex_wakeup, code

/*
 * void mutex_exit(kmutex_t *mtx);
 *
 */

LEAF_ENTRY(mutex_exit)
	/*
	 * If it's a spin mutex or unowned, we have to take the slow path.
	 */
	ldi	MUTEX_ADAPTIVE_UNOWNED, %t1
	ldw	MTX_OWNER(%arg0),%t2
	depi	0, 27, 1, %t2			/* bit27 = 0 */
	comb,=	%t1, %t2, .Lexit_slowpath
	 nop

	/*
	 * We know that it's an adapative mutex.  Clear the owners
	 * field and release the lock.
	 */

	ldi	__SIMPLELOCK_RAW_UNLOCKED, %t2	/* unlocked = 1 */
	ldo	(MTX_LOCK + HPPA_LDCW_ALIGN - 1)(%arg0), %t3
	depi	0, 31, 4, %t3			/* bits[28-31] = 0 */
	stw	%t1, MTX_OWNER(%arg0)
	stw	%t2, 0(%t3)			/* %t3 is properly aligned */
	sync

	/*
	 * We have posted a read memory barrier so the check of mtx_waiters
	 * will happen in sequence.  If it's set then trap into mutex_wakeup()
	 * to wake up any threads sleeping on the lock.
	 */
	ldb	MTX_WAITERS(%arg0), %t4
	comib,=	0, %t4, .Lexit_done
	 nop

	ldil	L%mutex_wakeup, %t1
	ldo	R%mutex_wakeup(%t1), %t1
	.call
	bv,n	%r0(%t1)

.Lexit_slowpath:
	ldil	L%mutex_vector_exit, %t1
	ldo	R%mutex_vector_exit(%t1), %t1
	.call
	bv,n	%r0(%t1)
.Lexit_done:
	bv,n	%r0(%rp)

EXIT(mutex_exit)

/*
 * void mutex_enter(kmutex_t *mtx)
 */

LEAF_ENTRY(mutex_enter)

	/*
	 * It might be a spin lock (c.f. MUTEX_SPIN_FLAG) or might be
	 * already owned.  We short circuit the request and go straight
	 * into mutex_vector_enter() if the owners field is not 'unowned'.
	 */

	ldi	MUTEX_ADAPTIVE_UNOWNED, %t1
	ldw	MTX_OWNER(%arg0), %t2
	comb,=,n %t1, %t2, .Lmutexunowned
.Lenter_slowpath:

	ldil	L%mutex_vector_enter, %t1
	ldo	R%mutex_vector_enter(%t1), %t1
	.call
	bv,n	%r0(%t1)
	 nop

	/*
	 * We now know that it's an adaptive mutex.  Grab the spin
	 * lock, which is an atomic operation.  Once we have that,
	 * we can set the owner field.  If we can't get it, we
	 * need to go the slow path.
	 *
	 * Even if we are preempted between acquiring the lock and
	 * setting the owners field, there is no problem - noone
	 * else can acquire the mutex while the lock is held.
	 */
.Lmutexunowned:
	ldo	(MTX_LOCK + HPPA_LDCW_ALIGN - 1)(%arg0), %t1
	depi	0, 31, 4, %t1
	ldcw	0(%t1), %ret0

mutex_enter_crit_start:
	comib,=	0, %ret0, .Lenter_slowpath

	GET_CURLWP(%t2)

	bv	%r0(%rp)
	 stw	%t2, MTX_OWNER(%arg0)
mutex_enter_crit_end:
EXIT(mutex_enter)

#endif	/* !LOCKDEBUG */

#ifndef MULTIPROCESSOR

	.global _lock_cas_ras_start
	.global _lock_cas_ras_end

LEAF_ENTRY(_lock_cas)
_lock_cas_ras_start:
	ldw	0(%arg0),%t1
	comb,<>	%arg1, %t1, 1f
	 copy	%t1,%ret0
	stw	%arg2,0(%arg0)
_lock_cas_ras_end:
	copy	%arg1,%ret0
1:
	bv,n	%r0(%rp)

EXIT(_lock_cas)

STRONG_ALIAS(_atomic_cas_ulong,_lock_cas)
STRONG_ALIAS(atomic_cas_ulong,_lock_cas)
STRONG_ALIAS(_atomic_cas_32,_lock_cas)
STRONG_ALIAS(atomic_cas_32,_lock_cas)
STRONG_ALIAS(_atomic_cas_uint,_lock_cas)
STRONG_ALIAS(atomic_cas_uint,_lock_cas)
STRONG_ALIAS(_atomic_cas_ptr,_lock_cas)
STRONG_ALIAS(atomic_cas_ptr,_lock_cas)

STRONG_ALIAS(_atomic_cas_ulong_ni,_lock_cas)
STRONG_ALIAS(atomic_cas_ulong_ni,_lock_cas)
STRONG_ALIAS(_atomic_cas_32_ni,_lock_cas)
STRONG_ALIAS(atomic_cas_32_ni,_lock_cas)
STRONG_ALIAS(_atomic_cas_uint_ni,_lock_cas)
STRONG_ALIAS(atomic_cas_uint_ni,_lock_cas)
STRONG_ALIAS(_atomic_cas_ptr_ni,_lock_cas)
STRONG_ALIAS(atomic_cas_ptr_ni,_lock_cas)

#else  /* !MULTIPROCESSOR */

/*
 * uintptr_t _lock_cas(volatile uintptr_t *ptr, uintptr_t old, uintptr_t new);
 *
 * Perform an atomic compare-and-swap operation.
 *
 * On multi-CPU systems, this has to use an interlock and disable interrupts.
 * The interlock is to protect against another CPU attempting to perform the
 * cas.  Disabling interrupts is to prevent deadlocks on the current CPU.  That
 * is, we don't want an interrupts attempting to perform a cas on the interlock
 * at the same time.
 *
 */

#define IL	\
	.word	__SIMPLELOCK_RAW_UNLOCKED ! \
	.word	__SIMPLELOCK_RAW_UNLOCKED ! \
	.word	__SIMPLELOCK_RAW_UNLOCKED ! \
	.word	__SIMPLELOCK_RAW_UNLOCKED ! \

#define I8	\
	IL IL IL IL IL IL IL IL

#define I64	\
	I8 I8 I8 I8 I8 I8 I8 I8

	.section .data
	.align 4096
	.export _lock_hash, data
_lock_hash:
	I64 I64
	I64 I64
	I64 I64
	I64 I64
	I64 I64
	I64 I64
	I64 I64
	I64 I64

LEAF_ENTRY(_lock_cas)
ALTENTRY(_lock_cas_mp)

	mfctl	%eiem, %t1
	mtctl	%r0, %eiem			/* disable interrupts */

	extru	%arg0, 21+8-1, 8, %ret0
	ldil	L%_lock_hash, %r1
	zdep	%ret0, 27, 28, %ret0
	ldo	R%_lock_hash(%r1), %r1

	addl	%ret0, %r1, %ret0
	ldo	15(%ret0), %ret0
	copy	%ret0, %t3
	depi	0, 31, 4, %t3

	/* %t3 is the interlock address */
	ldcw	0(%t3), %ret0
	comib,<>,n	0, %ret0, _lock_cas_mp_interlocked
_lock_cas_mp_spin:
	 ldw	0(%t3), %ret0
	comib,= 0, %ret0, _lock_cas_mp_spin
	 nop
	ldcw	0(%t3), %ret0
	comib,= 0, %ret0, _lock_cas_mp_spin
	 nop

_lock_cas_mp_interlocked:
	ldw	0(%arg0), %ret0
	comclr,<> %arg1, %ret0, %r0	/* If *ptr != old, then nullify */
	stw	%arg2, 0(%arg0)

	sync

	ldi	__SIMPLELOCK_RAW_UNLOCKED, %t4
	stw	%t4, 0(%t3)
	bv	%r0(%r2)
	 mtctl	%t1, %eiem		/* enable interrupts */

EXIT(_lock_cas)

STRONG_ALIAS(_atomic_cas_ulong,_lock_cas_mp)
STRONG_ALIAS(atomic_cas_ulong,_lock_cas_mp)
STRONG_ALIAS(_atomic_cas_32,_lock_cas_mp)
STRONG_ALIAS(atomic_cas_32,_lock_cas_mp)
STRONG_ALIAS(_atomic_cas_uint,_lock_cas_mp)
STRONG_ALIAS(atomic_cas_uint,_lock_cas_mp)
STRONG_ALIAS(_atomic_cas_ptr,_lock_cas_mp)
STRONG_ALIAS(atomic_cas_ptr,_lock_cas_mp)

STRONG_ALIAS(_atomic_cas_ulong_ni,_lock_cas_mp)
STRONG_ALIAS(atomic_cas_ulong_ni,_lock_cas_mp)
STRONG_ALIAS(_atomic_cas_32_ni,_lock_cas_mp)
STRONG_ALIAS(atomic_cas_32_ni,_lock_cas_mp)
STRONG_ALIAS(_atomic_cas_uint_ni,_lock_cas_mp)
STRONG_ALIAS(atomic_cas_uint_ni,_lock_cas_mp)
STRONG_ALIAS(_atomic_cas_ptr_ni,_lock_cas_mp)
STRONG_ALIAS(atomic_cas_ptr_ni,_lock_cas_mp)

#endif /* MULTIPROCESSOR */