/* $NetBSD: sgmap_typedep.c,v 1.44 2021/07/19 16:25:54 thorpej Exp $ */

/*-
 * Copyright (c) 1997, 1998, 2001 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(1, "$NetBSD: sgmap_typedep.c,v 1.44 2021/07/19 16:25:54 thorpej Exp $");

#include "opt_ddb.h"

#include <sys/evcnt.h>
#include <uvm/uvm_extern.h>

#define	DMA_COUNT_DECL(cnt)	_DMA_COUNT_DECL(dma_sgmap, cnt)
#define	DMA_COUNT(cnt)		_DMA_COUNT(dma_sgmap, cnt)

#ifdef SGMAP_DEBUG
int			__C(SGMAP_TYPE,_debug) = 0;
#endif

SGMAP_PTE_TYPE		__C(SGMAP_TYPE,_prefetch_spill_page_pte);

static void		__C(SGMAP_TYPE,_do_unload)(bus_dma_tag_t, bus_dmamap_t,
			    struct alpha_sgmap *);

void
__C(SGMAP_TYPE,_init_spill_page_pte)(void)
{

	__C(SGMAP_TYPE,_prefetch_spill_page_pte) =
	    (alpha_sgmap_prefetch_spill_page_pa >>
	     SGPTE_PGADDR_SHIFT) | SGPTE_VALID;
}

DMA_COUNT_DECL(spill_page);
DMA_COUNT_DECL(extra_segment);
DMA_COUNT_DECL(extra_segment_and_spill);

static int
__C(SGMAP_TYPE,_load_buffer)(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
    size_t buflen, struct vmspace *vm, int flags, int * const segp,
    struct alpha_sgmap *sgmap)
{
	vaddr_t endva, va = (vaddr_t)buf;
	paddr_t pa;
	bus_addr_t dmaoffset, sgva, extra_sgva;
	bus_size_t sgvalen, extra_sgvalen, boundary, alignment;
	SGMAP_PTE_TYPE *pte, *page_table = sgmap->aps_pt;
	int pteidx, error, spill, seg = *segp;
	bool address_is_valid __diagused;

	/* Initialize the spill page PTE if it hasn't been already. */
	if (__C(SGMAP_TYPE,_prefetch_spill_page_pte) == 0)
		__C(SGMAP_TYPE,_init_spill_page_pte)();

	if (seg == map->_dm_segcnt) {
		/* Ran of segments. */
		return EFBIG;
	}
	KASSERT(seg < map->_dm_segcnt);

	/*
	 * Remember the offset into the first page and the total
	 * transfer length.
	 */
	dmaoffset = ((u_long)buf) & PGOFSET;

#ifdef SGMAP_DEBUG
	if (__C(SGMAP_TYPE,_debug)) {
		printf("sgmap_load: ----- buf = %p -----\n", buf);
		printf("sgmap_load: dmaoffset = 0x%lx, buflen = 0x%lx\n",
		    dmaoffset, buflen);
	}
#endif

	/*
	 * Allocate the necessary virtual address space for the
	 * mapping.  Round the size, since we deal with whole pages.
	 */

	/*
	 * XXX Always allocate a spill page for now.  Note
	 * the spill page is not needed for an in-bound-only
	 * transfer.
	 */
	if ((flags & BUS_DMA_READ) == 0)
		spill = 1;
	else
		spill = 0;

	boundary = map->_dm_boundary;

	/*
	 * Caller's mistake if the requested length is larger than
	 * their own boundary constraint.
	 */
	if (__predict_false(boundary != 0 && buflen > boundary)) {
		return EINVAL;
	}

	endva = round_page(va + buflen);
	va = trunc_page(va);

	const vm_flag_t vmflags = VM_INSTANTFIT |
	    ((flags & BUS_DMA_NOWAIT) ? VM_NOSLEEP : VM_SLEEP);

	KASSERT(t->_sgmap_minalign != 0);
	alignment = t->_sgmap_minalign;
	sgvalen = (endva - va);

	SGMAP_PTE_TYPE spill_pte_v = __C(SGMAP_TYPE,_prefetch_spill_page_pte);

	/*
	 * If we have a boundary constraint, it's possible to end up in
	 * a situation where sgvalen > boundary if the caller's buffer
	 * is not page aligned.  In this case, we will have to allocate
	 * an extra SG segment and split the buffer.
	 */
	if (__predict_false(boundary != 0 && boundary < sgvalen)) {
#ifdef SGMAP_DEBUG
		if (__C(SGMAP_TYPE,_debug)) {
			printf("sgmap_load: extra segment needed\n");
		}
#endif
		DMA_COUNT(extra_segment);

		/* This should only ever happen for unaligned buffers. */
		KASSERT(dmaoffset != 0);

		extra_sgvalen = sgvalen - boundary;
		KASSERT(extra_sgvalen == PAGE_SIZE);

		/*
		 * Adjust the lengths of the first segment.  The length
		 * of the second segment will be dmaoffset.
		 */
		sgvalen -= extra_sgvalen;
		endva -= extra_sgvalen;
		buflen -= dmaoffset;

		if (spill) {
			DMA_COUNT(extra_segment_and_spill);
			extra_sgvalen += PAGE_SIZE;
		}

		error = vmem_xalloc(sgmap->aps_arena, extra_sgvalen,
				    alignment,		/* alignment */
				    0,			/* phase */
				    boundary,		/* nocross */
				    VMEM_ADDR_MIN,	/* minaddr */
				    VMEM_ADDR_MAX,	/* maxaddr */
				    vmflags,
				    &extra_sgva);
		if (error) {
			return error;
		}
	} else {
		extra_sgvalen = 0;
		extra_sgva = 0;
	}


	if (spill) {
		DMA_COUNT(spill_page);
		sgvalen += PAGE_SIZE;

		/*
		 * ARGH!  If the addition of the spill page bumped us
		 * over our boundary, we have to 2x the boundary limit.
		 * To compensate (and enforce the original boundary
		 * constraint), we force our alignment to be at least the
		 * previous boundary, thus ensuring that the only boundary
		 * violation is the pre-fetch that the SGMAP controller
		 * performs that necessitates the spill page in the first
		 * place.
		 */
		if (boundary && boundary < sgvalen) {
			if (alignment < boundary) {
				alignment = boundary;
			}
			do {
				boundary <<= 1;
			} while (boundary < sgvalen);
		}
	}

#ifdef SGMAP_DEBUG
	if (__C(SGMAP_TYPE,_debug)) {
		printf("sgmap_load: va:endva = 0x%lx:0x%lx\n", va, endva);
		printf("sgmap_load: sgvalen = 0x%lx, boundary = 0x%lx\n",
		       sgvalen, boundary);
	}
#endif

	error = vmem_xalloc(sgmap->aps_arena, sgvalen,
			    alignment,		/* alignment */
			    0,			/* phase */
			    boundary,		/* nocross */
			    VMEM_ADDR_MIN,	/* minaddr */
			    VMEM_ADDR_MAX,	/* maxaddr */
			    vmflags,
			    &sgva);
	if (error) {
		if (extra_sgvalen != 0) {
			vmem_xfree(sgmap->aps_arena, extra_sgva, extra_sgvalen);
		}
		return error;
	}

	pteidx = sgva >> SGMAP_ADDR_PTEIDX_SHIFT;
	pte = &page_table[pteidx * SGMAP_PTE_SPACING];

#ifdef SGMAP_DEBUG
	if (__C(SGMAP_TYPE,_debug))
		printf("sgmap_load: sgva = 0x%lx, pteidx = %d, "
		    "pte = %p (pt = %p)\n", sgva, pteidx, pte,
		    page_table);
#endif

	/* Generate the DMA address. */
	map->dm_segs[seg].ds_addr = sgmap->aps_wbase | sgva | dmaoffset;
	map->dm_segs[seg].ds_len = buflen;
	if (__predict_false(extra_sgvalen != 0)) {
		if (++seg == map->_dm_segcnt) {
			/* Boo! Ran out of segments! */
			vmem_xfree(sgmap->aps_arena, extra_sgva, extra_sgvalen);
			vmem_xfree(sgmap->aps_arena, sgva, sgvalen);
			return EFBIG;
		}
		map->dm_segs[seg].ds_addr = sgmap->aps_wbase | extra_sgva;
		map->dm_segs[seg].ds_len = dmaoffset;
		*segp = seg;
	}

#ifdef SGMAP_DEBUG
	if (__C(SGMAP_TYPE,_debug))
		printf("sgmap_load: wbase = 0x%lx, vpage = 0x%lx, "
		    "DMA addr = 0x%lx\n", sgmap->aps_wbase, (uint64_t)sgva,
		    map->dm_segs[seg].ds_addr);
#endif

	for (; va < endva; va += PAGE_SIZE, pteidx++,
	     pte = &page_table[pteidx * SGMAP_PTE_SPACING]) {
		/* Get the physical address for this segment. */
		address_is_valid = pmap_extract(vm->vm_map.pmap, va, &pa);
		KASSERT(address_is_valid);

		/* Load the current PTE with this page. */
		*pte = (pa >> SGPTE_PGADDR_SHIFT) | SGPTE_VALID;
#ifdef SGMAP_DEBUG
		if (__C(SGMAP_TYPE,_debug))
			printf("sgmap_load:     pa = 0x%lx, pte = %p, "
			    "*pte = 0x%lx\n", pa, pte, (u_long)(*pte));
#endif
	}

	if (__predict_false(extra_sgvalen != 0)) {
		int extra_pteidx = extra_sgva >> SGMAP_ADDR_PTEIDX_SHIFT;
		SGMAP_PTE_TYPE *extra_pte =
		    &page_table[extra_pteidx * SGMAP_PTE_SPACING];

		/* va == endva == address of extra page */
		KASSERT(va == endva);
		address_is_valid = pmap_extract(vm->vm_map.pmap, va, &pa);
		KASSERT(address_is_valid);

		/*
		 * If a spill page is needed, the previous segment will
		 * need to use this PTE value for it.
		 */
		spill_pte_v = (pa >> SGPTE_PGADDR_SHIFT) | SGPTE_VALID;
		*extra_pte = spill_pte_v;

		/* ...but the extra segment uses the real spill PTE. */
		if (spill) {
			extra_pteidx++;
			extra_pte =
			    &page_table[extra_pteidx * SGMAP_PTE_SPACING];
			*extra_pte = __C(SGMAP_TYPE,_prefetch_spill_page_pte);
		}
	}

	if (spill) {
		/* ...and the prefetch-spill page. */
		*pte = spill_pte_v;
#ifdef SGMAP_DEBUG
		if (__C(SGMAP_TYPE,_debug)) {
			printf("sgmap_load:     spill page, pte = %p, "
			    "*pte = 0x%lx\n", pte, (uint64_t)*pte);
		}
#endif
	}

	return (0);
}

DMA_COUNT_DECL(load);
DMA_COUNT_DECL(load_next_window);

int
__C(SGMAP_TYPE,_load)(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
    bus_size_t buflen, struct proc *p, int flags, struct alpha_sgmap *sgmap)
{
	int seg, error;
	struct vmspace *vm;

	/*
	 * Make sure that on error condition we return "no valid mappings".
	 */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;

	if (buflen > map->_dm_size)
		return (EINVAL);

	KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0);
	KASSERT((flags & (BUS_DMA_READ|BUS_DMA_WRITE)) !=
	    (BUS_DMA_READ|BUS_DMA_WRITE));

	map->_dm_flags |= flags & (BUS_DMA_READ|BUS_DMA_WRITE);

	if (p != NULL) {
		vm = p->p_vmspace;
	} else {
		vm = vmspace_kernel();
	}
	seg = 0;
	error = __C(SGMAP_TYPE,_load_buffer)(t, map, buf, buflen, vm,
	    flags, &seg, sgmap);

	alpha_mb();

#if defined(SGMAP_DEBUG) && defined(DDB)
	if (__C(SGMAP_TYPE,_debug) > 1)
		Debugger();
#endif

	if (error == 0) {
		DMA_COUNT(load);
		map->dm_mapsize = buflen;
		map->dm_nsegs = seg + 1;
		map->_dm_window = t;
	} else {
		map->_dm_flags &= ~(BUS_DMA_READ|BUS_DMA_WRITE);
		if (t->_next_window != NULL) {
			/* Give the next window a chance. */
			DMA_COUNT(load_next_window);
			error = bus_dmamap_load(t->_next_window, map, buf,
			    buflen, p, flags);
		}
	}
	return (error);
}

DMA_COUNT_DECL(load_mbuf);
DMA_COUNT_DECL(load_mbuf_next_window);

int
__C(SGMAP_TYPE,_load_mbuf)(bus_dma_tag_t t, bus_dmamap_t map,
    struct mbuf *m0, int flags, struct alpha_sgmap *sgmap)
{
	struct mbuf *m;
	int seg, error;

	/*
	 * Make sure that on error condition we return "no valid mappings".
	 */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;

#ifdef DIAGNOSTIC
	if ((m0->m_flags & M_PKTHDR) == 0)
		panic(__S(__C(SGMAP_TYPE,_load_mbuf)) ": no packet header");
#endif

	if (m0->m_pkthdr.len > map->_dm_size)
		return (EINVAL);

	KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0);
	KASSERT((flags & (BUS_DMA_READ|BUS_DMA_WRITE)) !=
	    (BUS_DMA_READ|BUS_DMA_WRITE));

	map->_dm_flags |= flags & (BUS_DMA_READ|BUS_DMA_WRITE);

	seg = 0;
	error = 0;
	for (m = m0; m != NULL && error == 0; m = m->m_next) {
		if (m->m_len == 0)
			continue;
		error = __C(SGMAP_TYPE,_load_buffer)(t, map,
		    m->m_data, m->m_len, vmspace_kernel(), flags, &seg, sgmap);
		seg++;
	}

	alpha_mb();

#if defined(SGMAP_DEBUG) && defined(DDB)
	if (__C(SGMAP_TYPE,_debug) > 1)
		Debugger();
#endif

	if (error == 0) {
		DMA_COUNT(load_mbuf);
		map->dm_mapsize = m0->m_pkthdr.len;
		map->dm_nsegs = seg;
		map->_dm_window = t;
	} else {
		/* Need to back out what we've done so far. */
		map->dm_nsegs = seg - 1;
		__C(SGMAP_TYPE,_do_unload)(t, map, sgmap);
		map->_dm_flags &= ~(BUS_DMA_READ|BUS_DMA_WRITE);
		if (t->_next_window != NULL) {
			/* Give the next window a chance. */
			DMA_COUNT(load_mbuf_next_window);
			error = bus_dmamap_load_mbuf(t->_next_window, map,
			    m0, flags);
		}
	}

	return (error);
}

DMA_COUNT_DECL(load_uio);
DMA_COUNT_DECL(load_uio_next_window);

int
__C(SGMAP_TYPE,_load_uio)(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio,
    int flags, struct alpha_sgmap *sgmap)
{
	bus_size_t minlen, resid;
	struct vmspace *vm;
	struct iovec *iov;
	void *addr;
	int i, seg, error;

	/*
	 * Make sure that on error condition we return "no valid mappings".
	 */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;

	KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0);
	KASSERT((flags & (BUS_DMA_READ|BUS_DMA_WRITE)) !=
	    (BUS_DMA_READ|BUS_DMA_WRITE));

	map->_dm_flags |= flags & (BUS_DMA_READ|BUS_DMA_WRITE);

	resid = uio->uio_resid;
	iov = uio->uio_iov;

	vm = uio->uio_vmspace;

	seg = 0;
	error = 0;
	for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
		/*
		 * Now at the first iovec to load.  Load each iovec
		 * until we have exhausted the residual count.
		 */
		minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
		addr = (void *)iov[i].iov_base;

		error = __C(SGMAP_TYPE,_load_buffer)(t, map,
		    addr, minlen, vm, flags, &seg, sgmap);
		seg++;

		resid -= minlen;
	}

	alpha_mb();

#if defined(SGMAP_DEBUG) && defined(DDB)
	if (__C(SGMAP_TYPE,_debug) > 1)
		Debugger();
#endif

	if (error == 0) {
		DMA_COUNT(load_uio);
		map->dm_mapsize = uio->uio_resid;
		map->dm_nsegs = seg;
		map->_dm_window = t;
	} else {
		/* Need to back out what we've done so far. */
		map->dm_nsegs = seg - 1;
		__C(SGMAP_TYPE,_do_unload)(t, map, sgmap);
		map->_dm_flags &= ~(BUS_DMA_READ|BUS_DMA_WRITE);
		if (t->_next_window != NULL) {
			/* Give the next window a chance. */
			DMA_COUNT(load_uio_next_window);
			error = bus_dmamap_load_uio(t->_next_window, map,
			    uio, flags);
		}
	}

	return (error);
}

int
__C(SGMAP_TYPE,_load_raw)(bus_dma_tag_t t, bus_dmamap_t map,
    bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags,
    struct alpha_sgmap *sgmap)
{

	KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0);
	KASSERT((flags & (BUS_DMA_READ|BUS_DMA_WRITE)) !=
	    (BUS_DMA_READ|BUS_DMA_WRITE));

	panic(__S(__C(SGMAP_TYPE,_load_raw)) ": not implemented");
}

static void
__C(SGMAP_TYPE,_do_unload)(bus_dma_tag_t t, bus_dmamap_t map,
    struct alpha_sgmap *sgmap)
{
	SGMAP_PTE_TYPE *pte, *page_table = sgmap->aps_pt;
	bus_addr_t osgva, sgva, esgva;
	int spill, seg, pteidx;

	for (seg = 0; seg < map->dm_nsegs; seg++) {
		/*
		 * XXX Always allocate a spill page for now.  Note
		 * the spill page is not needed for an in-bound-only
		 * transfer.
		 */
		if ((map->_dm_flags & BUS_DMA_READ) == 0)
			spill = 1;
		else
			spill = 0;

		sgva = map->dm_segs[seg].ds_addr & ~sgmap->aps_wbase;

		esgva = round_page(sgva + map->dm_segs[seg].ds_len);
		osgva = sgva = trunc_page(sgva);

		if (spill)
			esgva += PAGE_SIZE;

		/* Invalidate the PTEs for the mapping. */
		for (pteidx = sgva >> SGMAP_ADDR_PTEIDX_SHIFT;
		     sgva < esgva; sgva += PAGE_SIZE, pteidx++) {
			pte = &page_table[pteidx * SGMAP_PTE_SPACING];
#ifdef SGMAP_DEBUG
			if (__C(SGMAP_TYPE,_debug))
				printf("sgmap_unload:     pte = %p, "
				    "*pte = 0x%lx\n", pte, (u_long)(*pte));
#endif
			*pte = 0;
		}

		alpha_mb();

		/* Free the virtual address space used by the mapping. */
		vmem_xfree(sgmap->aps_arena, osgva, (esgva - osgva));
	}

	map->_dm_flags &= ~(BUS_DMA_READ|BUS_DMA_WRITE);

	/* Mark the mapping invalid. */
	map->dm_mapsize = 0;
	map->dm_nsegs = 0;
	map->_dm_window = NULL;
}

DMA_COUNT_DECL(unload);

void
__C(SGMAP_TYPE,_unload)(bus_dma_tag_t t, bus_dmamap_t map,
    struct alpha_sgmap *sgmap)
{
	KASSERT(map->_dm_window == t);
	DMA_COUNT(unload);
	__C(SGMAP_TYPE,_do_unload)(t, map, sgmap);
}