* Copyright (c) 2008, 2012, 2016 KIYOHARA Takashi

/* $NetBSD: gtidmac.c,v 1.21 2024/06/12 20:03:56 andvar Exp $ */
/*
* Copyright (c) 2008, 2012, 2016 KIYOHARA Takashi
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: gtidmac.c,v 1.21 2024/06/12 20:03:56 andvar Exp $");

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/device.h>
#include <sys/errno.h>
#include <sys/endian.h>
#include <sys/kmem.h>

#include <uvm/uvm_param.h> /* For PAGE_SIZE */

#include <dev/dmover/dmovervar.h>

#include <dev/marvell/gtidmacreg.h>
#include <dev/marvell/gtidmacvar.h>
#include <dev/marvell/marvellreg.h>
#include <dev/marvell/marvellvar.h>

#include <prop/proplib.h>

#include "locators.h"

#ifdef GTIDMAC_DEBUG
#define DPRINTF(x) if (gtidmac_debug) printf x
int gtidmac_debug = 0;
#else
#define DPRINTF(x)
#endif

#define GTIDMAC_NDESC 64
#define GTIDMAC_MAXCHAN 8
#define MVXORE_NDESC 128
#define MVXORE_MAXCHAN 2

#define GTIDMAC_NSEGS ((GTIDMAC_MAXXFER + PAGE_SIZE - 1) / PAGE_SIZE)
#define MVXORE_NSEGS ((MVXORE_MAXXFER + PAGE_SIZE - 1) / PAGE_SIZE)

struct gtidmac_softc;

struct gtidmac_function {
int (*chan_alloc)(void *, bus_dmamap_t **, bus_dmamap_t **, void *);
void (*chan_free)(void *, int);
int (*dma_setup)(void *, int, int, bus_dmamap_t *, bus_dmamap_t *,
bus_size_t);
void (*dma_start)(void *, int,
void (*dma_done_cb)(void *, int, bus_dmamap_t *,
bus_dmamap_t *, int));
uint32_t (*dma_finish)(void *, int, int);
};

struct gtidmac_dma_desc {
int dd_index;
union {
struct gtidmac_desc *idmac_vaddr;
struct mvxore_desc *xore_vaddr;
} dd_vaddr;
#define dd_idmac_vaddr dd_vaddr.idmac_vaddr
#define dd_xore_vaddr dd_vaddr.xore_vaddr
paddr_t dd_paddr;
SLIST_ENTRY(gtidmac_dma_desc) dd_next;
};

struct gtidmac_softc {
device_t sc_dev;

bus_space_tag_t sc_iot;
bus_space_handle_t sc_ioh;

bus_dma_tag_t sc_dmat;
struct gtidmac_dma_desc *sc_dd_buffer;
bus_dma_segment_t sc_pattern_segment;
struct {
u_char pbuf[16]; /* 16byte/pattern */
} *sc_pbuf; /* x256 pattern */

int sc_gtidmac_nchan;
struct gtidmac_desc *sc_dbuf;
bus_dmamap_t sc_dmap;
SLIST_HEAD(, gtidmac_dma_desc) sc_dlist;
struct {
bus_dmamap_t chan_in; /* In dmamap */
bus_dmamap_t chan_out; /* Out dmamap */
uint64_t chan_totalcnt; /* total transferred byte */
int chan_ddidx;
void *chan_running; /* opaque object data */
void (*chan_dma_done)(void *, int, bus_dmamap_t *,
bus_dmamap_t *, int);
} sc_cdesc[GTIDMAC_MAXCHAN];
struct gtidmac_intr_arg {
struct gtidmac_softc *ia_sc;
uint32_t ia_cause;
uint32_t ia_mask;
uint32_t ia_eaddr;
uint32_t ia_eselect;
} sc_intrarg[GTIDMAC_NINTRRUPT];

int sc_mvxore_nchan;
struct mvxore_desc *sc_dbuf_xore;
bus_dmamap_t sc_dmap_xore;
SLIST_HEAD(, gtidmac_dma_desc) sc_dlist_xore;
struct {
bus_dmamap_t chan_in[MVXORE_NSRC]; /* In dmamap */
bus_dmamap_t chan_out; /* Out dmamap */
uint64_t chan_totalcnt; /* total transferred */
int chan_ddidx;
void *chan_running; /* opaque object data */
void (*chan_dma_done)(void *, int, bus_dmamap_t *,
bus_dmamap_t *, int);
} sc_cdesc_xore[MVXORE_MAXCHAN];

struct dmover_backend sc_dmb;
struct dmover_backend sc_dmb_xore;
int sc_dmb_busy;
};
struct gtidmac_softc *gtidmac_softc = NULL;

static int gtidmac_match(device_t, struct cfdata *, void *);
static void gtidmac_attach(device_t, device_t, void *);

static int gtidmac_intr(void *);
static int mvxore_port0_intr(void *);
static int mvxore_port1_intr(void *);
static int mvxore_intr(struct gtidmac_softc *, int);

static void gtidmac_process(struct dmover_backend *);
static void gtidmac_dmover_run(struct dmover_backend *);
static void gtidmac_dmover_done(void *, int, bus_dmamap_t *, bus_dmamap_t *,
int);
static __inline int gtidmac_dmmap_load(struct gtidmac_softc *, bus_dmamap_t,
dmover_buffer_type, dmover_buffer *, int);
static __inline void gtidmac_dmmap_unload(struct gtidmac_softc *, bus_dmamap_t, int);

static uint32_t gtidmac_finish(void *, int, int);
static uint32_t mvxore_finish(void *, int, int);

static void gtidmac_wininit(struct gtidmac_softc *, enum marvell_tags *);
static void mvxore_wininit(struct gtidmac_softc *, enum marvell_tags *);

static int gtidmac_buffer_setup(struct gtidmac_softc *);
static int mvxore_buffer_setup(struct gtidmac_softc *);

#ifdef GTIDMAC_DEBUG
static void gtidmac_dump_idmacreg(struct gtidmac_softc *, int);
static void gtidmac_dump_idmacdesc(struct gtidmac_softc *,
struct gtidmac_dma_desc *, uint32_t, int);
static void gtidmac_dump_xorereg(struct gtidmac_softc *, int);
static void gtidmac_dump_xoredesc(struct gtidmac_softc *,
struct gtidmac_dma_desc *, uint32_t, int);
#endif

static struct gtidmac_function gtidmac_functions = {
.chan_alloc = gtidmac_chan_alloc,
.chan_free = gtidmac_chan_free,
.dma_setup = gtidmac_setup,
.dma_start = gtidmac_start,
.dma_finish = gtidmac_finish,
};

static struct gtidmac_function mvxore_functions = {
.chan_alloc = mvxore_chan_alloc,
.chan_free = mvxore_chan_free,
.dma_setup = mvxore_setup,
.dma_start = mvxore_start,
.dma_finish = mvxore_finish,
};

static const struct dmover_algdesc gtidmac_algdescs[] = {
{
.dad_name = DMOVER_FUNC_ZERO,
.dad_data = &gtidmac_functions,
.dad_ninputs = 0
},
{
.dad_name = DMOVER_FUNC_FILL8,
.dad_data = &gtidmac_functions,
.dad_ninputs = 0
},
{
.dad_name = DMOVER_FUNC_COPY,
.dad_data = &gtidmac_functions,
.dad_ninputs = 1
},
};

static const struct dmover_algdesc mvxore_algdescs[] = {
#if 0
/*
* As for these operations, there are a lot of restrictions. It is
* necessary to use IDMAC.
*/
{
.dad_name = DMOVER_FUNC_ZERO,
.dad_data = &mvxore_functions,
.dad_ninputs = 0
},
{
.dad_name = DMOVER_FUNC_FILL8,
.dad_data = &mvxore_functions,
.dad_ninputs = 0
},
#endif
{
.dad_name = DMOVER_FUNC_COPY,
.dad_data = &mvxore_functions,
.dad_ninputs = 1
},
{
.dad_name = DMOVER_FUNC_ISCSI_CRC32C,
.dad_data = &mvxore_functions,
.dad_ninputs = 1
},
{
.dad_name = DMOVER_FUNC_XOR2,
.dad_data = &mvxore_functions,
.dad_ninputs = 2
},
{
.dad_name = DMOVER_FUNC_XOR3,
.dad_data = &mvxore_functions,
.dad_ninputs = 3
},
{
.dad_name = DMOVER_FUNC_XOR4,
.dad_data = &mvxore_functions,
.dad_ninputs = 4
},
{
.dad_name = DMOVER_FUNC_XOR5,
.dad_data = &mvxore_functions,
.dad_ninputs = 5
},
{
.dad_name = DMOVER_FUNC_XOR6,
.dad_data = &mvxore_functions,
.dad_ninputs = 6
},
{
.dad_name = DMOVER_FUNC_XOR7,
.dad_data = &mvxore_functions,
.dad_ninputs = 7
},
{
.dad_name = DMOVER_FUNC_XOR8,
.dad_data = &mvxore_functions,
.dad_ninputs = 8
},
};

static int orion_88f5182_xore_irqs[] = { 30, 31 };
static int kirkwood_xore_irqs[] = { 5, 6, 7, 8 };
static int dove_xore_irqs[] = { 39, 40, 42, 43 };
static int armadaxp_xore_irqs0[] = { 51, 52 };
static int armadaxp_xore_irqs1[] = { 94, 95 };

static struct {
int model;
int idmac_nchan;
int idmac_irq;
int xore_nchan;
int *xore_irqs;
} channels[] = {
/*
* Marvell System Controllers:
* need irqs in attach_args.
*/
{ MARVELL_DISCOVERY, 8, -1, 0, NULL },
{ MARVELL_DISCOVERY_II, 8, -1, 0, NULL },
{ MARVELL_DISCOVERY_III, 8, -1, 0, NULL },
#if 0
{ MARVELL_DISCOVERY_LT, 4, -1, 2, NULL },
{ MARVELL_DISCOVERY_V, 4, -1, 2, NULL },
{ MARVELL_DISCOVERY_VI, 4, -1, 2, NULL }, ????
#endif

/*
* Marvell System on Chips:
* No need irqs in attach_args. We always connecting to interrupt-pin
* statically.
*/
{ MARVELL_ORION_1_88F1181, 4, 24, 0, NULL },
{ MARVELL_ORION_2_88F1281, 4, 24, 0, NULL },
{ MARVELL_ORION_1_88F5082, 4, 24, 0, NULL },
{ MARVELL_ORION_1_88F5180N, 4, 24, 0, NULL },
{ MARVELL_ORION_1_88F5181, 4, 24, 0, NULL },
{ MARVELL_ORION_1_88F5182, 4, 24, 2, orion_88f5182_xore_irqs },
{ MARVELL_ORION_2_88F5281, 4, 24, 0, NULL },
{ MARVELL_ORION_1_88W8660, 4, 24, 0, NULL },
{ MARVELL_KIRKWOOD_88F6180, 0, -1, 4, kirkwood_xore_irqs },
{ MARVELL_KIRKWOOD_88F6192, 0, -1, 4, kirkwood_xore_irqs },
{ MARVELL_KIRKWOOD_88F6281, 0, -1, 4, kirkwood_xore_irqs },
{ MARVELL_KIRKWOOD_88F6282, 0, -1, 4, kirkwood_xore_irqs },
{ MARVELL_DOVE_88AP510, 0, -1, 4, dove_xore_irqs },
{ MARVELL_ARMADAXP_MV78130, 4, 33, 2, armadaxp_xore_irqs0 },
{ MARVELL_ARMADAXP_MV78130, 0, -1, 2, armadaxp_xore_irqs1 },
{ MARVELL_ARMADAXP_MV78160, 4, 33, 2, armadaxp_xore_irqs0 },
{ MARVELL_ARMADAXP_MV78160, 0, -1, 2, armadaxp_xore_irqs1 },
{ MARVELL_ARMADAXP_MV78230, 4, 33, 2, armadaxp_xore_irqs0 },
{ MARVELL_ARMADAXP_MV78230, 0, -1, 2, armadaxp_xore_irqs1 },
{ MARVELL_ARMADAXP_MV78260, 4, 33, 2, armadaxp_xore_irqs0 },
{ MARVELL_ARMADAXP_MV78260, 0, -1, 2, armadaxp_xore_irqs1 },
{ MARVELL_ARMADAXP_MV78460, 4, 33, 2, armadaxp_xore_irqs0 },
{ MARVELL_ARMADAXP_MV78460, 0, -1, 2, armadaxp_xore_irqs1 },
};

struct gtidmac_winacctbl *gtidmac_winacctbl;
struct gtidmac_winacctbl *mvxore_winacctbl;

CFATTACH_DECL_NEW(gtidmac_gt, sizeof(struct gtidmac_softc),
gtidmac_match, gtidmac_attach, NULL, NULL);
CFATTACH_DECL_NEW(gtidmac_mbus, sizeof(struct gtidmac_softc),
gtidmac_match, gtidmac_attach, NULL, NULL);

/* ARGSUSED */
static int
gtidmac_match(device_t parent, struct cfdata *match, void *aux)
{
struct marvell_attach_args *mva = aux;
int unit, i;

if (strcmp(mva->mva_name, match->cf_name) != 0)
return 0;
if (mva->mva_offset == MVA_OFFSET_DEFAULT)
return 0;
unit = 0;
for (i = 0; i < __arraycount(channels); i++)
if (mva->mva_model == channels[i].model) {
if (mva->mva_unit == unit) {
mva->mva_size = GTIDMAC_SIZE;
return 1;
}
unit++;
}
return 0;
}

/* ARGSUSED */
static void
gtidmac_attach(device_t parent, device_t self, void *aux)
{
struct gtidmac_softc *sc = device_private(self);
struct marvell_attach_args *mva = aux;
prop_dictionary_t dict = device_properties(self);
uint32_t idmac_irq, xore_irq, *xore_irqs, dmb_speed;
int unit, idmac_nchan, xore_nchan, nsegs, i, j, n;

unit = 0;
for (i = 0; i < __arraycount(channels); i++)
if (mva->mva_model == channels[i].model) {
if (mva->mva_unit == unit)
break;
unit++;
}
idmac_nchan = channels[i].idmac_nchan;
idmac_irq = channels[i].idmac_irq;
if (idmac_nchan != 0) {
if (idmac_irq == -1)
idmac_irq = mva->mva_irq;
if (idmac_irq == -1)
/* Discovery */
if (!prop_dictionary_get_uint32(dict,
"idmac-irq", &idmac_irq)) {
aprint_error(": no idmac-irq property\n");
return;
}
}
xore_nchan = channels[i].xore_nchan;
xore_irqs = channels[i].xore_irqs;
xore_irq = MVA_IRQ_DEFAULT;
if (xore_nchan != 0) {
if (xore_irqs == NULL)
xore_irq = mva->mva_irq;
if (xore_irqs == NULL && xore_irq == MVA_IRQ_DEFAULT)
/* Discovery LT/V/VI */
if (!prop_dictionary_get_uint32(dict,
"xore-irq", &xore_irq)) {
aprint_error(": no xore-irq property\n");
return;
}
}

aprint_naive("\n");
aprint_normal(": Marvell IDMA Controller%s\n",
xore_nchan ? "/XOR Engine" : "");
if (idmac_nchan > 0)
aprint_normal_dev(self,
"IDMA Controller %d channels, intr %d...%d\n",
idmac_nchan, idmac_irq, idmac_irq + GTIDMAC_NINTRRUPT - 1);
if (xore_nchan > 0) {
aprint_normal_dev(self, "XOR Engine %d channels", xore_nchan);
if (xore_irqs == NULL)
aprint_normal(", intr %d...%d\n",
xore_irq, xore_irq + xore_nchan - 1);
else {
aprint_normal(", intr %d", xore_irqs[0]);
for (i = 1; i < xore_nchan; i++)
aprint_normal(", %d", xore_irqs[i]);
aprint_normal("\n");
}
}

sc->sc_dev = self;
sc->sc_iot = mva->mva_iot;

/* Map I/O registers */
if (bus_space_subregion(mva->mva_iot, mva->mva_ioh, mva->mva_offset,
mva->mva_size, &sc->sc_ioh)) {
aprint_error_dev(self, "can't map registers\n");
return;
}

/*
* Initialise DMA descriptors and associated metadata
*/
sc->sc_dmat = mva->mva_dmat;
n = idmac_nchan * GTIDMAC_NDESC + xore_nchan * MVXORE_NDESC;
sc->sc_dd_buffer =
kmem_alloc(sizeof(struct gtidmac_dma_desc) * n, KM_SLEEP);

/* pattern buffer */
if (bus_dmamem_alloc(sc->sc_dmat, PAGE_SIZE, PAGE_SIZE, 0,
&sc->sc_pattern_segment, 1, &nsegs, BUS_DMA_NOWAIT)) {
aprint_error_dev(self,
"bus_dmamem_alloc failed: pattern buffer\n");
goto fail2;
}
if (bus_dmamem_map(sc->sc_dmat, &sc->sc_pattern_segment, 1, PAGE_SIZE,
(void **)&sc->sc_pbuf, BUS_DMA_NOWAIT)) {
aprint_error_dev(self,
"bus_dmamem_map failed: pattern buffer\n");
goto fail3;
}
for (i = 0; i < 0x100; i++)
for (j = 0; j < sizeof(sc->sc_pbuf[i].pbuf); j++)
sc->sc_pbuf[i].pbuf[j] = i;

if (!prop_dictionary_get_uint32(dict, "dmb_speed", &dmb_speed)) {
aprint_error_dev(self, "no dmb_speed property\n");
dmb_speed = 10; /* More than fast swdmover perhaps. */
}

/* IDMAC DMA descriptor buffer */
sc->sc_gtidmac_nchan = idmac_nchan;
if (sc->sc_gtidmac_nchan > 0) {
if (gtidmac_buffer_setup(sc) != 0)
goto fail4;

if (mva->mva_model != MARVELL_DISCOVERY)
gtidmac_wininit(sc, mva->mva_tags);

/* Setup interrupt */
for (i = 0; i < GTIDMAC_NINTRRUPT; i++) {
j = i * idmac_nchan / GTIDMAC_NINTRRUPT;

sc->sc_intrarg[i].ia_sc = sc;
sc->sc_intrarg[i].ia_cause = GTIDMAC_ICR(j);
sc->sc_intrarg[i].ia_eaddr = GTIDMAC_EAR(j);
sc->sc_intrarg[i].ia_eselect = GTIDMAC_ESR(j);
marvell_intr_establish(idmac_irq + i, IPL_BIO,
gtidmac_intr, &sc->sc_intrarg[i]);
}

/* Register us with dmover. */
sc->sc_dmb.dmb_name = device_xname(self);
sc->sc_dmb.dmb_speed = dmb_speed;
sc->sc_dmb.dmb_cookie = sc;
sc->sc_dmb.dmb_algdescs = gtidmac_algdescs;
sc->sc_dmb.dmb_nalgdescs = __arraycount(gtidmac_algdescs);
sc->sc_dmb.dmb_process = gtidmac_process;
dmover_backend_register(&sc->sc_dmb);
sc->sc_dmb_busy = 0;
}

/* XORE DMA descriptor buffer */
sc->sc_mvxore_nchan = xore_nchan;
if (sc->sc_mvxore_nchan > 0) {
if (mvxore_buffer_setup(sc) != 0)
goto fail5;

/* Setup interrupt */
for (i = 0; i < sc->sc_mvxore_nchan; i++)
marvell_intr_establish(
xore_irqs != NULL ? xore_irqs[i] : xore_irq + i,
IPL_BIO,
(i & 0x2) ? mvxore_port1_intr : mvxore_port0_intr,
sc);

mvxore_wininit(sc, mva->mva_tags);

/* Register us with dmover. */
sc->sc_dmb_xore.dmb_name = device_xname(sc->sc_dev);
sc->sc_dmb_xore.dmb_speed = dmb_speed;
sc->sc_dmb_xore.dmb_cookie = sc;
sc->sc_dmb_xore.dmb_algdescs = mvxore_algdescs;
sc->sc_dmb_xore.dmb_nalgdescs = __arraycount(mvxore_algdescs);
sc->sc_dmb_xore.dmb_process = gtidmac_process;
dmover_backend_register(&sc->sc_dmb_xore);
}

gtidmac_softc = sc;

return;

fail5:
for (i = sc->sc_gtidmac_nchan - 1; i >= 0; i--) {
bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc[i].chan_in);
bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc[i].chan_out);
}
bus_dmamap_unload(sc->sc_dmat, sc->sc_dmap);
bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmap);
bus_dmamem_unmap(sc->sc_dmat, sc->sc_dbuf,
sizeof(struct gtidmac_desc) * GTIDMAC_NDESC);
bus_dmamem_free(sc->sc_dmat,
sc->sc_dmap->dm_segs, sc->sc_dmap->dm_nsegs);
fail4:
bus_dmamem_unmap(sc->sc_dmat, sc->sc_pbuf, PAGE_SIZE);
fail3:
bus_dmamem_free(sc->sc_dmat, &sc->sc_pattern_segment, 1);
fail2:
kmem_free(sc->sc_dd_buffer, sizeof(struct gtidmac_dma_desc) * n);
bus_space_unmap(sc->sc_iot, sc->sc_ioh, mva->mva_size);
return;
}

static int
gtidmac_intr(void *arg)
{
struct gtidmac_intr_arg *ia = arg;
struct gtidmac_softc *sc = ia->ia_sc;
uint32_t cause;
int handled = 0, chan, error;

cause = bus_space_read_4(sc->sc_iot, sc->sc_ioh, ia->ia_cause);
DPRINTF(("IDMAC intr: cause=0x%x\n", cause));
bus_space_write_4(sc->sc_iot, sc->sc_ioh, ia->ia_cause, ~cause);

chan = 0;
while (cause) {
error = 0;
if (cause & GTIDMAC_I_ADDRMISS) {
aprint_error_dev(sc->sc_dev, "Address Miss");
error = EINVAL;
}
if (cause & GTIDMAC_I_ACCPROT) {
aprint_error_dev(sc->sc_dev,
"Access Protect Violation");
error = EACCES;
}
if (cause & GTIDMAC_I_WRPROT) {
aprint_error_dev(sc->sc_dev, "Write Protect");
error = EACCES;
}
if (cause & GTIDMAC_I_OWN) {
aprint_error_dev(sc->sc_dev, "Ownership Violation");
error = EINVAL;
}

#define GTIDMAC_I_ERROR \
(GTIDMAC_I_ADDRMISS | \
GTIDMAC_I_ACCPROT | \
GTIDMAC_I_WRPROT | \
GTIDMAC_I_OWN)
if (cause & GTIDMAC_I_ERROR) {
uint32_t sel;
int select;

sel = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
ia->ia_eselect) & GTIDMAC_ESR_SEL;
select = sel - chan * GTIDMAC_I_BITS;
if (select >= 0 && select < GTIDMAC_I_BITS) {
uint32_t ear;

ear = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
ia->ia_eaddr);
aprint_error(": Error Address 0x%x\n", ear);
} else
aprint_error(": lost Error Address\n");
}

if (cause & (GTIDMAC_I_COMP | GTIDMAC_I_ERROR)) {
sc->sc_cdesc[chan].chan_dma_done(
sc->sc_cdesc[chan].chan_running, chan,
&sc->sc_cdesc[chan].chan_in,
&sc->sc_cdesc[chan].chan_out, error);
handled++;
}

cause >>= GTIDMAC_I_BITS;
}
DPRINTF(("IDMAC intr: %shandled\n", handled ? "" : "not "));

return handled;
}

static int
mvxore_port0_intr(void *arg)
{
struct gtidmac_softc *sc = arg;

return mvxore_intr(sc, 0);
}

static int
mvxore_port1_intr(void *arg)
{
struct gtidmac_softc *sc = arg;

return mvxore_intr(sc, 1);
}

static int
mvxore_intr(struct gtidmac_softc *sc, int port)
{
uint32_t cause;
int handled = 0, chan, error;

cause =
bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEICR(sc, port));
DPRINTF(("XORE port %d intr: cause=0x%x\n", port, cause));
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEICR(sc, port), ~cause);

chan = 0;
while (cause) {
error = 0;
if (cause & MVXORE_I_ADDRDECODE) {
aprint_error_dev(sc->sc_dev, "Failed address decoding");
error = EINVAL;
}
if (cause & MVXORE_I_ACCPROT) {
aprint_error_dev(sc->sc_dev,
"Access Protect Violation");
error = EACCES;
}
if (cause & MVXORE_I_WRPROT) {
aprint_error_dev(sc->sc_dev, "Write Protect");
error = EACCES;
}
if (cause & MVXORE_I_OWN) {
aprint_error_dev(sc->sc_dev, "Ownership Violation");
error = EINVAL;
}
if (cause & MVXORE_I_INTPARITY) {
aprint_error_dev(sc->sc_dev, "Parity Error");
error = EIO;
}
if (cause & MVXORE_I_XBAR) {
aprint_error_dev(sc->sc_dev, "Crossbar Parity Error");
error = EINVAL;
}

#define MVXORE_I_ERROR \
(MVXORE_I_ADDRDECODE | \
MVXORE_I_ACCPROT | \
MVXORE_I_WRPROT | \
MVXORE_I_OWN | \
MVXORE_I_INTPARITY | \
MVXORE_I_XBAR)
if (cause & MVXORE_I_ERROR) {
uint32_t type;
int event;

type = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEECR(sc, port));
type &= MVXORE_XEECR_ERRORTYPE_MASK;
event = type - chan * MVXORE_I_BITS;
if (event >= 0 && event < MVXORE_I_BITS) {
uint32_t xeear;

xeear = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEEAR(sc, port));
aprint_error(": Error Address 0x%x\n", xeear);
} else
aprint_error(": lost Error Address\n");
}

if (cause & (MVXORE_I_EOC | MVXORE_I_ERROR)) {
sc->sc_cdesc_xore[chan].chan_dma_done(
sc->sc_cdesc_xore[chan].chan_running, chan,
sc->sc_cdesc_xore[chan].chan_in,
&sc->sc_cdesc_xore[chan].chan_out, error);
handled++;
}

cause >>= MVXORE_I_BITS;
}
DPRINTF(("XORE port %d intr: %shandled\n",
port, handled ? "" : "not "));

return handled;
}

/*
* dmover(9) backend function.
*/
static void
gtidmac_process(struct dmover_backend *dmb)
{
struct gtidmac_softc *sc = dmb->dmb_cookie;
int s;

/* If the backend is currently idle, go process the queue. */
s = splbio();
if (!sc->sc_dmb_busy)
gtidmac_dmover_run(dmb);
splx(s);
}

static void
gtidmac_dmover_run(struct dmover_backend *dmb)
{
struct gtidmac_softc *sc = dmb->dmb_cookie;
struct dmover_request *dreq;
const struct dmover_algdesc *algdesc;
struct gtidmac_function *df;
bus_dmamap_t *dmamap_in, *dmamap_out;
int chan, ninputs, error, i;

sc->sc_dmb_busy = 1;

for (;;) {
dreq = TAILQ_FIRST(&dmb->dmb_pendreqs);
if (dreq == NULL)
break;
algdesc = dreq->dreq_assignment->das_algdesc;
df = algdesc->dad_data;
chan = (*df->chan_alloc)(sc, &dmamap_in, &dmamap_out, dreq);
if (chan == -1)
return;

dmover_backend_remque(dmb, dreq);
dreq->dreq_flags |= DMOVER_REQ_RUNNING;

/* XXXUNLOCK */

error = 0;

/* Load in/out buffers of dmover to bus_dmamap. */
ninputs = dreq->dreq_assignment->das_algdesc->dad_ninputs;
if (ninputs == 0) {
int pno = 0;

if (algdesc->dad_name == DMOVER_FUNC_FILL8)
pno = dreq->dreq_immediate[0];

i = 0;
error = bus_dmamap_load(sc->sc_dmat, *dmamap_in,
&sc->sc_pbuf[pno], sizeof(sc->sc_pbuf[pno]), NULL,
BUS_DMA_NOWAIT | BUS_DMA_STREAMING | BUS_DMA_WRITE);
if (error == 0) {
bus_dmamap_sync(sc->sc_dmat, *dmamap_in, 0,
sizeof(sc->sc_pbuf[pno]),
BUS_DMASYNC_PREWRITE);

/*
* We will call gtidmac_dmmap_unload() when
* becoming an error.
*/
i = 1;
}
} else
for (i = 0; i < ninputs; i++) {
error = gtidmac_dmmap_load(sc,
*(dmamap_in + i), dreq->dreq_inbuf_type,
&dreq->dreq_inbuf[i], 0/*write*/);
if (error != 0)
break;
}
if (algdesc->dad_name != DMOVER_FUNC_ISCSI_CRC32C) {
if (error == 0)
error = gtidmac_dmmap_load(sc, *dmamap_out,
dreq->dreq_outbuf_type, &dreq->dreq_outbuf,
1/*read*/);

if (error == 0) {
/*
* The size of outbuf is always believed to be
* DMA transfer size in dmover request.
*/
error = (*df->dma_setup)(sc, chan, ninputs,
dmamap_in, dmamap_out,
(*dmamap_out)->dm_mapsize);
if (error != 0)
gtidmac_dmmap_unload(sc, *dmamap_out,
1);
}
} else
if (error == 0)
error = (*df->dma_setup)(sc, chan, ninputs,
dmamap_in, dmamap_out,
(*dmamap_in)->dm_mapsize);

/* XXXLOCK */

if (error != 0) {
for (; i-- > 0;)
gtidmac_dmmap_unload(sc, *(dmamap_in + i), 0);
(*df->chan_free)(sc, chan);

dreq->dreq_flags |= DMOVER_REQ_ERROR;
dreq->dreq_error = error;
/* XXXUNLOCK */
dmover_done(dreq);
/* XXXLOCK */
continue;
}

(*df->dma_start)(sc, chan, gtidmac_dmover_done);
break;
}

/* All done */
sc->sc_dmb_busy = 0;
}

static void
gtidmac_dmover_done(void *object, int chan, bus_dmamap_t *dmamap_in,
bus_dmamap_t *dmamap_out, int error)
{
struct gtidmac_softc *sc;
struct dmover_request *dreq = object;
struct dmover_backend *dmb;
struct gtidmac_function *df;
uint32_t result;
int ninputs, i;

KASSERT(dreq != NULL);

dmb = dreq->dreq_assignment->das_backend;
df = dreq->dreq_assignment->das_algdesc->dad_data;
ninputs = dreq->dreq_assignment->das_algdesc->dad_ninputs;
sc = dmb->dmb_cookie;

result = (*df->dma_finish)(sc, chan, error);
for (i = 0; i < ninputs; i++)
gtidmac_dmmap_unload(sc, *(dmamap_in + i), 0);
if (dreq->dreq_assignment->das_algdesc->dad_name ==
DMOVER_FUNC_ISCSI_CRC32C)
memcpy(dreq->dreq_immediate, &result, sizeof(result));
else
gtidmac_dmmap_unload(sc, *dmamap_out, 1);

(*df->chan_free)(sc, chan);

if (error) {
dreq->dreq_error = error;
dreq->dreq_flags |= DMOVER_REQ_ERROR;
}

dmover_done(dreq);

/*
* See if we can start some more dmover(9) requests.
*
* Note: We're already at splbio() here.
*/
if (!sc->sc_dmb_busy)
gtidmac_dmover_run(dmb);
}

static __inline int
gtidmac_dmmap_load(struct gtidmac_softc *sc, bus_dmamap_t dmamap,
dmover_buffer_type dmbuf_type, dmover_buffer *dmbuf,
int read)
{
int error, flags;

flags = BUS_DMA_NOWAIT | BUS_DMA_STREAMING |
read ? BUS_DMA_READ : BUS_DMA_WRITE;

switch (dmbuf_type) {
case DMOVER_BUF_LINEAR:
error = bus_dmamap_load(sc->sc_dmat, dmamap,
dmbuf->dmbuf_linear.l_addr, dmbuf->dmbuf_linear.l_len,
NULL, flags);
break;

case DMOVER_BUF_UIO:
if ((read && dmbuf->dmbuf_uio->uio_rw != UIO_READ) ||
(!read && dmbuf->dmbuf_uio->uio_rw == UIO_READ))
return (EINVAL);

error = bus_dmamap_load_uio(sc->sc_dmat, dmamap,
dmbuf->dmbuf_uio, flags);
break;

default:
error = EINVAL;
}

if (error == 0)
bus_dmamap_sync(sc->sc_dmat, dmamap, 0, dmamap->dm_mapsize,
read ? BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);

return error;
}

static __inline void
gtidmac_dmmap_unload(struct gtidmac_softc *sc, bus_dmamap_t dmamap, int read)
{

bus_dmamap_sync(sc->sc_dmat, dmamap, 0, dmamap->dm_mapsize,
read ? BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);

bus_dmamap_unload(sc->sc_dmat, dmamap);
}

/*
* IDMAC functions
*/
int
gtidmac_chan_alloc(void *tag, bus_dmamap_t **dmamap_in,
bus_dmamap_t **dmamap_out, void *object)
{
struct gtidmac_softc *sc = tag;
int chan;

/* maybe need lock */

for (chan = 0; chan < sc->sc_gtidmac_nchan; chan++)
if (sc->sc_cdesc[chan].chan_running == NULL)
break;
if (chan >= sc->sc_gtidmac_nchan)
return -1;

sc->sc_cdesc[chan].chan_running = object;

/* unlock */

*dmamap_in = &sc->sc_cdesc[chan].chan_in;
*dmamap_out = &sc->sc_cdesc[chan].chan_out;

return chan;
}

void
gtidmac_chan_free(void *tag, int chan)
{
struct gtidmac_softc *sc = tag;

/* maybe need lock */

sc->sc_cdesc[chan].chan_running = NULL;

/* unlock */
}

/* ARGSUSED */
int
gtidmac_setup(void *tag, int chan, int ninputs, bus_dmamap_t *dmamap_in,
bus_dmamap_t *dmamap_out, bus_size_t size)
{
struct gtidmac_softc *sc = tag;
struct gtidmac_dma_desc *dd, *fstdd, *nxtdd;
struct gtidmac_desc *desc;
uint32_t ccl, bcnt, ires, ores;
int n = 0, iidx, oidx;

KASSERT(ninputs == 0 || ninputs == 1);

ccl = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan));
#ifdef DIAGNOSTIC
if (ccl & GTIDMAC_CCLR_CHANACT)
panic("gtidmac_setup: chan%d already active", chan);
#endif

/* We always Chain-mode and max (16M - 1)byte/desc */
ccl = (GTIDMAC_CCLR_DESCMODE_16M |
#ifdef GTIDMAC_DEBUG
GTIDMAC_CCLR_CDEN |
#endif
GTIDMAC_CCLR_TRANSFERMODE_B /* Transfer Mode: Block */ |
GTIDMAC_CCLR_INTMODE_NULL /* Intr Mode: Next Desc NULL */ |
GTIDMAC_CCLR_CHAINMODE_C /* Chain Mode: Chaind */);
if (size != (*dmamap_in)->dm_mapsize) {
ccl |= GTIDMAC_CCLR_SRCHOLD;
if ((*dmamap_in)->dm_mapsize == 8)
ccl |= GTIDMAC_CCLR_SBL_8B;
else if ((*dmamap_in)->dm_mapsize == 16)
ccl |= GTIDMAC_CCLR_SBL_16B;
else if ((*dmamap_in)->dm_mapsize == 32)
ccl |= GTIDMAC_CCLR_SBL_32B;
else if ((*dmamap_in)->dm_mapsize == 64)
ccl |= GTIDMAC_CCLR_SBL_64B;
else if ((*dmamap_in)->dm_mapsize == 128)
ccl |= GTIDMAC_CCLR_SBL_128B;
else
panic("gtidmac_setup: chan%d source:"
" unsupported hold size", chan);
} else
ccl |= GTIDMAC_CCLR_SBL_128B;
if (size != (*dmamap_out)->dm_mapsize) {
ccl |= GTIDMAC_CCLR_DESTHOLD;
if ((*dmamap_out)->dm_mapsize == 8)
ccl |= GTIDMAC_CCLR_DBL_8B;
else if ((*dmamap_out)->dm_mapsize == 16)
ccl |= GTIDMAC_CCLR_DBL_16B;
else if ((*dmamap_out)->dm_mapsize == 32)
ccl |= GTIDMAC_CCLR_DBL_32B;
else if ((*dmamap_out)->dm_mapsize == 64)
ccl |= GTIDMAC_CCLR_DBL_64B;
else if ((*dmamap_out)->dm_mapsize == 128)
ccl |= GTIDMAC_CCLR_DBL_128B;
else
panic("gtidmac_setup: chan%d destination:"
" unsupport hold size", chan);
} else
ccl |= GTIDMAC_CCLR_DBL_128B;

fstdd = SLIST_FIRST(&sc->sc_dlist);
if (fstdd == NULL) {
aprint_error_dev(sc->sc_dev, "no descriptor\n");
return ENOMEM;
}
SLIST_REMOVE_HEAD(&sc->sc_dlist, dd_next);
sc->sc_cdesc[chan].chan_ddidx = fstdd->dd_index;

dd = fstdd;
ires = ores = 0;
iidx = oidx = 0;
while (1 /*CONSTCOND*/) {
if (ccl & GTIDMAC_CCLR_SRCHOLD) {
if (ccl & GTIDMAC_CCLR_DESTHOLD)
bcnt = size; /* src/dst hold */
else
bcnt = (*dmamap_out)->dm_segs[oidx].ds_len;
} else if (ccl & GTIDMAC_CCLR_DESTHOLD)
bcnt = (*dmamap_in)->dm_segs[iidx].ds_len;
else
bcnt = uimin((*dmamap_in)->dm_segs[iidx].ds_len - ires,
(*dmamap_out)->dm_segs[oidx].ds_len - ores);

desc = dd->dd_idmac_vaddr;
desc->bc.mode16m.bcnt =
bcnt | GTIDMAC_CIDMABCR_BCLEFT | GTIDMAC_CIDMABCR_OWN;
desc->srcaddr = (*dmamap_in)->dm_segs[iidx].ds_addr + ires;
desc->dstaddr = (*dmamap_out)->dm_segs[oidx].ds_addr + ores;

n += bcnt;
if (n >= size)
break;
if (!(ccl & GTIDMAC_CCLR_SRCHOLD)) {
ires += bcnt;
if (ires >= (*dmamap_in)->dm_segs[iidx].ds_len) {
ires = 0;
iidx++;
KASSERT(iidx < (*dmamap_in)->dm_nsegs);
}
}
if (!(ccl & GTIDMAC_CCLR_DESTHOLD)) {
ores += bcnt;
if (ores >= (*dmamap_out)->dm_segs[oidx].ds_len) {
ores = 0;
oidx++;
KASSERT(oidx < (*dmamap_out)->dm_nsegs);
}
}

nxtdd = SLIST_FIRST(&sc->sc_dlist);
if (nxtdd == NULL) {
aprint_error_dev(sc->sc_dev, "no descriptor\n");
return ENOMEM;
}
SLIST_REMOVE_HEAD(&sc->sc_dlist, dd_next);

desc->nextdp = (uint32_t)nxtdd->dd_paddr;
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
dd->dd_index * sizeof(*desc), sizeof(*desc),
#ifdef GTIDMAC_DEBUG
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
#else
BUS_DMASYNC_PREWRITE);
#endif

SLIST_INSERT_AFTER(dd, nxtdd, dd_next);
dd = nxtdd;
}
desc->nextdp = (uint32_t)NULL;
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap, dd->dd_index * sizeof(*desc),
#ifdef GTIDMAC_DEBUG
sizeof(*desc), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
#else
sizeof(*desc), BUS_DMASYNC_PREWRITE);
#endif

/* Set paddr of descriptor to Channel Next Descriptor Pointer */
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CNDPR(chan),
fstdd->dd_paddr);

#if BYTE_ORDER == LITTLE_ENDIAN
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCHR(chan),
GTIDMAC_CCHR_DESCBYTESWAP | GTIDMAC_CCHR_ENDIAN_LE);
#else
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCHR(chan),
GTIDMAC_CCHR_DESCBYTESWAP | GTIDMAC_CCHR_ENDIAN_BE);
#endif
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan), ccl);

#ifdef GTIDMAC_DEBUG
gtidmac_dump_idmacdesc(sc, fstdd, ccl, 0/*pre*/);
#endif

sc->sc_cdesc[chan].chan_totalcnt += size;

return 0;
}

void
gtidmac_start(void *tag, int chan,
void (*dma_done_cb)(void *, int, bus_dmamap_t *, bus_dmamap_t *,
int))
{
struct gtidmac_softc *sc = tag;
uint32_t ccl;

DPRINTF(("%s:%d: starting\n", device_xname(sc->sc_dev), chan));

#ifdef GTIDMAC_DEBUG
gtidmac_dump_idmacreg(sc, chan);
#endif

sc->sc_cdesc[chan].chan_dma_done = dma_done_cb;

ccl = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan));
/* Start and 'Fetch Next Descriptor' */
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan),
ccl | GTIDMAC_CCLR_CHANEN | GTIDMAC_CCLR_FETCHND);
}

static uint32_t
gtidmac_finish(void *tag, int chan, int error)
{
struct gtidmac_softc *sc = tag;
struct gtidmac_dma_desc *dd, *fstdd, *nxtdd;
struct gtidmac_desc *desc;

fstdd = &sc->sc_dd_buffer[sc->sc_cdesc[chan].chan_ddidx];

#ifdef GTIDMAC_DEBUG
if (error || gtidmac_debug > 1) {
uint32_t ccl;

gtidmac_dump_idmacreg(sc, chan);
ccl = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
GTIDMAC_CCLR(chan));
gtidmac_dump_idmacdesc(sc, fstdd, ccl, 1/*post*/);
}
#endif

dd = fstdd;
do {
desc = dd->dd_idmac_vaddr;

bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
dd->dd_index * sizeof(*desc), sizeof(*desc),
#ifdef GTIDMAC_DEBUG
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
#else
BUS_DMASYNC_POSTWRITE);
#endif

nxtdd = SLIST_NEXT(dd, dd_next);
SLIST_INSERT_HEAD(&sc->sc_dlist, dd, dd_next);
dd = nxtdd;
} while (desc->nextdp);

return 0;
}

/*
* XORE functions
*/
int
mvxore_chan_alloc(void *tag, bus_dmamap_t **dmamap_in,
bus_dmamap_t **dmamap_out, void *object)
{
struct gtidmac_softc *sc = tag;
int chan;

/* maybe need lock */

for (chan = 0; chan < sc->sc_mvxore_nchan; chan++)
if (sc->sc_cdesc_xore[chan].chan_running == NULL)
break;
if (chan >= sc->sc_mvxore_nchan)
return -1;

sc->sc_cdesc_xore[chan].chan_running = object;

/* unlock */

*dmamap_in = sc->sc_cdesc_xore[chan].chan_in;
*dmamap_out = &sc->sc_cdesc_xore[chan].chan_out;

return chan;
}

void
mvxore_chan_free(void *tag, int chan)
{
struct gtidmac_softc *sc = tag;

/* maybe need lock */

sc->sc_cdesc_xore[chan].chan_running = NULL;

/* unlock */
}

/* ARGSUSED */
int
mvxore_setup(void *tag, int chan, int ninputs, bus_dmamap_t *dmamap_in,
bus_dmamap_t *dmamap_out, bus_size_t size)
{
struct gtidmac_softc *sc = tag;
struct gtidmac_dma_desc *dd, *fstdd, *nxtdd;
struct mvxore_desc *desc;
uint32_t xexc, bcnt, cmd, lastcmd;
int n = 0, i;
uint32_t ires[MVXORE_NSRC] = { 0, 0, 0, 0, 0, 0, 0, 0 }, ores = 0;
int iidx[MVXORE_NSRC] = { 0, 0, 0, 0, 0, 0, 0, 0 }, oidx = 0;

#ifdef DIAGNOSTIC
uint32_t xexact =
bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXACTR(sc, chan));

if ((xexact & MVXORE_XEXACTR_XESTATUS_MASK) ==
MVXORE_XEXACTR_XESTATUS_ACT)
panic("mvxore_setup: chan%d already active."
" mvxore not support hot insertion", chan);
#endif

xexc =
(MVXORE_XEXCR_REGACCPROTECT |
MVXORE_XEXCR_DBL_128B |
MVXORE_XEXCR_SBL_128B);
cmd = lastcmd = 0;
if (ninputs > 1) {
xexc |= MVXORE_XEXCR_OM_XOR;
lastcmd = cmd = (1 << ninputs) - 1;
} else if (ninputs == 1) {
if ((*dmamap_out)->dm_nsegs == 0) {
xexc |= MVXORE_XEXCR_OM_CRC32;
lastcmd = MVXORE_DESC_CMD_CRCLAST;
} else
xexc |= MVXORE_XEXCR_OM_DMA;
} else if (ninputs == 0) {
if ((*dmamap_out)->dm_nsegs != 1) {
aprint_error_dev(sc->sc_dev,
"XORE not supports %d DMA segments\n",
(*dmamap_out)->dm_nsegs);
return EINVAL;
}

if ((*dmamap_in)->dm_mapsize == 0) {
xexc |= MVXORE_XEXCR_OM_ECC;

/* XXXXX: Maybe need to set Timer Mode registers? */

#if 0
} else if ((*dmamap_in)->dm_mapsize == 8 ||
(*dmamap_in)->dm_mapsize == 16) { /* in case dmover */
uint64_t pattern;

/* XXXX: Get pattern data */

KASSERT((*dmamap_in)->dm_mapsize == 8 ||
(void *)((uint32_t)(*dmamap_in)->_dm_origbuf &
~PAGE_MASK) == sc->sc_pbuf);
pattern = *(uint64_t *)(*dmamap_in)->_dm_origbuf;

/* XXXXX: XORE has a IVR. We should get this first. */
bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEIVRL,
pattern);
bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEIVRH,
pattern >> 32);

xexc |= MVXORE_XEXCR_OM_MEMINIT;
#endif
} else {
aprint_error_dev(sc->sc_dev,
"XORE not supports DMA mapsize %zd\n",
(*dmamap_in)->dm_mapsize);
return EINVAL;
}
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEXDPR(sc, chan), (*dmamap_out)->dm_segs[0].ds_addr);
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEXBSR(sc, chan), (*dmamap_out)->dm_mapsize);

bus_space_write_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEXCR(sc, chan), xexc);
sc->sc_cdesc_xore[chan].chan_totalcnt += size;

return 0;
}

/* Make descriptor for DMA/CRC32/XOR */

fstdd = SLIST_FIRST(&sc->sc_dlist_xore);
if (fstdd == NULL) {
aprint_error_dev(sc->sc_dev, "no xore descriptor\n");
return ENOMEM;
}
SLIST_REMOVE_HEAD(&sc->sc_dlist_xore, dd_next);
sc->sc_cdesc_xore[chan].chan_ddidx =
fstdd->dd_index + GTIDMAC_NDESC * sc->sc_gtidmac_nchan;

dd = fstdd;
while (1 /*CONSTCOND*/) {
desc = dd->dd_xore_vaddr;
desc->stat = MVXORE_DESC_STAT_OWN;
desc->cmd = cmd;
if ((*dmamap_out)->dm_nsegs != 0) {
desc->dstaddr =
(*dmamap_out)->dm_segs[oidx].ds_addr + ores;
bcnt = (*dmamap_out)->dm_segs[oidx].ds_len - ores;
} else {
desc->dstaddr = 0;
bcnt = MVXORE_MAXXFER; /* XXXXX */
}
for (i = 0; i < ninputs; i++) {
desc->srcaddr[i] =
(*dmamap_in[i]).dm_segs[iidx[i]].ds_addr + ires[i];
bcnt = uimin(bcnt,
(*dmamap_in[i]).dm_segs[iidx[i]].ds_len - ires[i]);
}
desc->bcnt = bcnt;

n += bcnt;
if (n >= size)
break;
ores += bcnt;
if ((*dmamap_out)->dm_nsegs != 0 &&
ores >= (*dmamap_out)->dm_segs[oidx].ds_len) {
ores = 0;
oidx++;
KASSERT(oidx < (*dmamap_out)->dm_nsegs);
}
for (i = 0; i < ninputs; i++) {
ires[i] += bcnt;
if (ires[i] >=
(*dmamap_in[i]).dm_segs[iidx[i]].ds_len) {
ires[i] = 0;
iidx[i]++;
KASSERT(iidx[i] < (*dmamap_in[i]).dm_nsegs);
}
}

nxtdd = SLIST_FIRST(&sc->sc_dlist_xore);
if (nxtdd == NULL) {
aprint_error_dev(sc->sc_dev, "no xore descriptor\n");
return ENOMEM;
}
SLIST_REMOVE_HEAD(&sc->sc_dlist_xore, dd_next);

desc->nextda = (uint32_t)nxtdd->dd_paddr;
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
dd->dd_index * sizeof(*desc), sizeof(*desc),
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

SLIST_INSERT_AFTER(dd, nxtdd, dd_next);
dd = nxtdd;
}
desc->cmd = lastcmd;
desc->nextda = (uint32_t)NULL;
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
dd->dd_index * sizeof(*desc), sizeof(*desc),
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

/* Set paddr of descriptor to Channel Next Descriptor Pointer */
bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXNDPR(sc, chan),
fstdd->dd_paddr);

bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXCR(sc, chan), xexc);

#ifdef GTIDMAC_DEBUG
gtidmac_dump_xoredesc(sc, fstdd, xexc, 0/*pre*/);
#endif

sc->sc_cdesc_xore[chan].chan_totalcnt += size;

return 0;
}

void
mvxore_start(void *tag, int chan,
void (*dma_done_cb)(void *, int, bus_dmamap_t *, bus_dmamap_t *,
int))
{
struct gtidmac_softc *sc = tag;
uint32_t xexact;

DPRINTF(("%s:%d: xore starting\n", device_xname(sc->sc_dev), chan));

#ifdef GTIDMAC_DEBUG
gtidmac_dump_xorereg(sc, chan);
#endif

sc->sc_cdesc_xore[chan].chan_dma_done = dma_done_cb;

xexact =
bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXACTR(sc, chan));
bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXACTR(sc, chan),
xexact | MVXORE_XEXACTR_XESTART);
}

static uint32_t
mvxore_finish(void *tag, int chan, int error)
{
struct gtidmac_softc *sc = tag;
struct gtidmac_dma_desc *dd, *fstdd, *nxtdd;
struct mvxore_desc *desc;
uint32_t xexc;

#ifdef GTIDMAC_DEBUG
if (error || gtidmac_debug > 1)
gtidmac_dump_xorereg(sc, chan);
#endif

xexc = bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXCR(sc, chan));
if ((xexc & MVXORE_XEXCR_OM_MASK) == MVXORE_XEXCR_OM_ECC ||
(xexc & MVXORE_XEXCR_OM_MASK) == MVXORE_XEXCR_OM_MEMINIT)
return 0;

fstdd = &sc->sc_dd_buffer[sc->sc_cdesc_xore[chan].chan_ddidx];

#ifdef GTIDMAC_DEBUG
if (error || gtidmac_debug > 1)
gtidmac_dump_xoredesc(sc, fstdd, xexc, 1/*post*/);
#endif

dd = fstdd;
do {
desc = dd->dd_xore_vaddr;

bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
dd->dd_index * sizeof(*desc), sizeof(*desc),
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

nxtdd = SLIST_NEXT(dd, dd_next);
SLIST_INSERT_HEAD(&sc->sc_dlist_xore, dd, dd_next);
dd = nxtdd;
} while (desc->nextda);

if ((xexc & MVXORE_XEXCR_OM_MASK) == MVXORE_XEXCR_OM_CRC32)
return desc->result;
return 0;
}

static void
gtidmac_wininit(struct gtidmac_softc *sc, enum marvell_tags *tags)
{
device_t pdev = device_parent(sc->sc_dev);
uint64_t base;
uint32_t size, cxap, en, winacc;
int window, target, attr, rv, i, j;

en = 0xff;
cxap = 0;
for (window = 0, i = 0;
tags[i] != MARVELL_TAG_UNDEFINED && window < GTIDMAC_NWINDOW; i++) {
rv = marvell_winparams_by_tag(pdev, tags[i],
&target, &attr, &base, &size);
if (rv != 0 || size == 0)
continue;

if (base > 0xffffffffULL) {
if (window >= GTIDMAC_NREMAP) {
aprint_error_dev(sc->sc_dev,
"can't remap window %d\n", window);
continue;
}
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
GTIDMAC_HARXR(window), (base >> 32) & 0xffffffff);
}
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_BARX(window),
GTIDMAC_BARX_TARGET(target) |
GTIDMAC_BARX_ATTR(attr) |
GTIDMAC_BARX_BASE(base));
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_SRX(window),
GTIDMAC_SRX_SIZE(size));
en &= ~GTIDMAC_BAER_EN(window);

winacc = GTIDMAC_CXAPR_WINACC_FA;
if (gtidmac_winacctbl != NULL)
for (j = 0;
gtidmac_winacctbl[j].tag != MARVELL_TAG_UNDEFINED;
j++) {
if (gtidmac_winacctbl[j].tag != tags[i])
continue;

switch (gtidmac_winacctbl[j].winacc) {
case GTIDMAC_WINACC_NOACCESSALLOWED:
winacc = GTIDMAC_CXAPR_WINACC_NOAA;
break;
case GTIDMAC_WINACC_READONLY:
winacc = GTIDMAC_CXAPR_WINACC_RO;
break;
case GTIDMAC_WINACC_FULLACCESS:
default: /* XXXX: default is full access */
break;
}
break;
}
cxap |= GTIDMAC_CXAPR_WINACC(window, winacc);

window++;
}
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_BAER, en);

for (i = 0; i < GTIDMAC_NACCPROT; i++)
bus_space_write_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CXAPR(i),
cxap);
}

static void
mvxore_wininit(struct gtidmac_softc *sc, enum marvell_tags *tags)
{
device_t pdev = device_parent(sc->sc_dev);
uint64_t base;
uint32_t target, attr, size, xexwc, winacc;
int window, rv, i, j, p;

xexwc = 0;
for (window = 0, i = 0;
tags[i] != MARVELL_TAG_UNDEFINED && window < MVXORE_NWINDOW; i++) {
rv = marvell_winparams_by_tag(pdev, tags[i],
&target, &attr, &base, &size);
if (rv != 0 || size == 0)
continue;

if (base > 0xffffffffULL) {
if (window >= MVXORE_NREMAP) {
aprint_error_dev(sc->sc_dev,
"can't remap window %d\n", window);
continue;
}
for (p = 0; p < sc->sc_mvxore_nchan >> 1; p++)
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEHARRX(sc, p, window),
(base >> 32) & 0xffffffff);
}

for (p = 0; p < sc->sc_mvxore_nchan >> 1; p++) {
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEBARX(sc, p, window),
MVXORE_XEBARX_TARGET(target) |
MVXORE_XEBARX_ATTR(attr) |
MVXORE_XEBARX_BASE(base));
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XESMRX(sc, p, window),
MVXORE_XESMRX_SIZE(size));
}

winacc = MVXORE_XEXWCR_WINACC_FA;
if (mvxore_winacctbl != NULL)
for (j = 0;
mvxore_winacctbl[j].tag != MARVELL_TAG_UNDEFINED;
j++) {
if (gtidmac_winacctbl[j].tag != tags[i])
continue;

switch (gtidmac_winacctbl[j].winacc) {
case GTIDMAC_WINACC_NOACCESSALLOWED:
winacc = MVXORE_XEXWCR_WINACC_NOAA;
break;
case GTIDMAC_WINACC_READONLY:
winacc = MVXORE_XEXWCR_WINACC_RO;
break;
case GTIDMAC_WINACC_FULLACCESS:
default: /* XXXX: default is full access */
break;
}
break;
}
xexwc |= (MVXORE_XEXWCR_WINEN(window) |
MVXORE_XEXWCR_WINACC(window, winacc));
window++;
}

for (i = 0; i < sc->sc_mvxore_nchan; i++) {
bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXWCR(sc, i),
xexwc);

/* XXXXX: reset... */
bus_space_write_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXAOCR(sc, 0),
0);
}
}

static int
gtidmac_buffer_setup(struct gtidmac_softc *sc)
{
bus_dma_segment_t segs;
struct gtidmac_dma_desc *dd;
uint32_t mask;
int nchan, nsegs, i;

nchan = sc->sc_gtidmac_nchan;

if (bus_dmamem_alloc(sc->sc_dmat,
sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan,
PAGE_SIZE, 0, &segs, 1, &nsegs, BUS_DMA_NOWAIT)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamem_alloc failed: descriptor buffer\n");
goto fail0;
}
if (bus_dmamem_map(sc->sc_dmat, &segs, 1,
sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan,
(void **)&sc->sc_dbuf, BUS_DMA_NOWAIT)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamem_map failed: descriptor buffer\n");
goto fail1;
}
if (bus_dmamap_create(sc->sc_dmat,
sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan, 1,
sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan, 0,
BUS_DMA_NOWAIT, &sc->sc_dmap)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamap_create failed: descriptor buffer\n");
goto fail2;
}
if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmap, sc->sc_dbuf,
sizeof(struct gtidmac_desc) * GTIDMAC_NDESC * nchan,
NULL, BUS_DMA_NOWAIT)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamap_load failed: descriptor buffer\n");
goto fail3;
}
SLIST_INIT(&sc->sc_dlist);
for (i = 0; i < GTIDMAC_NDESC * nchan; i++) {
dd = &sc->sc_dd_buffer[i];
dd->dd_index = i;
dd->dd_idmac_vaddr = &sc->sc_dbuf[i];
dd->dd_paddr = sc->sc_dmap->dm_segs[0].ds_addr +
(sizeof(struct gtidmac_desc) * i);
SLIST_INSERT_HEAD(&sc->sc_dlist, dd, dd_next);
}

/* Initialize IDMAC DMA channels */
mask = 0;
for (i = 0; i < nchan; i++) {
if (i > 0 && ((i * GTIDMAC_I_BITS) & 31 /*bit*/) == 0) {
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
GTIDMAC_IMR(i - 1), mask);
mask = 0;
}

if (bus_dmamap_create(sc->sc_dmat, GTIDMAC_MAXXFER,
GTIDMAC_NSEGS, GTIDMAC_MAXXFER, 0, BUS_DMA_NOWAIT,
&sc->sc_cdesc[i].chan_in)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamap_create failed: chan%d in\n", i);
goto fail4;
}
if (bus_dmamap_create(sc->sc_dmat, GTIDMAC_MAXXFER,
GTIDMAC_NSEGS, GTIDMAC_MAXXFER, 0, BUS_DMA_NOWAIT,
&sc->sc_cdesc[i].chan_out)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamap_create failed: chan%d out\n", i);
bus_dmamap_destroy(sc->sc_dmat,
sc->sc_cdesc[i].chan_in);
goto fail4;
}
sc->sc_cdesc[i].chan_totalcnt = 0;
sc->sc_cdesc[i].chan_running = NULL;

/* Ignore bits overflow. The mask is 32bit. */
mask |= GTIDMAC_I(i,
GTIDMAC_I_COMP |
GTIDMAC_I_ADDRMISS |
GTIDMAC_I_ACCPROT |
GTIDMAC_I_WRPROT |
GTIDMAC_I_OWN);

/* 8bits/channel * 4channels => 32bit */
if ((i & 0x3) == 0x3) {
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
GTIDMAC_IMR(i), mask);
mask = 0;
}
}

return 0;

fail4:
for (; i-- > 0;) {
bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc[i].chan_in);
bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc[i].chan_out);
}
bus_dmamap_unload(sc->sc_dmat, sc->sc_dmap);
fail3:
bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmap);
fail2:
bus_dmamem_unmap(sc->sc_dmat, sc->sc_dbuf,
sizeof(struct gtidmac_desc) * GTIDMAC_NDESC);
fail1:
bus_dmamem_free(sc->sc_dmat, &segs, 1);
fail0:
return -1;
}

static int
mvxore_buffer_setup(struct gtidmac_softc *sc)
{
bus_dma_segment_t segs;
struct gtidmac_dma_desc *dd;
uint32_t mask;
int nchan, nsegs, i, j;

nchan = sc->sc_mvxore_nchan;

if (bus_dmamem_alloc(sc->sc_dmat,
sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan,
PAGE_SIZE, 0, &segs, 1, &nsegs, BUS_DMA_NOWAIT)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamem_alloc failed: xore descriptor buffer\n");
goto fail0;
}
if (bus_dmamem_map(sc->sc_dmat, &segs, 1,
sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan,
(void **)&sc->sc_dbuf_xore, BUS_DMA_NOWAIT)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamem_map failed: xore descriptor buffer\n");
goto fail1;
}
if (bus_dmamap_create(sc->sc_dmat,
sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan, 1,
sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan, 0,
BUS_DMA_NOWAIT, &sc->sc_dmap_xore)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamap_create failed: xore descriptor buffer\n");
goto fail2;
}
if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmap_xore, sc->sc_dbuf_xore,
sizeof(struct mvxore_desc) * MVXORE_NDESC * nchan,
NULL, BUS_DMA_NOWAIT)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamap_load failed: xore descriptor buffer\n");
goto fail3;
}
SLIST_INIT(&sc->sc_dlist_xore);
for (i = 0; i < MVXORE_NDESC * nchan; i++) {
dd =
&sc->sc_dd_buffer[i + GTIDMAC_NDESC * sc->sc_gtidmac_nchan];
dd->dd_index = i;
dd->dd_xore_vaddr = &sc->sc_dbuf_xore[i];
dd->dd_paddr = sc->sc_dmap_xore->dm_segs[0].ds_addr +
(sizeof(struct mvxore_desc) * i);
SLIST_INSERT_HEAD(&sc->sc_dlist_xore, dd, dd_next);
}

/* Initialize XORE DMA channels */
mask = 0;
for (i = 0; i < nchan; i++) {
for (j = 0; j < MVXORE_NSRC; j++) {
if (bus_dmamap_create(sc->sc_dmat,
MVXORE_MAXXFER, MVXORE_NSEGS,
MVXORE_MAXXFER, 0, BUS_DMA_NOWAIT,
&sc->sc_cdesc_xore[i].chan_in[j])) {
aprint_error_dev(sc->sc_dev,
"bus_dmamap_create failed:"
" xore chan%d in[%d]\n", i, j);
goto fail4;
}
}
if (bus_dmamap_create(sc->sc_dmat, MVXORE_MAXXFER,
MVXORE_NSEGS, MVXORE_MAXXFER, 0,
BUS_DMA_NOWAIT, &sc->sc_cdesc_xore[i].chan_out)) {
aprint_error_dev(sc->sc_dev,
"bus_dmamap_create failed: chan%d out\n", i);
goto fail5;
}
sc->sc_cdesc_xore[i].chan_totalcnt = 0;
sc->sc_cdesc_xore[i].chan_running = NULL;

mask |= MVXORE_I(i,
MVXORE_I_EOC |
MVXORE_I_ADDRDECODE |
MVXORE_I_ACCPROT |
MVXORE_I_WRPROT |
MVXORE_I_OWN |
MVXORE_I_INTPARITY |
MVXORE_I_XBAR);

/* 16bits/channel * 2channels => 32bit */
if (i & 0x1) {
bus_space_write_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEIMR(sc, i >> 1), mask);
mask = 0;
}
}

return 0;

for (; i-- > 0;) {
bus_dmamap_destroy(sc->sc_dmat, sc->sc_cdesc_xore[i].chan_out);

fail5:
j = MVXORE_NSRC;
fail4:
for (; j-- > 0;)
bus_dmamap_destroy(sc->sc_dmat,
sc->sc_cdesc_xore[i].chan_in[j]);
}
bus_dmamap_unload(sc->sc_dmat, sc->sc_dmap_xore);
fail3:
bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmap_xore);
fail2:
bus_dmamem_unmap(sc->sc_dmat, sc->sc_dbuf_xore,
sizeof(struct mvxore_desc) * MVXORE_NDESC);
fail1:
bus_dmamem_free(sc->sc_dmat, &segs, 1);
fail0:
return -1;
}

#ifdef GTIDMAC_DEBUG
static void
gtidmac_dump_idmacreg(struct gtidmac_softc *sc, int chan)
{
uint32_t val;
char buf[256];

printf("IDMAC Registers\n");

val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CIDMABCR(chan));
snprintb(buf, sizeof(buf), "\177\020b\037Own\0b\036BCLeft\0", val);
printf(" Byte Count : %s\n", buf);
printf(" ByteCnt : 0x%06x\n",
val & GTIDMAC_CIDMABCR_BYTECNT_MASK);
printf(" Source Address : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CIDMASAR(chan)));
printf(" Destination Address : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CIDMADAR(chan)));
printf(" Next Descriptor Pointer : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CNDPR(chan)));
printf(" Current Descriptor Pointer : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCDPR(chan)));

val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCLR(chan));
snprintb(buf, sizeof(buf),
"\177\020b\024Abr\0b\021CDEn\0b\016ChanAct\0b\015FetchND\0"
"b\014ChanEn\0b\012IntMode\0b\005DestHold\0b\003SrcHold\0",
val);
printf(" Channel Control (Low) : %s\n", buf);
printf(" SrcBurstLimit : %s Bytes\n",
(val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_128B ? "128" :
(val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_64B ? "64" :
(val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_32B ? "32" :
(val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_16B ? "16" :
(val & GTIDMAC_CCLR_SBL_MASK) == GTIDMAC_CCLR_SBL_8B ? "8" :
"unknown");
printf(" DstBurstLimit : %s Bytes\n",
(val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_128B ? "128" :
(val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_64B ? "64" :
(val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_32B ? "32" :
(val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_16B ? "16" :
(val & GTIDMAC_CCLR_DBL_MASK) == GTIDMAC_CCLR_DBL_8B ? "8" :
"unknown");
printf(" ChainMode : %sChained\n",
val & GTIDMAC_CCLR_CHAINMODE_NC ? "Non-" : "");
printf(" TransferMode : %s\n",
val & GTIDMAC_CCLR_TRANSFERMODE_B ? "Block" : "Demand");
printf(" DescMode : %s\n",
val & GTIDMAC_CCLR_DESCMODE_16M ? "16M" : "64k");
val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, GTIDMAC_CCHR(chan));
snprintb(buf, sizeof(buf),
"\177\020b\001DescByteSwap\0b\000Endianness\0", val);
printf(" Channel Control (High) : %s\n", buf);
}

static void
gtidmac_dump_idmacdesc(struct gtidmac_softc *sc, struct gtidmac_dma_desc *dd,
uint32_t mode, int post)
{
struct gtidmac_desc *desc;
int i;
char buf[256];

printf("IDMAC Descriptor\n");

i = 0;
while (1 /*CONSTCOND*/) {
if (post)
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
dd->dd_index * sizeof(*desc), sizeof(*desc),
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

desc = dd->dd_idmac_vaddr;

printf("%d (0x%lx)\n", i, dd->dd_paddr);
if (mode & GTIDMAC_CCLR_DESCMODE_16M) {
snprintb(buf, sizeof(buf),
"\177\020b\037Own\0b\036BCLeft\0",
desc->bc.mode16m.bcnt);
printf(" Byte Count : %s\n", buf);
printf(" ByteCount : 0x%06x\n",
desc->bc.mode16m.bcnt &
GTIDMAC_CIDMABCR_BYTECNT_MASK);
} else {
printf(" Byte Count : 0x%04x\n",
desc->bc.mode64k.bcnt);
printf(" Remind Byte Count : 0x%04x\n",
desc->bc.mode64k.rbc);
}
printf(" Source Address : 0x%08x\n", desc->srcaddr);
printf(" Destination Address : 0x%08x\n", desc->dstaddr);
printf(" Next Descriptor Pointer : 0x%08x\n", desc->nextdp);

if (desc->nextdp == (uint32_t)NULL)
break;

if (!post)
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
dd->dd_index * sizeof(*desc), sizeof(*desc),
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

i++;
dd = SLIST_NEXT(dd, dd_next);
}
if (!post)
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap,
dd->dd_index * sizeof(*desc), sizeof(*desc),
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}

static void
gtidmac_dump_xorereg(struct gtidmac_softc *sc, int chan)
{
uint32_t val, opmode;
char buf[64];

printf("XORE Registers\n");

val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXCR(sc, chan));
snprintb(buf, sizeof(buf),
"\177\020"
"b\017RegAccProtect\0b\016DesSwp\0b\015DwrReqSwp\0b\014DrdResSwp\0",
val);
printf(" Configuration : %s\n", buf);
opmode = val & MVXORE_XEXCR_OM_MASK;
printf(" OperationMode : %s operation\n",
opmode == MVXORE_XEXCR_OM_XOR ? "XOR calculate" :
opmode == MVXORE_XEXCR_OM_CRC32 ? "CRC-32 calculate" :
opmode == MVXORE_XEXCR_OM_DMA ? "DMA" :
opmode == MVXORE_XEXCR_OM_ECC ? "ECC cleanup" :
opmode == MVXORE_XEXCR_OM_MEMINIT ? "Memory Initialization" :
"unknown");
printf(" SrcBurstLimit : %s Bytes\n",
(val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_128B ? "128" :
(val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_64B ? "64" :
(val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_32B ? "32" :
"unknown");
printf(" DstBurstLimit : %s Bytes\n",
(val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_128B ? "128" :
(val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_64B ? "64" :
(val & MVXORE_XEXCR_SBL_MASK) == MVXORE_XEXCR_SBL_32B ? "32" :
"unknown");
val =
bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXACTR(sc, chan));
printf(" Activation : 0x%08x\n", val);
val &= MVXORE_XEXACTR_XESTATUS_MASK;
printf(" XEstatus : %s\n",
val == MVXORE_XEXACTR_XESTATUS_NA ? "Channel not active" :
val == MVXORE_XEXACTR_XESTATUS_ACT ? "Channel active" :
val == MVXORE_XEXACTR_XESTATUS_P ? "Channel paused" : "???");

if (opmode == MVXORE_XEXCR_OM_XOR ||
opmode == MVXORE_XEXCR_OM_CRC32 ||
opmode == MVXORE_XEXCR_OM_DMA) {
printf(" NextDescPtr : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEXNDPR(sc, chan)));
printf(" CurrentDescPtr : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEXCDPR(sc, chan)));
}
printf(" ByteCnt : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh, MVXORE_XEXBCR(sc, chan)));

if (opmode == MVXORE_XEXCR_OM_ECC ||
opmode == MVXORE_XEXCR_OM_MEMINIT) {
printf(" DstPtr : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEXDPR(sc, chan)));
printf(" BlockSize : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEXBSR(sc, chan)));

if (opmode == MVXORE_XEXCR_OM_ECC) {
val = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XETMCR);
if (val & MVXORE_XETMCR_TIMEREN) {
val >>= MVXORE_XETMCR_SECTIONSIZECTRL_SHIFT;
val &= MVXORE_XETMCR_SECTIONSIZECTRL_MASK;
printf(" SectionSizeCtrl : 0x%08x\n", 2 ^ val);
printf(" TimerInitVal : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XETMIVR));
printf(" TimerCrntVal : 0x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XETMCVR));
}
} else /* MVXORE_XEXCR_OM_MEMINIT */
printf(" InitVal : 0x%08x%08x\n",
bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEIVRH),
bus_space_read_4(sc->sc_iot, sc->sc_ioh,
MVXORE_XEIVRL));
}
}

static void
gtidmac_dump_xoredesc(struct gtidmac_softc *sc, struct gtidmac_dma_desc *dd,
uint32_t mode, int post)
{
struct mvxore_desc *desc;
int i, j;
char buf[256];

printf("XORE Descriptor\n");

mode &= MVXORE_XEXCR_OM_MASK;

i = 0;
while (1 /*CONSTCOND*/) {
if (post)
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
dd->dd_index * sizeof(*desc), sizeof(*desc),
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);

desc = dd->dd_xore_vaddr;

printf("%d (0x%lx)\n", i, dd->dd_paddr);

snprintb(buf, sizeof(buf), "\177\020b\037Own\0b\036Success\0",
desc->stat);
printf(" Status : %s\n", buf);
if (desc->cmd & MVXORE_DESC_CMD_CRCLAST && post)
printf(" CRC-32 Result : 0x%08x\n",
desc->result);
snprintb(buf, sizeof(buf),
"\177\020b\037EODIntEn\0b\036CRCLast\0"
"b\007Src7Cmd\0b\006Src6Cmd\0b\005Src5Cmd\0b\004Src4Cmd\0"
"b\003Src3Cmd\0b\002Src2Cmd\0b\001Src1Cmd\0b\000Src0Cmd\0",
desc->cmd);
printf(" Command : %s\n", buf);
printf(" Next Descriptor Address : 0x%08x\n", desc->nextda);
printf(" Byte Count : 0x%06x\n", desc->bcnt);
printf(" Destination Address : 0x%08x\n", desc->dstaddr);
if (mode == MVXORE_XEXCR_OM_XOR) {
for (j = 0; j < MVXORE_NSRC; j++)
if (desc->cmd & MVXORE_DESC_CMD_SRCCMD(j))
printf(" Source Address#%d :"
" 0x%08x\n", j, desc->srcaddr[j]);
} else
printf(" Source Address : 0x%08x\n",
desc->srcaddr[0]);

if (desc->nextda == (uint32_t)NULL)
break;

if (!post)
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
dd->dd_index * sizeof(*desc), sizeof(*desc),
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);

i++;
dd = SLIST_NEXT(dd, dd_next);
}
if (!post)
bus_dmamap_sync(sc->sc_dmat, sc->sc_dmap_xore,
dd->dd_index * sizeof(*desc), sizeof(*desc),
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}
#endif