* Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.

/* $NetBSD: pq3etsec.c,v 1.61 2025/01/07 17:39:45 andvar Exp $ */
/*-
* Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
* Agency and which was developed by Matt Thomas of 3am Software Foundry.
*
* This material is based upon work supported by the Defense Advanced Research
* Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
* Contract No. N66001-09-C-2073.
* Approved for Public Release, Distribution Unlimited
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: pq3etsec.c,v 1.61 2025/01/07 17:39:45 andvar Exp $");

#ifdef _KERNEL_OPT
#include "opt_inet.h"
#include "opt_mpc85xx.h"
#include "opt_multiprocessor.h"
#include "opt_net_mpsafe.h"
#endif

#include <sys/param.h>
#include <sys/cpu.h>
#include <sys/device.h>
#include <sys/mbuf.h>
#include <sys/ioctl.h>
#include <sys/intr.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/proc.h>
#include <sys/atomic.h>
#include <sys/callout.h>
#include <sys/sysctl.h>

#include <sys/rndsource.h>

#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_ether.h>
#include <net/if_media.h>
#include <net/bpf.h>

#include <dev/mii/miivar.h>

#ifdef INET
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/in_offload.h>
#endif /* INET */
#ifdef INET6
#include <netinet6/in6.h>
#include <netinet/ip6.h>
#endif
#include <netinet6/in6_offload.h>

#include <powerpc/spr.h>
#include <powerpc/booke/spr.h>
#include <powerpc/booke/cpuvar.h>
#include <powerpc/booke/e500var.h>
#include <powerpc/booke/e500reg.h>
#include <powerpc/booke/etsecreg.h>

#define M_HASFCB M_LINK2 /* tx packet has FCB prepended */

#define ETSEC_MAXTXMBUFS 30
#define ETSEC_NTXSEGS 30
#define ETSEC_MAXRXMBUFS 511
#define ETSEC_MINRXMBUFS 32
#define ETSEC_NRXSEGS 1

#define IFCAP_RCTRL_IPCSEN IFCAP_CSUM_IPv4_Rx
#define IFCAP_RCTRL_TUCSEN (IFCAP_CSUM_TCPv4_Rx \
| IFCAP_CSUM_UDPv4_Rx \
| IFCAP_CSUM_TCPv6_Rx \
| IFCAP_CSUM_UDPv6_Rx)

#define IFCAP_TCTRL_IPCSEN IFCAP_CSUM_IPv4_Tx
#define IFCAP_TCTRL_TUCSEN (IFCAP_CSUM_TCPv4_Tx \
| IFCAP_CSUM_UDPv4_Tx \
| IFCAP_CSUM_TCPv6_Tx \
| IFCAP_CSUM_UDPv6_Tx)

#define IFCAP_ETSEC (IFCAP_RCTRL_IPCSEN | IFCAP_RCTRL_TUCSEN \
| IFCAP_TCTRL_IPCSEN | IFCAP_TCTRL_TUCSEN)

#define M_CSUM_IP (M_CSUM_CIP | M_CSUM_CTU)
#define M_CSUM_IP6 (M_CSUM_TCPv6 | M_CSUM_UDPv6)
#define M_CSUM_TUP (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6)
#define M_CSUM_UDP (M_CSUM_UDPv4 | M_CSUM_UDPv6)
#define M_CSUM_IP4 (M_CSUM_IPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv4)
#define M_CSUM_CIP (M_CSUM_IPv4)
#define M_CSUM_CTU (M_CSUM_TCPv4 | M_CSUM_UDPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv6)

struct pq3etsec_txqueue {
bus_dmamap_t txq_descmap;
volatile struct txbd *txq_consumer;
volatile struct txbd *txq_producer;
volatile struct txbd *txq_first;
volatile struct txbd *txq_last;
struct ifqueue txq_mbufs;
struct mbuf *txq_next;
#ifdef ETSEC_DEBUG
struct mbuf *txq_lmbufs[512];
#endif
uint32_t txq_qmask;
uint32_t txq_free;
uint32_t txq_threshold;
uint32_t txq_lastintr;
bus_size_t txq_reg_tbase;
bus_dma_segment_t txq_descmap_seg;
};

struct pq3etsec_rxqueue {
bus_dmamap_t rxq_descmap;
volatile struct rxbd *rxq_consumer;
volatile struct rxbd *rxq_producer;
volatile struct rxbd *rxq_first;
volatile struct rxbd *rxq_last;
struct mbuf *rxq_mhead;
struct mbuf **rxq_mtail;
struct mbuf *rxq_mconsumer;
#ifdef ETSEC_DEBUG
struct mbuf *rxq_mbufs[512];
#endif
uint32_t rxq_qmask;
uint32_t rxq_inuse;
uint32_t rxq_threshold;
bus_size_t rxq_reg_rbase;
bus_size_t rxq_reg_rbptr;
bus_dma_segment_t rxq_descmap_seg;
};

struct pq3etsec_mapcache {
u_int dmc_nmaps;
u_int dmc_maxseg;
u_int dmc_maxmaps;
u_int dmc_maxmapsize;
bus_dmamap_t dmc_maps[0];
};

struct pq3etsec_softc {
device_t sc_dev;
device_t sc_mdio_dev;
struct ethercom sc_ec;
#define sc_if sc_ec.ec_if
struct mii_data sc_mii;
bus_space_tag_t sc_bst;
bus_space_handle_t sc_bsh;
bus_space_handle_t sc_mdio_bsh;
bus_dma_tag_t sc_dmat;
int sc_phy_addr;
prop_dictionary_t sc_intrmap;
uint32_t sc_intrmask;

uint32_t sc_soft_flags;
#define SOFT_RESET 0x0001
#define SOFT_RXINTR 0x0010
#define SOFT_RXBSY 0x0020
#define SOFT_TXINTR 0x0100
#define SOFT_TXERROR 0x0200

struct pq3etsec_txqueue sc_txq;
struct pq3etsec_rxqueue sc_rxq;
uint32_t sc_txerrors;
uint32_t sc_rxerrors;

size_t sc_rx_adjlen;

/*
* Copies of various ETSEC registers.
*/
uint32_t sc_imask;
uint32_t sc_maccfg1;
uint32_t sc_maccfg2;
uint32_t sc_maxfrm;
uint32_t sc_ecntrl;
uint32_t sc_dmactrl;
uint32_t sc_macstnaddr1;
uint32_t sc_macstnaddr2;
uint32_t sc_tctrl;
uint32_t sc_rctrl;
uint32_t sc_gaddr[16];
uint64_t sc_macaddrs[15];

void *sc_tx_ih;
void *sc_rx_ih;
void *sc_error_ih;
void *sc_soft_ih;

kmutex_t *sc_lock;
kmutex_t *sc_hwlock;

struct evcnt sc_ev_tx_stall;
struct evcnt sc_ev_tx_intr;
struct evcnt sc_ev_rx_stall;
struct evcnt sc_ev_rx_intr;
struct evcnt sc_ev_error_intr;
struct evcnt sc_ev_soft_intr;
struct evcnt sc_ev_tx_pause;
struct evcnt sc_ev_rx_pause;
struct evcnt sc_ev_mii_ticks;

struct callout sc_mii_callout;
uint64_t sc_mii_last_tick;

struct ifqueue sc_rx_bufcache;
struct pq3etsec_mapcache *sc_rx_mapcache;
struct pq3etsec_mapcache *sc_tx_mapcache;

/* Interrupt Coalescing parameters */
int sc_ic_rx_time;
int sc_ic_rx_count;
int sc_ic_tx_time;
int sc_ic_tx_count;

krndsource_t rnd_source;
};

#define ETSEC_IC_RX_ENABLED(sc) \
((sc)->sc_ic_rx_time != 0 && (sc)->sc_ic_rx_count != 0)
#define ETSEC_IC_TX_ENABLED(sc) \
((sc)->sc_ic_tx_time != 0 && (sc)->sc_ic_tx_count != 0)

struct pq3mdio_softc {
device_t mdio_dev;

bus_space_tag_t mdio_bst;
bus_space_handle_t mdio_bsh;
};

static int pq3etsec_match(device_t, cfdata_t, void *);
static void pq3etsec_attach(device_t, device_t, void *);

static int pq3mdio_match(device_t, cfdata_t, void *);
static void pq3mdio_attach(device_t, device_t, void *);

static void pq3etsec_ifstart(struct ifnet *);
static void pq3etsec_ifwatchdog(struct ifnet *);
static int pq3etsec_ifinit(struct ifnet *);
static void pq3etsec_ifstop(struct ifnet *, int);
static int pq3etsec_ifioctl(struct ifnet *, u_long, void *);

static int pq3etsec_mapcache_create(struct pq3etsec_softc *,
struct pq3etsec_mapcache **, size_t, size_t, size_t);
static void pq3etsec_mapcache_destroy(struct pq3etsec_softc *,
struct pq3etsec_mapcache *);
static bus_dmamap_t pq3etsec_mapcache_get(struct pq3etsec_softc *,
struct pq3etsec_mapcache *);
static void pq3etsec_mapcache_put(struct pq3etsec_softc *,
struct pq3etsec_mapcache *, bus_dmamap_t);

static int pq3etsec_txq_attach(struct pq3etsec_softc *,
struct pq3etsec_txqueue *, u_int);
static void pq3etsec_txq_purge(struct pq3etsec_softc *,
struct pq3etsec_txqueue *);
static void pq3etsec_txq_reset(struct pq3etsec_softc *,
struct pq3etsec_txqueue *);
static bool pq3etsec_txq_consume(struct pq3etsec_softc *,
struct pq3etsec_txqueue *);
static bool pq3etsec_txq_produce(struct pq3etsec_softc *,
struct pq3etsec_txqueue *, struct mbuf *m);
static bool pq3etsec_txq_active_p(struct pq3etsec_softc *,
struct pq3etsec_txqueue *);

static int pq3etsec_rxq_attach(struct pq3etsec_softc *,
struct pq3etsec_rxqueue *, u_int);
static bool pq3etsec_rxq_produce(struct pq3etsec_softc *,
struct pq3etsec_rxqueue *);
static void pq3etsec_rxq_purge(struct pq3etsec_softc *,
struct pq3etsec_rxqueue *, bool);
static void pq3etsec_rxq_reset(struct pq3etsec_softc *,
struct pq3etsec_rxqueue *);

static void pq3etsec_mc_setup(struct pq3etsec_softc *);

static void pq3etsec_mii_tick(void *);
static int pq3etsec_rx_intr(void *);
static int pq3etsec_tx_intr(void *);
static int pq3etsec_error_intr(void *);
static void pq3etsec_soft_intr(void *);

static void pq3etsec_set_ic_rx(struct pq3etsec_softc *);
static void pq3etsec_set_ic_tx(struct pq3etsec_softc *);

static void pq3etsec_sysctl_setup(struct sysctllog **, struct pq3etsec_softc *);

CFATTACH_DECL_NEW(pq3etsec, sizeof(struct pq3etsec_softc),
pq3etsec_match, pq3etsec_attach, NULL, NULL);

CFATTACH_DECL_NEW(pq3mdio_tsec, sizeof(struct pq3mdio_softc),
pq3mdio_match, pq3mdio_attach, NULL, NULL);

CFATTACH_DECL_NEW(pq3mdio_cpunode, sizeof(struct pq3mdio_softc),
pq3mdio_match, pq3mdio_attach, NULL, NULL);

static inline uint32_t
etsec_mdio_read(struct pq3mdio_softc *mdio, bus_size_t off)
{
return bus_space_read_4(mdio->mdio_bst, mdio->mdio_bsh, off);
}

static inline void
etsec_mdio_write(struct pq3mdio_softc *mdio, bus_size_t off, uint32_t data)
{
bus_space_write_4(mdio->mdio_bst, mdio->mdio_bsh, off, data);
}

static inline uint32_t
etsec_read(struct pq3etsec_softc *sc, bus_size_t off)
{
return bus_space_read_4(sc->sc_bst, sc->sc_bsh, off);
}

static int
pq3mdio_find(device_t parent, cfdata_t cf, const int *ldesc, void *aux)
{
return strcmp(cf->cf_name, "mdio") == 0;
}

static int
pq3mdio_match(device_t parent, cfdata_t cf, void *aux)
{
const uint16_t svr = (mfspr(SPR_SVR) & ~0x80000) >> 16;
const bool p1025_p = (svr == (SVR_P1025v1 >> 16)
|| svr == (SVR_P1016v1 >> 16));

if (device_is_a(parent, "cpunode")) {
if (!p1025_p
|| !e500_cpunode_submatch(parent, cf, cf->cf_name, aux))
return 0;

return 1;
}

if (device_is_a(parent, "tsec")) {
if (p1025_p
|| !e500_cpunode_submatch(parent, cf, cf->cf_name, aux))
return 0;

return 1;
}

return 0;
}

static void
pq3mdio_attach(device_t parent, device_t self, void *aux)
{
struct pq3mdio_softc * const mdio = device_private(self);
struct cpunode_attach_args * const cna = aux;
struct cpunode_locators * const cnl = &cna->cna_locs;

mdio->mdio_dev = self;

if (device_is_a(parent, "cpunode")) {
struct cpunode_softc * const psc = device_private(parent);
psc->sc_children |= cna->cna_childmask;

mdio->mdio_bst = cna->cna_memt;
if (bus_space_map(mdio->mdio_bst, cnl->cnl_addr,
cnl->cnl_size, 0, &mdio->mdio_bsh) != 0) {
aprint_error(": error mapping registers @ %#x\n",
cnl->cnl_addr);
return;
}
} else {
struct pq3etsec_softc * const sc = device_private(parent);

KASSERT(device_is_a(parent, "tsec"));
KASSERTMSG(cnl->cnl_addr == ETSEC1_BASE
|| cnl->cnl_addr == ETSEC2_BASE
|| cnl->cnl_addr == ETSEC3_BASE
|| cnl->cnl_addr == ETSEC4_BASE,
"unknown tsec addr %x", cnl->cnl_addr);

mdio->mdio_bst = sc->sc_bst;
mdio->mdio_bsh = sc->sc_bsh;
}

aprint_normal("\n");
}

static int
pq3mdio_mii_readreg(device_t self, int phy, int reg, uint16_t *val)
{
struct pq3mdio_softc * const mdio = device_private(self);
uint32_t miimcom = etsec_mdio_read(mdio, MIIMCOM);

etsec_mdio_write(mdio, MIIMADD,
__SHIFTIN(phy, MIIMADD_PHY) | __SHIFTIN(reg, MIIMADD_REG));

etsec_mdio_write(mdio, MIIMCOM, 0); /* clear any past bits */
etsec_mdio_write(mdio, MIIMCOM, MIIMCOM_READ);

while (etsec_mdio_read(mdio, MIIMIND) != 0) {
delay(1);
}
*val = etsec_mdio_read(mdio, MIIMSTAT) &0xffff;

if (miimcom == MIIMCOM_SCAN)
etsec_mdio_write(mdio, MIIMCOM, miimcom);

#if 0
aprint_normal_dev(mdio->mdio_dev, "%s: phy %d reg %d: %#x\n",
__func__, phy, reg, data);
#endif
return 0;
}

static int
pq3mdio_mii_writereg(device_t self, int phy, int reg, uint16_t data)
{
struct pq3mdio_softc * const mdio = device_private(self);
uint32_t miimcom = etsec_mdio_read(mdio, MIIMCOM);

#if 0
aprint_normal_dev(mdio->mdio_dev, "%s: phy %d reg %d: %#x\n",
__func__, phy, reg, data);
#endif

etsec_mdio_write(mdio, MIIMADD,
__SHIFTIN(phy, MIIMADD_PHY) | __SHIFTIN(reg, MIIMADD_REG));
etsec_mdio_write(mdio, MIIMCOM, 0); /* clear any past bits */
etsec_mdio_write(mdio, MIIMCON, data);

int timo = 1000; /* 1ms */
while ((etsec_mdio_read(mdio, MIIMIND) & MIIMIND_BUSY) && --timo > 0) {
delay(1);
}

if (miimcom == MIIMCOM_SCAN)
etsec_mdio_write(mdio, MIIMCOM, miimcom);

return 0;
}

static inline void
etsec_write(struct pq3etsec_softc *sc, bus_size_t off, uint32_t data)
{
bus_space_write_4(sc->sc_bst, sc->sc_bsh, off, data);
}

static void
pq3etsec_mii_statchg(struct ifnet *ifp)
{
struct pq3etsec_softc * const sc = ifp->if_softc;
struct mii_data * const mii = &sc->sc_mii;

uint32_t maccfg1 = sc->sc_maccfg1;
uint32_t maccfg2 = sc->sc_maccfg2;
uint32_t ecntrl = sc->sc_ecntrl;

maccfg1 &= ~(MACCFG1_TX_FLOW | MACCFG1_RX_FLOW);
maccfg2 &= ~(MACCFG2_IFMODE | MACCFG2_FD);

if (sc->sc_mii.mii_media_active & IFM_FDX) {
maccfg2 |= MACCFG2_FD;
}

/*
* Now deal with the flow control bits.
*/
if (IFM_SUBTYPE(mii->mii_media.ifm_cur->ifm_media) == IFM_AUTO
&& (mii->mii_media_active & IFM_ETH_FMASK)) {
if (mii->mii_media_active & IFM_ETH_RXPAUSE)
maccfg1 |= MACCFG1_RX_FLOW;
if (mii->mii_media_active & IFM_ETH_TXPAUSE)
maccfg1 |= MACCFG1_TX_FLOW;
}

/*
* Now deal with the speed.
*/
if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T) {
maccfg2 |= MACCFG2_IFMODE_GMII;
} else {
maccfg2 |= MACCFG2_IFMODE_MII;
ecntrl &= ~ECNTRL_R100M;
if (IFM_SUBTYPE(mii->mii_media_active) != IFM_10_T) {
ecntrl |= ECNTRL_R100M;
}
}

/*
* If things are different, re-init things.
*/
if (maccfg1 != sc->sc_maccfg1
|| maccfg2 != sc->sc_maccfg2
|| ecntrl != sc->sc_ecntrl) {
if (sc->sc_if.if_flags & IFF_RUNNING)
atomic_or_uint(&sc->sc_soft_flags, SOFT_RESET);
sc->sc_maccfg1 = maccfg1;
sc->sc_maccfg2 = maccfg2;
sc->sc_ecntrl = ecntrl;
}
}

#if 0
static void
pq3etsec_mediastatus(struct ifnet *ifp, struct ifmediareq *ifmr)
{
struct pq3etsec_softc * const sc = ifp->if_softc;

mii_pollstat(&sc->sc_mii);
ether_mediastatus(ifp, ifmr);
ifmr->ifm_status = sc->sc_mii.mii_media_status;
ifmr->ifm_active = sc->sc_mii.mii_media_active;
}

static int
pq3etsec_mediachange(struct ifnet *ifp)
{
struct pq3etsec_softc * const sc = ifp->if_softc;

if ((ifp->if_flags & IFF_UP) == 0)
return 0;

int rv = mii_mediachg(&sc->sc_mii);
return (rv == ENXIO) ? 0 : rv;
}
#endif

static int
pq3etsec_match(device_t parent, cfdata_t cf, void *aux)
{

if (!e500_cpunode_submatch(parent, cf, cf->cf_name, aux))
return 0;

return 1;
}

static void
pq3etsec_attach(device_t parent, device_t self, void *aux)
{
struct cpunode_softc * const psc = device_private(parent);
struct pq3etsec_softc * const sc = device_private(self);
struct mii_data * const mii = &sc->sc_mii;
struct cpunode_attach_args * const cna = aux;
struct cpunode_locators * const cnl = &cna->cna_locs;
cfdata_t cf = device_cfdata(self);
int error;

psc->sc_children |= cna->cna_childmask;
sc->sc_dev = self;
sc->sc_bst = cna->cna_memt;
sc->sc_dmat = &booke_bus_dma_tag;

/*
* Pull out the mdio bus and phy we are supposed to use.
*/
const int mdio = cf->cf_loc[CPUNODECF_MDIO];
const int phy = cf->cf_loc[CPUNODECF_PHY];
if (mdio != CPUNODECF_MDIO_DEFAULT)
aprint_normal(" mdio %d", mdio);

/*
* See if the phy is in the config file...
*/
if (phy != CPUNODECF_PHY_DEFAULT) {
sc->sc_phy_addr = phy;
} else {
unsigned char prop_name[20];
snprintf(prop_name, sizeof(prop_name), "tsec%u-phy-addr",
cnl->cnl_instance);
sc->sc_phy_addr = board_info_get_number(prop_name);
}
if (sc->sc_phy_addr != MII_PHY_ANY)
aprint_normal(" phy %d", sc->sc_phy_addr);

error = bus_space_map(sc->sc_bst, cnl->cnl_addr, cnl->cnl_size, 0,
&sc->sc_bsh);
if (error) {
aprint_error(": error mapping registers: %d\n", error);
return;
}

/*
* Assume firmware has aready set the mac address and fetch it
* before we reinit it.
*/
sc->sc_macstnaddr2 = etsec_read(sc, MACSTNADDR2);
sc->sc_macstnaddr1 = etsec_read(sc, MACSTNADDR1);
sc->sc_rctrl = RCTRL_DEFAULT;
sc->sc_ecntrl = etsec_read(sc, ECNTRL);
sc->sc_maccfg1 = etsec_read(sc, MACCFG1);
sc->sc_maccfg2 = etsec_read(sc, MACCFG2) | MACCFG2_DEFAULT;

if (sc->sc_macstnaddr1 == 0 && sc->sc_macstnaddr2 == 0) {
size_t len;
const uint8_t *mac_addr =
board_info_get_data("tsec-mac-addr-base", &len);
KASSERT(len == ETHER_ADDR_LEN);
sc->sc_macstnaddr2 =
(mac_addr[1] << 24)
| (mac_addr[0] << 16);
sc->sc_macstnaddr1 =
((mac_addr[5] + cnl->cnl_instance - 1) << 24)
| (mac_addr[4] << 16)
| (mac_addr[3] << 8)
| (mac_addr[2] << 0);
#if 0
aprint_error(": mac-address unknown\n");
return;
#endif
}

sc->sc_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_SOFTNET);
sc->sc_hwlock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_VM);

callout_init(&sc->sc_mii_callout, CALLOUT_MPSAFE);
callout_setfunc(&sc->sc_mii_callout, pq3etsec_mii_tick, sc);

/* Disable interrupts */
etsec_write(sc, IMASK, 0);

error = pq3etsec_rxq_attach(sc, &sc->sc_rxq, 0);
if (error) {
aprint_error(": failed to init rxq: %d\n", error);
goto fail_1;
}

error = pq3etsec_txq_attach(sc, &sc->sc_txq, 0);
if (error) {
aprint_error(": failed to init txq: %d\n", error);
goto fail_2;
}

error = pq3etsec_mapcache_create(sc, &sc->sc_rx_mapcache,
ETSEC_MAXRXMBUFS, MCLBYTES, ETSEC_NRXSEGS);
if (error) {
aprint_error(": failed to allocate rx dmamaps: %d\n", error);
goto fail_3;
}

error = pq3etsec_mapcache_create(sc, &sc->sc_tx_mapcache,
ETSEC_MAXTXMBUFS, MCLBYTES, ETSEC_NTXSEGS);
if (error) {
aprint_error(": failed to allocate tx dmamaps: %d\n", error);
goto fail_4;
}

sc->sc_tx_ih = intr_establish(cnl->cnl_intrs[0], IPL_VM, IST_ONCHIP,
pq3etsec_tx_intr, sc);
if (sc->sc_tx_ih == NULL) {
aprint_error(": failed to establish tx interrupt: %d\n",
cnl->cnl_intrs[0]);
goto fail_5;
}

sc->sc_rx_ih = intr_establish(cnl->cnl_intrs[1], IPL_VM, IST_ONCHIP,
pq3etsec_rx_intr, sc);
if (sc->sc_rx_ih == NULL) {
aprint_error(": failed to establish rx interrupt: %d\n",
cnl->cnl_intrs[1]);
goto fail_6;
}

sc->sc_error_ih = intr_establish(cnl->cnl_intrs[2], IPL_VM, IST_ONCHIP,
pq3etsec_error_intr, sc);
if (sc->sc_error_ih == NULL) {
aprint_error(": failed to establish error interrupt: %d\n",
cnl->cnl_intrs[2]);
goto fail_7;
}

int softint_flags = SOFTINT_NET;
#if !defined(MULTIPROCESSOR) || defined(NET_MPSAFE)
softint_flags |= SOFTINT_MPSAFE;
#endif /* !MULTIPROCESSOR || NET_MPSAFE */
sc->sc_soft_ih = softint_establish(softint_flags,
pq3etsec_soft_intr, sc);
if (sc->sc_soft_ih == NULL) {
aprint_error(": failed to establish soft interrupt\n");
goto fail_8;
}

/*
* If there was no MDIO
*/
if (mdio == CPUNODECF_MDIO_DEFAULT) {
aprint_normal("\n");
cfdata_t mdio_cf = config_search(self, cna,
CFARGS(.submatch = pq3mdio_find));
if (mdio_cf != NULL) {
sc->sc_mdio_dev =
config_attach(self, mdio_cf, cna, NULL, CFARGS_NONE);
}
} else {
sc->sc_mdio_dev = device_find_by_driver_unit("mdio", mdio);
if (sc->sc_mdio_dev == NULL) {
aprint_error(": failed to locate mdio device\n");
goto fail_9;
}
aprint_normal("\n");
}

etsec_write(sc, ATTR, ATTR_DEFAULT);
etsec_write(sc, ATTRELI, ATTRELI_DEFAULT);

/* Enable interrupt coalescing */
sc->sc_ic_rx_time = 768;
sc->sc_ic_rx_count = 16;
sc->sc_ic_tx_time = 768;
sc->sc_ic_tx_count = 16;
pq3etsec_set_ic_rx(sc);
pq3etsec_set_ic_tx(sc);

char enaddr[ETHER_ADDR_LEN] = {
[0] = sc->sc_macstnaddr2 >> 16,
[1] = sc->sc_macstnaddr2 >> 24,
[2] = sc->sc_macstnaddr1 >> 0,
[3] = sc->sc_macstnaddr1 >> 8,
[4] = sc->sc_macstnaddr1 >> 16,
[5] = sc->sc_macstnaddr1 >> 24,
};
aprint_normal_dev(sc->sc_dev, "Ethernet address %s\n",
ether_sprintf(enaddr));

const char * const xname = device_xname(sc->sc_dev);
struct ethercom * const ec = &sc->sc_ec;
struct ifnet * const ifp = &ec->ec_if;

ec->ec_mii = mii;

mii->mii_ifp = ifp;
mii->mii_readreg = pq3mdio_mii_readreg;
mii->mii_writereg = pq3mdio_mii_writereg;
mii->mii_statchg = pq3etsec_mii_statchg;

ifmedia_init(&mii->mii_media, 0, ether_mediachange, ether_mediastatus);

if (sc->sc_mdio_dev != NULL && sc->sc_phy_addr < 32) {
mii_attach(sc->sc_mdio_dev, mii, 0xffffffff,
sc->sc_phy_addr, MII_OFFSET_ANY, MIIF_DOPAUSE);

if (LIST_FIRST(&mii->mii_phys) == NULL) {
ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_NONE,
0, NULL);
ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_NONE);
} else {
callout_schedule(&sc->sc_mii_callout, hz);
ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_AUTO);
}
} else {
ifmedia_add(&mii->mii_media, IFM_ETHER | IFM_1000_T | IFM_FDX,
0, NULL);
ifmedia_set(&mii->mii_media, IFM_ETHER | IFM_1000_T | IFM_FDX);
}

ec->ec_capabilities = ETHERCAP_VLAN_MTU | ETHERCAP_VLAN_HWTAGGING
| ETHERCAP_JUMBO_MTU;
ec->ec_capenable = ETHERCAP_VLAN_HWTAGGING;

strlcpy(ifp->if_xname, xname, IFNAMSIZ);
ifp->if_softc = sc;
ifp->if_capabilities = IFCAP_ETSEC;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_ioctl = pq3etsec_ifioctl;
ifp->if_start = pq3etsec_ifstart;
ifp->if_watchdog = pq3etsec_ifwatchdog;
ifp->if_init = pq3etsec_ifinit;
ifp->if_stop = pq3etsec_ifstop;
IFQ_SET_READY(&ifp->if_snd);

/*
* Attach the interface.
*/
if_initialize(ifp);
pq3etsec_sysctl_setup(NULL, sc);
if_attach(ifp);
if_deferred_start_init(ifp, NULL);
ether_ifattach(ifp, enaddr);

rnd_attach_source(&sc->rnd_source, xname, RND_TYPE_NET,
RND_FLAG_DEFAULT);

pq3etsec_ifstop(ifp, true);

evcnt_attach_dynamic(&sc->sc_ev_rx_stall, EVCNT_TYPE_MISC,
NULL, xname, "rx stall");
evcnt_attach_dynamic(&sc->sc_ev_tx_stall, EVCNT_TYPE_MISC,
NULL, xname, "tx stall");
evcnt_attach_dynamic(&sc->sc_ev_tx_intr, EVCNT_TYPE_INTR,
NULL, xname, "tx intr");
evcnt_attach_dynamic(&sc->sc_ev_rx_intr, EVCNT_TYPE_INTR,
NULL, xname, "rx intr");
evcnt_attach_dynamic(&sc->sc_ev_error_intr, EVCNT_TYPE_INTR,
NULL, xname, "error intr");
evcnt_attach_dynamic(&sc->sc_ev_soft_intr, EVCNT_TYPE_INTR,
NULL, xname, "soft intr");
evcnt_attach_dynamic(&sc->sc_ev_tx_pause, EVCNT_TYPE_MISC,
NULL, xname, "tx pause");
evcnt_attach_dynamic(&sc->sc_ev_rx_pause, EVCNT_TYPE_MISC,
NULL, xname, "rx pause");
evcnt_attach_dynamic(&sc->sc_ev_mii_ticks, EVCNT_TYPE_MISC,
NULL, xname, "mii ticks");
return;

fail_9:
softint_disestablish(sc->sc_soft_ih);
fail_8:
intr_disestablish(sc->sc_error_ih);
fail_7:
intr_disestablish(sc->sc_rx_ih);
fail_6:
intr_disestablish(sc->sc_tx_ih);
fail_5:
pq3etsec_mapcache_destroy(sc, sc->sc_tx_mapcache);
fail_4:
pq3etsec_mapcache_destroy(sc, sc->sc_rx_mapcache);
fail_3:
#if 0 /* notyet */
pq3etsec_txq_detach(sc);
#endif
fail_2:
#if 0 /* notyet */
pq3etsec_rxq_detach(sc);
#endif
fail_1:
callout_destroy(&sc->sc_mii_callout);
mutex_obj_free(sc->sc_lock);
mutex_obj_free(sc->sc_hwlock);
bus_space_unmap(sc->sc_bst, sc->sc_bsh, cnl->cnl_size);
}

static uint64_t
pq3etsec_macaddr_create(const uint8_t *lladdr)
{
uint64_t macaddr = 0;

lladdr += ETHER_ADDR_LEN;
for (u_int i = ETHER_ADDR_LEN; i-- > 0; ) {
macaddr = (macaddr << 8) | *--lladdr;
}
return macaddr << 16;
}

static int
pq3etsec_ifinit(struct ifnet *ifp)
{
struct pq3etsec_softc * const sc = ifp->if_softc;
int error = 0;

sc->sc_maxfrm = uimax(ifp->if_mtu + 32, MCLBYTES);
if (ifp->if_mtu > ETHERMTU_JUMBO)
return error;

KASSERT(ifp->if_flags & IFF_UP);

/*
* Stop the interface (steps 1 to 4 in the Soft Reset and
* Reconfigurating Procedure.
*/
pq3etsec_ifstop(ifp, 0);

/*
* If our frame size has changed (or it's our first time through)
* destroy the existing transmit mapcache.
*/
if (sc->sc_tx_mapcache != NULL
&& sc->sc_maxfrm != sc->sc_tx_mapcache->dmc_maxmapsize) {
pq3etsec_mapcache_destroy(sc, sc->sc_tx_mapcache);
sc->sc_tx_mapcache = NULL;
}

if (sc->sc_tx_mapcache == NULL) {
error = pq3etsec_mapcache_create(sc, &sc->sc_tx_mapcache,
ETSEC_MAXTXMBUFS, sc->sc_maxfrm, ETSEC_NTXSEGS);
if (error)
return error;
}

sc->sc_ev_mii_ticks.ev_count++;
mii_tick(&sc->sc_mii);

if (ifp->if_flags & IFF_PROMISC) {
sc->sc_rctrl |= RCTRL_PROM;
} else {
sc->sc_rctrl &= ~RCTRL_PROM;
}

uint32_t rctrl_prsdep = 0;
sc->sc_rctrl &=
~(RCTRL_IPCSEN | RCTRL_TUCSEN | RCTRL_VLEX | RCTRL_PRSDEP);
if (VLAN_ATTACHED(&sc->sc_ec)) {
sc->sc_rctrl |= RCTRL_VLEX;
rctrl_prsdep = RCTRL_PRSDEP_L2;
}
if (ifp->if_capenable & IFCAP_RCTRL_IPCSEN) {
sc->sc_rctrl |= RCTRL_IPCSEN;
rctrl_prsdep = RCTRL_PRSDEP_L3;
}
if (ifp->if_capenable & IFCAP_RCTRL_TUCSEN) {
sc->sc_rctrl |= RCTRL_TUCSEN;
rctrl_prsdep = RCTRL_PRSDEP_L4;
}
sc->sc_rctrl |= rctrl_prsdep;
#if 0
if (sc->sc_rctrl
& (RCTRL_IPCSEN | RCTRL_TUCSEN | RCTRL_VLEX | RCTRL_PRSDEP))
aprint_normal_dev(sc->sc_dev,
"rctrl=%#x ipcsen=%"PRIuMAX" tucsen=%"PRIuMAX" vlex=%"PRIuMAX" prsdep=%"PRIuMAX"\n",
sc->sc_rctrl,
__SHIFTOUT(sc->sc_rctrl, RCTRL_IPCSEN),
__SHIFTOUT(sc->sc_rctrl, RCTRL_TUCSEN),
__SHIFTOUT(sc->sc_rctrl, RCTRL_VLEX),
__SHIFTOUT(sc->sc_rctrl, RCTRL_PRSDEP));
#endif

sc->sc_tctrl &= ~(TCTRL_IPCSEN | TCTRL_TUCSEN | TCTRL_VLINS);
if (VLAN_ATTACHED(&sc->sc_ec)) /* is this really true */
sc->sc_tctrl |= TCTRL_VLINS;
if (ifp->if_capenable & IFCAP_TCTRL_IPCSEN)
sc->sc_tctrl |= TCTRL_IPCSEN;
if (ifp->if_capenable & IFCAP_TCTRL_TUCSEN)
sc->sc_tctrl |= TCTRL_TUCSEN;
#if 0
if (sc->sc_tctrl & (TCTRL_IPCSEN | TCTRL_TUCSEN | TCTRL_VLINS))
aprint_normal_dev(sc->sc_dev,
"tctrl=%#x ipcsen=%"PRIuMAX" tucsen=%"PRIuMAX" vlins=%"PRIuMAX"\n",
sc->sc_tctrl,
__SHIFTOUT(sc->sc_tctrl, TCTRL_IPCSEN),
__SHIFTOUT(sc->sc_tctrl, TCTRL_TUCSEN),
__SHIFTOUT(sc->sc_tctrl, TCTRL_VLINS));
#endif

sc->sc_maccfg1 &= ~(MACCFG1_TX_EN | MACCFG1_RX_EN);

const uint64_t macstnaddr =
pq3etsec_macaddr_create(CLLADDR(ifp->if_sadl));

sc->sc_imask = IEVENT_DPE;

/* 5. Load TDBPH, TBASEH, TBASE0-TBASE7 with new Tx BD pointers */
pq3etsec_rxq_reset(sc, &sc->sc_rxq);
pq3etsec_rxq_produce(sc, &sc->sc_rxq); /* fill with rx buffers */

/* 6. Load RDBPH, RBASEH, RBASE0-RBASE7 with new Rx BD pointers */
pq3etsec_txq_reset(sc, &sc->sc_txq);

/* 7. Setup other MAC registers (MACCFG2, MAXFRM, etc.) */
KASSERT(MACCFG2_PADCRC & sc->sc_maccfg2);
etsec_write(sc, MAXFRM, sc->sc_maxfrm);
etsec_write(sc, MACSTNADDR1, (uint32_t)(macstnaddr >> 32));
etsec_write(sc, MACSTNADDR2, (uint32_t)(macstnaddr >> 0));
etsec_write(sc, MACCFG1, sc->sc_maccfg1);
etsec_write(sc, MACCFG2, sc->sc_maccfg2);
etsec_write(sc, ECNTRL, sc->sc_ecntrl);

/* 8. Setup group address hash table (GADDR0-GADDR15) */
pq3etsec_mc_setup(sc);

/* 9. Setup receive frame filer table (via RQFAR, RQFCR, and RQFPR) */
etsec_write(sc, MRBLR, MCLBYTES);

/* 10. Setup WWR, WOP, TOD bits in DMACTRL register */
sc->sc_dmactrl |= DMACTRL_DEFAULT;
etsec_write(sc, DMACTRL, sc->sc_dmactrl);

/* 11. Enable transmit queues in TQUEUE, and ensure that the transmit scheduling mode is correctly set in TCTRL. */
etsec_write(sc, TQUEUE, TQUEUE_EN0);
sc->sc_imask |= IEVENT_TXF | IEVENT_TXE | IEVENT_TXC;

etsec_write(sc, TCTRL, sc->sc_tctrl); /* for TOE stuff */

/* 12. Enable receive queues in RQUEUE, */
etsec_write(sc, RQUEUE, RQUEUE_EN0 | RQUEUE_EX0);
sc->sc_imask |= IEVENT_RXF | IEVENT_BSY | IEVENT_RXC;

/* and optionally set TOE functionality in RCTRL. */
etsec_write(sc, RCTRL, sc->sc_rctrl);
sc->sc_rx_adjlen = __SHIFTOUT(sc->sc_rctrl, RCTRL_PAL);
if ((sc->sc_rctrl & RCTRL_PRSDEP) != RCTRL_PRSDEP_OFF)
sc->sc_rx_adjlen += sizeof(struct rxfcb);

/* 13. Clear THLT and TXF bits in TSTAT register by writing 1 to them */
etsec_write(sc, TSTAT, TSTAT_THLT | TSTAT_TXF);

/* 14. Clear QHLT and RXF bits in RSTAT register by writing 1 to them.*/
etsec_write(sc, RSTAT, RSTAT_QHLT | RSTAT_RXF);

/* 15. Clear GRS/GTS bits in DMACTRL (do not change other bits) */
sc->sc_dmactrl &= ~(DMACTRL_GRS | DMACTRL_GTS);
etsec_write(sc, DMACTRL, sc->sc_dmactrl);

/* 16. Enable Tx_EN/Rx_EN in MACCFG1 register */
etsec_write(sc, MACCFG1, sc->sc_maccfg1 | MACCFG1_TX_EN|MACCFG1_RX_EN);
etsec_write(sc, MACCFG1, sc->sc_maccfg1 | MACCFG1_TX_EN|MACCFG1_RX_EN);

sc->sc_soft_flags = 0;

etsec_write(sc, IMASK, sc->sc_imask);

ifp->if_flags |= IFF_RUNNING;

return error;
}

static void
pq3etsec_ifstop(struct ifnet *ifp, int disable)
{
struct pq3etsec_softc * const sc = ifp->if_softc;

KASSERT(!cpu_intr_p());
const uint32_t imask_gsc_mask = IEVENT_GTSC | IEVENT_GRSC;
/*
* Clear the GTSC and GRSC from the interrupt mask until
* we are ready for them. Then clear them from IEVENT,
* request the graceful shutdown, and then enable the
* GTSC and GRSC bits in the mask. This should cause the
* error interrupt to fire which will issue a wakeup to
* allow us to resume.
*/

/*
* 1. Set GRS/GTS bits in DMACTRL register
*/
sc->sc_dmactrl |= DMACTRL_GRS | DMACTRL_GTS;
etsec_write(sc, IMASK, sc->sc_imask & ~imask_gsc_mask);
etsec_write(sc, IEVENT, imask_gsc_mask);
etsec_write(sc, DMACTRL, sc->sc_dmactrl);

if (etsec_read(sc, MACCFG1) & (MACCFG1_TX_EN | MACCFG1_RX_EN)) {
/*
* 2. Poll GRSC/GTSC bits in IEVENT register until both are set
*/
etsec_write(sc, IMASK, sc->sc_imask | imask_gsc_mask);

u_int timo = 1000;
uint32_t ievent = etsec_read(sc, IEVENT);
while ((ievent & imask_gsc_mask) != imask_gsc_mask) {
if (--timo == 0) {
aprint_error_dev(sc->sc_dev,
"WARNING: "
"request to stop failed (IEVENT=%#x)\n",
ievent);
break;
}
delay(10);
ievent = etsec_read(sc, IEVENT);
}
}

/*
* Now reset the controller.
*
* 3. Set SOFT_RESET bit in MACCFG1 register
* 4. Clear SOFT_RESET bit in MACCFG1 register
*/
etsec_write(sc, MACCFG1, MACCFG1_SOFT_RESET);
etsec_write(sc, MACCFG1, 0);
etsec_write(sc, IMASK, 0);
etsec_write(sc, IEVENT, ~0);
sc->sc_imask = 0;
ifp->if_flags &= ~IFF_RUNNING;

uint32_t tbipa = etsec_read(sc, TBIPA);
if (tbipa == sc->sc_phy_addr) {
aprint_normal_dev(sc->sc_dev, "relocating TBI\n");
etsec_write(sc, TBIPA, 0x1f);
}
uint32_t miimcfg = etsec_read(sc, MIIMCFG);
etsec_write(sc, MIIMCFG, MIIMCFG_RESET);
etsec_write(sc, MIIMCFG, miimcfg);

/*
* Let's consume any remaining transmitted packets. And if we are
* disabling the interface, purge ourselves of any untransmitted
* packets. But don't consume any received packets, just drop them.
* If we aren't disabling the interface, save the mbufs in the
* receive queue for reuse.
*/
pq3etsec_rxq_purge(sc, &sc->sc_rxq, disable);
pq3etsec_txq_consume(sc, &sc->sc_txq);
if (disable) {
pq3etsec_txq_purge(sc, &sc->sc_txq);
IFQ_PURGE(&ifp->if_snd);
}
}

static void
pq3etsec_ifwatchdog(struct ifnet *ifp)
{
}

static void
pq3etsec_mc_setup(
struct pq3etsec_softc *sc)
{
struct ethercom * const ec = &sc->sc_ec;
struct ifnet * const ifp = &sc->sc_if;
struct ether_multi *enm;
struct ether_multistep step;
uint32_t *gaddr = sc->sc_gaddr + ((sc->sc_rctrl & RCTRL_GHTX) ? 0 : 8);
const uint32_t crc_shift = 32 - ((sc->sc_rctrl & RCTRL_GHTX) ? 9 : 8);

memset(sc->sc_gaddr, 0, sizeof(sc->sc_gaddr));
memset(sc->sc_macaddrs, 0, sizeof(sc->sc_macaddrs));

ifp->if_flags &= ~IFF_ALLMULTI;

ETHER_LOCK(ec);
ETHER_FIRST_MULTI(step, ec, enm);
for (u_int i = 0; enm != NULL; ) {
const char *addr = enm->enm_addrlo;
if (memcmp(addr, enm->enm_addrhi, ETHER_ADDR_LEN) != 0) {
ifp->if_flags |= IFF_ALLMULTI;
memset(gaddr, 0xff, 32 << (crc_shift & 1));
memset(sc->sc_macaddrs, 0, sizeof(sc->sc_macaddrs));
break;
}
if ((sc->sc_rctrl & RCTRL_EMEN)
&& i < __arraycount(sc->sc_macaddrs)) {
sc->sc_macaddrs[i++] = pq3etsec_macaddr_create(addr);
} else {
uint32_t crc = ether_crc32_be(addr, ETHER_ADDR_LEN);
#if 0
printf("%s: %s: crc=%#x: %#x: [%u,%u]=%#x\n", __func__,
ether_sprintf(addr), crc,
crc >> crc_shift,
crc >> (crc_shift + 5),
(crc >> crc_shift) & 31,
1 << (((crc >> crc_shift) & 31) ^ 31));
#endif
/*
* The documentation doesn't completely follow PowerPC
* bit order. The BE crc32 (H) for 01:00:5E:00:00:01
* is 0x7fa32d9b. By empirical testing, the
* corresponding hash bit is word 3, bit 31 (ppc bit
* order). Since 3 << 31 | 31 is 0x7f, we deduce
* H[0:2] selects the register while H[3:7] selects
* the bit (ppc bit order).
*/
crc >>= crc_shift;
gaddr[crc / 32] |= 1 << ((crc & 31) ^ 31);
}
ETHER_NEXT_MULTI(step, enm);
}
ETHER_UNLOCK(ec);
for (u_int i = 0; i < 8; i++) {
etsec_write(sc, IGADDR(i), sc->sc_gaddr[i]);
etsec_write(sc, GADDR(i), sc->sc_gaddr[i+8]);
#if 0
if (sc->sc_gaddr[i] || sc->sc_gaddr[i+8])
printf("%s: IGADDR%u(%#x)=%#x GADDR%u(%#x)=%#x\n", __func__,
i, IGADDR(i), etsec_read(sc, IGADDR(i)),
i, GADDR(i), etsec_read(sc, GADDR(i)));
#endif
}
for (u_int i = 0; i < __arraycount(sc->sc_macaddrs); i++) {
uint64_t macaddr = sc->sc_macaddrs[i];
etsec_write(sc, MACnADDR1(i), (uint32_t)(macaddr >> 32));
etsec_write(sc, MACnADDR2(i), (uint32_t)(macaddr >> 0));
#if 0
if (macaddr)
printf("%s: MAC%02uADDR2(%08x)=%#x MAC%02uADDR2(%#x)=%08x\n", __func__,
i+1, MACnADDR1(i), etsec_read(sc, MACnADDR1(i)),
i+1, MACnADDR2(i), etsec_read(sc, MACnADDR2(i)));
#endif
}
}

static int
pq3etsec_ifioctl(struct ifnet *ifp, u_long cmd, void *data)
{
struct pq3etsec_softc *sc = ifp->if_softc;
struct ifreq * const ifr = data;
const int s = splnet();
int error;

switch (cmd) {
case SIOCSIFMEDIA:
/* Flow control requires full-duplex mode. */
if (IFM_SUBTYPE(ifr->ifr_media) == IFM_AUTO ||
(ifr->ifr_media & IFM_FDX) == 0)
ifr->ifr_media &= ~IFM_ETH_FMASK;
if (IFM_SUBTYPE(ifr->ifr_media) != IFM_AUTO) {
if ((ifr->ifr_media & IFM_ETH_FMASK) == IFM_FLOW) {
/* We can do both TXPAUSE and RXPAUSE. */
ifr->ifr_media |=
IFM_ETH_TXPAUSE | IFM_ETH_RXPAUSE;
}
}
error = ifmedia_ioctl(ifp, ifr, &sc->sc_mii.mii_media, cmd);
break;

default:
error = ether_ioctl(ifp, cmd, data);
if (error != ENETRESET)
break;

if (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI) {
error = 0;
if (ifp->if_flags & IFF_RUNNING)
pq3etsec_mc_setup(sc);
break;
}
error = pq3etsec_ifinit(ifp);
break;
}

splx(s);
return error;
}

static void
pq3etsec_rxq_desc_presync(
struct pq3etsec_softc *sc,
struct pq3etsec_rxqueue *rxq,
volatile struct rxbd *rxbd,
size_t count)
{
bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap,
(rxbd - rxq->rxq_first) * sizeof(*rxbd), count * sizeof(*rxbd),
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}

static void
pq3etsec_rxq_desc_postsync(
struct pq3etsec_softc *sc,
struct pq3etsec_rxqueue *rxq,
volatile struct rxbd *rxbd,
size_t count)
{
bus_dmamap_sync(sc->sc_dmat, rxq->rxq_descmap,
(rxbd - rxq->rxq_first) * sizeof(*rxbd), count * sizeof(*rxbd),
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
}

static void
pq3etsec_txq_desc_presync(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq,
volatile struct txbd *txbd,
size_t count)
{
bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap,
(txbd - txq->txq_first) * sizeof(*txbd), count * sizeof(*txbd),
BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
}

static void
pq3etsec_txq_desc_postsync(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq,
volatile struct txbd *txbd,
size_t count)
{
bus_dmamap_sync(sc->sc_dmat, txq->txq_descmap,
(txbd - txq->txq_first) * sizeof(*txbd), count * sizeof(*txbd),
BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
}

static bus_dmamap_t
pq3etsec_mapcache_get(
struct pq3etsec_softc *sc,
struct pq3etsec_mapcache *dmc)
{
KASSERT(dmc->dmc_nmaps > 0);
KASSERT(dmc->dmc_maps[dmc->dmc_nmaps-1] != NULL);
return dmc->dmc_maps[--dmc->dmc_nmaps];
}

static void
pq3etsec_mapcache_put(
struct pq3etsec_softc *sc,
struct pq3etsec_mapcache *dmc,
bus_dmamap_t map)
{
KASSERT(map != NULL);
KASSERT(dmc->dmc_nmaps < dmc->dmc_maxmaps);
dmc->dmc_maps[dmc->dmc_nmaps++] = map;
}

static void
pq3etsec_mapcache_destroy(
struct pq3etsec_softc *sc,
struct pq3etsec_mapcache *dmc)
{
const size_t dmc_size =
offsetof(struct pq3etsec_mapcache, dmc_maps[dmc->dmc_maxmaps]);

for (u_int i = 0; i < dmc->dmc_maxmaps; i++) {
bus_dmamap_destroy(sc->sc_dmat, dmc->dmc_maps[i]);
}
kmem_intr_free(dmc, dmc_size);
}

static int
pq3etsec_mapcache_create(
struct pq3etsec_softc *sc,
struct pq3etsec_mapcache **dmc_p,
size_t maxmaps,
size_t maxmapsize,
size_t maxseg)
{
const size_t dmc_size =
offsetof(struct pq3etsec_mapcache, dmc_maps[maxmaps]);
struct pq3etsec_mapcache * const dmc =
kmem_intr_zalloc(dmc_size, KM_NOSLEEP);

dmc->dmc_maxmaps = maxmaps;
dmc->dmc_nmaps = maxmaps;
dmc->dmc_maxmapsize = maxmapsize;
dmc->dmc_maxseg = maxseg;

for (u_int i = 0; i < maxmaps; i++) {
int error = bus_dmamap_create(sc->sc_dmat, dmc->dmc_maxmapsize,
dmc->dmc_maxseg, dmc->dmc_maxmapsize, 0,
BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dmc->dmc_maps[i]);
if (error) {
aprint_error_dev(sc->sc_dev,
"failed to creat dma map cache "
"entry %u of %zu: %d\n",
i, maxmaps, error);
while (i-- > 0) {
bus_dmamap_destroy(sc->sc_dmat,
dmc->dmc_maps[i]);
}
kmem_intr_free(dmc, dmc_size);
return error;
}
KASSERT(dmc->dmc_maps[i] != NULL);
}

*dmc_p = dmc;

return 0;
}

#if 0
static void
pq3etsec_dmamem_free(
bus_dma_tag_t dmat,
size_t map_size,
bus_dma_segment_t *seg,
bus_dmamap_t map,
void *kvap)
{
bus_dmamap_destroy(dmat, map);
bus_dmamem_unmap(dmat, kvap, map_size);
bus_dmamem_free(dmat, seg, 1);
}
#endif

static int
pq3etsec_dmamem_alloc(
bus_dma_tag_t dmat,
size_t map_size,
bus_dma_segment_t *seg,
bus_dmamap_t *map,
void **kvap)
{
int error;
int nseg;

*kvap = NULL;
*map = NULL;

error = bus_dmamem_alloc(dmat, map_size, PAGE_SIZE, 0,
seg, 1, &nseg, 0);
if (error)
return error;

KASSERT(nseg == 1);

error = bus_dmamem_map(dmat, seg, nseg, map_size, (void **)kvap,
BUS_DMA_COHERENT);
if (error == 0) {
error = bus_dmamap_create(dmat, map_size, 1, map_size, 0, 0,
map);
if (error == 0) {
error = bus_dmamap_load(dmat, *map, *kvap, map_size,
NULL, 0);
if (error == 0)
return 0;
bus_dmamap_destroy(dmat, *map);
*map = NULL;
}
bus_dmamem_unmap(dmat, *kvap, map_size);
*kvap = NULL;
}
bus_dmamem_free(dmat, seg, nseg);
return 0;
}

static struct mbuf *
pq3etsec_rx_buf_alloc(
struct pq3etsec_softc *sc)
{
struct mbuf *m = m_gethdr(M_DONTWAIT, MT_DATA);
if (m == NULL) {
printf("%s:%d: %s\n", __func__, __LINE__, "m_gethdr");
return NULL;
}
MCLGET(m, M_DONTWAIT);
if ((m->m_flags & M_EXT) == 0) {
printf("%s:%d: %s\n", __func__, __LINE__, "MCLGET");
m_freem(m);
return NULL;
}
m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;

bus_dmamap_t map = pq3etsec_mapcache_get(sc, sc->sc_rx_mapcache);
if (map == NULL) {
printf("%s:%d: %s\n", __func__, __LINE__, "map get");
m_freem(m);
return NULL;
}
M_SETCTX(m, map);
m->m_len = m->m_pkthdr.len = MCLBYTES;
int error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
BUS_DMA_READ | BUS_DMA_NOWAIT);
if (error) {
aprint_error_dev(sc->sc_dev, "fail to load rx dmamap: %d\n",
error);
M_SETCTX(m, NULL);
m_freem(m);
pq3etsec_mapcache_put(sc, sc->sc_rx_mapcache, map);
return NULL;
}
KASSERT(map->dm_mapsize == MCLBYTES);
bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
BUS_DMASYNC_PREREAD);

return m;
}

static void
pq3etsec_rx_map_unload(
struct pq3etsec_softc *sc,
struct mbuf *m)
{
KASSERT(m);
for (; m != NULL; m = m->m_next) {
bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t);
KASSERT(map);
KASSERT(map->dm_mapsize == MCLBYTES);
bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_len,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->sc_dmat, map);
pq3etsec_mapcache_put(sc, sc->sc_rx_mapcache, map);
M_SETCTX(m, NULL);
}
}

static bool
pq3etsec_rxq_produce(
struct pq3etsec_softc *sc,
struct pq3etsec_rxqueue *rxq)
{
volatile struct rxbd *producer = rxq->rxq_producer;
#if 0
size_t inuse = rxq->rxq_inuse;
#endif
while (rxq->rxq_inuse < rxq->rxq_threshold) {
struct mbuf *m;
IF_DEQUEUE(&sc->sc_rx_bufcache, m);
if (m == NULL) {
m = pq3etsec_rx_buf_alloc(sc);
if (m == NULL) {
printf("%s: pq3etsec_rx_buf_alloc failed\n", __func__);
break;
}
}
bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t);
KASSERT(map);

#ifdef ETSEC_DEBUG
KASSERT(rxq->rxq_mbufs[producer-rxq->rxq_first] == NULL);
rxq->rxq_mbufs[producer-rxq->rxq_first] = m;
#endif

/* rxbd_len is write-only by the ETSEC */
producer->rxbd_bufptr = map->dm_segs[0].ds_addr;
membar_producer();
producer->rxbd_flags |= RXBD_E;
if (__predict_false(rxq->rxq_mhead == NULL)) {
KASSERT(producer == rxq->rxq_consumer);
rxq->rxq_mconsumer = m;
}
*rxq->rxq_mtail = m;
rxq->rxq_mtail = &m->m_next;
m->m_len = MCLBYTES;
m->m_next = NULL;
rxq->rxq_inuse++;
if (++producer == rxq->rxq_last) {
membar_producer();
pq3etsec_rxq_desc_presync(sc, rxq, rxq->rxq_producer,
rxq->rxq_last - rxq->rxq_producer);
producer = rxq->rxq_producer = rxq->rxq_first;
}
}
if (producer != rxq->rxq_producer) {
membar_producer();
pq3etsec_rxq_desc_presync(sc, rxq, rxq->rxq_producer,
producer - rxq->rxq_producer);
rxq->rxq_producer = producer;
}
uint32_t qhlt = etsec_read(sc, RSTAT) & RSTAT_QHLT;
if (qhlt) {
KASSERT(qhlt & rxq->rxq_qmask);
sc->sc_ev_rx_stall.ev_count++;
etsec_write(sc, RSTAT, RSTAT_QHLT & rxq->rxq_qmask);
}
#if 0
aprint_normal_dev(sc->sc_dev,
"%s: buffers inuse went from %zu to %zu\n",
__func__, inuse, rxq->rxq_inuse);
#endif
return true;
}

static bool
pq3etsec_rx_offload(
struct pq3etsec_softc *sc,
struct mbuf *m,
const struct rxfcb *fcb)
{
if (fcb->rxfcb_flags & RXFCB_VLN) {
vlan_set_tag(m, fcb->rxfcb_vlctl);
}
if ((fcb->rxfcb_flags & RXFCB_IP) == 0
|| (fcb->rxfcb_flags & (RXFCB_CIP | RXFCB_CTU)) == 0)
return true;
int csum_flags = 0;
if ((fcb->rxfcb_flags & (RXFCB_IP6 | RXFCB_CIP)) == RXFCB_CIP) {
csum_flags |= M_CSUM_IPv4;
if (fcb->rxfcb_flags & RXFCB_EIP)
csum_flags |= M_CSUM_IPv4_BAD;
}
if ((fcb->rxfcb_flags & RXFCB_CTU) == RXFCB_CTU) {
int ipv_flags;
if (fcb->rxfcb_flags & RXFCB_IP6)
ipv_flags = M_CSUM_TCPv6 | M_CSUM_UDPv6;
else
ipv_flags = M_CSUM_TCPv4 | M_CSUM_UDPv4;
if (fcb->rxfcb_pro == IPPROTO_TCP) {
csum_flags |= (M_CSUM_TCPv4 |M_CSUM_TCPv6) & ipv_flags;
} else {
csum_flags |= (M_CSUM_UDPv4 |M_CSUM_UDPv6) & ipv_flags;
}
if (fcb->rxfcb_flags & RXFCB_ETU)
csum_flags |= M_CSUM_TCP_UDP_BAD;
}

m->m_pkthdr.csum_flags = csum_flags;
return true;
}

static void
pq3etsec_rx_input(
struct pq3etsec_softc *sc,
struct mbuf *m,
uint16_t rxbd_flags)
{
struct ifnet * const ifp = &sc->sc_if;

pq3etsec_rx_map_unload(sc, m);

if ((sc->sc_rctrl & RCTRL_PRSDEP) != RCTRL_PRSDEP_OFF) {
struct rxfcb fcb = *mtod(m, struct rxfcb *);
if (!pq3etsec_rx_offload(sc, m, &fcb))
return;
}
m_adj(m, sc->sc_rx_adjlen);

if (rxbd_flags & RXBD_M)
m->m_flags |= M_PROMISC;
if (rxbd_flags & RXBD_BC)
m->m_flags |= M_BCAST;
if (rxbd_flags & RXBD_MC)
m->m_flags |= M_MCAST;
m->m_flags |= M_HASFCS;
m_set_rcvif(m, &sc->sc_if);

/*
* Let's give it to the network subsystm to deal with.
*/
if_percpuq_enqueue(ifp->if_percpuq, m);
}

static void
pq3etsec_rxq_consume(
struct pq3etsec_softc *sc,
struct pq3etsec_rxqueue *rxq)
{
struct ifnet * const ifp = &sc->sc_if;
volatile struct rxbd *consumer = rxq->rxq_consumer;
size_t rxconsumed = 0;

etsec_write(sc, RSTAT, RSTAT_RXF & rxq->rxq_qmask);

for (;;) {
if (consumer == rxq->rxq_producer) {
rxq->rxq_consumer = consumer;
rxq->rxq_inuse -= rxconsumed;
KASSERT(rxq->rxq_inuse == 0);
break;
}
pq3etsec_rxq_desc_postsync(sc, rxq, consumer, 1);
const uint16_t rxbd_flags = consumer->rxbd_flags;
if (rxbd_flags & RXBD_E) {
rxq->rxq_consumer = consumer;
rxq->rxq_inuse -= rxconsumed;
break;
}
KASSERT(rxq->rxq_mconsumer != NULL);
#ifdef ETSEC_DEBUG
KASSERT(rxq->rxq_mbufs[consumer - rxq->rxq_first] == rxq->rxq_mconsumer);
#endif
#if 0
printf("%s: rxdb[%u]: flags=%#x len=%#x: %08x %08x %08x %08x\n",
__func__,
consumer - rxq->rxq_first, rxbd_flags, consumer->rxbd_len,
mtod(rxq->rxq_mconsumer, int *)[0],
mtod(rxq->rxq_mconsumer, int *)[1],
mtod(rxq->rxq_mconsumer, int *)[2],
mtod(rxq->rxq_mconsumer, int *)[3]);
#endif
/*
* We own this packet again. Clear all flags except wrap.
*/
rxconsumed++;
consumer->rxbd_flags = rxbd_flags & (RXBD_W | RXBD_I);

/*
* If this descriptor has the LAST bit set and no errors,
* it's a valid input packet.
*/
if ((rxbd_flags & (RXBD_L | RXBD_ERRORS)) == RXBD_L) {
size_t rxbd_len = consumer->rxbd_len;
struct mbuf *m = rxq->rxq_mhead;
struct mbuf *m_last = rxq->rxq_mconsumer;
if ((rxq->rxq_mhead = m_last->m_next) == NULL)
rxq->rxq_mtail = &rxq->rxq_mhead;
rxq->rxq_mconsumer = rxq->rxq_mhead;
m_last->m_next = NULL;
m_last->m_len = rxbd_len & (MCLBYTES - 1);
m->m_pkthdr.len = rxbd_len;
pq3etsec_rx_input(sc, m, rxbd_flags);
} else if (rxbd_flags & RXBD_L) {
KASSERT(rxbd_flags & RXBD_ERRORS);
struct mbuf *m;
/*
* We encountered an error, take the mbufs and add
* then to the rx bufcache so we can reuse them.
*/
if_statinc(ifp, if_ierrors);
for (m = rxq->rxq_mhead;
m != rxq->rxq_mconsumer;
m = m->m_next) {
IF_ENQUEUE(&sc->sc_rx_bufcache, m);
}
m = rxq->rxq_mconsumer;
if ((rxq->rxq_mhead = m->m_next) == NULL)
rxq->rxq_mtail = &rxq->rxq_mhead;
rxq->rxq_mconsumer = m->m_next;
IF_ENQUEUE(&sc->sc_rx_bufcache, m);
} else {
rxq->rxq_mconsumer = rxq->rxq_mconsumer->m_next;
}
#ifdef ETSEC_DEBUG
rxq->rxq_mbufs[consumer - rxq->rxq_first] = NULL;
#endif

/*
* Wrap at the last entry!
*/
if (rxbd_flags & RXBD_W) {
KASSERT(consumer + 1 == rxq->rxq_last);
consumer = rxq->rxq_first;
} else {
consumer++;
}
#ifdef ETSEC_DEBUG
KASSERT(rxq->rxq_mbufs[consumer - rxq->rxq_first] == rxq->rxq_mconsumer);
#endif
}

if (rxconsumed != 0)
rnd_add_uint32(&sc->rnd_source, rxconsumed);
}

static void
pq3etsec_rxq_purge(
struct pq3etsec_softc *sc,
struct pq3etsec_rxqueue *rxq,
bool discard)
{
struct mbuf *m;

if ((m = rxq->rxq_mhead) != NULL) {
#ifdef ETSEC_DEBUG
memset(rxq->rxq_mbufs, 0, sizeof(rxq->rxq_mbufs));
#endif

if (discard) {
pq3etsec_rx_map_unload(sc, m);
m_freem(m);
} else {
while (m != NULL) {
struct mbuf *m0 = m->m_next;
m->m_next = NULL;
IF_ENQUEUE(&sc->sc_rx_bufcache, m);
m = m0;
}
}
}

rxq->rxq_mconsumer = NULL;
rxq->rxq_mhead = NULL;
rxq->rxq_mtail = &rxq->rxq_mhead;
rxq->rxq_inuse = 0;
}

static void
pq3etsec_rxq_reset(
struct pq3etsec_softc *sc,
struct pq3etsec_rxqueue *rxq)
{
/*
* sync all the descriptors
*/
pq3etsec_rxq_desc_postsync(sc, rxq, rxq->rxq_first,
rxq->rxq_last - rxq->rxq_first);

/*
* Make sure we own all descriptors in the ring.
*/
volatile struct rxbd *rxbd;
for (rxbd = rxq->rxq_first; rxbd < rxq->rxq_last - 1; rxbd++) {
rxbd->rxbd_flags = RXBD_I;
}

/*
* Last descriptor has the wrap flag.
*/
rxbd->rxbd_flags = RXBD_W | RXBD_I;

/*
* Reset the producer consumer indexes.
*/
rxq->rxq_consumer = rxq->rxq_first;
rxq->rxq_producer = rxq->rxq_first;
rxq->rxq_inuse = 0;
if (rxq->rxq_threshold < ETSEC_MINRXMBUFS)
rxq->rxq_threshold = ETSEC_MINRXMBUFS;

sc->sc_imask |= IEVENT_RXF | IEVENT_BSY;

/*
* Restart the transmit at the first descriptor
*/
etsec_write(sc, rxq->rxq_reg_rbase, rxq->rxq_descmap->dm_segs->ds_addr);
}

static int
pq3etsec_rxq_attach(
struct pq3etsec_softc *sc,
struct pq3etsec_rxqueue *rxq,
u_int qno)
{
size_t map_size = PAGE_SIZE;
size_t desc_count = map_size / sizeof(struct rxbd);
int error;
void *descs;

error = pq3etsec_dmamem_alloc(sc->sc_dmat, map_size,
&rxq->rxq_descmap_seg, &rxq->rxq_descmap, &descs);
if (error)
return error;

memset(descs, 0, map_size);
rxq->rxq_first = descs;
rxq->rxq_last = rxq->rxq_first + desc_count;
rxq->rxq_consumer = descs;
rxq->rxq_producer = descs;

pq3etsec_rxq_purge(sc, rxq, true);
pq3etsec_rxq_reset(sc, rxq);

rxq->rxq_reg_rbase = RBASEn(qno);
rxq->rxq_qmask = RSTAT_QHLTn(qno) | RSTAT_RXFn(qno);

return 0;
}

static bool
pq3etsec_txq_active_p(
struct pq3etsec_softc * const sc,
struct pq3etsec_txqueue *txq)
{
return !IF_IS_EMPTY(&txq->txq_mbufs);
}

static bool
pq3etsec_txq_fillable_p(
struct pq3etsec_softc * const sc,
struct pq3etsec_txqueue *txq)
{
return txq->txq_free >= txq->txq_threshold;
}

static int
pq3etsec_txq_attach(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq,
u_int qno)
{
size_t map_size = PAGE_SIZE;
size_t desc_count = map_size / sizeof(struct txbd);
int error;
void *descs;

error = pq3etsec_dmamem_alloc(sc->sc_dmat, map_size,
&txq->txq_descmap_seg, &txq->txq_descmap, &descs);
if (error)
return error;

memset(descs, 0, map_size);
txq->txq_first = descs;
txq->txq_last = txq->txq_first + desc_count;
txq->txq_consumer = descs;
txq->txq_producer = descs;

IFQ_SET_MAXLEN(&txq->txq_mbufs, ETSEC_MAXTXMBUFS);

txq->txq_reg_tbase = TBASEn(qno);
txq->txq_qmask = TSTAT_THLTn(qno) | TSTAT_TXFn(qno);

pq3etsec_txq_reset(sc, txq);

return 0;
}

static int
pq3etsec_txq_map_load(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq,
struct mbuf *m)
{
bus_dmamap_t map;
int error;

map = M_GETCTX(m, bus_dmamap_t);
if (map != NULL)
return 0;

map = pq3etsec_mapcache_get(sc, sc->sc_tx_mapcache);
if (map == NULL)
return ENOMEM;

error = bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
BUS_DMA_WRITE | BUS_DMA_NOWAIT);
if (error)
return error;

bus_dmamap_sync(sc->sc_dmat, map, 0, m->m_pkthdr.len,
BUS_DMASYNC_PREWRITE);
M_SETCTX(m, map);
return 0;
}

static void
pq3etsec_txq_map_unload(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq,
struct mbuf *m)
{
KASSERT(m);
bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t);
bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, map);
pq3etsec_mapcache_put(sc, sc->sc_tx_mapcache, map);
}

static bool
pq3etsec_txq_produce(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq,
struct mbuf *m)
{
bus_dmamap_t map = M_GETCTX(m, bus_dmamap_t);

if (map->dm_nsegs > txq->txq_free)
return false;

/*
* TCP Offload flag must be set in the first descriptor.
*/
volatile struct txbd *producer = txq->txq_producer;
uint16_t last_flags = TXBD_L;
uint16_t first_flags = TXBD_R
| ((m->m_flags & M_HASFCB) ? TXBD_TOE : 0);

/*
* If we've produced enough descriptors without consuming any
* we need to ask for an interrupt to reclaim some.
*/
txq->txq_lastintr += map->dm_nsegs;
if (ETSEC_IC_TX_ENABLED(sc)
|| txq->txq_lastintr >= txq->txq_threshold
|| txq->txq_mbufs.ifq_len + 1 == txq->txq_mbufs.ifq_maxlen) {
txq->txq_lastintr = 0;
last_flags |= TXBD_I;
}

#ifdef ETSEC_DEBUG
KASSERT(txq->txq_lmbufs[producer - txq->txq_first] == NULL);
#endif
KASSERT(producer != txq->txq_last);
producer->txbd_bufptr = map->dm_segs[0].ds_addr;
producer->txbd_len = map->dm_segs[0].ds_len;

if (map->dm_nsegs > 1) {
volatile struct txbd *start = producer + 1;
size_t count = map->dm_nsegs - 1;
for (u_int i = 1; i < map->dm_nsegs; i++) {
if (__predict_false(++producer == txq->txq_last)) {
producer = txq->txq_first;
if (start < txq->txq_last) {
pq3etsec_txq_desc_presync(sc, txq,
start, txq->txq_last - start);
count -= txq->txq_last - start;
}
start = txq->txq_first;
}
#ifdef ETSEC_DEBUG
KASSERT(txq->txq_lmbufs[producer - txq->txq_first] == NULL);
#endif
producer->txbd_bufptr = map->dm_segs[i].ds_addr;
producer->txbd_len = map->dm_segs[i].ds_len;
producer->txbd_flags = TXBD_R
| (producer->txbd_flags & TXBD_W)
| (i == map->dm_nsegs - 1 ? last_flags : 0);
#if 0
printf("%s: txbd[%u]=%#x/%u/%#x\n", __func__, producer - txq->txq_first,
producer->txbd_flags, producer->txbd_len, producer->txbd_bufptr);
#endif
}
pq3etsec_txq_desc_presync(sc, txq, start, count);
} else {
first_flags |= last_flags;
}

membar_producer();
txq->txq_producer->txbd_flags =
first_flags | (txq->txq_producer->txbd_flags & TXBD_W);
#if 0
printf("%s: txbd[%u]=%#x/%u/%#x\n", __func__,
txq->txq_producer - txq->txq_first, txq->txq_producer->txbd_flags,
txq->txq_producer->txbd_len, txq->txq_producer->txbd_bufptr);
#endif
pq3etsec_txq_desc_presync(sc, txq, txq->txq_producer, 1);

/*
* Reduce free count by the number of segments we consumed.
*/
txq->txq_free -= map->dm_nsegs;
KASSERT(map->dm_nsegs == 1 || txq->txq_producer != producer);
KASSERT(map->dm_nsegs == 1 || (txq->txq_producer->txbd_flags & TXBD_L) == 0);
KASSERT(producer->txbd_flags & TXBD_L);
#ifdef ETSEC_DEBUG
txq->txq_lmbufs[producer - txq->txq_first] = m;
#endif

#if 0
printf("%s: mbuf %p: produced a %u byte packet in %u segments (%u..%u)\n",
__func__, m, m->m_pkthdr.len, map->dm_nsegs,
txq->txq_producer - txq->txq_first, producer - txq->txq_first);
#endif

if (++producer == txq->txq_last)
txq->txq_producer = txq->txq_first;
else
txq->txq_producer = producer;
IF_ENQUEUE(&txq->txq_mbufs, m);

/*
* Restart the transmitter.
*/
etsec_write(sc, TSTAT, txq->txq_qmask & TSTAT_THLT); /* W1C */

return true;
}

static void
pq3etsec_tx_offload(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq,
struct mbuf **mp)
{
struct mbuf *m = *mp;
u_int csum_flags = m->m_pkthdr.csum_flags;
bool have_vtag;
uint16_t vtag;

KASSERT(m->m_flags & M_PKTHDR);

have_vtag = vlan_has_tag(m);
vtag = (have_vtag) ? vlan_get_tag(m) : 0;

/*
* Let see if we are doing any offload first.
*/
if (csum_flags == 0 && !have_vtag) {
m->m_flags &= ~M_HASFCB;
return;
}

uint16_t flags = 0;
if (csum_flags & M_CSUM_IP) {
flags |= TXFCB_IP
| ((csum_flags & M_CSUM_IP6) ? TXFCB_IP6 : 0)
| ((csum_flags & M_CSUM_TUP) ? TXFCB_TUP : 0)
| ((csum_flags & M_CSUM_UDP) ? TXFCB_UDP : 0)
| ((csum_flags & M_CSUM_CIP) ? TXFCB_CIP : 0)
| ((csum_flags & M_CSUM_CTU) ? TXFCB_CTU : 0);
}
if (have_vtag) {
flags |= TXFCB_VLN;
}
if (flags == 0) {
m->m_flags &= ~M_HASFCB;
return;
}

struct txfcb fcb;
fcb.txfcb_flags = flags;
if (csum_flags & M_CSUM_IPv4)
fcb.txfcb_l4os = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data);
else
fcb.txfcb_l4os = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data);
fcb.txfcb_l3os = ETHER_HDR_LEN;
fcb.txfcb_phcs = 0;
fcb.txfcb_vlctl = vtag;

#if 0
printf("%s: csum_flags=%#x: txfcb flags=%#x lsos=%u l4os=%u phcs=%u vlctl=%#x\n",
__func__, csum_flags, fcb.txfcb_flags, fcb.txfcb_l3os, fcb.txfcb_l4os,
fcb.txfcb_phcs, fcb.txfcb_vlctl);
#endif

if (M_LEADINGSPACE(m) >= sizeof(fcb)) {
m->m_data -= sizeof(fcb);
m->m_len += sizeof(fcb);
} else if (!(m->m_flags & M_EXT) && MHLEN - m->m_len >= sizeof(fcb)) {
memmove(m->m_pktdat + sizeof(fcb), m->m_data, m->m_len);
m->m_data = m->m_pktdat;
m->m_len += sizeof(fcb);
} else {
struct mbuf *mn;
MGET(mn, M_DONTWAIT, m->m_type);
if (mn == NULL) {
if (csum_flags & M_CSUM_IP4) {
#ifdef INET
in_undefer_cksum(m, ETHER_HDR_LEN,
csum_flags & M_CSUM_IP4);
#else
panic("%s: impossible M_CSUM flags %#x",
device_xname(sc->sc_dev), csum_flags);
#endif
} else if (csum_flags & M_CSUM_IP6) {
#ifdef INET6
in6_undefer_cksum(m, ETHER_HDR_LEN,
csum_flags & M_CSUM_IP6);
#else
panic("%s: impossible M_CSUM flags %#x",
device_xname(sc->sc_dev), csum_flags);
#endif
}

m->m_flags &= ~M_HASFCB;
return;
}

m_move_pkthdr(mn, m);
mn->m_next = m;
m = mn;
m_align(m, sizeof(fcb));
m->m_len = sizeof(fcb);
*mp = m;
}
m->m_pkthdr.len += sizeof(fcb);
m->m_flags |= M_HASFCB;
*mtod(m, struct txfcb *) = fcb;
return;
}

static bool
pq3etsec_txq_enqueue(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq)
{
for (;;) {
if (IF_QFULL(&txq->txq_mbufs))
return false;
struct mbuf *m = txq->txq_next;
if (m == NULL) {
int s = splnet();
IFQ_DEQUEUE(&sc->sc_if.if_snd, m);
splx(s);
if (m == NULL)
return true;
M_SETCTX(m, NULL);
pq3etsec_tx_offload(sc, txq, &m);
} else {
txq->txq_next = NULL;
}
int error = pq3etsec_txq_map_load(sc, txq, m);
if (error) {
aprint_error_dev(sc->sc_dev,
"discarded packet due to "
"dmamap load failure: %d\n", error);
m_freem(m);
continue;
}
KASSERT(txq->txq_next == NULL);
if (!pq3etsec_txq_produce(sc, txq, m)) {
txq->txq_next = m;
return false;
}
KASSERT(txq->txq_next == NULL);
}
}

static bool
pq3etsec_txq_consume(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq)
{
struct ifnet * const ifp = &sc->sc_if;
volatile struct txbd *consumer = txq->txq_consumer;
size_t txfree = 0;
bool ret;

#if 0
printf("%s: entry: free=%zu\n", __func__, txq->txq_free);
#endif
etsec_write(sc, TSTAT, TSTAT_TXF & txq->txq_qmask);

for (;;) {
if (consumer == txq->txq_producer) {
txq->txq_consumer = consumer;
txq->txq_free += txfree;
txq->txq_lastintr -= uimin(txq->txq_lastintr, txfree);
KASSERT(txq->txq_lastintr == 0);
KASSERT(txq->txq_free ==
txq->txq_last - txq->txq_first - 1);
ret = true;
break;
}
pq3etsec_txq_desc_postsync(sc, txq, consumer, 1);
const uint16_t txbd_flags = consumer->txbd_flags;
if (txbd_flags & TXBD_R) {
txq->txq_consumer = consumer;
txq->txq_free += txfree;
txq->txq_lastintr -= uimin(txq->txq_lastintr, txfree);
ret = pq3etsec_txq_fillable_p(sc, txq);
break;
}

/*
* If this is the last descriptor in the chain, get the
* mbuf, free its dmamap, and free the mbuf chain itself.
*/
if (txbd_flags & TXBD_L) {
struct mbuf *m;

IF_DEQUEUE(&txq->txq_mbufs, m);
#ifdef ETSEC_DEBUG
KASSERTMSG(
m == txq->txq_lmbufs[consumer-txq->txq_first],
"%s: %p [%u]: flags %#x m (%p) != %p (%p)",
__func__, consumer, consumer - txq->txq_first,
txbd_flags, m,
&txq->txq_lmbufs[consumer-txq->txq_first],
txq->txq_lmbufs[consumer-txq->txq_first]);
#endif
KASSERT(m);
pq3etsec_txq_map_unload(sc, txq, m);
#if 0
printf("%s: mbuf %p: consumed a %u byte packet\n",
__func__, m, m->m_pkthdr.len);
#endif
if (m->m_flags & M_HASFCB)
m_adj(m, sizeof(struct txfcb));
bpf_mtap(ifp, m, BPF_D_OUT);
net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
if_statinc_ref(ifp, nsr, if_opackets);
if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len);
if (m->m_flags & M_MCAST)
if_statinc_ref(ifp, nsr, if_omcasts);
if (txbd_flags & TXBD_ERRORS)
if_statinc_ref(ifp, nsr, if_oerrors);
IF_STAT_PUTREF(ifp);
m_freem(m);
#ifdef ETSEC_DEBUG
txq->txq_lmbufs[consumer - txq->txq_first] = NULL;
#endif
} else {
#ifdef ETSEC_DEBUG
KASSERT(txq->txq_lmbufs[consumer-txq->txq_first] == NULL);
#endif
}

/*
* We own this packet again. Clear all flags except wrap.
*/
txfree++;
//consumer->txbd_flags = txbd_flags & TXBD_W;

/*
* Wrap at the last entry!
*/
if (txbd_flags & TXBD_W) {
KASSERT(consumer + 1 == txq->txq_last);
consumer = txq->txq_first;
} else {
consumer++;
KASSERT(consumer < txq->txq_last);
}
}

if (txfree != 0)
rnd_add_uint32(&sc->rnd_source, txfree);
return ret;
}

static void
pq3etsec_txq_purge(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq)
{
struct mbuf *m;
KASSERT((etsec_read(sc, MACCFG1) & MACCFG1_TX_EN) == 0);

for (;;) {
IF_DEQUEUE(&txq->txq_mbufs, m);
if (m == NULL)
break;
pq3etsec_txq_map_unload(sc, txq, m);
m_freem(m);
}
if ((m = txq->txq_next) != NULL) {
txq->txq_next = NULL;
pq3etsec_txq_map_unload(sc, txq, m);
m_freem(m);
}
#ifdef ETSEC_DEBUG
memset(txq->txq_lmbufs, 0, sizeof(txq->txq_lmbufs));
#endif
}

static void
pq3etsec_txq_reset(
struct pq3etsec_softc *sc,
struct pq3etsec_txqueue *txq)
{
/*
* sync all the descriptors
*/
pq3etsec_txq_desc_postsync(sc, txq, txq->txq_first,
txq->txq_last - txq->txq_first);

/*
* Make sure we own all descriptors in the ring.
*/
volatile struct txbd *txbd;
for (txbd = txq->txq_first; txbd < txq->txq_last - 1; txbd++) {
txbd->txbd_flags = 0;
}

/*
* Last descriptor has the wrap flag.
*/
txbd->txbd_flags = TXBD_W;

/*
* Reset the producer consumer indexes.
*/
txq->txq_consumer = txq->txq_first;
txq->txq_producer = txq->txq_first;
txq->txq_free = txq->txq_last - txq->txq_first - 1;
txq->txq_threshold = txq->txq_free / 2;
txq->txq_lastintr = 0;

/*
* What do we want to get interrupted on?
*/
sc->sc_imask |= IEVENT_TXF | IEVENT_TXE;

/*
* Restart the transmit at the first descriptor
*/
etsec_write(sc, txq->txq_reg_tbase, txq->txq_descmap->dm_segs->ds_addr);
}

static void
pq3etsec_ifstart(struct ifnet *ifp)
{
struct pq3etsec_softc * const sc = ifp->if_softc;

if (__predict_false((ifp->if_flags & IFF_RUNNING) == 0)) {
return;
}

atomic_or_uint(&sc->sc_soft_flags, SOFT_TXINTR);
softint_schedule(sc->sc_soft_ih);
}

static void
pq3etsec_tx_error(
struct pq3etsec_softc * const sc)
{
struct pq3etsec_txqueue * const txq = &sc->sc_txq;

pq3etsec_txq_consume(sc, txq);

if (sc->sc_txerrors
& (IEVENT_LC | IEVENT_CRL | IEVENT_XFUN | IEVENT_BABT)) {
} else if (sc->sc_txerrors & IEVENT_EBERR) {
}

if (pq3etsec_txq_active_p(sc, txq))
etsec_write(sc, TSTAT, TSTAT_THLT & txq->txq_qmask);
if (!pq3etsec_txq_enqueue(sc, txq)) {
sc->sc_ev_tx_stall.ev_count++;
}

sc->sc_txerrors = 0;
}

int
pq3etsec_tx_intr(void *arg)
{
struct pq3etsec_softc * const sc = arg;

mutex_enter(sc->sc_hwlock);

sc->sc_ev_tx_intr.ev_count++;

uint32_t ievent = etsec_read(sc, IEVENT);
ievent &= IEVENT_TXF | IEVENT_TXB;
etsec_write(sc, IEVENT, ievent); /* write 1 to clear */

#if 0
aprint_normal_dev(sc->sc_dev, "%s: ievent=%#x imask=%#x\n",
__func__, ievent, etsec_read(sc, IMASK));
#endif

if (ievent == 0) {
mutex_exit(sc->sc_hwlock);
return 0;
}

sc->sc_imask &= ~(IEVENT_TXF | IEVENT_TXB);
atomic_or_uint(&sc->sc_soft_flags, SOFT_TXINTR);
etsec_write(sc, IMASK, sc->sc_imask);
softint_schedule(sc->sc_soft_ih);

mutex_exit(sc->sc_hwlock);

return 1;
}

int
pq3etsec_rx_intr(void *arg)
{
struct pq3etsec_softc * const sc = arg;

mutex_enter(sc->sc_hwlock);

sc->sc_ev_rx_intr.ev_count++;

uint32_t ievent = etsec_read(sc, IEVENT);
ievent &= IEVENT_RXF | IEVENT_RXB;
etsec_write(sc, IEVENT, ievent); /* write 1 to clear */
if (ievent == 0) {
mutex_exit(sc->sc_hwlock);
return 0;
}

#if 0
aprint_normal_dev(sc->sc_dev, "%s: ievent=%#x\n", __func__, ievent);
#endif

sc->sc_imask &= ~(IEVENT_RXF | IEVENT_RXB);
atomic_or_uint(&sc->sc_soft_flags, SOFT_RXINTR);
etsec_write(sc, IMASK, sc->sc_imask);
softint_schedule(sc->sc_soft_ih);

mutex_exit(sc->sc_hwlock);

return 1;
}

int
pq3etsec_error_intr(void *arg)
{
struct pq3etsec_softc * const sc = arg;

mutex_enter(sc->sc_hwlock);

sc->sc_ev_error_intr.ev_count++;

for (int rv = 0, soft_flags = 0;; rv = 1) {
uint32_t ievent = etsec_read(sc, IEVENT);
ievent &= ~(IEVENT_RXF | IEVENT_RXB | IEVENT_TXF | IEVENT_TXB);
etsec_write(sc, IEVENT, ievent); /* write 1 to clear */
if (ievent == 0) {
if (soft_flags) {
atomic_or_uint(&sc->sc_soft_flags, soft_flags);
softint_schedule(sc->sc_soft_ih);
}
mutex_exit(sc->sc_hwlock);
return rv;
}
#if 0
aprint_normal_dev(sc->sc_dev, "%s: ievent=%#x imask=%#x\n",
__func__, ievent, etsec_read(sc, IMASK));
#endif

if (ievent & (IEVENT_GRSC | IEVENT_GTSC)) {
sc->sc_imask &= ~(IEVENT_GRSC | IEVENT_GTSC);
etsec_write(sc, IMASK, sc->sc_imask);
wakeup(sc);
}
if (ievent & (IEVENT_MMRD | IEVENT_MMWR)) {
sc->sc_imask &= ~(IEVENT_MMRD | IEVENT_MMWR);
etsec_write(sc, IMASK, sc->sc_imask);
wakeup(&sc->sc_mii);
}
if (ievent & IEVENT_BSY) {
soft_flags |= SOFT_RXBSY;
sc->sc_imask &= ~IEVENT_BSY;
etsec_write(sc, IMASK, sc->sc_imask);
}
if (ievent & IEVENT_TXE) {
soft_flags |= SOFT_TXERROR;
sc->sc_imask &= ~IEVENT_TXE;
sc->sc_txerrors |= ievent;
}
if (ievent & IEVENT_TXC) {
sc->sc_ev_tx_pause.ev_count++;
}
if (ievent & IEVENT_RXC) {
sc->sc_ev_rx_pause.ev_count++;
}
if (ievent & IEVENT_DPE) {
soft_flags |= SOFT_RESET;
sc->sc_imask &= ~IEVENT_DPE;
etsec_write(sc, IMASK, sc->sc_imask);
}
}
}

void
pq3etsec_soft_intr(void *arg)
{
struct pq3etsec_softc * const sc = arg;
struct ifnet * const ifp = &sc->sc_if;
uint32_t imask = 0;

mutex_enter(sc->sc_lock);

u_int soft_flags = atomic_swap_uint(&sc->sc_soft_flags, 0);

sc->sc_ev_soft_intr.ev_count++;

if (soft_flags & SOFT_RESET) {
int s = splnet();
pq3etsec_ifinit(ifp);
splx(s);
soft_flags = 0;
}

if (soft_flags & SOFT_RXBSY) {
struct pq3etsec_rxqueue * const rxq = &sc->sc_rxq;
size_t threshold = 5 * rxq->rxq_threshold / 4;
if (threshold >= rxq->rxq_last - rxq->rxq_first) {
threshold = rxq->rxq_last - rxq->rxq_first - 1;
} else {
imask |= IEVENT_BSY;
}
aprint_normal_dev(sc->sc_dev,
"increasing receive buffers from %zu to %zu\n",
rxq->rxq_threshold, threshold);
rxq->rxq_threshold = threshold;
}

if ((soft_flags & SOFT_TXINTR)
|| pq3etsec_txq_active_p(sc, &sc->sc_txq)) {
/*
* Let's do what we came here for. Consume transmitted
* packets off the transmit ring.
*/
if (!pq3etsec_txq_consume(sc, &sc->sc_txq)
|| !pq3etsec_txq_enqueue(sc, &sc->sc_txq)) {
sc->sc_ev_tx_stall.ev_count++;
}
imask |= IEVENT_TXF;
}

if (soft_flags & (SOFT_RXINTR | SOFT_RXBSY)) {
/* Let's consume */
pq3etsec_rxq_consume(sc, &sc->sc_rxq);
imask |= IEVENT_RXF;
}

if (soft_flags & SOFT_TXERROR) {
pq3etsec_tx_error(sc);
imask |= IEVENT_TXE;
}

if (ifp->if_flags & IFF_RUNNING) {
pq3etsec_rxq_produce(sc, &sc->sc_rxq);
mutex_spin_enter(sc->sc_hwlock);
sc->sc_imask |= imask;
etsec_write(sc, IMASK, sc->sc_imask);
mutex_spin_exit(sc->sc_hwlock);
} else {
KASSERT((soft_flags & SOFT_RXBSY) == 0);
}

mutex_exit(sc->sc_lock);
}

static void
pq3etsec_mii_tick(void *arg)
{
struct pq3etsec_softc * const sc = arg;
mutex_enter(sc->sc_lock);
callout_ack(&sc->sc_mii_callout);
sc->sc_ev_mii_ticks.ev_count++;
#ifdef DEBUG
uint64_t now = mftb();
if (now - sc->sc_mii_last_tick < cpu_timebase - 5000) {
aprint_debug_dev(sc->sc_dev, "%s: diff=%"PRIu64"\n",
__func__, now - sc->sc_mii_last_tick);
callout_stop(&sc->sc_mii_callout);
}
#endif
mii_tick(&sc->sc_mii);
int s = splnet();
if (sc->sc_soft_flags & SOFT_RESET)
softint_schedule(sc->sc_soft_ih);
splx(s);
callout_schedule(&sc->sc_mii_callout, hz);
#ifdef DEBUG
sc->sc_mii_last_tick = now;
#endif
mutex_exit(sc->sc_lock);
}

static void
pq3etsec_set_ic_rx(struct pq3etsec_softc *sc)
{
uint32_t reg;

if (ETSEC_IC_RX_ENABLED(sc)) {
reg = RXIC_ICEN;
reg |= RXIC_ICFT_SET(sc->sc_ic_rx_count);
reg |= RXIC_ICTT_SET(sc->sc_ic_rx_time);
} else {
/* Disable RX interrupt coalescing */
reg = 0;
}

etsec_write(sc, RXIC, reg);
}

static void
pq3etsec_set_ic_tx(struct pq3etsec_softc *sc)
{
uint32_t reg;

if (ETSEC_IC_TX_ENABLED(sc)) {
reg = TXIC_ICEN;
reg |= TXIC_ICFT_SET(sc->sc_ic_tx_count);
reg |= TXIC_ICTT_SET(sc->sc_ic_tx_time);
} else {
/* Disable TX interrupt coalescing */
reg = 0;
}

etsec_write(sc, TXIC, reg);
}

/*
* sysctl
*/
static int
pq3etsec_sysctl_ic_time_helper(SYSCTLFN_ARGS, int *valuep)
{
struct sysctlnode node = *rnode;
struct pq3etsec_softc *sc = rnode->sysctl_data;
int value = *valuep;
int error;

node.sysctl_data = &value;
error = sysctl_lookup(SYSCTLFN_CALL(&node));
if (error != 0 || newp == NULL)
return error;

if (value < 0 || value > 65535)
return EINVAL;

mutex_enter(sc->sc_lock);
*valuep = value;
if (valuep == &sc->sc_ic_rx_time)
pq3etsec_set_ic_rx(sc);
else
pq3etsec_set_ic_tx(sc);
mutex_exit(sc->sc_lock);

return 0;
}

static int
pq3etsec_sysctl_ic_count_helper(SYSCTLFN_ARGS, int *valuep)
{
struct sysctlnode node = *rnode;
struct pq3etsec_softc *sc = rnode->sysctl_data;
int value = *valuep;
int error;

node.sysctl_data = &value;
error = sysctl_lookup(SYSCTLFN_CALL(&node));
if (error != 0 || newp == NULL)
return error;

if (value < 0 || value > 255)
return EINVAL;

mutex_enter(sc->sc_lock);
*valuep = value;
if (valuep == &sc->sc_ic_rx_count)
pq3etsec_set_ic_rx(sc);
else
pq3etsec_set_ic_tx(sc);
mutex_exit(sc->sc_lock);

return 0;
}

static int
pq3etsec_sysctl_ic_rx_time_helper(SYSCTLFN_ARGS)
{
struct pq3etsec_softc *sc = rnode->sysctl_data;

return pq3etsec_sysctl_ic_time_helper(SYSCTLFN_CALL(rnode),
&sc->sc_ic_rx_time);
}

static int
pq3etsec_sysctl_ic_rx_count_helper(SYSCTLFN_ARGS)
{
struct pq3etsec_softc *sc = rnode->sysctl_data;

return pq3etsec_sysctl_ic_count_helper(SYSCTLFN_CALL(rnode),
&sc->sc_ic_rx_count);
}

static int
pq3etsec_sysctl_ic_tx_time_helper(SYSCTLFN_ARGS)
{
struct pq3etsec_softc *sc = rnode->sysctl_data;

return pq3etsec_sysctl_ic_time_helper(SYSCTLFN_CALL(rnode),
&sc->sc_ic_tx_time);
}

static int
pq3etsec_sysctl_ic_tx_count_helper(SYSCTLFN_ARGS)
{
struct pq3etsec_softc *sc = rnode->sysctl_data;

return pq3etsec_sysctl_ic_count_helper(SYSCTLFN_CALL(rnode),
&sc->sc_ic_tx_count);
}

static void pq3etsec_sysctl_setup(struct sysctllog **clog,
struct pq3etsec_softc *sc)
{
const struct sysctlnode *cnode, *rnode;

if (sysctl_createv(clog, 0, NULL, &rnode,
CTLFLAG_PERMANENT,
CTLTYPE_NODE, device_xname(sc->sc_dev),
SYSCTL_DESCR("TSEC interface"),
NULL, 0, NULL, 0,
CTL_HW, CTL_CREATE, CTL_EOL) != 0)
goto bad;

if (sysctl_createv(clog, 0, &rnode, &rnode,
CTLFLAG_PERMANENT,
CTLTYPE_NODE, "int_coal",
SYSCTL_DESCR("Interrupts coalescing"),
NULL, 0, NULL, 0,
CTL_CREATE, CTL_EOL) != 0)
goto bad;

if (sysctl_createv(clog, 0, &rnode, &cnode,
CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
CTLTYPE_INT, "rx_time",
SYSCTL_DESCR("RX time threshold (0-65535)"),
pq3etsec_sysctl_ic_rx_time_helper, 0, (void *)sc, 0,
CTL_CREATE, CTL_EOL) != 0)
goto bad;

if (sysctl_createv(clog, 0, &rnode, &cnode,
CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
CTLTYPE_INT, "rx_count",
SYSCTL_DESCR("RX frame count threshold (0-255)"),
pq3etsec_sysctl_ic_rx_count_helper, 0, (void *)sc, 0,
CTL_CREATE, CTL_EOL) != 0)
goto bad;

if (sysctl_createv(clog, 0, &rnode, &cnode,
CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
CTLTYPE_INT, "tx_time",
SYSCTL_DESCR("TX time threshold (0-65535)"),
pq3etsec_sysctl_ic_tx_time_helper, 0, (void *)sc, 0,
CTL_CREATE, CTL_EOL) != 0)
goto bad;

if (sysctl_createv(clog, 0, &rnode, &cnode,
CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
CTLTYPE_INT, "tx_count",
SYSCTL_DESCR("TX frame count threshold (0-255)"),
pq3etsec_sysctl_ic_tx_count_helper, 0, (void *)sc, 0,
CTL_CREATE, CTL_EOL) != 0)
goto bad;

return;

bad:
aprint_error_dev(sc->sc_dev, "could not attach sysctl nodes\n");
}