/* $NetBSD: if_ixl.c,v 1.100 2025/03/25 19:24:07 christos Exp $ */

/* $NetBSD: if_ixl.c,v 1.100 2025/03/25 19:24:07 christos Exp $ */

/*
* Copyright (c) 2013-2015, Intel Corporation
* All rights reserved.

* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. Neither the name of the Intel Corporation nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

/*
* Copyright (c) 2016,2017 David Gwynne <[email protected]>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

/*
* Copyright (c) 2019 Internet Initiative Japan, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: if_ixl.c,v 1.100 2025/03/25 19:24:07 christos Exp $");

#ifdef _KERNEL_OPT
#include "opt_if_ixl.h"
#endif

#include <sys/param.h>
#include <sys/types.h>

#include <sys/bitops.h>
#include <sys/cpu.h>
#include <sys/device.h>
#include <sys/evcnt.h>
#include <sys/interrupt.h>
#include <sys/kmem.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/pcq.h>
#include <sys/syslog.h>
#include <sys/workqueue.h>
#include <sys/xcall.h>

#include <sys/bus.h>

#include <net/bpf.h>
#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_media.h>
#include <net/if_ether.h>
#include <net/rss_config.h>

#include <netinet/tcp.h> /* for struct tcphdr */
#include <netinet/udp.h> /* for struct udphdr */

#include <dev/pci/pcivar.h>
#include <dev/pci/pcidevs.h>

#include <dev/pci/if_ixlreg.h>
#include <dev/pci/if_ixlvar.h>

#include <prop/proplib.h>

struct ixl_softc; /* defined */

#define I40E_PF_RESET_WAIT_COUNT 200
#define I40E_AQ_LARGE_BUF 512

/* bitfields for Tx queue mapping in QTX_CTL */
#define I40E_QTX_CTL_VF_QUEUE 0x0
#define I40E_QTX_CTL_VM_QUEUE 0x1
#define I40E_QTX_CTL_PF_QUEUE 0x2

#define I40E_QUEUE_TYPE_EOL 0x7ff
#define I40E_INTR_NOTX_QUEUE 0

#define I40E_QUEUE_TYPE_RX 0x0
#define I40E_QUEUE_TYPE_TX 0x1
#define I40E_QUEUE_TYPE_PE_CEQ 0x2
#define I40E_QUEUE_TYPE_UNKNOWN 0x3

#define I40E_ITR_INDEX_RX 0x0
#define I40E_ITR_INDEX_TX 0x1
#define I40E_ITR_INDEX_OTHER 0x2
#define I40E_ITR_INDEX_NONE 0x3
#define IXL_ITR_RX 0x7a /* 4K intrs/sec */
#define IXL_ITR_TX 0x7a /* 4K intrs/sec */

#define I40E_INTR_NOTX_QUEUE 0
#define I40E_INTR_NOTX_INTR 0
#define I40E_INTR_NOTX_RX_QUEUE 0
#define I40E_INTR_NOTX_TX_QUEUE 1
#define I40E_INTR_NOTX_RX_MASK I40E_PFINT_ICR0_QUEUE_0_MASK
#define I40E_INTR_NOTX_TX_MASK I40E_PFINT_ICR0_QUEUE_1_MASK

#define I40E_HASH_LUT_SIZE_128 0

#define IXL_ICR0_CRIT_ERR_MASK \
(I40E_PFINT_ICR0_PCI_EXCEPTION_MASK | \
I40E_PFINT_ICR0_ECC_ERR_MASK | \
I40E_PFINT_ICR0_PE_CRITERR_MASK)

#define IXL_QUEUE_MAX_XL710 64
#define IXL_QUEUE_MAX_X722 128

#define IXL_TX_PKT_DESCS 8
#define IXL_TX_PKT_MAXSIZE (MCLBYTES * IXL_TX_PKT_DESCS)
#define IXL_TX_QUEUE_ALIGN 128
#define IXL_RX_QUEUE_ALIGN 128

#define IXL_MCLBYTES (MCLBYTES - ETHER_ALIGN)
#define IXL_MTU_ETHERLEN ETHER_HDR_LEN \
+ ETHER_CRC_LEN \
+ ETHER_VLAN_ENCAP_LEN
#if 0
#define IXL_MAX_MTU (9728 - IXL_MTU_ETHERLEN)
#else
/* (dbuff * 5) - ETHER_HDR_LEN - ETHER_CRC_LEN */
#define IXL_MAX_MTU (9600 - IXL_MTU_ETHERLEN)
#endif
#define IXL_MIN_MTU (ETHER_MIN_LEN - ETHER_CRC_LEN)

#define IXL_PCIREG PCI_MAPREG_START

#define IXL_ITR0 0x0
#define IXL_ITR1 0x1
#define IXL_ITR2 0x2
#define IXL_NOITR 0x3

#define IXL_AQ_NUM 256
#define IXL_AQ_MASK (IXL_AQ_NUM - 1)
#define IXL_AQ_ALIGN 64 /* lol */
#define IXL_AQ_BUFLEN 4096

#define IXL_HMC_ROUNDUP 512
#define IXL_HMC_PGSIZE 4096
#define IXL_HMC_DVASZ sizeof(uint64_t)
#define IXL_HMC_PGS (IXL_HMC_PGSIZE / IXL_HMC_DVASZ)
#define IXL_HMC_L2SZ (IXL_HMC_PGSIZE * IXL_HMC_PGS)
#define IXL_HMC_PDVALID 1ULL

#define IXL_ATQ_EXEC_TIMEOUT (10 * hz)

#define IXL_SRRD_SRCTL_ATTEMPTS 100000

struct ixl_aq_regs {
bus_size_t atq_tail;
bus_size_t atq_head;
bus_size_t atq_len;
bus_size_t atq_bal;
bus_size_t atq_bah;

bus_size_t arq_tail;
bus_size_t arq_head;
bus_size_t arq_len;
bus_size_t arq_bal;
bus_size_t arq_bah;

uint32_t atq_len_enable;
uint32_t atq_tail_mask;
uint32_t atq_head_mask;

uint32_t arq_len_enable;
uint32_t arq_tail_mask;
uint32_t arq_head_mask;
};

struct ixl_phy_type {
uint64_t phy_type;
uint64_t ifm_type;
};

struct ixl_speed_type {
uint8_t dev_speed;
uint64_t net_speed;
};

struct ixl_hmc_entry {
uint64_t hmc_base;
uint32_t hmc_count;
uint64_t hmc_size;
};

enum ixl_hmc_types {
IXL_HMC_LAN_TX = 0,
IXL_HMC_LAN_RX,
IXL_HMC_FCOE_CTX,
IXL_HMC_FCOE_FILTER,
IXL_HMC_COUNT
};

struct ixl_hmc_pack {
uint16_t offset;
uint16_t width;
uint16_t lsb;
};

/*
* these hmc objects have weird sizes and alignments, so these are abstract
* representations of them that are nice for c to populate.
*
* the packing code relies on little-endian values being stored in the fields,
* no high bits in the fields being set, and the fields must be packed in the
* same order as they are in the ctx structure.
*/

struct ixl_hmc_rxq {
uint16_t head;
uint8_t cpuid;
uint64_t base;
#define IXL_HMC_RXQ_BASE_UNIT 128
uint16_t qlen;
uint16_t dbuff;
#define IXL_HMC_RXQ_DBUFF_UNIT 128
uint8_t hbuff;
#define IXL_HMC_RXQ_HBUFF_UNIT 64
uint8_t dtype;
#define IXL_HMC_RXQ_DTYPE_NOSPLIT 0x0
#define IXL_HMC_RXQ_DTYPE_HSPLIT 0x1
#define IXL_HMC_RXQ_DTYPE_SPLIT_ALWAYS 0x2
uint8_t dsize;
#define IXL_HMC_RXQ_DSIZE_16 0
#define IXL_HMC_RXQ_DSIZE_32 1
uint8_t crcstrip;
uint8_t fc_ena;
uint8_t l2sel;
uint8_t hsplit_0;
uint8_t hsplit_1;
uint8_t showiv;
uint16_t rxmax;
uint8_t tphrdesc_ena;
uint8_t tphwdesc_ena;
uint8_t tphdata_ena;
uint8_t tphhead_ena;
uint8_t lrxqthresh;
uint8_t prefena;
};

static const struct ixl_hmc_pack ixl_hmc_pack_rxq[] = {
{ offsetof(struct ixl_hmc_rxq, head), 13, 0 },
{ offsetof(struct ixl_hmc_rxq, cpuid), 8, 13 },
{ offsetof(struct ixl_hmc_rxq, base), 57, 32 },
{ offsetof(struct ixl_hmc_rxq, qlen), 13, 89 },
{ offsetof(struct ixl_hmc_rxq, dbuff), 7, 102 },
{ offsetof(struct ixl_hmc_rxq, hbuff), 5, 109 },
{ offsetof(struct ixl_hmc_rxq, dtype), 2, 114 },
{ offsetof(struct ixl_hmc_rxq, dsize), 1, 116 },
{ offsetof(struct ixl_hmc_rxq, crcstrip), 1, 117 },
{ offsetof(struct ixl_hmc_rxq, fc_ena), 1, 118 },
{ offsetof(struct ixl_hmc_rxq, l2sel), 1, 119 },
{ offsetof(struct ixl_hmc_rxq, hsplit_0), 4, 120 },
{ offsetof(struct ixl_hmc_rxq, hsplit_1), 2, 124 },
{ offsetof(struct ixl_hmc_rxq, showiv), 1, 127 },
{ offsetof(struct ixl_hmc_rxq, rxmax), 14, 174 },
{ offsetof(struct ixl_hmc_rxq, tphrdesc_ena), 1, 193 },
{ offsetof(struct ixl_hmc_rxq, tphwdesc_ena), 1, 194 },
{ offsetof(struct ixl_hmc_rxq, tphdata_ena), 1, 195 },
{ offsetof(struct ixl_hmc_rxq, tphhead_ena), 1, 196 },
{ offsetof(struct ixl_hmc_rxq, lrxqthresh), 3, 198 },
{ offsetof(struct ixl_hmc_rxq, prefena), 1, 201 },
};

#define IXL_HMC_RXQ_MINSIZE (201 + 1)

struct ixl_hmc_txq {
uint16_t head;
uint8_t new_context;
uint64_t base;
#define IXL_HMC_TXQ_BASE_UNIT 128
uint8_t fc_ena;
uint8_t timesync_ena;
uint8_t fd_ena;
uint8_t alt_vlan_ena;
uint8_t cpuid;
uint16_t thead_wb;
uint8_t head_wb_ena;
#define IXL_HMC_TXQ_DESC_WB 0
#define IXL_HMC_TXQ_HEAD_WB 1
uint16_t qlen;
uint8_t tphrdesc_ena;
uint8_t tphrpacket_ena;
uint8_t tphwdesc_ena;
uint64_t head_wb_addr;
uint32_t crc;
uint16_t rdylist;
uint8_t rdylist_act;
};

static const struct ixl_hmc_pack ixl_hmc_pack_txq[] = {
{ offsetof(struct ixl_hmc_txq, head), 13, 0 },
{ offsetof(struct ixl_hmc_txq, new_context), 1, 30 },
{ offsetof(struct ixl_hmc_txq, base), 57, 32 },
{ offsetof(struct ixl_hmc_txq, fc_ena), 1, 89 },
{ offsetof(struct ixl_hmc_txq, timesync_ena), 1, 90 },
{ offsetof(struct ixl_hmc_txq, fd_ena), 1, 91 },
{ offsetof(struct ixl_hmc_txq, alt_vlan_ena), 1, 92 },
{ offsetof(struct ixl_hmc_txq, cpuid), 8, 96 },
/* line 1 */
{ offsetof(struct ixl_hmc_txq, thead_wb), 13, 0 + 128 },
{ offsetof(struct ixl_hmc_txq, head_wb_ena), 1, 32 + 128 },
{ offsetof(struct ixl_hmc_txq, qlen), 13, 33 + 128 },
{ offsetof(struct ixl_hmc_txq, tphrdesc_ena), 1, 46 + 128 },
{ offsetof(struct ixl_hmc_txq, tphrpacket_ena), 1, 47 + 128 },
{ offsetof(struct ixl_hmc_txq, tphwdesc_ena), 1, 48 + 128 },
{ offsetof(struct ixl_hmc_txq, head_wb_addr), 64, 64 + 128 },
/* line 7 */
{ offsetof(struct ixl_hmc_txq, crc), 32, 0 + (7*128) },
{ offsetof(struct ixl_hmc_txq, rdylist), 10, 84 + (7*128) },
{ offsetof(struct ixl_hmc_txq, rdylist_act), 1, 94 + (7*128) },
};

#define IXL_HMC_TXQ_MINSIZE (94 + (7*128) + 1)

struct ixl_work {
struct work ixw_cookie;
void (*ixw_func)(void *);
void *ixw_arg;
unsigned int ixw_added;
};
#define IXL_WORKQUEUE_PRI PRI_SOFTNET

struct ixl_tx_map {
struct mbuf *txm_m;
bus_dmamap_t txm_map;
unsigned int txm_eop;
};

struct ixl_tx_ring {
kmutex_t txr_lock;
struct ixl_softc *txr_sc;

unsigned int txr_prod;
unsigned int txr_cons;

struct ixl_tx_map *txr_maps;
struct ixl_dmamem txr_mem;

bus_size_t txr_tail;
unsigned int txr_qid;
pcq_t *txr_intrq;
void *txr_si;

struct evcnt txr_defragged;
struct evcnt txr_defrag_failed;
struct evcnt txr_pcqdrop;
struct evcnt txr_transmitdef;
struct evcnt txr_intr;
struct evcnt txr_defer;
};

struct ixl_rx_map {
struct mbuf *rxm_m;
bus_dmamap_t rxm_map;
};

struct ixl_rx_ring {
kmutex_t rxr_lock;

unsigned int rxr_prod;
unsigned int rxr_cons;

struct ixl_rx_map *rxr_maps;
struct ixl_dmamem rxr_mem;

struct mbuf *rxr_m_head;
struct mbuf **rxr_m_tail;

bus_size_t rxr_tail;
unsigned int rxr_qid;

struct evcnt rxr_mgethdr_failed;
struct evcnt rxr_mgetcl_failed;
struct evcnt rxr_mbuf_load_failed;
struct evcnt rxr_intr;
struct evcnt rxr_defer;
};

struct ixl_queue_pair {
struct ixl_softc *qp_sc;
struct ixl_tx_ring *qp_txr;
struct ixl_rx_ring *qp_rxr;

char qp_name[16];

void *qp_si;
struct work qp_work;
bool qp_workqueue;
};

struct ixl_atq {
struct ixl_aq_desc iatq_desc;
void (*iatq_fn)(struct ixl_softc *,
const struct ixl_aq_desc *);
bool iatq_inuse;
};
SIMPLEQ_HEAD(ixl_atq_list, ixl_atq);

struct ixl_product {
unsigned int vendor_id;
unsigned int product_id;
};

struct ixl_stats_counters {
bool isc_has_offset;
struct evcnt isc_crc_errors;
uint64_t isc_crc_errors_offset;
struct evcnt isc_illegal_bytes;
uint64_t isc_illegal_bytes_offset;
struct evcnt isc_rx_bytes;
uint64_t isc_rx_bytes_offset;
struct evcnt isc_rx_discards;
uint64_t isc_rx_discards_offset;
struct evcnt isc_rx_unicast;
uint64_t isc_rx_unicast_offset;
struct evcnt isc_rx_multicast;
uint64_t isc_rx_multicast_offset;
struct evcnt isc_rx_broadcast;
uint64_t isc_rx_broadcast_offset;
struct evcnt isc_rx_size_64;
uint64_t isc_rx_size_64_offset;
struct evcnt isc_rx_size_127;
uint64_t isc_rx_size_127_offset;
struct evcnt isc_rx_size_255;
uint64_t isc_rx_size_255_offset;
struct evcnt isc_rx_size_511;
uint64_t isc_rx_size_511_offset;
struct evcnt isc_rx_size_1023;
uint64_t isc_rx_size_1023_offset;
struct evcnt isc_rx_size_1522;
uint64_t isc_rx_size_1522_offset;
struct evcnt isc_rx_size_big;
uint64_t isc_rx_size_big_offset;
struct evcnt isc_rx_undersize;
uint64_t isc_rx_undersize_offset;
struct evcnt isc_rx_oversize;
uint64_t isc_rx_oversize_offset;
struct evcnt isc_rx_fragments;
uint64_t isc_rx_fragments_offset;
struct evcnt isc_rx_jabber;
uint64_t isc_rx_jabber_offset;
struct evcnt isc_tx_bytes;
uint64_t isc_tx_bytes_offset;
struct evcnt isc_tx_dropped_link_down;
uint64_t isc_tx_dropped_link_down_offset;
struct evcnt isc_tx_unicast;
uint64_t isc_tx_unicast_offset;
struct evcnt isc_tx_multicast;
uint64_t isc_tx_multicast_offset;
struct evcnt isc_tx_broadcast;
uint64_t isc_tx_broadcast_offset;
struct evcnt isc_tx_size_64;
uint64_t isc_tx_size_64_offset;
struct evcnt isc_tx_size_127;
uint64_t isc_tx_size_127_offset;
struct evcnt isc_tx_size_255;
uint64_t isc_tx_size_255_offset;
struct evcnt isc_tx_size_511;
uint64_t isc_tx_size_511_offset;
struct evcnt isc_tx_size_1023;
uint64_t isc_tx_size_1023_offset;
struct evcnt isc_tx_size_1522;
uint64_t isc_tx_size_1522_offset;
struct evcnt isc_tx_size_big;
uint64_t isc_tx_size_big_offset;
struct evcnt isc_mac_local_faults;
uint64_t isc_mac_local_faults_offset;
struct evcnt isc_mac_remote_faults;
uint64_t isc_mac_remote_faults_offset;
struct evcnt isc_link_xon_rx;
uint64_t isc_link_xon_rx_offset;
struct evcnt isc_link_xon_tx;
uint64_t isc_link_xon_tx_offset;
struct evcnt isc_link_xoff_rx;
uint64_t isc_link_xoff_rx_offset;
struct evcnt isc_link_xoff_tx;
uint64_t isc_link_xoff_tx_offset;
struct evcnt isc_vsi_rx_discards;
uint64_t isc_vsi_rx_discards_offset;
struct evcnt isc_vsi_rx_bytes;
uint64_t isc_vsi_rx_bytes_offset;
struct evcnt isc_vsi_rx_unicast;
uint64_t isc_vsi_rx_unicast_offset;
struct evcnt isc_vsi_rx_multicast;
uint64_t isc_vsi_rx_multicast_offset;
struct evcnt isc_vsi_rx_broadcast;
uint64_t isc_vsi_rx_broadcast_offset;
struct evcnt isc_vsi_tx_errors;
uint64_t isc_vsi_tx_errors_offset;
struct evcnt isc_vsi_tx_bytes;
uint64_t isc_vsi_tx_bytes_offset;
struct evcnt isc_vsi_tx_unicast;
uint64_t isc_vsi_tx_unicast_offset;
struct evcnt isc_vsi_tx_multicast;
uint64_t isc_vsi_tx_multicast_offset;
struct evcnt isc_vsi_tx_broadcast;
uint64_t isc_vsi_tx_broadcast_offset;
};

/*
* Locking notes:
* + a field in ixl_tx_ring is protected by txr_lock (a spin mutex), and
* a field in ixl_rx_ring is protected by rxr_lock (a spin mutex).
* - more than one lock of them cannot be held at once.
* + a field named sc_atq_* in ixl_softc is protected by sc_atq_lock
* (a spin mutex).
* - the lock cannot held with txr_lock or rxr_lock.
* + a field named sc_arq_* is not protected by any lock.
* - operations for sc_arq_* is done in one context related to
* sc_arq_task.
* + other fields in ixl_softc is protected by sc_cfg_lock
* (an adaptive mutex)
* - It must be held before another lock is held, and It can be
* released after the other lock is released.
* */

struct ixl_softc {
device_t sc_dev;
struct ethercom sc_ec;
bool sc_attached;
bool sc_dead;
uint32_t sc_port;
struct sysctllog *sc_sysctllog;
struct workqueue *sc_workq;
struct workqueue *sc_workq_txrx;
int sc_stats_intval;
callout_t sc_stats_callout;
struct ixl_work sc_stats_task;
struct ixl_stats_counters
sc_stats_counters;
uint8_t sc_enaddr[ETHER_ADDR_LEN];
struct ifmedia sc_media;
uint64_t sc_media_status;
uint64_t sc_media_active;
uint64_t sc_phy_types;
uint8_t sc_phy_abilities;
uint8_t sc_phy_linkspeed;
uint8_t sc_phy_fec_cfg;
uint16_t sc_eee_cap;
uint32_t sc_eeer_val;
uint8_t sc_d3_lpan;
kmutex_t sc_cfg_lock;
enum i40e_mac_type sc_mac_type;
uint32_t sc_rss_table_size;
uint32_t sc_rss_table_entry_width;
bool sc_txrx_workqueue;
u_int sc_tx_process_limit;
u_int sc_rx_process_limit;
u_int sc_tx_intr_process_limit;
u_int sc_rx_intr_process_limit;

int sc_cur_ec_capenable;

struct pci_attach_args sc_pa;
pci_intr_handle_t *sc_ihp;
void **sc_ihs;
unsigned int sc_nintrs;

bus_dma_tag_t sc_dmat;
bus_space_tag_t sc_memt;
bus_space_handle_t sc_memh;
bus_size_t sc_mems;

uint8_t sc_pf_id;
uint16_t sc_uplink_seid; /* le */
uint16_t sc_downlink_seid; /* le */
uint16_t sc_vsi_number;
uint16_t sc_vsi_stat_counter_idx;
uint16_t sc_seid;
unsigned int sc_base_queue;

pci_intr_type_t sc_intrtype;
unsigned int sc_msix_vector_queue;

struct ixl_dmamem sc_scratch;
struct ixl_dmamem sc_aqbuf;

const struct ixl_aq_regs *
sc_aq_regs;
uint32_t sc_aq_flags;
#define IXL_SC_AQ_FLAG_RXCTL __BIT(0)
#define IXL_SC_AQ_FLAG_NVMLOCK __BIT(1)
#define IXL_SC_AQ_FLAG_NVMREAD __BIT(2)
#define IXL_SC_AQ_FLAG_RSS __BIT(3)

kmutex_t sc_atq_lock;
kcondvar_t sc_atq_cv;
struct ixl_dmamem sc_atq;
unsigned int sc_atq_prod;
unsigned int sc_atq_cons;

struct ixl_dmamem sc_arq;
struct ixl_work sc_arq_task;
struct ixl_aq_bufs sc_arq_idle;
struct ixl_aq_buf *sc_arq_live[IXL_AQ_NUM];
unsigned int sc_arq_prod;
unsigned int sc_arq_cons;

struct ixl_work sc_link_state_task;
struct ixl_work sc_link_state_done_task;
struct ixl_atq sc_link_state_atq;

struct ixl_dmamem sc_hmc_sd;
struct ixl_dmamem sc_hmc_pd;
struct ixl_hmc_entry sc_hmc_entries[IXL_HMC_COUNT];

struct if_percpuq *sc_ipq;
unsigned int sc_tx_ring_ndescs;
unsigned int sc_rx_ring_ndescs;
unsigned int sc_nqueue_pairs;
unsigned int sc_nqueue_pairs_max;
unsigned int sc_nqueue_pairs_device;
struct ixl_queue_pair *sc_qps;
uint32_t sc_itr_rx;
uint32_t sc_itr_tx;

struct evcnt sc_event_atq;
struct evcnt sc_event_link;
struct evcnt sc_event_ecc_err;
struct evcnt sc_event_pci_exception;
struct evcnt sc_event_crit_err;
};

#define IXL_TXRX_PROCESS_UNLIMIT UINT_MAX
#define IXL_TX_PROCESS_LIMIT 256
#define IXL_RX_PROCESS_LIMIT 256
#define IXL_TX_INTR_PROCESS_LIMIT 256
#define IXL_RX_INTR_PROCESS_LIMIT 0U

#define IXL_IFCAP_RXCSUM (IFCAP_CSUM_IPv4_Rx | \
IFCAP_CSUM_TCPv4_Rx | \
IFCAP_CSUM_UDPv4_Rx | \
IFCAP_CSUM_TCPv6_Rx | \
IFCAP_CSUM_UDPv6_Rx)
#define IXL_IFCAP_TXCSUM (IFCAP_CSUM_IPv4_Tx | \
IFCAP_CSUM_TCPv4_Tx | \
IFCAP_CSUM_UDPv4_Tx | \
IFCAP_CSUM_TCPv6_Tx | \
IFCAP_CSUM_UDPv6_Tx)
#define IXL_CSUM_ALL_OFFLOAD (M_CSUM_IPv4 | \
M_CSUM_TCPv4 | M_CSUM_TCPv6 | \
M_CSUM_UDPv4 | M_CSUM_UDPv6)

#define delaymsec(_x) DELAY(1000 * (_x))
#ifdef IXL_DEBUG
#define DDPRINTF(sc, fmt, args...) \
do { \
if ((sc) != NULL) { \
device_printf( \
((struct ixl_softc *)(sc))->sc_dev, \
""); \
} \
printf("%s:\t" fmt, __func__, ##args); \
} while (0)
#else
#define DDPRINTF(sc, fmt, args...) __nothing
#endif
#ifndef IXL_STATS_INTERVAL_MSEC
#define IXL_STATS_INTERVAL_MSEC 10000
#endif
#ifndef IXL_QUEUE_NUM
#define IXL_QUEUE_NUM 0
#endif

enum ixl_link_flags {
IXL_LINK_NOFLAGS = 0,
IXL_LINK_FLAG_WAITDONE = __BIT(0),
};

static bool ixl_param_nomsix = false;
static int ixl_param_stats_interval = IXL_STATS_INTERVAL_MSEC;
static int ixl_param_nqps_limit = IXL_QUEUE_NUM;
static unsigned int ixl_param_tx_ndescs = 512;
static unsigned int ixl_param_rx_ndescs = 512;

static enum i40e_mac_type
ixl_mactype(pci_product_id_t);
static void ixl_pci_csr_setup(pci_chipset_tag_t, pcitag_t);
static void ixl_clear_hw(struct ixl_softc *);
static int ixl_pf_reset(struct ixl_softc *);

static int ixl_dmamem_alloc(struct ixl_softc *, struct ixl_dmamem *,
bus_size_t, bus_size_t);
static void ixl_dmamem_free(struct ixl_softc *, struct ixl_dmamem *);

static int ixl_arq_fill(struct ixl_softc *);
static void ixl_arq_unfill(struct ixl_softc *);

static int ixl_atq_poll(struct ixl_softc *, struct ixl_aq_desc *,
unsigned int);
static void ixl_atq_set(struct ixl_atq *,
void (*)(struct ixl_softc *, const struct ixl_aq_desc *));
static void ixl_wakeup(struct ixl_softc *, const struct ixl_aq_desc *);
static int ixl_atq_post_locked(struct ixl_softc *, struct ixl_atq *);
static void ixl_atq_done(struct ixl_softc *);
static int ixl_atq_exec(struct ixl_softc *, struct ixl_atq *);
static int ixl_atq_exec_locked(struct ixl_softc *, struct ixl_atq *);
static int ixl_get_version(struct ixl_softc *);
static int ixl_get_nvm_version(struct ixl_softc *);
static int ixl_get_hw_capabilities(struct ixl_softc *);
static int ixl_pxe_clear(struct ixl_softc *);
static int ixl_lldp_shut(struct ixl_softc *);
static int ixl_get_mac(struct ixl_softc *);
static int ixl_get_switch_config(struct ixl_softc *);
static int ixl_phy_mask_ints(struct ixl_softc *);
static int ixl_get_phy_info(struct ixl_softc *);
static int ixl_set_phy_config(struct ixl_softc *, uint8_t, uint8_t, bool);
static int ixl_set_phy_autoselect(struct ixl_softc *);
static int ixl_restart_an(struct ixl_softc *);
static int ixl_hmc(struct ixl_softc *);
static void ixl_hmc_free(struct ixl_softc *);
static int ixl_get_vsi(struct ixl_softc *);
static int ixl_set_vsi(struct ixl_softc *);
static void ixl_set_filter_control(struct ixl_softc *);
static int ixl_get_link_status(struct ixl_softc *, enum ixl_link_flags);
static void ixl_get_link_status_work(void *);
static int ixl_get_link_status_poll(struct ixl_softc *, int *);
static void ixl_get_link_status_done(struct ixl_softc *,
const struct ixl_aq_desc *);
static void ixl_get_link_status_done_work(void *);
static int ixl_set_link_status_locked(struct ixl_softc *,
const struct ixl_aq_desc *);
static uint64_t ixl_search_link_speed(uint8_t);
static uint8_t ixl_search_baudrate(uint64_t);
static void ixl_config_rss(struct ixl_softc *);
static int ixl_add_macvlan(struct ixl_softc *, const uint8_t *,
uint16_t, uint16_t);
static int ixl_remove_macvlan(struct ixl_softc *, const uint8_t *,
uint16_t, uint16_t);
static void ixl_arq(void *);
static void ixl_hmc_pack(void *, const void *,
const struct ixl_hmc_pack *, unsigned int);
static uint32_t ixl_rd_rx_csr(struct ixl_softc *, uint32_t);
static void ixl_wr_rx_csr(struct ixl_softc *, uint32_t, uint32_t);
static int ixl_rd16_nvm(struct ixl_softc *, uint16_t, uint16_t *);

static int ixl_match(device_t, cfdata_t, void *);
static void ixl_attach(device_t, device_t, void *);
static int ixl_detach(device_t, int);

static void ixl_media_add(struct ixl_softc *);
static int ixl_media_change(struct ifnet *);
static void ixl_media_status(struct ifnet *, struct ifmediareq *);
static int ixl_ioctl(struct ifnet *, u_long, void *);
static void ixl_start(struct ifnet *);
static int ixl_transmit(struct ifnet *, struct mbuf *);
static void ixl_deferred_transmit(void *);
static int ixl_intr(void *);
static int ixl_queue_intr(void *);
static int ixl_other_intr(void *);
static void ixl_handle_queue(void *);
static void ixl_handle_queue_wk(struct work *, void *);
static void ixl_sched_handle_queue(struct ixl_softc *,
struct ixl_queue_pair *);
static int ixl_init(struct ifnet *);
static int ixl_init_locked(struct ixl_softc *);
static void ixl_stop(struct ifnet *, int);
static void ixl_stop_locked(struct ixl_softc *);
static int ixl_iff(struct ixl_softc *);
static int ixl_ifflags_cb(struct ethercom *);
static int ixl_setup_interrupts(struct ixl_softc *);
static int ixl_establish_intx(struct ixl_softc *);
static int ixl_establish_msix(struct ixl_softc *);
static void ixl_enable_queue_intr(struct ixl_softc *,
struct ixl_queue_pair *);
static void ixl_disable_queue_intr(struct ixl_softc *,
struct ixl_queue_pair *);
static void ixl_enable_other_intr(struct ixl_softc *);
static void ixl_disable_other_intr(struct ixl_softc *);
static void ixl_config_queue_intr(struct ixl_softc *);
static void ixl_config_other_intr(struct ixl_softc *);

static struct ixl_tx_ring *
ixl_txr_alloc(struct ixl_softc *, unsigned int);
static void ixl_txr_qdis(struct ixl_softc *, struct ixl_tx_ring *, int);
static void ixl_txr_config(struct ixl_softc *, struct ixl_tx_ring *);
static int ixl_txr_enabled(struct ixl_softc *, struct ixl_tx_ring *);
static int ixl_txr_disabled(struct ixl_softc *, struct ixl_tx_ring *);
static void ixl_txr_unconfig(struct ixl_softc *, struct ixl_tx_ring *);
static void ixl_txr_clean(struct ixl_softc *, struct ixl_tx_ring *);
static void ixl_txr_free(struct ixl_softc *, struct ixl_tx_ring *);
static int ixl_txeof(struct ixl_softc *, struct ixl_tx_ring *, u_int);

static struct ixl_rx_ring *
ixl_rxr_alloc(struct ixl_softc *, unsigned int);
static void ixl_rxr_config(struct ixl_softc *, struct ixl_rx_ring *);
static int ixl_rxr_enabled(struct ixl_softc *, struct ixl_rx_ring *);
static int ixl_rxr_disabled(struct ixl_softc *, struct ixl_rx_ring *);
static void ixl_rxr_unconfig(struct ixl_softc *, struct ixl_rx_ring *);
static void ixl_rxr_clean(struct ixl_softc *, struct ixl_rx_ring *);
static void ixl_rxr_free(struct ixl_softc *, struct ixl_rx_ring *);
static int ixl_rxeof(struct ixl_softc *, struct ixl_rx_ring *, u_int);
static int ixl_rxfill(struct ixl_softc *, struct ixl_rx_ring *);

static struct workqueue *
ixl_workq_create(const char *, pri_t, int, int);
static void ixl_workq_destroy(struct workqueue *);
static int ixl_workqs_teardown(device_t);
static void ixl_work_set(struct ixl_work *, void (*)(void *), void *);
static void ixl_work_add(struct workqueue *, struct ixl_work *);
static void ixl_work_wait(struct workqueue *, struct ixl_work *);
static void ixl_workq_work(struct work *, void *);
static const struct ixl_product *
ixl_lookup(const struct pci_attach_args *pa);
static void ixl_link_state_update(struct ixl_softc *,
const struct ixl_aq_desc *);
static int ixl_vlan_cb(struct ethercom *, uint16_t, bool);
static int ixl_setup_vlan_hwfilter(struct ixl_softc *);
static void ixl_teardown_vlan_hwfilter(struct ixl_softc *);
static int ixl_update_macvlan(struct ixl_softc *);
static int ixl_setup_interrupts(struct ixl_softc *);
static void ixl_teardown_interrupts(struct ixl_softc *);
static int ixl_setup_stats(struct ixl_softc *);
static void ixl_teardown_stats(struct ixl_softc *);
static void ixl_stats_callout(void *);
static void ixl_stats_update(void *);
static int ixl_setup_sysctls(struct ixl_softc *);
static void ixl_teardown_sysctls(struct ixl_softc *);
static int ixl_sysctl_itr_handler(SYSCTLFN_PROTO);
static int ixl_queue_pairs_alloc(struct ixl_softc *);
static void ixl_queue_pairs_free(struct ixl_softc *);

static const struct ixl_phy_type ixl_phy_type_map[] = {
{ 1ULL << IXL_PHY_TYPE_SGMII, IFM_1000_SGMII },
{ 1ULL << IXL_PHY_TYPE_1000BASE_KX, IFM_1000_KX },
{ 1ULL << IXL_PHY_TYPE_10GBASE_KX4, IFM_10G_KX4 },
{ 1ULL << IXL_PHY_TYPE_10GBASE_KR, IFM_10G_KR },
{ 1ULL << IXL_PHY_TYPE_40GBASE_KR4, IFM_40G_KR4 },
{ 1ULL << IXL_PHY_TYPE_XAUI |
1ULL << IXL_PHY_TYPE_XFI, IFM_10G_CX4 },
{ 1ULL << IXL_PHY_TYPE_SFI, IFM_10G_SFI },
{ 1ULL << IXL_PHY_TYPE_XLAUI |
1ULL << IXL_PHY_TYPE_XLPPI, IFM_40G_XLPPI },
{ 1ULL << IXL_PHY_TYPE_40GBASE_CR4_CU |
1ULL << IXL_PHY_TYPE_40GBASE_CR4, IFM_40G_CR4 },
{ 1ULL << IXL_PHY_TYPE_10GBASE_CR1_CU |
1ULL << IXL_PHY_TYPE_10GBASE_CR1, IFM_10G_CR1 },
{ 1ULL << IXL_PHY_TYPE_10GBASE_AOC, IFM_10G_AOC },
{ 1ULL << IXL_PHY_TYPE_40GBASE_AOC, IFM_40G_AOC },
{ 1ULL << IXL_PHY_TYPE_100BASE_TX, IFM_100_TX },
{ 1ULL << IXL_PHY_TYPE_1000BASE_T_OPTICAL |
1ULL << IXL_PHY_TYPE_1000BASE_T, IFM_1000_T },
{ 1ULL << IXL_PHY_TYPE_10GBASE_T, IFM_10G_T },
{ 1ULL << IXL_PHY_TYPE_10GBASE_SR, IFM_10G_SR },
{ 1ULL << IXL_PHY_TYPE_10GBASE_LR, IFM_10G_LR },
{ 1ULL << IXL_PHY_TYPE_10GBASE_SFPP_CU, IFM_10G_TWINAX },
{ 1ULL << IXL_PHY_TYPE_40GBASE_SR4, IFM_40G_SR4 },
{ 1ULL << IXL_PHY_TYPE_40GBASE_LR4, IFM_40G_LR4 },
{ 1ULL << IXL_PHY_TYPE_1000BASE_SX, IFM_1000_SX },
{ 1ULL << IXL_PHY_TYPE_1000BASE_LX, IFM_1000_LX },
{ 1ULL << IXL_PHY_TYPE_20GBASE_KR2, IFM_20G_KR2 },
{ 1ULL << IXL_PHY_TYPE_25GBASE_KR, IFM_25G_KR },
{ 1ULL << IXL_PHY_TYPE_25GBASE_CR, IFM_25G_CR },
{ 1ULL << IXL_PHY_TYPE_25GBASE_SR, IFM_25G_SR },
{ 1ULL << IXL_PHY_TYPE_25GBASE_LR, IFM_25G_LR },
{ 1ULL << IXL_PHY_TYPE_25GBASE_AOC, IFM_25G_AOC },
{ 1ULL << IXL_PHY_TYPE_25GBASE_ACC, IFM_25G_ACC },
{ 1ULL << IXL_PHY_TYPE_2500BASE_T_1, IFM_2500_T },
{ 1ULL << IXL_PHY_TYPE_5000BASE_T_1, IFM_5000_T },
{ 1ULL << IXL_PHY_TYPE_2500BASE_T_2, IFM_2500_T },
{ 1ULL << IXL_PHY_TYPE_5000BASE_T_2, IFM_5000_T },
};

static const struct ixl_speed_type ixl_speed_type_map[] = {
{ IXL_AQ_LINK_SPEED_40GB, IF_Gbps(40) },
{ IXL_AQ_LINK_SPEED_25GB, IF_Gbps(25) },
{ IXL_AQ_LINK_SPEED_10GB, IF_Gbps(10) },
{ IXL_AQ_LINK_SPEED_5000MB, IF_Mbps(5000) },
{ IXL_AQ_LINK_SPEED_2500MB, IF_Mbps(2500) },
{ IXL_AQ_LINK_SPEED_1000MB, IF_Mbps(1000) },
{ IXL_AQ_LINK_SPEED_100MB, IF_Mbps(100)},
};

static const struct ixl_aq_regs ixl_pf_aq_regs = {
.atq_tail = I40E_PF_ATQT,
.atq_tail_mask = I40E_PF_ATQT_ATQT_MASK,
.atq_head = I40E_PF_ATQH,
.atq_head_mask = I40E_PF_ATQH_ATQH_MASK,
.atq_len = I40E_PF_ATQLEN,
.atq_bal = I40E_PF_ATQBAL,
.atq_bah = I40E_PF_ATQBAH,
.atq_len_enable = I40E_PF_ATQLEN_ATQENABLE_MASK,

.arq_tail = I40E_PF_ARQT,
.arq_tail_mask = I40E_PF_ARQT_ARQT_MASK,
.arq_head = I40E_PF_ARQH,
.arq_head_mask = I40E_PF_ARQH_ARQH_MASK,
.arq_len = I40E_PF_ARQLEN,
.arq_bal = I40E_PF_ARQBAL,
.arq_bah = I40E_PF_ARQBAH,
.arq_len_enable = I40E_PF_ARQLEN_ARQENABLE_MASK,
};

#define ixl_rd(_s, _r) \
bus_space_read_4((_s)->sc_memt, (_s)->sc_memh, (_r))
#define ixl_wr(_s, _r, _v) \
bus_space_write_4((_s)->sc_memt, (_s)->sc_memh, (_r), (_v))
#define ixl_barrier(_s, _r, _l, _o) \
bus_space_barrier((_s)->sc_memt, (_s)->sc_memh, (_r), (_l), (_o))
#define ixl_flush(_s) (void)ixl_rd((_s), I40E_GLGEN_STAT)
#define ixl_nqueues(_sc) (1 << ((_sc)->sc_nqueue_pairs - 1))

CFATTACH_DECL3_NEW(ixl, sizeof(struct ixl_softc),
ixl_match, ixl_attach, ixl_detach, NULL, NULL, NULL,
DVF_DETACH_SHUTDOWN);

static const struct ixl_product ixl_products[] = {
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_SFP },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_B },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_KX_C },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_A },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_B },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_QSFP_C },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_1 },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_T_2 },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_1 },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XL710_20G_BP_2 },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_T4_10G },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_BP },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_XXV710_25G_SFP28 },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_KX },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_QSFP },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_SFP },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_1G_BASET },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_10G_BASET },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X722_I_SFP },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_SFP },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_X710_10G_BP },
{ PCI_VENDOR_INTEL, PCI_PRODUCT_INTEL_V710_5G_T},
/* required last entry */
{0, 0}
};

static const struct ixl_product *
ixl_lookup(const struct pci_attach_args *pa)
{
const struct ixl_product *ixlp;

for (ixlp = ixl_products; ixlp->vendor_id != 0; ixlp++) {
if (PCI_VENDOR(pa->pa_id) == ixlp->vendor_id &&
PCI_PRODUCT(pa->pa_id) == ixlp->product_id)
return ixlp;
}

return NULL;
}

static void
ixl_intr_barrier(void)
{

/* wait for finish of all handler */
xc_barrier(0);
}

static int
ixl_match(device_t parent, cfdata_t match, void *aux)
{
const struct pci_attach_args *pa = aux;

return (ixl_lookup(pa) != NULL) ? 1 : 0;
}

static void
ixl_attach(device_t parent, device_t self, void *aux)
{
struct ixl_softc *sc;
struct pci_attach_args *pa = aux;
struct ifnet *ifp;
pcireg_t memtype;
uint32_t firstq, port, ari, func;
char xnamebuf[32];
int tries, rv, link;

sc = device_private(self);
sc->sc_dev = self;
ifp = &sc->sc_ec.ec_if;

sc->sc_pa = *pa;
sc->sc_dmat = (pci_dma64_available(pa)) ?
pa->pa_dmat64 : pa->pa_dmat;
sc->sc_aq_regs = &ixl_pf_aq_regs;

sc->sc_mac_type = ixl_mactype(PCI_PRODUCT(pa->pa_id));

ixl_pci_csr_setup(pa->pa_pc, pa->pa_tag);

pci_aprint_devinfo(pa, "Ethernet controller");

memtype = pci_mapreg_type(pa->pa_pc, pa->pa_tag, IXL_PCIREG);
if (pci_mapreg_map(pa, IXL_PCIREG, memtype, 0,
&sc->sc_memt, &sc->sc_memh, NULL, &sc->sc_mems)) {
aprint_error(": unable to map registers\n");
return;
}

mutex_init(&sc->sc_cfg_lock, MUTEX_DEFAULT, IPL_SOFTNET);

firstq = ixl_rd(sc, I40E_PFLAN_QALLOC);
firstq &= I40E_PFLAN_QALLOC_FIRSTQ_MASK;
firstq >>= I40E_PFLAN_QALLOC_FIRSTQ_SHIFT;
sc->sc_base_queue = firstq;

ixl_clear_hw(sc);
if (ixl_pf_reset(sc) == -1) {
/* error printed by ixl pf_reset */
goto unmap;
}

port = ixl_rd(sc, I40E_PFGEN_PORTNUM);
port &= I40E_PFGEN_PORTNUM_PORT_NUM_MASK;
port >>= I40E_PFGEN_PORTNUM_PORT_NUM_SHIFT;
sc->sc_port = port;
aprint_normal_dev(self, "port %u", sc->sc_port);

ari = ixl_rd(sc, I40E_GLPCI_CAPSUP);
ari &= I40E_GLPCI_CAPSUP_ARI_EN_MASK;
ari >>= I40E_GLPCI_CAPSUP_ARI_EN_SHIFT;

func = ixl_rd(sc, I40E_PF_FUNC_RID);
sc->sc_pf_id = func & (ari ? 0xff : 0x7);

/* initialise the adminq */

mutex_init(&sc->sc_atq_lock, MUTEX_DEFAULT, IPL_NET);

if (ixl_dmamem_alloc(sc, &sc->sc_atq,
sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) {
aprint_error("\n" "%s: unable to allocate atq\n",
device_xname(self));
goto unmap;
}

SIMPLEQ_INIT(&sc->sc_arq_idle);
ixl_work_set(&sc->sc_arq_task, ixl_arq, sc);
sc->sc_arq_cons = 0;
sc->sc_arq_prod = 0;

if (ixl_dmamem_alloc(sc, &sc->sc_arq,
sizeof(struct ixl_aq_desc) * IXL_AQ_NUM, IXL_AQ_ALIGN) != 0) {
aprint_error("\n" "%s: unable to allocate arq\n",
device_xname(self));
goto free_atq;
}

if (!ixl_arq_fill(sc)) {
aprint_error("\n" "%s: unable to fill arq descriptors\n",
device_xname(self));
goto free_arq;
}

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq),
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq),
0, IXL_DMA_LEN(&sc->sc_arq),
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);

for (tries = 0; tries < 10; tries++) {
sc->sc_atq_cons = 0;
sc->sc_atq_prod = 0;

ixl_wr(sc, sc->sc_aq_regs->atq_head, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_head, 0);
ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0);

ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE);

ixl_wr(sc, sc->sc_aq_regs->atq_bal,
ixl_dmamem_lo(&sc->sc_atq));
ixl_wr(sc, sc->sc_aq_regs->atq_bah,
ixl_dmamem_hi(&sc->sc_atq));
ixl_wr(sc, sc->sc_aq_regs->atq_len,
sc->sc_aq_regs->atq_len_enable | IXL_AQ_NUM);

ixl_wr(sc, sc->sc_aq_regs->arq_bal,
ixl_dmamem_lo(&sc->sc_arq));
ixl_wr(sc, sc->sc_aq_regs->arq_bah,
ixl_dmamem_hi(&sc->sc_arq));
ixl_wr(sc, sc->sc_aq_regs->arq_len,
sc->sc_aq_regs->arq_len_enable | IXL_AQ_NUM);

rv = ixl_get_version(sc);
if (rv == 0)
break;
if (rv != ETIMEDOUT) {
aprint_error(", unable to get firmware version\n");
goto shutdown;
}

delaymsec(100);
}

ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod);

if (ixl_dmamem_alloc(sc, &sc->sc_aqbuf, IXL_AQ_BUFLEN, 0) != 0) {
aprint_error_dev(self, ", unable to allocate nvm buffer\n");
goto shutdown;
}

ixl_get_nvm_version(sc);

if (sc->sc_mac_type == I40E_MAC_X722)
sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_X722;
else
sc->sc_nqueue_pairs_device = IXL_QUEUE_MAX_XL710;

rv = ixl_get_hw_capabilities(sc);
if (rv != 0) {
aprint_error(", GET HW CAPABILITIES %s\n",
rv == ETIMEDOUT ? "timeout" : "error");
goto free_aqbuf;
}

sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_device, ncpu);
if (ixl_param_nqps_limit > 0) {
sc->sc_nqueue_pairs_max = MIN((int)sc->sc_nqueue_pairs_max,
ixl_param_nqps_limit);
}

sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max;
sc->sc_tx_ring_ndescs = ixl_param_tx_ndescs;
sc->sc_rx_ring_ndescs = ixl_param_rx_ndescs;

KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_rx_ring_ndescs);
KASSERT(IXL_TXRX_PROCESS_UNLIMIT > sc->sc_tx_ring_ndescs);
KASSERT(sc->sc_rx_ring_ndescs ==
(1U << (fls32(sc->sc_rx_ring_ndescs) - 1)));
KASSERT(sc->sc_tx_ring_ndescs ==
(1U << (fls32(sc->sc_tx_ring_ndescs) - 1)));

if (ixl_get_mac(sc) != 0) {
/* error printed by ixl_get_mac */
goto free_aqbuf;
}

aprint_normal("\n");
aprint_naive("\n");

aprint_normal_dev(self, "Ethernet address %s\n",
ether_sprintf(sc->sc_enaddr));

rv = ixl_pxe_clear(sc);
if (rv != 0) {
aprint_debug_dev(self, "CLEAR PXE MODE %s\n",
rv == ETIMEDOUT ? "timeout" : "error");
}

ixl_set_filter_control(sc);

if (ixl_hmc(sc) != 0) {
/* error printed by ixl_hmc */
goto free_aqbuf;
}

if (ixl_lldp_shut(sc) != 0) {
/* error printed by ixl_lldp_shut */
goto free_hmc;
}

if (ixl_phy_mask_ints(sc) != 0) {
/* error printed by ixl_phy_mask_ints */
goto free_hmc;
}

if (ixl_restart_an(sc) != 0) {
/* error printed by ixl_restart_an */
goto free_hmc;
}

if (ixl_get_switch_config(sc) != 0) {
/* error printed by ixl_get_switch_config */
goto free_hmc;
}

rv = ixl_get_link_status_poll(sc, NULL);
if (rv != 0) {
aprint_error_dev(self, "GET LINK STATUS %s\n",
rv == ETIMEDOUT ? "timeout" : "error");
goto free_hmc;
}

/*
* The FW often returns EIO in "Get PHY Abilities" command
* if there is no delay
*/
DELAY(500);
if (ixl_get_phy_info(sc) != 0) {
/* error printed by ixl_get_phy_info */
goto free_hmc;
}

if (ixl_dmamem_alloc(sc, &sc->sc_scratch,
sizeof(struct ixl_aq_vsi_data), 8) != 0) {
aprint_error_dev(self, "unable to allocate scratch buffer\n");
goto free_hmc;
}

rv = ixl_get_vsi(sc);
if (rv != 0) {
aprint_error_dev(self, "GET VSI %s %d\n",
rv == ETIMEDOUT ? "timeout" : "error", rv);
goto free_scratch;
}

rv = ixl_set_vsi(sc);
if (rv != 0) {
aprint_error_dev(self, "UPDATE VSI error %s %d\n",
rv == ETIMEDOUT ? "timeout" : "error", rv);
goto free_scratch;
}

if (ixl_queue_pairs_alloc(sc) != 0) {
/* error printed by ixl_queue_pairs_alloc */
goto free_scratch;
}

if (ixl_setup_interrupts(sc) != 0) {
/* error printed by ixl_setup_interrupts */
goto free_queue_pairs;
}

if (ixl_setup_stats(sc) != 0) {
aprint_error_dev(self, "failed to setup event counters\n");
goto teardown_intrs;
}

if (ixl_setup_sysctls(sc) != 0) {
/* error printed by ixl_setup_sysctls */
goto teardown_stats;
}

snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_cfg", device_xname(self));
sc->sc_workq = ixl_workq_create(xnamebuf, IXL_WORKQUEUE_PRI,
IPL_NET, WQ_MPSAFE);
if (sc->sc_workq == NULL)
goto teardown_sysctls;

snprintf(xnamebuf, sizeof(xnamebuf), "%s_wq_txrx", device_xname(self));
rv = workqueue_create(&sc->sc_workq_txrx, xnamebuf, ixl_handle_queue_wk,
sc, IXL_WORKQUEUE_PRI, IPL_NET, WQ_PERCPU | WQ_MPSAFE);
if (rv != 0) {
sc->sc_workq_txrx = NULL;
goto teardown_wqs;
}

snprintf(xnamebuf, sizeof(xnamebuf), "%s_atq_cv", device_xname(self));
cv_init(&sc->sc_atq_cv, xnamebuf);

strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);

ifp->if_softc = sc;
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
ifp->if_extflags = IFEF_MPSAFE;
ifp->if_ioctl = ixl_ioctl;
ifp->if_start = ixl_start;
ifp->if_transmit = ixl_transmit;
ifp->if_init = ixl_init;
ifp->if_stop = ixl_stop;
IFQ_SET_MAXLEN(&ifp->if_snd, sc->sc_tx_ring_ndescs);
IFQ_SET_READY(&ifp->if_snd);
ifp->if_capabilities |= IXL_IFCAP_RXCSUM;
ifp->if_capabilities |= IXL_IFCAP_TXCSUM;
#if 0
ifp->if_capabilities |= IFCAP_TSOv4 | IFCAP_TSOv6;
#endif
ether_set_vlan_cb(&sc->sc_ec, ixl_vlan_cb);
sc->sc_ec.ec_capabilities |= ETHERCAP_JUMBO_MTU;
sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_MTU;
sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWTAGGING;
sc->sc_ec.ec_capabilities |= ETHERCAP_VLAN_HWFILTER;

sc->sc_ec.ec_capenable = sc->sc_ec.ec_capabilities;
/* Disable VLAN_HWFILTER by default */
CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER);

sc->sc_cur_ec_capenable = sc->sc_ec.ec_capenable;

sc->sc_ec.ec_ifmedia = &sc->sc_media;
ifmedia_init_with_lock(&sc->sc_media, IFM_IMASK, ixl_media_change,
ixl_media_status, &sc->sc_cfg_lock);

ixl_media_add(sc);
ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_AUTO, 0, NULL);
if (ISSET(sc->sc_phy_abilities,
(IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) {
ifmedia_add(&sc->sc_media,
IFM_ETHER | IFM_AUTO | IFM_FLOW, 0, NULL);
}
ifmedia_add(&sc->sc_media, IFM_ETHER | IFM_NONE, 0, NULL);
ifmedia_set(&sc->sc_media, IFM_ETHER | IFM_AUTO);

if_initialize(ifp);

sc->sc_ipq = if_percpuq_create(ifp);
if_deferred_start_init(ifp, NULL);
ether_ifattach(ifp, sc->sc_enaddr);
ether_set_ifflags_cb(&sc->sc_ec, ixl_ifflags_cb);

rv = ixl_get_link_status_poll(sc, &link);
if (rv != 0)
link = LINK_STATE_UNKNOWN;
if_link_state_change(ifp, link);

ixl_atq_set(&sc->sc_link_state_atq, ixl_get_link_status_done);
ixl_work_set(&sc->sc_link_state_task,
ixl_get_link_status_work, sc);
ixl_work_set(&sc->sc_link_state_done_task,
ixl_get_link_status_done_work, sc);

ixl_config_other_intr(sc);
ixl_enable_other_intr(sc);

ixl_set_phy_autoselect(sc);

/* remove default mac filter and replace it so we can see vlans */
rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0, 0);
if (rv != ENOENT) {
aprint_debug_dev(self,
"unable to remove macvlan %u\n", rv);
}
rv = ixl_remove_macvlan(sc, sc->sc_enaddr, 0,
IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN);
if (rv != ENOENT) {
aprint_debug_dev(self,
"unable to remove macvlan, ignore vlan %u\n", rv);
}

if (ixl_update_macvlan(sc) != 0) {
aprint_debug_dev(self,
"couldn't enable vlan hardware filter\n");
CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER);
CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER);
}

sc->sc_txrx_workqueue = true;
sc->sc_tx_process_limit = IXL_TX_PROCESS_LIMIT;
sc->sc_rx_process_limit = IXL_RX_PROCESS_LIMIT;
sc->sc_tx_intr_process_limit = IXL_TX_INTR_PROCESS_LIMIT;
sc->sc_rx_intr_process_limit = IXL_RX_INTR_PROCESS_LIMIT;

ixl_stats_update(sc);
sc->sc_stats_counters.isc_has_offset = true;

if (pmf_device_register(self, NULL, NULL) != true)
aprint_debug_dev(self, "couldn't establish power handler\n");
sc->sc_itr_rx = IXL_ITR_RX;
sc->sc_itr_tx = IXL_ITR_TX;
sc->sc_attached = true;
if_register(ifp);

return;

teardown_wqs:
config_finalize_register(self, ixl_workqs_teardown);
teardown_sysctls:
ixl_teardown_sysctls(sc);
teardown_stats:
ixl_teardown_stats(sc);
teardown_intrs:
ixl_teardown_interrupts(sc);
free_queue_pairs:
ixl_queue_pairs_free(sc);
free_scratch:
ixl_dmamem_free(sc, &sc->sc_scratch);
free_hmc:
ixl_hmc_free(sc);
free_aqbuf:
ixl_dmamem_free(sc, &sc->sc_aqbuf);
shutdown:
ixl_wr(sc, sc->sc_aq_regs->atq_head, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_head, 0);
ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0);

ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0);
ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0);
ixl_wr(sc, sc->sc_aq_regs->atq_len, 0);

ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_len, 0);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq),
0, IXL_DMA_LEN(&sc->sc_arq),
BUS_DMASYNC_POSTREAD);
bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq),
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);

ixl_arq_unfill(sc);
free_arq:
ixl_dmamem_free(sc, &sc->sc_arq);
free_atq:
ixl_dmamem_free(sc, &sc->sc_atq);
unmap:
mutex_destroy(&sc->sc_atq_lock);
bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
mutex_destroy(&sc->sc_cfg_lock);
sc->sc_mems = 0;

sc->sc_attached = false;
}

static int
ixl_detach(device_t self, int flags)
{
struct ixl_softc *sc = device_private(self);
struct ifnet *ifp = &sc->sc_ec.ec_if;

if (!sc->sc_attached)
return 0;

ixl_stop(ifp, 1);

callout_halt(&sc->sc_stats_callout, NULL);
ixl_work_wait(sc->sc_workq, &sc->sc_stats_task);

/* detach the I/F before stop adminq due to callbacks */
ether_ifdetach(ifp);
if_detach(ifp);
ifmedia_fini(&sc->sc_media);
if_percpuq_destroy(sc->sc_ipq);

ixl_disable_other_intr(sc);
ixl_intr_barrier();
ixl_work_wait(sc->sc_workq, &sc->sc_arq_task);
ixl_work_wait(sc->sc_workq, &sc->sc_link_state_task);

if (sc->sc_workq != NULL) {
ixl_workq_destroy(sc->sc_workq);
sc->sc_workq = NULL;
}

if (sc->sc_workq_txrx != NULL) {
workqueue_destroy(sc->sc_workq_txrx);
sc->sc_workq_txrx = NULL;
}

ixl_teardown_interrupts(sc);
ixl_teardown_stats(sc);
ixl_teardown_sysctls(sc);

ixl_queue_pairs_free(sc);

ixl_dmamem_free(sc, &sc->sc_scratch);
ixl_hmc_free(sc);

/* shutdown */
ixl_wr(sc, sc->sc_aq_regs->atq_head, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_head, 0);
ixl_wr(sc, sc->sc_aq_regs->atq_tail, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_tail, 0);

ixl_wr(sc, sc->sc_aq_regs->atq_bal, 0);
ixl_wr(sc, sc->sc_aq_regs->atq_bah, 0);
ixl_wr(sc, sc->sc_aq_regs->atq_len, 0);

ixl_wr(sc, sc->sc_aq_regs->arq_bal, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_bah, 0);
ixl_wr(sc, sc->sc_aq_regs->arq_len, 0);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq),
0, IXL_DMA_LEN(&sc->sc_arq),
BUS_DMASYNC_POSTREAD);
bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq),
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);

ixl_arq_unfill(sc);

ixl_dmamem_free(sc, &sc->sc_arq);
ixl_dmamem_free(sc, &sc->sc_atq);
ixl_dmamem_free(sc, &sc->sc_aqbuf);

cv_destroy(&sc->sc_atq_cv);
mutex_destroy(&sc->sc_atq_lock);

if (sc->sc_mems != 0) {
bus_space_unmap(sc->sc_memt, sc->sc_memh, sc->sc_mems);
sc->sc_mems = 0;
}

mutex_destroy(&sc->sc_cfg_lock);

return 0;
}

static int
ixl_workqs_teardown(device_t self)
{
struct ixl_softc *sc = device_private(self);

if (sc->sc_workq != NULL) {
ixl_workq_destroy(sc->sc_workq);
sc->sc_workq = NULL;
}

if (sc->sc_workq_txrx != NULL) {
workqueue_destroy(sc->sc_workq_txrx);
sc->sc_workq_txrx = NULL;
}

return 0;
}

static int
ixl_vlan_cb(struct ethercom *ec, uint16_t vid, bool set)
{
struct ifnet *ifp = &ec->ec_if;
struct ixl_softc *sc = ifp->if_softc;
int rv;

if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) {
return 0;
}

if (set) {
rv = ixl_add_macvlan(sc, sc->sc_enaddr, vid,
IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH);
if (rv == 0) {
rv = ixl_add_macvlan(sc, etherbroadcastaddr,
vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH);
}
} else {
rv = ixl_remove_macvlan(sc, sc->sc_enaddr, vid,
IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH);
(void)ixl_remove_macvlan(sc, etherbroadcastaddr, vid,
IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH);
}

return rv;
}

static void
ixl_media_add(struct ixl_softc *sc)
{
struct ifmedia *ifm = &sc->sc_media;
const struct ixl_phy_type *itype;
unsigned int i;
bool flow;

if (ISSET(sc->sc_phy_abilities,
(IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX))) {
flow = true;
} else {
flow = false;
}

for (i = 0; i < __arraycount(ixl_phy_type_map); i++) {
itype = &ixl_phy_type_map[i];

if (ISSET(sc->sc_phy_types, itype->phy_type)) {
ifmedia_add(ifm,
IFM_ETHER | IFM_FDX | itype->ifm_type, 0, NULL);

if (flow) {
ifmedia_add(ifm,
IFM_ETHER | IFM_FDX | IFM_FLOW |
itype->ifm_type, 0, NULL);
}

if (itype->ifm_type != IFM_100_TX)
continue;

ifmedia_add(ifm, IFM_ETHER | itype->ifm_type,
0, NULL);
if (flow) {
ifmedia_add(ifm,
IFM_ETHER | IFM_FLOW | itype->ifm_type,
0, NULL);
}
}
}
}

static void
ixl_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
{
struct ixl_softc *sc = ifp->if_softc;

KASSERT(mutex_owned(&sc->sc_cfg_lock));

ifmr->ifm_status = sc->sc_media_status;
ifmr->ifm_active = sc->sc_media_active;
}

static int
ixl_media_change(struct ifnet *ifp)
{
struct ixl_softc *sc = ifp->if_softc;
struct ifmedia *ifm = &sc->sc_media;
uint64_t ifm_active = sc->sc_media_active;
uint8_t link_speed, abilities;

switch (IFM_SUBTYPE(ifm_active)) {
case IFM_1000_SGMII:
case IFM_1000_KX:
case IFM_10G_KX4:
case IFM_10G_KR:
case IFM_40G_KR4:
case IFM_20G_KR2:
case IFM_25G_KR:
/* backplanes */
return EINVAL;
}

abilities = IXL_PHY_ABILITY_AUTONEGO | IXL_PHY_ABILITY_LINKUP;

switch (IFM_SUBTYPE(ifm->ifm_media)) {
case IFM_AUTO:
link_speed = sc->sc_phy_linkspeed;
break;
case IFM_NONE:
link_speed = 0;
CLR(abilities, IXL_PHY_ABILITY_LINKUP);
break;
default:
link_speed = ixl_search_baudrate(
ifmedia_baudrate(ifm->ifm_media));
}

if (ISSET(abilities, IXL_PHY_ABILITY_LINKUP)) {
if (ISSET(link_speed, sc->sc_phy_linkspeed) == 0)
return EINVAL;
}

if (ifm->ifm_media & IFM_FLOW) {
abilities |= sc->sc_phy_abilities &
(IXL_PHY_ABILITY_PAUSE_TX | IXL_PHY_ABILITY_PAUSE_RX);
}

return ixl_set_phy_config(sc, link_speed, abilities, false);
}

static void
ixl_del_all_multiaddr(struct ixl_softc *sc)
{
struct ethercom *ec = &sc->sc_ec;
struct ether_multi *enm;
struct ether_multistep step;

ETHER_LOCK(ec);
for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
ETHER_NEXT_MULTI(step, enm)) {
ixl_remove_macvlan(sc, enm->enm_addrlo, 0,
IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN);
}
ETHER_UNLOCK(ec);
}

static int
ixl_add_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi)
{
struct ifnet *ifp = &sc->sc_ec.ec_if;
int rv;

if (ISSET(ifp->if_flags, IFF_ALLMULTI))
return 0;

if (memcmp(addrlo, addrhi, ETHER_ADDR_LEN) != 0) {
ixl_del_all_multiaddr(sc);
SET(ifp->if_flags, IFF_ALLMULTI);
return ENETRESET;
}

/* multicast address can not use VLAN HWFILTER */
rv = ixl_add_macvlan(sc, addrlo, 0,
IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN);

if (rv == ENOSPC) {
ixl_del_all_multiaddr(sc);
SET(ifp->if_flags, IFF_ALLMULTI);
return ENETRESET;
}

return rv;
}

static int
ixl_del_multi(struct ixl_softc *sc, uint8_t *addrlo, uint8_t *addrhi)
{
struct ifnet *ifp = &sc->sc_ec.ec_if;
struct ethercom *ec = &sc->sc_ec;
struct ether_multi *enm, *enm_last;
struct ether_multistep step;
int error, rv = 0;

if (!ISSET(ifp->if_flags, IFF_ALLMULTI)) {
ixl_remove_macvlan(sc, addrlo, 0,
IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN);
return 0;
}

ETHER_LOCK(ec);
for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
ETHER_NEXT_MULTI(step, enm)) {
if (memcmp(enm->enm_addrlo, enm->enm_addrhi,
ETHER_ADDR_LEN) != 0) {
goto out;
}
}

for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
ETHER_NEXT_MULTI(step, enm)) {
error = ixl_add_macvlan(sc, enm->enm_addrlo, 0,
IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN);
if (error != 0)
break;
}

if (enm != NULL) {
enm_last = enm;
for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
ETHER_NEXT_MULTI(step, enm)) {
if (enm == enm_last)
break;

ixl_remove_macvlan(sc, enm->enm_addrlo, 0,
IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN);
}
} else {
CLR(ifp->if_flags, IFF_ALLMULTI);
rv = ENETRESET;
}

out:
ETHER_UNLOCK(ec);
return rv;
}

static int
ixl_ioctl(struct ifnet *ifp, u_long cmd, void *data)
{
struct ifreq *ifr = (struct ifreq *)data;
struct ixl_softc *sc = (struct ixl_softc *)ifp->if_softc;
const struct sockaddr *sa;
uint8_t addrhi[ETHER_ADDR_LEN], addrlo[ETHER_ADDR_LEN];
int s, error = 0;
unsigned int nmtu;

switch (cmd) {
case SIOCSIFMTU:
nmtu = ifr->ifr_mtu;

if (nmtu < IXL_MIN_MTU || nmtu > IXL_MAX_MTU) {
error = EINVAL;
break;
}
if (ifp->if_mtu != nmtu) {
s = splnet();
error = ether_ioctl(ifp, cmd, data);
splx(s);
if (error == ENETRESET)
error = ixl_init(ifp);
}
break;
case SIOCADDMULTI:
sa = ifreq_getaddr(SIOCADDMULTI, ifr);
if (ether_addmulti(sa, &sc->sc_ec) == ENETRESET) {
error = ether_multiaddr(sa, addrlo, addrhi);
if (error != 0)
return error;

error = ixl_add_multi(sc, addrlo, addrhi);
if (error != 0 && error != ENETRESET) {
ether_delmulti(sa, &sc->sc_ec);
error = EIO;
}
}
break;

case SIOCDELMULTI:
sa = ifreq_getaddr(SIOCDELMULTI, ifr);
if (ether_delmulti(sa, &sc->sc_ec) == ENETRESET) {
error = ether_multiaddr(sa, addrlo, addrhi);
if (error != 0)
return error;

error = ixl_del_multi(sc, addrlo, addrhi);
}
break;

default:
s = splnet();
error = ether_ioctl(ifp, cmd, data);
splx(s);
}

if (error == ENETRESET)
error = ixl_iff(sc);

return error;
}

static enum i40e_mac_type
ixl_mactype(pci_product_id_t id)
{

switch (id) {
case PCI_PRODUCT_INTEL_XL710_SFP:
case PCI_PRODUCT_INTEL_XL710_KX_B:
case PCI_PRODUCT_INTEL_XL710_KX_C:
case PCI_PRODUCT_INTEL_XL710_QSFP_A:
case PCI_PRODUCT_INTEL_XL710_QSFP_B:
case PCI_PRODUCT_INTEL_XL710_QSFP_C:
case PCI_PRODUCT_INTEL_X710_10G_T_1:
case PCI_PRODUCT_INTEL_X710_10G_T_2:
case PCI_PRODUCT_INTEL_XL710_20G_BP_1:
case PCI_PRODUCT_INTEL_XL710_20G_BP_2:
case PCI_PRODUCT_INTEL_X710_T4_10G:
case PCI_PRODUCT_INTEL_XXV710_25G_BP:
case PCI_PRODUCT_INTEL_XXV710_25G_SFP28:
case PCI_PRODUCT_INTEL_X710_10G_SFP:
case PCI_PRODUCT_INTEL_X710_10G_BP:
return I40E_MAC_XL710;

case PCI_PRODUCT_INTEL_X722_KX:
case PCI_PRODUCT_INTEL_X722_QSFP:
case PCI_PRODUCT_INTEL_X722_SFP:
case PCI_PRODUCT_INTEL_X722_1G_BASET:
case PCI_PRODUCT_INTEL_X722_10G_BASET:
case PCI_PRODUCT_INTEL_X722_I_SFP:
return I40E_MAC_X722;
}

return I40E_MAC_GENERIC;
}

static void
ixl_pci_csr_setup(pci_chipset_tag_t pc, pcitag_t tag)
{
pcireg_t csr;

csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG);
csr |= (PCI_COMMAND_MASTER_ENABLE |
PCI_COMMAND_MEM_ENABLE);
pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr);
}

static inline void *
ixl_hmc_kva(struct ixl_softc *sc, enum ixl_hmc_types type, unsigned int i)
{
uint8_t *kva = IXL_DMA_KVA(&sc->sc_hmc_pd);
struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type];

if (i >= e->hmc_count)
return NULL;

kva += e->hmc_base;
kva += i * e->hmc_size;

return kva;
}

static inline size_t
ixl_hmc_len(struct ixl_softc *sc, enum ixl_hmc_types type)
{
struct ixl_hmc_entry *e = &sc->sc_hmc_entries[type];

return e->hmc_size;
}

static void
ixl_enable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp)
{
struct ixl_rx_ring *rxr = qp->qp_rxr;

ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid),
I40E_PFINT_DYN_CTLN_INTENA_MASK |
I40E_PFINT_DYN_CTLN_CLEARPBA_MASK |
(IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT));
ixl_flush(sc);
}

static void
ixl_disable_queue_intr(struct ixl_softc *sc, struct ixl_queue_pair *qp)
{
struct ixl_rx_ring *rxr = qp->qp_rxr;

ixl_wr(sc, I40E_PFINT_DYN_CTLN(rxr->rxr_qid),
(IXL_NOITR << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT));
ixl_flush(sc);
}

static void
ixl_enable_other_intr(struct ixl_softc *sc)
{

ixl_wr(sc, I40E_PFINT_DYN_CTL0,
I40E_PFINT_DYN_CTL0_INTENA_MASK |
I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
(IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT));
ixl_flush(sc);
}

static void
ixl_disable_other_intr(struct ixl_softc *sc)
{

ixl_wr(sc, I40E_PFINT_DYN_CTL0,
(IXL_NOITR << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT));
ixl_flush(sc);
}

static int
ixl_reinit(struct ixl_softc *sc)
{
struct ixl_rx_ring *rxr;
struct ixl_tx_ring *txr;
unsigned int i;
uint32_t reg;

KASSERT(mutex_owned(&sc->sc_cfg_lock));

if (ixl_get_vsi(sc) != 0)
return EIO;

if (ixl_set_vsi(sc) != 0)
return EIO;

for (i = 0; i < sc->sc_nqueue_pairs; i++) {
txr = sc->sc_qps[i].qp_txr;
rxr = sc->sc_qps[i].qp_rxr;

ixl_txr_config(sc, txr);
ixl_rxr_config(sc, rxr);
}

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd),
0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_PREWRITE);

for (i = 0; i < sc->sc_nqueue_pairs; i++) {
txr = sc->sc_qps[i].qp_txr;
rxr = sc->sc_qps[i].qp_rxr;

ixl_wr(sc, I40E_QTX_CTL(i), I40E_QTX_CTL_PF_QUEUE |
(sc->sc_pf_id << I40E_QTX_CTL_PF_INDX_SHIFT));
ixl_flush(sc);

ixl_wr(sc, txr->txr_tail, txr->txr_prod);
ixl_wr(sc, rxr->rxr_tail, rxr->rxr_prod);

/* ixl_rxfill() needs lock held */
mutex_enter(&rxr->rxr_lock);
ixl_rxfill(sc, rxr);
mutex_exit(&rxr->rxr_lock);

reg = ixl_rd(sc, I40E_QRX_ENA(i));
SET(reg, I40E_QRX_ENA_QENA_REQ_MASK);
ixl_wr(sc, I40E_QRX_ENA(i), reg);
if (ixl_rxr_enabled(sc, rxr) != 0)
goto stop;

ixl_txr_qdis(sc, txr, 1);

reg = ixl_rd(sc, I40E_QTX_ENA(i));
SET(reg, I40E_QTX_ENA_QENA_REQ_MASK);
ixl_wr(sc, I40E_QTX_ENA(i), reg);

if (ixl_txr_enabled(sc, txr) != 0)
goto stop;
}

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd),
0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE);

return 0;

stop:
bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd),
0, IXL_DMA_LEN(&sc->sc_hmc_pd), BUS_DMASYNC_POSTWRITE);

return ETIMEDOUT;
}

static int
ixl_init_locked(struct ixl_softc *sc)
{
struct ifnet *ifp = &sc->sc_ec.ec_if;
unsigned int i;
int error, eccap_change;

KASSERT(mutex_owned(&sc->sc_cfg_lock));

if (ISSET(ifp->if_flags, IFF_RUNNING))
ixl_stop_locked(sc);

if (sc->sc_dead) {
return ENXIO;
}

eccap_change = sc->sc_ec.ec_capenable ^ sc->sc_cur_ec_capenable;
if (ISSET(eccap_change, ETHERCAP_VLAN_HWTAGGING))
sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWTAGGING;

if (ISSET(eccap_change, ETHERCAP_VLAN_HWFILTER)) {
if (ixl_update_macvlan(sc) == 0) {
sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER;
} else {
CLR(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_HWFILTER);
CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER);
}
}

if (sc->sc_intrtype != PCI_INTR_TYPE_MSIX)
sc->sc_nqueue_pairs = 1;
else
sc->sc_nqueue_pairs = sc->sc_nqueue_pairs_max;

error = ixl_reinit(sc);
if (error) {
ixl_stop_locked(sc);
return error;
}

SET(ifp->if_flags, IFF_RUNNING);
CLR(ifp->if_flags, IFF_OACTIVE);

ixl_config_rss(sc);
ixl_config_queue_intr(sc);

for (i = 0; i < sc->sc_nqueue_pairs; i++) {
ixl_enable_queue_intr(sc, &sc->sc_qps[i]);
}

error = ixl_iff(sc);
if (error) {
ixl_stop_locked(sc);
return error;
}

callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval));

return 0;
}

static int
ixl_init(struct ifnet *ifp)
{
struct ixl_softc *sc = ifp->if_softc;
int error;

mutex_enter(&sc->sc_cfg_lock);
error = ixl_init_locked(sc);
mutex_exit(&sc->sc_cfg_lock);

if (error == 0) {
error = ixl_get_link_status(sc,
IXL_LINK_FLAG_WAITDONE);
}

return error;
}

static int
ixl_iff(struct ixl_softc *sc)
{
struct ifnet *ifp = &sc->sc_ec.ec_if;
struct ixl_atq iatq;
struct ixl_aq_desc *iaq;
struct ixl_aq_vsi_promisc_param *param;
uint16_t flag_add, flag_del;
int error;

if (!ISSET(ifp->if_flags, IFF_RUNNING))
return 0;

memset(&iatq, 0, sizeof(iatq));

iaq = &iatq.iatq_desc;
iaq->iaq_opcode = htole16(IXL_AQ_OP_SET_VSI_PROMISC);

param = (struct ixl_aq_vsi_promisc_param *)&iaq->iaq_param;
param->flags = htole16(0);

if (!ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)
|| ISSET(ifp->if_flags, IFF_PROMISC)) {
param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_BCAST |
IXL_AQ_VSI_PROMISC_FLAG_VLAN);
}

if (ISSET(ifp->if_flags, IFF_PROMISC)) {
param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST |
IXL_AQ_VSI_PROMISC_FLAG_MCAST);
} else if (ISSET(ifp->if_flags, IFF_ALLMULTI)) {
param->flags |= htole16(IXL_AQ_VSI_PROMISC_FLAG_MCAST);
}
param->valid_flags = htole16(IXL_AQ_VSI_PROMISC_FLAG_UCAST |
IXL_AQ_VSI_PROMISC_FLAG_MCAST | IXL_AQ_VSI_PROMISC_FLAG_BCAST |
IXL_AQ_VSI_PROMISC_FLAG_VLAN);
param->seid = sc->sc_seid;

error = ixl_atq_exec(sc, &iatq);
if (error)
return error;

if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK))
return EIO;

if (memcmp(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN) != 0) {
if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER)) {
flag_add = IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH;
flag_del = IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH;
} else {
flag_add = IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN;
flag_del = IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN;
}

ixl_remove_macvlan(sc, sc->sc_enaddr, 0, flag_del);

memcpy(sc->sc_enaddr, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
ixl_add_macvlan(sc, sc->sc_enaddr, 0, flag_add);
}
return 0;
}

static void
ixl_stop_locked(struct ixl_softc *sc)
{
struct ifnet *ifp = &sc->sc_ec.ec_if;
struct ixl_rx_ring *rxr;
struct ixl_tx_ring *txr;
unsigned int i;
uint32_t reg;

KASSERT(mutex_owned(&sc->sc_cfg_lock));

CLR(ifp->if_flags, IFF_RUNNING | IFF_OACTIVE);
callout_stop(&sc->sc_stats_callout);

for (i = 0; i < sc->sc_nqueue_pairs; i++) {
txr = sc->sc_qps[i].qp_txr;
rxr = sc->sc_qps[i].qp_rxr;

ixl_disable_queue_intr(sc, &sc->sc_qps[i]);

mutex_enter(&txr->txr_lock);
ixl_txr_qdis(sc, txr, 0);
mutex_exit(&txr->txr_lock);
}

/* XXX wait at least 400 usec for all tx queues in one go */
ixl_flush(sc);
DELAY(500);

for (i = 0; i < sc->sc_nqueue_pairs; i++) {
txr = sc->sc_qps[i].qp_txr;
rxr = sc->sc_qps[i].qp_rxr;

mutex_enter(&txr->txr_lock);
reg = ixl_rd(sc, I40E_QTX_ENA(i));
CLR(reg, I40E_QTX_ENA_QENA_REQ_MASK);
ixl_wr(sc, I40E_QTX_ENA(i), reg);
mutex_exit(&txr->txr_lock);

mutex_enter(&rxr->rxr_lock);
reg = ixl_rd(sc, I40E_QRX_ENA(i));
CLR(reg, I40E_QRX_ENA_QENA_REQ_MASK);
ixl_wr(sc, I40E_QRX_ENA(i), reg);
mutex_exit(&rxr->rxr_lock);
}

/* XXX short wait for all queue disables to settle */
ixl_flush(sc);
DELAY(50);

for (i = 0; i < sc->sc_nqueue_pairs; i++) {
txr = sc->sc_qps[i].qp_txr;
rxr = sc->sc_qps[i].qp_rxr;

mutex_enter(&txr->txr_lock);
if (ixl_txr_disabled(sc, txr) != 0) {
mutex_exit(&txr->txr_lock);
goto die;
}
mutex_exit(&txr->txr_lock);

mutex_enter(&rxr->rxr_lock);
if (ixl_rxr_disabled(sc, rxr) != 0) {
mutex_exit(&rxr->rxr_lock);
goto die;
}
mutex_exit(&rxr->rxr_lock);
}

for (i = 0; i < sc->sc_nqueue_pairs; i++) {
sc->sc_qps[i].qp_workqueue = false;
workqueue_wait(sc->sc_workq_txrx,
&sc->sc_qps[i].qp_work);
}

for (i = 0; i < sc->sc_nqueue_pairs; i++) {
txr = sc->sc_qps[i].qp_txr;
rxr = sc->sc_qps[i].qp_rxr;

mutex_enter(&txr->txr_lock);
ixl_txr_unconfig(sc, txr);
mutex_exit(&txr->txr_lock);

mutex_enter(&rxr->rxr_lock);
ixl_rxr_unconfig(sc, rxr);
mutex_exit(&rxr->rxr_lock);

ixl_txr_clean(sc, txr);
ixl_rxr_clean(sc, rxr);
}

return;
die:
sc->sc_dead = true;
log(LOG_CRIT, "%s: failed to shut down rings",
device_xname(sc->sc_dev));
return;
}

static void
ixl_stop(struct ifnet *ifp, int disable)
{
struct ixl_softc *sc = ifp->if_softc;

mutex_enter(&sc->sc_cfg_lock);
ixl_stop_locked(sc);
mutex_exit(&sc->sc_cfg_lock);
}

static int
ixl_queue_pairs_alloc(struct ixl_softc *sc)
{
struct ixl_queue_pair *qp;
unsigned int i;
size_t sz;

sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max;
sc->sc_qps = kmem_zalloc(sz, KM_SLEEP);

for (i = 0; i < sc->sc_nqueue_pairs_max; i++) {
qp = &sc->sc_qps[i];

qp->qp_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
ixl_handle_queue, qp);
if (qp->qp_si == NULL)
goto free;

qp->qp_txr = ixl_txr_alloc(sc, i);
if (qp->qp_txr == NULL)
goto free;

qp->qp_rxr = ixl_rxr_alloc(sc, i);
if (qp->qp_rxr == NULL)
goto free;

qp->qp_sc = sc;
snprintf(qp->qp_name, sizeof(qp->qp_name),
"%s-TXRX%d", device_xname(sc->sc_dev), i);
}

return 0;
free:
if (sc->sc_qps != NULL) {
for (i = 0; i < sc->sc_nqueue_pairs_max; i++) {
qp = &sc->sc_qps[i];

if (qp->qp_txr != NULL)
ixl_txr_free(sc, qp->qp_txr);
if (qp->qp_rxr != NULL)
ixl_rxr_free(sc, qp->qp_rxr);
if (qp->qp_si != NULL)
softint_disestablish(qp->qp_si);
}

sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max;
kmem_free(sc->sc_qps, sz);
sc->sc_qps = NULL;
}

return -1;
}

static void
ixl_queue_pairs_free(struct ixl_softc *sc)
{
struct ixl_queue_pair *qp;
unsigned int i;
size_t sz;

for (i = 0; i < sc->sc_nqueue_pairs_max; i++) {
qp = &sc->sc_qps[i];
ixl_txr_free(sc, qp->qp_txr);
ixl_rxr_free(sc, qp->qp_rxr);
softint_disestablish(qp->qp_si);
}

sz = sizeof(sc->sc_qps[0]) * sc->sc_nqueue_pairs_max;
kmem_free(sc->sc_qps, sz);
sc->sc_qps = NULL;
}

static struct ixl_tx_ring *
ixl_txr_alloc(struct ixl_softc *sc, unsigned int qid)
{
struct ixl_tx_ring *txr = NULL;
struct ixl_tx_map *maps = NULL, *txm;
unsigned int i;

txr = kmem_zalloc(sizeof(*txr), KM_SLEEP);
maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_tx_ring_ndescs,
KM_SLEEP);

if (ixl_dmamem_alloc(sc, &txr->txr_mem,
sizeof(struct ixl_tx_desc) * sc->sc_tx_ring_ndescs,
IXL_TX_QUEUE_ALIGN) != 0)
goto free;

for (i = 0; i < sc->sc_tx_ring_ndescs; i++) {
txm = &maps[i];

if (bus_dmamap_create(sc->sc_dmat, IXL_TX_PKT_MAXSIZE,
IXL_TX_PKT_DESCS, IXL_TX_PKT_MAXSIZE, 0,
BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &txm->txm_map) != 0)
goto uncreate;

txm->txm_eop = -1;
txm->txm_m = NULL;
}

txr->txr_cons = txr->txr_prod = 0;
txr->txr_maps = maps;

txr->txr_intrq = pcq_create(sc->sc_tx_ring_ndescs, KM_NOSLEEP);
if (txr->txr_intrq == NULL)
goto uncreate;

txr->txr_si = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE,
ixl_deferred_transmit, txr);
if (txr->txr_si == NULL)
goto destroy_pcq;

txr->txr_tail = I40E_QTX_TAIL(qid);
txr->txr_qid = qid;
txr->txr_sc = sc;
mutex_init(&txr->txr_lock, MUTEX_DEFAULT, IPL_NET);

return txr;

destroy_pcq:
pcq_destroy(txr->txr_intrq);
uncreate:
for (i = 0; i < sc->sc_tx_ring_ndescs; i++) {
txm = &maps[i];

if (txm->txm_map == NULL)
continue;

bus_dmamap_destroy(sc->sc_dmat, txm->txm_map);
}

ixl_dmamem_free(sc, &txr->txr_mem);
free:
kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs);
kmem_free(txr, sizeof(*txr));

return NULL;
}

static void
ixl_txr_qdis(struct ixl_softc *sc, struct ixl_tx_ring *txr, int enable)
{
unsigned int qid;
bus_size_t reg;
uint32_t r;

qid = txr->txr_qid + sc->sc_base_queue;
reg = I40E_GLLAN_TXPRE_QDIS(qid / 128);
qid %= 128;

r = ixl_rd(sc, reg);
CLR(r, I40E_GLLAN_TXPRE_QDIS_QINDX_MASK);
SET(r, qid << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT);
SET(r, enable ? I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK :
I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK);
ixl_wr(sc, reg, r);
}

static void
ixl_txr_config(struct ixl_softc *sc, struct ixl_tx_ring *txr)
{
struct ixl_hmc_txq txq;
struct ixl_aq_vsi_data *data = IXL_DMA_KVA(&sc->sc_scratch);
void *hmc;

memset(&txq, 0, sizeof(txq));
txq.head = htole16(txr->txr_cons);
txq.new_context = 1;
txq.base = htole64(IXL_DMA_DVA(&txr->txr_mem) / IXL_HMC_TXQ_BASE_UNIT);
txq.head_wb_ena = IXL_HMC_TXQ_DESC_WB;
txq.qlen = htole16(sc->sc_tx_ring_ndescs);
txq.tphrdesc_ena = 0;
txq.tphrpacket_ena = 0;
txq.tphwdesc_ena = 0;
txq.rdylist = data->qs_handle[0];

hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid);
memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX));
ixl_hmc_pack(hmc, &txq, ixl_hmc_pack_txq,
__arraycount(ixl_hmc_pack_txq));
}

static void
ixl_txr_unconfig(struct ixl_softc *sc, struct ixl_tx_ring *txr)
{
void *hmc;

hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_TX, txr->txr_qid);
memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_TX));
txr->txr_cons = txr->txr_prod = 0;
}

static void
ixl_txr_clean(struct ixl_softc *sc, struct ixl_tx_ring *txr)
{
struct ixl_tx_map *maps, *txm;
bus_dmamap_t map;
unsigned int i;

maps = txr->txr_maps;
for (i = 0; i < sc->sc_tx_ring_ndescs; i++) {
txm = &maps[i];

if (txm->txm_m == NULL)
continue;

map = txm->txm_map;
bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, map);

m_freem(txm->txm_m);
txm->txm_m = NULL;
}
}

static int
ixl_txr_enabled(struct ixl_softc *sc, struct ixl_tx_ring *txr)
{
bus_size_t ena = I40E_QTX_ENA(txr->txr_qid);
uint32_t reg;
int i;

for (i = 0; i < 10; i++) {
reg = ixl_rd(sc, ena);
if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK))
return 0;

delaymsec(10);
}

return ETIMEDOUT;
}

static int
ixl_txr_disabled(struct ixl_softc *sc, struct ixl_tx_ring *txr)
{
bus_size_t ena = I40E_QTX_ENA(txr->txr_qid);
uint32_t reg;
int i;

KASSERT(mutex_owned(&txr->txr_lock));

for (i = 0; i < 10; i++) {
reg = ixl_rd(sc, ena);
if (ISSET(reg, I40E_QTX_ENA_QENA_STAT_MASK) == 0)
return 0;

delaymsec(10);
}

return ETIMEDOUT;
}

static void
ixl_txr_free(struct ixl_softc *sc, struct ixl_tx_ring *txr)
{
struct ixl_tx_map *maps, *txm;
struct mbuf *m;
unsigned int i;

softint_disestablish(txr->txr_si);

maps = txr->txr_maps;
for (i = 0; i < sc->sc_tx_ring_ndescs; i++) {
txm = &maps[i];

bus_dmamap_destroy(sc->sc_dmat, txm->txm_map);
}

while ((m = pcq_get(txr->txr_intrq)) != NULL)
m_freem(m);
pcq_destroy(txr->txr_intrq);

ixl_dmamem_free(sc, &txr->txr_mem);
mutex_destroy(&txr->txr_lock);
kmem_free(maps, sizeof(maps[0]) * sc->sc_tx_ring_ndescs);
kmem_free(txr, sizeof(*txr));
}

static inline int
ixl_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf **m0,
struct ixl_tx_ring *txr)
{
struct mbuf *m;
int error;

KASSERT(mutex_owned(&txr->txr_lock));

m = *m0;

error = bus_dmamap_load_mbuf(dmat, map, m,
BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT);
if (error != EFBIG)
return error;

m = m_defrag(m, M_DONTWAIT);
if (m != NULL) {
*m0 = m;
txr->txr_defragged.ev_count++;

error = bus_dmamap_load_mbuf(dmat, map, m,
BUS_DMA_STREAMING | BUS_DMA_WRITE | BUS_DMA_NOWAIT);
} else {
txr->txr_defrag_failed.ev_count++;
error = ENOBUFS;
}

return error;
}

static inline int
ixl_tx_setup_offloads(struct mbuf *m, uint64_t *cmd_txd)
{
struct ether_header *eh;
size_t len;
uint64_t cmd;

cmd = 0;

eh = mtod(m, struct ether_header *);
switch (htons(eh->ether_type)) {
case ETHERTYPE_IP:
case ETHERTYPE_IPV6:
len = ETHER_HDR_LEN;
break;
case ETHERTYPE_VLAN:
len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
break;
default:
len = 0;
}
cmd |= ((len >> 1) << IXL_TX_DESC_MACLEN_SHIFT);

if (m->m_pkthdr.csum_flags &
(M_CSUM_TSOv4 | M_CSUM_TCPv4 | M_CSUM_UDPv4)) {
cmd |= IXL_TX_DESC_CMD_IIPT_IPV4;
}
if (m->m_pkthdr.csum_flags & M_CSUM_IPv4) {
cmd |= IXL_TX_DESC_CMD_IIPT_IPV4_CSUM;
}

if (m->m_pkthdr.csum_flags &
(M_CSUM_TSOv6 | M_CSUM_TCPv6 | M_CSUM_UDPv6)) {
cmd |= IXL_TX_DESC_CMD_IIPT_IPV6;
}

switch (cmd & IXL_TX_DESC_CMD_IIPT_MASK) {
case IXL_TX_DESC_CMD_IIPT_IPV4:
case IXL_TX_DESC_CMD_IIPT_IPV4_CSUM:
len = M_CSUM_DATA_IPv4_IPHL(m->m_pkthdr.csum_data);
break;
case IXL_TX_DESC_CMD_IIPT_IPV6:
len = M_CSUM_DATA_IPv6_IPHL(m->m_pkthdr.csum_data);
break;
default:
len = 0;
}
cmd |= ((len >> 2) << IXL_TX_DESC_IPLEN_SHIFT);

if (m->m_pkthdr.csum_flags &
(M_CSUM_TSOv4 | M_CSUM_TSOv6 | M_CSUM_TCPv4 | M_CSUM_TCPv6)) {
len = sizeof(struct tcphdr);
cmd |= IXL_TX_DESC_CMD_L4T_EOFT_TCP;
} else if (m->m_pkthdr.csum_flags & (M_CSUM_UDPv4 | M_CSUM_UDPv6)) {
len = sizeof(struct udphdr);
cmd |= IXL_TX_DESC_CMD_L4T_EOFT_UDP;
} else {
len = 0;
}
cmd |= ((len >> 2) << IXL_TX_DESC_L4LEN_SHIFT);

*cmd_txd |= cmd;
return 0;
}

static void
ixl_tx_common_locked(struct ifnet *ifp, struct ixl_tx_ring *txr,
bool is_transmit)
{
struct ixl_softc *sc = ifp->if_softc;
struct ixl_tx_desc *ring, *txd;
struct ixl_tx_map *txm;
bus_dmamap_t map;
struct mbuf *m;
uint64_t cmd, cmd_txd;
unsigned int prod, free, last, i;
unsigned int mask;
int post = 0;

KASSERT(mutex_owned(&txr->txr_lock));

if (!ISSET(ifp->if_flags, IFF_RUNNING)
|| (!is_transmit && ISSET(ifp->if_flags, IFF_OACTIVE))) {
if (!is_transmit)
IFQ_PURGE(&ifp->if_snd);
return;
}

prod = txr->txr_prod;
free = txr->txr_cons;
if (free <= prod)
free += sc->sc_tx_ring_ndescs;
free -= prod;

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem),
0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTWRITE);

ring = IXL_DMA_KVA(&txr->txr_mem);
mask = sc->sc_tx_ring_ndescs - 1;
last = prod;
cmd = 0;
txd = NULL;

for (;;) {
if (free <= IXL_TX_PKT_DESCS) {
if (!is_transmit)
SET(ifp->if_flags, IFF_OACTIVE);
break;
}

if (is_transmit)
m = pcq_get(txr->txr_intrq);
else
IFQ_DEQUEUE(&ifp->if_snd, m);

if (m == NULL)
break;

txm = &txr->txr_maps[prod];
map = txm->txm_map;

if (ixl_load_mbuf(sc->sc_dmat, map, &m, txr) != 0) {
if_statinc(ifp, if_oerrors);
m_freem(m);
continue;
}

cmd_txd = 0;
if (m->m_pkthdr.csum_flags & IXL_CSUM_ALL_OFFLOAD) {
ixl_tx_setup_offloads(m, &cmd_txd);
}

if (vlan_has_tag(m)) {
uint16_t vtag;
vtag = htole16(vlan_get_tag(m));
cmd_txd |= (uint64_t)vtag <<
IXL_TX_DESC_L2TAG1_SHIFT;
cmd_txd |= IXL_TX_DESC_CMD_IL2TAG1;
}

bus_dmamap_sync(sc->sc_dmat, map, 0,
map->dm_mapsize, BUS_DMASYNC_PREWRITE);

for (i = 0; i < (unsigned int)map->dm_nsegs; i++) {
txd = &ring[prod];

cmd = (uint64_t)map->dm_segs[i].ds_len <<
IXL_TX_DESC_BSIZE_SHIFT;
cmd |= IXL_TX_DESC_DTYPE_DATA | IXL_TX_DESC_CMD_ICRC;
cmd |= cmd_txd;

txd->addr = htole64(map->dm_segs[i].ds_addr);
txd->cmd = htole64(cmd);

last = prod;

prod++;
prod &= mask;
}
cmd |= IXL_TX_DESC_CMD_EOP | IXL_TX_DESC_CMD_RS;
txd->cmd = htole64(cmd);

txm->txm_m = m;
txm->txm_eop = last;

bpf_mtap(ifp, m, BPF_D_OUT);

free -= i;
post = 1;
}

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem),
0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREWRITE);

if (post) {
txr->txr_prod = prod;
ixl_wr(sc, txr->txr_tail, prod);
}
}

static int
ixl_txeof(struct ixl_softc *sc, struct ixl_tx_ring *txr, u_int txlimit)
{
struct ifnet *ifp = &sc->sc_ec.ec_if;
struct ixl_tx_desc *ring, *txd;
struct ixl_tx_map *txm;
struct mbuf *m;
bus_dmamap_t map;
unsigned int cons, prod, last;
unsigned int mask;
uint64_t dtype;
int done = 0, more = 0;

KASSERT(mutex_owned(&txr->txr_lock));

prod = txr->txr_prod;
cons = txr->txr_cons;

if (cons == prod)
return 0;

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem),
0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_POSTREAD);

ring = IXL_DMA_KVA(&txr->txr_mem);
mask = sc->sc_tx_ring_ndescs - 1;

net_stat_ref_t nsr = IF_STAT_GETREF(ifp);

do {
if (txlimit-- <= 0) {
more = 1;
break;
}

txm = &txr->txr_maps[cons];
last = txm->txm_eop;
txd = &ring[last];

dtype = txd->cmd & htole64(IXL_TX_DESC_DTYPE_MASK);
if (dtype != htole64(IXL_TX_DESC_DTYPE_DONE))
break;

map = txm->txm_map;

bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, map);

m = txm->txm_m;
if (m != NULL) {
if_statinc_ref(ifp, nsr, if_opackets);
if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len);
if (ISSET(m->m_flags, M_MCAST))
if_statinc_ref(ifp, nsr, if_omcasts);
m_freem(m);
}

txm->txm_m = NULL;
txm->txm_eop = -1;

cons = last + 1;
cons &= mask;
done = 1;
} while (cons != prod);

IF_STAT_PUTREF(ifp);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&txr->txr_mem),
0, IXL_DMA_LEN(&txr->txr_mem), BUS_DMASYNC_PREREAD);

txr->txr_cons = cons;

if (done) {
softint_schedule(txr->txr_si);
if (txr->txr_qid == 0) {
CLR(ifp->if_flags, IFF_OACTIVE);
if_schedule_deferred_start(ifp);
}
}

return more;
}

static void
ixl_start(struct ifnet *ifp)
{
struct ixl_softc *sc;
struct ixl_tx_ring *txr;

sc = ifp->if_softc;
txr = sc->sc_qps[0].qp_txr;

mutex_enter(&txr->txr_lock);
ixl_tx_common_locked(ifp, txr, false);
mutex_exit(&txr->txr_lock);
}

static inline unsigned int
ixl_select_txqueue(struct ixl_softc *sc, struct mbuf *m)
{
u_int cpuid;

cpuid = cpu_index(curcpu());

return (unsigned int)(cpuid % sc->sc_nqueue_pairs);
}

static int
ixl_transmit(struct ifnet *ifp, struct mbuf *m)
{
struct ixl_softc *sc;
struct ixl_tx_ring *txr;
unsigned int qid;

sc = ifp->if_softc;
qid = ixl_select_txqueue(sc, m);

txr = sc->sc_qps[qid].qp_txr;

if (__predict_false(!pcq_put(txr->txr_intrq, m))) {
mutex_enter(&txr->txr_lock);
txr->txr_pcqdrop.ev_count++;
mutex_exit(&txr->txr_lock);

m_freem(m);
return ENOBUFS;
}

#ifdef IXL_ALWAYS_TXDEFER
kpreempt_disable();
softint_schedule(txr->txr_si);
kpreempt_enable();
#else
if (mutex_tryenter(&txr->txr_lock)) {
ixl_tx_common_locked(ifp, txr, true);
mutex_exit(&txr->txr_lock);
} else {
kpreempt_disable();
softint_schedule(txr->txr_si);
kpreempt_enable();
}
#endif

return 0;
}

static void
ixl_deferred_transmit(void *xtxr)
{
struct ixl_tx_ring *txr = xtxr;
struct ixl_softc *sc = txr->txr_sc;
struct ifnet *ifp = &sc->sc_ec.ec_if;

mutex_enter(&txr->txr_lock);
txr->txr_transmitdef.ev_count++;
if (pcq_peek(txr->txr_intrq) != NULL)
ixl_tx_common_locked(ifp, txr, true);
mutex_exit(&txr->txr_lock);
}

static struct ixl_rx_ring *
ixl_rxr_alloc(struct ixl_softc *sc, unsigned int qid)
{
struct ixl_rx_ring *rxr = NULL;
struct ixl_rx_map *maps = NULL, *rxm;
unsigned int i;

rxr = kmem_zalloc(sizeof(*rxr), KM_SLEEP);
maps = kmem_zalloc(sizeof(maps[0]) * sc->sc_rx_ring_ndescs,
KM_SLEEP);

if (ixl_dmamem_alloc(sc, &rxr->rxr_mem,
sizeof(struct ixl_rx_rd_desc_32) * sc->sc_rx_ring_ndescs,
IXL_RX_QUEUE_ALIGN) != 0)
goto free;

for (i = 0; i < sc->sc_rx_ring_ndescs; i++) {
rxm = &maps[i];

if (bus_dmamap_create(sc->sc_dmat,
IXL_MCLBYTES, 1, IXL_MCLBYTES, 0,
BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &rxm->rxm_map) != 0)
goto uncreate;

rxm->rxm_m = NULL;
}

rxr->rxr_cons = rxr->rxr_prod = 0;
rxr->rxr_m_head = NULL;
rxr->rxr_m_tail = &rxr->rxr_m_head;
rxr->rxr_maps = maps;

rxr->rxr_tail = I40E_QRX_TAIL(qid);
rxr->rxr_qid = qid;
mutex_init(&rxr->rxr_lock, MUTEX_DEFAULT, IPL_NET);

return rxr;

uncreate:
for (i = 0; i < sc->sc_rx_ring_ndescs; i++) {
rxm = &maps[i];

if (rxm->rxm_map == NULL)
continue;

bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map);
}

ixl_dmamem_free(sc, &rxr->rxr_mem);
free:
kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs);
kmem_free(rxr, sizeof(*rxr));

return NULL;
}

static void
ixl_rxr_clean(struct ixl_softc *sc, struct ixl_rx_ring *rxr)
{
struct ixl_rx_map *maps, *rxm;
bus_dmamap_t map;
unsigned int i;

maps = rxr->rxr_maps;
for (i = 0; i < sc->sc_rx_ring_ndescs; i++) {
rxm = &maps[i];

if (rxm->rxm_m == NULL)
continue;

map = rxm->rxm_map;
bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
BUS_DMASYNC_POSTWRITE);
bus_dmamap_unload(sc->sc_dmat, map);

m_freem(rxm->rxm_m);
rxm->rxm_m = NULL;
}

m_freem(rxr->rxr_m_head);
rxr->rxr_m_head = NULL;
rxr->rxr_m_tail = &rxr->rxr_m_head;

rxr->rxr_prod = rxr->rxr_cons = 0;
}

static int
ixl_rxr_enabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr)
{
bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid);
uint32_t reg;
int i;

for (i = 0; i < 10; i++) {
reg = ixl_rd(sc, ena);
if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK))
return 0;

delaymsec(10);
}

return ETIMEDOUT;
}

static int
ixl_rxr_disabled(struct ixl_softc *sc, struct ixl_rx_ring *rxr)
{
bus_size_t ena = I40E_QRX_ENA(rxr->rxr_qid);
uint32_t reg;
int i;

KASSERT(mutex_owned(&rxr->rxr_lock));

for (i = 0; i < 10; i++) {
reg = ixl_rd(sc, ena);
if (ISSET(reg, I40E_QRX_ENA_QENA_STAT_MASK) == 0)
return 0;

delaymsec(10);
}

return ETIMEDOUT;
}

static void
ixl_rxr_config(struct ixl_softc *sc, struct ixl_rx_ring *rxr)
{
struct ixl_hmc_rxq rxq;
struct ifnet *ifp = &sc->sc_ec.ec_if;
uint16_t rxmax;
void *hmc;

memset(&rxq, 0, sizeof(rxq));
rxmax = ifp->if_mtu + IXL_MTU_ETHERLEN;
if (!ISSET(sc->sc_ec.ec_capenable, ETHERCAP_VLAN_MTU))
rxmax -= ETHER_VLAN_ENCAP_LEN;

rxq.head = htole16(rxr->rxr_cons);
rxq.base = htole64(IXL_DMA_DVA(&rxr->rxr_mem) / IXL_HMC_RXQ_BASE_UNIT);
rxq.qlen = htole16(sc->sc_rx_ring_ndescs);
rxq.dbuff = htole16(IXL_MCLBYTES / IXL_HMC_RXQ_DBUFF_UNIT);
rxq.hbuff = 0;
rxq.dtype = IXL_HMC_RXQ_DTYPE_NOSPLIT;
rxq.dsize = IXL_HMC_RXQ_DSIZE_32;
rxq.crcstrip = 1;
rxq.l2sel = 1;
rxq.showiv = 1;
rxq.rxmax = htole16(rxmax);
rxq.tphrdesc_ena = 0;
rxq.tphwdesc_ena = 0;
rxq.tphdata_ena = 0;
rxq.tphhead_ena = 0;
rxq.lrxqthresh = 0;
rxq.prefena = 1;

hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid);
memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX));
ixl_hmc_pack(hmc, &rxq, ixl_hmc_pack_rxq,
__arraycount(ixl_hmc_pack_rxq));
}

static void
ixl_rxr_unconfig(struct ixl_softc *sc, struct ixl_rx_ring *rxr)
{
void *hmc;

hmc = ixl_hmc_kva(sc, IXL_HMC_LAN_RX, rxr->rxr_qid);
memset(hmc, 0, ixl_hmc_len(sc, IXL_HMC_LAN_RX));
rxr->rxr_cons = rxr->rxr_prod = 0;
}

static void
ixl_rxr_free(struct ixl_softc *sc, struct ixl_rx_ring *rxr)
{
struct ixl_rx_map *maps, *rxm;
unsigned int i;

maps = rxr->rxr_maps;
for (i = 0; i < sc->sc_rx_ring_ndescs; i++) {
rxm = &maps[i];

bus_dmamap_destroy(sc->sc_dmat, rxm->rxm_map);
}

ixl_dmamem_free(sc, &rxr->rxr_mem);
mutex_destroy(&rxr->rxr_lock);
kmem_free(maps, sizeof(maps[0]) * sc->sc_rx_ring_ndescs);
kmem_free(rxr, sizeof(*rxr));
}

static inline void
ixl_rx_csum(struct mbuf *m, uint64_t qword)
{
int flags_mask;

if (!ISSET(qword, IXL_RX_DESC_L3L4P)) {
/* No L3 or L4 checksum was calculated */
return;
}

switch (__SHIFTOUT(qword, IXL_RX_DESC_PTYPE_MASK)) {
case IXL_RX_DESC_PTYPE_IPV4FRAG:
case IXL_RX_DESC_PTYPE_IPV4:
case IXL_RX_DESC_PTYPE_SCTPV4:
case IXL_RX_DESC_PTYPE_ICMPV4:
flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD;
break;
case IXL_RX_DESC_PTYPE_TCPV4:
flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD;
flags_mask |= M_CSUM_TCPv4 | M_CSUM_TCP_UDP_BAD;
break;
case IXL_RX_DESC_PTYPE_UDPV4:
flags_mask = M_CSUM_IPv4 | M_CSUM_IPv4_BAD;
flags_mask |= M_CSUM_UDPv4 | M_CSUM_TCP_UDP_BAD;
break;
case IXL_RX_DESC_PTYPE_TCPV6:
flags_mask = M_CSUM_TCPv6 | M_CSUM_TCP_UDP_BAD;
break;
case IXL_RX_DESC_PTYPE_UDPV6:
flags_mask = M_CSUM_UDPv6 | M_CSUM_TCP_UDP_BAD;
break;
default:
flags_mask = 0;
}

m->m_pkthdr.csum_flags |= (flags_mask & (M_CSUM_IPv4 |
M_CSUM_TCPv4 | M_CSUM_TCPv6 | M_CSUM_UDPv4 | M_CSUM_UDPv6));

if (ISSET(qword, IXL_RX_DESC_IPE)) {
m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_IPv4_BAD);
}

if (ISSET(qword, IXL_RX_DESC_L4E)) {
m->m_pkthdr.csum_flags |= (flags_mask & M_CSUM_TCP_UDP_BAD);
}
}

static int
ixl_rxeof(struct ixl_softc *sc, struct ixl_rx_ring *rxr, u_int rxlimit)
{
struct ifnet *ifp = &sc->sc_ec.ec_if;
struct ixl_rx_wb_desc_32 *ring, *rxd;
struct ixl_rx_map *rxm;
bus_dmamap_t map;
unsigned int cons, prod;
struct mbuf *m;
uint64_t word, word0;
unsigned int len;
unsigned int mask;
int done = 0, more = 0;

KASSERT(mutex_owned(&rxr->rxr_lock));

if (!ISSET(ifp->if_flags, IFF_RUNNING))
return 0;

prod = rxr->rxr_prod;
cons = rxr->rxr_cons;

if (cons == prod)
return 0;

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem),
0, IXL_DMA_LEN(&rxr->rxr_mem),
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);

ring = IXL_DMA_KVA(&rxr->rxr_mem);
mask = sc->sc_rx_ring_ndescs - 1;

net_stat_ref_t nsr = IF_STAT_GETREF(ifp);

do {
if (rxlimit-- <= 0) {
more = 1;
break;
}

rxd = &ring[cons];

word = le64toh(rxd->qword1);

if (!ISSET(word, IXL_RX_DESC_DD))
break;

rxm = &rxr->rxr_maps[cons];

map = rxm->rxm_map;
bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
BUS_DMASYNC_POSTREAD);
bus_dmamap_unload(sc->sc_dmat, map);

m = rxm->rxm_m;
rxm->rxm_m = NULL;

KASSERT(m != NULL);

len = (word & IXL_RX_DESC_PLEN_MASK) >> IXL_RX_DESC_PLEN_SHIFT;
m->m_len = len;
m->m_pkthdr.len = 0;

m->m_next = NULL;
*rxr->rxr_m_tail = m;
rxr->rxr_m_tail = &m->m_next;

m = rxr->rxr_m_head;
m->m_pkthdr.len += len;

if (ISSET(word, IXL_RX_DESC_EOP)) {
word0 = le64toh(rxd->qword0);

if (ISSET(word, IXL_RX_DESC_L2TAG1P)) {
uint16_t vtag;
vtag = __SHIFTOUT(word0, IXL_RX_DESC_L2TAG1_MASK);
vlan_set_tag(m, le16toh(vtag));
}

if ((ifp->if_capenable & IXL_IFCAP_RXCSUM) != 0)
ixl_rx_csum(m, word);

if (!ISSET(word,
IXL_RX_DESC_RXE | IXL_RX_DESC_OVERSIZE)) {
m_set_rcvif(m, ifp);
if_statinc_ref(ifp, nsr, if_ipackets);
if_statadd_ref(ifp, nsr, if_ibytes,
m->m_pkthdr.len);
if_percpuq_enqueue(sc->sc_ipq, m);
} else {
if_statinc_ref(ifp, nsr, if_ierrors);
m_freem(m);
}

rxr->rxr_m_head = NULL;
rxr->rxr_m_tail = &rxr->rxr_m_head;
}

cons++;
cons &= mask;

done = 1;
} while (cons != prod);

if (done) {
rxr->rxr_cons = cons;
if (ixl_rxfill(sc, rxr) == -1)
if_statinc_ref(ifp, nsr, if_iqdrops);
}

IF_STAT_PUTREF(ifp);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&rxr->rxr_mem),
0, IXL_DMA_LEN(&rxr->rxr_mem),
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);

return more;
}

static int
ixl_rxfill(struct ixl_softc *sc, struct ixl_rx_ring *rxr)
{
struct ixl_rx_rd_desc_32 *ring, *rxd;
struct ixl_rx_map *rxm;
bus_dmamap_t map;
struct mbuf *m;
unsigned int prod;
unsigned int slots;
unsigned int mask;
int post = 0, error = 0;

KASSERT(mutex_owned(&rxr->rxr_lock));

prod = rxr->rxr_prod;
slots = ixl_rxr_unrefreshed(rxr->rxr_prod, rxr->rxr_cons,
sc->sc_rx_ring_ndescs);

ring = IXL_DMA_KVA(&rxr->rxr_mem);
mask = sc->sc_rx_ring_ndescs - 1;

if (__predict_false(slots <= 0))
return -1;

do {
rxm = &rxr->rxr_maps[prod];

MGETHDR(m, M_DONTWAIT, MT_DATA);
if (m == NULL) {
rxr->rxr_mgethdr_failed.ev_count++;
error = -1;
break;
}

MCLGET(m, M_DONTWAIT);
if (!ISSET(m->m_flags, M_EXT)) {
rxr->rxr_mgetcl_failed.ev_count++;
error = -1;
m_freem(m);
break;
}

m->m_len = m->m_pkthdr.len = MCLBYTES;
m_adj(m, ETHER_ALIGN);

map = rxm->rxm_map;

if (bus_dmamap_load_mbuf(sc->sc_dmat, map, m,
BUS_DMA_READ | BUS_DMA_NOWAIT) != 0) {
rxr->rxr_mbuf_load_failed.ev_count++;
error = -1;
m_freem(m);
break;
}

rxm->rxm_m = m;

bus_dmamap_sync(sc->sc_dmat, map, 0, map->dm_mapsize,
BUS_DMASYNC_PREREAD);

rxd = &ring[prod];

rxd->paddr = htole64(map->dm_segs[0].ds_addr);
rxd->haddr = htole64(0);

prod++;
prod &= mask;

post = 1;

} while (--slots);

if (post) {
rxr->rxr_prod = prod;
ixl_wr(sc, rxr->rxr_tail, prod);
}

return error;
}

static inline int
ixl_handle_queue_common(struct ixl_softc *sc, struct ixl_queue_pair *qp,
u_int txlimit, struct evcnt *txevcnt,
u_int rxlimit, struct evcnt *rxevcnt)
{
struct ixl_tx_ring *txr = qp->qp_txr;
struct ixl_rx_ring *rxr = qp->qp_rxr;
int txmore, rxmore;
int rv;

mutex_enter(&txr->txr_lock);
txevcnt->ev_count++;
txmore = ixl_txeof(sc, txr, txlimit);
mutex_exit(&txr->txr_lock);

mutex_enter(&rxr->rxr_lock);
rxevcnt->ev_count++;
rxmore = ixl_rxeof(sc, rxr, rxlimit);
mutex_exit(&rxr->rxr_lock);

rv = txmore | (rxmore << 1);

return rv;
}

static void
ixl_sched_handle_queue(struct ixl_softc *sc, struct ixl_queue_pair *qp)
{

if (qp->qp_workqueue)
workqueue_enqueue(sc->sc_workq_txrx, &qp->qp_work, NULL);
else
softint_schedule(qp->qp_si);
}

static int
ixl_intr(void *xsc)
{
struct ixl_softc *sc = xsc;
struct ixl_tx_ring *txr;
struct ixl_rx_ring *rxr;
uint32_t icr, rxintr, txintr;
int rv = 0;
unsigned int i;

KASSERT(sc != NULL);

ixl_enable_other_intr(sc);
icr = ixl_rd(sc, I40E_PFINT_ICR0);

if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) {
atomic_inc_64(&sc->sc_event_atq.ev_count);
ixl_atq_done(sc);
ixl_work_add(sc->sc_workq, &sc->sc_arq_task);
rv = 1;
}

if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) {
atomic_inc_64(&sc->sc_event_link.ev_count);
ixl_work_add(sc->sc_workq, &sc->sc_link_state_task);
rv = 1;
}

rxintr = icr & I40E_INTR_NOTX_RX_MASK;
txintr = icr & I40E_INTR_NOTX_TX_MASK;

if (txintr || rxintr) {
for (i = 0; i < sc->sc_nqueue_pairs; i++) {
txr = sc->sc_qps[i].qp_txr;
rxr = sc->sc_qps[i].qp_rxr;

ixl_handle_queue_common(sc, &sc->sc_qps[i],
IXL_TXRX_PROCESS_UNLIMIT, &txr->txr_intr,
IXL_TXRX_PROCESS_UNLIMIT, &rxr->rxr_intr);
}
rv = 1;
}

return rv;
}

static int
ixl_queue_intr(void *xqp)
{
struct ixl_queue_pair *qp = xqp;
struct ixl_tx_ring *txr = qp->qp_txr;
struct ixl_rx_ring *rxr = qp->qp_rxr;
struct ixl_softc *sc = qp->qp_sc;
u_int txlimit, rxlimit;
int more;

txlimit = sc->sc_tx_intr_process_limit;
rxlimit = sc->sc_rx_intr_process_limit;
qp->qp_workqueue = sc->sc_txrx_workqueue;

more = ixl_handle_queue_common(sc, qp,
txlimit, &txr->txr_intr, rxlimit, &rxr->rxr_intr);

if (more != 0) {
ixl_sched_handle_queue(sc, qp);
} else {
/* for ALTQ */
if (txr->txr_qid == 0)
if_schedule_deferred_start(&sc->sc_ec.ec_if);
softint_schedule(txr->txr_si);

ixl_enable_queue_intr(sc, qp);
}

return 1;
}

static void
ixl_handle_queue_wk(struct work *wk, void *xsc)
{
struct ixl_queue_pair *qp;

qp = container_of(wk, struct ixl_queue_pair, qp_work);
ixl_handle_queue(qp);
}

static void
ixl_handle_queue(void *xqp)
{
struct ixl_queue_pair *qp = xqp;
struct ixl_softc *sc = qp->qp_sc;
struct ixl_tx_ring *txr = qp->qp_txr;
struct ixl_rx_ring *rxr = qp->qp_rxr;
u_int txlimit, rxlimit;
int more;

txlimit = sc->sc_tx_process_limit;
rxlimit = sc->sc_rx_process_limit;

more = ixl_handle_queue_common(sc, qp,
txlimit, &txr->txr_defer, rxlimit, &rxr->rxr_defer);

if (more != 0)
ixl_sched_handle_queue(sc, qp);
else
ixl_enable_queue_intr(sc, qp);
}

static inline void
ixl_print_hmc_error(struct ixl_softc *sc, uint32_t reg)
{
uint32_t hmc_idx, hmc_isvf;
uint32_t hmc_errtype, hmc_objtype, hmc_data;

hmc_idx = reg & I40E_PFHMC_ERRORINFO_PMF_INDEX_MASK;
hmc_idx = hmc_idx >> I40E_PFHMC_ERRORINFO_PMF_INDEX_SHIFT;
hmc_isvf = reg & I40E_PFHMC_ERRORINFO_PMF_ISVF_MASK;
hmc_isvf = hmc_isvf >> I40E_PFHMC_ERRORINFO_PMF_ISVF_SHIFT;
hmc_errtype = reg & I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_MASK;
hmc_errtype = hmc_errtype >> I40E_PFHMC_ERRORINFO_HMC_ERROR_TYPE_SHIFT;
hmc_objtype = reg & I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_MASK;
hmc_objtype = hmc_objtype >> I40E_PFHMC_ERRORINFO_HMC_OBJECT_TYPE_SHIFT;
hmc_data = ixl_rd(sc, I40E_PFHMC_ERRORDATA);

device_printf(sc->sc_dev,
"HMC Error (idx=0x%x, isvf=0x%x, err=0x%x, obj=0x%x, data=0x%x)\n",
hmc_idx, hmc_isvf, hmc_errtype, hmc_objtype, hmc_data);
}

static int
ixl_other_intr(void *xsc)
{
struct ixl_softc *sc = xsc;
uint32_t icr, mask, reg;
int rv = 0;

icr = ixl_rd(sc, I40E_PFINT_ICR0);
mask = ixl_rd(sc, I40E_PFINT_ICR0_ENA);

if (ISSET(icr, I40E_PFINT_ICR0_ADMINQ_MASK)) {
atomic_inc_64(&sc->sc_event_atq.ev_count);
ixl_atq_done(sc);
ixl_work_add(sc->sc_workq, &sc->sc_arq_task);
rv = 1;
}

if (ISSET(icr, I40E_PFINT_ICR0_LINK_STAT_CHANGE_MASK)) {
if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG))
device_printf(sc->sc_dev, "link stat changed\n");

atomic_inc_64(&sc->sc_event_link.ev_count);
ixl_work_add(sc->sc_workq, &sc->sc_link_state_task);
rv = 1;
}

if (ISSET(icr, I40E_PFINT_ICR0_GRST_MASK)) {
CLR(mask, I40E_PFINT_ICR0_ENA_GRST_MASK);
reg = ixl_rd(sc, I40E_GLGEN_RSTAT);
reg = reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK;
reg = reg >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;

device_printf(sc->sc_dev, "GRST: %s\n",
reg == I40E_RESET_CORER ? "CORER" :
reg == I40E_RESET_GLOBR ? "GLOBR" :
reg == I40E_RESET_EMPR ? "EMPR" :
"POR");
}

if (ISSET(icr, I40E_PFINT_ICR0_ECC_ERR_MASK))
atomic_inc_64(&sc->sc_event_ecc_err.ev_count);
if (ISSET(icr, I40E_PFINT_ICR0_PCI_EXCEPTION_MASK))
atomic_inc_64(&sc->sc_event_pci_exception.ev_count);
if (ISSET(icr, I40E_PFINT_ICR0_PE_CRITERR_MASK))
atomic_inc_64(&sc->sc_event_crit_err.ev_count);

if (ISSET(icr, IXL_ICR0_CRIT_ERR_MASK)) {
CLR(mask, IXL_ICR0_CRIT_ERR_MASK);
device_printf(sc->sc_dev, "critical error\n");
}

if (ISSET(icr, I40E_PFINT_ICR0_HMC_ERR_MASK)) {
reg = ixl_rd(sc, I40E_PFHMC_ERRORINFO);
if (ISSET(reg, I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK))
ixl_print_hmc_error(sc, reg);
ixl_wr(sc, I40E_PFHMC_ERRORINFO, 0);
}

ixl_wr(sc, I40E_PFINT_ICR0_ENA, mask);
ixl_flush(sc);
ixl_enable_other_intr(sc);
return rv;
}

static void
ixl_get_link_status_done_work(void *xsc)
{
struct ixl_softc *sc = xsc;
struct ixl_aq_desc *iaq, iaq_buf;

mutex_enter(&sc->sc_atq_lock);
iaq = &sc->sc_link_state_atq.iatq_desc;
iaq_buf = *iaq;
mutex_exit(&sc->sc_atq_lock);

ixl_link_state_update(sc, &iaq_buf);

mutex_enter(&sc->sc_atq_lock);
CLR(iaq->iaq_flags, htole16(IXL_AQ_DD));
ixl_wakeup(sc, iaq);
mutex_exit(&sc->sc_atq_lock);
}

static void
ixl_get_link_status_done(struct ixl_softc *sc,
const struct ixl_aq_desc *iaq)
{

ixl_work_add(sc->sc_workq, &sc->sc_link_state_done_task);
}

static int
ixl_get_link_status(struct ixl_softc *sc, enum ixl_link_flags flags)
{
struct ixl_atq *iatq;
struct ixl_aq_desc *iaq;
struct ixl_aq_link_param *param;
int error;

mutex_enter(&sc->sc_atq_lock);

iatq = &sc->sc_link_state_atq;
iaq = &iatq->iatq_desc;

if (!sc->sc_link_state_atq.iatq_inuse &&
!ISSET(iaq->iaq_flags, htole16(IXL_AQ_DD))) {
memset(iaq, 0, sizeof(*iaq));
iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS);
param = (struct ixl_aq_link_param *)iaq->iaq_param;
param->notify = IXL_AQ_LINK_NOTIFY;

KASSERT(iatq->iatq_fn == ixl_get_link_status_done);
error = ixl_atq_post_locked(sc, iatq);
if (error != 0)
goto out;
} else {
/* the previous command is not completed */
error = EBUSY;
}

if (ISSET(flags, IXL_LINK_FLAG_WAITDONE)) {
do {
error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock,
IXL_ATQ_EXEC_TIMEOUT);
if (error == EWOULDBLOCK)
break;
} while (iatq->iatq_inuse ||
ISSET(iaq->iaq_flags, htole16(IXL_AQ_DD)));
}

out:
mutex_exit(&sc->sc_atq_lock);

return error;
}

static void
ixl_get_link_status_work(void *xsc)
{
struct ixl_softc *sc = xsc;

/*
* IXL_LINK_FLAG_WAITDONE causes deadlock
* because of doing ixl_gt_link_status_done_work()
* in the same workqueue.
*/
(void)ixl_get_link_status(sc, IXL_LINK_NOFLAGS);
}

static void
ixl_link_state_update(struct ixl_softc *sc, const struct ixl_aq_desc *iaq)
{
struct ifnet *ifp = &sc->sc_ec.ec_if;
int link_state;

mutex_enter(&sc->sc_cfg_lock);
link_state = ixl_set_link_status_locked(sc, iaq);
mutex_exit(&sc->sc_cfg_lock);

if (ifp->if_link_state != link_state)
if_link_state_change(ifp, link_state);

if (link_state != LINK_STATE_DOWN) {
kpreempt_disable();
if_schedule_deferred_start(ifp);
kpreempt_enable();
}
}

static void
ixl_aq_dump(const struct ixl_softc *sc, const struct ixl_aq_desc *iaq,
const char *msg)
{
char buf[512];
size_t len;

len = sizeof(buf);
buf[--len] = '\0';

device_printf(sc->sc_dev, "%s\n", msg);
snprintb(buf, len, IXL_AQ_FLAGS_FMT, le16toh(iaq->iaq_flags));
device_printf(sc->sc_dev, "flags %s opcode %04x\n",
buf, le16toh(iaq->iaq_opcode));
device_printf(sc->sc_dev, "datalen %u retval %u\n",
le16toh(iaq->iaq_datalen), le16toh(iaq->iaq_retval));
device_printf(sc->sc_dev, "cookie %016" PRIx64 "\n", iaq->iaq_cookie);
device_printf(sc->sc_dev, "%08x %08x %08x %08x\n",
le32toh(iaq->iaq_param[0]), le32toh(iaq->iaq_param[1]),
le32toh(iaq->iaq_param[2]), le32toh(iaq->iaq_param[3]));
}

static void
ixl_arq(void *xsc)
{
struct ixl_softc *sc = xsc;
struct ixl_aq_desc *arq, *iaq;
struct ixl_aq_buf *aqb;
unsigned int cons = sc->sc_arq_cons;
unsigned int prod;
int done = 0;

prod = ixl_rd(sc, sc->sc_aq_regs->arq_head) &
sc->sc_aq_regs->arq_head_mask;

if (cons == prod)
goto done;

arq = IXL_DMA_KVA(&sc->sc_arq);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq),
0, IXL_DMA_LEN(&sc->sc_arq),
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);

do {
iaq = &arq[cons];
aqb = sc->sc_arq_live[cons];

KASSERT(aqb != NULL);

bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, IXL_AQ_BUFLEN,
BUS_DMASYNC_POSTREAD);

if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG))
ixl_aq_dump(sc, iaq, "arq event");

switch (iaq->iaq_opcode) {
case htole16(IXL_AQ_OP_PHY_LINK_STATUS):
ixl_link_state_update(sc, iaq);
break;
}

memset(iaq, 0, sizeof(*iaq));
sc->sc_arq_live[cons] = NULL;
SIMPLEQ_INSERT_TAIL(&sc->sc_arq_idle, aqb, aqb_entry);

cons++;
cons &= IXL_AQ_MASK;

done = 1;
} while (cons != prod);

if (done) {
sc->sc_arq_cons = cons;
ixl_arq_fill(sc);
bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_arq),
0, IXL_DMA_LEN(&sc->sc_arq),
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
}

done:
ixl_enable_other_intr(sc);
}

static void
ixl_atq_set(struct ixl_atq *iatq,
void (*fn)(struct ixl_softc *, const struct ixl_aq_desc *))
{

iatq->iatq_fn = fn;
}

static int
ixl_atq_post_locked(struct ixl_softc *sc, struct ixl_atq *iatq)
{
struct ixl_aq_desc *atq, *slot;
unsigned int prod, cons, prod_next;

/* assert locked */
KASSERT(mutex_owned(&sc->sc_atq_lock));

atq = IXL_DMA_KVA(&sc->sc_atq);
prod = sc->sc_atq_prod;
cons = sc->sc_atq_cons;
prod_next = (prod +1) & IXL_AQ_MASK;

if (cons == prod_next)
return ENOMEM;

slot = &atq[prod];

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE);

KASSERT(iatq->iatq_fn != NULL);
*slot = iatq->iatq_desc;
slot->iaq_cookie = (uint64_t)((intptr_t)iatq);

if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG))
ixl_aq_dump(sc, slot, "atq command");

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE);

sc->sc_atq_prod = prod_next;
ixl_wr(sc, sc->sc_aq_regs->atq_tail, sc->sc_atq_prod);
iatq->iatq_inuse = true;

return 0;
}

static void
ixl_atq_done_locked(struct ixl_softc *sc)
{
struct ixl_aq_desc *atq, *slot;
struct ixl_atq *iatq;
unsigned int cons;
unsigned int prod;

KASSERT(mutex_owned(&sc->sc_atq_lock));

prod = sc->sc_atq_prod;
cons = sc->sc_atq_cons;

if (prod == cons)
return;

atq = IXL_DMA_KVA(&sc->sc_atq);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq),
BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);

do {
slot = &atq[cons];
if (!ISSET(slot->iaq_flags, htole16(IXL_AQ_DD)))
break;

iatq = (struct ixl_atq *)((intptr_t)slot->iaq_cookie);
iatq->iatq_desc = *slot;
iatq->iatq_inuse = false;

memset(slot, 0, sizeof(*slot));

if (ISSET(sc->sc_ec.ec_if.if_flags, IFF_DEBUG))
ixl_aq_dump(sc, &iatq->iatq_desc, "atq response");

(*iatq->iatq_fn)(sc, &iatq->iatq_desc);

cons++;
cons &= IXL_AQ_MASK;
} while (cons != prod);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq),
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);

sc->sc_atq_cons = cons;
}

static void
ixl_atq_done(struct ixl_softc *sc)
{

mutex_enter(&sc->sc_atq_lock);
ixl_atq_done_locked(sc);
mutex_exit(&sc->sc_atq_lock);
}

static void
ixl_wakeup(struct ixl_softc *sc, const struct ixl_aq_desc *iaq)
{

KASSERT(mutex_owned(&sc->sc_atq_lock));

cv_broadcast(&sc->sc_atq_cv);
}

static int
ixl_atq_exec(struct ixl_softc *sc, struct ixl_atq *iatq)
{
int error;

mutex_enter(&sc->sc_atq_lock);
error = ixl_atq_exec_locked(sc, iatq);
mutex_exit(&sc->sc_atq_lock);

return error;
}

static int
ixl_atq_exec_locked(struct ixl_softc *sc, struct ixl_atq *iatq)
{
int error;

KASSERT(mutex_owned(&sc->sc_atq_lock));
KASSERT(iatq->iatq_desc.iaq_cookie == 0);

ixl_atq_set(iatq, ixl_wakeup);

error = ixl_atq_post_locked(sc, iatq);
if (error)
return error;

do {
error = cv_timedwait(&sc->sc_atq_cv, &sc->sc_atq_lock,
IXL_ATQ_EXEC_TIMEOUT);
if (error == EWOULDBLOCK)
break;
} while (iatq->iatq_inuse);

return error;
}

static int
ixl_atq_poll(struct ixl_softc *sc, struct ixl_aq_desc *iaq, unsigned int tm)
{
struct ixl_aq_desc *atq, *slot;
unsigned int prod;
unsigned int t = 0;

mutex_enter(&sc->sc_atq_lock);

atq = IXL_DMA_KVA(&sc->sc_atq);
prod = sc->sc_atq_prod;
slot = atq + prod;

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTWRITE);

*slot = *iaq;
slot->iaq_flags |= htole16(IXL_AQ_SI);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREWRITE);

prod++;
prod &= IXL_AQ_MASK;
sc->sc_atq_prod = prod;
ixl_wr(sc, sc->sc_aq_regs->atq_tail, prod);

while (ixl_rd(sc, sc->sc_aq_regs->atq_head) != prod) {
delaymsec(1);

if (t++ > tm) {
mutex_exit(&sc->sc_atq_lock);
return ETIMEDOUT;
}
}

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_POSTREAD);
*iaq = *slot;
memset(slot, 0, sizeof(*slot));
bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_atq),
0, IXL_DMA_LEN(&sc->sc_atq), BUS_DMASYNC_PREREAD);

sc->sc_atq_cons = prod;

mutex_exit(&sc->sc_atq_lock);

return 0;
}

static int
ixl_get_version(struct ixl_softc *sc)
{
struct ixl_aq_desc iaq;
uint32_t fwbuild, fwver, apiver;
uint16_t api_maj_ver, api_min_ver;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VERSION);

iaq.iaq_retval = le16toh(23);

if (ixl_atq_poll(sc, &iaq, 2000) != 0)
return ETIMEDOUT;
if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK))
return EIO;

fwbuild = le32toh(iaq.iaq_param[1]);
fwver = le32toh(iaq.iaq_param[2]);
apiver = le32toh(iaq.iaq_param[3]);

api_maj_ver = (uint16_t)apiver;
api_min_ver = (uint16_t)(apiver >> 16);

aprint_normal(", FW %hu.%hu.%05u API %hu.%hu", (uint16_t)fwver,
(uint16_t)(fwver >> 16), fwbuild, api_maj_ver, api_min_ver);

if (sc->sc_mac_type == I40E_MAC_X722) {
SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK |
IXL_SC_AQ_FLAG_NVMREAD);
SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL);
SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS);
}

#define IXL_API_VER(maj, min) (((uint32_t)(maj) << 16) | (min))
if (IXL_API_VER(api_maj_ver, api_min_ver) >= IXL_API_VER(1, 5)) {
SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL);
SET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK);
}
#undef IXL_API_VER

return 0;
}

static int
ixl_get_nvm_version(struct ixl_softc *sc)
{
uint16_t nvmver, cfg_ptr, eetrack_hi, eetrack_lo, oem_hi, oem_lo;
uint32_t eetrack, oem;
uint16_t nvm_maj_ver, nvm_min_ver, oem_build;
uint8_t oem_ver, oem_patch;

nvmver = cfg_ptr = eetrack_hi = eetrack_lo = oem_hi = oem_lo = 0;
ixl_rd16_nvm(sc, I40E_SR_NVM_DEV_STARTER_VERSION, &nvmver);
ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_HI, &eetrack_hi);
ixl_rd16_nvm(sc, I40E_SR_NVM_EETRACK_LO, &eetrack_lo);
ixl_rd16_nvm(sc, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr);
ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF, &oem_hi);
ixl_rd16_nvm(sc, cfg_ptr + I40E_NVM_OEM_VER_OFF + 1, &oem_lo);

nvm_maj_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_HI_MASK);
nvm_min_ver = (uint16_t)__SHIFTOUT(nvmver, IXL_NVM_VERSION_LO_MASK);
eetrack = ((uint32_t)eetrack_hi << 16) | eetrack_lo;
oem = ((uint32_t)oem_hi << 16) | oem_lo;
oem_ver = __SHIFTOUT(oem, IXL_NVM_OEMVERSION_MASK);
oem_build = __SHIFTOUT(oem, IXL_NVM_OEMBUILD_MASK);
oem_patch = __SHIFTOUT(oem, IXL_NVM_OEMPATCH_MASK);

aprint_normal(" nvm %x.%02x etid %08x oem %d.%d.%d",
nvm_maj_ver, nvm_min_ver, eetrack,
oem_ver, oem_build, oem_patch);

return 0;
}

static int
ixl_pxe_clear(struct ixl_softc *sc)
{
struct ixl_aq_desc iaq;
int rv;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_CLEAR_PXE_MODE);
iaq.iaq_param[0] = htole32(0x2);

rv = ixl_atq_poll(sc, &iaq, 250);

ixl_wr(sc, I40E_GLLAN_RCTL_0, 0x1);

if (rv != 0)
return ETIMEDOUT;

switch (iaq.iaq_retval) {
case htole16(IXL_AQ_RC_OK):
case htole16(IXL_AQ_RC_EEXIST):
break;
default:
return EIO;
}

return 0;
}

static int
ixl_lldp_shut(struct ixl_softc *sc)
{
struct ixl_aq_desc iaq;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_LLDP_STOP_AGENT);
iaq.iaq_param[0] = htole32(IXL_LLDP_SHUTDOWN);

if (ixl_atq_poll(sc, &iaq, 250) != 0) {
aprint_error_dev(sc->sc_dev, "STOP LLDP AGENT timeout\n");
return -1;
}

switch (iaq.iaq_retval) {
case htole16(IXL_AQ_RC_EMODE):
case htole16(IXL_AQ_RC_EPERM):
/* ignore silently */
default:
break;
}

return 0;
}

static void
ixl_parse_hw_capability(struct ixl_softc *sc, struct ixl_aq_capability *cap)
{
uint16_t id;
uint32_t number, logical_id;

id = le16toh(cap->cap_id);
number = le32toh(cap->number);
logical_id = le32toh(cap->logical_id);

switch (id) {
case IXL_AQ_CAP_RSS:
sc->sc_rss_table_size = number;
sc->sc_rss_table_entry_width = logical_id;
break;
case IXL_AQ_CAP_RXQ:
case IXL_AQ_CAP_TXQ:
sc->sc_nqueue_pairs_device = MIN(number,
sc->sc_nqueue_pairs_device);
break;
}
}

static int
ixl_get_hw_capabilities(struct ixl_softc *sc)
{
struct ixl_dmamem idm;
struct ixl_aq_desc iaq;
struct ixl_aq_capability *caps;
size_t i, ncaps;
bus_size_t caps_size;
uint16_t status;
int rv;

caps_size = sizeof(caps[0]) * 40;
memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_LIST_FUNC_CAP);

do {
if (ixl_dmamem_alloc(sc, &idm, caps_size, 0) != 0) {
return -1;
}

iaq.iaq_flags = htole16(IXL_AQ_BUF |
(caps_size > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0));
iaq.iaq_datalen = htole16(caps_size);
ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm));

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0,
IXL_DMA_LEN(&idm), BUS_DMASYNC_PREREAD);

rv = ixl_atq_poll(sc, &iaq, 250);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0,
IXL_DMA_LEN(&idm), BUS_DMASYNC_POSTREAD);

if (rv != 0) {
aprint_error(", HW capabilities timeout\n");
goto done;
}

status = le16toh(iaq.iaq_retval);

if (status == IXL_AQ_RC_ENOMEM) {
caps_size = le16toh(iaq.iaq_datalen);
ixl_dmamem_free(sc, &idm);
}
} while (status == IXL_AQ_RC_ENOMEM);

if (status != IXL_AQ_RC_OK) {
aprint_error(", HW capabilities error\n");
goto done;
}

caps = IXL_DMA_KVA(&idm);
ncaps = le16toh(iaq.iaq_param[1]);

for (i = 0; i < ncaps; i++) {
ixl_parse_hw_capability(sc, &caps[i]);
}

done:
ixl_dmamem_free(sc, &idm);
return rv;
}

static int
ixl_get_mac(struct ixl_softc *sc)
{
struct ixl_dmamem idm;
struct ixl_aq_desc iaq;
struct ixl_aq_mac_addresses *addrs;
int rv;

if (ixl_dmamem_alloc(sc, &idm, sizeof(*addrs), 0) != 0) {
aprint_error(", unable to allocate mac addresses\n");
return -1;
}

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_flags = htole16(IXL_AQ_BUF);
iaq.iaq_opcode = htole16(IXL_AQ_OP_MAC_ADDRESS_READ);
iaq.iaq_datalen = htole16(sizeof(*addrs));
ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm));

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm),
BUS_DMASYNC_PREREAD);

rv = ixl_atq_poll(sc, &iaq, 250);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm),
BUS_DMASYNC_POSTREAD);

if (rv != 0) {
aprint_error(", MAC ADDRESS READ timeout\n");
rv = -1;
goto done;
}
if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) {
aprint_error(", MAC ADDRESS READ error\n");
rv = -1;
goto done;
}

addrs = IXL_DMA_KVA(&idm);
if (!ISSET(iaq.iaq_param[0], htole32(IXL_AQ_MAC_PORT_VALID))) {
printf(", port address is not valid\n");
goto done;
}

memcpy(sc->sc_enaddr, addrs->port, ETHER_ADDR_LEN);
rv = 0;

done:
ixl_dmamem_free(sc, &idm);
return rv;
}

static int
ixl_get_switch_config(struct ixl_softc *sc)
{
struct ixl_dmamem idm;
struct ixl_aq_desc iaq;
struct ixl_aq_switch_config *hdr;
struct ixl_aq_switch_config_element *elms, *elm;
unsigned int nelm, i;
int rv;

if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) {
aprint_error_dev(sc->sc_dev,
"unable to allocate switch config buffer\n");
return -1;
}

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_flags = htole16(IXL_AQ_BUF |
(IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0));
iaq.iaq_opcode = htole16(IXL_AQ_OP_SWITCH_GET_CONFIG);
iaq.iaq_datalen = htole16(IXL_AQ_BUFLEN);
ixl_aq_dva(&iaq, IXL_DMA_DVA(&idm));

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm),
BUS_DMASYNC_PREREAD);

rv = ixl_atq_poll(sc, &iaq, 250);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&idm), 0, IXL_DMA_LEN(&idm),
BUS_DMASYNC_POSTREAD);

if (rv != 0) {
aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG timeout\n");
rv = -1;
goto done;
}
if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) {
aprint_error_dev(sc->sc_dev, "GET SWITCH CONFIG error\n");
rv = -1;
goto done;
}

hdr = IXL_DMA_KVA(&idm);
elms = (struct ixl_aq_switch_config_element *)(hdr + 1);

nelm = le16toh(hdr->num_reported);
if (nelm < 1) {
aprint_error_dev(sc->sc_dev, "no switch config available\n");
rv = -1;
goto done;
}

for (i = 0; i < nelm; i++) {
elm = &elms[i];

aprint_debug_dev(sc->sc_dev,
"type %x revision %u seid %04x\n",
elm->type, elm->revision, le16toh(elm->seid));
aprint_debug_dev(sc->sc_dev,
"uplink %04x downlink %04x\n",
le16toh(elm->uplink_seid),
le16toh(elm->downlink_seid));
aprint_debug_dev(sc->sc_dev,
"conntype %x scheduler %04x extra %04x\n",
elm->connection_type,
le16toh(elm->scheduler_id),
le16toh(elm->element_info));
}

elm = &elms[0];

sc->sc_uplink_seid = elm->uplink_seid;
sc->sc_downlink_seid = elm->downlink_seid;
sc->sc_seid = elm->seid;

if ((sc->sc_uplink_seid == htole16(0)) !=
(sc->sc_downlink_seid == htole16(0))) {
aprint_error_dev(sc->sc_dev, "SEIDs are misconfigured\n");
rv = -1;
goto done;
}

done:
ixl_dmamem_free(sc, &idm);
return rv;
}

static int
ixl_phy_mask_ints(struct ixl_softc *sc)
{
struct ixl_aq_desc iaq;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_EVENT_MASK);
iaq.iaq_param[2] = htole32(IXL_AQ_PHY_EV_MASK &
~(IXL_AQ_PHY_EV_LINK_UPDOWN | IXL_AQ_PHY_EV_MODULE_QUAL_FAIL |
IXL_AQ_PHY_EV_MEDIA_NA));

if (ixl_atq_poll(sc, &iaq, 250) != 0) {
aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK timeout\n");
return -1;
}
if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) {
aprint_error_dev(sc->sc_dev, "SET PHY EVENT MASK error\n");
return -1;
}

return 0;
}

static int
ixl_get_phy_abilities(struct ixl_softc *sc, struct ixl_dmamem *idm)
{
struct ixl_aq_desc iaq;
int rv;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_flags = htole16(IXL_AQ_BUF |
(IXL_DMA_LEN(idm) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0));
iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_GET_ABILITIES);
iaq.iaq_datalen = htole16(IXL_DMA_LEN(idm));
iaq.iaq_param[0] = htole32(IXL_AQ_PHY_REPORT_INIT);
ixl_aq_dva(&iaq, IXL_DMA_DVA(idm));

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm),
BUS_DMASYNC_PREREAD);

rv = ixl_atq_poll(sc, &iaq, 250);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm),
BUS_DMASYNC_POSTREAD);

if (rv != 0)
return -1;

return le16toh(iaq.iaq_retval);
}

static int
ixl_get_phy_info(struct ixl_softc *sc)
{
struct ixl_dmamem idm;
struct ixl_aq_phy_abilities *phy;
int rv;

if (ixl_dmamem_alloc(sc, &idm, IXL_AQ_BUFLEN, 0) != 0) {
aprint_error_dev(sc->sc_dev,
"unable to allocate phy abilities buffer\n");
return -1;
}

rv = ixl_get_phy_abilities(sc, &idm);
switch (rv) {
case -1:
aprint_error_dev(sc->sc_dev, "GET PHY ABILITIES timeout\n");
goto done;
case IXL_AQ_RC_OK:
break;
case IXL_AQ_RC_EIO:
aprint_error_dev(sc->sc_dev,"unable to query phy types\n");
goto done;
default:
aprint_error_dev(sc->sc_dev,
"GET PHY ABILITIES error %u\n", rv);
goto done;
}

phy = IXL_DMA_KVA(&idm);

sc->sc_phy_types = le32toh(phy->phy_type);
sc->sc_phy_types |= (uint64_t)le32toh(phy->phy_type_ext) << 32;

sc->sc_phy_abilities = phy->abilities;
sc->sc_phy_linkspeed = phy->link_speed;
sc->sc_phy_fec_cfg = phy->fec_cfg_curr_mod_ext_info &
(IXL_AQ_ENABLE_FEC_KR | IXL_AQ_ENABLE_FEC_RS |
IXL_AQ_REQUEST_FEC_KR | IXL_AQ_REQUEST_FEC_RS);
sc->sc_eee_cap = phy->eee_capability;
sc->sc_eeer_val = phy->eeer_val;
sc->sc_d3_lpan = phy->d3_lpan;

rv = 0;

done:
ixl_dmamem_free(sc, &idm);
return rv;
}

static int
ixl_set_phy_config(struct ixl_softc *sc,
uint8_t link_speed, uint8_t abilities, bool polling)
{
struct ixl_aq_phy_param *param;
struct ixl_atq iatq;
struct ixl_aq_desc *iaq;
int error;

memset(&iatq, 0, sizeof(iatq));

iaq = &iatq.iatq_desc;
iaq->iaq_opcode = htole16(IXL_AQ_OP_PHY_SET_CONFIG);
param = (struct ixl_aq_phy_param *)&iaq->iaq_param;
param->phy_types = htole32((uint32_t)sc->sc_phy_types);
param->phy_type_ext = (uint8_t)(sc->sc_phy_types >> 32);
param->link_speed = link_speed;
param->abilities = abilities | IXL_AQ_PHY_ABILITY_AUTO_LINK;
param->fec_cfg = sc->sc_phy_fec_cfg;
param->eee_capability = sc->sc_eee_cap;
param->eeer_val = sc->sc_eeer_val;
param->d3_lpan = sc->sc_d3_lpan;

if (polling)
error = ixl_atq_poll(sc, iaq, 250);
else
error = ixl_atq_exec(sc, &iatq);

if (error != 0)
return error;

switch (le16toh(iaq->iaq_retval)) {
case IXL_AQ_RC_OK:
break;
case IXL_AQ_RC_EPERM:
return EPERM;
default:
return EIO;
}

return 0;
}

static int
ixl_set_phy_autoselect(struct ixl_softc *sc)
{
uint8_t link_speed, abilities;

link_speed = sc->sc_phy_linkspeed;
abilities = IXL_PHY_ABILITY_LINKUP | IXL_PHY_ABILITY_AUTONEGO;

return ixl_set_phy_config(sc, link_speed, abilities, true);
}

static int
ixl_get_link_status_poll(struct ixl_softc *sc, int *l)
{
struct ixl_aq_desc iaq;
struct ixl_aq_link_param *param;
int link;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_LINK_STATUS);
param = (struct ixl_aq_link_param *)iaq.iaq_param;
param->notify = IXL_AQ_LINK_NOTIFY;

if (ixl_atq_poll(sc, &iaq, 250) != 0) {
return ETIMEDOUT;
}
if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) {
return EIO;
}

/* It is unnecessary to hold lock */
link = ixl_set_link_status_locked(sc, &iaq);

if (l != NULL)
*l = link;

return 0;
}

static int
ixl_get_vsi(struct ixl_softc *sc)
{
struct ixl_dmamem *vsi = &sc->sc_scratch;
struct ixl_aq_desc iaq;
struct ixl_aq_vsi_param *param;
struct ixl_aq_vsi_reply *reply;
struct ixl_aq_vsi_data *data;
int rv;

/* grumble, vsi info isn't "known" at compile time */

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_flags = htole16(IXL_AQ_BUF |
(IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0));
iaq.iaq_opcode = htole16(IXL_AQ_OP_GET_VSI_PARAMS);
iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi));
ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi));

param = (struct ixl_aq_vsi_param *)iaq.iaq_param;
param->uplink_seid = sc->sc_seid;

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi),
BUS_DMASYNC_PREREAD);

rv = ixl_atq_poll(sc, &iaq, 250);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi),
BUS_DMASYNC_POSTREAD);

if (rv != 0) {
return ETIMEDOUT;
}

switch (le16toh(iaq.iaq_retval)) {
case IXL_AQ_RC_OK:
break;
case IXL_AQ_RC_ENOENT:
return ENOENT;
case IXL_AQ_RC_EACCES:
return EACCES;
default:
return EIO;
}

reply = (struct ixl_aq_vsi_reply *)iaq.iaq_param;
sc->sc_vsi_number = le16toh(reply->vsi_number);
data = IXL_DMA_KVA(vsi);
sc->sc_vsi_stat_counter_idx = le16toh(data->stat_counter_idx);

return 0;
}

static int
ixl_set_vsi(struct ixl_softc *sc)
{
struct ixl_dmamem *vsi = &sc->sc_scratch;
struct ixl_aq_desc iaq;
struct ixl_aq_vsi_param *param;
struct ixl_aq_vsi_data *data = IXL_DMA_KVA(vsi);
unsigned int qnum;
uint16_t val;
int rv;

qnum = sc->sc_nqueue_pairs - 1;

data->valid_sections = htole16(IXL_AQ_VSI_VALID_QUEUE_MAP |
IXL_AQ_VSI_VALID_VLAN);

CLR(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_MASK));
SET(data->mapping_flags, htole16(IXL_AQ_VSI_QUE_MAP_CONTIG));
data->queue_mapping[0] = htole16(0);
data->tc_mapping[0] = htole16((0 << IXL_AQ_VSI_TC_Q_OFFSET_SHIFT) |
(qnum << IXL_AQ_VSI_TC_Q_NUMBER_SHIFT));

val = le16toh(data->port_vlan_flags);
CLR(val, IXL_AQ_VSI_PVLAN_MODE_MASK | IXL_AQ_VSI_PVLAN_EMOD_MASK);
SET(val, IXL_AQ_VSI_PVLAN_MODE_ALL);

if (ISSET(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWTAGGING)) {
SET(val, IXL_AQ_VSI_PVLAN_EMOD_STR_BOTH);
} else {
SET(val, IXL_AQ_VSI_PVLAN_EMOD_NOTHING);
}

data->port_vlan_flags = htole16(val);

/* grumble, vsi info isn't "known" at compile time */

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD |
(IXL_DMA_LEN(vsi) > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0));
iaq.iaq_opcode = htole16(IXL_AQ_OP_UPD_VSI_PARAMS);
iaq.iaq_datalen = htole16(IXL_DMA_LEN(vsi));
ixl_aq_dva(&iaq, IXL_DMA_DVA(vsi));

param = (struct ixl_aq_vsi_param *)iaq.iaq_param;
param->uplink_seid = sc->sc_seid;

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi),
BUS_DMASYNC_PREWRITE);

rv = ixl_atq_poll(sc, &iaq, 250);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(vsi), 0, IXL_DMA_LEN(vsi),
BUS_DMASYNC_POSTWRITE);

if (rv != 0) {
return ETIMEDOUT;
}

switch (le16toh(iaq.iaq_retval)) {
case IXL_AQ_RC_OK:
break;
case IXL_AQ_RC_ENOENT:
return ENOENT;
case IXL_AQ_RC_EACCES:
return EACCES;
default:
return EIO;
}

return 0;
}

static void
ixl_set_filter_control(struct ixl_softc *sc)
{
uint32_t reg;

reg = ixl_rd_rx_csr(sc, I40E_PFQF_CTL_0);

CLR(reg, I40E_PFQF_CTL_0_HASHLUTSIZE_MASK);
SET(reg, I40E_HASH_LUT_SIZE_128 << I40E_PFQF_CTL_0_HASHLUTSIZE_SHIFT);

SET(reg, I40E_PFQF_CTL_0_FD_ENA_MASK);
SET(reg, I40E_PFQF_CTL_0_ETYPE_ENA_MASK);
SET(reg, I40E_PFQF_CTL_0_MACVLAN_ENA_MASK);

ixl_wr_rx_csr(sc, I40E_PFQF_CTL_0, reg);
}

static inline void
ixl_get_default_rss_key(uint32_t *buf, size_t len)
{
size_t cplen;
uint8_t rss_seed[RSS_KEYSIZE];

rss_getkey(rss_seed);
memset(buf, 0, len);

cplen = MIN(len, sizeof(rss_seed));
memcpy(buf, rss_seed, cplen);
}

static int
ixl_set_rss_key(struct ixl_softc *sc, uint8_t *key, size_t keylen)
{
struct ixl_dmamem *idm;
struct ixl_atq iatq;
struct ixl_aq_desc *iaq;
struct ixl_aq_rss_key_param *param;
struct ixl_aq_rss_key_data *data;
size_t len, datalen, stdlen, extlen;
uint16_t vsi_id;
int rv;

memset(&iatq, 0, sizeof(iatq));
iaq = &iatq.iatq_desc;
idm = &sc->sc_aqbuf;

datalen = sizeof(*data);

/*XXX The buf size has to be less than the size of the register */
datalen = MIN(IXL_RSS_KEY_SIZE_REG * sizeof(uint32_t), datalen);

iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD |
(datalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0));
iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_KEY);
iaq->iaq_datalen = htole16(datalen);

param = (struct ixl_aq_rss_key_param *)iaq->iaq_param;
vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSKEY_VSI_ID_SHIFT) |
IXL_AQ_RSSKEY_VSI_VALID;
param->vsi_id = htole16(vsi_id);

memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm));
data = IXL_DMA_KVA(idm);

len = MIN(keylen, datalen);
stdlen = MIN(sizeof(data->standard_rss_key), len);
memcpy(data->standard_rss_key, key, stdlen);
len = (len > stdlen) ? (len - stdlen) : 0;

extlen = MIN(sizeof(data->extended_hash_key), len);
extlen = (stdlen < keylen) ? 0 : keylen - stdlen;
memcpy(data->extended_hash_key, key + stdlen, extlen);

ixl_aq_dva(iaq, IXL_DMA_DVA(idm));

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0,
IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE);

rv = ixl_atq_exec(sc, &iatq);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0,
IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE);

if (rv != 0) {
return ETIMEDOUT;
}

if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) {
return EIO;
}

return 0;
}

static int
ixl_set_rss_lut(struct ixl_softc *sc, uint8_t *lut, size_t lutlen)
{
struct ixl_dmamem *idm;
struct ixl_atq iatq;
struct ixl_aq_desc *iaq;
struct ixl_aq_rss_lut_param *param;
uint16_t vsi_id;
uint8_t *data;
size_t dmalen;
int rv;

memset(&iatq, 0, sizeof(iatq));
iaq = &iatq.iatq_desc;
idm = &sc->sc_aqbuf;

dmalen = MIN(lutlen, IXL_DMA_LEN(idm));

iaq->iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD |
(dmalen > I40E_AQ_LARGE_BUF ? IXL_AQ_LB : 0));
iaq->iaq_opcode = htole16(IXL_AQ_OP_RSS_SET_LUT);
iaq->iaq_datalen = htole16(dmalen);

memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm));
data = IXL_DMA_KVA(idm);
memcpy(data, lut, dmalen);
ixl_aq_dva(iaq, IXL_DMA_DVA(idm));

param = (struct ixl_aq_rss_lut_param *)iaq->iaq_param;
vsi_id = (sc->sc_vsi_number << IXL_AQ_RSSLUT_VSI_ID_SHIFT) |
IXL_AQ_RSSLUT_VSI_VALID;
param->vsi_id = htole16(vsi_id);
param->flags = htole16(IXL_AQ_RSSLUT_TABLE_TYPE_PF <<
IXL_AQ_RSSLUT_TABLE_TYPE_SHIFT);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0,
IXL_DMA_LEN(idm), BUS_DMASYNC_PREWRITE);

rv = ixl_atq_exec(sc, &iatq);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0,
IXL_DMA_LEN(idm), BUS_DMASYNC_POSTWRITE);

if (rv != 0) {
return ETIMEDOUT;
}

if (iaq->iaq_retval != htole16(IXL_AQ_RC_OK)) {
return EIO;
}

return 0;
}

static int
ixl_register_rss_key(struct ixl_softc *sc)
{
uint32_t rss_seed[IXL_RSS_KEY_SIZE_REG];
int rv;
size_t i;

ixl_get_default_rss_key(rss_seed, sizeof(rss_seed));

if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) {
rv = ixl_set_rss_key(sc, (uint8_t*)rss_seed,
sizeof(rss_seed));
} else {
rv = 0;
for (i = 0; i < IXL_RSS_KEY_SIZE_REG; i++) {
ixl_wr_rx_csr(sc, I40E_PFQF_HKEY(i), rss_seed[i]);
}
}

return rv;
}

static void
ixl_register_rss_pctype(struct ixl_softc *sc)
{
uint64_t set_hena = 0;
uint32_t hena0, hena1;

/*
* We use TCP/UDP with IPv4/IPv6 by default.
* Note: the device can not use just IP header in each
* TCP/UDP packets for the RSS hash calculation.
*/
if (sc->sc_mac_type == I40E_MAC_X722)
set_hena = IXL_RSS_HENA_DEFAULT_X722;
else
set_hena = IXL_RSS_HENA_DEFAULT_XL710;

hena0 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(0));
hena1 = ixl_rd_rx_csr(sc, I40E_PFQF_HENA(1));

SET(hena0, set_hena);
SET(hena1, set_hena >> 32);

ixl_wr_rx_csr(sc, I40E_PFQF_HENA(0), hena0);
ixl_wr_rx_csr(sc, I40E_PFQF_HENA(1), hena1);
}

static int
ixl_register_rss_hlut(struct ixl_softc *sc)
{
unsigned int qid;
uint8_t hlut_buf[512], lut_mask;
uint32_t *hluts;
size_t i, hluts_num;
int rv;

lut_mask = (0x01 << sc->sc_rss_table_entry_width) - 1;

for (i = 0; i < sc->sc_rss_table_size; i++) {
qid = i % sc->sc_nqueue_pairs;
hlut_buf[i] = qid & lut_mask;
}

if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RSS)) {
rv = ixl_set_rss_lut(sc, hlut_buf, sizeof(hlut_buf));
} else {
rv = 0;
hluts = (uint32_t *)hlut_buf;
hluts_num = sc->sc_rss_table_size >> 2;
for (i = 0; i < hluts_num; i++) {
ixl_wr(sc, I40E_PFQF_HLUT(i), hluts[i]);
}
ixl_flush(sc);
}

return rv;
}

static void
ixl_config_rss(struct ixl_softc *sc)
{

KASSERT(mutex_owned(&sc->sc_cfg_lock));

ixl_register_rss_key(sc);
ixl_register_rss_pctype(sc);
ixl_register_rss_hlut(sc);
}

static const struct ixl_phy_type *
ixl_search_phy_type(uint8_t phy_type)
{
const struct ixl_phy_type *itype;
uint64_t mask;
unsigned int i;

if (phy_type >= 64)
return NULL;

mask = 1ULL << phy_type;

for (i = 0; i < __arraycount(ixl_phy_type_map); i++) {
itype = &ixl_phy_type_map[i];

if (ISSET(itype->phy_type, mask))
return itype;
}

return NULL;
}

static uint64_t
ixl_search_link_speed(uint8_t link_speed)
{
const struct ixl_speed_type *type;
unsigned int i;

for (i = 0; i < __arraycount(ixl_speed_type_map); i++) {
type = &ixl_speed_type_map[i];

if (ISSET(type->dev_speed, link_speed))
return type->net_speed;
}

return 0;
}

static uint8_t
ixl_search_baudrate(uint64_t baudrate)
{
const struct ixl_speed_type *type;
unsigned int i;

for (i = 0; i < __arraycount(ixl_speed_type_map); i++) {
type = &ixl_speed_type_map[i];

if (type->net_speed == baudrate) {
return type->dev_speed;
}
}

return 0;
}

static int
ixl_restart_an(struct ixl_softc *sc)
{
struct ixl_aq_desc iaq;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_PHY_RESTART_AN);
iaq.iaq_param[0] =
htole32(IXL_AQ_PHY_RESTART_AN | IXL_AQ_PHY_LINK_ENABLE);

if (ixl_atq_poll(sc, &iaq, 250) != 0) {
aprint_error_dev(sc->sc_dev, "RESTART AN timeout\n");
return -1;
}
if (iaq.iaq_retval != htole16(IXL_AQ_RC_OK)) {
aprint_error_dev(sc->sc_dev, "RESTART AN error\n");
return -1;
}

return 0;
}

static int
ixl_add_macvlan(struct ixl_softc *sc, const uint8_t *macaddr,
uint16_t vlan, uint16_t flags)
{
struct ixl_aq_desc iaq;
struct ixl_aq_add_macvlan *param;
struct ixl_aq_add_macvlan_elem *elem;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD);
iaq.iaq_opcode = htole16(IXL_AQ_OP_ADD_MACVLAN);
iaq.iaq_datalen = htole16(sizeof(*elem));
ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch));

param = (struct ixl_aq_add_macvlan *)&iaq.iaq_param;
param->num_addrs = htole16(1);
param->seid0 = htole16(0x8000) | sc->sc_seid;
param->seid1 = 0;
param->seid2 = 0;

elem = IXL_DMA_KVA(&sc->sc_scratch);
memset(elem, 0, sizeof(*elem));
memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN);
elem->flags = htole16(IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH | flags);
elem->vlan = htole16(vlan);

if (ixl_atq_poll(sc, &iaq, 250) != 0) {
return IXL_AQ_RC_EINVAL;
}

switch (le16toh(iaq.iaq_retval)) {
case IXL_AQ_RC_OK:
break;
case IXL_AQ_RC_ENOSPC:
return ENOSPC;
case IXL_AQ_RC_ENOENT:
return ENOENT;
case IXL_AQ_RC_EACCES:
return EACCES;
case IXL_AQ_RC_EEXIST:
return EEXIST;
case IXL_AQ_RC_EINVAL:
return EINVAL;
default:
return EIO;
}

return 0;
}

static int
ixl_remove_macvlan(struct ixl_softc *sc, const uint8_t *macaddr,
uint16_t vlan, uint16_t flags)
{
struct ixl_aq_desc iaq;
struct ixl_aq_remove_macvlan *param;
struct ixl_aq_remove_macvlan_elem *elem;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_flags = htole16(IXL_AQ_BUF | IXL_AQ_RD);
iaq.iaq_opcode = htole16(IXL_AQ_OP_REMOVE_MACVLAN);
iaq.iaq_datalen = htole16(sizeof(*elem));
ixl_aq_dva(&iaq, IXL_DMA_DVA(&sc->sc_scratch));

param = (struct ixl_aq_remove_macvlan *)&iaq.iaq_param;
param->num_addrs = htole16(1);
param->seid0 = htole16(0x8000) | sc->sc_seid;
param->seid1 = 0;
param->seid2 = 0;

elem = IXL_DMA_KVA(&sc->sc_scratch);
memset(elem, 0, sizeof(*elem));
memcpy(elem->macaddr, macaddr, ETHER_ADDR_LEN);
elem->flags = htole16(IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH | flags);
elem->vlan = htole16(vlan);

if (ixl_atq_poll(sc, &iaq, 250) != 0) {
return EINVAL;
}

switch (le16toh(iaq.iaq_retval)) {
case IXL_AQ_RC_OK:
break;
case IXL_AQ_RC_ENOENT:
return ENOENT;
case IXL_AQ_RC_EACCES:
return EACCES;
case IXL_AQ_RC_EINVAL:
return EINVAL;
default:
return EIO;
}

return 0;
}

static int
ixl_hmc(struct ixl_softc *sc)
{
struct {
uint32_t count;
uint32_t minsize;
bus_size_t objsiz;
bus_size_t setoff;
bus_size_t setcnt;
} regs[] = {
{
0,
IXL_HMC_TXQ_MINSIZE,
I40E_GLHMC_LANTXOBJSZ,
I40E_GLHMC_LANTXBASE(sc->sc_pf_id),
I40E_GLHMC_LANTXCNT(sc->sc_pf_id),
},
{
0,
IXL_HMC_RXQ_MINSIZE,
I40E_GLHMC_LANRXOBJSZ,
I40E_GLHMC_LANRXBASE(sc->sc_pf_id),
I40E_GLHMC_LANRXCNT(sc->sc_pf_id),
},
{
0,
0,
I40E_GLHMC_FCOEDDPOBJSZ,
I40E_GLHMC_FCOEDDPBASE(sc->sc_pf_id),
I40E_GLHMC_FCOEDDPCNT(sc->sc_pf_id),
},
{
0,
0,
I40E_GLHMC_FCOEFOBJSZ,
I40E_GLHMC_FCOEFBASE(sc->sc_pf_id),
I40E_GLHMC_FCOEFCNT(sc->sc_pf_id),
},
};
struct ixl_hmc_entry *e;
uint64_t size, dva;
uint8_t *kva;
uint64_t *sdpage;
unsigned int i;
int npages, tables;
uint32_t reg;

CTASSERT(__arraycount(regs) <= __arraycount(sc->sc_hmc_entries));

regs[IXL_HMC_LAN_TX].count = regs[IXL_HMC_LAN_RX].count =
ixl_rd(sc, I40E_GLHMC_LANQMAX);

size = 0;
for (i = 0; i < __arraycount(regs); i++) {
e = &sc->sc_hmc_entries[i];

e->hmc_count = regs[i].count;
reg = ixl_rd(sc, regs[i].objsiz);
e->hmc_size = IXL_BIT_ULL(0x3F & reg);
e->hmc_base = size;

if ((e->hmc_size * 8) < regs[i].minsize) {
aprint_error_dev(sc->sc_dev,
"kernel hmc entry is too big\n");
return -1;
}

size += roundup(e->hmc_size * e->hmc_count, IXL_HMC_ROUNDUP);
}
size = roundup(size, IXL_HMC_PGSIZE);
npages = size / IXL_HMC_PGSIZE;

tables = roundup(size, IXL_HMC_L2SZ) / IXL_HMC_L2SZ;

if (ixl_dmamem_alloc(sc, &sc->sc_hmc_pd, size, IXL_HMC_PGSIZE) != 0) {
aprint_error_dev(sc->sc_dev,
"unable to allocate hmc pd memory\n");
return -1;
}

if (ixl_dmamem_alloc(sc, &sc->sc_hmc_sd, tables * IXL_HMC_PGSIZE,
IXL_HMC_PGSIZE) != 0) {
aprint_error_dev(sc->sc_dev,
"unable to allocate hmc sd memory\n");
ixl_dmamem_free(sc, &sc->sc_hmc_pd);
return -1;
}

kva = IXL_DMA_KVA(&sc->sc_hmc_pd);
memset(kva, 0, IXL_DMA_LEN(&sc->sc_hmc_pd));

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_pd),
0, IXL_DMA_LEN(&sc->sc_hmc_pd),
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);

dva = IXL_DMA_DVA(&sc->sc_hmc_pd);
sdpage = IXL_DMA_KVA(&sc->sc_hmc_sd);
memset(sdpage, 0, IXL_DMA_LEN(&sc->sc_hmc_sd));

for (i = 0; (int)i < npages; i++) {
*sdpage = htole64(dva | IXL_HMC_PDVALID);
sdpage++;

dva += IXL_HMC_PGSIZE;
}

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(&sc->sc_hmc_sd),
0, IXL_DMA_LEN(&sc->sc_hmc_sd),
BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);

dva = IXL_DMA_DVA(&sc->sc_hmc_sd);
for (i = 0; (int)i < tables; i++) {
uint32_t count;

KASSERT(npages >= 0);

count = ((unsigned int)npages > IXL_HMC_PGS) ?
IXL_HMC_PGS : (unsigned int)npages;

ixl_wr(sc, I40E_PFHMC_SDDATAHIGH, dva >> 32);
ixl_wr(sc, I40E_PFHMC_SDDATALOW, dva |
(count << I40E_PFHMC_SDDATALOW_PMSDBPCOUNT_SHIFT) |
(1U << I40E_PFHMC_SDDATALOW_PMSDVALID_SHIFT));
ixl_barrier(sc, 0, sc->sc_mems, BUS_SPACE_BARRIER_WRITE);
ixl_wr(sc, I40E_PFHMC_SDCMD,
(1U << I40E_PFHMC_SDCMD_PMSDWR_SHIFT) | i);

npages -= IXL_HMC_PGS;
dva += IXL_HMC_PGSIZE;
}

for (i = 0; i < __arraycount(regs); i++) {
e = &sc->sc_hmc_entries[i];

ixl_wr(sc, regs[i].setoff, e->hmc_base / IXL_HMC_ROUNDUP);
ixl_wr(sc, regs[i].setcnt, e->hmc_count);
}

return 0;
}

static void
ixl_hmc_free(struct ixl_softc *sc)
{
ixl_dmamem_free(sc, &sc->sc_hmc_sd);
ixl_dmamem_free(sc, &sc->sc_hmc_pd);
}

static void
ixl_hmc_pack(void *d, const void *s, const struct ixl_hmc_pack *packing,
unsigned int npacking)
{
uint8_t *dst = d;
const uint8_t *src = s;
unsigned int i;

for (i = 0; i < npacking; i++) {
const struct ixl_hmc_pack *pack = &packing[i];
unsigned int offset = pack->lsb / 8;
unsigned int align = pack->lsb % 8;
const uint8_t *in = src + pack->offset;
uint8_t *out = dst + offset;
int width = pack->width;
unsigned int inbits = 0;

if (align) {
inbits = (*in++) << align;
*out++ |= (inbits & 0xff);
inbits >>= 8;

width -= 8 - align;
}

while (width >= 8) {
inbits |= (*in++) << align;
*out++ = (inbits & 0xff);
inbits >>= 8;

width -= 8;
}

if (width > 0) {
inbits |= (*in) << align;
*out |= (inbits & ((1 << width) - 1));
}
}
}

static struct ixl_aq_buf *
ixl_aqb_alloc(struct ixl_softc *sc)
{
struct ixl_aq_buf *aqb;

aqb = kmem_alloc(sizeof(*aqb), KM_SLEEP);

aqb->aqb_size = IXL_AQ_BUFLEN;

if (bus_dmamap_create(sc->sc_dmat, aqb->aqb_size, 1,
aqb->aqb_size, 0,
BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &aqb->aqb_map) != 0)
goto free;
if (bus_dmamem_alloc(sc->sc_dmat, aqb->aqb_size,
IXL_AQ_ALIGN, 0, &aqb->aqb_seg, 1, &aqb->aqb_nsegs,
BUS_DMA_WAITOK) != 0)
goto destroy;
if (bus_dmamem_map(sc->sc_dmat, &aqb->aqb_seg, aqb->aqb_nsegs,
aqb->aqb_size, &aqb->aqb_data, BUS_DMA_WAITOK) != 0)
goto dma_free;
if (bus_dmamap_load(sc->sc_dmat, aqb->aqb_map, aqb->aqb_data,
aqb->aqb_size, NULL, BUS_DMA_WAITOK) != 0)
goto unmap;

return aqb;
unmap:
bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size);
dma_free:
bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1);
destroy:
bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map);
free:
kmem_free(aqb, sizeof(*aqb));

return NULL;
}

static void
ixl_aqb_free(struct ixl_softc *sc, struct ixl_aq_buf *aqb)
{

bus_dmamap_unload(sc->sc_dmat, aqb->aqb_map);
bus_dmamem_unmap(sc->sc_dmat, aqb->aqb_data, aqb->aqb_size);
bus_dmamem_free(sc->sc_dmat, &aqb->aqb_seg, 1);
bus_dmamap_destroy(sc->sc_dmat, aqb->aqb_map);
kmem_free(aqb, sizeof(*aqb));
}

static int
ixl_arq_fill(struct ixl_softc *sc)
{
struct ixl_aq_buf *aqb;
struct ixl_aq_desc *arq, *iaq;
unsigned int prod = sc->sc_arq_prod;
unsigned int n;
int post = 0;

n = ixl_rxr_unrefreshed(sc->sc_arq_prod, sc->sc_arq_cons,
IXL_AQ_NUM);
arq = IXL_DMA_KVA(&sc->sc_arq);

if (__predict_false(n <= 0))
return 0;

do {
aqb = sc->sc_arq_live[prod];
iaq = &arq[prod];

if (aqb == NULL) {
aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle);
if (aqb != NULL) {
SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb,
ixl_aq_buf, aqb_entry);
} else if ((aqb = ixl_aqb_alloc(sc)) == NULL) {
break;
}

sc->sc_arq_live[prod] = aqb;
memset(aqb->aqb_data, 0, aqb->aqb_size);

bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0,
aqb->aqb_size, BUS_DMASYNC_PREREAD);

iaq->iaq_flags = htole16(IXL_AQ_BUF |
(IXL_AQ_BUFLEN > I40E_AQ_LARGE_BUF ?
IXL_AQ_LB : 0));
iaq->iaq_opcode = 0;
iaq->iaq_datalen = htole16(aqb->aqb_size);
iaq->iaq_retval = 0;
iaq->iaq_cookie = 0;
iaq->iaq_param[0] = 0;
iaq->iaq_param[1] = 0;
ixl_aq_dva(iaq, aqb->aqb_map->dm_segs[0].ds_addr);
}

prod++;
prod &= IXL_AQ_MASK;

post = 1;

} while (--n);

if (post) {
sc->sc_arq_prod = prod;
ixl_wr(sc, sc->sc_aq_regs->arq_tail, sc->sc_arq_prod);
}

return post;
}

static void
ixl_arq_unfill(struct ixl_softc *sc)
{
struct ixl_aq_buf *aqb;
unsigned int i;

for (i = 0; i < __arraycount(sc->sc_arq_live); i++) {
aqb = sc->sc_arq_live[i];
if (aqb == NULL)
continue;

sc->sc_arq_live[i] = NULL;
bus_dmamap_sync(sc->sc_dmat, aqb->aqb_map, 0, aqb->aqb_size,
BUS_DMASYNC_POSTREAD);
ixl_aqb_free(sc, aqb);
}

while ((aqb = SIMPLEQ_FIRST(&sc->sc_arq_idle)) != NULL) {
SIMPLEQ_REMOVE(&sc->sc_arq_idle, aqb,
ixl_aq_buf, aqb_entry);
ixl_aqb_free(sc, aqb);
}
}

static void
ixl_clear_hw(struct ixl_softc *sc)
{
uint32_t num_queues, base_queue;
uint32_t num_pf_int;
uint32_t num_vf_int;
uint32_t num_vfs;
uint32_t i, j;
uint32_t val;
uint32_t eol = 0x7ff;

/* get number of interrupts, queues, and vfs */
val = ixl_rd(sc, I40E_GLPCI_CNF2);
num_pf_int = (val & I40E_GLPCI_CNF2_MSI_X_PF_N_MASK) >>
I40E_GLPCI_CNF2_MSI_X_PF_N_SHIFT;
num_vf_int = (val & I40E_GLPCI_CNF2_MSI_X_VF_N_MASK) >>
I40E_GLPCI_CNF2_MSI_X_VF_N_SHIFT;

val = ixl_rd(sc, I40E_PFLAN_QALLOC);
base_queue = (val & I40E_PFLAN_QALLOC_FIRSTQ_MASK) >>
I40E_PFLAN_QALLOC_FIRSTQ_SHIFT;
j = (val & I40E_PFLAN_QALLOC_LASTQ_MASK) >>
I40E_PFLAN_QALLOC_LASTQ_SHIFT;
if (val & I40E_PFLAN_QALLOC_VALID_MASK)
num_queues = (j - base_queue) + 1;
else
num_queues = 0;

val = ixl_rd(sc, I40E_PF_VT_PFALLOC);
i = (val & I40E_PF_VT_PFALLOC_FIRSTVF_MASK) >>
I40E_PF_VT_PFALLOC_FIRSTVF_SHIFT;
j = (val & I40E_PF_VT_PFALLOC_LASTVF_MASK) >>
I40E_PF_VT_PFALLOC_LASTVF_SHIFT;
if (val & I40E_PF_VT_PFALLOC_VALID_MASK)
num_vfs = (j - i) + 1;
else
num_vfs = 0;

/* stop all the interrupts */
ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0);
ixl_flush(sc);
val = 0x3 << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT;
for (i = 0; i < num_pf_int - 2; i++)
ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), val);
ixl_flush(sc);

/* Set the FIRSTQ_INDX field to 0x7FF in PFINT_LNKLSTx */
val = eol << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
ixl_wr(sc, I40E_PFINT_LNKLST0, val);
for (i = 0; i < num_pf_int - 2; i++)
ixl_wr(sc, I40E_PFINT_LNKLSTN(i), val);
val = eol << I40E_VPINT_LNKLST0_FIRSTQ_INDX_SHIFT;
for (i = 0; i < num_vfs; i++)
ixl_wr(sc, I40E_VPINT_LNKLST0(i), val);
for (i = 0; i < num_vf_int - 2; i++)
ixl_wr(sc, I40E_VPINT_LNKLSTN(i), val);

/* warn the HW of the coming Tx disables */
for (i = 0; i < num_queues; i++) {
uint32_t abs_queue_idx = base_queue + i;
uint32_t reg_block = 0;

if (abs_queue_idx >= 128) {
reg_block = abs_queue_idx / 128;
abs_queue_idx %= 128;
}

val = ixl_rd(sc, I40E_GLLAN_TXPRE_QDIS(reg_block));
val &= ~I40E_GLLAN_TXPRE_QDIS_QINDX_MASK;
val |= (abs_queue_idx << I40E_GLLAN_TXPRE_QDIS_QINDX_SHIFT);
val |= I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK;

ixl_wr(sc, I40E_GLLAN_TXPRE_QDIS(reg_block), val);
}
delaymsec(400);

/* stop all the queues */
for (i = 0; i < num_queues; i++) {
ixl_wr(sc, I40E_QINT_TQCTL(i), 0);
ixl_wr(sc, I40E_QTX_ENA(i), 0);
ixl_wr(sc, I40E_QINT_RQCTL(i), 0);
ixl_wr(sc, I40E_QRX_ENA(i), 0);
}

/* short wait for all queue disables to settle */
delaymsec(50);
}

static int
ixl_pf_reset(struct ixl_softc *sc)
{
uint32_t cnt = 0;
uint32_t cnt1 = 0;
uint32_t reg = 0, reg0 = 0;
uint32_t grst_del;

/*
* Poll for Global Reset steady state in case of recent GRST.
* The grst delay value is in 100ms units, and we'll wait a
* couple counts longer to be sure we don't just miss the end.
*/
grst_del = ixl_rd(sc, I40E_GLGEN_RSTCTL);
grst_del &= I40E_GLGEN_RSTCTL_GRSTDEL_MASK;
grst_del >>= I40E_GLGEN_RSTCTL_GRSTDEL_SHIFT;

grst_del = grst_del * 20;

for (cnt = 0; cnt < grst_del; cnt++) {
reg = ixl_rd(sc, I40E_GLGEN_RSTAT);
if (!(reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK))
break;
delaymsec(100);
}
if (reg & I40E_GLGEN_RSTAT_DEVSTATE_MASK) {
aprint_error(", Global reset polling failed to complete\n");
return -1;
}

/* Now Wait for the FW to be ready */
for (cnt1 = 0; cnt1 < I40E_PF_RESET_WAIT_COUNT; cnt1++) {
reg = ixl_rd(sc, I40E_GLNVM_ULD);
reg &= (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK |
I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK);
if (reg == (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK |
I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))
break;

delaymsec(10);
}
if (!(reg & (I40E_GLNVM_ULD_CONF_CORE_DONE_MASK |
I40E_GLNVM_ULD_CONF_GLOBAL_DONE_MASK))) {
aprint_error(", wait for FW Reset complete timed out "
"(I40E_GLNVM_ULD = 0x%x)\n", reg);
return -1;
}

/*
* If there was a Global Reset in progress when we got here,
* we don't need to do the PF Reset
*/
if (cnt == 0) {
reg = ixl_rd(sc, I40E_PFGEN_CTRL);
ixl_wr(sc, I40E_PFGEN_CTRL, reg | I40E_PFGEN_CTRL_PFSWR_MASK);
for (cnt = 0; cnt < I40E_PF_RESET_WAIT_COUNT; cnt++) {
reg = ixl_rd(sc, I40E_PFGEN_CTRL);
if (!(reg & I40E_PFGEN_CTRL_PFSWR_MASK))
break;
delaymsec(1);

reg0 = ixl_rd(sc, I40E_GLGEN_RSTAT);
if (reg0 & I40E_GLGEN_RSTAT_DEVSTATE_MASK) {
aprint_error(", Core reset upcoming."
" Skipping PF reset reset request\n");
return -1;
}
}
if (reg & I40E_PFGEN_CTRL_PFSWR_MASK) {
aprint_error(", PF reset polling failed to complete"
"(I40E_PFGEN_CTRL= 0x%x)\n", reg);
return -1;
}
}

return 0;
}

static int
ixl_dmamem_alloc(struct ixl_softc *sc, struct ixl_dmamem *ixm,
bus_size_t size, bus_size_t align)
{
ixm->ixm_size = size;

if (bus_dmamap_create(sc->sc_dmat, ixm->ixm_size, 1,
ixm->ixm_size, 0,
BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW,
&ixm->ixm_map) != 0)
return 1;
if (bus_dmamem_alloc(sc->sc_dmat, ixm->ixm_size,
align, 0, &ixm->ixm_seg, 1, &ixm->ixm_nsegs,
BUS_DMA_WAITOK) != 0)
goto destroy;
if (bus_dmamem_map(sc->sc_dmat, &ixm->ixm_seg, ixm->ixm_nsegs,
ixm->ixm_size, &ixm->ixm_kva, BUS_DMA_WAITOK) != 0)
goto free;
if (bus_dmamap_load(sc->sc_dmat, ixm->ixm_map, ixm->ixm_kva,
ixm->ixm_size, NULL, BUS_DMA_WAITOK) != 0)
goto unmap;

memset(ixm->ixm_kva, 0, ixm->ixm_size);

return 0;
unmap:
bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size);
free:
bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1);
destroy:
bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map);
return 1;
}

static void
ixl_dmamem_free(struct ixl_softc *sc, struct ixl_dmamem *ixm)
{
bus_dmamap_unload(sc->sc_dmat, ixm->ixm_map);
bus_dmamem_unmap(sc->sc_dmat, ixm->ixm_kva, ixm->ixm_size);
bus_dmamem_free(sc->sc_dmat, &ixm->ixm_seg, 1);
bus_dmamap_destroy(sc->sc_dmat, ixm->ixm_map);
}

static int
ixl_setup_vlan_hwfilter(struct ixl_softc *sc)
{
struct ethercom *ec = &sc->sc_ec;
struct vlanid_list *vlanidp;
int rv;

ixl_remove_macvlan(sc, sc->sc_enaddr, 0,
IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN);
ixl_remove_macvlan(sc, etherbroadcastaddr, 0,
IXL_AQ_OP_REMOVE_MACVLAN_IGNORE_VLAN);

rv = ixl_add_macvlan(sc, sc->sc_enaddr, 0,
IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH);
if (rv != 0)
return rv;
rv = ixl_add_macvlan(sc, etherbroadcastaddr, 0,
IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH);
if (rv != 0)
return rv;

ETHER_LOCK(ec);
SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) {
rv = ixl_add_macvlan(sc, sc->sc_enaddr,
vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH);
if (rv != 0)
break;
rv = ixl_add_macvlan(sc, etherbroadcastaddr,
vlanidp->vid, IXL_AQ_OP_ADD_MACVLAN_PERFECT_MATCH);
if (rv != 0)
break;
}
ETHER_UNLOCK(ec);

return rv;
}

static void
ixl_teardown_vlan_hwfilter(struct ixl_softc *sc)
{
struct vlanid_list *vlanidp;
struct ethercom *ec = &sc->sc_ec;

ixl_remove_macvlan(sc, sc->sc_enaddr, 0,
IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH);
ixl_remove_macvlan(sc, etherbroadcastaddr, 0,
IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH);

ETHER_LOCK(ec);
SIMPLEQ_FOREACH(vlanidp, &ec->ec_vids, vid_list) {
ixl_remove_macvlan(sc, sc->sc_enaddr,
vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH);
ixl_remove_macvlan(sc, etherbroadcastaddr,
vlanidp->vid, IXL_AQ_OP_REMOVE_MACVLAN_PERFECT_MATCH);
}
ETHER_UNLOCK(ec);

ixl_add_macvlan(sc, sc->sc_enaddr, 0,
IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN);
ixl_add_macvlan(sc, etherbroadcastaddr, 0,
IXL_AQ_OP_ADD_MACVLAN_IGNORE_VLAN);
}

static int
ixl_update_macvlan(struct ixl_softc *sc)
{
int rv = 0;
int next_ec_capenable = sc->sc_ec.ec_capenable;

if (ISSET(next_ec_capenable, ETHERCAP_VLAN_HWFILTER)) {
rv = ixl_setup_vlan_hwfilter(sc);
if (rv != 0)
ixl_teardown_vlan_hwfilter(sc);
} else {
ixl_teardown_vlan_hwfilter(sc);
}

return rv;
}

static int
ixl_ifflags_cb(struct ethercom *ec)
{
struct ifnet *ifp = &ec->ec_if;
struct ixl_softc *sc = ifp->if_softc;
int rv, change, reset_bits;

mutex_enter(&sc->sc_cfg_lock);

change = ec->ec_capenable ^ sc->sc_cur_ec_capenable;
reset_bits = change & (ETHERCAP_VLAN_HWTAGGING | ETHERCAP_VLAN_MTU);
if (reset_bits != 0) {
sc->sc_cur_ec_capenable ^= reset_bits;
rv = ENETRESET;
goto out;
}

if (ISSET(change, ETHERCAP_VLAN_HWFILTER)) {
rv = ixl_update_macvlan(sc);
if (rv == 0) {
sc->sc_cur_ec_capenable ^= ETHERCAP_VLAN_HWFILTER;
} else {
CLR(ec->ec_capenable, ETHERCAP_VLAN_HWFILTER);
CLR(sc->sc_cur_ec_capenable, ETHERCAP_VLAN_HWFILTER);
}
}

rv = ixl_iff(sc);
out:
mutex_exit(&sc->sc_cfg_lock);

return rv;
}

static int
ixl_set_link_status_locked(struct ixl_softc *sc, const struct ixl_aq_desc *iaq)
{
const struct ixl_aq_link_status *status;
const struct ixl_phy_type *itype;

uint64_t ifm_active = IFM_ETHER;
uint64_t ifm_status = IFM_AVALID;
int link_state = LINK_STATE_DOWN;
uint64_t baudrate = 0;

status = (const struct ixl_aq_link_status *)iaq->iaq_param;
if (!ISSET(status->link_info, IXL_AQ_LINK_UP_FUNCTION)) {
ifm_active |= IFM_NONE;
goto done;
}

ifm_active |= IFM_FDX;
ifm_status |= IFM_ACTIVE;
link_state = LINK_STATE_UP;

itype = ixl_search_phy_type(status->phy_type);
if (itype != NULL)
ifm_active |= itype->ifm_type;

if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_TX))
ifm_active |= IFM_ETH_TXPAUSE;
if (ISSET(status->an_info, IXL_AQ_LINK_PAUSE_RX))
ifm_active |= IFM_ETH_RXPAUSE;

baudrate = ixl_search_link_speed(status->link_speed);

done:
/* sc->sc_cfg_lock held expect during attach */
sc->sc_media_active = ifm_active;
sc->sc_media_status = ifm_status;

sc->sc_ec.ec_if.if_baudrate = baudrate;

return link_state;
}

static int
ixl_establish_intx(struct ixl_softc *sc)
{
pci_chipset_tag_t pc = sc->sc_pa.pa_pc;
pci_intr_handle_t *intr;
char xnamebuf[32];
char intrbuf[PCI_INTRSTR_LEN];
char const *intrstr;

KASSERT(sc->sc_nintrs == 1);

intr = &sc->sc_ihp[0];

intrstr = pci_intr_string(pc, *intr, intrbuf, sizeof(intrbuf));
snprintf(xnamebuf, sizeof(xnamebuf), "%s:legacy",
device_xname(sc->sc_dev));

sc->sc_ihs[0] = pci_intr_establish_xname(pc, *intr, IPL_NET, ixl_intr,
sc, xnamebuf);

if (sc->sc_ihs[0] == NULL) {
aprint_error_dev(sc->sc_dev,
"unable to establish interrupt at %s\n", intrstr);
return -1;
}

aprint_normal_dev(sc->sc_dev, "interrupting at %s\n", intrstr);
return 0;
}

static int
ixl_establish_msix(struct ixl_softc *sc)
{
pci_chipset_tag_t pc = sc->sc_pa.pa_pc;
kcpuset_t *affinity;
unsigned int vector = 0;
unsigned int i;
int affinity_to, r;
char xnamebuf[32];
char intrbuf[PCI_INTRSTR_LEN];
char const *intrstr;

kcpuset_create(&affinity, false);

/* the "other" intr is mapped to vector 0 */
vector = 0;
intrstr = pci_intr_string(pc, sc->sc_ihp[vector],
intrbuf, sizeof(intrbuf));
snprintf(xnamebuf, sizeof(xnamebuf), "%s others",
device_xname(sc->sc_dev));
sc->sc_ihs[vector] = pci_intr_establish_xname(pc,
sc->sc_ihp[vector], IPL_NET, ixl_other_intr,
sc, xnamebuf);
if (sc->sc_ihs[vector] == NULL) {
aprint_error_dev(sc->sc_dev,
"unable to establish interrupt at %s\n", intrstr);
goto fail;
}

aprint_normal_dev(sc->sc_dev, "other interrupt at %s", intrstr);

affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0;
affinity_to = (affinity_to + sc->sc_nqueue_pairs_max) % ncpu;

kcpuset_zero(affinity);
kcpuset_set(affinity, affinity_to);
r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL);
if (r == 0) {
aprint_normal(", affinity to %u", affinity_to);
}
aprint_normal("\n");
vector++;

sc->sc_msix_vector_queue = vector;
affinity_to = ncpu > (int)sc->sc_nqueue_pairs_max ? 1 : 0;

for (i = 0; i < sc->sc_nqueue_pairs_max; i++) {
intrstr = pci_intr_string(pc, sc->sc_ihp[vector],
intrbuf, sizeof(intrbuf));
snprintf(xnamebuf, sizeof(xnamebuf), "%s TXRX%d",
device_xname(sc->sc_dev), i);

sc->sc_ihs[vector] = pci_intr_establish_xname(pc,
sc->sc_ihp[vector], IPL_NET, ixl_queue_intr,
(void *)&sc->sc_qps[i], xnamebuf);

if (sc->sc_ihs[vector] == NULL) {
aprint_error_dev(sc->sc_dev,
"unable to establish interrupt at %s\n", intrstr);
goto fail;
}

aprint_normal_dev(sc->sc_dev,
"for TXRX%d interrupt at %s", i, intrstr);

kcpuset_zero(affinity);
kcpuset_set(affinity, affinity_to);
r = interrupt_distribute(sc->sc_ihs[vector], affinity, NULL);
if (r == 0) {
aprint_normal(", affinity to %u", affinity_to);
affinity_to = (affinity_to + 1) % ncpu;
}
aprint_normal("\n");
vector++;
}

kcpuset_destroy(affinity);

return 0;
fail:
for (i = 0; i < vector; i++) {
pci_intr_disestablish(pc, sc->sc_ihs[i]);
}

sc->sc_msix_vector_queue = 0;
sc->sc_msix_vector_queue = 0;
kcpuset_destroy(affinity);

return -1;
}

static void
ixl_config_queue_intr(struct ixl_softc *sc)
{
unsigned int i, vector;

if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) {
vector = sc->sc_msix_vector_queue;
} else {
vector = I40E_INTR_NOTX_INTR;

ixl_wr(sc, I40E_PFINT_LNKLST0,
(I40E_INTR_NOTX_QUEUE <<
I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT) |
(I40E_QUEUE_TYPE_RX <<
I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));
}

for (i = 0; i < sc->sc_nqueue_pairs; i++) {
ixl_wr(sc, I40E_PFINT_DYN_CTLN(i), 0);
ixl_flush(sc);

ixl_wr(sc, I40E_PFINT_LNKLSTN(i),
((i) << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) |
(I40E_QUEUE_TYPE_RX <<
I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT));

ixl_wr(sc, I40E_QINT_RQCTL(i),
(vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
(I40E_ITR_INDEX_RX <<
I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
(I40E_INTR_NOTX_RX_QUEUE <<
I40E_QINT_RQCTL_MSIX0_INDX_SHIFT) |
(i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
(I40E_QUEUE_TYPE_TX <<
I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT) |
I40E_QINT_RQCTL_CAUSE_ENA_MASK);

ixl_wr(sc, I40E_QINT_TQCTL(i),
(vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
(I40E_ITR_INDEX_TX <<
I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
(I40E_INTR_NOTX_TX_QUEUE <<
I40E_QINT_TQCTL_MSIX0_INDX_SHIFT) |
(I40E_QUEUE_TYPE_EOL <<
I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
(I40E_QUEUE_TYPE_RX <<
I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT) |
I40E_QINT_TQCTL_CAUSE_ENA_MASK);

if (sc->sc_intrtype == PCI_INTR_TYPE_MSIX) {
ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_RX, i),
sc->sc_itr_rx);
ixl_wr(sc, I40E_PFINT_ITRN(I40E_ITR_INDEX_TX, i),
sc->sc_itr_tx);
vector++;
}
}
ixl_flush(sc);

ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_RX), sc->sc_itr_rx);
ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_TX), sc->sc_itr_tx);
ixl_flush(sc);
}

static void
ixl_config_other_intr(struct ixl_softc *sc)
{
ixl_wr(sc, I40E_PFINT_ICR0_ENA, 0);
(void)ixl_rd(sc, I40E_PFINT_ICR0);

ixl_wr(sc, I40E_PFINT_ICR0_ENA,
I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
I40E_PFINT_ICR0_ENA_GRST_MASK |
I40E_PFINT_ICR0_ENA_ADMINQ_MASK |
I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
I40E_PFINT_ICR0_ENA_VFLR_MASK |
I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK |
I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
I40E_PFINT_ICR0_ENA_LINK_STAT_CHANGE_MASK);

ixl_wr(sc, I40E_PFINT_LNKLST0, 0x7FF);
ixl_wr(sc, I40E_PFINT_ITR0(I40E_ITR_INDEX_OTHER), 0);
ixl_wr(sc, I40E_PFINT_STAT_CTL0,
(I40E_ITR_INDEX_OTHER <<
I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT));
ixl_flush(sc);
}

static int
ixl_setup_interrupts(struct ixl_softc *sc)
{
struct pci_attach_args *pa = &sc->sc_pa;
pci_intr_type_t max_type, intr_type;
int counts[PCI_INTR_TYPE_SIZE];
int error;
unsigned int i;
bool retry;

memset(counts, 0, sizeof(counts));
max_type = PCI_INTR_TYPE_MSIX;
/* QPs + other interrupt */
counts[PCI_INTR_TYPE_MSIX] = sc->sc_nqueue_pairs_max + 1;
counts[PCI_INTR_TYPE_INTX] = 1;

if (ixl_param_nomsix)
counts[PCI_INTR_TYPE_MSIX] = 0;

do {
retry = false;
error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type);
if (error != 0) {
aprint_error_dev(sc->sc_dev,
"couldn't map interrupt\n");
break;
}

intr_type = pci_intr_type(pa->pa_pc, sc->sc_ihp[0]);
sc->sc_nintrs = counts[intr_type];
KASSERT(sc->sc_nintrs > 0);

for (i = 0; i < sc->sc_nintrs; i++) {
pci_intr_setattr(pa->pa_pc, &sc->sc_ihp[i],
PCI_INTR_MPSAFE, true);
}

sc->sc_ihs = kmem_zalloc(sizeof(sc->sc_ihs[0]) * sc->sc_nintrs,
KM_SLEEP);

if (intr_type == PCI_INTR_TYPE_MSIX) {
error = ixl_establish_msix(sc);
if (error) {
counts[PCI_INTR_TYPE_MSIX] = 0;
retry = true;
}
} else if (intr_type == PCI_INTR_TYPE_INTX) {
error = ixl_establish_intx(sc);
} else {
error = -1;
}

if (error) {
kmem_free(sc->sc_ihs,
sizeof(sc->sc_ihs[0]) * sc->sc_nintrs);
pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs);
} else {
sc->sc_intrtype = intr_type;
}
} while (retry);

return error;
}

static void
ixl_teardown_interrupts(struct ixl_softc *sc)
{
struct pci_attach_args *pa = &sc->sc_pa;
unsigned int i;

for (i = 0; i < sc->sc_nintrs; i++) {
pci_intr_disestablish(pa->pa_pc, sc->sc_ihs[i]);
}

pci_intr_release(pa->pa_pc, sc->sc_ihp, sc->sc_nintrs);

kmem_free(sc->sc_ihs, sizeof(sc->sc_ihs[0]) * sc->sc_nintrs);
sc->sc_ihs = NULL;
sc->sc_nintrs = 0;
}

static int
ixl_setup_stats(struct ixl_softc *sc)
{
struct ixl_queue_pair *qp;
struct ixl_tx_ring *txr;
struct ixl_rx_ring *rxr;
struct ixl_stats_counters *isc;
unsigned int i;

for (i = 0; i < sc->sc_nqueue_pairs_max; i++) {
qp = &sc->sc_qps[i];
txr = qp->qp_txr;
rxr = qp->qp_rxr;

evcnt_attach_dynamic(&txr->txr_defragged, EVCNT_TYPE_MISC,
NULL, qp->qp_name, "m_defrag successed");
evcnt_attach_dynamic(&txr->txr_defrag_failed, EVCNT_TYPE_MISC,
NULL, qp->qp_name, "m_defrag_failed");
evcnt_attach_dynamic(&txr->txr_pcqdrop, EVCNT_TYPE_MISC,
NULL, qp->qp_name, "Dropped in pcq");
evcnt_attach_dynamic(&txr->txr_transmitdef, EVCNT_TYPE_MISC,
NULL, qp->qp_name, "Deferred transmit");
evcnt_attach_dynamic(&txr->txr_intr, EVCNT_TYPE_INTR,
NULL, qp->qp_name, "Interrupt on queue");
evcnt_attach_dynamic(&txr->txr_defer, EVCNT_TYPE_MISC,
NULL, qp->qp_name, "Handled queue in softint/workqueue");

evcnt_attach_dynamic(&rxr->rxr_mgethdr_failed, EVCNT_TYPE_MISC,
NULL, qp->qp_name, "MGETHDR failed");
evcnt_attach_dynamic(&rxr->rxr_mgetcl_failed, EVCNT_TYPE_MISC,
NULL, qp->qp_name, "MCLGET failed");
evcnt_attach_dynamic(&rxr->rxr_mbuf_load_failed,
EVCNT_TYPE_MISC, NULL, qp->qp_name,
"bus_dmamap_load_mbuf failed");
evcnt_attach_dynamic(&rxr->rxr_intr, EVCNT_TYPE_INTR,
NULL, qp->qp_name, "Interrupt on queue");
evcnt_attach_dynamic(&rxr->rxr_defer, EVCNT_TYPE_MISC,
NULL, qp->qp_name, "Handled queue in softint/workqueue");
}

evcnt_attach_dynamic(&sc->sc_event_atq, EVCNT_TYPE_INTR,
NULL, device_xname(sc->sc_dev), "Interrupt for other events");
evcnt_attach_dynamic(&sc->sc_event_link, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Link status event");
evcnt_attach_dynamic(&sc->sc_event_ecc_err, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "ECC error");
evcnt_attach_dynamic(&sc->sc_event_pci_exception, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "PCI exception");
evcnt_attach_dynamic(&sc->sc_event_crit_err, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Critical error");

isc = &sc->sc_stats_counters;
evcnt_attach_dynamic(&isc->isc_crc_errors, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "CRC errors");
evcnt_attach_dynamic(&isc->isc_illegal_bytes, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Illegal bytes");
evcnt_attach_dynamic(&isc->isc_mac_local_faults, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Mac local faults");
evcnt_attach_dynamic(&isc->isc_mac_remote_faults, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Mac remote faults");
evcnt_attach_dynamic(&isc->isc_link_xon_rx, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx xon");
evcnt_attach_dynamic(&isc->isc_link_xon_tx, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx xon");
evcnt_attach_dynamic(&isc->isc_link_xoff_rx, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx xoff");
evcnt_attach_dynamic(&isc->isc_link_xoff_tx, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx xoff");
evcnt_attach_dynamic(&isc->isc_rx_fragments, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx fragments");
evcnt_attach_dynamic(&isc->isc_rx_jabber, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx jabber");

evcnt_attach_dynamic(&isc->isc_rx_size_64, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx size 64");
evcnt_attach_dynamic(&isc->isc_rx_size_127, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx size 127");
evcnt_attach_dynamic(&isc->isc_rx_size_255, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx size 255");
evcnt_attach_dynamic(&isc->isc_rx_size_511, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx size 511");
evcnt_attach_dynamic(&isc->isc_rx_size_1023, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx size 1023");
evcnt_attach_dynamic(&isc->isc_rx_size_1522, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx size 1522");
evcnt_attach_dynamic(&isc->isc_rx_size_big, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx jumbo packets");
evcnt_attach_dynamic(&isc->isc_rx_undersize, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx under size");
evcnt_attach_dynamic(&isc->isc_rx_oversize, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx over size");

evcnt_attach_dynamic(&isc->isc_rx_bytes, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx bytes / port");
evcnt_attach_dynamic(&isc->isc_rx_discards, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx discards / port");
evcnt_attach_dynamic(&isc->isc_rx_unicast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx unicast / port");
evcnt_attach_dynamic(&isc->isc_rx_multicast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx multicast / port");
evcnt_attach_dynamic(&isc->isc_rx_broadcast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx broadcast / port");

evcnt_attach_dynamic(&isc->isc_vsi_rx_bytes, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx bytes / vsi");
evcnt_attach_dynamic(&isc->isc_vsi_rx_discards, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx discards / vsi");
evcnt_attach_dynamic(&isc->isc_vsi_rx_unicast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx unicast / vsi");
evcnt_attach_dynamic(&isc->isc_vsi_rx_multicast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx multicast / vsi");
evcnt_attach_dynamic(&isc->isc_vsi_rx_broadcast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Rx broadcast / vsi");

evcnt_attach_dynamic(&isc->isc_tx_size_64, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx size 64");
evcnt_attach_dynamic(&isc->isc_tx_size_127, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx size 127");
evcnt_attach_dynamic(&isc->isc_tx_size_255, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx size 255");
evcnt_attach_dynamic(&isc->isc_tx_size_511, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx size 511");
evcnt_attach_dynamic(&isc->isc_tx_size_1023, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx size 1023");
evcnt_attach_dynamic(&isc->isc_tx_size_1522, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx size 1522");
evcnt_attach_dynamic(&isc->isc_tx_size_big, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx jumbo packets");

evcnt_attach_dynamic(&isc->isc_tx_bytes, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx bytes / port");
evcnt_attach_dynamic(&isc->isc_tx_dropped_link_down, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev),
"Tx dropped due to link down / port");
evcnt_attach_dynamic(&isc->isc_tx_unicast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx unicast / port");
evcnt_attach_dynamic(&isc->isc_tx_multicast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx multicast / port");
evcnt_attach_dynamic(&isc->isc_tx_broadcast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx broadcast / port");

evcnt_attach_dynamic(&isc->isc_vsi_tx_bytes, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx bytes / vsi");
evcnt_attach_dynamic(&isc->isc_vsi_tx_errors, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx errors / vsi");
evcnt_attach_dynamic(&isc->isc_vsi_tx_unicast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx unicast / vsi");
evcnt_attach_dynamic(&isc->isc_vsi_tx_multicast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx multicast / vsi");
evcnt_attach_dynamic(&isc->isc_vsi_tx_broadcast, EVCNT_TYPE_MISC,
NULL, device_xname(sc->sc_dev), "Tx broadcast / vsi");

sc->sc_stats_intval = ixl_param_stats_interval;
callout_init(&sc->sc_stats_callout, CALLOUT_MPSAFE);
callout_setfunc(&sc->sc_stats_callout, ixl_stats_callout, sc);
ixl_work_set(&sc->sc_stats_task, ixl_stats_update, sc);

return 0;
}

static void
ixl_teardown_stats(struct ixl_softc *sc)
{
struct ixl_tx_ring *txr;
struct ixl_rx_ring *rxr;
struct ixl_stats_counters *isc;
unsigned int i;

for (i = 0; i < sc->sc_nqueue_pairs_max; i++) {
txr = sc->sc_qps[i].qp_txr;
rxr = sc->sc_qps[i].qp_rxr;

evcnt_detach(&txr->txr_defragged);
evcnt_detach(&txr->txr_defrag_failed);
evcnt_detach(&txr->txr_pcqdrop);
evcnt_detach(&txr->txr_transmitdef);
evcnt_detach(&txr->txr_intr);
evcnt_detach(&txr->txr_defer);

evcnt_detach(&rxr->rxr_mgethdr_failed);
evcnt_detach(&rxr->rxr_mgetcl_failed);
evcnt_detach(&rxr->rxr_mbuf_load_failed);
evcnt_detach(&rxr->rxr_intr);
evcnt_detach(&rxr->rxr_defer);
}

isc = &sc->sc_stats_counters;
evcnt_detach(&isc->isc_crc_errors);
evcnt_detach(&isc->isc_illegal_bytes);
evcnt_detach(&isc->isc_mac_local_faults);
evcnt_detach(&isc->isc_mac_remote_faults);
evcnt_detach(&isc->isc_link_xon_rx);
evcnt_detach(&isc->isc_link_xon_tx);
evcnt_detach(&isc->isc_link_xoff_rx);
evcnt_detach(&isc->isc_link_xoff_tx);
evcnt_detach(&isc->isc_rx_fragments);
evcnt_detach(&isc->isc_rx_jabber);
evcnt_detach(&isc->isc_rx_bytes);
evcnt_detach(&isc->isc_rx_discards);
evcnt_detach(&isc->isc_rx_unicast);
evcnt_detach(&isc->isc_rx_multicast);
evcnt_detach(&isc->isc_rx_broadcast);
evcnt_detach(&isc->isc_rx_size_64);
evcnt_detach(&isc->isc_rx_size_127);
evcnt_detach(&isc->isc_rx_size_255);
evcnt_detach(&isc->isc_rx_size_511);
evcnt_detach(&isc->isc_rx_size_1023);
evcnt_detach(&isc->isc_rx_size_1522);
evcnt_detach(&isc->isc_rx_size_big);
evcnt_detach(&isc->isc_rx_undersize);
evcnt_detach(&isc->isc_rx_oversize);
evcnt_detach(&isc->isc_tx_bytes);
evcnt_detach(&isc->isc_tx_dropped_link_down);
evcnt_detach(&isc->isc_tx_unicast);
evcnt_detach(&isc->isc_tx_multicast);
evcnt_detach(&isc->isc_tx_broadcast);
evcnt_detach(&isc->isc_tx_size_64);
evcnt_detach(&isc->isc_tx_size_127);
evcnt_detach(&isc->isc_tx_size_255);
evcnt_detach(&isc->isc_tx_size_511);
evcnt_detach(&isc->isc_tx_size_1023);
evcnt_detach(&isc->isc_tx_size_1522);
evcnt_detach(&isc->isc_tx_size_big);
evcnt_detach(&isc->isc_vsi_rx_discards);
evcnt_detach(&isc->isc_vsi_rx_bytes);
evcnt_detach(&isc->isc_vsi_rx_unicast);
evcnt_detach(&isc->isc_vsi_rx_multicast);
evcnt_detach(&isc->isc_vsi_rx_broadcast);
evcnt_detach(&isc->isc_vsi_tx_errors);
evcnt_detach(&isc->isc_vsi_tx_bytes);
evcnt_detach(&isc->isc_vsi_tx_unicast);
evcnt_detach(&isc->isc_vsi_tx_multicast);
evcnt_detach(&isc->isc_vsi_tx_broadcast);

evcnt_detach(&sc->sc_event_atq);
evcnt_detach(&sc->sc_event_link);
evcnt_detach(&sc->sc_event_ecc_err);
evcnt_detach(&sc->sc_event_pci_exception);
evcnt_detach(&sc->sc_event_crit_err);

callout_destroy(&sc->sc_stats_callout);
}

static void
ixl_stats_callout(void *xsc)
{
struct ixl_softc *sc = xsc;

ixl_work_add(sc->sc_workq, &sc->sc_stats_task);
callout_schedule(&sc->sc_stats_callout, mstohz(sc->sc_stats_intval));
}

static uint64_t
ixl_stat_delta(struct ixl_softc *sc, uint32_t reg_hi, uint32_t reg_lo,
uint64_t *offset, bool has_offset)
{
uint64_t value, delta;
int bitwidth;

bitwidth = reg_hi == 0 ? 32 : 48;

value = ixl_rd(sc, reg_lo);

if (bitwidth > 32) {
value |= ((uint64_t)ixl_rd(sc, reg_hi) << 32);
}

if (__predict_true(has_offset)) {
delta = value;
if (value < *offset)
delta += ((uint64_t)1 << bitwidth);
delta -= *offset;
} else {
delta = 0;
}
atomic_swap_64(offset, value);

return delta;
}

static void
ixl_stats_update(void *xsc)
{
struct ixl_softc *sc = xsc;
struct ixl_stats_counters *isc;
uint64_t delta;

isc = &sc->sc_stats_counters;

/* errors */
delta = ixl_stat_delta(sc,
0, I40E_GLPRT_CRCERRS(sc->sc_port),
&isc->isc_crc_errors_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_crc_errors.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_ILLERRC(sc->sc_port),
&isc->isc_illegal_bytes_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_illegal_bytes.ev_count, delta);

/* rx */
delta = ixl_stat_delta(sc,
I40E_GLPRT_GORCH(sc->sc_port), I40E_GLPRT_GORCL(sc->sc_port),
&isc->isc_rx_bytes_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_bytes.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_RDPC(sc->sc_port),
&isc->isc_rx_discards_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_discards.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_UPRCH(sc->sc_port), I40E_GLPRT_UPRCL(sc->sc_port),
&isc->isc_rx_unicast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_unicast.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_MPRCH(sc->sc_port), I40E_GLPRT_MPRCL(sc->sc_port),
&isc->isc_rx_multicast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_multicast.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_BPRCH(sc->sc_port), I40E_GLPRT_BPRCL(sc->sc_port),
&isc->isc_rx_broadcast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_broadcast.ev_count, delta);

/* Packet size stats rx */
delta = ixl_stat_delta(sc,
I40E_GLPRT_PRC64H(sc->sc_port), I40E_GLPRT_PRC64L(sc->sc_port),
&isc->isc_rx_size_64_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_size_64.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PRC127H(sc->sc_port), I40E_GLPRT_PRC127L(sc->sc_port),
&isc->isc_rx_size_127_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_size_127.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PRC255H(sc->sc_port), I40E_GLPRT_PRC255L(sc->sc_port),
&isc->isc_rx_size_255_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_size_255.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PRC511H(sc->sc_port), I40E_GLPRT_PRC511L(sc->sc_port),
&isc->isc_rx_size_511_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_size_511.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PRC1023H(sc->sc_port), I40E_GLPRT_PRC1023L(sc->sc_port),
&isc->isc_rx_size_1023_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_size_1023.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PRC1522H(sc->sc_port), I40E_GLPRT_PRC1522L(sc->sc_port),
&isc->isc_rx_size_1522_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_size_1522.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PRC9522H(sc->sc_port), I40E_GLPRT_PRC9522L(sc->sc_port),
&isc->isc_rx_size_big_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_size_big.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_RUC(sc->sc_port),
&isc->isc_rx_undersize_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_undersize.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_ROC(sc->sc_port),
&isc->isc_rx_oversize_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_oversize.ev_count, delta);

/* tx */
delta = ixl_stat_delta(sc,
I40E_GLPRT_GOTCH(sc->sc_port), I40E_GLPRT_GOTCL(sc->sc_port),
&isc->isc_tx_bytes_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_bytes.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_TDOLD(sc->sc_port),
&isc->isc_tx_dropped_link_down_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_dropped_link_down.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_UPTCH(sc->sc_port), I40E_GLPRT_UPTCL(sc->sc_port),
&isc->isc_tx_unicast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_unicast.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_MPTCH(sc->sc_port), I40E_GLPRT_MPTCL(sc->sc_port),
&isc->isc_tx_multicast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_multicast.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_BPTCH(sc->sc_port), I40E_GLPRT_BPTCL(sc->sc_port),
&isc->isc_tx_broadcast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_broadcast.ev_count, delta);

/* Packet size stats tx */
delta = ixl_stat_delta(sc,
I40E_GLPRT_PTC64L(sc->sc_port), I40E_GLPRT_PTC64L(sc->sc_port),
&isc->isc_tx_size_64_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_size_64.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PTC127H(sc->sc_port), I40E_GLPRT_PTC127L(sc->sc_port),
&isc->isc_tx_size_127_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_size_127.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PTC255H(sc->sc_port), I40E_GLPRT_PTC255L(sc->sc_port),
&isc->isc_tx_size_255_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_size_255.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PTC511H(sc->sc_port), I40E_GLPRT_PTC511L(sc->sc_port),
&isc->isc_tx_size_511_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_size_511.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PTC1023H(sc->sc_port), I40E_GLPRT_PTC1023L(sc->sc_port),
&isc->isc_tx_size_1023_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_size_1023.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PTC1522H(sc->sc_port), I40E_GLPRT_PTC1522L(sc->sc_port),
&isc->isc_tx_size_1522_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_size_1522.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLPRT_PTC9522H(sc->sc_port), I40E_GLPRT_PTC9522L(sc->sc_port),
&isc->isc_tx_size_big_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_tx_size_big.ev_count, delta);

/* mac faults */
delta = ixl_stat_delta(sc,
0, I40E_GLPRT_MLFC(sc->sc_port),
&isc->isc_mac_local_faults_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_mac_local_faults.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_MRFC(sc->sc_port),
&isc->isc_mac_remote_faults_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_mac_remote_faults.ev_count, delta);

/* Flow control (LFC) stats */
delta = ixl_stat_delta(sc,
0, I40E_GLPRT_LXONRXC(sc->sc_port),
&isc->isc_link_xon_rx_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_link_xon_rx.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_LXONTXC(sc->sc_port),
&isc->isc_link_xon_tx_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_link_xon_tx.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_LXOFFRXC(sc->sc_port),
&isc->isc_link_xoff_rx_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_link_xoff_rx.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_LXOFFTXC(sc->sc_port),
&isc->isc_link_xoff_tx_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_link_xoff_tx.ev_count, delta);

/* fragments */
delta = ixl_stat_delta(sc,
0, I40E_GLPRT_RFC(sc->sc_port),
&isc->isc_rx_fragments_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_fragments.ev_count, delta);

delta = ixl_stat_delta(sc,
0, I40E_GLPRT_RJC(sc->sc_port),
&isc->isc_rx_jabber_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_rx_jabber.ev_count, delta);

/* VSI rx counters */
delta = ixl_stat_delta(sc,
0, I40E_GLV_RDPC(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_rx_discards_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_rx_discards.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLV_GORCH(sc->sc_vsi_stat_counter_idx),
I40E_GLV_GORCL(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_rx_bytes_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_rx_bytes.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLV_UPRCH(sc->sc_vsi_stat_counter_idx),
I40E_GLV_UPRCL(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_rx_unicast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_rx_unicast.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLV_MPRCH(sc->sc_vsi_stat_counter_idx),
I40E_GLV_MPRCL(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_rx_multicast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_rx_multicast.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLV_BPRCH(sc->sc_vsi_stat_counter_idx),
I40E_GLV_BPRCL(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_rx_broadcast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_rx_broadcast.ev_count, delta);

/* VSI tx counters */
delta = ixl_stat_delta(sc,
0, I40E_GLV_TEPC(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_tx_errors_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_tx_errors.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLV_GOTCH(sc->sc_vsi_stat_counter_idx),
I40E_GLV_GOTCL(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_tx_bytes_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_tx_bytes.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLV_UPTCH(sc->sc_vsi_stat_counter_idx),
I40E_GLV_UPTCL(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_tx_unicast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_tx_unicast.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLV_MPTCH(sc->sc_vsi_stat_counter_idx),
I40E_GLV_MPTCL(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_tx_multicast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_tx_multicast.ev_count, delta);

delta = ixl_stat_delta(sc,
I40E_GLV_BPTCH(sc->sc_vsi_stat_counter_idx),
I40E_GLV_BPTCL(sc->sc_vsi_stat_counter_idx),
&isc->isc_vsi_tx_broadcast_offset, isc->isc_has_offset);
atomic_add_64(&isc->isc_vsi_tx_broadcast.ev_count, delta);
}

static int
ixl_setup_sysctls(struct ixl_softc *sc)
{
const char *devname;
struct sysctllog **log;
const struct sysctlnode *rnode, *rxnode, *txnode;
int error;

log = &sc->sc_sysctllog;
devname = device_xname(sc->sc_dev);

error = sysctl_createv(log, 0, NULL, &rnode,
0, CTLTYPE_NODE, devname,
SYSCTL_DESCR("ixl information and settings"),
NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &rnode, NULL,
CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
SYSCTL_DESCR("Use workqueue for packet processing"),
NULL, 0, &sc->sc_txrx_workqueue, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &rnode, NULL,
CTLFLAG_READONLY, CTLTYPE_INT, "stats_interval",
SYSCTL_DESCR("Statistics collection interval in milliseconds"),
NULL, 0, &sc->sc_stats_intval, 0, CTL_CREATE, CTL_EOL);

error = sysctl_createv(log, 0, &rnode, &rxnode,
0, CTLTYPE_NODE, "rx",
SYSCTL_DESCR("ixl information and settings for Rx"),
NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &rxnode, NULL,
CTLFLAG_READWRITE, CTLTYPE_INT, "itr",
SYSCTL_DESCR("Interrupt Throttling"),
ixl_sysctl_itr_handler, 0,
(void *)sc, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &rxnode, NULL,
CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num",
SYSCTL_DESCR("the number of rx descriptors"),
NULL, 0, &sc->sc_rx_ring_ndescs, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &rxnode, NULL,
CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
SYSCTL_DESCR("max number of Rx packets"
" to process for interrupt processing"),
NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &rxnode, NULL,
CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
SYSCTL_DESCR("max number of Rx packets"
" to process for deferred processing"),
NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &rnode, &txnode,
0, CTLTYPE_NODE, "tx",
SYSCTL_DESCR("ixl information and settings for Tx"),
NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &txnode, NULL,
CTLFLAG_READWRITE, CTLTYPE_INT, "itr",
SYSCTL_DESCR("Interrupt Throttling"),
ixl_sysctl_itr_handler, 0,
(void *)sc, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &txnode, NULL,
CTLFLAG_READONLY, CTLTYPE_INT, "descriptor_num",
SYSCTL_DESCR("the number of tx descriptors"),
NULL, 0, &sc->sc_tx_ring_ndescs, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &txnode, NULL,
CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
SYSCTL_DESCR("max number of Tx packets"
" to process for interrupt processing"),
NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

error = sysctl_createv(log, 0, &txnode, NULL,
CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
SYSCTL_DESCR("max number of Tx packets"
" to process for deferred processing"),
NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
if (error)
goto out;

out:
if (error) {
aprint_error_dev(sc->sc_dev,
"unable to create sysctl node\n");
sysctl_teardown(log);
}

return error;
}

static void
ixl_teardown_sysctls(struct ixl_softc *sc)
{

sysctl_teardown(&sc->sc_sysctllog);
}

static bool
ixl_sysctlnode_is_rx(struct sysctlnode *node)
{

if (strstr(node->sysctl_parent->sysctl_name, "rx") != NULL)
return true;

return false;
}

static int
ixl_sysctl_itr_handler(SYSCTLFN_ARGS)
{
struct sysctlnode node = *rnode;
struct ixl_softc *sc = (struct ixl_softc *)node.sysctl_data;
struct ifnet *ifp = &sc->sc_ec.ec_if;
uint32_t newitr, *itrptr;
int error;

if (ixl_sysctlnode_is_rx(&node)) {
itrptr = &sc->sc_itr_rx;
} else {
itrptr = &sc->sc_itr_tx;
}

newitr = *itrptr;
node.sysctl_data = &newitr;
node.sysctl_size = sizeof(newitr);

error = sysctl_lookup(SYSCTLFN_CALL(&node));

if (error || newp == NULL)
return error;

/* ITRs are applied in ixl_init() for simple implementation */
if (ISSET(ifp->if_flags, IFF_RUNNING))
return EBUSY;

if (newitr > 0x07ff)
return EINVAL;

*itrptr = newitr;

return 0;
}

static struct workqueue *
ixl_workq_create(const char *name, pri_t prio, int ipl, int flags)
{
struct workqueue *wq;
int error;

error = workqueue_create(&wq, name, ixl_workq_work, NULL,
prio, ipl, flags);

if (error)
return NULL;

return wq;
}

static void
ixl_workq_destroy(struct workqueue *wq)
{

workqueue_destroy(wq);
}

static void
ixl_work_set(struct ixl_work *work, void (*func)(void *), void *arg)
{

memset(work, 0, sizeof(*work));
work->ixw_func = func;
work->ixw_arg = arg;
}

static void
ixl_work_add(struct workqueue *wq, struct ixl_work *work)
{
if (atomic_cas_uint(&work->ixw_added, 0, 1) != 0)
return;

kpreempt_disable();
workqueue_enqueue(wq, &work->ixw_cookie, NULL);
kpreempt_enable();
}

static void
ixl_work_wait(struct workqueue *wq, struct ixl_work *work)
{

workqueue_wait(wq, &work->ixw_cookie);
}

static void
ixl_workq_work(struct work *wk, void *context)
{
struct ixl_work *work;

work = container_of(wk, struct ixl_work, ixw_cookie);

atomic_swap_uint(&work->ixw_added, 0);
work->ixw_func(work->ixw_arg);
}

static int
ixl_rx_ctl_read(struct ixl_softc *sc, uint32_t reg, uint32_t *rv)
{
struct ixl_aq_desc iaq;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_READ);
iaq.iaq_param[1] = htole32(reg);

if (ixl_atq_poll(sc, &iaq, 250) != 0)
return ETIMEDOUT;

switch (htole16(iaq.iaq_retval)) {
case IXL_AQ_RC_OK:
/* success */
break;
case IXL_AQ_RC_EACCES:
return EPERM;
case IXL_AQ_RC_EAGAIN:
return EAGAIN;
default:
return EIO;
}

*rv = htole32(iaq.iaq_param[3]);
return 0;
}

static uint32_t
ixl_rd_rx_csr(struct ixl_softc *sc, uint32_t reg)
{
uint32_t val;
int rv, retry, retry_limit;

if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) {
retry_limit = 5;
} else {
retry_limit = 0;
}

for (retry = 0; retry < retry_limit; retry++) {
rv = ixl_rx_ctl_read(sc, reg, &val);
if (rv == 0)
return val;
else if (rv == EAGAIN)
delaymsec(1);
else
break;
}

val = ixl_rd(sc, reg);

return val;
}

static int
ixl_rx_ctl_write(struct ixl_softc *sc, uint32_t reg, uint32_t value)
{
struct ixl_aq_desc iaq;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_RX_CTL_REG_WRITE);
iaq.iaq_param[1] = htole32(reg);
iaq.iaq_param[3] = htole32(value);

if (ixl_atq_poll(sc, &iaq, 250) != 0)
return ETIMEDOUT;

switch (htole16(iaq.iaq_retval)) {
case IXL_AQ_RC_OK:
/* success */
break;
case IXL_AQ_RC_EACCES:
return EPERM;
case IXL_AQ_RC_EAGAIN:
return EAGAIN;
default:
return EIO;
}

return 0;
}

static void
ixl_wr_rx_csr(struct ixl_softc *sc, uint32_t reg, uint32_t value)
{
int rv, retry, retry_limit;

if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_RXCTL)) {
retry_limit = 5;
} else {
retry_limit = 0;
}

for (retry = 0; retry < retry_limit; retry++) {
rv = ixl_rx_ctl_write(sc, reg, value);
if (rv == 0)
return;
else if (rv == EAGAIN)
delaymsec(1);
else
break;
}

ixl_wr(sc, reg, value);
}

static int
ixl_nvm_lock(struct ixl_softc *sc, char rw)
{
struct ixl_aq_desc iaq;
struct ixl_aq_req_resource_param *param;
int rv;

if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK))
return 0;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_REQUEST_RESOURCE);

param = (struct ixl_aq_req_resource_param *)&iaq.iaq_param;
param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM);
if (rw == 'R') {
param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_READ);
} else {
param->access_type = htole16(IXL_AQ_RESOURCE_ACCES_WRITE);
}

rv = ixl_atq_poll(sc, &iaq, 250);

if (rv != 0)
return ETIMEDOUT;

switch (le16toh(iaq.iaq_retval)) {
case IXL_AQ_RC_OK:
break;
case IXL_AQ_RC_EACCES:
return EACCES;
case IXL_AQ_RC_EBUSY:
return EBUSY;
case IXL_AQ_RC_EPERM:
return EPERM;
}

return 0;
}

static int
ixl_nvm_unlock(struct ixl_softc *sc)
{
struct ixl_aq_desc iaq;
struct ixl_aq_rel_resource_param *param;
int rv;

if (!ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMLOCK))
return 0;

memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_RELEASE_RESOURCE);

param = (struct ixl_aq_rel_resource_param *)&iaq.iaq_param;
param->resource_id = htole16(IXL_AQ_RESOURCE_ID_NVM);

rv = ixl_atq_poll(sc, &iaq, 250);

if (rv != 0)
return ETIMEDOUT;

switch (le16toh(iaq.iaq_retval)) {
case IXL_AQ_RC_OK:
break;
default:
return EIO;
}
return 0;
}

static int
ixl_srdone_poll(struct ixl_softc *sc)
{
int wait_count;
uint32_t reg;

for (wait_count = 0; wait_count < IXL_SRRD_SRCTL_ATTEMPTS;
wait_count++) {
reg = ixl_rd(sc, I40E_GLNVM_SRCTL);
if (ISSET(reg, I40E_GLNVM_SRCTL_DONE_MASK))
break;

delaymsec(5);
}

if (wait_count == IXL_SRRD_SRCTL_ATTEMPTS)
return -1;

return 0;
}

static int
ixl_nvm_read_srctl(struct ixl_softc *sc, uint16_t offset, uint16_t *data)
{
uint32_t reg;

if (ixl_srdone_poll(sc) != 0)
return ETIMEDOUT;

reg = ((uint32_t)offset << I40E_GLNVM_SRCTL_ADDR_SHIFT) |
__BIT(I40E_GLNVM_SRCTL_START_SHIFT);
ixl_wr(sc, I40E_GLNVM_SRCTL, reg);

if (ixl_srdone_poll(sc) != 0) {
aprint_debug("NVM read error: couldn't access "
"Shadow RAM address: 0x%x\n", offset);
return ETIMEDOUT;
}

reg = ixl_rd(sc, I40E_GLNVM_SRDATA);
*data = (uint16_t)__SHIFTOUT(reg, I40E_GLNVM_SRDATA_RDDATA_MASK);

return 0;
}

static int
ixl_nvm_read_aq(struct ixl_softc *sc, uint16_t offset_word,
void *data, size_t len)
{
struct ixl_dmamem *idm;
struct ixl_aq_desc iaq;
struct ixl_aq_nvm_param *param;
uint32_t offset_bytes;
int rv;

idm = &sc->sc_aqbuf;
if (len > IXL_DMA_LEN(idm))
return ENOMEM;

memset(IXL_DMA_KVA(idm), 0, IXL_DMA_LEN(idm));
memset(&iaq, 0, sizeof(iaq));
iaq.iaq_opcode = htole16(IXL_AQ_OP_NVM_READ);
iaq.iaq_flags = htole16(IXL_AQ_BUF |
((len > I40E_AQ_LARGE_BUF) ? IXL_AQ_LB : 0));
iaq.iaq_datalen = htole16(len);
ixl_aq_dva(&iaq, IXL_DMA_DVA(idm));

param = (struct ixl_aq_nvm_param *)iaq.iaq_param;
param->command_flags = IXL_AQ_NVM_LAST_CMD;
param->module_pointer = 0;
param->length = htole16(len);
offset_bytes = (uint32_t)offset_word * 2;
offset_bytes &= 0x00FFFFFF;
param->offset = htole32(offset_bytes);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm),
BUS_DMASYNC_PREREAD);

rv = ixl_atq_poll(sc, &iaq, 250);

bus_dmamap_sync(sc->sc_dmat, IXL_DMA_MAP(idm), 0, IXL_DMA_LEN(idm),
BUS_DMASYNC_POSTREAD);

if (rv != 0) {
return ETIMEDOUT;
}

switch (le16toh(iaq.iaq_retval)) {
case IXL_AQ_RC_OK:
break;
case IXL_AQ_RC_EPERM:
return EPERM;
case IXL_AQ_RC_EINVAL:
return EINVAL;
case IXL_AQ_RC_EBUSY:
return EBUSY;
case IXL_AQ_RC_EIO:
default:
return EIO;
}

memcpy(data, IXL_DMA_KVA(idm), len);

return 0;
}

static int
ixl_rd16_nvm(struct ixl_softc *sc, uint16_t offset, uint16_t *data)
{
int error;
uint16_t buf;

error = ixl_nvm_lock(sc, 'R');
if (error)
return error;

if (ISSET(sc->sc_aq_flags, IXL_SC_AQ_FLAG_NVMREAD)) {
error = ixl_nvm_read_aq(sc, offset,
&buf, sizeof(buf));
if (error == 0)
*data = le16toh(buf);
} else {
error = ixl_nvm_read_srctl(sc, offset, &buf);
if (error == 0)
*data = buf;
}

ixl_nvm_unlock(sc);

return error;
}

MODULE(MODULE_CLASS_DRIVER, if_ixl, "pci");

#ifdef _MODULE
#include "ioconf.c"
#endif

#ifdef _MODULE
static void
ixl_parse_modprop(prop_dictionary_t dict)
{
prop_object_t obj;
int64_t val;
uint64_t uval;

if (dict == NULL)
return;

obj = prop_dictionary_get(dict, "nomsix");
if (obj != NULL && prop_object_type(obj) == PROP_TYPE_BOOL) {
ixl_param_nomsix = prop_bool_true((prop_bool_t)obj);
}

obj = prop_dictionary_get(dict, "stats_interval");
if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) {
val = prop_number_signed_value((prop_number_t)obj);

/* the range has no reason */
if (100 < val && val < 180000) {
ixl_param_stats_interval = val;
}
}

obj = prop_dictionary_get(dict, "nqps_limit");
if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) {
val = prop_number_signed_value((prop_number_t)obj);

if (val <= INT32_MAX)
ixl_param_nqps_limit = val;
}

obj = prop_dictionary_get(dict, "rx_ndescs");
if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) {
uval = prop_number_unsigned_integer_value((prop_number_t)obj);

if (uval > 8)
ixl_param_rx_ndescs = uval;
}

obj = prop_dictionary_get(dict, "tx_ndescs");
if (obj != NULL && prop_object_type(obj) == PROP_TYPE_NUMBER) {
uval = prop_number_unsigned_integer_value((prop_number_t)obj);

if (uval > IXL_TX_PKT_DESCS)
ixl_param_tx_ndescs = uval;
}

}
#endif

static int
if_ixl_modcmd(modcmd_t cmd, void *opaque)
{
int error = 0;

#ifdef _MODULE
switch (cmd) {
case MODULE_CMD_INIT:
ixl_parse_modprop((prop_dictionary_t)opaque);
error = config_init_component(cfdriver_ioconf_if_ixl,
cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl);
break;
case MODULE_CMD_FINI:
error = config_fini_component(cfdriver_ioconf_if_ixl,
cfattach_ioconf_if_ixl, cfdata_ioconf_if_ixl);
break;
default:
error = ENOTTY;
break;
}
#endif

return error;
}