* Copyright (c) 2015 Internet Initiative Japan Inc.

/* $NetBSD: msipic.c,v 1.27 2022/05/24 14:00:23 bouyer Exp $ */

/*
* Copyright (c) 2015 Internet Initiative Japan Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: msipic.c,v 1.27 2022/05/24 14:00:23 bouyer Exp $");

#include "opt_intrdebug.h"

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/kmem.h>
#include <sys/mutex.h>
#include <sys/bitops.h>

#include <dev/pci/pcivar.h>

#include <machine/i82489reg.h>
#include <machine/i82489var.h>
#include <machine/i82093reg.h>
#include <machine/i82093var.h>
#include <machine/pic.h>
#include <machine/lock.h>

#include <x86/pci/msipic.h>

#ifdef INTRDEBUG
#define MSIPICDEBUG
#endif

#ifdef MSIPICDEBUG
#define DPRINTF(msg) printf msg
#else
#define DPRINTF(msg)
#endif

#define BUS_SPACE_WRITE_FLUSH(pc, tag) (void)bus_space_read_4(pc, tag, 0)

#define MSIPICNAMEBUF 16

/*
* A Pseudo pic for single MSI/MSI-X device.
* The pic and MSI/MSI-X device are distinbuished by "devid". The "devid"
* is managed by below "dev_seqs".
*/
struct msipic {
struct msipic_pci_info mp_i;

int mp_devid; /* The device id for the MSI/MSI-X device. */

char mp_pic_name[MSIPICNAMEBUF]; /* The MSI/MSI-X device's name. */

struct pci_attach_args mp_pa;
bus_space_tag_t mp_bstag;
bus_space_handle_t mp_bshandle;
bus_size_t mp_bssize;
struct pic *mp_pic;

LIST_ENTRY(msipic) mp_list;
};

static kmutex_t msipic_list_lock;

static LIST_HEAD(, msipic) msipic_list =
LIST_HEAD_INITIALIZER(msipic_list);

/*
* This struct managements "devid" to use the same "devid" for the device
* re-attached. If the device's bus number and device number and function
* number are equal, it is assumed re-attached.
*/
struct dev_last_used_seq {
bool ds_using;
int ds_bus;
int ds_dev;
int ds_fun;
};
/* The number of MSI/MSI-X devices supported by system. */
#define NUM_MSI_DEVS 256
/* Record devids to use the same devid when the device is re-attached. */
static struct dev_last_used_seq dev_seqs[NUM_MSI_DEVS];

static int msipic_allocate_common_msi_devid(const struct pci_attach_args *);
static void msipic_release_common_msi_devid(int);

static struct pic *msipic_find_msi_pic_locked(int);
static struct pic *msipic_construct_common_msi_pic(const struct pci_attach_args *,
const struct pic *);
static void msipic_destruct_common_msi_pic(struct pic *);

static void msi_set_msictl_enablebit(struct pic *, int, int);
static void msi_hwmask(struct pic *, int);
static void msi_hwunmask(struct pic *, int);
static void msi_addroute(struct pic *, struct cpu_info *, int, int, int);
static void msi_delroute(struct pic *, struct cpu_info *, int, int, int);

static void msix_set_vecctl_mask(struct pic *, int, int);
static void msix_hwmask(struct pic *, int);
static void msix_hwunmask(struct pic *, int);
static void msix_addroute(struct pic *, struct cpu_info *, int, int, int);
static void msix_delroute(struct pic *, struct cpu_info *, int, int, int);

/*
* Return new "devid" for the device attached first.
* Return the same "devid" for the device re-attached after dettached once.
* Return -1 if the number of attached MSI/MSI-X devices is over NUM_MSI_DEVS.
*/
static int
msipic_allocate_common_msi_devid(const struct pci_attach_args *pa)
{
pci_chipset_tag_t pc;
pcitag_t tag;
int bus, dev, fun, i;

KASSERT(mutex_owned(&msipic_list_lock));

pc = pa->pa_pc;
tag = pa->pa_tag;
pci_decompose_tag(pc, tag, &bus, &dev, &fun);

/* if the device was once attached, use same devid */
for (i = 0; i < NUM_MSI_DEVS; i++) {
/* skip host bridge */
if (dev_seqs[i].ds_bus == 0
&& dev_seqs[i].ds_dev == 0
&& dev_seqs[i].ds_fun == 0)
break;

if (dev_seqs[i].ds_bus == bus
&& dev_seqs[i].ds_dev == dev
&& dev_seqs[i].ds_fun == fun) {
dev_seqs[i].ds_using = true;
return i;
}
}

for (i = 0; i < NUM_MSI_DEVS; i++) {
if (dev_seqs[i].ds_using == 0) {
dev_seqs[i].ds_using = true;
dev_seqs[i].ds_bus = bus;
dev_seqs[i].ds_dev = dev;
dev_seqs[i].ds_fun = fun;
return i;
}
}

DPRINTF(("too many MSI devices.\n"));
return -1;
}

/*
* Set the "devid" unused, but keep reserving the "devid" to reuse when
* the device is re-attached.
*/
static void
msipic_release_common_msi_devid(int devid)
{

KASSERT(mutex_owned(&msipic_list_lock));

if (devid < 0 || NUM_MSI_DEVS <= devid) {
DPRINTF(("%s: invalid devid.\n", __func__));
return;
}

dev_seqs[devid].ds_using = false;
/* Keep ds_* to reuse the same devid for the same device. */
}

static struct pic *
msipic_find_msi_pic_locked(int devid)
{
struct msipic *mpp;

KASSERT(mutex_owned(&msipic_list_lock));

LIST_FOREACH(mpp, &msipic_list, mp_list) {
if (mpp->mp_devid == devid)
return mpp->mp_pic;
}
return NULL;
}

/*
* Return the msi_pic whose device is already registered.
* If the device is not registered yet, return NULL.
*/
struct pic *
msipic_find_msi_pic(int devid)
{
struct pic *msipic;

mutex_enter(&msipic_list_lock);
msipic = msipic_find_msi_pic_locked(devid);
mutex_exit(&msipic_list_lock);

return msipic;
}

/*
* A common construct process of MSI and MSI-X.
*/
static struct pic *
msipic_construct_common_msi_pic(const struct pci_attach_args *pa,
const struct pic *pic_tmpl)
{
struct pic *pic;
struct msipic *msipic;
int devid;

pic = kmem_alloc(sizeof(*pic), KM_SLEEP);
msipic = kmem_zalloc(sizeof(*msipic), KM_SLEEP);

mutex_enter(&msipic_list_lock);

devid = msipic_allocate_common_msi_devid(pa);
if (devid == -1) {
mutex_exit(&msipic_list_lock);
kmem_free(pic, sizeof(*pic));
kmem_free(msipic, sizeof(*msipic));
return NULL;
}

memcpy(pic, pic_tmpl, sizeof(*pic));
pic->pic_edge_stubs
= x2apic_mode ? x2apic_edge_stubs : ioapic_edge_stubs;
pic->pic_msipic = msipic;
msipic->mp_pic = pic;
pci_decompose_tag(pa->pa_pc, pa->pa_tag,
&msipic->mp_i.mp_bus, &msipic->mp_i.mp_dev, &msipic->mp_i.mp_fun);
memcpy(&msipic->mp_pa, pa, sizeof(msipic->mp_pa));
msipic->mp_devid = devid;
/*
* pci_msi{,x}_alloc() must be called only once in the device driver.
*/
KASSERT(msipic_find_msi_pic_locked(msipic->mp_devid) == NULL);

LIST_INSERT_HEAD(&msipic_list, msipic, mp_list);

mutex_exit(&msipic_list_lock);

return pic;
}

static void
msipic_destruct_common_msi_pic(struct pic *msi_pic)
{
struct msipic *msipic;

if (msi_pic == NULL)
return;

msipic = msi_pic->pic_msipic;
mutex_enter(&msipic_list_lock);
LIST_REMOVE(msipic, mp_list);
msipic_release_common_msi_devid(msipic->mp_devid);
mutex_exit(&msipic_list_lock);

if (msipic->mp_i.mp_xen_pirq != NULL) {
KASSERT(msipic->mp_i.mp_veccnt > 0);
#ifdef DIAGNOSTIC
for (int i = 0; i < msipic->mp_i.mp_veccnt; i++) {
KASSERT(msipic->mp_i.mp_xen_pirq[i] == 0);
}
#endif
kmem_free(msipic->mp_i.mp_xen_pirq,
sizeof(*msipic->mp_i.mp_xen_pirq) * msipic->mp_i.mp_veccnt);
}
kmem_free(msipic, sizeof(*msipic));
kmem_free(msi_pic, sizeof(*msi_pic));
}

/*
* The pic is MSI/MSI-X pic or not.
*/
bool
msipic_is_msi_pic(struct pic *pic)
{

return (pic->pic_msipic != NULL);
}

/*
* Return the MSI/MSI-X devid which is unique for each devices.
*/
int
msipic_get_devid(struct pic *pic)
{

KASSERT(msipic_is_msi_pic(pic));

return pic->pic_msipic->mp_devid;
}

/*
* Return the PCI bus/dev/func info for the device.
*/
const struct msipic_pci_info *
msipic_get_pci_info(struct pic *pic)
{
KASSERT(msipic_is_msi_pic(pic));

return &pic->pic_msipic->mp_i;
}

#define MSI_MSICTL_ENABLE 1
#define MSI_MSICTL_DISABLE 0
static void
msi_set_msictl_enablebit(struct pic *pic, int msi_vec, int flag)
{
pci_chipset_tag_t pc;
struct pci_attach_args *pa;
pcitag_t tag;
pcireg_t ctl;
int off, err __diagused;

pc = NULL;
pa = &pic->pic_msipic->mp_pa;
tag = pa->pa_tag;
err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
KASSERT(err != 0);

/*
* MSI can establish only one vector at once.
* So, use whole device mask bit instead of a vector mask bit.
*/
ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
if (flag == MSI_MSICTL_ENABLE)
ctl |= PCI_MSI_CTL_MSI_ENABLE;
else
ctl &= ~PCI_MSI_CTL_MSI_ENABLE;

#ifdef XENPV
pci_conf_write16(pc, tag, off + PCI_MSI_CTL + 2, ctl >> 16);
#else
pci_conf_write(pc, tag, off, ctl);
#endif
}

static void
msi_hwmask(struct pic *pic, int msi_vec)
{

msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_DISABLE);
}

/*
* Do not use pic->hwunmask() immediately after pic->delroute().
* It is required to use pic->addroute() before pic->hwunmask().
*/
static void
msi_hwunmask(struct pic *pic, int msi_vec)
{

msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_ENABLE);
}

static void
msi_addroute(struct pic *pic, struct cpu_info *ci,
int unused, int idt_vec, int type)
{
pci_chipset_tag_t pc;
struct pci_attach_args *pa;
pcitag_t tag;
#ifndef XENPV
pcireg_t addr, data;
#endif
pcireg_t ctl;
int off, err __diagused;

pc = NULL;
pa = &pic->pic_msipic->mp_pa;
tag = pa->pa_tag;
err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
KASSERT(err != 0);

ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
#ifndef XENPV
/*
* See Intel 64 and IA-32 Architectures Software Developer's Manual
* Volume 3 10.11 Message Signalled Interrupts.
*/
/*
* "cpuid" for MSI address is local APIC ID. In NetBSD, the ID is
* the same as ci->ci_cpuid.
*/
addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid,
LAPIC_MSIADDR_DSTID_MASK);
/* If trigger mode is edge, it don't care level for trigger mode. */
data = __SHIFTIN(idt_vec, LAPIC_VECTOR_MASK)
| LAPIC_TRIGMODE_EDGE | LAPIC_DLMODE_FIXED;

/*
* The size of the message data register is 16bit if the extended
* message data is not implemented. If it's 16bit and the per-vector
* masking is not capable, the location of the upper 16bit is out of
* the MSI capability structure's range. The PCI spec says the upper
* 16bit is driven to 0 if the message data register is 16bit. It's the
* spec, so it's OK just to write it regardless of the value of the
* upper 16bit.
*/
if (ctl & PCI_MSI_CTL_64BIT_ADDR) {
pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, addr);
pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 0);
pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data);
} else {
pci_conf_write(pc, tag, off + PCI_MSI_MADDR, addr);
pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data);
}
#endif /* !XENPV */
ctl |= PCI_MSI_CTL_MSI_ENABLE;
#ifdef XENPV
pci_conf_write16(pc, tag, off + PCI_MSI_CTL + 2, ctl >> 16);
#else
pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
#endif
}

/*
* Do not use pic->hwunmask() immediately after pic->delroute().
* It is required to use pic->addroute() before pic->hwunmask().
*/
static void
msi_delroute(struct pic *pic, struct cpu_info *ci,
int msi_vec, int idt_vec, int type)
{

msi_hwmask(pic, msi_vec);
}

/*
* Template for MSI pic.
* .pic_msipic is set later in construct_msi_pic().
*/
static const struct pic msi_pic_tmpl = {
.pic_type = PIC_MSI,
.pic_vecbase = 0,
.pic_apicid = 0,
.pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msi_pic */
.pic_hwmask = msi_hwmask,
.pic_hwunmask = msi_hwunmask,
.pic_addroute = msi_addroute,
.pic_delroute = msi_delroute,
.pic_intr_get_devname = x86_intr_get_devname,
.pic_intr_get_assigned = x86_intr_get_assigned,
.pic_intr_get_count = x86_intr_get_count,
};

/*
* Create pseudo pic for a MSI device.
*/
struct pic *
msipic_construct_msi_pic(const struct pci_attach_args *pa)
{
struct pic *msi_pic;
char pic_name_buf[MSIPICNAMEBUF];

msi_pic = msipic_construct_common_msi_pic(pa, &msi_pic_tmpl);
if (msi_pic == NULL) {
DPRINTF(("cannot allocate MSI pic.\n"));
return NULL;
}

memset(pic_name_buf, 0, MSIPICNAMEBUF);
snprintf(pic_name_buf, MSIPICNAMEBUF, "msi%d",
msi_pic->pic_msipic->mp_devid);
strncpy(msi_pic->pic_msipic->mp_pic_name, pic_name_buf,
MSIPICNAMEBUF - 1);
msi_pic->pic_name = msi_pic->pic_msipic->mp_pic_name;

return msi_pic;
}

/*
* Delete pseudo pic for a MSI device.
*/
void
msipic_destruct_msi_pic(struct pic *msi_pic)
{

msipic_destruct_common_msi_pic(msi_pic);
}

#define MSIX_VECCTL_HWMASK 1
#define MSIX_VECCTL_HWUNMASK 0
static void
msix_set_vecctl_mask(struct pic *pic, int msix_vec, int flag)
{
bus_space_tag_t bstag;
bus_space_handle_t bshandle;
uint64_t entry_base;
uint32_t vecctl;

if (msix_vec < 0) {
DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d",
__func__, msipic_get_devid(pic), msix_vec));
return;
}

entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;

bstag = pic->pic_msipic->mp_bstag;
bshandle = pic->pic_msipic->mp_bshandle;
vecctl = bus_space_read_4(bstag, bshandle,
entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL);
if (flag == MSIX_VECCTL_HWMASK)
vecctl |= PCI_MSIX_VECTCTL_MASK;
else
vecctl &= ~PCI_MSIX_VECTCTL_MASK;

bus_space_write_4(bstag, bshandle,
entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, vecctl);
BUS_SPACE_WRITE_FLUSH(bstag, bshandle);
}

static void
msix_hwmask(struct pic *pic, int msix_vec)
{

msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWMASK);
}

/*
* Do not use pic->hwunmask() immediately after pic->delroute().
* It is required to use pic->addroute() before pic->hwunmask().
*/
static void
msix_hwunmask(struct pic *pic, int msix_vec)
{

msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWUNMASK);
}

static void
msix_addroute(struct pic *pic, struct cpu_info *ci,
int msix_vec, int idt_vec, int type)
{
pci_chipset_tag_t pc;
struct pci_attach_args *pa;
pcitag_t tag;
#ifndef XENPV
bus_space_tag_t bstag;
bus_space_handle_t bshandle;
uint64_t entry_base;
pcireg_t addr, data;
#endif
pcireg_t ctl;
int off, err __diagused;

if (msix_vec < 0) {
DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d",
__func__, msipic_get_devid(pic), msix_vec));
return;
}

pa = &pic->pic_msipic->mp_pa;
pc = pa->pa_pc;
tag = pa->pa_tag;
err = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL);
KASSERT(err != 0);

#ifndef XENPV
/* Disable MSI-X before writing MSI-X table */
ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
ctl &= ~PCI_MSIX_CTL_ENABLE;
pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);

bstag = pic->pic_msipic->mp_bstag;
bshandle = pic->pic_msipic->mp_bshandle;
entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec;

/*
* See Intel 64 and IA-32 Architectures Software Developer's Manual
* Volume 3 10.11 Message Signalled Interrupts.
*/
/*
* "cpuid" for MSI-X address is local APIC ID. In NetBSD, the ID is
* the same as ci->ci_cpuid.
*/
addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid,
LAPIC_MSIADDR_DSTID_MASK);
/* If trigger mode is edge, it don't care level for trigger mode. */
data = __SHIFTIN(idt_vec, LAPIC_VECTOR_MASK)
| LAPIC_TRIGMODE_EDGE | LAPIC_DLMODE_FIXED;

bus_space_write_4(bstag, bshandle,
entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, addr);
bus_space_write_4(bstag, bshandle,
entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 0);
bus_space_write_4(bstag, bshandle,
entry_base + PCI_MSIX_TABLE_ENTRY_DATA, data);
BUS_SPACE_WRITE_FLUSH(bstag, bshandle);
#endif /* !XENPV */

ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL);
if (ctl & PCI_MSIX_CTL_FUNCMASK) {
ctl &= ~PCI_MSIX_CTL_FUNCMASK;
}
ctl |= PCI_MSIX_CTL_ENABLE;
#ifdef XENPV
pci_conf_write16(pc, tag, off + PCI_MSIX_CTL + 2, ctl >> 16);
#else
pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl);
#endif
}

/*
* Do not use pic->hwunmask() immediately after pic->delroute().
* It is required to use pic->addroute() before pic->hwunmask().
*/
static void
msix_delroute(struct pic *pic, struct cpu_info *ci,
int msix_vec, int vec, int type)
{

msix_hwmask(pic, msix_vec);
}

/*
* Template for MSI-X pic.
* .pic_msipic is set later in construct_msix_pic().
*/
static const struct pic msix_pic_tmpl = {
.pic_type = PIC_MSIX,
.pic_vecbase = 0,
.pic_apicid = 0,
.pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msix_pic */
.pic_hwmask = msix_hwmask,
.pic_hwunmask = msix_hwunmask,
.pic_addroute = msix_addroute,
.pic_delroute = msix_delroute,
.pic_intr_get_devname = x86_intr_get_devname,
.pic_intr_get_assigned = x86_intr_get_assigned,
.pic_intr_get_count = x86_intr_get_count,
};

struct pic *
msipic_construct_msix_pic(const struct pci_attach_args *pa)
{
struct pic *msix_pic;
pci_chipset_tag_t pc;
pcitag_t tag;
pcireg_t tbl;
bus_space_tag_t bstag;
bus_space_handle_t bshandle;
bus_size_t bssize;
size_t table_size;
uint32_t table_offset;
u_int memtype;
bus_addr_t memaddr;
int flags;
int bir, bar, err, off, table_nentry;
char pic_name_buf[MSIPICNAMEBUF];

table_nentry = pci_msix_count(pa->pa_pc, pa->pa_tag);
if (table_nentry == 0) {
DPRINTF(("MSI-X table entry is 0.\n"));
return NULL;
}

pc = pa->pa_pc;
tag = pa->pa_tag;
if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL) == 0) {
DPRINTF(("%s: no msix capability", __func__));
return NULL;
}

msix_pic = msipic_construct_common_msi_pic(pa, &msix_pic_tmpl);
if (msix_pic == NULL) {
DPRINTF(("cannot allocate MSI-X pic.\n"));
return NULL;
}

memset(pic_name_buf, 0, MSIPICNAMEBUF);
snprintf(pic_name_buf, MSIPICNAMEBUF, "msix%d",
msix_pic->pic_msipic->mp_devid);
strncpy(msix_pic->pic_msipic->mp_pic_name, pic_name_buf,
MSIPICNAMEBUF - 1);
msix_pic->pic_name = msix_pic->pic_msipic->mp_pic_name;

tbl = pci_conf_read(pc, tag, off + PCI_MSIX_TBLOFFSET);
table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK;
bir = tbl & PCI_MSIX_TBLBIR_MASK;
switch (bir) {
case 0:
bar = PCI_BAR0;
break;
case 1:
bar = PCI_BAR1;
break;
case 2:
bar = PCI_BAR2;
break;
case 3:
bar = PCI_BAR3;
break;
case 4:
bar = PCI_BAR4;
break;
case 5:
bar = PCI_BAR5;
break;
default:
aprint_error("detect an illegal device! "
"The device use reserved BIR values.\n");
msipic_destruct_common_msi_pic(msix_pic);
return NULL;
}
memtype = pci_mapreg_type(pc, tag, bar);
/*
* PCI_MSIX_TABLE_ENTRY_SIZE consists below
* - Vector Control (32bit)
* - Message Data (32bit)
* - Message Upper Address (32bit)
* - Message Lower Address (32bit)
*/
table_size = table_nentry * PCI_MSIX_TABLE_ENTRY_SIZE;
#if 0
err = pci_mapreg_submap(pa, bar, memtype, BUS_SPACE_MAP_LINEAR,
roundup(table_size, PAGE_SIZE), table_offset,
&bstag, &bshandle, NULL, &bssize);
#else
/*
* Workaround for PCI prefetchable bit. Some chips (e.g. Intel 82599)
* report SERR and MSI-X doesn't work. This problem might not be the
* driver's bug but our PCI common part or VMs' bug. Until we find a
* real reason, we ignore the prefetchable bit.
*/
if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, bar, memtype,
&memaddr, NULL, &flags) != 0) {
DPRINTF(("cannot get a map info.\n"));
msipic_destruct_common_msi_pic(msix_pic);
return NULL;
}
if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) {
DPRINTF(( "clear prefetchable bit\n"));
flags &= ~BUS_SPACE_MAP_PREFETCHABLE;
}
bssize = roundup(table_size, PAGE_SIZE);
err = _x86_memio_map(pa->pa_memt, memaddr + table_offset, bssize, flags,
&bshandle);
bstag = pa->pa_memt;
#endif
if (err) {
DPRINTF(("cannot map msix table.\n"));
msipic_destruct_common_msi_pic(msix_pic);
return NULL;
}
msix_pic->pic_msipic->mp_bstag = bstag;
msix_pic->pic_msipic->mp_bshandle = bshandle;
msix_pic->pic_msipic->mp_bssize = bssize;
msix_pic->pic_msipic->mp_i.mp_table_base = memaddr;

return msix_pic;
}

/*
* Delete pseudo pic for a MSI-X device.
*/
void
msipic_destruct_msix_pic(struct pic *msix_pic)
{
struct msipic *msipic;

KASSERT(msipic_is_msi_pic(msix_pic));
KASSERT(msix_pic->pic_type == PIC_MSIX);

msipic = msix_pic->pic_msipic;
_x86_memio_unmap(msipic->mp_bstag, msipic->mp_bshandle,
msipic->mp_bssize, NULL);

msipic_destruct_common_msi_pic(msix_pic);
}

/*
* Set the number of MSI vectors for pseudo MSI pic.
*/
int
msipic_set_msi_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs,
int count)
{

KASSERT(msipic_is_msi_pic(msi_pic));

if (msi_pic->pic_type == PIC_MSI) {
pci_chipset_tag_t pc;
struct pci_attach_args *pa;
pcitag_t tag;
int off, err __diagused;
pcireg_t ctl;

pc = NULL;
pa = &msi_pic->pic_msipic->mp_pa;
tag = pa->pa_tag;
err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL);
KASSERT(err != 0);

ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL);
ctl &= ~PCI_MSI_CTL_MME_MASK;
ctl |= __SHIFTIN(ilog2(count), PCI_MSI_CTL_MME_MASK);
pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl);
}

msi_pic->pic_msipic->mp_i.mp_veccnt = count;
#ifdef XENPV
msi_pic->pic_msipic->mp_i.mp_xen_pirq =
kmem_zalloc(sizeof(*msi_pic->pic_msipic->mp_i.mp_xen_pirq) * count,
KM_SLEEP);
#endif
return 0;
}

/*
* Initialize the system to use MSI/MSI-X.
*/
void
msipic_init(void)
{

mutex_init(&msipic_list_lock, MUTEX_DEFAULT, IPL_NONE);
}