/*      $NetBSD: pci_msi_machdep.c,v 1.19 2023/11/21 23:22:23 gutteridge Exp $  */

/*
* Copyright (c) 2015 Internet Initiative Japan Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

/*
* TODO
*
*     - PBA (Pending Bit Array) support
*     - HyperTransport mapping support
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: pci_msi_machdep.c,v 1.19 2023/11/21 23:22:23 gutteridge Exp $");

#include "opt_intrdebug.h"
#include "ioapic.h"

#include <sys/types.h>
#include <sys/param.h>
#include <sys/time.h>
#include <sys/systm.h>
#include <sys/cpu.h>
#include <sys/errno.h>
#include <sys/device.h>
#include <sys/intr.h>
#include <sys/kmem.h>

#include <dev/pci/pcivar.h>

#include <machine/i82093var.h>
#include <machine/pic.h>

#include <x86/pci/msipic.h>
#include <x86/pci/pci_msi_machdep.h>

#ifdef INTRDEBUG
#define MSIDEBUG
#endif

#ifdef MSIDEBUG
#define DPRINTF(msg) printf msg
#else
#define DPRINTF(msg)
#endif

static pci_intr_handle_t
pci_msi_calculate_handle(struct pic *msi_pic, int vector)
{
       pci_intr_handle_t pih;

       KASSERT(msipic_is_msi_pic(msi_pic));

       pih = __SHIFTIN((uint64_t)msipic_get_devid(msi_pic), MSI_INT_DEV_MASK)
           | __SHIFTIN((uint64_t)vector, MSI_INT_VEC_MASK)
           | APIC_INT_VIA_MSI;
       if (msi_pic->pic_type == PIC_MSI)
               MSI_INT_MAKE_MSI(pih);
       else if (msi_pic->pic_type == PIC_MSIX)
               MSI_INT_MAKE_MSIX(pih);
       else
               panic("%s: Unexpected pic_type: %d\n", __func__,
                   msi_pic->pic_type);

       return pih;
}

#ifdef __HAVE_PCI_MSI_MSIX
static pci_intr_handle_t *
pci_msi_alloc_vectors(struct pic *msi_pic, uint *table_indexes, int *count)
{
       struct intrsource *isp;
       pci_intr_handle_t *vectors, pih;
       int i;
       const char *intrstr;
       char intrstr_buf[INTRIDBUF];

       vectors = kmem_zalloc(sizeof(vectors[0]) * (*count), KM_SLEEP);
       mutex_enter(&cpu_lock);
       for (i = 0; i < *count; i++) {
               u_int table_index;

               if (table_indexes == NULL)
                       table_index = i;
               else
                       table_index = table_indexes[i];

               pih = pci_msi_calculate_handle(msi_pic, table_index);

               intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf,
                   sizeof(intrstr_buf));
               isp = intr_allocate_io_intrsource(intrstr);
               if (isp == NULL) {
                       mutex_exit(&cpu_lock);
                       DPRINTF(("can't allocate io_intersource\n"));
                       kmem_free(vectors, sizeof(vectors[0]) * (*count));
                       return NULL;
               }

               vectors[i] = pih;
       }
       mutex_exit(&cpu_lock);

       return vectors;
}
#endif /* __HAVE_PCI_MSI_MSIX */

static void
pci_msi_free_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, int count)
{
       pci_intr_handle_t pih;
       int i;
       const char *intrstr;
       char intrstr_buf[INTRIDBUF];

       mutex_enter(&cpu_lock);
       for (i = 0; i < count; i++) {
               pih = pci_msi_calculate_handle(msi_pic, i);
               intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf,
                   sizeof(intrstr_buf));
               intr_free_io_intrsource(intrstr);
       }
       mutex_exit(&cpu_lock);

       kmem_free(pihs, sizeof(pihs[0]) * count);
}

#ifdef __HAVE_PCI_MSI_MSIX
static int
pci_msi_alloc_common(pci_intr_handle_t **ihps, int *count,
   const struct pci_attach_args *pa, bool exact)
{
       struct pic *msi_pic;
       pci_intr_handle_t *vectors;
       int error, i;

#if NIOAPIC > 0
       if (nioapics == 0) {
               DPRINTF(("no IOAPIC.\n"));
               return ENODEV;
       }
#endif

       if ((pa->pa_flags & PCI_FLAGS_MSI_OKAY) == 0) {
               DPRINTF(("PCI host bridge does not support MSI.\n"));
               return ENODEV;
       }

       msi_pic = msipic_construct_msi_pic(pa);
       if (msi_pic == NULL) {
               DPRINTF(("cannot allocate MSI pic.\n"));
               return EINVAL;
       }

       vectors = NULL;
       while (*count > 0) {
               vectors = pci_msi_alloc_vectors(msi_pic, NULL, count);
               if (vectors != NULL)
                       break;

               if (exact) {
                       DPRINTF(("cannot allocate MSI vectors.\n"));
                       msipic_destruct_msi_pic(msi_pic);
                       return ENOMEM;
               } else {
                       (*count) >>= 1; /* must be power of 2. */
                       continue;
               }
       }
       if (vectors == NULL) {
               DPRINTF(("cannot allocate MSI vectors.\n"));
               msipic_destruct_msi_pic(msi_pic);
               return ENOMEM;
       }

       for (i = 0; i < *count; i++) {
               MSI_INT_MAKE_MSI(vectors[i]);
       }

       error = msipic_set_msi_vectors(msi_pic, NULL, *count);
       if (error) {
               pci_msi_free_vectors(msi_pic, vectors, *count);
               msipic_destruct_msi_pic(msi_pic);
               return error;
       }

       *ihps = vectors;
#ifdef XENPV
       if (xen_map_msi_pirq(msi_pic, *count)) {
               DPRINTF(("xen_map_msi_pirq() failed\n"));
               pci_msi_free_vectors(msi_pic, vectors, *count);
               msipic_destruct_msi_pic(msi_pic);
               return EINVAL;
       }
#endif

       return 0;
}
#endif /* __HAVE_PCI_MSI_MSIX */

static void *
pci_msi_common_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
   int level, int (*func)(void *), void *arg, struct pic *pic,
   const char *xname)
{
       int irq, pin;
       bool mpsafe;

       KASSERT(INT_VIA_MSI(ih));

       irq = -1;
       pin = MSI_INT_VEC(ih);
       mpsafe = ((ih & MPSAFE_MASK) != 0);

       return intr_establish_xname(irq, pic, pin, IST_EDGE, level, func, arg,
           mpsafe, xname);
}

static void
pci_msi_common_disestablish(pci_chipset_tag_t pc, void *cookie)
{

       intr_disestablish(cookie);
}

#ifdef __HAVE_PCI_MSI_MSIX
static int
pci_msix_alloc_common(pci_intr_handle_t **ihps, u_int *table_indexes,
   int *count, const struct pci_attach_args *pa, bool exact)
{
       struct pic *msix_pic;
       pci_intr_handle_t *vectors;
       int error, i;

#if NIOAPIC > 0
       if (nioapics == 0) {
               DPRINTF(("no IOAPIC.\n"));
               return ENODEV;
       }
#endif

       if ((pa->pa_flags & PCI_FLAGS_MSIX_OKAY) == 0) {
               DPRINTF(("PCI host bridge does not support MSI-X.\n"));
               return ENODEV;
       }

       msix_pic = msipic_construct_msix_pic(pa);
       if (msix_pic == NULL)
               return EINVAL;

       vectors = NULL;
       while (*count > 0) {
               vectors = pci_msi_alloc_vectors(msix_pic, table_indexes, count);
               if (vectors != NULL)
                       break;

               if (exact) {
                       DPRINTF(("cannot allocate MSI-X vectors.\n"));
                       msipic_destruct_msix_pic(msix_pic);
                       return ENOMEM;
               } else {
                       (*count)--;
                       continue;
               }
       }
       if (vectors == NULL) {
               DPRINTF(("cannot allocate MSI-X vectors.\n"));
               msipic_destruct_msix_pic(msix_pic);
               return ENOMEM;
       }

       for (i = 0; i < *count; i++) {
               MSI_INT_MAKE_MSIX(vectors[i]);
       }

       error = msipic_set_msi_vectors(msix_pic, vectors, *count);
       if (error) {
               pci_msi_free_vectors(msix_pic, vectors, *count);
               msipic_destruct_msix_pic(msix_pic);
               return error;
       }

       *ihps = vectors;

#ifdef XENPV
       if (xen_map_msix_pirq(msix_pic, *count)) {
               DPRINTF(("xen_map_msi_pirq() failed\n"));
               pci_msi_free_vectors(msix_pic, vectors, *count);
               msipic_destruct_msix_pic(msix_pic);
               return EINVAL;
       }
#endif
       return 0;
}

static int
x86_pci_msi_alloc(pci_intr_handle_t **ihps, int *count,
   const struct pci_attach_args *pa)
{

       return pci_msi_alloc_common(ihps, count, pa, false);
}

static int
x86_pci_msi_alloc_exact(pci_intr_handle_t **ihps, int count,
   const struct pci_attach_args *pa)
{

       return pci_msi_alloc_common(ihps, &count, pa, true);
}
#endif /* __HAVE_PCI_MSI_MSIX */

static void
x86_pci_msi_release_internal(pci_intr_handle_t *pihs, int count)
{
       struct pic *pic;

       pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0]));
       if (pic == NULL)
               return;

#ifdef XENPV
       xen_pci_msi_release(pic, count);
#endif
       pci_msi_free_vectors(pic, pihs, count);
       msipic_destruct_msi_pic(pic);
}

#ifdef __HAVE_PCI_MSI_MSIX
static int
x86_pci_msix_alloc(pci_intr_handle_t **ihps, int *count,
   const struct pci_attach_args *pa)
{

       return pci_msix_alloc_common(ihps, NULL, count, pa, false);
}

static int
x86_pci_msix_alloc_exact(pci_intr_handle_t **ihps, int count,
   const struct pci_attach_args *pa)
{

       return pci_msix_alloc_common(ihps, NULL, &count, pa, true);
}

static int
x86_pci_msix_alloc_map(pci_intr_handle_t **ihps, u_int *table_indexes,
   int count, const struct pci_attach_args *pa)
{

       return pci_msix_alloc_common(ihps, table_indexes, &count, pa, true);
}
#endif /* __HAVE_PCI_MSI_MSIX */

static void
x86_pci_msix_release_internal(pci_intr_handle_t *pihs, int count)
{
       struct pic *pic;

       pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0]));
       if (pic == NULL)
               return;

#ifdef XENPV
       xen_pci_msi_release(pic, count);
#endif
       pci_msi_free_vectors(pic, pihs, count);
       msipic_destruct_msix_pic(pic);
}

/*****************************************************************************/
/*
* extern for MD code.
*/

/*
* Return intrid for an MSI/MSI-X device.
* "buf" must be allocated by caller.
*/
const char *
x86_pci_msi_string(pci_chipset_tag_t pc, pci_intr_handle_t ih, char *buf,
   size_t len)
{
       int dev, vec;

       KASSERT(INT_VIA_MSI(ih));

       dev = MSI_INT_DEV(ih);
       vec = MSI_INT_VEC(ih);
       if (MSI_INT_IS_MSIX(ih))
               snprintf(buf, len, "msix%d vec %d", dev, vec);
       else
               snprintf(buf, len, "msi%d vec %d", dev, vec);

       return buf;
}

/*
* Release MSI handles.
*/
void
x86_pci_msi_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count)
{

       if (count < 1)
               return;

       return x86_pci_msi_release_internal(pihs, count);
}

/*
* Establish an MSI handle.
* If multiple MSI handles are required to be established, a device driver
* must call this function for each handle.
*/
void *
x86_pci_msi_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
   int level, int (*func)(void *), void *arg, const char *xname)
{
       struct pic *pic;

       pic = msipic_find_msi_pic(MSI_INT_DEV(ih));
       if (pic == NULL) {
               DPRINTF(("pci_intr_handler has no msi_pic\n"));
               return NULL;
       }

       return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname);
}

/*
* Disestablish an MSI handle.
* If multiple MSI handles are required to be disestablished, a device driver
* must call this function for each handle.
*/
void
x86_pci_msi_disestablish(pci_chipset_tag_t pc, void *cookie)
{

       pci_msi_common_disestablish(pc, cookie);
}

/*
* Release MSI-X handles.
*/
void
x86_pci_msix_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count)
{

       if (count < 1)
               return;

       return x86_pci_msix_release_internal(pihs, count);
}

/*
* Establish an MSI-X handle.
* If multiple MSI-X handles are required to be established, a device driver
* must call this function for each handle.
*/
void *
x86_pci_msix_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
   int level, int (*func)(void *), void *arg, const char *xname)
{
       struct pic *pic;

       pic = msipic_find_msi_pic(MSI_INT_DEV(ih));
       if (pic == NULL) {
               DPRINTF(("pci_intr_handler has no msi_pic\n"));
               return NULL;
       }

       return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname);
}

/*
* Disestablish an MSI-X handle.
* If multiple MSI-X handles are required to be disestablished, a device driver
* must call this function for each handle.
*/
void
x86_pci_msix_disestablish(pci_chipset_tag_t pc, void *cookie)
{

       pci_msi_common_disestablish(pc, cookie);
}

#ifdef __HAVE_PCI_MSI_MSIX
/*****************************************************************************/
/*
* extern for MI code.
*/

/*
* This function is used by device drivers like pci_intr_map().
*
* "ihps" is the array of vector numbers which MSI used instead of IRQ number.
* "count" must be power of 2.
* "count" can decrease if struct intrsource cannot be allocated.
* if count == 0, return non-zero value.
*/
int
pci_msi_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
   int *count)
{
       int hw_max;

       /* MSI vector count must be power of 2. */
       KASSERT(*count > 0);
       KASSERT(((*count - 1) & *count) == 0);

       hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag);
       if (hw_max == 0)
               return ENODEV;

       if (*count > hw_max) {
               DPRINTF(("cut off MSI count to %d\n", hw_max));
               *count = hw_max; /* cut off hw_max */
       }

       return x86_pci_msi_alloc(ihps, count, pa);
}

/*
* This function is used by device drivers like pci_intr_map().
*
* "ihps" is the array  of vector numbers which MSI used instead of IRQ number.
* "count" must be power of 2.
* "count" can not decrease.
* If "count" struct intrsources cannot be allocated, return non-zero value.
*/
int
pci_msi_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
   int count)
{
       int hw_max;

       /* MSI vector count must be power of 2. */
       KASSERT(count > 0);
       KASSERT(((count - 1) & count) == 0);

       hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag);
       if (hw_max == 0)
               return ENODEV;

       if (count > hw_max) {
               DPRINTF(("over hardware max MSI count %d\n", hw_max));
               return EINVAL;
       }

       return x86_pci_msi_alloc_exact(ihps, count, pa);
}

/*
* This function is used by device drivers like pci_intr_map().
*
* "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
* "count" can decrease if enough struct intrsources cannot be allocated.
* if count == 0, return non-zero value.
*/
int
pci_msix_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
   int *count)
{
       int hw_max;

       KASSERT(*count > 0);

       hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
       if (hw_max == 0)
               return ENODEV;

       if (*count > hw_max) {
               DPRINTF(("cut off MSI-X count to %d\n", hw_max));
               *count = hw_max; /* cut off hw_max */
       }

       return x86_pci_msix_alloc(ihps, count, pa);
}

/*
* This function is used by device drivers like pci_intr_map().
*
* "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
* "count" can not decrease.
* If "count" struct intrsource cannot be allocated, return non-zero value.
*/
int
pci_msix_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
   int count)
{
       int hw_max;

       KASSERT(count > 0);

       hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
       if (hw_max == 0)
               return ENODEV;

       if (count > hw_max) {
               DPRINTF(("over hardware max MSI-X count %d\n", hw_max));
               return EINVAL;
       }

       return x86_pci_msix_alloc_exact(ihps, count, pa);
}

/*
* This function is used by device drivers like pci_intr_map().
* Furthermore, this function can map each handle to an MSI-X table index.
*
* "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
* "count" can not decrease.
* "map" size must be equal to "count".
* If "count" struct intrsource cannot be allocated, return non-zero value.
* e.g.
*     If "map" = { 1, 4, 0 },
*     1st handle is bound to MSI-X index 1
*     2nd handle is bound to MSI-X index 4
*     3rd handle is bound to MSI-X index 0
*/
int
pci_msix_alloc_map(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
   u_int *table_indexes, int count)
{
       int hw_max, i, j;

       KASSERT(count > 0);

       hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
       if (hw_max == 0)
               return ENODEV;

       if (count > hw_max) {
               DPRINTF(("over hardware max MSI-X count %d\n", hw_max));
               return EINVAL;
       }

       /* check not to duplicate table_index */
       for (i = 0; i < count; i++) {
               u_int basing = table_indexes[i];

               KASSERT(table_indexes[i] < PCI_MSIX_MAX_VECTORS);
               if (basing >= hw_max) {
                       DPRINTF(("table index is over hardware max MSI-X index %d\n",
                               hw_max - 1));
                       return EINVAL;
               }

               for (j = i + 1; j < count; j++) {
                       if (basing == table_indexes[j]) {
                               DPRINTF(("MSI-X table index duplicated\n"));
                               return EINVAL;
                       }
               }
       }

       return x86_pci_msix_alloc_map(ihps, table_indexes, count, pa);
}
#endif /* __HAVE_PCI_MSI_MSIX */