/*      $NetBSD: subr_ipi.c,v 1.11 2023/02/24 11:02:27 riastradh Exp $  */

/*-
* Copyright (c) 2014 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Mindaugas Rasiukevicius.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

/*
* Inter-processor interrupt (IPI) interface: asynchronous IPIs to
* invoke functions with a constant argument and synchronous IPIs
* with the cross-call support.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.11 2023/02/24 11:02:27 riastradh Exp $");

#include <sys/param.h>
#include <sys/types.h>

#include <sys/atomic.h>
#include <sys/evcnt.h>
#include <sys/cpu.h>
#include <sys/ipi.h>
#include <sys/intr.h>
#include <sys/kcpuset.h>
#include <sys/kmem.h>
#include <sys/lock.h>
#include <sys/mutex.h>

/*
* An array of the IPI handlers used for asynchronous invocation.
* The lock protects the slot allocation.
*/

typedef struct {
       ipi_func_t      func;
       void *          arg;
} ipi_intr_t;

static kmutex_t         ipi_mngmt_lock;
static ipi_intr_t       ipi_intrs[IPI_MAXREG]   __cacheline_aligned;

/*
* Per-CPU mailbox for IPI messages: it is a single cache line storing
* up to IPI_MSG_MAX messages.  This interface is built on top of the
* synchronous IPIs.
*/

#define IPI_MSG_SLOTS   (CACHE_LINE_SIZE / sizeof(ipi_msg_t *))
#define IPI_MSG_MAX     IPI_MSG_SLOTS

typedef struct {
       ipi_msg_t *     msg[IPI_MSG_SLOTS];
} ipi_mbox_t;


/* Mailboxes for the synchronous IPIs. */
static ipi_mbox_t *     ipi_mboxes      __read_mostly;
static struct evcnt     ipi_mboxfull_ev __cacheline_aligned;
static void             ipi_msg_cpu_handler(void *);

/* Handler for the synchronous IPIs - it must be zero. */
#define IPI_SYNCH_ID    0

#ifndef MULTIPROCESSOR
#define cpu_ipi(ci)     KASSERT(ci == NULL)
#endif

void
ipi_sysinit(void)
{

       mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE);
       memset(ipi_intrs, 0, sizeof(ipi_intrs));

       /*
        * Register the handler for synchronous IPIs.  This mechanism
        * is built on top of the asynchronous interface.  Slot zero is
        * reserved permanently; it is also handy to use zero as a failure
        * for other registers (as it is potentially less error-prone).
        */
       ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler;

       evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL,
          "ipi", "full");
}

void
ipi_percpu_init(void)
{
       const size_t len = ncpu * sizeof(ipi_mbox_t);

       /* Initialise the per-CPU bit fields. */
       for (u_int i = 0; i < ncpu; i++) {
               struct cpu_info *ci = cpu_lookup(i);
               memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend));
       }

       /* Allocate per-CPU IPI mailboxes. */
       ipi_mboxes = kmem_zalloc(len, KM_SLEEP);
       KASSERT(ipi_mboxes != NULL);
}

/*
* ipi_register: register an asynchronous IPI handler.
*
* => Returns IPI ID which is greater than zero; on failure - zero.
*/
u_int
ipi_register(ipi_func_t func, void *arg)
{
       mutex_enter(&ipi_mngmt_lock);
       for (u_int i = 0; i < IPI_MAXREG; i++) {
               if (ipi_intrs[i].func == NULL) {
                       /* Register the function. */
                       ipi_intrs[i].func = func;
                       ipi_intrs[i].arg = arg;
                       mutex_exit(&ipi_mngmt_lock);

                       KASSERT(i != IPI_SYNCH_ID);
                       return i;
               }
       }
       mutex_exit(&ipi_mngmt_lock);
       printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n");
       return 0;
}

/*
* ipi_unregister: release the IPI handler given the ID.
*/
void
ipi_unregister(u_int ipi_id)
{
       ipi_msg_t ipimsg = { .func = __FPTRCAST(ipi_func_t, nullop) };

       KASSERT(ipi_id != IPI_SYNCH_ID);
       KASSERT(ipi_id < IPI_MAXREG);

       /* Release the slot. */
       mutex_enter(&ipi_mngmt_lock);
       KASSERT(ipi_intrs[ipi_id].func != NULL);
       ipi_intrs[ipi_id].func = NULL;

       /* Ensure that there are no IPIs in flight. */
       kpreempt_disable();
       ipi_broadcast(&ipimsg, false);
       ipi_wait(&ipimsg);
       kpreempt_enable();
       mutex_exit(&ipi_mngmt_lock);
}

/*
* ipi_mark_pending: internal routine to mark an IPI pending on the
* specified CPU (which might be curcpu()).
*/
static bool
ipi_mark_pending(u_int ipi_id, struct cpu_info *ci)
{
       const u_int i = ipi_id >> IPI_BITW_SHIFT;
       const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK);

       KASSERT(ipi_id < IPI_MAXREG);
       KASSERT(kpreempt_disabled());

       /* Mark as pending and return true if not previously marked. */
       if ((atomic_load_acquire(&ci->ci_ipipend[i]) & bitm) == 0) {
               membar_release();
               atomic_or_32(&ci->ci_ipipend[i], bitm);
               return true;
       }
       return false;
}

/*
* ipi_trigger: asynchronously send an IPI to the specified CPU.
*/
void
ipi_trigger(u_int ipi_id, struct cpu_info *ci)
{

       KASSERT(curcpu() != ci);
       if (ipi_mark_pending(ipi_id, ci)) {
               cpu_ipi(ci);
       }
}

/*
* ipi_trigger_multi_internal: the guts of ipi_trigger_multi() and
* ipi_trigger_broadcast().
*/
static void
ipi_trigger_multi_internal(u_int ipi_id, const kcpuset_t *target,
   bool skip_self)
{
       const cpuid_t selfid = cpu_index(curcpu());
       CPU_INFO_ITERATOR cii;
       struct cpu_info *ci;

       KASSERT(kpreempt_disabled());
       KASSERT(target != NULL);

       for (CPU_INFO_FOREACH(cii, ci)) {
               const cpuid_t cpuid = cpu_index(ci);

               if (!kcpuset_isset(target, cpuid) || cpuid == selfid) {
                       continue;
               }
               ipi_trigger(ipi_id, ci);
       }
       if (!skip_self && kcpuset_isset(target, selfid)) {
               ipi_mark_pending(ipi_id, curcpu());
               int s = splhigh();
               ipi_cpu_handler();
               splx(s);
       }
}

/*
* ipi_trigger_multi: same as ipi_trigger() but sends to the multiple
* CPUs given the target CPU set.
*/
void
ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target)
{
       ipi_trigger_multi_internal(ipi_id, target, false);
}

/*
* ipi_trigger_broadcast: same as ipi_trigger_multi() to kcpuset_attached,
* optionally skipping the sending CPU.
*/
void
ipi_trigger_broadcast(u_int ipi_id, bool skip_self)
{
       ipi_trigger_multi_internal(ipi_id, kcpuset_attached, skip_self);
}

/*
* put_msg: insert message into the mailbox.
*
* Caller is responsible for issuing membar_release first.
*/
static inline void
put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg)
{
       int count = SPINLOCK_BACKOFF_MIN;
again:
       for (u_int i = 0; i < IPI_MSG_MAX; i++) {
               if (atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) {
                       return;
               }
       }

       /* All slots are full: we have to spin-wait. */
       ipi_mboxfull_ev.ev_count++;
       SPINLOCK_BACKOFF(count);
       goto again;
}

/*
* ipi_cpu_handler: the IPI handler.
*/
void
ipi_cpu_handler(void)
{
       struct cpu_info * const ci = curcpu();

       /*
        * Handle asynchronous IPIs: inspect per-CPU bit field, extract
        * IPI ID numbers and execute functions in those slots.
        */
       for (u_int i = 0; i < IPI_BITWORDS; i++) {
               uint32_t pending, bit;

               if (atomic_load_relaxed(&ci->ci_ipipend[i]) == 0) {
                       continue;
               }
               pending = atomic_swap_32(&ci->ci_ipipend[i], 0);
               membar_acquire();
               while ((bit = ffs(pending)) != 0) {
                       const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit;
                       ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id];

                       pending &= ~(1U << bit);
                       KASSERT(ipi_hdl->func != NULL);
                       ipi_hdl->func(ipi_hdl->arg);
               }
       }
}

/*
* ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox,
* execute the passed functions and acknowledge the messages.
*/
static void
ipi_msg_cpu_handler(void *arg __unused)
{
       const struct cpu_info * const ci = curcpu();
       ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)];

       for (u_int i = 0; i < IPI_MSG_MAX; i++) {
               ipi_msg_t *msg;

               /* Get the message. */
               if ((msg = atomic_load_acquire(&mbox->msg[i])) == NULL) {
                       continue;
               }
               atomic_store_relaxed(&mbox->msg[i], NULL);

               /* Execute the handler. */
               KASSERT(msg->func);
               msg->func(msg->arg);

               /* Ack the request. */
               membar_release();
               atomic_dec_uint(&msg->_pending);
       }
}

/*
* ipi_unicast: send an IPI to a single CPU.
*
* => The CPU must be remote; must not be local.
* => The caller must ipi_wait() on the message for completion.
*/
void
ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci)
{
       const cpuid_t id = cpu_index(ci);

       KASSERT(msg->func != NULL);
       KASSERT(kpreempt_disabled());
       KASSERT(curcpu() != ci);

       msg->_pending = 1;
       membar_release();

       put_msg(&ipi_mboxes[id], msg);
       ipi_trigger(IPI_SYNCH_ID, ci);
}

/*
* ipi_multicast: send an IPI to each CPU in the specified set.
*
* => The caller must ipi_wait() on the message for completion.
*/
void
ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target)
{
       const struct cpu_info * const self = curcpu();
       CPU_INFO_ITERATOR cii;
       struct cpu_info *ci;
       u_int local;

       KASSERT(msg->func != NULL);
       KASSERT(kpreempt_disabled());

       local = !!kcpuset_isset(target, cpu_index(self));
       msg->_pending = kcpuset_countset(target) - local;
       membar_release();

       for (CPU_INFO_FOREACH(cii, ci)) {
               cpuid_t id;

               if (__predict_false(ci == self)) {
                       continue;
               }
               id = cpu_index(ci);
               if (!kcpuset_isset(target, id)) {
                       continue;
               }
               put_msg(&ipi_mboxes[id], msg);
               ipi_trigger(IPI_SYNCH_ID, ci);
       }
       if (local) {
               msg->func(msg->arg);
       }
}

/*
* ipi_broadcast: send an IPI to all CPUs.
*
* => The caller must ipi_wait() on the message for completion.
*/
void
ipi_broadcast(ipi_msg_t *msg, bool skip_self)
{
       const struct cpu_info * const self = curcpu();
       CPU_INFO_ITERATOR cii;
       struct cpu_info *ci;

       KASSERT(msg->func != NULL);
       KASSERT(kpreempt_disabled());

       msg->_pending = ncpu - 1;
       membar_release();

       /* Broadcast IPIs for remote CPUs. */
       for (CPU_INFO_FOREACH(cii, ci)) {
               cpuid_t id;

               if (__predict_false(ci == self)) {
                       continue;
               }
               id = cpu_index(ci);
               put_msg(&ipi_mboxes[id], msg);
               ipi_trigger(IPI_SYNCH_ID, ci);
       }

       if (!skip_self) {
               /* Finally, execute locally. */
               msg->func(msg->arg);
       }
}

/*
* ipi_wait: spin-wait until the message is processed.
*/
void
ipi_wait(ipi_msg_t *msg)
{
       int count = SPINLOCK_BACKOFF_MIN;

       while (atomic_load_acquire(&msg->_pending)) {
               KASSERT(atomic_load_relaxed(&msg->_pending) < ncpu);
               SPINLOCK_BACKOFF(count);
       }
}

You are viewing proxied material from ftp.icm.edu.pl. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.