/*      $NetBSD: subr_xcall.c,v 1.39 2025/04/01 03:16:41 ozaki-r Exp $  */

/*-
* Copyright (c) 2007-2010, 2019 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Andrew Doran and Mindaugas Rasiukevicius.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

/*
* Cross call support
*
* Background
*
*      Sometimes it is necessary to modify hardware state that is tied
*      directly to individual CPUs (such as a CPU's local timer), and
*      these updates can not be done remotely by another CPU.  The LWP
*      requesting the update may be unable to guarantee that it will be
*      running on the CPU where the update must occur, when the update
*      occurs.
*
*      Additionally, it's sometimes necessary to modify per-CPU software
*      state from a remote CPU.  Where these update operations are so
*      rare or the access to the per-CPU data so frequent that the cost
*      of using locking or atomic operations to provide coherency is
*      prohibitive, another way must be found.
*
*      Cross calls help to solve these types of problem by allowing
*      any LWP in the system to request that an arbitrary function be
*      executed on a specific CPU.
*
* Implementation
*
*      A slow mechanism for making low priority cross calls is
*      provided.  The function to be executed runs on the remote CPU
*      within a bound kthread.  No queueing is provided, and the
*      implementation uses global state.  The function being called may
*      block briefly on locks, but in doing so must be careful to not
*      interfere with other cross calls in the system.  The function is
*      called with thread context and not from a soft interrupt, so it
*      can ensure that it is not interrupting other code running on the
*      CPU, and so has exclusive access to the CPU.  Since this facility
*      is heavyweight, it's expected that it will not be used often.
*
*      Cross calls must not allocate memory, as the pagedaemon uses cross
*      calls (and memory allocation may need to wait on the pagedaemon).
*
*      A low-overhead mechanism for high priority calls (XC_HIGHPRI) is
*      also provided.  The function to be executed runs in software
*      interrupt context at IPL_SOFTSERIAL level, and is expected to
*      be very lightweight, e.g. avoid blocking.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.39 2025/04/01 03:16:41 ozaki-r Exp $");

#include <sys/types.h>
#include <sys/param.h>
#include <sys/xcall.h>
#include <sys/mutex.h>
#include <sys/condvar.h>
#include <sys/evcnt.h>
#include <sys/kthread.h>
#include <sys/cpu.h>
#include <sys/atomic.h>

#ifdef _RUMPKERNEL
#include "rump_private.h"
#endif

/* Cross-call state box. */
typedef struct {
       kmutex_t        xc_lock;
       kcondvar_t      xc_busy;
       xcfunc_t        xc_func;
       void *          xc_arg1;
       void *          xc_arg2;
       uint64_t        xc_headp;
       uint64_t        xc_donep;
       unsigned int    xc_ipl;
} xc_state_t;

/* Bit indicating high (1) or low (0) priority. */
#define XC_PRI_BIT      (1ULL << 63)

/* Low priority xcall structures. */
static xc_state_t       xc_low_pri      __cacheline_aligned;

/* High priority xcall structures. */
static xc_state_t       xc_high_pri     __cacheline_aligned;
static void *           xc_sihs[4]      __cacheline_aligned;

/* Event counters. */
static struct evcnt     xc_unicast_ev   __cacheline_aligned;
static struct evcnt     xc_broadcast_ev __cacheline_aligned;

static void             xc_init(void);
static void             xc_thread(void *);

static inline uint64_t  xc_highpri(xcfunc_t, void *, void *, struct cpu_info *,
                           unsigned int);
static inline uint64_t  xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *);

/* The internal form of IPL */
#define XC_IPL_MASK             0xff00
/*
* Assign 0 to XC_IPL_SOFTSERIAL to treat IPL_SOFTSERIAL as the default value
* (just XC_HIGHPRI).
*/
#define XC_IPL_SOFTSERIAL       0
#define XC_IPL_SOFTNET          1
#define XC_IPL_SOFTBIO          2
#define XC_IPL_SOFTCLOCK        3
#define XC_IPL_MAX              XC_IPL_SOFTCLOCK

CTASSERT(XC_IPL_MAX <= __arraycount(xc_sihs));

/*
* xc_init:
*
*      Initialize low and high priority cross-call structures.
*/
static void
xc_init(void)
{
       xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri;

       memset(xclo, 0, sizeof(xc_state_t));
       mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE);
       cv_init(&xclo->xc_busy, "xclow");

       memset(xchi, 0, sizeof(xc_state_t));
       mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
       cv_init(&xchi->xc_busy, "xchigh");

       /* Set up a softint for each IPL_SOFT*. */
#define SETUP_SOFTINT(xipl, sipl) do {                                  \
               xc_sihs[(xipl)] = softint_establish( (sipl) | SOFTINT_MPSAFE,\
                   xc__highpri_intr, NULL);                            \
               KASSERT(xc_sihs[(xipl)] != NULL);                       \
       } while (0)

       SETUP_SOFTINT(XC_IPL_SOFTSERIAL, SOFTINT_SERIAL);
       /*
        * If a IPL_SOFTXXX have the same value of the previous, we don't use
        * the IPL (see xc_encode_ipl).  So we don't need to allocate a softint
        * for it.
        */
#if IPL_SOFTNET != IPL_SOFTSERIAL
       SETUP_SOFTINT(XC_IPL_SOFTNET, SOFTINT_NET);
#endif
#if IPL_SOFTBIO != IPL_SOFTNET
       SETUP_SOFTINT(XC_IPL_SOFTBIO, SOFTINT_BIO);
#endif
#if IPL_SOFTCLOCK != IPL_SOFTBIO
       SETUP_SOFTINT(XC_IPL_SOFTCLOCK, SOFTINT_CLOCK);
#endif

#undef SETUP_SOFTINT

       evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL,
          "crosscall", "unicast");
       evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL,
          "crosscall", "broadcast");
}

/*
* Encode an IPL to a form that can be embedded into flags of xc_broadcast
* or xc_unicast.
*/
unsigned int
xc_encode_ipl(int ipl)
{

       switch (ipl) {
       case IPL_SOFTSERIAL:
               return __SHIFTIN(XC_IPL_SOFTSERIAL, XC_IPL_MASK);
       /* IPL_SOFT* can be the same value (e.g., on sparc or mips). */
#if IPL_SOFTNET != IPL_SOFTSERIAL
       case IPL_SOFTNET:
               return __SHIFTIN(XC_IPL_SOFTNET, XC_IPL_MASK);
#endif
#if IPL_SOFTBIO != IPL_SOFTNET
       case IPL_SOFTBIO:
               return __SHIFTIN(XC_IPL_SOFTBIO, XC_IPL_MASK);
#endif
#if IPL_SOFTCLOCK != IPL_SOFTBIO
       case IPL_SOFTCLOCK:
               return __SHIFTIN(XC_IPL_SOFTCLOCK, XC_IPL_MASK);
#endif
       }

       panic("Invalid IPL: %d", ipl);
}

/*
* Extract an XC_IPL from flags of xc_broadcast or xc_unicast.
*/
static inline unsigned int
xc_extract_ipl(unsigned int flags)
{

       return __SHIFTOUT(flags, XC_IPL_MASK);
}

/*
* xc_init_cpu:
*
*      Initialize the cross-call subsystem.  Called once for each CPU
*      in the system as they are attached.
*/
void
xc_init_cpu(struct cpu_info *ci)
{
       static bool again = false;
       int error __diagused;

       if (!again) {
               /* Autoconfiguration will prevent re-entry. */
               xc_init();
               again = true;
       }
       cv_init(&ci->ci_data.cpu_xcall, "xcall");
       error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread,
           NULL, NULL, "xcall/%u", ci->ci_index);
       KASSERT(error == 0);
}

/*
* xc_broadcast:
*
*      Trigger a call on all CPUs in the system.
*/
uint64_t
xc_broadcast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2)
{

       KASSERT(!cpu_intr_p());
       KASSERT(!cpu_softintr_p());
       ASSERT_SLEEPABLE();

       if (__predict_false(!mp_online)) {
               int s, bound;

               if (flags & XC_HIGHPRI)
                       s = splsoftserial();
               else
                       bound = curlwp_bind();
               (*func)(arg1, arg2);
               if (flags & XC_HIGHPRI)
                       splx(s);
               else
                       curlwp_bindx(bound);
               return 0;
       }

       if ((flags & XC_HIGHPRI) != 0) {
               unsigned int ipl = xc_extract_ipl(flags);
               return xc_highpri(func, arg1, arg2, NULL, ipl);
       } else {
               return xc_lowpri(func, arg1, arg2, NULL);
       }
}

static void
xc_nop(void *arg1, void *arg2)
{

       return;
}

/*
* xc_barrier:
*
*      Broadcast a nop to all CPUs in the system.
*/
void
xc_barrier(unsigned int flags)
{
       uint64_t where;

       where = xc_broadcast(flags, xc_nop, NULL, NULL);
       xc_wait(where);
}

/*
* xc_unicast:
*
*      Trigger a call on one CPU.
*/
uint64_t
xc_unicast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2,
   struct cpu_info *ci)
{

       KASSERT(ci != NULL);
       KASSERT(!cpu_intr_p());
       KASSERT(!cpu_softintr_p());
       ASSERT_SLEEPABLE();

       if (__predict_false(!mp_online)) {
               int s, bound;

               KASSERT(ci == curcpu());

               if (flags & XC_HIGHPRI)
                       s = splsoftserial();
               else
                       bound = curlwp_bind();
               (*func)(arg1, arg2);
               if (flags & XC_HIGHPRI)
                       splx(s);
               else
                       curlwp_bindx(bound);

               return 0;
       }

       if ((flags & XC_HIGHPRI) != 0) {
               unsigned int ipl = xc_extract_ipl(flags);
               return xc_highpri(func, arg1, arg2, ci, ipl);
       } else {
               return xc_lowpri(func, arg1, arg2, ci);
       }
}

/*
* xc_wait:
*
*      Wait for a cross call to complete.
*/
void
xc_wait(uint64_t where)
{
       xc_state_t *xc;

       KASSERT(!cpu_intr_p());
       KASSERT(!cpu_softintr_p());
       ASSERT_SLEEPABLE();

       if (__predict_false(!mp_online)) {
               return;
       }

       /* Determine whether it is high or low priority cross-call. */
       if ((where & XC_PRI_BIT) != 0) {
               xc = &xc_high_pri;
               where &= ~XC_PRI_BIT;
       } else {
               xc = &xc_low_pri;
       }

#ifdef __HAVE_ATOMIC64_LOADSTORE
       /* Fast path, if already done. */
       if (atomic_load_acquire(&xc->xc_donep) >= where) {
               return;
       }
#endif

       /* Slow path: block until awoken. */
       mutex_enter(&xc->xc_lock);
       while (xc->xc_donep < where) {
               cv_wait(&xc->xc_busy, &xc->xc_lock);
       }
       mutex_exit(&xc->xc_lock);
}

/*
* xc_lowpri:
*
*      Trigger a low priority call on one or more CPUs.
*/
static inline uint64_t
xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci)
{
       xc_state_t *xc = &xc_low_pri;
       CPU_INFO_ITERATOR cii;
       uint64_t where;

       mutex_enter(&xc->xc_lock);
       while (xc->xc_headp != xc->xc_donep) {
               cv_wait(&xc->xc_busy, &xc->xc_lock);
       }
       xc->xc_arg1 = arg1;
       xc->xc_arg2 = arg2;
       xc->xc_func = func;
       if (ci == NULL) {
               xc_broadcast_ev.ev_count++;
               for (CPU_INFO_FOREACH(cii, ci)) {
                       if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0)
                               continue;
                       xc->xc_headp += 1;
                       ci->ci_data.cpu_xcall_pending = true;
                       cv_signal(&ci->ci_data.cpu_xcall);
               }
       } else {
               xc_unicast_ev.ev_count++;
               xc->xc_headp += 1;
               ci->ci_data.cpu_xcall_pending = true;
               cv_signal(&ci->ci_data.cpu_xcall);
       }
       KASSERT(xc->xc_donep < xc->xc_headp);
       where = xc->xc_headp;
       mutex_exit(&xc->xc_lock);

       /* Return a low priority ticket. */
       KASSERT((where & XC_PRI_BIT) == 0);
       return where;
}

/*
* xc_thread:
*
*      One thread per-CPU to dispatch low priority calls.
*/
static void
xc_thread(void *cookie)
{
       struct cpu_info *ci = curcpu();
       xc_state_t *xc = &xc_low_pri;
       void *arg1, *arg2;
       xcfunc_t func;
       struct lwp *l = curlwp;

       KASSERTMSG(l->l_nopreempt == 0, "lwp %p nopreempt %d",
           l, l->l_nopreempt);

       mutex_enter(&xc->xc_lock);
       for (;;) {
               while (!ci->ci_data.cpu_xcall_pending) {
                       if (xc->xc_headp == xc->xc_donep) {
                               cv_broadcast(&xc->xc_busy);
                       }
                       cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock);
                       KASSERT(ci == curcpu());
               }
               ci->ci_data.cpu_xcall_pending = false;
               func = xc->xc_func;
               arg1 = xc->xc_arg1;
               arg2 = xc->xc_arg2;
               mutex_exit(&xc->xc_lock);

               KASSERT(func != NULL);
               (*func)(arg1, arg2);

               KASSERTMSG(l->l_nopreempt == 0, "lwp %p nopreempt %d func %p",
                   l, l->l_nopreempt, func);

               mutex_enter(&xc->xc_lock);
#ifdef __HAVE_ATOMIC64_LOADSTORE
               atomic_store_release(&xc->xc_donep, xc->xc_donep + 1);
#else
               xc->xc_donep++;
#endif
       }
       /* NOTREACHED */
}

/*
* xc_ipi_handler:
*
*      Handler of cross-call IPI.
*/
void
xc_ipi_handler(void)
{
       xc_state_t *xc = & xc_high_pri;

       KASSERT(xc->xc_ipl < __arraycount(xc_sihs));
       KASSERT(xc_sihs[xc->xc_ipl] != NULL);

       /* Executes xc__highpri_intr() via software interrupt. */
       softint_schedule(xc_sihs[xc->xc_ipl]);
}

/*
* xc__highpri_intr:
*
*      A software interrupt handler for high priority calls.
*/
void
xc__highpri_intr(void *dummy)
{
       xc_state_t *xc = &xc_high_pri;
       void *arg1, *arg2;
       xcfunc_t func;

       KASSERTMSG(!cpu_intr_p(), "high priority xcall for function %p",
           xc->xc_func);
       /*
        * Lock-less fetch of function and its arguments.
        * Safe since it cannot change at this point.
        */
       func = xc->xc_func;
       arg1 = xc->xc_arg1;
       arg2 = xc->xc_arg2;

       KASSERT(func != NULL);
       (*func)(arg1, arg2);

       /*
        * Note the request as done, and if we have reached the head,
        * cross-call has been processed - notify waiters, if any.
        */
       mutex_enter(&xc->xc_lock);
       KASSERT(xc->xc_donep < xc->xc_headp);
#ifdef __HAVE_ATOMIC64_LOADSTORE
       atomic_store_release(&xc->xc_donep, xc->xc_donep + 1);
#else
       xc->xc_donep++;
#endif
       if (xc->xc_donep == xc->xc_headp) {
               cv_broadcast(&xc->xc_busy);
       }
       mutex_exit(&xc->xc_lock);
}

/*
* xc_highpri:
*
*      Trigger a high priority call on one or more CPUs.
*/
static inline uint64_t
xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci,
   unsigned int ipl)
{
       xc_state_t *xc = &xc_high_pri;
       uint64_t where;

       mutex_enter(&xc->xc_lock);
       while (xc->xc_headp != xc->xc_donep) {
               cv_wait(&xc->xc_busy, &xc->xc_lock);
       }
       xc->xc_func = func;
       xc->xc_arg1 = arg1;
       xc->xc_arg2 = arg2;
       xc->xc_headp += (ci ? 1 : ncpu);
       xc->xc_ipl = ipl;
       where = xc->xc_headp;
       mutex_exit(&xc->xc_lock);

       /*
        * Send the IPI once lock is released.
        * Note: it will handle the local CPU case.
        */

#ifdef _RUMPKERNEL
       rump_xc_highpri(ci);
#else
#ifdef MULTIPROCESSOR
       kpreempt_disable();
       if (curcpu() == ci) {
               /* Unicast: local CPU. */
               xc_ipi_handler();
       } else if (ci) {
               /* Unicast: remote CPU. */
               xc_send_ipi(ci);
       } else {
               /* Broadcast: all, including local. */
               xc_send_ipi(NULL);
               xc_ipi_handler();
       }
       kpreempt_enable();
#else
       KASSERT(ci == NULL || curcpu() == ci);
       xc_ipi_handler();
#endif
#endif

       /* Indicate a high priority ticket. */
       return (where | XC_PRI_BIT);
}

You are viewing proxied material from ftp.icm.edu.pl. The copyright of proxied material belongs to its original authors. Any comments or complaints in relation to proxied material should be directed to the original authors of the content concerned. Please see the disclaimer for more details.