/*-
* Copyright (c)2007,2008 YAMAMOTO Takashi,
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* percpu_init_cpu: cpu initialization
*
* => should be called before the cpu appears on the list for CPU_INFO_FOREACH.
* => may be called for static CPUs afterward (typically just primary CPU)
*/
/*
* For the primary CPU, prior percpu_create may have already
* triggered allocation, so there's nothing more for us to do
* here.
*/
if (pcc->pcc_size)
return;
KASSERT(pcc->pcc_data == NULL);
/*
* Otherwise, allocate storage and, while the constructor list
* is locked, run constructors for all percpus on this CPU.
*/
pcc->pcc_size = size;
if (size) {
pcc->pcc_data = kmem_zalloc(pcc->pcc_size, KM_SLEEP);
mutex_enter(&percpu_allocation.lock);
while (percpu_allocation.busy)
cv_wait(&percpu_allocation.cv,
&percpu_allocation.lock);
percpu_allocation.busy = curlwp;
LIST_FOREACH(pc, &percpu_allocation.ctor_list, pc_list) {
KASSERT(pc->pc_ctor);
mutex_exit(&percpu_allocation.lock);
(*pc->pc_ctor)((char *)pcc->pcc_data + pc->pc_offset,
pc->pc_cookie, ci);
mutex_enter(&percpu_allocation.lock);
}
KASSERT(percpu_allocation.busy == curlwp);
percpu_allocation.busy = NULL;
cv_broadcast(&percpu_allocation.cv);
mutex_exit(&percpu_allocation.lock);
}
}
/*
* percpu_alloc: allocate percpu storage
*
* => called in thread context.
* => considered as an expensive and rare operation.
* => allocated storage is initialized with zeros.
*/
percpu_t *
percpu_alloc(size_t size)
{
return percpu_create(size, NULL, NULL, NULL);
}
/*
* percpu_create: allocate percpu storage and associate ctor/dtor with it
*
* => called in thread context.
* => considered as an expensive and rare operation.
* => allocated storage is initialized by ctor, or zeros if ctor is null
* => percpu_free will call dtor first, if dtor is nonnull
* => ctor or dtor may sleep, even on allocation
*/
if (ctor) {
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
void *buf;
/*
* Wait until nobody is using the list of percpus with
* constructors.
*/
mutex_enter(&percpu_allocation.lock);
while (percpu_allocation.busy)
cv_wait(&percpu_allocation.cv,
&percpu_allocation.lock);
percpu_allocation.busy = curlwp;
mutex_exit(&percpu_allocation.lock);
/*
* Run the constructor for all CPUs. We use a
* temporary buffer wo that we need not hold the
* percpu_swap_lock while running the constructor.
*/
buf = kmem_alloc(size, KM_SLEEP);
for (CPU_INFO_FOREACH(cii, ci)) {
memset(buf, 0, size);
(*ctor)(buf, cookie, ci);
percpu_traverse_enter();
memcpy(percpu_getptr_remote(pc, ci), buf, size);
percpu_traverse_exit();
}
explicit_memset(buf, 0, size);
kmem_free(buf, size);
/*
* Insert the percpu into the list of percpus with
* constructors. We are now done using the list, so it
* is safe for concurrent percpu_create or concurrent
* percpu_init_cpu to run.
*/
mutex_enter(&percpu_allocation.lock);
KASSERT(percpu_allocation.busy == curlwp);
percpu_allocation.busy = NULL;
cv_broadcast(&percpu_allocation.cv);
LIST_INSERT_HEAD(&percpu_allocation.ctor_list, pc, pc_list);
mutex_exit(&percpu_allocation.lock);
} else {
percpu_zero(pc, size);
}
return pc;
}
/*
* percpu_free: free percpu storage
*
* => called in thread context.
* => considered as an expensive and rare operation.
*/
void
percpu_free(percpu_t *pc, size_t size)
{
ASSERT_SLEEPABLE();
KASSERT(size == pc->pc_size);
/*
* If there's a constructor, take the percpu off the list of
* percpus with constructors, but first wait until nobody is
* using the list.
*/
if (pc->pc_ctor) {
mutex_enter(&percpu_allocation.lock);
while (percpu_allocation.busy)
cv_wait(&percpu_allocation.cv,
&percpu_allocation.lock);
LIST_REMOVE(pc, pc_list);
mutex_exit(&percpu_allocation.lock);
}
/* If there's a destructor, run it now for all CPUs. */
if (pc->pc_dtor) {
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
void *buf;
/*
* percpu_foreach: call the specified callback function for each cpus.
*
* => must be called from thread context.
* => callback executes on **current** CPU (or, really, arbitrary CPU,
* in case of preemption)
* => caller should not rely on the cpu iteration order.
* => the callback function should be minimum because it is executed with
* holding a global lock, which can block low-priority xcalls.
* eg. it's illegal for a callback function to sleep for memory allocation.
*/
void
percpu_foreach(percpu_t *pc, percpu_callback_t cb, void *arg)
{
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
/*
* percpu_foreach_xcall: call the specified callback function for each
* cpu. This version uses an xcall to run the callback on each cpu.
*
* => must be called from thread context.
* => callback executes on **remote** CPU in soft-interrupt context
* (at the specified soft interrupt priority).
* => caller should not rely on the cpu iteration order.
* => the callback function should be minimum because it may be
* executed in soft-interrupt context. eg. it's illegal for
* a callback function to sleep for memory allocation.
*/
void
percpu_foreach_xcall(percpu_t *pc, u_int xcflags, percpu_callback_t cb,
void *arg)
{
struct percpu_xcall_ctx ctx = {
.ctx_cb = cb,
.ctx_arg = arg,
};
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
