* Copyright (c)2008,2009,2010 YAMAMOTO Takashi,

/* $NetBSD: tprof.c,v 1.23 2023/04/11 10:07:12 msaitoh Exp $ */

/*-
* Copyright (c)2008,2009,2010 YAMAMOTO Takashi,
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: tprof.c,v 1.23 2023/04/11 10:07:12 msaitoh Exp $");

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>

#include <sys/callout.h>
#include <sys/conf.h>
#include <sys/cpu.h>
#include <sys/kmem.h>
#include <sys/module.h>
#include <sys/percpu.h>
#include <sys/poll.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/select.h>
#include <sys/workqueue.h>
#include <sys/xcall.h>

#include <dev/tprof/tprof.h>
#include <dev/tprof/tprof_ioctl.h>

#include "ioconf.h"

#ifndef TPROF_HZ
#define TPROF_HZ 10000
#endif

/*
* locking order:
* tprof_reader_lock -> tprof_lock
* tprof_startstop_lock -> tprof_lock
*/

/*
* protected by:
* L: tprof_lock
* R: tprof_reader_lock
* S: tprof_startstop_lock
* s: writer should hold tprof_startstop_lock and tprof_lock
* reader should hold tprof_startstop_lock or tprof_lock
*/

typedef struct tprof_buf {
u_int b_used;
u_int b_size;
u_int b_overflow;
u_int b_unused;
STAILQ_ENTRY(tprof_buf) b_list;
tprof_sample_t b_data[];
} tprof_buf_t;
#define TPROF_BUF_BYTESIZE(sz) \
(sizeof(tprof_buf_t) + (sz) * sizeof(tprof_sample_t))
#define TPROF_MAX_SAMPLES_PER_BUF TPROF_HZ

typedef struct {
tprof_buf_t *c_buf;
uint32_t c_cpuid;
struct work c_work;
callout_t c_callout;
} __aligned(CACHE_LINE_SIZE) tprof_cpu_t;

typedef struct tprof_backend {
/*
* tprof_backend_softc_t must be passed as an argument to the interrupt
* handler, but since this is difficult to implement in armv7/v8. Then,
* tprof_backend is exposed. Additionally, softc must be placed at the
* beginning of struct tprof_backend.
*/
tprof_backend_softc_t tb_softc;

const char *tb_name;
const tprof_backend_ops_t *tb_ops;
LIST_ENTRY(tprof_backend) tb_list;
} tprof_backend_t;

static kmutex_t tprof_lock;
static u_int tprof_nworker; /* L: # of running worker LWPs */
static lwp_t *tprof_owner;
static STAILQ_HEAD(, tprof_buf) tprof_list; /* L: global buffer list */
static u_int tprof_nbuf_on_list; /* L: # of buffers on tprof_list */
static struct workqueue *tprof_wq;
static struct percpu *tprof_cpus __read_mostly; /* tprof_cpu_t * */
static u_int tprof_samples_per_buf;
static u_int tprof_max_buf;

tprof_backend_t *tprof_backend; /* S: */
static LIST_HEAD(, tprof_backend) tprof_backends =
LIST_HEAD_INITIALIZER(tprof_backend); /* S: */

static kmutex_t tprof_reader_lock;
static kcondvar_t tprof_reader_cv; /* L: */
static off_t tprof_reader_offset; /* R: */

static kmutex_t tprof_startstop_lock;
static kcondvar_t tprof_cv; /* L: */
static struct selinfo tprof_selp; /* L: */

static struct tprof_stat tprof_stat; /* L: */

static tprof_cpu_t *
tprof_cpu_direct(struct cpu_info *ci)
{
tprof_cpu_t **cp;

cp = percpu_getptr_remote(tprof_cpus, ci);
return *cp;
}

static tprof_cpu_t *
tprof_cpu(struct cpu_info *ci)
{
tprof_cpu_t *c;

/*
* As long as xcalls are blocked -- e.g., by kpreempt_disable
* -- the percpu object will not be swapped and destroyed. We
* can't write to it, because the data may have already been
* moved to a new buffer, but we can safely read from it.
*/
kpreempt_disable();
c = tprof_cpu_direct(ci);
kpreempt_enable();

return c;
}

static tprof_cpu_t *
tprof_curcpu(void)
{

return tprof_cpu(curcpu());
}

static tprof_buf_t *
tprof_buf_alloc(void)
{
tprof_buf_t *new;
u_int size = tprof_samples_per_buf;

new = kmem_alloc(TPROF_BUF_BYTESIZE(size), KM_SLEEP);
new->b_used = 0;
new->b_size = size;
new->b_overflow = 0;
return new;
}

static void
tprof_buf_free(tprof_buf_t *buf)
{

kmem_free(buf, TPROF_BUF_BYTESIZE(buf->b_size));
}

static tprof_buf_t *
tprof_buf_switch(tprof_cpu_t *c, tprof_buf_t *new)
{
tprof_buf_t *old;

old = c->c_buf;
c->c_buf = new;
return old;
}

static tprof_buf_t *
tprof_buf_refresh(void)
{
tprof_cpu_t * const c = tprof_curcpu();
tprof_buf_t *new;

new = tprof_buf_alloc();
return tprof_buf_switch(c, new);
}

static void
tprof_worker(struct work *wk, void *dummy)
{
tprof_cpu_t * const c = tprof_curcpu();
tprof_buf_t *buf;
tprof_backend_t *tb;
bool shouldstop;

KASSERT(wk == &c->c_work);
KASSERT(dummy == NULL);

/*
* Get a per cpu buffer.
*/
buf = tprof_buf_refresh();

/*
* and put it on the global list for read(2).
*/
mutex_enter(&tprof_lock);
tb = tprof_backend;
shouldstop = (tb == NULL || tb->tb_softc.sc_ctr_running_mask == 0);
if (shouldstop) {
KASSERT(tprof_nworker > 0);
tprof_nworker--;
cv_broadcast(&tprof_cv);
cv_broadcast(&tprof_reader_cv);
}
if (buf->b_used == 0) {
tprof_stat.ts_emptybuf++;
} else if (tprof_nbuf_on_list < tprof_max_buf) {
tprof_stat.ts_sample += buf->b_used;
tprof_stat.ts_overflow += buf->b_overflow;
tprof_stat.ts_buf++;
STAILQ_INSERT_TAIL(&tprof_list, buf, b_list);
tprof_nbuf_on_list++;
buf = NULL;
selnotify(&tprof_selp, 0, NOTE_SUBMIT);
cv_broadcast(&tprof_reader_cv);
} else {
tprof_stat.ts_dropbuf_sample += buf->b_used;
tprof_stat.ts_dropbuf++;
}
mutex_exit(&tprof_lock);
if (buf)
tprof_buf_free(buf);

if (!shouldstop)
callout_schedule(&c->c_callout, hz / 8);
}

static void
tprof_kick(void *vp)
{
struct cpu_info * const ci = vp;
tprof_cpu_t * const c = tprof_cpu(ci);

workqueue_enqueue(tprof_wq, &c->c_work, ci);
}

static void
tprof_stop1(void)
{
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;

KASSERT(mutex_owned(&tprof_startstop_lock));
KASSERT(tprof_nworker == 0);

for (CPU_INFO_FOREACH(cii, ci)) {
tprof_cpu_t * const c = tprof_cpu(ci);
tprof_buf_t *old;

old = tprof_buf_switch(c, NULL);
if (old != NULL)
tprof_buf_free(old);

callout_destroy(&c->c_callout);
}
workqueue_destroy(tprof_wq);
}

static void
tprof_getinfo(struct tprof_info *info)
{
tprof_backend_t *tb;

KASSERT(mutex_owned(&tprof_startstop_lock));

memset(info, 0, sizeof(*info));
info->ti_version = TPROF_VERSION;
if ((tb = tprof_backend) != NULL)
info->ti_ident = tb->tb_ops->tbo_ident();
}

static int
tprof_getncounters(u_int *ncounters)
{
tprof_backend_t *tb;

tb = tprof_backend;
if (tb == NULL)
return ENOENT;

*ncounters = tb->tb_ops->tbo_ncounters();
return 0;
}

static void
tprof_start_cpu(void *arg1, void *arg2)
{
tprof_backend_t *tb = arg1;
tprof_countermask_t runmask = (uintptr_t)arg2;

tb->tb_ops->tbo_start(runmask);
}

static void
tprof_stop_cpu(void *arg1, void *arg2)
{
tprof_backend_t *tb = arg1;
tprof_countermask_t stopmask = (uintptr_t)arg2;

tb->tb_ops->tbo_stop(stopmask);
}

static int
tprof_start(tprof_countermask_t runmask)
{
CPU_INFO_ITERATOR cii;
struct cpu_info *ci;
tprof_backend_t *tb;
uint64_t xc;
int error;
bool firstrun;

KASSERT(mutex_owned(&tprof_startstop_lock));

tb = tprof_backend;
if (tb == NULL) {
error = ENOENT;
goto done;
}

runmask &= ~tb->tb_softc.sc_ctr_running_mask;
runmask &= tb->tb_softc.sc_ctr_configured_mask;
if (runmask == 0) {
/*
* Targets are already running.
* Unconfigured counters are ignored.
*/
error = 0;
goto done;
}

firstrun = (tb->tb_softc.sc_ctr_running_mask == 0);
if (firstrun) {
if (tb->tb_ops->tbo_establish != NULL) {
error = tb->tb_ops->tbo_establish(&tb->tb_softc);
if (error != 0)
goto done;
}

tprof_samples_per_buf = TPROF_MAX_SAMPLES_PER_BUF;
tprof_max_buf = ncpu * 3;
error = workqueue_create(&tprof_wq, "tprofmv", tprof_worker,
NULL, PRI_NONE, IPL_SOFTCLOCK, WQ_MPSAFE | WQ_PERCPU);
if (error != 0) {
if (tb->tb_ops->tbo_disestablish != NULL)
tb->tb_ops->tbo_disestablish(&tb->tb_softc);
goto done;
}

for (CPU_INFO_FOREACH(cii, ci)) {
tprof_cpu_t * const c = tprof_cpu(ci);
tprof_buf_t *new;
tprof_buf_t *old;

new = tprof_buf_alloc();
old = tprof_buf_switch(c, new);
if (old != NULL) {
tprof_buf_free(old);
}
callout_init(&c->c_callout, CALLOUT_MPSAFE);
callout_setfunc(&c->c_callout, tprof_kick, ci);
}
}

runmask &= tb->tb_softc.sc_ctr_configured_mask;
xc = xc_broadcast(0, tprof_start_cpu, tb, (void *)(uintptr_t)runmask);
xc_wait(xc);
mutex_enter(&tprof_lock);
tb->tb_softc.sc_ctr_running_mask |= runmask;
mutex_exit(&tprof_lock);

if (firstrun) {
for (CPU_INFO_FOREACH(cii, ci)) {
tprof_cpu_t * const c = tprof_cpu(ci);

mutex_enter(&tprof_lock);
tprof_nworker++;
mutex_exit(&tprof_lock);
workqueue_enqueue(tprof_wq, &c->c_work, ci);
}
}
error = 0;

done:
return error;
}

static void
tprof_stop(tprof_countermask_t stopmask)
{
tprof_backend_t *tb;
uint64_t xc;

tb = tprof_backend;
if (tb == NULL)
return;

KASSERT(mutex_owned(&tprof_startstop_lock));
stopmask &= tb->tb_softc.sc_ctr_running_mask;
if (stopmask == 0) {
/* Targets are not running */
goto done;
}

xc = xc_broadcast(0, tprof_stop_cpu, tb, (void *)(uintptr_t)stopmask);
xc_wait(xc);
mutex_enter(&tprof_lock);
tb->tb_softc.sc_ctr_running_mask &= ~stopmask;
mutex_exit(&tprof_lock);

/* All counters have stopped? */
if (tb->tb_softc.sc_ctr_running_mask == 0) {
mutex_enter(&tprof_lock);
cv_broadcast(&tprof_reader_cv);
while (tprof_nworker > 0)
cv_wait(&tprof_cv, &tprof_lock);

mutex_exit(&tprof_lock);

tprof_stop1();
if (tb->tb_ops->tbo_disestablish != NULL)
tb->tb_ops->tbo_disestablish(&tb->tb_softc);
}
done:
;
}

static void
tprof_init_percpu_counters_offset(void *vp, void *vp2, struct cpu_info *ci)
{
uint64_t *counters_offset = vp;
u_int counter = (uintptr_t)vp2;

tprof_backend_t *tb = tprof_backend;
tprof_param_t *param = &tb->tb_softc.sc_count[counter].ctr_param;
counters_offset[counter] = param->p_value;
}

static void
tprof_configure_event_cpu(void *arg1, void *arg2)
{
tprof_backend_t *tb = arg1;
u_int counter = (uintptr_t)arg2;
tprof_param_t *param = &tb->tb_softc.sc_count[counter].ctr_param;

tb->tb_ops->tbo_configure_event(counter, param);
}

static int
tprof_configure_event(const tprof_param_t *param)
{
tprof_backend_t *tb;
tprof_backend_softc_t *sc;
tprof_param_t *sc_param;
uint64_t xc;
int c, error;

if ((param->p_flags & (TPROF_PARAM_USER | TPROF_PARAM_KERN)) == 0) {
error = EINVAL;
goto done;
}

tb = tprof_backend;
if (tb == NULL) {
error = ENOENT;
goto done;
}
sc = &tb->tb_softc;

c = param->p_counter;
if (c >= tb->tb_softc.sc_ncounters) {
error = EINVAL;
goto done;
}

if (tb->tb_ops->tbo_valid_event != NULL) {
error = tb->tb_ops->tbo_valid_event(param->p_counter, param);
if (error != 0)
goto done;
}

/* if already running, stop the counter */
if (ISSET(c, tb->tb_softc.sc_ctr_running_mask))
tprof_stop(__BIT(c));

sc->sc_count[c].ctr_bitwidth =
tb->tb_ops->tbo_counter_bitwidth(param->p_counter);

sc_param = &sc->sc_count[c].ctr_param;
memcpy(sc_param, param, sizeof(*sc_param)); /* save copy of param */

if (ISSET(param->p_flags, TPROF_PARAM_PROFILE)) {
uint64_t freq, inum, dnum;

freq = tb->tb_ops->tbo_counter_estimate_freq(c);
sc->sc_count[c].ctr_counter_val = freq / TPROF_HZ;
if (sc->sc_count[c].ctr_counter_val == 0) {
printf("%s: counter#%d frequency (%"PRIu64") is"
" very low relative to TPROF_HZ (%u)\n", __func__,
c, freq, TPROF_HZ);
sc->sc_count[c].ctr_counter_val =
4000000000ULL / TPROF_HZ;
}

switch (param->p_flags & TPROF_PARAM_VALUE2_MASK) {
case TPROF_PARAM_VALUE2_SCALE:
if (sc_param->p_value2 == 0)
break;
/*
* p_value2 is 64-bit fixed-point
* upper 32 bits are the integer part
* lower 32 bits are the decimal part
*/
inum = sc_param->p_value2 >> 32;
dnum = sc_param->p_value2 & __BITS(31, 0);
sc->sc_count[c].ctr_counter_val =
sc->sc_count[c].ctr_counter_val * inum +
(sc->sc_count[c].ctr_counter_val * dnum >> 32);
if (sc->sc_count[c].ctr_counter_val == 0)
sc->sc_count[c].ctr_counter_val = 1;
break;
case TPROF_PARAM_VALUE2_TRIGGERCOUNT:
if (sc_param->p_value2 == 0)
sc_param->p_value2 = 1;
if (sc_param->p_value2 >
__BITS(sc->sc_count[c].ctr_bitwidth - 1, 0)) {
sc_param->p_value2 =
__BITS(sc->sc_count[c].ctr_bitwidth - 1, 0);
}
sc->sc_count[c].ctr_counter_val = sc_param->p_value2;
break;
default:
break;
}
sc->sc_count[c].ctr_counter_reset_val =
-sc->sc_count[c].ctr_counter_val;
sc->sc_count[c].ctr_counter_reset_val &=
__BITS(sc->sc_count[c].ctr_bitwidth - 1, 0);
} else {
sc->sc_count[c].ctr_counter_val = 0;
sc->sc_count[c].ctr_counter_reset_val = 0;
}

/* At this point, p_value is used as an initial value */
percpu_foreach(tb->tb_softc.sc_ctr_offset_percpu,
tprof_init_percpu_counters_offset, (void *)(uintptr_t)c);
/* On the backend side, p_value is used as the reset value */
sc_param->p_value = tb->tb_softc.sc_count[c].ctr_counter_reset_val;

xc = xc_broadcast(0, tprof_configure_event_cpu,
tb, (void *)(uintptr_t)c);
xc_wait(xc);

mutex_enter(&tprof_lock);
/* update counters bitmasks */
SET(tb->tb_softc.sc_ctr_configured_mask, __BIT(c));
CLR(tb->tb_softc.sc_ctr_prof_mask, __BIT(c));
CLR(tb->tb_softc.sc_ctr_ovf_mask, __BIT(c));
/* profiled counter requires overflow handling */
if (ISSET(param->p_flags, TPROF_PARAM_PROFILE)) {
SET(tb->tb_softc.sc_ctr_prof_mask, __BIT(c));
SET(tb->tb_softc.sc_ctr_ovf_mask, __BIT(c));
}
/* counters with less than 64bits also require overflow handling */
if (sc->sc_count[c].ctr_bitwidth != 64)
SET(tb->tb_softc.sc_ctr_ovf_mask, __BIT(c));
mutex_exit(&tprof_lock);

error = 0;

done:
return error;
}

static void
tprof_getcounts_cpu(void *arg1, void *arg2)
{
tprof_backend_t *tb = arg1;
tprof_backend_softc_t *sc = &tb->tb_softc;
uint64_t *counters = arg2;
uint64_t *counters_offset;
unsigned int c;

tprof_countermask_t configmask = sc->sc_ctr_configured_mask;
counters_offset = percpu_getref(sc->sc_ctr_offset_percpu);
for (c = 0; c < sc->sc_ncounters; c++) {
if (ISSET(configmask, __BIT(c))) {
uint64_t ctr = tb->tb_ops->tbo_counter_read(c);
counters[c] = counters_offset[c] +
((ctr - sc->sc_count[c].ctr_counter_reset_val) &
__BITS(sc->sc_count[c].ctr_bitwidth - 1, 0));
} else
counters[c] = 0;
}
percpu_putref(sc->sc_ctr_offset_percpu);
}

static int
tprof_getcounts(tprof_counts_t *counts)
{
struct cpu_info *ci;
tprof_backend_t *tb;
uint64_t xc;

tb = tprof_backend;
if (tb == NULL)
return ENOENT;

if (counts->c_cpu >= ncpu)
return ESRCH;
ci = cpu_lookup(counts->c_cpu);
if (ci == NULL)
return ESRCH;

xc = xc_unicast(0, tprof_getcounts_cpu, tb, counts->c_count, ci);
xc_wait(xc);

counts->c_ncounters = tb->tb_softc.sc_ncounters;
counts->c_runningmask = tb->tb_softc.sc_ctr_running_mask;
return 0;
}

/*
* tprof_clear: drain unread samples.
*/

static void
tprof_clear(void)
{
tprof_buf_t *buf;

mutex_enter(&tprof_reader_lock);
mutex_enter(&tprof_lock);
while ((buf = STAILQ_FIRST(&tprof_list)) != NULL) {
if (buf != NULL) {
STAILQ_REMOVE_HEAD(&tprof_list, b_list);
KASSERT(tprof_nbuf_on_list > 0);
tprof_nbuf_on_list--;
mutex_exit(&tprof_lock);
tprof_buf_free(buf);
mutex_enter(&tprof_lock);
}
}
KASSERT(tprof_nbuf_on_list == 0);
mutex_exit(&tprof_lock);
tprof_reader_offset = 0;
mutex_exit(&tprof_reader_lock);

memset(&tprof_stat, 0, sizeof(tprof_stat));
}

static tprof_backend_t *
tprof_backend_lookup(const char *name)
{
tprof_backend_t *tb;

KASSERT(mutex_owned(&tprof_startstop_lock));

LIST_FOREACH(tb, &tprof_backends, tb_list) {
if (!strcmp(tb->tb_name, name)) {
return tb;
}
}
return NULL;
}

/* -------------------- backend interfaces */

/*
* tprof_sample: record a sample on the per-cpu buffer.
*
* be careful; can be called in NMI context.
* we are bluntly assuming the followings are safe.
* curcpu()
* curlwp->l_lid
* curlwp->l_proc->p_pid
*/

void
tprof_sample(void *unused, const tprof_frame_info_t *tfi)
{
tprof_cpu_t * const c = tprof_cpu_direct(curcpu());
tprof_buf_t * const buf = c->c_buf;
tprof_sample_t *sp;
const uintptr_t pc = tfi->tfi_pc;
const lwp_t * const l = curlwp;
u_int idx;

idx = buf->b_used;
if (__predict_false(idx >= buf->b_size)) {
buf->b_overflow++;
return;
}
sp = &buf->b_data[idx];
sp->s_pid = l->l_proc->p_pid;
sp->s_lwpid = l->l_lid;
sp->s_cpuid = c->c_cpuid;
sp->s_flags = ((tfi->tfi_inkernel) ? TPROF_SAMPLE_INKERNEL : 0) |
__SHIFTIN(tfi->tfi_counter, TPROF_SAMPLE_COUNTER_MASK);
sp->s_pc = pc;
buf->b_used = idx + 1;
}

/*
* tprof_backend_register:
*/

int
tprof_backend_register(const char *name, const tprof_backend_ops_t *ops,
int vers)
{
tprof_backend_t *tb;

if (vers != TPROF_BACKEND_VERSION)
return EINVAL;

mutex_enter(&tprof_startstop_lock);
tb = tprof_backend_lookup(name);
if (tb != NULL) {
mutex_exit(&tprof_startstop_lock);
return EEXIST;
}
#if 1 /* XXX for now */
if (!LIST_EMPTY(&tprof_backends)) {
mutex_exit(&tprof_startstop_lock);
return ENOTSUP;
}
#endif
tb = kmem_zalloc(sizeof(*tb), KM_SLEEP);
tb->tb_name = name;
tb->tb_ops = ops;
LIST_INSERT_HEAD(&tprof_backends, tb, tb_list);
#if 1 /* XXX for now */
if (tprof_backend == NULL) {
tprof_backend = tb;
}
#endif
mutex_exit(&tprof_startstop_lock);

/* Init backend softc */
tb->tb_softc.sc_ncounters = tb->tb_ops->tbo_ncounters();
tb->tb_softc.sc_ctr_offset_percpu_size =
sizeof(uint64_t) * tb->tb_softc.sc_ncounters;
tb->tb_softc.sc_ctr_offset_percpu =
percpu_alloc(tb->tb_softc.sc_ctr_offset_percpu_size);

return 0;
}

/*
* tprof_backend_unregister:
*/

int
tprof_backend_unregister(const char *name)
{
tprof_backend_t *tb;

mutex_enter(&tprof_startstop_lock);
tb = tprof_backend_lookup(name);
#if defined(DIAGNOSTIC)
if (tb == NULL) {
mutex_exit(&tprof_startstop_lock);
panic("%s: not found '%s'", __func__, name);
}
#endif /* defined(DIAGNOSTIC) */
if (tb->tb_softc.sc_ctr_running_mask != 0) {
mutex_exit(&tprof_startstop_lock);
return EBUSY;
}
#if 1 /* XXX for now */
if (tprof_backend == tb)
tprof_backend = NULL;
#endif
LIST_REMOVE(tb, tb_list);
mutex_exit(&tprof_startstop_lock);

/* fini backend softc */
percpu_free(tb->tb_softc.sc_ctr_offset_percpu,
tb->tb_softc.sc_ctr_offset_percpu_size);

/* Free backend */
kmem_free(tb, sizeof(*tb));

return 0;
}

/* -------------------- cdevsw interfaces */

static int
tprof_open(dev_t dev, int flags, int type, struct lwp *l)
{

if (minor(dev) != 0)
return EXDEV;

mutex_enter(&tprof_lock);
if (tprof_owner != NULL) {
mutex_exit(&tprof_lock);
return EBUSY;
}
tprof_owner = curlwp;
mutex_exit(&tprof_lock);

return 0;
}

static int
tprof_close(dev_t dev, int flags, int type, struct lwp *l)
{

KASSERT(minor(dev) == 0);

mutex_enter(&tprof_startstop_lock);
mutex_enter(&tprof_lock);
tprof_owner = NULL;
mutex_exit(&tprof_lock);
tprof_stop(TPROF_COUNTERMASK_ALL);
tprof_clear();

tprof_backend_t *tb = tprof_backend;
if (tb != NULL) {
KASSERT(tb->tb_softc.sc_ctr_running_mask == 0);
tb->tb_softc.sc_ctr_configured_mask = 0;
tb->tb_softc.sc_ctr_prof_mask = 0;
tb->tb_softc.sc_ctr_ovf_mask = 0;
}

mutex_exit(&tprof_startstop_lock);

return 0;
}

static int
tprof_poll(dev_t dev, int events, struct lwp *l)
{
int revents;

revents = events & (POLLIN | POLLRDNORM);
if (revents == 0)
return 0;

mutex_enter(&tprof_lock);
if (STAILQ_EMPTY(&tprof_list)) {
revents = 0;
selrecord(l, &tprof_selp);
}
mutex_exit(&tprof_lock);

return revents;
}

static void
filt_tprof_read_detach(struct knote *kn)
{
mutex_enter(&tprof_lock);
selremove_knote(&tprof_selp, kn);
mutex_exit(&tprof_lock);
}

static int
filt_tprof_read_event(struct knote *kn, long hint)
{
int rv = 0;

if ((hint & NOTE_SUBMIT) == 0)
mutex_enter(&tprof_lock);

if (!STAILQ_EMPTY(&tprof_list)) {
tprof_buf_t *buf;
int64_t n = 0;

STAILQ_FOREACH(buf, &tprof_list, b_list) {
n += buf->b_used;
}
kn->kn_data = n * sizeof(tprof_sample_t);

rv = 1;
}

if ((hint & NOTE_SUBMIT) == 0)
mutex_exit(&tprof_lock);

return rv;
}

static const struct filterops tprof_read_filtops = {
.f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
.f_attach = NULL,
.f_detach = filt_tprof_read_detach,
.f_event = filt_tprof_read_event,
};

static int
tprof_kqfilter(dev_t dev, struct knote *kn)
{
switch (kn->kn_filter) {
case EVFILT_READ:
kn->kn_fop = &tprof_read_filtops;
mutex_enter(&tprof_lock);
selrecord_knote(&tprof_selp, kn);
mutex_exit(&tprof_lock);
break;
default:
return EINVAL;
}

return 0;
}

static int
tprof_read(dev_t dev, struct uio *uio, int flags)
{
tprof_buf_t *buf;
size_t bytes;
size_t resid;
size_t done = 0;
int error = 0;

KASSERT(minor(dev) == 0);
mutex_enter(&tprof_reader_lock);
while (uio->uio_resid > 0 && error == 0) {
/*
* Take the first buffer from the list.
*/
mutex_enter(&tprof_lock);
buf = STAILQ_FIRST(&tprof_list);
if (buf == NULL) {
if (tprof_nworker == 0 || done != 0) {
mutex_exit(&tprof_lock);
error = 0;
break;
}
mutex_exit(&tprof_reader_lock);
error = cv_wait_sig(&tprof_reader_cv, &tprof_lock);
mutex_exit(&tprof_lock);
mutex_enter(&tprof_reader_lock);
continue;
}
STAILQ_REMOVE_HEAD(&tprof_list, b_list);
KASSERT(tprof_nbuf_on_list > 0);
tprof_nbuf_on_list--;
mutex_exit(&tprof_lock);

/*
* Copy it out.
*/
bytes = MIN(buf->b_used * sizeof(tprof_sample_t) -
tprof_reader_offset, uio->uio_resid);
resid = uio->uio_resid;
error = uiomove((char *)buf->b_data + tprof_reader_offset,
bytes, uio);
done = resid - uio->uio_resid;
tprof_reader_offset += done;

/*
* If we didn't consume the whole buffer,
* put it back to the list.
*/
if (tprof_reader_offset <
buf->b_used * sizeof(tprof_sample_t)) {
mutex_enter(&tprof_lock);
STAILQ_INSERT_HEAD(&tprof_list, buf, b_list);
tprof_nbuf_on_list++;
cv_broadcast(&tprof_reader_cv);
mutex_exit(&tprof_lock);
} else {
tprof_buf_free(buf);
tprof_reader_offset = 0;
}
}
mutex_exit(&tprof_reader_lock);

return error;
}

static int
tprof_ioctl(dev_t dev, u_long cmd, void *data, int flags, struct lwp *l)
{
const tprof_param_t *param;
tprof_counts_t *counts;
int error = 0;

KASSERT(minor(dev) == 0);

switch (cmd) {
case TPROF_IOC_GETINFO:
mutex_enter(&tprof_startstop_lock);
tprof_getinfo(data);
mutex_exit(&tprof_startstop_lock);
break;
case TPROF_IOC_GETNCOUNTERS:
mutex_enter(&tprof_lock);
error = tprof_getncounters((u_int *)data);
mutex_exit(&tprof_lock);
break;
case TPROF_IOC_START:
mutex_enter(&tprof_startstop_lock);
error = tprof_start(*(tprof_countermask_t *)data);
mutex_exit(&tprof_startstop_lock);
break;
case TPROF_IOC_STOP:
mutex_enter(&tprof_startstop_lock);
tprof_stop(*(tprof_countermask_t *)data);
mutex_exit(&tprof_startstop_lock);
break;
case TPROF_IOC_GETSTAT:
mutex_enter(&tprof_lock);
memcpy(data, &tprof_stat, sizeof(tprof_stat));
mutex_exit(&tprof_lock);
break;
case TPROF_IOC_CONFIGURE_EVENT:
param = data;
mutex_enter(&tprof_startstop_lock);
error = tprof_configure_event(param);
mutex_exit(&tprof_startstop_lock);
break;
case TPROF_IOC_GETCOUNTS:
counts = data;
mutex_enter(&tprof_startstop_lock);
error = tprof_getcounts(counts);
mutex_exit(&tprof_startstop_lock);
break;
default:
error = EINVAL;
break;
}

return error;
}

const struct cdevsw tprof_cdevsw = {
.d_open = tprof_open,
.d_close = tprof_close,
.d_read = tprof_read,
.d_write = nowrite,
.d_ioctl = tprof_ioctl,
.d_stop = nostop,
.d_tty = notty,
.d_poll = tprof_poll,
.d_mmap = nommap,
.d_kqfilter = tprof_kqfilter,
.d_discard = nodiscard,
.d_flag = D_OTHER | D_MPSAFE
};

void
tprofattach(int nunits)
{

/* Nothing */
}

MODULE(MODULE_CLASS_DRIVER, tprof, NULL);

static void
tprof_cpu_init(void *vcp, void *vcookie, struct cpu_info *ci)
{
tprof_cpu_t **cp = vcp, *c;

c = kmem_zalloc(sizeof(*c), KM_SLEEP);
c->c_buf = NULL;
c->c_cpuid = cpu_index(ci);
*cp = c;
}

static void
tprof_cpu_fini(void *vcp, void *vcookie, struct cpu_info *ci)
{
tprof_cpu_t **cp = vcp, *c;

c = *cp;
KASSERT(c->c_cpuid == cpu_index(ci));
KASSERT(c->c_buf == NULL);
kmem_free(c, sizeof(*c));
*cp = NULL;
}

static void
tprof_driver_init(void)
{

mutex_init(&tprof_lock, MUTEX_DEFAULT, IPL_NONE);
mutex_init(&tprof_reader_lock, MUTEX_DEFAULT, IPL_NONE);
mutex_init(&tprof_startstop_lock, MUTEX_DEFAULT, IPL_NONE);
selinit(&tprof_selp);
cv_init(&tprof_cv, "tprof");
cv_init(&tprof_reader_cv, "tprof_rd");
STAILQ_INIT(&tprof_list);
tprof_cpus = percpu_create(sizeof(tprof_cpu_t *),
tprof_cpu_init, tprof_cpu_fini, NULL);
}

static void
tprof_driver_fini(void)
{

percpu_free(tprof_cpus, sizeof(tprof_cpu_t *));
mutex_destroy(&tprof_lock);
mutex_destroy(&tprof_reader_lock);
mutex_destroy(&tprof_startstop_lock);
seldestroy(&tprof_selp);
cv_destroy(&tprof_cv);
cv_destroy(&tprof_reader_cv);
}

static int
tprof_modcmd(modcmd_t cmd, void *arg)
{

switch (cmd) {
case MODULE_CMD_INIT:
tprof_driver_init();
#if defined(_MODULE)
{
devmajor_t bmajor = NODEVMAJOR;
devmajor_t cmajor = NODEVMAJOR;
int error;

error = devsw_attach("tprof", NULL, &bmajor,
&tprof_cdevsw, &cmajor);
if (error) {
tprof_driver_fini();
return error;
}
}
#endif /* defined(_MODULE) */
return 0;

case MODULE_CMD_FINI:
#if defined(_MODULE)
devsw_detach(NULL, &tprof_cdevsw);
#endif /* defined(_MODULE) */
tprof_driver_fini();
return 0;

default:
return ENOTTY;
}
}