/*-
* Copyright (c) 2019 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Taylor R. Campbell.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* Entropy subsystem
*
* * Each CPU maintains a per-CPU entropy pool so that gathering
* entropy requires no interprocessor synchronization, except
* early at boot when we may be scrambling to gather entropy as
* soon as possible.
*
* - entropy_enter gathers entropy and never drops it on the
* floor, at the cost of sometimes having to do cryptography.
*
* - entropy_enter_intr gathers entropy or drops it on the
* floor, with low latency. Work to stir the pool or kick the
* housekeeping thread is scheduled in soft interrupts.
*
* * entropy_enter immediately enters into the global pool if it
* can transition to full entropy in one swell foop. Otherwise,
* it defers to a housekeeping thread that consolidates entropy,
* but only when the CPUs collectively have full entropy, in
* order to mitigate iterative-guessing attacks.
*
* * The entropy housekeeping thread continues to consolidate
* entropy even after we think we have full entropy, in case we
* are wrong, but is limited to one discretionary consolidation
* per minute, and only when new entropy is actually coming in,
* to limit performance impact.
*
* * The entropy epoch is the number that changes when we
* transition from partial entropy to full entropy, so that
* users can easily determine when to reseed. This also
* facilitates an operator explicitly causing everything to
* reseed by sysctl -w kern.entropy.consolidate=1.
*
* * Entropy depletion is available for testing (or if you're into
* that sort of thing), with sysctl -w kern.entropy.depletion=1;
* the logic to support it is small, to minimize chance of bugs.
*
* * While cold, a single global entropy pool is available for
* entering and extracting, serialized through splhigh/splx.
* The per-CPU entropy pool data structures are initialized in
* entropy_init and entropy_init_late (separated mainly for
* hysterical raisins at this point), but are not used until the
* system is warm, at which point access to the global entropy
* pool is limited to thread and softint context and serialized
* by E->lock.
*/
/*
* struct entropy_cpu
*
* Per-CPU entropy state. The pool is allocated separately
* because percpu(9) sometimes moves per-CPU objects around
* without zeroing them, which would lead to unwanted copies of
* sensitive secrets. The evcnt is allocated separately because
* evcnt(9) assumes it stays put in memory.
*/
struct entropy_cpu {
struct entropy_cpu_evcnt {
struct evcnt softint;
struct evcnt intrdrop;
struct evcnt intrtrunc;
} *ec_evcnt;
struct entpool *ec_pool;
unsigned ec_bitspending;
unsigned ec_samplespending;
bool ec_locked;
};
/*
* struct entropy_cpu_lock
*
* State for locking the per-CPU entropy state.
*/
struct entropy_cpu_lock {
int ecl_s;
long ecl_pctr;
};
/*
* entropy_global (a.k.a. E for short in this file)
*
* Global entropy state. Writes protected by the global lock.
* Some fields, marked (A), can be read outside the lock, and are
* maintained with atomic_load/store_relaxed.
*/
struct {
kmutex_t lock; /* covers all global state */
struct entpool pool; /* global pool for extraction */
unsigned bitsneeded; /* (A) needed globally */
unsigned bitspending; /* pending in per-CPU pools */
unsigned samplesneeded; /* (A) needed globally */
unsigned samplespending; /* pending in per-CPU pools */
unsigned timestamp; /* (A) time of last consolidation */
unsigned epoch; /* (A) changes when needed -> 0 */
kcondvar_t cv; /* notifies state changes */
struct selinfo selq; /* notifies needed -> 0 */
struct lwp *sourcelock; /* lock on list of sources */
kcondvar_t sourcelock_cv; /* notifies sourcelock release */
LIST_HEAD(,krndsource) sources; /* list of entropy sources */
bool consolidate; /* kick thread to consolidate */
bool seed_rndsource; /* true if seed source is attached */
bool seeded; /* true if seed file already loaded */
} entropy_global __cacheline_aligned = {
/* Fields that must be initialized when the kernel is loaded. */
.bitsneeded = MINENTROPYBITS,
.samplesneeded = MINSAMPLES,
.epoch = (unsigned)-1, /* -1 means entropy never consolidated */
.sources = LIST_HEAD_INITIALIZER(entropy_global.sources),
};
/*
* entropy_timer()
*
* Cycle counter, time counter, or anything that changes a wee bit
* unpredictably.
*/
static inline uint32_t
entropy_timer(void)
{
struct bintime bt;
uint32_t v;
/* If we have a CPU cycle counter, use the low 32 bits. */
#ifdef __HAVE_CPU_COUNTER
if (__predict_true(cpu_hascounter()))
return cpu_counter32();
#endif /* __HAVE_CPU_COUNTER */
/* If we're cold, tough. Can't binuptime while cold. */
if (__predict_false(cold))
return 0;
/* Fold the 128 bits of binuptime into 32 bits. */
binuptime(&bt);
v = bt.frac;
v ^= bt.frac >> 32;
v ^= bt.sec;
v ^= bt.sec >> 32;
return v;
}
/*
* entropy_init()
*
* Initialize the entropy subsystem. Panic on failure.
*
* Requires percpu(9) and sysctl(9) to be initialized. Must run
* while cold.
*/
static void
entropy_init(void)
{
uint32_t extra[2];
struct krndsource *rs;
unsigned i = 0;
KASSERT(cold);
/* Grab some cycle counts early at boot. */
extra[i++] = entropy_timer();
/* Run the entropy pool cryptography self-test. */
if (entpool_selftest() == -1)
panic("entropy pool crypto self-test failed");
/* Create the sysctl knobs. */
/* XXX These shouldn't be writable at securelevel>0. */
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "collection",
SYSCTL_DESCR("Automatically collect entropy from hardware"),
NULL, 0, &entropy_collection, 0, CTL_CREATE, CTL_EOL);
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_BOOL, "depletion",
SYSCTL_DESCR("`Deplete' entropy pool when observed"),
NULL, 0, &entropy_depletion, 0, CTL_CREATE, CTL_EOL);
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "consolidate",
SYSCTL_DESCR("Trigger entropy consolidation now"),
sysctl_entropy_consolidate, 0, NULL, 0, CTL_CREATE, CTL_EOL);
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_INT, "gather",
SYSCTL_DESCR("Trigger entropy gathering from sources now"),
sysctl_entropy_gather, 0, NULL, 0, CTL_CREATE, CTL_EOL);
/* XXX These should maybe not be readable at securelevel>0. */
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
"needed",
SYSCTL_DESCR("Systemwide entropy deficit (bits of entropy)"),
NULL, 0, &E->bitsneeded, 0, CTL_CREATE, CTL_EOL);
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
"pending",
SYSCTL_DESCR("Number of bits of entropy pending on CPUs"),
NULL, 0, &E->bitspending, 0, CTL_CREATE, CTL_EOL);
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
"samplesneeded",
SYSCTL_DESCR("Systemwide entropy deficit (samples)"),
NULL, 0, &E->samplesneeded, 0, CTL_CREATE, CTL_EOL);
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READONLY|CTLFLAG_PRIVATE, CTLTYPE_INT,
"samplespending",
SYSCTL_DESCR("Number of samples pending on CPUs"),
NULL, 0, &E->samplespending, 0, CTL_CREATE, CTL_EOL);
sysctl_createv(&entropy_sysctllog, 0, &entropy_sysctlroot, NULL,
CTLFLAG_PERMANENT|CTLFLAG_READONLY, CTLTYPE_INT,
"epoch", SYSCTL_DESCR("Entropy epoch"),
NULL, 0, &E->epoch, 0, KERN_ENTROPY_EPOCH, CTL_EOL);
/* Initialize the global state for multithreaded operation. */
mutex_init(&E->lock, MUTEX_DEFAULT, IPL_SOFTSERIAL);
cv_init(&E->cv, "entropy");
selinit(&E->selq);
cv_init(&E->sourcelock_cv, "entsrclock");
/* Make sure the seed source is attached. */
attach_seed_rndsource();
/* Note if the bootloader didn't provide a seed. */
if (!E->seeded)
aprint_debug("entropy: no seed from bootloader\n");
/* Allocate the per-CPU records for all early entropy sources. */
LIST_FOREACH(rs, &E->sources, list)
rs->state = percpu_alloc(sizeof(struct rndsource_cpu));
/* Allocate and initialize the per-CPU state. */
entropy_percpu = percpu_create(sizeof(struct entropy_cpu),
entropy_init_cpu, entropy_fini_cpu, NULL);
/* Enter the boot cycle count to get started. */
extra[i++] = entropy_timer();
KASSERT(i == __arraycount(extra));
entropy_enter(extra, sizeof extra, /*nbits*/0, /*count*/false);
explicit_memset(extra, 0, sizeof extra);
}
/*
* entropy_init_late()
*
* Late initialization. Panic on failure.
*
* Requires CPUs to have been detected and LWPs to have started.
* Must run while cold.
*/
static void
entropy_init_late(void)
{
int error;
KASSERT(cold);
/*
* Establish the softint at the highest softint priority level.
* Must happen after CPU detection.
*/
entropy_sih = softint_establish(SOFTINT_SERIAL|SOFTINT_MPSAFE,
&entropy_softintr, NULL);
if (entropy_sih == NULL)
panic("unable to establish entropy softint");
/*
* Create the entropy housekeeping thread. Must happen after
* lwpinit.
*/
error = kthread_create(PRI_NONE, KTHREAD_MPSAFE|KTHREAD_TS, NULL,
entropy_thread, NULL, &entropy_lwp, "entbutler");
if (error)
panic("unable to create entropy housekeeping thread: %d",
error);
}
/*
* Zero any lingering data. Disclosure of the per-CPU pool
* shouldn't retroactively affect the security of any keys
* generated, because entpool(9) erases whatever we have just
* drawn out of any pool, but better safe than sorry.
*/
explicit_memset(ec->ec_pool, 0, sizeof(*ec->ec_pool));
/*
* ec = entropy_cpu_get(&lock)
* entropy_cpu_put(&lock, ec)
*
* Lock and unlock the per-CPU entropy state. This only prevents
* access on the same CPU -- by hard interrupts, by soft
* interrupts, or by other threads.
*
* Blocks soft interrupts and preemption altogether; doesn't block
* hard interrupts, but causes samples in hard interrupts to be
* dropped.
*/
static struct entropy_cpu *
entropy_cpu_get(struct entropy_cpu_lock *lock)
{
struct entropy_cpu *ec;
/*
* entropy_seed(seed)
*
* Seed the entropy pool with seed. Meant to be called as early
* as possible by the bootloader; may be called before or after
* entropy_init. Must be called before system reaches userland.
* Must be called in thread or soft interrupt context, not in hard
* interrupt context. Must be called at most once.
*
* Overwrites the seed in place. Caller may then free the memory.
*/
static void
entropy_seed(rndsave_t *seed)
{
SHA1_CTX ctx;
uint8_t digest[SHA1_DIGEST_LENGTH];
bool seeded;
/*
* Verify the checksum. If the checksum fails, take the data
* but ignore the entropy estimate -- the file may have been
* incompletely written with garbage, which is harmless to add
* but may not be as unpredictable as alleged.
*/
SHA1Init(&ctx);
SHA1Update(&ctx, (const void *)&seed->entropy, sizeof(seed->entropy));
SHA1Update(&ctx, seed->data, sizeof(seed->data));
SHA1Final(digest, &ctx);
CTASSERT(sizeof(seed->digest) == sizeof(digest));
if (!consttime_memequal(digest, seed->digest, sizeof(digest))) {
printf("entropy: invalid seed checksum\n");
seed->entropy = 0;
}
explicit_memset(&ctx, 0, sizeof ctx);
explicit_memset(digest, 0, sizeof digest);
/*
* If the entropy is insensibly large, try byte-swapping.
* Otherwise assume the file is corrupted and act as though it
* has zero entropy.
*/
if (howmany(seed->entropy, NBBY) > sizeof(seed->data)) {
seed->entropy = bswap32(seed->entropy);
if (howmany(seed->entropy, NBBY) > sizeof(seed->data))
seed->entropy = 0;
}
/* Make sure the seed source is attached. */
attach_seed_rndsource();
/* Test and set E->seeded. */
seeded = E->seeded;
E->seeded = (seed->entropy > 0);
/*
* If we've been seeded, may be re-entering the same seed
* (e.g., bootloader vs module init, or something). No harm in
* entering it twice, but it contributes no additional entropy.
*/
if (seeded) {
printf("entropy: double-seeded by bootloader\n");
seed->entropy = 0;
} else {
printf("entropy: entering seed from bootloader"
" with %u bits of entropy\n", (unsigned)seed->entropy);
}
/* Enter it into the pool and promptly zero it. */
rnd_add_data(&seed_rndsource, seed->data, sizeof(seed->data),
seed->entropy);
explicit_memset(seed, 0, sizeof(*seed));
}
/*
* entropy_bootrequest()
*
* Request entropy from all sources at boot, once config is
* complete and interrupts are running but we are still cold.
*/
void
entropy_bootrequest(void)
{
int error;
/*
* Request enough to satisfy the maximum entropy shortage.
* This is harmless overkill if the bootloader provided a seed.
*/
error = entropy_request(MINENTROPYBYTES, ENTROPY_WAIT);
KASSERTMSG(error == 0, "error=%d", error);
}
/*
* entropy_epoch()
*
* Returns the current entropy epoch. If this changes, you should
* reseed. If -1, means system entropy has not yet reached full
* entropy or been explicitly consolidated; never reverts back to
* -1. Never zero, so you can always use zero as an uninitialized
* sentinel value meaning `reseed ASAP'.
*
* Usage model:
*
* struct foo {
* struct crypto_prng prng;
* unsigned epoch;
* } *foo;
*
* unsigned epoch = entropy_epoch();
* if (__predict_false(epoch != foo->epoch)) {
* uint8_t seed[32];
* if (entropy_extract(seed, sizeof seed, 0) != 0)
* warn("no entropy");
* crypto_prng_reseed(&foo->prng, seed, sizeof seed);
* foo->epoch = epoch;
* }
*/
unsigned
entropy_epoch(void)
{
/*
* Unsigned int, so no need for seqlock for an atomic read, but
* make sure we read it afresh each time.
*/
return atomic_load_relaxed(&E->epoch);
}
/*
* entropy_ready()
*
* True if the entropy pool has full entropy.
*/
bool
entropy_ready(void)
{
/*
* entropy_account_cpu(ec)
*
* Consider whether to consolidate entropy into the global pool
* after we just added some into the current CPU's pending pool.
*
* - If this CPU can provide enough entropy now, do so.
*
* - If this and whatever else is available on other CPUs can
* provide enough entropy, kick the consolidation thread.
*
* - Otherwise, do as little as possible, except maybe consolidate
* entropy at most once a minute.
*
* Caller must be bound to a CPU and therefore have exclusive
* access to ec. Will acquire and release the global lock.
*/
static void
entropy_account_cpu(struct entropy_cpu *ec)
{
struct entropy_cpu_lock lock;
struct entropy_cpu *ec0;
unsigned bitsdiff, samplesdiff;
/*
* If there's no entropy needed, and entropy has been
* consolidated in the last minute, do nothing.
*/
if (__predict_true(atomic_load_relaxed(&E->bitsneeded) == 0) &&
__predict_true(!atomic_load_relaxed(&entropy_depletion)) &&
__predict_true((time_uptime - E->timestamp) <= 60))
return;
/*
* Consider consolidation, under the global lock and with the
* per-CPU state locked.
*/
mutex_enter(&E->lock);
ec0 = entropy_cpu_get(&lock);
KASSERT(ec0 == ec);
if (ec->ec_bitspending == 0 && ec->ec_samplespending == 0) {
/* Raced with consolidation xcall. Nothing to do. */
} else if (E->bitsneeded != 0 && E->bitsneeded <= ec->ec_bitspending) {
/*
* If we have not yet attained full entropy but we can
* now, do so. This way we disseminate entropy
* promptly when it becomes available early at boot;
* otherwise we leave it to the entropy consolidation
* thread, which is rate-limited to mitigate side
* channels and abuse.
*/
uint8_t buf[ENTPOOL_CAPACITY];
/* Transfer from the local pool to the global pool. */
entpool_extract(ec->ec_pool, buf, sizeof buf);
entpool_enter(&E->pool, buf, sizeof buf);
atomic_store_relaxed(&ec->ec_bitspending, 0);
atomic_store_relaxed(&ec->ec_samplespending, 0);
atomic_store_relaxed(&E->bitsneeded, 0);
atomic_store_relaxed(&E->samplesneeded, 0);
/* Notify waiters that we now have full entropy. */
entropy_notify();
entropy_immediate_evcnt.ev_count++;
} else {
/* Determine how much we can add to the global pool. */
KASSERTMSG(E->bitspending <= MINENTROPYBITS,
"E->bitspending=%u", E->bitspending);
bitsdiff = MIN(ec->ec_bitspending,
MINENTROPYBITS - E->bitspending);
KASSERTMSG(E->samplespending <= MINSAMPLES,
"E->samplespending=%u", E->samplespending);
samplesdiff = MIN(ec->ec_samplespending,
MINSAMPLES - E->samplespending);
/*
* This should make a difference unless we are already
* saturated.
*/
KASSERTMSG((bitsdiff || samplesdiff ||
E->bitspending == MINENTROPYBITS ||
E->samplespending == MINSAMPLES),
"bitsdiff=%u E->bitspending=%u ec->ec_bitspending=%u"
"samplesdiff=%u E->samplespending=%u"
" ec->ec_samplespending=%u"
" minentropybits=%u minsamples=%u",
bitsdiff, E->bitspending, ec->ec_bitspending,
samplesdiff, E->samplespending, ec->ec_samplespending,
(unsigned)MINENTROPYBITS, (unsigned)MINSAMPLES);
/* Add to the global, subtract from the local. */
E->bitspending += bitsdiff;
KASSERTMSG(E->bitspending <= MINENTROPYBITS,
"E->bitspending=%u", E->bitspending);
atomic_store_relaxed(&ec->ec_bitspending,
ec->ec_bitspending - bitsdiff);
/* One or the other must have gone up from zero. */
KASSERT(E->bitspending || E->samplespending);
if (E->bitsneeded <= E->bitspending ||
E->samplesneeded <= E->samplespending) {
/*
* Enough bits or at least samples between all
* the per-CPU pools. Leave a note for the
* housekeeping thread to consolidate entropy
* next time it wakes up -- and wake it up if
* this is the first time, to speed things up.
*
* If we don't need any entropy, this doesn't
* mean much, but it is the only time we ever
* gather additional entropy in case the
* accounting has been overly optimistic. This
* happens at most once a minute, so there's
* negligible performance cost.
*/
E->consolidate = true;
if (E->epoch == (unsigned)-1)
cv_broadcast(&E->cv);
if (E->bitsneeded == 0)
entropy_discretionary_evcnt.ev_count++;
} else {
/* Can't get full entropy. Keep gathering. */
entropy_partial_evcnt.ev_count++;
}
}
/*
* entropy_enter_early(buf, len, nbits)
*
* Do entropy bookkeeping globally, before we have established
* per-CPU pools. Enter directly into the global pool in the hope
* that we enter enough before the first entropy_extract to thwart
* iterative-guessing attacks; entropy_extract will warn if not.
*/
static void
entropy_enter_early(const void *buf, size_t len, unsigned nbits)
{
bool notify = false;
int s;
KASSERT(cold);
/*
* We're early at boot before multithreading and multi-CPU
* operation, and we don't have softints yet to defer
* processing from interrupt context, so we have to enter the
* samples directly into the global pool. But interrupts may
* be enabled, and we enter this path from interrupt context,
* so block interrupts until we're done.
*/
s = splhigh();
/* Enter it into the pool. */
entpool_enter(&E->pool, buf, len);
/*
* Decide whether to notify reseed -- we will do so if either:
* (a) we transition from partial entropy to full entropy, or
* (b) we get a batch of full entropy all at once.
* We don't count timing samples because we assume, while cold,
* there's not likely to be much jitter yet.
*/
notify |= (E->bitsneeded && E->bitsneeded <= nbits);
notify |= (nbits >= MINENTROPYBITS);
/*
* Subtract from the needed count and notify if appropriate.
* We don't count samples here because entropy_timer might
* still be returning zero at this point if there's no CPU
* cycle counter.
*/
E->bitsneeded -= MIN(E->bitsneeded, nbits);
if (notify) {
entropy_notify();
entropy_immediate_evcnt.ev_count++;
}
splx(s);
}
/*
* entropy_enter(buf, len, nbits, count)
*
* Enter len bytes of data from buf into the system's entropy
* pool, stirring as necessary when the internal buffer fills up.
* nbits is a lower bound on the number of bits of entropy in the
* process that led to this sample.
*/
static void
entropy_enter(const void *buf, size_t len, unsigned nbits, bool count)
{
struct entropy_cpu_lock lock;
struct entropy_cpu *ec;
unsigned bitspending, samplespending;
int bound;
KASSERTMSG(!cpu_intr_p(),
"use entropy_enter_intr from interrupt context");
KASSERTMSG(howmany(nbits, NBBY) <= len,
"impossible entropy rate: %u bits in %zu-byte string", nbits, len);
/*
* If we're still cold, just use entropy_enter_early to put
* samples directly into the global pool.
*/
if (__predict_false(cold)) {
entropy_enter_early(buf, len, nbits);
return;
}
/*
* Bind ourselves to the current CPU so we don't switch CPUs
* between entering data into the current CPU's pool (and
* updating the pending count) and transferring it to the
* global pool in entropy_account_cpu.
*/
bound = curlwp_bind();
/*
* With the per-CPU state locked, enter into the per-CPU pool
* and count up what we can add.
*
* We don't count samples while cold because entropy_timer
* might still be returning zero if there's no CPU cycle
* counter.
*/
ec = entropy_cpu_get(&lock);
entpool_enter(ec->ec_pool, buf, len);
bitspending = ec->ec_bitspending;
bitspending += MIN(MINENTROPYBITS - bitspending, nbits);
atomic_store_relaxed(&ec->ec_bitspending, bitspending);
samplespending = ec->ec_samplespending;
if (__predict_true(count)) {
samplespending += MIN(MINSAMPLES - samplespending, 1);
atomic_store_relaxed(&ec->ec_samplespending, samplespending);
}
entropy_cpu_put(&lock, ec);
/* Consolidate globally if appropriate based on what we added. */
if (bitspending > 0 || samplespending >= MINSAMPLES)
entropy_account_cpu(ec);
curlwp_bindx(bound);
}
/*
* entropy_enter_intr(buf, len, nbits, count)
*
* Enter up to len bytes of data from buf into the system's
* entropy pool without stirring. nbits is a lower bound on the
* number of bits of entropy in the process that led to this
* sample. If the sample could be entered completely, assume
* nbits of entropy pending; otherwise assume none, since we don't
* know whether some parts of the sample are constant, for
* instance. Schedule a softint to stir the entropy pool if
* needed. Return true if used fully, false if truncated at all.
*
* Using this in thread or softint context with no spin locks held
* will work, but you might as well use entropy_enter in that
* case.
*/
static bool
entropy_enter_intr(const void *buf, size_t len, unsigned nbits, bool count)
{
struct entropy_cpu *ec;
bool fullyused = false;
uint32_t bitspending, samplespending;
int s;
/*
* If we're still cold, just use entropy_enter_early to put
* samples directly into the global pool.
*/
if (__predict_false(cold)) {
entropy_enter_early(buf, len, nbits);
return true;
}
/*
* In case we were called in thread or interrupt context with
* interrupts unblocked, block soft interrupts up to
* IPL_SOFTSERIAL. This way logic that is safe in interrupt
* context or under a spin lock is also safe in less
* restrictive contexts.
*/
s = splsoftserial();
/*
* Acquire the per-CPU state. If someone is in the middle of
* using it, drop the sample. Otherwise, take the lock so that
* higher-priority interrupts will drop their samples.
*/
ec = percpu_getref(entropy_percpu);
if (ec->ec_locked) {
ec->ec_evcnt->intrdrop.ev_count++;
goto out0;
}
ec->ec_locked = true;
__insn_barrier();
/*
* Enter as much as we can into the per-CPU pool. If it was
* truncated, schedule a softint to stir the pool and stop.
*/
if (!entpool_enter_nostir(ec->ec_pool, buf, len)) {
if (__predict_true(!cold))
softint_schedule(entropy_sih);
ec->ec_evcnt->intrtrunc.ev_count++;
goto out1;
}
fullyused = true;
/*
* Count up what we can contribute.
*
* We don't count samples while cold because entropy_timer
* might still be returning zero if there's no CPU cycle
* counter.
*/
bitspending = ec->ec_bitspending;
bitspending += MIN(MINENTROPYBITS - bitspending, nbits);
atomic_store_relaxed(&ec->ec_bitspending, bitspending);
if (__predict_true(count)) {
samplespending = ec->ec_samplespending;
samplespending += MIN(MINSAMPLES - samplespending, 1);
atomic_store_relaxed(&ec->ec_samplespending, samplespending);
}
/* Schedule a softint if we added anything and it matters. */
if (__predict_false(atomic_load_relaxed(&E->bitsneeded) ||
atomic_load_relaxed(&entropy_depletion)) &&
(nbits != 0 || count) &&
__predict_true(!cold))
softint_schedule(entropy_sih);
/*
* entropy_softintr(cookie)
*
* Soft interrupt handler for entering entropy. Takes care of
* stirring the local CPU's entropy pool if it filled up during
* hard interrupts, and promptly crediting entropy from the local
* CPU's entropy pool to the global entropy pool if needed.
*/
static void
entropy_softintr(void *cookie)
{
struct entropy_cpu_lock lock;
struct entropy_cpu *ec;
unsigned bitspending, samplespending;
/*
* With the per-CPU state locked, stir the pool if necessary
* and determine if there's any pending entropy on this CPU to
* account globally.
*/
ec = entropy_cpu_get(&lock);
ec->ec_evcnt->softint.ev_count++;
entpool_stir(ec->ec_pool);
bitspending = ec->ec_bitspending;
samplespending = ec->ec_samplespending;
entropy_cpu_put(&lock, ec);
/* Consolidate globally if appropriate based on what we added. */
if (bitspending > 0 || samplespending >= MINSAMPLES)
entropy_account_cpu(ec);
}
for (;;) {
/*
* Wait until there's full entropy somewhere among the
* CPUs, as confirmed at most once per minute, or
* someone wants to consolidate.
*/
if (entropy_pending()) {
consolidate = true;
} else {
mutex_enter(&E->lock);
if (!E->consolidate)
cv_timedwait(&E->cv, &E->lock, 60*hz);
consolidate = E->consolidate;
E->consolidate = false;
mutex_exit(&E->lock);
}
if (consolidate) {
/* Do it. */
entropy_do_consolidate();
/*
* entropy_do_consolidate()
*
* Issue a cross-call to gather entropy on all CPUs and advance
* the entropy epoch.
*/
static void
entropy_do_consolidate(void)
{
static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
static struct timeval lasttime; /* serialized by E->lock */
struct entpool pool;
uint8_t buf[ENTPOOL_CAPACITY];
unsigned bitsdiff, samplesdiff;
uint64_t ticket;
KASSERT(!cold);
ASSERT_SLEEPABLE();
/* Gather entropy on all CPUs into a temporary pool. */
memset(&pool, 0, sizeof pool);
ticket = xc_broadcast(0, &entropy_consolidate_xc, &pool, NULL);
xc_wait(ticket);
/* Acquire the lock to notify waiters. */
mutex_enter(&E->lock);
/* Count another consolidation. */
entropy_consolidate_evcnt.ev_count++;
/* Note when we last consolidated, i.e. now. */
E->timestamp = time_uptime;
/* Mix what we gathered into the global pool. */
entpool_extract(&pool, buf, sizeof buf);
entpool_enter(&E->pool, buf, sizeof buf);
explicit_memset(&pool, 0, sizeof pool);
/* Count the entropy that was gathered. */
bitsdiff = MIN(E->bitsneeded, E->bitspending);
atomic_store_relaxed(&E->bitsneeded, E->bitsneeded - bitsdiff);
E->bitspending -= bitsdiff;
if (__predict_false(E->bitsneeded > 0) && bitsdiff != 0) {
if ((boothowto & AB_DEBUG) != 0 &&
ratecheck(&lasttime, &interval)) {
printf("WARNING:"
" consolidating less than full entropy\n");
}
}
/* Advance the epoch and notify waiters. */
entropy_notify();
/* Release the lock. */
mutex_exit(&E->lock);
}
/*
* entropy_consolidate_xc(vpool, arg2)
*
* Extract output from the local CPU's input pool and enter it
* into a temporary pool passed as vpool.
*/
static void
entropy_consolidate_xc(void *vpool, void *arg2 __unused)
{
struct entpool *pool = vpool;
struct entropy_cpu_lock lock;
struct entropy_cpu *ec;
uint8_t buf[ENTPOOL_CAPACITY];
uint32_t extra[7];
unsigned i = 0;
/* Grab CPU number and cycle counter to mix extra into the pool. */
extra[i++] = cpu_number();
extra[i++] = entropy_timer();
/*
* With the per-CPU state locked, extract from the per-CPU pool
* and count it as no longer pending.
*/
ec = entropy_cpu_get(&lock);
extra[i++] = entropy_timer();
entpool_extract(ec->ec_pool, buf, sizeof buf);
atomic_store_relaxed(&ec->ec_bitspending, 0);
atomic_store_relaxed(&ec->ec_samplespending, 0);
extra[i++] = entropy_timer();
entropy_cpu_put(&lock, ec);
extra[i++] = entropy_timer();
/*
* Copy over statistics, and enter the per-CPU extract and the
* extra timing into the temporary pool, under the global lock.
*/
mutex_enter(&E->lock);
extra[i++] = entropy_timer();
entpool_enter(pool, buf, sizeof buf);
explicit_memset(buf, 0, sizeof buf);
extra[i++] = entropy_timer();
KASSERT(i == __arraycount(extra));
entpool_enter(pool, extra, sizeof extra);
explicit_memset(extra, 0, sizeof extra);
mutex_exit(&E->lock);
}
/*
* entropy_notify()
*
* Caller just contributed entropy to the global pool. Advance
* the entropy epoch and notify waiters.
*
* Caller must hold the global entropy lock.
*/
static void
entropy_notify(void)
{
static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
static struct timeval lasttime; /* serialized by E->lock */
static bool ready = false, besteffort = false;
unsigned epoch;
/*
* If this is the first time, print a message to the console
* that we're ready so operators can compare it to the timing
* of other events.
*
* If we didn't get full entropy from reliable sources, report
* instead that we are running on fumes with best effort. (If
* we ever do get full entropy after that, print the ready
* message once.)
*/
if (__predict_false(!ready)) {
if (E->bitsneeded == 0) {
printf("entropy: ready\n");
ready = true;
} else if (E->samplesneeded == 0 && !besteffort) {
printf("entropy: best effort\n");
besteffort = true;
}
}
/* Set the epoch; roll over from UINTMAX-1 to 1. */
if (__predict_true(!atomic_load_relaxed(&entropy_depletion)) ||
ratecheck(&lasttime, &interval)) {
epoch = E->epoch + 1;
if (epoch == 0 || epoch == (unsigned)-1)
epoch = 1;
atomic_store_relaxed(&E->epoch, epoch);
}
KASSERT(E->epoch != (unsigned)-1);
/* Count another notification. */
entropy_notify_evcnt.ev_count++;
}
/*
* entropy_consolidate()
*
* Trigger entropy consolidation and wait for it to complete, or
* return EINTR if interrupted by a signal.
*
* This should be used sparingly, not periodically -- requiring
* conscious intervention by the operator or a clear policy
* decision. Otherwise, the kernel will automatically consolidate
* when enough entropy has been gathered into per-CPU pools to
* transition to full entropy.
*/
int
entropy_consolidate(void)
{
uint64_t ticket;
int error;
/*
* sysctl -w kern.entropy.consolidate=1
*
* Trigger entropy consolidation and wait for it to complete.
* Writable only by superuser. This, writing to /dev/random, and
* ioctl(RNDADDDATA) are the only ways for the system to
* consolidate entropy if the operator knows something the kernel
* doesn't about how unpredictable the pending entropy pools are.
*/
static int
sysctl_entropy_consolidate(SYSCTLFN_ARGS)
{
struct sysctlnode node = *rnode;
int arg = 0;
int error;
node.sysctl_data = &arg;
error = sysctl_lookup(SYSCTLFN_CALL(&node));
if (error || newp == NULL)
return error;
if (arg)
error = entropy_consolidate();
return error;
}
/*
* entropy_gather()
*
* Trigger gathering entropy from all on-demand sources, and, if
* requested, wait for synchronous sources (but not asynchronous
* sources) to complete, or fail with EINTR if interrupted by a
* signal.
*/
int
entropy_gather(void)
{
int error;
/*
* sysctl -w kern.entropy.gather=1
*
* Trigger gathering entropy from all on-demand sources, and wait
* for synchronous sources (but not asynchronous sources) to
* complete. Writable only by superuser.
*/
static int
sysctl_entropy_gather(SYSCTLFN_ARGS)
{
struct sysctlnode node = *rnode;
int arg = 0;
int error;
node.sysctl_data = &arg;
error = sysctl_lookup(SYSCTLFN_CALL(&node));
if (error || newp == NULL)
return error;
if (arg)
error = entropy_gather();
return error;
}
/*
* entropy_extract(buf, len, flags)
*
* Extract len bytes from the global entropy pool into buf.
*
* Caller MUST NOT expose these bytes directly -- must use them
* ONLY to seed a cryptographic pseudorandom number generator
* (`CPRNG'), a.k.a. deterministic random bit generator (`DRBG'),
* and then erase them. entropy_extract does not, on its own,
* provide backtracking resistance -- it must be combined with a
* PRNG/DRBG that does.
*
* This may be used very early at boot, before even entropy_init
* has been called.
*
* You generally shouldn't use this directly -- use cprng(9)
* instead.
*
* Flags may have:
*
* ENTROPY_WAIT Wait for entropy if not available yet.
* ENTROPY_SIG Allow interruption by a signal during wait.
* ENTROPY_HARDFAIL Either fill the buffer with full entropy,
* or fail without filling it at all.
*
* Return zero on success, or error on failure:
*
* EWOULDBLOCK No entropy and ENTROPY_WAIT not set.
* EINTR/ERESTART No entropy, ENTROPY_SIG set, and interrupted.
*
* If ENTROPY_WAIT is set, allowed only in thread context. If
* ENTROPY_WAIT is not set, allowed also in softint context -- may
* sleep on an adaptive lock up to IPL_SOFTSERIAL. Forbidden in
* hard interrupt context.
*/
int
entropy_extract(void *buf, size_t len, int flags)
{
static const struct timeval interval = {.tv_sec = 60, .tv_usec = 0};
static struct timeval lasttime; /* serialized by E->lock */
bool printed = false;
int s = -1/*XXXGCC*/, error;
if (ISSET(flags, ENTROPY_WAIT)) {
ASSERT_SLEEPABLE();
KASSERT(!cold);
}
/* Refuse to operate in interrupt context. */
KASSERT(!cpu_intr_p());
/*
* If we're cold, we are only contending with interrupts on the
* current CPU, so block them. Otherwise, we are _not_
* contending with interrupts on the current CPU, but we are
* contending with other threads, to exclude them with a mutex.
*/
if (__predict_false(cold))
s = splhigh();
else
mutex_enter(&E->lock);
/* Wait until there is enough entropy in the system. */
error = 0;
if (E->bitsneeded > 0 && E->samplesneeded == 0) {
/*
* We don't have full entropy from reliable sources,
* but we gathered a plausible number of samples from
* other sources such as timers. Try asking for more
* from any sources we can, but don't worry if it
* fails -- best effort.
*/
(void)entropy_request(ENTROPY_CAPACITY, flags);
} else while (E->bitsneeded > 0 && E->samplesneeded > 0) {
/* Ask for more, synchronously if possible. */
error = entropy_request(len, flags);
if (error)
break;
/* If we got enough, we're done. */
if (E->bitsneeded == 0 || E->samplesneeded == 0) {
KASSERT(error == 0);
break;
}
/* If not waiting, stop here. */
if (!ISSET(flags, ENTROPY_WAIT)) {
error = EWOULDBLOCK;
break;
}
/* Wait for some entropy to come in and try again. */
KASSERT(!cold);
if (!printed) {
printf("entropy: pid %d (%s) waiting for entropy(7)\n",
curproc->p_pid, curproc->p_comm);
printed = true;
}
/*
* Count failure -- but fill the buffer nevertheless, unless
* the caller specified ENTROPY_HARDFAIL.
*/
if (error) {
if (ISSET(flags, ENTROPY_HARDFAIL))
goto out;
entropy_extract_fail_evcnt.ev_count++;
}
/*
* Report a warning if we haven't yet reached full entropy.
* This is the only case where we consider entropy to be
* `depleted' without kern.entropy.depletion enabled -- when we
* only have partial entropy, an adversary may be able to
* narrow the state of the pool down to a small number of
* possibilities; the output then enables them to confirm a
* guess, reducing its entropy from the adversary's perspective
* to zero.
*
* This should only happen if the operator has chosen to
* consolidate, either through sysctl kern.entropy.consolidate
* or by writing less than full entropy to /dev/random as root
* (which /dev/random promises will immediately affect
* subsequent output, for better or worse).
*/
if (E->bitsneeded > 0 && E->samplesneeded > 0) {
if (__predict_false(E->epoch == (unsigned)-1) &&
ratecheck(&lasttime, &interval)) {
printf("WARNING:"
" system needs entropy for security;"
" see entropy(7)\n");
}
atomic_store_relaxed(&E->bitsneeded, MINENTROPYBITS);
atomic_store_relaxed(&E->samplesneeded, MINSAMPLES);
}
/* Extract data from the pool, and `deplete' if we're doing that. */
entpool_extract(&E->pool, buf, len);
if (__predict_false(atomic_load_relaxed(&entropy_depletion)) &&
error == 0) {
unsigned cost = MIN(len, ENTROPY_CAPACITY)*NBBY;
unsigned bitsneeded = E->bitsneeded;
unsigned samplesneeded = E->samplesneeded;
out: /* Release the global lock and return the error. */
if (__predict_false(cold))
splx(s);
else
mutex_exit(&E->lock);
return error;
}
/*
* entropy_poll(events)
*
* Return the subset of events ready, and if it is not all of
* events, record curlwp as waiting for entropy.
*/
int
entropy_poll(int events)
{
int revents = 0;
/* Narrow it down to reads. */
events &= POLLIN|POLLRDNORM;
if (events == 0)
return revents;
/*
* If we have reached full entropy and we're not depleting
* entropy, we are forever ready.
*/
if (__predict_true(atomic_load_relaxed(&E->bitsneeded) == 0 ||
atomic_load_relaxed(&E->samplesneeded) == 0) &&
__predict_true(!atomic_load_relaxed(&entropy_depletion)))
return revents | events;
/*
* Otherwise, check whether we need entropy under the lock. If
* we don't, we're ready; if we do, add ourselves to the queue.
*/
mutex_enter(&E->lock);
if (E->bitsneeded == 0 || E->samplesneeded == 0)
revents |= events;
else
selrecord(curlwp, &E->selq);
mutex_exit(&E->lock);
return revents;
}
/*
* filt_entropy_read_detach(kn)
*
* struct filterops::f_detach callback for entropy read events:
* remove kn from the list of waiters.
*/
static void
filt_entropy_read_detach(struct knote *kn)
{
/*
* filt_entropy_read_event(kn, hint)
*
* struct filterops::f_event callback for entropy read events:
* poll for entropy. Caller must hold the global entropy lock if
* hint is NOTE_SUBMIT, and must not if hint is not NOTE_SUBMIT.
*/
static int
filt_entropy_read_event(struct knote *kn, long hint)
{
int ret;
KASSERT(!cold);
/* Acquire the lock, if caller is outside entropy subsystem. */
if (hint == NOTE_SUBMIT)
KASSERT(mutex_owned(&E->lock));
else
mutex_enter(&E->lock);
/*
* If we still need entropy, can't read anything; if not, can
* read arbitrarily much.
*/
if (E->bitsneeded != 0 && E->samplesneeded != 0) {
ret = 0;
} else {
if (atomic_load_relaxed(&entropy_depletion))
kn->kn_data = ENTROPY_CAPACITY; /* bytes */
else
kn->kn_data = MIN(INT64_MAX, SSIZE_MAX);
ret = 1;
}
/* Release the lock, if caller is outside entropy subsystem. */
if (hint == NOTE_SUBMIT)
KASSERT(mutex_owned(&E->lock));
else
mutex_exit(&E->lock);
return ret;
}
/* XXX Makes sense only for /dev/u?random. */
static const struct filterops entropy_read_filtops = {
.f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
.f_attach = NULL,
.f_detach = filt_entropy_read_detach,
.f_event = filt_entropy_read_event,
};
/*
* entropy_kqfilter(kn)
*
* Register kn to receive entropy event notifications. May be
* EVFILT_READ or EVFILT_WRITE; anything else yields EINVAL.
*/
int
entropy_kqfilter(struct knote *kn)
{
KASSERT(!cold);
switch (kn->kn_filter) {
case EVFILT_READ:
/* Enter into the global select queue. */
mutex_enter(&E->lock);
kn->kn_fop = &entropy_read_filtops;
selrecord_knote(&E->selq, kn);
mutex_exit(&E->lock);
return 0;
case EVFILT_WRITE:
/* Can always dump entropy into the system. */
kn->kn_fop = &seltrue_filtops;
return 0;
default:
return EINVAL;
}
}
/*
* rndsource_setcb(rs, get, getarg)
*
* Set the request callback for the entropy source rs, if it can
* provide entropy on demand. Must precede rnd_attach_source.
*/
void
rndsource_setcb(struct krndsource *rs, void (*get)(size_t, void *),
void *getarg)
{
rs->get = get;
rs->getarg = getarg;
}
/*
* rnd_attach_source(rs, name, type, flags)
*
* Attach the entropy source rs. Must be done after
* rndsource_setcb, if any, and before any calls to rnd_add_data.
*/
void
rnd_attach_source(struct krndsource *rs, const char *name, uint32_t type,
uint32_t flags)
{
uint32_t extra[4];
unsigned i = 0;
KASSERTMSG(name[0] != '\0', "rndsource must have nonempty name");
/* Grab cycle counter to mix extra into the pool. */
extra[i++] = entropy_timer();
/*
* Apply some standard flags:
*
* - We do not bother with network devices by default, for
* hysterical raisins (perhaps: because it is often the case
* that an adversary can influence network packet timings).
*/
switch (type) {
case RND_TYPE_NET:
flags |= RND_FLAG_NO_COLLECT;
break;
}
/* Sanity-check the callback if RND_FLAG_HASCB is set. */
KASSERT(!ISSET(flags, RND_FLAG_HASCB) || rs->get != NULL);
/* Wire it into the global list of random sources. */
if (__predict_true(!cold))
mutex_enter(&E->lock);
LIST_INSERT_HEAD(&E->sources, rs, list);
if (__predict_true(!cold))
mutex_exit(&E->lock);
extra[i++] = entropy_timer();
/* Request that it provide entropy ASAP, if we can. */
if (ISSET(flags, RND_FLAG_HASCB))
(*rs->get)(ENTROPY_CAPACITY, rs->getarg);
extra[i++] = entropy_timer();
/* Mix the extra into the pool. */
KASSERT(i == __arraycount(extra));
entropy_enter(extra, sizeof extra, 0, /*count*/__predict_true(!cold));
explicit_memset(extra, 0, sizeof extra);
}
/*
* rnd_detach_source(rs)
*
* Detach the entropy source rs. May sleep waiting for users to
* drain. Further use is not allowed.
*/
void
rnd_detach_source(struct krndsource *rs)
{
/*
* If we're cold (shouldn't happen, but hey), just remove it
* from the list -- there's nothing allocated.
*/
if (__predict_false(cold) && entropy_percpu == NULL) {
LIST_REMOVE(rs, list);
return;
}
/* We may have to wait for entropy_request. */
ASSERT_SLEEPABLE();
/* Wait until the source list is not in use, and remove it. */
mutex_enter(&E->lock);
while (E->sourcelock)
cv_wait(&E->sourcelock_cv, &E->lock);
LIST_REMOVE(rs, list);
mutex_exit(&E->lock);
/* Free the per-CPU data. */
percpu_free(rs->state, sizeof(struct rndsource_cpu));
}
/*
* rnd_lock_sources(flags)
*
* Lock the list of entropy sources. Caller must hold the global
* entropy lock. If successful, no rndsource will go away until
* rnd_unlock_sources even while the caller releases the global
* entropy lock.
*
* May be called very early at boot, before entropy_init.
*
* If flags & ENTROPY_WAIT, wait for concurrent access to finish.
* If flags & ENTROPY_SIG, allow interruption by signal.
*/
static int __attribute__((warn_unused_result))
rnd_lock_sources(int flags)
{
int error;
while (E->sourcelock) {
KASSERT(!cold);
if (!ISSET(flags, ENTROPY_WAIT))
return EWOULDBLOCK;
if (ISSET(flags, ENTROPY_SIG)) {
error = cv_wait_sig(&E->sourcelock_cv, &E->lock);
if (error)
return error;
} else {
cv_wait(&E->sourcelock_cv, &E->lock);
}
}
E->sourcelock = curlwp;
return 0;
}
/*
* rnd_unlock_sources()
*
* Unlock the list of sources after rnd_lock_sources. Caller must
* hold the global entropy lock.
*
* May be called very early at boot, before entropy_init.
*/
static void
rnd_unlock_sources(void)
{
KASSERTMSG(E->sourcelock == curlwp, "lwp %p releasing lock held by %p",
curlwp, E->sourcelock);
E->sourcelock = NULL;
if (__predict_true(!cold))
cv_signal(&E->sourcelock_cv);
}
/*
* rnd_sources_locked()
*
* True if we hold the list of rndsources locked, for diagnostic
* assertions.
*
* May be called very early at boot, before entropy_init.
*/
static bool __diagused
rnd_sources_locked(void)
{
return E->sourcelock == curlwp;
}
/*
* entropy_request(nbytes, flags)
*
* Request nbytes bytes of entropy from all sources in the system.
* OK if we overdo it. Caller must hold the global entropy lock;
* will release and re-acquire it.
*
* May be called very early at boot, before entropy_init.
*
* If flags & ENTROPY_WAIT, wait for concurrent access to finish.
* If flags & ENTROPY_SIG, allow interruption by signal.
*/
static int
entropy_request(size_t nbytes, int flags)
{
struct krndsource *rs;
int error;
/*
* Lock the list of entropy sources to block rnd_detach_source
* until we're done, and to serialize calls to the entropy
* callbacks as guaranteed to drivers.
*/
error = rnd_lock_sources(flags);
if (error)
return error;
entropy_request_evcnt.ev_count++;
/* Clamp to the maximum reasonable request. */
nbytes = MIN(nbytes, ENTROPY_CAPACITY);
/* Walk the list of sources. */
LIST_FOREACH(rs, &E->sources, list) {
/* Skip sources without callbacks. */
if (!ISSET(rs->flags, RND_FLAG_HASCB))
continue;
/*
* Skip sources that are disabled altogether -- we
* would just ignore their samples anyway.
*/
if (ISSET(rs->flags, RND_FLAG_NO_COLLECT))
continue;
/* Drop the lock while we call the callback. */
if (__predict_true(!cold))
mutex_exit(&E->lock);
(*rs->get)(nbytes, rs->getarg);
if (__predict_true(!cold))
mutex_enter(&E->lock);
}
/* Request done; unlock the list of entropy sources. */
rnd_unlock_sources();
return 0;
}
/*
* rnd_add_uint32(rs, value)
*
* Enter 32 bits of data from an entropy source into the pool.
*
* May be called from any context or with spin locks held, but may
* drop data.
*
* This is meant for cheaply taking samples from devices that
* aren't designed to be hardware random number generators.
*/
void
rnd_add_uint32(struct krndsource *rs, uint32_t value)
{
bool intr_p = true;
/*
* rnd_add_data(rs, buf, len, entropybits)
*
* Enter data from an entropy source into the pool, with a
* driver's estimate of how much entropy the physical source of
* the data has. If RND_FLAG_NO_ESTIMATE, we ignore the driver's
* estimate and treat it as zero.
*
* rs MAY but SHOULD NOT be NULL. If rs is NULL, MUST NOT be
* called from interrupt context or with spin locks held.
*
* If rs is non-NULL, MAY but SHOULD NOT be called from interrupt
* context, in which case act like rnd_add_data_intr -- if the
* sample buffer is full, schedule a softint and drop any
* additional data on the floor. (This may change later once we
* fix drivers that still call this from interrupt context to use
* rnd_add_data_intr instead.) MUST NOT be called with spin locks
* held if not in hard interrupt context -- i.e., MUST NOT be
* called in thread context or softint context with spin locks
* held.
*/
void
rnd_add_data(struct krndsource *rs, const void *buf, uint32_t len,
uint32_t entropybits)
{
bool intr_p = cpu_intr_p(); /* XXX make this unconditionally false */
/*
* Weird legacy exception that we should rip out and replace by
* creating new rndsources to attribute entropy to the callers:
* If there's no rndsource, just enter the data and time now.
*/
if (rs == NULL) {
uint32_t extra;
/*
* rnd_add_data_intr(rs, buf, len, entropybits)
*
* Try to enter data from an entropy source into the pool, with a
* driver's estimate of how much entropy the physical source of
* the data has. If RND_FLAG_NO_ESTIMATE, we ignore the driver's
* estimate and treat it as zero. If the sample buffer is full,
* schedule a softint and drop any additional data on the floor.
*/
void
rnd_add_data_intr(struct krndsource *rs, const void *buf, uint32_t len,
uint32_t entropybits)
{
bool intr_p = true;
/*
* rnd_add_data_internal(rs, buf, len, entropybits, intr_p)
*
* Internal subroutine to decide whether or not to enter data or
* timing for a particular rndsource, and if so, to enter it.
*
* intr_p is true for callers from interrupt context or spin locks
* held, and false for callers from thread or soft interrupt
* context and no spin locks held.
*/
static void
rnd_add_data_internal(struct krndsource *rs, const void *buf, uint32_t len,
uint32_t entropybits, bool intr_p)
{
uint32_t flags;
/*
* Hold up the reset xcall before it zeroes the entropy counts
* on this CPU or globally. Otherwise, we might leave some
* nonzero entropy attributed to an untrusted source in the
* event of a race with a change to flags.
*/
kpreempt_disable();
/* Load a snapshot of the flags. Ioctl may change them under us. */
flags = atomic_load_relaxed(&rs->flags);
/*
* Skip if:
* - we're not collecting entropy, or
* - the operator doesn't want to collect entropy from this, or
* - neither data nor timings are being collected from this.
*/
if (!atomic_load_relaxed(&entropy_collection) ||
ISSET(flags, RND_FLAG_NO_COLLECT) ||
!ISSET(flags, RND_FLAG_COLLECT_VALUE|RND_FLAG_COLLECT_TIME))
goto out;
/* If asked, ignore the estimate. */
if (ISSET(flags, RND_FLAG_NO_ESTIMATE))
entropybits = 0;
/* If we are collecting data, enter them. */
if (ISSET(flags, RND_FLAG_COLLECT_VALUE)) {
rnd_add_data_1(rs, buf, len, entropybits, /*count*/false,
RND_FLAG_COLLECT_VALUE, intr_p);
}
/* If we are collecting timings, enter one. */
if (ISSET(flags, RND_FLAG_COLLECT_TIME)) {
uint32_t extra;
bool count;
/* Sample a timer. */
extra = entropy_timer();
/* If asked, do entropy estimation on the time. */
if ((flags & (RND_FLAG_ESTIMATE_TIME|RND_FLAG_NO_ESTIMATE)) ==
RND_FLAG_ESTIMATE_TIME && __predict_true(!cold))
count = rnd_dt_estimate(rs, extra);
else
count = false;
out: /* Allow concurrent changes to flags to finish. */
kpreempt_enable();
}
static unsigned
add_sat(unsigned a, unsigned b)
{
unsigned c = a + b;
return (c < a ? UINT_MAX : c);
}
/*
* rnd_add_data_1(rs, buf, len, entropybits, count, flag)
*
* Internal subroutine to call either entropy_enter_intr, if we're
* in interrupt context, or entropy_enter if not, and to count the
* entropy in an rndsource.
*/
static void
rnd_add_data_1(struct krndsource *rs, const void *buf, uint32_t len,
uint32_t entropybits, bool count, uint32_t flag, bool intr_p)
{
bool fullyused;
/*
* For the interrupt-like path, use entropy_enter_intr and take
* note of whether it consumed the full sample; otherwise, use
* entropy_enter, which always consumes the full sample.
*/
if (intr_p) {
fullyused = entropy_enter_intr(buf, len, entropybits, count);
} else {
entropy_enter(buf, len, entropybits, count);
fullyused = true;
}
/*
* If we used the full sample, note how many bits were
* contributed from this source.
*/
if (fullyused) {
if (__predict_false(cold)) {
const int s = splhigh();
rs->total = add_sat(rs->total, entropybits);
switch (flag) {
case RND_FLAG_COLLECT_TIME:
rs->time_delta.insamples =
add_sat(rs->time_delta.insamples, 1);
break;
case RND_FLAG_COLLECT_VALUE:
rs->value_delta.insamples =
add_sat(rs->value_delta.insamples, 1);
break;
}
splx(s);
} else {
struct rndsource_cpu *rc = percpu_getref(rs->state);
/*
* rnd_add_data_sync(rs, buf, len, entropybits)
*
* Same as rnd_add_data. Originally used in rndsource callbacks,
* to break an unnecessary cycle; no longer really needed.
*/
void
rnd_add_data_sync(struct krndsource *rs, const void *buf, uint32_t len,
uint32_t entropybits)
{
rnd_add_data(rs, buf, len, entropybits);
}
/*
* rndsource_entropybits(rs)
*
* Return approximately the number of bits of entropy that have
* been contributed via rs so far. Approximate if other CPUs may
* be calling rnd_add_data concurrently.
*/
static unsigned
rndsource_entropybits(struct krndsource *rs)
{
unsigned nbits = rs->total;
/*
* rndsource_to_user(rs, urs)
*
* Copy a description of rs out to urs for userland.
*/
static void
rndsource_to_user(struct krndsource *rs, rndsource_t *urs)
{
/*
* rndsource_to_user_est(rs, urse)
*
* Copy a description of rs and estimation statistics out to urse
* for userland.
*/
static void
rndsource_to_user_est(struct krndsource *rs, rndsource_est_t *urse)
{
/*
* entropy_reset_xc(arg1, arg2)
*
* Reset the current CPU's pending entropy to zero.
*/
static void
entropy_reset_xc(void *arg1 __unused, void *arg2 __unused)
{
uint32_t extra = entropy_timer();
struct entropy_cpu_lock lock;
struct entropy_cpu *ec;
/*
* With the per-CPU state locked, zero the pending count and
* enter a cycle count for fun.
*/
ec = entropy_cpu_get(&lock);
ec->ec_bitspending = 0;
ec->ec_samplespending = 0;
entpool_enter(ec->ec_pool, &extra, sizeof extra);
entropy_cpu_put(&lock, ec);
}
/*
* entropy_reset()
*
* Assume the entropy pool has been exposed, e.g. because the VM
* has been cloned. Nix all the pending entropy and set the
* needed to maximum.
*/
void
entropy_reset(void)
{
/* state */
pstat->added = 0; /* XXX total entropy_enter count */
pstat->curentropy = MINENTROPYBITS - E->bitsneeded; /* bits */
pstat->removed = 0; /* XXX total entropy_extract count */
pstat->discarded = 0; /* XXX bits of entropy beyond capacity */
/*
* This used to be bits of data fabricated in some
* sense; we'll take it to mean number of samples,
* excluding the bits of entropy from HWRNG or seed.
*/
pstat->generated = MINSAMPLES - E->samplesneeded;
pstat->generated -= MIN(pstat->generated, pstat->curentropy);
mutex_exit(&E->lock);
break;
}
case RNDGETSRCNUM: { /* Get entropy sources by number. */
rndstat_t *stat = data;
uint32_t start = 0, i = 0;
/* Skip if none requested; fail if too many requested. */
if (stat->count == 0)
break;
if (stat->count > RND_MAXSTATCOUNT)
return EINVAL;
/*
* Under the lock, find the first one, copy out as many
* as requested, and report how many we copied out.
*/
mutex_enter(&E->lock);
error = rnd_lock_sources(ENTROPY_WAIT|ENTROPY_SIG);
if (error) {
mutex_exit(&E->lock);
return error;
}
LIST_FOREACH(rs, &E->sources, list) {
if (start++ == stat->start)
break;
}
while (i < stat->count && rs != NULL) {
mutex_exit(&E->lock);
rndsource_to_user(rs, &stat->source[i++]);
mutex_enter(&E->lock);
rs = LIST_NEXT(rs, list);
}
KASSERT(i <= stat->count);
stat->count = i;
rnd_unlock_sources();
mutex_exit(&E->lock);
break;
}
case RNDGETESTNUM: { /* Get sources and estimates by number. */
rndstat_est_t *estat = data;
uint32_t start = 0, i = 0;
/* Skip if none requested; fail if too many requested. */
if (estat->count == 0)
break;
if (estat->count > RND_MAXSTATCOUNT)
return EINVAL;
/*
* Under the lock, find the first one, copy out as many
* as requested, and report how many we copied out.
*/
mutex_enter(&E->lock);
error = rnd_lock_sources(ENTROPY_WAIT|ENTROPY_SIG);
if (error) {
mutex_exit(&E->lock);
return error;
}
LIST_FOREACH(rs, &E->sources, list) {
if (start++ == estat->start)
break;
}
while (i < estat->count && rs != NULL) {
mutex_exit(&E->lock);
rndsource_to_user_est(rs, &estat->source[i++]);
mutex_enter(&E->lock);
rs = LIST_NEXT(rs, list);
}
KASSERT(i <= estat->count);
estat->count = i;
rnd_unlock_sources();
mutex_exit(&E->lock);
break;
}
case RNDGETSRCNAME: { /* Get entropy sources by name. */
rndstat_name_t *nstat = data;
const size_t n = sizeof(rs->name);
/* Whitelist the flags that user can change. */
rndctl->mask &= RND_FLAG_NO_ESTIMATE|RND_FLAG_NO_COLLECT;
/*
* For each matching rndsource, either by type if
* specified or by name if not, set the masked flags.
*/
mutex_enter(&E->lock);
LIST_FOREACH(rs, &E->sources, list) {
if (rndctl->type != 0xff) {
if (rs->type != rndctl->type)
continue;
} else if (rndctl->name[0] != '\0') {
if (strncmp(rs->name, rndctl->name, n) != 0)
continue;
}
flags = rs->flags & ~rndctl->mask;
flags |= rndctl->flags & rndctl->mask;
if ((rs->flags & resetflags) == 0 &&
(flags & resetflags) != 0)
reset = true;
if ((rs->flags ^ flags) & resetflags)
request = true;
atomic_store_relaxed(&rs->flags, flags);
}
mutex_exit(&E->lock);
/*
* If we disabled estimation or collection, nix all the
* pending entropy and set needed to the maximum.
*/
if (reset)
entropy_reset();
/*
* If we changed any of the estimation or collection
* flags, request new samples from everyone -- either
* to make up for what we just lost, or to get new
* samples from what we just added.
*
* Failing on signal, while waiting for another process
* to finish requesting entropy, is OK here even though
* we have committed side effects, because this ioctl
* command is idempotent, so repeating it is safe.
*/
if (request)
error = entropy_gather();
break;
}
case RNDADDDATA: { /* Enter seed into entropy pool. */
rnddata_t *rdata = data;
unsigned entropybits = 0;
if (!atomic_load_relaxed(&entropy_collection))
break; /* thanks but no thanks */
if (rdata->len > MIN(sizeof(rdata->data), UINT32_MAX/NBBY))
return EINVAL;
/*
* This ioctl serves as the userland alternative a
* bootloader-provided seed -- typically furnished by
* /etc/rc.d/random_seed. We accept the user's entropy
* claim only if
*
* (a) the user is privileged, and
* (b) we have not entered a bootloader seed.
*
* under the assumption that the user may use this to
* load a seed from disk that we have already loaded
* from the bootloader, so we don't double-count it.
*/
if (privileged && rdata->entropy && rdata->len) {
mutex_enter(&E->lock);
if (!E->seeded) {
entropybits = MIN(rdata->entropy,
MIN(rdata->len, ENTROPY_CAPACITY)*NBBY);
E->seeded = true;
}
mutex_exit(&E->lock);
}
/* Enter the data and consolidate entropy. */
rnd_add_data(&seed_rndsource, rdata->data, rdata->len,
entropybits);
error = entropy_consolidate();
break;
}
default:
error = ENOTTY;
}
/* Return any error that may have come up. */
return error;
}
