diff options
| author | riastradh <riastradh@NetBSD.org> | 2022-03-20 00:19:11 +0000 |
|---|---|---|
| committer | riastradh <riastradh@NetBSD.org> | 2022-03-20 00:19:11 +0000 |
| commit | 30afedb017fa787130b2f2cacc2920d02fcd1e9c (patch) | |
| tree | 78bfb2d2556348238adf5b4549de39cdc26b35f0 /sys | |
| parent | e1a7e1e1f8ff50be095ecb82bf7d3741c118fd29 (diff) | |
entropy(9): Avoid reentrance to per-CPU state from sleeping on lock.
Changing the global entropy lock from IPL_VM to IPL_SOFTSERIAL meant
it went from being a spin lock, which blocks preemption, to being an
adaptive lock, which might sleep -- and allow other threads to run
concurrently with the softint, even if those threads have softints
blocked with splsoftserial.
This manifested as KASSERT(!ec->ec_locked) triggering in
entropy_consolidate_xc -- presumably entropy_softintr slept on the
global entropy lock while holding the per-CPU state locked with
ec->ec_locked, and then entropy_consolidate_xc ran.
Instead, to protect access to the per-CPU state without taking a
global lock, defer entropy_account_cpu until after ec->ec_locked is
cleared. This way, we never sleep while holding ec->ec_locked, nor
do we incur any contention on shared memory when entering entropy
unless we're about to distribute it. To verify this, sprinkle in
assertions that curlwp->l_ncsw hasn't changed by the time we release
ec->ec_locked.
Diffstat (limited to 'sys')
| -rw-r--r-- | sys/kern/kern_entropy.c | 41 |
1 files changed, 29 insertions, 12 deletions
diff --git a/sys/kern/kern_entropy.c b/sys/kern/kern_entropy.c index b1f9676988e..9e1aac08430 100644 --- a/sys/kern/kern_entropy.c +++ b/sys/kern/kern_entropy.c @@ -1,4 +1,4 @@ -/* $NetBSD: kern_entropy.c,v 1.41 2022/03/19 14:35:08 riastradh Exp $ */ +/* $NetBSD: kern_entropy.c,v 1.42 2022/03/20 00:19:11 riastradh Exp $ */ /*- * Copyright (c) 2019 The NetBSD Foundation, Inc. @@ -75,7 +75,7 @@ */ #include <sys/cdefs.h> -__KERNEL_RCSID(0, "$NetBSD: kern_entropy.c,v 1.41 2022/03/19 14:35:08 riastradh Exp $"); +__KERNEL_RCSID(0, "$NetBSD: kern_entropy.c,v 1.42 2022/03/20 00:19:11 riastradh Exp $"); #include <sys/param.h> #include <sys/types.h> @@ -678,6 +678,7 @@ static void entropy_account_cpu(struct entropy_cpu *ec) { unsigned diff; + int s; KASSERT(E->stage >= ENTROPY_WARM); @@ -690,12 +691,9 @@ entropy_account_cpu(struct entropy_cpu *ec) __predict_true((time_uptime - E->timestamp) <= 60)) return; - /* If there's nothing pending, stop here. */ - if (ec->ec_pending == 0) - return; - /* Consider consolidation, under the lock. */ mutex_enter(&E->lock); + s = splsoftserial(); if (E->needed != 0 && E->needed <= ec->ec_pending) { /* * If we have not yet attained full entropy but we can @@ -750,6 +748,7 @@ entropy_account_cpu(struct entropy_cpu *ec) entropy_partial_evcnt.ev_count++; } } + splx(s); mutex_exit(&E->lock); } @@ -799,7 +798,8 @@ static void entropy_enter(const void *buf, size_t len, unsigned nbits) { struct entropy_cpu *ec; - uint32_t pending; + unsigned pending; + uint64_t ncsw; int s; KASSERTMSG(!cpu_intr_p(), @@ -821,6 +821,7 @@ entropy_enter(const void *buf, size_t len, unsigned nbits) s = splsoftserial(); KASSERT(!ec->ec_locked); ec->ec_locked = true; + ncsw = curlwp->l_ncsw; __insn_barrier(); /* Enter into the per-CPU pool. */ @@ -831,15 +832,17 @@ entropy_enter(const void *buf, size_t len, unsigned nbits) pending += MIN(ENTROPY_CAPACITY*NBBY - pending, nbits); atomic_store_relaxed(&ec->ec_pending, pending); - /* Consolidate globally if appropriate based on what we added. */ - entropy_account_cpu(ec); - /* Release the per-CPU state. */ KASSERT(ec->ec_locked); __insn_barrier(); + KASSERT(ncsw == curlwp->l_ncsw); ec->ec_locked = false; splx(s); percpu_putref(entropy_percpu); + + /* Consolidate globally if appropriate based on what we added. */ + if (pending) + entropy_account_cpu(ec); } /* @@ -935,6 +938,8 @@ static void entropy_softintr(void *cookie) { struct entropy_cpu *ec; + unsigned pending; + uint64_t ncsw; /* * Acquire the per-CPU state. Other users can lock this only @@ -944,6 +949,7 @@ entropy_softintr(void *cookie) ec = percpu_getref(entropy_percpu); KASSERT(!ec->ec_locked); ec->ec_locked = true; + ncsw = curlwp->l_ncsw; __insn_barrier(); /* Count statistics. */ @@ -952,14 +958,19 @@ entropy_softintr(void *cookie) /* Stir the pool if necessary. */ entpool_stir(ec->ec_pool); - /* Consolidate globally if appropriate based on what we added. */ - entropy_account_cpu(ec); + /* Determine if there's anything pending on this CPU. */ + pending = ec->ec_pending; /* Release the per-CPU state. */ KASSERT(ec->ec_locked); __insn_barrier(); + KASSERT(ncsw == curlwp->l_ncsw); ec->ec_locked = false; percpu_putref(entropy_percpu); + + /* Consolidate globally if appropriate based on what we added. */ + if (pending) + entropy_account_cpu(ec); } /* @@ -1092,6 +1103,7 @@ entropy_consolidate_xc(void *vpool, void *arg2 __unused) uint8_t buf[ENTPOOL_CAPACITY]; uint32_t extra[7]; unsigned i = 0; + uint64_t ncsw; int s; /* Grab CPU number and cycle counter to mix extra into the pool. */ @@ -1107,6 +1119,7 @@ entropy_consolidate_xc(void *vpool, void *arg2 __unused) s = splsoftserial(); KASSERT(!ec->ec_locked); ec->ec_locked = true; + ncsw = curlwp->l_ncsw; __insn_barrier(); extra[i++] = entropy_timer(); @@ -1118,6 +1131,7 @@ entropy_consolidate_xc(void *vpool, void *arg2 __unused) /* Release the per-CPU state. */ KASSERT(ec->ec_locked); __insn_barrier(); + KASSERT(ncsw == curlwp->l_ncsw); ec->ec_locked = false; splx(s); percpu_putref(entropy_percpu); @@ -2051,6 +2065,7 @@ entropy_reset_xc(void *arg1 __unused, void *arg2 __unused) { uint32_t extra = entropy_timer(); struct entropy_cpu *ec; + uint64_t ncsw; int s; /* @@ -2061,6 +2076,7 @@ entropy_reset_xc(void *arg1 __unused, void *arg2 __unused) s = splsoftserial(); KASSERT(!ec->ec_locked); ec->ec_locked = true; + ncsw = curlwp->l_ncsw; __insn_barrier(); /* Zero the pending count and enter a cycle count for fun. */ @@ -2070,6 +2086,7 @@ entropy_reset_xc(void *arg1 __unused, void *arg2 __unused) /* Release the per-CPU state. */ KASSERT(ec->ec_locked); __insn_barrier(); + KASSERT(ncsw == curlwp->l_ncsw); ec->ec_locked = false; splx(s); percpu_putref(entropy_percpu); |
