History

random: use static branch for crng_ready()

Since crng_ready() is only false briefly during initialization and then forever after becomes true, we don't need to evaluate it after, making it a prime candidate for a static branch. One complication, however, is that it changes state in a particular call to credit_init_bits(), which might be made from atomic context, which means we must kick off a workqueue to change the static key. Further complicating things, credit_init_bits() may be called sufficiently early on in system initialization such that system_wq is NULL. Fortunately, there exists the nice function execute_in_process_context(), which will immediately execute the function if !in_interrupt(), and otherwise defer it to a workqueue. During early init, before workqueues are available, in_interrupt() is always false, because interrupts haven't even been enabled yet, which means the function in that case executes immediately. Later on, after workqueues are available, in_interrupt() might be true, but in that case, the work is queued in system_wq and all goes well.

Jason A. Donenfeld committed 4 years ago

Commit f5bda351 parent 12e45a21 file · +12 −4Browse files

Unified · Split

drivers/char/random.c+12−4
@@ -78,8 +78,9 @@ static enum { · 77 unmodified lines
78 CRNG_EMPTY = 0, /* Little to no entropy collected */78 CRNG_EMPTY = 0, /* Little to no entropy collected */
79 CRNG_EARLY = 1, /* At least POOL_EARLY_BITS collected */79 CRNG_EARLY = 1, /* At least POOL_EARLY_BITS collected */
80 CRNG_READY = 2 /* Fully initialized with POOL_READY_BITS collected */80 CRNG_READY = 2 /* Fully initialized with POOL_READY_BITS collected */
81} crng_init = CRNG_EMPTY;81} crng_init __read_mostly = CRNG_EMPTY;
82#define crng_ready() (likely(crng_init >= CRNG_READY))82static DEFINE_STATIC_KEY_FALSE(crng_is_ready);
83#define crng_ready() (static_branch_likely(&crng_is_ready) || crng_init >= CRNG_READY)
83/* Various types of waiters for crng_init->CRNG_READY transition. */84/* Various types of waiters for crng_init->CRNG_READY transition. */
84static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);85static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
85static struct fasync_struct *fasync;86static struct fasync_struct *fasync;
@@ -109,6 +110,11 @@ bool rng_is_initialized(void) · 23 unmodified lines
109}110}
110EXPORT_SYMBOL(rng_is_initialized);111EXPORT_SYMBOL(rng_is_initialized);
111 112
113static void crng_set_ready(struct work_struct *work)
114{
115 static_branch_enable(&crng_is_ready);
116}
117
112/* Used by wait_for_random_bytes(), and considered an entropy collector, below. */118/* Used by wait_for_random_bytes(), and considered an entropy collector, below. */
113static void try_to_generate_entropy(void);119static void try_to_generate_entropy(void);
114 120
@@ -268,7 +274,7 @@ static void crng_reseed(void) · 153 unmodified lines
268 ++next_gen;274 ++next_gen;
269 WRITE_ONCE(base_crng.generation, next_gen);275 WRITE_ONCE(base_crng.generation, next_gen);
270 WRITE_ONCE(base_crng.birth, jiffies);276 WRITE_ONCE(base_crng.birth, jiffies);
271 if (!crng_ready())277 if (!static_branch_likely(&crng_is_ready))
272 crng_init = CRNG_READY;278 crng_init = CRNG_READY;
273 spin_unlock_irqrestore(&base_crng.lock, flags);279 spin_unlock_irqrestore(&base_crng.lock, flags);
274 memzero_explicit(key, sizeof(key));280 memzero_explicit(key, sizeof(key));
@@ -786,6 +792,7 @@ static void extract_entropy(void *buf, size_t nbytes) · 511 unmodified lines
786 792
787static void credit_init_bits(size_t nbits)793static void credit_init_bits(size_t nbits)
788{794{
795 static struct execute_work set_ready;
789 unsigned int new, orig, add;796 unsigned int new, orig, add;
790 unsigned long flags;797 unsigned long flags;
791 798
@@ -801,6 +808,7 @@ static void credit_init_bits(size_t nbits) · 9 unmodified lines
801 808
802 if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {809 if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {
803 crng_reseed(); /* Sets crng_init to CRNG_READY under base_crng.lock. */810 crng_reseed(); /* Sets crng_init to CRNG_READY under base_crng.lock. */
811 execute_in_process_context(crng_set_ready, &set_ready);
804 process_random_ready_list();812 process_random_ready_list();
805 wake_up_interruptible(&crng_init_wait);813 wake_up_interruptible(&crng_init_wait);
806 kill_fasync(&fasync, SIGIO, POLL_IN);814 kill_fasync(&fasync, SIGIO, POLL_IN);
@@ -1396,7 +1404,7 @@ SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count, unsigned int, · 589 unmodified lines
1396 if (count > INT_MAX)1404 if (count > INT_MAX)
1397 count = INT_MAX;1405 count = INT_MAX;
1398 1406
1399 if (!(flags & GRND_INSECURE) && !crng_ready()) {1407 if (!crng_ready() && !(flags & GRND_INSECURE)) {
1400 int ret;1408 int ret;
1401 1409
1402 if (flags & GRND_NONBLOCK)1410 if (flags & GRND_NONBLOCK)