From 5cbe0f13b51ac2fb2fd55902cff8d0077fc084c0 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 10 Feb 2020 13:00:12 +0000 Subject: [PATCH 1/7] random: split primary/secondary crng init paths Currently crng_initialize() is used for both the primary CRNG and secondary CRNGs. While we wish to share common logic, we need to do a number of additional things for the primary CRNG, and this would be easier to deal with were these handled in separate functions. This patch splits crng_initialize() into crng_initialize_primary() and crng_initialize_secondary(), with common logic factored out into a crng_init_try_arch() helper. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Cc: Mark Brown Cc: Theodore Ts'o Link: https://lore.kernel.org/r/20200210130015.17664-2-mark.rutland@arm.com Signed-off-by: Theodore Ts'o --- drivers/char/random.c | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index c7f9584de2c8..62d32e62f2da 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -781,27 +781,37 @@ static int __init parse_trust_cpu(char *arg) } early_param("random.trust_cpu", parse_trust_cpu); -static void crng_initialize(struct crng_state *crng) +static bool crng_init_try_arch(struct crng_state *crng) { int i; - int arch_init = 1; + bool arch_init = true; unsigned long rv; - memcpy(&crng->state[0], "expand 32-byte k", 16); - if (crng == &primary_crng) - _extract_entropy(&input_pool, &crng->state[4], - sizeof(__u32) * 12, 0); - else - _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); for (i = 4; i < 16; i++) { if (!arch_get_random_seed_long(&rv) && !arch_get_random_long(&rv)) { rv = random_get_entropy(); - arch_init = 0; + arch_init = false; } crng->state[i] ^= rv; } - if (trust_cpu && arch_init && crng == &primary_crng) { + + return arch_init; +} + +static void crng_initialize_secondary(struct crng_state *crng) +{ + memcpy(&crng->state[0], "expand 32-byte k", 16); + _get_random_bytes(&crng->state[4], sizeof(__u32) * 12); + crng_init_try_arch(crng); + crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1; +} + +static void __init crng_initialize_primary(struct crng_state *crng) +{ + memcpy(&crng->state[0], "expand 32-byte k", 16); + _extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0); + if (crng_init_try_arch(crng) && trust_cpu) { invalidate_batched_entropy(); numa_crng_init(); crng_init = 2; @@ -822,7 +832,7 @@ static void do_numa_crng_init(struct work_struct *work) crng = kmalloc_node(sizeof(struct crng_state), GFP_KERNEL | __GFP_NOFAIL, i); spin_lock_init(&crng->lock); - crng_initialize(crng); + crng_initialize_secondary(crng); pool[i] = crng; } mb(); @@ -1771,7 +1781,7 @@ static void __init init_std_data(struct entropy_store *r) int __init rand_initialize(void) { init_std_data(&input_pool); - crng_initialize(&primary_crng); + crng_initialize_primary(&primary_crng); crng_global_init_time = jiffies; if (ratelimit_disable) { urandom_warning.interval = 0; From 253d3194c2b58152fe830fd27c2fd83ebc6fe5ee Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 10 Feb 2020 13:00:13 +0000 Subject: [PATCH 2/7] random: add arch_get_random_*long_early() Some architectures (e.g. arm64) can have heterogeneous CPUs, and the boot CPU may be able to provide entropy while secondary CPUs cannot. On such systems, arch_get_random_long() and arch_get_random_seed_long() will fail unless support for RNG instructions has been detected on all CPUs. This prevents the boot CPU from being able to provide (potentially) trusted entropy when seeding the primary CRNG. To make it possible to seed the primary CRNG from the boot CPU without adversely affecting the runtime versions of arch_get_random_long() and arch_get_random_seed_long(), this patch adds new early versions of the functions used when initializing the primary CRNG. Default implementations are provided atop of the existing arch_get_random_long() and arch_get_random_seed_long() so that only architectures with such constraints need to provide the new helpers. There should be no functional change as a result of this patch. Signed-off-by: Mark Rutland Cc: Mark Brown Cc: Theodore Ts'o Link: https://lore.kernel.org/r/20200210130015.17664-3-mark.rutland@arm.com Signed-off-by: Theodore Ts'o --- drivers/char/random.c | 20 +++++++++++++++++++- include/linux/random.h | 22 ++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index 62d32e62f2da..02a85b87b993 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -799,6 +799,24 @@ static bool crng_init_try_arch(struct crng_state *crng) return arch_init; } +static bool __init crng_init_try_arch_early(struct crng_state *crng) +{ + int i; + bool arch_init = true; + unsigned long rv; + + for (i = 4; i < 16; i++) { + if (!arch_get_random_seed_long_early(&rv) && + !arch_get_random_long_early(&rv)) { + rv = random_get_entropy(); + arch_init = false; + } + crng->state[i] ^= rv; + } + + return arch_init; +} + static void crng_initialize_secondary(struct crng_state *crng) { memcpy(&crng->state[0], "expand 32-byte k", 16); @@ -811,7 +829,7 @@ static void __init crng_initialize_primary(struct crng_state *crng) { memcpy(&crng->state[0], "expand 32-byte k", 16); _extract_entropy(&input_pool, &crng->state[4], sizeof(__u32) * 12, 0); - if (crng_init_try_arch(crng) && trust_cpu) { + if (crng_init_try_arch_early(crng) && trust_cpu) { invalidate_batched_entropy(); numa_crng_init(); crng_init = 2; diff --git a/include/linux/random.h b/include/linux/random.h index d319f9a1e429..45e1f8fa742b 100644 --- a/include/linux/random.h +++ b/include/linux/random.h @@ -7,6 +7,8 @@ #ifndef _LINUX_RANDOM_H #define _LINUX_RANDOM_H +#include +#include #include #include @@ -185,6 +187,26 @@ static inline bool __must_check arch_get_random_seed_int(unsigned int *v) } #endif +/* + * Called from the boot CPU during startup; not valid to call once + * secondary CPUs are up and preemption is possible. + */ +#ifndef arch_get_random_seed_long_early +static inline bool __init arch_get_random_seed_long_early(unsigned long *v) +{ + WARN_ON(system_state != SYSTEM_BOOTING); + return arch_get_random_seed_long(v); +} +#endif + +#ifndef arch_get_random_long_early +static inline bool __init arch_get_random_long_early(unsigned long *v) +{ + WARN_ON(system_state != SYSTEM_BOOTING); + return arch_get_random_long(v); +} +#endif + /* Pseudo random number generator from numerical recipes. */ static inline u32 next_pseudo_random32(u32 seed) { From ead5084cdf5af51445d219800c2ac8b01eb85f2f Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 10 Feb 2020 13:00:14 +0000 Subject: [PATCH 3/7] arm64: add credited/trusted RNG support Currently arm64 doesn't initialize the primary CRNG in a (potentially) trusted manner as we only detect the presence of the RNG once secondary CPUs are up. Now that the core RNG code distinguishes the early initialization of the primary CRNG, we can implement arch_get_random_seed_long_early() to support this. This patch does so. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Mark Brown Cc: Theodore Ts'o Cc: Will Deacon Link: https://lore.kernel.org/r/20200210130015.17664-4-mark.rutland@arm.com Signed-off-by: Theodore Ts'o --- arch/arm64/include/asm/archrandom.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/arm64/include/asm/archrandom.h b/arch/arm64/include/asm/archrandom.h index 3fe02da70004..fc1594a0710e 100644 --- a/arch/arm64/include/asm/archrandom.h +++ b/arch/arm64/include/asm/archrandom.h @@ -4,6 +4,8 @@ #ifdef CONFIG_ARCH_RANDOM +#include +#include #include #include @@ -66,6 +68,18 @@ static inline bool __init __early_cpu_has_rndr(void) return (ftr >> ID_AA64ISAR0_RNDR_SHIFT) & 0xf; } +static inline bool __init __must_check +arch_get_random_seed_long_early(unsigned long *v) +{ + WARN_ON(system_state != SYSTEM_BOOTING); + + if (!__early_cpu_has_rndr()) + return false; + + return __arm64_rndr(v); +} +#define arch_get_random_seed_long_early arch_get_random_seed_long_early + #else static inline bool __arm64_rndr(unsigned long *v) { return false; } From 23ae0c17b89cfeb511aa0770735a319d35597072 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 10 Feb 2020 13:00:15 +0000 Subject: [PATCH 4/7] random: Make RANDOM_TRUST_CPU depend on ARCH_RANDOM Listing the set of host architectures does not scale. Depend instead on the existence of the architecture rng. This will allow RANDOM_TRUST_CPU to be selected on arm64. Today ARCH_RANDOM is only selected by x86, s390, and powerpc, so this does not adversely affect other architectures. Signed-off-by: Richard Henderson Signed-off-by: Mark Rutland Link: https://lore.kernel.org/r/20200210130015.17664-5-mark.rutland@arm.com Signed-off-by: Theodore Ts'o --- drivers/char/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig index 26956c006987..84207d5a9bb0 100644 --- a/drivers/char/Kconfig +++ b/drivers/char/Kconfig @@ -539,7 +539,7 @@ endmenu config RANDOM_TRUST_CPU bool "Trust the CPU manufacturer to initialize Linux's CRNG" - depends on X86 || S390 || PPC + depends on ARCH_RANDOM default n help Assume that CPU manufacturer (e.g., Intel or AMD for RDSEED or From 69efea712f5b0489e67d07565aad5c94e09a3e52 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Fri, 21 Feb 2020 21:10:37 +0100 Subject: [PATCH 5/7] random: always use batched entropy for get_random_u{32,64} It turns out that RDRAND is pretty slow. Comparing these two constructions: for (i = 0; i < CHACHA_BLOCK_SIZE; i += sizeof(ret)) arch_get_random_long(&ret); and long buf[CHACHA_BLOCK_SIZE / sizeof(long)]; extract_crng((u8 *)buf); it amortizes out to 352 cycles per long for the top one and 107 cycles per long for the bottom one, on Coffee Lake Refresh, Intel Core i9-9880H. And importantly, the top one has the drawback of not benefiting from the real rng, whereas the bottom one has all the nice benefits of using our own chacha rng. As get_random_u{32,64} gets used in more places (perhaps beyond what it was originally intended for when it was introduced as get_random_{int,long} back in the md5 monstrosity era), it seems like it might be a good thing to strengthen its posture a tiny bit. Doing this should only be stronger and not any weaker because that pool is already initialized with a bunch of rdrand data (when available). This way, we get the benefits of the hardware rng as well as our own rng. Another benefit of this is that we no longer hit pitfalls of the recent stream of AMD bugs in RDRAND. One often used code pattern for various things is: do { val = get_random_u32(); } while (hash_table_contains_key(val)); That recent AMD bug rendered that pattern useless, whereas we're really very certain that chacha20 output will give pretty distributed numbers, no matter what. So, this simplification seems better both from a security perspective and from a performance perspective. Signed-off-by: Jason A. Donenfeld Reviewed-by: Greg Kroah-Hartman Link: https://lore.kernel.org/r/20200221201037.30231-1-Jason@zx2c4.com Signed-off-by: Theodore Ts'o --- drivers/char/random.c | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index 02a85b87b993..c33e7a3cdb13 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -2177,11 +2177,11 @@ struct batched_entropy { /* * Get a random word for internal kernel use only. The quality of the random - * number is either as good as RDRAND or as good as /dev/urandom, with the - * goal of being quite fast and not depleting entropy. In order to ensure + * number is good as /dev/urandom, but there is no backtrack protection, with + * the goal of being quite fast and not depleting entropy. In order to ensure * that the randomness provided by this function is okay, the function - * wait_for_random_bytes() should be called and return 0 at least once - * at any point prior. + * wait_for_random_bytes() should be called and return 0 at least once at any + * point prior. */ static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), @@ -2194,15 +2194,6 @@ u64 get_random_u64(void) struct batched_entropy *batch; static void *previous; -#if BITS_PER_LONG == 64 - if (arch_get_random_long((unsigned long *)&ret)) - return ret; -#else - if (arch_get_random_long((unsigned long *)&ret) && - arch_get_random_long((unsigned long *)&ret + 1)) - return ret; -#endif - warn_unseeded_randomness(&previous); batch = raw_cpu_ptr(&batched_entropy_u64); @@ -2227,9 +2218,6 @@ u32 get_random_u32(void) struct batched_entropy *batch; static void *previous; - if (arch_get_random_int(&ret)) - return ret; - warn_unseeded_randomness(&previous); batch = raw_cpu_ptr(&batched_entropy_u32); From e00d996a4317aff5351c4338dd97d390225412c2 Mon Sep 17 00:00:00 2001 From: Qian Cai Date: Tue, 25 Feb 2020 11:27:04 -0500 Subject: [PATCH 6/7] random: fix data races at timer_rand_state Fields in "struct timer_rand_state" could be accessed concurrently. Lockless plain reads and writes result in data races. Fix them by adding pairs of READ|WRITE_ONCE(). The data races were reported by KCSAN, BUG: KCSAN: data-race in add_timer_randomness / add_timer_randomness write to 0xffff9f320a0a01d0 of 8 bytes by interrupt on cpu 22: add_timer_randomness+0x100/0x190 add_timer_randomness at drivers/char/random.c:1152 add_disk_randomness+0x85/0x280 scsi_end_request+0x43a/0x4a0 scsi_io_completion+0xb7/0x7e0 scsi_finish_command+0x1ed/0x2a0 scsi_softirq_done+0x1c9/0x1d0 blk_done_softirq+0x181/0x1d0 __do_softirq+0xd9/0x57c irq_exit+0xa2/0xc0 do_IRQ+0x8b/0x190 ret_from_intr+0x0/0x42 cpuidle_enter_state+0x15e/0x980 cpuidle_enter+0x69/0xc0 call_cpuidle+0x23/0x40 do_idle+0x248/0x280 cpu_startup_entry+0x1d/0x1f start_secondary+0x1b2/0x230 secondary_startup_64+0xb6/0xc0 no locks held by swapper/22/0. irq event stamp: 32871382 _raw_spin_unlock_irqrestore+0x53/0x60 _raw_spin_lock_irqsave+0x21/0x60 _local_bh_enable+0x21/0x30 irq_exit+0xa2/0xc0 read to 0xffff9f320a0a01d0 of 8 bytes by interrupt on cpu 2: add_timer_randomness+0xe8/0x190 add_disk_randomness+0x85/0x280 scsi_end_request+0x43a/0x4a0 scsi_io_completion+0xb7/0x7e0 scsi_finish_command+0x1ed/0x2a0 scsi_softirq_done+0x1c9/0x1d0 blk_done_softirq+0x181/0x1d0 __do_softirq+0xd9/0x57c irq_exit+0xa2/0xc0 do_IRQ+0x8b/0x190 ret_from_intr+0x0/0x42 cpuidle_enter_state+0x15e/0x980 cpuidle_enter+0x69/0xc0 call_cpuidle+0x23/0x40 do_idle+0x248/0x280 cpu_startup_entry+0x1d/0x1f start_secondary+0x1b2/0x230 secondary_startup_64+0xb6/0xc0 no locks held by swapper/2/0. irq event stamp: 37846304 _raw_spin_unlock_irqrestore+0x53/0x60 _raw_spin_lock_irqsave+0x21/0x60 _local_bh_enable+0x21/0x30 irq_exit+0xa2/0xc0 Reported by Kernel Concurrency Sanitizer on: Hardware name: HP ProLiant BL660c Gen9, BIOS I38 10/17/2018 Link: https://lore.kernel.org/r/1582648024-13111-1-git-send-email-cai@lca.pw Signed-off-by: Qian Cai Signed-off-by: Theodore Ts'o --- drivers/char/random.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index c33e7a3cdb13..f43f65c2195d 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1170,14 +1170,14 @@ static void add_timer_randomness(struct timer_rand_state *state, unsigned num) * We take into account the first, second and third-order deltas * in order to make our estimate. */ - delta = sample.jiffies - state->last_time; - state->last_time = sample.jiffies; + delta = sample.jiffies - READ_ONCE(state->last_time); + WRITE_ONCE(state->last_time, sample.jiffies); - delta2 = delta - state->last_delta; - state->last_delta = delta; + delta2 = delta - READ_ONCE(state->last_delta); + WRITE_ONCE(state->last_delta, delta); - delta3 = delta2 - state->last_delta2; - state->last_delta2 = delta2; + delta3 = delta2 - READ_ONCE(state->last_delta2); + WRITE_ONCE(state->last_delta2, delta2); if (delta < 0) delta = -delta; From ab9a7e27044b87ff2be47b8f8e095400e7fccc44 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 10 Mar 2020 12:09:12 +0000 Subject: [PATCH 7/7] random: avoid warnings for !CONFIG_NUMA builds As crng_initialize_secondary() is only called by do_numa_crng_init(), and the latter is under ifdeffery for CONFIG_NUMA, when CONFIG_NUMA is not selected the compiler will warn that the former is unused: | drivers/char/random.c:820:13: warning: 'crng_initialize_secondary' defined but not used [-Wunused-function] | 820 | static void crng_initialize_secondary(struct crng_state *crng) | | ^~~~~~~~~~~~~~~~~~~~~~~~~ Stephen reports that this happens for x86_64 noallconfig builds. We could move crng_initialize_secondary() and crng_init_try_arch() under the CONFIG_NUMA ifdeffery, but this has the unfortunate property of separating them from crng_initialize_primary() and crng_init_try_arch_early() respectively. Instead, let's mark crng_initialize_secondary() as __maybe_unused. Link: https://lore.kernel.org/r/20200310121747.GA49602@lakrids.cambridge.arm.com Fixes: 5cbe0f13b51a ("random: split primary/secondary crng init paths") Reported-by: Stephen Rothwell Signed-off-by: Mark Rutland Cc: Theodore Ts'o Signed-off-by: Theodore Ts'o --- drivers/char/random.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/char/random.c b/drivers/char/random.c index f43f65c2195d..0d10e31fd342 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -817,7 +817,7 @@ static bool __init crng_init_try_arch_early(struct crng_state *crng) return arch_init; } -static void crng_initialize_secondary(struct crng_state *crng) +static void __maybe_unused crng_initialize_secondary(struct crng_state *crng) { memcpy(&crng->state[0], "expand 32-byte k", 16); _get_random_bytes(&crng->state[4], sizeof(__u32) * 12);