From fc4d24c9b47150245b3eb5bebc2ad4764c754ef4 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:49:57 +0100 Subject: [PATCH 01/60] fs/buffer: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: linux-fsdevel@vger.kernel.org Cc: Alexander Viro Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-2-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- fs/buffer.c | 16 ++++++---------- include/linux/cpuhotplug.h | 1 + 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index b205a629001d..1613656028d6 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -3403,7 +3403,7 @@ void free_buffer_head(struct buffer_head *bh) } EXPORT_SYMBOL(free_buffer_head); -static void buffer_exit_cpu(int cpu) +static int buffer_exit_cpu_dead(unsigned int cpu) { int i; struct bh_lru *b = &per_cpu(bh_lrus, cpu); @@ -3414,14 +3414,7 @@ static void buffer_exit_cpu(int cpu) } this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr); per_cpu(bh_accounting, cpu).nr = 0; -} - -static int buffer_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) - buffer_exit_cpu((unsigned long)hcpu); - return NOTIFY_OK; + return 0; } /** @@ -3471,6 +3464,7 @@ EXPORT_SYMBOL(bh_submit_read); void __init buffer_init(void) { unsigned long nrpages; + int ret; bh_cachep = kmem_cache_create("buffer_head", sizeof(struct buffer_head), 0, @@ -3483,5 +3477,7 @@ void __init buffer_init(void) */ nrpages = (nr_free_buffer_pages() * 10) / 100; max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head)); - hotcpu_notifier(buffer_cpu_notify, 0); + ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead", + NULL, buffer_exit_cpu_dead); + WARN_ON(ret < 0); } diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index afe641c02dca..69b74fa0da60 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -30,6 +30,7 @@ enum cpuhp_state { CPUHP_ACPI_CPUDRV_DEAD, CPUHP_S390_PFAULT_DEAD, CPUHP_BLK_MQ_DEAD, + CPUHP_FS_BUFF_DEAD, CPUHP_WORKQUEUE_PREP, CPUHP_POWER_NUMA_PREPARE, CPUHP_HRTIMERS_PREPARE, From 90b14889d2f9b29d7e5b4b2d36251c13ce3dd13f Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:49:58 +0100 Subject: [PATCH 02/60] kernel/printk: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Andrew Morton Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-3-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + kernel/printk/printk.c | 27 ++++++++++++--------------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 69b74fa0da60..4174083280d7 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -31,6 +31,7 @@ enum cpuhp_state { CPUHP_S390_PFAULT_DEAD, CPUHP_BLK_MQ_DEAD, CPUHP_FS_BUFF_DEAD, + CPUHP_PRINTK_DEAD, CPUHP_WORKQUEUE_PREP, CPUHP_POWER_NUMA_PREPARE, CPUHP_HRTIMERS_PREPARE, diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index de08fc90baaf..4487ffcd42d5 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2185,27 +2185,18 @@ void resume_console(void) /** * console_cpu_notify - print deferred console messages after CPU hotplug - * @self: notifier struct - * @action: CPU hotplug event - * @hcpu: unused + * @cpu: unused * * If printk() is called from a CPU that is not online yet, the messages * will be spooled but will not show up on the console. This function is * called when a new CPU comes online (or fails to come up), and ensures * that any such output gets printed. */ -static int console_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) +static int console_cpu_notify(unsigned int cpu) { - switch (action) { - case CPU_ONLINE: - case CPU_DEAD: - case CPU_DOWN_FAILED: - case CPU_UP_CANCELED: - console_lock(); - console_unlock(); - } - return NOTIFY_OK; + console_lock(); + console_unlock(); + return 0; } /** @@ -2843,6 +2834,7 @@ EXPORT_SYMBOL(unregister_console); static int __init printk_late_init(void) { struct console *con; + int ret; for_each_console(con) { if (!keep_bootcon && con->flags & CON_BOOT) { @@ -2857,7 +2849,12 @@ static int __init printk_late_init(void) unregister_console(con); } } - hotcpu_notifier(console_cpu_notify, 0); + ret = cpuhp_setup_state_nocalls(CPUHP_PRINTK_DEAD, "printk:dead", NULL, + console_cpu_notify); + WARN_ON(ret < 0); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "printk:online", + console_cpu_notify, NULL); + WARN_ON(ret < 0); return 0; } late_initcall(printk_late_init); From 308167fcb330296fc80505a6b11ba0661f38a4cc Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:49:59 +0100 Subject: [PATCH 03/60] mm/memcg: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Michal Hocko Cc: linux-mm@kvack.org Cc: rt@linutronix.de Cc: Johannes Weiner Cc: cgroups@vger.kernel.org Link: http://lkml.kernel.org/r/20161103145021.28528-4-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + mm/memcontrol.c | 24 ++++++++---------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 4174083280d7..c622ab349af3 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -32,6 +32,7 @@ enum cpuhp_state { CPUHP_BLK_MQ_DEAD, CPUHP_FS_BUFF_DEAD, CPUHP_PRINTK_DEAD, + CPUHP_MM_MEMCQ_DEAD, CPUHP_WORKQUEUE_PREP, CPUHP_POWER_NUMA_PREPARE, CPUHP_HRTIMERS_PREPARE, diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 0f870ba43942..6c2043509fb5 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -1816,22 +1816,13 @@ static void drain_all_stock(struct mem_cgroup *root_memcg) mutex_unlock(&percpu_charge_mutex); } -static int memcg_cpu_hotplug_callback(struct notifier_block *nb, - unsigned long action, - void *hcpu) +static int memcg_hotplug_cpu_dead(unsigned int cpu) { - int cpu = (unsigned long)hcpu; struct memcg_stock_pcp *stock; - if (action == CPU_ONLINE) - return NOTIFY_OK; - - if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) - return NOTIFY_OK; - stock = &per_cpu(memcg_stock, cpu); drain_stock(stock); - return NOTIFY_OK; + return 0; } static void reclaim_high(struct mem_cgroup *memcg, @@ -5774,16 +5765,17 @@ __setup("cgroup.memory=", cgroup_memory); /* * subsys_initcall() for memory controller. * - * Some parts like hotcpu_notifier() have to be initialized from this context - * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically - * everything that doesn't depend on a specific mem_cgroup structure should - * be initialized from here. + * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this + * context because of lock dependencies (cgroup_lock -> cpu hotplug) but + * basically everything that doesn't depend on a specific mem_cgroup structure + * should be initialized from here. */ static int __init mem_cgroup_init(void) { int cpu, node; - hotcpu_notifier(memcg_cpu_hotplug_callback, 0); + cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, + memcg_hotplug_cpu_dead); for_each_possible_cpu(cpu) INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, From 5588f5afb4cfc33eb377b751ba4b97184373e8d6 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:00 +0100 Subject: [PATCH 04/60] lib/percpu_counter: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-5-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + lib/percpu_counter.c | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index c622ab349af3..04e5f99ffc70 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -33,6 +33,7 @@ enum cpuhp_state { CPUHP_FS_BUFF_DEAD, CPUHP_PRINTK_DEAD, CPUHP_MM_MEMCQ_DEAD, + CPUHP_PERCPU_CNT_DEAD, CPUHP_WORKQUEUE_PREP, CPUHP_POWER_NUMA_PREPARE, CPUHP_HRTIMERS_PREPARE, diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index 72d36113ccaa..c8cebb137076 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c @@ -158,25 +158,21 @@ EXPORT_SYMBOL(percpu_counter_destroy); int percpu_counter_batch __read_mostly = 32; EXPORT_SYMBOL(percpu_counter_batch); -static void compute_batch_value(void) +static int compute_batch_value(unsigned int cpu) { int nr = num_online_cpus(); percpu_counter_batch = max(32, nr*2); + return 0; } -static int percpu_counter_hotcpu_callback(struct notifier_block *nb, - unsigned long action, void *hcpu) +static int percpu_counter_cpu_dead(unsigned int cpu) { #ifdef CONFIG_HOTPLUG_CPU - unsigned int cpu; struct percpu_counter *fbc; - compute_batch_value(); - if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) - return NOTIFY_OK; + compute_batch_value(cpu); - cpu = (unsigned long)hcpu; spin_lock_irq(&percpu_counters_lock); list_for_each_entry(fbc, &percpu_counters, list) { s32 *pcount; @@ -190,7 +186,7 @@ static int percpu_counter_hotcpu_callback(struct notifier_block *nb, } spin_unlock_irq(&percpu_counters_lock); #endif - return NOTIFY_OK; + return 0; } /* @@ -222,8 +218,15 @@ EXPORT_SYMBOL(__percpu_counter_compare); static int __init percpu_counter_startup(void) { - compute_batch_value(); - hotcpu_notifier(percpu_counter_hotcpu_callback, 0); + int ret; + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "lib/percpu_cnt:online", + compute_batch_value, NULL); + WARN_ON(ret < 0); + ret = cpuhp_setup_state_nocalls(CPUHP_PERCPU_CNT_DEAD, + "lib/percpu_cnt:dead", NULL, + percpu_counter_cpu_dead); + WARN_ON(ret < 0); return 0; } module_init(percpu_counter_startup); From d544abd5ff7d8b07c0c67682a63e4939c3c82914 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:01 +0100 Subject: [PATCH 05/60] lib/radix-tree: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Andrew Morton Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-6-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + lib/radix-tree.c | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 04e5f99ffc70..89310fb1031d 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -34,6 +34,7 @@ enum cpuhp_state { CPUHP_PRINTK_DEAD, CPUHP_MM_MEMCQ_DEAD, CPUHP_PERCPU_CNT_DEAD, + CPUHP_RADIX_DEAD, CPUHP_WORKQUEUE_PREP, CPUHP_POWER_NUMA_PREPARE, CPUHP_HRTIMERS_PREPARE, diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 8e6d552c40dd..4b8bb3618b83 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c @@ -1642,32 +1642,31 @@ static __init void radix_tree_init_maxnodes(void) } } -static int radix_tree_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) +static int radix_tree_cpu_dead(unsigned int cpu) { - int cpu = (long)hcpu; struct radix_tree_preload *rtp; struct radix_tree_node *node; /* Free per-cpu pool of preloaded nodes */ - if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { - rtp = &per_cpu(radix_tree_preloads, cpu); - while (rtp->nr) { - node = rtp->nodes; - rtp->nodes = node->private_data; - kmem_cache_free(radix_tree_node_cachep, node); - rtp->nr--; - } + rtp = &per_cpu(radix_tree_preloads, cpu); + while (rtp->nr) { + node = rtp->nodes; + rtp->nodes = node->private_data; + kmem_cache_free(radix_tree_node_cachep, node); + rtp->nr--; } - return NOTIFY_OK; + return 0; } void __init radix_tree_init(void) { + int ret; radix_tree_node_cachep = kmem_cache_create("radix_tree_node", sizeof(struct radix_tree_node), 0, SLAB_PANIC | SLAB_RECLAIM_ACCOUNT, radix_tree_node_ctor); radix_tree_init_maxnodes(); - hotcpu_notifier(radix_tree_callback, 0); + ret = cpuhp_setup_state_nocalls(CPUHP_RADIX_DEAD, "lib/radix:dead", + NULL, radix_tree_cpu_dead); + WARN_ON(ret < 0); } From 005fd4bbef168e9dea896085b001d64369e9834a Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:02 +0100 Subject: [PATCH 06/60] mm/page_alloc: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: linux-mm@kvack.org Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-7-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + mm/page_alloc.c | 49 +++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 89310fb1031d..31c58f6ec3c6 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -35,6 +35,7 @@ enum cpuhp_state { CPUHP_MM_MEMCQ_DEAD, CPUHP_PERCPU_CNT_DEAD, CPUHP_RADIX_DEAD, + CPUHP_PAGE_ALLOC_DEAD, CPUHP_WORKQUEUE_PREP, CPUHP_POWER_NUMA_PREPARE, CPUHP_HRTIMERS_PREPARE, diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 072d791dce2d..fc98c2bae905 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -6491,38 +6491,39 @@ void __init free_area_init(unsigned long *zones_size) __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); } -static int page_alloc_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) +static int page_alloc_cpu_dead(unsigned int cpu) { - int cpu = (unsigned long)hcpu; - if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { - lru_add_drain_cpu(cpu); - drain_pages(cpu); + lru_add_drain_cpu(cpu); + drain_pages(cpu); - /* - * Spill the event counters of the dead processor - * into the current processors event counters. - * This artificially elevates the count of the current - * processor. - */ - vm_events_fold_cpu(cpu); + /* + * Spill the event counters of the dead processor + * into the current processors event counters. + * This artificially elevates the count of the current + * processor. + */ + vm_events_fold_cpu(cpu); - /* - * Zero the differential counters of the dead processor - * so that the vm statistics are consistent. - * - * This is only okay since the processor is dead and cannot - * race with what we are doing. - */ - cpu_vm_stats_fold(cpu); - } - return NOTIFY_OK; + /* + * Zero the differential counters of the dead processor + * so that the vm statistics are consistent. + * + * This is only okay since the processor is dead and cannot + * race with what we are doing. + */ + cpu_vm_stats_fold(cpu); + return 0; } void __init page_alloc_init(void) { - hotcpu_notifier(page_alloc_cpu_notify, 0); + int ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD, + "mm/page_alloc:dead", NULL, + page_alloc_cpu_dead); + WARN_ON(ret < 0); } /* From 517bbed90635a17cf27450fb385af0e502413bea Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:03 +0100 Subject: [PATCH 07/60] mm/vmscan: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: linux-mm@kvack.org Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-8-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- mm/vmscan.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 76fda2268148..b8404d32caf0 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3556,24 +3556,21 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim) not required for correctness. So if the last cpu in a node goes away, we get changed to run anywhere: as the first one comes back, restore their cpu bindings. */ -static int cpu_callback(struct notifier_block *nfb, unsigned long action, - void *hcpu) +static int kswapd_cpu_online(unsigned int cpu) { int nid; - if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) { - for_each_node_state(nid, N_MEMORY) { - pg_data_t *pgdat = NODE_DATA(nid); - const struct cpumask *mask; + for_each_node_state(nid, N_MEMORY) { + pg_data_t *pgdat = NODE_DATA(nid); + const struct cpumask *mask; - mask = cpumask_of_node(pgdat->node_id); + mask = cpumask_of_node(pgdat->node_id); - if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids) - /* One of our CPUs online: restore mask */ - set_cpus_allowed_ptr(pgdat->kswapd, mask); - } + if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids) + /* One of our CPUs online: restore mask */ + set_cpus_allowed_ptr(pgdat->kswapd, mask); } - return NOTIFY_OK; + return 0; } /* @@ -3615,12 +3612,15 @@ void kswapd_stop(int nid) static int __init kswapd_init(void) { - int nid; + int nid, ret; swap_setup(); for_each_node_state(nid, N_MEMORY) kswapd_run(nid); - hotcpu_notifier(cpu_callback, 0); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "mm/vmscan:online", kswapd_cpu_online, + NULL); + WARN_ON(ret < 0); return 0; } From f0bf90def3528cebed45ebd81d9b5d0fa17d7422 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:04 +0100 Subject: [PATCH 08/60] net/dev: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: netdev@vger.kernel.org Cc: "David S. Miller" Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-9-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + net/core/dev.c | 16 ++++++---------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 31c58f6ec3c6..394eb7ed53be 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -36,6 +36,7 @@ enum cpuhp_state { CPUHP_PERCPU_CNT_DEAD, CPUHP_RADIX_DEAD, CPUHP_PAGE_ALLOC_DEAD, + CPUHP_NET_DEV_DEAD, CPUHP_WORKQUEUE_PREP, CPUHP_POWER_NUMA_PREPARE, CPUHP_HRTIMERS_PREPARE, diff --git a/net/core/dev.c b/net/core/dev.c index 820bac239738..8e909b2a5f2f 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -7953,18 +7953,13 @@ int dev_change_net_namespace(struct net_device *dev, struct net *net, const char } EXPORT_SYMBOL_GPL(dev_change_net_namespace); -static int dev_cpu_callback(struct notifier_block *nfb, - unsigned long action, - void *ocpu) +static int dev_cpu_dead(unsigned int oldcpu) { struct sk_buff **list_skb; struct sk_buff *skb; - unsigned int cpu, oldcpu = (unsigned long)ocpu; + unsigned int cpu; struct softnet_data *sd, *oldsd; - if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) - return NOTIFY_OK; - local_irq_disable(); cpu = smp_processor_id(); sd = &per_cpu(softnet_data, cpu); @@ -8014,10 +8009,9 @@ static int dev_cpu_callback(struct notifier_block *nfb, input_queue_head_incr(oldsd); } - return NOTIFY_OK; + return 0; } - /** * netdev_increment_features - increment feature set by one * @all: current feature set @@ -8351,7 +8345,9 @@ static int __init net_dev_init(void) open_softirq(NET_TX_SOFTIRQ, net_tx_action); open_softirq(NET_RX_SOFTIRQ, net_rx_action); - hotcpu_notifier(dev_cpu_callback, 0); + rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead", + NULL, dev_cpu_dead); + WARN_ON(rc < 0); dst_subsys_init(); rc = 0; out: From a4fc1bfc42062e8bc7b2271a90d17403b096ce5d Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:05 +0100 Subject: [PATCH 09/60] net/flowcache: Convert to hotplug state machine Install the callbacks via the state machine. Use multi state support to avoid custom list handling for the multiple instances. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Steffen Klassert Cc: Herbert Xu Cc: netdev@vger.kernel.org Cc: rt@linutronix.de Cc: "David S. Miller" Link: http://lkml.kernel.org/r/20161103145021.28528-10-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + include/net/flow.h | 1 + include/net/flowcache.h | 2 +- net/core/flow.c | 60 +++++++++++++++++--------------------- net/xfrm/xfrm_policy.c | 1 + 5 files changed, 30 insertions(+), 35 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 394eb7ed53be..86b940f19df8 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -56,6 +56,7 @@ enum cpuhp_state { CPUHP_ARM_SHMOBILE_SCU_PREPARE, CPUHP_SH_SH3X_PREPARE, CPUHP_BLK_MQ_PREPARE, + CPUHP_NET_FLOW_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, diff --git a/include/net/flow.h b/include/net/flow.h index 035aa7716967..2e386bd6ee63 100644 --- a/include/net/flow.h +++ b/include/net/flow.h @@ -239,6 +239,7 @@ struct flow_cache_object *flow_cache_lookup(struct net *net, void *ctx); int flow_cache_init(struct net *net); void flow_cache_fini(struct net *net); +void flow_cache_hp_init(void); void flow_cache_flush(struct net *net); void flow_cache_flush_deferred(struct net *net); diff --git a/include/net/flowcache.h b/include/net/flowcache.h index c8f665ec6e0d..9caf3bfc8d2d 100644 --- a/include/net/flowcache.h +++ b/include/net/flowcache.h @@ -17,7 +17,7 @@ struct flow_cache_percpu { struct flow_cache { u32 hash_shift; struct flow_cache_percpu __percpu *percpu; - struct notifier_block hotcpu_notifier; + struct hlist_node node; int low_watermark; int high_watermark; struct timer_list rnd_timer; diff --git a/net/core/flow.c b/net/core/flow.c index 3937b1b68d5b..841fd7f87b30 100644 --- a/net/core/flow.c +++ b/net/core/flow.c @@ -419,28 +419,20 @@ static int flow_cache_cpu_prepare(struct flow_cache *fc, int cpu) return 0; } -static int flow_cache_cpu(struct notifier_block *nfb, - unsigned long action, - void *hcpu) +static int flow_cache_cpu_up_prep(unsigned int cpu, struct hlist_node *node) { - struct flow_cache *fc = container_of(nfb, struct flow_cache, - hotcpu_notifier); - int res, cpu = (unsigned long) hcpu; + struct flow_cache *fc = hlist_entry_safe(node, struct flow_cache, node); + + return flow_cache_cpu_prepare(fc, cpu); +} + +static int flow_cache_cpu_dead(unsigned int cpu, struct hlist_node *node) +{ + struct flow_cache *fc = hlist_entry_safe(node, struct flow_cache, node); struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu); - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - res = flow_cache_cpu_prepare(fc, cpu); - if (res) - return notifier_from_errno(res); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - __flow_cache_shrink(fc, fcp, 0); - break; - } - return NOTIFY_OK; + __flow_cache_shrink(fc, fcp, 0); + return 0; } int flow_cache_init(struct net *net) @@ -467,18 +459,8 @@ int flow_cache_init(struct net *net) if (!fc->percpu) return -ENOMEM; - cpu_notifier_register_begin(); - - for_each_online_cpu(i) { - if (flow_cache_cpu_prepare(fc, i)) - goto err; - } - fc->hotcpu_notifier = (struct notifier_block){ - .notifier_call = flow_cache_cpu, - }; - __register_hotcpu_notifier(&fc->hotcpu_notifier); - - cpu_notifier_register_done(); + if (cpuhp_state_add_instance(CPUHP_NET_FLOW_PREPARE, &fc->node)) + goto err; setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd, (unsigned long) fc); @@ -494,8 +476,6 @@ int flow_cache_init(struct net *net) fcp->hash_table = NULL; } - cpu_notifier_register_done(); - free_percpu(fc->percpu); fc->percpu = NULL; @@ -509,7 +489,8 @@ void flow_cache_fini(struct net *net) struct flow_cache *fc = &net->xfrm.flow_cache_global; del_timer_sync(&fc->rnd_timer); - unregister_hotcpu_notifier(&fc->hotcpu_notifier); + + cpuhp_state_remove_instance_nocalls(CPUHP_NET_FLOW_PREPARE, &fc->node); for_each_possible_cpu(i) { struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i); @@ -521,3 +502,14 @@ void flow_cache_fini(struct net *net) fc->percpu = NULL; } EXPORT_SYMBOL(flow_cache_fini); + +void __init flow_cache_hp_init(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_NET_FLOW_PREPARE, + "net/flow:prepare", + flow_cache_cpu_up_prep, + flow_cache_cpu_dead); + WARN_ON(ret < 0); +} diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c index fd6986634e6f..4a8eff11bdad 100644 --- a/net/xfrm/xfrm_policy.c +++ b/net/xfrm/xfrm_policy.c @@ -3111,6 +3111,7 @@ static struct pernet_operations __net_initdata xfrm_net_ops = { void __init xfrm_init(void) { + flow_cache_hp_init(); register_pernet_subsys(&xfrm_net_ops); seqcount_init(&xfrm_policy_hash_generation); xfrm_input_init(); From ef65d45cbfbb438a5fbe8ef3fc424314ff1e8b7c Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 4 Nov 2016 15:41:41 +0100 Subject: [PATCH 10/60] s390/smp: Make cpu notifier symetric There is no reason to remove the sysfs cpu files when the CPU is dead, they can be removed when the cpu is prepared to go down. Doing it at DOWN_PREPARE allows us to convert it to a symetric hotplug state in the next step. Signed-off-by: Thomas Gleixner Signed-off-by: Sebastian Andrzej Siewior Acked-by: Heiko Carstens Cc: Martin Schwidefsky Cc: linux-s390@vger.kernel.org Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161104144140.lcee6kwmwlx37m7g@linutronix.de Signed-off-by: Thomas Gleixner --- arch/s390/kernel/smp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 35531fe1c5ea..2a9c03df95e9 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -1056,9 +1056,10 @@ static int smp_cpu_notify(struct notifier_block *self, unsigned long action, switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: + case CPU_DOWN_FAILED: err = sysfs_create_group(&s->kobj, &cpu_online_attr_group); break; - case CPU_DEAD: + case CPU_DOWN_PREPARE: sysfs_remove_group(&s->kobj, &cpu_online_attr_group); break; } From dfbbd86a0f1c3ceec15b64c8f2149a903806ed8c Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Fri, 4 Nov 2016 15:45:03 +0100 Subject: [PATCH 11/60] s390/smp: Convert to hotplug state machine cpuhp_setup_state() invokes the startup callback on all online cpus with the proper protection, so we can remove the cpu hotplug protection from the init function and the creation of the per cpu files for online cpus in smp_add_present_cpu(). smp_add_present_cpu() is called also called from __smp_rescan_cpus(), but this callpath never adds an online cpu, it merily adds newly present cpus, so the creation of the cpu files is not required. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Acked-by: Heiko Carstens Cc: Martin Schwidefsky Cc: linux-s390@vger.kernel.org Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161104144502.7kd4bxz2rxqvtack@linutronix.de Signed-off-by: Thomas Gleixner --- arch/s390/kernel/smp.c | 37 +++++++++++-------------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c index 2a9c03df95e9..870fb5a3d3f0 100644 --- a/arch/s390/kernel/smp.c +++ b/arch/s390/kernel/smp.c @@ -1047,23 +1047,18 @@ static struct attribute_group cpu_online_attr_group = { .attrs = cpu_online_attrs, }; -static int smp_cpu_notify(struct notifier_block *self, unsigned long action, - void *hcpu) +static int smp_cpu_online(unsigned int cpu) { - unsigned int cpu = (unsigned int)(long)hcpu; struct device *s = &per_cpu(cpu_device, cpu)->dev; - int err = 0; - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_ONLINE: - case CPU_DOWN_FAILED: - err = sysfs_create_group(&s->kobj, &cpu_online_attr_group); - break; - case CPU_DOWN_PREPARE: - sysfs_remove_group(&s->kobj, &cpu_online_attr_group); - break; - } - return notifier_from_errno(err); + return sysfs_create_group(&s->kobj, &cpu_online_attr_group); +} +static int smp_cpu_pre_down(unsigned int cpu) +{ + struct device *s = &per_cpu(cpu_device, cpu)->dev; + + sysfs_remove_group(&s->kobj, &cpu_online_attr_group); + return 0; } static int smp_add_present_cpu(int cpu) @@ -1084,20 +1079,12 @@ static int smp_add_present_cpu(int cpu) rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group); if (rc) goto out_cpu; - if (cpu_online(cpu)) { - rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group); - if (rc) - goto out_online; - } rc = topology_cpu_init(c); if (rc) goto out_topology; return 0; out_topology: - if (cpu_online(cpu)) - sysfs_remove_group(&s->kobj, &cpu_online_attr_group); -out_online: sysfs_remove_group(&s->kobj, &cpu_common_attr_group); out_cpu: #ifdef CONFIG_HOTPLUG_CPU @@ -1150,17 +1137,15 @@ static int __init s390_smp_init(void) if (rc) return rc; #endif - cpu_notifier_register_begin(); for_each_present_cpu(cpu) { rc = smp_add_present_cpu(cpu); if (rc) goto out; } - __hotcpu_notifier(smp_cpu_notify, 0); - + rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online", + smp_cpu_online, smp_cpu_pre_down); out: - cpu_notifier_register_done(); return rc; } subsys_initcall(s390_smp_init); From 7cc277b489b4fe91f42eb596b282879c2d13152e Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:08 +0100 Subject: [PATCH 12/60] drivers base/cacheinfo: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. No functional change. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Greg Kroah-Hartman Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-13-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/base/cacheinfo.c | 57 +++++++++++----------------------------- 1 file changed, 15 insertions(+), 42 deletions(-) diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c index e9fd32e91668..47983a2ba621 100644 --- a/drivers/base/cacheinfo.c +++ b/drivers/base/cacheinfo.c @@ -498,57 +498,30 @@ static int cache_add_dev(unsigned int cpu) return rc; } -static void cache_remove_dev(unsigned int cpu) +static int cacheinfo_cpu_online(unsigned int cpu) { - if (!cpumask_test_cpu(cpu, &cache_dev_map)) - return; - cpumask_clear_cpu(cpu, &cache_dev_map); + int rc = detect_cache_attributes(cpu); - cpu_cache_sysfs_exit(cpu); + if (rc) + return rc; + rc = cache_add_dev(cpu); + if (rc) + free_cache_attributes(cpu); + return rc; } -static int cacheinfo_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) +static int cacheinfo_cpu_pre_down(unsigned int cpu) { - unsigned int cpu = (unsigned long)hcpu; - int rc = 0; + if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map)) + cpu_cache_sysfs_exit(cpu); - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_ONLINE: - rc = detect_cache_attributes(cpu); - if (!rc) - rc = cache_add_dev(cpu); - break; - case CPU_DEAD: - cache_remove_dev(cpu); - free_cache_attributes(cpu); - break; - } - return notifier_from_errno(rc); + free_cache_attributes(cpu); + return 0; } static int __init cacheinfo_sysfs_init(void) { - int cpu, rc = 0; - - cpu_notifier_register_begin(); - - for_each_online_cpu(cpu) { - rc = detect_cache_attributes(cpu); - if (rc) - goto out; - rc = cache_add_dev(cpu); - if (rc) { - free_cache_attributes(cpu); - pr_err("error populating cacheinfo..cpu%d\n", cpu); - goto out; - } - } - __hotcpu_notifier(cacheinfo_cpu_callback, 0); - -out: - cpu_notifier_register_done(); - return rc; + return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "base/cacheinfo:online", + cacheinfo_cpu_online, cacheinfo_cpu_pre_down); } - device_initcall(cacheinfo_sysfs_init); From 38643a0e691ec947d311eb2db011b289cf95014e Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:09 +0100 Subject: [PATCH 13/60] drivers base/topology: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. No functional change Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Greg Kroah-Hartman Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-14-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/base/topology.c | 42 +++++--------------------------------- include/linux/cpuhotplug.h | 1 + 2 files changed, 6 insertions(+), 37 deletions(-) diff --git a/drivers/base/topology.c b/drivers/base/topology.c index df3c97cb4c99..d6ec1c546f5b 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -118,51 +118,19 @@ static int topology_add_dev(unsigned int cpu) return sysfs_create_group(&dev->kobj, &topology_attr_group); } -static void topology_remove_dev(unsigned int cpu) +static int topology_remove_dev(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); sysfs_remove_group(&dev->kobj, &topology_attr_group); -} - -static int topology_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - int rc = 0; - - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - rc = topology_add_dev(cpu); - break; - case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: - case CPU_DEAD: - case CPU_DEAD_FROZEN: - topology_remove_dev(cpu); - break; - } - return notifier_from_errno(rc); + return 0; } static int topology_sysfs_init(void) { - int cpu; - int rc = 0; - - cpu_notifier_register_begin(); - - for_each_online_cpu(cpu) { - rc = topology_add_dev(cpu); - if (rc) - goto out; - } - __hotcpu_notifier(topology_cpu_callback, 0); - -out: - cpu_notifier_register_done(); - return rc; + return cpuhp_setup_state(CPUHP_TOPOLOGY_PREPARE, + "base/topology:prepare", topology_add_dev, + topology_remove_dev); } device_initcall(topology_sysfs_init); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 86b940f19df8..3410d83cc2e2 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -57,6 +57,7 @@ enum cpuhp_state { CPUHP_SH_SH3X_PREPARE, CPUHP_BLK_MQ_PREPARE, CPUHP_NET_FLOW_PREPARE, + CPUHP_TOPOLOGY_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, From 122231445f2517de1d3dd30acd28801b66d72cab Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:10 +0100 Subject: [PATCH 14/60] ia64/err-inject: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. The removal of the files happens now in the prepare down stage as there is no reason to keep them around until the cpu has actually died. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Fenghua Yu Cc: Tony Luck Cc: linux-ia64@vger.kernel.org Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-15-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/ia64/kernel/err_inject.c | 78 +++++++++-------------------------- 1 file changed, 19 insertions(+), 59 deletions(-) diff --git a/arch/ia64/kernel/err_inject.c b/arch/ia64/kernel/err_inject.c index 5ed0ea92c5bf..85bba43e7d5d 100644 --- a/arch/ia64/kernel/err_inject.c +++ b/arch/ia64/kernel/err_inject.c @@ -224,85 +224,45 @@ static struct attribute_group err_inject_attr_group = { .name = "err_inject" }; /* Add/Remove err_inject interface for CPU device */ -static int err_inject_add_dev(struct device *sys_dev) +static int err_inject_add_dev(unsigned int cpu) { + struct device *sys_dev = get_cpu_device(cpu); + return sysfs_create_group(&sys_dev->kobj, &err_inject_attr_group); } -static int err_inject_remove_dev(struct device *sys_dev) +static int err_inject_remove_dev(unsigned int cpu) { + struct device *sys_dev = get_cpu_device(cpu); + sysfs_remove_group(&sys_dev->kobj, &err_inject_attr_group); return 0; } -static int err_inject_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) + +static enum cpuhp_state hp_online; + +static int __init err_inject_init(void) { - unsigned int cpu = (unsigned long)hcpu; - struct device *sys_dev; - - sys_dev = get_cpu_device(cpu); - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - err_inject_add_dev(sys_dev); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - err_inject_remove_dev(sys_dev); - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block err_inject_cpu_notifier = -{ - .notifier_call = err_inject_cpu_callback, -}; - -static int __init -err_inject_init(void) -{ - int i; - + int ret; #ifdef ERR_INJ_DEBUG printk(KERN_INFO "Enter error injection driver.\n"); #endif - cpu_notifier_register_begin(); - - for_each_online_cpu(i) { - err_inject_cpu_callback(&err_inject_cpu_notifier, CPU_ONLINE, - (void *)(long)i); + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/err_inj:online", + err_inject_add_dev, err_inject_remove_dev); + if (ret >= 0) { + hp_online = ret; + ret = 0; } - - __register_hotcpu_notifier(&err_inject_cpu_notifier); - - cpu_notifier_register_done(); - - return 0; + return ret; } -static void __exit -err_inject_exit(void) +static void __exit err_inject_exit(void) { - int i; - struct device *sys_dev; - #ifdef ERR_INJ_DEBUG printk(KERN_INFO "Exit error injection driver.\n"); #endif - - cpu_notifier_register_begin(); - - for_each_online_cpu(i) { - sys_dev = get_cpu_device(i); - sysfs_remove_group(&sys_dev->kobj, &err_inject_attr_group); - } - - __unregister_hotcpu_notifier(&err_inject_cpu_notifier); - - cpu_notifier_register_done(); + cpuhp_remove_state(hp_online); } module_init(err_inject_init); From 715c32116fa9cfa298170646be02ae5c0135ba0f Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 15:50:11 +0100 Subject: [PATCH 15/60] ia64/palinfo: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. The removal of the files happens now in the prepare down stage as there is no reason to keep them around until the cpu has actually died. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Fenghua Yu Cc: Tony Luck Cc: linux-ia64@vger.kernel.org Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161103145021.28528-16-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/ia64/kernel/palinfo.c | 60 ++++++++++---------------------------- 1 file changed, 16 insertions(+), 44 deletions(-) diff --git a/arch/ia64/kernel/palinfo.c b/arch/ia64/kernel/palinfo.c index c39c3cd3ac34..b6e597860888 100644 --- a/arch/ia64/kernel/palinfo.c +++ b/arch/ia64/kernel/palinfo.c @@ -932,8 +932,7 @@ static const struct file_operations proc_palinfo_fops = { .release = single_release, }; -static void -create_palinfo_proc_entries(unsigned int cpu) +static int palinfo_add_proc(unsigned int cpu) { pal_func_cpu_u_t f; struct proc_dir_entry *cpu_dir; @@ -943,7 +942,7 @@ create_palinfo_proc_entries(unsigned int cpu) cpu_dir = proc_mkdir(cpustr, palinfo_dir); if (!cpu_dir) - return; + return -EINVAL; f.req_cpu = cpu; @@ -952,42 +951,21 @@ create_palinfo_proc_entries(unsigned int cpu) proc_create_data(palinfo_entries[j].name, 0, cpu_dir, &proc_palinfo_fops, (void *)f.value); } + return 0; } -static void -remove_palinfo_proc_entries(unsigned int hcpu) +static int palinfo_del_proc(unsigned int hcpu) { char cpustr[3+4+1]; /* cpu numbers are up to 4095 on itanic */ + sprintf(cpustr, "cpu%d", hcpu); remove_proc_subtree(cpustr, palinfo_dir); + return 0; } -static int palinfo_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int hotcpu = (unsigned long)hcpu; +static enum cpuhp_state hp_online; - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - create_palinfo_proc_entries(hotcpu); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - remove_palinfo_proc_entries(hotcpu); - break; - } - return NOTIFY_OK; -} - -static struct notifier_block __refdata palinfo_cpu_notifier = -{ - .notifier_call = palinfo_cpu_callback, - .priority = 0, -}; - -static int __init -palinfo_init(void) +static int __init palinfo_init(void) { int i = 0; @@ -996,25 +974,19 @@ palinfo_init(void) if (!palinfo_dir) return -ENOMEM; - cpu_notifier_register_begin(); - - /* Create palinfo dirs in /proc for all online cpus */ - for_each_online_cpu(i) { - create_palinfo_proc_entries(i); + i = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/palinfo:online", + palinfo_add_proc, palinfo_del_proc); + if (i < 0) { + remove_proc_subtree("pal", NULL); + return i; } - - /* Register for future delivery via notify registration */ - __register_hotcpu_notifier(&palinfo_cpu_notifier); - - cpu_notifier_register_done(); - + hp_online = i; return 0; } -static void __exit -palinfo_exit(void) +static void __exit palinfo_exit(void) { - unregister_hotcpu_notifier(&palinfo_cpu_notifier); + cpuhp_remove_state(hp_online); remove_proc_subtree("pal", NULL); } From aa17662bc26d2555e1f79e7bcd259043582f32ed Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 18:31:29 +0100 Subject: [PATCH 16/60] ia64/salinfo: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Fenghua Yu Cc: Tony Luck Cc: linux-ia64@vger.kernel.org Cc: kbuild test robot Cc: rt@linutronix.de Cc: kbuild-all@01.org Link: http://lkml.kernel.org/r/20161103173128.xuulg4nius46dng5@linutronix.de Signed-off-by: Thomas Gleixner --- arch/ia64/kernel/salinfo.c | 83 +++++++++++++++----------------------- 1 file changed, 32 insertions(+), 51 deletions(-) diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c index 5313007d5423..aaf74f36cfa1 100644 --- a/arch/ia64/kernel/salinfo.c +++ b/arch/ia64/kernel/salinfo.c @@ -550,52 +550,40 @@ static const struct file_operations salinfo_data_fops = { .llseek = default_llseek, }; -static int -salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) +static int salinfo_cpu_online(unsigned int cpu) { - unsigned int i, cpu = (unsigned long)hcpu; - unsigned long flags; + unsigned int i, end = ARRAY_SIZE(salinfo_data); struct salinfo_data *data; - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - spin_lock_irqsave(&data_saved_lock, flags); - for (i = 0, data = salinfo_data; - i < ARRAY_SIZE(salinfo_data); - ++i, ++data) { - cpumask_set_cpu(cpu, &data->cpu_event); - wake_up_interruptible(&data->read_wait); - } - spin_unlock_irqrestore(&data_saved_lock, flags); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - spin_lock_irqsave(&data_saved_lock, flags); - for (i = 0, data = salinfo_data; - i < ARRAY_SIZE(salinfo_data); - ++i, ++data) { - struct salinfo_data_saved *data_saved; - int j; - for (j = ARRAY_SIZE(data->data_saved) - 1, data_saved = data->data_saved + j; - j >= 0; - --j, --data_saved) { - if (data_saved->buffer && data_saved->cpu == cpu) { - shift1_data_saved(data, j); - } - } - cpumask_clear_cpu(cpu, &data->cpu_event); - } - spin_unlock_irqrestore(&data_saved_lock, flags); - break; + + spin_lock_irq(&data_saved_lock); + for (i = 0, data = salinfo_data; i < end; ++i, ++data) { + cpumask_set_cpu(cpu, &data->cpu_event); + wake_up_interruptible(&data->read_wait); } - return NOTIFY_OK; + spin_unlock_irq(&data_saved_lock); + return 0; } -static struct notifier_block salinfo_cpu_notifier = +static int salinfo_cpu_pre_down(unsigned int cpu) { - .notifier_call = salinfo_cpu_callback, - .priority = 0, -}; + unsigned int i, end = ARRAY_SIZE(salinfo_data); + struct salinfo_data *data; + + spin_lock_irq(&data_saved_lock); + for (i = 0, data = salinfo_data; i < end; ++i, ++data) { + struct salinfo_data_saved *data_saved; + int j = ARRAY_SIZE(data->data_saved) - 1; + + for (data_saved = data->data_saved + j; j >= 0; + --j, --data_saved) { + if (data_saved->buffer && data_saved->cpu == cpu) + shift1_data_saved(data, j); + } + cpumask_clear_cpu(cpu, &data->cpu_event); + } + spin_unlock_irq(&data_saved_lock); + return 0; +} static int __init salinfo_init(void) @@ -604,7 +592,7 @@ salinfo_init(void) struct proc_dir_entry **sdir = salinfo_proc_entries; /* keeps track of every entry */ struct proc_dir_entry *dir, *entry; struct salinfo_data *data; - int i, j; + int i; salinfo_dir = proc_mkdir("sal", NULL); if (!salinfo_dir) @@ -617,8 +605,6 @@ salinfo_init(void) (void *)salinfo_entries[i].feature); } - cpu_notifier_register_begin(); - for (i = 0; i < ARRAY_SIZE(salinfo_log_name); i++) { data = salinfo_data + i; data->type = i; @@ -639,10 +625,6 @@ salinfo_init(void) continue; *sdir++ = entry; - /* we missed any events before now */ - for_each_online_cpu(j) - cpumask_set_cpu(j, &data->cpu_event); - *sdir++ = dir; } @@ -653,10 +635,9 @@ salinfo_init(void) salinfo_timer.function = &salinfo_timeout; add_timer(&salinfo_timer); - __register_hotcpu_notifier(&salinfo_cpu_notifier); - - cpu_notifier_register_done(); - + i = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/salinfo:online", + salinfo_cpu_online, salinfo_cpu_pre_down); + WARN_ON(i < 0); return 0; } From 5c584dd58ea878027ed067996b306416a9c356b8 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 3 Nov 2016 18:33:53 +0100 Subject: [PATCH 17/60] ia64/topology: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Fenghua Yu Cc: Tony Luck Cc: linux-ia64@vger.kernel.org Cc: kbuild test robot Cc: rt@linutronix.de Cc: kbuild-all@01.org Link: http://lkml.kernel.org/r/20161103173353.dudhkpioitghd74x@linutronix.de Signed-off-by: Thomas Gleixner --- arch/ia64/kernel/topology.c | 54 +++++-------------------------------- 1 file changed, 7 insertions(+), 47 deletions(-) diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c index c01fe8991244..1a68f012a6dc 100644 --- a/arch/ia64/kernel/topology.c +++ b/arch/ia64/kernel/topology.c @@ -349,9 +349,9 @@ static int cpu_cache_sysfs_init(unsigned int cpu) } /* Add cache interface for CPU device */ -static int cache_add_dev(struct device *sys_dev) +static int cache_add_dev(unsigned int cpu) { - unsigned int cpu = sys_dev->id; + struct device *sys_dev = get_cpu_device(cpu); unsigned long i, j; struct cache_info *this_object; int retval = 0; @@ -399,9 +399,8 @@ static int cache_add_dev(struct device *sys_dev) } /* Remove cache interface for CPU device */ -static int cache_remove_dev(struct device *sys_dev) +static int cache_remove_dev(unsigned int cpu) { - unsigned int cpu = sys_dev->id; unsigned long i; for (i = 0; i < all_cpu_cache_info[cpu].num_cache_leaves; i++) @@ -419,52 +418,13 @@ static int cache_remove_dev(struct device *sys_dev) return 0; } -/* - * When a cpu is hot-plugged, do a check and initiate - * cache kobject if necessary - */ -static int cache_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - struct device *sys_dev; - - sys_dev = get_cpu_device(cpu); - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - cache_add_dev(sys_dev); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - cache_remove_dev(sys_dev); - break; - } - return NOTIFY_OK; -} - -static struct notifier_block cache_cpu_notifier = -{ - .notifier_call = cache_cpu_callback -}; - static int __init cache_sysfs_init(void) { - int i; - - cpu_notifier_register_begin(); - - for_each_online_cpu(i) { - struct device *sys_dev = get_cpu_device((unsigned int)i); - cache_add_dev(sys_dev); - } - - __register_hotcpu_notifier(&cache_cpu_notifier); - - cpu_notifier_register_done(); + int ret; + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/topology:online", + cache_add_dev, cache_remove_dev); + WARN_ON(ret < 0); return 0; } - device_initcall(cache_sysfs_init); - From 0943637293a05aa3d8fb47bc30c7c0104480829a Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 10 Nov 2016 18:44:41 +0100 Subject: [PATCH 18/60] x86/mcheck: Move threshold_create_device() Move the threshold_create_device() so it can use threshold_remove_device() without a forward declaration. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Borislav Petkov Cc: Tony Luck Cc: rt@linutronix.de Cc: linux-edac@vger.kernel.org Link: http://lkml.kernel.org/r/20161110174447.11848-2-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/kernel/cpu/mcheck/mce_amd.c | 50 ++++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c index 9b5403462936..75a3e3eab81e 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c @@ -1010,31 +1010,6 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank) return err; } -/* create dir/files for all valid threshold banks */ -static int threshold_create_device(unsigned int cpu) -{ - unsigned int bank; - struct threshold_bank **bp; - int err = 0; - - bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks, - GFP_KERNEL); - if (!bp) - return -ENOMEM; - - per_cpu(threshold_banks, cpu) = bp; - - for (bank = 0; bank < mca_cfg.banks; ++bank) { - if (!(per_cpu(bank_map, cpu) & (1 << bank))) - continue; - err = threshold_create_bank(cpu, bank); - if (err) - return err; - } - - return err; -} - static void deallocate_threshold_block(unsigned int cpu, unsigned int bank) { @@ -1114,6 +1089,31 @@ static void threshold_remove_device(unsigned int cpu) kfree(per_cpu(threshold_banks, cpu)); } +/* create dir/files for all valid threshold banks */ +static int threshold_create_device(unsigned int cpu) +{ + unsigned int bank; + struct threshold_bank **bp; + int err = 0; + + bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks, + GFP_KERNEL); + if (!bp) + return -ENOMEM; + + per_cpu(threshold_banks, cpu) = bp; + + for (bank = 0; bank < mca_cfg.banks; ++bank) { + if (!(per_cpu(bank_map, cpu) & (1 << bank))) + continue; + err = threshold_create_bank(cpu, bank); + if (err) + return err; + } + + return err; +} + /* get notified when a cpu comes on/off */ static void amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu) From ec553abb318d75b1eafd275c71125569fa13b33b Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 10 Nov 2016 18:44:42 +0100 Subject: [PATCH 19/60] x86/mcheck: Explicit cleanup on failure in mce_amd If the ONLINE callback fails, the driver does not any clean up right away instead it waits to get to the DEAD stage to do it. Yes, it waits. Since we don't pass the error code back to the caller, no one knows. Do the clean up right away so it does not look like a leak. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Borislav Petkov Cc: Tony Luck Cc: rt@linutronix.de Cc: linux-edac@vger.kernel.org Link: http://lkml.kernel.org/r/20161110174447.11848-3-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/kernel/cpu/mcheck/mce_amd.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c index 75a3e3eab81e..55cd018bc1ae 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c @@ -1087,6 +1087,7 @@ static void threshold_remove_device(unsigned int cpu) threshold_remove_bank(cpu, bank); } kfree(per_cpu(threshold_banks, cpu)); + per_cpu(threshold_banks, cpu) = NULL; } /* create dir/files for all valid threshold banks */ @@ -1108,9 +1109,11 @@ static int threshold_create_device(unsigned int cpu) continue; err = threshold_create_bank(cpu, bank); if (err) - return err; + goto err; } - + return err; +err: + threshold_remove_device(cpu); return err; } From 7f34b935e8bf2f5304fce273a8fa98c63886d686 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 10 Nov 2016 18:44:43 +0100 Subject: [PATCH 20/60] x86/mcheck: Be prepared for a rollback back to the ONLINE state If we try a CPU down and fail in the middle then we roll back to the online state. This means we would perform CPU_ONLINE / mce_device_create() without invoking CPU_DEAD / mce_device_remove() for the cleanup of what was allocated in CPU_ONLINE. Be prepared for this and don't allocate the struct if we have it already. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Borislav Petkov Cc: Tony Luck Cc: rt@linutronix.de Cc: linux-edac@vger.kernel.org Link: http://lkml.kernel.org/r/20161110174447.11848-4-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/kernel/cpu/mcheck/mce.c | 4 ++++ arch/x86/kernel/cpu/mcheck/mce_amd.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index a7fdf453d895..e9ffd6d9e32d 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -2409,6 +2409,10 @@ static int mce_device_create(unsigned int cpu) if (!mce_available(&boot_cpu_data)) return -EIO; + dev = per_cpu(mce_device, cpu); + if (dev) + return 0; + dev = kzalloc(sizeof *dev, GFP_KERNEL); if (!dev) return -ENOMEM; diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c index 55cd018bc1ae..e93580c82ef0 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c @@ -1097,6 +1097,10 @@ static int threshold_create_device(unsigned int cpu) struct threshold_bank **bp; int err = 0; + bp = per_cpu(threshold_banks, cpu); + if (bp) + return 0; + bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks, GFP_KERNEL); if (!bp) From 4d7b02d58c4000597d08930193d7aed81fba6b7c Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 10 Nov 2016 18:44:44 +0100 Subject: [PATCH 21/60] x86/mcheck: Split threshold_cpu_callback into two callbacks The threshold_cpu_callback callbacks looks like one of the notifier and its arguments are almost the same. Split this out and have one ONLINE and one DEAD callback. This will come handy later once the main code gets changed to use the callback mechanism. Also, handle threshold_cpu_callback_online() return value so we don't continue if the function fails. Boris Petkov removed the callback pointer and replaced it with proper functions. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Borislav Petkov Cc: Tony Luck Cc: rt@linutronix.de Cc: linux-edac@vger.kernel.org Link: http://lkml.kernel.org/r/20161110174447.11848-5-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/include/asm/mce.h | 7 +++-- arch/x86/kernel/cpu/mcheck/mce.c | 14 +++++----- arch/x86/kernel/cpu/mcheck/mce_amd.c | 38 +++++++++++----------------- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index 9bd7ff5ffbcc..caafad4a211e 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -293,9 +293,7 @@ void do_machine_check(struct pt_regs *, long); /* * Threshold handler */ - extern void (*mce_threshold_vector)(void); -extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu); /* Deferred error interrupt handler */ extern void (*deferred_error_int_vector)(void); @@ -377,7 +375,12 @@ struct smca_bank_info { }; extern struct smca_bank_info smca_banks[MAX_NR_BANKS]; +extern int mce_threshold_create_device(unsigned int cpu); +extern int mce_threshold_remove_device(unsigned int cpu); +#else +static inline int mce_threshold_create_device(unsigned int cpu) { return 0; }; +static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; }; #endif #endif /* _ASM_X86_MCE_H */ diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index e9ffd6d9e32d..052b5e05c3c4 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -2255,8 +2255,6 @@ static struct bus_type mce_subsys = { DEFINE_PER_CPU(struct device *, mce_device); -void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu); - static inline struct mce_bank *attr_to_bank(struct device_attribute *attr) { return container_of(attr, struct mce_bank, attr); @@ -2512,13 +2510,17 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: + mce_device_create(cpu); - if (threshold_cpu_callback) - threshold_cpu_callback(action, cpu); + + if (mce_threshold_create_device(cpu)) { + mce_device_remove(cpu); + return NOTIFY_BAD; + } + break; case CPU_DEAD: - if (threshold_cpu_callback) - threshold_cpu_callback(action, cpu); + mce_threshold_remove_device(cpu); mce_device_remove(cpu); mce_intel_hcpu_update(cpu); diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c index e93580c82ef0..c33a3ee2e383 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c @@ -55,6 +55,8 @@ /* Threshold LVT offset is at MSR0xC0000410[15:12] */ #define SMCA_THR_LVT_OFF 0xF000 +static bool thresholding_en; + static const char * const th_names[] = { "load_store", "insn_fetch", @@ -1077,10 +1079,13 @@ static void threshold_remove_bank(unsigned int cpu, int bank) per_cpu(threshold_banks, cpu)[bank] = NULL; } -static void threshold_remove_device(unsigned int cpu) +int mce_threshold_remove_device(unsigned int cpu) { unsigned int bank; + if (!thresholding_en) + return 0; + for (bank = 0; bank < mca_cfg.banks; ++bank) { if (!(per_cpu(bank_map, cpu) & (1 << bank))) continue; @@ -1088,15 +1093,19 @@ static void threshold_remove_device(unsigned int cpu) } kfree(per_cpu(threshold_banks, cpu)); per_cpu(threshold_banks, cpu) = NULL; + return 0; } /* create dir/files for all valid threshold banks */ -static int threshold_create_device(unsigned int cpu) +int mce_threshold_create_device(unsigned int cpu) { unsigned int bank; struct threshold_bank **bp; int err = 0; + if (!thresholding_en) + return 0; + bp = per_cpu(threshold_banks, cpu); if (bp) return 0; @@ -1117,40 +1126,23 @@ static int threshold_create_device(unsigned int cpu) } return err; err: - threshold_remove_device(cpu); + mce_threshold_remove_device(cpu); return err; } -/* get notified when a cpu comes on/off */ -static void -amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu) -{ - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - threshold_create_device(cpu); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - threshold_remove_device(cpu); - break; - default: - break; - } -} - static __init int threshold_init_device(void) { unsigned lcpu = 0; /* to hit CPUs online before the notifier is up */ for_each_online_cpu(lcpu) { - int err = threshold_create_device(lcpu); + int err = mce_threshold_create_device(lcpu); if (err) return err; } - threshold_cpu_callback = amd_64_threshold_cpu_callback; + + thresholding_en = true; return 0; } From 39f152ffbfedb42b57b6e0c896eeae51dbe83b7a Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 10 Nov 2016 18:44:45 +0100 Subject: [PATCH 22/60] x86/mcheck: Reorganize the hotplug callbacks Initially I wanted to remove mcheck_cpu_init() from identify_cpu() and let it become an independent early hotplug callback. The main problem here was that the init on the boot CPU may happen too late (device_initcall_sync(mcheck_init_device)) and nobody wanted to risk receiving and MCE event at boot time leading to a shutdown (if the MCE feature is not yet enabled). Here is attempt two: the timming stays as-is but the ordering of the functions is changed: - mcheck_cpu_init() (which is run from identify_cpu()) will setup the timer struct but won't fire the timer. This is moved to CPU_ONLINE since its cleanup part is in CPU_DOWN_PREPARE. So if it is okay to stop the timer early in the shutdown phase, it should be okay to start it late in the bring up phase. - CPU_DOWN_PREPARE disables the MCE feature flags for !INTEL CPUs in mce_disable_cpu(). If a failure occures it would be re-enabled on all vendor CPUs (including Intel where it was not disabled during shutdown). To keep this working I am moving it to CPU_ONLINE. smp_call_function_single() is dropped beause the notifier runs nowdays on the target CPU. - CPU_ONLINE is invoking mce_device_create() + mce_threshold_create_device() but its cleanup part is in CPU_DEAD (mce_threshold_remove_device() and mce_device_remove()). In order to keep this symmetrical I am moving the clean up from CPU_DEAD to CPU_DOWN_PREPARE. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Borislav Petkov Cc: Tony Luck Cc: rt@linutronix.de Cc: linux-edac@vger.kernel.org Link: http://lkml.kernel.org/r/20161110174447.11848-6-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/kernel/cpu/mcheck/mce.c | 36 ++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 052b5e05c3c4..a524faa51400 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -1745,6 +1745,14 @@ static void mce_start_timer(unsigned int cpu, struct timer_list *t) add_timer_on(t, cpu); } +static void __mcheck_cpu_setup_timer(void) +{ + struct timer_list *t = this_cpu_ptr(&mce_timer); + unsigned int cpu = smp_processor_id(); + + setup_pinned_timer(t, mce_timer_fn, cpu); +} + static void __mcheck_cpu_init_timer(void) { struct timer_list *t = this_cpu_ptr(&mce_timer); @@ -1796,7 +1804,7 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c) __mcheck_cpu_init_generic(); __mcheck_cpu_init_vendor(c); __mcheck_cpu_init_clear_banks(); - __mcheck_cpu_init_timer(); + __mcheck_cpu_setup_timer(); } /* @@ -2470,28 +2478,25 @@ static void mce_device_remove(unsigned int cpu) } /* Make sure there are no machine checks on offlined CPUs. */ -static void mce_disable_cpu(void *h) +static void mce_disable_cpu(void) { - unsigned long action = *(unsigned long *)h; - if (!mce_available(raw_cpu_ptr(&cpu_info))) return; - if (!(action & CPU_TASKS_FROZEN)) + if (!cpuhp_tasks_frozen) cmci_clear(); vendor_disable_error_reporting(); } -static void mce_reenable_cpu(void *h) +static void mce_reenable_cpu(void) { - unsigned long action = *(unsigned long *)h; int i; if (!mce_available(raw_cpu_ptr(&cpu_info))) return; - if (!(action & CPU_TASKS_FROZEN)) + if (!cpuhp_tasks_frozen) cmci_reenable(); for (i = 0; i < mca_cfg.banks; i++) { struct mce_bank *b = &mce_banks[i]; @@ -2510,6 +2515,7 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: + case CPU_DOWN_FAILED: mce_device_create(cpu); @@ -2517,11 +2523,10 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) mce_device_remove(cpu); return NOTIFY_BAD; } - + mce_reenable_cpu(); + mce_start_timer(cpu, t); break; case CPU_DEAD: - mce_threshold_remove_device(cpu); - mce_device_remove(cpu); mce_intel_hcpu_update(cpu); /* intentionally ignoring frozen here */ @@ -2529,12 +2534,11 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) cmci_rediscover(); break; case CPU_DOWN_PREPARE: - smp_call_function_single(cpu, mce_disable_cpu, &action, 1); + mce_disable_cpu(); del_timer_sync(t); - break; - case CPU_DOWN_FAILED: - smp_call_function_single(cpu, mce_reenable_cpu, &action, 1); - mce_start_timer(cpu, t); + + mce_threshold_remove_device(cpu); + mce_device_remove(cpu); break; } From 8c0eeac819c85e4c1143f7a874d87b4594739208 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 10 Nov 2016 18:44:46 +0100 Subject: [PATCH 23/60] x86/mcheck: Move CPU_ONLINE and CPU_DOWN_PREPARE to hotplug state machine The CPU_ONLINE and CPU_DOWN_PREPARE look fully symmetrical and could be move to the hotplug state machine. On a failure during registration we have the tear down callback invoked (mce_cpu_pre_down()) so there should be no timer around and so no need to need keep notifier installed (this was the reason according to the comment why the notifier was registered despite of errors). Signed-off-by: Sebastian Andrzej Siewior Acked-by: Borislav Petkov Cc: Tony Luck Cc: rt@linutronix.de Cc: linux-edac@vger.kernel.org Link: http://lkml.kernel.org/r/20161110174447.11848-7-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/kernel/cpu/mcheck/mce.c | 78 +++++++++++++++----------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index a524faa51400..78955f501ff2 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -2511,21 +2511,8 @@ static int mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) { unsigned int cpu = (unsigned long)hcpu; - struct timer_list *t = &per_cpu(mce_timer, cpu); switch (action & ~CPU_TASKS_FROZEN) { - case CPU_ONLINE: - case CPU_DOWN_FAILED: - - mce_device_create(cpu); - - if (mce_threshold_create_device(cpu)) { - mce_device_remove(cpu); - return NOTIFY_BAD; - } - mce_reenable_cpu(); - mce_start_timer(cpu, t); - break; case CPU_DEAD: mce_intel_hcpu_update(cpu); @@ -2534,17 +2521,41 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) cmci_rediscover(); break; case CPU_DOWN_PREPARE: - mce_disable_cpu(); - del_timer_sync(t); - mce_threshold_remove_device(cpu); - mce_device_remove(cpu); break; } return NOTIFY_OK; } +static int mce_cpu_online(unsigned int cpu) +{ + struct timer_list *t = &per_cpu(mce_timer, cpu); + int ret; + + mce_device_create(cpu); + + ret = mce_threshold_create_device(cpu); + if (ret) { + mce_device_remove(cpu); + return ret; + } + mce_reenable_cpu(); + mce_start_timer(cpu, t); + return 0; +} + +static int mce_cpu_pre_down(unsigned int cpu) +{ + struct timer_list *t = &per_cpu(mce_timer, cpu); + + mce_disable_cpu(); + del_timer_sync(t); + mce_threshold_remove_device(cpu); + mce_device_remove(cpu); + return 0; +} + static struct notifier_block mce_cpu_notifier = { .notifier_call = mce_cpu_callback, }; @@ -2569,8 +2580,8 @@ static __init void mce_init_banks(void) static __init int mcheck_init_device(void) { + enum cpuhp_state hp_online; int err; - int i = 0; if (!mce_available(&boot_cpu_data)) { err = -EIO; @@ -2588,21 +2599,13 @@ static __init int mcheck_init_device(void) if (err) goto err_out_mem; - cpu_notifier_register_begin(); - for_each_online_cpu(i) { - err = mce_device_create(i); - if (err) { - /* - * Register notifier anyway (and do not unreg it) so - * that we don't leave undeleted timers, see notifier - * callback above. - */ - __register_hotcpu_notifier(&mce_cpu_notifier); - cpu_notifier_register_done(); - goto err_device_create; - } - } + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online", + mce_cpu_online, mce_cpu_pre_down); + if (err < 0) + goto err_out_mem; + hp_online = err; + cpu_notifier_register_begin(); __register_hotcpu_notifier(&mce_cpu_notifier); cpu_notifier_register_done(); @@ -2617,16 +2620,7 @@ static __init int mcheck_init_device(void) err_register: unregister_syscore_ops(&mce_syscore_ops); - -err_device_create: - /* - * We didn't keep track of which devices were created above, but - * even if we had, the set of online cpus might have changed. - * Play safe and remove for every possible cpu, since - * mce_device_remove() will do the right thing. - */ - for_each_possible_cpu(i) - mce_device_remove(i); + cpuhp_remove_state(hp_online); err_out_mem: free_cpumask_var(mce_device_initialized); From 0e285d36bd2bfee0b95433ccc9065c878164f5b2 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 10 Nov 2016 18:44:47 +0100 Subject: [PATCH 24/60] x86/mcheck: Move CPU_DEAD to hotplug state machine This moves the last piece of the old hotplug notifier code in MCE to the new hotplug state machine. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Borislav Petkov Cc: Tony Luck Cc: rt@linutronix.de Cc: linux-edac@vger.kernel.org Link: http://lkml.kernel.org/r/20161110174447.11848-8-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/kernel/cpu/mcheck/mce.c | 42 ++++++++++++-------------------- include/linux/cpuhotplug.h | 1 + 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c index 78955f501ff2..b888e2f6af41 100644 --- a/arch/x86/kernel/cpu/mcheck/mce.c +++ b/arch/x86/kernel/cpu/mcheck/mce.c @@ -2506,26 +2506,14 @@ static void mce_reenable_cpu(void) } } -/* Get notified when a cpu comes on/off. Be hotplug friendly. */ -static int -mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu) +static int mce_cpu_dead(unsigned int cpu) { - unsigned int cpu = (unsigned long)hcpu; + mce_intel_hcpu_update(cpu); - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_DEAD: - mce_intel_hcpu_update(cpu); - - /* intentionally ignoring frozen here */ - if (!(action & CPU_TASKS_FROZEN)) - cmci_rediscover(); - break; - case CPU_DOWN_PREPARE: - - break; - } - - return NOTIFY_OK; + /* intentionally ignoring frozen here */ + if (!cpuhp_tasks_frozen) + cmci_rediscover(); + return 0; } static int mce_cpu_online(unsigned int cpu) @@ -2556,10 +2544,6 @@ static int mce_cpu_pre_down(unsigned int cpu) return 0; } -static struct notifier_block mce_cpu_notifier = { - .notifier_call = mce_cpu_callback, -}; - static __init void mce_init_banks(void) { int i; @@ -2599,16 +2583,17 @@ static __init int mcheck_init_device(void) if (err) goto err_out_mem; + err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL, + mce_cpu_dead); + if (err) + goto err_out_mem; + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online", mce_cpu_online, mce_cpu_pre_down); if (err < 0) - goto err_out_mem; + goto err_out_online; hp_online = err; - cpu_notifier_register_begin(); - __register_hotcpu_notifier(&mce_cpu_notifier); - cpu_notifier_register_done(); - register_syscore_ops(&mce_syscore_ops); /* register character device /dev/mcelog */ @@ -2622,6 +2607,9 @@ static __init int mcheck_init_device(void) unregister_syscore_ops(&mce_syscore_ops); cpuhp_remove_state(hp_online); +err_out_online: + cpuhp_remove_state(CPUHP_X86_MCE_DEAD); + err_out_mem: free_cpumask_var(mce_device_initialized); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 3410d83cc2e2..79b96f647d64 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -16,6 +16,7 @@ enum cpuhp_state { CPUHP_PERF_SUPERH, CPUHP_X86_HPET_DEAD, CPUHP_X86_APB_DEAD, + CPUHP_X86_MCE_DEAD, CPUHP_VIRT_NET_DEAD, CPUHP_SLUB_DEAD, CPUHP_MM_WRITEBACK_DEAD, From f97960fbddd98e1b7bfba469b8510b49a62b4bc5 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Thu, 17 Nov 2016 17:31:55 +0100 Subject: [PATCH 25/60] kernel/printk: Block cpuhotplug callback when tasks are frozen The recent conversion of the console hotplug notifier to the state machine missed the fact, that the notifier only operated on the non frozen transitions. As a consequence the console_lock/unlock() pair is also invoked during suspend, which results in a lockdep warning. Restore the previous state by making the lock/unlock conditional on !tasks_frozen. Fixes: 90b14889d2f9 ("kernel/printk: Convert to hotplug state machine") Reported-and-tested-by: Borislav Petkov Link: http://lkml.kernel.org/r/alpine.DEB.2.20.1611171729320.3645@nanos Signed-off-by: Thomas Gleixner Cc: Peter Zijlstra Cc: Sebastian Andrzej Siewior --- kernel/printk/printk.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 4487ffcd42d5..37893da9bae8 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -2194,8 +2194,10 @@ void resume_console(void) */ static int console_cpu_notify(unsigned int cpu) { - console_lock(); - console_unlock(); + if (!cpuhp_tasks_frozen) { + console_lock(); + console_unlock(); + } return 0; } From 254fe9c7a4187ebf5ce23e0ca0e9ba98b1dbef18 Mon Sep 17 00:00:00 2001 From: Borislav Petkov Date: Sat, 19 Nov 2016 11:34:02 +0100 Subject: [PATCH 26/60] x86/MCE/AMD: Fix thinko about thresholding_en So adding thresholding_en et al was a good thing for removing the per-CPU thresholding callback, i.e., threshold_cpu_callback. But, in order for it to work and especially that test in mce_threshold_create_device() so that all thresholding banks get properly created and not the whole thing to fail with a NULL ptr dereference at mce_cpu_pre_down() when we offline the CPUs, we need to set the thresholding_en flag *before* we start creating the devices. Yap, it failed because thresholding_en wasn't set at the time we were creating the banks so we didn't create any and then at mce_cpu_pre_down() -> mce_threshold_remove_device() time, we would blow up. And the fix is actually easy: we have thresholding on the system when we have managed to set the thresholding vector to amd_threshold_interrupt() earlier in mce_amd_feature_init() while we were picking apart the thresholding banks and what is set and what not. So let's do that. Signed-off-by: Borislav Petkov Cc: Linus Torvalds Cc: Peter Zijlstra Cc: Sebastian Andrzej Siewior Cc: Thomas Gleixner Cc: Yazen Ghannam Fixes: 4d7b02d58c40 ("x86/mcheck: Split threshold_cpu_callback into two callbacks") Link: http://lkml.kernel.org/r/20161119103402.5227-1-bp@alien8.de Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/mcheck/mce_amd.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c index c33a3ee2e383..4e82e5324063 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c @@ -1134,6 +1134,9 @@ static __init int threshold_init_device(void) { unsigned lcpu = 0; + if (mce_threshold_vector == amd_threshold_interrupt) + thresholding_en = true; + /* to hit CPUs online before the notifier is up */ for_each_online_cpu(lcpu) { int err = mce_threshold_create_device(lcpu); @@ -1142,8 +1145,6 @@ static __init int threshold_init_device(void) return err; } - thresholding_en = true; - return 0; } /* From d6526e73dbbbc4c382c1b16942413eab77ed5e1a Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:22 +0100 Subject: [PATCH 27/60] x86/mce/therm_throt: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Cc: Tony Luck Cc: rt@linuxtronix.de Cc: Borislav Petkov Cc: linux-edac@vger.kernel.org Link: http://lkml.kernel.org/r/20161117183541.8588-2-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/kernel/cpu/mcheck/therm_throt.c | 53 ++++++------------------ include/linux/cpuhotplug.h | 1 + 2 files changed, 13 insertions(+), 41 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c index 6b9dc4d18ccc..7f56620735ca 100644 --- a/arch/x86/kernel/cpu/mcheck/therm_throt.c +++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c @@ -271,58 +271,29 @@ static void thermal_throttle_remove_dev(struct device *dev) } /* Get notified when a cpu comes on/off. Be hotplug friendly. */ -static int -thermal_throttle_cpu_callback(struct notifier_block *nfb, - unsigned long action, - void *hcpu) +static int thermal_throttle_prepare(unsigned int cpu) { - unsigned int cpu = (unsigned long)hcpu; - struct device *dev; - int err = 0; + struct device *dev = get_cpu_device(cpu); - dev = get_cpu_device(cpu); - - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - err = thermal_throttle_add_dev(dev, cpu); - WARN_ON(err); - break; - case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: - case CPU_DEAD: - case CPU_DEAD_FROZEN: - thermal_throttle_remove_dev(dev); - break; - } - return notifier_from_errno(err); + return thermal_throttle_add_dev(dev, cpu); } -static struct notifier_block thermal_throttle_cpu_notifier = +static int thermal_throttle_dead(unsigned int cpu) { - .notifier_call = thermal_throttle_cpu_callback, -}; + struct device *dev = get_cpu_device(cpu); + + thermal_throttle_remove_dev(dev); + return 0; +} static __init int thermal_throttle_init_device(void) { - unsigned int cpu = 0; - int err; - if (!atomic_read(&therm_throt_en)) return 0; - cpu_notifier_register_begin(); - - /* connect live CPUs to sysfs */ - for_each_online_cpu(cpu) { - err = thermal_throttle_add_dev(get_cpu_device(cpu), cpu); - WARN_ON(err); - } - - __register_hotcpu_notifier(&thermal_throttle_cpu_notifier); - cpu_notifier_register_done(); - - return 0; + return cpuhp_setup_state(CPUHP_X86_THERM_PREPARE, "x86/therm:prepare", + thermal_throttle_prepare, + thermal_throttle_dead); } device_initcall(thermal_throttle_init_device); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 79b96f647d64..aea6c6a63139 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -59,6 +59,7 @@ enum cpuhp_state { CPUHP_BLK_MQ_PREPARE, CPUHP_NET_FLOW_PREPARE, CPUHP_TOPOLOGY_PREPARE, + CPUHP_X86_THERM_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, From 33d97302eb502b72b76107d3122afbf18b09b3ec Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 21 Nov 2016 13:15:32 +0100 Subject: [PATCH 28/60] x86/mce/therm_throt: Move hotplug callbacks to online No point to have the sysfs files around before the cpu is online and no point to have them around until the cpu is dead. Get rid of the explicit state. Signed-off-by: Thomas Gleixner Cc: Sebastian Siewior Cc: Tony Luck Cc: Borislav Petkov --- arch/x86/kernel/cpu/mcheck/therm_throt.c | 13 ++++++++----- include/linux/cpuhotplug.h | 1 - 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c index 7f56620735ca..e1d74fd79d5f 100644 --- a/arch/x86/kernel/cpu/mcheck/therm_throt.c +++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c @@ -271,14 +271,14 @@ static void thermal_throttle_remove_dev(struct device *dev) } /* Get notified when a cpu comes on/off. Be hotplug friendly. */ -static int thermal_throttle_prepare(unsigned int cpu) +static int thermal_throttle_online(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); return thermal_throttle_add_dev(dev, cpu); } -static int thermal_throttle_dead(unsigned int cpu) +static int thermal_throttle_offline(unsigned int cpu) { struct device *dev = get_cpu_device(cpu); @@ -288,12 +288,15 @@ static int thermal_throttle_dead(unsigned int cpu) static __init int thermal_throttle_init_device(void) { + int ret; + if (!atomic_read(&therm_throt_en)) return 0; - return cpuhp_setup_state(CPUHP_X86_THERM_PREPARE, "x86/therm:prepare", - thermal_throttle_prepare, - thermal_throttle_dead); + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/therm:online", + thermal_throttle_online, + thermal_throttle_offline); + return ret < 0 ? ret : 0; } device_initcall(thermal_throttle_init_device); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index aea6c6a63139..79b96f647d64 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -59,7 +59,6 @@ enum cpuhp_state { CPUHP_BLK_MQ_PREPARE, CPUHP_NET_FLOW_PREPARE, CPUHP_TOPOLOGY_PREPARE, - CPUHP_X86_THERM_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, From 8c07b494ab2859bc7efb27c40d6faff255f2d2ae Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:23 +0100 Subject: [PATCH 29/60] x86/cpuid: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Cc: rt@linuxtronix.de Link: http://lkml.kernel.org/r/20161117183541.8588-3-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/kernel/cpuid.c | 64 +++++++------------------------------- include/linux/cpuhotplug.h | 1 + 2 files changed, 12 insertions(+), 53 deletions(-) diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c index 2836de390f95..fd85e93e0691 100644 --- a/arch/x86/kernel/cpuid.c +++ b/arch/x86/kernel/cpuid.c @@ -115,7 +115,7 @@ static const struct file_operations cpuid_fops = { .open = cpuid_open, }; -static int cpuid_device_create(int cpu) +static int cpuid_device_create(unsigned int cpu) { struct device *dev; @@ -124,35 +124,12 @@ static int cpuid_device_create(int cpu) return PTR_ERR_OR_ZERO(dev); } -static void cpuid_device_destroy(int cpu) +static int cpuid_device_destroy(unsigned int cpu) { device_destroy(cpuid_class, MKDEV(CPUID_MAJOR, cpu)); + return 0; } -static int cpuid_class_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - int err = 0; - - switch (action) { - case CPU_UP_PREPARE: - err = cpuid_device_create(cpu); - break; - case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: - case CPU_DEAD: - cpuid_device_destroy(cpu); - break; - } - return notifier_from_errno(err); -} - -static struct notifier_block cpuid_class_cpu_notifier = -{ - .notifier_call = cpuid_class_cpu_callback, -}; - static char *cpuid_devnode(struct device *dev, umode_t *mode) { return kasprintf(GFP_KERNEL, "cpu/%u/cpuid", MINOR(dev->devt)); @@ -160,15 +137,13 @@ static char *cpuid_devnode(struct device *dev, umode_t *mode) static int __init cpuid_init(void) { - int i, err = 0; - i = 0; + int err; if (__register_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid", &cpuid_fops)) { printk(KERN_ERR "cpuid: unable to get major %d for cpuid\n", CPUID_MAJOR); - err = -EBUSY; - goto out; + return -EBUSY; } cpuid_class = class_create(THIS_MODULE, "cpuid"); if (IS_ERR(cpuid_class)) { @@ -177,42 +152,25 @@ static int __init cpuid_init(void) } cpuid_class->devnode = cpuid_devnode; - cpu_notifier_register_begin(); - for_each_online_cpu(i) { - err = cpuid_device_create(i); - if (err != 0) - goto out_class; - } - __register_hotcpu_notifier(&cpuid_class_cpu_notifier); - cpu_notifier_register_done(); + err = cpuhp_setup_state(CPUHP_X86_CPUID_PREPARE, "x86/cpuid:prepare", + cpuid_device_create, cpuid_device_destroy); + if (err) + goto out_class; - err = 0; - goto out; + return 0; out_class: - i = 0; - for_each_online_cpu(i) { - cpuid_device_destroy(i); - } - cpu_notifier_register_done(); class_destroy(cpuid_class); out_chrdev: __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid"); -out: return err; } static void __exit cpuid_exit(void) { - int cpu = 0; - - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) - cpuid_device_destroy(cpu); + cpuhp_remove_state(CPUHP_X86_CPUID_PREPARE); class_destroy(cpuid_class); __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid"); - __unregister_hotcpu_notifier(&cpuid_class_cpu_notifier); - cpu_notifier_register_done(); } module_init(cpuid_init); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 79b96f647d64..bc340ef2f200 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -59,6 +59,7 @@ enum cpuhp_state { CPUHP_BLK_MQ_PREPARE, CPUHP_NET_FLOW_PREPARE, CPUHP_TOPOLOGY_PREPARE, + CPUHP_X86_CPUID_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, From ee92be9b0d7ad34cb58b61d5c0933d2e5ff7c31d Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Mon, 21 Nov 2016 13:07:34 +0100 Subject: [PATCH 30/60] x86/cpuid: Move the hotplug callbacks to online No point to have this file around before the cpu is online and no point to have it around until the cpu is dead. Get rid of the explicit state. Signed-off-by: Thomas Gleixner Cc: Sebastian Siewior --- arch/x86/kernel/cpuid.c | 11 ++++++----- include/linux/cpuhotplug.h | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c index fd85e93e0691..373e372a0e1c 100644 --- a/arch/x86/kernel/cpuid.c +++ b/arch/x86/kernel/cpuid.c @@ -45,6 +45,7 @@ #include static struct class *cpuid_class; +static enum cpuhp_state cpuhp_cpuid_state; struct cpuid_regs { u32 eax, ebx, ecx, edx; @@ -152,11 +153,12 @@ static int __init cpuid_init(void) } cpuid_class->devnode = cpuid_devnode; - err = cpuhp_setup_state(CPUHP_X86_CPUID_PREPARE, "x86/cpuid:prepare", + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/cpuid:online", cpuid_device_create, cpuid_device_destroy); - if (err) + if (err < 0) goto out_class; + cpuhp_cpuid_state = err; return 0; out_class: @@ -165,15 +167,14 @@ static int __init cpuid_init(void) __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid"); return err; } +module_init(cpuid_init); static void __exit cpuid_exit(void) { - cpuhp_remove_state(CPUHP_X86_CPUID_PREPARE); + cpuhp_remove_state(cpuhp_cpuid_state); class_destroy(cpuid_class); __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid"); } - -module_init(cpuid_init); module_exit(cpuid_exit); MODULE_AUTHOR("H. Peter Anvin "); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index bc340ef2f200..79b96f647d64 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -59,7 +59,6 @@ enum cpuhp_state { CPUHP_BLK_MQ_PREPARE, CPUHP_NET_FLOW_PREPARE, CPUHP_TOPOLOGY_PREPARE, - CPUHP_X86_CPUID_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, From 8fba38c937cd585bcf562fda8db3decb25509506 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:24 +0100 Subject: [PATCH 31/60] x86/msr: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Move the callbacks to online/offline as there is no point in having the files around before the cpu is online and until its completely gone. [ tglx: Move the callbacks to online/offline ] Signed-off-by: Sebastian Andrzej Siewior Cc: rt@linuxtronix.de Link: http://lkml.kernel.org/r/20161117183541.8588-4-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/kernel/msr.c | 69 ++++++++++--------------------------------- 1 file changed, 15 insertions(+), 54 deletions(-) diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c index 7f3550acde1b..f5e3ff835cc8 100644 --- a/arch/x86/kernel/msr.c +++ b/arch/x86/kernel/msr.c @@ -44,6 +44,7 @@ #include static struct class *msr_class; +static enum cpuhp_state cpuhp_msr_state; static ssize_t msr_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) @@ -180,7 +181,7 @@ static const struct file_operations msr_fops = { .compat_ioctl = msr_ioctl, }; -static int msr_device_create(int cpu) +static int msr_device_create(unsigned int cpu) { struct device *dev; @@ -189,34 +190,12 @@ static int msr_device_create(int cpu) return PTR_ERR_OR_ZERO(dev); } -static void msr_device_destroy(int cpu) +static int msr_device_destroy(unsigned int cpu) { device_destroy(msr_class, MKDEV(MSR_MAJOR, cpu)); + return 0; } -static int msr_class_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - int err = 0; - - switch (action) { - case CPU_UP_PREPARE: - err = msr_device_create(cpu); - break; - case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: - case CPU_DEAD: - msr_device_destroy(cpu); - break; - } - return notifier_from_errno(err); -} - -static struct notifier_block __refdata msr_class_cpu_notifier = { - .notifier_call = msr_class_cpu_callback, -}; - static char *msr_devnode(struct device *dev, umode_t *mode) { return kasprintf(GFP_KERNEL, "cpu/%u/msr", MINOR(dev->devt)); @@ -224,13 +203,11 @@ static char *msr_devnode(struct device *dev, umode_t *mode) static int __init msr_init(void) { - int i, err = 0; - i = 0; + int err; if (__register_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr", &msr_fops)) { pr_err("unable to get major %d for msr\n", MSR_MAJOR); - err = -EBUSY; - goto out; + return -EBUSY; } msr_class = class_create(THIS_MODULE, "msr"); if (IS_ERR(msr_class)) { @@ -239,44 +216,28 @@ static int __init msr_init(void) } msr_class->devnode = msr_devnode; - cpu_notifier_register_begin(); - for_each_online_cpu(i) { - err = msr_device_create(i); - if (err != 0) - goto out_class; - } - __register_hotcpu_notifier(&msr_class_cpu_notifier); - cpu_notifier_register_done(); - - err = 0; - goto out; + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/msr:online", + msr_device_create, msr_device_destroy); + if (err < 0) + goto out_class; + cpuhp_msr_state = err; + return 0; out_class: - i = 0; - for_each_online_cpu(i) - msr_device_destroy(i); - cpu_notifier_register_done(); + cpuhp_remove_state(cpuhp_msr_state); class_destroy(msr_class); out_chrdev: __unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr"); -out: return err; } +module_init(msr_init); static void __exit msr_exit(void) { - int cpu = 0; - - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) - msr_device_destroy(cpu); + cpuhp_remove_state(cpuhp_msr_state); class_destroy(msr_class); __unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr"); - __unregister_hotcpu_notifier(&msr_class_cpu_notifier); - cpu_notifier_register_done(); } - -module_init(msr_init); module_exit(msr_exit) MODULE_AUTHOR("H. Peter Anvin "); From 9c248f8896e6bf0c77abb98bfea8d69b5a7cd11d Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:28 +0100 Subject: [PATCH 32/60] PCI/xgene-msi: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Cc: linux-pci@vger.kernel.org Cc: Duc Dang Cc: rt@linuxtronix.de Cc: Bjorn Helgaas Cc: linux-arm-kernel@lists.infradead.org Link: http://lkml.kernel.org/r/20161117183541.8588-8-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/pci/host/pci-xgene-msi.c | 69 +++++++++----------------------- include/linux/cpuhotplug.h | 1 + 2 files changed, 20 insertions(+), 50 deletions(-) diff --git a/drivers/pci/host/pci-xgene-msi.c b/drivers/pci/host/pci-xgene-msi.c index a6456b578269..1f38d0836751 100644 --- a/drivers/pci/host/pci-xgene-msi.c +++ b/drivers/pci/host/pci-xgene-msi.c @@ -360,16 +360,16 @@ static void xgene_msi_isr(struct irq_desc *desc) chained_irq_exit(chip, desc); } +static enum cpuhp_state pci_xgene_online; + static int xgene_msi_remove(struct platform_device *pdev) { - int virq, i; struct xgene_msi *msi = platform_get_drvdata(pdev); - for (i = 0; i < NR_HW_IRQS; i++) { - virq = msi->msi_groups[i].gic_irq; - if (virq != 0) - irq_set_chained_handler_and_data(virq, NULL, NULL); - } + if (pci_xgene_online) + cpuhp_remove_state(pci_xgene_online); + cpuhp_remove_state(CPUHP_PCI_XGENE_DEAD); + kfree(msi->msi_groups); kfree(msi->bitmap); @@ -427,7 +427,7 @@ static int xgene_msi_hwirq_alloc(unsigned int cpu) return 0; } -static void xgene_msi_hwirq_free(unsigned int cpu) +static int xgene_msi_hwirq_free(unsigned int cpu) { struct xgene_msi *msi = &xgene_msi_ctrl; struct xgene_msi_group *msi_group; @@ -441,33 +441,9 @@ static void xgene_msi_hwirq_free(unsigned int cpu) irq_set_chained_handler_and_data(msi_group->gic_irq, NULL, NULL); } + return 0; } -static int xgene_msi_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned cpu = (unsigned long)hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - xgene_msi_hwirq_alloc(cpu); - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - xgene_msi_hwirq_free(cpu); - break; - default: - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block xgene_msi_cpu_notifier = { - .notifier_call = xgene_msi_cpu_callback, -}; - static const struct of_device_id xgene_msi_match_table[] = { {.compatible = "apm,xgene1-msi"}, {}, @@ -478,7 +454,6 @@ static int xgene_msi_probe(struct platform_device *pdev) struct resource *res; int rc, irq_index; struct xgene_msi *xgene_msi; - unsigned int cpu; int virt_msir; u32 msi_val, msi_idx; @@ -540,28 +515,22 @@ static int xgene_msi_probe(struct platform_device *pdev) } } - cpu_notifier_register_begin(); - - for_each_online_cpu(cpu) - if (xgene_msi_hwirq_alloc(cpu)) { - dev_err(&pdev->dev, "failed to register MSI handlers\n"); - cpu_notifier_register_done(); - goto error; - } - - rc = __register_hotcpu_notifier(&xgene_msi_cpu_notifier); - if (rc) { - dev_err(&pdev->dev, "failed to add CPU MSI notifier\n"); - cpu_notifier_register_done(); - goto error; - } - - cpu_notifier_register_done(); + rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/xgene:online", + xgene_msi_hwirq_alloc, NULL); + if (rc) + goto err_cpuhp; + pci_xgene_online = rc; + rc = cpuhp_setup_state(CPUHP_PCI_XGENE_DEAD, "pci/xgene:dead", NULL, + xgene_msi_hwirq_free); + if (rc) + goto err_cpuhp; dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n"); return 0; +err_cpuhp: + dev_err(&pdev->dev, "failed to add CPU MSI notifier\n"); error: xgene_msi_remove(pdev); return rc; diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 79b96f647d64..94c6a189421f 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -38,6 +38,7 @@ enum cpuhp_state { CPUHP_RADIX_DEAD, CPUHP_PAGE_ALLOC_DEAD, CPUHP_NET_DEV_DEAD, + CPUHP_PCI_XGENE_DEAD, CPUHP_WORKQUEUE_PREP, CPUHP_POWER_NUMA_PREPARE, CPUHP_HRTIMERS_PREPARE, From 948b9c60cb3e044ed4a56fd170ef0d0617ab840f Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:32 +0100 Subject: [PATCH 33/60] watchdog/octeon: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Cc: Wim Van Sebroeck Cc: rt@linuxtronix.de Cc: Guenter Roeck Cc: linux-watchdog@vger.kernel.org Link: http://lkml.kernel.org/r/20161117183541.8588-12-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/watchdog/octeon-wdt-main.c | 62 ++++++++---------------------- 1 file changed, 15 insertions(+), 47 deletions(-) diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c index b55981f88a08..529182d7d8a7 100644 --- a/drivers/watchdog/octeon-wdt-main.c +++ b/drivers/watchdog/octeon-wdt-main.c @@ -374,7 +374,7 @@ void octeon_wdt_nmi_stage3(u64 reg[32]) octeon_wdt_write_string("*** Chip soft reset soon ***\r\n"); } -static void octeon_wdt_disable_interrupt(int cpu) +static int octeon_wdt_cpu_pre_down(unsigned int cpu) { unsigned int core; unsigned int irq; @@ -392,9 +392,10 @@ static void octeon_wdt_disable_interrupt(int cpu) cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64); free_irq(irq, octeon_wdt_poke_irq); + return 0; } -static void octeon_wdt_setup_interrupt(int cpu) +static int octeon_wdt_cpu_online(unsigned int cpu) { unsigned int core; unsigned int irq; @@ -424,25 +425,8 @@ static void octeon_wdt_setup_interrupt(int cpu) ciu_wdog.s.len = timeout_cnt; ciu_wdog.s.mode = 3; /* 3 = Interrupt + NMI + Soft-Reset */ cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64); -} -static int octeon_wdt_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_DOWN_PREPARE: - octeon_wdt_disable_interrupt(cpu); - break; - case CPU_ONLINE: - case CPU_DOWN_FAILED: - octeon_wdt_setup_interrupt(cpu); - break; - default: - break; - } - return NOTIFY_OK; + return 0; } static int octeon_wdt_ping(struct watchdog_device __always_unused *wdog) @@ -531,10 +515,6 @@ static int octeon_wdt_stop(struct watchdog_device *wdog) return 0; } -static struct notifier_block octeon_wdt_cpu_notifier = { - .notifier_call = octeon_wdt_cpu_callback, -}; - static const struct watchdog_info octeon_wdt_info = { .options = WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING, .identity = "OCTEON", @@ -553,6 +533,7 @@ static struct watchdog_device octeon_wdt = { .ops = &octeon_wdt_ops, }; +static enum cpuhp_state octeon_wdt_online; /** * Module/ driver initialization. * @@ -562,7 +543,6 @@ static int __init octeon_wdt_init(void) { int i; int ret; - int cpu; u64 *ptr; /* @@ -610,14 +590,16 @@ static int __init octeon_wdt_init(void) cpumask_clear(&irq_enabled_cpus); - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) - octeon_wdt_setup_interrupt(cpu); - - __register_hotcpu_notifier(&octeon_wdt_cpu_notifier); - cpu_notifier_register_done(); - + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "watchdog/octeon:online", + octeon_wdt_cpu_online, octeon_wdt_cpu_pre_down); + if (ret < 0) + goto err; + octeon_wdt_online = ret; return 0; +err: + cvmx_write_csr(CVMX_MIO_BOOT_LOC_CFGX(0), 0); + watchdog_unregister_device(&octeon_wdt); + return ret; } /** @@ -625,22 +607,8 @@ static int __init octeon_wdt_init(void) */ static void __exit octeon_wdt_cleanup(void) { - int cpu; - watchdog_unregister_device(&octeon_wdt); - - cpu_notifier_register_begin(); - __unregister_hotcpu_notifier(&octeon_wdt_cpu_notifier); - - for_each_online_cpu(cpu) { - int core = cpu2core(cpu); - /* Disable the watchdog */ - cvmx_write_csr(CVMX_CIU_WDOGX(core), 0); - /* Free the interrupt handler */ - free_irq(OCTEON_IRQ_WDOG0 + core, octeon_wdt_poke_irq); - } - - cpu_notifier_register_done(); + cpuhp_remove_state(octeon_wdt_online); /* * Disable the boot-bus memory, the code it points to is soon From 38b482929e8f96dfe459d2ef757d0a5c3a74cea3 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:33 +0100 Subject: [PATCH 34/60] net/iucv: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. The smp function calls in the online/downprep callbacks are not required as the callback is guaranteed to be invoked on the upcoming/outgoing cpu. Signed-off-by: Sebastian Andrzej Siewior Cc: "David S. Miller" Cc: linux-s390@vger.kernel.org Cc: netdev@vger.kernel.org Cc: Ursula Braun Cc: rt@linuxtronix.de Link: http://lkml.kernel.org/r/20161117183541.8588-13-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + net/iucv/iucv.c | 118 ++++++++++++++----------------------- 2 files changed, 45 insertions(+), 74 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 94c6a189421f..12bbcf3ded70 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -60,6 +60,7 @@ enum cpuhp_state { CPUHP_BLK_MQ_PREPARE, CPUHP_NET_FLOW_PREPARE, CPUHP_TOPOLOGY_PREPARE, + CPUHP_NET_IUCV_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index 88a2a3ba4212..f0d6afc5d4a9 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -639,7 +639,7 @@ static void iucv_disable(void) put_online_cpus(); } -static void free_iucv_data(int cpu) +static int iucv_cpu_dead(unsigned int cpu) { kfree(iucv_param_irq[cpu]); iucv_param_irq[cpu] = NULL; @@ -647,9 +647,10 @@ static void free_iucv_data(int cpu) iucv_param[cpu] = NULL; kfree(iucv_irq_data[cpu]); iucv_irq_data[cpu] = NULL; + return 0; } -static int alloc_iucv_data(int cpu) +static int iucv_cpu_prepare(unsigned int cpu) { /* Note: GFP_DMA used to get memory below 2G */ iucv_irq_data[cpu] = kmalloc_node(sizeof(struct iucv_irq_data), @@ -671,58 +672,38 @@ static int alloc_iucv_data(int cpu) return 0; out_free: - free_iucv_data(cpu); + iucv_cpu_dead(cpu); return -ENOMEM; } -static int iucv_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) +static int iucv_cpu_online(unsigned int cpu) { - cpumask_t cpumask; - long cpu = (long) hcpu; - - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - if (alloc_iucv_data(cpu)) - return notifier_from_errno(-ENOMEM); - break; - case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: - case CPU_DEAD: - case CPU_DEAD_FROZEN: - free_iucv_data(cpu); - break; - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - case CPU_DOWN_FAILED: - case CPU_DOWN_FAILED_FROZEN: - if (!iucv_path_table) - break; - smp_call_function_single(cpu, iucv_declare_cpu, NULL, 1); - break; - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - if (!iucv_path_table) - break; - cpumask_copy(&cpumask, &iucv_buffer_cpumask); - cpumask_clear_cpu(cpu, &cpumask); - if (cpumask_empty(&cpumask)) - /* Can't offline last IUCV enabled cpu. */ - return notifier_from_errno(-EINVAL); - smp_call_function_single(cpu, iucv_retrieve_cpu, NULL, 1); - if (cpumask_empty(&iucv_irq_cpumask)) - smp_call_function_single( - cpumask_first(&iucv_buffer_cpumask), - iucv_allow_cpu, NULL, 1); - break; - } - return NOTIFY_OK; + if (!iucv_path_table) + return 0; + iucv_declare_cpu(NULL); + return 0; } -static struct notifier_block __refdata iucv_cpu_notifier = { - .notifier_call = iucv_cpu_notify, -}; +static int iucv_cpu_down_prep(unsigned int cpu) +{ + cpumask_t cpumask; + + if (!iucv_path_table) + return 0; + + cpumask_copy(&cpumask, &iucv_buffer_cpumask); + cpumask_clear_cpu(cpu, &cpumask); + if (cpumask_empty(&cpumask)) + /* Can't offline last IUCV enabled cpu. */ + return -EINVAL; + + iucv_retrieve_cpu(NULL); + if (!cpumask_empty(&iucv_irq_cpumask)) + return 0; + smp_call_function_single(cpumask_first(&iucv_buffer_cpumask), + iucv_allow_cpu, NULL, 1); + return 0; +} /** * iucv_sever_pathid @@ -2027,6 +2008,7 @@ struct iucv_interface iucv_if = { }; EXPORT_SYMBOL(iucv_if); +static enum cpuhp_state iucv_online; /** * iucv_init * @@ -2035,7 +2017,6 @@ EXPORT_SYMBOL(iucv_if); static int __init iucv_init(void) { int rc; - int cpu; if (!MACHINE_IS_VM) { rc = -EPROTONOSUPPORT; @@ -2054,23 +2035,19 @@ static int __init iucv_init(void) goto out_int; } - cpu_notifier_register_begin(); - - for_each_online_cpu(cpu) { - if (alloc_iucv_data(cpu)) { - rc = -ENOMEM; - goto out_free; - } - } - rc = __register_hotcpu_notifier(&iucv_cpu_notifier); + rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare", + iucv_cpu_prepare, iucv_cpu_dead); if (rc) goto out_free; - - cpu_notifier_register_done(); + rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online", + iucv_cpu_online, iucv_cpu_down_prep); + if (rc < 0) + goto out_free; + iucv_online = rc; rc = register_reboot_notifier(&iucv_reboot_notifier); if (rc) - goto out_cpu; + goto out_free; ASCEBC(iucv_error_no_listener, 16); ASCEBC(iucv_error_no_memory, 16); ASCEBC(iucv_error_pathid, 16); @@ -2084,14 +2061,10 @@ static int __init iucv_init(void) out_reboot: unregister_reboot_notifier(&iucv_reboot_notifier); -out_cpu: - cpu_notifier_register_begin(); - __unregister_hotcpu_notifier(&iucv_cpu_notifier); out_free: - for_each_possible_cpu(cpu) - free_iucv_data(cpu); - - cpu_notifier_register_done(); + if (iucv_online) + cpuhp_remove_state(iucv_online); + cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE); root_device_unregister(iucv_root); out_int: @@ -2110,7 +2083,6 @@ static int __init iucv_init(void) static void __exit iucv_exit(void) { struct iucv_irq_list *p, *n; - int cpu; spin_lock_irq(&iucv_queue_lock); list_for_each_entry_safe(p, n, &iucv_task_queue, list) @@ -2119,11 +2091,9 @@ static void __exit iucv_exit(void) kfree(p); spin_unlock_irq(&iucv_queue_lock); unregister_reboot_notifier(&iucv_reboot_notifier); - cpu_notifier_register_begin(); - __unregister_hotcpu_notifier(&iucv_cpu_notifier); - for_each_possible_cpu(cpu) - free_iucv_data(cpu); - cpu_notifier_register_done(); + + cpuhp_remove_state_nocalls(iucv_online); + cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE); root_device_unregister(iucv_root); bus_unregister(&iucv_bus); unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt); From 31eff2434db542763a00074a8368d7bd78d14ea1 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:34 +0100 Subject: [PATCH 35/60] sched/nohz: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Cc: rt@linuxtronix.de Link: http://lkml.kernel.org/r/20161117183541.8588-14-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- kernel/time/tick-sched.c | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 3bcb61b52f6c..71496a20e670 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -390,24 +390,16 @@ static int __init tick_nohz_full_setup(char *str) } __setup("nohz_full=", tick_nohz_full_setup); -static int tick_nohz_cpu_down_callback(struct notifier_block *nfb, - unsigned long action, - void *hcpu) +static int tick_nohz_cpu_down(unsigned int cpu) { - unsigned int cpu = (unsigned long)hcpu; - - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_DOWN_PREPARE: - /* - * The boot CPU handles housekeeping duty (unbound timers, - * workqueues, timekeeping, ...) on behalf of full dynticks - * CPUs. It must remain online when nohz full is enabled. - */ - if (tick_nohz_full_running && tick_do_timer_cpu == cpu) - return NOTIFY_BAD; - break; - } - return NOTIFY_OK; + /* + * The boot CPU handles housekeeping duty (unbound timers, + * workqueues, timekeeping, ...) on behalf of full dynticks + * CPUs. It must remain online when nohz full is enabled. + */ + if (tick_nohz_full_running && tick_do_timer_cpu == cpu) + return -EBUSY; + return 0; } static int tick_nohz_init_all(void) @@ -428,7 +420,7 @@ static int tick_nohz_init_all(void) void __init tick_nohz_init(void) { - int cpu; + int cpu, ret; if (!tick_nohz_full_running) { if (tick_nohz_init_all() < 0) @@ -469,7 +461,10 @@ void __init tick_nohz_init(void) for_each_cpu(cpu, tick_nohz_full_mask) context_tracking_cpu_set(cpu); - cpu_notifier(tick_nohz_cpu_down_callback, 0); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "kernel/nohz:predown", NULL, + tick_nohz_cpu_down); + WARN_ON(ret < 0); pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n", cpumask_pr_args(tick_nohz_full_mask)); From a3c9b14f6f151ee4c2a119fab14f9a60d1684d60 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:35 +0100 Subject: [PATCH 36/60] arm/bL_switcher: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Cc: rt@linuxtronix.de Cc: linux-arm-kernel@lists.infradead.org Cc: Russell King Link: http://lkml.kernel.org/r/20161117183541.8588-15-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/arm/common/bL_switcher.c | 34 ++++++++++++++++++++-------------- include/linux/cpuhotplug.h | 1 + 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c index 37dc0fe1093f..46730017b3c5 100644 --- a/arch/arm/common/bL_switcher.c +++ b/arch/arm/common/bL_switcher.c @@ -757,19 +757,18 @@ EXPORT_SYMBOL_GPL(bL_switcher_put_enabled); * while the switcher is active. * We're just not ready to deal with that given the trickery involved. */ -static int bL_switcher_hotplug_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) +static int bL_switcher_cpu_pre(unsigned int cpu) { - if (bL_switcher_active) { - int pairing = bL_switcher_cpu_pairing[(unsigned long)hcpu]; - switch (action & 0xf) { - case CPU_UP_PREPARE: - case CPU_DOWN_PREPARE: - if (pairing == -1) - return NOTIFY_BAD; - } - } - return NOTIFY_DONE; + int pairing; + + if (!bL_switcher_active) + return 0; + + pairing = bL_switcher_cpu_pairing[cpu]; + + if (pairing == -1) + return -EINVAL; + return 0; } static bool no_bL_switcher; @@ -782,8 +781,15 @@ static int __init bL_switcher_init(void) if (!mcpm_is_available()) return -ENODEV; - cpu_notifier(bL_switcher_hotplug_callback, 0); - + cpuhp_setup_state_nocalls(CPUHP_ARM_BL_PREPARE, "arm/bl:prepare", + bL_switcher_cpu_pre, NULL); + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/bl:predown", + NULL, bL_switcher_cpu_pre); + if (ret < 0) { + cpuhp_remove_state_nocalls(CPUHP_ARM_BL_PREPARE); + pr_err("bL_switcher: Failed to allocate a hotplug state\n"); + return ret; + } if (!no_bL_switcher) { ret = bL_switcher_enable(); if (ret) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 12bbcf3ded70..e3771fb959c0 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -61,6 +61,7 @@ enum cpuhp_state { CPUHP_NET_FLOW_PREPARE, CPUHP_TOPOLOGY_PREPARE, CPUHP_NET_IUCV_PREPARE, + CPUHP_ARM_BL_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, From 9b377e217f0bd07f972d89ed0963df92818beffd Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:36 +0100 Subject: [PATCH 37/60] ARM/hw_breakpoint: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. smp_call_function_single() has been removed because the function is already invoked on the target CPU. [ tglx: Added protection agaist hotplug back according to discussion with Will ] Signed-off-by: Sebastian Andrzej Siewior Cc: Mark Rutland Cc: rt@linuxtronix.de Cc: Will Deacon Cc: Russell King Cc: linux-arm-kernel@lists.infradead.org Link: http://lkml.kernel.org/r/20161117183541.8588-16-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/arm/kernel/hw_breakpoint.c | 47 +++++++++++++++------------------ 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index b8df45883cf7..188180b5523d 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -925,9 +925,9 @@ static bool core_has_os_save_restore(void) } } -static void reset_ctrl_regs(void *unused) +static void reset_ctrl_regs(unsigned int cpu) { - int i, raw_num_brps, err = 0, cpu = smp_processor_id(); + int i, raw_num_brps, err = 0; u32 val; /* @@ -1020,25 +1020,20 @@ static void reset_ctrl_regs(void *unused) cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu)); } -static int dbg_reset_notify(struct notifier_block *self, - unsigned long action, void *cpu) +static int dbg_reset_online(unsigned int cpu) { - if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE) - smp_call_function_single((int)cpu, reset_ctrl_regs, NULL, 1); - - return NOTIFY_OK; + local_irq_disable(); + reset_ctrl_regs(cpu); + local_irq_enable(); + return 0; } -static struct notifier_block dbg_reset_nb = { - .notifier_call = dbg_reset_notify, -}; - #ifdef CONFIG_CPU_PM static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action, void *v) { if (action == CPU_PM_EXIT) - reset_ctrl_regs(NULL); + reset_ctrl_regs(smp_processor_id()); return NOTIFY_OK; } @@ -1059,6 +1054,8 @@ static inline void pm_init(void) static int __init arch_hw_breakpoint_init(void) { + int ret; + debug_arch = get_debug_arch(); if (!debug_arch_supported()) { @@ -1072,25 +1069,28 @@ static int __init arch_hw_breakpoint_init(void) core_num_brps = get_num_brps(); core_num_wrps = get_num_wrps(); - cpu_notifier_register_begin(); - /* * We need to tread carefully here because DBGSWENABLE may be * driven low on this core and there isn't an architected way to * determine that. */ + get_online_cpus(); register_undef_hook(&debug_reg_hook); /* - * Reset the breakpoint resources. We assume that a halting - * debugger will leave the world in a nice state for us. + * Register CPU notifier which resets the breakpoint resources. We + * assume that a halting debugger will leave the world in a nice state + * for us. */ - on_each_cpu(reset_ctrl_regs, NULL, 1); + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "arm/hw_breakpoint:online", + dbg_reset_online, NULL); unregister_undef_hook(&debug_reg_hook); - if (!cpumask_empty(&debug_err_mask)) { + if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) { core_num_brps = 0; core_num_wrps = 0; - cpu_notifier_register_done(); + if (ret > 0) + cpuhp_remove_state_nocalls(ret); + put_online_cpus(); return 0; } @@ -1108,12 +1108,9 @@ static int __init arch_hw_breakpoint_init(void) TRAP_HWBKPT, "watchpoint debug exception"); hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT, "breakpoint debug exception"); + put_online_cpus(); - /* Register hotplug and PM notifiers. */ - __register_cpu_notifier(&dbg_reset_nb); - - cpu_notifier_register_done(); - + /* Register PM notifiers. */ pm_init(); return 0; } From 977ab257a2b327f161728ab08bc618d770cc92ad Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:37 +0100 Subject: [PATCH 38/60] powerpc/sysfs: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. The previous convention of keeping the files around until the CPU is dead has not been preserved as there is no point to keep them available when the cpu is going down. This makes the hotplug call symmetric. Signed-off-by: Sebastian Andrzej Siewior Cc: Benjamin Herrenschmidt Cc: rt@linuxtronix.de Cc: Paul Mackerras Cc: Michael Ellerman Cc: linuxppc-dev@lists.ozlabs.org Link: http://lkml.kernel.org/r/20161117183541.8588-17-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/powerpc/kernel/sysfs.c | 50 ++++++++----------------------------- 1 file changed, 10 insertions(+), 40 deletions(-) diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c index c4f1d1f7bae0..c1fb255a60d6 100644 --- a/arch/powerpc/kernel/sysfs.c +++ b/arch/powerpc/kernel/sysfs.c @@ -703,7 +703,7 @@ static struct device_attribute pa6t_attrs[] = { #endif /* HAS_PPC_PMC_PA6T */ #endif /* HAS_PPC_PMC_CLASSIC */ -static void register_cpu_online(unsigned int cpu) +static int register_cpu_online(unsigned int cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); struct device *s = &c->dev; @@ -782,11 +782,12 @@ static void register_cpu_online(unsigned int cpu) } #endif cacheinfo_cpu_online(cpu); + return 0; } -#ifdef CONFIG_HOTPLUG_CPU -static void unregister_cpu_online(unsigned int cpu) +static int unregister_cpu_online(unsigned int cpu) { +#ifdef CONFIG_HOTPLUG_CPU struct cpu *c = &per_cpu(cpu_devices, cpu); struct device *s = &c->dev; struct device_attribute *attrs, *pmc_attrs; @@ -863,6 +864,8 @@ static void unregister_cpu_online(unsigned int cpu) } #endif cacheinfo_cpu_offline(cpu); +#endif /* CONFIG_HOTPLUG_CPU */ + return 0; } #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE @@ -883,32 +886,6 @@ ssize_t arch_cpu_release(const char *buf, size_t count) } #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */ -#endif /* CONFIG_HOTPLUG_CPU */ - -static int sysfs_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned int)(long)hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - register_cpu_online(cpu); - break; -#ifdef CONFIG_HOTPLUG_CPU - case CPU_DEAD: - case CPU_DEAD_FROZEN: - unregister_cpu_online(cpu); - break; -#endif - } - return NOTIFY_OK; -} - -static struct notifier_block sysfs_cpu_nb = { - .notifier_call = sysfs_cpu_notify, -}; - static DEFINE_MUTEX(cpu_mutex); int cpu_add_dev_attr(struct device_attribute *attr) @@ -1023,12 +1000,10 @@ static DEVICE_ATTR(physical_id, 0444, show_physical_id, NULL); static int __init topology_init(void) { - int cpu; + int cpu, r; register_nodes(); - cpu_notifier_register_begin(); - for_each_possible_cpu(cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); @@ -1047,15 +1022,10 @@ static int __init topology_init(void) device_create_file(&c->dev, &dev_attr_physical_id); } - - if (cpu_online(cpu)) - register_cpu_online(cpu); } - - __register_cpu_notifier(&sysfs_cpu_nb); - - cpu_notifier_register_done(); - + r = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powerpc/topology:online", + register_cpu_online, unregister_cpu_online); + WARN_ON(r < 0); #ifdef CONFIG_PPC64 sysfs_create_dscr_default(); #endif /* CONFIG_PPC64 */ From e5355cd6e78c096b66c26de8c9e5680ddba71c1f Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:38 +0100 Subject: [PATCH 39/60] sparc/sysfs: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. The previous convention of keeping the files around until the CPU is dead has not been preserved as there is no point to keep them available when the cpu is going down. This makes the hotplug call symmetric. Signed-off-by: Sebastian Andrzej Siewior Acked-by: "David S. Miller" Cc: sparclinux@vger.kernel.org Cc: rt@linuxtronix.de Link: http://lkml.kernel.org/r/20161117183541.8588-18-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/sparc/kernel/sysfs.c | 45 ++++++++------------------------------- 1 file changed, 9 insertions(+), 36 deletions(-) diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c index fa8e21abb5e0..4808b6d23455 100644 --- a/arch/sparc/kernel/sysfs.c +++ b/arch/sparc/kernel/sysfs.c @@ -221,7 +221,7 @@ static struct device_attribute cpu_core_attrs[] = { static DEFINE_PER_CPU(struct cpu, cpu_devices); -static void register_cpu_online(unsigned int cpu) +static int register_cpu_online(unsigned int cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); struct device *s = &c->dev; @@ -231,11 +231,12 @@ static void register_cpu_online(unsigned int cpu) device_create_file(s, &cpu_core_attrs[i]); register_mmu_stats(s); + return 0; } -#ifdef CONFIG_HOTPLUG_CPU -static void unregister_cpu_online(unsigned int cpu) +static int unregister_cpu_online(unsigned int cpu) { +#ifdef CONFIG_HOTPLUG_CPU struct cpu *c = &per_cpu(cpu_devices, cpu); struct device *s = &c->dev; int i; @@ -243,33 +244,10 @@ static void unregister_cpu_online(unsigned int cpu) unregister_mmu_stats(s); for (i = 0; i < ARRAY_SIZE(cpu_core_attrs); i++) device_remove_file(s, &cpu_core_attrs[i]); -} #endif - -static int sysfs_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned int)(long)hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - register_cpu_online(cpu); - break; -#ifdef CONFIG_HOTPLUG_CPU - case CPU_DEAD: - case CPU_DEAD_FROZEN: - unregister_cpu_online(cpu); - break; -#endif - } - return NOTIFY_OK; + return 0; } -static struct notifier_block sysfs_cpu_nb = { - .notifier_call = sysfs_cpu_notify, -}; - static void __init check_mmu_stats(void) { unsigned long dummy1, err; @@ -294,26 +272,21 @@ static void register_nodes(void) static int __init topology_init(void) { - int cpu; + int cpu, ret; register_nodes(); check_mmu_stats(); - cpu_notifier_register_begin(); - for_each_possible_cpu(cpu) { struct cpu *c = &per_cpu(cpu_devices, cpu); register_cpu(c, cpu); - if (cpu_online(cpu)) - register_cpu_online(cpu); } - __register_cpu_notifier(&sysfs_cpu_nb); - - cpu_notifier_register_done(); - + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "sparc/topology:online", + register_cpu_online, unregister_cpu_online); + WARN_ON(ret < 0); return 0; } From 08ed487c819d285c3f6ceafd156094102acdfec2 Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Thu, 17 Nov 2016 19:35:39 +0100 Subject: [PATCH 40/60] x86/oprofile/nmi: Remove superfluous smp_function_call_single() Since commit 1cf4f629d9d2 ("cpu/hotplug: Move online calls to hotplugged cpu") the CPU_ONLINE and CPU_DOWN_PREPARE notifiers are always run on the hot plugged CPU, and as of commit 3b9d6da67e11 ("cpu/hotplug: Fix rollback during error-out in __cpu_disable()") the CPU_DOWN_FAILED notifier also runs on the hot plugged CPU. This patch converts the SMP functional calls into direct calls. smp_call_function_single() executes the function with interrupts disabled. This calling convention is preserved. Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior Cc: Robert Richter Cc: rt@linuxtronix.de Cc: oprofile-list@lists.sf.net Link: http://lkml.kernel.org/r/20161117183541.8588-19-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/oprofile/nmi_int.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c index 28c04123b6dd..c39172cd6c87 100644 --- a/arch/x86/oprofile/nmi_int.c +++ b/arch/x86/oprofile/nmi_int.c @@ -387,20 +387,24 @@ static void nmi_cpu_shutdown(void *dummy) nmi_cpu_restore_registers(msrs); } -static void nmi_cpu_up(void *dummy) +static void nmi_cpu_up(void) { + local_irq_disable(); if (nmi_enabled) - nmi_cpu_setup(dummy); + nmi_cpu_setup(NULL); if (ctr_running) - nmi_cpu_start(dummy); + nmi_cpu_start(NULL); + local_irq_enable(); } -static void nmi_cpu_down(void *dummy) +static void nmi_cpu_down(void) { + local_irq_disable(); if (ctr_running) - nmi_cpu_stop(dummy); + nmi_cpu_stop(NULL); if (nmi_enabled) - nmi_cpu_shutdown(dummy); + nmi_cpu_shutdown(NULL); + local_irq_enable(); } static int nmi_create_files(struct dentry *root) @@ -436,15 +440,13 @@ static int nmi_create_files(struct dentry *root) static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action, void *data) { - int cpu = (unsigned long)data; - switch (action & ~CPU_TASKS_FROZEN) { case CPU_DOWN_FAILED: case CPU_ONLINE: - smp_call_function_single(cpu, nmi_cpu_up, NULL, 0); + nmi_cpu_up(); break; case CPU_DOWN_PREPARE: - smp_call_function_single(cpu, nmi_cpu_down, NULL, 1); + nmi_cpu_down(); break; } return NOTIFY_DONE; From 89666c50472263fba97b7edbfd2a642d1d9d6f74 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:40 +0100 Subject: [PATCH 41/60] x86/oprofile/nmi: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Cc: Robert Richter Cc: oprofile-list@lists.sf.net Cc: rt@linuxtronix.de Link: http://lkml.kernel.org/r/20161117183541.8588-20-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/oprofile/nmi_int.c | 61 +++++++++++-------------------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c index c39172cd6c87..ffdbc4836b4f 100644 --- a/arch/x86/oprofile/nmi_int.c +++ b/arch/x86/oprofile/nmi_int.c @@ -339,10 +339,11 @@ static int allocate_msrs(void) return 0; } -static void nmi_cpu_setup(void *dummy) +static void nmi_cpu_setup(void) { int cpu = smp_processor_id(); struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu); + nmi_cpu_save_registers(msrs); raw_spin_lock(&oprofilefs_lock); model->setup_ctrs(model, msrs); @@ -369,7 +370,7 @@ static void nmi_cpu_restore_registers(struct op_msrs *msrs) } } -static void nmi_cpu_shutdown(void *dummy) +static void nmi_cpu_shutdown(void) { unsigned int v; int cpu = smp_processor_id(); @@ -387,24 +388,26 @@ static void nmi_cpu_shutdown(void *dummy) nmi_cpu_restore_registers(msrs); } -static void nmi_cpu_up(void) +static int nmi_cpu_online(unsigned int cpu) { local_irq_disable(); if (nmi_enabled) - nmi_cpu_setup(NULL); + nmi_cpu_setup(); if (ctr_running) nmi_cpu_start(NULL); local_irq_enable(); + return 0; } -static void nmi_cpu_down(void) +static int nmi_cpu_down_prep(unsigned int cpu) { local_irq_disable(); if (ctr_running) nmi_cpu_stop(NULL); if (nmi_enabled) - nmi_cpu_shutdown(NULL); + nmi_cpu_shutdown(); local_irq_enable(); + return 0; } static int nmi_create_files(struct dentry *root) @@ -437,24 +440,7 @@ static int nmi_create_files(struct dentry *root) return 0; } -static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action, - void *data) -{ - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_DOWN_FAILED: - case CPU_ONLINE: - nmi_cpu_up(); - break; - case CPU_DOWN_PREPARE: - nmi_cpu_down(); - break; - } - return NOTIFY_DONE; -} - -static struct notifier_block oprofile_cpu_nb = { - .notifier_call = oprofile_cpu_notifier -}; +static enum cpuhp_state cpuhp_nmi_online; static int nmi_setup(void) { @@ -497,20 +483,17 @@ static int nmi_setup(void) if (err) goto fail; - cpu_notifier_register_begin(); - - /* Use get/put_online_cpus() to protect 'nmi_enabled' */ - get_online_cpus(); nmi_enabled = 1; /* make nmi_enabled visible to the nmi handler: */ smp_mb(); - on_each_cpu(nmi_cpu_setup, NULL, 1); - __register_cpu_notifier(&oprofile_cpu_nb); - put_online_cpus(); - - cpu_notifier_register_done(); - + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/oprofile:online", + nmi_cpu_online, nmi_cpu_down_prep); + if (err < 0) + goto fail_nmi; + cpuhp_nmi_online = err; return 0; +fail_nmi: + unregister_nmi_handler(NMI_LOCAL, "oprofile"); fail: free_msrs(); return err; @@ -520,17 +503,9 @@ static void nmi_shutdown(void) { struct op_msrs *msrs; - cpu_notifier_register_begin(); - - /* Use get/put_online_cpus() to protect 'nmi_enabled' & 'ctr_running' */ - get_online_cpus(); - on_each_cpu(nmi_cpu_shutdown, NULL, 1); + cpuhp_remove_state(cpuhp_nmi_online); nmi_enabled = 0; ctr_running = 0; - __unregister_cpu_notifier(&oprofile_cpu_nb); - put_online_cpus(); - - cpu_notifier_register_done(); /* make variables visible to the nmi handler: */ smp_mb(); From c8b877a5e58a132c5efb05f0c404585b9789fe5c Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 17 Nov 2016 19:35:41 +0100 Subject: [PATCH 42/60] x86/pci/amd-bus: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. The smp_call_function_single() is dropped because the ONLINE callback is invoked on the target CPU since commit 1cf4f629d9d2 ("cpu/hotplug: Move online calls to hotplugged cpu"). smp_call_function_single() invokes the invoked function with interrupts disabled, but this calling convention is not preserved as the MSR is not modified by anything else than this code. Signed-off-by: Sebastian Andrzej Siewior Cc: Bjorn Helgaas Cc: linux-pci@vger.kernel.org Cc: rt@linuxtronix.de Link: http://lkml.kernel.org/r/20161117183541.8588-21-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/x86/pci/amd_bus.c | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c index c20d2cc7ef64..ae387e5ee6f7 100644 --- a/arch/x86/pci/amd_bus.c +++ b/arch/x86/pci/amd_bus.c @@ -327,35 +327,18 @@ static int __init early_root_info_init(void) #define ENABLE_CF8_EXT_CFG (1ULL << 46) -static void enable_pci_io_ecs(void *unused) +static int amd_bus_cpu_online(unsigned int cpu) { u64 reg; + rdmsrl(MSR_AMD64_NB_CFG, reg); if (!(reg & ENABLE_CF8_EXT_CFG)) { reg |= ENABLE_CF8_EXT_CFG; wrmsrl(MSR_AMD64_NB_CFG, reg); } + return 0; } -static int amd_cpu_notify(struct notifier_block *self, unsigned long action, - void *hcpu) -{ - int cpu = (long)hcpu; - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - smp_call_function_single(cpu, enable_pci_io_ecs, NULL, 0); - break; - default: - break; - } - return NOTIFY_OK; -} - -static struct notifier_block amd_cpu_notifier = { - .notifier_call = amd_cpu_notify, -}; - static void __init pci_enable_pci_io_ecs(void) { #ifdef CONFIG_AMD_NB @@ -385,7 +368,7 @@ static void __init pci_enable_pci_io_ecs(void) static int __init pci_io_ecs_init(void) { - int cpu; + int ret; /* assume all cpus from fam10h have IO ECS */ if (boot_cpu_data.x86 < 0x10) @@ -395,12 +378,9 @@ static int __init pci_io_ecs_init(void) if (early_pci_allowed()) pci_enable_pci_io_ecs(); - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) - amd_cpu_notify(&amd_cpu_notifier, (unsigned long)CPU_ONLINE, - (void *)(long)cpu); - __register_cpu_notifier(&amd_cpu_notifier); - cpu_notifier_register_done(); + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/amd_bus:online", + amd_bus_cpu_online, NULL); + WARN_ON(ret < 0); pci_probe |= PCI_HAS_IO_ECS; From 9c6bafab03dec222237b6eb8b5adf5c18ec76264 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Thu, 24 Nov 2016 17:10:13 +0100 Subject: [PATCH 43/60] net/iucv: Use explicit clean up labels in iucv_init() Ursula suggested to use explicit labels for clean up in the error path instead of one `out_free' label, which handles multiple exits, introduced in commit 38b482929e8f ("net/iucv: Convert to hotplug state machine"). Suggested-by: Ursula Braun Signed-off-by: Sebastian Andrzej Siewior Cc: linux-s390@vger.kernel.org Cc: netdev@vger.kernel.org Cc: rt@linutronix.de Cc: "David S. Miller" Link: http://lkml.kernel.org/r/20161124161013.dukr42y2nwscosk6@linutronix.de Signed-off-by: Thomas Gleixner --- net/iucv/iucv.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index f0d6afc5d4a9..8f7ef167c45a 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -2038,16 +2038,16 @@ static int __init iucv_init(void) rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare", iucv_cpu_prepare, iucv_cpu_dead); if (rc) - goto out_free; + goto out_dev; rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online", iucv_cpu_online, iucv_cpu_down_prep); if (rc < 0) - goto out_free; + goto out_prep; iucv_online = rc; rc = register_reboot_notifier(&iucv_reboot_notifier); if (rc) - goto out_free; + goto out_remove_hp; ASCEBC(iucv_error_no_listener, 16); ASCEBC(iucv_error_no_memory, 16); ASCEBC(iucv_error_pathid, 16); @@ -2061,11 +2061,11 @@ static int __init iucv_init(void) out_reboot: unregister_reboot_notifier(&iucv_reboot_notifier); -out_free: - if (iucv_online) - cpuhp_remove_state(iucv_online); +out_remove_hp: + cpuhp_remove_state(iucv_online); +out_prep: cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE); - +out_dev: root_device_unregister(iucv_root); out_int: unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt); From 14660b7ea3ca628410bb999d53926ca77973892b Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Sun, 27 Nov 2016 00:13:33 +0100 Subject: [PATCH 44/60] oprofile/nmi timer: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Cc: oprofile-list@lists.sf.net Cc: Robert Richter Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161126231350.10321-6-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/oprofile/nmi_timer_int.c | 58 +++++++++++--------------------- 1 file changed, 19 insertions(+), 39 deletions(-) diff --git a/drivers/oprofile/nmi_timer_int.c b/drivers/oprofile/nmi_timer_int.c index 9559829fb234..e65a576e4032 100644 --- a/drivers/oprofile/nmi_timer_int.c +++ b/drivers/oprofile/nmi_timer_int.c @@ -59,25 +59,16 @@ static void nmi_timer_stop_cpu(int cpu) perf_event_disable(event); } -static int nmi_timer_cpu_notifier(struct notifier_block *b, unsigned long action, - void *data) +static int nmi_timer_cpu_online(unsigned int cpu) { - int cpu = (unsigned long)data; - switch (action) { - case CPU_DOWN_FAILED: - case CPU_ONLINE: - nmi_timer_start_cpu(cpu); - break; - case CPU_DOWN_PREPARE: - nmi_timer_stop_cpu(cpu); - break; - } - return NOTIFY_DONE; + nmi_timer_start_cpu(cpu); + return 0; +} +static int nmi_timer_cpu_predown(unsigned int cpu) +{ + nmi_timer_stop_cpu(cpu); + return 0; } - -static struct notifier_block nmi_timer_cpu_nb = { - .notifier_call = nmi_timer_cpu_notifier -}; static int nmi_timer_start(void) { @@ -103,13 +94,14 @@ static void nmi_timer_stop(void) put_online_cpus(); } +static enum cpuhp_state hp_online; + static void nmi_timer_shutdown(void) { struct perf_event *event; int cpu; - cpu_notifier_register_begin(); - __unregister_cpu_notifier(&nmi_timer_cpu_nb); + cpuhp_remove_state(hp_online); for_each_possible_cpu(cpu) { event = per_cpu(nmi_timer_events, cpu); if (!event) @@ -118,13 +110,11 @@ static void nmi_timer_shutdown(void) per_cpu(nmi_timer_events, cpu) = NULL; perf_event_release_kernel(event); } - - cpu_notifier_register_done(); } static int nmi_timer_setup(void) { - int cpu, err; + int err; u64 period; /* clock cycles per tick: */ @@ -132,24 +122,14 @@ static int nmi_timer_setup(void) do_div(period, HZ); nmi_timer_attr.sample_period = period; - cpu_notifier_register_begin(); - err = __register_cpu_notifier(&nmi_timer_cpu_nb); - if (err) - goto out; - - /* can't attach events to offline cpus: */ - for_each_online_cpu(cpu) { - err = nmi_timer_start_cpu(cpu); - if (err) { - cpu_notifier_register_done(); - nmi_timer_shutdown(); - return err; - } + err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "oprofile/nmi:online", + nmi_timer_cpu_online, nmi_timer_cpu_predown); + if (err < 0) { + nmi_timer_shutdown(); + return err; } - -out: - cpu_notifier_register_done(); - return err; + hp_online = err; + return 0; } int __init op_nmi_timer_init(struct oprofile_operations *ops) From b32614c03413f8a6025d8677c2b7c0ee976e63d4 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Sun, 27 Nov 2016 00:13:34 +0100 Subject: [PATCH 45/60] tracing/rb: Convert to hotplug state machine Install the callbacks via the state machine. The notifier in struct ring_buffer is replaced by the multi instance interface. Upon __ring_buffer_alloc() invocation, cpuhp_state_add_instance() will invoke the trace_rb_cpu_prepare() on each CPU. This callback may now fail. This means __ring_buffer_alloc() will fail and cleanup (like previously) and during a CPU up event this failure will not allow the CPU to come up. Signed-off-by: Sebastian Andrzej Siewior Cc: Steven Rostedt Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161126231350.10321-7-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + include/linux/ring_buffer.h | 6 ++ kernel/trace/ring_buffer.c | 133 ++++++++++++------------------------ kernel/trace/trace.c | 15 +++- 4 files changed, 65 insertions(+), 90 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index e3771fb959c0..18bcfeb2463e 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -62,6 +62,7 @@ enum cpuhp_state { CPUHP_TOPOLOGY_PREPARE, CPUHP_NET_IUCV_PREPARE, CPUHP_ARM_BL_PREPARE, + CPUHP_TRACE_RB_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index 4acc552e9279..b6d4568795a7 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -198,4 +198,10 @@ enum ring_buffer_flags { RB_FL_OVERWRITE = 1 << 0, }; +#ifdef CONFIG_RING_BUFFER +int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node); +#else +#define trace_rb_cpu_prepare NULL +#endif + #endif /* _LINUX_RING_BUFFER_H */ diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 9c143739b8d7..a7a055f167c7 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -479,9 +479,7 @@ struct ring_buffer { struct ring_buffer_per_cpu **buffers; -#ifdef CONFIG_HOTPLUG_CPU - struct notifier_block cpu_notify; -#endif + struct hlist_node node; u64 (*clock)(void); struct rb_irq_work irq_work; @@ -1274,11 +1272,6 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) kfree(cpu_buffer); } -#ifdef CONFIG_HOTPLUG_CPU -static int rb_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu); -#endif - /** * __ring_buffer_alloc - allocate a new ring_buffer * @size: the size in bytes per cpu that is needed. @@ -1296,6 +1289,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, long nr_pages; int bsize; int cpu; + int ret; /* keep it in its own cache line */ buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), @@ -1318,17 +1312,6 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, if (nr_pages < 2) nr_pages = 2; - /* - * In case of non-hotplug cpu, if the ring-buffer is allocated - * in early initcall, it will not be notified of secondary cpus. - * In that off case, we need to allocate for all possible cpus. - */ -#ifdef CONFIG_HOTPLUG_CPU - cpu_notifier_register_begin(); - cpumask_copy(buffer->cpumask, cpu_online_mask); -#else - cpumask_copy(buffer->cpumask, cpu_possible_mask); -#endif buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; @@ -1337,19 +1320,15 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, if (!buffer->buffers) goto fail_free_cpumask; - for_each_buffer_cpu(buffer, cpu) { - buffer->buffers[cpu] = - rb_allocate_cpu_buffer(buffer, nr_pages, cpu); - if (!buffer->buffers[cpu]) - goto fail_free_buffers; - } + cpu = raw_smp_processor_id(); + cpumask_set_cpu(cpu, buffer->cpumask); + buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); + if (!buffer->buffers[cpu]) + goto fail_free_buffers; -#ifdef CONFIG_HOTPLUG_CPU - buffer->cpu_notify.notifier_call = rb_cpu_notify; - buffer->cpu_notify.priority = 0; - __register_cpu_notifier(&buffer->cpu_notify); - cpu_notifier_register_done(); -#endif + ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); + if (ret < 0) + goto fail_free_buffers; mutex_init(&buffer->mutex); @@ -1364,9 +1343,6 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, fail_free_cpumask: free_cpumask_var(buffer->cpumask); -#ifdef CONFIG_HOTPLUG_CPU - cpu_notifier_register_done(); -#endif fail_free_buffer: kfree(buffer); @@ -1383,18 +1359,11 @@ ring_buffer_free(struct ring_buffer *buffer) { int cpu; -#ifdef CONFIG_HOTPLUG_CPU - cpu_notifier_register_begin(); - __unregister_cpu_notifier(&buffer->cpu_notify); -#endif + cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); for_each_buffer_cpu(buffer, cpu) rb_free_cpu_buffer(buffer->buffers[cpu]); -#ifdef CONFIG_HOTPLUG_CPU - cpu_notifier_register_done(); -#endif - kfree(buffer->buffers); free_cpumask_var(buffer->cpumask); @@ -4633,62 +4602,48 @@ int ring_buffer_read_page(struct ring_buffer *buffer, } EXPORT_SYMBOL_GPL(ring_buffer_read_page); -#ifdef CONFIG_HOTPLUG_CPU -static int rb_cpu_notify(struct notifier_block *self, - unsigned long action, void *hcpu) +/* + * We only allocate new buffers, never free them if the CPU goes down. + * If we were to free the buffer, then the user would lose any trace that was in + * the buffer. + */ +int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node) { - struct ring_buffer *buffer = - container_of(self, struct ring_buffer, cpu_notify); - long cpu = (long)hcpu; + struct ring_buffer *buffer; long nr_pages_same; int cpu_i; unsigned long nr_pages; - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - if (cpumask_test_cpu(cpu, buffer->cpumask)) - return NOTIFY_OK; + buffer = container_of(node, struct ring_buffer, node); + if (cpumask_test_cpu(cpu, buffer->cpumask)) + return 0; - nr_pages = 0; - nr_pages_same = 1; - /* check if all cpu sizes are same */ - for_each_buffer_cpu(buffer, cpu_i) { - /* fill in the size from first enabled cpu */ - if (nr_pages == 0) - nr_pages = buffer->buffers[cpu_i]->nr_pages; - if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { - nr_pages_same = 0; - break; - } + nr_pages = 0; + nr_pages_same = 1; + /* check if all cpu sizes are same */ + for_each_buffer_cpu(buffer, cpu_i) { + /* fill in the size from first enabled cpu */ + if (nr_pages == 0) + nr_pages = buffer->buffers[cpu_i]->nr_pages; + if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { + nr_pages_same = 0; + break; } - /* allocate minimum pages, user can later expand it */ - if (!nr_pages_same) - nr_pages = 2; - buffer->buffers[cpu] = - rb_allocate_cpu_buffer(buffer, nr_pages, cpu); - if (!buffer->buffers[cpu]) { - WARN(1, "failed to allocate ring buffer on CPU %ld\n", - cpu); - return NOTIFY_OK; - } - smp_wmb(); - cpumask_set_cpu(cpu, buffer->cpumask); - break; - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - /* - * Do nothing. - * If we were to free the buffer, then the user would - * lose any trace that was in the buffer. - */ - break; - default: - break; } - return NOTIFY_OK; + /* allocate minimum pages, user can later expand it */ + if (!nr_pages_same) + nr_pages = 2; + buffer->buffers[cpu] = + rb_allocate_cpu_buffer(buffer, nr_pages, cpu); + if (!buffer->buffers[cpu]) { + WARN(1, "failed to allocate ring buffer on CPU %u\n", + cpu); + return -ENOMEM; + } + smp_wmb(); + cpumask_set_cpu(cpu, buffer->cpumask); + return 0; } -#endif #ifdef CONFIG_RING_BUFFER_STARTUP_TEST /* diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 8696ce6bf2f6..465d56febc5b 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -7659,10 +7659,21 @@ __init static int tracer_alloc_buffers(void) raw_spin_lock_init(&global_trace.start_lock); + /* + * The prepare callbacks allocates some memory for the ring buffer. We + * don't free the buffer if the if the CPU goes down. If we were to free + * the buffer, then the user would lose any trace that was in the + * buffer. The memory will be removed once the "instance" is removed. + */ + ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE, + "trace/RB:preapre", trace_rb_cpu_prepare, + NULL); + if (ret < 0) + goto out_free_cpumask; /* Used for event triggers */ temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE); if (!temp_buffer) - goto out_free_cpumask; + goto out_rm_hp_state; if (trace_create_savedcmd() < 0) goto out_free_temp_buffer; @@ -7723,6 +7734,8 @@ __init static int tracer_alloc_buffers(void) free_saved_cmdlines_buffer(savedcmd); out_free_temp_buffer: ring_buffer_free(temp_buffer); +out_rm_hp_state: + cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE); out_free_cpumask: free_cpumask_var(global_trace.tracing_cpumask); out_free_buffer_mask: From 76f290935b1a395ee6c9a14697f16593591d1fe8 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Sun, 27 Nov 2016 00:13:35 +0100 Subject: [PATCH 46/60] mm/vmstat: Drop get_online_cpus() from init_cpu_node_state/vmstat_cpu_dead() Both functions are called with protection against cpu hotplug already so *_online_cpus() could be dropped. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Michal Hocko Cc: linux-mm@kvack.org Cc: rt@linutronix.de Cc: Johannes Weiner Cc: Andrew Morton Cc: Mel Gorman Cc: Vlastimil Babka Link: http://lkml.kernel.org/r/20161126231350.10321-8-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- mm/vmstat.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/mm/vmstat.c b/mm/vmstat.c index 604f26a4f696..0b63ffb5c407 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1722,24 +1722,19 @@ static void __init init_cpu_node_state(void) { int cpu; - get_online_cpus(); for_each_online_cpu(cpu) node_set_state(cpu_to_node(cpu), N_CPU); - put_online_cpus(); } static void vmstat_cpu_dead(int node) { int cpu; - get_online_cpus(); for_each_online_cpu(cpu) if (cpu_to_node(cpu) == node) - goto end; + return; node_clear_state(node, N_CPU); -end: - put_online_cpus(); } /* From 4c501327b4c67f2b874219f20c41d2619b49c78c Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 29 Nov 2016 15:51:14 +0100 Subject: [PATCH 47/60] mm/vmstat: Avoid on each online CPU loops Both iterations over online cpus can be replaced by the proper node specific functions. Signed-off-by: Sebastian Andrzej Siewior Acked-by: Michal Hocko Cc: linux-mm@kvack.org Cc: rt@linutronix.de Cc: Johannes Weiner Cc: Andrew Morton Cc: Mel Gorman Cc: Vlastimil Babka Link: http://lkml.kernel.org/r/20161129145113.fn3lw5aazjjvdrr3@linutronix.de Signed-off-by: Thomas Gleixner --- mm/vmstat.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mm/vmstat.c b/mm/vmstat.c index 0b63ffb5c407..5152cd1c490f 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1720,19 +1720,21 @@ static void __init start_shepherd_timer(void) static void __init init_cpu_node_state(void) { - int cpu; + int node; - for_each_online_cpu(cpu) - node_set_state(cpu_to_node(cpu), N_CPU); + for_each_online_node(node) { + if (cpumask_weight(cpumask_of_node(node)) > 0) + node_set_state(node, N_CPU); + } } static void vmstat_cpu_dead(int node) { - int cpu; + const struct cpumask *node_cpus; - for_each_online_cpu(cpu) - if (cpu_to_node(cpu) == node) - return; + node_cpus = cpumask_of_node(node); + if (cpumask_weight(node_cpus) > 0) + return; node_clear_state(node, N_CPU); } From 5438da977f83c945d4e72ee4f9c4508c0eb64e15 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Tue, 29 Nov 2016 15:52:21 +0100 Subject: [PATCH 48/60] mm/vmstat: Convert to hotplug state machine Install the callbacks via the state machine, but do not invoke them as we can initialize the node state without calling the callbacks on all online CPUs. start_shepherd_timer() is now called outside the get_online_cpus() block which is safe as it only operates on cpu possible mask. Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: linux-mm@kvack.org Cc: rt@linutronix.de Cc: Johannes Weiner Cc: Andrew Morton Cc: Mel Gorman Cc: Vlastimil Babka Link: http://lkml.kernel.org/r/20161129145221.ffc3kg3hd7lxiwj6@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + mm/vmstat.c | 76 ++++++++++++++++++-------------------- 2 files changed, 36 insertions(+), 41 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 18bcfeb2463e..4ebd1bc27f8d 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -20,6 +20,7 @@ enum cpuhp_state { CPUHP_VIRT_NET_DEAD, CPUHP_SLUB_DEAD, CPUHP_MM_WRITEBACK_DEAD, + CPUHP_MM_VMSTAT_DEAD, CPUHP_SOFTIRQ_DEAD, CPUHP_NET_MVNETA_DEAD, CPUHP_CPUIDLE_DEAD, diff --git a/mm/vmstat.c b/mm/vmstat.c index 5152cd1c490f..7c28df36f50f 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -1728,64 +1728,58 @@ static void __init init_cpu_node_state(void) } } -static void vmstat_cpu_dead(int node) +static int vmstat_cpu_online(unsigned int cpu) +{ + refresh_zone_stat_thresholds(); + node_set_state(cpu_to_node(cpu), N_CPU); + return 0; +} + +static int vmstat_cpu_down_prep(unsigned int cpu) +{ + cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu)); + return 0; +} + +static int vmstat_cpu_dead(unsigned int cpu) { const struct cpumask *node_cpus; + int node; + node = cpu_to_node(cpu); + + refresh_zone_stat_thresholds(); node_cpus = cpumask_of_node(node); if (cpumask_weight(node_cpus) > 0) - return; + return 0; node_clear_state(node, N_CPU); + return 0; } -/* - * Use the cpu notifier to insure that the thresholds are recalculated - * when necessary. - */ -static int vmstat_cpuup_callback(struct notifier_block *nfb, - unsigned long action, - void *hcpu) -{ - long cpu = (long)hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - refresh_zone_stat_thresholds(); - node_set_state(cpu_to_node(cpu), N_CPU); - break; - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu)); - break; - case CPU_DOWN_FAILED: - case CPU_DOWN_FAILED_FROZEN: - break; - case CPU_DEAD: - case CPU_DEAD_FROZEN: - refresh_zone_stat_thresholds(); - vmstat_cpu_dead(cpu_to_node(cpu)); - break; - default: - break; - } - return NOTIFY_OK; -} - -static struct notifier_block vmstat_notifier = - { &vmstat_cpuup_callback, NULL, 0 }; #endif static int __init setup_vmstat(void) { #ifdef CONFIG_SMP - cpu_notifier_register_begin(); - __register_cpu_notifier(&vmstat_notifier); + int ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead", + NULL, vmstat_cpu_dead); + if (ret < 0) + pr_err("vmstat: failed to register 'dead' hotplug state\n"); + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online", + vmstat_cpu_online, + vmstat_cpu_down_prep); + if (ret < 0) + pr_err("vmstat: failed to register 'online' hotplug state\n"); + + get_online_cpus(); init_cpu_node_state(); + put_online_cpus(); start_shepherd_timer(); - cpu_notifier_register_done(); #endif #ifdef CONFIG_PROC_FS proc_create("buddyinfo", S_IRUGO, NULL, &fragmentation_file_operations); From 215c89d055e085c42cea4fd571a1d0db0b6d5648 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Sun, 27 Nov 2016 00:13:38 +0100 Subject: [PATCH 49/60] mm/zsmalloc: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Cc: Sergey Senozhatsky Cc: linux-mm@kvack.org Cc: Minchan Kim Cc: rt@linutronix.de Cc: Nitin Gupta Link: http://lkml.kernel.org/r/20161126231350.10321-11-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + mm/zsmalloc.c | 67 ++++++++------------------------------ 2 files changed, 14 insertions(+), 54 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 4ebd1bc27f8d..9f29dd996088 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -64,6 +64,7 @@ enum cpuhp_state { CPUHP_NET_IUCV_PREPARE, CPUHP_ARM_BL_PREPARE, CPUHP_TRACE_RB_PREPARE, + CPUHP_MM_ZS_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index b0bc023d25c5..9cc3c0b2c2c1 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -1284,61 +1284,21 @@ static void __zs_unmap_object(struct mapping_area *area, #endif /* CONFIG_PGTABLE_MAPPING */ -static int zs_cpu_notifier(struct notifier_block *nb, unsigned long action, - void *pcpu) +static int zs_cpu_prepare(unsigned int cpu) { - int ret, cpu = (long)pcpu; struct mapping_area *area; - switch (action) { - case CPU_UP_PREPARE: - area = &per_cpu(zs_map_area, cpu); - ret = __zs_cpu_up(area); - if (ret) - return notifier_from_errno(ret); - break; - case CPU_DEAD: - case CPU_UP_CANCELED: - area = &per_cpu(zs_map_area, cpu); - __zs_cpu_down(area); - break; - } - - return NOTIFY_OK; + area = &per_cpu(zs_map_area, cpu); + return __zs_cpu_up(area); } -static struct notifier_block zs_cpu_nb = { - .notifier_call = zs_cpu_notifier -}; - -static int zs_register_cpu_notifier(void) +static int zs_cpu_dead(unsigned int cpu) { - int cpu, uninitialized_var(ret); + struct mapping_area *area; - cpu_notifier_register_begin(); - - __register_cpu_notifier(&zs_cpu_nb); - for_each_online_cpu(cpu) { - ret = zs_cpu_notifier(NULL, CPU_UP_PREPARE, (void *)(long)cpu); - if (notifier_to_errno(ret)) - break; - } - - cpu_notifier_register_done(); - return notifier_to_errno(ret); -} - -static void zs_unregister_cpu_notifier(void) -{ - int cpu; - - cpu_notifier_register_begin(); - - for_each_online_cpu(cpu) - zs_cpu_notifier(NULL, CPU_DEAD, (void *)(long)cpu); - __unregister_cpu_notifier(&zs_cpu_nb); - - cpu_notifier_register_done(); + area = &per_cpu(zs_map_area, cpu); + __zs_cpu_down(area); + return 0; } static void __init init_zs_size_classes(void) @@ -2534,10 +2494,10 @@ static int __init zs_init(void) if (ret) goto out; - ret = zs_register_cpu_notifier(); - + ret = cpuhp_setup_state(CPUHP_MM_ZS_PREPARE, "mm/zsmalloc:prepare", + zs_cpu_prepare, zs_cpu_dead); if (ret) - goto notifier_fail; + goto hp_setup_fail; init_zs_size_classes(); @@ -2549,8 +2509,7 @@ static int __init zs_init(void) return 0; -notifier_fail: - zs_unregister_cpu_notifier(); +hp_setup_fail: zsmalloc_unmount(); out: return ret; @@ -2562,7 +2521,7 @@ static void __exit zs_exit(void) zpool_unregister_driver(&zs_zpool_driver); #endif zsmalloc_unmount(); - zs_unregister_cpu_notifier(); + cpuhp_remove_state(CPUHP_MM_ZS_PREPARE); zs_stat_exit(); } From ad7ed7708db9ff388450935645816d44bf08a56d Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Sun, 27 Nov 2016 00:13:39 +0100 Subject: [PATCH 50/60] mm/zswap: Convert dst-mem to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Sebastian Andrzej Siewior Cc: linux-mm@kvack.org Cc: Seth Jennings Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161126231350.10321-12-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + mm/zswap.c | 75 +++++++++----------------------------- 2 files changed, 19 insertions(+), 57 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 9f29dd996088..62f51a4e8676 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -65,6 +65,7 @@ enum cpuhp_state { CPUHP_ARM_BL_PREPARE, CPUHP_TRACE_RB_PREPARE, CPUHP_MM_ZS_PREPARE, + CPUHP_MM_ZSWP_MEM_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, diff --git a/mm/zswap.c b/mm/zswap.c index 275b22cc8df4..b13aa5706348 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -352,70 +352,28 @@ static struct zswap_entry *zswap_entry_find_get(struct rb_root *root, **********************************/ static DEFINE_PER_CPU(u8 *, zswap_dstmem); -static int __zswap_cpu_dstmem_notifier(unsigned long action, unsigned long cpu) +static int zswap_dstmem_prepare(unsigned int cpu) { u8 *dst; - switch (action) { - case CPU_UP_PREPARE: - dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); - if (!dst) { - pr_err("can't allocate compressor buffer\n"); - return NOTIFY_BAD; - } - per_cpu(zswap_dstmem, cpu) = dst; - break; - case CPU_DEAD: - case CPU_UP_CANCELED: - dst = per_cpu(zswap_dstmem, cpu); - kfree(dst); - per_cpu(zswap_dstmem, cpu) = NULL; - break; - default: - break; + dst = kmalloc_node(PAGE_SIZE * 2, GFP_KERNEL, cpu_to_node(cpu)); + if (!dst) { + pr_err("can't allocate compressor buffer\n"); + return -ENOMEM; } - return NOTIFY_OK; -} - -static int zswap_cpu_dstmem_notifier(struct notifier_block *nb, - unsigned long action, void *pcpu) -{ - return __zswap_cpu_dstmem_notifier(action, (unsigned long)pcpu); -} - -static struct notifier_block zswap_dstmem_notifier = { - .notifier_call = zswap_cpu_dstmem_notifier, -}; - -static int __init zswap_cpu_dstmem_init(void) -{ - unsigned long cpu; - - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) - if (__zswap_cpu_dstmem_notifier(CPU_UP_PREPARE, cpu) == - NOTIFY_BAD) - goto cleanup; - __register_cpu_notifier(&zswap_dstmem_notifier); - cpu_notifier_register_done(); + per_cpu(zswap_dstmem, cpu) = dst; return 0; - -cleanup: - for_each_online_cpu(cpu) - __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu); - cpu_notifier_register_done(); - return -ENOMEM; } -static void zswap_cpu_dstmem_destroy(void) +static int zswap_dstmem_dead(unsigned int cpu) { - unsigned long cpu; + u8 *dst; - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) - __zswap_cpu_dstmem_notifier(CPU_UP_CANCELED, cpu); - __unregister_cpu_notifier(&zswap_dstmem_notifier); - cpu_notifier_register_done(); + dst = per_cpu(zswap_dstmem, cpu); + kfree(dst); + per_cpu(zswap_dstmem, cpu) = NULL; + + return 0; } static int __zswap_cpu_comp_notifier(struct zswap_pool *pool, @@ -1238,6 +1196,7 @@ static void __exit zswap_debugfs_exit(void) { } static int __init init_zswap(void) { struct zswap_pool *pool; + int ret; zswap_init_started = true; @@ -1246,7 +1205,9 @@ static int __init init_zswap(void) goto cache_fail; } - if (zswap_cpu_dstmem_init()) { + ret = cpuhp_setup_state(CPUHP_MM_ZSWP_MEM_PREPARE, "mm/zswap:prepare", + zswap_dstmem_prepare, zswap_dstmem_dead); + if (ret) { pr_err("dstmem alloc failed\n"); goto dstmem_fail; } @@ -1267,7 +1228,7 @@ static int __init init_zswap(void) return 0; pool_fail: - zswap_cpu_dstmem_destroy(); + cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE); dstmem_fail: zswap_entry_cache_destroy(); cache_fail: From cab7a7e5b6c5c2638b00c72559ff9fb715583c98 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Sun, 27 Nov 2016 00:13:40 +0100 Subject: [PATCH 51/60] mm/zswap: Convert pool to hotplug state machine Install the callbacks via the state machine. Multi state is used to address the per-pool notifier. Uppon adding of the intance the callback is invoked for all online CPUs so the manual init can go. Signed-off-by: Sebastian Andrzej Siewior Cc: linux-mm@kvack.org Cc: Seth Jennings Cc: rt@linutronix.de Link: http://lkml.kernel.org/r/20161126231350.10321-13-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/cpuhotplug.h | 1 + mm/zswap.c | 99 +++++++++++++------------------------- 2 files changed, 35 insertions(+), 65 deletions(-) diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 62f51a4e8676..c7d0d76ef0ee 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -66,6 +66,7 @@ enum cpuhp_state { CPUHP_TRACE_RB_PREPARE, CPUHP_MM_ZS_PREPARE, CPUHP_MM_ZSWP_MEM_PREPARE, + CPUHP_MM_ZSWP_POOL_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, diff --git a/mm/zswap.c b/mm/zswap.c index b13aa5706348..067a0d62f318 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -118,7 +118,7 @@ struct zswap_pool { struct kref kref; struct list_head list; struct work_struct work; - struct notifier_block notifier; + struct hlist_node node; char tfm_name[CRYPTO_MAX_ALG_NAME]; }; @@ -376,77 +376,34 @@ static int zswap_dstmem_dead(unsigned int cpu) return 0; } -static int __zswap_cpu_comp_notifier(struct zswap_pool *pool, - unsigned long action, unsigned long cpu) +static int zswap_cpu_comp_prepare(unsigned int cpu, struct hlist_node *node) { + struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); struct crypto_comp *tfm; - switch (action) { - case CPU_UP_PREPARE: - if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu))) - break; - tfm = crypto_alloc_comp(pool->tfm_name, 0, 0); - if (IS_ERR_OR_NULL(tfm)) { - pr_err("could not alloc crypto comp %s : %ld\n", - pool->tfm_name, PTR_ERR(tfm)); - return NOTIFY_BAD; - } - *per_cpu_ptr(pool->tfm, cpu) = tfm; - break; - case CPU_DEAD: - case CPU_UP_CANCELED: - tfm = *per_cpu_ptr(pool->tfm, cpu); - if (!IS_ERR_OR_NULL(tfm)) - crypto_free_comp(tfm); - *per_cpu_ptr(pool->tfm, cpu) = NULL; - break; - default: - break; + if (WARN_ON(*per_cpu_ptr(pool->tfm, cpu))) + return 0; + + tfm = crypto_alloc_comp(pool->tfm_name, 0, 0); + if (IS_ERR_OR_NULL(tfm)) { + pr_err("could not alloc crypto comp %s : %ld\n", + pool->tfm_name, PTR_ERR(tfm)); + return -ENOMEM; } - return NOTIFY_OK; -} - -static int zswap_cpu_comp_notifier(struct notifier_block *nb, - unsigned long action, void *pcpu) -{ - unsigned long cpu = (unsigned long)pcpu; - struct zswap_pool *pool = container_of(nb, typeof(*pool), notifier); - - return __zswap_cpu_comp_notifier(pool, action, cpu); -} - -static int zswap_cpu_comp_init(struct zswap_pool *pool) -{ - unsigned long cpu; - - memset(&pool->notifier, 0, sizeof(pool->notifier)); - pool->notifier.notifier_call = zswap_cpu_comp_notifier; - - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) - if (__zswap_cpu_comp_notifier(pool, CPU_UP_PREPARE, cpu) == - NOTIFY_BAD) - goto cleanup; - __register_cpu_notifier(&pool->notifier); - cpu_notifier_register_done(); + *per_cpu_ptr(pool->tfm, cpu) = tfm; return 0; - -cleanup: - for_each_online_cpu(cpu) - __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu); - cpu_notifier_register_done(); - return -ENOMEM; } -static void zswap_cpu_comp_destroy(struct zswap_pool *pool) +static int zswap_cpu_comp_dead(unsigned int cpu, struct hlist_node *node) { - unsigned long cpu; + struct zswap_pool *pool = hlist_entry(node, struct zswap_pool, node); + struct crypto_comp *tfm; - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) - __zswap_cpu_comp_notifier(pool, CPU_UP_CANCELED, cpu); - __unregister_cpu_notifier(&pool->notifier); - cpu_notifier_register_done(); + tfm = *per_cpu_ptr(pool->tfm, cpu); + if (!IS_ERR_OR_NULL(tfm)) + crypto_free_comp(tfm); + *per_cpu_ptr(pool->tfm, cpu) = NULL; + return 0; } /********************************* @@ -527,6 +484,7 @@ static struct zswap_pool *zswap_pool_create(char *type, char *compressor) struct zswap_pool *pool; char name[38]; /* 'zswap' + 32 char (max) num + \0 */ gfp_t gfp = __GFP_NORETRY | __GFP_NOWARN | __GFP_KSWAPD_RECLAIM; + int ret; pool = kzalloc(sizeof(*pool), GFP_KERNEL); if (!pool) { @@ -551,7 +509,9 @@ static struct zswap_pool *zswap_pool_create(char *type, char *compressor) goto error; } - if (zswap_cpu_comp_init(pool)) + ret = cpuhp_state_add_instance(CPUHP_MM_ZSWP_POOL_PREPARE, + &pool->node); + if (ret) goto error; pr_debug("using %s compressor\n", pool->tfm_name); @@ -605,7 +565,7 @@ static void zswap_pool_destroy(struct zswap_pool *pool) { zswap_pool_debug("destroying", pool); - zswap_cpu_comp_destroy(pool); + cpuhp_state_remove_instance(CPUHP_MM_ZSWP_POOL_PREPARE, &pool->node); free_percpu(pool->tfm); zpool_destroy_pool(pool->zpool); kfree(pool); @@ -1212,6 +1172,13 @@ static int __init init_zswap(void) goto dstmem_fail; } + ret = cpuhp_setup_state_multi(CPUHP_MM_ZSWP_POOL_PREPARE, + "mm/zswap_pool:prepare", + zswap_cpu_comp_prepare, + zswap_cpu_comp_dead); + if (ret) + goto hp_fail; + pool = __zswap_pool_create_fallback(); if (!pool) { pr_err("pool creation failed\n"); @@ -1228,6 +1195,8 @@ static int __init init_zswap(void) return 0; pool_fail: + cpuhp_remove_state_nocalls(CPUHP_MM_ZSWP_POOL_PREPARE); +hp_fail: cpuhp_remove_state(CPUHP_MM_ZSWP_MEM_PREPARE); dstmem_fail: zswap_entry_cache_destroy(); From 21647615db288d9dacad0de6a5df846b39d51bea Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Sun, 27 Nov 2016 00:13:41 +0100 Subject: [PATCH 52/60] iommu/vt-d: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior Cc: Joerg Roedel Cc: iommu@lists.linux-foundation.org Cc: rt@linutronix.de Cc: David Woodhouse Link: http://lkml.kernel.org/r/20161126231350.10321-14-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/iommu/intel-iommu.c | 24 ++++++------------------ include/linux/cpuhotplug.h | 1 + 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a4407eabf0e6..fd7962560e56 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4665,25 +4665,13 @@ static void free_all_cpu_cached_iovas(unsigned int cpu) } } -static int intel_iommu_cpu_notifier(struct notifier_block *nfb, - unsigned long action, void *v) +static int intel_iommu_cpu_dead(unsigned int cpu) { - unsigned int cpu = (unsigned long)v; - - switch (action) { - case CPU_DEAD: - case CPU_DEAD_FROZEN: - free_all_cpu_cached_iovas(cpu); - flush_unmaps_timeout(cpu); - break; - } - return NOTIFY_OK; + free_all_cpu_cached_iovas(cpu); + flush_unmaps_timeout(cpu); + return 0; } -static struct notifier_block intel_iommu_cpu_nb = { - .notifier_call = intel_iommu_cpu_notifier, -}; - static ssize_t intel_iommu_show_version(struct device *dev, struct device_attribute *attr, char *buf) @@ -4832,8 +4820,8 @@ int __init intel_iommu_init(void) bus_register_notifier(&pci_bus_type, &device_nb); if (si_domain && !hw_pass_through) register_memory_notifier(&intel_iommu_memory_nb); - register_hotcpu_notifier(&intel_iommu_cpu_nb); - + cpuhp_setup_state(CPUHP_IOMMU_INTEL_DEAD, "iommu/intel:dead", NULL, + intel_iommu_cpu_dead); intel_iommu_enabled = 1; return 0; diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index c7d0d76ef0ee..853f8176594d 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -40,6 +40,7 @@ enum cpuhp_state { CPUHP_PAGE_ALLOC_DEAD, CPUHP_NET_DEV_DEAD, CPUHP_PCI_XGENE_DEAD, + CPUHP_IOMMU_INTEL_DEAD, CPUHP_WORKQUEUE_PREP, CPUHP_POWER_NUMA_PREPARE, CPUHP_HRTIMERS_PREPARE, From e46b1db249d83bb19b6bdbed2b2e2ff3858cd211 Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Sun, 27 Nov 2016 00:13:42 +0100 Subject: [PATCH 53/60] mm/compaction: Convert to hotplug state machine Install the callbacks via the state machine. Should the hotplug init fail then no threads are spawned. Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior Cc: Michal Hocko Cc: Mel Gorman Cc: linux-mm@kvack.org Cc: rt@linutronix.de Cc: Andrew Morton Cc: Vlastimil Babka Link: http://lkml.kernel.org/r/20161126231350.10321-15-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- mm/compaction.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/mm/compaction.c b/mm/compaction.c index 0409a4ad6ea1..0d37192d9423 100644 --- a/mm/compaction.c +++ b/mm/compaction.c @@ -2043,33 +2043,38 @@ void kcompactd_stop(int nid) * away, we get changed to run anywhere: as the first one comes back, * restore their cpu bindings. */ -static int cpu_callback(struct notifier_block *nfb, unsigned long action, - void *hcpu) +static int kcompactd_cpu_online(unsigned int cpu) { int nid; - if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) { - for_each_node_state(nid, N_MEMORY) { - pg_data_t *pgdat = NODE_DATA(nid); - const struct cpumask *mask; + for_each_node_state(nid, N_MEMORY) { + pg_data_t *pgdat = NODE_DATA(nid); + const struct cpumask *mask; - mask = cpumask_of_node(pgdat->node_id); + mask = cpumask_of_node(pgdat->node_id); - if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids) - /* One of our CPUs online: restore mask */ - set_cpus_allowed_ptr(pgdat->kcompactd, mask); - } + if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids) + /* One of our CPUs online: restore mask */ + set_cpus_allowed_ptr(pgdat->kcompactd, mask); } - return NOTIFY_OK; + return 0; } static int __init kcompactd_init(void) { int nid; + int ret; + + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "mm/compaction:online", + kcompactd_cpu_online, NULL); + if (ret < 0) { + pr_err("kcompactd: failed to register hotplug callbacks.\n"); + return ret; + } for_each_node_state(nid, N_MEMORY) kcompactd_run(nid); - hotcpu_notifier(cpu_callback, 0); return 0; } subsys_initcall(kcompactd_init) From 914fb85f01571233af199a78ed1f88d9268421fd Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Sun, 27 Nov 2016 00:13:43 +0100 Subject: [PATCH 54/60] arm64/cpuinfo: Make hotplug notifier symmetric There is no requirement to keep the sysfs files around until the CPU is completely dead. Remove them during the DOWN_PREPARE notification. This is a preparatory patch for converting to the hotplug state machine. Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior Reviewed-by: Suzuki K Poulose Cc: Catalin Marinas Cc: Will Deacon Cc: rt@linutronix.de Cc: linux-arm-kernel@lists.infradead.org Link: http://lkml.kernel.org/r/20161126231350.10321-16-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/arm64/kernel/cpuinfo.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index b3d5b3e8fbcb..19aad7041e14 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -272,9 +272,10 @@ static int cpuid_callback(struct notifier_block *nb, switch (action & ~CPU_TASKS_FROZEN) { case CPU_ONLINE: + case CPU_DOWN_FAILED: rc = cpuid_add_regs(cpu); break; - case CPU_DEAD: + case CPU_DOWN_PREPARE: rc = cpuid_remove_regs(cpu); break; } From a7ce95e1741f2d866481c8138a84ce5af61eb219 Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Sun, 27 Nov 2016 00:13:44 +0100 Subject: [PATCH 55/60] arm64/cpuinfo: Convert to hotplug state machine Install the callbacks via the state machine and let the core invoke the callbacks on the already online CPUs. Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior Reviewed-by: Suzuki K Poulose Cc: Catalin Marinas Cc: Will Deacon Cc: rt@linutronix.de Cc: linux-arm-kernel@lists.infradead.org Link: http://lkml.kernel.org/r/20161126231350.10321-17-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/arm64/kernel/cpuinfo.c | 37 +++++++++---------------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 19aad7041e14..7b7be71e87bf 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -227,7 +227,7 @@ static struct attribute_group cpuregs_attr_group = { .name = "identification" }; -static int cpuid_add_regs(int cpu) +static int cpuid_cpu_online(unsigned int cpu) { int rc; struct device *dev; @@ -248,7 +248,7 @@ static int cpuid_add_regs(int cpu) return rc; } -static int cpuid_remove_regs(int cpu) +static int cpuid_cpu_offline(unsigned int cpu) { struct device *dev; struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu); @@ -264,41 +264,22 @@ static int cpuid_remove_regs(int cpu) return 0; } -static int cpuid_callback(struct notifier_block *nb, - unsigned long action, void *hcpu) -{ - int rc = 0; - unsigned long cpu = (unsigned long)hcpu; - - switch (action & ~CPU_TASKS_FROZEN) { - case CPU_ONLINE: - case CPU_DOWN_FAILED: - rc = cpuid_add_regs(cpu); - break; - case CPU_DOWN_PREPARE: - rc = cpuid_remove_regs(cpu); - break; - } - - return notifier_from_errno(rc); -} - static int __init cpuinfo_regs_init(void) { - int cpu; - - cpu_notifier_register_begin(); + int cpu, ret; for_each_possible_cpu(cpu) { struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu); kobject_init(&info->kobj, &cpuregs_kobj_type); - if (cpu_online(cpu)) - cpuid_add_regs(cpu); } - __hotcpu_notifier(cpuid_callback, 0); - cpu_notifier_register_done(); + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "arm64/cpuinfo:online", + cpuid_cpu_online, cpuid_cpu_offline); + if (ret < 0) { + pr_err("cpuinfo: failed to register hotplug callbacks.\n"); + return ret; + } return 0; } static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) From 3f7cd919f3df05918535de39273174710409eb40 Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Sun, 27 Nov 2016 00:13:45 +0100 Subject: [PATCH 56/60] KVM/PPC/Book3S HV: Convert to hotplug state machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Install the callbacks via the state machine. Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior Cc: kvm@vger.kernel.org Cc: Radim Krčmář Cc: Benjamin Herrenschmidt Cc: linuxppc-dev@lists.ozlabs.org Cc: kvm-ppc@vger.kernel.org Cc: Paul Mackerras Cc: rt@linutronix.de Cc: Michael Ellerman Cc: Paolo Bonzini Cc: Alexander Graf Link: http://lkml.kernel.org/r/20161126231350.10321-18-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- arch/powerpc/kvm/book3s_hv.c | 48 +++++++++--------------------------- include/linux/cpuhotplug.h | 1 + 2 files changed, 12 insertions(+), 37 deletions(-) diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 3686471be32b..39ef1f4a7b02 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -2254,12 +2254,12 @@ static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) * enter the guest. Only do this if it is the primary thread of the * core (not if a subcore) that is entering the guest. */ -static inline void kvmppc_clear_host_core(int cpu) +static inline int kvmppc_clear_host_core(unsigned int cpu) { int core; if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) - return; + return 0; /* * Memory barrier can be omitted here as we will do a smp_wmb() * later in kvmppc_start_thread and we need ensure that state is @@ -2267,6 +2267,7 @@ static inline void kvmppc_clear_host_core(int cpu) */ core = cpu >> threads_shift; kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 0; + return 0; } /* @@ -2274,12 +2275,12 @@ static inline void kvmppc_clear_host_core(int cpu) * Only need to do this if it is the primary thread of the core that is * exiting. */ -static inline void kvmppc_set_host_core(int cpu) +static inline int kvmppc_set_host_core(unsigned int cpu) { int core; if (!kvmppc_host_rm_ops_hv || cpu_thread_in_core(cpu)) - return; + return 0; /* * Memory barrier can be omitted here because we do a spin_unlock @@ -2287,6 +2288,7 @@ static inline void kvmppc_set_host_core(int cpu) */ core = cpu >> threads_shift; kvmppc_host_rm_ops_hv->rm_core[core].rm_state.in_host = 1; + return 0; } /* @@ -3094,36 +3096,6 @@ static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu) } #ifdef CONFIG_KVM_XICS -static int kvmppc_cpu_notify(struct notifier_block *self, unsigned long action, - void *hcpu) -{ - unsigned long cpu = (long)hcpu; - - switch (action) { - case CPU_UP_PREPARE: - case CPU_UP_PREPARE_FROZEN: - kvmppc_set_host_core(cpu); - break; - -#ifdef CONFIG_HOTPLUG_CPU - case CPU_DEAD: - case CPU_DEAD_FROZEN: - case CPU_UP_CANCELED: - case CPU_UP_CANCELED_FROZEN: - kvmppc_clear_host_core(cpu); - break; -#endif - default: - break; - } - - return NOTIFY_OK; -} - -static struct notifier_block kvmppc_cpu_notifier = { - .notifier_call = kvmppc_cpu_notify, -}; - /* * Allocate a per-core structure for managing state about which cores are * running in the host versus the guest and for exchanging data between @@ -3185,15 +3157,17 @@ void kvmppc_alloc_host_rm_ops(void) return; } - register_cpu_notifier(&kvmppc_cpu_notifier); - + cpuhp_setup_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE, + "ppc/kvm_book3s:prepare", + kvmppc_set_host_core, + kvmppc_clear_host_core); put_online_cpus(); } void kvmppc_free_host_rm_ops(void) { if (kvmppc_host_rm_ops_hv) { - unregister_cpu_notifier(&kvmppc_cpu_notifier); + cpuhp_remove_state_nocalls(CPUHP_KVM_PPC_BOOK3S_PREPARE); kfree(kvmppc_host_rm_ops_hv->rm_core); kfree(kvmppc_host_rm_ops_hv); kvmppc_host_rm_ops_hv = NULL; diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 853f8176594d..71c6822dd5be 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -68,6 +68,7 @@ enum cpuhp_state { CPUHP_MM_ZS_PREPARE, CPUHP_MM_ZSWP_MEM_PREPARE, CPUHP_MM_ZSWP_POOL_PREPARE, + CPUHP_KVM_PPC_BOOK3S_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, From 1dd6c834fa4a75a86fecefb6d1f1525f1cb755c7 Mon Sep 17 00:00:00 2001 From: Anna-Maria Gleixner Date: Sun, 27 Nov 2016 00:13:46 +0100 Subject: [PATCH 57/60] zram: Convert to hotplug state machine Install the callbacks via the state machine with multi instance support and let the core invoke the callbacks on the already online CPUs. [bigeasy: wire up the multi instance stuff] Signed-off-by: Anna-Maria Gleixner Signed-off-by: Sebastian Andrzej Siewior Signed-off-by: Thomas Gleixner Cc: Sergey Senozhatsky Cc: Minchan Kim Cc: rt@linutronix.de Cc: Nitin Gupta Link: http://lkml.kernel.org/r/20161126231350.10321-19-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/block/zram/zcomp.c | 76 ++++++++++++----------------------- drivers/block/zram/zcomp.h | 5 ++- drivers/block/zram/zram_drv.c | 9 +++++ include/linux/cpuhotplug.h | 1 + 4 files changed, 38 insertions(+), 53 deletions(-) diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c index 4b5cd3a7b2b6..12046f4f00e4 100644 --- a/drivers/block/zram/zcomp.c +++ b/drivers/block/zram/zcomp.c @@ -160,82 +160,56 @@ int zcomp_decompress(struct zcomp_strm *zstrm, dst, &dst_len); } -static int __zcomp_cpu_notifier(struct zcomp *comp, - unsigned long action, unsigned long cpu) +int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) { + struct zcomp *comp = hlist_entry(node, struct zcomp, node); struct zcomp_strm *zstrm; - switch (action) { - case CPU_UP_PREPARE: - if (WARN_ON(*per_cpu_ptr(comp->stream, cpu))) - break; - zstrm = zcomp_strm_alloc(comp); - if (IS_ERR_OR_NULL(zstrm)) { - pr_err("Can't allocate a compression stream\n"); - return NOTIFY_BAD; - } - *per_cpu_ptr(comp->stream, cpu) = zstrm; - break; - case CPU_DEAD: - case CPU_UP_CANCELED: - zstrm = *per_cpu_ptr(comp->stream, cpu); - if (!IS_ERR_OR_NULL(zstrm)) - zcomp_strm_free(zstrm); - *per_cpu_ptr(comp->stream, cpu) = NULL; - break; - default: - break; + if (WARN_ON(*per_cpu_ptr(comp->stream, cpu))) + return 0; + + zstrm = zcomp_strm_alloc(comp); + if (IS_ERR_OR_NULL(zstrm)) { + pr_err("Can't allocate a compression stream\n"); + return -ENOMEM; } - return NOTIFY_OK; + *per_cpu_ptr(comp->stream, cpu) = zstrm; + return 0; } -static int zcomp_cpu_notifier(struct notifier_block *nb, - unsigned long action, void *pcpu) +int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node) { - unsigned long cpu = (unsigned long)pcpu; - struct zcomp *comp = container_of(nb, typeof(*comp), notifier); + struct zcomp *comp = hlist_entry(node, struct zcomp, node); + struct zcomp_strm *zstrm; - return __zcomp_cpu_notifier(comp, action, cpu); + zstrm = *per_cpu_ptr(comp->stream, cpu); + if (!IS_ERR_OR_NULL(zstrm)) + zcomp_strm_free(zstrm); + *per_cpu_ptr(comp->stream, cpu) = NULL; + return 0; } static int zcomp_init(struct zcomp *comp) { - unsigned long cpu; int ret; - comp->notifier.notifier_call = zcomp_cpu_notifier; - comp->stream = alloc_percpu(struct zcomp_strm *); if (!comp->stream) return -ENOMEM; - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) { - ret = __zcomp_cpu_notifier(comp, CPU_UP_PREPARE, cpu); - if (ret == NOTIFY_BAD) - goto cleanup; - } - __register_cpu_notifier(&comp->notifier); - cpu_notifier_register_done(); + ret = cpuhp_state_add_instance(CPUHP_ZCOMP_PREPARE, &comp->node); + if (ret < 0) + goto cleanup; return 0; cleanup: - for_each_online_cpu(cpu) - __zcomp_cpu_notifier(comp, CPU_UP_CANCELED, cpu); - cpu_notifier_register_done(); - return -ENOMEM; + free_percpu(comp->stream); + return ret; } void zcomp_destroy(struct zcomp *comp) { - unsigned long cpu; - - cpu_notifier_register_begin(); - for_each_online_cpu(cpu) - __zcomp_cpu_notifier(comp, CPU_UP_CANCELED, cpu); - __unregister_cpu_notifier(&comp->notifier); - cpu_notifier_register_done(); - + cpuhp_state_remove_instance(CPUHP_ZCOMP_PREPARE, &comp->node); free_percpu(comp->stream); kfree(comp); } diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h index 478cac2ed465..41c1002a7d7d 100644 --- a/drivers/block/zram/zcomp.h +++ b/drivers/block/zram/zcomp.h @@ -19,11 +19,12 @@ struct zcomp_strm { /* dynamic per-device compression frontend */ struct zcomp { struct zcomp_strm * __percpu *stream; - struct notifier_block notifier; - const char *name; + struct hlist_node node; }; +int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node); +int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node); ssize_t zcomp_available_show(const char *comp, char *buf); bool zcomp_available_algorithm(const char *comp); diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 04365b17ee67..511c35fd5c6f 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "zram_drv.h" @@ -1436,15 +1437,22 @@ static void destroy_devices(void) idr_for_each(&zram_index_idr, &zram_remove_cb, NULL); idr_destroy(&zram_index_idr); unregister_blkdev(zram_major, "zram"); + cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE); } static int __init zram_init(void) { int ret; + ret = cpuhp_setup_state_multi(CPUHP_ZCOMP_PREPARE, "block/zram:prepare", + zcomp_cpu_up_prepare, zcomp_cpu_dead); + if (ret < 0) + return ret; + ret = class_register(&zram_control_class); if (ret) { pr_err("Unable to register zram-control class\n"); + cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE); return ret; } @@ -1452,6 +1460,7 @@ static int __init zram_init(void) if (zram_major <= 0) { pr_err("Unable to get major number\n"); class_unregister(&zram_control_class); + cpuhp_remove_multi_state(CPUHP_ZCOMP_PREPARE); return -EBUSY; } diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 71c6822dd5be..22acee76cf4c 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -69,6 +69,7 @@ enum cpuhp_state { CPUHP_MM_ZSWP_MEM_PREPARE, CPUHP_MM_ZSWP_POOL_PREPARE, CPUHP_KVM_PPC_BOOK3S_PREPARE, + CPUHP_ZCOMP_PREPARE, CPUHP_TIMERS_DEAD, CPUHP_NOTF_ERR_INJ_PREPARE, CPUHP_MIPS_SOC_PREPARE, From fce77c3562eccdd8002c9c7947f00af70e00a375 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Sun, 27 Nov 2016 00:13:47 +0100 Subject: [PATCH 58/60] soc/fsl/qbman: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Cc: Scott Wood Cc: Claudiu Manoil Cc: rt@linutronix.de Cc: Roy Pledge Link: http://lkml.kernel.org/r/20161126231350.10321-20-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/soc/fsl/qbman/bman_portal.c | 46 ++++++++++------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/drivers/soc/fsl/qbman/bman_portal.c b/drivers/soc/fsl/qbman/bman_portal.c index 6579cc18811a..986f64690e6e 100644 --- a/drivers/soc/fsl/qbman/bman_portal.c +++ b/drivers/soc/fsl/qbman/bman_portal.c @@ -53,58 +53,38 @@ static struct bman_portal *init_pcfg(struct bm_portal_config *pcfg) return p; } -static void bman_offline_cpu(unsigned int cpu) +static int bman_offline_cpu(unsigned int cpu) { struct bman_portal *p = affine_bportals[cpu]; const struct bm_portal_config *pcfg; if (!p) - return; + return 0; pcfg = bman_get_bm_portal_config(p); if (!pcfg) - return; + return 0; irq_set_affinity(pcfg->irq, cpumask_of(0)); + return 0; } -static void bman_online_cpu(unsigned int cpu) +static int bman_online_cpu(unsigned int cpu) { struct bman_portal *p = affine_bportals[cpu]; const struct bm_portal_config *pcfg; if (!p) - return; + return 0; pcfg = bman_get_bm_portal_config(p); if (!pcfg) - return; + return 0; irq_set_affinity(pcfg->irq, cpumask_of(cpu)); + return 0; } -static int bman_hotplug_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - bman_online_cpu(cpu); - break; - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - bman_offline_cpu(cpu); - } - - return NOTIFY_OK; -} - -static struct notifier_block bman_hotplug_cpu_notifier = { - .notifier_call = bman_hotplug_cpu_callback, -}; - static int bman_portal_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -210,8 +190,14 @@ static int __init bman_portal_driver_register(struct platform_driver *drv) if (ret < 0) return ret; - register_hotcpu_notifier(&bman_hotplug_cpu_notifier); - + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "soc/qbman_portal:online", + bman_online_cpu, bman_offline_cpu); + if (ret < 0) { + pr_err("bman: failed to register hotplug callbacks.\n"); + platform_driver_unregister(drv); + return ret; + } return 0; } From 1f759d328f6d8c06b5db55374b028565187deaca Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Sun, 27 Nov 2016 00:13:48 +0100 Subject: [PATCH 59/60] soc/fsl/qbman: Convert to hotplug state machine Install the callbacks via the state machine. Signed-off-by: Sebastian Andrzej Siewior Cc: Scott Wood Cc: Claudiu Manoil Cc: rt@linutronix.de Cc: Roy Pledge Link: http://lkml.kernel.org/r/20161126231350.10321-21-bigeasy@linutronix.de Signed-off-by: Thomas Gleixner --- drivers/soc/fsl/qbman/qman_portal.c | 39 +++++++++-------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c index 148614388fca..d068e4820f49 100644 --- a/drivers/soc/fsl/qbman/qman_portal.c +++ b/drivers/soc/fsl/qbman/qman_portal.c @@ -179,7 +179,7 @@ static void qman_portal_update_sdest(const struct qm_portal_config *pcfg, qman_set_sdest(pcfg->channel, cpu); } -static void qman_offline_cpu(unsigned int cpu) +static int qman_offline_cpu(unsigned int cpu) { struct qman_portal *p; const struct qm_portal_config *pcfg; @@ -192,9 +192,10 @@ static void qman_offline_cpu(unsigned int cpu) qman_portal_update_sdest(pcfg, 0); } } + return 0; } -static void qman_online_cpu(unsigned int cpu) +static int qman_online_cpu(unsigned int cpu) { struct qman_portal *p; const struct qm_portal_config *pcfg; @@ -207,31 +208,9 @@ static void qman_online_cpu(unsigned int cpu) qman_portal_update_sdest(pcfg, cpu); } } + return 0; } -static int qman_hotplug_cpu_callback(struct notifier_block *nfb, - unsigned long action, void *hcpu) -{ - unsigned int cpu = (unsigned long)hcpu; - - switch (action) { - case CPU_ONLINE: - case CPU_ONLINE_FROZEN: - qman_online_cpu(cpu); - break; - case CPU_DOWN_PREPARE: - case CPU_DOWN_PREPARE_FROZEN: - qman_offline_cpu(cpu); - default: - break; - } - return NOTIFY_OK; -} - -static struct notifier_block qman_hotplug_cpu_notifier = { - .notifier_call = qman_hotplug_cpu_callback, -}; - static int qman_portal_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -346,8 +325,14 @@ static int __init qman_portal_driver_register(struct platform_driver *drv) if (ret < 0) return ret; - register_hotcpu_notifier(&qman_hotplug_cpu_notifier); - + ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "soc/qman_portal:online", + qman_online_cpu, qman_offline_cpu); + if (ret < 0) { + pr_err("qman: failed to register hotplug callbacks.\n"); + platform_driver_unregister(drv); + return ret; + } return 0; } From b18cc3de00ec3442cf40ac60787dbe0703b99e24 Mon Sep 17 00:00:00 2001 From: Sebastian Andrzej Siewior Date: Wed, 7 Dec 2016 14:31:33 +0100 Subject: [PATCH 60/60] tracing/rb: Init the CPU mask on allocation Before commit b32614c03413 ("tracing/rb: Convert to hotplug state machine") the allocated cpumask was initialized to the mask of online or possible CPUs. After the CPU hotplug changes the buffer initialization moved to trace_rb_cpu_prepare() but the cpumask is allocated with alloc_cpumask() and therefor has random content. As a consequence the cpu buffers are not initialized and a later access dereferences a NULL pointer. Use zalloc_cpumask() instead so trace_rb_cpu_prepare() initializes the buffers properly. Fixes: b32614c03413 ("tracing/rb: Convert to hotplug state machine") Reported-by: Tetsuo Handa Signed-off-by: Sebastian Andrzej Siewior Cc: rostedt@goodmis.org Link: http://lkml.kernel.org/r/20161207133133.hzkcqfllxcdi3joz@linutronix.de Signed-off-by: Thomas Gleixner --- kernel/trace/ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index a7a055f167c7..89a2611a1635 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1297,7 +1297,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags, if (!buffer) return NULL; - if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) + if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) goto fail_free_buffer; nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);