thermal/x86 pkg temp: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Replace the wrmsr/rdmrs_on_cpu() calls in the hotplug callbacks as they are
guaranteed to be invoked on the incoming/outgoing cpu.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
Signed-off-by: Zhang Rui <rui.zhang@intel.com>

drivers/thermal/x86_pkg_temp_thermal.c

@@ -79,6 +79,9 @@ static DEFINE_SPINLOCK(pkg_temp_lock);
 /* Protects zone operation in the work function against hotplug removal */
 static DEFINE_MUTEX(thermal_zone_mutex);
 
+/* The dynamically assigned cpu hotplug state for module_exit() */
+static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
+
 /* Debug counters to show using debugfs */
 static struct dentry *debugfs;
 static unsigned int pkg_interrupt_cnt;
@@ -386,9 +389,8 @@ static int pkg_temp_thermal_device_add(unsigned int cpu)
 		return err;
 	}
 	/* Store MSR value for package thermal interrupt, to restore at exit */
-	rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
+	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, pkgdev->msr_pkg_therm_low,
-		     &pkgdev->msr_pkg_therm_low,
+	      pkgdev->msr_pkg_therm_high);
-		     &pkgdev->msr_pkg_therm_high);
 
 	cpumask_set_cpu(cpu, &pkgdev->cpumask);
 	spin_lock_irq(&pkg_temp_lock);
@@ -397,14 +399,14 @@ static int pkg_temp_thermal_device_add(unsigned int cpu)
 	return 0;
 }
 
-static void put_core_offline(unsigned int cpu)
+static int pkg_thermal_cpu_offline(unsigned int cpu)
 {
 	struct pkg_device *pkgdev = pkg_temp_thermal_get_dev(cpu);
 	bool lastcpu, was_target;
 	int target;
 
 	if (!pkgdev)
-		return;
+		return 0;
 
 	target = cpumask_any_but(&pkgdev->cpumask, cpu);
 	cpumask_clear_cpu(cpu, &pkgdev->cpumask);
@@ -448,16 +450,9 @@ static void put_core_offline(unsigned int cpu)
 	 */
 	if (lastcpu) {
 		packages[topology_logical_package_id(cpu)] = NULL;
-		/*
+		/* After this point nothing touches the MSR anymore. */
-		 * After this point nothing touches the MSR anymore. We
+		wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
-		 * must drop the lock to make the cross cpu call. This goes
+		      pkgdev->msr_pkg_therm_low, pkgdev->msr_pkg_therm_high);
-		 * away once we move that code to the hotplug state machine.
-		 */
-		spin_unlock_irq(&pkg_temp_lock);
-		wrmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
-			     pkgdev->msr_pkg_therm_low,
-			     pkgdev->msr_pkg_therm_high);
-		spin_lock_irq(&pkg_temp_lock);
 	}
 
 	/*
@@ -487,9 +482,10 @@ static void put_core_offline(unsigned int cpu)
 	/* Final cleanup if this is the last cpu */
 	if (lastcpu)
 		kfree(pkgdev);
+	return 0;
 }
 
-static int get_core_online(unsigned int cpu)
+static int pkg_thermal_cpu_online(unsigned int cpu)
 {
 	struct pkg_device *pkgdev = pkg_temp_thermal_get_dev(cpu);
 	struct cpuinfo_x86 *c = &cpu_data(cpu);
@@ -506,27 +502,6 @@ static int get_core_online(unsigned int cpu)
 	return pkg_temp_thermal_device_add(cpu);
 }
 
-static int pkg_temp_thermal_cpu_callback(struct notifier_block *nfb,
-				 unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long) hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		get_core_online(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-		put_core_offline(cpu);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block pkg_temp_thermal_notifier __refdata = {
-	.notifier_call = pkg_temp_thermal_cpu_callback,
-};
-
 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
 	{ X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_PTS },
 	{}
@@ -535,7 +510,7 @@ MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
 
 static int __init pkg_temp_thermal_init(void)
 {
-	int i;
+	int ret;
 
 	if (!x86_match_cpu(pkg_temp_thermal_ids))
 		return -ENODEV;
@@ -545,12 +520,13 @@ static int __init pkg_temp_thermal_init(void)
 	if (!packages)
 		return -ENOMEM;
 
-	cpu_notifier_register_begin();
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
-	for_each_online_cpu(i)
+				pkg_thermal_cpu_online,	pkg_thermal_cpu_offline);
-		if (get_core_online(i))
+	if (ret < 0)
-			goto err_ret;
+		goto err;
-	__register_hotcpu_notifier(&pkg_temp_thermal_notifier);
+
-	cpu_notifier_register_done();
+	/* Store the state for module exit */
+	pkg_thermal_hp_state = ret;
 
 	platform_thermal_package_notify = pkg_thermal_notify;
 	platform_thermal_package_rate_control = pkg_thermal_rate_control;
@@ -559,28 +535,18 @@ static int __init pkg_temp_thermal_init(void)
 	pkg_temp_debugfs_init();
 	return 0;
 
-err_ret:
+err:
-	for_each_online_cpu(i)
-		put_core_offline(i);
-	cpu_notifier_register_done();
 	kfree(packages);
-	return -ENODEV;
+	return ret;
 }
 module_init(pkg_temp_thermal_init)
 
 static void __exit pkg_temp_thermal_exit(void)
 {
-	int i;
-
 	platform_thermal_package_notify = NULL;
 	platform_thermal_package_rate_control = NULL;
 
-	cpu_notifier_register_begin();
+	cpuhp_remove_state(pkg_thermal_hp_state);
-	__unregister_hotcpu_notifier(&pkg_temp_thermal_notifier);
-	for_each_online_cpu(i)
-		put_core_offline(i);
-	cpu_notifier_register_done();
-
 	debugfs_remove_recursive(debugfs);
 	kfree(packages);
 }