2012-03-05 19:49:30 +08:00
|
|
|
/*
|
|
|
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
|
|
|
*
|
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
|
*
|
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef __ASM_SMP_H
|
|
|
|
|
#define __ASM_SMP_H
|
|
|
|
|
|
2016-02-23 18:31:42 +08:00
|
|
|
/* Values for secondary_data.status */
|
|
|
|
|
|
|
|
|
|
#define CPU_MMU_OFF (-1)
|
|
|
|
|
#define CPU_BOOT_SUCCESS (0)
|
|
|
|
|
/* The cpu invoked ops->cpu_die, synchronise it with cpu_kill */
|
|
|
|
|
#define CPU_KILL_ME (1)
|
|
|
|
|
/* The cpu couldn't die gracefully and is looping in the kernel */
|
|
|
|
|
#define CPU_STUCK_IN_KERNEL (2)
|
|
|
|
|
/* Fatal system error detected by secondary CPU, crash the system */
|
|
|
|
|
#define CPU_PANIC_KERNEL (3)
|
|
|
|
|
|
|
|
|
|
#ifndef __ASSEMBLY__
|
|
|
|
|
|
arm64: make cpu number a percpu variable
In the absence of CONFIG_THREAD_INFO_IN_TASK, core code maintains
thread_info::cpu, and low-level architecture code can access this to
build raw_smp_processor_id(). With CONFIG_THREAD_INFO_IN_TASK, core code
maintains task_struct::cpu, which for reasons of hte header soup is not
accessible to low-level arch code.
Instead, we can maintain a percpu variable containing the cpu number.
For both the old and new implementation of raw_smp_processor_id(), we
read a syreg into a GPR, add an offset, and load the result. As the
offset is now larger, it may not be folded into the load, but otherwise
the assembly shouldn't change much.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: James Morse <james.morse@arm.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:11 +08:00
|
|
|
#include <asm/percpu.h>
|
|
|
|
|
|
2012-03-05 19:49:30 +08:00
|
|
|
#include <linux/threads.h>
|
|
|
|
|
#include <linux/cpumask.h>
|
|
|
|
|
#include <linux/thread_info.h>
|
|
|
|
|
|
arm64: make cpu number a percpu variable
In the absence of CONFIG_THREAD_INFO_IN_TASK, core code maintains
thread_info::cpu, and low-level architecture code can access this to
build raw_smp_processor_id(). With CONFIG_THREAD_INFO_IN_TASK, core code
maintains task_struct::cpu, which for reasons of hte header soup is not
accessible to low-level arch code.
Instead, we can maintain a percpu variable containing the cpu number.
For both the old and new implementation of raw_smp_processor_id(), we
read a syreg into a GPR, add an offset, and load the result. As the
offset is now larger, it may not be folded into the load, but otherwise
the assembly shouldn't change much.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: James Morse <james.morse@arm.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:11 +08:00
|
|
|
DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We don't use this_cpu_read(cpu_number) as that has implicit writes to
|
|
|
|
|
* preempt_count, and associated (compiler) barriers, that we'd like to avoid
|
|
|
|
|
* the expense of. If we're preemptible, the value can be stale at use anyway.
|
2016-12-01 23:55:13 +08:00
|
|
|
* And we can't use this_cpu_ptr() either, as that winds up recursing back
|
|
|
|
|
* here under CONFIG_DEBUG_PREEMPT=y.
|
arm64: make cpu number a percpu variable
In the absence of CONFIG_THREAD_INFO_IN_TASK, core code maintains
thread_info::cpu, and low-level architecture code can access this to
build raw_smp_processor_id(). With CONFIG_THREAD_INFO_IN_TASK, core code
maintains task_struct::cpu, which for reasons of hte header soup is not
accessible to low-level arch code.
Instead, we can maintain a percpu variable containing the cpu number.
For both the old and new implementation of raw_smp_processor_id(), we
read a syreg into a GPR, add an offset, and load the result. As the
offset is now larger, it may not be folded into the load, but otherwise
the assembly shouldn't change much.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: James Morse <james.morse@arm.com>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:11 +08:00
|
|
|
*/
|
2016-12-01 23:55:13 +08:00
|
|
|
#define raw_smp_processor_id() (*raw_cpu_ptr(&cpu_number))
|
2012-03-05 19:49:30 +08:00
|
|
|
|
|
|
|
|
struct seq_file;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* generate IPI list text
|
|
|
|
|
*/
|
|
|
|
|
extern void show_ipi_list(struct seq_file *p, int prec);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Called from C code, this handles an IPI.
|
|
|
|
|
*/
|
|
|
|
|
extern void handle_IPI(int ipinr, struct pt_regs *regs);
|
|
|
|
|
|
|
|
|
|
/*
|
2015-03-24 22:02:45 +08:00
|
|
|
* Discover the set of possible CPUs and determine their
|
|
|
|
|
* SMP operations.
|
2012-03-05 19:49:30 +08:00
|
|
|
*/
|
2015-05-13 21:12:47 +08:00
|
|
|
extern void smp_init_cpus(void);
|
2012-03-05 19:49:30 +08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Provide a function to raise an IPI cross call on CPUs in callmap.
|
|
|
|
|
*/
|
|
|
|
|
extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
|
|
|
|
|
|
2014-08-17 00:48:05 +08:00
|
|
|
extern void (*__smp_cross_call)(const struct cpumask *, unsigned int);
|
|
|
|
|
|
2012-03-05 19:49:30 +08:00
|
|
|
/*
|
|
|
|
|
* Called from the secondary holding pen, this is the secondary CPU entry point.
|
|
|
|
|
*/
|
|
|
|
|
asmlinkage void secondary_start_kernel(void);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Initial data for bringing up a secondary CPU.
|
2016-02-23 18:31:42 +08:00
|
|
|
* @stack - sp for the secondary CPU
|
|
|
|
|
* @status - Result passed back from the secondary CPU to
|
|
|
|
|
* indicate failure.
|
2012-03-05 19:49:30 +08:00
|
|
|
*/
|
|
|
|
|
struct secondary_data {
|
|
|
|
|
void *stack;
|
arm64: split thread_info from task stack
This patch moves arm64's struct thread_info from the task stack into
task_struct. This protects thread_info from corruption in the case of
stack overflows, and makes its address harder to determine if stack
addresses are leaked, making a number of attacks more difficult. Precise
detection and handling of overflow is left for subsequent patches.
Largely, this involves changing code to store the task_struct in sp_el0,
and acquire the thread_info from the task struct. Core code now
implements current_thread_info(), and as noted in <linux/sched.h> this
relies on offsetof(task_struct, thread_info) == 0, enforced by core
code.
This change means that the 'tsk' register used in entry.S now points to
a task_struct, rather than a thread_info as it used to. To make this
clear, the TI_* field offsets are renamed to TSK_TI_*, with asm-offsets
appropriately updated to account for the structural change.
Userspace clobbers sp_el0, and we can no longer restore this from the
stack. Instead, the current task is cached in a per-cpu variable that we
can safely access from early assembly as interrupts are disabled (and we
are thus not preemptible).
Both secondary entry and idle are updated to stash the sp and task
pointer separately.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Laura Abbott <labbott@redhat.com>
Cc: AKASHI Takahiro <takahiro.akashi@linaro.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Cc: James Morse <james.morse@arm.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Suzuki K Poulose <suzuki.poulose@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2016-11-04 04:23:13 +08:00
|
|
|
struct task_struct *task;
|
2016-02-23 18:31:42 +08:00
|
|
|
long status;
|
2012-03-05 19:49:30 +08:00
|
|
|
};
|
2016-02-23 18:31:42 +08:00
|
|
|
|
2012-03-05 19:49:30 +08:00
|
|
|
extern struct secondary_data secondary_data;
|
2016-02-23 18:31:42 +08:00
|
|
|
extern long __early_cpu_boot_status;
|
arm64: factor out spin-table boot method
The arm64 kernel has an internal holding pen, which is necessary for
some systems where we can't bring CPUs online individually and must hold
multiple CPUs in a safe area until the kernel is able to handle them.
The current SMP infrastructure for arm64 is closely coupled to this
holding pen, and alternative boot methods must launch CPUs into the pen,
where they sit before they are launched into the kernel proper.
With PSCI (and possibly other future boot methods), we can bring CPUs
online individually, and need not perform the secondary_holding_pen
dance. Instead, this patch factors the holding pen management code out
to the spin-table boot method code, as it is the only boot method
requiring the pen.
A new entry point for secondaries, secondary_entry is added for other
boot methods to use, which bypasses the holding pen and its associated
overhead when bringing CPUs online. The smp.pen.text section is also
removed, as the pen can live in head.text without problem.
The cpu_operations structure is extended with two new functions,
cpu_boot and cpu_postboot, for bringing a cpu into the kernel and
performing any post-boot cleanup required by a bootmethod (e.g.
resetting the secondary_holding_pen_release to INVALID_HWID).
Documentation is added for cpu_operations.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
2013-10-25 03:30:16 +08:00
|
|
|
extern void secondary_entry(void);
|
2012-03-05 19:49:30 +08:00
|
|
|
|
|
|
|
|
extern void arch_send_call_function_single_ipi(int cpu);
|
|
|
|
|
extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
|
|
|
|
|
|
2016-01-26 19:10:38 +08:00
|
|
|
#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
|
|
|
|
|
extern void arch_send_wakeup_ipi_mask(const struct cpumask *mask);
|
|
|
|
|
#else
|
|
|
|
|
static inline void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
|
|
|
|
|
{
|
|
|
|
|
BUILD_BUG();
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
2013-10-25 03:30:18 +08:00
|
|
|
extern int __cpu_disable(void);
|
|
|
|
|
|
|
|
|
|
extern void __cpu_die(unsigned int cpu);
|
|
|
|
|
extern void cpu_die(void);
|
2016-02-23 18:31:41 +08:00
|
|
|
extern void cpu_die_early(void);
|
2013-10-25 03:30:18 +08:00
|
|
|
|
2016-02-23 18:31:39 +08:00
|
|
|
static inline void cpu_park_loop(void)
|
|
|
|
|
{
|
|
|
|
|
for (;;) {
|
|
|
|
|
wfe();
|
|
|
|
|
wfi();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2016-02-23 18:31:42 +08:00
|
|
|
static inline void update_cpu_boot_status(int val)
|
|
|
|
|
{
|
|
|
|
|
WRITE_ONCE(secondary_data.status, val);
|
|
|
|
|
/* Ensure the visibility of the status update */
|
|
|
|
|
dsb(ishst);
|
|
|
|
|
}
|
|
|
|
|
|
2016-04-12 22:46:00 +08:00
|
|
|
/*
|
|
|
|
|
* The calling secondary CPU has detected serious configuration mismatch,
|
|
|
|
|
* which calls for a kernel panic. Update the boot status and park the calling
|
|
|
|
|
* CPU.
|
|
|
|
|
*/
|
|
|
|
|
static inline void cpu_panic_kernel(void)
|
|
|
|
|
{
|
|
|
|
|
update_cpu_boot_status(CPU_PANIC_KERNEL);
|
|
|
|
|
cpu_park_loop();
|
|
|
|
|
}
|
|
|
|
|
|
2016-06-22 17:06:12 +08:00
|
|
|
/*
|
|
|
|
|
* If a secondary CPU enters the kernel but fails to come online,
|
|
|
|
|
* (e.g. due to mismatched features), and cannot exit the kernel,
|
|
|
|
|
* we increment cpus_stuck_in_kernel and leave the CPU in a
|
|
|
|
|
* quiesecent loop within the kernel text. The memory containing
|
|
|
|
|
* this loop must not be re-used for anything else as the 'stuck'
|
|
|
|
|
* core is executing it.
|
|
|
|
|
*
|
|
|
|
|
* This function is used to inhibit features like kexec and hibernate.
|
|
|
|
|
*/
|
|
|
|
|
bool cpus_are_stuck_in_kernel(void);
|
|
|
|
|
|
2017-08-17 10:24:27 +08:00
|
|
|
extern void crash_smp_send_stop(void);
|
2017-04-03 10:24:36 +08:00
|
|
|
extern bool smp_crash_stop_failed(void);
|
|
|
|
|
|
2016-02-23 18:31:42 +08:00
|
|
|
#endif /* ifndef __ASSEMBLY__ */
|
|
|
|
|
|
2012-03-05 19:49:30 +08:00
|
|
|
#endif /* ifndef __ASM_SMP_H */
|