openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

- CPU suspend support on top of PSCI (firmware Power State Coordination 

Interface)
 - Jump label support
 - CMA can now be enabled on arm64
 - HWCAP bits for crypto and CRC32 extensions
 - Optimised percpu using tpidr_el1 register
 - Code cleanup
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1.4.9 (GNU/Linux)
 
 iQIcBAABAgAGBQJS3WLBAAoJEGvWsS0AyF7xxx8P+wUYzu04rKtz4BOj5IHl8TxB
 xsRw8ce9MdxIVgdtCjDzmdpkd0s8ZPTEweJnVGYYlB9O9Pmz0VSX4Z1y6W6k0P1f
 GCKDMa+hn2uQYnw3bS022Zji6OjUfad9XUfe3f61YdA7GrSdjTVMapXuloASRcfl
 0XkfpXwbfLPGpuNp4q/QaA9K/y93T/gc6O/ctJh3OUJDOWJXZGsUTRIKXTF9GrWn
 /gPEK9MiatAPpcS7iO283a3vllDalNoEGpt+a4cYCc8il2kCWUpX6W2c9m3Ua26k
 mAvkoUErfb3cW/PzqDZzr8M3XbnXb8Je99HBbcjQluL6zyw+0hdUHJpCFOamsz5m
 pEpT1e0Hvxb6yNbjqyituiYFPwOUZHP/HeZpH4l1njhN7sIyZP5cUEV4f53VN4lB
 KL3HSGzUTaNzT5UpD35CA4vXRwRKrV8YsAVhB0p53KgkUreKA6wbJHSXHorjBZaE
 uuP7kqOMGQ494+f6h+yvZqwIcObQGaHYNQJLY3Yhzg3WAs59s/bX/s4yWhgkte0U
 yfxKpxTSiLjv5LmZrVQer04DIf9duNkEpI/DAKUbXagHJ7RCHjOneg9F5ZvJ598o
 umCo+ok9hV+vLUhagh4t5guSk2ehW7qoZOG44XkYcCLXTIlMV1AQA6oJr804DUm2
 71UbGFi01OY0Jtp8prO3
 =TPaC
 -----END PGP SIGNATURE-----

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull ARM64 updates from Catalin Marinas:
 - CPU suspend support on top of PSCI (firmware Power State Coordination
   Interface)
 - jump label support
 - CMA can now be enabled on arm64
 - HWCAP bits for crypto and CRC32 extensions
 - optimised percpu using tpidr_el1 register
 - code cleanup

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (42 commits)
  arm64: fix typo in entry.S
  arm64: kernel: restore HW breakpoint registers in cpu_suspend
  jump_label: use defined macros instead of hard-coding for better readability
  arm64, jump label: optimize jump label implementation
  arm64, jump label: detect %c support for ARM64
  arm64: introduce aarch64_insn_gen_{nop|branch_imm}() helper functions
  arm64: move encode_insn_immediate() from module.c to insn.c
  arm64: introduce interfaces to hotpatch kernel and module code
  arm64: introduce basic aarch64 instruction decoding helpers
  arm64: dts: Reduce size of virtio block device for foundation model
  arm64: Remove unused __data_loc variable
  arm64: Enable CMA
  arm64: Warn on NULL device structure for dma APIs
  arm64: Add hwcaps for crypto and CRC32 extensions.
  arm64: drop redundant macros from read_cpuid()
  arm64: Remove outdated comment
  arm64: cmpxchg: update macros to prevent warnings
  arm64: support single-step and breakpoint handler hooks
  ARM64: fix framepointer check in unwind_frame
  ARM64: check stack pointer in get_wchan
  ...
This commit is contained in:
Linus Torvalds 2014-01-20 15:40:44 -08:00
parent 15c8102620 883c057367
commit 82b51734b4
50 changed files with 1859 additions and 403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
arch/arm/kvm/arm.c
View File

@ -17,6 +17,7 @@
*/
#include <linux/cpu.h>
#include <linux/cpu_pm.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/kvm_host.h>
@ -853,6 +854,33 @@ static struct notifier_block hyp_init_cpu_nb = {
.notifier_call = hyp_init_cpu_notify,
};
#ifdef CONFIG_CPU_PM
static int hyp_init_cpu_pm_notifier(struct notifier_block *self,
unsigned long cmd,
void *v)
{
if (cmd == CPU_PM_EXIT) {
cpu_init_hyp_mode(NULL);
return NOTIFY_OK;
}
return NOTIFY_DONE;
}
static struct notifier_block hyp_init_cpu_pm_nb = {
.notifier_call = hyp_init_cpu_pm_notifier,
};
static void __init hyp_cpu_pm_init(void)
{
cpu_pm_register_notifier(&hyp_init_cpu_pm_nb);
}
#else
static inline void hyp_cpu_pm_init(void)
{
}
#endif
/**
* Inits Hyp-mode on all online CPUs
*/
@ -1013,6 +1041,8 @@ int kvm_arch_init(void *opaque)
goto out_err;
}
hyp_cpu_pm_init();
kvm_coproc_table_init();
return 0;
out_err:

27
arch/arm64/Kconfig
View File

@ -2,6 +2,7 @@ config ARM64
def_bool y
select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE
select ARCH_USE_CMPXCHG_LOCKREF
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
select ARCH_WANT_OPTIONAL_GPIOLIB
select ARCH_WANT_COMPAT_IPC_PARSE_VERSION
select ARCH_WANT_FRAME_POINTERS
@ -11,19 +12,27 @@ config ARM64
select BUILDTIME_EXTABLE_SORT
select CLONE_BACKWARDS
select COMMON_CLK
select CPU_PM if (SUSPEND || CPU_IDLE)
select DCACHE_WORD_ACCESS
select GENERIC_CLOCKEVENTS
select GENERIC_CLOCKEVENTS_BROADCAST if SMP
select GENERIC_IOMAP
select GENERIC_IRQ_PROBE
select GENERIC_IRQ_SHOW
select GENERIC_SCHED_CLOCK
select GENERIC_SMP_IDLE_THREAD
select GENERIC_STRNCPY_FROM_USER
select GENERIC_STRNLEN_USER
select GENERIC_TIME_VSYSCALL
select HARDIRQS_SW_RESEND
select HAVE_ARCH_JUMP_LABEL
select HAVE_ARCH_TRACEHOOK
select HAVE_DEBUG_BUGVERBOSE
select HAVE_DEBUG_KMEMLEAK
select HAVE_DMA_API_DEBUG
select HAVE_DMA_ATTRS
select HAVE_DMA_CONTIGUOUS
select HAVE_EFFICIENT_UNALIGNED_ACCESS
select HAVE_GENERIC_DMA_COHERENT
select HAVE_HW_BREAKPOINT if PERF_EVENTS
select HAVE_MEMBLOCK
@ -275,6 +284,24 @@ config SYSVIPC_COMPAT
endmenu
menu "Power management options"
source "kernel/power/Kconfig"
config ARCH_SUSPEND_POSSIBLE
def_bool y
config ARM64_CPU_SUSPEND
def_bool PM_SLEEP
endmenu
menu "CPU Power Management"
source "drivers/cpuidle/Kconfig"
endmenu
source "net/Kconfig"
source "drivers/Kconfig"

2
arch/arm64/boot/dts/foundation-v8.dts
View File

@ -224,7 +224,7 @@ v2m_serial3: uart@0c0000 {
virtio_block@0130000 {
compatible = "virtio,mmio";
reg = <0x130000 0x1000>;
reg = <0x130000 0x200>;
interrupts = <42>;
};
};

6
arch/arm64/boot/dts/rtsm_ve-motherboard.dtsi
View File

@ -183,6 +183,12 @@ clcd@1f0000 {
clocks = <&v2m_oscclk1>, <&v2m_clk24mhz>;
clock-names = "clcdclk", "apb_pclk";
};
virtio_block@0130000 {
compatible = "virtio,mmio";
reg = <0x130000 0x200>;
interrupts = <42>;
};
};
v2m_fixed_3v3: fixedregulator@0 {

1
arch/arm64/include/asm/Kbuild
View File

@ -26,7 +26,6 @@ generic-y += mman.h
generic-y += msgbuf.h
generic-y += mutex.h
generic-y += pci.h
generic-y += percpu.h
generic-y += poll.h
generic-y += posix_types.h
generic-y += resource.h

26
arch/arm64/include/asm/cmpxchg.h
View File

@ -158,17 +158,23 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
return ret;
}
#define cmpxchg(ptr,o,n) \
((__typeof__(*(ptr)))__cmpxchg_mb((ptr), \
(unsigned long)(o), \
(unsigned long)(n), \
sizeof(*(ptr))))
#define cmpxchg(ptr, o, n) \
({ \
__typeof__(*(ptr)) __ret; \
__ret = (__typeof__(*(ptr))) \
__cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
sizeof(*(ptr))); \
__ret; \
})
#define cmpxchg_local(ptr,o,n) \
((__typeof__(*(ptr)))__cmpxchg((ptr), \
(unsigned long)(o), \
(unsigned long)(n), \
sizeof(*(ptr))))
#define cmpxchg_local(ptr, o, n) \
({ \
__typeof__(*(ptr)) __ret; \
__ret = (__typeof__(*(ptr))) \
__cmpxchg((ptr), (unsigned long)(o), \
(unsigned long)(n), sizeof(*(ptr))); \
__ret; \
})
#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n))
#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))

6
arch/arm64/include/asm/cpu_ops.h
View File

@ -39,6 +39,9 @@ struct device_node;
* from the cpu to be killed.
* @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the
* cpu being killed.
* @cpu_suspend: Suspends a cpu and saves the required context. May fail owing
* to wrong parameters or error conditions. Called from the
* CPU being suspended. Must be called with IRQs disabled.
*/
struct cpu_operations {
const char *name;
@ -50,6 +53,9 @@ struct cpu_operations {
int (*cpu_disable)(unsigned int cpu);
void (*cpu_die)(unsigned int cpu);
#endif
#ifdef CONFIG_ARM64_CPU_SUSPEND
int (*cpu_suspend)(unsigned long);
#endif
};
extern const struct cpu_operations *cpu_ops[NR_CPUS];

28
arch/arm64/include/asm/cputype.h
View File

@ -16,23 +16,23 @@
#ifndef __ASM_CPUTYPE_H
#define __ASM_CPUTYPE_H
#define ID_MIDR_EL1 "midr_el1"
#define ID_MPIDR_EL1 "mpidr_el1"
#define ID_CTR_EL0 "ctr_el0"
#define ID_AA64PFR0_EL1 "id_aa64pfr0_el1"
#define ID_AA64DFR0_EL1 "id_aa64dfr0_el1"
#define ID_AA64AFR0_EL1 "id_aa64afr0_el1"
#define ID_AA64ISAR0_EL1 "id_aa64isar0_el1"
#define ID_AA64MMFR0_EL1 "id_aa64mmfr0_el1"
#define INVALID_HWID ULONG_MAX
#define MPIDR_HWID_BITMASK 0xff00ffffff
#define MPIDR_LEVEL_BITS_SHIFT 3
#define MPIDR_LEVEL_BITS (1 << MPIDR_LEVEL_BITS_SHIFT)
#define MPIDR_LEVEL_MASK ((1 << MPIDR_LEVEL_BITS) - 1)
#define MPIDR_LEVEL_SHIFT(level) \
(((1 << level) >> 1) << MPIDR_LEVEL_BITS_SHIFT)
#define MPIDR_AFFINITY_LEVEL(mpidr, level) \
((mpidr >> MPIDR_LEVEL_SHIFT(level)) & MPIDR_LEVEL_MASK)
#define read_cpuid(reg) ({ \
u64 __val; \
asm("mrs %0, " reg : "=r" (__val)); \
asm("mrs %0, " #reg : "=r" (__val)); \
__val; \
})
@ -54,12 +54,12 @@
*/
static inline u32 __attribute_const__ read_cpuid_id(void)
{
return read_cpuid(ID_MIDR_EL1);
return read_cpuid(MIDR_EL1);
}
static inline u64 __attribute_const__ read_cpuid_mpidr(void)
{
return read_cpuid(ID_MPIDR_EL1);
return read_cpuid(MPIDR_EL1);
}
static inline unsigned int __attribute_const__ read_cpuid_implementor(void)
@ -74,7 +74,7 @@ static inline unsigned int __attribute_const__ read_cpuid_part_number(void)
static inline u32 __attribute_const__ read_cpuid_cachetype(void)
{
return read_cpuid(ID_CTR_EL0);
return read_cpuid(CTR_EL0);
}
#endif /* __ASSEMBLY__ */

21
arch/arm64/include/asm/debug-monitors.h
View File

@ -62,6 +62,27 @@ struct task_struct;
#define DBG_ARCH_ID_RESERVED 0 /* In case of ptrace ABI updates. */
#define DBG_HOOK_HANDLED 0
#define DBG_HOOK_ERROR 1
struct step_hook {
struct list_head node;
int (*fn)(struct pt_regs *regs, unsigned int esr);
};
void register_step_hook(struct step_hook *hook);
void unregister_step_hook(struct step_hook *hook);
struct break_hook {
struct list_head node;
u32 esr_val;
u32 esr_mask;
int (*fn)(struct pt_regs *regs, unsigned int esr);
};
void register_break_hook(struct break_hook *hook);
void unregister_break_hook(struct break_hook *hook);
u8 debug_monitors_arch(void);
void enable_debug_monitors(enum debug_el el);

29
arch/arm64/include/asm/dma-contiguous.h Normal file
View File

@ -0,0 +1,29 @@
/*
* Copyright (c) 2013, The Linux Foundation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 and
* only version 2 as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*/
#ifndef _ASM_DMA_CONTIGUOUS_H
#define _ASM_DMA_CONTIGUOUS_H
#ifdef __KERNEL__
#ifdef CONFIG_DMA_CMA
#include <linux/types.h>
#include <asm-generic/dma-contiguous.h>
static inline void
dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) { }
#endif
#endif
#endif

1
arch/arm64/include/asm/futex.h
View File

@ -30,6 +30,7 @@
" cbnz %w3, 1b\n" \
"3:\n" \
" .pushsection .fixup,\"ax\"\n" \
" .align 2\n" \
"4: mov %w0, %w5\n" \
" b 3b\n" \
" .popsection\n" \

2
arch/arm64/include/asm/hardirq.h
View File

@ -20,7 +20,7 @@
#include <linux/threads.h>
#include <asm/irq.h>
#define NR_IPI 4
#define NR_IPI 5
typedef struct {
unsigned int __softirq_pending;

108
arch/arm64/include/asm/insn.h Normal file
View File

@ -0,0 +1,108 @@
/*
* Copyright (C) 2013 Huawei Ltd.
* Author: Jiang Liu <liuj97@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_INSN_H
#define __ASM_INSN_H
#include <linux/types.h>
/* A64 instructions are always 32 bits. */
#define AARCH64_INSN_SIZE 4
/*
* ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
* Section C3.1 "A64 instruction index by encoding":
* AArch64 main encoding table
* Bit position
* 28 27 26 25 Encoding Group
* 0 0 - - Unallocated
* 1 0 0 - Data processing, immediate
* 1 0 1 - Branch, exception generation and system instructions
* - 1 - 0 Loads and stores
* - 1 0 1 Data processing - register
* 0 1 1 1 Data processing - SIMD and floating point
* 1 1 1 1 Data processing - SIMD and floating point
* "-" means "don't care"
*/
enum aarch64_insn_encoding_class {
AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */
AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */
AARCH64_INSN_CLS_DP_REG, /* Data processing - register */
AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */
AARCH64_INSN_CLS_LDST, /* Loads and stores */
AARCH64_INSN_CLS_BR_SYS, /* Branch, exception generation and
* system instructions */
};
enum aarch64_insn_hint_op {
AARCH64_INSN_HINT_NOP = 0x0 << 5,
AARCH64_INSN_HINT_YIELD = 0x1 << 5,
AARCH64_INSN_HINT_WFE = 0x2 << 5,
AARCH64_INSN_HINT_WFI = 0x3 << 5,
AARCH64_INSN_HINT_SEV = 0x4 << 5,
AARCH64_INSN_HINT_SEVL = 0x5 << 5,
};
enum aarch64_insn_imm_type {
AARCH64_INSN_IMM_ADR,
AARCH64_INSN_IMM_26,
AARCH64_INSN_IMM_19,
AARCH64_INSN_IMM_16,
AARCH64_INSN_IMM_14,
AARCH64_INSN_IMM_12,
AARCH64_INSN_IMM_9,
AARCH64_INSN_IMM_MAX
};
enum aarch64_insn_branch_type {
AARCH64_INSN_BRANCH_NOLINK,
AARCH64_INSN_BRANCH_LINK,
};
#define __AARCH64_INSN_FUNCS(abbr, mask, val) \
static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
{ return (code & (mask)) == (val); } \
static __always_inline u32 aarch64_insn_get_##abbr##_value(void) \
{ return (val); }
__AARCH64_INSN_FUNCS(b, 0xFC000000, 0x14000000)
__AARCH64_INSN_FUNCS(bl, 0xFC000000, 0x94000000)
__AARCH64_INSN_FUNCS(svc, 0xFFE0001F, 0xD4000001)
__AARCH64_INSN_FUNCS(hvc, 0xFFE0001F, 0xD4000002)
__AARCH64_INSN_FUNCS(smc, 0xFFE0001F, 0xD4000003)
__AARCH64_INSN_FUNCS(brk, 0xFFE0001F, 0xD4200000)
__AARCH64_INSN_FUNCS(hint, 0xFFFFF01F, 0xD503201F)
#undef __AARCH64_INSN_FUNCS
bool aarch64_insn_is_nop(u32 insn);
int aarch64_insn_read(void *addr, u32 *insnp);
int aarch64_insn_write(void *addr, u32 insn);
enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
u32 aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
u32 insn, u64 imm);
u32 aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
enum aarch64_insn_branch_type type);
u32 aarch64_insn_gen_hint(enum aarch64_insn_hint_op op);
u32 aarch64_insn_gen_nop(void);
bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt);
int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
#endif /* __ASM_INSN_H */

52
arch/arm64/include/asm/jump_label.h Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright (C) 2013 Huawei Ltd.
* Author: Jiang Liu <liuj97@gmail.com>
*
* Based on arch/arm/include/asm/jump_label.h
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_JUMP_LABEL_H
#define __ASM_JUMP_LABEL_H
#include <linux/types.h>
#include <asm/insn.h>
#ifdef __KERNEL__
#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
static __always_inline bool arch_static_branch(struct static_key *key)
{
asm goto("1: nop\n\t"
".pushsection __jump_table, \"aw\"\n\t"
".align 3\n\t"
".quad 1b, %l[l_yes], %c0\n\t"
".popsection\n\t"
: : "i"(key) : : l_yes);
return false;
l_yes:
return true;
}
#endif /* __KERNEL__ */
typedef u64 jump_label_t;
struct jump_entry {
jump_label_t code;
jump_label_t target;
jump_label_t key;
};
#endif /* __ASM_JUMP_LABEL_H */

3
arch/arm64/include/asm/memory.h
View File

@ -146,8 +146,7 @@ static inline void *phys_to_virt(phys_addr_t x)
#define ARCH_PFN_OFFSET PHYS_PFN_OFFSET
#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
#define virt_addr_valid(kaddr) (((void *)(kaddr) >= (void *)PAGE_OFFSET) && \
((void *)(kaddr) < (void *)high_memory))
#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
#endif

41
arch/arm64/include/asm/percpu.h Normal file
View File

@ -0,0 +1,41 @@
/*
* Copyright (C) 2013 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_PERCPU_H
#define __ASM_PERCPU_H
static inline void set_my_cpu_offset(unsigned long off)
{
asm volatile("msr tpidr_el1, %0" :: "r" (off) : "memory");
}
static inline unsigned long __my_cpu_offset(void)
{
unsigned long off;
register unsigned long *sp asm ("sp");
/*
* We want to allow caching the value, so avoid using volatile and
* instead use a fake stack read to hazard against barrier().
*/
asm("mrs %0, tpidr_el1" : "=r" (off) : "Q" (*sp));
return off;
}
#define __my_cpu_offset __my_cpu_offset()
#include <asm-generic/percpu.h>
#endif /* __ASM_PERCPU_H */

3
arch/arm64/include/asm/proc-fns.h
View File

@ -26,11 +26,14 @@
#include <asm/page.h>
struct mm_struct;
struct cpu_suspend_ctx;
extern void cpu_cache_off(void);
extern void cpu_do_idle(void);
extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
extern void cpu_do_suspend(struct cpu_suspend_ctx *ptr);
extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr);
#include <asm/memory.h>

13
arch/arm64/include/asm/smp_plat.h
View File

@ -21,6 +21,19 @@
#include <asm/types.h>
struct mpidr_hash {
u64 mask;
u32 shift_aff[4];
u32 bits;
};
extern struct mpidr_hash mpidr_hash;
static inline u32 mpidr_hash_size(void)
{
return 1 << mpidr_hash.bits;
}
/*
* Logical CPU mapping.
*/

27
arch/arm64/include/asm/suspend.h Normal file
View File

@ -0,0 +1,27 @@
#ifndef __ASM_SUSPEND_H
#define __ASM_SUSPEND_H
#define NR_CTX_REGS 11
/*
* struct cpu_suspend_ctx must be 16-byte aligned since it is allocated on
* the stack, which must be 16-byte aligned on v8
*/
struct cpu_suspend_ctx {
/*
* This struct must be kept in sync with
* cpu_do_{suspend/resume} in mm/proc.S
*/
u64 ctx_regs[NR_CTX_REGS];
u64 sp;
} __aligned(16);
struct sleep_save_sp {
phys_addr_t *save_ptr_stash;
phys_addr_t save_ptr_stash_phys;
};
extern void cpu_resume(void);
extern int cpu_suspend(unsigned long);
#endif

25
arch/arm64/include/asm/uaccess.h
View File

@ -100,6 +100,7 @@ static inline void set_fs(mm_segment_t fs)
})
#define access_ok(type, addr, size) __range_ok(addr, size)
#define user_addr_max get_fs
/*
* The "__xxx" versions of the user access functions do not verify the address
@ -240,9 +241,6 @@ extern unsigned long __must_check __copy_to_user(void __user *to, const void *fr
extern unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n);
extern unsigned long __must_check __clear_user(void __user *addr, unsigned long n);
extern unsigned long __must_check __strncpy_from_user(char *to, const char __user *from, unsigned long count);
extern unsigned long __must_check __strnlen_user(const char __user *s, long n);
static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n)
{
if (access_ok(VERIFY_READ, from, n))
@ -276,24 +274,9 @@ static inline unsigned long __must_check clear_user(void __user *to, unsigned lo
return n;
}
static inline long __must_check strncpy_from_user(char *dst, const char __user *src, long count)
{
long res = -EFAULT;
if (access_ok(VERIFY_READ, src, 1))
res = __strncpy_from_user(dst, src, count);
return res;
}
extern long strncpy_from_user(char *dest, const char __user *src, long count);
#define strlen_user(s) strnlen_user(s, ~0UL >> 1)
static inline long __must_check strnlen_user(const char __user *s, long n)
{
unsigned long res = 0;
if (__addr_ok(s))
res = __strnlen_user(s, n);
return res;
}
extern __must_check long strlen_user(const char __user *str);
extern __must_check long strnlen_user(const char __user *str, long n);
#endif /* __ASM_UACCESS_H */

94
arch/arm64/include/asm/word-at-a-time.h Normal file
View File

@ -0,0 +1,94 @@
/*
* Copyright (C) 2013 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_WORD_AT_A_TIME_H
#define __ASM_WORD_AT_A_TIME_H
#ifndef __AARCH64EB__
#include <linux/kernel.h>
struct word_at_a_time {
const unsigned long one_bits, high_bits;
};
#define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
static inline unsigned long has_zero(unsigned long a, unsigned long *bits,
const struct word_at_a_time *c)
{
unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
*bits = mask;
return mask;
}
#define prep_zero_mask(a, bits, c) (bits)
static inline unsigned long create_zero_mask(unsigned long bits)
{
bits = (bits - 1) & ~bits;
return bits >> 7;
}
static inline unsigned long find_zero(unsigned long mask)
{
return fls64(mask) >> 3;
}
#define zero_bytemask(mask) (mask)
#else /* __AARCH64EB__ */
#include <asm-generic/word-at-a-time.h>
#endif
/*
* Load an unaligned word from kernel space.
*
* In the (very unlikely) case of the word being a page-crosser
* and the next page not being mapped, take the exception and
* return zeroes in the non-existing part.
*/
static inline unsigned long load_unaligned_zeropad(const void *addr)
{
unsigned long ret, offset;
/* Load word from unaligned pointer addr */
asm(
"1: ldr %0, %3\n"
"2:\n"
" .pushsection .fixup,\"ax\"\n"
" .align 2\n"
"3: and %1, %2, #0x7\n"
" bic %2, %2, #0x7\n"
" ldr %0, [%2]\n"
" lsl %1, %1, #0x3\n"
#ifndef __AARCH64EB__
" lsr %0, %0, %1\n"
#else
" lsl %0, %0, %1\n"
#endif
" b 2b\n"
" .popsection\n"
" .pushsection __ex_table,\"a\"\n"
" .align 3\n"
" .quad 1b, 3b\n"
" .popsection"
: "=&r" (ret), "=&r" (offset)
: "r" (addr), "Q" (*(unsigned long *)addr));
return ret;
}
#endif /* __ASM_WORD_AT_A_TIME_H */

6
arch/arm64/include/uapi/asm/hwcap.h
View File

@ -22,6 +22,10 @@
#define HWCAP_FP (1 << 0)
#define HWCAP_ASIMD (1 << 1)
#define HWCAP_EVTSTRM (1 << 2)
#define HWCAP_AES (1 << 3)
#define HWCAP_PMULL (1 << 4)
#define HWCAP_SHA1 (1 << 5)
#define HWCAP_SHA2 (1 << 6)
#define HWCAP_CRC32 (1 << 7)
#endif /* _UAPI__ASM_HWCAP_H */

4
arch/arm64/kernel/Makefile
View File

@ -9,7 +9,7 @@ AFLAGS_head.o := -DTEXT_OFFSET=$(TEXT_OFFSET)
arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \
entry-fpsimd.o process.o ptrace.o setup.o signal.o \
sys.o stacktrace.o time.o traps.o io.o vdso.o \
hyp-stub.o psci.o cpu_ops.o
hyp-stub.o psci.o cpu_ops.o insn.o
arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
sys_compat.o
@ -18,6 +18,8 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o
arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT)+= hw_breakpoint.o
arm64-obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o
arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o
obj-y += $(arm64-obj-y) vdso/
obj-m += $(arm64-obj-m)

5
arch/arm64/kernel/arm64ksyms.c
View File

@ -29,13 +29,10 @@
#include <asm/checksum.h>
/* user mem (segment) */
EXPORT_SYMBOL(__strnlen_user);
EXPORT_SYMBOL(__strncpy_from_user);
EXPORT_SYMBOL(copy_page);
EXPORT_SYMBOL(clear_page);
/* user mem (segment) */
EXPORT_SYMBOL(__copy_from_user);
EXPORT_SYMBOL(__copy_to_user);
EXPORT_SYMBOL(__clear_user);

11
arch/arm64/kernel/asm-offsets.c
View File

@ -25,6 +25,8 @@
#include <asm/thread_info.h>
#include <asm/memory.h>
#include <asm/cputable.h>
#include <asm/smp_plat.h>
#include <asm/suspend.h>
#include <asm/vdso_datapage.h>
#include <linux/kbuild.h>
@ -137,6 +139,15 @@ int main(void)
DEFINE(VGIC_CPU_NR_LR, offsetof(struct vgic_cpu, nr_lr));
DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
#endif
#ifdef CONFIG_ARM64_CPU_SUSPEND
DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));
DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp));
DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask));
DEFINE(MPIDR_HASH_SHIFTS, offsetof(struct mpidr_hash, shift_aff));
DEFINE(SLEEP_SAVE_SP_SZ, sizeof(struct sleep_save_sp));
DEFINE(SLEEP_SAVE_SP_PHYS, offsetof(struct sleep_save_sp, save_ptr_stash_phys));
DEFINE(SLEEP_SAVE_SP_VIRT, offsetof(struct sleep_save_sp, save_ptr_stash));
#endif
return 0;
}

88
arch/arm64/kernel/debug-monitors.c
View File

@ -187,6 +187,48 @@ static void clear_regs_spsr_ss(struct pt_regs *regs)
regs->pstate = spsr;
}
/* EL1 Single Step Handler hooks */
static LIST_HEAD(step_hook);
DEFINE_RWLOCK(step_hook_lock);
void register_step_hook(struct step_hook *hook)
{
write_lock(&step_hook_lock);
list_add(&hook->node, &step_hook);
write_unlock(&step_hook_lock);
}
void unregister_step_hook(struct step_hook *hook)
{
write_lock(&step_hook_lock);
list_del(&hook->node);
write_unlock(&step_hook_lock);
}
/*
* Call registered single step handers
* There is no Syndrome info to check for determining the handler.
* So we call all the registered handlers, until the right handler is
* found which returns zero.
*/
static int call_step_hook(struct pt_regs *regs, unsigned int esr)
{
struct step_hook *hook;
int retval = DBG_HOOK_ERROR;
read_lock(&step_hook_lock);
list_for_each_entry(hook, &step_hook, node) {
retval = hook->fn(regs, esr);
if (retval == DBG_HOOK_HANDLED)
break;
}
read_unlock(&step_hook_lock);
return retval;
}
static int single_step_handler(unsigned long addr, unsigned int esr,
struct pt_regs *regs)
{
@ -214,7 +256,9 @@ static int single_step_handler(unsigned long addr, unsigned int esr,
*/
user_rewind_single_step(current);
} else {
/* TODO: route to KGDB */
if (call_step_hook(regs, esr) == DBG_HOOK_HANDLED)
return 0;
pr_warning("Unexpected kernel single-step exception at EL1\n");
/*
* Re-enable stepping since we know that we will be
@ -226,11 +270,53 @@ static int single_step_handler(unsigned long addr, unsigned int esr,
return 0;
}
/*
* Breakpoint handler is re-entrant as another breakpoint can
* hit within breakpoint handler, especically in kprobes.
* Use reader/writer locks instead of plain spinlock.
*/
static LIST_HEAD(break_hook);
DEFINE_RWLOCK(break_hook_lock);
void register_break_hook(struct break_hook *hook)
{
write_lock(&break_hook_lock);
list_add(&hook->node, &break_hook);
write_unlock(&break_hook_lock);
}
void unregister_break_hook(struct break_hook *hook)
{
write_lock(&break_hook_lock);
list_del(&hook->node);
write_unlock(&break_hook_lock);
}
static int call_break_hook(struct pt_regs *regs, unsigned int esr)
{
struct break_hook *hook;
int (*fn)(struct pt_regs *regs, unsigned int esr) = NULL;
read_lock(&break_hook_lock);
list_for_each_entry(hook, &break_hook, node)
if ((esr & hook->esr_mask) == hook->esr_val)
fn = hook->fn;
read_unlock(&break_hook_lock);
return fn ? fn(regs, esr) : DBG_HOOK_ERROR;
}
static int brk_handler(unsigned long addr, unsigned int esr,
struct pt_regs *regs)
{
siginfo_t info;
if (call_break_hook(regs, esr) == DBG_HOOK_HANDLED)
return 0;
pr_warn("unexpected brk exception at %lx, esr=0x%x\n",
(long)instruction_pointer(regs), esr);
if (!user_mode(regs))
return -EFAULT;

4
arch/arm64/kernel/entry.S
View File

@ -288,6 +288,8 @@ el1_dbg:
/*
* Debug exception handling
*/
cmp x24, #ESR_EL1_EC_BRK64 // if BRK64
cinc x24, x24, eq // set bit '0'
tbz x24, #0, el1_inv // EL1 only
mrs x0, far_el1
mov x2, sp // struct pt_regs
@ -314,7 +316,7 @@ el1_irq:
#ifdef CONFIG_PREEMPT
get_thread_info tsk
ldr w24, [tsk, #TI_PREEMPT] // restore preempt count
ldr w24, [tsk, #TI_PREEMPT] // get preempt count
cbnz w24, 1f // preempt count != 0
ldr x0, [tsk, #TI_FLAGS] // get flags
tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?

36
arch/arm64/kernel/fpsimd.c
View File

@ -17,6 +17,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/cpu_pm.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/sched.h>
@ -113,6 +114,39 @@ EXPORT_SYMBOL(kernel_neon_end);
#endif /* CONFIG_KERNEL_MODE_NEON */
#ifdef CONFIG_CPU_PM
static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
unsigned long cmd, void *v)
{
switch (cmd) {
case CPU_PM_ENTER:
if (current->mm)
fpsimd_save_state(&current->thread.fpsimd_state);
break;
case CPU_PM_EXIT:
if (current->mm)
fpsimd_load_state(&current->thread.fpsimd_state);
break;
case CPU_PM_ENTER_FAILED:
default:
return NOTIFY_DONE;
}
return NOTIFY_OK;
}
static struct notifier_block fpsimd_cpu_pm_notifier_block = {
.notifier_call = fpsimd_cpu_pm_notifier,
};
static void fpsimd_pm_init(void)
{
cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
}
#else
static inline void fpsimd_pm_init(void) { }
#endif /* CONFIG_CPU_PM */
/*
* FP/SIMD support code initialisation.
*/
@ -131,6 +165,8 @@ static int __init fpsimd_init(void)
else
elf_hwcap |= HWCAP_ASIMD;
fpsimd_pm_init();
return 0;
}
late_initcall(fpsimd_init);

10
arch/arm64/kernel/head.S
View File

@ -482,8 +482,6 @@ ENDPROC(__create_page_tables)
.type __switch_data, %object
__switch_data:
.quad __mmap_switched
.quad __data_loc // x4
.quad _data // x5
.quad __bss_start // x6
.quad _end // x7
.quad processor_id // x4
@ -498,15 +496,7 @@ __switch_data:
__mmap_switched:
adr x3, __switch_data + 8
ldp x4, x5, [x3], #16
ldp x6, x7, [x3], #16
cmp x4, x5 // Copy data segment if needed
1: ccmp x5, x6, #4, ne
b.eq 2f
ldr x16, [x4], #8
str x16, [x5], #8
b 1b
2:
1: cmp x6, x7
b.hs 2f
str xzr, [x6], #8 // Clear BSS

205
arch/arm64/kernel/hw_breakpoint.c
View File

@ -20,6 +20,7 @@
#define pr_fmt(fmt) "hw-breakpoint: " fmt
#include <linux/cpu_pm.h>
#include <linux/errno.h>
#include <linux/hw_breakpoint.h>
#include <linux/perf_event.h>
@ -169,15 +170,68 @@ static enum debug_el debug_exception_level(int privilege)
}
}
/*
* Install a perf counter breakpoint.
enum hw_breakpoint_ops {
HW_BREAKPOINT_INSTALL,
HW_BREAKPOINT_UNINSTALL,
HW_BREAKPOINT_RESTORE
};
/**
* hw_breakpoint_slot_setup - Find and setup a perf slot according to
* operations
*
* @slots: pointer to array of slots
* @max_slots: max number of slots
* @bp: perf_event to setup
* @ops: operation to be carried out on the slot
*
* Return:
* slot index on success
* -ENOSPC if no slot is available/matches
* -EINVAL on wrong operations parameter
*/
int arch_install_hw_breakpoint(struct perf_event *bp)
static int hw_breakpoint_slot_setup(struct perf_event **slots, int max_slots,
struct perf_event *bp,
enum hw_breakpoint_ops ops)
{
int i;
struct perf_event **slot;
for (i = 0; i < max_slots; ++i) {
slot = &slots[i];
switch (ops) {
case HW_BREAKPOINT_INSTALL:
if (!*slot) {
*slot = bp;
return i;
}
break;
case HW_BREAKPOINT_UNINSTALL:
if (*slot == bp) {
*slot = NULL;
return i;
}
break;
case HW_BREAKPOINT_RESTORE:
if (*slot == bp)
return i;
break;
default:
pr_warn_once("Unhandled hw breakpoint ops %d\n", ops);
return -EINVAL;
}
}
return -ENOSPC;
}
static int hw_breakpoint_control(struct perf_event *bp,
enum hw_breakpoint_ops ops)
{
struct arch_hw_breakpoint *info = counter_arch_bp(bp);
struct perf_event **slot, **slots;
struct perf_event **slots;
struct debug_info *debug_info = &current->thread.debug;
int i, max_slots, ctrl_reg, val_reg, reg_enable;
enum debug_el dbg_el = debug_exception_level(info->ctrl.privilege);
u32 ctrl;
if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
@ -196,67 +250,54 @@ int arch_install_hw_breakpoint(struct perf_event *bp)
reg_enable = !debug_info->wps_disabled;
}
for (i = 0; i < max_slots; ++i) {
slot = &slots[i];
i = hw_breakpoint_slot_setup(slots, max_slots, bp, ops);
if (!*slot) {
*slot = bp;
break;
}
if (WARN_ONCE(i < 0, "Can't find any breakpoint slot"))
return i;
switch (ops) {
case HW_BREAKPOINT_INSTALL:
/*
* Ensure debug monitors are enabled at the correct exception
* level.
*/
enable_debug_monitors(dbg_el);
/* Fall through */
case HW_BREAKPOINT_RESTORE:
/* Setup the address register. */
write_wb_reg(val_reg, i, info->address);
/* Setup the control register. */
ctrl = encode_ctrl_reg(info->ctrl);
write_wb_reg(ctrl_reg, i,
reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
break;
case HW_BREAKPOINT_UNINSTALL:
/* Reset the control register. */
write_wb_reg(ctrl_reg, i, 0);
/*
* Release the debug monitors for the correct exception
* level.
*/
disable_debug_monitors(dbg_el);
break;
}
if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot"))
return -ENOSPC;
/* Ensure debug monitors are enabled at the correct exception level. */
enable_debug_monitors(debug_exception_level(info->ctrl.privilege));
/* Setup the address register. */
write_wb_reg(val_reg, i, info->address);
/* Setup the control register. */
ctrl = encode_ctrl_reg(info->ctrl);
write_wb_reg(ctrl_reg, i, reg_enable ? ctrl | 0x1 : ctrl & ~0x1);
return 0;
}
/*
* Install a perf counter breakpoint.
*/
int arch_install_hw_breakpoint(struct perf_event *bp)
{
return hw_breakpoint_control(bp, HW_BREAKPOINT_INSTALL);
}
void arch_uninstall_hw_breakpoint(struct perf_event *bp)
{
struct arch_hw_breakpoint *info = counter_arch_bp(bp);
struct perf_event **slot, **slots;
int i, max_slots, base;
if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
/* Breakpoint */
base = AARCH64_DBG_REG_BCR;
slots = this_cpu_ptr(bp_on_reg);
max_slots = core_num_brps;
} else {
/* Watchpoint */
base = AARCH64_DBG_REG_WCR;
slots = this_cpu_ptr(wp_on_reg);
max_slots = core_num_wrps;
}
/* Remove the breakpoint. */
for (i = 0; i < max_slots; ++i) {
slot = &slots[i];
if (*slot == bp) {
*slot = NULL;
break;
}
}
if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot"))
return;
/* Reset the control register. */
write_wb_reg(base, i, 0);
/* Release the debug monitors for the correct exception level. */
disable_debug_monitors(debug_exception_level(info->ctrl.privilege));
hw_breakpoint_control(bp, HW_BREAKPOINT_UNINSTALL);
}
static int get_hbp_len(u8 hbp_len)
@ -806,18 +847,36 @@ void hw_breakpoint_thread_switch(struct task_struct *next)
/*
* CPU initialisation.
*/
static void reset_ctrl_regs(void *unused)
static void hw_breakpoint_reset(void *unused)
{
int i;
for (i = 0; i < core_num_brps; ++i) {
write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
struct perf_event **slots;
/*
* When a CPU goes through cold-boot, it does not have any installed
* slot, so it is safe to share the same function for restoring and
* resetting breakpoints; when a CPU is hotplugged in, it goes
* through the slots, which are all empty, hence it just resets control
* and value for debug registers.
* When this function is triggered on warm-boot through a CPU PM
* notifier some slots might be initialized; if so they are
* reprogrammed according to the debug slots content.
*/
for (slots = this_cpu_ptr(bp_on_reg), i = 0; i < core_num_brps; ++i) {
if (slots[i]) {
hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
} else {
write_wb_reg(AARCH64_DBG_REG_BCR, i, 0UL);
write_wb_reg(AARCH64_DBG_REG_BVR, i, 0UL);
}
}
for (i = 0; i < core_num_wrps; ++i) {
write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
for (slots = this_cpu_ptr(wp_on_reg), i = 0; i < core_num_wrps; ++i) {
if (slots[i]) {
hw_breakpoint_control(slots[i], HW_BREAKPOINT_RESTORE);
} else {
write_wb_reg(AARCH64_DBG_REG_WCR, i, 0UL);
write_wb_reg(AARCH64_DBG_REG_WVR, i, 0UL);
}
}
}
@ -827,7 +886,7 @@ static int hw_breakpoint_reset_notify(struct notifier_block *self,
{
int cpu = (long)hcpu;
if (action == CPU_ONLINE)
smp_call_function_single(cpu, reset_ctrl_regs, NULL, 1);
smp_call_function_single(cpu, hw_breakpoint_reset, NULL, 1);
return NOTIFY_OK;
}
@ -835,6 +894,14 @@ static struct notifier_block hw_breakpoint_reset_nb = {
.notifier_call = hw_breakpoint_reset_notify,
};
#ifdef CONFIG_ARM64_CPU_SUSPEND
extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *));
#else
static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
{
}
#endif
/*
* One-time initialisation.
*/
@ -850,8 +917,8 @@ static int __init arch_hw_breakpoint_init(void)
* Reset the breakpoint resources. We assume that a halting
* debugger will leave the world in a nice state for us.
*/
smp_call_function(reset_ctrl_regs, NULL, 1);
reset_ctrl_regs(NULL);
smp_call_function(hw_breakpoint_reset, NULL, 1);
hw_breakpoint_reset(NULL);
/* Register debug fault handlers. */
hook_debug_fault_code(DBG_ESR_EVT_HWBP, breakpoint_handler, SIGTRAP,
@ -861,6 +928,8 @@ static int __init arch_hw_breakpoint_init(void)
/* Register hotplug notifier. */
register_cpu_notifier(&hw_breakpoint_reset_nb);
/* Register cpu_suspend hw breakpoint restore hook */
cpu_suspend_set_dbg_restorer(hw_breakpoint_reset);
return 0;
}

304
arch/arm64/kernel/insn.c Normal file
View File

@ -0,0 +1,304 @@
/*
* Copyright (C) 2013 Huawei Ltd.
* Author: Jiang Liu <liuj97@gmail.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/bitops.h>
#include <linux/compiler.h>
#include <linux/kernel.h>
#include <linux/smp.h>
#include <linux/stop_machine.h>
#include <linux/uaccess.h>
#include <asm/cacheflush.h>
#include <asm/insn.h>
static int aarch64_insn_encoding_class[] = {
AARCH64_INSN_CLS_UNKNOWN,
AARCH64_INSN_CLS_UNKNOWN,
AARCH64_INSN_CLS_UNKNOWN,
AARCH64_INSN_CLS_UNKNOWN,
AARCH64_INSN_CLS_LDST,
AARCH64_INSN_CLS_DP_REG,
AARCH64_INSN_CLS_LDST,
AARCH64_INSN_CLS_DP_FPSIMD,
AARCH64_INSN_CLS_DP_IMM,
AARCH64_INSN_CLS_DP_IMM,
AARCH64_INSN_CLS_BR_SYS,
AARCH64_INSN_CLS_BR_SYS,
AARCH64_INSN_CLS_LDST,
AARCH64_INSN_CLS_DP_REG,
AARCH64_INSN_CLS_LDST,
AARCH64_INSN_CLS_DP_FPSIMD,
};
enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
{
return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
}
/* NOP is an alias of HINT */
bool __kprobes aarch64_insn_is_nop(u32 insn)
{
if (!aarch64_insn_is_hint(insn))
return false;
switch (insn & 0xFE0) {
case AARCH64_INSN_HINT_YIELD:
case AARCH64_INSN_HINT_WFE:
case AARCH64_INSN_HINT_WFI:
case AARCH64_INSN_HINT_SEV:
case AARCH64_INSN_HINT_SEVL:
return false;
default:
return true;
}
}
/*
* In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
* little-endian.
*/
int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
{
int ret;
u32 val;
ret = probe_kernel_read(&val, addr, AARCH64_INSN_SIZE);
if (!ret)
*insnp = le32_to_cpu(val);
return ret;
}
int __kprobes aarch64_insn_write(void *addr, u32 insn)
{
insn = cpu_to_le32(insn);
return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE);
}
static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
{
if (aarch64_get_insn_class(insn) != AARCH64_INSN_CLS_BR_SYS)
return false;
return aarch64_insn_is_b(insn) ||
aarch64_insn_is_bl(insn) ||
aarch64_insn_is_svc(insn) ||
aarch64_insn_is_hvc(insn) ||
aarch64_insn_is_smc(insn) ||
aarch64_insn_is_brk(insn) ||
aarch64_insn_is_nop(insn);
}
/*
* ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
* Section B2.6.5 "Concurrent modification and execution of instructions":
* Concurrent modification and execution of instructions can lead to the
* resulting instruction performing any behavior that can be achieved by
* executing any sequence of instructions that can be executed from the
* same Exception level, except where the instruction before modification
* and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC,
* or SMC instruction.
*/
bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
{
return __aarch64_insn_hotpatch_safe(old_insn) &&
__aarch64_insn_hotpatch_safe(new_insn);
}
int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
{
u32 *tp = addr;
int ret;
/* A64 instructions must be word aligned */
if ((uintptr_t)tp & 0x3)
return -EINVAL;
ret = aarch64_insn_write(tp, insn);
if (ret == 0)
flush_icache_range((uintptr_t)tp,
(uintptr_t)tp + AARCH64_INSN_SIZE);
return ret;
}
struct aarch64_insn_patch {
void **text_addrs;
u32 *new_insns;
int insn_cnt;
atomic_t cpu_count;
};
static int __kprobes aarch64_insn_patch_text_cb(void *arg)
{
int i, ret = 0;
struct aarch64_insn_patch *pp = arg;
/* The first CPU becomes master */
if (atomic_inc_return(&pp->cpu_count) == 1) {
for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
pp->new_insns[i]);
/*
* aarch64_insn_patch_text_nosync() calls flush_icache_range(),
* which ends with "dsb; isb" pair guaranteeing global
* visibility.
*/
atomic_set(&pp->cpu_count, -1);
} else {
while (atomic_read(&pp->cpu_count) != -1)
cpu_relax();
isb();
}
return ret;
}
int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
{
struct aarch64_insn_patch patch = {
.text_addrs = addrs,
.new_insns = insns,
.insn_cnt = cnt,
.cpu_count = ATOMIC_INIT(0),
};
if (cnt <= 0)
return -EINVAL;
return stop_machine(aarch64_insn_patch_text_cb, &patch,
cpu_online_mask);
}
int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
{
int ret;
u32 insn;
/* Unsafe to patch multiple instructions without synchronizaiton */
if (cnt == 1) {
ret = aarch64_insn_read(addrs[0], &insn);
if (ret)
return ret;
if (aarch64_insn_hotpatch_safe(insn, insns[0])) {
/*
* ARMv8 architecture doesn't guarantee all CPUs see
* the new instruction after returning from function
* aarch64_insn_patch_text_nosync(). So send IPIs to
* all other CPUs to achieve instruction
* synchronization.
*/
ret = aarch64_insn_patch_text_nosync(addrs[0], insns[0]);
kick_all_cpus_sync();
return ret;
}
}
return aarch64_insn_patch_text_sync(addrs, insns, cnt);
}
u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
u32 insn, u64 imm)
{
u32 immlo, immhi, lomask, himask, mask;
int shift;
switch (type) {
case AARCH64_INSN_IMM_ADR:
lomask = 0x3;
himask = 0x7ffff;
immlo = imm & lomask;
imm >>= 2;
immhi = imm & himask;
imm = (immlo << 24) | (immhi);
mask = (lomask << 24) | (himask);
shift = 5;
break;
case AARCH64_INSN_IMM_26:
mask = BIT(26) - 1;
shift = 0;
break;
case AARCH64_INSN_IMM_19:
mask = BIT(19) - 1;
shift = 5;
break;
case AARCH64_INSN_IMM_16:
mask = BIT(16) - 1;
shift = 5;
break;
case AARCH64_INSN_IMM_14:
mask = BIT(14) - 1;
shift = 5;
break;
case AARCH64_INSN_IMM_12:
mask = BIT(12) - 1;
shift = 10;
break;
case AARCH64_INSN_IMM_9:
mask = BIT(9) - 1;
shift = 12;
break;
default:
pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n",
type);
return 0;
}
/* Update the immediate field. */
insn &= ~(mask << shift);
insn |= (imm & mask) << shift;
return insn;
}
u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
enum aarch64_insn_branch_type type)
{
u32 insn;
long offset;
/*
* PC: A 64-bit Program Counter holding the address of the current
* instruction. A64 instructions must be word-aligned.
*/
BUG_ON((pc & 0x3) || (addr & 0x3));
/*
* B/BL support [-128M, 128M) offset
* ARM64 virtual address arrangement guarantees all kernel and module
* texts are within +/-128M.
*/
offset = ((long)addr - (long)pc);
BUG_ON(offset < -SZ_128M || offset >= SZ_128M);
if (type == AARCH64_INSN_BRANCH_LINK)
insn = aarch64_insn_get_bl_value();
else
insn = aarch64_insn_get_b_value();
return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
offset >> 2);
}
u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
{
return aarch64_insn_get_hint_value() | op;
}
u32 __kprobes aarch64_insn_gen_nop(void)
{
return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
}

58
arch/arm64/kernel/jump_label.c Normal file
View File

@ -0,0 +1,58 @@
/*
* Copyright (C) 2013 Huawei Ltd.
* Author: Jiang Liu <liuj97@gmail.com>
*
* Based on arch/arm/kernel/jump_label.c
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/kernel.h>
#include <linux/jump_label.h>
#include <asm/insn.h>
#ifdef HAVE_JUMP_LABEL
static void __arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type,
bool is_static)
{
void *addr = (void *)entry->code;
u32 insn;
if (type == JUMP_LABEL_ENABLE) {
insn = aarch64_insn_gen_branch_imm(entry->code,
entry->target,
AARCH64_INSN_BRANCH_NOLINK);
} else {
insn = aarch64_insn_gen_nop();
}
if (is_static)
aarch64_insn_patch_text_nosync(addr, insn);
else
aarch64_insn_patch_text(&addr, &insn, 1);
}
void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
{
__arch_jump_label_transform(entry, type, false);
}
void arch_jump_label_transform_static(struct jump_entry *entry,
enum jump_label_type type)
{
__arch_jump_label_transform(entry, type, true);
}
#endif /* HAVE_JUMP_LABEL */

157
arch/arm64/kernel/module.c
View File

@ -25,6 +25,10 @@
#include <linux/mm.h>
#include <linux/moduleloader.h>
#include <linux/vmalloc.h>
#include <asm/insn.h>
#define AARCH64_INSN_IMM_MOVNZ AARCH64_INSN_IMM_MAX
#define AARCH64_INSN_IMM_MOVK AARCH64_INSN_IMM_16
void *module_alloc(unsigned long size)
{
@ -94,28 +98,18 @@ static int reloc_data(enum aarch64_reloc_op op, void *place, u64 val, int len)
return 0;
}
enum aarch64_imm_type {
INSN_IMM_MOVNZ,
INSN_IMM_MOVK,
INSN_IMM_ADR,
INSN_IMM_26,
INSN_IMM_19,
INSN_IMM_16,
INSN_IMM_14,
INSN_IMM_12,
INSN_IMM_9,
};
static u32 encode_insn_immediate(enum aarch64_imm_type type, u32 insn, u64 imm)
static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val,
int lsb, enum aarch64_insn_imm_type imm_type)
{
u32 immlo, immhi, lomask, himask, mask;
int shift;
u64 imm, limit = 0;
s64 sval;
u32 insn = le32_to_cpu(*(u32 *)place);
/* The instruction stream is always little endian. */
insn = le32_to_cpu(insn);
sval = do_reloc(op, place, val);
sval >>= lsb;
imm = sval & 0xffff;
switch (type) {
case INSN_IMM_MOVNZ:
if (imm_type == AARCH64_INSN_IMM_MOVNZ) {
/*
* For signed MOVW relocations, we have to manipulate the
* instruction encoding depending on whether or not the
@ -134,70 +128,12 @@ static u32 encode_insn_immediate(enum aarch64_imm_type type, u32 insn, u64 imm)
*/
imm = ~imm;
}
case INSN_IMM_MOVK:
mask = BIT(16) - 1;
shift = 5;
break;
case INSN_IMM_ADR:
lomask = 0x3;
himask = 0x7ffff;
immlo = imm & lomask;
imm >>= 2;
immhi = imm & himask;
imm = (immlo << 24) | (immhi);
mask = (lomask << 24) | (himask);
shift = 5;
break;
case INSN_IMM_26:
mask = BIT(26) - 1;
shift = 0;
break;
case INSN_IMM_19:
mask = BIT(19) - 1;
shift = 5;
break;
case INSN_IMM_16:
mask = BIT(16) - 1;
shift = 5;
break;
case INSN_IMM_14:
mask = BIT(14) - 1;
shift = 5;
break;
case INSN_IMM_12:
mask = BIT(12) - 1;
shift = 10;
break;
case INSN_IMM_9:
mask = BIT(9) - 1;
shift = 12;
break;
default:
pr_err("encode_insn_immediate: unknown immediate encoding %d\n",
type);
return 0;
imm_type = AARCH64_INSN_IMM_MOVK;
}
/* Update the immediate field. */
insn &= ~(mask << shift);
insn |= (imm & mask) << shift;
return cpu_to_le32(insn);
}
static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val,
int lsb, enum aarch64_imm_type imm_type)
{
u64 imm, limit = 0;
s64 sval;
u32 insn = *(u32 *)place;
sval = do_reloc(op, place, val);
sval >>= lsb;
imm = sval & 0xffff;
/* Update the instruction with the new encoding. */
*(u32 *)place = encode_insn_immediate(imm_type, insn, imm);
insn = aarch64_insn_encode_immediate(imm_type, insn, imm);
*(u32 *)place = cpu_to_le32(insn);
/* Shift out the immediate field. */
sval >>= 16;
@ -206,9 +142,9 @@ static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val,
* For unsigned immediates, the overflow check is straightforward.
* For signed immediates, the sign bit is actually the bit past the
* most significant bit of the field.
* The INSN_IMM_16 immediate type is unsigned.
* The AARCH64_INSN_IMM_16 immediate type is unsigned.
*/
if (imm_type != INSN_IMM_16) {
if (imm_type != AARCH64_INSN_IMM_16) {
sval++;
limit++;
}
@ -221,11 +157,11 @@ static int reloc_insn_movw(enum aarch64_reloc_op op, void *place, u64 val,
}
static int reloc_insn_imm(enum aarch64_reloc_op op, void *place, u64 val,
int lsb, int len, enum aarch64_imm_type imm_type)
int lsb, int len, enum aarch64_insn_imm_type imm_type)
{
u64 imm, imm_mask;
s64 sval;
u32 insn = *(u32 *)place;
u32 insn = le32_to_cpu(*(u32 *)place);
/* Calculate the relocation value. */
sval = do_reloc(op, place, val);
@ -236,7 +172,8 @@ static int reloc_insn_imm(enum aarch64_reloc_op op, void *place, u64 val,
imm = sval & imm_mask;
/* Update the instruction's immediate field. */
*(u32 *)place = encode_insn_immediate(imm_type, insn, imm);
insn = aarch64_insn_encode_immediate(imm_type, insn, imm);
*(u32 *)place = cpu_to_le32(insn);
/*
* Extract the upper value bits (including the sign bit) and
@ -318,125 +255,125 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
overflow_check = false;
case R_AARCH64_MOVW_UABS_G0:
ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0,
INSN_IMM_16);
AARCH64_INSN_IMM_16);
break;
case R_AARCH64_MOVW_UABS_G1_NC:
overflow_check = false;
case R_AARCH64_MOVW_UABS_G1:
ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16,
INSN_IMM_16);
AARCH64_INSN_IMM_16);
break;
case R_AARCH64_MOVW_UABS_G2_NC:
overflow_check = false;
case R_AARCH64_MOVW_UABS_G2:
ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32,
INSN_IMM_16);
AARCH64_INSN_IMM_16);
break;
case R_AARCH64_MOVW_UABS_G3:
/* We're using the top bits so we can't overflow. */
overflow_check = false;
ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 48,
INSN_IMM_16);
AARCH64_INSN_IMM_16);
break;
case R_AARCH64_MOVW_SABS_G0:
ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 0,
INSN_IMM_MOVNZ);
AARCH64_INSN_IMM_MOVNZ);
break;
case R_AARCH64_MOVW_SABS_G1:
ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 16,
INSN_IMM_MOVNZ);
AARCH64_INSN_IMM_MOVNZ);
break;
case R_AARCH64_MOVW_SABS_G2:
ovf = reloc_insn_movw(RELOC_OP_ABS, loc, val, 32,
INSN_IMM_MOVNZ);
AARCH64_INSN_IMM_MOVNZ);
break;
case R_AARCH64_MOVW_PREL_G0_NC:
overflow_check = false;
ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0,
INSN_IMM_MOVK);
AARCH64_INSN_IMM_MOVK);
break;
case R_AARCH64_MOVW_PREL_G0:
ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 0,
INSN_IMM_MOVNZ);
AARCH64_INSN_IMM_MOVNZ);
break;
case R_AARCH64_MOVW_PREL_G1_NC:
overflow_check = false;
ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16,
INSN_IMM_MOVK);
AARCH64_INSN_IMM_MOVK);
break;
case R_AARCH64_MOVW_PREL_G1:
ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 16,
INSN_IMM_MOVNZ);
AARCH64_INSN_IMM_MOVNZ);
break;
case R_AARCH64_MOVW_PREL_G2_NC:
overflow_check = false;
ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32,
INSN_IMM_MOVK);
AARCH64_INSN_IMM_MOVK);
break;
case R_AARCH64_MOVW_PREL_G2:
ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 32,
INSN_IMM_MOVNZ);
AARCH64_INSN_IMM_MOVNZ);
break;
case R_AARCH64_MOVW_PREL_G3:
/* We're using the top bits so we can't overflow. */
overflow_check = false;
ovf = reloc_insn_movw(RELOC_OP_PREL, loc, val, 48,
INSN_IMM_MOVNZ);
AARCH64_INSN_IMM_MOVNZ);
break;
/* Immediate instruction relocations. */
case R_AARCH64_LD_PREL_LO19:
ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19,
INSN_IMM_19);
AARCH64_INSN_IMM_19);
break;
case R_AARCH64_ADR_PREL_LO21:
ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 0, 21,
INSN_IMM_ADR);
AARCH64_INSN_IMM_ADR);
break;
case R_AARCH64_ADR_PREL_PG_HI21_NC:
overflow_check = false;
case R_AARCH64_ADR_PREL_PG_HI21:
ovf = reloc_insn_imm(RELOC_OP_PAGE, loc, val, 12, 21,
INSN_IMM_ADR);
AARCH64_INSN_IMM_ADR);
break;
case R_AARCH64_ADD_ABS_LO12_NC:
case R_AARCH64_LDST8_ABS_LO12_NC:
overflow_check = false;
ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 0, 12,
INSN_IMM_12);
AARCH64_INSN_IMM_12);
break;
case R_AARCH64_LDST16_ABS_LO12_NC:
overflow_check = false;
ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 1, 11,
INSN_IMM_12);
AARCH64_INSN_IMM_12);
break;
case R_AARCH64_LDST32_ABS_LO12_NC:
overflow_check = false;
ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 2, 10,
INSN_IMM_12);
AARCH64_INSN_IMM_12);
break;
case R_AARCH64_LDST64_ABS_LO12_NC:
overflow_check = false;
ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 3, 9,
INSN_IMM_12);
AARCH64_INSN_IMM_12);
break;
case R_AARCH64_LDST128_ABS_LO12_NC:
overflow_check = false;
ovf = reloc_insn_imm(RELOC_OP_ABS, loc, val, 4, 8,
INSN_IMM_12);
AARCH64_INSN_IMM_12);
break;
case R_AARCH64_TSTBR14:
ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 14,
INSN_IMM_14);
AARCH64_INSN_IMM_14);
break;
case R_AARCH64_CONDBR19:
ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 19,
INSN_IMM_19);
AARCH64_INSN_IMM_19);
break;
case R_AARCH64_JUMP26:
case R_AARCH64_CALL26:
ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2, 26,
INSN_IMM_26);
AARCH64_INSN_IMM_26);
break;
default:

108
arch/arm64/kernel/perf_event.c
View File

@ -22,6 +22,7 @@
#include <linux/bitmap.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include <linux/perf_event.h>
@ -362,27 +363,54 @@ validate_group(struct perf_event *event)
return 0;
}
static void
armpmu_disable_percpu_irq(void *data)
{
unsigned int irq = *(unsigned int *)data;
disable_percpu_irq(irq);
}
static void
armpmu_release_hardware(struct arm_pmu *armpmu)
{
int i, irq, irqs;
int irq;
unsigned int i, irqs;
struct platform_device *pmu_device = armpmu->plat_device;
irqs = min(pmu_device->num_resources, num_possible_cpus());
if (!irqs)
return;
for (i = 0; i < irqs; ++i) {
if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs))
continue;
irq = platform_get_irq(pmu_device, i);
if (irq >= 0)
free_irq(irq, armpmu);
irq = platform_get_irq(pmu_device, 0);
if (irq <= 0)
return;
if (irq_is_percpu(irq)) {
on_each_cpu(armpmu_disable_percpu_irq, &irq, 1);
free_percpu_irq(irq, &cpu_hw_events);
} else {
for (i = 0; i < irqs; ++i) {
if (!cpumask_test_and_clear_cpu(i, &armpmu->active_irqs))
continue;
irq = platform_get_irq(pmu_device, i);
if (irq > 0)
free_irq(irq, armpmu);
}
}
}
static void
armpmu_enable_percpu_irq(void *data)
{
unsigned int irq = *(unsigned int *)data;
enable_percpu_irq(irq, IRQ_TYPE_NONE);
}
static int
armpmu_reserve_hardware(struct arm_pmu *armpmu)
{
int i, err, irq, irqs;
int err, irq;
unsigned int i, irqs;
struct platform_device *pmu_device = armpmu->plat_device;
if (!pmu_device) {
@ -391,39 +419,59 @@ armpmu_reserve_hardware(struct arm_pmu *armpmu)
}
irqs = min(pmu_device->num_resources, num_possible_cpus());
if (irqs < 1) {
if (!irqs) {
pr_err("no irqs for PMUs defined\n");
return -ENODEV;
}
for (i = 0; i < irqs; ++i) {
err = 0;
irq = platform_get_irq(pmu_device, i);
if (irq < 0)
continue;
irq = platform_get_irq(pmu_device, 0);
if (irq <= 0) {
pr_err("failed to get valid irq for PMU device\n");
return -ENODEV;
}
/*
* If we have a single PMU interrupt that we can't shift,
* assume that we're running on a uniprocessor machine and
* continue. Otherwise, continue without this interrupt.
*/
if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) {
pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n",
irq, i);
continue;
}
if (irq_is_percpu(irq)) {
err = request_percpu_irq(irq, armpmu->handle_irq,
"arm-pmu", &cpu_hw_events);
err = request_irq(irq, armpmu->handle_irq,
IRQF_NOBALANCING,
"arm-pmu", armpmu);
if (err) {
pr_err("unable to request IRQ%d for ARM PMU counters\n",
irq);
pr_err("unable to request percpu IRQ%d for ARM PMU counters\n",
irq);
armpmu_release_hardware(armpmu);
return err;
}
cpumask_set_cpu(i, &armpmu->active_irqs);
on_each_cpu(armpmu_enable_percpu_irq, &irq, 1);
} else {
for (i = 0; i < irqs; ++i) {
err = 0;
irq = platform_get_irq(pmu_device, i);
if (irq <= 0)
continue;
/*
* If we have a single PMU interrupt that we can't shift,
* assume that we're running on a uniprocessor machine and
* continue. Otherwise, continue without this interrupt.
*/
if (irq_set_affinity(irq, cpumask_of(i)) && irqs > 1) {
pr_warning("unable to set irq affinity (irq=%d, cpu=%u)\n",
irq, i);
continue;
}
err = request_irq(irq, armpmu->handle_irq,
IRQF_NOBALANCING,
"arm-pmu", armpmu);
if (err) {
pr_err("unable to request IRQ%d for ARM PMU counters\n",
irq);
armpmu_release_hardware(armpmu);
return err;
}
cpumask_set_cpu(i, &armpmu->active_irqs);
}
}
return 0;

14
arch/arm64/kernel/process.c
View File

@ -33,6 +33,7 @@
#include <linux/kallsyms.h>
#include <linux/init.h>
#include <linux/cpu.h>
#include <linux/cpuidle.h>
#include <linux/elfcore.h>
#include <linux/pm.h>
#include <linux/tick.h>
@ -98,8 +99,10 @@ void arch_cpu_idle(void)
* This should do all the clock switching and wait for interrupt
* tricks
*/
cpu_do_idle();
local_irq_enable();
if (cpuidle_idle_call()) {
cpu_do_idle();
local_irq_enable();
}
}
#ifdef CONFIG_HOTPLUG_CPU
@ -308,6 +311,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
unsigned long get_wchan(struct task_struct *p)
{
struct stackframe frame;
unsigned long stack_page;
int count = 0;
if (!p || p == current || p->state == TASK_RUNNING)
return 0;
@ -315,9 +319,11 @@ unsigned long get_wchan(struct task_struct *p)
frame.fp = thread_saved_fp(p);
frame.sp = thread_saved_sp(p);
frame.pc = thread_saved_pc(p);
stack_page = (unsigned long)task_stack_page(p);
do {
int ret = unwind_frame(&frame);
if (ret < 0)
if (frame.sp < stack_page ||
frame.sp >= stack_page + THREAD_SIZE ||
unwind_frame(&frame))
return 0;
if (!in_sched_functions(frame.pc))
return frame.pc;

122
arch/arm64/kernel/setup.c
View File

@ -108,20 +108,95 @@ void __init early_print(const char *str, ...)
printk("%s", buf);
}
void __init smp_setup_processor_id(void)
{
/*
* clear __my_cpu_offset on boot CPU to avoid hang caused by
* using percpu variable early, for example, lockdep will
* access percpu variable inside lock_release
*/
set_my_cpu_offset(0);
}
bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
{
return phys_id == cpu_logical_map(cpu);
}
struct mpidr_hash mpidr_hash;
#ifdef CONFIG_SMP
/**
* smp_build_mpidr_hash - Pre-compute shifts required at each affinity
* level in order to build a linear index from an
* MPIDR value. Resulting algorithm is a collision
* free hash carried out through shifting and ORing
*/
static void __init smp_build_mpidr_hash(void)
{
u32 i, affinity, fs[4], bits[4], ls;
u64 mask = 0;
/*
* Pre-scan the list of MPIDRS and filter out bits that do
* not contribute to affinity levels, ie they never toggle.
*/
for_each_possible_cpu(i)
mask |= (cpu_logical_map(i) ^ cpu_logical_map(0));
pr_debug("mask of set bits %#llx\n", mask);
/*
* Find and stash the last and first bit set at all affinity levels to
* check how many bits are required to represent them.
*/
for (i = 0; i < 4; i++) {
affinity = MPIDR_AFFINITY_LEVEL(mask, i);
/*
* Find the MSB bit and LSB bits position
* to determine how many bits are required
* to express the affinity level.
*/
ls = fls(affinity);
fs[i] = affinity ? ffs(affinity) - 1 : 0;
bits[i] = ls - fs[i];
}
/*
* An index can be created from the MPIDR_EL1 by isolating the
* significant bits at each affinity level and by shifting
* them in order to compress the 32 bits values space to a
* compressed set of values. This is equivalent to hashing
* the MPIDR_EL1 through shifting and ORing. It is a collision free
* hash though not minimal since some levels might contain a number
* of CPUs that is not an exact power of 2 and their bit
* representation might contain holes, eg MPIDR_EL1[7:0] = {0x2, 0x80}.
*/
mpidr_hash.shift_aff[0] = MPIDR_LEVEL_SHIFT(0) + fs[0];
mpidr_hash.shift_aff[1] = MPIDR_LEVEL_SHIFT(1) + fs[1] - bits[0];
mpidr_hash.shift_aff[2] = MPIDR_LEVEL_SHIFT(2) + fs[2] -
(bits[1] + bits[0]);
mpidr_hash.shift_aff[3] = MPIDR_LEVEL_SHIFT(3) +
fs[3] - (bits[2] + bits[1] + bits[0]);
mpidr_hash.mask = mask;
mpidr_hash.bits = bits[3] + bits[2] + bits[1] + bits[0];
pr_debug("MPIDR hash: aff0[%u] aff1[%u] aff2[%u] aff3[%u] mask[%#llx] bits[%u]\n",
mpidr_hash.shift_aff[0],
mpidr_hash.shift_aff[1],
mpidr_hash.shift_aff[2],
mpidr_hash.shift_aff[3],
mpidr_hash.mask,
mpidr_hash.bits);
/*
* 4x is an arbitrary value used to warn on a hash table much bigger
* than expected on most systems.
*/
if (mpidr_hash_size() > 4 * num_possible_cpus())
pr_warn("Large number of MPIDR hash buckets detected\n");
__flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash));
}
#endif
static void __init setup_processor(void)
{
struct cpu_info *cpu_info;
u64 features, block;
/*
* locate processor in the list of supported processor
* types. The linker builds this table for us from the
* entries in arch/arm/mm/proc.S
*/
cpu_info = lookup_processor_type(read_cpuid_id());
if (!cpu_info) {
printk("CPU configuration botched (ID %08x), unable to continue.\n",
@ -136,6 +211,37 @@ static void __init setup_processor(void)
sprintf(init_utsname()->machine, ELF_PLATFORM);
elf_hwcap = 0;
/*
* ID_AA64ISAR0_EL1 contains 4-bit wide signed feature blocks.
* The blocks we test below represent incremental functionality
* for non-negative values. Negative values are reserved.
*/
features = read_cpuid(ID_AA64ISAR0_EL1);
block = (features >> 4) & 0xf;
if (!(block & 0x8)) {
switch (block) {
default:
case 2:
elf_hwcap |= HWCAP_PMULL;
case 1:
elf_hwcap |= HWCAP_AES;
case 0:
break;
}
}
block = (features >> 8) & 0xf;
if (block && !(block & 0x8))
elf_hwcap |= HWCAP_SHA1;
block = (features >> 12) & 0xf;
if (block && !(block & 0x8))
elf_hwcap |= HWCAP_SHA2;
block = (features >> 16) & 0xf;
if (block && !(block & 0x8))
elf_hwcap |= HWCAP_CRC32;
}
static void __init setup_machine_fdt(phys_addr_t dt_phys)
@ -236,6 +342,7 @@ void __init setup_arch(char **cmdline_p)
cpu_read_bootcpu_ops();
#ifdef CONFIG_SMP
smp_init_cpus();
smp_build_mpidr_hash();
#endif
#ifdef CONFIG_VT
@ -275,6 +382,11 @@ static const char *hwcap_str[] = {
"fp",
"asimd",
"evtstrm",
"aes",
"pmull",
"sha1",
"sha2",
"crc32",
NULL
};

184
arch/arm64/kernel/sleep.S Normal file
View File

@ -0,0 +1,184 @@
#include <linux/errno.h>
#include <linux/linkage.h>
#include <asm/asm-offsets.h>
#include <asm/assembler.h>
.text
/*
* Implementation of MPIDR_EL1 hash algorithm through shifting
* and OR'ing.
*
* @dst: register containing hash result
* @rs0: register containing affinity level 0 bit shift
* @rs1: register containing affinity level 1 bit shift
* @rs2: register containing affinity level 2 bit shift
* @rs3: register containing affinity level 3 bit shift
* @mpidr: register containing MPIDR_EL1 value
* @mask: register containing MPIDR mask
*
* Pseudo C-code:
*
*u32 dst;
*
*compute_mpidr_hash(u32 rs0, u32 rs1, u32 rs2, u32 rs3, u64 mpidr, u64 mask) {
* u32 aff0, aff1, aff2, aff3;
* u64 mpidr_masked = mpidr & mask;
* aff0 = mpidr_masked & 0xff;
* aff1 = mpidr_masked & 0xff00;
* aff2 = mpidr_masked & 0xff0000;
* aff2 = mpidr_masked & 0xff00000000;
* dst = (aff0 >> rs0 | aff1 >> rs1 | aff2 >> rs2 | aff3 >> rs3);
*}
* Input registers: rs0, rs1, rs2, rs3, mpidr, mask
* Output register: dst
* Note: input and output registers must be disjoint register sets
(eg: a macro instance with mpidr = x1 and dst = x1 is invalid)
*/
.macro compute_mpidr_hash dst, rs0, rs1, rs2, rs3, mpidr, mask
and \mpidr, \mpidr, \mask // mask out MPIDR bits
and \dst, \mpidr, #0xff // mask=aff0
lsr \dst ,\dst, \rs0 // dst=aff0>>rs0
and \mask, \mpidr, #0xff00 // mask = aff1
lsr \mask ,\mask, \rs1
orr \dst, \dst, \mask // dst|=(aff1>>rs1)
and \mask, \mpidr, #0xff0000 // mask = aff2
lsr \mask ,\mask, \rs2
orr \dst, \dst, \mask // dst|=(aff2>>rs2)
and \mask, \mpidr, #0xff00000000 // mask = aff3
lsr \mask ,\mask, \rs3
orr \dst, \dst, \mask // dst|=(aff3>>rs3)
.endm
/*
* Save CPU state for a suspend. This saves callee registers, and allocates
* space on the kernel stack to save the CPU specific registers + some
* other data for resume.
*
* x0 = suspend finisher argument
*/
ENTRY(__cpu_suspend)
stp x29, lr, [sp, #-96]!
stp x19, x20, [sp,#16]
stp x21, x22, [sp,#32]
stp x23, x24, [sp,#48]
stp x25, x26, [sp,#64]
stp x27, x28, [sp,#80]
mov x2, sp
sub sp, sp, #CPU_SUSPEND_SZ // allocate cpu_suspend_ctx
mov x1, sp
/*
* x1 now points to struct cpu_suspend_ctx allocated on the stack
*/
str x2, [x1, #CPU_CTX_SP]
ldr x2, =sleep_save_sp
ldr x2, [x2, #SLEEP_SAVE_SP_VIRT]
#ifdef CONFIG_SMP
mrs x7, mpidr_el1
ldr x9, =mpidr_hash
ldr x10, [x9, #MPIDR_HASH_MASK]
/*
* Following code relies on the struct mpidr_hash
* members size.
*/
ldp w3, w4, [x9, #MPIDR_HASH_SHIFTS]
ldp w5, w6, [x9, #(MPIDR_HASH_SHIFTS + 8)]
compute_mpidr_hash x8, x3, x4, x5, x6, x7, x10
add x2, x2, x8, lsl #3
#endif
bl __cpu_suspend_finisher
/*
* Never gets here, unless suspend fails.
* Successful cpu_suspend should return from cpu_resume, returning
* through this code path is considered an error
* If the return value is set to 0 force x0 = -EOPNOTSUPP
* to make sure a proper error condition is propagated
*/
cmp x0, #0
mov x3, #-EOPNOTSUPP
csel x0, x3, x0, eq
add sp, sp, #CPU_SUSPEND_SZ // rewind stack pointer
ldp x19, x20, [sp, #16]
ldp x21, x22, [sp, #32]
ldp x23, x24, [sp, #48]
ldp x25, x26, [sp, #64]
ldp x27, x28, [sp, #80]
ldp x29, lr, [sp], #96
ret
ENDPROC(__cpu_suspend)
.ltorg
/*
* x0 must contain the sctlr value retrieved from restored context
*/
ENTRY(cpu_resume_mmu)
ldr x3, =cpu_resume_after_mmu
msr sctlr_el1, x0 // restore sctlr_el1
isb
br x3 // global jump to virtual address
ENDPROC(cpu_resume_mmu)
cpu_resume_after_mmu:
mov x0, #0 // return zero on success
ldp x19, x20, [sp, #16]
ldp x21, x22, [sp, #32]
ldp x23, x24, [sp, #48]
ldp x25, x26, [sp, #64]
ldp x27, x28, [sp, #80]
ldp x29, lr, [sp], #96
ret
ENDPROC(cpu_resume_after_mmu)
.data
ENTRY(cpu_resume)
bl el2_setup // if in EL2 drop to EL1 cleanly
#ifdef CONFIG_SMP
mrs x1, mpidr_el1
adr x4, mpidr_hash_ptr
ldr x5, [x4]
add x8, x4, x5 // x8 = struct mpidr_hash phys address
/* retrieve mpidr_hash members to compute the hash */
ldr x2, [x8, #MPIDR_HASH_MASK]
ldp w3, w4, [x8, #MPIDR_HASH_SHIFTS]
ldp w5, w6, [x8, #(MPIDR_HASH_SHIFTS + 8)]
compute_mpidr_hash x7, x3, x4, x5, x6, x1, x2
/* x7 contains hash index, let's use it to grab context pointer */
#else
mov x7, xzr
#endif
adr x0, sleep_save_sp
ldr x0, [x0, #SLEEP_SAVE_SP_PHYS]
ldr x0, [x0, x7, lsl #3]
/* load sp from context */
ldr x2, [x0, #CPU_CTX_SP]
adr x1, sleep_idmap_phys
/* load physical address of identity map page table in x1 */
ldr x1, [x1]
mov sp, x2
/*
* cpu_do_resume expects x0 to contain context physical address
* pointer and x1 to contain physical address of 1:1 page tables
*/
bl cpu_do_resume // PC relative jump, MMU off
b cpu_resume_mmu // Resume MMU, never returns
ENDPROC(cpu_resume)
.align 3
mpidr_hash_ptr:
/*
* offset of mpidr_hash symbol from current location
* used to obtain run-time mpidr_hash address with MMU off
*/
.quad mpidr_hash - .
/*
* physical address of identity mapped page tables
*/
.type sleep_idmap_phys, #object
ENTRY(sleep_idmap_phys)
.quad 0
/*
* struct sleep_save_sp {
* phys_addr_t *save_ptr_stash;
* phys_addr_t save_ptr_stash_phys;
* };
*/
.type sleep_save_sp, #object
ENTRY(sleep_save_sp)
.space SLEEP_SAVE_SP_SZ // struct sleep_save_sp

23
arch/arm64/kernel/smp.c
View File

@ -61,6 +61,7 @@ enum ipi_msg_type {
IPI_CALL_FUNC,
IPI_CALL_FUNC_SINGLE,
IPI_CPU_STOP,
IPI_TIMER,
};
/*
@ -122,8 +123,6 @@ asmlinkage void secondary_start_kernel(void)
struct mm_struct *mm = &init_mm;
unsigned int cpu = smp_processor_id();
printk("CPU%u: Booted secondary processor\n", cpu);
/*
* All kernel threads share the same mm context; grab a
* reference and switch to it.
@ -132,6 +131,9 @@ asmlinkage void secondary_start_kernel(void)
current->active_mm = mm;
cpumask_set_cpu(cpu, mm_cpumask(mm));
set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
printk("CPU%u: Booted secondary processor\n", cpu);
/*
* TTBR0 is only used for the identity mapping at this stage. Make it
* point to zero page to avoid speculatively fetching new entries.
@ -271,6 +273,7 @@ void __init smp_cpus_done(unsigned int max_cpus)
void __init smp_prepare_boot_cpu(void)
{
set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
}
static void (*smp_cross_call)(const struct cpumask *, unsigned int);
@ -447,6 +450,7 @@ static const char *ipi_types[NR_IPI] = {
S(IPI_CALL_FUNC, "Function call interrupts"),
S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
S(IPI_CPU_STOP, "CPU stop interrupts"),
S(IPI_TIMER, "Timer broadcast interrupts"),
};
void show_ipi_list(struct seq_file *p, int prec)
@ -532,6 +536,14 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
irq_exit();
break;
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
case IPI_TIMER:
irq_enter();
tick_receive_broadcast();
irq_exit();
break;
#endif
default:
pr_crit("CPU%u: Unknown IPI message 0x%x\n", cpu, ipinr);
break;
@ -544,6 +556,13 @@ void smp_send_reschedule(int cpu)
smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
}
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
void tick_broadcast(const struct cpumask *mask)
{
smp_cross_call(mask, IPI_TIMER);
}
#endif
void smp_send_stop(void)
{
unsigned long timeout;

2
arch/arm64/kernel/stacktrace.c
View File

@ -43,7 +43,7 @@ int unwind_frame(struct stackframe *frame)
low = frame->sp;
high = ALIGN(low, THREAD_SIZE);
if (fp < low || fp > high || fp & 0xf)
if (fp < low || fp > high - 0x18 || fp & 0xf)
return -EINVAL;
frame->sp = fp + 0x10;

132
arch/arm64/kernel/suspend.c Normal file
View File

@ -0,0 +1,132 @@
#include <linux/slab.h>
#include <asm/cacheflush.h>
#include <asm/cpu_ops.h>
#include <asm/debug-monitors.h>
#include <asm/pgtable.h>
#include <asm/memory.h>
#include <asm/smp_plat.h>
#include <asm/suspend.h>
#include <asm/tlbflush.h>
extern int __cpu_suspend(unsigned long);
/*
* This is called by __cpu_suspend() to save the state, and do whatever
* flushing is required to ensure that when the CPU goes to sleep we have
* the necessary data available when the caches are not searched.
*
* @arg: Argument to pass to suspend operations
* @ptr: CPU context virtual address
* @save_ptr: address of the location where the context physical address
* must be saved
*/
int __cpu_suspend_finisher(unsigned long arg, struct cpu_suspend_ctx *ptr,
phys_addr_t *save_ptr)
{
int cpu = smp_processor_id();
*save_ptr = virt_to_phys(ptr);
cpu_do_suspend(ptr);
/*
* Only flush the context that must be retrieved with the MMU
* off. VA primitives ensure the flush is applied to all
* cache levels so context is pushed to DRAM.
*/
__flush_dcache_area(ptr, sizeof(*ptr));
__flush_dcache_area(save_ptr, sizeof(*save_ptr));
return cpu_ops[cpu]->cpu_suspend(arg);
}
/*
* This hook is provided so that cpu_suspend code can restore HW
* breakpoints as early as possible in the resume path, before reenabling
* debug exceptions. Code cannot be run from a CPU PM notifier since by the
* time the notifier runs debug exceptions might have been enabled already,
* with HW breakpoints registers content still in an unknown state.
*/
void (*hw_breakpoint_restore)(void *);
void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
{
/* Prevent multiple restore hook initializations */
if (WARN_ON(hw_breakpoint_restore))
return;
hw_breakpoint_restore = hw_bp_restore;
}
/**
* cpu_suspend
*
* @arg: argument to pass to the finisher function
*/
int cpu_suspend(unsigned long arg)
{
struct mm_struct *mm = current->active_mm;
int ret, cpu = smp_processor_id();
unsigned long flags;
/*
* If cpu_ops have not been registered or suspend
* has not been initialized, cpu_suspend call fails early.
*/
if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
return -EOPNOTSUPP;
/*
* From this point debug exceptions are disabled to prevent
* updates to mdscr register (saved and restored along with
* general purpose registers) from kernel debuggers.
*/
local_dbg_save(flags);
/*
* mm context saved on the stack, it will be restored when
* the cpu comes out of reset through the identity mapped
* page tables, so that the thread address space is properly
* set-up on function return.
*/
ret = __cpu_suspend(arg);
if (ret == 0) {
cpu_switch_mm(mm->pgd, mm);
flush_tlb_all();
/*
* Restore HW breakpoint registers to sane values
* before debug exceptions are possibly reenabled
* through local_dbg_restore.
*/
if (hw_breakpoint_restore)
hw_breakpoint_restore(NULL);
}
/*
* Restore pstate flags. OS lock and mdscr have been already
* restored, so from this point onwards, debugging is fully
* renabled if it was enabled when core started shutdown.
*/
local_dbg_restore(flags);
return ret;
}
extern struct sleep_save_sp sleep_save_sp;
extern phys_addr_t sleep_idmap_phys;
static int cpu_suspend_init(void)
{
void *ctx_ptr;
/* ctx_ptr is an array of physical addresses */
ctx_ptr = kcalloc(mpidr_hash_size(), sizeof(phys_addr_t), GFP_KERNEL);
if (WARN_ON(!ctx_ptr))
return -ENOMEM;
sleep_save_sp.save_ptr_stash = ctx_ptr;
sleep_save_sp.save_ptr_stash_phys = virt_to_phys(ctx_ptr);
sleep_idmap_phys = virt_to_phys(idmap_pg_dir);
__flush_dcache_area(&sleep_save_sp, sizeof(struct sleep_save_sp));
__flush_dcache_area(&sleep_idmap_phys, sizeof(sleep_idmap_phys));
return 0;
}
early_initcall(cpu_suspend_init);

3
arch/arm64/kernel/vmlinux.lds.S
View File

@ -99,17 +99,14 @@ SECTIONS
. = ALIGN(PAGE_SIZE);
_data = .;
__data_loc = _data - LOAD_OFFSET;
_sdata = .;
RW_DATA_SECTION(64, PAGE_SIZE, THREAD_SIZE)
_edata = .;
_edata_loc = __data_loc + SIZEOF(.data);
BSS_SECTION(0, 0, 0)
_end = .;
STABS_DEBUG
.comment 0 : { *(.comment) }
}
/*

8
arch/arm64/lib/Makefile
View File

@ -1,6 +1,4 @@
lib-y := bitops.o delay.o \
strncpy_from_user.o strnlen_user.o clear_user.o \
copy_from_user.o copy_to_user.o copy_in_user.o \
copy_page.o clear_page.o \
memchr.o memcpy.o memmove.o memset.o \
lib-y := bitops.o clear_user.o delay.o copy_from_user.o \
copy_to_user.o copy_in_user.o copy_page.o \
clear_page.o memchr.o memcpy.o memmove.o memset.o \
strchr.o strrchr.o

50
arch/arm64/lib/strncpy_from_user.S
View File

@ -1,50 +0,0 @@
/*
* Based on arch/arm/lib/strncpy_from_user.S
*
* Copyright (C) 1995-2000 Russell King
* Copyright (C) 2012 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/errno.h>
.text
.align 5
/*
* Copy a string from user space to kernel space.
* x0 = dst, x1 = src, x2 = byte length
* returns the number of characters copied (strlen of copied string),
* -EFAULT on exception, or "len" if we fill the whole buffer
*/
ENTRY(__strncpy_from_user)
mov x4, x1
1: subs x2, x2, #1
bmi 2f
USER(9f, ldrb w3, [x1], #1 )
strb w3, [x0], #1
cbnz w3, 1b
sub x1, x1, #1 // take NUL character out of count
2: sub x0, x1, x4
ret
ENDPROC(__strncpy_from_user)
.section .fixup,"ax"
.align 0
9: strb wzr, [x0] // null terminate
mov x0, #-EFAULT
ret
.previous

47
arch/arm64/lib/strnlen_user.S
View File

@ -1,47 +0,0 @@
/*
* Based on arch/arm/lib/strnlen_user.S
*
* Copyright (C) 1995-2000 Russell King
* Copyright (C) 2012 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/errno.h>
.text
.align 5
/* Prototype: unsigned long __strnlen_user(const char *str, long n)
* Purpose : get length of a string in user memory
* Params : str - address of string in user memory
* Returns : length of string *including terminator*
* or zero on exception, or n if too long
*/
ENTRY(__strnlen_user)
mov x2, x0
1: subs x1, x1, #1
b.mi 2f
USER(9f, ldrb w3, [x0], #1 )
cbnz w3, 1b
2: sub x0, x0, x2
ret
ENDPROC(__strnlen_user)
.section .fixup,"ax"
.align 0
9: mov x0, #0
ret
.previous

35
arch/arm64/mm/dma-mapping.c
View File

@ -21,6 +21,7 @@
#include <linux/export.h>
#include <linux/slab.h>
#include <linux/dma-mapping.h>
#include <linux/dma-contiguous.h>
#include <linux/vmalloc.h>
#include <linux/swiotlb.h>
@ -33,17 +34,47 @@ static void *arm64_swiotlb_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
struct dma_attrs *attrs)
{
if (dev == NULL) {
WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
return NULL;
}
if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
dev->coherent_dma_mask <= DMA_BIT_MASK(32))
flags |= GFP_DMA32;
return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
if (IS_ENABLED(CONFIG_DMA_CMA)) {
struct page *page;
page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
get_order(size));
if (!page)
return NULL;
*dma_handle = phys_to_dma(dev, page_to_phys(page));
return page_address(page);
} else {
return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
}
}
static void arm64_swiotlb_free_coherent(struct device *dev, size_t size,
void *vaddr, dma_addr_t dma_handle,
struct dma_attrs *attrs)
{
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
if (dev == NULL) {
WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
return;
}
if (IS_ENABLED(CONFIG_DMA_CMA)) {
phys_addr_t paddr = dma_to_phys(dev, dma_handle);
dma_release_from_contiguous(dev,
phys_to_page(paddr),
size >> PAGE_SHIFT);
} else {
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
}
}
static struct dma_map_ops arm64_swiotlb_dma_ops = {

3
arch/arm64/mm/init.c
View File

@ -30,6 +30,7 @@
#include <linux/memblock.h>
#include <linux/sort.h>
#include <linux/of_fdt.h>
#include <linux/dma-contiguous.h>
#include <asm/sections.h>
#include <asm/setup.h>
@ -159,6 +160,8 @@ void __init arm64_memblock_init(void)
memblock_reserve(base, size);
}
dma_contiguous_reserve(0);
memblock_allow_resize();
memblock_dump_all();
}

69
arch/arm64/mm/proc.S
View File

@ -80,6 +80,75 @@ ENTRY(cpu_do_idle)
ret
ENDPROC(cpu_do_idle)
#ifdef CONFIG_ARM64_CPU_SUSPEND
/**
* cpu_do_suspend - save CPU registers context
*
* x0: virtual address of context pointer
*/
ENTRY(cpu_do_suspend)
mrs x2, tpidr_el0
mrs x3, tpidrro_el0
mrs x4, contextidr_el1
mrs x5, mair_el1
mrs x6, cpacr_el1
mrs x7, ttbr1_el1
mrs x8, tcr_el1
mrs x9, vbar_el1
mrs x10, mdscr_el1
mrs x11, oslsr_el1
mrs x12, sctlr_el1
stp x2, x3, [x0]
stp x4, x5, [x0, #16]
stp x6, x7, [x0, #32]
stp x8, x9, [x0, #48]
stp x10, x11, [x0, #64]
str x12, [x0, #80]
ret
ENDPROC(cpu_do_suspend)
/**
* cpu_do_resume - restore CPU register context
*
* x0: Physical address of context pointer
* x1: ttbr0_el1 to be restored
*
* Returns:
* sctlr_el1 value in x0
*/
ENTRY(cpu_do_resume)
/*
* Invalidate local tlb entries before turning on MMU
*/
tlbi vmalle1
ldp x2, x3, [x0]
ldp x4, x5, [x0, #16]
ldp x6, x7, [x0, #32]
ldp x8, x9, [x0, #48]
ldp x10, x11, [x0, #64]
ldr x12, [x0, #80]
msr tpidr_el0, x2
msr tpidrro_el0, x3
msr contextidr_el1, x4
msr mair_el1, x5
msr cpacr_el1, x6
msr ttbr0_el1, x1
msr ttbr1_el1, x7
msr tcr_el1, x8
msr vbar_el1, x9
msr mdscr_el1, x10
/*
* Restore oslsr_el1 by writing oslar_el1
*/
ubfx x11, x11, #1, #1
msr oslar_el1, x11
mov x0, x12
dsb nsh // Make sure local tlb invalidation completed
isb
ret
ENDPROC(cpu_do_resume)
#endif
/*
* cpu_switch_mm(pgd_phys, tsk)
*

8
include/linux/irqdesc.h
View File

@ -152,6 +152,14 @@ static inline int irq_balancing_disabled(unsigned int irq)
return desc->status_use_accessors & IRQ_NO_BALANCING_MASK;
}
static inline int irq_is_percpu(unsigned int irq)
{
struct irq_desc *desc;
desc = irq_to_desc(irq);
return desc->status_use_accessors & IRQ_PER_CPU;
}
static inline void
irq_set_lockdep_class(unsigned int irq, struct lock_class_key *class)
{

19
include/linux/jump_label.h
View File

@ -81,18 +81,21 @@ struct module;
#include <linux/atomic.h>
#ifdef HAVE_JUMP_LABEL
#define JUMP_LABEL_TRUE_BRANCH 1UL
#define JUMP_LABEL_TYPE_FALSE_BRANCH 0UL
#define JUMP_LABEL_TYPE_TRUE_BRANCH 1UL
#define JUMP_LABEL_TYPE_MASK 1UL
static
inline struct jump_entry *jump_label_get_entries(struct static_key *key)
{
return (struct jump_entry *)((unsigned long)key->entries
& ~JUMP_LABEL_TRUE_BRANCH);
& ~JUMP_LABEL_TYPE_MASK);
}
static inline bool jump_label_get_branch_default(struct static_key *key)
{
if ((unsigned long)key->entries & JUMP_LABEL_TRUE_BRANCH)
if (((unsigned long)key->entries & JUMP_LABEL_TYPE_MASK) ==
JUMP_LABEL_TYPE_TRUE_BRANCH)
return true;
return false;
}
@ -122,10 +125,12 @@ extern void static_key_slow_inc(struct static_key *key);
extern void static_key_slow_dec(struct static_key *key);
extern void jump_label_apply_nops(struct module *mod);
#define STATIC_KEY_INIT_TRUE ((struct static_key) \
{ .enabled = ATOMIC_INIT(1), .entries = (void *)1 })
#define STATIC_KEY_INIT_FALSE ((struct static_key) \
{ .enabled = ATOMIC_INIT(0), .entries = (void *)0 })
#define STATIC_KEY_INIT_TRUE ((struct static_key) \
{ .enabled = ATOMIC_INIT(1), \
.entries = (void *)JUMP_LABEL_TYPE_TRUE_BRANCH })
#define STATIC_KEY_INIT_FALSE ((struct static_key) \
{ .enabled = ATOMIC_INIT(0), \
.entries = (void *)JUMP_LABEL_TYPE_FALSE_BRANCH })
#else /* !HAVE_JUMP_LABEL */

2
scripts/gcc-goto.sh
View File

@ -5,7 +5,7 @@
cat << "END" | $@ -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y"
int main(void)
{
#ifdef __arm__
#if defined(__arm__) || defined(__aarch64__)
/*
* Not related to asm goto, but used by jump label
* and broken on some ARM GCC versions (see GCC Bug 48637).