openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

arm64 festive updates for 4.21 

In the end, we ended up with quite a lot more than I expected:
 
 - Support for ARMv8.3 Pointer Authentication in userspace (CRIU and
   kernel-side support to come later)
 
 - Support for per-thread stack canaries, pending an update to GCC that
   is currently undergoing review
 
 - Support for kexec_file_load(), which permits secure boot of a kexec
   payload but also happens to improve the performance of kexec
   dramatically because we can avoid the sucky purgatory code from
   userspace. Kdump will come later (requires updates to libfdt).
 
 - Optimisation of our dynamic CPU feature framework, so that all
   detected features are enabled via a single stop_machine() invocation
 
 - KPTI whitelisting of Cortex-A CPUs unaffected by Meltdown, so that
   they can benefit from global TLB entries when KASLR is not in use
 
 - 52-bit virtual addressing for userspace (kernel remains 48-bit)
 
 - Patch in LSE atomics for per-cpu atomic operations
 
 - Custom preempt.h implementation to avoid unconditional calls to
   preempt_schedule() from preempt_enable()
 
 - Support for the new 'SB' Speculation Barrier instruction
 
 - Vectorised implementation of XOR checksumming and CRC32 optimisations
 
 - Workaround for Cortex-A76 erratum #1165522
 
 - Improved compatibility with Clang/LLD
 
 - Support for TX2 system PMUS for profiling the L3 cache and DMC
 
 - Reflect read-only permissions in the linear map by default
 
 - Ensure MMIO reads are ordered with subsequent calls to Xdelay()
 
 - Initial support for memory hotplug
 
 - Tweak the threshold when we invalidate the TLB by-ASID, so that
   mremap() performance is improved for ranges spanning multiple PMDs.
 
 - Minor refactoring and cleanups
 -----BEGIN PGP SIGNATURE-----
 Version: GnuPG v1
 
 iQEcBAABCgAGBQJcE4TmAAoJELescNyEwWM0Nr0H/iaU7/wQSzHyNXtZoImyKTul
 Blu2ga4/EqUrTU7AVVfmkl/3NBILWlgQVpY6tH6EfXQuvnxqD7CizbHyLdyO+z0S
 B5PsFUH2GLMNAi48AUNqGqkgb2knFbg+T+9IimijDBkKg1G/KhQnRg6bXX32mLJv
 Une8oshUPBVJMsHN1AcQknzKariuoE3u0SgJ+eOZ9yA2ZwKxP4yy1SkDt3xQrtI0
 lojeRjxcyjTP1oGRNZC+BWUtGOT35p7y6cGTnBd/4TlqBGz5wVAJUcdoxnZ6JYVR
 O8+ob9zU+4I0+SKt80s7pTLqQiL9rxkKZ5joWK1pr1g9e0s5N5yoETXKFHgJYP8=
 =sYdt
 -----END PGP SIGNATURE-----

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 festive updates from Will Deacon:
 "In the end, we ended up with quite a lot more than I expected:

   - Support for ARMv8.3 Pointer Authentication in userspace (CRIU and
     kernel-side support to come later)

   - Support for per-thread stack canaries, pending an update to GCC
     that is currently undergoing review

   - Support for kexec_file_load(), which permits secure boot of a kexec
     payload but also happens to improve the performance of kexec
     dramatically because we can avoid the sucky purgatory code from
     userspace. Kdump will come later (requires updates to libfdt).

   - Optimisation of our dynamic CPU feature framework, so that all
     detected features are enabled via a single stop_machine()
     invocation

   - KPTI whitelisting of Cortex-A CPUs unaffected by Meltdown, so that
     they can benefit from global TLB entries when KASLR is not in use

   - 52-bit virtual addressing for userspace (kernel remains 48-bit)

   - Patch in LSE atomics for per-cpu atomic operations

   - Custom preempt.h implementation to avoid unconditional calls to
     preempt_schedule() from preempt_enable()

   - Support for the new 'SB' Speculation Barrier instruction

   - Vectorised implementation of XOR checksumming and CRC32
     optimisations

   - Workaround for Cortex-A76 erratum #1165522

   - Improved compatibility with Clang/LLD

   - Support for TX2 system PMUS for profiling the L3 cache and DMC

   - Reflect read-only permissions in the linear map by default

   - Ensure MMIO reads are ordered with subsequent calls to Xdelay()

   - Initial support for memory hotplug

   - Tweak the threshold when we invalidate the TLB by-ASID, so that
     mremap() performance is improved for ranges spanning multiple PMDs.

   - Minor refactoring and cleanups"

* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (125 commits)
  arm64: kaslr: print PHYS_OFFSET in dump_kernel_offset()
  arm64: sysreg: Use _BITUL() when defining register bits
  arm64: cpufeature: Rework ptr auth hwcaps using multi_entry_cap_matches
  arm64: cpufeature: Reduce number of pointer auth CPU caps from 6 to 4
  arm64: docs: document pointer authentication
  arm64: ptr auth: Move per-thread keys from thread_info to thread_struct
  arm64: enable pointer authentication
  arm64: add prctl control for resetting ptrauth keys
  arm64: perf: strip PAC when unwinding userspace
  arm64: expose user PAC bit positions via ptrace
  arm64: add basic pointer authentication support
  arm64/cpufeature: detect pointer authentication
  arm64: Don't trap host pointer auth use to EL2
  arm64/kvm: hide ptrauth from guests
  arm64/kvm: consistently handle host HCR_EL2 flags
  arm64: add pointer authentication register bits
  arm64: add comments about EC exception levels
  arm64: perf: Treat EXCLUDE_EL* bit definitions as unsigned
  arm64: kpti: Whitelist Cortex-A CPUs that don't implement the CSV3 field
  arm64: enable per-task stack canaries
  ...
This commit is contained in:
Linus Torvalds 2018-12-25 17:41:56 -08:00
parent 13e1ad2be3 12f799c8c7
commit 5694cecdb0
138 changed files with 4288 additions and 1198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Documentation/arm64/booting.txt
View File

@ -205,6 +205,14 @@ Before jumping into the kernel, the following conditions must be met:
ICC_SRE_EL2.SRE (bit 0) must be initialised to 0b0.
- The DT or ACPI tables must describe a GICv2 interrupt controller.
For CPUs with pointer authentication functionality:
- If EL3 is present:
SCR_EL3.APK (bit 16) must be initialised to 0b1
SCR_EL3.API (bit 17) must be initialised to 0b1
- If the kernel is entered at EL1:
HCR_EL2.APK (bit 40) must be initialised to 0b1
HCR_EL2.API (bit 41) must be initialised to 0b1
The requirements described above for CPU mode, caches, MMUs, architected
timers, coherency and system registers apply to all CPUs. All CPUs must
enter the kernel in the same exception level.

8
Documentation/arm64/cpu-feature-registers.txt
View File

@ -184,12 +184,20 @@ infrastructure:
x--------------------------------------------------x
| Name | bits | visible |
|--------------------------------------------------|
| GPI | [31-28] | y |
|--------------------------------------------------|
| GPA | [27-24] | y |
|--------------------------------------------------|
| LRCPC | [23-20] | y |
|--------------------------------------------------|
| FCMA | [19-16] | y |
|--------------------------------------------------|
| JSCVT | [15-12] | y |
|--------------------------------------------------|
| API | [11-8] | y |
|--------------------------------------------------|
| APA | [7-4] | y |
|--------------------------------------------------|
| DPB | [3-0] | y |
x--------------------------------------------------x

12
Documentation/arm64/elf_hwcaps.txt
View File

@ -182,3 +182,15 @@ HWCAP_FLAGM
HWCAP_SSBS
Functionality implied by ID_AA64PFR1_EL1.SSBS == 0b0010.
HWCAP_PACA
Functionality implied by ID_AA64ISAR1_EL1.APA == 0b0001 or
ID_AA64ISAR1_EL1.API == 0b0001, as described by
Documentation/arm64/pointer-authentication.txt.
HWCAP_PACG
Functionality implied by ID_AA64ISAR1_EL1.GPA == 0b0001 or
ID_AA64ISAR1_EL1.GPI == 0b0001, as described by
Documentation/arm64/pointer-authentication.txt.

88
Documentation/arm64/pointer-authentication.txt Normal file
View File

@ -0,0 +1,88 @@
Pointer authentication in AArch64 Linux
=======================================
Author: Mark Rutland <mark.rutland@arm.com>
Date: 2017-07-19
This document briefly describes the provision of pointer authentication
functionality in AArch64 Linux.
Architecture overview
---------------------
The ARMv8.3 Pointer Authentication extension adds primitives that can be
used to mitigate certain classes of attack where an attacker can corrupt
the contents of some memory (e.g. the stack).
The extension uses a Pointer Authentication Code (PAC) to determine
whether pointers have been modified unexpectedly. A PAC is derived from
a pointer, another value (such as the stack pointer), and a secret key
held in system registers.
The extension adds instructions to insert a valid PAC into a pointer,
and to verify/remove the PAC from a pointer. The PAC occupies a number
of high-order bits of the pointer, which varies dependent on the
configured virtual address size and whether pointer tagging is in use.
A subset of these instructions have been allocated from the HINT
encoding space. In the absence of the extension (or when disabled),
these instructions behave as NOPs. Applications and libraries using
these instructions operate correctly regardless of the presence of the
extension.
The extension provides five separate keys to generate PACs - two for
instruction addresses (APIAKey, APIBKey), two for data addresses
(APDAKey, APDBKey), and one for generic authentication (APGAKey).
Basic support
-------------
When CONFIG_ARM64_PTR_AUTH is selected, and relevant HW support is
present, the kernel will assign random key values to each process at
exec*() time. The keys are shared by all threads within the process, and
are preserved across fork().
Presence of address authentication functionality is advertised via
HWCAP_PACA, and generic authentication functionality via HWCAP_PACG.
The number of bits that the PAC occupies in a pointer is 55 minus the
virtual address size configured by the kernel. For example, with a
virtual address size of 48, the PAC is 7 bits wide.
Recent versions of GCC can compile code with APIAKey-based return
address protection when passed the -msign-return-address option. This
uses instructions in the HINT space (unless -march=armv8.3-a or higher
is also passed), and such code can run on systems without the pointer
authentication extension.
In addition to exec(), keys can also be reinitialized to random values
using the PR_PAC_RESET_KEYS prctl. A bitmask of PR_PAC_APIAKEY,
PR_PAC_APIBKEY, PR_PAC_APDAKEY, PR_PAC_APDBKEY and PR_PAC_APGAKEY
specifies which keys are to be reinitialized; specifying 0 means "all
keys".
Debugging
---------
When CONFIG_ARM64_PTR_AUTH is selected, and HW support for address
authentication is present, the kernel will expose the position of TTBR0
PAC bits in the NT_ARM_PAC_MASK regset (struct user_pac_mask), which
userspace can acquire via PTRACE_GETREGSET.
The regset is exposed only when HWCAP_PACA is set. Separate masks are
exposed for data pointers and instruction pointers, as the set of PAC
bits can vary between the two. Note that the masks apply to TTBR0
addresses, and are not valid to apply to TTBR1 addresses (e.g. kernel
pointers).
Virtualization
--------------
Pointer authentication is not currently supported in KVM guests. KVM
will mask the feature bits from ID_AA64ISAR1_EL1, and attempted use of
the feature will result in an UNDEFINED exception being injected into
the guest.

1
Documentation/arm64/silicon-errata.txt
View File

@ -57,6 +57,7 @@ stable kernels.
| ARM | Cortex-A73 | #858921 | ARM64_ERRATUM_858921 |
| ARM | Cortex-A55 | #1024718 | ARM64_ERRATUM_1024718 |
| ARM | Cortex-A76 | #1188873 | ARM64_ERRATUM_1188873 |
| ARM | Cortex-A76 | #1165522 | ARM64_ERRATUM_1165522 |
| ARM | Cortex-A76 | #1286807 | ARM64_ERRATUM_1286807 |
| ARM | MMU-500 | #841119,#826419 | N/A |
| | | | |

41
Documentation/perf/thunderx2-pmu.txt Normal file
View File

@ -0,0 +1,41 @@
Cavium ThunderX2 SoC Performance Monitoring Unit (PMU UNCORE)
=============================================================
The ThunderX2 SoC PMU consists of independent, system-wide, per-socket
PMUs such as the Level 3 Cache (L3C) and DDR4 Memory Controller (DMC).
The DMC has 8 interleaved channels and the L3C has 16 interleaved tiles.
Events are counted for the default channel (i.e. channel 0) and prorated
to the total number of channels/tiles.
The DMC and L3C support up to 4 counters. Counters are independently
programmable and can be started and stopped individually. Each counter
can be set to a different event. Counters are 32-bit and do not support
an overflow interrupt; they are read every 2 seconds.
PMU UNCORE (perf) driver:
The thunderx2_pmu driver registers per-socket perf PMUs for the DMC and
L3C devices. Each PMU can be used to count up to 4 events
simultaneously. The PMUs provide a description of their available events
and configuration options under sysfs, see
/sys/devices/uncore_<l3c_S/dmc_S/>; S is the socket id.
The driver does not support sampling, therefore "perf record" will not
work. Per-task perf sessions are also not supported.
Examples:
# perf stat -a -e uncore_dmc_0/cnt_cycles/ sleep 1
# perf stat -a -e \
uncore_dmc_0/cnt_cycles/,\
uncore_dmc_0/data_transfers/,\
uncore_dmc_0/read_txns/,\
uncore_dmc_0/write_txns/ sleep 1
# perf stat -a -e \
uncore_l3c_0/read_request/,\
uncore_l3c_0/read_hit/,\
uncore_l3c_0/inv_request/,\
uncore_l3c_0/inv_hit/ sleep 1

2
arch/arm/include/asm/kvm_host.h
View File

@ -285,7 +285,7 @@ void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
static inline bool kvm_arch_check_sve_has_vhe(void) { return true; }
static inline bool kvm_arch_requires_vhe(void) { return false; }
static inline void kvm_arch_hardware_unsetup(void) {}
static inline void kvm_arch_sync_events(struct kvm *kvm) {}
static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}

131
arch/arm64/Kconfig
View File

@ -261,6 +261,9 @@ config ZONE_DMA32
config HAVE_GENERIC_GUP
def_bool y
config ARCH_ENABLE_MEMORY_HOTPLUG
def_bool y
config SMP
def_bool y
@ -274,7 +277,7 @@ config PGTABLE_LEVELS
int
default 2 if ARM64_16K_PAGES && ARM64_VA_BITS_36
default 2 if ARM64_64K_PAGES && ARM64_VA_BITS_42
default 3 if ARM64_64K_PAGES && ARM64_VA_BITS_48
default 3 if ARM64_64K_PAGES && (ARM64_VA_BITS_48 || ARM64_USER_VA_BITS_52)
default 3 if ARM64_4K_PAGES && ARM64_VA_BITS_39
default 3 if ARM64_16K_PAGES && ARM64_VA_BITS_47
default 4 if !ARM64_64K_PAGES && ARM64_VA_BITS_48
@ -313,9 +316,13 @@ menu "Kernel Features"
menu "ARM errata workarounds via the alternatives framework"
config ARM64_WORKAROUND_CLEAN_CACHE
def_bool n
config ARM64_ERRATUM_826319
bool "Cortex-A53: 826319: System might deadlock if a write cannot complete until read data is accepted"
default y
select ARM64_WORKAROUND_CLEAN_CACHE
help
This option adds an alternative code sequence to work around ARM
erratum 826319 on Cortex-A53 parts up to r0p2 with an AMBA 4 ACE or
@ -337,6 +344,7 @@ config ARM64_ERRATUM_826319
config ARM64_ERRATUM_827319
bool "Cortex-A53: 827319: Data cache clean instructions might cause overlapping transactions to the interconnect"
default y
select ARM64_WORKAROUND_CLEAN_CACHE
help
This option adds an alternative code sequence to work around ARM
erratum 827319 on Cortex-A53 parts up to r0p2 with an AMBA 5 CHI
@ -358,6 +366,7 @@ config ARM64_ERRATUM_827319
config ARM64_ERRATUM_824069
bool "Cortex-A53: 824069: Cache line might not be marked as clean after a CleanShared snoop"
default y
select ARM64_WORKAROUND_CLEAN_CACHE
help
This option adds an alternative code sequence to work around ARM
erratum 824069 on Cortex-A53 parts up to r0p2 when it is connected
@ -380,6 +389,7 @@ config ARM64_ERRATUM_824069
config ARM64_ERRATUM_819472
bool "Cortex-A53: 819472: Store exclusive instructions might cause data corruption"
default y
select ARM64_WORKAROUND_CLEAN_CACHE
help
This option adds an alternative code sequence to work around ARM
erratum 819472 on Cortex-A53 parts up to r0p1 with an L2 cache
@ -497,6 +507,18 @@ config ARM64_ERRATUM_1188873
If unsure, say Y.
config ARM64_ERRATUM_1165522
bool "Cortex-A76: Speculative AT instruction using out-of-context translation regime could cause subsequent request to generate an incorrect translation"
default y
help
This option adds work arounds for ARM Cortex-A76 erratum 1165522
Affected Cortex-A76 cores (r0p0, r1p0, r2p0) could end-up with
corrupted TLBs by speculating an AT instruction during a guest
context switch.
If unsure, say Y.
config ARM64_ERRATUM_1286807
bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation"
default y
@ -700,15 +722,43 @@ config ARM64_VA_BITS_47
config ARM64_VA_BITS_48
bool "48-bit"
config ARM64_USER_VA_BITS_52
bool "52-bit (user)"
depends on ARM64_64K_PAGES && (ARM64_PAN || !ARM64_SW_TTBR0_PAN)
help
Enable 52-bit virtual addressing for userspace when explicitly
requested via a hint to mmap(). The kernel will continue to
use 48-bit virtual addresses for its own mappings.
NOTE: Enabling 52-bit virtual addressing in conjunction with
ARMv8.3 Pointer Authentication will result in the PAC being
reduced from 7 bits to 3 bits, which may have a significant
impact on its susceptibility to brute-force attacks.
If unsure, select 48-bit virtual addressing instead.
endchoice
config ARM64_FORCE_52BIT
bool "Force 52-bit virtual addresses for userspace"
depends on ARM64_USER_VA_BITS_52 && EXPERT
help
For systems with 52-bit userspace VAs enabled, the kernel will attempt
to maintain compatibility with older software by providing 48-bit VAs
unless a hint is supplied to mmap.
This configuration option disables the 48-bit compatibility logic, and
forces all userspace addresses to be 52-bit on HW that supports it. One
should only enable this configuration option for stress testing userspace
memory management code. If unsure say N here.
config ARM64_VA_BITS
int
default 36 if ARM64_VA_BITS_36
default 39 if ARM64_VA_BITS_39
default 42 if ARM64_VA_BITS_42
default 47 if ARM64_VA_BITS_47
default 48 if ARM64_VA_BITS_48
default 48 if ARM64_VA_BITS_48 || ARM64_USER_VA_BITS_52
choice
prompt "Physical address space size"
@ -883,6 +933,39 @@ config KEXEC
but it is independent of the system firmware. And like a reboot
you can start any kernel with it, not just Linux.
config KEXEC_FILE
bool "kexec file based system call"
select KEXEC_CORE
help
This is new version of kexec system call. This system call is
file based and takes file descriptors as system call argument
for kernel and initramfs as opposed to list of segments as
accepted by previous system call.
config KEXEC_VERIFY_SIG
bool "Verify kernel signature during kexec_file_load() syscall"
depends on KEXEC_FILE
help
Select this option to verify a signature with loaded kernel
image. If configured, any attempt of loading a image without
valid signature will fail.
In addition to that option, you need to enable signature
verification for the corresponding kernel image type being
loaded in order for this to work.
config KEXEC_IMAGE_VERIFY_SIG
bool "Enable Image signature verification support"
default y
depends on KEXEC_VERIFY_SIG
depends on EFI && SIGNED_PE_FILE_VERIFICATION
help
Enable Image signature verification support.
comment "Support for PE file signature verification disabled"
depends on KEXEC_VERIFY_SIG
depends on !EFI || !SIGNED_PE_FILE_VERIFICATION
config CRASH_DUMP
bool "Build kdump crash kernel"
help
@ -983,6 +1066,20 @@ config ARM64_SSBD
If unsure, say Y.
config RODATA_FULL_DEFAULT_ENABLED
bool "Apply r/o permissions of VM areas also to their linear aliases"
default y
help
Apply read-only attributes of VM areas to the linear alias of
the backing pages as well. This prevents code or read-only data
from being modified (inadvertently or intentionally) via another
mapping of the same memory page. This additional enhancement can
be turned off at runtime by passing rodata=[off|on] (and turned on
with rodata=full if this option is set to 'n')
This requires the linear region to be mapped down to pages,
which may adversely affect performance in some cases.
menuconfig ARMV8_DEPRECATED
bool "Emulate deprecated/obsolete ARMv8 instructions"
depends on COMPAT
@ -1188,6 +1285,29 @@ config ARM64_CNP
endmenu
menu "ARMv8.3 architectural features"
config ARM64_PTR_AUTH
bool "Enable support for pointer authentication"
default y
help
Pointer authentication (part of the ARMv8.3 Extensions) provides
instructions for signing and authenticating pointers against secret
keys, which can be used to mitigate Return Oriented Programming (ROP)
and other attacks.
This option enables these instructions at EL0 (i.e. for userspace).
Choosing this option will cause the kernel to initialise secret keys
for each process at exec() time, with these keys being
context-switched along with the process.
The feature is detected at runtime. If the feature is not present in
hardware it will not be advertised to userspace nor will it be
enabled.
endmenu
config ARM64_SVE
bool "ARM Scalable Vector Extension support"
default y
@ -1272,6 +1392,13 @@ config RANDOMIZE_MODULE_REGION_FULL
a limited range that contains the [_stext, _etext] interval of the
core kernel, so branch relocations are always in range.
config CC_HAVE_STACKPROTECTOR_SYSREG
def_bool $(cc-option,-mstack-protector-guard=sysreg -mstack-protector-guard-reg=sp_el0 -mstack-protector-guard-offset=0)
config STACKPROTECTOR_PER_TASK
def_bool y
depends on STACKPROTECTOR && CC_HAVE_STACKPROTECTOR_SYSREG
endmenu
menu "Boot options"

12
arch/arm64/Makefile
View File

@ -18,7 +18,7 @@ ifeq ($(CONFIG_RELOCATABLE), y)
# Pass --no-apply-dynamic-relocs to restore pre-binutils-2.27 behaviour
# for relative relocs, since this leads to better Image compression
# with the relocation offsets always being zero.
LDFLAGS_vmlinux += -pie -shared -Bsymbolic \
LDFLAGS_vmlinux += -shared -Bsymbolic -z notext -z norelro \
$(call ld-option, --no-apply-dynamic-relocs)
endif
@ -56,6 +56,16 @@ KBUILD_AFLAGS += $(lseinstr) $(brokengasinst)
KBUILD_CFLAGS += $(call cc-option,-mabi=lp64)
KBUILD_AFLAGS += $(call cc-option,-mabi=lp64)
ifeq ($(CONFIG_STACKPROTECTOR_PER_TASK),y)
prepare: stack_protector_prepare
stack_protector_prepare: prepare0
$(eval KBUILD_CFLAGS += -mstack-protector-guard=sysreg \
-mstack-protector-guard-reg=sp_el0 \
-mstack-protector-guard-offset=$(shell \
awk '{if ($$2 == "TSK_STACK_CANARY") print $$3;}' \
include/generated/asm-offsets.h))
endif
ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
KBUILD_CPPFLAGS += -mbig-endian
CHECKFLAGS += -D__AARCH64EB__

2
arch/arm64/include/asm/Kbuild
View File

@ -14,7 +14,6 @@ generic-y += local64.h
generic-y += mcs_spinlock.h
generic-y += mm-arch-hooks.h
generic-y += msi.h
generic-y += preempt.h
generic-y += qrwlock.h
generic-y += qspinlock.h
generic-y += rwsem.h
@ -27,4 +26,3 @@ generic-y += trace_clock.h
generic-y += unaligned.h
generic-y += user.h
generic-y += vga.h
generic-y += xor.h

19
arch/arm64/include/asm/acpi.h
View File

@ -22,12 +22,23 @@
#include <asm/tlbflush.h>
/* Macros for consistency checks of the GICC subtable of MADT */
#define ACPI_MADT_GICC_LENGTH \
(acpi_gbl_FADT.header.revision < 6 ? 76 : 80)
/*
* MADT GICC minimum length refers to the MADT GICC structure table length as
* defined in the earliest ACPI version supported on arm64, ie ACPI 5.1.
*
* The efficiency_class member was added to the
* struct acpi_madt_generic_interrupt to represent the MADT GICC structure
* "Processor Power Efficiency Class" field, added in ACPI 6.0 whose offset
* is therefore used to delimit the MADT GICC structure minimum length
* appropriately.
*/
#define ACPI_MADT_GICC_MIN_LENGTH ACPI_OFFSET( \
struct acpi_madt_generic_interrupt, efficiency_class)
#define BAD_MADT_GICC_ENTRY(entry, end) \
(!(entry) || (entry)->header.length != ACPI_MADT_GICC_LENGTH || \
(unsigned long)(entry) + ACPI_MADT_GICC_LENGTH > (end))
(!(entry) || (entry)->header.length < ACPI_MADT_GICC_MIN_LENGTH || \
(unsigned long)(entry) + (entry)->header.length > (end))
/* Basic configuration for ACPI */
#ifdef CONFIG_ACPI

26
arch/arm64/include/asm/asm-prototypes.h Normal file
View File

@ -0,0 +1,26 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_PROTOTYPES_H
#define __ASM_PROTOTYPES_H
/*
* CONFIG_MODEVERIONS requires a C declaration to generate the appropriate CRC
* for each symbol. Since commit:
*
* 4efca4ed05cbdfd1 ("kbuild: modversions for EXPORT_SYMBOL() for asm")
*
* ... kbuild will automatically pick these up from <asm/asm-prototypes.h> and
* feed this to genksyms when building assembly files.
*/
#include <linux/arm-smccc.h>
#include <asm/ftrace.h>
#include <asm/page.h>
#include <asm/string.h>
#include <asm/uaccess.h>
#include <asm-generic/asm-prototypes.h>
long long __ashlti3(long long a, int b);
long long __ashrti3(long long a, int b);
long long __lshrti3(long long a, int b);
#endif /* __ASM_PROTOTYPES_H */

90
arch/arm64/include/asm/assembler.h
View File

@ -23,6 +23,8 @@
#ifndef __ASM_ASSEMBLER_H
#define __ASM_ASSEMBLER_H
#include <asm-generic/export.h>
#include <asm/asm-offsets.h>
#include <asm/cpufeature.h>
#include <asm/debug-monitors.h>
@ -122,6 +124,19 @@
hint #20
.endm
/*
* Speculation barrier
*/
.macro sb
alternative_if_not ARM64_HAS_SB
dsb nsh
isb
alternative_else
SB_BARRIER_INSN
nop
alternative_endif
.endm
/*
* Sanitise a 64-bit bounded index wrt speculation, returning zero if out
* of bounds.
@ -342,11 +357,10 @@ alternative_endif
.endm
/*
* tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map
* tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map
*/
.macro tcr_set_idmap_t0sz, valreg, tmpreg
ldr_l \tmpreg, idmap_t0sz
bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
.macro tcr_set_t0sz, valreg, t0sz
bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
.endm
/*
@ -377,27 +391,33 @@ alternative_endif
* size: size of the region
* Corrupts: kaddr, size, tmp1, tmp2
*/
.macro __dcache_op_workaround_clean_cache, op, kaddr
alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
dc \op, \kaddr
alternative_else
dc civac, \kaddr
alternative_endif
.endm
.macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
dcache_line_size \tmp1, \tmp2
add \size, \kaddr, \size
sub \tmp2, \tmp1, #1
bic \kaddr, \kaddr, \tmp2
9998:
.if (\op == cvau || \op == cvac)
alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
dc \op, \kaddr
alternative_else
dc civac, \kaddr
alternative_endif
.elseif (\op == cvap)
alternative_if ARM64_HAS_DCPOP
sys 3, c7, c12, 1, \kaddr // dc cvap
alternative_else
dc cvac, \kaddr
alternative_endif
.ifc \op, cvau
__dcache_op_workaround_clean_cache \op, \kaddr
.else
.ifc \op, cvac
__dcache_op_workaround_clean_cache \op, \kaddr
.else
.ifc \op, cvap
sys 3, c7, c12, 1, \kaddr // dc cvap
.else
dc \op, \kaddr
.endif
.endif
.endif
add \kaddr, \kaddr, \tmp1
cmp \kaddr, \size
b.lo 9998b
@ -477,6 +497,13 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU
#else
#define NOKPROBE(x)
#endif
#ifdef CONFIG_KASAN
#define EXPORT_SYMBOL_NOKASAN(name)
#else
#define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name)
#endif
/*
* Emit a 64-bit absolute little endian symbol reference in a way that
* ensures that it will be resolved at build time, even when building a
@ -515,6 +542,29 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU
mrs \rd, sp_el0
.endm
/*
* Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD.
* orr is used as it can cover the immediate value (and is idempotent).
* In future this may be nop'ed out when dealing with 52-bit kernel VAs.
* ttbr: Value of ttbr to set, modified.
*/
.macro offset_ttbr1, ttbr
#ifdef CONFIG_ARM64_USER_VA_BITS_52
orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
#endif
.endm
/*
* Perform the reverse of offset_ttbr1.
* bic is used as it can cover the immediate value and, in future, won't need
* to be nop'ed out when dealing with 52-bit kernel VAs.
*/
.macro restore_ttbr1, ttbr
#ifdef CONFIG_ARM64_USER_VA_BITS_52
bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
#endif
.endm
/*
* Arrange a physical address in a TTBR register, taking care of 52-bit
* addresses.
@ -672,11 +722,9 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU
.macro if_will_cond_yield_neon
#ifdef CONFIG_PREEMPT
get_thread_info x0
ldr w1, [x0, #TSK_TI_PREEMPT]
ldr x0, [x0, #TSK_TI_FLAGS]
cmp w1, #PREEMPT_DISABLE_OFFSET
csel x0, x0, xzr, eq
tbnz x0, #TIF_NEED_RESCHED, .Lyield_\@ // needs rescheduling?
ldr x0, [x0, #TSK_TI_PREEMPT]
sub x0, x0, #PREEMPT_DISABLE_OFFSET
cbz x0, .Lyield_\@
/* fall through to endif_yield_neon */
.subsection 1
.Lyield_\@ :

63
arch/arm64/include/asm/atomic_ll_sc.h
View File

@ -248,48 +248,57 @@ __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
}
__LL_SC_EXPORT(atomic64_dec_if_positive);
#define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl) \
__LL_SC_INLINE unsigned long \
__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
unsigned long old, \
unsigned long new)) \
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl) \
__LL_SC_INLINE u##sz \
__LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \
unsigned long old, \
u##sz new)) \
{ \
unsigned long tmp, oldval; \
unsigned long tmp; \
u##sz oldval; \
\
/* \
* Sub-word sizes require explicit casting so that the compare \
* part of the cmpxchg doesn't end up interpreting non-zero \
* upper bits of the register containing "old". \
*/ \
if (sz < 32) \
old = (u##sz)old; \
\
asm volatile( \
" prfm pstl1strm, %[v]\n" \
"1: ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n" \
"1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \
" eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
" cbnz %" #w "[tmp], 2f\n" \
" st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
" st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \
" cbnz %w[tmp], 1b\n" \
" " #mb "\n" \
"2:" \
: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \
[v] "+Q" (*(unsigned long *)ptr) \
: [old] "Lr" (old), [new] "r" (new) \
[v] "+Q" (*(u##sz *)ptr) \
: [old] "Kr" (old), [new] "r" (new) \
: cl); \
\
return oldval; \
} \
__LL_SC_EXPORT(__cmpxchg_case_##name);
__LL_SC_EXPORT(__cmpxchg_case_##name##sz);
__CMPXCHG_CASE(w, b, 1, , , , )
__CMPXCHG_CASE(w, h, 2, , , , )
__CMPXCHG_CASE(w, , 4, , , , )
__CMPXCHG_CASE( , , 8, , , , )
__CMPXCHG_CASE(w, b, acq_1, , a, , "memory")
__CMPXCHG_CASE(w, h, acq_2, , a, , "memory")
__CMPXCHG_CASE(w, , acq_4, , a, , "memory")
__CMPXCHG_CASE( , , acq_8, , a, , "memory")
__CMPXCHG_CASE(w, b, rel_1, , , l, "memory")
__CMPXCHG_CASE(w, h, rel_2, , , l, "memory")
__CMPXCHG_CASE(w, , rel_4, , , l, "memory")
__CMPXCHG_CASE( , , rel_8, , , l, "memory")
__CMPXCHG_CASE(w, b, mb_1, dmb ish, , l, "memory")
__CMPXCHG_CASE(w, h, mb_2, dmb ish, , l, "memory")
__CMPXCHG_CASE(w, , mb_4, dmb ish, , l, "memory")
__CMPXCHG_CASE( , , mb_8, dmb ish, , l, "memory")
__CMPXCHG_CASE(w, b, , 8, , , , )
__CMPXCHG_CASE(w, h, , 16, , , , )
__CMPXCHG_CASE(w, , , 32, , , , )
__CMPXCHG_CASE( , , , 64, , , , )
__CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory")
__CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory")
__CMPXCHG_CASE(w, , acq_, 32, , a, , "memory")
__CMPXCHG_CASE( , , acq_, 64, , a, , "memory")
__CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory")
__CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory")
__CMPXCHG_CASE(w, , rel_, 32, , , l, "memory")
__CMPXCHG_CASE( , , rel_, 64, , , l, "memory")
__CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory")
__CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory")
__CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory")
__CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory")
#undef __CMPXCHG_CASE

48
arch/arm64/include/asm/atomic_lse.h
View File

@ -446,22 +446,22 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)
#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
unsigned long old, \
unsigned long new) \
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...) \
static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr, \
u##sz old, \
u##sz new) \
{ \
register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
register unsigned long x1 asm ("x1") = old; \
register unsigned long x2 asm ("x2") = new; \
register u##sz x1 asm ("x1") = old; \
register u##sz x2 asm ("x2") = new; \
\
asm volatile(ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
__LL_SC_CMPXCHG(name) \
__LL_SC_CMPXCHG(name##sz) \
__nops(2), \
/* LSE atomics */ \
" mov " #w "30, %" #w "[old]\n" \
" cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
" cas" #mb #sfx "\t" #w "30, %" #w "[new], %[v]\n" \
" mov %" #w "[ret], " #w "30") \
: [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
: [old] "r" (x1), [new] "r" (x2) \
@ -470,22 +470,22 @@ static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
return x0; \
}
__CMPXCHG_CASE(w, b, 1, )
__CMPXCHG_CASE(w, h, 2, )
__CMPXCHG_CASE(w, , 4, )
__CMPXCHG_CASE(x, , 8, )
__CMPXCHG_CASE(w, b, acq_1, a, "memory")
__CMPXCHG_CASE(w, h, acq_2, a, "memory")
__CMPXCHG_CASE(w, , acq_4, a, "memory")
__CMPXCHG_CASE(x, , acq_8, a, "memory")
__CMPXCHG_CASE(w, b, rel_1, l, "memory")
__CMPXCHG_CASE(w, h, rel_2, l, "memory")
__CMPXCHG_CASE(w, , rel_4, l, "memory")
__CMPXCHG_CASE(x, , rel_8, l, "memory")
__CMPXCHG_CASE(w, b, mb_1, al, "memory")
__CMPXCHG_CASE(w, h, mb_2, al, "memory")
__CMPXCHG_CASE(w, , mb_4, al, "memory")
__CMPXCHG_CASE(x, , mb_8, al, "memory")
__CMPXCHG_CASE(w, b, , 8, )
__CMPXCHG_CASE(w, h, , 16, )
__CMPXCHG_CASE(w, , , 32, )
__CMPXCHG_CASE(x, , , 64, )
__CMPXCHG_CASE(w, b, acq_, 8, a, "memory")
__CMPXCHG_CASE(w, h, acq_, 16, a, "memory")
__CMPXCHG_CASE(w, , acq_, 32, a, "memory")
__CMPXCHG_CASE(x, , acq_, 64, a, "memory")
__CMPXCHG_CASE(w, b, rel_, 8, l, "memory")
__CMPXCHG_CASE(w, h, rel_, 16, l, "memory")
__CMPXCHG_CASE(w, , rel_, 32, l, "memory")
__CMPXCHG_CASE(x, , rel_, 64, l, "memory")
__CMPXCHG_CASE(w, b, mb_, 8, al, "memory")
__CMPXCHG_CASE(w, h, mb_, 16, al, "memory")
__CMPXCHG_CASE(w, , mb_, 32, al, "memory")
__CMPXCHG_CASE(x, , mb_, 64, al, "memory")
#undef __LL_SC_CMPXCHG
#undef __CMPXCHG_CASE

4
arch/arm64/include/asm/barrier.h
View File

@ -34,6 +34,10 @@
#define psb_csync() asm volatile("hint #17" : : : "memory")
#define csdb() asm volatile("hint #20" : : : "memory")
#define spec_bar() asm volatile(ALTERNATIVE("dsb nsh\nisb\n", \
SB_BARRIER_INSN"nop\n", \
ARM64_HAS_SB))
#define mb() dsb(sy)
#define rmb() dsb(ld)
#define wmb() dsb(st)

124
arch/arm64/include/asm/cmpxchg.h
View File

@ -30,46 +30,46 @@
* barrier case is generated as release+dmb for the former and
* acquire+release for the latter.
*/
#define __XCHG_CASE(w, sz, name, mb, nop_lse, acq, acq_lse, rel, cl) \
static inline unsigned long __xchg_case_##name(unsigned long x, \
volatile void *ptr) \
{ \
unsigned long ret, tmp; \
\
asm volatile(ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
" prfm pstl1strm, %2\n" \
"1: ld" #acq "xr" #sz "\t%" #w "0, %2\n" \
" st" #rel "xr" #sz "\t%w1, %" #w "3, %2\n" \
" cbnz %w1, 1b\n" \
" " #mb, \
/* LSE atomics */ \
" swp" #acq_lse #rel #sz "\t%" #w "3, %" #w "0, %2\n" \
__nops(3) \
" " #nop_lse) \
: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned long *)ptr) \
: "r" (x) \
: cl); \
\
return ret; \
#define __XCHG_CASE(w, sfx, name, sz, mb, nop_lse, acq, acq_lse, rel, cl) \
static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr) \
{ \
u##sz ret; \
unsigned long tmp; \
\
asm volatile(ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
" prfm pstl1strm, %2\n" \
"1: ld" #acq "xr" #sfx "\t%" #w "0, %2\n" \
" st" #rel "xr" #sfx "\t%w1, %" #w "3, %2\n" \
" cbnz %w1, 1b\n" \
" " #mb, \
/* LSE atomics */ \
" swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n" \
__nops(3) \
" " #nop_lse) \
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u##sz *)ptr) \
: "r" (x) \
: cl); \
\
return ret; \
}
__XCHG_CASE(w, b, 1, , , , , , )
__XCHG_CASE(w, h, 2, , , , , , )
__XCHG_CASE(w, , 4, , , , , , )
__XCHG_CASE( , , 8, , , , , , )
__XCHG_CASE(w, b, acq_1, , , a, a, , "memory")
__XCHG_CASE(w, h, acq_2, , , a, a, , "memory")
__XCHG_CASE(w, , acq_4, , , a, a, , "memory")
__XCHG_CASE( , , acq_8, , , a, a, , "memory")
__XCHG_CASE(w, b, rel_1, , , , , l, "memory")
__XCHG_CASE(w, h, rel_2, , , , , l, "memory")
__XCHG_CASE(w, , rel_4, , , , , l, "memory")
__XCHG_CASE( , , rel_8, , , , , l, "memory")
__XCHG_CASE(w, b, mb_1, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, h, mb_2, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, , mb_4, dmb ish, nop, , a, l, "memory")
__XCHG_CASE( , , mb_8, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, b, , 8, , , , , , )
__XCHG_CASE(w, h, , 16, , , , , , )
__XCHG_CASE(w, , , 32, , , , , , )
__XCHG_CASE( , , , 64, , , , , , )
__XCHG_CASE(w, b, acq_, 8, , , a, a, , "memory")
__XCHG_CASE(w, h, acq_, 16, , , a, a, , "memory")
__XCHG_CASE(w, , acq_, 32, , , a, a, , "memory")
__XCHG_CASE( , , acq_, 64, , , a, a, , "memory")
__XCHG_CASE(w, b, rel_, 8, , , , , l, "memory")
__XCHG_CASE(w, h, rel_, 16, , , , , l, "memory")
__XCHG_CASE(w, , rel_, 32, , , , , l, "memory")
__XCHG_CASE( , , rel_, 64, , , , , l, "memory")
__XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory")
__XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory")
#undef __XCHG_CASE
@ -80,13 +80,13 @@ static inline unsigned long __xchg##sfx(unsigned long x, \
{ \
switch (size) { \
case 1: \
return __xchg_case##sfx##_1(x, ptr); \
case 2: \
return __xchg_case##sfx##_2(x, ptr); \
case 4: \
return __xchg_case##sfx##_4(x, ptr); \
case 8: \
return __xchg_case##sfx##_8(x, ptr); \
case 2: \
return __xchg_case##sfx##_16(x, ptr); \
case 4: \
return __xchg_case##sfx##_32(x, ptr); \
case 8: \
return __xchg_case##sfx##_64(x, ptr); \
default: \
BUILD_BUG(); \
} \
@ -123,13 +123,13 @@ static inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
{ \
switch (size) { \
case 1: \
return __cmpxchg_case##sfx##_1(ptr, (u8)old, new); \
case 2: \
return __cmpxchg_case##sfx##_2(ptr, (u16)old, new); \
case 4: \
return __cmpxchg_case##sfx##_4(ptr, old, new); \
case 8: \
return __cmpxchg_case##sfx##_8(ptr, old, new); \
case 2: \
return __cmpxchg_case##sfx##_16(ptr, old, new); \
case 4: \
return __cmpxchg_case##sfx##_32(ptr, old, new); \
case 8: \
return __cmpxchg_case##sfx##_64(ptr, old, new); \
default: \
BUILD_BUG(); \
} \
@ -197,16 +197,16 @@ __CMPXCHG_GEN(_mb)
__ret; \
})
#define __CMPWAIT_CASE(w, sz, name) \
static inline void __cmpwait_case_##name(volatile void *ptr, \
unsigned long val) \
#define __CMPWAIT_CASE(w, sfx, sz) \
static inline void __cmpwait_case_##sz(volatile void *ptr, \
unsigned long val) \
{ \
unsigned long tmp; \
\
asm volatile( \
" sevl\n" \
" wfe\n" \
" ldxr" #sz "\t%" #w "[tmp], %[v]\n" \
" ldxr" #sfx "\t%" #w "[tmp], %[v]\n" \
" eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
" cbnz %" #w "[tmp], 1f\n" \
" wfe\n" \
@ -215,10 +215,10 @@ static inline void __cmpwait_case_##name(volatile void *ptr, \
: [val] "r" (val)); \
}
__CMPWAIT_CASE(w, b, 1);
__CMPWAIT_CASE(w, h, 2);
__CMPWAIT_CASE(w, , 4);
__CMPWAIT_CASE( , , 8);
__CMPWAIT_CASE(w, b, 8);
__CMPWAIT_CASE(w, h, 16);
__CMPWAIT_CASE(w, , 32);
__CMPWAIT_CASE( , , 64);
#undef __CMPWAIT_CASE
@ -229,13 +229,13 @@ static inline void __cmpwait##sfx(volatile void *ptr, \
{ \
switch (size) { \
case 1: \
return __cmpwait_case##sfx##_1(ptr, (u8)val); \
return __cmpwait_case##sfx##_8(ptr, (u8)val); \
case 2: \
return __cmpwait_case##sfx##_2(ptr, (u16)val); \
return __cmpwait_case##sfx##_16(ptr, (u16)val); \
case 4: \
return __cmpwait_case##sfx##_4(ptr, val); \
return __cmpwait_case##sfx##_32(ptr, val); \
case 8: \
return __cmpwait_case##sfx##_8(ptr, val); \
return __cmpwait_case##sfx##_64(ptr, val); \
default: \
BUILD_BUG(); \
} \

8
arch/arm64/include/asm/cpucaps.h
View File

@ -54,7 +54,13 @@
#define ARM64_HAS_CRC32 33
#define ARM64_SSBS 34
#define ARM64_WORKAROUND_1188873 35
#define ARM64_HAS_SB 36
#define ARM64_WORKAROUND_1165522 37
#define ARM64_HAS_ADDRESS_AUTH_ARCH 38
#define ARM64_HAS_ADDRESS_AUTH_IMP_DEF 39
#define ARM64_HAS_GENERIC_AUTH_ARCH 40
#define ARM64_HAS_GENERIC_AUTH_IMP_DEF 41
#define ARM64_NCAPS 36
#define ARM64_NCAPS 42
#endif /* __ASM_CPUCAPS_H */

124
arch/arm64/include/asm/cpufeature.h
View File

@ -321,19 +321,20 @@ struct arm64_cpu_capabilities {
bool sign;
unsigned long hwcap;
};
/*
* A list of "matches/cpu_enable" pair for the same
* "capability" of the same "type" as described by the parent.
* Only matches(), cpu_enable() and fields relevant to these
* methods are significant in the list. The cpu_enable is
* invoked only if the corresponding entry "matches()".
* However, if a cpu_enable() method is associated
* with multiple matches(), care should be taken that either
* the match criteria are mutually exclusive, or that the
* method is robust against being called multiple times.
*/
const struct arm64_cpu_capabilities *match_list;
};
/*
* An optional list of "matches/cpu_enable" pair for the same
* "capability" of the same "type" as described by the parent.
* Only matches(), cpu_enable() and fields relevant to these
* methods are significant in the list. The cpu_enable is
* invoked only if the corresponding entry "matches()".
* However, if a cpu_enable() method is associated
* with multiple matches(), care should be taken that either
* the match criteria are mutually exclusive, or that the
* method is robust against being called multiple times.
*/
const struct arm64_cpu_capabilities *match_list;
};
static inline int cpucap_default_scope(const struct arm64_cpu_capabilities *cap)
@ -353,10 +354,46 @@ cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap)
return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU);
}
/*
* Generic helper for handling capabilties with multiple (match,enable) pairs
* of call backs, sharing the same capability bit.
* Iterate over each entry to see if at least one matches.
*/
static inline bool
cpucap_multi_entry_cap_matches(const struct arm64_cpu_capabilities *entry,
int scope)
{
const struct arm64_cpu_capabilities *caps;
for (caps = entry->match_list; caps->matches; caps++)
if (caps->matches(caps, scope))
return true;
return false;
}
/*
* Take appropriate action for all matching entries in the shared capability
* entry.
*/
static inline void
cpucap_multi_entry_cap_cpu_enable(const struct arm64_cpu_capabilities *entry)
{
const struct arm64_cpu_capabilities *caps;
for (caps = entry->match_list; caps->matches; caps++)
if (caps->matches(caps, SCOPE_LOCAL_CPU) &&
caps->cpu_enable)
caps->cpu_enable(caps);
}
extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
extern struct static_key_false arm64_const_caps_ready;
#define for_each_available_cap(cap) \
for_each_set_bit(cap, cpu_hwcaps, ARM64_NCAPS)
bool this_cpu_has_cap(unsigned int cap);
static inline bool cpu_have_feature(unsigned int num)
@ -473,7 +510,6 @@ static inline bool id_aa64pfr0_sve(u64 pfr0)
void __init setup_cpu_features(void);
void check_local_cpu_capabilities(void);
u64 read_sanitised_ftr_reg(u32 id);
static inline bool cpu_supports_mixed_endian_el0(void)
@ -486,11 +522,59 @@ static inline bool system_supports_32bit_el0(void)
return cpus_have_const_cap(ARM64_HAS_32BIT_EL0);
}
static inline bool system_supports_4kb_granule(void)
{
u64 mmfr0;
u32 val;
mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
val = cpuid_feature_extract_unsigned_field(mmfr0,
ID_AA64MMFR0_TGRAN4_SHIFT);
return val == ID_AA64MMFR0_TGRAN4_SUPPORTED;
}
static inline bool system_supports_64kb_granule(void)
{
u64 mmfr0;
u32 val;
mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
val = cpuid_feature_extract_unsigned_field(mmfr0,
ID_AA64MMFR0_TGRAN64_SHIFT);
return val == ID_AA64MMFR0_TGRAN64_SUPPORTED;
}
static inline bool system_supports_16kb_granule(void)
{
u64 mmfr0;
u32 val;
mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
val = cpuid_feature_extract_unsigned_field(mmfr0,
ID_AA64MMFR0_TGRAN16_SHIFT);
return val == ID_AA64MMFR0_TGRAN16_SUPPORTED;
}
static inline bool system_supports_mixed_endian_el0(void)
{
return id_aa64mmfr0_mixed_endian_el0(read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1));
}
static inline bool system_supports_mixed_endian(void)
{
u64 mmfr0;
u32 val;
mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
val = cpuid_feature_extract_unsigned_field(mmfr0,
ID_AA64MMFR0_BIGENDEL_SHIFT);
return val == 0x1;
}
static inline bool system_supports_fpsimd(void)
{
return !cpus_have_const_cap(ARM64_HAS_NO_FPSIMD);
@ -514,6 +598,20 @@ static inline bool system_supports_cnp(void)
cpus_have_const_cap(ARM64_HAS_CNP);
}
static inline bool system_supports_address_auth(void)
{
return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) &&
(cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH_ARCH) ||
cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH_IMP_DEF));
}
static inline bool system_supports_generic_auth(void)
{
return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) &&
(cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH_ARCH) ||
cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF));
}
#define ARM64_SSBD_UNKNOWN -1
#define ARM64_SSBD_FORCE_DISABLE 0
#define ARM64_SSBD_KERNEL 1

2
arch/arm64/include/asm/cputype.h
View File

@ -151,6 +151,8 @@ struct midr_range {
.rv_max = MIDR_CPU_VAR_REV(v_max, r_max), \
}
#define MIDR_REV_RANGE(m, v, r_min, r_max) MIDR_RANGE(m, v, r_min, v, r_max)
#define MIDR_REV(m, v, r) MIDR_RANGE(m, v, r, v, r)
#define MIDR_ALL_VERSIONS(m) MIDR_RANGE(m, 0, 0, 0xf, 0xf)
static inline bool is_midr_in_range(u32 midr, struct midr_range const *range)

4
arch/arm64/include/asm/elf.h
View File

@ -117,7 +117,11 @@
* 64-bit, this is above 4GB to leave the entire 32-bit address
* space open for things that want to use the area for 32-bit pointers.
*/
#ifdef CONFIG_ARM64_FORCE_52BIT
#define ELF_ET_DYN_BASE (2 * TASK_SIZE_64 / 3)
#else
#define ELF_ET_DYN_BASE (2 * DEFAULT_MAP_WINDOW_64 / 3)
#endif /* CONFIG_ARM64_FORCE_52BIT */
#ifndef __ASSEMBLY__

17
arch/arm64/include/asm/esr.h
View File

@ -29,23 +29,24 @@
#define ESR_ELx_EC_CP14_MR (0x05)
#define ESR_ELx_EC_CP14_LS (0x06)
#define ESR_ELx_EC_FP_ASIMD (0x07)
#define ESR_ELx_EC_CP10_ID (0x08)
/* Unallocated EC: 0x09 - 0x0B */
#define ESR_ELx_EC_CP10_ID (0x08) /* EL2 only */
#define ESR_ELx_EC_PAC (0x09) /* EL2 and above */
/* Unallocated EC: 0x0A - 0x0B */
#define ESR_ELx_EC_CP14_64 (0x0C)
/* Unallocated EC: 0x0d */
#define ESR_ELx_EC_ILL (0x0E)
/* Unallocated EC: 0x0F - 0x10 */
#define ESR_ELx_EC_SVC32 (0x11)
#define ESR_ELx_EC_HVC32 (0x12)
#define ESR_ELx_EC_SMC32 (0x13)
#define ESR_ELx_EC_HVC32 (0x12) /* EL2 only */
#define ESR_ELx_EC_SMC32 (0x13) /* EL2 and above */
/* Unallocated EC: 0x14 */
#define ESR_ELx_EC_SVC64 (0x15)
#define ESR_ELx_EC_HVC64 (0x16)
#define ESR_ELx_EC_SMC64 (0x17)
#define ESR_ELx_EC_HVC64 (0x16) /* EL2 and above */
#define ESR_ELx_EC_SMC64 (0x17) /* EL2 and above */
#define ESR_ELx_EC_SYS64 (0x18)
#define ESR_ELx_EC_SVE (0x19)
/* Unallocated EC: 0x1A - 0x1E */
#define ESR_ELx_EC_IMP_DEF (0x1f)
#define ESR_ELx_EC_IMP_DEF (0x1f) /* EL3 only */
#define ESR_ELx_EC_IABT_LOW (0x20)
#define ESR_ELx_EC_IABT_CUR (0x21)
#define ESR_ELx_EC_PC_ALIGN (0x22)
@ -68,7 +69,7 @@
/* Unallocated EC: 0x36 - 0x37 */
#define ESR_ELx_EC_BKPT32 (0x38)
/* Unallocated EC: 0x39 */
#define ESR_ELx_EC_VECTOR32 (0x3A)
#define ESR_ELx_EC_VECTOR32 (0x3A) /* EL2 only */
/* Unallocted EC: 0x3B */
#define ESR_ELx_EC_BRK64 (0x3C)
/* Unallocated EC: 0x3D - 0x3F */

1
arch/arm64/include/asm/ftrace.h
View File

@ -13,6 +13,7 @@
#include <asm/insn.h>
#define HAVE_FUNCTION_GRAPH_FP_TEST
#define MCOUNT_ADDR ((unsigned long)_mcount)
#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE

59
arch/arm64/include/asm/image.h Normal file
View File

@ -0,0 +1,59 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_IMAGE_H
#define __ASM_IMAGE_H
#define ARM64_IMAGE_MAGIC "ARM\x64"
#define ARM64_IMAGE_FLAG_BE_SHIFT 0
#define ARM64_IMAGE_FLAG_PAGE_SIZE_SHIFT (ARM64_IMAGE_FLAG_BE_SHIFT + 1)
#define ARM64_IMAGE_FLAG_PHYS_BASE_SHIFT \
(ARM64_IMAGE_FLAG_PAGE_SIZE_SHIFT + 2)
#define ARM64_IMAGE_FLAG_BE_MASK 0x1
#define ARM64_IMAGE_FLAG_PAGE_SIZE_MASK 0x3
#define ARM64_IMAGE_FLAG_PHYS_BASE_MASK 0x1
#define ARM64_IMAGE_FLAG_LE 0
#define ARM64_IMAGE_FLAG_BE 1
#define ARM64_IMAGE_FLAG_PAGE_SIZE_4K 1
#define ARM64_IMAGE_FLAG_PAGE_SIZE_16K 2
#define ARM64_IMAGE_FLAG_PAGE_SIZE_64K 3
#define ARM64_IMAGE_FLAG_PHYS_BASE 1
#ifndef __ASSEMBLY__
#define arm64_image_flag_field(flags, field) \
(((flags) >> field##_SHIFT) & field##_MASK)
/*
* struct arm64_image_header - arm64 kernel image header
* See Documentation/arm64/booting.txt for details
*
* @code0: Executable code, or
* @mz_header alternatively used for part of MZ header
* @code1: Executable code
* @text_offset: Image load offset
* @image_size: Effective Image size
* @flags: kernel flags
* @reserved: reserved
* @magic: Magic number
* @reserved5: reserved, or
* @pe_header: alternatively used for PE COFF offset
*/
struct arm64_image_header {
__le32 code0;
__le32 code1;
__le64 text_offset;
__le64 image_size;
__le64 flags;
__le64 res2;
__le64 res3;
__le64 res4;
__le32 magic;
__le32 res5;
};
#endif /* __ASSEMBLY__ */
#endif /* __ASM_IMAGE_H */

8
arch/arm64/include/asm/insn.h
View File

@ -261,6 +261,11 @@ enum aarch64_insn_prfm_policy {
AARCH64_INSN_PRFM_POLICY_STRM,
};
enum aarch64_insn_adr_type {
AARCH64_INSN_ADR_TYPE_ADRP,
AARCH64_INSN_ADR_TYPE_ADR,
};
#define __AARCH64_INSN_FUNCS(abbr, mask, val) \
static __always_inline bool aarch64_insn_is_##abbr(u32 code) \
{ return (code & (mask)) == (val); } \
@ -393,6 +398,9 @@ u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
enum aarch64_insn_register src,
int imm, enum aarch64_insn_variant variant,
enum aarch64_insn_adsb_type type);
u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr,
enum aarch64_insn_register reg,
enum aarch64_insn_adr_type type);
u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,
enum aarch64_insn_register src,
int immr, int imms,

32
arch/arm64/include/asm/io.h
View File

@ -104,7 +104,23 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
}
/* IO barriers */
#define __iormb() rmb()
#define __iormb(v) \
({ \
unsigned long tmp; \
\
rmb(); \
\
/* \
* Create a dummy control dependency from the IO read to any \
* later instructions. This ensures that a subsequent call to \
* udelay() will be ordered due to the ISB in get_cycles(). \
*/ \
asm volatile("eor %0, %1, %1\n" \
"cbnz %0, ." \
: "=r" (tmp) : "r" ((unsigned long)(v)) \
: "memory"); \
})
#define __iowmb() wmb()
#define mmiowb() do { } while (0)
@ -129,10 +145,10 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
* following Normal memory access. Writes are ordered relative to any prior
* Normal memory access.
*/
#define readb(c) ({ u8 __v = readb_relaxed(c); __iormb(); __v; })
#define readw(c) ({ u16 __v = readw_relaxed(c); __iormb(); __v; })
#define readl(c) ({ u32 __v = readl_relaxed(c); __iormb(); __v; })
#define readq(c) ({ u64 __v = readq_relaxed(c); __iormb(); __v; })
#define readb(c) ({ u8 __v = readb_relaxed(c); __iormb(__v); __v; })
#define readw(c) ({ u16 __v = readw_relaxed(c); __iormb(__v); __v; })
#define readl(c) ({ u32 __v = readl_relaxed(c); __iormb(__v); __v; })
#define readq(c) ({ u64 __v = readq_relaxed(c); __iormb(__v); __v; })
#define writeb(v,c) ({ __iowmb(); writeb_relaxed((v),(c)); })
#define writew(v,c) ({ __iowmb(); writew_relaxed((v),(c)); })
@ -183,9 +199,9 @@ extern void __iomem *ioremap_cache(phys_addr_t phys_addr, size_t size);
/*
* io{read,write}{16,32,64}be() macros
*/
#define ioread16be(p) ({ __u16 __v = be16_to_cpu((__force __be16)__raw_readw(p)); __iormb(); __v; })
#define ioread32be(p) ({ __u32 __v = be32_to_cpu((__force __be32)__raw_readl(p)); __iormb(); __v; })
#define ioread64be(p) ({ __u64 __v = be64_to_cpu((__force __be64)__raw_readq(p)); __iormb(); __v; })
#define ioread16be(p) ({ __u16 __v = be16_to_cpu((__force __be16)__raw_readw(p)); __iormb(__v); __v; })
#define ioread32be(p) ({ __u32 __v = be32_to_cpu((__force __be32)__raw_readl(p)); __iormb(__v); __v; })
#define ioread64be(p) ({ __u64 __v = be64_to_cpu((__force __be64)__raw_readq(p)); __iormb(__v); __v; })
#define iowrite16be(v,p) ({ __iowmb(); __raw_writew((__force __u16)cpu_to_be16(v), p); })
#define iowrite32be(v,p) ({ __iowmb(); __raw_writel((__force __u32)cpu_to_be32(v), p); })

19
arch/arm64/include/asm/kexec.h
View File

@ -93,6 +93,25 @@ static inline void crash_prepare_suspend(void) {}
static inline void crash_post_resume(void) {}
#endif
#ifdef CONFIG_KEXEC_FILE
#define ARCH_HAS_KIMAGE_ARCH
struct kimage_arch {
void *dtb;
unsigned long dtb_mem;
};
extern const struct kexec_file_ops kexec_image_ops;
struct kimage;
extern int arch_kimage_file_post_load_cleanup(struct kimage *image);
extern int load_other_segments(struct kimage *image,
unsigned long kernel_load_addr, unsigned long kernel_size,
char *initrd, unsigned long initrd_len,
char *cmdline);
#endif
#endif /* __ASSEMBLY__ */
#endif

3
arch/arm64/include/asm/kvm_arm.h
View File

@ -24,6 +24,8 @@
/* Hyp Configuration Register (HCR) bits */
#define HCR_FWB (UL(1) << 46)
#define HCR_API (UL(1) << 41)
#define HCR_APK (UL(1) << 40)
#define HCR_TEA (UL(1) << 37)
#define HCR_TERR (UL(1) << 36)
#define HCR_TLOR (UL(1) << 35)
@ -87,6 +89,7 @@
HCR_AMO | HCR_SWIO | HCR_TIDCP | HCR_RW | HCR_TLOR | \
HCR_FMO | HCR_IMO)
#define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF)
#define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK)
#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
/* TCR_EL2 Registers bits */

10
arch/arm64/include/asm/kvm_host.h
View File

@ -422,7 +422,7 @@ static inline void __cpu_init_hyp_mode(phys_addr_t pgd_ptr,
}
}
static inline bool kvm_arch_check_sve_has_vhe(void)
static inline bool kvm_arch_requires_vhe(void)
{
/*
* The Arm architecture specifies that implementation of SVE
@ -430,9 +430,13 @@ static inline bool kvm_arch_check_sve_has_vhe(void)
* relies on this when SVE is present:
*/
if (system_supports_sve())
return has_vhe();
else
return true;
/* Some implementations have defects that confine them to VHE */
if (cpus_have_cap(ARM64_WORKAROUND_1165522))
return true;
return false;
}
static inline void kvm_arch_hardware_unsetup(void) {}

8
arch/arm64/include/asm/kvm_hyp.h
View File

@ -20,6 +20,7 @@
#include <linux/compiler.h>
#include <linux/kvm_host.h>
#include <asm/alternative.h>
#include <asm/sysreg.h>
#define __hyp_text __section(.hyp.text) notrace
@ -163,6 +164,13 @@ static __always_inline void __hyp_text __load_guest_stage2(struct kvm *kvm)
{
write_sysreg(kvm->arch.vtcr, vtcr_el2);
write_sysreg(kvm->arch.vttbr, vttbr_el2);
/*
* ARM erratum 1165522 requires the actual execution of the above
* before we can switch to the EL1/EL0 translation regime used by
* the guest.
*/
asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_1165522));
}
#endif /* __ARM64_KVM_HYP_H__ */

16
arch/arm64/include/asm/memory.h
View File

@ -64,15 +64,26 @@
#define KERNEL_START _text
#define KERNEL_END _end
#ifdef CONFIG_ARM64_USER_VA_BITS_52
#define MAX_USER_VA_BITS 52
#else
#define MAX_USER_VA_BITS VA_BITS
#endif
/*
* KASAN requires 1/8th of the kernel virtual address space for the shadow
* region. KASAN can bloat the stack significantly, so double the (minimum)
* stack size when KASAN is in use.
* stack size when KASAN is in use, and then double it again if KASAN_EXTRA is
* on.
*/
#ifdef CONFIG_KASAN
#define KASAN_SHADOW_SCALE_SHIFT 3
#define KASAN_SHADOW_SIZE (UL(1) << (VA_BITS - KASAN_SHADOW_SCALE_SHIFT))
#ifdef CONFIG_KASAN_EXTRA
#define KASAN_THREAD_SHIFT 2
#else
#define KASAN_THREAD_SHIFT 1
#endif /* CONFIG_KASAN_EXTRA */
#else
#define KASAN_SHADOW_SIZE (0)
#define KASAN_THREAD_SHIFT 0
@ -187,6 +198,9 @@ static inline unsigned long kaslr_offset(void)
return kimage_vaddr - KIMAGE_VADDR;
}
/* the actual size of a user virtual address */
extern u64 vabits_user;
/*
* Allow all memory at the discovery stage. We will clip it later.
*/

5
arch/arm64/include/asm/mmu_context.h
View File

@ -35,6 +35,8 @@
#include <asm/sysreg.h>
#include <asm/tlbflush.h>
extern bool rodata_full;
static inline void contextidr_thread_switch(struct task_struct *next)
{
if (!IS_ENABLED(CONFIG_PID_IN_CONTEXTIDR))
@ -72,6 +74,9 @@ extern u64 idmap_ptrs_per_pgd;
static inline bool __cpu_uses_extended_idmap(void)
{
if (IS_ENABLED(CONFIG_ARM64_USER_VA_BITS_52))
return false;
return unlikely(idmap_t0sz != TCR_T0SZ(VA_BITS));
}

44
arch/arm64/include/asm/module.h
View File

@ -22,7 +22,7 @@
#ifdef CONFIG_ARM64_MODULE_PLTS
struct mod_plt_sec {
struct elf64_shdr *plt;
int plt_shndx;
int plt_num_entries;
int plt_max_entries;
};
@ -36,10 +36,12 @@ struct mod_arch_specific {
};
#endif
u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela,
u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs,
void *loc, const Elf64_Rela *rela,
Elf64_Sym *sym);
u64 module_emit_veneer_for_adrp(struct module *mod, void *loc, u64 val);
u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs,
void *loc, u64 val);
#ifdef CONFIG_RANDOMIZE_BASE
extern u64 module_alloc_base;
@ -56,39 +58,19 @@ struct plt_entry {
* is exactly what we are dealing with here, we are free to use x16
* as a scratch register in the PLT veneers.
*/
__le32 mov0; /* movn x16, #0x.... */
__le32 mov1; /* movk x16, #0x...., lsl #16 */
__le32 mov2; /* movk x16, #0x...., lsl #32 */
__le32 adrp; /* adrp x16, .... */
__le32 add; /* add x16, x16, #0x.... */
__le32 br; /* br x16 */
};
static inline struct plt_entry get_plt_entry(u64 val)
static inline bool is_forbidden_offset_for_adrp(void *place)
{
/*
* MOVK/MOVN/MOVZ opcode:
* +--------+------------+--------+-----------+-------------+---------+
* | sf[31] | opc[30:29] | 100101 | hw[22:21] | imm16[20:5] | Rd[4:0] |
* +--------+------------+--------+-----------+-------------+---------+
*
* Rd := 0x10 (x16)
* hw := 0b00 (no shift), 0b01 (lsl #16), 0b10 (lsl #32)
* opc := 0b11 (MOVK), 0b00 (MOVN), 0b10 (MOVZ)
* sf := 1 (64-bit variant)
*/
return (struct plt_entry){
cpu_to_le32(0x92800010 | (((~val ) & 0xffff)) << 5),
cpu_to_le32(0xf2a00010 | ((( val >> 16) & 0xffff)) << 5),
cpu_to_le32(0xf2c00010 | ((( val >> 32) & 0xffff)) << 5),
cpu_to_le32(0xd61f0200)
};
return IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) &&
cpus_have_const_cap(ARM64_WORKAROUND_843419) &&
((u64)place & 0xfff) >= 0xff8;
}
static inline bool plt_entries_equal(const struct plt_entry *a,
const struct plt_entry *b)
{
return a->mov0 == b->mov0 &&
a->mov1 == b->mov1 &&
a->mov2 == b->mov2;
}
struct plt_entry get_plt_entry(u64 dst, void *pc);
bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b);
#endif /* __ASM_MODULE_H */

39
arch/arm64/include/asm/neon-intrinsics.h Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (C) 2018 Linaro, Ltd. <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#ifndef __ASM_NEON_INTRINSICS_H
#define __ASM_NEON_INTRINSICS_H
#include <asm-generic/int-ll64.h>
/*
* In the kernel, u64/s64 are [un]signed long long, not [un]signed long.
* So by redefining these macros to the former, we can force gcc-stdint.h
* to define uint64_t / in64_t in a compatible manner.
*/
#ifdef __INT64_TYPE__
#undef __INT64_TYPE__
#define __INT64_TYPE__ long long
#endif
#ifdef __UINT64_TYPE__
#undef __UINT64_TYPE__
#define __UINT64_TYPE__ unsigned long long
#endif
/*
* genksyms chokes on the ARM NEON instrinsics system header, but we
* don't export anything it defines anyway, so just disregard when
* genksyms execute.
*/
#ifndef __GENKSYMS__
#include <arm_neon.h>
#endif
#endif /* __ASM_NEON_INTRINSICS_H */

382
arch/arm64/include/asm/percpu.h
View File

@ -48,263 +48,193 @@ static inline unsigned long __my_cpu_offset(void)
}
#define __my_cpu_offset __my_cpu_offset()
#define PERCPU_OP(op, asm_op) \
static inline unsigned long __percpu_##op(void *ptr, \
unsigned long val, int size) \
#define PERCPU_RW_OPS(sz) \
static inline unsigned long __percpu_read_##sz(void *ptr) \
{ \
unsigned long loop, ret; \
return READ_ONCE(*(u##sz *)ptr); \
} \
\
switch (size) { \
case 1: \
asm ("//__per_cpu_" #op "_1\n" \
"1: ldxrb %w[ret], %[ptr]\n" \
#asm_op " %w[ret], %w[ret], %w[val]\n" \
" stxrb %w[loop], %w[ret], %[ptr]\n" \
" cbnz %w[loop], 1b" \
: [loop] "=&r" (loop), [ret] "=&r" (ret), \
[ptr] "+Q"(*(u8 *)ptr) \
: [val] "Ir" (val)); \
break; \
case 2: \
asm ("//__per_cpu_" #op "_2\n" \
"1: ldxrh %w[ret], %[ptr]\n" \
#asm_op " %w[ret], %w[ret], %w[val]\n" \
" stxrh %w[loop], %w[ret], %[ptr]\n" \
" cbnz %w[loop], 1b" \
: [loop] "=&r" (loop), [ret] "=&r" (ret), \
[ptr] "+Q"(*(u16 *)ptr) \
: [val] "Ir" (val)); \
break; \
case 4: \
asm ("//__per_cpu_" #op "_4\n" \
"1: ldxr %w[ret], %[ptr]\n" \
#asm_op " %w[ret], %w[ret], %w[val]\n" \
" stxr %w[loop], %w[ret], %[ptr]\n" \
" cbnz %w[loop], 1b" \
: [loop] "=&r" (loop), [ret] "=&r" (ret), \
[ptr] "+Q"(*(u32 *)ptr) \
: [val] "Ir" (val)); \
break; \
case 8: \
asm ("//__per_cpu_" #op "_8\n" \
"1: ldxr %[ret], %[ptr]\n" \
#asm_op " %[ret], %[ret], %[val]\n" \
" stxr %w[loop], %[ret], %[ptr]\n" \
" cbnz %w[loop], 1b" \
: [loop] "=&r" (loop), [ret] "=&r" (ret), \
[ptr] "+Q"(*(u64 *)ptr) \
: [val] "Ir" (val)); \
break; \
default: \
ret = 0; \
BUILD_BUG(); \
} \
static inline void __percpu_write_##sz(void *ptr, unsigned long val) \
{ \
WRITE_ONCE(*(u##sz *)ptr, (u##sz)val); \
}
#define __PERCPU_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \
static inline void \
__percpu_##name##_case_##sz(void *ptr, unsigned long val) \
{ \
unsigned int loop; \
u##sz tmp; \
\
asm volatile (ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
"1: ldxr" #sfx "\t%" #w "[tmp], %[ptr]\n" \
#op_llsc "\t%" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
" stxr" #sfx "\t%w[loop], %" #w "[tmp], %[ptr]\n" \
" cbnz %w[loop], 1b", \
/* LSE atomics */ \
#op_lse "\t%" #w "[val], %[ptr]\n" \
__nops(3)) \
: [loop] "=&r" (loop), [tmp] "=&r" (tmp), \
[ptr] "+Q"(*(u##sz *)ptr) \
: [val] "r" ((u##sz)(val))); \
}
#define __PERCPU_RET_OP_CASE(w, sfx, name, sz, op_llsc, op_lse) \
static inline u##sz \
__percpu_##name##_return_case_##sz(void *ptr, unsigned long val) \
{ \
unsigned int loop; \
u##sz ret; \
\
asm volatile (ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \
"1: ldxr" #sfx "\t%" #w "[ret], %[ptr]\n" \
#op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \
" stxr" #sfx "\t%w[loop], %" #w "[ret], %[ptr]\n" \
" cbnz %w[loop], 1b", \
/* LSE atomics */ \
#op_lse "\t%" #w "[val], %" #w "[ret], %[ptr]\n" \
#op_llsc "\t%" #w "[ret], %" #w "[ret], %" #w "[val]\n" \
__nops(2)) \
: [loop] "=&r" (loop), [ret] "=&r" (ret), \
[ptr] "+Q"(*(u##sz *)ptr) \
: [val] "r" ((u##sz)(val))); \
\
return ret; \
}
PERCPU_OP(add, add)
PERCPU_OP(and, and)
PERCPU_OP(or, orr)
#define PERCPU_OP(name, op_llsc, op_lse) \
__PERCPU_OP_CASE(w, b, name, 8, op_llsc, op_lse) \
__PERCPU_OP_CASE(w, h, name, 16, op_llsc, op_lse) \
__PERCPU_OP_CASE(w, , name, 32, op_llsc, op_lse) \
__PERCPU_OP_CASE( , , name, 64, op_llsc, op_lse)
#define PERCPU_RET_OP(name, op_llsc, op_lse) \
__PERCPU_RET_OP_CASE(w, b, name, 8, op_llsc, op_lse) \
__PERCPU_RET_OP_CASE(w, h, name, 16, op_llsc, op_lse) \
__PERCPU_RET_OP_CASE(w, , name, 32, op_llsc, op_lse) \
__PERCPU_RET_OP_CASE( , , name, 64, op_llsc, op_lse)
PERCPU_RW_OPS(8)
PERCPU_RW_OPS(16)
PERCPU_RW_OPS(32)
PERCPU_RW_OPS(64)
PERCPU_OP(add, add, stadd)
PERCPU_OP(andnot, bic, stclr)
PERCPU_OP(or, orr, stset)
PERCPU_RET_OP(add, add, ldadd)
#undef PERCPU_RW_OPS
#undef __PERCPU_OP_CASE
#undef __PERCPU_RET_OP_CASE
#undef PERCPU_OP
#undef PERCPU_RET_OP
static inline unsigned long __percpu_read(void *ptr, int size)
{
unsigned long ret;
switch (size) {
case 1:
ret = READ_ONCE(*(u8 *)ptr);
break;
case 2:
ret = READ_ONCE(*(u16 *)ptr);
break;
case 4:
ret = READ_ONCE(*(u32 *)ptr);
break;
case 8:
ret = READ_ONCE(*(u64 *)ptr);
break;
default:
ret = 0;
BUILD_BUG();
}
return ret;
}
static inline void __percpu_write(void *ptr, unsigned long val, int size)
{
switch (size) {
case 1:
WRITE_ONCE(*(u8 *)ptr, (u8)val);
break;
case 2:
WRITE_ONCE(*(u16 *)ptr, (u16)val);
break;
case 4:
WRITE_ONCE(*(u32 *)ptr, (u32)val);
break;
case 8:
WRITE_ONCE(*(u64 *)ptr, (u64)val);
break;
default:
BUILD_BUG();
}
}
static inline unsigned long __percpu_xchg(void *ptr, unsigned long val,
int size)
{
unsigned long ret, loop;
switch (size) {
case 1:
asm ("//__percpu_xchg_1\n"
"1: ldxrb %w[ret], %[ptr]\n"
" stxrb %w[loop], %w[val], %[ptr]\n"
" cbnz %w[loop], 1b"
: [loop] "=&r"(loop), [ret] "=&r"(ret),
[ptr] "+Q"(*(u8 *)ptr)
: [val] "r" (val));
break;
case 2:
asm ("//__percpu_xchg_2\n"
"1: ldxrh %w[ret], %[ptr]\n"
" stxrh %w[loop], %w[val], %[ptr]\n"
" cbnz %w[loop], 1b"
: [loop] "=&r"(loop), [ret] "=&r"(ret),
[ptr] "+Q"(*(u16 *)ptr)
: [val] "r" (val));
break;
case 4:
asm ("//__percpu_xchg_4\n"
"1: ldxr %w[ret], %[ptr]\n"
" stxr %w[loop], %w[val], %[ptr]\n"
" cbnz %w[loop], 1b"
: [loop] "=&r"(loop), [ret] "=&r"(ret),
[ptr] "+Q"(*(u32 *)ptr)
: [val] "r" (val));
break;
case 8:
asm ("//__percpu_xchg_8\n"
"1: ldxr %[ret], %[ptr]\n"
" stxr %w[loop], %[val], %[ptr]\n"
" cbnz %w[loop], 1b"
: [loop] "=&r"(loop), [ret] "=&r"(ret),
[ptr] "+Q"(*(u64 *)ptr)
: [val] "r" (val));
break;
default:
ret = 0;
BUILD_BUG();
}
return ret;
}
/* this_cpu_cmpxchg */
#define _protect_cmpxchg_local(pcp, o, n) \
({ \
typeof(*raw_cpu_ptr(&(pcp))) __ret; \
preempt_disable(); \
__ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
preempt_enable(); \
__ret; \
})
#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
/*
* It would be nice to avoid the conditional call into the scheduler when
* re-enabling preemption for preemptible kernels, but doing that in a way
* which builds inside a module would mean messing directly with the preempt
* count. If you do this, peterz and tglx will hunt you down.
*/
#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
({ \
int __ret; \
preempt_disable(); \
preempt_disable_notrace(); \
__ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \
raw_cpu_ptr(&(ptr2)), \
o1, o2, n1, n2); \
preempt_enable(); \
preempt_enable_notrace(); \
__ret; \
})
#define _percpu_read(pcp) \
#define _pcp_protect(op, pcp, ...) \
({ \
preempt_disable_notrace(); \
op(raw_cpu_ptr(&(pcp)), __VA_ARGS__); \
preempt_enable_notrace(); \
})
#define _pcp_protect_return(op, pcp, args...) \
({ \
typeof(pcp) __retval; \
preempt_disable_notrace(); \
__retval = (typeof(pcp))__percpu_read(raw_cpu_ptr(&(pcp)), \
sizeof(pcp)); \
__retval = (typeof(pcp))op(raw_cpu_ptr(&(pcp)), ##args); \
preempt_enable_notrace(); \
__retval; \
})
#define _percpu_write(pcp, val) \
do { \
preempt_disable_notrace(); \
__percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), \
sizeof(pcp)); \
preempt_enable_notrace(); \
} while(0) \
#define this_cpu_read_1(pcp) \
_pcp_protect_return(__percpu_read_8, pcp)
#define this_cpu_read_2(pcp) \
_pcp_protect_return(__percpu_read_16, pcp)
#define this_cpu_read_4(pcp) \
_pcp_protect_return(__percpu_read_32, pcp)
#define this_cpu_read_8(pcp) \
_pcp_protect_return(__percpu_read_64, pcp)
#define _pcp_protect(operation, pcp, val) \
({ \
typeof(pcp) __retval; \
preempt_disable(); \
__retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \
(val), sizeof(pcp)); \
preempt_enable(); \
__retval; \
})
#define this_cpu_write_1(pcp, val) \
_pcp_protect(__percpu_write_8, pcp, (unsigned long)val)
#define this_cpu_write_2(pcp, val) \
_pcp_protect(__percpu_write_16, pcp, (unsigned long)val)
#define this_cpu_write_4(pcp, val) \
_pcp_protect(__percpu_write_32, pcp, (unsigned long)val)
#define this_cpu_write_8(pcp, val) \
_pcp_protect(__percpu_write_64, pcp, (unsigned long)val)
#define _percpu_add(pcp, val) \
_pcp_protect(__percpu_add, pcp, val)
#define this_cpu_add_1(pcp, val) \
_pcp_protect(__percpu_add_case_8, pcp, val)
#define this_cpu_add_2(pcp, val) \
_pcp_protect(__percpu_add_case_16, pcp, val)
#define this_cpu_add_4(pcp, val) \
_pcp_protect(__percpu_add_case_32, pcp, val)
#define this_cpu_add_8(pcp, val) \
_pcp_protect(__percpu_add_case_64, pcp, val)
#define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_return_1(pcp, val) \
_pcp_protect_return(__percpu_add_return_case_8, pcp, val)
#define this_cpu_add_return_2(pcp, val) \
_pcp_protect_return(__percpu_add_return_case_16, pcp, val)
#define this_cpu_add_return_4(pcp, val) \
_pcp_protect_return(__percpu_add_return_case_32, pcp, val)
#define this_cpu_add_return_8(pcp, val) \
_pcp_protect_return(__percpu_add_return_case_64, pcp, val)
#define _percpu_and(pcp, val) \
_pcp_protect(__percpu_and, pcp, val)
#define this_cpu_and_1(pcp, val) \
_pcp_protect(__percpu_andnot_case_8, pcp, ~val)
#define this_cpu_and_2(pcp, val) \
_pcp_protect(__percpu_andnot_case_16, pcp, ~val)
#define this_cpu_and_4(pcp, val) \
_pcp_protect(__percpu_andnot_case_32, pcp, ~val)
#define this_cpu_and_8(pcp, val) \
_pcp_protect(__percpu_andnot_case_64, pcp, ~val)
#define _percpu_or(pcp, val) \
_pcp_protect(__percpu_or, pcp, val)
#define this_cpu_or_1(pcp, val) \
_pcp_protect(__percpu_or_case_8, pcp, val)
#define this_cpu_or_2(pcp, val) \
_pcp_protect(__percpu_or_case_16, pcp, val)
#define this_cpu_or_4(pcp, val) \
_pcp_protect(__percpu_or_case_32, pcp, val)
#define this_cpu_or_8(pcp, val) \
_pcp_protect(__percpu_or_case_64, pcp, val)
#define _percpu_xchg(pcp, val) (typeof(pcp)) \
_pcp_protect(__percpu_xchg, pcp, (unsigned long)(val))
#define this_cpu_xchg_1(pcp, val) \
_pcp_protect_return(xchg_relaxed, pcp, val)
#define this_cpu_xchg_2(pcp, val) \
_pcp_protect_return(xchg_relaxed, pcp, val)
#define this_cpu_xchg_4(pcp, val) \
_pcp_protect_return(xchg_relaxed, pcp, val)
#define this_cpu_xchg_8(pcp, val) \
_pcp_protect_return(xchg_relaxed, pcp, val)
#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val)
#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val)
#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val)
#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val)
#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val)
#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val)
#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
#define this_cpu_read_1(pcp) _percpu_read(pcp)
#define this_cpu_read_2(pcp) _percpu_read(pcp)
#define this_cpu_read_4(pcp) _percpu_read(pcp)
#define this_cpu_read_8(pcp) _percpu_read(pcp)
#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
#define this_cpu_cmpxchg_1(pcp, o, n) \
_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
#define this_cpu_cmpxchg_2(pcp, o, n) \
_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
#define this_cpu_cmpxchg_4(pcp, o, n) \
_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
#define this_cpu_cmpxchg_8(pcp, o, n) \
_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
#include <asm-generic/percpu.h>

170
arch/arm64/include/asm/perf_event.h
View File

@ -23,6 +23,160 @@
#define ARMV8_PMU_MAX_COUNTERS 32
#define ARMV8_PMU_COUNTER_MASK (ARMV8_PMU_MAX_COUNTERS - 1)
/*
* Common architectural and microarchitectural event numbers.
*/
#define ARMV8_PMUV3_PERFCTR_SW_INCR 0x00
#define ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL 0x01
#define ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL 0x02
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL 0x03
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE 0x04
#define ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL 0x05
#define ARMV8_PMUV3_PERFCTR_LD_RETIRED 0x06
#define ARMV8_PMUV3_PERFCTR_ST_RETIRED 0x07
#define ARMV8_PMUV3_PERFCTR_INST_RETIRED 0x08
#define ARMV8_PMUV3_PERFCTR_EXC_TAKEN 0x09
#define ARMV8_PMUV3_PERFCTR_EXC_RETURN 0x0A
#define ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED 0x0B
#define ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED 0x0C
#define ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED 0x0D
#define ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED 0x0E
#define ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED 0x0F
#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED 0x10
#define ARMV8_PMUV3_PERFCTR_CPU_CYCLES 0x11
#define ARMV8_PMUV3_PERFCTR_BR_PRED 0x12
#define ARMV8_PMUV3_PERFCTR_MEM_ACCESS 0x13
#define ARMV8_PMUV3_PERFCTR_L1I_CACHE 0x14
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB 0x15
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE 0x16
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL 0x17
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB 0x18
#define ARMV8_PMUV3_PERFCTR_BUS_ACCESS 0x19
#define ARMV8_PMUV3_PERFCTR_MEMORY_ERROR 0x1A
#define ARMV8_PMUV3_PERFCTR_INST_SPEC 0x1B
#define ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED 0x1C
#define ARMV8_PMUV3_PERFCTR_BUS_CYCLES 0x1D
#define ARMV8_PMUV3_PERFCTR_CHAIN 0x1E
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE 0x1F
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE 0x20
#define ARMV8_PMUV3_PERFCTR_BR_RETIRED 0x21
#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED 0x22
#define ARMV8_PMUV3_PERFCTR_STALL_FRONTEND 0x23
#define ARMV8_PMUV3_PERFCTR_STALL_BACKEND 0x24
#define ARMV8_PMUV3_PERFCTR_L1D_TLB 0x25
#define ARMV8_PMUV3_PERFCTR_L1I_TLB 0x26
#define ARMV8_PMUV3_PERFCTR_L2I_CACHE 0x27
#define ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL 0x28
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE 0x29
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL 0x2A
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE 0x2B
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB 0x2C
#define ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL 0x2D
#define ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL 0x2E
#define ARMV8_PMUV3_PERFCTR_L2D_TLB 0x2F
#define ARMV8_PMUV3_PERFCTR_L2I_TLB 0x30
#define ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS 0x31
#define ARMV8_PMUV3_PERFCTR_LL_CACHE 0x32
#define ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS 0x33
#define ARMV8_PMUV3_PERFCTR_DTLB_WALK 0x34
#define ARMV8_PMUV3_PERFCTR_ITLB_WALK 0x35
#define ARMV8_PMUV3_PERFCTR_LL_CACHE_RD 0x36
#define ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD 0x37
#define ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS_RD 0x38
/* Statistical profiling extension microarchitectural events */
#define ARMV8_SPE_PERFCTR_SAMPLE_POP 0x4000
#define ARMV8_SPE_PERFCTR_SAMPLE_FEED 0x4001
#define ARMV8_SPE_PERFCTR_SAMPLE_FILTRATE 0x4002
#define ARMV8_SPE_PERFCTR_SAMPLE_COLLISION 0x4003
/* ARMv8 recommended implementation defined event types */
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD 0x40
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR 0x41
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD 0x42
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR 0x43
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_INNER 0x44
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_OUTER 0x45
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_VICTIM 0x46
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_CLEAN 0x47
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_INVAL 0x48
#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD 0x4C
#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR 0x4D
#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD 0x4E
#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR 0x4F
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_RD 0x50
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WR 0x51
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_RD 0x52
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_WR 0x53
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_VICTIM 0x56
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_CLEAN 0x57
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_INVAL 0x58
#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_RD 0x5C
#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_WR 0x5D
#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_RD 0x5E
#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_WR 0x5F
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD 0x60
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR 0x61
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_SHARED 0x62
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NOT_SHARED 0x63
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NORMAL 0x64
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_PERIPH 0x65
#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_RD 0x66
#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_WR 0x67
#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LD_SPEC 0x68
#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_ST_SPEC 0x69
#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LDST_SPEC 0x6A
#define ARMV8_IMPDEF_PERFCTR_LDREX_SPEC 0x6C
#define ARMV8_IMPDEF_PERFCTR_STREX_PASS_SPEC 0x6D
#define ARMV8_IMPDEF_PERFCTR_STREX_FAIL_SPEC 0x6E
#define ARMV8_IMPDEF_PERFCTR_STREX_SPEC 0x6F
#define ARMV8_IMPDEF_PERFCTR_LD_SPEC 0x70
#define ARMV8_IMPDEF_PERFCTR_ST_SPEC 0x71
#define ARMV8_IMPDEF_PERFCTR_LDST_SPEC 0x72
#define ARMV8_IMPDEF_PERFCTR_DP_SPEC 0x73
#define ARMV8_IMPDEF_PERFCTR_ASE_SPEC 0x74
#define ARMV8_IMPDEF_PERFCTR_VFP_SPEC 0x75
#define ARMV8_IMPDEF_PERFCTR_PC_WRITE_SPEC 0x76
#define ARMV8_IMPDEF_PERFCTR_CRYPTO_SPEC 0x77
#define ARMV8_IMPDEF_PERFCTR_BR_IMMED_SPEC 0x78
#define ARMV8_IMPDEF_PERFCTR_BR_RETURN_SPEC 0x79
#define ARMV8_IMPDEF_PERFCTR_BR_INDIRECT_SPEC 0x7A
#define ARMV8_IMPDEF_PERFCTR_ISB_SPEC 0x7C
#define ARMV8_IMPDEF_PERFCTR_DSB_SPEC 0x7D
#define ARMV8_IMPDEF_PERFCTR_DMB_SPEC 0x7E
#define ARMV8_IMPDEF_PERFCTR_EXC_UNDEF 0x81
#define ARMV8_IMPDEF_PERFCTR_EXC_SVC 0x82
#define ARMV8_IMPDEF_PERFCTR_EXC_PABORT 0x83
#define ARMV8_IMPDEF_PERFCTR_EXC_DABORT 0x84
#define ARMV8_IMPDEF_PERFCTR_EXC_IRQ 0x86
#define ARMV8_IMPDEF_PERFCTR_EXC_FIQ 0x87
#define ARMV8_IMPDEF_PERFCTR_EXC_SMC 0x88
#define ARMV8_IMPDEF_PERFCTR_EXC_HVC 0x8A
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_PABORT 0x8B
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_DABORT 0x8C
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_OTHER 0x8D
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_IRQ 0x8E
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_FIQ 0x8F
#define ARMV8_IMPDEF_PERFCTR_RC_LD_SPEC 0x90
#define ARMV8_IMPDEF_PERFCTR_RC_ST_SPEC 0x91
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_RD 0xA0
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WR 0xA1
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_RD 0xA2
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_WR 0xA3
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_VICTIM 0xA6
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_CLEAN 0xA7
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_INVAL 0xA8
/*
* Per-CPU PMCR: config reg
*/
@ -49,22 +203,12 @@
#define ARMV8_PMU_EVTYPE_MASK 0xc800ffff /* Mask for writable bits */
#define ARMV8_PMU_EVTYPE_EVENT 0xffff /* Mask for EVENT bits */
/*
* PMUv3 event types: required events
*/
#define ARMV8_PMUV3_PERFCTR_SW_INCR 0x00
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL 0x03
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE 0x04
#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED 0x10
#define ARMV8_PMUV3_PERFCTR_CPU_CYCLES 0x11
#define ARMV8_PMUV3_PERFCTR_BR_PRED 0x12
/*
* Event filters for PMUv3
*/
#define ARMV8_PMU_EXCLUDE_EL1 (1 << 31)
#define ARMV8_PMU_EXCLUDE_EL0 (1 << 30)
#define ARMV8_PMU_INCLUDE_EL2 (1 << 27)
#define ARMV8_PMU_EXCLUDE_EL1 (1U << 31)
#define ARMV8_PMU_EXCLUDE_EL0 (1U << 30)
#define ARMV8_PMU_INCLUDE_EL2 (1U << 27)
/*
* PMUSERENR: user enable reg

12
arch/arm64/include/asm/pgtable-hwdef.h
View File

@ -80,7 +80,7 @@
#define PGDIR_SHIFT ARM64_HW_PGTABLE_LEVEL_SHIFT(4 - CONFIG_PGTABLE_LEVELS)
#define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT)
#define PGDIR_MASK (~(PGDIR_SIZE-1))
#define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT))
#define PTRS_PER_PGD (1 << (MAX_USER_VA_BITS - PGDIR_SHIFT))
/*
* Section address mask and size definitions.
@ -224,6 +224,8 @@
#define TCR_TxSZ_WIDTH 6
#define TCR_T0SZ_MASK (((UL(1) << TCR_TxSZ_WIDTH) - 1) << TCR_T0SZ_OFFSET)
#define TCR_EPD0_SHIFT 7
#define TCR_EPD0_MASK (UL(1) << TCR_EPD0_SHIFT)
#define TCR_IRGN0_SHIFT 8
#define TCR_IRGN0_MASK (UL(3) << TCR_IRGN0_SHIFT)
#define TCR_IRGN0_NC (UL(0) << TCR_IRGN0_SHIFT)
@ -231,6 +233,8 @@
#define TCR_IRGN0_WT (UL(2) << TCR_IRGN0_SHIFT)
#define TCR_IRGN0_WBnWA (UL(3) << TCR_IRGN0_SHIFT)
#define TCR_EPD1_SHIFT 23
#define TCR_EPD1_MASK (UL(1) << TCR_EPD1_SHIFT)
#define TCR_IRGN1_SHIFT 24
#define TCR_IRGN1_MASK (UL(3) << TCR_IRGN1_SHIFT)
#define TCR_IRGN1_NC (UL(0) << TCR_IRGN1_SHIFT)
@ -306,4 +310,10 @@
#define TTBR_BADDR_MASK_52 (((UL(1) << 46) - 1) << 2)
#endif
#ifdef CONFIG_ARM64_USER_VA_BITS_52
/* Must be at least 64-byte aligned to prevent corruption of the TTBR */
#define TTBR1_BADDR_4852_OFFSET (((UL(1) << (52 - PGDIR_SHIFT)) - \
(UL(1) << (48 - PGDIR_SHIFT))) * 8)
#endif
#endif

22
arch/arm64/include/asm/pgtable.h
View File

@ -22,6 +22,7 @@
#include <asm/memory.h>
#include <asm/pgtable-hwdef.h>
#include <asm/pgtable-prot.h>
#include <asm/tlbflush.h>
/*
* VMALLOC range.
@ -685,6 +686,27 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
return __ptep_test_and_clear_young(ptep);
}
#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
int young = ptep_test_and_clear_young(vma, address, ptep);
if (young) {
/*
* We can elide the trailing DSB here since the worst that can
* happen is that a CPU continues to use the young entry in its
* TLB and we mistakenly reclaim the associated page. The
* window for such an event is bounded by the next
* context-switch, which provides a DSB to complete the TLB
* invalidation.
*/
flush_tlb_page_nosync(vma, address);
}
return young;
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,

97
arch/arm64/include/asm/pointer_auth.h Normal file
View File

@ -0,0 +1,97 @@
// SPDX-License-Identifier: GPL-2.0
#ifndef __ASM_POINTER_AUTH_H
#define __ASM_POINTER_AUTH_H
#include <linux/bitops.h>
#include <linux/random.h>
#include <asm/cpufeature.h>
#include <asm/memory.h>
#include <asm/sysreg.h>
#ifdef CONFIG_ARM64_PTR_AUTH
/*
* Each key is a 128-bit quantity which is split across a pair of 64-bit
* registers (Lo and Hi).
*/
struct ptrauth_key {
unsigned long lo, hi;
};
/*
* We give each process its own keys, which are shared by all threads. The keys
* are inherited upon fork(), and reinitialised upon exec*().
*/
struct ptrauth_keys {
struct ptrauth_key apia;
struct ptrauth_key apib;
struct ptrauth_key apda;
struct ptrauth_key apdb;
struct ptrauth_key apga;
};
static inline void ptrauth_keys_init(struct ptrauth_keys *keys)
{
if (system_supports_address_auth()) {
get_random_bytes(&keys->apia, sizeof(keys->apia));
get_random_bytes(&keys->apib, sizeof(keys->apib));
get_random_bytes(&keys->apda, sizeof(keys->apda));
get_random_bytes(&keys->apdb, sizeof(keys->apdb));
}
if (system_supports_generic_auth())
get_random_bytes(&keys->apga, sizeof(keys->apga));
}
#define __ptrauth_key_install(k, v) \
do { \
struct ptrauth_key __pki_v = (v); \
write_sysreg_s(__pki_v.lo, SYS_ ## k ## KEYLO_EL1); \
write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1); \
} while (0)
static inline void ptrauth_keys_switch(struct ptrauth_keys *keys)
{
if (system_supports_address_auth()) {
__ptrauth_key_install(APIA, keys->apia);
__ptrauth_key_install(APIB, keys->apib);
__ptrauth_key_install(APDA, keys->apda);
__ptrauth_key_install(APDB, keys->apdb);
}
if (system_supports_generic_auth())
__ptrauth_key_install(APGA, keys->apga);
}
extern int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg);
/*
* The EL0 pointer bits used by a pointer authentication code.
* This is dependent on TBI0 being enabled, or bits 63:56 would also apply.
*/
#define ptrauth_user_pac_mask() GENMASK(54, vabits_user)
/* Only valid for EL0 TTBR0 instruction pointers */
static inline unsigned long ptrauth_strip_insn_pac(unsigned long ptr)
{
return ptr & ~ptrauth_user_pac_mask();
}
#define ptrauth_thread_init_user(tsk) \
do { \
struct task_struct *__ptiu_tsk = (tsk); \
ptrauth_keys_init(&__ptiu_tsk->thread.keys_user); \
ptrauth_keys_switch(&__ptiu_tsk->thread.keys_user); \
} while (0)
#define ptrauth_thread_switch(tsk) \
ptrauth_keys_switch(&(tsk)->thread.keys_user)
#else /* CONFIG_ARM64_PTR_AUTH */
#define ptrauth_prctl_reset_keys(tsk, arg) (-EINVAL)
#define ptrauth_strip_insn_pac(lr) (lr)
#define ptrauth_thread_init_user(tsk)
#define ptrauth_thread_switch(tsk)
#endif /* CONFIG_ARM64_PTR_AUTH */
#endif /* __ASM_POINTER_AUTH_H */

89
arch/arm64/include/asm/preempt.h Normal file
View File

@ -0,0 +1,89 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __ASM_PREEMPT_H
#define __ASM_PREEMPT_H
#include <linux/thread_info.h>
#define PREEMPT_NEED_RESCHED BIT(32)
#define PREEMPT_ENABLED (PREEMPT_NEED_RESCHED)
static inline int preempt_count(void)
{
return READ_ONCE(current_thread_info()->preempt.count);
}
static inline void preempt_count_set(u64 pc)
{
/* Preserve existing value of PREEMPT_NEED_RESCHED */
WRITE_ONCE(current_thread_info()->preempt.count, pc);
}
#define init_task_preempt_count(p) do { \
task_thread_info(p)->preempt_count = FORK_PREEMPT_COUNT; \
} while (0)
#define init_idle_preempt_count(p, cpu) do { \
task_thread_info(p)->preempt_count = PREEMPT_ENABLED; \
} while (0)
static inline void set_preempt_need_resched(void)
{
current_thread_info()->preempt.need_resched = 0;
}
static inline void clear_preempt_need_resched(void)
{
current_thread_info()->preempt.need_resched = 1;
}
static inline bool test_preempt_need_resched(void)
{
return !current_thread_info()->preempt.need_resched;
}
static inline void __preempt_count_add(int val)
{
u32 pc = READ_ONCE(current_thread_info()->preempt.count);
pc += val;
WRITE_ONCE(current_thread_info()->preempt.count, pc);
}
static inline void __preempt_count_sub(int val)
{
u32 pc = READ_ONCE(current_thread_info()->preempt.count);
pc -= val;
WRITE_ONCE(current_thread_info()->preempt.count, pc);
}
static inline bool __preempt_count_dec_and_test(void)
{
struct thread_info *ti = current_thread_info();
u64 pc = READ_ONCE(ti->preempt_count);
/* Update only the count field, leaving need_resched unchanged */
WRITE_ONCE(ti->preempt.count, --pc);
/*
* If we wrote back all zeroes, then we're preemptible and in
* need of a reschedule. Otherwise, we need to reload the
* preempt_count in case the need_resched flag was cleared by an
* interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE
* pair.
*/
return !pc || !READ_ONCE(ti->preempt_count);
}
static inline bool should_resched(int preempt_offset)
{
u64 pc = READ_ONCE(current_thread_info()->preempt_count);
return pc == preempt_offset;
}
#ifdef CONFIG_PREEMPT
void preempt_schedule(void);
#define __preempt_schedule() preempt_schedule()
void preempt_schedule_notrace(void);
#define __preempt_schedule_notrace() preempt_schedule_notrace()
#endif /* CONFIG_PREEMPT */
#endif /* __ASM_PREEMPT_H */

38
arch/arm64/include/asm/processor.h
View File

@ -19,10 +19,8 @@
#ifndef __ASM_PROCESSOR_H
#define __ASM_PROCESSOR_H
#define TASK_SIZE_64 (UL(1) << VA_BITS)
#define KERNEL_DS UL(-1)
#define USER_DS (TASK_SIZE_64 - 1)
#define KERNEL_DS UL(-1)
#define USER_DS ((UL(1) << MAX_USER_VA_BITS) - 1)
/*
* On arm64 systems, unaligned accesses by the CPU are cheap, and so there is
@ -46,6 +44,7 @@
#include <asm/hw_breakpoint.h>
#include <asm/lse.h>
#include <asm/pgtable-hwdef.h>
#include <asm/pointer_auth.h>
#include <asm/ptrace.h>
#include <asm/types.h>
@ -53,19 +52,31 @@
* TASK_SIZE - the maximum size of a user space task.
* TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
*/
#define DEFAULT_MAP_WINDOW_64 (UL(1) << VA_BITS)
#define TASK_SIZE_64 (UL(1) << vabits_user)
#ifdef CONFIG_COMPAT
#define TASK_SIZE_32 UL(0x100000000)
#define TASK_SIZE (test_thread_flag(TIF_32BIT) ? \
TASK_SIZE_32 : TASK_SIZE_64)
#define TASK_SIZE_OF(tsk) (test_tsk_thread_flag(tsk, TIF_32BIT) ? \
TASK_SIZE_32 : TASK_SIZE_64)
#define DEFAULT_MAP_WINDOW (test_thread_flag(TIF_32BIT) ? \
TASK_SIZE_32 : DEFAULT_MAP_WINDOW_64)
#else
#define TASK_SIZE TASK_SIZE_64
#define DEFAULT_MAP_WINDOW DEFAULT_MAP_WINDOW_64
#endif /* CONFIG_COMPAT */
#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
#ifdef CONFIG_ARM64_FORCE_52BIT
#define STACK_TOP_MAX TASK_SIZE_64
#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
#else
#define STACK_TOP_MAX DEFAULT_MAP_WINDOW_64
#define TASK_UNMAPPED_BASE (PAGE_ALIGN(DEFAULT_MAP_WINDOW / 4))
#endif /* CONFIG_ARM64_FORCE_52BIT */
#ifdef CONFIG_COMPAT
#define AARCH32_VECTORS_BASE 0xffff0000
#define STACK_TOP (test_thread_flag(TIF_32BIT) ? \
@ -74,6 +85,15 @@
#define STACK_TOP STACK_TOP_MAX
#endif /* CONFIG_COMPAT */
#ifndef CONFIG_ARM64_FORCE_52BIT
#define arch_get_mmap_end(addr) ((addr > DEFAULT_MAP_WINDOW) ? TASK_SIZE :\
DEFAULT_MAP_WINDOW)
#define arch_get_mmap_base(addr, base) ((addr > DEFAULT_MAP_WINDOW) ? \
base + TASK_SIZE - DEFAULT_MAP_WINDOW :\
base)
#endif /* CONFIG_ARM64_FORCE_52BIT */
extern phys_addr_t arm64_dma_phys_limit;
#define ARCH_LOW_ADDRESS_LIMIT (arm64_dma_phys_limit - 1)
@ -127,6 +147,9 @@ struct thread_struct {
unsigned long fault_address; /* fault info */
unsigned long fault_code; /* ESR_EL1 value */
struct debug_info debug; /* debugging */
#ifdef CONFIG_ARM64_PTR_AUTH
struct ptrauth_keys keys_user;
#endif
};
static inline void arch_thread_struct_whitelist(unsigned long *offset,
@ -270,6 +293,9 @@ extern void __init minsigstksz_setup(void);
#define SVE_SET_VL(arg) sve_set_current_vl(arg)
#define SVE_GET_VL() sve_get_current_vl()
/* PR_PAC_RESET_KEYS prctl */
#define PAC_RESET_KEYS(tsk, arg) ptrauth_prctl_reset_keys(tsk, arg)
/*
* For CONFIG_GCC_PLUGIN_STACKLEAK
*

15
arch/arm64/include/asm/smp.h
View File

@ -17,15 +17,20 @@
#define __ASM_SMP_H
/* Values for secondary_data.status */
#define CPU_STUCK_REASON_SHIFT (8)
#define CPU_BOOT_STATUS_MASK ((1U << CPU_STUCK_REASON_SHIFT) - 1)
#define CPU_MMU_OFF (-1)
#define CPU_BOOT_SUCCESS (0)
#define CPU_MMU_OFF (-1)
#define CPU_BOOT_SUCCESS (0)
/* The cpu invoked ops->cpu_die, synchronise it with cpu_kill */
#define CPU_KILL_ME (1)
#define CPU_KILL_ME (1)
/* The cpu couldn't die gracefully and is looping in the kernel */
#define CPU_STUCK_IN_KERNEL (2)
#define CPU_STUCK_IN_KERNEL (2)
/* Fatal system error detected by secondary CPU, crash the system */
#define CPU_PANIC_KERNEL (3)
#define CPU_PANIC_KERNEL (3)
#define CPU_STUCK_REASON_52_BIT_VA (1U << CPU_STUCK_REASON_SHIFT)
#define CPU_STUCK_REASON_NO_GRAN (2U << CPU_STUCK_REASON_SHIFT)
#ifndef __ASSEMBLY__

3
arch/arm64/include/asm/stackprotector.h
View File

@ -34,7 +34,8 @@ static __always_inline void boot_init_stack_canary(void)
canary &= CANARY_MASK;
current->stack_canary = canary;
__stack_chk_guard = current->stack_canary;
if (!IS_ENABLED(CONFIG_STACKPROTECTOR_PER_TASK))
__stack_chk_guard = current->stack_canary;
}
#endif /* _ASM_STACKPROTECTOR_H */

107
arch/arm64/include/asm/sysreg.h
View File

@ -20,6 +20,7 @@
#ifndef __ASM_SYSREG_H
#define __ASM_SYSREG_H
#include <linux/const.h>
#include <linux/stringify.h>
/*
@ -104,6 +105,11 @@
#define SET_PSTATE_UAO(x) __emit_inst(0xd500401f | PSTATE_UAO | ((!!x) << PSTATE_Imm_shift))
#define SET_PSTATE_SSBS(x) __emit_inst(0xd500401f | PSTATE_SSBS | ((!!x) << PSTATE_Imm_shift))
#define __SYS_BARRIER_INSN(CRm, op2, Rt) \
__emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f))
#define SB_BARRIER_INSN __SYS_BARRIER_INSN(0, 7, 31)
#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)
#define SYS_DC_CSW sys_insn(1, 0, 7, 10, 2)
#define SYS_DC_CISW sys_insn(1, 0, 7, 14, 2)
@ -183,6 +189,19 @@
#define SYS_TTBR1_EL1 sys_reg(3, 0, 2, 0, 1)
#define SYS_TCR_EL1 sys_reg(3, 0, 2, 0, 2)
#define SYS_APIAKEYLO_EL1 sys_reg(3, 0, 2, 1, 0)
#define SYS_APIAKEYHI_EL1 sys_reg(3, 0, 2, 1, 1)
#define SYS_APIBKEYLO_EL1 sys_reg(3, 0, 2, 1, 2)
#define SYS_APIBKEYHI_EL1 sys_reg(3, 0, 2, 1, 3)
#define SYS_APDAKEYLO_EL1 sys_reg(3, 0, 2, 2, 0)
#define SYS_APDAKEYHI_EL1 sys_reg(3, 0, 2, 2, 1)
#define SYS_APDBKEYLO_EL1 sys_reg(3, 0, 2, 2, 2)
#define SYS_APDBKEYHI_EL1 sys_reg(3, 0, 2, 2, 3)
#define SYS_APGAKEYLO_EL1 sys_reg(3, 0, 2, 3, 0)
#define SYS_APGAKEYHI_EL1 sys_reg(3, 0, 2, 3, 1)
#define SYS_ICC_PMR_EL1 sys_reg(3, 0, 4, 6, 0)
#define SYS_AFSR0_EL1 sys_reg(3, 0, 5, 1, 0)
@ -431,27 +450,31 @@
#define SYS_ICH_LR15_EL2 __SYS__LR8_EL2(7)
/* Common SCTLR_ELx flags. */
#define SCTLR_ELx_DSSBS (1UL << 44)
#define SCTLR_ELx_EE (1 << 25)
#define SCTLR_ELx_IESB (1 << 21)
#define SCTLR_ELx_WXN (1 << 19)
#define SCTLR_ELx_I (1 << 12)
#define SCTLR_ELx_SA (1 << 3)
#define SCTLR_ELx_C (1 << 2)
#define SCTLR_ELx_A (1 << 1)
#define SCTLR_ELx_M 1
#define SCTLR_ELx_DSSBS (_BITUL(44))
#define SCTLR_ELx_ENIA (_BITUL(31))
#define SCTLR_ELx_ENIB (_BITUL(30))
#define SCTLR_ELx_ENDA (_BITUL(27))
#define SCTLR_ELx_EE (_BITUL(25))
#define SCTLR_ELx_IESB (_BITUL(21))
#define SCTLR_ELx_WXN (_BITUL(19))
#define SCTLR_ELx_ENDB (_BITUL(13))
#define SCTLR_ELx_I (_BITUL(12))
#define SCTLR_ELx_SA (_BITUL(3))
#define SCTLR_ELx_C (_BITUL(2))
#define SCTLR_ELx_A (_BITUL(1))
#define SCTLR_ELx_M (_BITUL(0))
#define SCTLR_ELx_FLAGS (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \
SCTLR_ELx_SA | SCTLR_ELx_I | SCTLR_ELx_IESB)
/* SCTLR_EL2 specific flags. */
#define SCTLR_EL2_RES1 ((1 << 4) | (1 << 5) | (1 << 11) | (1 << 16) | \
(1 << 18) | (1 << 22) | (1 << 23) | (1 << 28) | \
(1 << 29))
#define SCTLR_EL2_RES0 ((1 << 6) | (1 << 7) | (1 << 8) | (1 << 9) | \
(1 << 10) | (1 << 13) | (1 << 14) | (1 << 15) | \
(1 << 17) | (1 << 20) | (1 << 24) | (1 << 26) | \
(1 << 27) | (1 << 30) | (1 << 31) | \
#define SCTLR_EL2_RES1 ((_BITUL(4)) | (_BITUL(5)) | (_BITUL(11)) | (_BITUL(16)) | \
(_BITUL(18)) | (_BITUL(22)) | (_BITUL(23)) | (_BITUL(28)) | \
(_BITUL(29)))
#define SCTLR_EL2_RES0 ((_BITUL(6)) | (_BITUL(7)) | (_BITUL(8)) | (_BITUL(9)) | \
(_BITUL(10)) | (_BITUL(13)) | (_BITUL(14)) | (_BITUL(15)) | \
(_BITUL(17)) | (_BITUL(20)) | (_BITUL(24)) | (_BITUL(26)) | \
(_BITUL(27)) | (_BITUL(30)) | (_BITUL(31)) | \
(0xffffefffUL << 32))
#ifdef CONFIG_CPU_BIG_ENDIAN
@ -473,23 +496,23 @@
#endif
/* SCTLR_EL1 specific flags. */
#define SCTLR_EL1_UCI (1 << 26)
#define SCTLR_EL1_E0E (1 << 24)
#define SCTLR_EL1_SPAN (1 << 23)
#define SCTLR_EL1_NTWE (1 << 18)
#define SCTLR_EL1_NTWI (1 << 16)
#define SCTLR_EL1_UCT (1 << 15)
#define SCTLR_EL1_DZE (1 << 14)
#define SCTLR_EL1_UMA (1 << 9)
#define SCTLR_EL1_SED (1 << 8)
#define SCTLR_EL1_ITD (1 << 7)
#define SCTLR_EL1_CP15BEN (1 << 5)
#define SCTLR_EL1_SA0 (1 << 4)
#define SCTLR_EL1_UCI (_BITUL(26))
#define SCTLR_EL1_E0E (_BITUL(24))
#define SCTLR_EL1_SPAN (_BITUL(23))
#define SCTLR_EL1_NTWE (_BITUL(18))
#define SCTLR_EL1_NTWI (_BITUL(16))
#define SCTLR_EL1_UCT (_BITUL(15))
#define SCTLR_EL1_DZE (_BITUL(14))
#define SCTLR_EL1_UMA (_BITUL(9))
#define SCTLR_EL1_SED (_BITUL(8))
#define SCTLR_EL1_ITD (_BITUL(7))
#define SCTLR_EL1_CP15BEN (_BITUL(5))
#define SCTLR_EL1_SA0 (_BITUL(4))
#define SCTLR_EL1_RES1 ((1 << 11) | (1 << 20) | (1 << 22) | (1 << 28) | \
(1 << 29))
#define SCTLR_EL1_RES0 ((1 << 6) | (1 << 10) | (1 << 13) | (1 << 17) | \
(1 << 27) | (1 << 30) | (1 << 31) | \
#define SCTLR_EL1_RES1 ((_BITUL(11)) | (_BITUL(20)) | (_BITUL(22)) | (_BITUL(28)) | \
(_BITUL(29)))
#define SCTLR_EL1_RES0 ((_BITUL(6)) | (_BITUL(10)) | (_BITUL(13)) | (_BITUL(17)) | \
(_BITUL(27)) | (_BITUL(30)) | (_BITUL(31)) | \
(0xffffefffUL << 32))
#ifdef CONFIG_CPU_BIG_ENDIAN
@ -528,11 +551,25 @@
#define ID_AA64ISAR0_AES_SHIFT 4
/* id_aa64isar1 */
#define ID_AA64ISAR1_SB_SHIFT 36
#define ID_AA64ISAR1_GPI_SHIFT 28
#define ID_AA64ISAR1_GPA_SHIFT 24
#define ID_AA64ISAR1_LRCPC_SHIFT 20
#define ID_AA64ISAR1_FCMA_SHIFT 16
#define ID_AA64ISAR1_JSCVT_SHIFT 12
#define ID_AA64ISAR1_API_SHIFT 8
#define ID_AA64ISAR1_APA_SHIFT 4
#define ID_AA64ISAR1_DPB_SHIFT 0
#define ID_AA64ISAR1_APA_NI 0x0
#define ID_AA64ISAR1_APA_ARCHITECTED 0x1
#define ID_AA64ISAR1_API_NI 0x0
#define ID_AA64ISAR1_API_IMP_DEF 0x1
#define ID_AA64ISAR1_GPA_NI 0x0
#define ID_AA64ISAR1_GPA_ARCHITECTED 0x1
#define ID_AA64ISAR1_GPI_NI 0x0
#define ID_AA64ISAR1_GPI_IMP_DEF 0x1
/* id_aa64pfr0 */
#define ID_AA64PFR0_CSV3_SHIFT 60
#define ID_AA64PFR0_CSV2_SHIFT 56
@ -676,13 +713,13 @@
#define ZCR_ELx_LEN_SIZE 9
#define ZCR_ELx_LEN_MASK 0x1ff
#define CPACR_EL1_ZEN_EL1EN (1 << 16) /* enable EL1 access */
#define CPACR_EL1_ZEN_EL0EN (1 << 17) /* enable EL0 access, if EL1EN set */
#define CPACR_EL1_ZEN_EL1EN (_BITUL(16)) /* enable EL1 access */
#define CPACR_EL1_ZEN_EL0EN (_BITUL(17)) /* enable EL0 access, if EL1EN set */
#define CPACR_EL1_ZEN (CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN)
/* Safe value for MPIDR_EL1: Bit31:RES1, Bit30:U:0, Bit24:MT:0 */
#define SYS_MPIDR_SAFE_VAL (1UL << 31)
#define SYS_MPIDR_SAFE_VAL (_BITUL(31))
#ifdef __ASSEMBLY__

13
arch/arm64/include/asm/thread_info.h
View File

@ -42,7 +42,18 @@ struct thread_info {
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
u64 ttbr0; /* saved TTBR0_EL1 */
#endif
int preempt_count; /* 0 => preemptable, <0 => bug */
union {
u64 preempt_count; /* 0 => preemptible, <0 => bug */
struct {
#ifdef CONFIG_CPU_BIG_ENDIAN
u32 need_resched;
u32 count;
#else
u32 count;
u32 need_resched;
#endif
} preempt;
};
};
#define thread_saved_pc(tsk) \

15
arch/arm64/include/asm/tlbflush.h
View File

@ -21,6 +21,7 @@
#ifndef __ASSEMBLY__
#include <linux/mm_types.h>
#include <linux/sched.h>
#include <asm/cputype.h>
#include <asm/mmu.h>
@ -164,14 +165,20 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
dsb(ish);
}
static inline void flush_tlb_page(struct vm_area_struct *vma,
unsigned long uaddr)
static inline void flush_tlb_page_nosync(struct vm_area_struct *vma,
unsigned long uaddr)
{
unsigned long addr = __TLBI_VADDR(uaddr, ASID(vma->vm_mm));
dsb(ishst);
__tlbi(vale1is, addr);
__tlbi_user(vale1is, addr);
}
static inline void flush_tlb_page(struct vm_area_struct *vma,
unsigned long uaddr)
{
flush_tlb_page_nosync(vma, uaddr);
dsb(ish);
}
@ -179,7 +186,7 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
* This is meant to avoid soft lock-ups on large TLB flushing ranges and not
* necessarily a performance improvement.
*/
#define MAX_TLBI_OPS 1024UL
#define MAX_TLBI_OPS PTRS_PER_PTE
static inline void __flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end,
@ -188,7 +195,7 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
unsigned long asid = ASID(vma->vm_mm);
unsigned long addr;
if ((end - start) > (MAX_TLBI_OPS * stride)) {
if ((end - start) >= (MAX_TLBI_OPS * stride)) {
flush_tlb_mm(vma->vm_mm);
return;
}

3
arch/arm64/include/asm/uaccess.h
View File

@ -45,8 +45,7 @@ static inline void set_fs(mm_segment_t fs)
* Prevent a mispredicted conditional call to set_fs from forwarding
* the wrong address limit to access_ok under speculation.
*/
dsb(nsh);
isb();
spec_bar();
/* On user-mode return, check fs is correct */
set_thread_flag(TIF_FSCHECK);

73
arch/arm64/include/asm/xor.h Normal file
View File

@ -0,0 +1,73 @@
/*
* arch/arm64/include/asm/xor.h
*
* Authors: Jackie Liu <liuyun01@kylinos.cn>
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/hardirq.h>
#include <asm-generic/xor.h>
#include <asm/hwcap.h>
#include <asm/neon.h>
#ifdef CONFIG_KERNEL_MODE_NEON
extern struct xor_block_template const xor_block_inner_neon;
static void
xor_neon_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
{
kernel_neon_begin();
xor_block_inner_neon.do_2(bytes, p1, p2);
kernel_neon_end();
}
static void
xor_neon_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3)
{
kernel_neon_begin();
xor_block_inner_neon.do_3(bytes, p1, p2, p3);
kernel_neon_end();
}
static void
xor_neon_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4)
{
kernel_neon_begin();
xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4);
kernel_neon_end();
}
static void
xor_neon_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
unsigned long *p3, unsigned long *p4, unsigned long *p5)
{
kernel_neon_begin();
xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5);
kernel_neon_end();
}
static struct xor_block_template xor_block_arm64 = {
.name = "arm64_neon",
.do_2 = xor_neon_2,
.do_3 = xor_neon_3,
.do_4 = xor_neon_4,
.do_5 = xor_neon_5
};
#undef XOR_TRY_TEMPLATES
#define XOR_TRY_TEMPLATES \
do { \
xor_speed(&xor_block_8regs); \
xor_speed(&xor_block_32regs); \
if (cpu_has_neon()) { \
xor_speed(&xor_block_arm64);\
} \
} while (0)
#endif /* ! CONFIG_KERNEL_MODE_NEON */

3
arch/arm64/include/uapi/asm/hwcap.h
View File

@ -49,5 +49,8 @@
#define HWCAP_ILRCPC (1 << 26)
#define HWCAP_FLAGM (1 << 27)
#define HWCAP_SSBS (1 << 28)
#define HWCAP_SB (1 << 29)
#define HWCAP_PACA (1 << 30)
#define HWCAP_PACG (1UL << 31)
#endif /* _UAPI__ASM_HWCAP_H */

7
arch/arm64/include/uapi/asm/ptrace.h
View File

@ -229,6 +229,13 @@ struct user_sve_header {
SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, flags) \
: SVE_PT_FPSIMD_OFFSET + SVE_PT_FPSIMD_SIZE(vq, flags))
/* pointer authentication masks (NT_ARM_PAC_MASK) */
struct user_pac_mask {
__u64 data_mask;
__u64 insn_mask;
};
#endif /* __ASSEMBLY__ */
#endif /* _UAPI__ASM_PTRACE_H */

6
arch/arm64/kernel/Makefile
View File

@ -30,7 +30,7 @@ $(obj)/%.stub.o: $(obj)/%.o FORCE
arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
sys_compat.o
arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
arm64-obj-$(CONFIG_MODULES) += module.o
arm64-obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o
arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o perf_callchain.o
arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
@ -49,14 +49,16 @@ arm64-obj-$(CONFIG_ARM64_ACPI_PARKING_PROTOCOL) += acpi_parking_protocol.o
arm64-obj-$(CONFIG_PARAVIRT) += paravirt.o
arm64-obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o
arm64-obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate-asm.o
arm64-obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o \
arm64-obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o \
cpu-reset.o
arm64-obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file.o kexec_image.o
arm64-obj-$(CONFIG_ARM64_RELOC_TEST) += arm64-reloc-test.o
arm64-reloc-test-y := reloc_test_core.o reloc_test_syms.o
arm64-obj-$(CONFIG_CRASH_DUMP) += crash_dump.o
arm64-obj-$(CONFIG_CRASH_CORE) += crash_core.o
arm64-obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o
arm64-obj-$(CONFIG_ARM64_SSBD) += ssbd.o
arm64-obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o
obj-y += $(arm64-obj-y) vdso/ probes/
obj-m += $(arm64-obj-m)

88
arch/arm64/kernel/arm64ksyms.c
View File

@ -1,88 +0,0 @@
/*
* Based on arch/arm/kernel/armksyms.c
*
* Copyright (C) 2000 Russell King
* Copyright (C) 2012 ARM Ltd.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/export.h>
#include <linux/sched.h>
#include <linux/string.h>
#include <linux/cryptohash.h>
#include <linux/delay.h>
#include <linux/in6.h>
#include <linux/syscalls.h>
#include <linux/uaccess.h>
#include <linux/io.h>
#include <linux/arm-smccc.h>
#include <linux/kprobes.h>
#include <asm/checksum.h>
EXPORT_SYMBOL(copy_page);
EXPORT_SYMBOL(clear_page);
/* user mem (segment) */
EXPORT_SYMBOL(__arch_copy_from_user);
EXPORT_SYMBOL(__arch_copy_to_user);
EXPORT_SYMBOL(__arch_clear_user);
EXPORT_SYMBOL(__arch_copy_in_user);
/* physical memory */
EXPORT_SYMBOL(memstart_addr);
/* string / mem functions */
#ifndef CONFIG_KASAN
EXPORT_SYMBOL(strchr);
EXPORT_SYMBOL(strrchr);
EXPORT_SYMBOL(strcmp);
EXPORT_SYMBOL(strncmp);
EXPORT_SYMBOL(strlen);
EXPORT_SYMBOL(strnlen);
EXPORT_SYMBOL(memcmp);
EXPORT_SYMBOL(memchr);
#endif
EXPORT_SYMBOL(memset);
EXPORT_SYMBOL(memcpy);
EXPORT_SYMBOL(memmove);
EXPORT_SYMBOL(__memset);
EXPORT_SYMBOL(__memcpy);
EXPORT_SYMBOL(__memmove);
/* atomic bitops */
EXPORT_SYMBOL(set_bit);
EXPORT_SYMBOL(test_and_set_bit);
EXPORT_SYMBOL(clear_bit);
EXPORT_SYMBOL(test_and_clear_bit);
EXPORT_SYMBOL(change_bit);
EXPORT_SYMBOL(test_and_change_bit);
#ifdef CONFIG_FUNCTION_TRACER
EXPORT_SYMBOL(_mcount);
NOKPROBE_SYMBOL(_mcount);
#endif
/* arm-smccc */
EXPORT_SYMBOL(__arm_smccc_smc);
EXPORT_SYMBOL(__arm_smccc_hvc);
/* tishift.S */
extern long long __ashlti3(long long a, int b);
EXPORT_SYMBOL(__ashlti3);
extern long long __ashrti3(long long a, int b);
EXPORT_SYMBOL(__ashrti3);
extern long long __lshrti3(long long a, int b);
EXPORT_SYMBOL(__lshrti3);

3
arch/arm64/kernel/asm-offsets.c
View File

@ -46,6 +46,9 @@ int main(void)
DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0));
#endif
DEFINE(TSK_STACK, offsetof(struct task_struct, stack));
#ifdef CONFIG_STACKPROTECTOR
DEFINE(TSK_STACK_CANARY, offsetof(struct task_struct, stack_canary));
#endif
BLANK();
DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context));
BLANK();

8
arch/arm64/kernel/cpu-reset.S
View File

@ -22,11 +22,11 @@
* __cpu_soft_restart(el2_switch, entry, arg0, arg1, arg2) - Helper for
* cpu_soft_restart.
*
* @el2_switch: Flag to indicate a swich to EL2 is needed.
* @el2_switch: Flag to indicate a switch to EL2 is needed.
* @entry: Location to jump to for soft reset.
* arg0: First argument passed to @entry.
* arg1: Second argument passed to @entry.
* arg2: Third argument passed to @entry.
* arg0: First argument passed to @entry. (relocation list)
* arg1: Second argument passed to @entry.(physical kernel entry)
* arg2: Third argument passed to @entry. (physical dtb address)
*
* Put the CPU into the same state as it would be if it had been reset, and
* branch to what would be the reset vector. It must be executed with the

149
arch/arm64/kernel/cpu_errata.c
View File

@ -135,7 +135,7 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
const char *hyp_vecs_start,
const char *hyp_vecs_end)
{
static DEFINE_SPINLOCK(bp_lock);
static DEFINE_RAW_SPINLOCK(bp_lock);
int cpu, slot = -1;
/*
@ -147,7 +147,7 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
return;
}
spin_lock(&bp_lock);
raw_spin_lock(&bp_lock);
for_each_possible_cpu(cpu) {
if (per_cpu(bp_hardening_data.fn, cpu) == fn) {
slot = per_cpu(bp_hardening_data.hyp_vectors_slot, cpu);
@ -163,7 +163,7 @@ static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
__this_cpu_write(bp_hardening_data.hyp_vectors_slot, slot);
__this_cpu_write(bp_hardening_data.fn, fn);
spin_unlock(&bp_lock);
raw_spin_unlock(&bp_lock);
}
#else
#define __smccc_workaround_1_smc_start NULL
@ -507,38 +507,6 @@ cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused)
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, \
CAP_MIDR_RANGE_LIST(midr_list)
/*
* Generic helper for handling capabilties with multiple (match,enable) pairs
* of call backs, sharing the same capability bit.
* Iterate over each entry to see if at least one matches.
*/
static bool __maybe_unused
multi_entry_cap_matches(const struct arm64_cpu_capabilities *entry, int scope)
{
const struct arm64_cpu_capabilities *caps;
for (caps = entry->match_list; caps->matches; caps++)
if (caps->matches(caps, scope))
return true;
return false;
}
/*
* Take appropriate action for all matching entries in the shared capability
* entry.
*/
static void __maybe_unused
multi_entry_cap_cpu_enable(const struct arm64_cpu_capabilities *entry)
{
const struct arm64_cpu_capabilities *caps;
for (caps = entry->match_list; caps->matches; caps++)
if (caps->matches(caps, SCOPE_LOCAL_CPU) &&
caps->cpu_enable)
caps->cpu_enable(caps);
}
#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
/*
@ -584,24 +552,63 @@ static const struct midr_range arm64_repeat_tlbi_cpus[] = {
#endif
const struct arm64_cpu_capabilities arm64_errata[] = {
#ifdef CONFIG_CAVIUM_ERRATUM_27456
static const struct midr_range cavium_erratum_27456_cpus[] = {
/* Cavium ThunderX, T88 pass 1.x - 2.1 */
MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 1),
/* Cavium ThunderX, T81 pass 1.0 */
MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
{},
};
#endif
#ifdef CONFIG_CAVIUM_ERRATUM_30115
static const struct midr_range cavium_erratum_30115_cpus[] = {
/* Cavium ThunderX, T88 pass 1.x - 2.2 */
MIDR_RANGE(MIDR_THUNDERX, 0, 0, 1, 2),
/* Cavium ThunderX, T81 pass 1.0 - 1.2 */
MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
/* Cavium ThunderX, T83 pass 1.0 */
MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
{},
};
#endif
#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
static const struct arm64_cpu_capabilities qcom_erratum_1003_list[] = {
{
ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
},
{
.midr_range.model = MIDR_QCOM_KRYO,
.matches = is_kryo_midr,
},
{},
};
#endif
#ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
static const struct midr_range workaround_clean_cache[] = {
#if defined(CONFIG_ARM64_ERRATUM_826319) || \
defined(CONFIG_ARM64_ERRATUM_827319) || \
defined(CONFIG_ARM64_ERRATUM_824069)
{
/* Cortex-A53 r0p[012] */
.desc = "ARM errata 826319, 827319, 824069",
.capability = ARM64_WORKAROUND_CLEAN_CACHE,
ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
.cpu_enable = cpu_enable_cache_maint_trap,
},
/* Cortex-A53 r0p[012]: ARM errata 826319, 827319, 824069 */
MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 2),
#endif
#ifdef CONFIG_ARM64_ERRATUM_819472
#ifdef CONFIG_ARM64_ERRATUM_819472
/* Cortex-A53 r0p[01] : ARM errata 819472 */
MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
#endif
{},
};
#endif
const struct arm64_cpu_capabilities arm64_errata[] = {
#ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
{
/* Cortex-A53 r0p[01] */
.desc = "ARM errata 819472",
.desc = "ARM errata 826319, 827319, 824069, 819472",
.capability = ARM64_WORKAROUND_CLEAN_CACHE,
ERRATA_MIDR_REV_RANGE(MIDR_CORTEX_A53, 0, 0, 1),
ERRATA_MIDR_RANGE_LIST(workaround_clean_cache),
.cpu_enable = cpu_enable_cache_maint_trap,
},
#endif
@ -652,40 +659,16 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
#endif
#ifdef CONFIG_CAVIUM_ERRATUM_27456
{
/* Cavium ThunderX, T88 pass 1.x - 2.1 */
.desc = "Cavium erratum 27456",
.capability = ARM64_WORKAROUND_CAVIUM_27456,
ERRATA_MIDR_RANGE(MIDR_THUNDERX,
0, 0,
1, 1),
},
{
/* Cavium ThunderX, T81 pass 1.0 */
.desc = "Cavium erratum 27456",
.capability = ARM64_WORKAROUND_CAVIUM_27456,
ERRATA_MIDR_REV(MIDR_THUNDERX_81XX, 0, 0),
ERRATA_MIDR_RANGE_LIST(cavium_erratum_27456_cpus),
},
#endif
#ifdef CONFIG_CAVIUM_ERRATUM_30115
{
/* Cavium ThunderX, T88 pass 1.x - 2.2 */
.desc = "Cavium erratum 30115",
.capability = ARM64_WORKAROUND_CAVIUM_30115,
ERRATA_MIDR_RANGE(MIDR_THUNDERX,
0, 0,
1, 2),
},
{
/* Cavium ThunderX, T81 pass 1.0 - 1.2 */
.desc = "Cavium erratum 30115",
.capability = ARM64_WORKAROUND_CAVIUM_30115,
ERRATA_MIDR_REV_RANGE(MIDR_THUNDERX_81XX, 0, 0, 2),
},
{
/* Cavium ThunderX, T83 pass 1.0 */
.desc = "Cavium erratum 30115",
.capability = ARM64_WORKAROUND_CAVIUM_30115,
ERRATA_MIDR_REV(MIDR_THUNDERX_83XX, 0, 0),
ERRATA_MIDR_RANGE_LIST(cavium_erratum_30115_cpus),
},
#endif
{
@ -697,16 +680,10 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
},
#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
{
.desc = "Qualcomm Technologies Falkor erratum 1003",
.desc = "Qualcomm Technologies Falkor/Kryo erratum 1003",
.capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
ERRATA_MIDR_REV(MIDR_QCOM_FALKOR_V1, 0, 0),
},
{
.desc = "Qualcomm Technologies Kryo erratum 1003",
.capability = ARM64_WORKAROUND_QCOM_FALKOR_E1003,
.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
.midr_range.model = MIDR_QCOM_KRYO,
.matches = is_kryo_midr,
.matches = cpucap_multi_entry_cap_matches,
.match_list = qcom_erratum_1003_list,
},
#endif
#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI
@ -753,6 +730,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
.capability = ARM64_WORKAROUND_1188873,
ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0),
},
#endif
#ifdef CONFIG_ARM64_ERRATUM_1165522
{
/* Cortex-A76 r0p0 to r2p0 */
.desc = "ARM erratum 1165522",
.capability = ARM64_WORKAROUND_1165522,
ERRATA_MIDR_RANGE(MIDR_CORTEX_A76, 0, 0, 2, 0),
},
#endif
{
}

314
arch/arm64/kernel/cpufeature.c
View File

@ -52,6 +52,7 @@ unsigned int compat_elf_hwcap2 __read_mostly;
DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
EXPORT_SYMBOL(cpu_hwcaps);
static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM64_NCAPS];
/*
* Flag to indicate if we have computed the system wide
@ -141,9 +142,18 @@ static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
};
static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_SB_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_GPI_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_GPA_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_LRCPC_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_FCMA_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_JSCVT_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_API_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_APA_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_DPB_SHIFT, 4, 0),
ARM64_FTR_END,
};
@ -518,6 +528,29 @@ static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
}
extern const struct arm64_cpu_capabilities arm64_errata[];
static const struct arm64_cpu_capabilities arm64_features[];
static void __init
init_cpu_hwcaps_indirect_list_from_array(const struct arm64_cpu_capabilities *caps)
{
for (; caps->matches; caps++) {
if (WARN(caps->capability >= ARM64_NCAPS,
"Invalid capability %d\n", caps->capability))
continue;
if (WARN(cpu_hwcaps_ptrs[caps->capability],
"Duplicate entry for capability %d\n",
caps->capability))
continue;
cpu_hwcaps_ptrs[caps->capability] = caps;
}
}
static void __init init_cpu_hwcaps_indirect_list(void)
{
init_cpu_hwcaps_indirect_list_from_array(arm64_features);
init_cpu_hwcaps_indirect_list_from_array(arm64_errata);
}
static void __init setup_boot_cpu_capabilities(void);
void __init init_cpu_features(struct cpuinfo_arm64 *info)
@ -563,6 +596,12 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
sve_init_vq_map();
}
/*
* Initialize the indirect array of CPU hwcaps capabilities pointers
* before we handle the boot CPU below.
*/
init_cpu_hwcaps_indirect_list();
/*
* Detect and enable early CPU capabilities based on the boot CPU,
* after we have initialised the CPU feature infrastructure.
@ -915,6 +954,12 @@ static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
static const struct midr_range kpti_safe_list[] = {
MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
{ /* sentinel */ }
};
char const *str = "command line option";
@ -1145,6 +1190,14 @@ static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused)
}
#endif /* CONFIG_ARM64_RAS_EXTN */
#ifdef CONFIG_ARM64_PTR_AUTH
static void cpu_enable_address_auth(struct arm64_cpu_capabilities const *cap)
{
sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ENIA | SCTLR_ELx_ENIB |
SCTLR_ELx_ENDA | SCTLR_ELx_ENDB);
}
#endif /* CONFIG_ARM64_PTR_AUTH */
static const struct arm64_cpu_capabilities arm64_features[] = {
{
.desc = "GIC system register CPU interface",
@ -1368,22 +1421,115 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.cpu_enable = cpu_enable_cnp,
},
#endif
{
.desc = "Speculation barrier (SB)",
.capability = ARM64_HAS_SB,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
.sys_reg = SYS_ID_AA64ISAR1_EL1,
.field_pos = ID_AA64ISAR1_SB_SHIFT,
.sign = FTR_UNSIGNED,
.min_field_value = 1,
},
#ifdef CONFIG_ARM64_PTR_AUTH
{
.desc = "Address authentication (architected algorithm)",
.capability = ARM64_HAS_ADDRESS_AUTH_ARCH,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.sys_reg = SYS_ID_AA64ISAR1_EL1,
.sign = FTR_UNSIGNED,
.field_pos = ID_AA64ISAR1_APA_SHIFT,
.min_field_value = ID_AA64ISAR1_APA_ARCHITECTED,
.matches = has_cpuid_feature,
.cpu_enable = cpu_enable_address_auth,
},
{
.desc = "Address authentication (IMP DEF algorithm)",
.capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.sys_reg = SYS_ID_AA64ISAR1_EL1,
.sign = FTR_UNSIGNED,
.field_pos = ID_AA64ISAR1_API_SHIFT,
.min_field_value = ID_AA64ISAR1_API_IMP_DEF,
.matches = has_cpuid_feature,
.cpu_enable = cpu_enable_address_auth,
},
{
.desc = "Generic authentication (architected algorithm)",
.capability = ARM64_HAS_GENERIC_AUTH_ARCH,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.sys_reg = SYS_ID_AA64ISAR1_EL1,
.sign = FTR_UNSIGNED,
.field_pos = ID_AA64ISAR1_GPA_SHIFT,
.min_field_value = ID_AA64ISAR1_GPA_ARCHITECTED,
.matches = has_cpuid_feature,
},
{
.desc = "Generic authentication (IMP DEF algorithm)",
.capability = ARM64_HAS_GENERIC_AUTH_IMP_DEF,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.sys_reg = SYS_ID_AA64ISAR1_EL1,
.sign = FTR_UNSIGNED,
.field_pos = ID_AA64ISAR1_GPI_SHIFT,
.min_field_value = ID_AA64ISAR1_GPI_IMP_DEF,
.matches = has_cpuid_feature,
},
#endif /* CONFIG_ARM64_PTR_AUTH */
{},
};
#define HWCAP_CAP(reg, field, s, min_value, cap_type, cap) \
{ \
.desc = #cap, \
.type = ARM64_CPUCAP_SYSTEM_FEATURE, \
.matches = has_cpuid_feature, \
.sys_reg = reg, \
.field_pos = field, \
.sign = s, \
.min_field_value = min_value, \
.hwcap_type = cap_type, \
.hwcap = cap, \
#define HWCAP_CPUID_MATCH(reg, field, s, min_value) \
.matches = has_cpuid_feature, \
.sys_reg = reg, \
.field_pos = field, \
.sign = s, \
.min_field_value = min_value,
#define __HWCAP_CAP(name, cap_type, cap) \
.desc = name, \
.type = ARM64_CPUCAP_SYSTEM_FEATURE, \
.hwcap_type = cap_type, \
.hwcap = cap, \
#define HWCAP_CAP(reg, field, s, min_value, cap_type, cap) \
{ \
__HWCAP_CAP(#cap, cap_type, cap) \
HWCAP_CPUID_MATCH(reg, field, s, min_value) \
}
#define HWCAP_MULTI_CAP(list, cap_type, cap) \
{ \
__HWCAP_CAP(#cap, cap_type, cap) \
.matches = cpucap_multi_entry_cap_matches, \
.match_list = list, \
}
#ifdef CONFIG_ARM64_PTR_AUTH
static const struct arm64_cpu_capabilities ptr_auth_hwcap_addr_matches[] = {
{
HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_APA_SHIFT,
FTR_UNSIGNED, ID_AA64ISAR1_APA_ARCHITECTED)
},
{
HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_API_SHIFT,
FTR_UNSIGNED, ID_AA64ISAR1_API_IMP_DEF)
},
{},
};
static const struct arm64_cpu_capabilities ptr_auth_hwcap_gen_matches[] = {
{
HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_GPA_SHIFT,
FTR_UNSIGNED, ID_AA64ISAR1_GPA_ARCHITECTED)
},
{
HWCAP_CPUID_MATCH(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_GPI_SHIFT,
FTR_UNSIGNED, ID_AA64ISAR1_GPI_IMP_DEF)
},
{},
};
#endif
static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_PMULL),
HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_AES_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_AES),
@ -1409,11 +1555,16 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_FCMA_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_FCMA),
HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_LRCPC),
HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_LRCPC_SHIFT, FTR_UNSIGNED, 2, CAP_HWCAP, HWCAP_ILRCPC),
HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_SB_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SB),
HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_USCAT),
#ifdef CONFIG_ARM64_SVE
HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, HWCAP_SVE),
#endif
HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, HWCAP_SSBS),
#ifdef CONFIG_ARM64_PTR_AUTH
HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, HWCAP_PACA),
HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, HWCAP_PACG),
#endif
{},
};
@ -1482,52 +1633,46 @@ static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
cap_set_elf_hwcap(hwcaps);
}
/*
* Check if the current CPU has a given feature capability.
* Should be called from non-preemptible context.
*/
static bool __this_cpu_has_cap(const struct arm64_cpu_capabilities *cap_array,
unsigned int cap)
static void update_cpu_capabilities(u16 scope_mask)
{
int i;
const struct arm64_cpu_capabilities *caps;
if (WARN_ON(preemptible()))
return false;
for (caps = cap_array; caps->matches; caps++)
if (caps->capability == cap)
return caps->matches(caps, SCOPE_LOCAL_CPU);
return false;
}
static void __update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
u16 scope_mask, const char *info)
{
scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
for (; caps->matches; caps++) {
if (!(caps->type & scope_mask) ||
for (i = 0; i < ARM64_NCAPS; i++) {
caps = cpu_hwcaps_ptrs[i];
if (!caps || !(caps->type & scope_mask) ||
cpus_have_cap(caps->capability) ||
!caps->matches(caps, cpucap_default_scope(caps)))
continue;
if (!cpus_have_cap(caps->capability) && caps->desc)
pr_info("%s %s\n", info, caps->desc);
if (caps->desc)
pr_info("detected: %s\n", caps->desc);
cpus_set_cap(caps->capability);
}
}
static void update_cpu_capabilities(u16 scope_mask)
/*
* Enable all the available capabilities on this CPU. The capabilities
* with BOOT_CPU scope are handled separately and hence skipped here.
*/
static int cpu_enable_non_boot_scope_capabilities(void *__unused)
{
__update_cpu_capabilities(arm64_errata, scope_mask,
"enabling workaround for");
__update_cpu_capabilities(arm64_features, scope_mask, "detected:");
}
int i;
u16 non_boot_scope = SCOPE_ALL & ~SCOPE_BOOT_CPU;
static int __enable_cpu_capability(void *arg)
{
const struct arm64_cpu_capabilities *cap = arg;
for_each_available_cap(i) {
const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[i];
cap->cpu_enable(cap);
if (WARN_ON(!cap))
continue;
if (!(cap->type & non_boot_scope))
continue;
if (cap->cpu_enable)
cap->cpu_enable(cap);
}
return 0;
}
@ -1535,21 +1680,29 @@ static int __enable_cpu_capability(void *arg)
* Run through the enabled capabilities and enable() it on all active
* CPUs
*/
static void __init
__enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
u16 scope_mask)
static void __init enable_cpu_capabilities(u16 scope_mask)
{
scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
for (; caps->matches; caps++) {
unsigned int num = caps->capability;
int i;
const struct arm64_cpu_capabilities *caps;
bool boot_scope;
if (!(caps->type & scope_mask) || !cpus_have_cap(num))
scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
boot_scope = !!(scope_mask & SCOPE_BOOT_CPU);
for (i = 0; i < ARM64_NCAPS; i++) {
unsigned int num;
caps = cpu_hwcaps_ptrs[i];
if (!caps || !(caps->type & scope_mask))
continue;
num = caps->capability;
if (!cpus_have_cap(num))
continue;
/* Ensure cpus_have_const_cap(num) works */
static_branch_enable(&cpu_hwcap_keys[num]);
if (caps->cpu_enable) {
if (boot_scope && caps->cpu_enable)
/*
* Capabilities with SCOPE_BOOT_CPU scope are finalised
* before any secondary CPU boots. Thus, each secondary
@ -1558,25 +1711,19 @@ __enable_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
* the boot CPU, for which the capability must be
* enabled here. This approach avoids costly
* stop_machine() calls for this case.
*
* Otherwise, use stop_machine() as it schedules the
* work allowing us to modify PSTATE, instead of
* on_each_cpu() which uses an IPI, giving us a PSTATE
* that disappears when we return.
*/
if (scope_mask & SCOPE_BOOT_CPU)
caps->cpu_enable(caps);
else
stop_machine(__enable_cpu_capability,
(void *)caps, cpu_online_mask);
}
caps->cpu_enable(caps);
}
}
static void __init enable_cpu_capabilities(u16 scope_mask)
{
__enable_cpu_capabilities(arm64_errata, scope_mask);
__enable_cpu_capabilities(arm64_features, scope_mask);
/*
* For all non-boot scope capabilities, use stop_machine()
* as it schedules the work allowing us to modify PSTATE,
* instead of on_each_cpu() which uses an IPI, giving us a
* PSTATE that disappears when we return.
*/
if (!boot_scope)
stop_machine(cpu_enable_non_boot_scope_capabilities,
NULL, cpu_online_mask);
}
/*
@ -1586,16 +1733,17 @@ static void __init enable_cpu_capabilities(u16 scope_mask)
*
* Returns "false" on conflicts.
*/
static bool
__verify_local_cpu_caps(const struct arm64_cpu_capabilities *caps,
u16 scope_mask)
static bool verify_local_cpu_caps(u16 scope_mask)
{
int i;
bool cpu_has_cap, system_has_cap;
const struct arm64_cpu_capabilities *caps;
scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
for (; caps->matches; caps++) {
if (!(caps->type & scope_mask))
for (i = 0; i < ARM64_NCAPS; i++) {
caps = cpu_hwcaps_ptrs[i];
if (!caps || !(caps->type & scope_mask))
continue;
cpu_has_cap = caps->matches(caps, SCOPE_LOCAL_CPU);
@ -1626,7 +1774,7 @@ __verify_local_cpu_caps(const struct arm64_cpu_capabilities *caps,
}
}
if (caps->matches) {
if (i < ARM64_NCAPS) {
pr_crit("CPU%d: Detected conflict for capability %d (%s), System: %d, CPU: %d\n",
smp_processor_id(), caps->capability,
caps->desc, system_has_cap, cpu_has_cap);
@ -1636,12 +1784,6 @@ __verify_local_cpu_caps(const struct arm64_cpu_capabilities *caps,
return true;
}
static bool verify_local_cpu_caps(u16 scope_mask)
{
return __verify_local_cpu_caps(arm64_errata, scope_mask) &&
__verify_local_cpu_caps(arm64_features, scope_mask);
}
/*
* Check for CPU features that are used in early boot
* based on the Boot CPU value.
@ -1750,12 +1892,16 @@ static void __init mark_const_caps_ready(void)
static_branch_enable(&arm64_const_caps_ready);
}
extern const struct arm64_cpu_capabilities arm64_errata[];
bool this_cpu_has_cap(unsigned int cap)
bool this_cpu_has_cap(unsigned int n)
{
return (__this_cpu_has_cap(arm64_features, cap) ||
__this_cpu_has_cap(arm64_errata, cap));
if (!WARN_ON(preemptible()) && n < ARM64_NCAPS) {
const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[n];
if (cap)
return cap->matches(cap, SCOPE_LOCAL_CPU);
}
return false;
}
static void __init setup_system_capabilities(void)

3
arch/arm64/kernel/cpuinfo.c
View File

@ -82,6 +82,9 @@ static const char *const hwcap_str[] = {
"ilrcpc",
"flagm",
"ssbs",
"sb",
"paca",
"pacg",
NULL
};

55
arch/arm64/kernel/entry-ftrace.S
View File

@ -79,7 +79,6 @@
.macro mcount_get_lr reg
ldr \reg, [x29]
ldr \reg, [\reg, #8]
mcount_adjust_addr \reg, \reg
.endm
.macro mcount_get_lr_addr reg
@ -121,6 +120,8 @@ skip_ftrace_call: // }
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
mcount_exit
ENDPROC(_mcount)
EXPORT_SYMBOL(_mcount)
NOKPROBE(_mcount)
#else /* CONFIG_DYNAMIC_FTRACE */
/*
@ -132,6 +133,8 @@ ENDPROC(_mcount)
ENTRY(_mcount)
ret
ENDPROC(_mcount)
EXPORT_SYMBOL(_mcount)
NOKPROBE(_mcount)
/*
* void ftrace_caller(unsigned long return_address)
@ -148,14 +151,12 @@ ENTRY(ftrace_caller)
mcount_get_pc0 x0 // function's pc
mcount_get_lr x1 // function's lr
.global ftrace_call
ftrace_call: // tracer(pc, lr);
GLOBAL(ftrace_call) // tracer(pc, lr);
nop // This will be replaced with "bl xxx"
// where xxx can be any kind of tracer.
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
.global ftrace_graph_call
ftrace_graph_call: // ftrace_graph_caller();
GLOBAL(ftrace_graph_call) // ftrace_graph_caller();
nop // If enabled, this will be replaced
// "b ftrace_graph_caller"
#endif
@ -169,24 +170,6 @@ ENTRY(ftrace_stub)
ENDPROC(ftrace_stub)
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* save return value regs*/
.macro save_return_regs
sub sp, sp, #64
stp x0, x1, [sp]
stp x2, x3, [sp, #16]
stp x4, x5, [sp, #32]
stp x6, x7, [sp, #48]
.endm
/* restore return value regs*/
.macro restore_return_regs
ldp x0, x1, [sp]
ldp x2, x3, [sp, #16]
ldp x4, x5, [sp, #32]
ldp x6, x7, [sp, #48]
add sp, sp, #64
.endm
/*
* void ftrace_graph_caller(void)
*
@ -197,10 +180,10 @@ ENDPROC(ftrace_stub)
* and run return_to_handler() later on its exit.
*/
ENTRY(ftrace_graph_caller)
mcount_get_lr_addr x0 // pointer to function's saved lr
mcount_get_pc x1 // function's pc
mcount_get_pc x0 // function's pc
mcount_get_lr_addr x1 // pointer to function's saved lr
mcount_get_parent_fp x2 // parent's fp
bl prepare_ftrace_return // prepare_ftrace_return(&lr, pc, fp)
bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp)
mcount_exit
ENDPROC(ftrace_graph_caller)
@ -209,15 +192,27 @@ ENDPROC(ftrace_graph_caller)
* void return_to_handler(void)
*
* Run ftrace_return_to_handler() before going back to parent.
* @fp is checked against the value passed by ftrace_graph_caller()
* only when HAVE_FUNCTION_GRAPH_FP_TEST is enabled.
* @fp is checked against the value passed by ftrace_graph_caller().
*/
ENTRY(return_to_handler)
save_return_regs
/* save return value regs */
sub sp, sp, #64
stp x0, x1, [sp]
stp x2, x3, [sp, #16]
stp x4, x5, [sp, #32]
stp x6, x7, [sp, #48]
mov x0, x29 // parent's fp
bl ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
mov x30, x0 // restore the original return address
restore_return_regs
/* restore return value regs */
ldp x0, x1, [sp]
ldp x2, x3, [sp, #16]
ldp x4, x5, [sp, #32]
ldp x6, x7, [sp, #48]
add sp, sp, #64
ret
END(return_to_handler)
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

12
arch/arm64/kernel/entry.S
View File

@ -344,10 +344,6 @@ alternative_else_nop_endif
ldp x28, x29, [sp, #16 * 14]
ldr lr, [sp, #S_LR]
add sp, sp, #S_FRAME_SIZE // restore sp
/*
* ARCH_HAS_MEMBARRIER_SYNC_CORE rely on eret context synchronization
* when returning from IPI handler, and when returning to user-space.
*/
.if \el == 0
alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
@ -363,6 +359,7 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
.else
eret
.endif
sb
.endm
.macro irq_stack_entry
@ -622,10 +619,8 @@ el1_irq:
irq_handler
#ifdef CONFIG_PREEMPT
ldr w24, [tsk, #TSK_TI_PREEMPT] // get preempt count
cbnz w24, 1f // preempt count != 0
ldr x0, [tsk, #TSK_TI_FLAGS] // get flags
tbz x0, #TIF_NEED_RESCHED, 1f // needs rescheduling?
ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count
cbnz x24, 1f // preempt count != 0
bl el1_preempt
1:
#endif
@ -1006,6 +1001,7 @@ alternative_insn isb, nop, ARM64_WORKAROUND_QCOM_FALKOR_E1003
mrs x30, far_el1
.endif
eret
sb
.endm
.align 11

4
arch/arm64/kernel/ftrace.c
View File

@ -104,7 +104,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
* is added in the future, but for now, the pr_err() below
* deals with a theoretical issue only.
*/
trampoline = get_plt_entry(addr);
trampoline = get_plt_entry(addr, mod->arch.ftrace_trampoline);
if (!plt_entries_equal(mod->arch.ftrace_trampoline,
&trampoline)) {
if (!plt_entries_equal(mod->arch.ftrace_trampoline,
@ -211,7 +211,7 @@ int __init ftrace_dyn_arch_init(void)
*
* Note that @frame_pointer is used only for sanity check later.
*/
void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
unsigned long frame_pointer)
{
unsigned long return_hooker = (unsigned long)&return_to_handler;

46
arch/arm64/kernel/head.S
View File

@ -31,6 +31,7 @@
#include <asm/cache.h>
#include <asm/cputype.h>
#include <asm/elf.h>
#include <asm/image.h>
#include <asm/kernel-pgtable.h>
#include <asm/kvm_arm.h>
#include <asm/memory.h>
@ -91,7 +92,7 @@ _head:
.quad 0 // reserved
.quad 0 // reserved
.quad 0 // reserved
.ascii "ARM\x64" // Magic number
.ascii ARM64_IMAGE_MAGIC // Magic number
#ifdef CONFIG_EFI
.long pe_header - _head // Offset to the PE header.
@ -318,6 +319,19 @@ __create_page_tables:
adrp x0, idmap_pg_dir
adrp x3, __idmap_text_start // __pa(__idmap_text_start)
#ifdef CONFIG_ARM64_USER_VA_BITS_52
mrs_s x6, SYS_ID_AA64MMFR2_EL1
and x6, x6, #(0xf << ID_AA64MMFR2_LVA_SHIFT)
mov x5, #52
cbnz x6, 1f
#endif
mov x5, #VA_BITS
1:
adr_l x6, vabits_user
str x5, [x6]
dmb sy
dc ivac, x6 // Invalidate potentially stale cache line
/*
* VA_BITS may be too small to allow for an ID mapping to be created
* that covers system RAM if that is located sufficiently high in the
@ -496,10 +510,9 @@ ENTRY(el2_setup)
#endif
/* Hyp configuration. */
mov x0, #HCR_RW // 64-bit EL1
mov_q x0, HCR_HOST_NVHE_FLAGS
cbz x2, set_hcr
orr x0, x0, #HCR_TGE // Enable Host Extensions
orr x0, x0, #HCR_E2H
mov_q x0, HCR_HOST_VHE_FLAGS
set_hcr:
msr hcr_el2, x0
isb
@ -707,6 +720,7 @@ secondary_startup:
/*
* Common entry point for secondary CPUs.
*/
bl __cpu_secondary_check52bitva
bl __cpu_setup // initialise processor
adrp x1, swapper_pg_dir
bl __enable_mmu
@ -769,6 +783,7 @@ ENTRY(__enable_mmu)
phys_to_ttbr x1, x1
phys_to_ttbr x2, x2
msr ttbr0_el1, x2 // load TTBR0
offset_ttbr1 x1
msr ttbr1_el1, x1 // load TTBR1
isb
msr sctlr_el1, x0
@ -784,9 +799,30 @@ ENTRY(__enable_mmu)
ret
ENDPROC(__enable_mmu)
ENTRY(__cpu_secondary_check52bitva)
#ifdef CONFIG_ARM64_USER_VA_BITS_52
ldr_l x0, vabits_user
cmp x0, #52
b.ne 2f
mrs_s x0, SYS_ID_AA64MMFR2_EL1
and x0, x0, #(0xf << ID_AA64MMFR2_LVA_SHIFT)
cbnz x0, 2f
update_early_cpu_boot_status \
CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_52_BIT_VA, x0, x1
1: wfe
wfi
b 1b
#endif
2: ret
ENDPROC(__cpu_secondary_check52bitva)
__no_granule_support:
/* Indicate that this CPU can't boot and is stuck in the kernel */
update_early_cpu_boot_status CPU_STUCK_IN_KERNEL, x1, x2
update_early_cpu_boot_status \
CPU_STUCK_IN_KERNEL | CPU_STUCK_REASON_NO_GRAN, x1, x2
1:
wfe
wfi

1
arch/arm64/kernel/hibernate-asm.S
View File

@ -40,6 +40,7 @@
tlbi vmalle1
dsb nsh
phys_to_ttbr \tmp, \page_table
offset_ttbr1 \tmp
msr ttbr1_el1, \tmp
isb
.endm

67
arch/arm64/kernel/image.h
View File

@ -15,13 +15,15 @@
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __ASM_IMAGE_H
#define __ASM_IMAGE_H
#ifndef __ARM64_KERNEL_IMAGE_H
#define __ARM64_KERNEL_IMAGE_H
#ifndef LINKER_SCRIPT
#error This file should only be included in vmlinux.lds.S
#endif
#include <asm/image.h>
/*
* There aren't any ELF relocations we can use to endian-swap values known only
* at link time (e.g. the subtraction of two symbol addresses), so we must get
@ -47,19 +49,22 @@
sym##_lo32 = DATA_LE32((data) & 0xffffffff); \
sym##_hi32 = DATA_LE32((data) >> 32)
#define __HEAD_FLAG(field) (__HEAD_FLAG_##field << \
ARM64_IMAGE_FLAG_##field##_SHIFT)
#ifdef CONFIG_CPU_BIG_ENDIAN
#define __HEAD_FLAG_BE 1
#define __HEAD_FLAG_BE ARM64_IMAGE_FLAG_BE
#else
#define __HEAD_FLAG_BE 0
#define __HEAD_FLAG_BE ARM64_IMAGE_FLAG_LE
#endif
#define __HEAD_FLAG_PAGE_SIZE ((PAGE_SHIFT - 10) / 2)
#define __HEAD_FLAG_PHYS_BASE 1
#define __HEAD_FLAGS ((__HEAD_FLAG_BE << 0) | \
(__HEAD_FLAG_PAGE_SIZE << 1) | \
(__HEAD_FLAG_PHYS_BASE << 3))
#define __HEAD_FLAGS (__HEAD_FLAG(BE) | \
__HEAD_FLAG(PAGE_SIZE) | \
__HEAD_FLAG(PHYS_BASE))
/*
* These will output as part of the Image header, which should be little-endian
@ -75,16 +80,6 @@
__efistub_stext_offset = stext - _text;
/*
* Prevent the symbol aliases below from being emitted into the kallsyms
* table, by forcing them to be absolute symbols (which are conveniently
* ignored by scripts/kallsyms) rather than section relative symbols.
* The distinction is only relevant for partial linking, and only for symbols
* that are defined within a section declaration (which is not the case for
* the definitions below) so the resulting values will be identical.
*/
#define KALLSYMS_HIDE(sym) ABSOLUTE(sym)
/*
* The EFI stub has its own symbol namespace prefixed by __efistub_, to
* isolate it from the kernel proper. The following symbols are legally
@ -94,29 +89,29 @@ __efistub_stext_offset = stext - _text;
* linked at. The routines below are all implemented in assembler in a
* position independent manner
*/
__efistub_memcmp = KALLSYMS_HIDE(__pi_memcmp);
__efistub_memchr = KALLSYMS_HIDE(__pi_memchr);
__efistub_memcpy = KALLSYMS_HIDE(__pi_memcpy);
__efistub_memmove = KALLSYMS_HIDE(__pi_memmove);
__efistub_memset = KALLSYMS_HIDE(__pi_memset);
__efistub_strlen = KALLSYMS_HIDE(__pi_strlen);
__efistub_strnlen = KALLSYMS_HIDE(__pi_strnlen);
__efistub_strcmp = KALLSYMS_HIDE(__pi_strcmp);
__efistub_strncmp = KALLSYMS_HIDE(__pi_strncmp);
__efistub_strrchr = KALLSYMS_HIDE(__pi_strrchr);
__efistub___flush_dcache_area = KALLSYMS_HIDE(__pi___flush_dcache_area);
__efistub_memcmp = __pi_memcmp;
__efistub_memchr = __pi_memchr;
__efistub_memcpy = __pi_memcpy;
__efistub_memmove = __pi_memmove;
__efistub_memset = __pi_memset;
__efistub_strlen = __pi_strlen;
__efistub_strnlen = __pi_strnlen;
__efistub_strcmp = __pi_strcmp;
__efistub_strncmp = __pi_strncmp;
__efistub_strrchr = __pi_strrchr;
__efistub___flush_dcache_area = __pi___flush_dcache_area;
#ifdef CONFIG_KASAN
__efistub___memcpy = KALLSYMS_HIDE(__pi_memcpy);
__efistub___memmove = KALLSYMS_HIDE(__pi_memmove);
__efistub___memset = KALLSYMS_HIDE(__pi_memset);
__efistub___memcpy = __pi_memcpy;
__efistub___memmove = __pi_memmove;
__efistub___memset = __pi_memset;
#endif
__efistub__text = KALLSYMS_HIDE(_text);
__efistub__end = KALLSYMS_HIDE(_end);
__efistub__edata = KALLSYMS_HIDE(_edata);
__efistub_screen_info = KALLSYMS_HIDE(screen_info);
__efistub__text = _text;
__efistub__end = _end;
__efistub__edata = _edata;
__efistub_screen_info = screen_info;
#endif
#endif /* __ASM_IMAGE_H */
#endif /* __ARM64_KERNEL_IMAGE_H */

29
arch/arm64/kernel/insn.c
View File

@ -1239,6 +1239,35 @@ u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
}
u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr,
enum aarch64_insn_register reg,
enum aarch64_insn_adr_type type)
{
u32 insn;
s32 offset;
switch (type) {
case AARCH64_INSN_ADR_TYPE_ADR:
insn = aarch64_insn_get_adr_value();
offset = addr - pc;
break;
case AARCH64_INSN_ADR_TYPE_ADRP:
insn = aarch64_insn_get_adrp_value();
offset = (addr - ALIGN_DOWN(pc, SZ_4K)) >> 12;
break;
default:
pr_err("%s: unknown adr encoding %d\n", __func__, type);
return AARCH64_BREAK_FAULT;
}
if (offset < -SZ_1M || offset >= SZ_1M)
return AARCH64_BREAK_FAULT;
insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, reg);
return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, offset);
}
/*
* Decode the imm field of a branch, and return the byte offset as a
* signed value (so it can be used when computing a new branch

130
arch/arm64/kernel/kexec_image.c Normal file
View File

@ -0,0 +1,130 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Kexec image loader
* Copyright (C) 2018 Linaro Limited
* Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
*/
#define pr_fmt(fmt) "kexec_file(Image): " fmt
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/kexec.h>
#include <linux/pe.h>
#include <linux/string.h>
#include <linux/verification.h>
#include <asm/byteorder.h>
#include <asm/cpufeature.h>
#include <asm/image.h>
#include <asm/memory.h>
static int image_probe(const char *kernel_buf, unsigned long kernel_len)
{
const struct arm64_image_header *h =
(const struct arm64_image_header *)(kernel_buf);
if (!h || (kernel_len < sizeof(*h)))
return -EINVAL;
if (memcmp(&h->magic, ARM64_IMAGE_MAGIC, sizeof(h->magic)))
return -EINVAL;
return 0;
}
static void *image_load(struct kimage *image,
char *kernel, unsigned long kernel_len,
char *initrd, unsigned long initrd_len,
char *cmdline, unsigned long cmdline_len)
{
struct arm64_image_header *h;
u64 flags, value;
bool be_image, be_kernel;
struct kexec_buf kbuf;
unsigned long text_offset;
struct kexec_segment *kernel_segment;
int ret;
/* We don't support crash kernels yet. */
if (image->type == KEXEC_TYPE_CRASH)
return ERR_PTR(-EOPNOTSUPP);
/*
* We require a kernel with an unambiguous Image header. Per
* Documentation/booting.txt, this is the case when image_size
* is non-zero (practically speaking, since v3.17).
*/
h = (struct arm64_image_header *)kernel;
if (!h->image_size)
return ERR_PTR(-EINVAL);
/* Check cpu features */
flags = le64_to_cpu(h->flags);
be_image = arm64_image_flag_field(flags, ARM64_IMAGE_FLAG_BE);
be_kernel = IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
if ((be_image != be_kernel) && !system_supports_mixed_endian())
return ERR_PTR(-EINVAL);
value = arm64_image_flag_field(flags, ARM64_IMAGE_FLAG_PAGE_SIZE);
if (((value == ARM64_IMAGE_FLAG_PAGE_SIZE_4K) &&
!system_supports_4kb_granule()) ||
((value == ARM64_IMAGE_FLAG_PAGE_SIZE_64K) &&
!system_supports_64kb_granule()) ||
((value == ARM64_IMAGE_FLAG_PAGE_SIZE_16K) &&
!system_supports_16kb_granule()))
return ERR_PTR(-EINVAL);
/* Load the kernel */
kbuf.image = image;
kbuf.buf_min = 0;
kbuf.buf_max = ULONG_MAX;
kbuf.top_down = false;
kbuf.buffer = kernel;
kbuf.bufsz = kernel_len;
kbuf.mem = 0;
kbuf.memsz = le64_to_cpu(h->image_size);
text_offset = le64_to_cpu(h->text_offset);
kbuf.buf_align = MIN_KIMG_ALIGN;
/* Adjust kernel segment with TEXT_OFFSET */
kbuf.memsz += text_offset;
ret = kexec_add_buffer(&kbuf);
if (ret)
return ERR_PTR(ret);
kernel_segment = &image->segment[image->nr_segments - 1];
kernel_segment->mem += text_offset;
kernel_segment->memsz -= text_offset;
image->start = kernel_segment->mem;
pr_debug("Loaded kernel at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
kernel_segment->mem, kbuf.bufsz,
kernel_segment->memsz);
/* Load additional data */
ret = load_other_segments(image,
kernel_segment->mem, kernel_segment->memsz,
initrd, initrd_len, cmdline);
return ERR_PTR(ret);
}
#ifdef CONFIG_KEXEC_IMAGE_VERIFY_SIG
static int image_verify_sig(const char *kernel, unsigned long kernel_len)
{
return verify_pefile_signature(kernel, kernel_len, NULL,
VERIFYING_KEXEC_PE_SIGNATURE);
}
#endif
const struct kexec_file_ops kexec_image_ops = {
.probe = image_probe,
.load = image_load,
#ifdef CONFIG_KEXEC_IMAGE_VERIFY_SIG
.verify_sig = image_verify_sig,
#endif
};

12
arch/arm64/kernel/machine_kexec.c
View File

@ -212,9 +212,17 @@ void machine_kexec(struct kimage *kimage)
* uses physical addressing to relocate the new image to its final
* position and transfers control to the image entry point when the
* relocation is complete.
* In kexec case, kimage->start points to purgatory assuming that
* kernel entry and dtb address are embedded in purgatory by
* userspace (kexec-tools).
* In kexec_file case, the kernel starts directly without purgatory.
*/
cpu_soft_restart(reboot_code_buffer_phys, kimage->head, kimage->start, 0);
cpu_soft_restart(reboot_code_buffer_phys, kimage->head, kimage->start,
#ifdef CONFIG_KEXEC_FILE
kimage->arch.dtb_mem);
#else
0);
#endif
BUG(); /* Should never get here. */
}

224
arch/arm64/kernel/machine_kexec_file.c Normal file
View File

@ -0,0 +1,224 @@
// SPDX-License-Identifier: GPL-2.0
/*
* kexec_file for arm64
*
* Copyright (C) 2018 Linaro Limited
* Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
*
* Most code is derived from arm64 port of kexec-tools
*/
#define pr_fmt(fmt) "kexec_file: " fmt
#include <linux/ioport.h>
#include <linux/kernel.h>
#include <linux/kexec.h>
#include <linux/libfdt.h>
#include <linux/memblock.h>
#include <linux/of_fdt.h>
#include <linux/random.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/vmalloc.h>
#include <asm/byteorder.h>
/* relevant device tree properties */
#define FDT_PROP_INITRD_START "linux,initrd-start"
#define FDT_PROP_INITRD_END "linux,initrd-end"
#define FDT_PROP_BOOTARGS "bootargs"
#define FDT_PROP_KASLR_SEED "kaslr-seed"
const struct kexec_file_ops * const kexec_file_loaders[] = {
&kexec_image_ops,
NULL
};
int arch_kimage_file_post_load_cleanup(struct kimage *image)
{
vfree(image->arch.dtb);
image->arch.dtb = NULL;
return kexec_image_post_load_cleanup_default(image);
}
static int setup_dtb(struct kimage *image,
unsigned long initrd_load_addr, unsigned long initrd_len,
char *cmdline, void *dtb)
{
int off, ret;
ret = fdt_path_offset(dtb, "/chosen");
if (ret < 0)
goto out;
off = ret;
/* add bootargs */
if (cmdline) {
ret = fdt_setprop_string(dtb, off, FDT_PROP_BOOTARGS, cmdline);
if (ret)
goto out;
} else {
ret = fdt_delprop(dtb, off, FDT_PROP_BOOTARGS);
if (ret && (ret != -FDT_ERR_NOTFOUND))
goto out;
}
/* add initrd-* */
if (initrd_load_addr) {
ret = fdt_setprop_u64(dtb, off, FDT_PROP_INITRD_START,
initrd_load_addr);
if (ret)
goto out;
ret = fdt_setprop_u64(dtb, off, FDT_PROP_INITRD_END,
initrd_load_addr + initrd_len);
if (ret)
goto out;
} else {
ret = fdt_delprop(dtb, off, FDT_PROP_INITRD_START);
if (ret && (ret != -FDT_ERR_NOTFOUND))
goto out;
ret = fdt_delprop(dtb, off, FDT_PROP_INITRD_END);
if (ret && (ret != -FDT_ERR_NOTFOUND))
goto out;
}
/* add kaslr-seed */
ret = fdt_delprop(dtb, off, FDT_PROP_KASLR_SEED);
if (ret && (ret != -FDT_ERR_NOTFOUND))
goto out;
if (rng_is_initialized()) {
u64 seed = get_random_u64();
ret = fdt_setprop_u64(dtb, off, FDT_PROP_KASLR_SEED, seed);
if (ret)
goto out;
} else {
pr_notice("RNG is not initialised: omitting \"%s\" property\n",
FDT_PROP_KASLR_SEED);
}
out:
if (ret)
return (ret == -FDT_ERR_NOSPACE) ? -ENOMEM : -EINVAL;
return 0;
}
/*
* More space needed so that we can add initrd, bootargs and kaslr-seed.
*/
#define DTB_EXTRA_SPACE 0x1000
static int create_dtb(struct kimage *image,
unsigned long initrd_load_addr, unsigned long initrd_len,
char *cmdline, void **dtb)
{
void *buf;
size_t buf_size;
int ret;
buf_size = fdt_totalsize(initial_boot_params)
+ strlen(cmdline) + DTB_EXTRA_SPACE;
for (;;) {
buf = vmalloc(buf_size);
if (!buf)
return -ENOMEM;
/* duplicate a device tree blob */
ret = fdt_open_into(initial_boot_params, buf, buf_size);
if (ret)
return -EINVAL;
ret = setup_dtb(image, initrd_load_addr, initrd_len,
cmdline, buf);
if (ret) {
vfree(buf);
if (ret == -ENOMEM) {
/* unlikely, but just in case */
buf_size += DTB_EXTRA_SPACE;
continue;
} else {
return ret;
}
}
/* trim it */
fdt_pack(buf);
*dtb = buf;
return 0;
}
}
int load_other_segments(struct kimage *image,
unsigned long kernel_load_addr,
unsigned long kernel_size,
char *initrd, unsigned long initrd_len,
char *cmdline)
{
struct kexec_buf kbuf;
void *dtb = NULL;
unsigned long initrd_load_addr = 0, dtb_len;
int ret = 0;
kbuf.image = image;
/* not allocate anything below the kernel */
kbuf.buf_min = kernel_load_addr + kernel_size;
/* load initrd */
if (initrd) {
kbuf.buffer = initrd;
kbuf.bufsz = initrd_len;
kbuf.mem = 0;
kbuf.memsz = initrd_len;
kbuf.buf_align = 0;
/* within 1GB-aligned window of up to 32GB in size */
kbuf.buf_max = round_down(kernel_load_addr, SZ_1G)
+ (unsigned long)SZ_1G * 32;
kbuf.top_down = false;
ret = kexec_add_buffer(&kbuf);
if (ret)
goto out_err;
initrd_load_addr = kbuf.mem;
pr_debug("Loaded initrd at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
initrd_load_addr, initrd_len, initrd_len);
}
/* load dtb */
ret = create_dtb(image, initrd_load_addr, initrd_len, cmdline, &dtb);
if (ret) {
pr_err("Preparing for new dtb failed\n");
goto out_err;
}
dtb_len = fdt_totalsize(dtb);
kbuf.buffer = dtb;
kbuf.bufsz = dtb_len;
kbuf.mem = 0;
kbuf.memsz = dtb_len;
/* not across 2MB boundary */
kbuf.buf_align = SZ_2M;
kbuf.buf_max = ULONG_MAX;
kbuf.top_down = true;
ret = kexec_add_buffer(&kbuf);
if (ret)
goto out_err;
image->arch.dtb = dtb;
image->arch.dtb_mem = kbuf.mem;
pr_debug("Loaded dtb at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
kbuf.mem, dtb_len, dtb_len);
return 0;
out_err:
vfree(dtb);
return ret;
}

135
arch/arm64/kernel/module-plts.c
View File

@ -11,31 +11,91 @@
#include <linux/module.h>
#include <linux/sort.h>
static struct plt_entry __get_adrp_add_pair(u64 dst, u64 pc,
enum aarch64_insn_register reg)
{
u32 adrp, add;
adrp = aarch64_insn_gen_adr(pc, dst, reg, AARCH64_INSN_ADR_TYPE_ADRP);
add = aarch64_insn_gen_add_sub_imm(reg, reg, dst % SZ_4K,
AARCH64_INSN_VARIANT_64BIT,
AARCH64_INSN_ADSB_ADD);
return (struct plt_entry){ cpu_to_le32(adrp), cpu_to_le32(add) };
}
struct plt_entry get_plt_entry(u64 dst, void *pc)
{
struct plt_entry plt;
static u32 br;
if (!br)
br = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_16,
AARCH64_INSN_BRANCH_NOLINK);
plt = __get_adrp_add_pair(dst, (u64)pc, AARCH64_INSN_REG_16);
plt.br = cpu_to_le32(br);
return plt;
}
bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b)
{
u64 p, q;
/*
* Check whether both entries refer to the same target:
* do the cheapest checks first.
* If the 'add' or 'br' opcodes are different, then the target
* cannot be the same.
*/
if (a->add != b->add || a->br != b->br)
return false;
p = ALIGN_DOWN((u64)a, SZ_4K);
q = ALIGN_DOWN((u64)b, SZ_4K);
/*
* If the 'adrp' opcodes are the same then we just need to check
* that they refer to the same 4k region.
*/
if (a->adrp == b->adrp && p == q)
return true;
return (p + aarch64_insn_adrp_get_offset(le32_to_cpu(a->adrp))) ==
(q + aarch64_insn_adrp_get_offset(le32_to_cpu(b->adrp)));
}
static bool in_init(const struct module *mod, void *loc)
{
return (u64)loc - (u64)mod->init_layout.base < mod->init_layout.size;
}
u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela,
u64 module_emit_plt_entry(struct module *mod, Elf64_Shdr *sechdrs,
void *loc, const Elf64_Rela *rela,
Elf64_Sym *sym)
{
struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
&mod->arch.init;
struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;
struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
int i = pltsec->plt_num_entries;
int j = i - 1;
u64 val = sym->st_value + rela->r_addend;
plt[i] = get_plt_entry(val);
if (is_forbidden_offset_for_adrp(&plt[i].adrp))
i++;
plt[i] = get_plt_entry(val, &plt[i]);
/*
* Check if the entry we just created is a duplicate. Given that the
* relocations are sorted, this will be the last entry we allocated.
* (if one exists).
*/
if (i > 0 && plt_entries_equal(plt + i, plt + i - 1))
return (u64)&plt[i - 1];
if (j >= 0 && plt_entries_equal(plt + i, plt + j))
return (u64)&plt[j];
pltsec->plt_num_entries++;
pltsec->plt_num_entries += i - j;
if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
return 0;
@ -43,41 +103,31 @@ u64 module_emit_plt_entry(struct module *mod, void *loc, const Elf64_Rela *rela,
}
#ifdef CONFIG_ARM64_ERRATUM_843419
u64 module_emit_veneer_for_adrp(struct module *mod, void *loc, u64 val)
u64 module_emit_veneer_for_adrp(struct module *mod, Elf64_Shdr *sechdrs,
void *loc, u64 val)
{
struct mod_plt_sec *pltsec = !in_init(mod, loc) ? &mod->arch.core :
&mod->arch.init;
struct plt_entry *plt = (struct plt_entry *)pltsec->plt->sh_addr;
struct plt_entry *plt = (struct plt_entry *)sechdrs[pltsec->plt_shndx].sh_addr;
int i = pltsec->plt_num_entries++;
u32 mov0, mov1, mov2, br;
u32 br;
int rd;
if (WARN_ON(pltsec->plt_num_entries > pltsec->plt_max_entries))
return 0;
if (is_forbidden_offset_for_adrp(&plt[i].adrp))
i = pltsec->plt_num_entries++;
/* get the destination register of the ADRP instruction */
rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD,
le32_to_cpup((__le32 *)loc));
/* generate the veneer instructions */
mov0 = aarch64_insn_gen_movewide(rd, (u16)~val, 0,
AARCH64_INSN_VARIANT_64BIT,
AARCH64_INSN_MOVEWIDE_INVERSE);
mov1 = aarch64_insn_gen_movewide(rd, (u16)(val >> 16), 16,
AARCH64_INSN_VARIANT_64BIT,
AARCH64_INSN_MOVEWIDE_KEEP);
mov2 = aarch64_insn_gen_movewide(rd, (u16)(val >> 32), 32,
AARCH64_INSN_VARIANT_64BIT,
AARCH64_INSN_MOVEWIDE_KEEP);
br = aarch64_insn_gen_branch_imm((u64)&plt[i].br, (u64)loc + 4,
AARCH64_INSN_BRANCH_NOLINK);
plt[i] = (struct plt_entry){
cpu_to_le32(mov0),
cpu_to_le32(mov1),
cpu_to_le32(mov2),
cpu_to_le32(br)
};
plt[i] = __get_adrp_add_pair(val, (u64)&plt[i], rd);
plt[i].br = cpu_to_le32(br);
return (u64)&plt[i];
}
@ -193,6 +243,15 @@ static unsigned int count_plts(Elf64_Sym *syms, Elf64_Rela *rela, int num,
break;
}
}
if (IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) &&
cpus_have_const_cap(ARM64_WORKAROUND_843419))
/*
* Add some slack so we can skip PLT slots that may trigger
* the erratum due to the placement of the ADRP instruction.
*/
ret += DIV_ROUND_UP(ret, (SZ_4K / sizeof(struct plt_entry)));
return ret;
}
@ -202,7 +261,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
unsigned long core_plts = 0;
unsigned long init_plts = 0;
Elf64_Sym *syms = NULL;
Elf_Shdr *tramp = NULL;
Elf_Shdr *pltsec, *tramp = NULL;
int i;
/*
@ -211,9 +270,9 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
*/
for (i = 0; i < ehdr->e_shnum; i++) {
if (!strcmp(secstrings + sechdrs[i].sh_name, ".plt"))
mod->arch.core.plt = sechdrs + i;
mod->arch.core.plt_shndx = i;
else if (!strcmp(secstrings + sechdrs[i].sh_name, ".init.plt"))
mod->arch.init.plt = sechdrs + i;
mod->arch.init.plt_shndx = i;
else if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE) &&
!strcmp(secstrings + sechdrs[i].sh_name,
".text.ftrace_trampoline"))
@ -222,7 +281,7 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
syms = (Elf64_Sym *)sechdrs[i].sh_addr;
}
if (!mod->arch.core.plt || !mod->arch.init.plt) {
if (!mod->arch.core.plt_shndx || !mod->arch.init.plt_shndx) {
pr_err("%s: module PLT section(s) missing\n", mod->name);
return -ENOEXEC;
}
@ -254,17 +313,19 @@ int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
sechdrs[i].sh_info, dstsec);
}
mod->arch.core.plt->sh_type = SHT_NOBITS;
mod->arch.core.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
mod->arch.core.plt->sh_addralign = L1_CACHE_BYTES;
mod->arch.core.plt->sh_size = (core_plts + 1) * sizeof(struct plt_entry);
pltsec = sechdrs + mod->arch.core.plt_shndx;
pltsec->sh_type = SHT_NOBITS;
pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
pltsec->sh_addralign = L1_CACHE_BYTES;
pltsec->sh_size = (core_plts + 1) * sizeof(struct plt_entry);
mod->arch.core.plt_num_entries = 0;
mod->arch.core.plt_max_entries = core_plts;
mod->arch.init.plt->sh_type = SHT_NOBITS;
mod->arch.init.plt->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
mod->arch.init.plt->sh_addralign = L1_CACHE_BYTES;
mod->arch.init.plt->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
pltsec = sechdrs + mod->arch.init.plt_shndx;
pltsec->sh_type = SHT_NOBITS;
pltsec->sh_flags = SHF_EXECINSTR | SHF_ALLOC;
pltsec->sh_addralign = L1_CACHE_BYTES;
pltsec->sh_size = (init_plts + 1) * sizeof(struct plt_entry);
mod->arch.init.plt_num_entries = 0;
mod->arch.init.plt_max_entries = init_plts;

13
arch/arm64/kernel/module.c
View File

@ -198,13 +198,12 @@ static int reloc_insn_imm(enum aarch64_reloc_op op, __le32 *place, u64 val,
return 0;
}
static int reloc_insn_adrp(struct module *mod, __le32 *place, u64 val)
static int reloc_insn_adrp(struct module *mod, Elf64_Shdr *sechdrs,
__le32 *place, u64 val)
{
u32 insn;
if (!IS_ENABLED(CONFIG_ARM64_ERRATUM_843419) ||
!cpus_have_const_cap(ARM64_WORKAROUND_843419) ||
((u64)place & 0xfff) < 0xff8)
if (!is_forbidden_offset_for_adrp(place))
return reloc_insn_imm(RELOC_OP_PAGE, place, val, 12, 21,
AARCH64_INSN_IMM_ADR);
@ -215,7 +214,7 @@ static int reloc_insn_adrp(struct module *mod, __le32 *place, u64 val)
insn &= ~BIT(31);
} else {
/* out of range for ADR -> emit a veneer */
val = module_emit_veneer_for_adrp(mod, place, val & ~0xfff);
val = module_emit_veneer_for_adrp(mod, sechdrs, place, val & ~0xfff);
if (!val)
return -ENOEXEC;
insn = aarch64_insn_gen_branch_imm((u64)place, val,
@ -368,7 +367,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
case R_AARCH64_ADR_PREL_PG_HI21_NC:
overflow_check = false;
case R_AARCH64_ADR_PREL_PG_HI21:
ovf = reloc_insn_adrp(me, loc, val);
ovf = reloc_insn_adrp(me, sechdrs, loc, val);
if (ovf && ovf != -ERANGE)
return ovf;
break;
@ -413,7 +412,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
if (IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) &&
ovf == -ERANGE) {
val = module_emit_plt_entry(me, loc, &rel[i], sym);
val = module_emit_plt_entry(me, sechdrs, loc, &rel[i], sym);
if (!val)
return -ENOEXEC;
ovf = reloc_insn_imm(RELOC_OP_PREL, loc, val, 2,

6
arch/arm64/kernel/perf_callchain.c
View File

@ -18,6 +18,7 @@
#include <linux/perf_event.h>
#include <linux/uaccess.h>
#include <asm/pointer_auth.h>
#include <asm/stacktrace.h>
struct frame_tail {
@ -35,6 +36,7 @@ user_backtrace(struct frame_tail __user *tail,
{
struct frame_tail buftail;
unsigned long err;
unsigned long lr;
/* Also check accessibility of one struct frame_tail beyond */
if (!access_ok(VERIFY_READ, tail, sizeof(buftail)))
@ -47,7 +49,9 @@ user_backtrace(struct frame_tail __user *tail,
if (err)
return NULL;
perf_callchain_store(entry, buftail.lr);
lr = ptrauth_strip_insn_pac(buftail.lr);
perf_callchain_store(entry, lr);
/*
* Frame pointers should strictly progress back up the stack

221
arch/arm64/kernel/perf_event.c
View File

@ -1,5 +1,5 @@
/*
* PMU support
* ARMv8 PMUv3 Performance Events handling code.
*
* Copyright (C) 2012 ARM Limited
* Author: Will Deacon <will.deacon@arm.com>
@ -30,149 +30,6 @@
#include <linux/perf/arm_pmu.h>
#include <linux/platform_device.h>
/*
* ARMv8 PMUv3 Performance Events handling code.
* Common event types (some are defined in asm/perf_event.h).
*/
/* At least one of the following is required. */
#define ARMV8_PMUV3_PERFCTR_INST_RETIRED 0x08
#define ARMV8_PMUV3_PERFCTR_INST_SPEC 0x1B
/* Common architectural events. */
#define ARMV8_PMUV3_PERFCTR_LD_RETIRED 0x06
#define ARMV8_PMUV3_PERFCTR_ST_RETIRED 0x07
#define ARMV8_PMUV3_PERFCTR_EXC_TAKEN 0x09
#define ARMV8_PMUV3_PERFCTR_EXC_RETURN 0x0A
#define ARMV8_PMUV3_PERFCTR_CID_WRITE_RETIRED 0x0B
#define ARMV8_PMUV3_PERFCTR_PC_WRITE_RETIRED 0x0C
#define ARMV8_PMUV3_PERFCTR_BR_IMMED_RETIRED 0x0D
#define ARMV8_PMUV3_PERFCTR_BR_RETURN_RETIRED 0x0E
#define ARMV8_PMUV3_PERFCTR_UNALIGNED_LDST_RETIRED 0x0F
#define ARMV8_PMUV3_PERFCTR_TTBR_WRITE_RETIRED 0x1C
#define ARMV8_PMUV3_PERFCTR_CHAIN 0x1E
#define ARMV8_PMUV3_PERFCTR_BR_RETIRED 0x21
/* Common microarchitectural events. */
#define ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL 0x01
#define ARMV8_PMUV3_PERFCTR_L1I_TLB_REFILL 0x02
#define ARMV8_PMUV3_PERFCTR_L1D_TLB_REFILL 0x05
#define ARMV8_PMUV3_PERFCTR_MEM_ACCESS 0x13
#define ARMV8_PMUV3_PERFCTR_L1I_CACHE 0x14
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_WB 0x15
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE 0x16
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_REFILL 0x17
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_WB 0x18
#define ARMV8_PMUV3_PERFCTR_BUS_ACCESS 0x19
#define ARMV8_PMUV3_PERFCTR_MEMORY_ERROR 0x1A
#define ARMV8_PMUV3_PERFCTR_BUS_CYCLES 0x1D
#define ARMV8_PMUV3_PERFCTR_L1D_CACHE_ALLOCATE 0x1F
#define ARMV8_PMUV3_PERFCTR_L2D_CACHE_ALLOCATE 0x20
#define ARMV8_PMUV3_PERFCTR_BR_MIS_PRED_RETIRED 0x22
#define ARMV8_PMUV3_PERFCTR_STALL_FRONTEND 0x23
#define ARMV8_PMUV3_PERFCTR_STALL_BACKEND 0x24
#define ARMV8_PMUV3_PERFCTR_L1D_TLB 0x25
#define ARMV8_PMUV3_PERFCTR_L1I_TLB 0x26
#define ARMV8_PMUV3_PERFCTR_L2I_CACHE 0x27
#define ARMV8_PMUV3_PERFCTR_L2I_CACHE_REFILL 0x28
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_ALLOCATE 0x29
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_REFILL 0x2A
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE 0x2B
#define ARMV8_PMUV3_PERFCTR_L3D_CACHE_WB 0x2C
#define ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL 0x2D
#define ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL 0x2E
#define ARMV8_PMUV3_PERFCTR_L2D_TLB 0x2F
#define ARMV8_PMUV3_PERFCTR_L2I_TLB 0x30
/* ARMv8 recommended implementation defined event types */
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_RD 0x40
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WR 0x41
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_RD 0x42
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_WR 0x43
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_INNER 0x44
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_REFILL_OUTER 0x45
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_VICTIM 0x46
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_WB_CLEAN 0x47
#define ARMV8_IMPDEF_PERFCTR_L1D_CACHE_INVAL 0x48
#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_RD 0x4C
#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_REFILL_WR 0x4D
#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_RD 0x4E
#define ARMV8_IMPDEF_PERFCTR_L1D_TLB_WR 0x4F
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_RD 0x50
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WR 0x51
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_RD 0x52
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_REFILL_WR 0x53
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_VICTIM 0x56
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_WB_CLEAN 0x57
#define ARMV8_IMPDEF_PERFCTR_L2D_CACHE_INVAL 0x58
#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_RD 0x5C
#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_REFILL_WR 0x5D
#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_RD 0x5E
#define ARMV8_IMPDEF_PERFCTR_L2D_TLB_WR 0x5F
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_RD 0x60
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_WR 0x61
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_SHARED 0x62
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NOT_SHARED 0x63
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_NORMAL 0x64
#define ARMV8_IMPDEF_PERFCTR_BUS_ACCESS_PERIPH 0x65
#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_RD 0x66
#define ARMV8_IMPDEF_PERFCTR_MEM_ACCESS_WR 0x67
#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LD_SPEC 0x68
#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_ST_SPEC 0x69
#define ARMV8_IMPDEF_PERFCTR_UNALIGNED_LDST_SPEC 0x6A
#define ARMV8_IMPDEF_PERFCTR_LDREX_SPEC 0x6C
#define ARMV8_IMPDEF_PERFCTR_STREX_PASS_SPEC 0x6D
#define ARMV8_IMPDEF_PERFCTR_STREX_FAIL_SPEC 0x6E
#define ARMV8_IMPDEF_PERFCTR_STREX_SPEC 0x6F
#define ARMV8_IMPDEF_PERFCTR_LD_SPEC 0x70
#define ARMV8_IMPDEF_PERFCTR_ST_SPEC 0x71
#define ARMV8_IMPDEF_PERFCTR_LDST_SPEC 0x72
#define ARMV8_IMPDEF_PERFCTR_DP_SPEC 0x73
#define ARMV8_IMPDEF_PERFCTR_ASE_SPEC 0x74
#define ARMV8_IMPDEF_PERFCTR_VFP_SPEC 0x75
#define ARMV8_IMPDEF_PERFCTR_PC_WRITE_SPEC 0x76
#define ARMV8_IMPDEF_PERFCTR_CRYPTO_SPEC 0x77
#define ARMV8_IMPDEF_PERFCTR_BR_IMMED_SPEC 0x78
#define ARMV8_IMPDEF_PERFCTR_BR_RETURN_SPEC 0x79
#define ARMV8_IMPDEF_PERFCTR_BR_INDIRECT_SPEC 0x7A
#define ARMV8_IMPDEF_PERFCTR_ISB_SPEC 0x7C
#define ARMV8_IMPDEF_PERFCTR_DSB_SPEC 0x7D
#define ARMV8_IMPDEF_PERFCTR_DMB_SPEC 0x7E
#define ARMV8_IMPDEF_PERFCTR_EXC_UNDEF 0x81
#define ARMV8_IMPDEF_PERFCTR_EXC_SVC 0x82
#define ARMV8_IMPDEF_PERFCTR_EXC_PABORT 0x83
#define ARMV8_IMPDEF_PERFCTR_EXC_DABORT 0x84
#define ARMV8_IMPDEF_PERFCTR_EXC_IRQ 0x86
#define ARMV8_IMPDEF_PERFCTR_EXC_FIQ 0x87
#define ARMV8_IMPDEF_PERFCTR_EXC_SMC 0x88
#define ARMV8_IMPDEF_PERFCTR_EXC_HVC 0x8A
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_PABORT 0x8B
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_DABORT 0x8C
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_OTHER 0x8D
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_IRQ 0x8E
#define ARMV8_IMPDEF_PERFCTR_EXC_TRAP_FIQ 0x8F
#define ARMV8_IMPDEF_PERFCTR_RC_LD_SPEC 0x90
#define ARMV8_IMPDEF_PERFCTR_RC_ST_SPEC 0x91
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_RD 0xA0
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WR 0xA1
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_RD 0xA2
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_REFILL_WR 0xA3
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_VICTIM 0xA6
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_WB_CLEAN 0xA7
#define ARMV8_IMPDEF_PERFCTR_L3D_CACHE_INVAL 0xA8
/* ARMv8 Cortex-A53 specific event types. */
#define ARMV8_A53_PERFCTR_PREF_LINEFILL 0xC2
@ -183,12 +40,10 @@
#define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_ACCESS 0xEC
#define ARMV8_THUNDER_PERFCTR_L1I_CACHE_PREF_MISS 0xED
/* PMUv3 HW events mapping. */
/*
* ARMv8 Architectural defined events, not all of these may
* be supported on any given implementation. Undefined events will
* be disabled at run-time.
* be supported on any given implementation. Unsupported events will
* be disabled at run-time based on the PMCEID registers.
*/
static const unsigned armv8_pmuv3_perf_map[PERF_COUNT_HW_MAX] = {
PERF_MAP_ALL_UNSUPPORTED,
@ -210,8 +65,6 @@ static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
[C(L1D)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
[C(L1D)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
[C(L1D)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE,
[C(L1D)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1D_CACHE_REFILL,
[C(L1I)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE,
[C(L1I)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_L1I_CACHE_REFILL,
@ -224,8 +77,6 @@ static const unsigned armv8_pmuv3_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
[C(BPU)][C(OP_READ)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
[C(BPU)][C(OP_READ)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
[C(BPU)][C(OP_WRITE)][C(RESULT_ACCESS)] = ARMV8_PMUV3_PERFCTR_BR_PRED,
[C(BPU)][C(OP_WRITE)][C(RESULT_MISS)] = ARMV8_PMUV3_PERFCTR_BR_MIS_PRED,
};
static const unsigned armv8_a53_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
@ -370,6 +221,18 @@ ARMV8_EVENT_ATTR(l2d_tlb_refill, ARMV8_PMUV3_PERFCTR_L2D_TLB_REFILL);
ARMV8_EVENT_ATTR(l2i_tlb_refill, ARMV8_PMUV3_PERFCTR_L2I_TLB_REFILL);
ARMV8_EVENT_ATTR(l2d_tlb, ARMV8_PMUV3_PERFCTR_L2D_TLB);
ARMV8_EVENT_ATTR(l2i_tlb, ARMV8_PMUV3_PERFCTR_L2I_TLB);
ARMV8_EVENT_ATTR(remote_access, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS);
ARMV8_EVENT_ATTR(ll_cache, ARMV8_PMUV3_PERFCTR_LL_CACHE);
ARMV8_EVENT_ATTR(ll_cache_miss, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS);
ARMV8_EVENT_ATTR(dtlb_walk, ARMV8_PMUV3_PERFCTR_DTLB_WALK);
ARMV8_EVENT_ATTR(itlb_walk, ARMV8_PMUV3_PERFCTR_ITLB_WALK);
ARMV8_EVENT_ATTR(ll_cache_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_RD);
ARMV8_EVENT_ATTR(ll_cache_miss_rd, ARMV8_PMUV3_PERFCTR_LL_CACHE_MISS_RD);
ARMV8_EVENT_ATTR(remote_access_rd, ARMV8_PMUV3_PERFCTR_REMOTE_ACCESS_RD);
ARMV8_EVENT_ATTR(sample_pop, ARMV8_SPE_PERFCTR_SAMPLE_POP);
ARMV8_EVENT_ATTR(sample_feed, ARMV8_SPE_PERFCTR_SAMPLE_FEED);
ARMV8_EVENT_ATTR(sample_filtrate, ARMV8_SPE_PERFCTR_SAMPLE_FILTRATE);
ARMV8_EVENT_ATTR(sample_collision, ARMV8_SPE_PERFCTR_SAMPLE_COLLISION);
static struct attribute *armv8_pmuv3_event_attrs[] = {
&armv8_event_attr_sw_incr.attr.attr,
@ -420,6 +283,18 @@ static struct attribute *armv8_pmuv3_event_attrs[] = {
&armv8_event_attr_l2i_tlb_refill.attr.attr,
&armv8_event_attr_l2d_tlb.attr.attr,
&armv8_event_attr_l2i_tlb.attr.attr,
&armv8_event_attr_remote_access.attr.attr,
&armv8_event_attr_ll_cache.attr.attr,
&armv8_event_attr_ll_cache_miss.attr.attr,
&armv8_event_attr_dtlb_walk.attr.attr,
&armv8_event_attr_itlb_walk.attr.attr,
&armv8_event_attr_ll_cache_rd.attr.attr,
&armv8_event_attr_ll_cache_miss_rd.attr.attr,
&armv8_event_attr_remote_access_rd.attr.attr,
&armv8_event_attr_sample_pop.attr.attr,
&armv8_event_attr_sample_feed.attr.attr,
&armv8_event_attr_sample_filtrate.attr.attr,
&armv8_event_attr_sample_collision.attr.attr,
NULL,
};
@ -434,7 +309,13 @@ armv8pmu_event_attr_is_visible(struct kobject *kobj,
pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
if (test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))
if (pmu_attr->id < ARMV8_PMUV3_MAX_COMMON_EVENTS &&
test_bit(pmu_attr->id, cpu_pmu->pmceid_bitmap))
return attr->mode;
pmu_attr->id -= ARMV8_PMUV3_EXT_COMMON_EVENT_BASE;
if (pmu_attr->id < ARMV8_PMUV3_MAX_COMMON_EVENTS &&
test_bit(pmu_attr->id, cpu_pmu->pmceid_ext_bitmap))
return attr->mode;
return 0;
@ -1009,7 +890,7 @@ static int __armv8_pmuv3_map_event(struct perf_event *event,
if (armv8pmu_event_is_64bit(event))
event->hw.flags |= ARMPMU_EVT_64BIT;
/* Onl expose micro/arch events supported by this PMU */
/* Only expose micro/arch events supported by this PMU */
if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
&& test_bit(hw_event_id, armpmu->pmceid_bitmap)) {
return hw_event_id;
@ -1061,6 +942,7 @@ static void __armv8pmu_probe_pmu(void *info)
struct armv8pmu_probe_info *probe = info;
struct arm_pmu *cpu_pmu = probe->pmu;
u64 dfr0;
u64 pmceid_raw[2];
u32 pmceid[2];
int pmuver;
@ -1079,11 +961,17 @@ static void __armv8pmu_probe_pmu(void *info)
/* Add the CPU cycles counter */
cpu_pmu->num_events += 1;
pmceid[0] = read_sysreg(pmceid0_el0);
pmceid[1] = read_sysreg(pmceid1_el0);
pmceid[0] = pmceid_raw[0] = read_sysreg(pmceid0_el0);
pmceid[1] = pmceid_raw[1] = read_sysreg(pmceid1_el0);
bitmap_from_arr32(cpu_pmu->pmceid_bitmap,
pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
pmceid[0] = pmceid_raw[0] >> 32;
pmceid[1] = pmceid_raw[1] >> 32;
bitmap_from_arr32(cpu_pmu->pmceid_ext_bitmap,
pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
}
static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu)
@ -1109,16 +997,16 @@ static int armv8_pmu_init(struct arm_pmu *cpu_pmu)
if (ret)
return ret;
cpu_pmu->handle_irq = armv8pmu_handle_irq,
cpu_pmu->enable = armv8pmu_enable_event,
cpu_pmu->disable = armv8pmu_disable_event,
cpu_pmu->read_counter = armv8pmu_read_counter,
cpu_pmu->write_counter = armv8pmu_write_counter,
cpu_pmu->get_event_idx = armv8pmu_get_event_idx,
cpu_pmu->clear_event_idx = armv8pmu_clear_event_idx,
cpu_pmu->start = armv8pmu_start,
cpu_pmu->stop = armv8pmu_stop,
cpu_pmu->reset = armv8pmu_reset,
cpu_pmu->handle_irq = armv8pmu_handle_irq;
cpu_pmu->enable = armv8pmu_enable_event;
cpu_pmu->disable = armv8pmu_disable_event;
cpu_pmu->read_counter = armv8pmu_read_counter;
cpu_pmu->write_counter = armv8pmu_write_counter;
cpu_pmu->get_event_idx = armv8pmu_get_event_idx;
cpu_pmu->clear_event_idx = armv8pmu_clear_event_idx;
cpu_pmu->start = armv8pmu_start;
cpu_pmu->stop = armv8pmu_stop;
cpu_pmu->reset = armv8pmu_reset;
cpu_pmu->set_event_filter = armv8pmu_set_event_filter;
cpu_pmu->filter_match = armv8pmu_filter_match;
@ -1274,6 +1162,7 @@ static struct platform_driver armv8_pmu_driver = {
.driver = {
.name = ARMV8_PMU_PDEV_NAME,
.of_match_table = armv8_pmu_of_device_ids,
.suppress_bind_attrs = true,
},
.probe = armv8_pmu_device_probe,
};

47
arch/arm64/kernel/pointer_auth.c Normal file
View File

@ -0,0 +1,47 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/errno.h>
#include <linux/prctl.h>
#include <linux/random.h>
#include <linux/sched.h>
#include <asm/cpufeature.h>
#include <asm/pointer_auth.h>
int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg)
{
struct ptrauth_keys *keys = &tsk->thread.keys_user;
unsigned long addr_key_mask = PR_PAC_APIAKEY | PR_PAC_APIBKEY |
PR_PAC_APDAKEY | PR_PAC_APDBKEY;
unsigned long key_mask = addr_key_mask | PR_PAC_APGAKEY;
if (!system_supports_address_auth() && !system_supports_generic_auth())
return -EINVAL;
if (!arg) {
ptrauth_keys_init(keys);
ptrauth_keys_switch(keys);
return 0;
}
if (arg & ~key_mask)
return -EINVAL;
if (((arg & addr_key_mask) && !system_supports_address_auth()) ||
((arg & PR_PAC_APGAKEY) && !system_supports_generic_auth()))
return -EINVAL;
if (arg & PR_PAC_APIAKEY)
get_random_bytes(&keys->apia, sizeof(keys->apia));
if (arg & PR_PAC_APIBKEY)
get_random_bytes(&keys->apib, sizeof(keys->apib));
if (arg & PR_PAC_APDAKEY)
get_random_bytes(&keys->apda, sizeof(keys->apda));
if (arg & PR_PAC_APDBKEY)
get_random_bytes(&keys->apdb, sizeof(keys->apdb));
if (arg & PR_PAC_APGAKEY)
get_random_bytes(&keys->apga, sizeof(keys->apga));
ptrauth_keys_switch(keys);
return 0;
}

6
arch/arm64/kernel/process.c
View File

@ -57,9 +57,10 @@
#include <asm/fpsimd.h>
#include <asm/mmu_context.h>
#include <asm/processor.h>
#include <asm/pointer_auth.h>
#include <asm/stacktrace.h>
#ifdef CONFIG_STACKPROTECTOR
#if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
#include <linux/stackprotector.h>
unsigned long __stack_chk_guard __read_mostly;
EXPORT_SYMBOL(__stack_chk_guard);
@ -429,6 +430,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
contextidr_thread_switch(next);
entry_task_switch(next);
uao_thread_switch(next);
ptrauth_thread_switch(next);
/*
* Complete any pending TLB or cache maintenance on this CPU in case
@ -496,4 +498,6 @@ unsigned long arch_randomize_brk(struct mm_struct *mm)
void arch_setup_new_exec(void)
{
current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0;
ptrauth_thread_init_user(current);
}

38
arch/arm64/kernel/ptrace.c
View File

@ -46,6 +46,7 @@
#include <asm/debug-monitors.h>
#include <asm/fpsimd.h>
#include <asm/pgtable.h>
#include <asm/pointer_auth.h>
#include <asm/stacktrace.h>
#include <asm/syscall.h>
#include <asm/traps.h>
@ -956,6 +957,30 @@ static int sve_set(struct task_struct *target,
#endif /* CONFIG_ARM64_SVE */
#ifdef CONFIG_ARM64_PTR_AUTH
static int pac_mask_get(struct task_struct *target,
const struct user_regset *regset,
unsigned int pos, unsigned int count,
void *kbuf, void __user *ubuf)
{
/*
* The PAC bits can differ across data and instruction pointers
* depending on TCR_EL1.TBID*, which we may make use of in future, so
* we expose separate masks.
*/
unsigned long mask = ptrauth_user_pac_mask();
struct user_pac_mask uregs = {
.data_mask = mask,
.insn_mask = mask,
};
if (!system_supports_address_auth())
return -EINVAL;
return user_regset_copyout(&pos, &count, &kbuf, &ubuf, &uregs, 0, -1);
}
#endif /* CONFIG_ARM64_PTR_AUTH */
enum aarch64_regset {
REGSET_GPR,
REGSET_FPR,
@ -968,6 +993,9 @@ enum aarch64_regset {
#ifdef CONFIG_ARM64_SVE
REGSET_SVE,
#endif
#ifdef CONFIG_ARM64_PTR_AUTH
REGSET_PAC_MASK,
#endif
};
static const struct user_regset aarch64_regsets[] = {
@ -1037,6 +1065,16 @@ static const struct user_regset aarch64_regsets[] = {
.get_size = sve_get_size,
},
#endif
#ifdef CONFIG_ARM64_PTR_AUTH
[REGSET_PAC_MASK] = {
.core_note_type = NT_ARM_PAC_MASK,
.n = sizeof(struct user_pac_mask) / sizeof(u64),
.size = sizeof(u64),
.align = sizeof(u64),
.get = pac_mask_get,
/* this cannot be set dynamically */
},
#endif
};
static const struct user_regset_view user_aarch64_view = {

3
arch/arm64/kernel/relocate_kernel.S
View File

@ -32,6 +32,7 @@
ENTRY(arm64_relocate_new_kernel)
/* Setup the list loop variables. */
mov x18, x2 /* x18 = dtb address */
mov x17, x1 /* x17 = kimage_start */
mov x16, x0 /* x16 = kimage_head */
raw_dcache_line_size x15, x0 /* x15 = dcache line size */
@ -107,7 +108,7 @@ ENTRY(arm64_relocate_new_kernel)
isb
/* Start new image. */
mov x0, xzr
mov x0, x18
mov x1, xzr
mov x2, xzr
mov x3, xzr

1
arch/arm64/kernel/setup.c
View File

@ -388,6 +388,7 @@ static int dump_kernel_offset(struct notifier_block *self, unsigned long v,
if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && offset > 0) {
pr_emerg("Kernel Offset: 0x%lx from 0x%lx\n",
offset, KIMAGE_VADDR);
pr_emerg("PHYS_OFFSET: 0x%llx\n", PHYS_OFFSET);
} else {
pr_emerg("Kernel Offset: disabled\n");
}

4
arch/arm64/kernel/smccc-call.S
View File

@ -13,7 +13,9 @@
*/
#include <linux/linkage.h>
#include <linux/arm-smccc.h>
#include <asm/asm-offsets.h>
#include <asm/assembler.h>
.macro SMCCC instr
.cfi_startproc
@ -40,6 +42,7 @@
ENTRY(__arm_smccc_smc)
SMCCC smc
ENDPROC(__arm_smccc_smc)
EXPORT_SYMBOL(__arm_smccc_smc)
/*
* void arm_smccc_hvc(unsigned long a0, unsigned long a1, unsigned long a2,
@ -50,3 +53,4 @@ ENDPROC(__arm_smccc_smc)
ENTRY(__arm_smccc_hvc)
SMCCC hvc
ENDPROC(__arm_smccc_hvc)
EXPORT_SYMBOL(__arm_smccc_hvc)

7
arch/arm64/kernel/smp.c
View File

@ -141,6 +141,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
}
} else {
pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
return ret;
}
secondary_data.task = NULL;
@ -151,7 +152,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
if (status == CPU_MMU_OFF)
status = READ_ONCE(__early_cpu_boot_status);
switch (status) {
switch (status & CPU_BOOT_STATUS_MASK) {
default:
pr_err("CPU%u: failed in unknown state : 0x%lx\n",
cpu, status);
@ -165,6 +166,10 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
pr_crit("CPU%u: may not have shut down cleanly\n", cpu);
case CPU_STUCK_IN_KERNEL:
pr_crit("CPU%u: is stuck in kernel\n", cpu);
if (status & CPU_STUCK_REASON_52_BIT_VA)
pr_crit("CPU%u: does not support 52-bit VAs\n", cpu);
if (status & CPU_STUCK_REASON_NO_GRAN)
pr_crit("CPU%u: does not support %luK granule \n", cpu, PAGE_SIZE / SZ_1K);
cpus_stuck_in_kernel++;
break;
case CPU_PANIC_KERNEL:

9
arch/arm64/kernel/vmlinux.lds.S
View File

@ -99,7 +99,8 @@ SECTIONS
*(.discard)
*(.discard.*)
*(.interp .dynamic)
*(.dynsym .dynstr .hash)
*(.dynsym .dynstr .hash .gnu.hash)
*(.eh_frame)
}
. = KIMAGE_VADDR + TEXT_OFFSET;
@ -192,12 +193,12 @@ SECTIONS
PERCPU_SECTION(L1_CACHE_BYTES)
.rela : ALIGN(8) {
.rela.dyn : ALIGN(8) {
*(.rela .rela*)
}
__rela_offset = ABSOLUTE(ADDR(.rela) - KIMAGE_VADDR);
__rela_size = SIZEOF(.rela);
__rela_offset = ABSOLUTE(ADDR(.rela.dyn) - KIMAGE_VADDR);
__rela_size = SIZEOF(.rela.dyn);
. = ALIGN(SEGMENT_ALIGN);
__initdata_end = .;

18
arch/arm64/kvm/handle_exit.c
View File

@ -173,6 +173,23 @@ static int handle_sve(struct kvm_vcpu *vcpu, struct kvm_run *run)
return 1;
}
/*
* Guest usage of a ptrauth instruction (which the guest EL1 did not turn into
* a NOP).
*/
static int kvm_handle_ptrauth(struct kvm_vcpu *vcpu, struct kvm_run *run)
{
/*
* We don't currently support ptrauth in a guest, and we mask the ID
* registers to prevent well-behaved guests from trying to make use of
* it.
*
* Inject an UNDEF, as if the feature really isn't present.
*/
kvm_inject_undefined(vcpu);
return 1;
}
static exit_handle_fn arm_exit_handlers[] = {
[0 ... ESR_ELx_EC_MAX] = kvm_handle_unknown_ec,
[ESR_ELx_EC_WFx] = kvm_handle_wfx,
@ -195,6 +212,7 @@ static exit_handle_fn arm_exit_handlers[] = {
[ESR_ELx_EC_BKPT32] = kvm_handle_guest_debug,
[ESR_ELx_EC_BRK64] = kvm_handle_guest_debug,
[ESR_ELx_EC_FP_ASIMD] = handle_no_fpsimd,
[ESR_ELx_EC_PAC] = kvm_handle_ptrauth,
};
static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)

1
arch/arm64/kvm/hyp/entry.S
View File

@ -83,6 +83,7 @@ ENTRY(__guest_enter)
// Do not touch any register after this!
eret
sb
ENDPROC(__guest_enter)
ENTRY(__guest_exit)

4
arch/arm64/kvm/hyp/hyp-entry.S
View File

@ -96,6 +96,7 @@ el1_sync: // Guest trapped into EL2
do_el2_call
eret
sb
el1_hvc_guest:
/*
@ -146,6 +147,7 @@ wa_epilogue:
mov x0, xzr
add sp, sp, #16
eret
sb
el1_trap:
get_vcpu_ptr x1, x0
@ -199,6 +201,7 @@ el2_error:
b.ne __hyp_panic
mov x0, #(1 << ARM_EXIT_WITH_SERROR_BIT)
eret
sb
ENTRY(__hyp_do_panic)
mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
@ -207,6 +210,7 @@ ENTRY(__hyp_do_panic)
ldr lr, =panic
msr elr_el2, lr
eret
sb
ENDPROC(__hyp_do_panic)
ENTRY(__hyp_panic)

25
arch/arm64/kvm/hyp/switch.c
View File

@ -143,6 +143,14 @@ static void deactivate_traps_vhe(void)
{
extern char vectors[]; /* kernel exception vectors */
write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
/*
* ARM erratum 1165522 requires the actual execution of the above
* before we can switch to the EL2/EL0 translation regime used by
* the host.
*/
asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_1165522));
write_sysreg(CPACR_EL1_DEFAULT, cpacr_el1);
write_sysreg(vectors, vbar_el1);
}
@ -157,7 +165,7 @@ static void __hyp_text __deactivate_traps_nvhe(void)
mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
write_sysreg(mdcr_el2, mdcr_el2);
write_sysreg(HCR_RW, hcr_el2);
write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2);
write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
}
@ -499,8 +507,19 @@ int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
sysreg_save_host_state_vhe(host_ctxt);
__activate_traps(vcpu);
/*
* ARM erratum 1165522 requires us to configure both stage 1 and
* stage 2 translation for the guest context before we clear
* HCR_EL2.TGE.
*
* We have already configured the guest's stage 1 translation in
* kvm_vcpu_load_sysregs above. We must now call __activate_vm
* before __activate_traps, because __activate_vm configures
* stage 2 translation, and __activate_traps clear HCR_EL2.TGE
* (among other things).
*/
__activate_vm(vcpu->kvm);
__activate_traps(vcpu);
sysreg_restore_guest_state_vhe(guest_ctxt);
__debug_switch_to_guest(vcpu);
@ -545,8 +564,8 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
__sysreg_save_state_nvhe(host_ctxt);
__activate_traps(vcpu);
__activate_vm(kern_hyp_va(vcpu->kvm));
__activate_traps(vcpu);
__hyp_vgic_restore_state(vcpu);
__timer_enable_traps(vcpu);

71
arch/arm64/kvm/hyp/tlb.c
View File

@ -15,20 +15,54 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <linux/irqflags.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <asm/tlbflush.h>
static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm)
struct tlb_inv_context {
unsigned long flags;
u64 tcr;
u64 sctlr;
};
static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm,
struct tlb_inv_context *cxt)
{
u64 val;
local_irq_save(cxt->flags);
if (cpus_have_const_cap(ARM64_WORKAROUND_1165522)) {
/*
* For CPUs that are affected by ARM erratum 1165522, we
* cannot trust stage-1 to be in a correct state at that
* point. Since we do not want to force a full load of the
* vcpu state, we prevent the EL1 page-table walker to
* allocate new TLBs. This is done by setting the EPD bits
* in the TCR_EL1 register. We also need to prevent it to
* allocate IPA->PA walks, so we enable the S1 MMU...
*/
val = cxt->tcr = read_sysreg_el1(tcr);
val |= TCR_EPD1_MASK | TCR_EPD0_MASK;
write_sysreg_el1(val, tcr);
val = cxt->sctlr = read_sysreg_el1(sctlr);
val |= SCTLR_ELx_M;
write_sysreg_el1(val, sctlr);
}
/*
* With VHE enabled, we have HCR_EL2.{E2H,TGE} = {1,1}, and
* most TLB operations target EL2/EL0. In order to affect the
* guest TLBs (EL1/EL0), we need to change one of these two
* bits. Changing E2H is impossible (goodbye TTBR1_EL2), so
* let's flip TGE before executing the TLB operation.
*
* ARM erratum 1165522 requires some special handling (again),
* as we need to make sure both stages of translation are in
* place before clearing TGE. __load_guest_stage2() already
* has an ISB in order to deal with this.
*/
__load_guest_stage2(kvm);
val = read_sysreg(hcr_el2);
@ -37,7 +71,8 @@ static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm)
isb();
}
static void __hyp_text __tlb_switch_to_guest_nvhe(struct kvm *kvm)
static void __hyp_text __tlb_switch_to_guest_nvhe(struct kvm *kvm,
struct tlb_inv_context *cxt)
{
__load_guest_stage2(kvm);
isb();
@ -48,7 +83,8 @@ static hyp_alternate_select(__tlb_switch_to_guest,
__tlb_switch_to_guest_vhe,
ARM64_HAS_VIRT_HOST_EXTN);
static void __hyp_text __tlb_switch_to_host_vhe(struct kvm *kvm)
static void __hyp_text __tlb_switch_to_host_vhe(struct kvm *kvm,
struct tlb_inv_context *cxt)
{
/*
* We're done with the TLB operation, let's restore the host's
@ -56,9 +92,19 @@ static void __hyp_text __tlb_switch_to_host_vhe(struct kvm *kvm)
*/
write_sysreg(0, vttbr_el2);
write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
isb();
if (cpus_have_const_cap(ARM64_WORKAROUND_1165522)) {
/* Restore the registers to what they were */
write_sysreg_el1(cxt->tcr, tcr);
write_sysreg_el1(cxt->sctlr, sctlr);
}
local_irq_restore(cxt->flags);
}
static void __hyp_text __tlb_switch_to_host_nvhe(struct kvm *kvm)
static void __hyp_text __tlb_switch_to_host_nvhe(struct kvm *kvm,
struct tlb_inv_context *cxt)
{
write_sysreg(0, vttbr_el2);
}
@ -70,11 +116,13 @@ static hyp_alternate_select(__tlb_switch_to_host,
void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
{
struct tlb_inv_context cxt;
dsb(ishst);
/* Switch to requested VMID */
kvm = kern_hyp_va(kvm);
__tlb_switch_to_guest()(kvm);
__tlb_switch_to_guest()(kvm, &cxt);
/*
* We could do so much better if we had the VA as well.
@ -117,36 +165,39 @@ void __hyp_text __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa)
if (!has_vhe() && icache_is_vpipt())
__flush_icache_all();
__tlb_switch_to_host()(kvm);
__tlb_switch_to_host()(kvm, &cxt);
}
void __hyp_text __kvm_tlb_flush_vmid(struct kvm *kvm)
{
struct tlb_inv_context cxt;
dsb(ishst);
/* Switch to requested VMID */
kvm = kern_hyp_va(kvm);
__tlb_switch_to_guest()(kvm);
__tlb_switch_to_guest()(kvm, &cxt);
__tlbi(vmalls12e1is);
dsb(ish);
isb();
__tlb_switch_to_host()(kvm);
__tlb_switch_to_host()(kvm, &cxt);
}
void __hyp_text __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu)
{
struct kvm *kvm = kern_hyp_va(kern_hyp_va(vcpu)->kvm);
struct tlb_inv_context cxt;
/* Switch to requested VMID */
__tlb_switch_to_guest()(kvm);
__tlb_switch_to_guest()(kvm, &cxt);
__tlbi(vmalle1);
dsb(nsh);
isb();
__tlb_switch_to_host()(kvm);
__tlb_switch_to_host()(kvm, &cxt);
}
void __hyp_text __kvm_flush_vm_context(void)

8
arch/arm64/kvm/sys_regs.c
View File

@ -1040,6 +1040,14 @@ static u64 read_id_reg(struct sys_reg_desc const *r, bool raz)
kvm_debug("SVE unsupported for guests, suppressing\n");
val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
} else if (id == SYS_ID_AA64ISAR1_EL1) {
const u64 ptrauth_mask = (0xfUL << ID_AA64ISAR1_APA_SHIFT) |
(0xfUL << ID_AA64ISAR1_API_SHIFT) |
(0xfUL << ID_AA64ISAR1_GPA_SHIFT) |
(0xfUL << ID_AA64ISAR1_GPI_SHIFT);
if (val & ptrauth_mask)
kvm_debug("ptrauth unsupported for guests, suppressing\n");
val &= ~ptrauth_mask;
} else if (id == SYS_ID_AA64MMFR1_EL1) {
if (val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))
kvm_debug("LORegions unsupported for guests, suppressing\n");

6
arch/arm64/lib/Makefile
View File

@ -5,6 +5,12 @@ lib-y := clear_user.o delay.o copy_from_user.o \
memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
strchr.o strrchr.o tishift.o
ifeq ($(CONFIG_KERNEL_MODE_NEON), y)
obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o
CFLAGS_REMOVE_xor-neon.o += -mgeneral-regs-only
CFLAGS_xor-neon.o += -ffreestanding
endif
# Tell the compiler to treat all general purpose registers (with the
# exception of the IP registers, which are already handled by the caller
# in case of a PLT) as callee-saved, which allows for efficient runtime

1
arch/arm64/lib/clear_page.S
View File

@ -37,3 +37,4 @@ ENTRY(clear_page)
b.ne 1b
ret
ENDPROC(clear_page)
EXPORT_SYMBOL(clear_page)

2
arch/arm64/lib/clear_user.S
View File

@ -18,6 +18,7 @@
#include <linux/linkage.h>
#include <asm/asm-uaccess.h>
#include <asm/assembler.h>
.text
@ -53,6 +54,7 @@ uao_user_alternative 9f, strb, sttrb, wzr, x0, 0
uaccess_disable_not_uao x2, x3
ret
ENDPROC(__arch_clear_user)
EXPORT_SYMBOL(__arch_clear_user)
.section .fixup,"ax"
.align 2

4
arch/arm64/lib/copy_from_user.S
View File

@ -16,8 +16,9 @@
#include <linux/linkage.h>
#include <asm/cache.h>
#include <asm/asm-uaccess.h>
#include <asm/assembler.h>
#include <asm/cache.h>
/*
* Copy from user space to a kernel buffer (alignment handled by the hardware)
@ -71,6 +72,7 @@ ENTRY(__arch_copy_from_user)
mov x0, #0 // Nothing to copy
ret
ENDPROC(__arch_copy_from_user)
EXPORT_SYMBOL(__arch_copy_from_user)
.section .fixup,"ax"
.align 2

4
arch/arm64/lib/copy_in_user.S
View File

@ -18,8 +18,9 @@
#include <linux/linkage.h>
#include <asm/cache.h>
#include <asm/asm-uaccess.h>
#include <asm/assembler.h>
#include <asm/cache.h>
/*
* Copy from user space to user space (alignment handled by the hardware)
@ -73,6 +74,7 @@ ENTRY(__arch_copy_in_user)
mov x0, #0
ret
ENDPROC(__arch_copy_in_user)
EXPORT_SYMBOL(__arch_copy_in_user)
.section .fixup,"ax"
.align 2

1
arch/arm64/lib/copy_page.S
View File

@ -87,3 +87,4 @@ alternative_else_nop_endif
ret
ENDPROC(copy_page)
EXPORT_SYMBOL(copy_page)

4
arch/arm64/lib/copy_to_user.S
View File

@ -16,8 +16,9 @@
#include <linux/linkage.h>
#include <asm/cache.h>
#include <asm/asm-uaccess.h>
#include <asm/assembler.h>
#include <asm/cache.h>
/*
* Copy to user space from a kernel buffer (alignment handled by the hardware)
@ -70,6 +71,7 @@ ENTRY(__arch_copy_to_user)
mov x0, #0
ret
ENDPROC(__arch_copy_to_user)
EXPORT_SYMBOL(__arch_copy_to_user)
.section .fixup,"ax"
.align 2

54
arch/arm64/lib/crc32.S
View File

@ -15,15 +15,59 @@
.cpu generic+crc
.macro __crc32, c
0: subs x2, x2, #16
b.mi 8f
ldp x3, x4, [x1], #16
cmp x2, #16
b.lt 8f // less than 16 bytes
and x7, x2, #0x1f
and x2, x2, #~0x1f
cbz x7, 32f // multiple of 32 bytes
and x8, x7, #0xf
ldp x3, x4, [x1]
add x8, x8, x1
add x1, x1, x7
ldp x5, x6, [x8]
CPU_BE( rev x3, x3 )
CPU_BE( rev x4, x4 )
CPU_BE( rev x5, x5 )
CPU_BE( rev x6, x6 )
tst x7, #8
crc32\c\()x w8, w0, x3
csel x3, x3, x4, eq
csel w0, w0, w8, eq
tst x7, #4
lsr x4, x3, #32
crc32\c\()w w8, w0, w3
csel x3, x3, x4, eq
csel w0, w0, w8, eq
tst x7, #2
lsr w4, w3, #16
crc32\c\()h w8, w0, w3
csel w3, w3, w4, eq
csel w0, w0, w8, eq
tst x7, #1
crc32\c\()b w8, w0, w3
csel w0, w0, w8, eq
tst x7, #16
crc32\c\()x w8, w0, x5
crc32\c\()x w8, w8, x6
csel w0, w0, w8, eq
cbz x2, 0f
32: ldp x3, x4, [x1], #32
sub x2, x2, #32
ldp x5, x6, [x1, #-16]
CPU_BE( rev x3, x3 )
CPU_BE( rev x4, x4 )
CPU_BE( rev x5, x5 )
CPU_BE( rev x6, x6 )
crc32\c\()x w0, w0, x3
crc32\c\()x w0, w0, x4
b.ne 0b
ret
crc32\c\()x w0, w0, x5
crc32\c\()x w0, w0, x6
cbnz x2, 32b
0: ret
8: tbz x2, #3, 4f
ldr x3, [x1], #8

1
arch/arm64/lib/memchr.S
View File

@ -42,3 +42,4 @@ WEAK(memchr)
2: mov x0, #0
ret
ENDPIPROC(memchr)
EXPORT_SYMBOL_NOKASAN(memchr)

1
arch/arm64/lib/memcmp.S
View File

@ -256,3 +256,4 @@ CPU_LE( rev data2, data2 )
mov result, #0
ret
ENDPIPROC(memcmp)
EXPORT_SYMBOL_NOKASAN(memcmp)

2
arch/arm64/lib/memcpy.S
View File

@ -74,4 +74,6 @@ ENTRY(memcpy)
#include "copy_template.S"
ret
ENDPIPROC(memcpy)
EXPORT_SYMBOL(memcpy)
ENDPROC(__memcpy)
EXPORT_SYMBOL(__memcpy)

2
arch/arm64/lib/memmove.S
View File

@ -197,4 +197,6 @@ ENTRY(memmove)
b.ne .Ltail63
ret
ENDPIPROC(memmove)
EXPORT_SYMBOL(memmove)
ENDPROC(__memmove)
EXPORT_SYMBOL(__memmove)

2
arch/arm64/lib/memset.S
View File

@ -216,4 +216,6 @@ ENTRY(memset)
b.ne .Ltail_maybe_long
ret
ENDPIPROC(memset)
EXPORT_SYMBOL(memset)
ENDPROC(__memset)
EXPORT_SYMBOL(__memset)

1
arch/arm64/lib/strchr.S
View File

@ -40,3 +40,4 @@ WEAK(strchr)
csel x0, x0, xzr, eq
ret
ENDPROC(strchr)
EXPORT_SYMBOL_NOKASAN(strchr)

1
arch/arm64/lib/strcmp.S
View File

@ -232,3 +232,4 @@ CPU_BE( orr syndrome, diff, has_nul )
sub result, data1, data2, lsr #56
ret
ENDPIPROC(strcmp)
EXPORT_SYMBOL_NOKASAN(strcmp)

Some files were not shown because too many files have changed in this diff Show More