2019-06-03 13:44:50 +08:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2013-02-07 03:54:04 +08:00
|
|
|
/*
|
|
|
|
|
* (not much of an) Emulation layer for 32bit guests.
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2012,2013 - ARM Ltd
|
|
|
|
|
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
|
|
|
|
*
|
|
|
|
|
* based on arch/arm/kvm/emulate.c
|
|
|
|
|
* Copyright (C) 2012 - Virtual Open Systems and Columbia University
|
|
|
|
|
* Author: Christoffer Dall <c.dall@virtualopensystems.com>
|
|
|
|
|
*/
|
|
|
|
|
|
KVM: arm/arm64: Correct CPSR on exception entry
When KVM injects an exception into a guest, it generates the CPSR value
from scratch, configuring CPSR.{M,A,I,T,E}, and setting all other
bits to zero.
This isn't correct, as the architecture specifies that some CPSR bits
are (conditionally) cleared or set upon an exception, and others are
unchanged from the original context.
This patch adds logic to match the architectural behaviour. To make this
simple to follow/audit/extend, documentation references are provided,
and bits are configured in order of their layout in SPSR_EL2. This
layout can be seen in the diagram on ARM DDI 0487E.a page C5-426.
Note that this code is used by both arm and arm64, and is intended to
fuction with the SPSR_EL2 and SPSR_HYP layouts.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Reviewed-by: Alexandru Elisei <alexandru.elisei@arm.com>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/r/20200108134324.46500-3-mark.rutland@arm.com
2020-01-08 21:43:23 +08:00
|
|
|
#include <linux/bits.h>
|
2013-02-07 03:54:04 +08:00
|
|
|
#include <linux/kvm_host.h>
|
|
|
|
|
#include <asm/kvm_emulate.h>
|
2016-09-06 16:28:44 +08:00
|
|
|
#include <asm/kvm_hyp.h>
|
2013-02-07 03:54:04 +08:00
|
|
|
|
2020-01-21 20:33:56 +08:00
|
|
|
#define DFSR_FSC_EXTABT_LPAE 0x10
|
|
|
|
|
#define DFSR_FSC_EXTABT_nLPAE 0x08
|
|
|
|
|
#define DFSR_LPAE BIT(9)
|
|
|
|
|
|
2017-10-29 10:18:09 +08:00
|
|
|
/*
|
|
|
|
|
* Table taken from ARMv8 ARM DDI0487B-B, table G1-10.
|
|
|
|
|
*/
|
|
|
|
|
static const u8 return_offsets[8][2] = {
|
|
|
|
|
[0] = { 0, 0 }, /* Reset, unused */
|
|
|
|
|
[1] = { 4, 2 }, /* Undefined */
|
|
|
|
|
[2] = { 0, 0 }, /* SVC, unused */
|
|
|
|
|
[3] = { 4, 4 }, /* Prefetch abort */
|
|
|
|
|
[4] = { 8, 8 }, /* Data abort */
|
|
|
|
|
[5] = { 0, 0 }, /* HVC, unused */
|
|
|
|
|
[6] = { 4, 4 }, /* IRQ, unused */
|
|
|
|
|
[7] = { 4, 4 }, /* FIQ, unused */
|
|
|
|
|
};
|
|
|
|
|
|
2020-06-09 15:50:29 +08:00
|
|
|
static bool pre_fault_synchronize(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
preempt_disable();
|
|
|
|
|
if (vcpu->arch.sysregs_loaded_on_cpu) {
|
|
|
|
|
kvm_arch_vcpu_put(vcpu);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
preempt_enable();
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void post_fault_synchronize(struct kvm_vcpu *vcpu, bool loaded)
|
|
|
|
|
{
|
|
|
|
|
if (loaded) {
|
|
|
|
|
kvm_arch_vcpu_load(vcpu, smp_processor_id());
|
|
|
|
|
preempt_enable();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
KVM: arm/arm64: Correct CPSR on exception entry
When KVM injects an exception into a guest, it generates the CPSR value
from scratch, configuring CPSR.{M,A,I,T,E}, and setting all other
bits to zero.
This isn't correct, as the architecture specifies that some CPSR bits
are (conditionally) cleared or set upon an exception, and others are
unchanged from the original context.
This patch adds logic to match the architectural behaviour. To make this
simple to follow/audit/extend, documentation references are provided,
and bits are configured in order of their layout in SPSR_EL2. This
layout can be seen in the diagram on ARM DDI 0487E.a page C5-426.
Note that this code is used by both arm and arm64, and is intended to
fuction with the SPSR_EL2 and SPSR_HYP layouts.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Reviewed-by: Alexandru Elisei <alexandru.elisei@arm.com>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/r/20200108134324.46500-3-mark.rutland@arm.com
2020-01-08 21:43:23 +08:00
|
|
|
/*
|
|
|
|
|
* When an exception is taken, most CPSR fields are left unchanged in the
|
|
|
|
|
* handler. However, some are explicitly overridden (e.g. M[4:0]).
|
|
|
|
|
*
|
|
|
|
|
* The SPSR/SPSR_ELx layouts differ, and the below is intended to work with
|
|
|
|
|
* either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was
|
|
|
|
|
* obsoleted by the ARMv7 virtualization extensions and is RES0.
|
|
|
|
|
*
|
|
|
|
|
* For the SPSR layout seen from AArch32, see:
|
|
|
|
|
* - ARM DDI 0406C.d, page B1-1148
|
|
|
|
|
* - ARM DDI 0487E.a, page G8-6264
|
|
|
|
|
*
|
|
|
|
|
* For the SPSR_ELx layout for AArch32 seen from AArch64, see:
|
|
|
|
|
* - ARM DDI 0487E.a, page C5-426
|
|
|
|
|
*
|
|
|
|
|
* Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from
|
|
|
|
|
* MSB to LSB.
|
|
|
|
|
*/
|
|
|
|
|
static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode)
|
|
|
|
|
{
|
|
|
|
|
u32 sctlr = vcpu_cp15(vcpu, c1_SCTLR);
|
|
|
|
|
unsigned long old, new;
|
|
|
|
|
|
|
|
|
|
old = *vcpu_cpsr(vcpu);
|
|
|
|
|
new = 0;
|
|
|
|
|
|
|
|
|
|
new |= (old & PSR_AA32_N_BIT);
|
|
|
|
|
new |= (old & PSR_AA32_Z_BIT);
|
|
|
|
|
new |= (old & PSR_AA32_C_BIT);
|
|
|
|
|
new |= (old & PSR_AA32_V_BIT);
|
|
|
|
|
new |= (old & PSR_AA32_Q_BIT);
|
|
|
|
|
|
|
|
|
|
// CPSR.IT[7:0] are set to zero upon any exception
|
|
|
|
|
// See ARM DDI 0487E.a, section G1.12.3
|
|
|
|
|
// See ARM DDI 0406C.d, section B1.8.3
|
|
|
|
|
|
|
|
|
|
new |= (old & PSR_AA32_DIT_BIT);
|
|
|
|
|
|
|
|
|
|
// CPSR.SSBS is set to SCTLR.DSSBS upon any exception
|
|
|
|
|
// See ARM DDI 0487E.a, page G8-6244
|
|
|
|
|
if (sctlr & BIT(31))
|
|
|
|
|
new |= PSR_AA32_SSBS_BIT;
|
|
|
|
|
|
|
|
|
|
// CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0
|
|
|
|
|
// SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented
|
|
|
|
|
// See ARM DDI 0487E.a, page G8-6246
|
|
|
|
|
new |= (old & PSR_AA32_PAN_BIT);
|
|
|
|
|
if (!(sctlr & BIT(23)))
|
|
|
|
|
new |= PSR_AA32_PAN_BIT;
|
|
|
|
|
|
|
|
|
|
// SS does not exist in AArch32, so ignore
|
|
|
|
|
|
|
|
|
|
// CPSR.IL is set to zero upon any exception
|
|
|
|
|
// See ARM DDI 0487E.a, page G1-5527
|
|
|
|
|
|
|
|
|
|
new |= (old & PSR_AA32_GE_MASK);
|
|
|
|
|
|
|
|
|
|
// CPSR.IT[7:0] are set to zero upon any exception
|
|
|
|
|
// See prior comment above
|
|
|
|
|
|
|
|
|
|
// CPSR.E is set to SCTLR.EE upon any exception
|
|
|
|
|
// See ARM DDI 0487E.a, page G8-6245
|
|
|
|
|
// See ARM DDI 0406C.d, page B4-1701
|
|
|
|
|
if (sctlr & BIT(25))
|
|
|
|
|
new |= PSR_AA32_E_BIT;
|
|
|
|
|
|
|
|
|
|
// CPSR.A is unchanged upon an exception to Undefined, Supervisor
|
|
|
|
|
// CPSR.A is set upon an exception to other modes
|
|
|
|
|
// See ARM DDI 0487E.a, pages G1-5515 to G1-5516
|
|
|
|
|
// See ARM DDI 0406C.d, page B1-1182
|
|
|
|
|
new |= (old & PSR_AA32_A_BIT);
|
|
|
|
|
if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC)
|
|
|
|
|
new |= PSR_AA32_A_BIT;
|
|
|
|
|
|
|
|
|
|
// CPSR.I is set upon any exception
|
|
|
|
|
// See ARM DDI 0487E.a, pages G1-5515 to G1-5516
|
|
|
|
|
// See ARM DDI 0406C.d, page B1-1182
|
|
|
|
|
new |= PSR_AA32_I_BIT;
|
|
|
|
|
|
|
|
|
|
// CPSR.F is set upon an exception to FIQ
|
|
|
|
|
// CPSR.F is unchanged upon an exception to other modes
|
|
|
|
|
// See ARM DDI 0487E.a, pages G1-5515 to G1-5516
|
|
|
|
|
// See ARM DDI 0406C.d, page B1-1182
|
|
|
|
|
new |= (old & PSR_AA32_F_BIT);
|
|
|
|
|
if (mode == PSR_AA32_MODE_FIQ)
|
|
|
|
|
new |= PSR_AA32_F_BIT;
|
|
|
|
|
|
|
|
|
|
// CPSR.T is set to SCTLR.TE upon any exception
|
|
|
|
|
// See ARM DDI 0487E.a, page G8-5514
|
|
|
|
|
// See ARM DDI 0406C.d, page B1-1181
|
|
|
|
|
if (sctlr & BIT(30))
|
|
|
|
|
new |= PSR_AA32_T_BIT;
|
|
|
|
|
|
|
|
|
|
new |= mode;
|
|
|
|
|
|
|
|
|
|
return new;
|
|
|
|
|
}
|
|
|
|
|
|
2017-10-29 10:18:09 +08:00
|
|
|
static void prepare_fault32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
|
|
|
|
|
{
|
KVM: arm/arm64: Correct AArch32 SPSR on exception entry
Confusingly, there are three SPSR layouts that a kernel may need to deal
with:
(1) An AArch64 SPSR_ELx view of an AArch64 pstate
(2) An AArch64 SPSR_ELx view of an AArch32 pstate
(3) An AArch32 SPSR_* view of an AArch32 pstate
When the KVM AArch32 support code deals with SPSR_{EL2,HYP}, it's either
dealing with #2 or #3 consistently. On arm64 the PSR_AA32_* definitions
match the AArch64 SPSR_ELx view, and on arm the PSR_AA32_* definitions
match the AArch32 SPSR_* view.
However, when we inject an exception into an AArch32 guest, we have to
synthesize the AArch32 SPSR_* that the guest will see. Thus, an AArch64
host needs to synthesize layout #3 from layout #2.
This patch adds a new host_spsr_to_spsr32() helper for this, and makes
use of it in the KVM AArch32 support code. For arm64 we need to shuffle
the DIT bit around, and remove the SS bit, while for arm we can use the
value as-is.
I've open-coded the bit manipulation for now to avoid having to rework
the existing PSR_* definitions into PSR64_AA32_* and PSR32_AA32_*
definitions. I hope to perform a more thorough refactoring in future so
that we can handle pstate view manipulation more consistently across the
kernel tree.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Reviewed-by: Alexandru Elisei <alexandru.elisei@arm.com>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/r/20200108134324.46500-4-mark.rutland@arm.com
2020-01-08 21:43:24 +08:00
|
|
|
unsigned long spsr = *vcpu_cpsr(vcpu);
|
|
|
|
|
bool is_thumb = (spsr & PSR_AA32_T_BIT);
|
2017-10-29 10:18:09 +08:00
|
|
|
u32 return_offset = return_offsets[vect_offset >> 2][is_thumb];
|
|
|
|
|
u32 sctlr = vcpu_cp15(vcpu, c1_SCTLR);
|
|
|
|
|
|
KVM: arm/arm64: Correct CPSR on exception entry
When KVM injects an exception into a guest, it generates the CPSR value
from scratch, configuring CPSR.{M,A,I,T,E}, and setting all other
bits to zero.
This isn't correct, as the architecture specifies that some CPSR bits
are (conditionally) cleared or set upon an exception, and others are
unchanged from the original context.
This patch adds logic to match the architectural behaviour. To make this
simple to follow/audit/extend, documentation references are provided,
and bits are configured in order of their layout in SPSR_EL2. This
layout can be seen in the diagram on ARM DDI 0487E.a page C5-426.
Note that this code is used by both arm and arm64, and is intended to
fuction with the SPSR_EL2 and SPSR_HYP layouts.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Reviewed-by: Alexandru Elisei <alexandru.elisei@arm.com>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/r/20200108134324.46500-3-mark.rutland@arm.com
2020-01-08 21:43:23 +08:00
|
|
|
*vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode);
|
2017-10-29 10:18:09 +08:00
|
|
|
|
|
|
|
|
/* Note: These now point to the banked copies */
|
KVM: arm/arm64: Correct AArch32 SPSR on exception entry
Confusingly, there are three SPSR layouts that a kernel may need to deal
with:
(1) An AArch64 SPSR_ELx view of an AArch64 pstate
(2) An AArch64 SPSR_ELx view of an AArch32 pstate
(3) An AArch32 SPSR_* view of an AArch32 pstate
When the KVM AArch32 support code deals with SPSR_{EL2,HYP}, it's either
dealing with #2 or #3 consistently. On arm64 the PSR_AA32_* definitions
match the AArch64 SPSR_ELx view, and on arm the PSR_AA32_* definitions
match the AArch32 SPSR_* view.
However, when we inject an exception into an AArch32 guest, we have to
synthesize the AArch32 SPSR_* that the guest will see. Thus, an AArch64
host needs to synthesize layout #3 from layout #2.
This patch adds a new host_spsr_to_spsr32() helper for this, and makes
use of it in the KVM AArch32 support code. For arm64 we need to shuffle
the DIT bit around, and remove the SS bit, while for arm we can use the
value as-is.
I've open-coded the bit manipulation for now to avoid having to rework
the existing PSR_* definitions into PSR64_AA32_* and PSR32_AA32_*
definitions. I hope to perform a more thorough refactoring in future so
that we can handle pstate view manipulation more consistently across the
kernel tree.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Reviewed-by: Alexandru Elisei <alexandru.elisei@arm.com>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/r/20200108134324.46500-4-mark.rutland@arm.com
2020-01-08 21:43:24 +08:00
|
|
|
vcpu_write_spsr(vcpu, host_spsr_to_spsr32(spsr));
|
2017-10-29 10:18:09 +08:00
|
|
|
*vcpu_reg32(vcpu, 14) = *vcpu_pc(vcpu) + return_offset;
|
|
|
|
|
|
|
|
|
|
/* Branch to exception vector */
|
|
|
|
|
if (sctlr & (1 << 13))
|
|
|
|
|
vect_offset += 0xffff0000;
|
|
|
|
|
else /* always have security exceptions */
|
|
|
|
|
vect_offset += vcpu_cp15(vcpu, c12_VBAR);
|
|
|
|
|
|
|
|
|
|
*vcpu_pc(vcpu) = vect_offset;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void kvm_inject_undef32(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
2020-06-09 15:50:29 +08:00
|
|
|
bool loaded = pre_fault_synchronize(vcpu);
|
|
|
|
|
|
2018-07-05 22:16:53 +08:00
|
|
|
prepare_fault32(vcpu, PSR_AA32_MODE_UND, 4);
|
2020-06-09 15:50:29 +08:00
|
|
|
post_fault_synchronize(vcpu, loaded);
|
2017-10-29 10:18:09 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Modelled after TakeDataAbortException() and TakePrefetchAbortException
|
|
|
|
|
* pseudocode.
|
|
|
|
|
*/
|
|
|
|
|
static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt,
|
|
|
|
|
unsigned long addr)
|
|
|
|
|
{
|
|
|
|
|
u32 vect_offset;
|
|
|
|
|
u32 *far, *fsr;
|
|
|
|
|
bool is_lpae;
|
2020-06-09 15:50:29 +08:00
|
|
|
bool loaded;
|
|
|
|
|
|
|
|
|
|
loaded = pre_fault_synchronize(vcpu);
|
2017-10-29 10:18:09 +08:00
|
|
|
|
|
|
|
|
if (is_pabt) {
|
|
|
|
|
vect_offset = 12;
|
|
|
|
|
far = &vcpu_cp15(vcpu, c6_IFAR);
|
|
|
|
|
fsr = &vcpu_cp15(vcpu, c5_IFSR);
|
|
|
|
|
} else { /* !iabt */
|
|
|
|
|
vect_offset = 16;
|
|
|
|
|
far = &vcpu_cp15(vcpu, c6_DFAR);
|
|
|
|
|
fsr = &vcpu_cp15(vcpu, c5_DFSR);
|
|
|
|
|
}
|
|
|
|
|
|
KVM: arm/arm64: Correct CPSR on exception entry
When KVM injects an exception into a guest, it generates the CPSR value
from scratch, configuring CPSR.{M,A,I,T,E}, and setting all other
bits to zero.
This isn't correct, as the architecture specifies that some CPSR bits
are (conditionally) cleared or set upon an exception, and others are
unchanged from the original context.
This patch adds logic to match the architectural behaviour. To make this
simple to follow/audit/extend, documentation references are provided,
and bits are configured in order of their layout in SPSR_EL2. This
layout can be seen in the diagram on ARM DDI 0487E.a page C5-426.
Note that this code is used by both arm and arm64, and is intended to
fuction with the SPSR_EL2 and SPSR_HYP layouts.
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>
Reviewed-by: Alexandru Elisei <alexandru.elisei@arm.com>
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/r/20200108134324.46500-3-mark.rutland@arm.com
2020-01-08 21:43:23 +08:00
|
|
|
prepare_fault32(vcpu, PSR_AA32_MODE_ABT, vect_offset);
|
2017-10-29 10:18:09 +08:00
|
|
|
|
|
|
|
|
*far = addr;
|
|
|
|
|
|
|
|
|
|
/* Give the guest an IMPLEMENTATION DEFINED exception */
|
|
|
|
|
is_lpae = (vcpu_cp15(vcpu, c2_TTBCR) >> 31);
|
2020-01-21 20:33:55 +08:00
|
|
|
if (is_lpae) {
|
2020-01-21 20:33:56 +08:00
|
|
|
*fsr = DFSR_LPAE | DFSR_FSC_EXTABT_LPAE;
|
2020-01-21 20:33:55 +08:00
|
|
|
} else {
|
2020-01-21 20:33:56 +08:00
|
|
|
/* no need to shuffle FS[4] into DFSR[10] as its 0 */
|
|
|
|
|
*fsr = DFSR_FSC_EXTABT_nLPAE;
|
2020-01-21 20:33:55 +08:00
|
|
|
}
|
2020-06-09 15:50:29 +08:00
|
|
|
|
|
|
|
|
post_fault_synchronize(vcpu, loaded);
|
2017-10-29 10:18:09 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void kvm_inject_dabt32(struct kvm_vcpu *vcpu, unsigned long addr)
|
|
|
|
|
{
|
|
|
|
|
inject_abt32(vcpu, false, addr);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void kvm_inject_pabt32(struct kvm_vcpu *vcpu, unsigned long addr)
|
|
|
|
|
{
|
|
|
|
|
inject_abt32(vcpu, true, addr);
|
|
|
|
|
}
|