2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* native hashtable management.
|
|
|
|
|
*
|
|
|
|
|
* SMP scalability work:
|
|
|
|
|
* Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
|
|
|
|
|
*
|
|
|
|
|
* This program is free software; you can redistribute it and/or
|
|
|
|
|
* modify it under the terms of the GNU General Public License
|
|
|
|
|
* as published by the Free Software Foundation; either version
|
|
|
|
|
* 2 of the License, or (at your option) any later version.
|
|
|
|
|
*/
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
|
#undef DEBUG_LOW
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/spinlock.h>
|
|
|
|
|
#include <linux/bitops.h>
|
2012-07-26 05:20:03 +08:00
|
|
|
#include <linux/of.h>
|
2017-06-06 21:08:32 +08:00
|
|
|
#include <linux/processor.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <linux/threads.h>
|
|
|
|
|
#include <linux/smp.h>
|
|
|
|
|
|
|
|
|
|
#include <asm/machdep.h>
|
|
|
|
|
#include <asm/mmu.h>
|
|
|
|
|
#include <asm/mmu_context.h>
|
|
|
|
|
#include <asm/pgtable.h>
|
2017-04-11 13:23:25 +08:00
|
|
|
#include <asm/trace.h>
|
2005-04-17 06:20:36 +08:00
|
|
|
#include <asm/tlb.h>
|
|
|
|
|
#include <asm/cputable.h>
|
2005-11-07 08:06:55 +08:00
|
|
|
#include <asm/udbg.h>
|
2007-05-02 22:19:11 +08:00
|
|
|
#include <asm/kexec.h>
|
2009-04-30 04:58:01 +08:00
|
|
|
#include <asm/ppc-opcode.h>
|
2018-07-06 00:25:01 +08:00
|
|
|
#include <asm/feature-fixups.h>
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2015-05-27 14:07:16 +08:00
|
|
|
#include <misc/cxl-base.h>
|
2014-10-08 16:55:00 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
#ifdef DEBUG_LOW
|
|
|
|
|
#define DBG_LOW(fmt...) udbg_printf(fmt)
|
|
|
|
|
#else
|
|
|
|
|
#define DBG_LOW(fmt...)
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2013-09-23 10:04:36 +08:00
|
|
|
#ifdef __BIG_ENDIAN__
|
2005-04-17 06:20:36 +08:00
|
|
|
#define HPTE_LOCK_BIT 3
|
2013-09-23 10:04:36 +08:00
|
|
|
#else
|
|
|
|
|
#define HPTE_LOCK_BIT (56+3)
|
|
|
|
|
#endif
|
2005-04-17 06:20:36 +08:00
|
|
|
|
KVM: PPC: book3s_hv: Add support for PPC970-family processors
This adds support for running KVM guests in supervisor mode on those
PPC970 processors that have a usable hypervisor mode. Unfortunately,
Apple G5 machines have supervisor mode disabled (MSR[HV] is forced to
1), but the YDL PowerStation does have a usable hypervisor mode.
There are several differences between the PPC970 and POWER7 in how
guests are managed. These differences are accommodated using the
CPU_FTR_ARCH_201 (PPC970) and CPU_FTR_ARCH_206 (POWER7) CPU feature
bits. Notably, on PPC970:
* The LPCR, LPID or RMOR registers don't exist, and the functions of
those registers are provided by bits in HID4 and one bit in HID0.
* External interrupts can be directed to the hypervisor, but unlike
POWER7 they are masked by MSR[EE] in non-hypervisor modes and use
SRR0/1 not HSRR0/1.
* There is no virtual RMA (VRMA) mode; the guest must use an RMO
(real mode offset) area.
* The TLB entries are not tagged with the LPID, so it is necessary to
flush the whole TLB on partition switch. Furthermore, when switching
partitions we have to ensure that no other CPU is executing the tlbie
or tlbsync instructions in either the old or the new partition,
otherwise undefined behaviour can occur.
* The PMU has 8 counters (PMC registers) rather than 6.
* The DSCR, PURR, SPURR, AMR, AMOR, UAMOR registers don't exist.
* The SLB has 64 entries rather than 32.
* There is no mediated external interrupt facility, so if we switch to
a guest that has a virtual external interrupt pending but the guest
has MSR[EE] = 0, we have to arrange to have an interrupt pending for
it so that we can get control back once it re-enables interrupts. We
do that by sending ourselves an IPI with smp_send_reschedule after
hard-disabling interrupts.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Alexander Graf <agraf@suse.de>
2011-06-29 08:40:08 +08:00
|
|
|
DEFINE_RAW_SPINLOCK(native_tlbie_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
powerpc/64s: Improve local TLB flush for boot and MCE on POWER9
There are several cases outside the normal address space management
where a CPU's entire local TLB is to be flushed:
1. Booting the kernel, in case something has left stale entries in
the TLB (e.g., kexec).
2. Machine check, to clean corrupted TLB entries.
One other place where the TLB is flushed, is waking from deep idle
states. The flush is a side-effect of calling ->cpu_restore with the
intention of re-setting various SPRs. The flush itself is unnecessary
because in the first case, the TLB should not acquire new corrupted
TLB entries as part of sleep/wake (though they may be lost).
This type of TLB flush is coded inflexibly, several times for each CPU
type, and they have a number of problems with ISA v3.0B:
- The current radix mode of the MMU is not taken into account, it is
always done as a hash flushn For IS=2 (LPID-matching flush from host)
and IS=3 with HV=0 (guest kernel flush), tlbie(l) is undefined if
the R field does not match the current radix mode.
- ISA v3.0B hash must flush the partition and process table caches as
well.
- ISA v3.0B radix must flush partition and process scoped translations,
partition and process table caches, and also the page walk cache.
So consolidate the flushing code and implement it in C and inline asm
under the mm/ directory with the rest of the flush code. Add ISA v3.0B
cases for radix and hash, and use the radix flush in radix environment.
Provide a way for IS=2 (LPID flush) to specify the radix mode of the
partition. Have KVM pass in the radix mode of the guest.
Take out the flushes from early cputable/dt_cpu_ftrs detection hooks,
and move it later in the boot process after, the MMU registers are set
up and before relocation is first turned on.
The TLB flush is no longer called when restoring from deep idle states.
This was not be done as a separate step because booting secondaries
uses the same cpu_restore as idle restore, which needs the TLB flush.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2017-12-23 23:15:50 +08:00
|
|
|
static inline void tlbiel_hash_set_isa206(unsigned int set, unsigned int is)
|
|
|
|
|
{
|
|
|
|
|
unsigned long rb;
|
|
|
|
|
|
|
|
|
|
rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
|
|
|
|
|
|
|
|
|
|
asm volatile("tlbiel %0" : : "r" (rb));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* tlbiel instruction for hash, set invalidation
|
|
|
|
|
* i.e., r=1 and is=01 or is=10 or is=11
|
|
|
|
|
*/
|
|
|
|
|
static inline void tlbiel_hash_set_isa300(unsigned int set, unsigned int is,
|
|
|
|
|
unsigned int pid,
|
|
|
|
|
unsigned int ric, unsigned int prs)
|
|
|
|
|
{
|
|
|
|
|
unsigned long rb;
|
|
|
|
|
unsigned long rs;
|
|
|
|
|
unsigned int r = 0; /* hash format */
|
|
|
|
|
|
|
|
|
|
rb = (set << PPC_BITLSHIFT(51)) | (is << PPC_BITLSHIFT(53));
|
|
|
|
|
rs = ((unsigned long)pid << PPC_BITLSHIFT(31));
|
|
|
|
|
|
|
|
|
|
asm volatile(PPC_TLBIEL(%0, %1, %2, %3, %4)
|
|
|
|
|
: : "r"(rb), "r"(rs), "i"(ric), "i"(prs), "r"(r)
|
|
|
|
|
: "memory");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
static void tlbiel_all_isa206(unsigned int num_sets, unsigned int is)
|
|
|
|
|
{
|
|
|
|
|
unsigned int set;
|
|
|
|
|
|
|
|
|
|
asm volatile("ptesync": : :"memory");
|
|
|
|
|
|
|
|
|
|
for (set = 0; set < num_sets; set++)
|
|
|
|
|
tlbiel_hash_set_isa206(set, is);
|
|
|
|
|
|
|
|
|
|
asm volatile("ptesync": : :"memory");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void tlbiel_all_isa300(unsigned int num_sets, unsigned int is)
|
|
|
|
|
{
|
|
|
|
|
unsigned int set;
|
|
|
|
|
|
|
|
|
|
asm volatile("ptesync": : :"memory");
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Flush the first set of the TLB, and any caching of partition table
|
|
|
|
|
* entries. Then flush the remaining sets of the TLB. Hash mode uses
|
|
|
|
|
* partition scoped TLB translations.
|
|
|
|
|
*/
|
|
|
|
|
tlbiel_hash_set_isa300(0, is, 0, 2, 0);
|
|
|
|
|
for (set = 1; set < num_sets; set++)
|
|
|
|
|
tlbiel_hash_set_isa300(set, is, 0, 0, 0);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Now invalidate the process table cache.
|
|
|
|
|
*
|
|
|
|
|
* From ISA v3.0B p. 1078:
|
|
|
|
|
* The following forms are invalid.
|
|
|
|
|
* * PRS=1, R=0, and RIC!=2 (The only process-scoped
|
|
|
|
|
* HPT caching is of the Process Table.)
|
|
|
|
|
*/
|
|
|
|
|
tlbiel_hash_set_isa300(0, is, 0, 2, 1);
|
|
|
|
|
|
|
|
|
|
asm volatile("ptesync": : :"memory");
|
2018-08-27 11:03:01 +08:00
|
|
|
|
|
|
|
|
asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
|
powerpc/64s: Improve local TLB flush for boot and MCE on POWER9
There are several cases outside the normal address space management
where a CPU's entire local TLB is to be flushed:
1. Booting the kernel, in case something has left stale entries in
the TLB (e.g., kexec).
2. Machine check, to clean corrupted TLB entries.
One other place where the TLB is flushed, is waking from deep idle
states. The flush is a side-effect of calling ->cpu_restore with the
intention of re-setting various SPRs. The flush itself is unnecessary
because in the first case, the TLB should not acquire new corrupted
TLB entries as part of sleep/wake (though they may be lost).
This type of TLB flush is coded inflexibly, several times for each CPU
type, and they have a number of problems with ISA v3.0B:
- The current radix mode of the MMU is not taken into account, it is
always done as a hash flushn For IS=2 (LPID-matching flush from host)
and IS=3 with HV=0 (guest kernel flush), tlbie(l) is undefined if
the R field does not match the current radix mode.
- ISA v3.0B hash must flush the partition and process table caches as
well.
- ISA v3.0B radix must flush partition and process scoped translations,
partition and process table caches, and also the page walk cache.
So consolidate the flushing code and implement it in C and inline asm
under the mm/ directory with the rest of the flush code. Add ISA v3.0B
cases for radix and hash, and use the radix flush in radix environment.
Provide a way for IS=2 (LPID flush) to specify the radix mode of the
partition. Have KVM pass in the radix mode of the guest.
Take out the flushes from early cputable/dt_cpu_ftrs detection hooks,
and move it later in the boot process after, the MMU registers are set
up and before relocation is first turned on.
The TLB flush is no longer called when restoring from deep idle states.
This was not be done as a separate step because booting secondaries
uses the same cpu_restore as idle restore, which needs the TLB flush.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2017-12-23 23:15:50 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void hash__tlbiel_all(unsigned int action)
|
|
|
|
|
{
|
|
|
|
|
unsigned int is;
|
|
|
|
|
|
|
|
|
|
switch (action) {
|
|
|
|
|
case TLB_INVAL_SCOPE_GLOBAL:
|
|
|
|
|
is = 3;
|
|
|
|
|
break;
|
|
|
|
|
case TLB_INVAL_SCOPE_LPID:
|
|
|
|
|
is = 2;
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
BUG();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (early_cpu_has_feature(CPU_FTR_ARCH_300))
|
|
|
|
|
tlbiel_all_isa300(POWER9_TLB_SETS_HASH, is);
|
|
|
|
|
else if (early_cpu_has_feature(CPU_FTR_ARCH_207S))
|
|
|
|
|
tlbiel_all_isa206(POWER8_TLB_SETS, is);
|
|
|
|
|
else if (early_cpu_has_feature(CPU_FTR_ARCH_206))
|
|
|
|
|
tlbiel_all_isa206(POWER7_TLB_SETS, is);
|
|
|
|
|
else
|
|
|
|
|
WARN(1, "%s called on pre-POWER7 CPU\n", __func__);
|
|
|
|
|
}
|
|
|
|
|
|
2017-11-23 01:32:07 +08:00
|
|
|
static inline unsigned long ___tlbie(unsigned long vpn, int psize,
|
|
|
|
|
int apsize, int ssize)
|
2005-11-07 08:06:55 +08:00
|
|
|
{
|
2012-09-10 10:52:50 +08:00
|
|
|
unsigned long va;
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned int penc;
|
2013-07-04 13:04:45 +08:00
|
|
|
unsigned long sllp;
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
/*
|
|
|
|
|
* We need 14 to 65 bits of va for a tlibe of 4K page
|
|
|
|
|
* With vpn we ignore the lower VPN_SHIFT bits already.
|
|
|
|
|
* And top two bits are already ignored because we can
|
2016-06-01 14:34:37 +08:00
|
|
|
* only accomodate 76 bits in a 64 bit vpn with a VPN_SHIFT
|
2012-09-10 10:52:50 +08:00
|
|
|
* of 12.
|
|
|
|
|
*/
|
|
|
|
|
va = vpn << VPN_SHIFT;
|
|
|
|
|
/*
|
|
|
|
|
* clear top 16 bits of 64bit va, non SLS segment
|
|
|
|
|
* Older versions of the architecture (2.02 and earler) require the
|
|
|
|
|
* masking of the top 16 bits.
|
|
|
|
|
*/
|
2016-07-13 17:35:24 +08:00
|
|
|
if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
|
|
|
|
|
va &= ~(0xffffULL << 48);
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
|
switch (psize) {
|
|
|
|
|
case MMU_PAGE_4K:
|
2013-04-28 17:37:39 +08:00
|
|
|
/* clear out bits after (52) [0....52.....63] */
|
|
|
|
|
va &= ~((1ul << (64 - 52)) - 1);
|
2007-10-11 18:37:10 +08:00
|
|
|
va |= ssize << 8;
|
2016-07-13 17:36:37 +08:00
|
|
|
sllp = get_sllp_encoding(apsize);
|
2013-07-04 13:04:45 +08:00
|
|
|
va |= sllp << 5;
|
2011-04-07 02:23:29 +08:00
|
|
|
asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
|
powerpc, KVM: Split HVMODE_206 cpu feature bit into separate HV and architecture bits
This replaces the single CPU_FTR_HVMODE_206 bit with two bits, one to
indicate that we have a usable hypervisor mode, and another to indicate
that the processor conforms to PowerISA version 2.06. We also add
another bit to indicate that the processor conforms to ISA version 2.01
and set that for PPC970 and derivatives.
Some PPC970 chips (specifically those in Apple machines) have a
hypervisor mode in that MSR[HV] is always 1, but the hypervisor mode
is not useful in the sense that there is no way to run any code in
supervisor mode (HV=0 PR=0). On these processors, the LPES0 and LPES1
bits in HID4 are always 0, and we use that as a way of detecting that
hypervisor mode is not useful.
Where we have a feature section in assembly code around code that
only applies on POWER7 in hypervisor mode, we use a construct like
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
The definition of END_FTR_SECTION_IFSET is such that the code will
be enabled (not overwritten with nops) only if all bits in the
provided mask are set.
Note that the CPU feature check in __tlbie() only needs to check the
ARCH_206 bit, not the HVMODE bit, because __tlbie() can only get called
if we are running bare-metal, i.e. in hypervisor mode.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Alexander Graf <agraf@suse.de>
2011-06-29 08:26:11 +08:00
|
|
|
: : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
|
2009-04-30 04:58:01 +08:00
|
|
|
: "memory");
|
2005-11-07 08:06:55 +08:00
|
|
|
break;
|
|
|
|
|
default:
|
2012-09-10 10:52:50 +08:00
|
|
|
/* We need 14 to 14 + i bits of va */
|
2013-04-28 17:37:35 +08:00
|
|
|
penc = mmu_psize_defs[psize].penc[apsize];
|
2013-04-28 17:37:39 +08:00
|
|
|
va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
|
2006-06-15 19:15:44 +08:00
|
|
|
va |= penc << 12;
|
2007-10-11 18:37:10 +08:00
|
|
|
va |= ssize << 8;
|
2014-04-21 13:07:36 +08:00
|
|
|
/*
|
|
|
|
|
* AVAL bits:
|
|
|
|
|
* We don't need all the bits, but rest of the bits
|
|
|
|
|
* must be ignored by the processor.
|
|
|
|
|
* vpn cover upto 65 bits of va. (0...65) and we need
|
|
|
|
|
* 58..64 bits of va.
|
|
|
|
|
*/
|
|
|
|
|
va |= (vpn & 0xfe); /* AVAL */
|
2009-04-30 04:58:01 +08:00
|
|
|
va |= 1; /* L */
|
2011-04-07 02:23:29 +08:00
|
|
|
asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
|
powerpc, KVM: Split HVMODE_206 cpu feature bit into separate HV and architecture bits
This replaces the single CPU_FTR_HVMODE_206 bit with two bits, one to
indicate that we have a usable hypervisor mode, and another to indicate
that the processor conforms to PowerISA version 2.06. We also add
another bit to indicate that the processor conforms to ISA version 2.01
and set that for PPC970 and derivatives.
Some PPC970 chips (specifically those in Apple machines) have a
hypervisor mode in that MSR[HV] is always 1, but the hypervisor mode
is not useful in the sense that there is no way to run any code in
supervisor mode (HV=0 PR=0). On these processors, the LPES0 and LPES1
bits in HID4 are always 0, and we use that as a way of detecting that
hypervisor mode is not useful.
Where we have a feature section in assembly code around code that
only applies on POWER7 in hypervisor mode, we use a construct like
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
The definition of END_FTR_SECTION_IFSET is such that the code will
be enabled (not overwritten with nops) only if all bits in the
provided mask are set.
Note that the CPU feature check in __tlbie() only needs to check the
ARCH_206 bit, not the HVMODE bit, because __tlbie() can only get called
if we are running bare-metal, i.e. in hypervisor mode.
Signed-off-by: Paul Mackerras <paulus@samba.org>
Signed-off-by: Alexander Graf <agraf@suse.de>
2011-06-29 08:26:11 +08:00
|
|
|
: : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
|
2009-04-30 04:58:01 +08:00
|
|
|
: "memory");
|
2005-11-07 08:06:55 +08:00
|
|
|
break;
|
|
|
|
|
}
|
2017-11-23 01:32:07 +08:00
|
|
|
return va;
|
|
|
|
|
}
|
|
|
|
|
|
2018-03-23 12:56:27 +08:00
|
|
|
static inline void fixup_tlbie(unsigned long vpn, int psize, int apsize, int ssize)
|
|
|
|
|
{
|
|
|
|
|
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
|
|
|
|
|
/* Need the extra ptesync to ensure we don't reorder tlbie*/
|
|
|
|
|
asm volatile("ptesync": : :"memory");
|
|
|
|
|
___tlbie(vpn, psize, apsize, ssize);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2017-11-23 01:32:07 +08:00
|
|
|
static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
|
|
|
|
|
{
|
|
|
|
|
unsigned long rb;
|
|
|
|
|
|
|
|
|
|
rb = ___tlbie(vpn, psize, apsize, ssize);
|
|
|
|
|
trace_tlbie(0, 0, rb, 0, 0, 0, 0);
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
|
|
|
|
|
2013-04-28 17:37:35 +08:00
|
|
|
static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
|
2005-11-07 08:06:55 +08:00
|
|
|
{
|
2012-09-10 10:52:50 +08:00
|
|
|
unsigned long va;
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned int penc;
|
2013-07-04 13:04:45 +08:00
|
|
|
unsigned long sllp;
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
/* VPN_SHIFT can be atmost 12 */
|
|
|
|
|
va = vpn << VPN_SHIFT;
|
|
|
|
|
/*
|
|
|
|
|
* clear top 16 bits of 64 bit va, non SLS segment
|
|
|
|
|
* Older versions of the architecture (2.02 and earler) require the
|
|
|
|
|
* masking of the top 16 bits.
|
|
|
|
|
*/
|
2016-07-13 17:35:24 +08:00
|
|
|
if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
|
|
|
|
|
va &= ~(0xffffULL << 48);
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
|
switch (psize) {
|
|
|
|
|
case MMU_PAGE_4K:
|
2013-04-28 17:37:39 +08:00
|
|
|
/* clear out bits after(52) [0....52.....63] */
|
|
|
|
|
va &= ~((1ul << (64 - 52)) - 1);
|
2007-10-11 18:37:10 +08:00
|
|
|
va |= ssize << 8;
|
2016-07-13 17:36:37 +08:00
|
|
|
sllp = get_sllp_encoding(apsize);
|
2013-07-04 13:04:45 +08:00
|
|
|
va |= sllp << 5;
|
powerpc/hash64: Be more careful when generating tlbiel
In ISA v2.05, the tlbiel instruction takes two arguments, RB and L:
tlbiel RB,L
+---------+---------+----+---------+---------+---------+----+
| 31 | / | L | / | RB | 274 | / |
| 31 - 26 | 25 - 22 | 21 | 20 - 16 | 15 - 11 | 10 - 1 | 0 |
+---------+---------+----+---------+---------+---------+----+
In ISA v2.06 tlbiel takes only one argument, RB:
tlbiel RB
+---------+---------+---------+---------+---------+----+
| 31 | / | / | RB | 274 | / |
| 31 - 26 | 25 - 21 | 20 - 16 | 15 - 11 | 10 - 1 | 0 |
+---------+---------+---------+---------+---------+----+
And in ISA v3.00 tlbiel takes five arguments:
tlbiel RB,RS,RIC,PRS,R
+---------+---------+----+---------+----+----+---------+---------+----+
| 31 | RS | / | RIC |PRS | R | RB | 274 | / |
| 31 - 26 | 25 - 21 | 20 | 19 - 18 | 17 | 16 | 15 - 11 | 10 - 1 | 0 |
+---------+---------+----+---------+----+----+---------+---------+----+
However the assembler also accepts "tlbiel RB", and generates
"tlbiel RB,r0,0,0,0".
As you can see above the L field from the v2.05 encoding overlaps with the
reserved field of the v2.06 encoding, and the low bit of the RS field of the
v3.00 encoding.
Currently in __tlbiel() we generate two tlbiel instructions manually using hex
constants. In the first case, for MMU_PAGE_4K, we generate "tlbiel RB,0", which
is safe in all cases, because the L bit is zero.
However in the default case we generate "tlbiel RB,1", therefore setting bit 21
to 1.
This is not an actual bug on v2.06 processors, because the CPU ignores the value
of the reserved field. However software is supposed to encode the reserved
fields as zero to enable forward compatibility.
On v3.00 processors setting bit 21 to 1 and no other bits of RS, means we are
using r1 for the value of RS.
Although it's not obvious, the code sets the IS field (bits 10-11) to 0 (by
omission), and L=1, in the va value, which is passed as RB. We also pass R=0 in
the instruction.
The combination of IS=0, L=1 and R=0 means the value of RS is not used, so even
on ISA v3.00 there is no actual bug.
We should still fix it, as setting a reserved bit on v2.06 is naughty, and we
are only avoiding a bug on v3.00 by accident rather than design. Use
ASM_FTR_IFSET() to generate the single argument form on ISA v2.06 and later, and
the two argument form on pre v2.06.
Although there may be very old toolchains which don't understand tlbiel, we have
other code in the tree which has been using tlbiel for over five years, and no
one has reported any build failures, so just let the assembler generate the
instructions.
Signed-off-by: Balbir Singh <bsingharora@gmail.com>
[mpe: Rewrite change log, use IFSET instead of IFCLR]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2016-09-28 15:25:52 +08:00
|
|
|
asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,0", %1)
|
|
|
|
|
: : "r" (va), "i" (CPU_FTR_ARCH_206)
|
|
|
|
|
: "memory");
|
2005-11-07 08:06:55 +08:00
|
|
|
break;
|
|
|
|
|
default:
|
2012-09-10 10:52:50 +08:00
|
|
|
/* We need 14 to 14 + i bits of va */
|
2013-04-28 17:37:35 +08:00
|
|
|
penc = mmu_psize_defs[psize].penc[apsize];
|
2013-04-28 17:37:39 +08:00
|
|
|
va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
|
2006-06-15 19:15:44 +08:00
|
|
|
va |= penc << 12;
|
2007-10-11 18:37:10 +08:00
|
|
|
va |= ssize << 8;
|
2014-04-21 13:07:36 +08:00
|
|
|
/*
|
|
|
|
|
* AVAL bits:
|
|
|
|
|
* We don't need all the bits, but rest of the bits
|
|
|
|
|
* must be ignored by the processor.
|
|
|
|
|
* vpn cover upto 65 bits of va. (0...65) and we need
|
|
|
|
|
* 58..64 bits of va.
|
|
|
|
|
*/
|
|
|
|
|
va |= (vpn & 0xfe);
|
2009-04-30 04:58:01 +08:00
|
|
|
va |= 1; /* L */
|
powerpc/hash64: Be more careful when generating tlbiel
In ISA v2.05, the tlbiel instruction takes two arguments, RB and L:
tlbiel RB,L
+---------+---------+----+---------+---------+---------+----+
| 31 | / | L | / | RB | 274 | / |
| 31 - 26 | 25 - 22 | 21 | 20 - 16 | 15 - 11 | 10 - 1 | 0 |
+---------+---------+----+---------+---------+---------+----+
In ISA v2.06 tlbiel takes only one argument, RB:
tlbiel RB
+---------+---------+---------+---------+---------+----+
| 31 | / | / | RB | 274 | / |
| 31 - 26 | 25 - 21 | 20 - 16 | 15 - 11 | 10 - 1 | 0 |
+---------+---------+---------+---------+---------+----+
And in ISA v3.00 tlbiel takes five arguments:
tlbiel RB,RS,RIC,PRS,R
+---------+---------+----+---------+----+----+---------+---------+----+
| 31 | RS | / | RIC |PRS | R | RB | 274 | / |
| 31 - 26 | 25 - 21 | 20 | 19 - 18 | 17 | 16 | 15 - 11 | 10 - 1 | 0 |
+---------+---------+----+---------+----+----+---------+---------+----+
However the assembler also accepts "tlbiel RB", and generates
"tlbiel RB,r0,0,0,0".
As you can see above the L field from the v2.05 encoding overlaps with the
reserved field of the v2.06 encoding, and the low bit of the RS field of the
v3.00 encoding.
Currently in __tlbiel() we generate two tlbiel instructions manually using hex
constants. In the first case, for MMU_PAGE_4K, we generate "tlbiel RB,0", which
is safe in all cases, because the L bit is zero.
However in the default case we generate "tlbiel RB,1", therefore setting bit 21
to 1.
This is not an actual bug on v2.06 processors, because the CPU ignores the value
of the reserved field. However software is supposed to encode the reserved
fields as zero to enable forward compatibility.
On v3.00 processors setting bit 21 to 1 and no other bits of RS, means we are
using r1 for the value of RS.
Although it's not obvious, the code sets the IS field (bits 10-11) to 0 (by
omission), and L=1, in the va value, which is passed as RB. We also pass R=0 in
the instruction.
The combination of IS=0, L=1 and R=0 means the value of RS is not used, so even
on ISA v3.00 there is no actual bug.
We should still fix it, as setting a reserved bit on v2.06 is naughty, and we
are only avoiding a bug on v3.00 by accident rather than design. Use
ASM_FTR_IFSET() to generate the single argument form on ISA v2.06 and later, and
the two argument form on pre v2.06.
Although there may be very old toolchains which don't understand tlbiel, we have
other code in the tree which has been using tlbiel for over five years, and no
one has reported any build failures, so just let the assembler generate the
instructions.
Signed-off-by: Balbir Singh <bsingharora@gmail.com>
[mpe: Rewrite change log, use IFSET instead of IFCLR]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2016-09-28 15:25:52 +08:00
|
|
|
asm volatile(ASM_FTR_IFSET("tlbiel %0", "tlbiel %0,1", %1)
|
|
|
|
|
: : "r" (va), "i" (CPU_FTR_ARCH_206)
|
|
|
|
|
: "memory");
|
2005-11-07 08:06:55 +08:00
|
|
|
break;
|
|
|
|
|
}
|
2017-04-11 13:23:25 +08:00
|
|
|
trace_tlbie(0, 1, va, 0, 0, 0, 0);
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
2013-04-28 17:37:35 +08:00
|
|
|
static inline void tlbie(unsigned long vpn, int psize, int apsize,
|
|
|
|
|
int ssize, int local)
|
2005-11-07 08:06:55 +08:00
|
|
|
{
|
2014-10-08 16:55:00 +08:00
|
|
|
unsigned int use_local;
|
2011-04-07 03:48:50 +08:00
|
|
|
int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2014-10-08 16:55:00 +08:00
|
|
|
use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) && !cxl_ctx_in_use();
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
if (use_local)
|
|
|
|
|
use_local = mmu_psize_defs[psize].tlbiel;
|
|
|
|
|
if (lock_tlbie && !use_local)
|
2010-02-18 10:22:35 +08:00
|
|
|
raw_spin_lock(&native_tlbie_lock);
|
2005-11-07 08:06:55 +08:00
|
|
|
asm volatile("ptesync": : :"memory");
|
|
|
|
|
if (use_local) {
|
2013-04-28 17:37:35 +08:00
|
|
|
__tlbiel(vpn, psize, apsize, ssize);
|
2005-11-07 08:06:55 +08:00
|
|
|
asm volatile("ptesync": : :"memory");
|
|
|
|
|
} else {
|
2013-04-28 17:37:35 +08:00
|
|
|
__tlbie(vpn, psize, apsize, ssize);
|
2018-03-23 12:56:27 +08:00
|
|
|
fixup_tlbie(vpn, psize, apsize, ssize);
|
2005-11-07 08:06:55 +08:00
|
|
|
asm volatile("eieio; tlbsync; ptesync": : :"memory");
|
|
|
|
|
}
|
|
|
|
|
if (lock_tlbie && !use_local)
|
2010-02-18 10:22:35 +08:00
|
|
|
raw_spin_unlock(&native_tlbie_lock);
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
|
|
|
|
|
2007-06-13 12:52:56 +08:00
|
|
|
static inline void native_lock_hpte(struct hash_pte *hptep)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2013-09-23 10:04:36 +08:00
|
|
|
unsigned long *word = (unsigned long *)&hptep->v;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
while (1) {
|
2010-02-10 09:03:06 +08:00
|
|
|
if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word))
|
2005-04-17 06:20:36 +08:00
|
|
|
break;
|
2017-06-06 21:08:32 +08:00
|
|
|
spin_begin();
|
2005-04-17 06:20:36 +08:00
|
|
|
while(test_bit(HPTE_LOCK_BIT, word))
|
2017-06-06 21:08:32 +08:00
|
|
|
spin_cpu_relax();
|
|
|
|
|
spin_end();
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2007-06-13 12:52:56 +08:00
|
|
|
static inline void native_unlock_hpte(struct hash_pte *hptep)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2013-09-23 10:04:36 +08:00
|
|
|
unsigned long *word = (unsigned long *)&hptep->v;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2010-02-10 09:03:06 +08:00
|
|
|
clear_bit_unlock(HPTE_LOCK_BIT, word);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn,
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned long pa, unsigned long rflags,
|
2013-04-28 17:37:35 +08:00
|
|
|
unsigned long vflags, int psize, int apsize, int ssize)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2007-06-13 12:52:56 +08:00
|
|
|
struct hash_pte *hptep = htab_address + hpte_group;
|
2005-07-13 16:11:42 +08:00
|
|
|
unsigned long hpte_v, hpte_r;
|
2005-04-17 06:20:36 +08:00
|
|
|
int i;
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
if (!(vflags & HPTE_V_BOLTED)) {
|
2012-09-10 10:52:50 +08:00
|
|
|
DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx,"
|
2005-11-07 08:06:55 +08:00
|
|
|
" rflags=%lx, vflags=%lx, psize=%d)\n",
|
2012-09-10 10:52:50 +08:00
|
|
|
hpte_group, vpn, pa, rflags, vflags, psize);
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
for (i = 0; i < HPTES_PER_GROUP; i++) {
|
2013-09-23 10:04:36 +08:00
|
|
|
if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
/* retry with lock held */
|
|
|
|
|
native_lock_hpte(hptep);
|
2013-09-23 10:04:36 +08:00
|
|
|
if (! (be64_to_cpu(hptep->v) & HPTE_V_VALID))
|
2005-04-17 06:20:36 +08:00
|
|
|
break;
|
|
|
|
|
native_unlock_hpte(hptep);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
hptep++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (i == HPTES_PER_GROUP)
|
|
|
|
|
return -1;
|
|
|
|
|
|
2013-04-28 17:37:35 +08:00
|
|
|
hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID;
|
2016-11-11 13:55:03 +08:00
|
|
|
hpte_r = hpte_encode_r(pa, psize, apsize) | rflags;
|
2005-11-07 08:06:55 +08:00
|
|
|
|
|
|
|
|
if (!(vflags & HPTE_V_BOLTED)) {
|
|
|
|
|
DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n",
|
|
|
|
|
i, hpte_v, hpte_r);
|
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2016-11-11 13:55:03 +08:00
|
|
|
if (cpu_has_feature(CPU_FTR_ARCH_300)) {
|
|
|
|
|
hpte_r = hpte_old_to_new_r(hpte_v, hpte_r);
|
|
|
|
|
hpte_v = hpte_old_to_new_v(hpte_v);
|
|
|
|
|
}
|
|
|
|
|
|
2013-09-23 10:04:36 +08:00
|
|
|
hptep->r = cpu_to_be64(hpte_r);
|
2005-04-17 06:20:36 +08:00
|
|
|
/* Guarantee the second dword is visible before the valid bit */
|
2007-07-10 12:49:09 +08:00
|
|
|
eieio();
|
2005-04-17 06:20:36 +08:00
|
|
|
/*
|
|
|
|
|
* Now set the first dword including the valid bit
|
|
|
|
|
* NOTE: this also unlocks the hpte
|
|
|
|
|
*/
|
2013-09-23 10:04:36 +08:00
|
|
|
hptep->v = cpu_to_be64(hpte_v);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
__asm__ __volatile__ ("ptesync" : : : "memory");
|
|
|
|
|
|
2005-07-13 16:11:42 +08:00
|
|
|
return i | (!!(vflags & HPTE_V_SECONDARY) << 3);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static long native_hpte_remove(unsigned long hpte_group)
|
|
|
|
|
{
|
2007-06-13 12:52:56 +08:00
|
|
|
struct hash_pte *hptep;
|
2005-04-17 06:20:36 +08:00
|
|
|
int i;
|
|
|
|
|
int slot_offset;
|
2005-07-13 16:11:42 +08:00
|
|
|
unsigned long hpte_v;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
DBG_LOW(" remove(group=%lx)\n", hpte_group);
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
/* pick a random entry to start at */
|
|
|
|
|
slot_offset = mftb() & 0x7;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < HPTES_PER_GROUP; i++) {
|
|
|
|
|
hptep = htab_address + hpte_group + slot_offset;
|
2013-09-23 10:04:36 +08:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2005-07-13 16:11:42 +08:00
|
|
|
if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) {
|
2005-04-17 06:20:36 +08:00
|
|
|
/* retry with lock held */
|
|
|
|
|
native_lock_hpte(hptep);
|
2013-09-23 10:04:36 +08:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2005-07-13 16:11:42 +08:00
|
|
|
if ((hpte_v & HPTE_V_VALID)
|
|
|
|
|
&& !(hpte_v & HPTE_V_BOLTED))
|
2005-04-17 06:20:36 +08:00
|
|
|
break;
|
|
|
|
|
native_unlock_hpte(hptep);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
slot_offset++;
|
|
|
|
|
slot_offset &= 0x7;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (i == HPTES_PER_GROUP)
|
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
|
|
/* Invalidate the hpte. NOTE: this also unlocks it */
|
2005-07-13 16:11:42 +08:00
|
|
|
hptep->v = 0;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
return i;
|
|
|
|
|
}
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
|
2013-06-20 17:00:13 +08:00
|
|
|
unsigned long vpn, int bpsize,
|
2014-12-04 13:30:14 +08:00
|
|
|
int apsize, int ssize, unsigned long flags)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2007-06-13 12:52:56 +08:00
|
|
|
struct hash_pte *hptep = htab_address + slot;
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned long hpte_v, want_v;
|
2014-12-04 13:30:14 +08:00
|
|
|
int ret = 0, local = 0;
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2013-06-20 17:00:13 +08:00
|
|
|
want_v = hpte_encode_avpn(vpn, bpsize, ssize);
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
|
|
|
|
|
vpn, want_v & HPTE_V_AVPN, slot, newpp);
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2018-06-29 16:36:30 +08:00
|
|
|
hpte_v = hpte_get_old_v(hptep);
|
2013-05-31 09:03:24 +08:00
|
|
|
/*
|
|
|
|
|
* We need to invalidate the TLB always because hpte_remove doesn't do
|
|
|
|
|
* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
|
|
|
|
|
* random entry from it. When we do that we don't invalidate the TLB
|
|
|
|
|
* (hpte_remove) because we assume the old translation is still
|
|
|
|
|
* technically "valid".
|
|
|
|
|
*/
|
2013-06-20 17:00:13 +08:00
|
|
|
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
|
2005-11-07 08:06:55 +08:00
|
|
|
DBG_LOW(" -> miss\n");
|
|
|
|
|
ret = -1;
|
|
|
|
|
} else {
|
2014-11-03 22:51:34 +08:00
|
|
|
native_lock_hpte(hptep);
|
|
|
|
|
/* recheck with locks held */
|
2018-06-29 16:36:30 +08:00
|
|
|
hpte_v = hpte_get_old_v(hptep);
|
2014-11-03 22:51:34 +08:00
|
|
|
if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) ||
|
|
|
|
|
!(hpte_v & HPTE_V_VALID))) {
|
|
|
|
|
ret = -1;
|
|
|
|
|
} else {
|
|
|
|
|
DBG_LOW(" -> hit\n");
|
|
|
|
|
/* Update the HPTE */
|
|
|
|
|
hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
|
2016-06-08 22:25:55 +08:00
|
|
|
~(HPTE_R_PPP | HPTE_R_N)) |
|
|
|
|
|
(newpp & (HPTE_R_PPP | HPTE_R_N |
|
2014-11-03 22:51:34 +08:00
|
|
|
HPTE_R_C)));
|
|
|
|
|
}
|
|
|
|
|
native_unlock_hpte(hptep);
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
2014-12-04 13:30:14 +08:00
|
|
|
|
|
|
|
|
if (flags & HPTE_LOCAL_UPDATE)
|
|
|
|
|
local = 1;
|
|
|
|
|
/*
|
|
|
|
|
* Ensure it is out of the tlb too if it is not a nohpte fault
|
|
|
|
|
*/
|
|
|
|
|
if (!(flags & HPTE_NOHPTE_UPDATE))
|
|
|
|
|
tlbie(vpn, bpsize, apsize, ssize, local);
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
return ret;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
static long native_hpte_find(unsigned long vpn, int psize, int ssize)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2007-06-13 12:52:56 +08:00
|
|
|
struct hash_pte *hptep;
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned long hash;
|
2007-10-11 18:37:10 +08:00
|
|
|
unsigned long i;
|
2005-04-17 06:20:36 +08:00
|
|
|
long slot;
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned long want_v, hpte_v;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize);
|
2013-04-28 17:37:34 +08:00
|
|
|
want_v = hpte_encode_avpn(vpn, psize, ssize);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-10-11 18:37:10 +08:00
|
|
|
/* Bolted mappings are only ever in the primary group */
|
|
|
|
|
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
|
|
|
|
|
for (i = 0; i < HPTES_PER_GROUP; i++) {
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2018-06-29 16:36:30 +08:00
|
|
|
hptep = htab_address + slot;
|
|
|
|
|
hpte_v = hpte_get_old_v(hptep);
|
2007-10-11 18:37:10 +08:00
|
|
|
if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
|
|
|
|
|
/* HPTE matches */
|
|
|
|
|
return slot;
|
|
|
|
|
++slot;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Update the page protection bits. Intended to be used to create
|
|
|
|
|
* guard pages for kernel data structures on pages which are bolted
|
|
|
|
|
* in the HPT. Assumes pages being operated on will not be stolen.
|
|
|
|
|
*
|
|
|
|
|
* No need to lock here because we should be the only user.
|
|
|
|
|
*/
|
2005-11-07 08:06:55 +08:00
|
|
|
static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
|
2007-10-11 18:37:10 +08:00
|
|
|
int psize, int ssize)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2012-09-10 10:52:50 +08:00
|
|
|
unsigned long vpn;
|
|
|
|
|
unsigned long vsid;
|
2005-04-17 06:20:36 +08:00
|
|
|
long slot;
|
2007-06-13 12:52:56 +08:00
|
|
|
struct hash_pte *hptep;
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2007-10-11 18:37:10 +08:00
|
|
|
vsid = get_kernel_vsid(ea, ssize);
|
2012-09-10 10:52:50 +08:00
|
|
|
vpn = hpt_vpn(ea, vsid, ssize);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
slot = native_hpte_find(vpn, psize, ssize);
|
2005-04-17 06:20:36 +08:00
|
|
|
if (slot == -1)
|
|
|
|
|
panic("could not find page to bolt\n");
|
|
|
|
|
hptep = htab_address + slot;
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/* Update the HPTE */
|
2013-09-23 10:04:36 +08:00
|
|
|
hptep->r = cpu_to_be64((be64_to_cpu(hptep->r) &
|
2016-06-08 22:25:55 +08:00
|
|
|
~(HPTE_R_PPP | HPTE_R_N)) |
|
|
|
|
|
(newpp & (HPTE_R_PPP | HPTE_R_N)));
|
2013-06-20 17:00:13 +08:00
|
|
|
/*
|
|
|
|
|
* Ensure it is out of the tlb too. Bolted entries base and
|
|
|
|
|
* actual page size will be same.
|
|
|
|
|
*/
|
|
|
|
|
tlbie(vpn, psize, psize, ssize, 0);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2017-06-28 09:32:35 +08:00
|
|
|
/*
|
|
|
|
|
* Remove a bolted kernel entry. Memory hotplug uses this.
|
|
|
|
|
*
|
|
|
|
|
* No need to lock here because we should be the only user.
|
|
|
|
|
*/
|
|
|
|
|
static int native_hpte_removebolted(unsigned long ea, int psize, int ssize)
|
|
|
|
|
{
|
|
|
|
|
unsigned long vpn;
|
|
|
|
|
unsigned long vsid;
|
|
|
|
|
long slot;
|
|
|
|
|
struct hash_pte *hptep;
|
|
|
|
|
|
|
|
|
|
vsid = get_kernel_vsid(ea, ssize);
|
|
|
|
|
vpn = hpt_vpn(ea, vsid, ssize);
|
|
|
|
|
|
|
|
|
|
slot = native_hpte_find(vpn, psize, ssize);
|
|
|
|
|
if (slot == -1)
|
|
|
|
|
return -ENOENT;
|
|
|
|
|
|
|
|
|
|
hptep = htab_address + slot;
|
|
|
|
|
|
|
|
|
|
VM_WARN_ON(!(be64_to_cpu(hptep->v) & HPTE_V_BOLTED));
|
|
|
|
|
|
|
|
|
|
/* Invalidate the hpte */
|
|
|
|
|
hptep->v = 0;
|
|
|
|
|
|
|
|
|
|
/* Invalidate the TLB */
|
|
|
|
|
tlbie(vpn, psize, psize, ssize, 0);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
|
2013-06-20 17:00:13 +08:00
|
|
|
int bpsize, int apsize, int ssize, int local)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2007-06-13 12:52:56 +08:00
|
|
|
struct hash_pte *hptep = htab_address + slot;
|
2005-07-13 16:11:42 +08:00
|
|
|
unsigned long hpte_v;
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned long want_v;
|
2005-04-17 06:20:36 +08:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);
|
2005-11-07 08:06:55 +08:00
|
|
|
|
2013-06-20 17:00:13 +08:00
|
|
|
want_v = hpte_encode_avpn(vpn, bpsize, ssize);
|
2018-06-29 16:36:30 +08:00
|
|
|
hpte_v = hpte_get_old_v(hptep);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
2018-06-29 16:36:31 +08:00
|
|
|
if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
|
|
|
|
|
native_lock_hpte(hptep);
|
|
|
|
|
/* recheck with locks held */
|
|
|
|
|
hpte_v = hpte_get_old_v(hptep);
|
|
|
|
|
|
|
|
|
|
if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
|
|
|
|
|
/* Invalidate the hpte. NOTE: this also unlocks it */
|
|
|
|
|
hptep->v = 0;
|
|
|
|
|
else
|
|
|
|
|
native_unlock_hpte(hptep);
|
|
|
|
|
}
|
2013-05-31 09:03:24 +08:00
|
|
|
/*
|
|
|
|
|
* We need to invalidate the TLB always because hpte_remove doesn't do
|
|
|
|
|
* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
|
|
|
|
|
* random entry from it. When we do that we don't invalidate the TLB
|
|
|
|
|
* (hpte_remove) because we assume the old translation is still
|
|
|
|
|
* technically "valid".
|
|
|
|
|
*/
|
2013-06-20 17:00:13 +08:00
|
|
|
tlbie(vpn, bpsize, apsize, ssize, local);
|
|
|
|
|
|
2005-04-17 06:20:36 +08:00
|
|
|
local_irq_restore(flags);
|
|
|
|
|
}
|
|
|
|
|
|
2015-12-01 11:36:53 +08:00
|
|
|
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
2014-08-13 15:01:58 +08:00
|
|
|
static void native_hugepage_invalidate(unsigned long vsid,
|
|
|
|
|
unsigned long addr,
|
2013-06-20 17:00:27 +08:00
|
|
|
unsigned char *hpte_slot_array,
|
2014-11-02 23:45:28 +08:00
|
|
|
int psize, int ssize, int local)
|
2013-06-20 17:00:27 +08:00
|
|
|
{
|
2014-08-13 15:02:01 +08:00
|
|
|
int i;
|
2013-06-20 17:00:27 +08:00
|
|
|
struct hash_pte *hptep;
|
|
|
|
|
int actual_psize = MMU_PAGE_16M;
|
|
|
|
|
unsigned int max_hpte_count, valid;
|
|
|
|
|
unsigned long flags, s_addr = addr;
|
|
|
|
|
unsigned long hpte_v, want_v, shift;
|
2014-08-13 15:01:58 +08:00
|
|
|
unsigned long hidx, vpn = 0, hash, slot;
|
2013-06-20 17:00:27 +08:00
|
|
|
|
|
|
|
|
shift = mmu_psize_defs[psize].shift;
|
|
|
|
|
max_hpte_count = 1U << (PMD_SHIFT - shift);
|
|
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
for (i = 0; i < max_hpte_count; i++) {
|
|
|
|
|
valid = hpte_valid(hpte_slot_array, i);
|
|
|
|
|
if (!valid)
|
|
|
|
|
continue;
|
|
|
|
|
hidx = hpte_hash_index(hpte_slot_array, i);
|
|
|
|
|
|
|
|
|
|
/* get the vpn */
|
|
|
|
|
addr = s_addr + (i * (1ul << shift));
|
|
|
|
|
vpn = hpt_vpn(addr, vsid, ssize);
|
|
|
|
|
hash = hpt_hash(vpn, shift, ssize);
|
|
|
|
|
if (hidx & _PTEIDX_SECONDARY)
|
|
|
|
|
hash = ~hash;
|
|
|
|
|
|
|
|
|
|
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
|
|
|
|
|
slot += hidx & _PTEIDX_GROUP_IX;
|
|
|
|
|
|
|
|
|
|
hptep = htab_address + slot;
|
|
|
|
|
want_v = hpte_encode_avpn(vpn, psize, ssize);
|
2018-06-29 16:36:30 +08:00
|
|
|
hpte_v = hpte_get_old_v(hptep);
|
2013-06-20 17:00:27 +08:00
|
|
|
|
|
|
|
|
/* Even if we miss, we need to invalidate the TLB */
|
2018-06-29 16:36:31 +08:00
|
|
|
if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
|
|
|
|
|
/* recheck with locks held */
|
|
|
|
|
native_lock_hpte(hptep);
|
|
|
|
|
hpte_v = hpte_get_old_v(hptep);
|
|
|
|
|
|
|
|
|
|
if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) {
|
|
|
|
|
/*
|
|
|
|
|
* Invalidate the hpte. NOTE: this also unlocks it
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
hptep->v = 0;
|
|
|
|
|
} else
|
|
|
|
|
native_unlock_hpte(hptep);
|
|
|
|
|
}
|
2014-08-13 15:02:01 +08:00
|
|
|
/*
|
|
|
|
|
* We need to do tlb invalidate for all the address, tlbie
|
|
|
|
|
* instruction compares entry_VA in tlb with the VA specified
|
|
|
|
|
* here
|
|
|
|
|
*/
|
2014-11-02 23:45:28 +08:00
|
|
|
tlbie(vpn, psize, actual_psize, ssize, local);
|
2013-06-20 17:00:27 +08:00
|
|
|
}
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
}
|
2015-12-01 11:36:53 +08:00
|
|
|
#else
|
|
|
|
|
static void native_hugepage_invalidate(unsigned long vsid,
|
|
|
|
|
unsigned long addr,
|
|
|
|
|
unsigned char *hpte_slot_array,
|
|
|
|
|
int psize, int ssize, int local)
|
|
|
|
|
{
|
|
|
|
|
WARN(1, "%s called without THP support\n", __func__);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2013-06-20 17:00:27 +08:00
|
|
|
|
2007-06-13 12:52:56 +08:00
|
|
|
static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
|
2013-04-28 17:37:35 +08:00
|
|
|
int *psize, int *apsize, int *ssize, unsigned long *vpn)
|
2007-05-02 22:19:11 +08:00
|
|
|
{
|
2012-09-10 10:52:49 +08:00
|
|
|
unsigned long avpn, pteg, vpi;
|
2013-09-23 10:04:36 +08:00
|
|
|
unsigned long hpte_v = be64_to_cpu(hpte->v);
|
|
|
|
|
unsigned long hpte_r = be64_to_cpu(hpte->r);
|
2012-09-10 10:52:49 +08:00
|
|
|
unsigned long vsid, seg_off;
|
2013-04-28 17:37:36 +08:00
|
|
|
int size, a_size, shift;
|
|
|
|
|
/* Look at the 8 bit LP value */
|
2013-09-23 10:04:36 +08:00
|
|
|
unsigned int lp = (hpte_r >> LP_SHIFT) & ((1 << LP_BITS) - 1);
|
2007-05-02 22:19:11 +08:00
|
|
|
|
2016-11-11 13:55:03 +08:00
|
|
|
if (cpu_has_feature(CPU_FTR_ARCH_300)) {
|
|
|
|
|
hpte_v = hpte_new_to_old_v(hpte_v, hpte_r);
|
|
|
|
|
hpte_r = hpte_new_to_old_r(hpte_r);
|
|
|
|
|
}
|
2013-04-28 17:37:35 +08:00
|
|
|
if (!(hpte_v & HPTE_V_LARGE)) {
|
|
|
|
|
size = MMU_PAGE_4K;
|
|
|
|
|
a_size = MMU_PAGE_4K;
|
|
|
|
|
} else {
|
2016-09-02 15:20:43 +08:00
|
|
|
size = hpte_page_sizes[lp] & 0xf;
|
|
|
|
|
a_size = hpte_page_sizes[lp] >> 4;
|
2007-05-02 22:19:11 +08:00
|
|
|
}
|
2007-05-10 13:28:44 +08:00
|
|
|
/* This works for all page sizes, and for 256M and 1T segments */
|
2016-11-11 13:55:03 +08:00
|
|
|
*ssize = hpte_v >> HPTE_V_SSIZE_SHIFT;
|
2007-05-02 22:19:11 +08:00
|
|
|
shift = mmu_psize_defs[size].shift;
|
|
|
|
|
|
2012-09-10 10:52:49 +08:00
|
|
|
avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm);
|
|
|
|
|
pteg = slot / HPTES_PER_GROUP;
|
|
|
|
|
if (hpte_v & HPTE_V_SECONDARY)
|
|
|
|
|
pteg = ~pteg;
|
|
|
|
|
|
|
|
|
|
switch (*ssize) {
|
|
|
|
|
case MMU_SEGSIZE_256M:
|
|
|
|
|
/* We only have 28 - 23 bits of seg_off in avpn */
|
|
|
|
|
seg_off = (avpn & 0x1f) << 23;
|
|
|
|
|
vsid = avpn >> 5;
|
|
|
|
|
/* We can find more bits from the pteg value */
|
|
|
|
|
if (shift < 23) {
|
|
|
|
|
vpi = (vsid ^ pteg) & htab_hash_mask;
|
|
|
|
|
seg_off |= vpi << shift;
|
|
|
|
|
}
|
2012-09-10 10:52:50 +08:00
|
|
|
*vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT;
|
2013-07-03 16:20:03 +08:00
|
|
|
break;
|
2012-09-10 10:52:49 +08:00
|
|
|
case MMU_SEGSIZE_1T:
|
|
|
|
|
/* We only have 40 - 23 bits of seg_off in avpn */
|
|
|
|
|
seg_off = (avpn & 0x1ffff) << 23;
|
|
|
|
|
vsid = avpn >> 17;
|
|
|
|
|
if (shift < 23) {
|
2007-05-10 13:28:44 +08:00
|
|
|
vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask;
|
2012-09-10 10:52:49 +08:00
|
|
|
seg_off |= vpi << shift;
|
2007-05-02 22:19:11 +08:00
|
|
|
}
|
2012-09-10 10:52:50 +08:00
|
|
|
*vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT;
|
2013-07-03 16:20:03 +08:00
|
|
|
break;
|
2012-09-10 10:52:49 +08:00
|
|
|
default:
|
2012-09-10 10:52:50 +08:00
|
|
|
*vpn = size = 0;
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
2013-04-28 17:37:35 +08:00
|
|
|
*psize = size;
|
|
|
|
|
*apsize = a_size;
|
2005-11-07 08:06:55 +08:00
|
|
|
}
|
|
|
|
|
|
2005-06-26 05:58:08 +08:00
|
|
|
/*
|
|
|
|
|
* clear all mappings on kexec. All cpus are in real mode (or they will
|
|
|
|
|
* be when they isi), and we are the only one left. We rely on our kernel
|
|
|
|
|
* mapping being 0xC0's and the hardware ignoring those two real bits.
|
|
|
|
|
*
|
2015-10-08 08:04:26 +08:00
|
|
|
* This must be called with interrupts disabled.
|
|
|
|
|
*
|
|
|
|
|
* Taking the native_tlbie_lock is unsafe here due to the possibility of
|
|
|
|
|
* lockdep being on. On pre POWER5 hardware, not taking the lock could
|
|
|
|
|
* cause deadlock. POWER5 and newer not taking the lock is fine. This only
|
|
|
|
|
* gets called during boot before secondary CPUs have come up and during
|
|
|
|
|
* crashdump and all bets are off anyway.
|
|
|
|
|
*
|
2005-06-26 05:58:08 +08:00
|
|
|
* TODO: add batching support when enabled. remember, no dynamic memory here,
|
2016-06-01 14:34:37 +08:00
|
|
|
* although there is the control page available...
|
2005-06-26 05:58:08 +08:00
|
|
|
*/
|
|
|
|
|
static void native_hpte_clear(void)
|
|
|
|
|
{
|
2012-09-10 10:52:50 +08:00
|
|
|
unsigned long vpn = 0;
|
2015-10-08 08:04:26 +08:00
|
|
|
unsigned long slot, slots;
|
2007-06-13 12:52:56 +08:00
|
|
|
struct hash_pte *hptep = htab_address;
|
2012-09-10 10:52:50 +08:00
|
|
|
unsigned long hpte_v;
|
2005-06-26 05:58:08 +08:00
|
|
|
unsigned long pteg_count;
|
2013-04-28 17:37:35 +08:00
|
|
|
int psize, apsize, ssize;
|
2005-06-26 05:58:08 +08:00
|
|
|
|
|
|
|
|
pteg_count = htab_hash_mask + 1;
|
|
|
|
|
|
|
|
|
|
slots = pteg_count * HPTES_PER_GROUP;
|
|
|
|
|
|
|
|
|
|
for (slot = 0; slot < slots; slot++, hptep++) {
|
|
|
|
|
/*
|
|
|
|
|
* we could lock the pte here, but we are the only cpu
|
|
|
|
|
* running, right? and for crash dump, we probably
|
|
|
|
|
* don't want to wait for a maybe bad cpu.
|
|
|
|
|
*/
|
2013-09-23 10:04:36 +08:00
|
|
|
hpte_v = be64_to_cpu(hptep->v);
|
2005-06-26 05:58:08 +08:00
|
|
|
|
2006-02-23 00:13:08 +08:00
|
|
|
/*
|
2015-10-08 08:04:26 +08:00
|
|
|
* Call __tlbie() here rather than tlbie() since we can't take the
|
|
|
|
|
* native_tlbie_lock.
|
2006-02-23 00:13:08 +08:00
|
|
|
*/
|
2005-07-13 16:11:42 +08:00
|
|
|
if (hpte_v & HPTE_V_VALID) {
|
2013-04-28 17:37:35 +08:00
|
|
|
hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn);
|
2005-07-13 16:11:42 +08:00
|
|
|
hptep->v = 0;
|
2017-11-23 01:32:07 +08:00
|
|
|
___tlbie(vpn, psize, apsize, ssize);
|
2005-06-26 05:58:08 +08:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2006-02-23 00:13:08 +08:00
|
|
|
asm volatile("eieio; tlbsync; ptesync":::"memory");
|
2005-06-26 05:58:08 +08:00
|
|
|
}
|
|
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
/*
|
|
|
|
|
* Batched hash table flush, we batch the tlbie's to avoid taking/releasing
|
|
|
|
|
* the lock all the time
|
|
|
|
|
*/
|
2005-09-20 11:52:50 +08:00
|
|
|
static void native_flush_hash_range(unsigned long number, int local)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2018-03-23 12:56:27 +08:00
|
|
|
unsigned long vpn = 0;
|
2012-09-10 10:52:50 +08:00
|
|
|
unsigned long hash, index, hidx, shift, slot;
|
2007-06-13 12:52:56 +08:00
|
|
|
struct hash_pte *hptep;
|
2005-07-13 16:11:42 +08:00
|
|
|
unsigned long hpte_v;
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned long want_v;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
real_pte_t pte;
|
powerpc: Replace __get_cpu_var uses
This still has not been merged and now powerpc is the only arch that does
not have this change. Sorry about missing linuxppc-dev before.
V2->V2
- Fix up to work against 3.18-rc1
__get_cpu_var() is used for multiple purposes in the kernel source. One of
them is address calculation via the form &__get_cpu_var(x). This calculates
the address for the instance of the percpu variable of the current processor
based on an offset.
Other use cases are for storing and retrieving data from the current
processors percpu area. __get_cpu_var() can be used as an lvalue when
writing data or on the right side of an assignment.
__get_cpu_var() is defined as :
__get_cpu_var() always only does an address determination. However, store
and retrieve operations could use a segment prefix (or global register on
other platforms) to avoid the address calculation.
this_cpu_write() and this_cpu_read() can directly take an offset into a
percpu area and use optimized assembly code to read and write per cpu
variables.
This patch converts __get_cpu_var into either an explicit address
calculation using this_cpu_ptr() or into a use of this_cpu operations that
use the offset. Thereby address calculations are avoided and less registers
are used when code is generated.
At the end of the patch set all uses of __get_cpu_var have been removed so
the macro is removed too.
The patch set includes passes over all arches as well. Once these operations
are used throughout then specialized macros can be defined in non -x86
arches as well in order to optimize per cpu access by f.e. using a global
register that may be set to the per cpu base.
Transformations done to __get_cpu_var()
1. Determine the address of the percpu instance of the current processor.
DEFINE_PER_CPU(int, y);
int *x = &__get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(&y);
2. Same as #1 but this time an array structure is involved.
DEFINE_PER_CPU(int, y[20]);
int *x = __get_cpu_var(y);
Converts to
int *x = this_cpu_ptr(y);
3. Retrieve the content of the current processors instance of a per cpu
variable.
DEFINE_PER_CPU(int, y);
int x = __get_cpu_var(y)
Converts to
int x = __this_cpu_read(y);
4. Retrieve the content of a percpu struct
DEFINE_PER_CPU(struct mystruct, y);
struct mystruct x = __get_cpu_var(y);
Converts to
memcpy(&x, this_cpu_ptr(&y), sizeof(x));
5. Assignment to a per cpu variable
DEFINE_PER_CPU(int, y)
__get_cpu_var(y) = x;
Converts to
__this_cpu_write(y, x);
6. Increment/Decrement etc of a per cpu variable
DEFINE_PER_CPU(int, y);
__get_cpu_var(y)++
Converts to
__this_cpu_inc(y)
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
CC: Paul Mackerras <paulus@samba.org>
Signed-off-by: Christoph Lameter <cl@linux.com>
[mpe: Fix build errors caused by set/or_softirq_pending(), and rework
assignment in __set_breakpoint() to use memcpy().]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
2014-10-22 04:23:25 +08:00
|
|
|
struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch);
|
2005-11-07 08:06:55 +08:00
|
|
|
unsigned long psize = batch->psize;
|
2007-10-11 18:37:10 +08:00
|
|
|
int ssize = batch->ssize;
|
2005-11-07 08:06:55 +08:00
|
|
|
int i;
|
2017-03-30 01:19:42 +08:00
|
|
|
unsigned int use_local;
|
|
|
|
|
|
|
|
|
|
use_local = local && mmu_has_feature(MMU_FTR_TLBIEL) &&
|
|
|
|
|
mmu_psize_defs[psize].tlbiel && !cxl_ctx_in_use();
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < number; i++) {
|
2012-09-10 10:52:50 +08:00
|
|
|
vpn = batch->vpn[i];
|
2005-11-07 08:06:55 +08:00
|
|
|
pte = batch->pte[i];
|
|
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
|
|
|
|
|
hash = hpt_hash(vpn, shift, ssize);
|
2005-11-07 08:06:55 +08:00
|
|
|
hidx = __rpte_to_hidx(pte, index);
|
|
|
|
|
if (hidx & _PTEIDX_SECONDARY)
|
|
|
|
|
hash = ~hash;
|
|
|
|
|
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
|
|
|
|
|
slot += hidx & _PTEIDX_GROUP_IX;
|
|
|
|
|
hptep = htab_address + slot;
|
2013-04-28 17:37:34 +08:00
|
|
|
want_v = hpte_encode_avpn(vpn, psize, ssize);
|
2018-06-29 16:36:31 +08:00
|
|
|
hpte_v = hpte_get_old_v(hptep);
|
|
|
|
|
|
|
|
|
|
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
|
|
|
|
|
continue;
|
|
|
|
|
/* lock and try again */
|
2005-11-07 08:06:55 +08:00
|
|
|
native_lock_hpte(hptep);
|
2018-06-29 16:36:30 +08:00
|
|
|
hpte_v = hpte_get_old_v(hptep);
|
2018-06-29 16:36:31 +08:00
|
|
|
|
|
|
|
|
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
|
2005-11-07 08:06:55 +08:00
|
|
|
native_unlock_hpte(hptep);
|
|
|
|
|
else
|
|
|
|
|
hptep->v = 0;
|
2018-06-29 16:36:31 +08:00
|
|
|
|
2005-11-07 08:06:55 +08:00
|
|
|
} pte_iterate_hashed_end();
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
2017-03-30 01:19:42 +08:00
|
|
|
if (use_local) {
|
2005-04-17 06:20:36 +08:00
|
|
|
asm volatile("ptesync":::"memory");
|
2005-11-07 08:06:55 +08:00
|
|
|
for (i = 0; i < number; i++) {
|
2012-09-10 10:52:50 +08:00
|
|
|
vpn = batch->vpn[i];
|
2005-11-07 08:06:55 +08:00
|
|
|
pte = batch->pte[i];
|
|
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
pte_iterate_hashed_subpages(pte, psize,
|
|
|
|
|
vpn, index, shift) {
|
2013-04-28 17:37:35 +08:00
|
|
|
__tlbiel(vpn, psize, psize, ssize);
|
2005-11-07 08:06:55 +08:00
|
|
|
} pte_iterate_hashed_end();
|
|
|
|
|
}
|
2005-04-17 06:20:36 +08:00
|
|
|
asm volatile("ptesync":::"memory");
|
|
|
|
|
} else {
|
2011-04-07 03:48:50 +08:00
|
|
|
int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
if (lock_tlbie)
|
2010-02-18 10:22:35 +08:00
|
|
|
raw_spin_lock(&native_tlbie_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
|
|
|
|
|
asm volatile("ptesync":::"memory");
|
2005-11-07 08:06:55 +08:00
|
|
|
for (i = 0; i < number; i++) {
|
2012-09-10 10:52:50 +08:00
|
|
|
vpn = batch->vpn[i];
|
2005-11-07 08:06:55 +08:00
|
|
|
pte = batch->pte[i];
|
|
|
|
|
|
2012-09-10 10:52:50 +08:00
|
|
|
pte_iterate_hashed_subpages(pte, psize,
|
|
|
|
|
vpn, index, shift) {
|
2013-04-28 17:37:35 +08:00
|
|
|
__tlbie(vpn, psize, psize, ssize);
|
2005-11-07 08:06:55 +08:00
|
|
|
} pte_iterate_hashed_end();
|
|
|
|
|
}
|
2018-03-23 12:56:27 +08:00
|
|
|
/*
|
|
|
|
|
* Just do one more with the last used values.
|
|
|
|
|
*/
|
|
|
|
|
fixup_tlbie(vpn, psize, psize, ssize);
|
2005-04-17 06:20:36 +08:00
|
|
|
asm volatile("eieio; tlbsync; ptesync":::"memory");
|
|
|
|
|
|
|
|
|
|
if (lock_tlbie)
|
2010-02-18 10:22:35 +08:00
|
|
|
raw_spin_unlock(&native_tlbie_lock);
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
}
|
|
|
|
|
|
2006-06-23 16:16:38 +08:00
|
|
|
void __init hpte_init_native(void)
|
2005-04-17 06:20:36 +08:00
|
|
|
{
|
2016-07-05 13:03:58 +08:00
|
|
|
mmu_hash_ops.hpte_invalidate = native_hpte_invalidate;
|
|
|
|
|
mmu_hash_ops.hpte_updatepp = native_hpte_updatepp;
|
|
|
|
|
mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp;
|
2017-06-28 09:32:35 +08:00
|
|
|
mmu_hash_ops.hpte_removebolted = native_hpte_removebolted;
|
2016-07-05 13:03:58 +08:00
|
|
|
mmu_hash_ops.hpte_insert = native_hpte_insert;
|
|
|
|
|
mmu_hash_ops.hpte_remove = native_hpte_remove;
|
|
|
|
|
mmu_hash_ops.hpte_clear_all = native_hpte_clear;
|
|
|
|
|
mmu_hash_ops.flush_hash_range = native_flush_hash_range;
|
|
|
|
|
mmu_hash_ops.hugepage_invalidate = native_hugepage_invalidate;
|
2005-04-17 06:20:36 +08:00
|
|
|
}
|