openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

ARM: Better virt_to_page() handling 

virt_to_page() is incredibly inefficient when virt-to-phys patching is
enabled.  This is because we end up with this calculation:

  page = &mem_map[asm virt_to_phys(addr) >> 12 - __pv_phys_offset >> 12]

in assembly.  The asm virt_to_phys() is equivalent this this operation:

  addr - PAGE_OFFSET + __pv_phys_offset

and we can see that because this is assembly, the compiler has no chance
to optimise some of that away.  This should reduce down to:

  page = &mem_map[(addr - PAGE_OFFSET) >> 12]

for the common cases.  Permit the compiler to make this optimisation by
giving it more of the information it needs - do this by providing a
virt_to_pfn() macro.

Another issue which makes this more complex is that __pv_phys_offset is
a 64-bit type on all platforms.  This is needlessly wasteful - if we
store the physical offset as a PFN, we can save a lot of work having
to deal with 64-bit values, which sometimes ends up producing incredibly
horrid code:

     a4c:       e3009000        movw    r9, #0
                        a4c: R_ARM_MOVW_ABS_NC  __pv_phys_offset
     a50:       e3409000        movt    r9, #0          ; r9 = &__pv_phys_offset
                        a50: R_ARM_MOVT_ABS     __pv_phys_offset
     a54:       e3002000        movw    r2, #0
                        a54: R_ARM_MOVW_ABS_NC  __pv_phys_offset
     a58:       e3402000        movt    r2, #0          ; r2 = &__pv_phys_offset
                        a58: R_ARM_MOVT_ABS     __pv_phys_offset
     a5c:       e5999004        ldr     r9, [r9, #4]    ; r9 = high word of __pv_phys_offset
     a60:       e3001000        movw    r1, #0
                        a60: R_ARM_MOVW_ABS_NC  mem_map
     a64:       e592c000        ldr     ip, [r2]        ; ip = low word of __pv_phys_offset

Reviewed-by: Nicolas Pitre <nico@linaro.org>
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
This commit is contained in:
Russell King 2014-03-25 19:45:31 +00:00
parent 83b3f64d46
commit e26a9e00af
3 changed files with 34 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
arch/arm/include/asm/memory.h
View File

@ -169,9 +169,17 @@
* Physical vs virtual RAM address space conversion. These are * Physical vs virtual RAM address space conversion. These are
* private definitions which should NOT be used outside memory.h * private definitions which should NOT be used outside memory.h
* files. Use virt_to_phys/phys_to_virt/__pa/__va instead. * files. Use virt_to_phys/phys_to_virt/__pa/__va instead.
*
* PFNs are used to describe any physical page; this means
* PFN 0 == physical address 0.
*/ */
#ifndef __virt_to_phys #if defined(__virt_to_phys)
#ifdef CONFIG_ARM_PATCH_PHYS_VIRT #define PHYS_OFFSET PLAT_PHYS_OFFSET
#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT))
#define virt_to_pfn(kaddr) (__pa(kaddr) >> PAGE_SHIFT)
#elif defined(CONFIG_ARM_PATCH_PHYS_VIRT)
/* /*
* Constants used to force the right instruction encodings and shifts * Constants used to force the right instruction encodings and shifts
@ -180,12 +188,17 @@
#define __PV_BITS_31_24 0x81000000 #define __PV_BITS_31_24 0x81000000
#define __PV_BITS_7_0 0x81 #define __PV_BITS_7_0 0x81
extern u64 __pv_phys_offset; extern unsigned long __pv_phys_pfn_offset;
extern u64 __pv_offset; extern u64 __pv_offset;
extern void fixup_pv_table(const void *, unsigned long); extern void fixup_pv_table(const void *, unsigned long);
extern const void *__pv_table_begin, *__pv_table_end; extern const void *__pv_table_begin, *__pv_table_end;
#define PHYS_OFFSET __pv_phys_offset #define PHYS_OFFSET ((phys_addr_t)__pv_phys_pfn_offset << PAGE_SHIFT)
#define PHYS_PFN_OFFSET (__pv_phys_pfn_offset)
#define virt_to_pfn(kaddr) \
((((unsigned long)(kaddr) - PAGE_OFFSET) >> PAGE_SHIFT) + \
PHYS_PFN_OFFSET)
#define __pv_stub(from,to,instr,type) \ #define __pv_stub(from,to,instr,type) \
__asm__("@ __pv_stub\n" \ __asm__("@ __pv_stub\n" \
@ -246,6 +259,7 @@ static inline unsigned long __phys_to_virt(phys_addr_t x)
#else #else
#define PHYS_OFFSET PLAT_PHYS_OFFSET #define PHYS_OFFSET PLAT_PHYS_OFFSET
#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT))
static inline phys_addr_t __virt_to_phys(unsigned long x) static inline phys_addr_t __virt_to_phys(unsigned long x)
{ {
@ -257,18 +271,11 @@ static inline unsigned long __phys_to_virt(phys_addr_t x)
return x - PHYS_OFFSET + PAGE_OFFSET; return x - PHYS_OFFSET + PAGE_OFFSET;
} }
#endif #define virt_to_pfn(kaddr) \
#endif ((((unsigned long)(kaddr) - PAGE_OFFSET) >> PAGE_SHIFT) + \
PHYS_PFN_OFFSET)
/* #endif
* PFNs are used to describe any physical page; this means
* PFN 0 == physical address 0.
*
* This is the PFN of the first RAM page in the kernel
* direct-mapped view. We assume this is the first page
* of RAM in the mem_map as well.
*/
#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT))
/* /*
* These are *only* valid on the kernel direct mapped RAM memory. * These are *only* valid on the kernel direct mapped RAM memory.
@ -346,9 +353,9 @@ static inline __deprecated void *bus_to_virt(unsigned long x)
*/ */
#define ARCH_PFN_OFFSET PHYS_PFN_OFFSET #define ARCH_PFN_OFFSET PHYS_PFN_OFFSET
#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) #define virt_to_page(kaddr) pfn_to_page(virt_to_pfn(kaddr))
#define virt_addr_valid(kaddr) (((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory) \ #define virt_addr_valid(kaddr) (((unsigned long)(kaddr) >= PAGE_OFFSET && (unsigned long)(kaddr) < (unsigned long)high_memory) \
&& pfn_valid(__pa(kaddr) >> PAGE_SHIFT) ) && pfn_valid(virt_to_pfn(kaddr)))
#endif #endif

2
arch/arm/kernel/armksyms.c
View File

@ -158,6 +158,6 @@ EXPORT_SYMBOL(__gnu_mcount_nc);
#endif #endif
#ifdef CONFIG_ARM_PATCH_PHYS_VIRT #ifdef CONFIG_ARM_PATCH_PHYS_VIRT
EXPORT_SYMBOL(__pv_phys_offset); EXPORT_SYMBOL(__pv_phys_pfn_offset);
EXPORT_SYMBOL(__pv_offset); EXPORT_SYMBOL(__pv_offset);
#endif #endif

17
arch/arm/kernel/head.S
View File

@ -584,9 +584,10 @@ __fixup_pv_table:
subs r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET subs r3, r0, r3 @ PHYS_OFFSET - PAGE_OFFSET
add r4, r4, r3 @ adjust table start address add r4, r4, r3 @ adjust table start address
add r5, r5, r3 @ adjust table end address add r5, r5, r3 @ adjust table end address
add r6, r6, r3 @ adjust __pv_phys_offset address add r6, r6, r3 @ adjust __pv_phys_pfn_offset address
add r7, r7, r3 @ adjust __pv_offset address add r7, r7, r3 @ adjust __pv_offset address
str r8, [r6, #LOW_OFFSET] @ save computed PHYS_OFFSET to __pv_phys_offset mov r0, r8, lsr #12 @ convert to PFN
str r0, [r6, #LOW_OFFSET] @ save computed PHYS_OFFSET to __pv_phys_pfn_offset
strcc ip, [r7, #HIGH_OFFSET] @ save to __pv_offset high bits strcc ip, [r7, #HIGH_OFFSET] @ save to __pv_offset high bits
mov r6, r3, lsr #24 @ constant for add/sub instructions mov r6, r3, lsr #24 @ constant for add/sub instructions
teq r3, r6, lsl #24 @ must be 16MiB aligned teq r3, r6, lsl #24 @ must be 16MiB aligned
@ -600,7 +601,7 @@ ENDPROC(__fixup_pv_table)
1: .long . 1: .long .
.long __pv_table_begin .long __pv_table_begin
.long __pv_table_end .long __pv_table_end
2: .long __pv_phys_offset 2: .long __pv_phys_pfn_offset
.long __pv_offset .long __pv_offset
.text .text
@ -688,11 +689,11 @@ ENTRY(fixup_pv_table)
ENDPROC(fixup_pv_table) ENDPROC(fixup_pv_table)
.data .data
.globl __pv_phys_offset .globl __pv_phys_pfn_offset
.type __pv_phys_offset, %object .type __pv_phys_pfn_offset, %object
__pv_phys_offset: __pv_phys_pfn_offset:
.quad 0 .word 0
.size __pv_phys_offset, . -__pv_phys_offset .size __pv_phys_pfn_offset, . -__pv_phys_pfn_offset
.globl __pv_offset .globl __pv_offset
.type __pv_offset, %object .type __pv_offset, %object