mirror of https://gitee.com/openkylin/linux.git
percpu, x86: Generic inc / dec percpu instructions
Optimize code generated for percpu access by checking for increment and
decrements.
tj: fix incorrect usage of __builtin_constant_p() and restructure
percpu_add_op() macro.
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
Christoph Lameter
Tejun Heo
parent
79615760f3
commit
5917dae83c
|
|
@ -103,6 +103,64 @@ do { \
|
|||
} \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* Generate a percpu add to memory instruction and optimize code
|
||||
* if a one is added or subtracted.
|
||||
*/
|
||||
#define percpu_add_op(var, val) \
|
||||
do { \
|
||||
typedef typeof(var) pao_T__; \
|
||||
const int pao_ID__ = (__builtin_constant_p(val) && \
|
||||
((val) == 1 || (val) == -1)) ? (val) : 0; \
|
||||
if (0) { \
|
||||
pao_T__ pao_tmp__; \
|
||||
pao_tmp__ = (val); \
|
||||
} \
|
||||
switch (sizeof(var)) { \
|
||||
case 1: \
|
||||
if (pao_ID__ == 1) \
|
||||
asm("incb "__percpu_arg(0) : "+m" (var)); \
|
||||
else if (pao_ID__ == -1) \
|
||||
asm("decb "__percpu_arg(0) : "+m" (var)); \
|
||||
else \
|
||||
asm("addb %1, "__percpu_arg(0) \
|
||||
: "+m" (var) \
|
||||
: "qi" ((pao_T__)(val))); \
|
||||
break; \
|
||||
case 2: \
|
||||
if (pao_ID__ == 1) \
|
||||
asm("incw "__percpu_arg(0) : "+m" (var)); \
|
||||
else if (pao_ID__ == -1) \
|
||||
asm("decw "__percpu_arg(0) : "+m" (var)); \
|
||||
else \
|
||||
asm("addw %1, "__percpu_arg(0) \
|
||||
: "+m" (var) \
|
||||
: "ri" ((pao_T__)(val))); \
|
||||
break; \
|
||||
case 4: \
|
||||
if (pao_ID__ == 1) \
|
||||
asm("incl "__percpu_arg(0) : "+m" (var)); \
|
||||
else if (pao_ID__ == -1) \
|
||||
asm("decl "__percpu_arg(0) : "+m" (var)); \
|
||||
else \
|
||||
asm("addl %1, "__percpu_arg(0) \
|
||||
: "+m" (var) \
|
||||
: "ri" ((pao_T__)(val))); \
|
||||
break; \
|
||||
case 8: \
|
||||
if (pao_ID__ == 1) \
|
||||
asm("incq "__percpu_arg(0) : "+m" (var)); \
|
||||
else if (pao_ID__ == -1) \
|
||||
asm("decq "__percpu_arg(0) : "+m" (var)); \
|
||||
else \
|
||||
asm("addq %1, "__percpu_arg(0) \
|
||||
: "+m" (var) \
|
||||
: "re" ((pao_T__)(val))); \
|
||||
break; \
|
||||
default: __bad_percpu_size(); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define percpu_from_op(op, var, constraint) \
|
||||
({ \
|
||||
typeof(var) pfo_ret__; \
|
||||
|
|
@ -144,8 +202,8 @@ do { \
|
|||
#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
|
||||
#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
|
||||
#define percpu_write(var, val) percpu_to_op("mov", var, val)
|
||||
#define percpu_add(var, val) percpu_to_op("add", var, val)
|
||||
#define percpu_sub(var, val) percpu_to_op("sub", var, val)
|
||||
#define percpu_add(var, val) percpu_add_op(var, val)
|
||||
#define percpu_sub(var, val) percpu_add_op(var, -(val))
|
||||
#define percpu_and(var, val) percpu_to_op("and", var, val)
|
||||
#define percpu_or(var, val) percpu_to_op("or", var, val)
|
||||
#define percpu_xor(var, val) percpu_to_op("xor", var, val)
|
||||
|
|
@ -157,9 +215,9 @@ do { \
|
|||
#define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
|
||||
#define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
|
||||
#define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
|
||||
#define __this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define __this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define __this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define __this_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
|
||||
#define __this_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
|
||||
#define __this_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
|
||||
#define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
#define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
#define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
|
|
@ -176,9 +234,9 @@ do { \
|
|||
#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
|
||||
#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
|
||||
#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
|
||||
#define this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define this_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
|
||||
#define this_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
|
||||
#define this_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
|
||||
#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
|
|
@ -189,9 +247,9 @@ do { \
|
|||
#define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val)
|
||||
#define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val)
|
||||
|
||||
#define irqsafe_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define irqsafe_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define irqsafe_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define irqsafe_cpu_add_1(pcp, val) percpu_add_op((pcp), val)
|
||||
#define irqsafe_cpu_add_2(pcp, val) percpu_add_op((pcp), val)
|
||||
#define irqsafe_cpu_add_4(pcp, val) percpu_add_op((pcp), val)
|
||||
#define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
#define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
#define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
|
|
@ -209,19 +267,19 @@ do { \
|
|||
#ifdef CONFIG_X86_64
|
||||
#define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
|
||||
#define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
|
||||
#define __this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define __this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
|
||||
#define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
#define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
|
||||
#define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
|
||||
|
||||
#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
|
||||
#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
|
||||
#define this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define this_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
|
||||
#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
|
||||
#define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
|
||||
|
||||
#define irqsafe_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val)
|
||||
#define irqsafe_cpu_add_8(pcp, val) percpu_add_op((pcp), val)
|
||||
#define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
|
||||
#define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
|
||||
#define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
|
||||
|
|
|
|||
Loading…
Reference in New Issue