2019-06-03 13:44:50 +08:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2014-08-27 12:15:30 +08:00
|
|
|
/*
|
|
|
|
|
* BPF JIT compiler for ARM64
|
|
|
|
|
*
|
arm64: bpf: implement bpf_tail_call() helper
Add support for JMP_CALL_X (tail call) introduced by commit 04fd61ab36ec
("bpf: allow bpf programs to tail-call other bpf programs").
bpf_tail_call() arguments:
ctx - context pointer passed to next program
array - pointer to map which type is BPF_MAP_TYPE_PROG_ARRAY
index - index inside array that selects specific program to run
In this implementation arm64 JIT jumps into callee program after prologue,
so callee program reuses the same stack. For tail_call_cnt, we use the
callee-saved R26 (which was already saved/restored but previously unused
by JIT).
With this patch a tail call generates the following code on arm64:
if (index >= array->map.max_entries)
goto out;
34: mov x10, #0x10 // #16
38: ldr w10, [x1,x10]
3c: cmp w2, w10
40: b.ge 0x0000000000000074
if (tail_call_cnt > MAX_TAIL_CALL_CNT)
goto out;
tail_call_cnt++;
44: mov x10, #0x20 // #32
48: cmp x26, x10
4c: b.gt 0x0000000000000074
50: add x26, x26, #0x1
prog = array->ptrs[index];
if (prog == NULL)
goto out;
54: mov x10, #0x68 // #104
58: ldr x10, [x1,x10]
5c: ldr x11, [x10,x2]
60: cbz x11, 0x0000000000000074
goto *(prog->bpf_func + prologue_size);
64: mov x10, #0x20 // #32
68: ldr x10, [x11,x10]
6c: add x10, x10, #0x20
70: br x10
74:
Signed-off-by: Zi Shen Lim <zlim.lnx@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-09 12:18:48 +08:00
|
|
|
* Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
|
2014-08-27 12:15:30 +08:00
|
|
|
*/
|
|
|
|
|
#ifndef _BPF_JIT_H
|
|
|
|
|
#define _BPF_JIT_H
|
|
|
|
|
|
|
|
|
|
#include <asm/insn.h>
|
|
|
|
|
|
|
|
|
|
/* 5-bit Register Operand */
|
|
|
|
|
#define A64_R(x) AARCH64_INSN_REG_##x
|
|
|
|
|
#define A64_FP AARCH64_INSN_REG_FP
|
|
|
|
|
#define A64_LR AARCH64_INSN_REG_LR
|
|
|
|
|
#define A64_ZR AARCH64_INSN_REG_ZR
|
|
|
|
|
#define A64_SP AARCH64_INSN_REG_SP
|
|
|
|
|
|
|
|
|
|
#define A64_VARIANT(sf) \
|
|
|
|
|
((sf) ? AARCH64_INSN_VARIANT_64BIT : AARCH64_INSN_VARIANT_32BIT)
|
|
|
|
|
|
|
|
|
|
/* Compare & branch (immediate) */
|
|
|
|
|
#define A64_COMP_BRANCH(sf, Rt, offset, type) \
|
|
|
|
|
aarch64_insn_gen_comp_branch_imm(0, offset, Rt, A64_VARIANT(sf), \
|
|
|
|
|
AARCH64_INSN_BRANCH_COMP_##type)
|
|
|
|
|
#define A64_CBZ(sf, Rt, imm19) A64_COMP_BRANCH(sf, Rt, (imm19) << 2, ZERO)
|
2015-11-04 14:56:44 +08:00
|
|
|
#define A64_CBNZ(sf, Rt, imm19) A64_COMP_BRANCH(sf, Rt, (imm19) << 2, NONZERO)
|
2014-08-27 12:15:30 +08:00
|
|
|
|
|
|
|
|
/* Conditional branch (immediate) */
|
|
|
|
|
#define A64_COND_BRANCH(cond, offset) \
|
|
|
|
|
aarch64_insn_gen_cond_branch_imm(0, offset, cond)
|
|
|
|
|
#define A64_COND_EQ AARCH64_INSN_COND_EQ /* == */
|
|
|
|
|
#define A64_COND_NE AARCH64_INSN_COND_NE /* != */
|
|
|
|
|
#define A64_COND_CS AARCH64_INSN_COND_CS /* unsigned >= */
|
|
|
|
|
#define A64_COND_HI AARCH64_INSN_COND_HI /* unsigned > */
|
2017-08-10 07:39:57 +08:00
|
|
|
#define A64_COND_LS AARCH64_INSN_COND_LS /* unsigned <= */
|
|
|
|
|
#define A64_COND_CC AARCH64_INSN_COND_CC /* unsigned < */
|
2014-08-27 12:15:30 +08:00
|
|
|
#define A64_COND_GE AARCH64_INSN_COND_GE /* signed >= */
|
|
|
|
|
#define A64_COND_GT AARCH64_INSN_COND_GT /* signed > */
|
2017-08-10 07:39:57 +08:00
|
|
|
#define A64_COND_LE AARCH64_INSN_COND_LE /* signed <= */
|
|
|
|
|
#define A64_COND_LT AARCH64_INSN_COND_LT /* signed < */
|
2014-08-27 12:15:30 +08:00
|
|
|
#define A64_B_(cond, imm19) A64_COND_BRANCH(cond, (imm19) << 2)
|
|
|
|
|
|
|
|
|
|
/* Unconditional branch (immediate) */
|
|
|
|
|
#define A64_BRANCH(offset, type) aarch64_insn_gen_branch_imm(0, offset, \
|
|
|
|
|
AARCH64_INSN_BRANCH_##type)
|
|
|
|
|
#define A64_B(imm26) A64_BRANCH((imm26) << 2, NOLINK)
|
|
|
|
|
#define A64_BL(imm26) A64_BRANCH((imm26) << 2, LINK)
|
|
|
|
|
|
|
|
|
|
/* Unconditional branch (register) */
|
arm64: bpf: implement bpf_tail_call() helper
Add support for JMP_CALL_X (tail call) introduced by commit 04fd61ab36ec
("bpf: allow bpf programs to tail-call other bpf programs").
bpf_tail_call() arguments:
ctx - context pointer passed to next program
array - pointer to map which type is BPF_MAP_TYPE_PROG_ARRAY
index - index inside array that selects specific program to run
In this implementation arm64 JIT jumps into callee program after prologue,
so callee program reuses the same stack. For tail_call_cnt, we use the
callee-saved R26 (which was already saved/restored but previously unused
by JIT).
With this patch a tail call generates the following code on arm64:
if (index >= array->map.max_entries)
goto out;
34: mov x10, #0x10 // #16
38: ldr w10, [x1,x10]
3c: cmp w2, w10
40: b.ge 0x0000000000000074
if (tail_call_cnt > MAX_TAIL_CALL_CNT)
goto out;
tail_call_cnt++;
44: mov x10, #0x20 // #32
48: cmp x26, x10
4c: b.gt 0x0000000000000074
50: add x26, x26, #0x1
prog = array->ptrs[index];
if (prog == NULL)
goto out;
54: mov x10, #0x68 // #104
58: ldr x10, [x1,x10]
5c: ldr x11, [x10,x2]
60: cbz x11, 0x0000000000000074
goto *(prog->bpf_func + prologue_size);
64: mov x10, #0x20 // #32
68: ldr x10, [x11,x10]
6c: add x10, x10, #0x20
70: br x10
74:
Signed-off-by: Zi Shen Lim <zlim.lnx@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-09 12:18:48 +08:00
|
|
|
#define A64_BR(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_NOLINK)
|
2014-08-27 12:15:30 +08:00
|
|
|
#define A64_BLR(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_LINK)
|
|
|
|
|
#define A64_RET(Rn) aarch64_insn_gen_branch_reg(Rn, AARCH64_INSN_BRANCH_RETURN)
|
|
|
|
|
|
|
|
|
|
/* Load/store register (register offset) */
|
|
|
|
|
#define A64_LS_REG(Rt, Rn, Rm, size, type) \
|
|
|
|
|
aarch64_insn_gen_load_store_reg(Rt, Rn, Rm, \
|
|
|
|
|
AARCH64_INSN_SIZE_##size, \
|
|
|
|
|
AARCH64_INSN_LDST_##type##_REG_OFFSET)
|
|
|
|
|
#define A64_STRB(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 8, STORE)
|
|
|
|
|
#define A64_LDRB(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 8, LOAD)
|
|
|
|
|
#define A64_STRH(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 16, STORE)
|
|
|
|
|
#define A64_LDRH(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 16, LOAD)
|
|
|
|
|
#define A64_STR32(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 32, STORE)
|
|
|
|
|
#define A64_LDR32(Wt, Xn, Xm) A64_LS_REG(Wt, Xn, Xm, 32, LOAD)
|
|
|
|
|
#define A64_STR64(Xt, Xn, Xm) A64_LS_REG(Xt, Xn, Xm, 64, STORE)
|
|
|
|
|
#define A64_LDR64(Xt, Xn, Xm) A64_LS_REG(Xt, Xn, Xm, 64, LOAD)
|
|
|
|
|
|
|
|
|
|
/* Load/store register pair */
|
|
|
|
|
#define A64_LS_PAIR(Rt, Rt2, Rn, offset, ls, type) \
|
|
|
|
|
aarch64_insn_gen_load_store_pair(Rt, Rt2, Rn, offset, \
|
|
|
|
|
AARCH64_INSN_VARIANT_64BIT, \
|
|
|
|
|
AARCH64_INSN_LDST_##ls##_PAIR_##type)
|
|
|
|
|
/* Rn -= 16; Rn[0] = Rt; Rn[8] = Rt2; */
|
|
|
|
|
#define A64_PUSH(Rt, Rt2, Rn) A64_LS_PAIR(Rt, Rt2, Rn, -16, STORE, PRE_INDEX)
|
|
|
|
|
/* Rt = Rn[0]; Rt2 = Rn[8]; Rn += 16; */
|
|
|
|
|
#define A64_POP(Rt, Rt2, Rn) A64_LS_PAIR(Rt, Rt2, Rn, 16, LOAD, POST_INDEX)
|
|
|
|
|
|
bpf, arm64: implement jiting of BPF_XADD
This work adds BPF_XADD for BPF_W/BPF_DW to the arm64 JIT and therefore
completes JITing of all BPF instructions, meaning we can thus also remove
the 'notyet' label and do not need to fall back to the interpreter when
BPF_XADD is used in a program!
This now also brings arm64 JIT in line with x86_64, s390x, ppc64, sparc64,
where all current eBPF features are supported.
BPF_W example from test_bpf:
.u.insns_int = {
BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
BPF_ST_MEM(BPF_W, R10, -40, 0x10),
BPF_STX_XADD(BPF_W, R10, R0, -40),
BPF_LDX_MEM(BPF_W, R0, R10, -40),
BPF_EXIT_INSN(),
},
[...]
00000020: 52800247 mov w7, #0x12 // #18
00000024: 928004eb mov x11, #0xffffffffffffffd8 // #-40
00000028: d280020a mov x10, #0x10 // #16
0000002c: b82b6b2a str w10, [x25,x11]
// start of xadd mapping:
00000030: 928004ea mov x10, #0xffffffffffffffd8 // #-40
00000034: 8b19014a add x10, x10, x25
00000038: f9800151 prfm pstl1strm, [x10]
0000003c: 885f7d4b ldxr w11, [x10]
00000040: 0b07016b add w11, w11, w7
00000044: 880b7d4b stxr w11, w11, [x10]
00000048: 35ffffab cbnz w11, 0x0000003c
// end of xadd mapping:
[...]
BPF_DW example from test_bpf:
.u.insns_int = {
BPF_ALU32_IMM(BPF_MOV, R0, 0x12),
BPF_ST_MEM(BPF_DW, R10, -40, 0x10),
BPF_STX_XADD(BPF_DW, R10, R0, -40),
BPF_LDX_MEM(BPF_DW, R0, R10, -40),
BPF_EXIT_INSN(),
},
[...]
00000020: 52800247 mov w7, #0x12 // #18
00000024: 928004eb mov x11, #0xffffffffffffffd8 // #-40
00000028: d280020a mov x10, #0x10 // #16
0000002c: f82b6b2a str x10, [x25,x11]
// start of xadd mapping:
00000030: 928004ea mov x10, #0xffffffffffffffd8 // #-40
00000034: 8b19014a add x10, x10, x25
00000038: f9800151 prfm pstl1strm, [x10]
0000003c: c85f7d4b ldxr x11, [x10]
00000040: 8b07016b add x11, x11, x7
00000044: c80b7d4b stxr w11, x11, [x10]
00000048: 35ffffab cbnz w11, 0x0000003c
// end of xadd mapping:
[...]
Tested on Cavium ThunderX ARMv8, test suite results after the patch:
No JIT: [ 3751.855362] test_bpf: Summary: 311 PASSED, 0 FAILED, [0/303 JIT'ed]
With JIT: [ 3573.759527] test_bpf: Summary: 311 PASSED, 0 FAILED, [303/303 JIT'ed]
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2017-05-01 08:57:20 +08:00
|
|
|
/* Load/store exclusive */
|
|
|
|
|
#define A64_SIZE(sf) \
|
|
|
|
|
((sf) ? AARCH64_INSN_SIZE_64 : AARCH64_INSN_SIZE_32)
|
|
|
|
|
#define A64_LSX(sf, Rt, Rn, Rs, type) \
|
|
|
|
|
aarch64_insn_gen_load_store_ex(Rt, Rn, Rs, A64_SIZE(sf), \
|
|
|
|
|
AARCH64_INSN_LDST_##type)
|
|
|
|
|
/* Rt = [Rn]; (atomic) */
|
|
|
|
|
#define A64_LDXR(sf, Rt, Rn) \
|
|
|
|
|
A64_LSX(sf, Rt, Rn, A64_ZR, LOAD_EX)
|
|
|
|
|
/* [Rn] = Rt; (atomic) Rs = [state] */
|
|
|
|
|
#define A64_STXR(sf, Rt, Rn, Rs) \
|
|
|
|
|
A64_LSX(sf, Rt, Rn, Rs, STORE_EX)
|
|
|
|
|
|
2019-04-27 03:48:22 +08:00
|
|
|
/* LSE atomics */
|
|
|
|
|
#define A64_STADD(sf, Rn, Rs) \
|
|
|
|
|
aarch64_insn_gen_stadd(Rn, Rs, A64_SIZE(sf))
|
|
|
|
|
|
2014-08-27 12:15:30 +08:00
|
|
|
/* Add/subtract (immediate) */
|
|
|
|
|
#define A64_ADDSUB_IMM(sf, Rd, Rn, imm12, type) \
|
|
|
|
|
aarch64_insn_gen_add_sub_imm(Rd, Rn, imm12, \
|
|
|
|
|
A64_VARIANT(sf), AARCH64_INSN_ADSB_##type)
|
|
|
|
|
/* Rd = Rn OP imm12 */
|
|
|
|
|
#define A64_ADD_I(sf, Rd, Rn, imm12) A64_ADDSUB_IMM(sf, Rd, Rn, imm12, ADD)
|
|
|
|
|
#define A64_SUB_I(sf, Rd, Rn, imm12) A64_ADDSUB_IMM(sf, Rd, Rn, imm12, SUB)
|
bpf, arm64: Optimize ADD,SUB,JMP BPF_K using arm64 add/sub immediates
The current code for BPF_{ADD,SUB} BPF_K loads the BPF immediate to a
temporary register before performing the addition/subtraction. Similarly,
BPF_JMP BPF_K cases load the immediate to a temporary register before
comparison.
This patch introduces optimizations that use arm64 immediate add, sub,
cmn, or cmp instructions when the BPF immediate fits. If the immediate
does not fit, it falls back to using a temporary register.
Example of generated code for BPF_ALU64_IMM(BPF_ADD, R0, 2):
without optimization:
24: mov x10, #0x2
28: add x7, x7, x10
with optimization:
24: add x7, x7, #0x2
The code could use A64_{ADD,SUB}_I directly and check if it returns
AARCH64_BREAK_FAULT, similar to how logical immediates are handled.
However, aarch64_insn_gen_add_sub_imm from insn.c prints error messages
when the immediate does not fit, and it's simpler to check if the
immediate fits ahead of time.
Co-developed-by: Xi Wang <xi.wang@gmail.com>
Signed-off-by: Xi Wang <xi.wang@gmail.com>
Signed-off-by: Luke Nelson <luke.r.nels@gmail.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/r/20200508181547.24783-4-luke.r.nels@gmail.com
Signed-off-by: Will Deacon <will@kernel.org>
2020-05-09 02:15:46 +08:00
|
|
|
#define A64_ADDS_I(sf, Rd, Rn, imm12) \
|
|
|
|
|
A64_ADDSUB_IMM(sf, Rd, Rn, imm12, ADD_SETFLAGS)
|
|
|
|
|
#define A64_SUBS_I(sf, Rd, Rn, imm12) \
|
|
|
|
|
A64_ADDSUB_IMM(sf, Rd, Rn, imm12, SUB_SETFLAGS)
|
|
|
|
|
/* Rn + imm12; set condition flags */
|
|
|
|
|
#define A64_CMN_I(sf, Rn, imm12) A64_ADDS_I(sf, A64_ZR, Rn, imm12)
|
|
|
|
|
/* Rn - imm12; set condition flags */
|
|
|
|
|
#define A64_CMP_I(sf, Rn, imm12) A64_SUBS_I(sf, A64_ZR, Rn, imm12)
|
2014-08-27 12:15:30 +08:00
|
|
|
/* Rd = Rn */
|
|
|
|
|
#define A64_MOV(sf, Rd, Rn) A64_ADD_I(sf, Rd, Rn, 0)
|
|
|
|
|
|
|
|
|
|
/* Bitfield move */
|
|
|
|
|
#define A64_BITFIELD(sf, Rd, Rn, immr, imms, type) \
|
|
|
|
|
aarch64_insn_gen_bitfield(Rd, Rn, immr, imms, \
|
|
|
|
|
A64_VARIANT(sf), AARCH64_INSN_BITFIELD_MOVE_##type)
|
|
|
|
|
/* Signed, with sign replication to left and zeros to right */
|
|
|
|
|
#define A64_SBFM(sf, Rd, Rn, ir, is) A64_BITFIELD(sf, Rd, Rn, ir, is, SIGNED)
|
|
|
|
|
/* Unsigned, with zeros to left and right */
|
|
|
|
|
#define A64_UBFM(sf, Rd, Rn, ir, is) A64_BITFIELD(sf, Rd, Rn, ir, is, UNSIGNED)
|
|
|
|
|
|
|
|
|
|
/* Rd = Rn << shift */
|
|
|
|
|
#define A64_LSL(sf, Rd, Rn, shift) ({ \
|
|
|
|
|
int sz = (sf) ? 64 : 32; \
|
|
|
|
|
A64_UBFM(sf, Rd, Rn, (unsigned)-(shift) % sz, sz - 1 - (shift)); \
|
|
|
|
|
})
|
|
|
|
|
/* Rd = Rn >> shift */
|
|
|
|
|
#define A64_LSR(sf, Rd, Rn, shift) A64_UBFM(sf, Rd, Rn, shift, (sf) ? 63 : 31)
|
|
|
|
|
/* Rd = Rn >> shift; signed */
|
|
|
|
|
#define A64_ASR(sf, Rd, Rn, shift) A64_SBFM(sf, Rd, Rn, shift, (sf) ? 63 : 31)
|
|
|
|
|
|
2015-06-26 09:39:15 +08:00
|
|
|
/* Zero extend */
|
|
|
|
|
#define A64_UXTH(sf, Rd, Rn) A64_UBFM(sf, Rd, Rn, 0, 15)
|
|
|
|
|
#define A64_UXTW(sf, Rd, Rn) A64_UBFM(sf, Rd, Rn, 0, 31)
|
|
|
|
|
|
2014-08-27 12:15:30 +08:00
|
|
|
/* Move wide (immediate) */
|
|
|
|
|
#define A64_MOVEW(sf, Rd, imm16, shift, type) \
|
|
|
|
|
aarch64_insn_gen_movewide(Rd, imm16, shift, \
|
|
|
|
|
A64_VARIANT(sf), AARCH64_INSN_MOVEWIDE_##type)
|
|
|
|
|
/* Rd = Zeros (for MOVZ);
|
|
|
|
|
* Rd |= imm16 << shift (where shift is {0, 16, 32, 48});
|
|
|
|
|
* Rd = ~Rd; (for MOVN); */
|
|
|
|
|
#define A64_MOVN(sf, Rd, imm16, shift) A64_MOVEW(sf, Rd, imm16, shift, INVERSE)
|
|
|
|
|
#define A64_MOVZ(sf, Rd, imm16, shift) A64_MOVEW(sf, Rd, imm16, shift, ZERO)
|
|
|
|
|
#define A64_MOVK(sf, Rd, imm16, shift) A64_MOVEW(sf, Rd, imm16, shift, KEEP)
|
|
|
|
|
|
|
|
|
|
/* Add/subtract (shifted register) */
|
|
|
|
|
#define A64_ADDSUB_SREG(sf, Rd, Rn, Rm, type) \
|
|
|
|
|
aarch64_insn_gen_add_sub_shifted_reg(Rd, Rn, Rm, 0, \
|
|
|
|
|
A64_VARIANT(sf), AARCH64_INSN_ADSB_##type)
|
|
|
|
|
/* Rd = Rn OP Rm */
|
|
|
|
|
#define A64_ADD(sf, Rd, Rn, Rm) A64_ADDSUB_SREG(sf, Rd, Rn, Rm, ADD)
|
|
|
|
|
#define A64_SUB(sf, Rd, Rn, Rm) A64_ADDSUB_SREG(sf, Rd, Rn, Rm, SUB)
|
|
|
|
|
#define A64_SUBS(sf, Rd, Rn, Rm) A64_ADDSUB_SREG(sf, Rd, Rn, Rm, SUB_SETFLAGS)
|
|
|
|
|
/* Rd = -Rm */
|
|
|
|
|
#define A64_NEG(sf, Rd, Rm) A64_SUB(sf, Rd, A64_ZR, Rm)
|
|
|
|
|
/* Rn - Rm; set condition flags */
|
|
|
|
|
#define A64_CMP(sf, Rn, Rm) A64_SUBS(sf, A64_ZR, Rn, Rm)
|
|
|
|
|
|
|
|
|
|
/* Data-processing (1 source) */
|
|
|
|
|
#define A64_DATA1(sf, Rd, Rn, type) aarch64_insn_gen_data1(Rd, Rn, \
|
|
|
|
|
A64_VARIANT(sf), AARCH64_INSN_DATA1_##type)
|
|
|
|
|
/* Rd = BSWAPx(Rn) */
|
|
|
|
|
#define A64_REV16(sf, Rd, Rn) A64_DATA1(sf, Rd, Rn, REVERSE_16)
|
|
|
|
|
#define A64_REV32(sf, Rd, Rn) A64_DATA1(sf, Rd, Rn, REVERSE_32)
|
|
|
|
|
#define A64_REV64(Rd, Rn) A64_DATA1(1, Rd, Rn, REVERSE_64)
|
|
|
|
|
|
|
|
|
|
/* Data-processing (2 source) */
|
|
|
|
|
/* Rd = Rn OP Rm */
|
2014-09-17 02:37:35 +08:00
|
|
|
#define A64_DATA2(sf, Rd, Rn, Rm, type) aarch64_insn_gen_data2(Rd, Rn, Rm, \
|
|
|
|
|
A64_VARIANT(sf), AARCH64_INSN_DATA2_##type)
|
|
|
|
|
#define A64_UDIV(sf, Rd, Rn, Rm) A64_DATA2(sf, Rd, Rn, Rm, UDIV)
|
|
|
|
|
#define A64_LSLV(sf, Rd, Rn, Rm) A64_DATA2(sf, Rd, Rn, Rm, LSLV)
|
|
|
|
|
#define A64_LSRV(sf, Rd, Rn, Rm) A64_DATA2(sf, Rd, Rn, Rm, LSRV)
|
|
|
|
|
#define A64_ASRV(sf, Rd, Rn, Rm) A64_DATA2(sf, Rd, Rn, Rm, ASRV)
|
2014-08-27 12:15:30 +08:00
|
|
|
|
|
|
|
|
/* Data-processing (3 source) */
|
|
|
|
|
/* Rd = Ra + Rn * Rm */
|
|
|
|
|
#define A64_MADD(sf, Rd, Ra, Rn, Rm) aarch64_insn_gen_data3(Rd, Ra, Rn, Rm, \
|
|
|
|
|
A64_VARIANT(sf), AARCH64_INSN_DATA3_MADD)
|
2019-09-02 14:14:48 +08:00
|
|
|
/* Rd = Ra - Rn * Rm */
|
|
|
|
|
#define A64_MSUB(sf, Rd, Ra, Rn, Rm) aarch64_insn_gen_data3(Rd, Ra, Rn, Rm, \
|
|
|
|
|
A64_VARIANT(sf), AARCH64_INSN_DATA3_MSUB)
|
2014-08-27 12:15:30 +08:00
|
|
|
/* Rd = Rn * Rm */
|
|
|
|
|
#define A64_MUL(sf, Rd, Rn, Rm) A64_MADD(sf, Rd, A64_ZR, Rn, Rm)
|
|
|
|
|
|
|
|
|
|
/* Logical (shifted register) */
|
|
|
|
|
#define A64_LOGIC_SREG(sf, Rd, Rn, Rm, type) \
|
|
|
|
|
aarch64_insn_gen_logical_shifted_reg(Rd, Rn, Rm, 0, \
|
|
|
|
|
A64_VARIANT(sf), AARCH64_INSN_LOGIC_##type)
|
|
|
|
|
/* Rd = Rn OP Rm */
|
|
|
|
|
#define A64_AND(sf, Rd, Rn, Rm) A64_LOGIC_SREG(sf, Rd, Rn, Rm, AND)
|
|
|
|
|
#define A64_ORR(sf, Rd, Rn, Rm) A64_LOGIC_SREG(sf, Rd, Rn, Rm, ORR)
|
|
|
|
|
#define A64_EOR(sf, Rd, Rn, Rm) A64_LOGIC_SREG(sf, Rd, Rn, Rm, EOR)
|
|
|
|
|
#define A64_ANDS(sf, Rd, Rn, Rm) A64_LOGIC_SREG(sf, Rd, Rn, Rm, AND_SETFLAGS)
|
|
|
|
|
/* Rn & Rm; set condition flags */
|
|
|
|
|
#define A64_TST(sf, Rn, Rm) A64_ANDS(sf, A64_ZR, Rn, Rm)
|
|
|
|
|
|
bpf, arm64: Optimize AND,OR,XOR,JSET BPF_K using arm64 logical immediates
The current code for BPF_{AND,OR,XOR,JSET} BPF_K loads the immediate to
a temporary register before use.
This patch changes the code to avoid using a temporary register
when the BPF immediate is encodable using an arm64 logical immediate
instruction. If the encoding fails (due to the immediate not being
encodable), it falls back to using a temporary register.
Example of generated code for BPF_ALU32_IMM(BPF_AND, R0, 0x80000001):
without optimization:
24: mov w10, #0x8000ffff
28: movk w10, #0x1
2c: and w7, w7, w10
with optimization:
24: and w7, w7, #0x80000001
Since the encoding process is quite complex, the JIT reuses existing
functionality in arch/arm64/kernel/insn.c for encoding logical immediates
rather than duplicate it in the JIT.
Co-developed-by: Xi Wang <xi.wang@gmail.com>
Signed-off-by: Xi Wang <xi.wang@gmail.com>
Signed-off-by: Luke Nelson <luke.r.nels@gmail.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/r/20200508181547.24783-3-luke.r.nels@gmail.com
Signed-off-by: Will Deacon <will@kernel.org>
2020-05-09 02:15:45 +08:00
|
|
|
/* Logical (immediate) */
|
|
|
|
|
#define A64_LOGIC_IMM(sf, Rd, Rn, imm, type) ({ \
|
|
|
|
|
u64 imm64 = (sf) ? (u64)imm : (u64)(u32)imm; \
|
|
|
|
|
aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_##type, \
|
|
|
|
|
A64_VARIANT(sf), Rn, Rd, imm64); \
|
|
|
|
|
})
|
|
|
|
|
/* Rd = Rn OP imm */
|
|
|
|
|
#define A64_AND_I(sf, Rd, Rn, imm) A64_LOGIC_IMM(sf, Rd, Rn, imm, AND)
|
|
|
|
|
#define A64_ORR_I(sf, Rd, Rn, imm) A64_LOGIC_IMM(sf, Rd, Rn, imm, ORR)
|
|
|
|
|
#define A64_EOR_I(sf, Rd, Rn, imm) A64_LOGIC_IMM(sf, Rd, Rn, imm, EOR)
|
|
|
|
|
#define A64_ANDS_I(sf, Rd, Rn, imm) A64_LOGIC_IMM(sf, Rd, Rn, imm, AND_SETFLAGS)
|
|
|
|
|
/* Rn & imm; set condition flags */
|
|
|
|
|
#define A64_TST_I(sf, Rn, imm) A64_ANDS_I(sf, A64_ZR, Rn, imm)
|
|
|
|
|
|
2020-05-07 03:51:32 +08:00
|
|
|
/* HINTs */
|
|
|
|
|
#define A64_HINT(x) aarch64_insn_gen_hint(x)
|
|
|
|
|
|
|
|
|
|
/* BTI */
|
|
|
|
|
#define A64_BTI_C A64_HINT(AARCH64_INSN_HINT_BTIC)
|
|
|
|
|
#define A64_BTI_J A64_HINT(AARCH64_INSN_HINT_BTIJ)
|
|
|
|
|
#define A64_BTI_JC A64_HINT(AARCH64_INSN_HINT_BTIJC)
|
|
|
|
|
|
2014-08-27 12:15:30 +08:00
|
|
|
#endif /* _BPF_JIT_H */
|