openkylin
/
platform_kernel-5.15
mirror of https://gitee.com/openkylin/platform_kernel-5.15.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next 

Daniel Borkmann says:

====================
pull-request: bpf-next 2020-11-14

1) Add BTF generation for kernel modules and extend BTF infra in kernel
   e.g. support for split BTF loading and validation, from Andrii Nakryiko.

2) Support for pointers beyond pkt_end to recognize LLVM generated patterns
   on inlined branch conditions, from Alexei Starovoitov.

3) Implements bpf_local_storage for task_struct for BPF LSM, from KP Singh.

4) Enable FENTRY/FEXIT/RAW_TP tracing program to use the bpf_sk_storage
   infra, from Martin KaFai Lau.

5) Add XDP bulk APIs that introduce a defer/flush mechanism to optimize the
   XDP_REDIRECT path, from Lorenzo Bianconi.

6) Fix a potential (although rather theoretical) deadlock of hashtab in NMI
   context, from Song Liu.

7) Fixes for cross and out-of-tree build of bpftool and runqslower allowing build
   for different target archs on same source tree, from Jean-Philippe Brucker.

8) Fix error path in htab_map_alloc() triggered from syzbot, from Eric Dumazet.

9) Move functionality from test_tcpbpf_user into the test_progs framework so it
   can run in BPF CI, from Alexander Duyck.

10) Lift hashtab key_size limit to be larger than MAX_BPF_STACK, from Florian Lehner.

Note that for the fix from Song we have seen a sparse report on context
imbalance which requires changes in sparse itself for proper annotation
detection where this is currently being discussed on linux-sparse among
developers [0]. Once we have more clarification/guidance after their fix,
Song will follow-up.

  [0] https://lore.kernel.org/linux-sparse/CAHk-=wh4bx8A8dHnX612MsDO13st6uzAz1mJ1PaHHVevJx_ZCw@mail.gmail.com/T/
      https://lore.kernel.org/linux-sparse/20201109221345.uklbp3lzgq6g42zb@ltop.local/T/

* git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (66 commits)
  net: mlx5: Add xdp tx return bulking support
  net: mvpp2: Add xdp tx return bulking support
  net: mvneta: Add xdp tx return bulking support
  net: page_pool: Add bulk support for ptr_ring
  net: xdp: Introduce bulking for xdp tx return path
  bpf: Expose bpf_d_path helper to sleepable LSM hooks
  bpf: Augment the set of sleepable LSM hooks
  bpf: selftest: Use bpf_sk_storage in FENTRY/FEXIT/RAW_TP
  bpf: Allow using bpf_sk_storage in FENTRY/FEXIT/RAW_TP
  bpf: Rename some functions in bpf_sk_storage
  bpf: Folding omem_charge() into sk_storage_charge()
  selftests/bpf: Add asm tests for pkt vs pkt_end comparison.
  selftests/bpf: Add skb_pkt_end test
  bpf: Support for pointers beyond pkt_end.
  tools/bpf: Always run the *-clean recipes
  tools/bpf: Add bootstrap/ to .gitignore
  bpf: Fix NULL dereference in bpf_task_storage
  tools/bpftool: Fix build slowdown
  tools/runqslower: Build bpftool using HOSTCC
  tools/runqslower: Enable out-of-tree build
  ...
====================

Link: https://lore.kernel.org/r/20201114020819.29584-1-daniel@iogearbox.net
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Jakub Kicinski 2020-11-14 09:13:40 -08:00
parent 774626fa44 c14d61fca0
commit 07cbce2e46
83 changed files with 3909 additions and 1273 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Documentation/ABI/testing/sysfs-kernel-btf
View File

@ -15,3 +15,11 @@ Description:
information with description of all internal kernel types. See
Documentation/bpf/btf.rst for detailed description of format
itself.
What: /sys/kernel/btf/<module-name>
Date: Nov 2020
KernelVersion: 5.11
Contact: bpf@vger.kernel.org
Description:
Read-only binary attribute exposing kernel module's BTF type
information as an add-on to the kernel's BTF (/sys/kernel/btf/vmlinux).

10
drivers/net/ethernet/marvell/mvneta.c
View File

@ -1834,8 +1834,13 @@ static void mvneta_txq_bufs_free(struct mvneta_port *pp,
struct netdev_queue *nq, bool napi)
{
unsigned int bytes_compl = 0, pkts_compl = 0;
struct xdp_frame_bulk bq;
int i;
xdp_frame_bulk_init(&bq);
rcu_read_lock(); /* need for xdp_return_frame_bulk */
for (i = 0; i < num; i++) {
struct mvneta_tx_buf *buf = &txq->buf[txq->txq_get_index];
struct mvneta_tx_desc *tx_desc = txq->descs +
@ -1857,9 +1862,12 @@ static void mvneta_txq_bufs_free(struct mvneta_port *pp,
if (napi && buf->type == MVNETA_TYPE_XDP_TX)
xdp_return_frame_rx_napi(buf->xdpf);
else
xdp_return_frame(buf->xdpf);
xdp_return_frame_bulk(buf->xdpf, &bq);
}
}
xdp_flush_frame_bulk(&bq);
rcu_read_unlock();
netdev_tx_completed_queue(nq, pkts_compl, bytes_compl);
}

10
drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
View File

@ -2440,8 +2440,13 @@ static void mvpp2_txq_bufs_free(struct mvpp2_port *port,
struct mvpp2_tx_queue *txq,
struct mvpp2_txq_pcpu *txq_pcpu, int num)
{
struct xdp_frame_bulk bq;
int i;
xdp_frame_bulk_init(&bq);
rcu_read_lock(); /* need for xdp_return_frame_bulk */
for (i = 0; i < num; i++) {
struct mvpp2_txq_pcpu_buf *tx_buf =
txq_pcpu->buffs + txq_pcpu->txq_get_index;
@ -2454,10 +2459,13 @@ static void mvpp2_txq_bufs_free(struct mvpp2_port *port,
dev_kfree_skb_any(tx_buf->skb);
else if (tx_buf->type == MVPP2_TYPE_XDP_TX ||
tx_buf->type == MVPP2_TYPE_XDP_NDO)
xdp_return_frame(tx_buf->xdpf);
xdp_return_frame_bulk(tx_buf->xdpf, &bq);
mvpp2_txq_inc_get(txq_pcpu);
}
xdp_flush_frame_bulk(&bq);
rcu_read_unlock();
}
static inline struct mvpp2_rx_queue *mvpp2_get_rx_queue(struct mvpp2_port *port,

22
drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
View File

@ -366,7 +366,8 @@ mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
struct mlx5e_xdp_wqe_info *wi,
u32 *xsk_frames,
bool recycle)
bool recycle,
struct xdp_frame_bulk *bq)
{
struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
u16 i;
@ -379,7 +380,7 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
/* XDP_TX from the XSK RQ and XDP_REDIRECT */
dma_unmap_single(sq->pdev, xdpi.frame.dma_addr,
xdpi.frame.xdpf->len, DMA_TO_DEVICE);
xdp_return_frame(xdpi.frame.xdpf);
xdp_return_frame_bulk(xdpi.frame.xdpf, bq);
break;
case MLX5E_XDP_XMIT_MODE_PAGE:
/* XDP_TX from the regular RQ */
@ -397,12 +398,15 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
{
struct xdp_frame_bulk bq;
struct mlx5e_xdpsq *sq;
struct mlx5_cqe64 *cqe;
u32 xsk_frames = 0;
u16 sqcc;
int i;
xdp_frame_bulk_init(&bq);
sq = container_of(cq, struct mlx5e_xdpsq, cq);
if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
@ -434,7 +438,7 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
sqcc += wi->num_wqebbs;
mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, true);
mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, true, &bq);
} while (!last_wqe);
if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
@ -447,6 +451,8 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
}
} while ((++i < MLX5E_TX_CQ_POLL_BUDGET) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
xdp_flush_frame_bulk(&bq);
if (xsk_frames)
xsk_tx_completed(sq->xsk_pool, xsk_frames);
@ -463,8 +469,13 @@ bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq)
void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
{
struct xdp_frame_bulk bq;
u32 xsk_frames = 0;
xdp_frame_bulk_init(&bq);
rcu_read_lock(); /* need for xdp_return_frame_bulk */
while (sq->cc != sq->pc) {
struct mlx5e_xdp_wqe_info *wi;
u16 ci;
@ -474,9 +485,12 @@ void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq)
sq->cc += wi->num_wqebbs;
mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, false);
mlx5e_free_xdpsq_desc(sq, wi, &xsk_frames, false, &bq);
}
xdp_flush_frame_bulk(&bq);
rcu_read_unlock();
if (xsk_frames)
xsk_tx_completed(sq->xsk_pool, xsk_frames);
}

8
include/linux/bpf.h
View File

@ -36,9 +36,11 @@ struct seq_operations;
struct bpf_iter_aux_info;
struct bpf_local_storage;
struct bpf_local_storage_map;
struct kobject;
extern struct idr btf_idr;
extern spinlock_t btf_idr_lock;
extern struct kobject *btf_kobj;
typedef int (*bpf_iter_init_seq_priv_t)(void *private_data,
struct bpf_iter_aux_info *aux);
@ -310,6 +312,7 @@ enum bpf_return_type {
RET_PTR_TO_BTF_ID_OR_NULL, /* returns a pointer to a btf_id or NULL */
RET_PTR_TO_MEM_OR_BTF_ID_OR_NULL, /* returns a pointer to a valid memory or a btf_id or NULL */
RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */
RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */
};
/* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs
@ -1294,6 +1297,10 @@ typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux,
typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux,
struct bpf_link_info *info);
enum bpf_iter_feature {
BPF_ITER_RESCHED = BIT(0),
};
#define BPF_ITER_CTX_ARG_MAX 2
struct bpf_iter_reg {
const char *target;
@ -1302,6 +1309,7 @@ struct bpf_iter_reg {
bpf_iter_show_fdinfo_t show_fdinfo;
bpf_iter_fill_link_info_t fill_link_info;
u32 ctx_arg_info_size;
u32 feature;
struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX];
const struct bpf_iter_seq_info *seq_info;
};

30
include/linux/bpf_lsm.h
View File

@ -7,6 +7,7 @@
#ifndef _LINUX_BPF_LSM_H
#define _LINUX_BPF_LSM_H
#include <linux/sched.h>
#include <linux/bpf.h>
#include <linux/lsm_hooks.h>
@ -26,6 +27,8 @@ extern struct lsm_blob_sizes bpf_lsm_blob_sizes;
int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
const struct bpf_prog *prog);
bool bpf_lsm_is_sleepable_hook(u32 btf_id);
static inline struct bpf_storage_blob *bpf_inode(
const struct inode *inode)
{
@ -35,12 +38,29 @@ static inline struct bpf_storage_blob *bpf_inode(
return inode->i_security + bpf_lsm_blob_sizes.lbs_inode;
}
static inline struct bpf_storage_blob *bpf_task(
const struct task_struct *task)
{
if (unlikely(!task->security))
return NULL;
return task->security + bpf_lsm_blob_sizes.lbs_task;
}
extern const struct bpf_func_proto bpf_inode_storage_get_proto;
extern const struct bpf_func_proto bpf_inode_storage_delete_proto;
extern const struct bpf_func_proto bpf_task_storage_get_proto;
extern const struct bpf_func_proto bpf_task_storage_delete_proto;
void bpf_inode_storage_free(struct inode *inode);
void bpf_task_storage_free(struct task_struct *task);
#else /* !CONFIG_BPF_LSM */
static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id)
{
return false;
}
static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog,
const struct bpf_prog *prog)
{
@ -53,10 +73,20 @@ static inline struct bpf_storage_blob *bpf_inode(
return NULL;
}
static inline struct bpf_storage_blob *bpf_task(
const struct task_struct *task)
{
return NULL;
}
static inline void bpf_inode_storage_free(struct inode *inode)
{
}
static inline void bpf_task_storage_free(struct task_struct *task)
{
}
#endif /* CONFIG_BPF_LSM */
#endif /* _LINUX_BPF_LSM_H */

1
include/linux/bpf_types.h
View File

@ -109,6 +109,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKHASH, sock_hash_ops)
#endif
#ifdef CONFIG_BPF_LSM
BPF_MAP_TYPE(BPF_MAP_TYPE_INODE_STORAGE, inode_storage_map_ops)
BPF_MAP_TYPE(BPF_MAP_TYPE_TASK_STORAGE, task_storage_map_ops)
#endif
BPF_MAP_TYPE(BPF_MAP_TYPE_CPUMAP, cpu_map_ops)
#if defined(CONFIG_XDP_SOCKETS)

2
include/linux/bpf_verifier.h
View File

@ -45,7 +45,7 @@ struct bpf_reg_state {
enum bpf_reg_type type;
union {
/* valid when type == PTR_TO_PACKET */
u16 range;
int range;
/* valid when type == CONST_PTR_TO_MAP | PTR_TO_MAP_VALUE |
* PTR_TO_MAP_VALUE_OR_NULL

4
include/linux/module.h
View File

@ -475,6 +475,10 @@ struct module {
unsigned int num_bpf_raw_events;
struct bpf_raw_event_map *bpf_raw_events;
#endif
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
unsigned int btf_data_size;
void *btf_data;
#endif
#ifdef CONFIG_JUMP_LABEL
struct jump_entry *jump_entries;
unsigned int num_jump_entries;

2
include/net/bpf_sk_storage.h
View File

@ -20,6 +20,8 @@ void bpf_sk_storage_free(struct sock *sk);
extern const struct bpf_func_proto bpf_sk_storage_get_proto;
extern const struct bpf_func_proto bpf_sk_storage_delete_proto;
extern const struct bpf_func_proto bpf_sk_storage_get_tracing_proto;
extern const struct bpf_func_proto bpf_sk_storage_delete_tracing_proto;
struct bpf_local_storage_elem;
struct bpf_sk_storage_diag;

26
include/net/page_pool.h
View File

@ -152,6 +152,8 @@ struct page_pool *page_pool_create(const struct page_pool_params *params);
void page_pool_destroy(struct page_pool *pool);
void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *));
void page_pool_release_page(struct page_pool *pool, struct page *page);
void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count);
#else
static inline void page_pool_destroy(struct page_pool *pool)
{
@ -165,6 +167,11 @@ static inline void page_pool_release_page(struct page_pool *pool,
struct page *page)
{
}
static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count)
{
}
#endif
void page_pool_put_page(struct page_pool *pool, struct page *page,
@ -215,4 +222,23 @@ static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid)
if (unlikely(pool->p.nid != new_nid))
page_pool_update_nid(pool, new_nid);
}
static inline void page_pool_ring_lock(struct page_pool *pool)
__acquires(&pool->ring.producer_lock)
{
if (in_serving_softirq())
spin_lock(&pool->ring.producer_lock);
else
spin_lock_bh(&pool->ring.producer_lock);
}
static inline void page_pool_ring_unlock(struct page_pool *pool)
__releases(&pool->ring.producer_lock)
{
if (in_serving_softirq())
spin_unlock(&pool->ring.producer_lock);
else
spin_unlock_bh(&pool->ring.producer_lock);
}
#endif /* _NET_PAGE_POOL_H */

17
include/net/xdp.h
View File

@ -104,6 +104,18 @@ struct xdp_frame {
struct net_device *dev_rx; /* used by cpumap */
};
#define XDP_BULK_QUEUE_SIZE 16
struct xdp_frame_bulk {
int count;
void *xa;
void *q[XDP_BULK_QUEUE_SIZE];
};
static __always_inline void xdp_frame_bulk_init(struct xdp_frame_bulk *bq)
{
/* bq->count will be zero'ed when bq->xa gets updated */
bq->xa = NULL;
}
static inline struct skb_shared_info *
xdp_get_shared_info_from_frame(struct xdp_frame *frame)
@ -194,6 +206,9 @@ struct xdp_frame *xdp_convert_buff_to_frame(struct xdp_buff *xdp)
void xdp_return_frame(struct xdp_frame *xdpf);
void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
void xdp_return_buff(struct xdp_buff *xdp);
void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq);
void xdp_return_frame_bulk(struct xdp_frame *xdpf,
struct xdp_frame_bulk *bq);
/* When sending xdp_frame into the network stack, then there is no
* return point callback, which is needed to release e.g. DMA-mapping
@ -245,6 +260,6 @@ bool xdp_attachment_flags_ok(struct xdp_attachment_info *info,
void xdp_attachment_setup(struct xdp_attachment_info *info,
struct netdev_bpf *bpf);
#define DEV_MAP_BULK_SIZE 16
#define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE
#endif /* __LINUX_NET_XDP_H__ */

51
include/uapi/linux/bpf.h
View File

@ -157,6 +157,7 @@ enum bpf_map_type {
BPF_MAP_TYPE_STRUCT_OPS,
BPF_MAP_TYPE_RINGBUF,
BPF_MAP_TYPE_INODE_STORAGE,
BPF_MAP_TYPE_TASK_STORAGE,
};
/* Note that tracing related programs such as
@ -3742,6 +3743,50 @@ union bpf_attr {
* Return
* The helper returns **TC_ACT_REDIRECT** on success or
* **TC_ACT_SHOT** on error.
*
* void *bpf_task_storage_get(struct bpf_map *map, struct task_struct *task, void *value, u64 flags)
* Description
* Get a bpf_local_storage from the *task*.
*
* Logically, it could be thought of as getting the value from
* a *map* with *task* as the **key**. From this
* perspective, the usage is not much different from
* **bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this
* helper enforces the key must be an task_struct and the map must also
* be a **BPF_MAP_TYPE_TASK_STORAGE**.
*
* Underneath, the value is stored locally at *task* instead of
* the *map*. The *map* is used as the bpf-local-storage
* "type". The bpf-local-storage "type" (i.e. the *map*) is
* searched against all bpf_local_storage residing at *task*.
*
* An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be
* used such that a new bpf_local_storage will be
* created if one does not exist. *value* can be used
* together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify
* the initial value of a bpf_local_storage. If *value* is
* **NULL**, the new bpf_local_storage will be zero initialized.
* Return
* A bpf_local_storage pointer is returned on success.
*
* **NULL** if not found or there was an error in adding
* a new bpf_local_storage.
*
* long bpf_task_storage_delete(struct bpf_map *map, struct task_struct *task)
* Description
* Delete a bpf_local_storage from a *task*.
* Return
* 0 on success.
*
* **-ENOENT** if the bpf_local_storage cannot be found.
*
* struct task_struct *bpf_get_current_task_btf(void)
* Description
* Return a BTF pointer to the "current" task.
* This pointer can also be used in helpers that accept an
* *ARG_PTR_TO_BTF_ID* of type *task_struct*.
* Return
* Pointer to the current task.
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@ -3900,6 +3945,9 @@ union bpf_attr {
FN(bpf_per_cpu_ptr), \
FN(bpf_this_cpu_ptr), \
FN(redirect_peer), \
FN(task_storage_get), \
FN(task_storage_delete), \
FN(get_current_task_btf), \
/* */
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@ -4418,6 +4466,9 @@ struct bpf_btf_info {
__aligned_u64 btf;
__u32 btf_size;
__u32 id;
__aligned_u64 name;
__u32 name_len;
__u32 kernel_btf;
} __attribute__((aligned(8)));
struct bpf_link_info {

1
kernel/bpf/Makefile
View File

@ -10,6 +10,7 @@ obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o bpf_i
obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o
obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o
obj-${CONFIG_BPF_LSM} += bpf_inode_storage.o
obj-${CONFIG_BPF_LSM} += bpf_task_storage.o
obj-$(CONFIG_BPF_SYSCALL) += disasm.o
obj-$(CONFIG_BPF_JIT) += trampoline.o
obj-$(CONFIG_BPF_SYSCALL) += btf.o

14
kernel/bpf/bpf_iter.c
View File

@ -67,6 +67,15 @@ static void bpf_iter_done_stop(struct seq_file *seq)
iter_priv->done_stop = true;
}
static bool bpf_iter_support_resched(struct seq_file *seq)
{
struct bpf_iter_priv_data *iter_priv;
iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
target_private);
return iter_priv->tinfo->reg_info->feature & BPF_ITER_RESCHED;
}
/* maximum visited objects before bailing out */
#define MAX_ITER_OBJECTS 1000000
@ -83,6 +92,7 @@ static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
struct seq_file *seq = file->private_data;
size_t n, offs, copied = 0;
int err = 0, num_objs = 0;
bool can_resched;
void *p;
mutex_lock(&seq->lock);
@ -135,6 +145,7 @@ static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
goto done;
}
can_resched = bpf_iter_support_resched(seq);
while (1) {
loff_t pos = seq->index;
@ -180,6 +191,9 @@ static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
}
break;
}
if (can_resched)
cond_resched();
}
stop:
offs = seq->count;

88
kernel/bpf/bpf_lsm.c
View File

@ -63,11 +63,99 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
return &bpf_sk_storage_get_proto;
case BPF_FUNC_sk_storage_delete:
return &bpf_sk_storage_delete_proto;
case BPF_FUNC_spin_lock:
return &bpf_spin_lock_proto;
case BPF_FUNC_spin_unlock:
return &bpf_spin_unlock_proto;
case BPF_FUNC_task_storage_get:
return &bpf_task_storage_get_proto;
case BPF_FUNC_task_storage_delete:
return &bpf_task_storage_delete_proto;
default:
return tracing_prog_func_proto(func_id, prog);
}
}
/* The set of hooks which are called without pagefaults disabled and are allowed
* to "sleep" and thus can be used for sleeable BPF programs.
*/
BTF_SET_START(sleepable_lsm_hooks)
BTF_ID(func, bpf_lsm_bpf)
BTF_ID(func, bpf_lsm_bpf_map)
BTF_ID(func, bpf_lsm_bpf_map_alloc_security)
BTF_ID(func, bpf_lsm_bpf_map_free_security)
BTF_ID(func, bpf_lsm_bpf_prog)
BTF_ID(func, bpf_lsm_bprm_check_security)
BTF_ID(func, bpf_lsm_bprm_committed_creds)
BTF_ID(func, bpf_lsm_bprm_committing_creds)
BTF_ID(func, bpf_lsm_bprm_creds_for_exec)
BTF_ID(func, bpf_lsm_bprm_creds_from_file)
BTF_ID(func, bpf_lsm_capget)
BTF_ID(func, bpf_lsm_capset)
BTF_ID(func, bpf_lsm_cred_prepare)
BTF_ID(func, bpf_lsm_file_ioctl)
BTF_ID(func, bpf_lsm_file_lock)
BTF_ID(func, bpf_lsm_file_open)
BTF_ID(func, bpf_lsm_file_receive)
BTF_ID(func, bpf_lsm_inet_conn_established)
BTF_ID(func, bpf_lsm_inode_create)
BTF_ID(func, bpf_lsm_inode_free_security)
BTF_ID(func, bpf_lsm_inode_getattr)
BTF_ID(func, bpf_lsm_inode_getxattr)
BTF_ID(func, bpf_lsm_inode_mknod)
BTF_ID(func, bpf_lsm_inode_need_killpriv)
BTF_ID(func, bpf_lsm_inode_post_setxattr)
BTF_ID(func, bpf_lsm_inode_readlink)
BTF_ID(func, bpf_lsm_inode_rename)
BTF_ID(func, bpf_lsm_inode_rmdir)
BTF_ID(func, bpf_lsm_inode_setattr)
BTF_ID(func, bpf_lsm_inode_setxattr)
BTF_ID(func, bpf_lsm_inode_symlink)
BTF_ID(func, bpf_lsm_inode_unlink)
BTF_ID(func, bpf_lsm_kernel_module_request)
BTF_ID(func, bpf_lsm_kernfs_init_security)
BTF_ID(func, bpf_lsm_key_free)
BTF_ID(func, bpf_lsm_mmap_file)
BTF_ID(func, bpf_lsm_netlink_send)
BTF_ID(func, bpf_lsm_path_notify)
BTF_ID(func, bpf_lsm_release_secctx)
BTF_ID(func, bpf_lsm_sb_alloc_security)
BTF_ID(func, bpf_lsm_sb_eat_lsm_opts)
BTF_ID(func, bpf_lsm_sb_kern_mount)
BTF_ID(func, bpf_lsm_sb_mount)
BTF_ID(func, bpf_lsm_sb_remount)
BTF_ID(func, bpf_lsm_sb_set_mnt_opts)
BTF_ID(func, bpf_lsm_sb_show_options)
BTF_ID(func, bpf_lsm_sb_statfs)
BTF_ID(func, bpf_lsm_sb_umount)
BTF_ID(func, bpf_lsm_settime)
BTF_ID(func, bpf_lsm_socket_accept)
BTF_ID(func, bpf_lsm_socket_bind)
BTF_ID(func, bpf_lsm_socket_connect)
BTF_ID(func, bpf_lsm_socket_create)
BTF_ID(func, bpf_lsm_socket_getpeername)
BTF_ID(func, bpf_lsm_socket_getpeersec_dgram)
BTF_ID(func, bpf_lsm_socket_getsockname)
BTF_ID(func, bpf_lsm_socket_getsockopt)
BTF_ID(func, bpf_lsm_socket_listen)
BTF_ID(func, bpf_lsm_socket_post_create)
BTF_ID(func, bpf_lsm_socket_recvmsg)
BTF_ID(func, bpf_lsm_socket_sendmsg)
BTF_ID(func, bpf_lsm_socket_shutdown)
BTF_ID(func, bpf_lsm_socket_socketpair)
BTF_ID(func, bpf_lsm_syslog)
BTF_ID(func, bpf_lsm_task_alloc)
BTF_ID(func, bpf_lsm_task_getsecid)
BTF_ID(func, bpf_lsm_task_prctl)
BTF_ID(func, bpf_lsm_task_setscheduler)
BTF_ID(func, bpf_lsm_task_to_inode)
BTF_SET_END(sleepable_lsm_hooks)
bool bpf_lsm_is_sleepable_hook(u32 btf_id)
{
return btf_id_set_contains(&sleepable_lsm_hooks, btf_id);
}
const struct bpf_prog_ops lsm_prog_ops = {
};

315
kernel/bpf/bpf_task_storage.c Normal file
View File

@ -0,0 +1,315 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (c) 2020 Facebook
* Copyright 2020 Google LLC.
*/
#include <linux/pid.h>
#include <linux/sched.h>
#include <linux/rculist.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/types.h>
#include <linux/spinlock.h>
#include <linux/bpf.h>
#include <linux/bpf_local_storage.h>
#include <linux/filter.h>
#include <uapi/linux/btf.h>
#include <linux/bpf_lsm.h>
#include <linux/btf_ids.h>
#include <linux/fdtable.h>
DEFINE_BPF_STORAGE_CACHE(task_cache);
static struct bpf_local_storage __rcu **task_storage_ptr(void *owner)
{
struct task_struct *task = owner;
struct bpf_storage_blob *bsb;
bsb = bpf_task(task);
if (!bsb)
return NULL;
return &bsb->storage;
}
static struct bpf_local_storage_data *
task_storage_lookup(struct task_struct *task, struct bpf_map *map,
bool cacheit_lockit)
{
struct bpf_local_storage *task_storage;
struct bpf_local_storage_map *smap;
struct bpf_storage_blob *bsb;
bsb = bpf_task(task);
if (!bsb)
return NULL;
task_storage = rcu_dereference(bsb->storage);
if (!task_storage)
return NULL;
smap = (struct bpf_local_storage_map *)map;
return bpf_local_storage_lookup(task_storage, smap, cacheit_lockit);
}
void bpf_task_storage_free(struct task_struct *task)
{
struct bpf_local_storage_elem *selem;
struct bpf_local_storage *local_storage;
bool free_task_storage = false;
struct bpf_storage_blob *bsb;
struct hlist_node *n;
bsb = bpf_task(task);
if (!bsb)
return;
rcu_read_lock();
local_storage = rcu_dereference(bsb->storage);
if (!local_storage) {
rcu_read_unlock();
return;
}
/* Neither the bpf_prog nor the bpf-map's syscall
* could be modifying the local_storage->list now.
* Thus, no elem can be added-to or deleted-from the
* local_storage->list by the bpf_prog or by the bpf-map's syscall.
*
* It is racing with bpf_local_storage_map_free() alone
* when unlinking elem from the local_storage->list and
* the map's bucket->list.
*/
raw_spin_lock_bh(&local_storage->lock);
hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {
/* Always unlink from map before unlinking from
* local_storage.
*/
bpf_selem_unlink_map(selem);
free_task_storage = bpf_selem_unlink_storage_nolock(
local_storage, selem, false);
}
raw_spin_unlock_bh(&local_storage->lock);
rcu_read_unlock();
/* free_task_storage should always be true as long as
* local_storage->list was non-empty.
*/
if (free_task_storage)
kfree_rcu(local_storage, rcu);
}
static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)
{
struct bpf_local_storage_data *sdata;
struct task_struct *task;
unsigned int f_flags;
struct pid *pid;
int fd, err;
fd = *(int *)key;
pid = pidfd_get_pid(fd, &f_flags);
if (IS_ERR(pid))
return ERR_CAST(pid);
/* We should be in an RCU read side critical section, it should be safe
* to call pid_task.
*/
WARN_ON_ONCE(!rcu_read_lock_held());
task = pid_task(pid, PIDTYPE_PID);
if (!task) {
err = -ENOENT;
goto out;
}
sdata = task_storage_lookup(task, map, true);
put_pid(pid);
return sdata ? sdata->data : NULL;
out:
put_pid(pid);
return ERR_PTR(err);
}
static int bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key,
void *value, u64 map_flags)
{
struct bpf_local_storage_data *sdata;
struct task_struct *task;
unsigned int f_flags;
struct pid *pid;
int fd, err;
fd = *(int *)key;
pid = pidfd_get_pid(fd, &f_flags);
if (IS_ERR(pid))
return PTR_ERR(pid);
/* We should be in an RCU read side critical section, it should be safe
* to call pid_task.
*/
WARN_ON_ONCE(!rcu_read_lock_held());
task = pid_task(pid, PIDTYPE_PID);
if (!task || !task_storage_ptr(task)) {
err = -ENOENT;
goto out;
}
sdata = bpf_local_storage_update(
task, (struct bpf_local_storage_map *)map, value, map_flags);
err = PTR_ERR_OR_ZERO(sdata);
out:
put_pid(pid);
return err;
}
static int task_storage_delete(struct task_struct *task, struct bpf_map *map)
{
struct bpf_local_storage_data *sdata;
sdata = task_storage_lookup(task, map, false);
if (!sdata)
return -ENOENT;
bpf_selem_unlink(SELEM(sdata));
return 0;
}
static int bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key)
{
struct task_struct *task;
unsigned int f_flags;
struct pid *pid;
int fd, err;
fd = *(int *)key;
pid = pidfd_get_pid(fd, &f_flags);
if (IS_ERR(pid))
return PTR_ERR(pid);
/* We should be in an RCU read side critical section, it should be safe
* to call pid_task.
*/
WARN_ON_ONCE(!rcu_read_lock_held());
task = pid_task(pid, PIDTYPE_PID);
if (!task) {
err = -ENOENT;
goto out;
}
err = task_storage_delete(task, map);
out:
put_pid(pid);
return err;
}
BPF_CALL_4(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
task, void *, value, u64, flags)
{
struct bpf_local_storage_data *sdata;
if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
return (unsigned long)NULL;
/* explicitly check that the task_storage_ptr is not
* NULL as task_storage_lookup returns NULL in this case and
* bpf_local_storage_update expects the owner to have a
* valid storage pointer.
*/
if (!task_storage_ptr(task))
return (unsigned long)NULL;
sdata = task_storage_lookup(task, map, true);
if (sdata)
return (unsigned long)sdata->data;
/* This helper must only be called from places where the lifetime of the task
* is guaranteed. Either by being refcounted or by being protected
* by an RCU read-side critical section.
*/
if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
sdata = bpf_local_storage_update(
task, (struct bpf_local_storage_map *)map, value,
BPF_NOEXIST);
return IS_ERR(sdata) ? (unsigned long)NULL :
(unsigned long)sdata->data;
}
return (unsigned long)NULL;
}
BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *,
task)
{
/* This helper must only be called from places where the lifetime of the task
* is guaranteed. Either by being refcounted or by being protected
* by an RCU read-side critical section.
*/
return task_storage_delete(task, map);
}
static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)
{
return -ENOTSUPP;
}
static struct bpf_map *task_storage_map_alloc(union bpf_attr *attr)
{
struct bpf_local_storage_map *smap;
smap = bpf_local_storage_map_alloc(attr);
if (IS_ERR(smap))
return ERR_CAST(smap);
smap->cache_idx = bpf_local_storage_cache_idx_get(&task_cache);
return &smap->map;
}
static void task_storage_map_free(struct bpf_map *map)
{
struct bpf_local_storage_map *smap;
smap = (struct bpf_local_storage_map *)map;
bpf_local_storage_cache_idx_free(&task_cache, smap->cache_idx);
bpf_local_storage_map_free(smap);
}
static int task_storage_map_btf_id;
const struct bpf_map_ops task_storage_map_ops = {
.map_meta_equal = bpf_map_meta_equal,
.map_alloc_check = bpf_local_storage_map_alloc_check,
.map_alloc = task_storage_map_alloc,
.map_free = task_storage_map_free,
.map_get_next_key = notsupp_get_next_key,
.map_lookup_elem = bpf_pid_task_storage_lookup_elem,
.map_update_elem = bpf_pid_task_storage_update_elem,
.map_delete_elem = bpf_pid_task_storage_delete_elem,
.map_check_btf = bpf_local_storage_map_check_btf,
.map_btf_name = "bpf_local_storage_map",
.map_btf_id = &task_storage_map_btf_id,
.map_owner_storage_ptr = task_storage_ptr,
};
BTF_ID_LIST_SINGLE(bpf_task_storage_btf_ids, struct, task_struct)
const struct bpf_func_proto bpf_task_storage_get_proto = {
.func = bpf_task_storage_get,
.gpl_only = false,
.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_BTF_ID,
.arg2_btf_id = &bpf_task_storage_btf_ids[0],
.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
.arg4_type = ARG_ANYTHING,
};
const struct bpf_func_proto bpf_task_storage_delete_proto = {
.func = bpf_task_storage_delete,
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_BTF_ID,
.arg2_btf_id = &bpf_task_storage_btf_ids[0],
};

413
kernel/bpf/btf.c
View File

@ -22,7 +22,8 @@
#include <linux/skmsg.h>
#include <linux/perf_event.h>
#include <linux/bsearch.h>
#include <linux/btf_ids.h>
#include <linux/kobject.h>
#include <linux/sysfs.h>
#include <net/sock.h>
/* BTF (BPF Type Format) is the meta data format which describes
@ -204,12 +205,19 @@ struct btf {
const char *strings;
void *nohdr_data;
struct btf_header hdr;
u32 nr_types;
u32 nr_types; /* includes VOID for base BTF */
u32 types_size;
u32 data_size;
refcount_t refcnt;
u32 id;
struct rcu_head rcu;
/* split BTF support */
struct btf *base_btf;
u32 start_id; /* first type ID in this BTF (0 for base BTF) */
u32 start_str_off; /* first string offset (0 for base BTF) */
char name[MODULE_NAME_LEN];
bool kernel_btf;
};
enum verifier_phase {
@ -450,14 +458,27 @@ static bool btf_type_is_datasec(const struct btf_type *t)
return BTF_INFO_KIND(t->info) == BTF_KIND_DATASEC;
}
static u32 btf_nr_types_total(const struct btf *btf)
{
u32 total = 0;
while (btf) {
total += btf->nr_types;
btf = btf->base_btf;
}
return total;
}
s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind)
{
const struct btf_type *t;
const char *tname;
u32 i;
u32 i, total;
for (i = 1; i <= btf->nr_types; i++) {
t = btf->types[i];
total = btf_nr_types_total(btf);
for (i = 1; i < total; i++) {
t = btf_type_by_id(btf, i);
if (BTF_INFO_KIND(t->info) != kind)
continue;
@ -600,8 +621,14 @@ static const struct btf_kind_operations *btf_type_ops(const struct btf_type *t)
static bool btf_name_offset_valid(const struct btf *btf, u32 offset)
{
return BTF_STR_OFFSET_VALID(offset) &&
offset < btf->hdr.str_len;
if (!BTF_STR_OFFSET_VALID(offset))
return false;
while (offset < btf->start_str_off)
btf = btf->base_btf;
offset -= btf->start_str_off;
return offset < btf->hdr.str_len;
}
static bool __btf_name_char_ok(char c, bool first, bool dot_ok)
@ -615,10 +642,22 @@ static bool __btf_name_char_ok(char c, bool first, bool dot_ok)
return true;
}
static const char *btf_str_by_offset(const struct btf *btf, u32 offset)
{
while (offset < btf->start_str_off)
btf = btf->base_btf;
offset -= btf->start_str_off;
if (offset < btf->hdr.str_len)
return &btf->strings[offset];
return NULL;
}
static bool __btf_name_valid(const struct btf *btf, u32 offset, bool dot_ok)
{
/* offset must be valid */
const char *src = &btf->strings[offset];
const char *src = btf_str_by_offset(btf, offset);
const char *src_limit;
if (!__btf_name_char_ok(*src, true, dot_ok))
@ -651,27 +690,28 @@ static bool btf_name_valid_section(const struct btf *btf, u32 offset)
static const char *__btf_name_by_offset(const struct btf *btf, u32 offset)
{
const char *name;
if (!offset)
return "(anon)";
else if (offset < btf->hdr.str_len)
return &btf->strings[offset];
else
return "(invalid-name-offset)";
name = btf_str_by_offset(btf, offset);
return name ?: "(invalid-name-offset)";
}
const char *btf_name_by_offset(const struct btf *btf, u32 offset)
{
if (offset < btf->hdr.str_len)
return &btf->strings[offset];
return NULL;
return btf_str_by_offset(btf, offset);
}
const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
{
if (type_id > btf->nr_types)
return NULL;
while (type_id < btf->start_id)
btf = btf->base_btf;
type_id -= btf->start_id;
if (type_id >= btf->nr_types)
return NULL;
return btf->types[type_id];
}
@ -1391,17 +1431,13 @@ static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
{
struct btf *btf = env->btf;
/* < 2 because +1 for btf_void which is always in btf->types[0].
* btf_void is not accounted in btf->nr_types because btf_void
* does not come from the BTF file.
*/
if (btf->types_size - btf->nr_types < 2) {
if (btf->types_size == btf->nr_types) {
/* Expand 'types' array */
struct btf_type **new_types;
u32 expand_by, new_size;
if (btf->types_size == BTF_MAX_TYPE) {
if (btf->start_id + btf->types_size == BTF_MAX_TYPE) {
btf_verifier_log(env, "Exceeded max num of types");
return -E2BIG;
}
@ -1415,18 +1451,23 @@ static int btf_add_type(struct btf_verifier_env *env, struct btf_type *t)
if (!new_types)
return -ENOMEM;
if (btf->nr_types == 0)
new_types[0] = &btf_void;
else
if (btf->nr_types == 0) {
if (!btf->base_btf) {
/* lazily init VOID type */
new_types[0] = &btf_void;
btf->nr_types++;
}
} else {
memcpy(new_types, btf->types,
sizeof(*btf->types) * (btf->nr_types + 1));
sizeof(*btf->types) * btf->nr_types);
}
kvfree(btf->types);
btf->types = new_types;
btf->types_size = new_size;
}
btf->types[++(btf->nr_types)] = t;
btf->types[btf->nr_types++] = t;
return 0;
}
@ -1499,18 +1540,17 @@ static int env_resolve_init(struct btf_verifier_env *env)
u32 *resolved_ids = NULL;
u8 *visit_states = NULL;
/* +1 for btf_void */
resolved_sizes = kvcalloc(nr_types + 1, sizeof(*resolved_sizes),
resolved_sizes = kvcalloc(nr_types, sizeof(*resolved_sizes),
GFP_KERNEL | __GFP_NOWARN);
if (!resolved_sizes)
goto nomem;
resolved_ids = kvcalloc(nr_types + 1, sizeof(*resolved_ids),
resolved_ids = kvcalloc(nr_types, sizeof(*resolved_ids),
GFP_KERNEL | __GFP_NOWARN);
if (!resolved_ids)
goto nomem;
visit_states = kvcalloc(nr_types + 1, sizeof(*visit_states),
visit_states = kvcalloc(nr_types, sizeof(*visit_states),
GFP_KERNEL | __GFP_NOWARN);
if (!visit_states)
goto nomem;
@ -1562,21 +1602,27 @@ static bool env_type_is_resolve_sink(const struct btf_verifier_env *env,
static bool env_type_is_resolved(const struct btf_verifier_env *env,
u32 type_id)
{
return env->visit_states[type_id] == RESOLVED;
/* base BTF types should be resolved by now */
if (type_id < env->btf->start_id)
return true;
return env->visit_states[type_id - env->btf->start_id] == RESOLVED;
}
static int env_stack_push(struct btf_verifier_env *env,
const struct btf_type *t, u32 type_id)
{
const struct btf *btf = env->btf;
struct resolve_vertex *v;
if (env->top_stack == MAX_RESOLVE_DEPTH)
return -E2BIG;
if (env->visit_states[type_id] != NOT_VISITED)
if (type_id < btf->start_id
|| env->visit_states[type_id - btf->start_id] != NOT_VISITED)
return -EEXIST;
env->visit_states[type_id] = VISITED;
env->visit_states[type_id - btf->start_id] = VISITED;
v = &env->stack[env->top_stack++];
v->t = t;
@ -1606,6 +1652,7 @@ static void env_stack_pop_resolved(struct btf_verifier_env *env,
u32 type_id = env->stack[--(env->top_stack)].type_id;
struct btf *btf = env->btf;
type_id -= btf->start_id; /* adjust to local type id */
btf->resolved_sizes[type_id] = resolved_size;
btf->resolved_ids[type_id] = resolved_type_id;
env->visit_states[type_id] = RESOLVED;
@ -1710,14 +1757,30 @@ btf_resolve_size(const struct btf *btf, const struct btf_type *type,
return __btf_resolve_size(btf, type, type_size, NULL, NULL, NULL, NULL);
}
static u32 btf_resolved_type_id(const struct btf *btf, u32 type_id)
{
while (type_id < btf->start_id)
btf = btf->base_btf;
return btf->resolved_ids[type_id - btf->start_id];
}
/* The input param "type_id" must point to a needs_resolve type */
static const struct btf_type *btf_type_id_resolve(const struct btf *btf,
u32 *type_id)
{
*type_id = btf->resolved_ids[*type_id];
*type_id = btf_resolved_type_id(btf, *type_id);
return btf_type_by_id(btf, *type_id);
}
static u32 btf_resolved_type_size(const struct btf *btf, u32 type_id)
{
while (type_id < btf->start_id)
btf = btf->base_btf;
return btf->resolved_sizes[type_id - btf->start_id];
}
const struct btf_type *btf_type_id_size(const struct btf *btf,
u32 *type_id, u32 *ret_size)
{
@ -1732,7 +1795,7 @@ const struct btf_type *btf_type_id_size(const struct btf *btf,
if (btf_type_has_size(size_type)) {
size = size_type->size;
} else if (btf_type_is_array(size_type)) {
size = btf->resolved_sizes[size_type_id];
size = btf_resolved_type_size(btf, size_type_id);
} else if (btf_type_is_ptr(size_type)) {
size = sizeof(void *);
} else {
@ -1740,14 +1803,14 @@ const struct btf_type *btf_type_id_size(const struct btf *btf,
!btf_type_is_var(size_type)))
return NULL;
size_type_id = btf->resolved_ids[size_type_id];
size_type_id = btf_resolved_type_id(btf, size_type_id);
size_type = btf_type_by_id(btf, size_type_id);
if (btf_type_nosize_or_null(size_type))
return NULL;
else if (btf_type_has_size(size_type))
size = size_type->size;
else if (btf_type_is_array(size_type))
size = btf->resolved_sizes[size_type_id];
size = btf_resolved_type_size(btf, size_type_id);
else if (btf_type_is_ptr(size_type))
size = sizeof(void *);
else
@ -3799,7 +3862,7 @@ static int btf_check_all_metas(struct btf_verifier_env *env)
cur = btf->nohdr_data + hdr->type_off;
end = cur + hdr->type_len;
env->log_type_id = 1;
env->log_type_id = btf->base_btf ? btf->start_id : 1;
while (cur < end) {
struct btf_type *t = cur;
s32 meta_size;
@ -3826,8 +3889,8 @@ static bool btf_resolve_valid(struct btf_verifier_env *env,
return false;
if (btf_type_is_struct(t) || btf_type_is_datasec(t))
return !btf->resolved_ids[type_id] &&
!btf->resolved_sizes[type_id];
return !btf_resolved_type_id(btf, type_id) &&
!btf_resolved_type_size(btf, type_id);
if (btf_type_is_modifier(t) || btf_type_is_ptr(t) ||
btf_type_is_var(t)) {
@ -3847,7 +3910,7 @@ static bool btf_resolve_valid(struct btf_verifier_env *env,
elem_type = btf_type_id_size(btf, &elem_type_id, &elem_size);
return elem_type && !btf_type_is_modifier(elem_type) &&
(array->nelems * elem_size ==
btf->resolved_sizes[type_id]);
btf_resolved_type_size(btf, type_id));
}
return false;
@ -3889,7 +3952,8 @@ static int btf_resolve(struct btf_verifier_env *env,
static int btf_check_all_types(struct btf_verifier_env *env)
{
struct btf *btf = env->btf;
u32 type_id;
const struct btf_type *t;
u32 type_id, i;
int err;
err = env_resolve_init(env);
@ -3897,8 +3961,9 @@ static int btf_check_all_types(struct btf_verifier_env *env)
return err;
env->phase++;
for (type_id = 1; type_id <= btf->nr_types; type_id++) {
const struct btf_type *t = btf_type_by_id(btf, type_id);
for (i = btf->base_btf ? 0 : 1; i < btf->nr_types; i++) {
type_id = btf->start_id + i;
t = btf_type_by_id(btf, type_id);
env->log_type_id = type_id;
if (btf_type_needs_resolve(t) &&
@ -3935,7 +4000,7 @@ static int btf_parse_type_sec(struct btf_verifier_env *env)
return -EINVAL;
}
if (!hdr->type_len) {
if (!env->btf->base_btf && !hdr->type_len) {
btf_verifier_log(env, "No type found");
return -EINVAL;
}
@ -3962,13 +4027,18 @@ static int btf_parse_str_sec(struct btf_verifier_env *env)
return -EINVAL;
}
if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET ||
start[0] || end[-1]) {
btf->strings = start;
if (btf->base_btf && !hdr->str_len)
return 0;
if (!hdr->str_len || hdr->str_len - 1 > BTF_MAX_NAME_OFFSET || end[-1]) {
btf_verifier_log(env, "Invalid string section");
return -EINVAL;
}
if (!btf->base_btf && start[0]) {
btf_verifier_log(env, "Invalid string section");
return -EINVAL;
}
btf->strings = start;
return 0;
}
@ -4363,6 +4433,8 @@ struct btf *btf_parse_vmlinux(void)
btf->data = __start_BTF;
btf->data_size = __stop_BTF - __start_BTF;
btf->kernel_btf = true;
snprintf(btf->name, sizeof(btf->name), "vmlinux");
err = btf_parse_hdr(env);
if (err)
@ -4388,8 +4460,13 @@ struct btf *btf_parse_vmlinux(void)
bpf_struct_ops_init(btf, log);
btf_verifier_env_free(env);
refcount_set(&btf->refcnt, 1);
err = btf_alloc_id(btf);
if (err)
goto errout;
btf_verifier_env_free(env);
return btf;
errout:
@ -4401,6 +4478,79 @@ struct btf *btf_parse_vmlinux(void)
return ERR_PTR(err);
}
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
static struct btf *btf_parse_module(const char *module_name, const void *data, unsigned int data_size)
{
struct btf_verifier_env *env = NULL;
struct bpf_verifier_log *log;
struct btf *btf = NULL, *base_btf;
int err;
base_btf = bpf_get_btf_vmlinux();
if (IS_ERR(base_btf))
return base_btf;
if (!base_btf)
return ERR_PTR(-EINVAL);
env = kzalloc(sizeof(*env), GFP_KERNEL | __GFP_NOWARN);
if (!env)
return ERR_PTR(-ENOMEM);
log = &env->log;
log->level = BPF_LOG_KERNEL;
btf = kzalloc(sizeof(*btf), GFP_KERNEL | __GFP_NOWARN);
if (!btf) {
err = -ENOMEM;
goto errout;
}
env->btf = btf;
btf->base_btf = base_btf;
btf->start_id = base_btf->nr_types;
btf->start_str_off = base_btf->hdr.str_len;
btf->kernel_btf = true;
snprintf(btf->name, sizeof(btf->name), "%s", module_name);
btf->data = kvmalloc(data_size, GFP_KERNEL | __GFP_NOWARN);
if (!btf->data) {
err = -ENOMEM;
goto errout;
}
memcpy(btf->data, data, data_size);
btf->data_size = data_size;
err = btf_parse_hdr(env);
if (err)
goto errout;
btf->nohdr_data = btf->data + btf->hdr.hdr_len;
err = btf_parse_str_sec(env);
if (err)
goto errout;
err = btf_check_all_metas(env);
if (err)
goto errout;
btf_verifier_env_free(env);
refcount_set(&btf->refcnt, 1);
return btf;
errout:
btf_verifier_env_free(env);
if (btf) {
kvfree(btf->data);
kvfree(btf->types);
kfree(btf);
}
return ERR_PTR(err);
}
#endif /* CONFIG_DEBUG_INFO_BTF_MODULES */
struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog)
{
struct bpf_prog *tgt_prog = prog->aux->dst_prog;
@ -4909,7 +5059,7 @@ static int __get_type_size(struct btf *btf, u32 btf_id,
while (t && btf_type_is_modifier(t))
t = btf_type_by_id(btf, t->type);
if (!t) {
*bad_type = btf->types[0];
*bad_type = btf_type_by_id(btf, 0);
return -EINVAL;
}
if (btf_type_is_ptr(t))
@ -5487,7 +5637,9 @@ int btf_get_info_by_fd(const struct btf *btf,
struct bpf_btf_info info;
u32 info_copy, btf_copy;
void __user *ubtf;
u32 uinfo_len;
char __user *uname;
u32 uinfo_len, uname_len, name_len;
int ret = 0;
uinfo = u64_to_user_ptr(attr->info.info);
uinfo_len = attr->info.info_len;
@ -5504,11 +5656,37 @@ int btf_get_info_by_fd(const struct btf *btf,
return -EFAULT;
info.btf_size = btf->data_size;
info.kernel_btf = btf->kernel_btf;
uname = u64_to_user_ptr(info.name);
uname_len = info.name_len;
if (!uname ^ !uname_len)
return -EINVAL;
name_len = strlen(btf->name);
info.name_len = name_len;
if (uname) {
if (uname_len >= name_len + 1) {
if (copy_to_user(uname, btf->name, name_len + 1))
return -EFAULT;
} else {
char zero = '\0';
if (copy_to_user(uname, btf->name, uname_len - 1))
return -EFAULT;
if (put_user(zero, uname + uname_len - 1))
return -EFAULT;
/* let user-space know about too short buffer */
ret = -ENOSPC;
}
}
if (copy_to_user(uinfo, &info, info_copy) ||
put_user(info_copy, &uattr->info.info_len))
return -EFAULT;
return 0;
return ret;
}
int btf_get_fd_by_id(u32 id)
@ -5548,3 +5726,126 @@ bool btf_id_set_contains(const struct btf_id_set *set, u32 id)
{
return bsearch(&id, set->ids, set->cnt, sizeof(u32), btf_id_cmp_func) != NULL;
}
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
struct btf_module {
struct list_head list;
struct module *module;
struct btf *btf;
struct bin_attribute *sysfs_attr;
};
static LIST_HEAD(btf_modules);
static DEFINE_MUTEX(btf_module_mutex);
static ssize_t
btf_module_read(struct file *file, struct kobject *kobj,
struct bin_attribute *bin_attr,
char *buf, loff_t off, size_t len)
{
const struct btf *btf = bin_attr->private;
memcpy(buf, btf->data + off, len);
return len;
}
static int btf_module_notify(struct notifier_block *nb, unsigned long op,
void *module)
{
struct btf_module *btf_mod, *tmp;
struct module *mod = module;
struct btf *btf;
int err = 0;
if (mod->btf_data_size == 0 ||
(op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
goto out;
switch (op) {
case MODULE_STATE_COMING:
btf_mod = kzalloc(sizeof(*btf_mod), GFP_KERNEL);
if (!btf_mod) {
err = -ENOMEM;
goto out;
}
btf = btf_parse_module(mod->name, mod->btf_data, mod->btf_data_size);
if (IS_ERR(btf)) {
pr_warn("failed to validate module [%s] BTF: %ld\n",
mod->name, PTR_ERR(btf));
kfree(btf_mod);
err = PTR_ERR(btf);
goto out;
}
err = btf_alloc_id(btf);
if (err) {
btf_free(btf);
kfree(btf_mod);
goto out;
}
mutex_lock(&btf_module_mutex);
btf_mod->module = module;
btf_mod->btf = btf;
list_add(&btf_mod->list, &btf_modules);
mutex_unlock(&btf_module_mutex);
if (IS_ENABLED(CONFIG_SYSFS)) {
struct bin_attribute *attr;
attr = kzalloc(sizeof(*attr), GFP_KERNEL);
if (!attr)
goto out;
sysfs_bin_attr_init(attr);
attr->attr.name = btf->name;
attr->attr.mode = 0444;
attr->size = btf->data_size;
attr->private = btf;
attr->read = btf_module_read;
err = sysfs_create_bin_file(btf_kobj, attr);
if (err) {
pr_warn("failed to register module [%s] BTF in sysfs: %d\n",
mod->name, err);
kfree(attr);
err = 0;
goto out;
}
btf_mod->sysfs_attr = attr;
}
break;
case MODULE_STATE_GOING:
mutex_lock(&btf_module_mutex);
list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) {
if (btf_mod->module != module)
continue;
list_del(&btf_mod->list);
if (btf_mod->sysfs_attr)
sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr);
btf_put(btf_mod->btf);
kfree(btf_mod->sysfs_attr);
kfree(btf_mod);
break;
}
mutex_unlock(&btf_module_mutex);
break;
}
out:
return notifier_from_errno(err);
}
static struct notifier_block btf_module_nb = {
.notifier_call = btf_module_notify,
};
static int __init btf_module_init(void)
{
register_module_notifier(&btf_module_nb);
return 0;
}
fs_initcall(btf_module_init);
#endif /* CONFIG_DEBUG_INFO_BTF_MODULES */

144
kernel/bpf/hashtab.c
View File

@ -86,6 +86,9 @@ struct bucket {
};
};
#define HASHTAB_MAP_LOCK_COUNT 8
#define HASHTAB_MAP_LOCK_MASK (HASHTAB_MAP_LOCK_COUNT - 1)
struct bpf_htab {
struct bpf_map map;
struct bucket *buckets;
@ -99,6 +102,8 @@ struct bpf_htab {
u32 n_buckets; /* number of hash buckets */
u32 elem_size; /* size of each element in bytes */
u32 hashrnd;
struct lock_class_key lockdep_key;
int __percpu *map_locked[HASHTAB_MAP_LOCK_COUNT];
};
/* each htab element is struct htab_elem + key + value */
@ -138,33 +143,53 @@ static void htab_init_buckets(struct bpf_htab *htab)
for (i = 0; i < htab->n_buckets; i++) {
INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i);
if (htab_use_raw_lock(htab))
if (htab_use_raw_lock(htab)) {
raw_spin_lock_init(&htab->buckets[i].raw_lock);
else
lockdep_set_class(&htab->buckets[i].raw_lock,
&htab->lockdep_key);
} else {
spin_lock_init(&htab->buckets[i].lock);
lockdep_set_class(&htab->buckets[i].lock,
&htab->lockdep_key);
}
}
}
static inline unsigned long htab_lock_bucket(const struct bpf_htab *htab,
struct bucket *b)
static inline int htab_lock_bucket(const struct bpf_htab *htab,
struct bucket *b, u32 hash,
unsigned long *pflags)
{
unsigned long flags;
hash = hash & HASHTAB_MAP_LOCK_MASK;
migrate_disable();
if (unlikely(__this_cpu_inc_return(*(htab->map_locked[hash])) != 1)) {
__this_cpu_dec(*(htab->map_locked[hash]));
migrate_enable();
return -EBUSY;
}
if (htab_use_raw_lock(htab))
raw_spin_lock_irqsave(&b->raw_lock, flags);
else
spin_lock_irqsave(&b->lock, flags);
return flags;
*pflags = flags;
return 0;
}
static inline void htab_unlock_bucket(const struct bpf_htab *htab,
struct bucket *b,
struct bucket *b, u32 hash,
unsigned long flags)
{
hash = hash & HASHTAB_MAP_LOCK_MASK;
if (htab_use_raw_lock(htab))
raw_spin_unlock_irqrestore(&b->raw_lock, flags);
else
spin_unlock_irqrestore(&b->lock, flags);
__this_cpu_dec(*(htab->map_locked[hash]));
migrate_enable();
}
static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node);
@ -390,17 +415,11 @@ static int htab_map_alloc_check(union bpf_attr *attr)
attr->value_size == 0)
return -EINVAL;
if (attr->key_size > MAX_BPF_STACK)
/* eBPF programs initialize keys on stack, so they cannot be
* larger than max stack size
*/
return -E2BIG;
if (attr->value_size >= KMALLOC_MAX_SIZE -
MAX_BPF_STACK - sizeof(struct htab_elem))
/* if value_size is bigger, the user space won't be able to
* access the elements via bpf syscall. This check also makes
* sure that the elem_size doesn't overflow and it's
if ((u64)attr->key_size + attr->value_size >= KMALLOC_MAX_SIZE -
sizeof(struct htab_elem))
/* if key_size + value_size is bigger, the user space won't be
* able to access the elements via bpf syscall. This check
* also makes sure that the elem_size doesn't overflow and it's
* kmalloc-able later in htab_map_update_elem()
*/
return -E2BIG;
@ -422,13 +441,15 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
bool percpu_lru = (attr->map_flags & BPF_F_NO_COMMON_LRU);
bool prealloc = !(attr->map_flags & BPF_F_NO_PREALLOC);
struct bpf_htab *htab;
int err, i;
u64 cost;
int err;
htab = kzalloc(sizeof(*htab), GFP_USER);
if (!htab)
return ERR_PTR(-ENOMEM);
lockdep_register_key(&htab->lockdep_key);
bpf_map_init_from_attr(&htab->map, attr);
if (percpu_lru) {
@ -480,6 +501,13 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
if (!htab->buckets)
goto free_charge;
for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++) {
htab->map_locked[i] = __alloc_percpu_gfp(sizeof(int),
sizeof(int), GFP_USER);
if (!htab->map_locked[i])
goto free_map_locked;
}
if (htab->map.map_flags & BPF_F_ZERO_SEED)
htab->hashrnd = 0;
else
@ -490,7 +518,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
if (prealloc) {
err = prealloc_init(htab);
if (err)
goto free_buckets;
goto free_map_locked;
if (!percpu && !lru) {
/* lru itself can remove the least used element, so
@ -506,11 +534,14 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
free_prealloc:
prealloc_destroy(htab);
free_buckets:
free_map_locked:
for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++)
free_percpu(htab->map_locked[i]);
bpf_map_area_free(htab->buckets);
free_charge:
bpf_map_charge_finish(&htab->map.memory);
free_htab:
lockdep_unregister_key(&htab->lockdep_key);
kfree(htab);
return ERR_PTR(err);
}
@ -687,12 +718,15 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
struct hlist_nulls_node *n;
unsigned long flags;
struct bucket *b;
int ret;
tgt_l = container_of(node, struct htab_elem, lru_node);
b = __select_bucket(htab, tgt_l->hash);
head = &b->head;
flags = htab_lock_bucket(htab, b);
ret = htab_lock_bucket(htab, b, tgt_l->hash, &flags);
if (ret)
return false;
hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
if (l == tgt_l) {
@ -700,7 +734,7 @@ static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node)
break;
}
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, tgt_l->hash, flags);
return l == tgt_l;
}
@ -998,7 +1032,9 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,
*/
}
flags = htab_lock_bucket(htab, b);
ret = htab_lock_bucket(htab, b, hash, &flags);
if (ret)
return ret;
l_old = lookup_elem_raw(head, hash, key, key_size);
@ -1039,7 +1075,7 @@ static int htab_map_update_elem(struct bpf_map *map, void *key, void *value,
}
ret = 0;
err:
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, hash, flags);
return ret;
}
@ -1077,7 +1113,9 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
return -ENOMEM;
memcpy(l_new->key + round_up(map->key_size, 8), value, map->value_size);
flags = htab_lock_bucket(htab, b);
ret = htab_lock_bucket(htab, b, hash, &flags);
if (ret)
return ret;
l_old = lookup_elem_raw(head, hash, key, key_size);
@ -1096,7 +1134,7 @@ static int htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value,
ret = 0;
err:
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, hash, flags);
if (ret)
bpf_lru_push_free(&htab->lru, &l_new->lru_node);
@ -1131,7 +1169,9 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,
b = __select_bucket(htab, hash);
head = &b->head;
flags = htab_lock_bucket(htab, b);
ret = htab_lock_bucket(htab, b, hash, &flags);
if (ret)
return ret;
l_old = lookup_elem_raw(head, hash, key, key_size);
@ -1154,7 +1194,7 @@ static int __htab_percpu_map_update_elem(struct bpf_map *map, void *key,
}
ret = 0;
err:
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, hash, flags);
return ret;
}
@ -1194,7 +1234,9 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
return -ENOMEM;
}
flags = htab_lock_bucket(htab, b);
ret = htab_lock_bucket(htab, b, hash, &flags);
if (ret)
return ret;
l_old = lookup_elem_raw(head, hash, key, key_size);
@ -1216,7 +1258,7 @@ static int __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key,
}
ret = 0;
err:
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, hash, flags);
if (l_new)
bpf_lru_push_free(&htab->lru, &l_new->lru_node);
return ret;
@ -1244,7 +1286,7 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key)
struct htab_elem *l;
unsigned long flags;
u32 hash, key_size;
int ret = -ENOENT;
int ret;
WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
@ -1254,17 +1296,20 @@ static int htab_map_delete_elem(struct bpf_map *map, void *key)
b = __select_bucket(htab, hash);
head = &b->head;
flags = htab_lock_bucket(htab, b);
ret = htab_lock_bucket(htab, b, hash, &flags);
if (ret)
return ret;
l = lookup_elem_raw(head, hash, key, key_size);
if (l) {
hlist_nulls_del_rcu(&l->hash_node);
free_htab_elem(htab, l);
ret = 0;
} else {
ret = -ENOENT;
}
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, hash, flags);
return ret;
}
@ -1276,7 +1321,7 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)
struct htab_elem *l;
unsigned long flags;
u32 hash, key_size;
int ret = -ENOENT;
int ret;
WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_trace_held());
@ -1286,16 +1331,18 @@ static int htab_lru_map_delete_elem(struct bpf_map *map, void *key)
b = __select_bucket(htab, hash);
head = &b->head;
flags = htab_lock_bucket(htab, b);
ret = htab_lock_bucket(htab, b, hash, &flags);
if (ret)
return ret;
l = lookup_elem_raw(head, hash, key, key_size);
if (l) {
if (l)
hlist_nulls_del_rcu(&l->hash_node);
ret = 0;
}
else
ret = -ENOENT;
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, hash, flags);
if (l)
bpf_lru_push_free(&htab->lru, &l->lru_node);
return ret;
@ -1321,6 +1368,7 @@ static void delete_all_elements(struct bpf_htab *htab)
static void htab_map_free(struct bpf_map *map)
{
struct bpf_htab *htab = container_of(map, struct bpf_htab, map);
int i;
/* bpf_free_used_maps() or close(map_fd) will trigger this map_free callback.
* bpf_free_used_maps() is called after bpf prog is no longer executing.
@ -1338,6 +1386,9 @@ static void htab_map_free(struct bpf_map *map)
free_percpu(htab->extra_elems);
bpf_map_area_free(htab->buckets);
for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++)
free_percpu(htab->map_locked[i]);
lockdep_unregister_key(&htab->lockdep_key);
kfree(htab);
}
@ -1441,8 +1492,11 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
b = &htab->buckets[batch];
head = &b->head;
/* do not grab the lock unless need it (bucket_cnt > 0). */
if (locked)
flags = htab_lock_bucket(htab, b);
if (locked) {
ret = htab_lock_bucket(htab, b, batch, &flags);
if (ret)
goto next_batch;
}
bucket_cnt = 0;
hlist_nulls_for_each_entry_rcu(l, n, head, hash_node)
@ -1459,7 +1513,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
/* Note that since bucket_cnt > 0 here, it is implicit
* that the locked was grabbed, so release it.
*/
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, batch, flags);
rcu_read_unlock();
bpf_enable_instrumentation();
goto after_loop;
@ -1470,7 +1524,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
/* Note that since bucket_cnt > 0 here, it is implicit
* that the locked was grabbed, so release it.
*/
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, batch, flags);
rcu_read_unlock();
bpf_enable_instrumentation();
kvfree(keys);
@ -1523,7 +1577,7 @@ __htab_map_lookup_and_delete_batch(struct bpf_map *map,
dst_val += value_size;
}
htab_unlock_bucket(htab, b, flags);
htab_unlock_bucket(htab, b, batch, flags);
locked = false;
while (node_to_free) {

3
kernel/bpf/syscall.c
View File

@ -773,7 +773,8 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf,
map->map_type != BPF_MAP_TYPE_ARRAY &&
map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
map->map_type != BPF_MAP_TYPE_SK_STORAGE &&
map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
map->map_type != BPF_MAP_TYPE_INODE_STORAGE &&
map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
return -ENOTSUPP;
if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
map->value_size) {

2
kernel/bpf/sysfs_btf.c
View File

@ -26,7 +26,7 @@ static struct bin_attribute bin_attr_btf_vmlinux __ro_after_init = {
.read = btf_vmlinux_read,
};
static struct kobject *btf_kobj;
struct kobject *btf_kobj;
static int __init btf_vmlinux_init(void)
{

2
kernel/bpf/task_iter.c
View File

@ -337,6 +337,7 @@ static const struct bpf_iter_seq_info task_seq_info = {
static struct bpf_iter_reg task_reg_info = {
.target = "task",
.feature = BPF_ITER_RESCHED,
.ctx_arg_info_size = 1,
.ctx_arg_info = {
{ offsetof(struct bpf_iter__task, task),
@ -354,6 +355,7 @@ static const struct bpf_iter_seq_info task_file_seq_info = {
static struct bpf_iter_reg task_file_reg_info = {
.target = "task_file",
.feature = BPF_ITER_RESCHED,
.ctx_arg_info_size = 2,
.ctx_arg_info = {
{ offsetof(struct bpf_iter__task_file, task),

182
kernel/bpf/verifier.c
View File

@ -2739,7 +2739,9 @@ static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
regno);
return -EACCES;
}
err = __check_mem_access(env, regno, off, size, reg->range,
err = reg->range < 0 ? -EINVAL :
__check_mem_access(env, regno, off, size, reg->range,
zero_size_allowed);
if (err) {
verbose(env, "R%d offset is outside of the packet\n", regno);
@ -4469,6 +4471,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
func_id != BPF_FUNC_inode_storage_delete)
goto error;
break;
case BPF_MAP_TYPE_TASK_STORAGE:
if (func_id != BPF_FUNC_task_storage_get &&
func_id != BPF_FUNC_task_storage_delete)
goto error;
break;
default:
break;
}
@ -4547,6 +4554,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
goto error;
break;
case BPF_FUNC_task_storage_get:
case BPF_FUNC_task_storage_delete:
if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
goto error;
break;
default:
break;
}
@ -4687,6 +4699,32 @@ static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
__clear_all_pkt_pointers(env, vstate->frame[i]);
}
enum {
AT_PKT_END = -1,
BEYOND_PKT_END = -2,
};
static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open)
{
struct bpf_func_state *state = vstate->frame[vstate->curframe];
struct bpf_reg_state *reg = &state->regs[regn];
if (reg->type != PTR_TO_PACKET)
/* PTR_TO_PACKET_META is not supported yet */
return;
/* The 'reg' is pkt > pkt_end or pkt >= pkt_end.
* How far beyond pkt_end it goes is unknown.
* if (!range_open) it's the case of pkt >= pkt_end
* if (range_open) it's the case of pkt > pkt_end
* hence this pointer is at least 1 byte bigger than pkt_end
*/
if (range_open)
reg->range = BEYOND_PKT_END;
else
reg->range = AT_PKT_END;
}
static void release_reg_references(struct bpf_verifier_env *env,
struct bpf_func_state *state,
int ref_obj_id)
@ -5176,11 +5214,14 @@ static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn
PTR_TO_BTF_ID : PTR_TO_BTF_ID_OR_NULL;
regs[BPF_REG_0].btf_id = meta.ret_btf_id;
}
} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL) {
} else if (fn->ret_type == RET_PTR_TO_BTF_ID_OR_NULL ||
fn->ret_type == RET_PTR_TO_BTF_ID) {
int ret_btf_id;
mark_reg_known_zero(env, regs, BPF_REG_0);
regs[BPF_REG_0].type = PTR_TO_BTF_ID_OR_NULL;
regs[BPF_REG_0].type = fn->ret_type == RET_PTR_TO_BTF_ID ?
PTR_TO_BTF_ID :
PTR_TO_BTF_ID_OR_NULL;
ret_btf_id = *fn->ret_btf_id;
if (ret_btf_id == 0) {
verbose(env, "invalid return type %d of func %s#%d\n",
@ -6695,7 +6736,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
static void __find_good_pkt_pointers(struct bpf_func_state *state,
struct bpf_reg_state *dst_reg,
enum bpf_reg_type type, u16 new_range)
enum bpf_reg_type type, int new_range)
{
struct bpf_reg_state *reg;
int i;
@ -6720,8 +6761,7 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
enum bpf_reg_type type,
bool range_right_open)
{
u16 new_range;
int i;
int new_range, i;
if (dst_reg->off < 0 ||
(dst_reg->off == 0 && range_right_open))
@ -6972,6 +7012,67 @@ static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
return is_branch64_taken(reg, val, opcode);
}
static int flip_opcode(u32 opcode)
{
/* How can we transform "a <op> b" into "b <op> a"? */
static const u8 opcode_flip[16] = {
/* these stay the same */
[BPF_JEQ >> 4] = BPF_JEQ,
[BPF_JNE >> 4] = BPF_JNE,
[BPF_JSET >> 4] = BPF_JSET,
/* these swap "lesser" and "greater" (L and G in the opcodes) */
[BPF_JGE >> 4] = BPF_JLE,
[BPF_JGT >> 4] = BPF_JLT,
[BPF_JLE >> 4] = BPF_JGE,
[BPF_JLT >> 4] = BPF_JGT,
[BPF_JSGE >> 4] = BPF_JSLE,
[BPF_JSGT >> 4] = BPF_JSLT,
[BPF_JSLE >> 4] = BPF_JSGE,
[BPF_JSLT >> 4] = BPF_JSGT
};
return opcode_flip[opcode >> 4];
}
static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
struct bpf_reg_state *src_reg,
u8 opcode)
{
struct bpf_reg_state *pkt;
if (src_reg->type == PTR_TO_PACKET_END) {
pkt = dst_reg;
} else if (dst_reg->type == PTR_TO_PACKET_END) {
pkt = src_reg;
opcode = flip_opcode(opcode);
} else {
return -1;
}
if (pkt->range >= 0)
return -1;
switch (opcode) {
case BPF_JLE:
/* pkt <= pkt_end */
fallthrough;
case BPF_JGT:
/* pkt > pkt_end */
if (pkt->range == BEYOND_PKT_END)
/* pkt has at last one extra byte beyond pkt_end */
return opcode == BPF_JGT;
break;
case BPF_JLT:
/* pkt < pkt_end */
fallthrough;
case BPF_JGE:
/* pkt >= pkt_end */
if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END)
return opcode == BPF_JGE;
break;
}
return -1;
}
/* Adjusts the register min/max values in the case that the dst_reg is the
* variable register that we are working on, and src_reg is a constant or we're
* simply doing a BPF_K check.
@ -7135,23 +7236,7 @@ static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
u64 val, u32 val32,
u8 opcode, bool is_jmp32)
{
/* How can we transform "a <op> b" into "b <op> a"? */
static const u8 opcode_flip[16] = {
/* these stay the same */
[BPF_JEQ >> 4] = BPF_JEQ,
[BPF_JNE >> 4] = BPF_JNE,
[BPF_JSET >> 4] = BPF_JSET,
/* these swap "lesser" and "greater" (L and G in the opcodes) */
[BPF_JGE >> 4] = BPF_JLE,
[BPF_JGT >> 4] = BPF_JLT,
[BPF_JLE >> 4] = BPF_JGE,
[BPF_JLT >> 4] = BPF_JGT,
[BPF_JSGE >> 4] = BPF_JSLE,
[BPF_JSGT >> 4] = BPF_JSLT,
[BPF_JSLE >> 4] = BPF_JSGE,
[BPF_JSLT >> 4] = BPF_JSGT
};
opcode = opcode_flip[opcode >> 4];
opcode = flip_opcode(opcode);
/* This uses zero as "not present in table"; luckily the zero opcode,
* BPF_JA, can't get here.
*/
@ -7333,6 +7418,7 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
/* pkt_data' > pkt_end, pkt_meta' > pkt_data */
find_good_pkt_pointers(this_branch, dst_reg,
dst_reg->type, false);
mark_pkt_end(other_branch, insn->dst_reg, true);
} else if ((dst_reg->type == PTR_TO_PACKET_END &&
src_reg->type == PTR_TO_PACKET) ||
(reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
@ -7340,6 +7426,7 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
/* pkt_end > pkt_data', pkt_data > pkt_meta' */
find_good_pkt_pointers(other_branch, src_reg,
src_reg->type, true);
mark_pkt_end(this_branch, insn->src_reg, false);
} else {
return false;
}
@ -7352,6 +7439,7 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
/* pkt_data' < pkt_end, pkt_meta' < pkt_data */
find_good_pkt_pointers(other_branch, dst_reg,
dst_reg->type, true);
mark_pkt_end(this_branch, insn->dst_reg, false);
} else if ((dst_reg->type == PTR_TO_PACKET_END &&
src_reg->type == PTR_TO_PACKET) ||
(reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
@ -7359,6 +7447,7 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
/* pkt_end < pkt_data', pkt_data > pkt_meta' */
find_good_pkt_pointers(this_branch, src_reg,
src_reg->type, false);
mark_pkt_end(other_branch, insn->src_reg, true);
} else {
return false;
}
@ -7371,6 +7460,7 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
/* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
find_good_pkt_pointers(this_branch, dst_reg,
dst_reg->type, true);
mark_pkt_end(other_branch, insn->dst_reg, false);
} else if ((dst_reg->type == PTR_TO_PACKET_END &&
src_reg->type == PTR_TO_PACKET) ||
(reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
@ -7378,6 +7468,7 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
/* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
find_good_pkt_pointers(other_branch, src_reg,
src_reg->type, false);
mark_pkt_end(this_branch, insn->src_reg, true);
} else {
return false;
}
@ -7390,6 +7481,7 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
/* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
find_good_pkt_pointers(other_branch, dst_reg,
dst_reg->type, false);
mark_pkt_end(this_branch, insn->dst_reg, true);
} else if ((dst_reg->type == PTR_TO_PACKET_END &&
src_reg->type == PTR_TO_PACKET) ||
(reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
@ -7397,6 +7489,7 @@ static bool try_match_pkt_pointers(const struct bpf_insn *insn,
/* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
find_good_pkt_pointers(this_branch, src_reg,
src_reg->type, true);
mark_pkt_end(other_branch, insn->src_reg, false);
} else {
return false;
}
@ -7496,6 +7589,10 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
src_reg->var_off.value,
opcode,
is_jmp32);
} else if (reg_is_pkt_pointer_any(dst_reg) &&
reg_is_pkt_pointer_any(src_reg) &&
!is_jmp32) {
pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode);
}
if (pred >= 0) {
@ -7504,7 +7601,8 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
*/
if (!__is_pointer_value(false, dst_reg))
err = mark_chain_precision(env, insn->dst_reg);
if (BPF_SRC(insn->code) == BPF_X && !err)
if (BPF_SRC(insn->code) == BPF_X && !err &&
!__is_pointer_value(false, src_reg))
err = mark_chain_precision(env, insn->src_reg);
if (err)
return err;
@ -9719,11 +9817,21 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env,
verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n");
}
if ((is_tracing_prog_type(prog_type) ||
prog_type == BPF_PROG_TYPE_SOCKET_FILTER) &&
map_value_has_spin_lock(map)) {
verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
return -EINVAL;
if (map_value_has_spin_lock(map)) {
if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
return -EINVAL;
}
if (is_tracing_prog_type(prog_type)) {
verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
return -EINVAL;
}
if (prog->aux->sleepable) {
verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n");
return -EINVAL;
}
}
if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
@ -11454,20 +11562,6 @@ static int check_attach_modify_return(unsigned long addr, const char *func_name)
return -EINVAL;
}
/* non exhaustive list of sleepable bpf_lsm_*() functions */
BTF_SET_START(btf_sleepable_lsm_hooks)
#ifdef CONFIG_BPF_LSM
BTF_ID(func, bpf_lsm_bprm_committed_creds)
#else
BTF_ID_UNUSED
#endif
BTF_SET_END(btf_sleepable_lsm_hooks)
static int check_sleepable_lsm_hook(u32 btf_id)
{
return btf_id_set_contains(&btf_sleepable_lsm_hooks, btf_id);
}
/* list of non-sleepable functions that are otherwise on
* ALLOW_ERROR_INJECTION list
*/
@ -11689,7 +11783,7 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
/* LSM progs check that they are attached to bpf_lsm_*() funcs.
* Only some of them are sleepable.
*/
if (check_sleepable_lsm_hook(btf_id))
if (bpf_lsm_is_sleepable_hook(btf_id))
ret = 0;
break;
default:

32
kernel/module.c
View File

@ -380,6 +380,35 @@ static void *section_objs(const struct load_info *info,
return (void *)info->sechdrs[sec].sh_addr;
}
/* Find a module section: 0 means not found. Ignores SHF_ALLOC flag. */
static unsigned int find_any_sec(const struct load_info *info, const char *name)
{
unsigned int i;
for (i = 1; i < info->hdr->e_shnum; i++) {
Elf_Shdr *shdr = &info->sechdrs[i];
if (strcmp(info->secstrings + shdr->sh_name, name) == 0)
return i;
}
return 0;
}
/*
* Find a module section, or NULL. Fill in number of "objects" in section.
* Ignores SHF_ALLOC flag.
*/
static __maybe_unused void *any_section_objs(const struct load_info *info,
const char *name,
size_t object_size,
unsigned int *num)
{
unsigned int sec = find_any_sec(info, name);
/* Section 0 has sh_addr 0 and sh_size 0. */
*num = info->sechdrs[sec].sh_size / object_size;
return (void *)info->sechdrs[sec].sh_addr;
}
/* Provided by the linker */
extern const struct kernel_symbol __start___ksymtab[];
extern const struct kernel_symbol __stop___ksymtab[];
@ -3250,6 +3279,9 @@ static int find_module_sections(struct module *mod, struct load_info *info)
sizeof(*mod->bpf_raw_events),
&mod->num_bpf_raw_events);
#endif
#ifdef CONFIG_DEBUG_INFO_BTF_MODULES
mod->btf_data = any_section_objs(info, ".BTF", 1, &mod->btf_data_size);
#endif
#ifdef CONFIG_JUMP_LABEL
mod->jump_entries = section_objs(info, "__jump_table",
sizeof(*mod->jump_entries),

29
kernel/trace/bpf_trace.c
View File

@ -16,6 +16,9 @@
#include <linux/syscalls.h>
#include <linux/error-injection.h>
#include <linux/btf_ids.h>
#include <linux/bpf_lsm.h>
#include <net/bpf_sk_storage.h>
#include <uapi/linux/bpf.h>
#include <uapi/linux/btf.h>
@ -1022,6 +1025,20 @@ const struct bpf_func_proto bpf_get_current_task_proto = {
.ret_type = RET_INTEGER,
};
BPF_CALL_0(bpf_get_current_task_btf)
{
return (unsigned long) current;
}
BTF_ID_LIST_SINGLE(bpf_get_current_btf_ids, struct, task_struct)
static const struct bpf_func_proto bpf_get_current_task_btf_proto = {
.func = bpf_get_current_task_btf,
.gpl_only = true,
.ret_type = RET_PTR_TO_BTF_ID,
.ret_btf_id = &bpf_get_current_btf_ids[0],
};
BPF_CALL_2(bpf_current_task_under_cgroup, struct bpf_map *, map, u32, idx)
{
struct bpf_array *array = container_of(map, struct bpf_array, map);
@ -1164,7 +1181,11 @@ BTF_SET_END(btf_allowlist_d_path)
static bool bpf_d_path_allowed(const struct bpf_prog *prog)
{
return btf_id_set_contains(&btf_allowlist_d_path, prog->aux->attach_btf_id);
if (prog->type == BPF_PROG_TYPE_LSM)
return bpf_lsm_is_sleepable_hook(prog->aux->attach_btf_id);
return btf_id_set_contains(&btf_allowlist_d_path,
prog->aux->attach_btf_id);
}
BTF_ID_LIST_SINGLE(bpf_d_path_btf_ids, struct, path)
@ -1265,6 +1286,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
return &bpf_get_current_pid_tgid_proto;
case BPF_FUNC_get_current_task:
return &bpf_get_current_task_proto;
case BPF_FUNC_get_current_task_btf:
return &bpf_get_current_task_btf_proto;
case BPF_FUNC_get_current_uid_gid:
return &bpf_get_current_uid_gid_proto;
case BPF_FUNC_get_current_comm:
@ -1719,6 +1742,10 @@ tracing_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
return &bpf_skc_to_tcp_request_sock_proto;
case BPF_FUNC_skc_to_udp6_sock:
return &bpf_skc_to_udp6_sock_proto;
case BPF_FUNC_sk_storage_get:
return &bpf_sk_storage_get_tracing_proto;
case BPF_FUNC_sk_storage_delete:
return &bpf_sk_storage_delete_tracing_proto;
#endif
case BPF_FUNC_seq_printf:
return prog->expected_attach_type == BPF_TRACE_ITER ?

9
lib/Kconfig.debug
View File

@ -274,6 +274,15 @@ config DEBUG_INFO_BTF
Turning this on expects presence of pahole tool, which will convert
DWARF type info into equivalent deduplicated BTF type info.
config PAHOLE_HAS_SPLIT_BTF
def_bool $(success, test `$(PAHOLE) --version | sed -E 's/v([0-9]+)\.([0-9]+)/\1\2/'` -ge "119")
config DEBUG_INFO_BTF_MODULES
def_bool y
depends on DEBUG_INFO_BTF && MODULES && PAHOLE_HAS_SPLIT_BTF
help
Generate compact split BTF type information for kernel modules.
config GDB_SCRIPTS
bool "Provide GDB scripts for kernel debugging"
help

135
net/core/bpf_sk_storage.c
View File

@ -6,6 +6,7 @@
#include <linux/types.h>
#include <linux/spinlock.h>
#include <linux/bpf.h>
#include <linux/btf.h>
#include <linux/btf_ids.h>
#include <linux/bpf_local_storage.h>
#include <net/bpf_sk_storage.h>
@ -15,20 +16,8 @@
DEFINE_BPF_STORAGE_CACHE(sk_cache);
static int omem_charge(struct sock *sk, unsigned int size)
{
/* same check as in sock_kmalloc() */
if (size <= sysctl_optmem_max &&
atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
atomic_add(size, &sk->sk_omem_alloc);
return 0;
}
return -ENOMEM;
}
static struct bpf_local_storage_data *
sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
bpf_sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
{
struct bpf_local_storage *sk_storage;
struct bpf_local_storage_map *smap;
@ -41,11 +30,11 @@ sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
return bpf_local_storage_lookup(sk_storage, smap, cacheit_lockit);
}
static int sk_storage_delete(struct sock *sk, struct bpf_map *map)
static int bpf_sk_storage_del(struct sock *sk, struct bpf_map *map)
{
struct bpf_local_storage_data *sdata;
sdata = sk_storage_lookup(sk, map, false);
sdata = bpf_sk_storage_lookup(sk, map, false);
if (!sdata)
return -ENOENT;
@ -94,7 +83,7 @@ void bpf_sk_storage_free(struct sock *sk)
kfree_rcu(sk_storage, rcu);
}
static void sk_storage_map_free(struct bpf_map *map)
static void bpf_sk_storage_map_free(struct bpf_map *map)
{
struct bpf_local_storage_map *smap;
@ -103,7 +92,7 @@ static void sk_storage_map_free(struct bpf_map *map)
bpf_local_storage_map_free(smap);
}
static struct bpf_map *sk_storage_map_alloc(union bpf_attr *attr)
static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
{
struct bpf_local_storage_map *smap;
@ -130,7 +119,7 @@ static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
fd = *(int *)key;
sock = sockfd_lookup(fd, &err);
if (sock) {
sdata = sk_storage_lookup(sock->sk, map, true);
sdata = bpf_sk_storage_lookup(sock->sk, map, true);
sockfd_put(sock);
return sdata ? sdata->data : NULL;
}
@ -166,7 +155,7 @@ static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
fd = *(int *)key;
sock = sockfd_lookup(fd, &err);
if (sock) {
err = sk_storage_delete(sock->sk, map);
err = bpf_sk_storage_del(sock->sk, map);
sockfd_put(sock);
return err;
}
@ -272,7 +261,7 @@ BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
if (!sk || !sk_fullsock(sk) || flags > BPF_SK_STORAGE_GET_F_CREATE)
return (unsigned long)NULL;
sdata = sk_storage_lookup(sk, map, true);
sdata = bpf_sk_storage_lookup(sk, map, true);
if (sdata)
return (unsigned long)sdata->data;
@ -305,7 +294,7 @@ BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
if (refcount_inc_not_zero(&sk->sk_refcnt)) {
int err;
err = sk_storage_delete(sk, map);
err = bpf_sk_storage_del(sk, map);
sock_put(sk);
return err;
}
@ -313,14 +302,23 @@ BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
return -ENOENT;
}
static int sk_storage_charge(struct bpf_local_storage_map *smap,
void *owner, u32 size)
static int bpf_sk_storage_charge(struct bpf_local_storage_map *smap,
void *owner, u32 size)
{
return omem_charge(owner, size);
struct sock *sk = (struct sock *)owner;
/* same check as in sock_kmalloc() */
if (size <= sysctl_optmem_max &&
atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
atomic_add(size, &sk->sk_omem_alloc);
return 0;
}
return -ENOMEM;
}
static void sk_storage_uncharge(struct bpf_local_storage_map *smap,
void *owner, u32 size)
static void bpf_sk_storage_uncharge(struct bpf_local_storage_map *smap,
void *owner, u32 size)
{
struct sock *sk = owner;
@ -328,7 +326,7 @@ static void sk_storage_uncharge(struct bpf_local_storage_map *smap,
}
static struct bpf_local_storage __rcu **
sk_storage_ptr(void *owner)
bpf_sk_storage_ptr(void *owner)
{
struct sock *sk = owner;
@ -339,8 +337,8 @@ static int sk_storage_map_btf_id;
const struct bpf_map_ops sk_storage_map_ops = {
.map_meta_equal = bpf_map_meta_equal,
.map_alloc_check = bpf_local_storage_map_alloc_check,
.map_alloc = sk_storage_map_alloc,
.map_free = sk_storage_map_free,
.map_alloc = bpf_sk_storage_map_alloc,
.map_free = bpf_sk_storage_map_free,
.map_get_next_key = notsupp_get_next_key,
.map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
.map_update_elem = bpf_fd_sk_storage_update_elem,
@ -348,9 +346,9 @@ const struct bpf_map_ops sk_storage_map_ops = {
.map_check_btf = bpf_local_storage_map_check_btf,
.map_btf_name = "bpf_local_storage_map",
.map_btf_id = &sk_storage_map_btf_id,
.map_local_storage_charge = sk_storage_charge,
.map_local_storage_uncharge = sk_storage_uncharge,
.map_owner_storage_ptr = sk_storage_ptr,
.map_local_storage_charge = bpf_sk_storage_charge,
.map_local_storage_uncharge = bpf_sk_storage_uncharge,
.map_owner_storage_ptr = bpf_sk_storage_ptr,
};
const struct bpf_func_proto bpf_sk_storage_get_proto = {
@ -381,6 +379,79 @@ const struct bpf_func_proto bpf_sk_storage_delete_proto = {
.arg2_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON,
};
static bool bpf_sk_storage_tracing_allowed(const struct bpf_prog *prog)
{
const struct btf *btf_vmlinux;
const struct btf_type *t;
const char *tname;
u32 btf_id;
if (prog->aux->dst_prog)
return false;
/* Ensure the tracing program is not tracing
* any bpf_sk_storage*() function and also
* use the bpf_sk_storage_(get|delete) helper.
*/
switch (prog->expected_attach_type) {
case BPF_TRACE_RAW_TP:
/* bpf_sk_storage has no trace point */
return true;
case BPF_TRACE_FENTRY:
case BPF_TRACE_FEXIT:
btf_vmlinux = bpf_get_btf_vmlinux();
btf_id = prog->aux->attach_btf_id;
t = btf_type_by_id(btf_vmlinux, btf_id);
tname = btf_name_by_offset(btf_vmlinux, t->name_off);
return !!strncmp(tname, "bpf_sk_storage",
strlen("bpf_sk_storage"));
default:
return false;
}
return false;
}
BPF_CALL_4(bpf_sk_storage_get_tracing, struct bpf_map *, map, struct sock *, sk,
void *, value, u64, flags)
{
if (!in_serving_softirq() && !in_task())
return (unsigned long)NULL;
return (unsigned long)____bpf_sk_storage_get(map, sk, value, flags);
}
BPF_CALL_2(bpf_sk_storage_delete_tracing, struct bpf_map *, map,
struct sock *, sk)
{
if (!in_serving_softirq() && !in_task())
return -EPERM;
return ____bpf_sk_storage_delete(map, sk);
}
const struct bpf_func_proto bpf_sk_storage_get_tracing_proto = {
.func = bpf_sk_storage_get_tracing,
.gpl_only = false,
.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_BTF_ID,
.arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
.arg4_type = ARG_ANYTHING,
.allowed = bpf_sk_storage_tracing_allowed,
};
const struct bpf_func_proto bpf_sk_storage_delete_tracing_proto = {
.func = bpf_sk_storage_delete_tracing,
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_BTF_ID,
.arg2_btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
.allowed = bpf_sk_storage_tracing_allowed,
};
struct bpf_sk_storage_diag {
u32 nr_maps;
struct bpf_map *maps[];

70
net/core/page_pool.c
View File

@ -11,6 +11,8 @@
#include <linux/device.h>
#include <net/page_pool.h>
#include <net/xdp.h>
#include <linux/dma-direction.h>
#include <linux/dma-mapping.h>
#include <linux/page-flags.h>
@ -362,8 +364,9 @@ static bool pool_page_reusable(struct page_pool *pool, struct page *page)
* If the page refcnt != 1, then the page will be returned to memory
* subsystem.
*/
void page_pool_put_page(struct page_pool *pool, struct page *page,
unsigned int dma_sync_size, bool allow_direct)
static __always_inline struct page *
__page_pool_put_page(struct page_pool *pool, struct page *page,
unsigned int dma_sync_size, bool allow_direct)
{
/* This allocator is optimized for the XDP mode that uses
* one-frame-per-page, but have fallbacks that act like the
@ -379,15 +382,12 @@ void page_pool_put_page(struct page_pool *pool, struct page *page,
page_pool_dma_sync_for_device(pool, page,
dma_sync_size);
if (allow_direct && in_serving_softirq())
if (page_pool_recycle_in_cache(page, pool))
return;
if (allow_direct && in_serving_softirq() &&
page_pool_recycle_in_cache(page, pool))
return NULL;
if (!page_pool_recycle_in_ring(pool, page)) {
/* Cache full, fallback to free pages */
page_pool_return_page(pool, page);
}
return;
/* Page found as candidate for recycling */
return page;
}
/* Fallback/non-XDP mode: API user have elevated refcnt.
*
@ -405,9 +405,59 @@ void page_pool_put_page(struct page_pool *pool, struct page *page,
/* Do not replace this with page_pool_return_page() */
page_pool_release_page(pool, page);
put_page(page);
return NULL;
}
void page_pool_put_page(struct page_pool *pool, struct page *page,
unsigned int dma_sync_size, bool allow_direct)
{
page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct);
if (page && !page_pool_recycle_in_ring(pool, page)) {
/* Cache full, fallback to free pages */
page_pool_return_page(pool, page);
}
}
EXPORT_SYMBOL(page_pool_put_page);
/* Caller must not use data area after call, as this function overwrites it */
void page_pool_put_page_bulk(struct page_pool *pool, void **data,
int count)
{
int i, bulk_len = 0;
for (i = 0; i < count; i++) {
struct page *page = virt_to_head_page(data[i]);
page = __page_pool_put_page(pool, page, -1, false);
/* Approved for bulk recycling in ptr_ring cache */
if (page)
data[bulk_len++] = page;
}
if (unlikely(!bulk_len))
return;
/* Bulk producer into ptr_ring page_pool cache */
page_pool_ring_lock(pool);
for (i = 0; i < bulk_len; i++) {
if (__ptr_ring_produce(&pool->ring, data[i]))
break; /* ring full */
}
page_pool_ring_unlock(pool);
/* Hopefully all pages was return into ptr_ring */
if (likely(i == bulk_len))
return;
/* ptr_ring cache full, free remaining pages outside producer lock
* since put_page() with refcnt == 1 can be an expensive operation
*/
for (; i < bulk_len; i++)
page_pool_return_page(pool, data[i]);
}
EXPORT_SYMBOL(page_pool_put_page_bulk);
static void page_pool_empty_ring(struct page_pool *pool)
{
struct page *page;

54
net/core/xdp.c
View File

@ -380,6 +380,60 @@ void xdp_return_frame_rx_napi(struct xdp_frame *xdpf)
}
EXPORT_SYMBOL_GPL(xdp_return_frame_rx_napi);
/* XDP bulk APIs introduce a defer/flush mechanism to return
* pages belonging to the same xdp_mem_allocator object
* (identified via the mem.id field) in bulk to optimize
* I-cache and D-cache.
* The bulk queue size is set to 16 to be aligned to how
* XDP_REDIRECT bulking works. The bulk is flushed when
* it is full or when mem.id changes.
* xdp_frame_bulk is usually stored/allocated on the function
* call-stack to avoid locking penalties.
*/
void xdp_flush_frame_bulk(struct xdp_frame_bulk *bq)
{
struct xdp_mem_allocator *xa = bq->xa;
if (unlikely(!xa || !bq->count))
return;
page_pool_put_page_bulk(xa->page_pool, bq->q, bq->count);
/* bq->xa is not cleared to save lookup, if mem.id same in next bulk */
bq->count = 0;
}
EXPORT_SYMBOL_GPL(xdp_flush_frame_bulk);
/* Must be called with rcu_read_lock held */
void xdp_return_frame_bulk(struct xdp_frame *xdpf,
struct xdp_frame_bulk *bq)
{
struct xdp_mem_info *mem = &xdpf->mem;
struct xdp_mem_allocator *xa;
if (mem->type != MEM_TYPE_PAGE_POOL) {
__xdp_return(xdpf->data, &xdpf->mem, false);
return;
}
xa = bq->xa;
if (unlikely(!xa)) {
xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
bq->count = 0;
bq->xa = xa;
}
if (bq->count == XDP_BULK_QUEUE_SIZE)
xdp_flush_frame_bulk(bq);
if (unlikely(mem->id != xa->mem.id)) {
xdp_flush_frame_bulk(bq);
bq->xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
}
bq->q[bq->count++] = xdpf->data;
}
EXPORT_SYMBOL_GPL(xdp_return_frame_bulk);
void xdp_return_buff(struct xdp_buff *xdp)
{
__xdp_return(xdp->data, &xdp->rxq->mem, true);

1
samples/bpf/hbm.c
View File

@ -51,7 +51,6 @@
#include "cgroup_helpers.h"
#include "hbm.h"
#include "bpf_util.h"
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
bool outFlag = true;

179
samples/bpf/test_ipip.sh
View File

@ -1,179 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
function config_device {
ip netns add at_ns0
ip netns add at_ns1
ip netns add at_ns2
ip link add veth0 type veth peer name veth0b
ip link add veth1 type veth peer name veth1b
ip link add veth2 type veth peer name veth2b
ip link set veth0b up
ip link set veth1b up
ip link set veth2b up
ip link set dev veth0b mtu 1500
ip link set dev veth1b mtu 1500
ip link set dev veth2b mtu 1500
ip link set veth0 netns at_ns0
ip link set veth1 netns at_ns1
ip link set veth2 netns at_ns2
ip netns exec at_ns0 ip addr add 172.16.1.100/24 dev veth0
ip netns exec at_ns0 ip addr add 2401:db00::1/64 dev veth0 nodad
ip netns exec at_ns0 ip link set dev veth0 up
ip netns exec at_ns1 ip addr add 172.16.1.101/24 dev veth1
ip netns exec at_ns1 ip addr add 2401:db00::2/64 dev veth1 nodad
ip netns exec at_ns1 ip link set dev veth1 up
ip netns exec at_ns2 ip addr add 172.16.1.200/24 dev veth2
ip netns exec at_ns2 ip addr add 2401:db00::3/64 dev veth2 nodad
ip netns exec at_ns2 ip link set dev veth2 up
ip link add br0 type bridge
ip link set br0 up
ip link set dev br0 mtu 1500
ip link set veth0b master br0
ip link set veth1b master br0
ip link set veth2b master br0
}
function add_ipip_tunnel {
ip netns exec at_ns0 \
ip link add dev $DEV_NS type ipip local 172.16.1.100 remote 172.16.1.200
ip netns exec at_ns0 ip link set dev $DEV_NS up
ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
ip netns exec at_ns1 \
ip link add dev $DEV_NS type ipip local 172.16.1.101 remote 172.16.1.200
ip netns exec at_ns1 ip link set dev $DEV_NS up
# same inner IP address in at_ns0 and at_ns1
ip netns exec at_ns1 ip addr add dev $DEV_NS 10.1.1.100/24
ip netns exec at_ns2 ip link add dev $DEV type ipip external
ip netns exec at_ns2 ip link set dev $DEV up
ip netns exec at_ns2 ip addr add dev $DEV 10.1.1.200/24
}
function add_ipip6_tunnel {
ip netns exec at_ns0 \
ip link add dev $DEV_NS type ip6tnl mode ipip6 local 2401:db00::1/64 remote 2401:db00::3/64
ip netns exec at_ns0 ip link set dev $DEV_NS up
ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
ip netns exec at_ns1 \
ip link add dev $DEV_NS type ip6tnl mode ipip6 local 2401:db00::2/64 remote 2401:db00::3/64
ip netns exec at_ns1 ip link set dev $DEV_NS up
# same inner IP address in at_ns0 and at_ns1
ip netns exec at_ns1 ip addr add dev $DEV_NS 10.1.1.100/24
ip netns exec at_ns2 ip link add dev $DEV type ip6tnl mode ipip6 external
ip netns exec at_ns2 ip link set dev $DEV up
ip netns exec at_ns2 ip addr add dev $DEV 10.1.1.200/24
}
function add_ip6ip6_tunnel {
ip netns exec at_ns0 \
ip link add dev $DEV_NS type ip6tnl mode ip6ip6 local 2401:db00::1/64 remote 2401:db00::3/64
ip netns exec at_ns0 ip link set dev $DEV_NS up
ip netns exec at_ns0 ip addr add dev $DEV_NS 2601:646::1/64
ip netns exec at_ns1 \
ip link add dev $DEV_NS type ip6tnl mode ip6ip6 local 2401:db00::2/64 remote 2401:db00::3/64
ip netns exec at_ns1 ip link set dev $DEV_NS up
# same inner IP address in at_ns0 and at_ns1
ip netns exec at_ns1 ip addr add dev $DEV_NS 2601:646::1/64
ip netns exec at_ns2 ip link add dev $DEV type ip6tnl mode ip6ip6 external
ip netns exec at_ns2 ip link set dev $DEV up
ip netns exec at_ns2 ip addr add dev $DEV 2601:646::2/64
}
function attach_bpf {
DEV=$1
SET_TUNNEL=$2
GET_TUNNEL=$3
ip netns exec at_ns2 tc qdisc add dev $DEV clsact
ip netns exec at_ns2 tc filter add dev $DEV egress bpf da obj tcbpf2_kern.o sec $SET_TUNNEL
ip netns exec at_ns2 tc filter add dev $DEV ingress bpf da obj tcbpf2_kern.o sec $GET_TUNNEL
}
function test_ipip {
DEV_NS=ipip_std
DEV=ipip_bpf
config_device
# tcpdump -nei br0 &
cat /sys/kernel/debug/tracing/trace_pipe &
add_ipip_tunnel
attach_bpf $DEV ipip_set_tunnel ipip_get_tunnel
ip netns exec at_ns0 ping -c 1 10.1.1.200
ip netns exec at_ns2 ping -c 1 10.1.1.100
ip netns exec at_ns0 iperf -sD -p 5200 > /dev/null
ip netns exec at_ns1 iperf -sD -p 5201 > /dev/null
sleep 0.2
# tcp check _same_ IP over different tunnels
ip netns exec at_ns2 iperf -c 10.1.1.100 -n 5k -p 5200
ip netns exec at_ns2 iperf -c 10.1.1.100 -n 5k -p 5201
cleanup
}
# IPv4 over IPv6 tunnel
function test_ipip6 {
DEV_NS=ipip_std
DEV=ipip_bpf
config_device
# tcpdump -nei br0 &
cat /sys/kernel/debug/tracing/trace_pipe &
add_ipip6_tunnel
attach_bpf $DEV ipip6_set_tunnel ipip6_get_tunnel
ip netns exec at_ns0 ping -c 1 10.1.1.200
ip netns exec at_ns2 ping -c 1 10.1.1.100
ip netns exec at_ns0 iperf -sD -p 5200 > /dev/null
ip netns exec at_ns1 iperf -sD -p 5201 > /dev/null
sleep 0.2
# tcp check _same_ IP over different tunnels
ip netns exec at_ns2 iperf -c 10.1.1.100 -n 5k -p 5200
ip netns exec at_ns2 iperf -c 10.1.1.100 -n 5k -p 5201
cleanup
}
# IPv6 over IPv6 tunnel
function test_ip6ip6 {
DEV_NS=ipip_std
DEV=ipip_bpf
config_device
# tcpdump -nei br0 &
cat /sys/kernel/debug/tracing/trace_pipe &
add_ip6ip6_tunnel
attach_bpf $DEV ip6ip6_set_tunnel ip6ip6_get_tunnel
ip netns exec at_ns0 ping -6 -c 1 2601:646::2
ip netns exec at_ns2 ping -6 -c 1 2601:646::1
ip netns exec at_ns0 iperf -6sD -p 5200 > /dev/null
ip netns exec at_ns1 iperf -6sD -p 5201 > /dev/null
sleep 0.2
# tcp check _same_ IP over different tunnels
ip netns exec at_ns2 iperf -6c 2601:646::1 -n 5k -p 5200
ip netns exec at_ns2 iperf -6c 2601:646::1 -n 5k -p 5201
cleanup
}
function cleanup {
set +ex
pkill iperf
ip netns delete at_ns0
ip netns delete at_ns1
ip netns delete at_ns2
ip link del veth0
ip link del veth1
ip link del veth2
ip link del br0
pkill tcpdump
pkill cat
set -ex
}
cleanup
echo "Testing IP tunnels..."
test_ipip
test_ipip6
test_ip6ip6
echo "*** PASS ***"

20
scripts/Makefile.modfinal
View File

@ -6,6 +6,7 @@
PHONY := __modfinal
__modfinal:
include include/config/auto.conf
include $(srctree)/scripts/Kbuild.include
# for c_flags
@ -36,8 +37,23 @@ quiet_cmd_ld_ko_o = LD [M] $@
-T scripts/module.lds -o $@ $(filter %.o, $^); \
$(if $(ARCH_POSTLINK), $(MAKE) -f $(ARCH_POSTLINK) $@, true)
$(modules): %.ko: %.o %.mod.o scripts/module.lds FORCE
+$(call if_changed,ld_ko_o)
quiet_cmd_btf_ko = BTF [M] $@
cmd_btf_ko = LLVM_OBJCOPY=$(OBJCOPY) $(PAHOLE) -J --btf_base vmlinux $@
# Same as newer-prereqs, but allows to exclude specified extra dependencies
newer_prereqs_except = $(filter-out $(PHONY) $(1),$?)
# Same as if_changed, but allows to exclude specified extra dependencies
if_changed_except = $(if $(call newer_prereqs_except,$(2))$(cmd-check), \
$(cmd); \
printf '%s\n' 'cmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:)
# Re-generate module BTFs if either module's .ko or vmlinux changed
$(modules): %.ko: %.o %.mod.o scripts/module.lds vmlinux FORCE
+$(call if_changed_except,ld_ko_o,vmlinux)
ifdef CONFIG_DEBUG_INFO_BTF_MODULES
+$(if $(newer-prereqs),$(call cmd,btf_ko))
endif
targets += $(modules) $(modules:.ko=.mod.o)

2
security/bpf/hooks.c
View File

@ -12,6 +12,7 @@ static struct security_hook_list bpf_lsm_hooks[] __lsm_ro_after_init = {
#include <linux/lsm_hook_defs.h>
#undef LSM_HOOK
LSM_HOOK_INIT(inode_free_security, bpf_inode_storage_free),
LSM_HOOK_INIT(task_free, bpf_task_storage_free),
};
static int __init bpf_lsm_init(void)
@ -23,6 +24,7 @@ static int __init bpf_lsm_init(void)
struct lsm_blob_sizes bpf_lsm_blob_sizes __lsm_ro_after_init = {
.lbs_inode = sizeof(struct bpf_storage_blob),
.lbs_task = sizeof(struct bpf_storage_blob),
};
DEFINE_LSM(bpf) = {

2
tools/bpf/bpftool/.gitignore vendored
View File

@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
*.d
/bpftool-bootstrap
/bootstrap/
/bpftool
bpftool*.8
bpf-helpers.*

3
tools/bpf/bpftool/Documentation/bpftool-map.rst
View File

@ -50,7 +50,8 @@ MAP COMMANDS
| | **lru_percpu_hash** | **lpm_trie** | **array_of_maps** | **hash_of_maps**
| | **devmap** | **devmap_hash** | **sockmap** | **cpumap** | **xskmap** | **sockhash**
| | **cgroup_storage** | **reuseport_sockarray** | **percpu_cgroup_storage**
| | **queue** | **stack** | **sk_storage** | **struct_ops** | **ringbuf** | **inode_storage** }
| | **queue** | **stack** | **sk_storage** | **struct_ops** | **ringbuf** | **inode_storage**
| **task_storage** }
DESCRIPTION
===========

44
tools/bpf/bpftool/Makefile
View File

@ -19,22 +19,39 @@ BPF_DIR = $(srctree)/tools/lib/bpf/
ifneq ($(OUTPUT),)
LIBBPF_OUTPUT = $(OUTPUT)/libbpf/
LIBBPF_PATH = $(LIBBPF_OUTPUT)
BOOTSTRAP_OUTPUT = $(OUTPUT)/bootstrap/
else
LIBBPF_OUTPUT =
LIBBPF_PATH = $(BPF_DIR)
BOOTSTRAP_OUTPUT = $(CURDIR)/bootstrap/
endif
LIBBPF = $(LIBBPF_PATH)libbpf.a
LIBBPF_BOOTSTRAP_OUTPUT = $(BOOTSTRAP_OUTPUT)libbpf/
LIBBPF_BOOTSTRAP = $(LIBBPF_BOOTSTRAP_OUTPUT)libbpf.a
BPFTOOL_VERSION ?= $(shell make -rR --no-print-directory -sC ../../.. kernelversion)
ifeq ($(BPFTOOL_VERSION),)
BPFTOOL_VERSION := $(shell make -rR --no-print-directory -sC ../../.. kernelversion)
endif
$(LIBBPF): FORCE
$(if $(LIBBPF_OUTPUT),@mkdir -p $(LIBBPF_OUTPUT))
$(LIBBPF_OUTPUT) $(BOOTSTRAP_OUTPUT) $(LIBBPF_BOOTSTRAP_OUTPUT):
$(QUIET_MKDIR)mkdir -p $@
$(LIBBPF): FORCE | $(LIBBPF_OUTPUT)
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_OUTPUT) $(LIBBPF_OUTPUT)libbpf.a
$(LIBBPF)-clean:
$(LIBBPF_BOOTSTRAP): FORCE | $(LIBBPF_BOOTSTRAP_OUTPUT)
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) \
ARCH= CC=$(HOSTCC) LD=$(HOSTLD) $@
$(LIBBPF)-clean: FORCE | $(LIBBPF_OUTPUT)
$(call QUIET_CLEAN, libbpf)
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_OUTPUT) clean >/dev/null
$(LIBBPF_BOOTSTRAP)-clean: FORCE | $(LIBBPF_BOOTSTRAP_OUTPUT)
$(call QUIET_CLEAN, libbpf-bootstrap)
$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) clean >/dev/null
prefix ?= /usr/local
bash_compdir ?= /usr/share/bash-completion/completions
@ -92,6 +109,7 @@ CFLAGS += -DCOMPAT_NEED_REALLOCARRAY
endif
LIBS = $(LIBBPF) -lelf -lz
LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf -lz
ifeq ($(feature-libcap), 1)
CFLAGS += -DUSE_LIBCAP
LIBS += -lcap
@ -118,9 +136,9 @@ CFLAGS += -DHAVE_LIBBFD_SUPPORT
SRCS += $(BFD_SRCS)
endif
BPFTOOL_BOOTSTRAP := $(if $(OUTPUT),$(OUTPUT)bpftool-bootstrap,./bpftool-bootstrap)
BPFTOOL_BOOTSTRAP := $(BOOTSTRAP_OUTPUT)bpftool
BOOTSTRAP_OBJS = $(addprefix $(OUTPUT),main.o common.o json_writer.o gen.o btf.o)
BOOTSTRAP_OBJS = $(addprefix $(BOOTSTRAP_OUTPUT),main.o common.o json_writer.o gen.o btf.o)
OBJS = $(patsubst %.c,$(OUTPUT)%.o,$(SRCS)) $(OUTPUT)disasm.o
VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \
@ -167,12 +185,16 @@ $(OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c
$(OUTPUT)feature.o: | zdep
$(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF)
$(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(BOOTSTRAP_OBJS) $(LIBS)
$(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF_BOOTSTRAP)
$(QUIET_LINK)$(HOSTCC) $(CFLAGS) $(LDFLAGS) -o $@ $(BOOTSTRAP_OBJS) \
$(LIBS_BOOTSTRAP)
$(OUTPUT)bpftool: $(OBJS) $(LIBBPF)
$(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
$(BOOTSTRAP_OUTPUT)%.o: %.c | $(BOOTSTRAP_OUTPUT)
$(QUIET_CC)$(HOSTCC) $(CFLAGS) -c -MMD -o $@ $<
$(OUTPUT)%.o: %.c
$(QUIET_CC)$(CC) $(CFLAGS) -c -MMD -o $@ $<
@ -180,11 +202,11 @@ feature-detect-clean:
$(call QUIET_CLEAN, feature-detect)
$(Q)$(MAKE) -C $(srctree)/tools/build/feature/ clean >/dev/null
clean: $(LIBBPF)-clean feature-detect-clean
clean: $(LIBBPF)-clean $(LIBBPF_BOOTSTRAP)-clean feature-detect-clean
$(call QUIET_CLEAN, bpftool)
$(Q)$(RM) -- $(OUTPUT)bpftool $(OUTPUT)*.o $(OUTPUT)*.d
$(Q)$(RM) -- $(BPFTOOL_BOOTSTRAP) $(OUTPUT)*.skel.h $(OUTPUT)vmlinux.h
$(Q)$(RM) -r -- $(OUTPUT)libbpf/
$(Q)$(RM) -- $(OUTPUT)*.skel.h $(OUTPUT)vmlinux.h
$(Q)$(RM) -r -- $(LIBBPF_OUTPUT) $(BOOTSTRAP_OUTPUT)
$(call QUIET_CLEAN, core-gen)
$(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.bpftool
$(Q)$(RM) -r -- $(OUTPUT)feature/

2
tools/bpf/bpftool/bash-completion/bpftool
View File

@ -705,7 +705,7 @@ _bpftool()
hash_of_maps devmap devmap_hash sockmap cpumap \
xskmap sockhash cgroup_storage reuseport_sockarray \
percpu_cgroup_storage queue stack sk_storage \
struct_ops inode_storage' -- \
struct_ops inode_storage task_storage' -- \
"$cur" ) )
return 0
;;

37
tools/bpf/bpftool/btf.c
View File

@ -358,8 +358,12 @@ static int dump_btf_raw(const struct btf *btf,
}
} else {
int cnt = btf__get_nr_types(btf);
int start_id = 1;
for (i = 1; i <= cnt; i++) {
if (base_btf)
start_id = btf__get_nr_types(base_btf) + 1;
for (i = start_id; i <= cnt; i++) {
t = btf__type_by_id(btf, i);
dump_btf_type(btf, i, t);
}
@ -438,7 +442,6 @@ static int do_dump(int argc, char **argv)
return -1;
}
src = GET_ARG();
if (is_prefix(src, "map")) {
struct bpf_map_info info = {};
__u32 len = sizeof(info);
@ -499,7 +502,7 @@ static int do_dump(int argc, char **argv)
}
NEXT_ARG();
} else if (is_prefix(src, "file")) {
btf = btf__parse(*argv, NULL);
btf = btf__parse_split(*argv, base_btf);
if (IS_ERR(btf)) {
err = -PTR_ERR(btf);
btf = NULL;
@ -739,9 +742,14 @@ show_btf_plain(struct bpf_btf_info *info, int fd,
struct btf_attach_table *btf_map_table)
{
struct btf_attach_point *obj;
const char *name = u64_to_ptr(info->name);
int n;
printf("%u: ", info->id);
if (info->kernel_btf)
printf("name [%s] ", name);
else if (name && name[0])
printf("name %s ", name);
printf("size %uB", info->btf_size);
n = 0;
@ -768,6 +776,7 @@ show_btf_json(struct bpf_btf_info *info, int fd,
struct btf_attach_table *btf_map_table)
{
struct btf_attach_point *obj;
const char *name = u64_to_ptr(info->name);
jsonw_start_object(json_wtr); /* btf object */
jsonw_uint_field(json_wtr, "id", info->id);
@ -793,6 +802,11 @@ show_btf_json(struct bpf_btf_info *info, int fd,
emit_obj_refs_json(&refs_table, info->id, json_wtr); /* pids */
jsonw_bool_field(json_wtr, "kernel", info->kernel_btf);
if (name && name[0])
jsonw_string_field(json_wtr, "name", name);
jsonw_end_object(json_wtr); /* btf object */
}
@ -800,15 +814,30 @@ static int
show_btf(int fd, struct btf_attach_table *btf_prog_table,
struct btf_attach_table *btf_map_table)
{
struct bpf_btf_info info = {};
struct bpf_btf_info info;
__u32 len = sizeof(info);
char name[64];
int err;
memset(&info, 0, sizeof(info));
err = bpf_obj_get_info_by_fd(fd, &info, &len);
if (err) {
p_err("can't get BTF object info: %s", strerror(errno));
return -1;
}
/* if kernel support emitting BTF object name, pass name pointer */
if (info.name_len) {
memset(&info, 0, sizeof(info));
info.name_len = sizeof(name);
info.name = ptr_to_u64(name);
len = sizeof(info);
err = bpf_obj_get_info_by_fd(fd, &info, &len);
if (err) {
p_err("can't get BTF object info: %s", strerror(errno));
return -1;
}
}
if (json_output)
show_btf_json(&info, fd, btf_prog_table, btf_map_table);

15
tools/bpf/bpftool/main.c
View File

@ -11,6 +11,7 @@
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include <bpf/btf.h>
#include "main.h"
@ -28,6 +29,7 @@ bool show_pinned;
bool block_mount;
bool verifier_logs;
bool relaxed_maps;
struct btf *base_btf;
struct pinned_obj_table prog_table;
struct pinned_obj_table map_table;
struct pinned_obj_table link_table;
@ -391,6 +393,7 @@ int main(int argc, char **argv)
{ "mapcompat", no_argument, NULL, 'm' },
{ "nomount", no_argument, NULL, 'n' },
{ "debug", no_argument, NULL, 'd' },
{ "base-btf", required_argument, NULL, 'B' },
{ 0 }
};
int opt, ret;
@ -407,7 +410,7 @@ int main(int argc, char **argv)
hash_init(link_table.table);
opterr = 0;
while ((opt = getopt_long(argc, argv, "Vhpjfmnd",
while ((opt = getopt_long(argc, argv, "VhpjfmndB:",
options, NULL)) >= 0) {
switch (opt) {
case 'V':
@ -441,6 +444,15 @@ int main(int argc, char **argv)
libbpf_set_print(print_all_levels);
verifier_logs = true;
break;
case 'B':
base_btf = btf__parse(optarg, NULL);
if (libbpf_get_error(base_btf)) {
p_err("failed to parse base BTF at '%s': %ld\n",
optarg, libbpf_get_error(base_btf));
base_btf = NULL;
return -1;
}
break;
default:
p_err("unrecognized option '%s'", argv[optind - 1]);
if (json_output)
@ -465,6 +477,7 @@ int main(int argc, char **argv)
delete_pinned_obj_table(&map_table);
delete_pinned_obj_table(&link_table);
}
btf__free(base_btf);
return ret;
}

1
tools/bpf/bpftool/main.h
View File

@ -90,6 +90,7 @@ extern bool show_pids;
extern bool block_mount;
extern bool verifier_logs;
extern bool relaxed_maps;
extern struct btf *base_btf;
extern struct pinned_obj_table prog_table;
extern struct pinned_obj_table map_table;
extern struct pinned_obj_table link_table;

4
tools/bpf/bpftool/map.c
View File

@ -51,6 +51,7 @@ const char * const map_type_name[] = {
[BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops",
[BPF_MAP_TYPE_RINGBUF] = "ringbuf",
[BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage",
[BPF_MAP_TYPE_TASK_STORAGE] = "task_storage",
};
const size_t map_type_name_size = ARRAY_SIZE(map_type_name);
@ -1464,7 +1465,8 @@ static int do_help(int argc, char **argv)
" lru_percpu_hash | lpm_trie | array_of_maps | hash_of_maps |\n"
" devmap | devmap_hash | sockmap | cpumap | xskmap | sockhash |\n"
" cgroup_storage | reuseport_sockarray | percpu_cgroup_storage |\n"
" queue | stack | sk_storage | struct_ops | ringbuf | inode_storage }\n"
" queue | stack | sk_storage | struct_ops | ringbuf | inode_storage |\n"
" task_storage }\n"
" " HELP_SPEC_OPTIONS "\n"
"",
bin_name, argv[-2]);

9
tools/bpf/resolve_btfids/Makefile
View File

@ -18,15 +18,6 @@ else
endif
# always use the host compiler
ifneq ($(LLVM),)
HOSTAR ?= llvm-ar
HOSTCC ?= clang
HOSTLD ?= ld.lld
else
HOSTAR ?= ar
HOSTCC ?= gcc
HOSTLD ?= ld
endif
AR = $(HOSTAR)
CC = $(HOSTCC)
LD = $(HOSTLD)

55
tools/bpf/runqslower/Makefile
View File

@ -1,13 +1,18 @@
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
OUTPUT := .output
include ../../scripts/Makefile.include
OUTPUT ?= $(abspath .output)/
CLANG ?= clang
LLC ?= llc
LLVM_STRIP ?= llvm-strip
DEFAULT_BPFTOOL := $(OUTPUT)/sbin/bpftool
BPFTOOL_OUTPUT := $(OUTPUT)bpftool/
DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bpftool
BPFTOOL ?= $(DEFAULT_BPFTOOL)
LIBBPF_SRC := $(abspath ../../lib/bpf)
BPFOBJ := $(OUTPUT)/libbpf.a
BPF_INCLUDE := $(OUTPUT)
BPFOBJ_OUTPUT := $(OUTPUT)libbpf/
BPFOBJ := $(BPFOBJ_OUTPUT)libbpf.a
BPF_INCLUDE := $(BPFOBJ_OUTPUT)
INCLUDES := -I$(OUTPUT) -I$(BPF_INCLUDE) -I$(abspath ../../lib) \
-I$(abspath ../../include/uapi)
CFLAGS := -g -Wall
@ -18,13 +23,10 @@ VMLINUX_BTF_PATHS := /sys/kernel/btf/vmlinux /boot/vmlinux-$(KERNEL_REL)
VMLINUX_BTF_PATH := $(or $(VMLINUX_BTF),$(firstword \
$(wildcard $(VMLINUX_BTF_PATHS))))
abs_out := $(abspath $(OUTPUT))
ifeq ($(V),1)
Q =
msg =
else
Q = @
msg = @printf ' %-8s %s%s\n' "$(1)" "$(notdir $(2))" "$(if $(3), $(3))";
MAKEFLAGS += --no-print-directory
submake_extras := feature_display=0
endif
@ -37,12 +39,15 @@ all: runqslower
runqslower: $(OUTPUT)/runqslower
clean:
$(call msg,CLEAN)
$(Q)rm -rf $(OUTPUT) runqslower
$(call QUIET_CLEAN, runqslower)
$(Q)$(RM) -r $(BPFOBJ_OUTPUT) $(BPFTOOL_OUTPUT)
$(Q)$(RM) $(OUTPUT)*.o $(OUTPUT)*.d
$(Q)$(RM) $(OUTPUT)*.skel.h $(OUTPUT)vmlinux.h
$(Q)$(RM) $(OUTPUT)runqslower
$(Q)$(RM) -r .output
$(OUTPUT)/runqslower: $(OUTPUT)/runqslower.o $(BPFOBJ)
$(call msg,BINARY,$@)
$(Q)$(CC) $(CFLAGS) $^ -lelf -lz -o $@
$(QUIET_LINK)$(CC) $(CFLAGS) $^ -lelf -lz -o $@
$(OUTPUT)/runqslower.o: runqslower.h $(OUTPUT)/runqslower.skel.h \
$(OUTPUT)/runqslower.bpf.o
@ -50,36 +55,30 @@ $(OUTPUT)/runqslower.o: runqslower.h $(OUTPUT)/runqslower.skel.h \
$(OUTPUT)/runqslower.bpf.o: $(OUTPUT)/vmlinux.h runqslower.h
$(OUTPUT)/%.skel.h: $(OUTPUT)/%.bpf.o | $(BPFTOOL)
$(call msg,GEN-SKEL,$@)
$(Q)$(BPFTOOL) gen skeleton $< > $@
$(QUIET_GEN)$(BPFTOOL) gen skeleton $< > $@
$(OUTPUT)/%.bpf.o: %.bpf.c $(BPFOBJ) | $(OUTPUT)
$(call msg,BPF,$@)
$(Q)$(CLANG) -g -O2 -target bpf $(INCLUDES) \
$(QUIET_GEN)$(CLANG) -g -O2 -target bpf $(INCLUDES) \
-c $(filter %.c,$^) -o $@ && \
$(LLVM_STRIP) -g $@
$(OUTPUT)/%.o: %.c | $(OUTPUT)
$(call msg,CC,$@)
$(Q)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
$(QUIET_CC)$(CC) $(CFLAGS) $(INCLUDES) -c $(filter %.c,$^) -o $@
$(OUTPUT):
$(call msg,MKDIR,$@)
$(Q)mkdir -p $(OUTPUT)
$(OUTPUT) $(BPFOBJ_OUTPUT) $(BPFTOOL_OUTPUT):
$(QUIET_MKDIR)mkdir -p $@
$(OUTPUT)/vmlinux.h: $(VMLINUX_BTF_PATH) | $(OUTPUT) $(BPFTOOL)
$(call msg,GEN,$@)
$(Q)if [ ! -e "$(VMLINUX_BTF_PATH)" ] ; then \
echo "Couldn't find kernel BTF; set VMLINUX_BTF to" \
"specify its location." >&2; \
exit 1;\
fi
$(Q)$(BPFTOOL) btf dump file $(VMLINUX_BTF_PATH) format c > $@
$(QUIET_GEN)$(BPFTOOL) btf dump file $(VMLINUX_BTF_PATH) format c > $@
$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(OUTPUT)
$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) \
OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
$(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(BPFOBJ_OUTPUT)
$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(BPFOBJ_OUTPUT) $@
$(DEFAULT_BPFTOOL):
$(Q)$(MAKE) $(submake_extras) -C ../bpftool \
prefix= OUTPUT=$(abs_out)/ DESTDIR=$(abs_out) install
$(DEFAULT_BPFTOOL): | $(BPFTOOL_OUTPUT)
$(Q)$(MAKE) $(submake_extras) -C ../bpftool OUTPUT=$(BPFTOOL_OUTPUT) \
CC=$(HOSTCC) LD=$(HOSTLD)

4
tools/build/Makefile
View File

@ -15,10 +15,6 @@ endef
$(call allow-override,CC,$(CROSS_COMPILE)gcc)
$(call allow-override,LD,$(CROSS_COMPILE)ld)
HOSTCC ?= gcc
HOSTLD ?= ld
HOSTAR ?= ar
export HOSTCC HOSTLD HOSTAR
ifeq ($(V),1)

51
tools/include/uapi/linux/bpf.h
View File

@ -157,6 +157,7 @@ enum bpf_map_type {
BPF_MAP_TYPE_STRUCT_OPS,
BPF_MAP_TYPE_RINGBUF,
BPF_MAP_TYPE_INODE_STORAGE,
BPF_MAP_TYPE_TASK_STORAGE,
};
/* Note that tracing related programs such as
@ -3742,6 +3743,50 @@ union bpf_attr {
* Return
* The helper returns **TC_ACT_REDIRECT** on success or
* **TC_ACT_SHOT** on error.
*
* void *bpf_task_storage_get(struct bpf_map *map, struct task_struct *task, void *value, u64 flags)
* Description
* Get a bpf_local_storage from the *task*.
*
* Logically, it could be thought of as getting the value from
* a *map* with *task* as the **key**. From this
* perspective, the usage is not much different from
* **bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this
* helper enforces the key must be an task_struct and the map must also
* be a **BPF_MAP_TYPE_TASK_STORAGE**.
*
* Underneath, the value is stored locally at *task* instead of
* the *map*. The *map* is used as the bpf-local-storage
* "type". The bpf-local-storage "type" (i.e. the *map*) is
* searched against all bpf_local_storage residing at *task*.
*
* An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be
* used such that a new bpf_local_storage will be
* created if one does not exist. *value* can be used
* together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify
* the initial value of a bpf_local_storage. If *value* is
* **NULL**, the new bpf_local_storage will be zero initialized.
* Return
* A bpf_local_storage pointer is returned on success.
*
* **NULL** if not found or there was an error in adding
* a new bpf_local_storage.
*
* long bpf_task_storage_delete(struct bpf_map *map, struct task_struct *task)
* Description
* Delete a bpf_local_storage from a *task*.
* Return
* 0 on success.
*
* **-ENOENT** if the bpf_local_storage cannot be found.
*
* struct task_struct *bpf_get_current_task_btf(void)
* Description
* Return a BTF pointer to the "current" task.
* This pointer can also be used in helpers that accept an
* *ARG_PTR_TO_BTF_ID* of type *task_struct*.
* Return
* Pointer to the current task.
*/
#define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \
@ -3900,6 +3945,9 @@ union bpf_attr {
FN(bpf_per_cpu_ptr), \
FN(bpf_this_cpu_ptr), \
FN(redirect_peer), \
FN(task_storage_get), \
FN(task_storage_delete), \
FN(get_current_task_btf), \
/* */
/* integer value in 'imm' field of BPF_CALL instruction selects which helper
@ -4418,6 +4466,9 @@ struct bpf_btf_info {
__aligned_u64 btf;
__u32 btf_size;
__u32 id;
__aligned_u64 name;
__u32 name_len;
__u32 kernel_btf;
} __attribute__((aligned(8)));
struct bpf_link_info {

809
tools/lib/bpf/btf.c
View File

File diff suppressed because it is too large Load Diff

8
tools/lib/bpf/btf.h
View File

@ -31,11 +31,19 @@ enum btf_endianness {
};
LIBBPF_API void btf__free(struct btf *btf);
LIBBPF_API struct btf *btf__new(const void *data, __u32 size);
LIBBPF_API struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf);
LIBBPF_API struct btf *btf__new_empty(void);
LIBBPF_API struct btf *btf__new_empty_split(struct btf *base_btf);
LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext);
LIBBPF_API struct btf *btf__parse_split(const char *path, struct btf *base_btf);
LIBBPF_API struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext);
LIBBPF_API struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf);
LIBBPF_API struct btf *btf__parse_raw(const char *path);
LIBBPF_API struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf);
LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf);
LIBBPF_API int btf__load(struct btf *btf);
LIBBPF_API __s32 btf__find_by_name(const struct btf *btf,

9
tools/lib/bpf/libbpf.map
View File

@ -337,3 +337,12 @@ LIBBPF_0.2.0 {
perf_buffer__consume_buffer;
xsk_socket__create_shared;
} LIBBPF_0.1.0;
LIBBPF_0.3.0 {
global:
btf__parse_elf_split;
btf__parse_raw_split;
btf__parse_split;
btf__new_empty_split;
btf__new_split;
} LIBBPF_0.2.0;

1
tools/lib/bpf/libbpf_probes.c
View File

@ -230,6 +230,7 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex)
break;
case BPF_MAP_TYPE_SK_STORAGE:
case BPF_MAP_TYPE_INODE_STORAGE:
case BPF_MAP_TYPE_TASK_STORAGE:
btf_key_type_id = 1;
btf_value_type_id = 3;
value_size = 8;

9
tools/objtool/Makefile
View File

@ -3,15 +3,6 @@ include ../scripts/Makefile.include
include ../scripts/Makefile.arch
# always use the host compiler
ifneq ($(LLVM),)
HOSTAR ?= llvm-ar
HOSTCC ?= clang
HOSTLD ?= ld.lld
else
HOSTAR ?= ar
HOSTCC ?= gcc
HOSTLD ?= ld
endif
AR = $(HOSTAR)
CC = $(HOSTCC)
LD = $(HOSTLD)

4
tools/perf/Makefile.perf
View File

@ -175,10 +175,6 @@ endef
LD += $(EXTRA_LDFLAGS)
HOSTCC ?= gcc
HOSTLD ?= ld
HOSTAR ?= ar
PKG_CONFIG = $(CROSS_COMPILE)pkg-config
LLVM_CONFIG ?= llvm-config

1
tools/power/acpi/Makefile.config
View File

@ -54,7 +54,6 @@ INSTALL_SCRIPT = ${INSTALL_PROGRAM}
CROSS = #/usr/i386-linux-uclibc/usr/bin/i386-uclibc-
CROSS_COMPILE ?= $(CROSS)
LD = $(CC)
HOSTCC = gcc
# check if compiler option is supported
cc-supports = ${shell if $(CC) ${1} -S -o /dev/null -x c /dev/null > /dev/null 2>&1; then echo "$(1)"; fi;}

10
tools/scripts/Makefile.include
View File

@ -59,6 +59,16 @@ $(call allow-override,LD,$(CROSS_COMPILE)ld)
$(call allow-override,CXX,$(CROSS_COMPILE)g++)
$(call allow-override,STRIP,$(CROSS_COMPILE)strip)
ifneq ($(LLVM),)
HOSTAR ?= llvm-ar
HOSTCC ?= clang
HOSTLD ?= ld.lld
else
HOSTAR ?= ar
HOSTCC ?= gcc
HOSTLD ?= ld
endif
ifeq ($(CC_NO_CLANG), 1)
EXTRA_WARNINGS += -Wstrict-aliasing=3
endif

1
tools/testing/selftests/bpf/.gitignore vendored
View File

@ -8,7 +8,6 @@ FEATURE-DUMP.libbpf
fixdep
test_dev_cgroup
/test_progs*
test_tcpbpf_user
test_verifier_log
feature
test_sock

5
tools/testing/selftests/bpf/Makefile
View File

@ -32,7 +32,7 @@ LDLIBS += -lcap -lelf -lz -lrt -lpthread
# Order correspond to 'make run_tests' order
TEST_GEN_PROGS = test_verifier test_tag test_maps test_lru_map test_lpm_map test_progs \
test_verifier_log test_dev_cgroup test_tcpbpf_user \
test_verifier_log test_dev_cgroup \
test_sock test_sockmap get_cgroup_id_user test_socket_cookie \
test_cgroup_storage \
test_netcnt test_tcpnotify_user test_sysctl \
@ -163,7 +163,6 @@ $(OUTPUT)/test_sock: cgroup_helpers.c
$(OUTPUT)/test_sock_addr: cgroup_helpers.c
$(OUTPUT)/test_socket_cookie: cgroup_helpers.c
$(OUTPUT)/test_sockmap: cgroup_helpers.c
$(OUTPUT)/test_tcpbpf_user: cgroup_helpers.c
$(OUTPUT)/test_tcpnotify_user: cgroup_helpers.c trace_helpers.c
$(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c
$(OUTPUT)/test_cgroup_storage: cgroup_helpers.c
@ -387,7 +386,7 @@ TRUNNER_TESTS_DIR := prog_tests
TRUNNER_BPF_PROGS_DIR := progs
TRUNNER_EXTRA_SOURCES := test_progs.c cgroup_helpers.c trace_helpers.c \
network_helpers.c testing_helpers.c \
flow_dissector_load.h
btf_helpers.c flow_dissector_load.h
TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read \
$(wildcard progs/btf_dump_test_case_*.c)
TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE

259
tools/testing/selftests/bpf/btf_helpers.c Normal file
View File

@ -0,0 +1,259 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <stdio.h>
#include <errno.h>
#include <bpf/btf.h>
#include <bpf/libbpf.h>
#include "test_progs.h"
static const char * const btf_kind_str_mapping[] = {
[BTF_KIND_UNKN] = "UNKNOWN",
[BTF_KIND_INT] = "INT",
[BTF_KIND_PTR] = "PTR",
[BTF_KIND_ARRAY] = "ARRAY",
[BTF_KIND_STRUCT] = "STRUCT",
[BTF_KIND_UNION] = "UNION",
[BTF_KIND_ENUM] = "ENUM",
[BTF_KIND_FWD] = "FWD",
[BTF_KIND_TYPEDEF] = "TYPEDEF",
[BTF_KIND_VOLATILE] = "VOLATILE",
[BTF_KIND_CONST] = "CONST",
[BTF_KIND_RESTRICT] = "RESTRICT",
[BTF_KIND_FUNC] = "FUNC",
[BTF_KIND_FUNC_PROTO] = "FUNC_PROTO",
[BTF_KIND_VAR] = "VAR",
[BTF_KIND_DATASEC] = "DATASEC",
};
static const char *btf_kind_str(__u16 kind)
{
if (kind > BTF_KIND_DATASEC)
return "UNKNOWN";
return btf_kind_str_mapping[kind];
}
static const char *btf_int_enc_str(__u8 encoding)
{
switch (encoding) {
case 0:
return "(none)";
case BTF_INT_SIGNED:
return "SIGNED";
case BTF_INT_CHAR:
return "CHAR";
case BTF_INT_BOOL:
return "BOOL";
default:
return "UNKN";
}
}
static const char *btf_var_linkage_str(__u32 linkage)
{
switch (linkage) {
case BTF_VAR_STATIC:
return "static";
case BTF_VAR_GLOBAL_ALLOCATED:
return "global-alloc";
default:
return "(unknown)";
}
}
static const char *btf_func_linkage_str(const struct btf_type *t)
{
switch (btf_vlen(t)) {
case BTF_FUNC_STATIC:
return "static";
case BTF_FUNC_GLOBAL:
return "global";
case BTF_FUNC_EXTERN:
return "extern";
default:
return "(unknown)";
}
}
static const char *btf_str(const struct btf *btf, __u32 off)
{
if (!off)
return "(anon)";
return btf__str_by_offset(btf, off) ?: "(invalid)";
}
int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id)
{
const struct btf_type *t;
int kind, i;
__u32 vlen;
t = btf__type_by_id(btf, id);
if (!t)
return -EINVAL;
vlen = btf_vlen(t);
kind = btf_kind(t);
fprintf(out, "[%u] %s '%s'", id, btf_kind_str(kind), btf_str(btf, t->name_off));
switch (kind) {
case BTF_KIND_INT:
fprintf(out, " size=%u bits_offset=%u nr_bits=%u encoding=%s",
t->size, btf_int_offset(t), btf_int_bits(t),
btf_int_enc_str(btf_int_encoding(t)));
break;
case BTF_KIND_PTR:
case BTF_KIND_CONST:
case BTF_KIND_VOLATILE:
case BTF_KIND_RESTRICT:
case BTF_KIND_TYPEDEF:
fprintf(out, " type_id=%u", t->type);
break;
case BTF_KIND_ARRAY: {
const struct btf_array *arr = btf_array(t);
fprintf(out, " type_id=%u index_type_id=%u nr_elems=%u",
arr->type, arr->index_type, arr->nelems);
break;
}
case BTF_KIND_STRUCT:
case BTF_KIND_UNION: {
const struct btf_member *m = btf_members(t);
fprintf(out, " size=%u vlen=%u", t->size, vlen);
for (i = 0; i < vlen; i++, m++) {
__u32 bit_off, bit_sz;
bit_off = btf_member_bit_offset(t, i);
bit_sz = btf_member_bitfield_size(t, i);
fprintf(out, "\n\t'%s' type_id=%u bits_offset=%u",
btf_str(btf, m->name_off), m->type, bit_off);
if (bit_sz)
fprintf(out, " bitfield_size=%u", bit_sz);
}
break;
}
case BTF_KIND_ENUM: {
const struct btf_enum *v = btf_enum(t);
fprintf(out, " size=%u vlen=%u", t->size, vlen);
for (i = 0; i < vlen; i++, v++) {
fprintf(out, "\n\t'%s' val=%u",
btf_str(btf, v->name_off), v->val);
}
break;
}
case BTF_KIND_FWD:
fprintf(out, " fwd_kind=%s", btf_kflag(t) ? "union" : "struct");
break;
case BTF_KIND_FUNC:
fprintf(out, " type_id=%u linkage=%s", t->type, btf_func_linkage_str(t));
break;
case BTF_KIND_FUNC_PROTO: {
const struct btf_param *p = btf_params(t);
fprintf(out, " ret_type_id=%u vlen=%u", t->type, vlen);
for (i = 0; i < vlen; i++, p++) {
fprintf(out, "\n\t'%s' type_id=%u",
btf_str(btf, p->name_off), p->type);
}
break;
}
case BTF_KIND_VAR:
fprintf(out, " type_id=%u, linkage=%s",
t->type, btf_var_linkage_str(btf_var(t)->linkage));
break;
case BTF_KIND_DATASEC: {
const struct btf_var_secinfo *v = btf_var_secinfos(t);
fprintf(out, " size=%u vlen=%u", t->size, vlen);
for (i = 0; i < vlen; i++, v++) {
fprintf(out, "\n\ttype_id=%u offset=%u size=%u",
v->type, v->offset, v->size);
}
break;
}
default:
break;
}
return 0;
}
/* Print raw BTF type dump into a local buffer and return string pointer back.
* Buffer *will* be overwritten by subsequent btf_type_raw_dump() calls
*/
const char *btf_type_raw_dump(const struct btf *btf, int type_id)
{
static char buf[16 * 1024];
FILE *buf_file;
buf_file = fmemopen(buf, sizeof(buf) - 1, "w");
if (!buf_file) {
fprintf(stderr, "Failed to open memstream: %d\n", errno);
return NULL;
}
fprintf_btf_type_raw(buf_file, btf, type_id);
fflush(buf_file);
fclose(buf_file);
return buf;
}
int btf_validate_raw(struct btf *btf, int nr_types, const char *exp_types[])
{
int i;
bool ok = true;
ASSERT_EQ(btf__get_nr_types(btf), nr_types, "btf_nr_types");
for (i = 1; i <= nr_types; i++) {
if (!ASSERT_STREQ(btf_type_raw_dump(btf, i), exp_types[i - 1], "raw_dump"))
ok = false;
}
return ok;
}
static void btf_dump_printf(void *ctx, const char *fmt, va_list args)
{
vfprintf(ctx, fmt, args);
}
/* Print BTF-to-C dump into a local buffer and return string pointer back.
* Buffer *will* be overwritten by subsequent btf_type_raw_dump() calls
*/
const char *btf_type_c_dump(const struct btf *btf)
{
static char buf[16 * 1024];
FILE *buf_file;
struct btf_dump *d = NULL;
struct btf_dump_opts opts = {};
int err, i;
buf_file = fmemopen(buf, sizeof(buf) - 1, "w");
if (!buf_file) {
fprintf(stderr, "Failed to open memstream: %d\n", errno);
return NULL;
}
opts.ctx = buf_file;
d = btf_dump__new(btf, NULL, &opts, btf_dump_printf);
if (libbpf_get_error(d)) {
fprintf(stderr, "Failed to create btf_dump instance: %ld\n", libbpf_get_error(d));
return NULL;
}
for (i = 1; i <= btf__get_nr_types(btf); i++) {
err = btf_dump__dump_type(d, i);
if (err) {
fprintf(stderr, "Failed to dump type [%d]: %d\n", i, err);
return NULL;
}
}
fflush(buf_file);
fclose(buf_file);
return buf;
}

19
tools/testing/selftests/bpf/btf_helpers.h Normal file
View File

@ -0,0 +1,19 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2020 Facebook */
#ifndef __BTF_HELPERS_H
#define __BTF_HELPERS_H
#include <stdio.h>
#include <bpf/btf.h>
int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id);
const char *btf_type_raw_dump(const struct btf *btf, int type_id);
int btf_validate_raw(struct btf *btf, int nr_types, const char *exp_types[]);
#define VALIDATE_RAW_BTF(btf, raw_types...) \
btf_validate_raw(btf, \
sizeof((const char *[]){raw_types})/sizeof(void *),\
(const char *[]){raw_types})
const char *btf_type_c_dump(const struct btf *btf);
#endif

40
tools/testing/selftests/bpf/prog_tests/btf.c
View File

@ -6652,7 +6652,7 @@ static void do_test_dedup(unsigned int test_num)
const void *test_btf_data, *expect_btf_data;
const char *ret_test_next_str, *ret_expect_next_str;
const char *test_strs, *expect_strs;
const char *test_str_cur, *test_str_end;
const char *test_str_cur;
const char *expect_str_cur, *expect_str_end;
unsigned int raw_btf_size;
void *raw_btf;
@ -6719,12 +6719,18 @@ static void do_test_dedup(unsigned int test_num)
goto done;
}
test_str_cur = test_strs;
test_str_end = test_strs + test_hdr->str_len;
expect_str_cur = expect_strs;
expect_str_end = expect_strs + expect_hdr->str_len;
while (test_str_cur < test_str_end && expect_str_cur < expect_str_end) {
while (expect_str_cur < expect_str_end) {
size_t test_len, expect_len;
int off;
off = btf__find_str(test_btf, expect_str_cur);
if (CHECK(off < 0, "exp str '%s' not found: %d\n", expect_str_cur, off)) {
err = -1;
goto done;
}
test_str_cur = btf__str_by_offset(test_btf, off);
test_len = strlen(test_str_cur);
expect_len = strlen(expect_str_cur);
@ -6741,15 +6747,8 @@ static void do_test_dedup(unsigned int test_num)
err = -1;
goto done;
}
test_str_cur += test_len + 1;
expect_str_cur += expect_len + 1;
}
if (CHECK(test_str_cur != test_str_end,
"test_str_cur:%p != test_str_end:%p",
test_str_cur, test_str_end)) {
err = -1;
goto done;
}
test_nr_types = btf__get_nr_types(test_btf);
expect_nr_types = btf__get_nr_types(expect_btf);
@ -6775,10 +6774,21 @@ static void do_test_dedup(unsigned int test_num)
err = -1;
goto done;
}
if (CHECK(memcmp((void *)test_type,
(void *)expect_type,
test_size),
"type #%d: contents differ", i)) {
if (CHECK(btf_kind(test_type) != btf_kind(expect_type),
"type %d kind: exp %d != got %u\n",
i, btf_kind(expect_type), btf_kind(test_type))) {
err = -1;
goto done;
}
if (CHECK(test_type->info != expect_type->info,
"type %d info: exp %d != got %u\n",
i, expect_type->info, test_type->info)) {
err = -1;
goto done;
}
if (CHECK(test_type->size != expect_type->size,
"type %d size/type: exp %d != got %u\n",
i, expect_type->size, test_type->size)) {
err = -1;
goto done;
}

325
tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c Normal file
View File

@ -0,0 +1,325 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <test_progs.h>
#include <bpf/btf.h>
#include "btf_helpers.h"
static void test_split_simple() {
const struct btf_type *t;
struct btf *btf1, *btf2;
int str_off, err;
btf1 = btf__new_empty();
if (!ASSERT_OK_PTR(btf1, "empty_main_btf"))
return;
btf__set_pointer_size(btf1, 8); /* enforce 64-bit arch */
btf__add_int(btf1, "int", 4, BTF_INT_SIGNED); /* [1] int */
btf__add_ptr(btf1, 1); /* [2] ptr to int */
btf__add_struct(btf1, "s1", 4); /* [3] struct s1 { */
btf__add_field(btf1, "f1", 1, 0, 0); /* int f1; */
/* } */
VALIDATE_RAW_BTF(
btf1,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=1",
"[3] STRUCT 's1' size=4 vlen=1\n"
"\t'f1' type_id=1 bits_offset=0");
ASSERT_STREQ(btf_type_c_dump(btf1), "\
struct s1 {\n\
int f1;\n\
};\n\n", "c_dump");
btf2 = btf__new_empty_split(btf1);
if (!ASSERT_OK_PTR(btf2, "empty_split_btf"))
goto cleanup;
/* pointer size should be "inherited" from main BTF */
ASSERT_EQ(btf__pointer_size(btf2), 8, "inherit_ptr_sz");
str_off = btf__find_str(btf2, "int");
ASSERT_NEQ(str_off, -ENOENT, "str_int_missing");
t = btf__type_by_id(btf2, 1);
if (!ASSERT_OK_PTR(t, "int_type"))
goto cleanup;
ASSERT_EQ(btf_is_int(t), true, "int_kind");
ASSERT_STREQ(btf__str_by_offset(btf2, t->name_off), "int", "int_name");
btf__add_struct(btf2, "s2", 16); /* [4] struct s2 { */
btf__add_field(btf2, "f1", 6, 0, 0); /* struct s1 f1; */
btf__add_field(btf2, "f2", 5, 32, 0); /* int f2; */
btf__add_field(btf2, "f3", 2, 64, 0); /* int *f3; */
/* } */
/* duplicated int */
btf__add_int(btf2, "int", 4, BTF_INT_SIGNED); /* [5] int */
/* duplicated struct s1 */
btf__add_struct(btf2, "s1", 4); /* [6] struct s1 { */
btf__add_field(btf2, "f1", 5, 0, 0); /* int f1; */
/* } */
VALIDATE_RAW_BTF(
btf2,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=1",
"[3] STRUCT 's1' size=4 vlen=1\n"
"\t'f1' type_id=1 bits_offset=0",
"[4] STRUCT 's2' size=16 vlen=3\n"
"\t'f1' type_id=6 bits_offset=0\n"
"\t'f2' type_id=5 bits_offset=32\n"
"\t'f3' type_id=2 bits_offset=64",
"[5] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[6] STRUCT 's1' size=4 vlen=1\n"
"\t'f1' type_id=5 bits_offset=0");
ASSERT_STREQ(btf_type_c_dump(btf2), "\
struct s1 {\n\
int f1;\n\
};\n\
\n\
struct s1___2 {\n\
int f1;\n\
};\n\
\n\
struct s2 {\n\
struct s1___2 f1;\n\
int f2;\n\
int *f3;\n\
};\n\n", "c_dump");
err = btf__dedup(btf2, NULL, NULL);
if (!ASSERT_OK(err, "btf_dedup"))
goto cleanup;
VALIDATE_RAW_BTF(
btf2,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=1",
"[3] STRUCT 's1' size=4 vlen=1\n"
"\t'f1' type_id=1 bits_offset=0",
"[4] STRUCT 's2' size=16 vlen=3\n"
"\t'f1' type_id=3 bits_offset=0\n"
"\t'f2' type_id=1 bits_offset=32\n"
"\t'f3' type_id=2 bits_offset=64");
ASSERT_STREQ(btf_type_c_dump(btf2), "\
struct s1 {\n\
int f1;\n\
};\n\
\n\
struct s2 {\n\
struct s1 f1;\n\
int f2;\n\
int *f3;\n\
};\n\n", "c_dump");
cleanup:
btf__free(btf2);
btf__free(btf1);
}
static void test_split_fwd_resolve() {
struct btf *btf1, *btf2;
int err;
btf1 = btf__new_empty();
if (!ASSERT_OK_PTR(btf1, "empty_main_btf"))
return;
btf__set_pointer_size(btf1, 8); /* enforce 64-bit arch */
btf__add_int(btf1, "int", 4, BTF_INT_SIGNED); /* [1] int */
btf__add_ptr(btf1, 4); /* [2] ptr to struct s1 */
btf__add_ptr(btf1, 5); /* [3] ptr to struct s2 */
btf__add_struct(btf1, "s1", 16); /* [4] struct s1 { */
btf__add_field(btf1, "f1", 2, 0, 0); /* struct s1 *f1; */
btf__add_field(btf1, "f2", 3, 64, 0); /* struct s2 *f2; */
/* } */
btf__add_struct(btf1, "s2", 4); /* [5] struct s2 { */
btf__add_field(btf1, "f1", 1, 0, 0); /* int f1; */
/* } */
VALIDATE_RAW_BTF(
btf1,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=4",
"[3] PTR '(anon)' type_id=5",
"[4] STRUCT 's1' size=16 vlen=2\n"
"\t'f1' type_id=2 bits_offset=0\n"
"\t'f2' type_id=3 bits_offset=64",
"[5] STRUCT 's2' size=4 vlen=1\n"
"\t'f1' type_id=1 bits_offset=0");
btf2 = btf__new_empty_split(btf1);
if (!ASSERT_OK_PTR(btf2, "empty_split_btf"))
goto cleanup;
btf__add_int(btf2, "int", 4, BTF_INT_SIGNED); /* [6] int */
btf__add_ptr(btf2, 10); /* [7] ptr to struct s1 */
btf__add_fwd(btf2, "s2", BTF_FWD_STRUCT); /* [8] fwd for struct s2 */
btf__add_ptr(btf2, 8); /* [9] ptr to fwd struct s2 */
btf__add_struct(btf2, "s1", 16); /* [10] struct s1 { */
btf__add_field(btf2, "f1", 7, 0, 0); /* struct s1 *f1; */
btf__add_field(btf2, "f2", 9, 64, 0); /* struct s2 *f2; */
/* } */
VALIDATE_RAW_BTF(
btf2,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=4",
"[3] PTR '(anon)' type_id=5",
"[4] STRUCT 's1' size=16 vlen=2\n"
"\t'f1' type_id=2 bits_offset=0\n"
"\t'f2' type_id=3 bits_offset=64",
"[5] STRUCT 's2' size=4 vlen=1\n"
"\t'f1' type_id=1 bits_offset=0",
"[6] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[7] PTR '(anon)' type_id=10",
"[8] FWD 's2' fwd_kind=struct",
"[9] PTR '(anon)' type_id=8",
"[10] STRUCT 's1' size=16 vlen=2\n"
"\t'f1' type_id=7 bits_offset=0\n"
"\t'f2' type_id=9 bits_offset=64");
err = btf__dedup(btf2, NULL, NULL);
if (!ASSERT_OK(err, "btf_dedup"))
goto cleanup;
VALIDATE_RAW_BTF(
btf2,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=4",
"[3] PTR '(anon)' type_id=5",
"[4] STRUCT 's1' size=16 vlen=2\n"
"\t'f1' type_id=2 bits_offset=0\n"
"\t'f2' type_id=3 bits_offset=64",
"[5] STRUCT 's2' size=4 vlen=1\n"
"\t'f1' type_id=1 bits_offset=0");
cleanup:
btf__free(btf2);
btf__free(btf1);
}
static void test_split_struct_duped() {
struct btf *btf1, *btf2;
int err;
btf1 = btf__new_empty();
if (!ASSERT_OK_PTR(btf1, "empty_main_btf"))
return;
btf__set_pointer_size(btf1, 8); /* enforce 64-bit arch */
btf__add_int(btf1, "int", 4, BTF_INT_SIGNED); /* [1] int */
btf__add_ptr(btf1, 5); /* [2] ptr to struct s1 */
btf__add_fwd(btf1, "s2", BTF_FWD_STRUCT); /* [3] fwd for struct s2 */
btf__add_ptr(btf1, 3); /* [4] ptr to fwd struct s2 */
btf__add_struct(btf1, "s1", 16); /* [5] struct s1 { */
btf__add_field(btf1, "f1", 2, 0, 0); /* struct s1 *f1; */
btf__add_field(btf1, "f2", 4, 64, 0); /* struct s2 *f2; */
/* } */
VALIDATE_RAW_BTF(
btf1,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=5",
"[3] FWD 's2' fwd_kind=struct",
"[4] PTR '(anon)' type_id=3",
"[5] STRUCT 's1' size=16 vlen=2\n"
"\t'f1' type_id=2 bits_offset=0\n"
"\t'f2' type_id=4 bits_offset=64");
btf2 = btf__new_empty_split(btf1);
if (!ASSERT_OK_PTR(btf2, "empty_split_btf"))
goto cleanup;
btf__add_int(btf2, "int", 4, BTF_INT_SIGNED); /* [6] int */
btf__add_ptr(btf2, 10); /* [7] ptr to struct s1 */
btf__add_fwd(btf2, "s2", BTF_FWD_STRUCT); /* [8] fwd for struct s2 */
btf__add_ptr(btf2, 11); /* [9] ptr to struct s2 */
btf__add_struct(btf2, "s1", 16); /* [10] struct s1 { */
btf__add_field(btf2, "f1", 7, 0, 0); /* struct s1 *f1; */
btf__add_field(btf2, "f2", 9, 64, 0); /* struct s2 *f2; */
/* } */
btf__add_struct(btf2, "s2", 40); /* [11] struct s2 { */
btf__add_field(btf2, "f1", 7, 0, 0); /* struct s1 *f1; */
btf__add_field(btf2, "f2", 9, 64, 0); /* struct s2 *f2; */
btf__add_field(btf2, "f3", 6, 128, 0); /* int f3; */
btf__add_field(btf2, "f4", 10, 192, 0); /* struct s1 f4; */
/* } */
btf__add_ptr(btf2, 8); /* [12] ptr to fwd struct s2 */
btf__add_struct(btf2, "s3", 8); /* [13] struct s3 { */
btf__add_field(btf2, "f1", 12, 0, 0); /* struct s2 *f1; (fwd) */
/* } */
VALIDATE_RAW_BTF(
btf2,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=5",
"[3] FWD 's2' fwd_kind=struct",
"[4] PTR '(anon)' type_id=3",
"[5] STRUCT 's1' size=16 vlen=2\n"
"\t'f1' type_id=2 bits_offset=0\n"
"\t'f2' type_id=4 bits_offset=64",
"[6] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[7] PTR '(anon)' type_id=10",
"[8] FWD 's2' fwd_kind=struct",
"[9] PTR '(anon)' type_id=11",
"[10] STRUCT 's1' size=16 vlen=2\n"
"\t'f1' type_id=7 bits_offset=0\n"
"\t'f2' type_id=9 bits_offset=64",
"[11] STRUCT 's2' size=40 vlen=4\n"
"\t'f1' type_id=7 bits_offset=0\n"
"\t'f2' type_id=9 bits_offset=64\n"
"\t'f3' type_id=6 bits_offset=128\n"
"\t'f4' type_id=10 bits_offset=192",
"[12] PTR '(anon)' type_id=8",
"[13] STRUCT 's3' size=8 vlen=1\n"
"\t'f1' type_id=12 bits_offset=0");
err = btf__dedup(btf2, NULL, NULL);
if (!ASSERT_OK(err, "btf_dedup"))
goto cleanup;
VALIDATE_RAW_BTF(
btf2,
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED",
"[2] PTR '(anon)' type_id=5",
"[3] FWD 's2' fwd_kind=struct",
"[4] PTR '(anon)' type_id=3",
"[5] STRUCT 's1' size=16 vlen=2\n"
"\t'f1' type_id=2 bits_offset=0\n"
"\t'f2' type_id=4 bits_offset=64",
"[6] PTR '(anon)' type_id=8",
"[7] PTR '(anon)' type_id=9",
"[8] STRUCT 's1' size=16 vlen=2\n"
"\t'f1' type_id=6 bits_offset=0\n"
"\t'f2' type_id=7 bits_offset=64",
"[9] STRUCT 's2' size=40 vlen=4\n"
"\t'f1' type_id=6 bits_offset=0\n"
"\t'f2' type_id=7 bits_offset=64\n"
"\t'f3' type_id=1 bits_offset=128\n"
"\t'f4' type_id=8 bits_offset=192",
"[10] STRUCT 's3' size=8 vlen=1\n"
"\t'f1' type_id=7 bits_offset=0");
cleanup:
btf__free(btf2);
btf__free(btf1);
}
void test_btf_dedup_split()
{
if (test__start_subtest("split_simple"))
test_split_simple();
if (test__start_subtest("split_struct_duped"))
test_split_struct_duped();
if (test__start_subtest("split_fwd_resolve"))
test_split_fwd_resolve();
}

2
tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c
View File

@ -17,7 +17,7 @@
#include "test_btf_skc_cls_ingress.skel.h"
static struct test_btf_skc_cls_ingress *skel;
struct sockaddr_in6 srv_sa6;
static struct sockaddr_in6 srv_sa6;
static __u32 duration;
#define PROG_PIN_FILE "/sys/fs/bpf/btf_skc_cls_ingress"

99
tools/testing/selftests/bpf/prog_tests/btf_split.c Normal file
View File

@ -0,0 +1,99 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <test_progs.h>
#include <bpf/btf.h>
static char *dump_buf;
static size_t dump_buf_sz;
static FILE *dump_buf_file;
static void btf_dump_printf(void *ctx, const char *fmt, va_list args)
{
vfprintf(ctx, fmt, args);
}
void test_btf_split() {
struct btf_dump_opts opts;
struct btf_dump *d = NULL;
const struct btf_type *t;
struct btf *btf1, *btf2;
int str_off, i, err;
btf1 = btf__new_empty();
if (!ASSERT_OK_PTR(btf1, "empty_main_btf"))
return;
btf__set_pointer_size(btf1, 8); /* enforce 64-bit arch */
btf__add_int(btf1, "int", 4, BTF_INT_SIGNED); /* [1] int */
btf__add_ptr(btf1, 1); /* [2] ptr to int */
btf__add_struct(btf1, "s1", 4); /* [3] struct s1 { */
btf__add_field(btf1, "f1", 1, 0, 0); /* int f1; */
/* } */
btf2 = btf__new_empty_split(btf1);
if (!ASSERT_OK_PTR(btf2, "empty_split_btf"))
goto cleanup;
/* pointer size should be "inherited" from main BTF */
ASSERT_EQ(btf__pointer_size(btf2), 8, "inherit_ptr_sz");
str_off = btf__find_str(btf2, "int");
ASSERT_NEQ(str_off, -ENOENT, "str_int_missing");
t = btf__type_by_id(btf2, 1);
if (!ASSERT_OK_PTR(t, "int_type"))
goto cleanup;
ASSERT_EQ(btf_is_int(t), true, "int_kind");
ASSERT_STREQ(btf__str_by_offset(btf2, t->name_off), "int", "int_name");
btf__add_struct(btf2, "s2", 16); /* [4] struct s2 { */
btf__add_field(btf2, "f1", 3, 0, 0); /* struct s1 f1; */
btf__add_field(btf2, "f2", 1, 32, 0); /* int f2; */
btf__add_field(btf2, "f3", 2, 64, 0); /* int *f3; */
/* } */
t = btf__type_by_id(btf1, 4);
ASSERT_NULL(t, "split_type_in_main");
t = btf__type_by_id(btf2, 4);
if (!ASSERT_OK_PTR(t, "split_struct_type"))
goto cleanup;
ASSERT_EQ(btf_is_struct(t), true, "split_struct_kind");
ASSERT_EQ(btf_vlen(t), 3, "split_struct_vlen");
ASSERT_STREQ(btf__str_by_offset(btf2, t->name_off), "s2", "split_struct_name");
/* BTF-to-C dump of split BTF */
dump_buf_file = open_memstream(&dump_buf, &dump_buf_sz);
if (!ASSERT_OK_PTR(dump_buf_file, "dump_memstream"))
return;
opts.ctx = dump_buf_file;
d = btf_dump__new(btf2, NULL, &opts, btf_dump_printf);
if (!ASSERT_OK_PTR(d, "btf_dump__new"))
goto cleanup;
for (i = 1; i <= btf__get_nr_types(btf2); i++) {
err = btf_dump__dump_type(d, i);
ASSERT_OK(err, "dump_type_ok");
}
fflush(dump_buf_file);
dump_buf[dump_buf_sz] = 0; /* some libc implementations don't do this */
ASSERT_STREQ(dump_buf,
"struct s1 {\n"
" int f1;\n"
"};\n"
"\n"
"struct s2 {\n"
" struct s1 f1;\n"
" int f2;\n"
" int *f3;\n"
"};\n\n", "c_dump");
cleanup:
if (dump_buf_file)
fclose(dump_buf_file);
free(dump_buf);
btf_dump__free(d);
btf__free(btf1);
btf__free(btf2);
}

43
tools/testing/selftests/bpf/prog_tests/btf_write.c
View File

@ -2,6 +2,7 @@
/* Copyright (c) 2020 Facebook */
#include <test_progs.h>
#include <bpf/btf.h>
#include "btf_helpers.h"
static int duration = 0;
@ -39,6 +40,8 @@ void test_btf_write() {
ASSERT_EQ(t->size, 4, "int_sz");
ASSERT_EQ(btf_int_encoding(t), BTF_INT_SIGNED, "int_enc");
ASSERT_EQ(btf_int_bits(t), 32, "int_bits");
ASSERT_STREQ(btf_type_raw_dump(btf, 1),
"[1] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", "raw_dump");
/* invalid int size */
id = btf__add_int(btf, "bad sz int", 7, 0);
@ -59,24 +62,32 @@ void test_btf_write() {
t = btf__type_by_id(btf, 2);
ASSERT_EQ(btf_kind(t), BTF_KIND_PTR, "ptr_kind");
ASSERT_EQ(t->type, 1, "ptr_type");
ASSERT_STREQ(btf_type_raw_dump(btf, 2),
"[2] PTR '(anon)' type_id=1", "raw_dump");
id = btf__add_const(btf, 5); /* points forward to restrict */
ASSERT_EQ(id, 3, "const_id");
t = btf__type_by_id(btf, 3);
ASSERT_EQ(btf_kind(t), BTF_KIND_CONST, "const_kind");
ASSERT_EQ(t->type, 5, "const_type");
ASSERT_STREQ(btf_type_raw_dump(btf, 3),
"[3] CONST '(anon)' type_id=5", "raw_dump");
id = btf__add_volatile(btf, 3);
ASSERT_EQ(id, 4, "volatile_id");
t = btf__type_by_id(btf, 4);
ASSERT_EQ(btf_kind(t), BTF_KIND_VOLATILE, "volatile_kind");
ASSERT_EQ(t->type, 3, "volatile_type");
ASSERT_STREQ(btf_type_raw_dump(btf, 4),
"[4] VOLATILE '(anon)' type_id=3", "raw_dump");
id = btf__add_restrict(btf, 4);
ASSERT_EQ(id, 5, "restrict_id");
t = btf__type_by_id(btf, 5);
ASSERT_EQ(btf_kind(t), BTF_KIND_RESTRICT, "restrict_kind");
ASSERT_EQ(t->type, 4, "restrict_type");
ASSERT_STREQ(btf_type_raw_dump(btf, 5),
"[5] RESTRICT '(anon)' type_id=4", "raw_dump");
/* ARRAY */
id = btf__add_array(btf, 1, 2, 10); /* int *[10] */
@ -86,6 +97,8 @@ void test_btf_write() {
ASSERT_EQ(btf_array(t)->index_type, 1, "array_index_type");
ASSERT_EQ(btf_array(t)->type, 2, "array_elem_type");
ASSERT_EQ(btf_array(t)->nelems, 10, "array_nelems");
ASSERT_STREQ(btf_type_raw_dump(btf, 6),
"[6] ARRAY '(anon)' type_id=2 index_type_id=1 nr_elems=10", "raw_dump");
/* STRUCT */
err = btf__add_field(btf, "field", 1, 0, 0);
@ -113,6 +126,10 @@ void test_btf_write() {
ASSERT_EQ(m->type, 1, "f2_type");
ASSERT_EQ(btf_member_bit_offset(t, 1), 32, "f2_bit_off");
ASSERT_EQ(btf_member_bitfield_size(t, 1), 16, "f2_bit_sz");
ASSERT_STREQ(btf_type_raw_dump(btf, 7),
"[7] STRUCT 's1' size=8 vlen=2\n"
"\t'f1' type_id=1 bits_offset=0\n"
"\t'f2' type_id=1 bits_offset=32 bitfield_size=16", "raw_dump");
/* UNION */
id = btf__add_union(btf, "u1", 8);
@ -136,6 +153,9 @@ void test_btf_write() {
ASSERT_EQ(m->type, 1, "f1_type");
ASSERT_EQ(btf_member_bit_offset(t, 0), 0, "f1_bit_off");
ASSERT_EQ(btf_member_bitfield_size(t, 0), 16, "f1_bit_sz");
ASSERT_STREQ(btf_type_raw_dump(btf, 8),
"[8] UNION 'u1' size=8 vlen=1\n"
"\t'f1' type_id=1 bits_offset=0 bitfield_size=16", "raw_dump");
/* ENUM */
id = btf__add_enum(btf, "e1", 4);
@ -156,6 +176,10 @@ void test_btf_write() {
v = btf_enum(t) + 1;
ASSERT_STREQ(btf__str_by_offset(btf, v->name_off), "v2", "v2_name");
ASSERT_EQ(v->val, 2, "v2_val");
ASSERT_STREQ(btf_type_raw_dump(btf, 9),
"[9] ENUM 'e1' size=4 vlen=2\n"
"\t'v1' val=1\n"
"\t'v2' val=2", "raw_dump");
/* FWDs */
id = btf__add_fwd(btf, "struct_fwd", BTF_FWD_STRUCT);
@ -164,6 +188,8 @@ void test_btf_write() {
ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "struct_fwd", "fwd_name");
ASSERT_EQ(btf_kind(t), BTF_KIND_FWD, "fwd_kind");
ASSERT_EQ(btf_kflag(t), 0, "fwd_kflag");
ASSERT_STREQ(btf_type_raw_dump(btf, 10),
"[10] FWD 'struct_fwd' fwd_kind=struct", "raw_dump");
id = btf__add_fwd(btf, "union_fwd", BTF_FWD_UNION);
ASSERT_EQ(id, 11, "union_fwd_id");
@ -171,6 +197,8 @@ void test_btf_write() {
ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "union_fwd", "fwd_name");
ASSERT_EQ(btf_kind(t), BTF_KIND_FWD, "fwd_kind");
ASSERT_EQ(btf_kflag(t), 1, "fwd_kflag");
ASSERT_STREQ(btf_type_raw_dump(btf, 11),
"[11] FWD 'union_fwd' fwd_kind=union", "raw_dump");
id = btf__add_fwd(btf, "enum_fwd", BTF_FWD_ENUM);
ASSERT_EQ(id, 12, "enum_fwd_id");
@ -179,6 +207,8 @@ void test_btf_write() {
ASSERT_EQ(btf_kind(t), BTF_KIND_ENUM, "enum_fwd_kind");
ASSERT_EQ(btf_vlen(t), 0, "enum_fwd_kind");
ASSERT_EQ(t->size, 4, "enum_fwd_sz");
ASSERT_STREQ(btf_type_raw_dump(btf, 12),
"[12] ENUM 'enum_fwd' size=4 vlen=0", "raw_dump");
/* TYPEDEF */
id = btf__add_typedef(btf, "typedef1", 1);
@ -187,6 +217,8 @@ void test_btf_write() {
ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "typedef1", "typedef_name");
ASSERT_EQ(btf_kind(t), BTF_KIND_TYPEDEF, "typedef_kind");
ASSERT_EQ(t->type, 1, "typedef_type");
ASSERT_STREQ(btf_type_raw_dump(btf, 13),
"[13] TYPEDEF 'typedef1' type_id=1", "raw_dump");
/* FUNC & FUNC_PROTO */
id = btf__add_func(btf, "func1", BTF_FUNC_GLOBAL, 15);
@ -196,6 +228,8 @@ void test_btf_write() {
ASSERT_EQ(t->type, 15, "func_type");
ASSERT_EQ(btf_kind(t), BTF_KIND_FUNC, "func_kind");
ASSERT_EQ(btf_vlen(t), BTF_FUNC_GLOBAL, "func_vlen");
ASSERT_STREQ(btf_type_raw_dump(btf, 14),
"[14] FUNC 'func1' type_id=15 linkage=global", "raw_dump");
id = btf__add_func_proto(btf, 1);
ASSERT_EQ(id, 15, "func_proto_id");
@ -214,6 +248,10 @@ void test_btf_write() {
p = btf_params(t) + 1;
ASSERT_STREQ(btf__str_by_offset(btf, p->name_off), "p2", "p2_name");
ASSERT_EQ(p->type, 2, "p2_type");
ASSERT_STREQ(btf_type_raw_dump(btf, 15),
"[15] FUNC_PROTO '(anon)' ret_type_id=1 vlen=2\n"
"\t'p1' type_id=1\n"
"\t'p2' type_id=2", "raw_dump");
/* VAR */
id = btf__add_var(btf, "var1", BTF_VAR_GLOBAL_ALLOCATED, 1);
@ -223,6 +261,8 @@ void test_btf_write() {
ASSERT_EQ(btf_kind(t), BTF_KIND_VAR, "var_kind");
ASSERT_EQ(t->type, 1, "var_type");
ASSERT_EQ(btf_var(t)->linkage, BTF_VAR_GLOBAL_ALLOCATED, "var_type");
ASSERT_STREQ(btf_type_raw_dump(btf, 16),
"[16] VAR 'var1' type_id=1, linkage=global-alloc", "raw_dump");
/* DATASECT */
id = btf__add_datasec(btf, "datasec1", 12);
@ -239,6 +279,9 @@ void test_btf_write() {
ASSERT_EQ(vi->type, 1, "v1_type");
ASSERT_EQ(vi->offset, 4, "v1_off");
ASSERT_EQ(vi->size, 8, "v1_sz");
ASSERT_STREQ(btf_type_raw_dump(btf, 17),
"[17] DATASEC 'datasec1' size=12 vlen=1\n"
"\ttype_id=1 offset=4 size=8", "raw_dump");
btf__free(btf);
}

43
tools/testing/selftests/bpf/prog_tests/hash_large_key.c Normal file
View File

@ -0,0 +1,43 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#include "test_hash_large_key.skel.h"
void test_hash_large_key(void)
{
int err, value = 21, duration = 0, hash_map_fd;
struct test_hash_large_key *skel;
struct bigelement {
int a;
char b[4096];
long long c;
} key;
bzero(&key, sizeof(key));
skel = test_hash_large_key__open_and_load();
if (CHECK(!skel, "skel_open_and_load", "skeleton open/load failed\n"))
return;
hash_map_fd = bpf_map__fd(skel->maps.hash_map);
if (CHECK(hash_map_fd < 0, "bpf_map__fd", "failed\n"))
goto cleanup;
err = test_hash_large_key__attach(skel);
if (CHECK(err, "attach_raw_tp", "err %d\n", err))
goto cleanup;
err = bpf_map_update_elem(hash_map_fd, &key, &value, BPF_ANY);
if (CHECK(err, "bpf_map_update_elem", "errno=%d\n", errno))
goto cleanup;
key.c = 1;
err = bpf_map_lookup_elem(hash_map_fd, &key, &value);
if (CHECK(err, "bpf_map_lookup_elem", "errno=%d\n", errno))
goto cleanup;
CHECK_FAIL(value != 42);
cleanup:
test_hash_large_key__destroy(skel);
}

135
tools/testing/selftests/bpf/prog_tests/sk_storage_tracing.c Normal file
View File

@ -0,0 +1,135 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <sys/types.h>
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include "test_progs.h"
#include "network_helpers.h"
#include "test_sk_storage_trace_itself.skel.h"
#include "test_sk_storage_tracing.skel.h"
#define LO_ADDR6 "::1"
#define TEST_COMM "test_progs"
struct sk_stg {
__u32 pid;
__u32 last_notclose_state;
char comm[16];
};
static struct test_sk_storage_tracing *skel;
static __u32 duration;
static pid_t my_pid;
static int check_sk_stg(int sk_fd, __u32 expected_state)
{
struct sk_stg sk_stg;
int err;
err = bpf_map_lookup_elem(bpf_map__fd(skel->maps.sk_stg_map), &sk_fd,
&sk_stg);
if (!ASSERT_OK(err, "map_lookup(sk_stg_map)"))
return -1;
if (!ASSERT_EQ(sk_stg.last_notclose_state, expected_state,
"last_notclose_state"))
return -1;
if (!ASSERT_EQ(sk_stg.pid, my_pid, "pid"))
return -1;
if (!ASSERT_STREQ(sk_stg.comm, skel->bss->task_comm, "task_comm"))
return -1;
return 0;
}
static void do_test(void)
{
int listen_fd = -1, passive_fd = -1, active_fd = -1, value = 1, err;
char abyte;
listen_fd = start_server(AF_INET6, SOCK_STREAM, LO_ADDR6, 0, 0);
if (CHECK(listen_fd == -1, "start_server",
"listen_fd:%d errno:%d\n", listen_fd, errno))
return;
active_fd = connect_to_fd(listen_fd, 0);
if (CHECK(active_fd == -1, "connect_to_fd", "active_fd:%d errno:%d\n",
active_fd, errno))
goto out;
err = bpf_map_update_elem(bpf_map__fd(skel->maps.del_sk_stg_map),
&active_fd, &value, 0);
if (!ASSERT_OK(err, "map_update(del_sk_stg_map)"))
goto out;
passive_fd = accept(listen_fd, NULL, 0);
if (CHECK(passive_fd == -1, "accept", "passive_fd:%d errno:%d\n",
passive_fd, errno))
goto out;
shutdown(active_fd, SHUT_WR);
err = read(passive_fd, &abyte, 1);
if (!ASSERT_OK(err, "read(passive_fd)"))
goto out;
shutdown(passive_fd, SHUT_WR);
err = read(active_fd, &abyte, 1);
if (!ASSERT_OK(err, "read(active_fd)"))
goto out;
err = bpf_map_lookup_elem(bpf_map__fd(skel->maps.del_sk_stg_map),
&active_fd, &value);
if (!ASSERT_ERR(err, "map_lookup(del_sk_stg_map)"))
goto out;
err = check_sk_stg(listen_fd, BPF_TCP_LISTEN);
if (!ASSERT_OK(err, "listen_fd sk_stg"))
goto out;
err = check_sk_stg(active_fd, BPF_TCP_FIN_WAIT2);
if (!ASSERT_OK(err, "active_fd sk_stg"))
goto out;
err = check_sk_stg(passive_fd, BPF_TCP_LAST_ACK);
ASSERT_OK(err, "passive_fd sk_stg");
out:
if (active_fd != -1)
close(active_fd);
if (passive_fd != -1)
close(passive_fd);
if (listen_fd != -1)
close(listen_fd);
}
void test_sk_storage_tracing(void)
{
struct test_sk_storage_trace_itself *skel_itself;
int err;
my_pid = getpid();
skel_itself = test_sk_storage_trace_itself__open_and_load();
if (!ASSERT_NULL(skel_itself, "test_sk_storage_trace_itself")) {
test_sk_storage_trace_itself__destroy(skel_itself);
return;
}
skel = test_sk_storage_tracing__open_and_load();
if (!ASSERT_OK_PTR(skel, "test_sk_storage_tracing"))
return;
err = test_sk_storage_tracing__attach(skel);
if (!ASSERT_OK(err, "test_sk_storage_tracing__attach")) {
test_sk_storage_tracing__destroy(skel);
return;
}
do_test();
test_sk_storage_tracing__destroy(skel);
}

12
tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c
View File

@ -18,12 +18,12 @@
#define LO_ADDR6 "::1"
#define CG_NAME "/tcpbpf-hdr-opt-test"
struct bpf_test_option exp_passive_estab_in;
struct bpf_test_option exp_active_estab_in;
struct bpf_test_option exp_passive_fin_in;
struct bpf_test_option exp_active_fin_in;
struct hdr_stg exp_passive_hdr_stg;
struct hdr_stg exp_active_hdr_stg = { .active = true, };
static struct bpf_test_option exp_passive_estab_in;
static struct bpf_test_option exp_active_estab_in;
static struct bpf_test_option exp_passive_fin_in;
static struct bpf_test_option exp_active_fin_in;
static struct hdr_stg exp_passive_hdr_stg;
static struct hdr_stg exp_active_hdr_stg = { .active = true, };
static struct test_misc_tcp_hdr_options *misc_skel;
static struct test_tcp_hdr_options *skel;

141
tools/testing/selftests/bpf/prog_tests/tcpbpf_user.c Normal file
View File

@ -0,0 +1,141 @@
// SPDX-License-Identifier: GPL-2.0
#include <test_progs.h>
#include <network_helpers.h>
#include "test_tcpbpf.h"
#include "test_tcpbpf_kern.skel.h"
#define LO_ADDR6 "::1"
#define CG_NAME "/tcpbpf-user-test"
static __u32 duration;
static void verify_result(struct tcpbpf_globals *result)
{
__u32 expected_events = ((1 << BPF_SOCK_OPS_TIMEOUT_INIT) |
(1 << BPF_SOCK_OPS_RWND_INIT) |
(1 << BPF_SOCK_OPS_TCP_CONNECT_CB) |
(1 << BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB) |
(1 << BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB) |
(1 << BPF_SOCK_OPS_NEEDS_ECN) |
(1 << BPF_SOCK_OPS_STATE_CB) |
(1 << BPF_SOCK_OPS_TCP_LISTEN_CB));
/* check global map */
CHECK(expected_events != result->event_map, "event_map",
"unexpected event_map: actual 0x%08x != expected 0x%08x\n",
result->event_map, expected_events);
ASSERT_EQ(result->bytes_received, 501, "bytes_received");
ASSERT_EQ(result->bytes_acked, 1002, "bytes_acked");
ASSERT_EQ(result->data_segs_in, 1, "data_segs_in");
ASSERT_EQ(result->data_segs_out, 1, "data_segs_out");
ASSERT_EQ(result->bad_cb_test_rv, 0x80, "bad_cb_test_rv");
ASSERT_EQ(result->good_cb_test_rv, 0, "good_cb_test_rv");
ASSERT_EQ(result->num_listen, 1, "num_listen");
/* 3 comes from one listening socket + both ends of the connection */
ASSERT_EQ(result->num_close_events, 3, "num_close_events");
/* check setsockopt for SAVE_SYN */
ASSERT_EQ(result->tcp_save_syn, 0, "tcp_save_syn");
/* check getsockopt for SAVED_SYN */
ASSERT_EQ(result->tcp_saved_syn, 1, "tcp_saved_syn");
}
static void run_test(struct tcpbpf_globals *result)
{
int listen_fd = -1, cli_fd = -1, accept_fd = -1;
char buf[1000];
int err = -1;
int i, rv;
listen_fd = start_server(AF_INET6, SOCK_STREAM, LO_ADDR6, 0, 0);
if (CHECK(listen_fd == -1, "start_server", "listen_fd:%d errno:%d\n",
listen_fd, errno))
goto done;
cli_fd = connect_to_fd(listen_fd, 0);
if (CHECK(cli_fd == -1, "connect_to_fd(listen_fd)",
"cli_fd:%d errno:%d\n", cli_fd, errno))
goto done;
accept_fd = accept(listen_fd, NULL, NULL);
if (CHECK(accept_fd == -1, "accept(listen_fd)",
"accept_fd:%d errno:%d\n", accept_fd, errno))
goto done;
/* Send 1000B of '+'s from cli_fd -> accept_fd */
for (i = 0; i < 1000; i++)
buf[i] = '+';
rv = send(cli_fd, buf, 1000, 0);
if (CHECK(rv != 1000, "send(cli_fd)", "rv:%d errno:%d\n", rv, errno))
goto done;
rv = recv(accept_fd, buf, 1000, 0);
if (CHECK(rv != 1000, "recv(accept_fd)", "rv:%d errno:%d\n", rv, errno))
goto done;
/* Send 500B of '.'s from accept_fd ->cli_fd */
for (i = 0; i < 500; i++)
buf[i] = '.';
rv = send(accept_fd, buf, 500, 0);
if (CHECK(rv != 500, "send(accept_fd)", "rv:%d errno:%d\n", rv, errno))
goto done;
rv = recv(cli_fd, buf, 500, 0);
if (CHECK(rv != 500, "recv(cli_fd)", "rv:%d errno:%d\n", rv, errno))
goto done;
/*
* shutdown accept first to guarantee correct ordering for
* bytes_received and bytes_acked when we go to verify the results.
*/
shutdown(accept_fd, SHUT_WR);
err = recv(cli_fd, buf, 1, 0);
if (CHECK(err, "recv(cli_fd) for fin", "err:%d errno:%d\n", err, errno))
goto done;
shutdown(cli_fd, SHUT_WR);
err = recv(accept_fd, buf, 1, 0);
CHECK(err, "recv(accept_fd) for fin", "err:%d errno:%d\n", err, errno);
done:
if (accept_fd != -1)
close(accept_fd);
if (cli_fd != -1)
close(cli_fd);
if (listen_fd != -1)
close(listen_fd);
if (!err)
verify_result(result);
}
void test_tcpbpf_user(void)
{
struct test_tcpbpf_kern *skel;
int cg_fd = -1;
skel = test_tcpbpf_kern__open_and_load();
if (CHECK(!skel, "open and load skel", "failed"))
return;
cg_fd = test__join_cgroup(CG_NAME);
if (CHECK(cg_fd < 0, "test__join_cgroup(" CG_NAME ")",
"cg_fd:%d errno:%d", cg_fd, errno))
goto err;
skel->links.bpf_testcb = bpf_program__attach_cgroup(skel->progs.bpf_testcb, cg_fd);
if (!ASSERT_OK_PTR(skel->links.bpf_testcb, "attach_cgroup(bpf_testcb)"))
goto err;
run_test(&skel->bss->global);
err:
if (cg_fd != -1)
close(cg_fd);
test_tcpbpf_kern__destroy(skel);
}

204
tools/testing/selftests/bpf/prog_tests/test_local_storage.c
View File

@ -4,30 +4,165 @@
* Copyright (C) 2020 Google LLC.
*/
#include <asm-generic/errno-base.h>
#include <sys/stat.h>
#include <test_progs.h>
#include <linux/limits.h>
#include "local_storage.skel.h"
#include "network_helpers.h"
int create_and_unlink_file(void)
#ifndef __NR_pidfd_open
#define __NR_pidfd_open 434
#endif
static inline int sys_pidfd_open(pid_t pid, unsigned int flags)
{
char fname[PATH_MAX] = "/tmp/fileXXXXXX";
int fd;
return syscall(__NR_pidfd_open, pid, flags);
}
fd = mkstemp(fname);
if (fd < 0)
return fd;
static inline ssize_t copy_file_range(int fd_in, loff_t *off_in, int fd_out,
loff_t *off_out, size_t len,
unsigned int flags)
{
return syscall(__NR_copy_file_range, fd_in, off_in, fd_out, off_out,
len, flags);
}
close(fd);
unlink(fname);
return 0;
static unsigned int duration;
#define TEST_STORAGE_VALUE 0xbeefdead
struct storage {
void *inode;
unsigned int value;
/* Lock ensures that spin locked versions of local stoage operations
* also work, most operations in this tests are still single threaded
*/
struct bpf_spin_lock lock;
};
/* Copies an rm binary to a temp file. dest is a mkstemp template */
static int copy_rm(char *dest)
{
int fd_in, fd_out = -1, ret = 0;
struct stat stat;
fd_in = open("/bin/rm", O_RDONLY);
if (fd_in < 0)
return -errno;
fd_out = mkstemp(dest);
if (fd_out < 0) {
ret = -errno;
goto out;
}
ret = fstat(fd_in, &stat);
if (ret == -1) {
ret = -errno;
goto out;
}
ret = copy_file_range(fd_in, NULL, fd_out, NULL, stat.st_size, 0);
if (ret == -1) {
ret = -errno;
goto out;
}
/* Set executable permission on the copied file */
ret = chmod(dest, 0100);
if (ret == -1)
ret = -errno;
out:
close(fd_in);
close(fd_out);
return ret;
}
/* Fork and exec the provided rm binary and return the exit code of the
* forked process and its pid.
*/
static int run_self_unlink(int *monitored_pid, const char *rm_path)
{
int child_pid, child_status, ret;
int null_fd;
child_pid = fork();
if (child_pid == 0) {
null_fd = open("/dev/null", O_WRONLY);
dup2(null_fd, STDOUT_FILENO);
dup2(null_fd, STDERR_FILENO);
close(null_fd);
*monitored_pid = getpid();
/* Use the copied /usr/bin/rm to delete itself
* /tmp/copy_of_rm /tmp/copy_of_rm.
*/
ret = execlp(rm_path, rm_path, rm_path, NULL);
if (ret)
exit(errno);
} else if (child_pid > 0) {
waitpid(child_pid, &child_status, 0);
return WEXITSTATUS(child_status);
}
return -EINVAL;
}
static bool check_syscall_operations(int map_fd, int obj_fd)
{
struct storage val = { .value = TEST_STORAGE_VALUE, .lock = { 0 } },
lookup_val = { .value = 0, .lock = { 0 } };
int err;
/* Looking up an existing element should fail initially */
err = bpf_map_lookup_elem_flags(map_fd, &obj_fd, &lookup_val,
BPF_F_LOCK);
if (CHECK(!err || errno != ENOENT, "bpf_map_lookup_elem",
"err:%d errno:%d\n", err, errno))
return false;
/* Create a new element */
err = bpf_map_update_elem(map_fd, &obj_fd, &val,
BPF_NOEXIST | BPF_F_LOCK);
if (CHECK(err < 0, "bpf_map_update_elem", "err:%d errno:%d\n", err,
errno))
return false;
/* Lookup the newly created element */
err = bpf_map_lookup_elem_flags(map_fd, &obj_fd, &lookup_val,
BPF_F_LOCK);
if (CHECK(err < 0, "bpf_map_lookup_elem", "err:%d errno:%d", err,
errno))
return false;
/* Check the value of the newly created element */
if (CHECK(lookup_val.value != val.value, "bpf_map_lookup_elem",
"value got = %x errno:%d", lookup_val.value, val.value))
return false;
err = bpf_map_delete_elem(map_fd, &obj_fd);
if (CHECK(err, "bpf_map_delete_elem()", "err:%d errno:%d\n", err,
errno))
return false;
/* The lookup should fail, now that the element has been deleted */
err = bpf_map_lookup_elem_flags(map_fd, &obj_fd, &lookup_val,
BPF_F_LOCK);
if (CHECK(!err || errno != ENOENT, "bpf_map_lookup_elem",
"err:%d errno:%d\n", err, errno))
return false;
return true;
}
void test_test_local_storage(void)
{
char tmp_exec_path[PATH_MAX] = "/tmp/copy_of_rmXXXXXX";
int err, serv_sk = -1, task_fd = -1, rm_fd = -1;
struct local_storage *skel = NULL;
int err, duration = 0, serv_sk = -1;
skel = local_storage__open_and_load();
if (CHECK(!skel, "skel_load", "lsm skeleton failed\n"))
@ -37,11 +172,45 @@ void test_test_local_storage(void)
if (CHECK(err, "attach", "lsm attach failed: %d\n", err))
goto close_prog;
task_fd = sys_pidfd_open(getpid(), 0);
if (CHECK(task_fd < 0, "pidfd_open",
"failed to get pidfd err:%d, errno:%d", task_fd, errno))
goto close_prog;
if (!check_syscall_operations(bpf_map__fd(skel->maps.task_storage_map),
task_fd))
goto close_prog;
err = copy_rm(tmp_exec_path);
if (CHECK(err < 0, "copy_rm", "err %d errno %d\n", err, errno))
goto close_prog;
rm_fd = open(tmp_exec_path, O_RDONLY);
if (CHECK(rm_fd < 0, "open", "failed to open %s err:%d, errno:%d",
tmp_exec_path, rm_fd, errno))
goto close_prog;
if (!check_syscall_operations(bpf_map__fd(skel->maps.inode_storage_map),
rm_fd))
goto close_prog;
/* Sets skel->bss->monitored_pid to the pid of the forked child
* forks a child process that executes tmp_exec_path and tries to
* unlink its executable. This operation should be denied by the loaded
* LSM program.
*/
err = run_self_unlink(&skel->bss->monitored_pid, tmp_exec_path);
if (CHECK(err != EPERM, "run_self_unlink", "err %d want EPERM\n", err))
goto close_prog_unlink;
/* Set the process being monitored to be the current process */
skel->bss->monitored_pid = getpid();
err = create_and_unlink_file();
if (CHECK(err < 0, "exec_cmd", "err %d errno %d\n", err, errno))
goto close_prog;
/* Remove the temporary created executable */
err = unlink(tmp_exec_path);
if (CHECK(err != 0, "unlink", "unable to unlink %s: %d", tmp_exec_path,
errno))
goto close_prog_unlink;
CHECK(skel->data->inode_storage_result != 0, "inode_storage_result",
"inode_local_storage not set\n");
@ -53,8 +222,15 @@ void test_test_local_storage(void)
CHECK(skel->data->sk_storage_result != 0, "sk_storage_result",
"sk_local_storage not set\n");
close(serv_sk);
if (!check_syscall_operations(bpf_map__fd(skel->maps.sk_storage_map),
serv_sk))
goto close_prog;
close_prog_unlink:
unlink(tmp_exec_path);
close_prog:
close(serv_sk);
close(rm_fd);
close(task_fd);
local_storage__destroy(skel);
}

41
tools/testing/selftests/bpf/prog_tests/test_skb_pkt_end.c Normal file
View File

@ -0,0 +1,41 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <test_progs.h>
#include <network_helpers.h>
#include "skb_pkt_end.skel.h"
static int sanity_run(struct bpf_program *prog)
{
__u32 duration, retval;
int err, prog_fd;
prog_fd = bpf_program__fd(prog);
err = bpf_prog_test_run(prog_fd, 1, &pkt_v4, sizeof(pkt_v4),
NULL, NULL, &retval, &duration);
if (CHECK(err || retval != 123, "test_run",
"err %d errno %d retval %d duration %d\n",
err, errno, retval, duration))
return -1;
return 0;
}
void test_test_skb_pkt_end(void)
{
struct skb_pkt_end *skb_pkt_end_skel = NULL;
__u32 duration = 0;
int err;
skb_pkt_end_skel = skb_pkt_end__open_and_load();
if (CHECK(!skb_pkt_end_skel, "skb_pkt_end_skel_load", "skb_pkt_end skeleton failed\n"))
goto cleanup;
err = skb_pkt_end__attach(skb_pkt_end_skel);
if (CHECK(err, "skb_pkt_end_attach", "skb_pkt_end attach failed: %d\n", err))
goto cleanup;
if (sanity_run(skb_pkt_end_skel->progs.main_prog))
goto cleanup;
cleanup:
skb_pkt_end__destroy(skb_pkt_end_skel);
}

103
tools/testing/selftests/bpf/progs/local_storage.c
View File

@ -4,9 +4,8 @@
* Copyright 2020 Google LLC.
*/
#include "vmlinux.h"
#include <errno.h>
#include <linux/bpf.h>
#include <stdbool.h>
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
@ -18,60 +17,68 @@ int monitored_pid = 0;
int inode_storage_result = -1;
int sk_storage_result = -1;
struct dummy_storage {
struct local_storage {
struct inode *exec_inode;
__u32 value;
struct bpf_spin_lock lock;
};
struct {
__uint(type, BPF_MAP_TYPE_INODE_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, struct dummy_storage);
__type(value, struct local_storage);
} inode_storage_map SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_SK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC | BPF_F_CLONE);
__type(key, int);
__type(value, struct dummy_storage);
__type(value, struct local_storage);
} sk_storage_map SEC(".maps");
/* TODO Use vmlinux.h once BTF pruning for embedded types is fixed.
*/
struct sock {} __attribute__((preserve_access_index));
struct sockaddr {} __attribute__((preserve_access_index));
struct socket {
struct sock *sk;
} __attribute__((preserve_access_index));
struct inode {} __attribute__((preserve_access_index));
struct dentry {
struct inode *d_inode;
} __attribute__((preserve_access_index));
struct file {
struct inode *f_inode;
} __attribute__((preserve_access_index));
struct {
__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, struct local_storage);
} task_storage_map SEC(".maps");
SEC("lsm/inode_unlink")
int BPF_PROG(unlink_hook, struct inode *dir, struct dentry *victim)
{
__u32 pid = bpf_get_current_pid_tgid() >> 32;
struct dummy_storage *storage;
struct local_storage *storage;
bool is_self_unlink;
int err;
if (pid != monitored_pid)
return 0;
storage = bpf_task_storage_get(&task_storage_map,
bpf_get_current_task_btf(), 0, 0);
if (storage) {
/* Don't let an executable delete itself */
bpf_spin_lock(&storage->lock);
is_self_unlink = storage->exec_inode == victim->d_inode;
bpf_spin_unlock(&storage->lock);
if (is_self_unlink)
return -EPERM;
}
storage = bpf_inode_storage_get(&inode_storage_map, victim->d_inode, 0,
BPF_SK_STORAGE_GET_F_CREATE);
BPF_LOCAL_STORAGE_GET_F_CREATE);
if (!storage)
return 0;
if (storage->value == DUMMY_STORAGE_VALUE)
bpf_spin_lock(&storage->lock);
if (storage->value != DUMMY_STORAGE_VALUE)
inode_storage_result = -1;
bpf_spin_unlock(&storage->lock);
inode_storage_result =
bpf_inode_storage_delete(&inode_storage_map, victim->d_inode);
err = bpf_inode_storage_delete(&inode_storage_map, victim->d_inode);
if (!err)
inode_storage_result = err;
return 0;
}
@ -81,20 +88,26 @@ int BPF_PROG(socket_bind, struct socket *sock, struct sockaddr *address,
int addrlen)
{
__u32 pid = bpf_get_current_pid_tgid() >> 32;
struct dummy_storage *storage;
struct local_storage *storage;
int err;
if (pid != monitored_pid)
return 0;
storage = bpf_sk_storage_get(&sk_storage_map, sock->sk, 0,
BPF_SK_STORAGE_GET_F_CREATE);
BPF_LOCAL_STORAGE_GET_F_CREATE);
if (!storage)
return 0;
if (storage->value == DUMMY_STORAGE_VALUE)
bpf_spin_lock(&storage->lock);
if (storage->value != DUMMY_STORAGE_VALUE)
sk_storage_result = -1;
bpf_spin_unlock(&storage->lock);
err = bpf_sk_storage_delete(&sk_storage_map, sock->sk);
if (!err)
sk_storage_result = err;
sk_storage_result = bpf_sk_storage_delete(&sk_storage_map, sock->sk);
return 0;
}
@ -103,17 +116,19 @@ int BPF_PROG(socket_post_create, struct socket *sock, int family, int type,
int protocol, int kern)
{
__u32 pid = bpf_get_current_pid_tgid() >> 32;
struct dummy_storage *storage;
struct local_storage *storage;
if (pid != monitored_pid)
return 0;
storage = bpf_sk_storage_get(&sk_storage_map, sock->sk, 0,
BPF_SK_STORAGE_GET_F_CREATE);
BPF_LOCAL_STORAGE_GET_F_CREATE);
if (!storage)
return 0;
bpf_spin_lock(&storage->lock);
storage->value = DUMMY_STORAGE_VALUE;
bpf_spin_unlock(&storage->lock);
return 0;
}
@ -122,7 +137,7 @@ SEC("lsm/file_open")
int BPF_PROG(file_open, struct file *file)
{
__u32 pid = bpf_get_current_pid_tgid() >> 32;
struct dummy_storage *storage;
struct local_storage *storage;
if (pid != monitored_pid)
return 0;
@ -131,10 +146,30 @@ int BPF_PROG(file_open, struct file *file)
return 0;
storage = bpf_inode_storage_get(&inode_storage_map, file->f_inode, 0,
BPF_LOCAL_STORAGE_GET_F_CREATE);
BPF_LOCAL_STORAGE_GET_F_CREATE);
if (!storage)
return 0;
bpf_spin_lock(&storage->lock);
storage->value = DUMMY_STORAGE_VALUE;
bpf_spin_unlock(&storage->lock);
return 0;
}
/* This uses the local storage to remember the inode of the binary that a
* process was originally executing.
*/
SEC("lsm/bprm_committed_creds")
void BPF_PROG(exec, struct linux_binprm *bprm)
{
struct local_storage *storage;
storage = bpf_task_storage_get(&task_storage_map,
bpf_get_current_task_btf(), 0,
BPF_LOCAL_STORAGE_GET_F_CREATE);
if (storage) {
bpf_spin_lock(&storage->lock);
storage->exec_inode = bprm->file->f_inode;
bpf_spin_unlock(&storage->lock);
}
}

54
tools/testing/selftests/bpf/progs/skb_pkt_end.c Normal file
View File

@ -0,0 +1,54 @@
// SPDX-License-Identifier: GPL-2.0
#define BPF_NO_PRESERVE_ACCESS_INDEX
#include <vmlinux.h>
#include <bpf/bpf_core_read.h>
#include <bpf/bpf_helpers.h>
#define NULL 0
#define INLINE __always_inline
#define skb_shorter(skb, len) ((void *)(long)(skb)->data + (len) > (void *)(long)skb->data_end)
#define ETH_IPV4_TCP_SIZE (14 + sizeof(struct iphdr) + sizeof(struct tcphdr))
static INLINE struct iphdr *get_iphdr(struct __sk_buff *skb)
{
struct iphdr *ip = NULL;
struct ethhdr *eth;
if (skb_shorter(skb, ETH_IPV4_TCP_SIZE))
goto out;
eth = (void *)(long)skb->data;
ip = (void *)(eth + 1);
out:
return ip;
}
SEC("classifier/cls")
int main_prog(struct __sk_buff *skb)
{
struct iphdr *ip = NULL;
struct tcphdr *tcp;
__u8 proto = 0;
if (!(ip = get_iphdr(skb)))
goto out;
proto = ip->protocol;
if (proto != IPPROTO_TCP)
goto out;
tcp = (void*)(ip + 1);
if (tcp->dest != 0)
goto out;
if (!tcp)
goto out;
return tcp->urg_ptr;
out:
return -1;
}
char _license[] SEC("license") = "GPL";

44
tools/testing/selftests/bpf/progs/test_hash_large_key.c Normal file
View File

@ -0,0 +1,44 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/bpf.h>
#include <bpf/bpf_helpers.h>
char _license[] SEC("license") = "GPL";
struct {
__uint(type, BPF_MAP_TYPE_HASH);
__uint(max_entries, 2);
__type(key, struct bigelement);
__type(value, __u32);
} hash_map SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
__uint(max_entries, 1);
__type(key, __u32);
__type(value, struct bigelement);
} key_map SEC(".maps");
struct bigelement {
int a;
char b[4096];
long long c;
};
SEC("raw_tracepoint/sys_enter")
int bpf_hash_large_key_test(void *ctx)
{
int zero = 0, err = 1, value = 42;
struct bigelement *key;
key = bpf_map_lookup_elem(&key_map, &zero);
if (!key)
return 0;
key->c = 1;
if (bpf_map_update_elem(&hash_map, key, &value, BPF_ANY))
return 0;
return 0;
}

29
tools/testing/selftests/bpf/progs/test_sk_storage_trace_itself.c Normal file
View File

@ -0,0 +1,29 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_helpers.h>
struct {
__uint(type, BPF_MAP_TYPE_SK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, int);
} sk_stg_map SEC(".maps");
SEC("fentry/bpf_sk_storage_free")
int BPF_PROG(trace_bpf_sk_storage_free, struct sock *sk)
{
int *value;
value = bpf_sk_storage_get(&sk_stg_map, sk, 0,
BPF_SK_STORAGE_GET_F_CREATE);
if (value)
*value = 1;
return 0;
}
char _license[] SEC("license") = "GPL";

95
tools/testing/selftests/bpf/progs/test_sk_storage_tracing.c Normal file
View File

@ -0,0 +1,95 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2020 Facebook */
#include <vmlinux.h>
#include <bpf/bpf_tracing.h>
#include <bpf/bpf_core_read.h>
#include <bpf/bpf_helpers.h>
struct sk_stg {
__u32 pid;
__u32 last_notclose_state;
char comm[16];
};
struct {
__uint(type, BPF_MAP_TYPE_SK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, struct sk_stg);
} sk_stg_map SEC(".maps");
/* Testing delete */
struct {
__uint(type, BPF_MAP_TYPE_SK_STORAGE);
__uint(map_flags, BPF_F_NO_PREALLOC);
__type(key, int);
__type(value, int);
} del_sk_stg_map SEC(".maps");
char task_comm[16] = "";
SEC("tp_btf/inet_sock_set_state")
int BPF_PROG(trace_inet_sock_set_state, struct sock *sk, int oldstate,
int newstate)
{
struct sk_stg *stg;
if (newstate == BPF_TCP_CLOSE)
return 0;
stg = bpf_sk_storage_get(&sk_stg_map, sk, 0,
BPF_SK_STORAGE_GET_F_CREATE);
if (!stg)
return 0;
stg->last_notclose_state = newstate;
bpf_sk_storage_delete(&del_sk_stg_map, sk);
return 0;
}
static void set_task_info(struct sock *sk)
{
struct task_struct *task;
struct sk_stg *stg;
stg = bpf_sk_storage_get(&sk_stg_map, sk, 0,
BPF_SK_STORAGE_GET_F_CREATE);
if (!stg)
return;
stg->pid = bpf_get_current_pid_tgid();
task = (struct task_struct *)bpf_get_current_task();
bpf_core_read_str(&stg->comm, sizeof(stg->comm), &task->comm);
bpf_core_read_str(&task_comm, sizeof(task_comm), &task->comm);
}
SEC("fentry/inet_csk_listen_start")
int BPF_PROG(trace_inet_csk_listen_start, struct sock *sk, int backlog)
{
set_task_info(sk);
return 0;
}
SEC("fentry/tcp_connect")
int BPF_PROG(trace_tcp_connect, struct sock *sk)
{
set_task_info(sk);
return 0;
}
SEC("fexit/inet_csk_accept")
int BPF_PROG(inet_csk_accept, struct sock *sk, int flags, int *err, bool kern,
struct sock *accepted_sk)
{
set_task_info(accepted_sk);
return 0;
}
char _license[] SEC("license") = "GPL";

86
tools/testing/selftests/bpf/progs/test_tcpbpf_kern.c
View File

@ -14,40 +14,7 @@
#include <bpf/bpf_endian.h>
#include "test_tcpbpf.h"
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, 4);
__type(key, __u32);
__type(value, struct tcpbpf_globals);
} global_map SEC(".maps");
struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__uint(max_entries, 2);
__type(key, __u32);
__type(value, int);
} sockopt_results SEC(".maps");
static inline void update_event_map(int event)
{
__u32 key = 0;
struct tcpbpf_globals g, *gp;
gp = bpf_map_lookup_elem(&global_map, &key);
if (gp == NULL) {
struct tcpbpf_globals g = {0};
g.event_map |= (1 << event);
bpf_map_update_elem(&global_map, &key, &g,
BPF_ANY);
} else {
g = *gp;
g.event_map |= (1 << event);
bpf_map_update_elem(&global_map, &key, &g,
BPF_ANY);
}
}
struct tcpbpf_globals global = {};
int _version SEC("version") = 1;
SEC("sockops")
@ -105,29 +72,15 @@ int bpf_testcb(struct bpf_sock_ops *skops)
op = (int) skops->op;
update_event_map(op);
global.event_map |= (1 << op);
switch (op) {
case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB:
/* Test failure to set largest cb flag (assumes not defined) */
bad_call_rv = bpf_sock_ops_cb_flags_set(skops, 0x80);
global.bad_cb_test_rv = bpf_sock_ops_cb_flags_set(skops, 0x80);
/* Set callback */
good_call_rv = bpf_sock_ops_cb_flags_set(skops,
global.good_cb_test_rv = bpf_sock_ops_cb_flags_set(skops,
BPF_SOCK_OPS_STATE_CB_FLAG);
/* Update results */
{
__u32 key = 0;
struct tcpbpf_globals g, *gp;
gp = bpf_map_lookup_elem(&global_map, &key);
if (!gp)
break;
g = *gp;
g.bad_cb_test_rv = bad_call_rv;
g.good_cb_test_rv = good_call_rv;
bpf_map_update_elem(&global_map, &key, &g,
BPF_ANY);
}
break;
case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB:
skops->sk_txhash = 0x12345f;
@ -143,10 +96,8 @@ int bpf_testcb(struct bpf_sock_ops *skops)
thdr = (struct tcphdr *)(header + offset);
v = thdr->syn;
__u32 key = 1;
bpf_map_update_elem(&sockopt_results, &key, &v,
BPF_ANY);
global.tcp_saved_syn = v;
}
}
break;
@ -156,25 +107,16 @@ int bpf_testcb(struct bpf_sock_ops *skops)
break;
case BPF_SOCK_OPS_STATE_CB:
if (skops->args[1] == BPF_TCP_CLOSE) {
__u32 key = 0;
struct tcpbpf_globals g, *gp;
gp = bpf_map_lookup_elem(&global_map, &key);
if (!gp)
break;
g = *gp;
if (skops->args[0] == BPF_TCP_LISTEN) {
g.num_listen++;
global.num_listen++;
} else {
g.total_retrans = skops->total_retrans;
g.data_segs_in = skops->data_segs_in;
g.data_segs_out = skops->data_segs_out;
g.bytes_received = skops->bytes_received;
g.bytes_acked = skops->bytes_acked;
global.total_retrans = skops->total_retrans;
global.data_segs_in = skops->data_segs_in;
global.data_segs_out = skops->data_segs_out;
global.bytes_received = skops->bytes_received;
global.bytes_acked = skops->bytes_acked;
}
g.num_close_events++;
bpf_map_update_elem(&global_map, &key, &g,
BPF_ANY);
global.num_close_events++;
}
break;
case BPF_SOCK_OPS_TCP_LISTEN_CB:
@ -182,9 +124,7 @@ int bpf_testcb(struct bpf_sock_ops *skops)
v = bpf_setsockopt(skops, IPPROTO_TCP, TCP_SAVE_SYN,
&save_syn, sizeof(save_syn));
/* Update global map w/ result of setsock opt */
__u32 key = 0;
bpf_map_update_elem(&sockopt_results, &key, &v, BPF_ANY);
global.tcp_save_syn = v;
break;
default:
rv = -1;

42
tools/testing/selftests/bpf/progs/test_tunnel_kern.c
View File

@ -15,7 +15,6 @@
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/types.h>
#include <linux/tcp.h>
#include <linux/socket.h>
#include <linux/pkt_cls.h>
#include <linux/erspan.h>
@ -528,12 +527,11 @@ int _ipip_set_tunnel(struct __sk_buff *skb)
struct bpf_tunnel_key key = {};
void *data = (void *)(long)skb->data;
struct iphdr *iph = data;
struct tcphdr *tcp = data + sizeof(*iph);
void *data_end = (void *)(long)skb->data_end;
int ret;
/* single length check */
if (data + sizeof(*iph) + sizeof(*tcp) > data_end) {
if (data + sizeof(*iph) > data_end) {
ERROR(1);
return TC_ACT_SHOT;
}
@ -541,16 +539,6 @@ int _ipip_set_tunnel(struct __sk_buff *skb)
key.tunnel_ttl = 64;
if (iph->protocol == IPPROTO_ICMP) {
key.remote_ipv4 = 0xac100164; /* 172.16.1.100 */
} else {
if (iph->protocol != IPPROTO_TCP || iph->ihl != 5)
return TC_ACT_SHOT;
if (tcp->dest == bpf_htons(5200))
key.remote_ipv4 = 0xac100164; /* 172.16.1.100 */
else if (tcp->dest == bpf_htons(5201))
key.remote_ipv4 = 0xac100165; /* 172.16.1.101 */
else
return TC_ACT_SHOT;
}
ret = bpf_skb_set_tunnel_key(skb, &key, sizeof(key), 0);
@ -585,19 +573,20 @@ int _ipip6_set_tunnel(struct __sk_buff *skb)
struct bpf_tunnel_key key = {};
void *data = (void *)(long)skb->data;
struct iphdr *iph = data;
struct tcphdr *tcp = data + sizeof(*iph);
void *data_end = (void *)(long)skb->data_end;
int ret;
/* single length check */
if (data + sizeof(*iph) + sizeof(*tcp) > data_end) {
if (data + sizeof(*iph) > data_end) {
ERROR(1);
return TC_ACT_SHOT;
}
__builtin_memset(&key, 0x0, sizeof(key));
key.remote_ipv6[3] = bpf_htonl(0x11); /* ::11 */
key.tunnel_ttl = 64;
if (iph->protocol == IPPROTO_ICMP) {
key.remote_ipv6[3] = bpf_htonl(0x11); /* ::11 */
}
ret = bpf_skb_set_tunnel_key(skb, &key, sizeof(key),
BPF_F_TUNINFO_IPV6);
@ -634,35 +623,18 @@ int _ip6ip6_set_tunnel(struct __sk_buff *skb)
struct bpf_tunnel_key key = {};
void *data = (void *)(long)skb->data;
struct ipv6hdr *iph = data;
struct tcphdr *tcp = data + sizeof(*iph);
void *data_end = (void *)(long)skb->data_end;
int ret;
/* single length check */
if (data + sizeof(*iph) + sizeof(*tcp) > data_end) {
if (data + sizeof(*iph) > data_end) {
ERROR(1);
return TC_ACT_SHOT;
}
key.remote_ipv6[0] = bpf_htonl(0x2401db00);
key.tunnel_ttl = 64;
if (iph->nexthdr == 58 /* NEXTHDR_ICMP */) {
key.remote_ipv6[3] = bpf_htonl(1);
} else {
if (iph->nexthdr != 6 /* NEXTHDR_TCP */) {
ERROR(iph->nexthdr);
return TC_ACT_SHOT;
}
if (tcp->dest == bpf_htons(5200)) {
key.remote_ipv6[3] = bpf_htonl(1);
} else if (tcp->dest == bpf_htons(5201)) {
key.remote_ipv6[3] = bpf_htonl(2);
} else {
ERROR(tcp->dest);
return TC_ACT_SHOT;
}
key.remote_ipv6[3] = bpf_htonl(0x11); /* ::11 */
}
ret = bpf_skb_set_tunnel_key(skb, &key, sizeof(key),

50
tools/testing/selftests/bpf/tcp_client.py
View File

@ -1,50 +0,0 @@
#!/usr/bin/env python3
#
# SPDX-License-Identifier: GPL-2.0
#
import sys, os, os.path, getopt
import socket, time
import subprocess
import select
def read(sock, n):
buf = b''
while len(buf) < n:
rem = n - len(buf)
try: s = sock.recv(rem)
except (socket.error) as e: return b''
buf += s
return buf
def send(sock, s):
total = len(s)
count = 0
while count < total:
try: n = sock.send(s)
except (socket.error) as e: n = 0
if n == 0:
return count;
count += n
return count
serverPort = int(sys.argv[1])
# create active socket
sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
try:
sock.connect(('::1', serverPort))
except socket.error as e:
sys.exit(1)
buf = b''
n = 0
while n < 1000:
buf += b'+'
n += 1
sock.settimeout(1);
n = send(sock, buf)
n = read(sock, 500)
sys.exit(0)

80
tools/testing/selftests/bpf/tcp_server.py
View File

@ -1,80 +0,0 @@
#!/usr/bin/env python3
#
# SPDX-License-Identifier: GPL-2.0
#
import sys, os, os.path, getopt
import socket, time
import subprocess
import select
def read(sock, n):
buf = b''
while len(buf) < n:
rem = n - len(buf)
try: s = sock.recv(rem)
except (socket.error) as e: return b''
buf += s
return buf
def send(sock, s):
total = len(s)
count = 0
while count < total:
try: n = sock.send(s)
except (socket.error) as e: n = 0
if n == 0:
return count;
count += n
return count
SERVER_PORT = 12877
MAX_PORTS = 2
serverPort = SERVER_PORT
serverSocket = None
# create passive socket
serverSocket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
try: serverSocket.bind(('::1', 0))
except socket.error as msg:
print('bind fails: ' + str(msg))
sn = serverSocket.getsockname()
serverPort = sn[1]
cmdStr = ("./tcp_client.py %d &") % (serverPort)
os.system(cmdStr)
buf = b''
n = 0
while n < 500:
buf += b'.'
n += 1
serverSocket.listen(MAX_PORTS)
readList = [serverSocket]
while True:
readyRead, readyWrite, inError = \
select.select(readList, [], [], 2)
if len(readyRead) > 0:
waitCount = 0
for sock in readyRead:
if sock == serverSocket:
(clientSocket, address) = serverSocket.accept()
address = str(address[0])
readList.append(clientSocket)
else:
sock.settimeout(1);
s = read(sock, 1000)
n = send(sock, buf)
sock.close()
serverSocket.close()
sys.exit(0)
else:
print('Select timeout!')
sys.exit(1)

3
tools/testing/selftests/bpf/test_maps.c
View File

@ -1223,9 +1223,10 @@ static void test_map_in_map(void)
static void test_map_large(void)
{
struct bigkey {
int a;
char b[116];
char b[4096];
long long c;
} key;
int fd, i, value;

11
tools/testing/selftests/bpf/test_progs.h
View File

@ -141,6 +141,17 @@ extern int test__join_cgroup(const char *path);
___ok; \
})
#define ASSERT_NEQ(actual, expected, name) ({ \
static int duration = 0; \
typeof(actual) ___act = (actual); \
typeof(expected) ___exp = (expected); \
bool ___ok = ___act != ___exp; \
CHECK(!___ok, (name), \
"unexpected %s: actual %lld == expected %lld\n", \
(name), (long long)(___act), (long long)(___exp)); \
___ok; \
})
#define ASSERT_STREQ(actual, expected, name) ({ \
static int duration = 0; \
const char *___act = actual; \

2
tools/testing/selftests/bpf/test_tcpbpf.h
View File

@ -14,5 +14,7 @@ struct tcpbpf_globals {
__u64 bytes_acked;
__u32 num_listen;
__u32 num_close_events;
__u32 tcp_save_syn;
__u32 tcp_saved_syn;
};
#endif

165
tools/testing/selftests/bpf/test_tcpbpf_user.c
View File

@ -1,165 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
#include <inttypes.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <string.h>
#include <linux/bpf.h>
#include <sys/types.h>
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include "bpf_rlimit.h"
#include "bpf_util.h"
#include "cgroup_helpers.h"
#include "test_tcpbpf.h"
/* 3 comes from one listening socket + both ends of the connection */
#define EXPECTED_CLOSE_EVENTS 3
#define EXPECT_EQ(expected, actual, fmt) \
do { \
if ((expected) != (actual)) { \
printf(" Value of: " #actual "\n" \
" Actual: %" fmt "\n" \
" Expected: %" fmt "\n", \
(actual), (expected)); \
ret--; \
} \
} while (0)
int verify_result(const struct tcpbpf_globals *result)
{
__u32 expected_events;
int ret = 0;
expected_events = ((1 << BPF_SOCK_OPS_TIMEOUT_INIT) |
(1 << BPF_SOCK_OPS_RWND_INIT) |
(1 << BPF_SOCK_OPS_TCP_CONNECT_CB) |
(1 << BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB) |
(1 << BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB) |
(1 << BPF_SOCK_OPS_NEEDS_ECN) |
(1 << BPF_SOCK_OPS_STATE_CB) |
(1 << BPF_SOCK_OPS_TCP_LISTEN_CB));
EXPECT_EQ(expected_events, result->event_map, "#" PRIx32);
EXPECT_EQ(501ULL, result->bytes_received, "llu");
EXPECT_EQ(1002ULL, result->bytes_acked, "llu");
EXPECT_EQ(1, result->data_segs_in, PRIu32);
EXPECT_EQ(1, result->data_segs_out, PRIu32);
EXPECT_EQ(0x80, result->bad_cb_test_rv, PRIu32);
EXPECT_EQ(0, result->good_cb_test_rv, PRIu32);
EXPECT_EQ(1, result->num_listen, PRIu32);
EXPECT_EQ(EXPECTED_CLOSE_EVENTS, result->num_close_events, PRIu32);
return ret;
}
int verify_sockopt_result(int sock_map_fd)
{
__u32 key = 0;
int ret = 0;
int res;
int rv;
/* check setsockopt for SAVE_SYN */
rv = bpf_map_lookup_elem(sock_map_fd, &key, &res);
EXPECT_EQ(0, rv, "d");
EXPECT_EQ(0, res, "d");
key = 1;
/* check getsockopt for SAVED_SYN */
rv = bpf_map_lookup_elem(sock_map_fd, &key, &res);
EXPECT_EQ(0, rv, "d");
EXPECT_EQ(1, res, "d");
return ret;
}
static int bpf_find_map(const char *test, struct bpf_object *obj,
const char *name)
{
struct bpf_map *map;
map = bpf_object__find_map_by_name(obj, name);
if (!map) {
printf("%s:FAIL:map '%s' not found\n", test, name);
return -1;
}
return bpf_map__fd(map);
}
int main(int argc, char **argv)
{
const char *file = "test_tcpbpf_kern.o";
int prog_fd, map_fd, sock_map_fd;
struct tcpbpf_globals g = {0};
const char *cg_path = "/foo";
int error = EXIT_FAILURE;
struct bpf_object *obj;
int cg_fd = -1;
int retry = 10;
__u32 key = 0;
int rv;
cg_fd = cgroup_setup_and_join(cg_path);
if (cg_fd < 0)
goto err;
if (bpf_prog_load(file, BPF_PROG_TYPE_SOCK_OPS, &obj, &prog_fd)) {
printf("FAILED: load_bpf_file failed for: %s\n", file);
goto err;
}
rv = bpf_prog_attach(prog_fd, cg_fd, BPF_CGROUP_SOCK_OPS, 0);
if (rv) {
printf("FAILED: bpf_prog_attach: %d (%s)\n",
error, strerror(errno));
goto err;
}
if (system("./tcp_server.py")) {
printf("FAILED: TCP server\n");
goto err;
}
map_fd = bpf_find_map(__func__, obj, "global_map");
if (map_fd < 0)
goto err;
sock_map_fd = bpf_find_map(__func__, obj, "sockopt_results");
if (sock_map_fd < 0)
goto err;
retry_lookup:
rv = bpf_map_lookup_elem(map_fd, &key, &g);
if (rv != 0) {
printf("FAILED: bpf_map_lookup_elem returns %d\n", rv);
goto err;
}
if (g.num_close_events != EXPECTED_CLOSE_EVENTS && retry--) {
printf("Unexpected number of close events (%d), retrying!\n",
g.num_close_events);
usleep(100);
goto retry_lookup;
}
if (verify_result(&g)) {
printf("FAILED: Wrong stats\n");
goto err;
}
if (verify_sockopt_result(sock_map_fd)) {
printf("FAILED: Wrong sockopt stats\n");
goto err;
}
printf("PASSED!\n");
error = 0;
err:
bpf_prog_detach(cg_fd, BPF_CGROUP_SOCK_OPS);
close(cg_fd);
cleanup_cgroup_environment();
return error;
}

43
tools/testing/selftests/bpf/test_tunnel.sh
View File

@ -24,12 +24,12 @@
# Root namespace with metadata-mode tunnel + BPF
# Device names and addresses:
# veth1 IP: 172.16.1.200, IPv6: 00::22 (underlay)
# tunnel dev <type>11, ex: gre11, IPv4: 10.1.1.200 (overlay)
# tunnel dev <type>11, ex: gre11, IPv4: 10.1.1.200, IPv6: 1::22 (overlay)
#
# Namespace at_ns0 with native tunnel
# Device names and addresses:
# veth0 IPv4: 172.16.1.100, IPv6: 00::11 (underlay)
# tunnel dev <type>00, ex: gre00, IPv4: 10.1.1.100 (overlay)
# tunnel dev <type>00, ex: gre00, IPv4: 10.1.1.100, IPv6: 1::11 (overlay)
#
#
# End-to-end ping packet flow
@ -250,7 +250,7 @@ add_ipip_tunnel()
ip addr add dev $DEV 10.1.1.200/24
}
add_ipip6tnl_tunnel()
add_ip6tnl_tunnel()
{
ip netns exec at_ns0 ip addr add ::11/96 dev veth0
ip netns exec at_ns0 ip link set dev veth0 up
@ -262,11 +262,13 @@ add_ipip6tnl_tunnel()
ip link add dev $DEV_NS type $TYPE \
local ::11 remote ::22
ip netns exec at_ns0 ip addr add dev $DEV_NS 10.1.1.100/24
ip netns exec at_ns0 ip addr add dev $DEV_NS 1::11/96
ip netns exec at_ns0 ip link set dev $DEV_NS up
# root namespace
ip link add dev $DEV type $TYPE external
ip addr add dev $DEV 10.1.1.200/24
ip addr add dev $DEV 1::22/96
ip link set dev $DEV up
}
@ -534,7 +536,7 @@ test_ipip6()
check $TYPE
config_device
add_ipip6tnl_tunnel
add_ip6tnl_tunnel
ip link set dev veth1 mtu 1500
attach_bpf $DEV ipip6_set_tunnel ipip6_get_tunnel
# underlay
@ -553,6 +555,34 @@ test_ipip6()
echo -e ${GREEN}"PASS: $TYPE"${NC}
}
test_ip6ip6()
{
TYPE=ip6tnl
DEV_NS=ip6ip6tnl00
DEV=ip6ip6tnl11
ret=0
check $TYPE
config_device
add_ip6tnl_tunnel
ip link set dev veth1 mtu 1500
attach_bpf $DEV ip6ip6_set_tunnel ip6ip6_get_tunnel
# underlay
ping6 $PING_ARG ::11
# ip6 over ip6
ping6 $PING_ARG 1::11
check_err $?
ip netns exec at_ns0 ping6 $PING_ARG 1::22
check_err $?
cleanup
if [ $ret -ne 0 ]; then
echo -e ${RED}"FAIL: ip6$TYPE"${NC}
return 1
fi
echo -e ${GREEN}"PASS: ip6$TYPE"${NC}
}
setup_xfrm_tunnel()
{
auth=0x$(printf '1%.0s' {1..40})
@ -646,6 +676,7 @@ cleanup()
ip link del veth1 2> /dev/null
ip link del ipip11 2> /dev/null
ip link del ipip6tnl11 2> /dev/null
ip link del ip6ip6tnl11 2> /dev/null
ip link del gretap11 2> /dev/null
ip link del ip6gre11 2> /dev/null
ip link del ip6gretap11 2> /dev/null
@ -742,6 +773,10 @@ bpf_tunnel_test()
test_ipip6
errors=$(( $errors + $? ))
echo "Testing IP6IP6 tunnel..."
test_ip6ip6
errors=$(( $errors + $? ))
echo "Testing IPSec tunnel..."
test_xfrm_tunnel
errors=$(( $errors + $? ))

42
tools/testing/selftests/bpf/verifier/ctx_skb.c
View File

@ -1089,3 +1089,45 @@
.errstr_unpriv = "R1 leaks addr",
.result = REJECT,
},
{
"pkt > pkt_end taken check",
.insns = {
BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, // 0. r2 = *(u32 *)(r1 + data_end)
offsetof(struct __sk_buff, data_end)),
BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_1, // 1. r4 = *(u32 *)(r1 + data)
offsetof(struct __sk_buff, data)),
BPF_MOV64_REG(BPF_REG_3, BPF_REG_4), // 2. r3 = r4
BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, 42), // 3. r3 += 42
BPF_MOV64_IMM(BPF_REG_1, 0), // 4. r1 = 0
BPF_JMP_REG(BPF_JGT, BPF_REG_3, BPF_REG_2, 2), // 5. if r3 > r2 goto 8
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 14), // 6. r4 += 14
BPF_MOV64_REG(BPF_REG_1, BPF_REG_4), // 7. r1 = r4
BPF_JMP_REG(BPF_JGT, BPF_REG_3, BPF_REG_2, 1), // 8. if r3 > r2 goto 10
BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1, 9), // 9. r2 = *(u8 *)(r1 + 9)
BPF_MOV64_IMM(BPF_REG_0, 0), // 10. r0 = 0
BPF_EXIT_INSN(), // 11. exit
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_SK_SKB,
},
{
"pkt_end < pkt taken check",
.insns = {
BPF_LDX_MEM(BPF_W, BPF_REG_2, BPF_REG_1, // 0. r2 = *(u32 *)(r1 + data_end)
offsetof(struct __sk_buff, data_end)),
BPF_LDX_MEM(BPF_W, BPF_REG_4, BPF_REG_1, // 1. r4 = *(u32 *)(r1 + data)
offsetof(struct __sk_buff, data)),
BPF_MOV64_REG(BPF_REG_3, BPF_REG_4), // 2. r3 = r4
BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, 42), // 3. r3 += 42
BPF_MOV64_IMM(BPF_REG_1, 0), // 4. r1 = 0
BPF_JMP_REG(BPF_JGT, BPF_REG_3, BPF_REG_2, 2), // 5. if r3 > r2 goto 8
BPF_ALU64_IMM(BPF_ADD, BPF_REG_4, 14), // 6. r4 += 14
BPF_MOV64_REG(BPF_REG_1, BPF_REG_4), // 7. r1 = r4
BPF_JMP_REG(BPF_JLT, BPF_REG_2, BPF_REG_3, 1), // 8. if r2 < r3 goto 10
BPF_LDX_MEM(BPF_H, BPF_REG_2, BPF_REG_1, 9), // 9. r2 = *(u8 *)(r1 + 9)
BPF_MOV64_IMM(BPF_REG_0, 0), // 10. r0 = 0
BPF_EXIT_INSN(), // 11. exit
},
.result = ACCEPT,
.prog_type = BPF_PROG_TYPE_SK_SKB,
},