p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'sk-local-storage' 

Martin KaFai Lau says:

====================
v4:
- Move checks to map_alloc_check in patch 1 (Stanislav Fomichev)
- Refactor BTF encoding macros to test_btf.h at
  a new patch 4 (Stanislav Fomichev)
- Refactor getenv and add print PASS message at the
  end of the test in patch 6 (Yonghong Song)

v3:
- Replace spinlock_types.h with spinlock.h in patch 1
  (kbuild test robot <lkp@intel.com>)

v2:
- Add the "test_maps.h" file in patch 5

This series introduces the BPF sk local storage.  The
details is in the patch 1 commit message.
====================

Acked-by: Yonghong Song <yhs@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Alexei Starovoitov 2019-04-27 09:07:49 -07:00
parent 3745dc24aa 263d0b3533
commit 9076c49bdc
26 changed files with 2089 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/linux/bpf.h
View File

@ -184,6 +184,7 @@ enum bpf_arg_type {
ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */
ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */
ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */ ARG_PTR_TO_UNINIT_MAP_VALUE, /* pointer to valid memory used to store a map value */
ARG_PTR_TO_MAP_VALUE_OR_NULL, /* pointer to stack used as map value or NULL */
/* the following constraints used to prototype bpf_memcmp() and other /* the following constraints used to prototype bpf_memcmp() and other
* functions that access data on eBPF program stack * functions that access data on eBPF program stack
@ -204,6 +205,7 @@ enum bpf_arg_type {
ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */
ARG_PTR_TO_INT, /* pointer to int */ ARG_PTR_TO_INT, /* pointer to int */
ARG_PTR_TO_LONG, /* pointer to long */ ARG_PTR_TO_LONG, /* pointer to long */
ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */
}; };
/* type of values returned from helper functions */ /* type of values returned from helper functions */

1
include/linux/bpf_types.h
View File

@ -61,6 +61,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_ARRAY_OF_MAPS, array_of_maps_map_ops)
BPF_MAP_TYPE(BPF_MAP_TYPE_HASH_OF_MAPS, htab_of_maps_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_HASH_OF_MAPS, htab_of_maps_map_ops)
#ifdef CONFIG_NET #ifdef CONFIG_NET
BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_DEVMAP, dev_map_ops)
BPF_MAP_TYPE(BPF_MAP_TYPE_SK_STORAGE, sk_storage_map_ops)
#if defined(CONFIG_BPF_STREAM_PARSER) #if defined(CONFIG_BPF_STREAM_PARSER)
BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKMAP, sock_map_ops)
BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKHASH, sock_hash_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_SOCKHASH, sock_hash_ops)

13
include/net/bpf_sk_storage.h Normal file
View File

@ -0,0 +1,13 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2019 Facebook */
#ifndef _BPF_SK_STORAGE_H
#define _BPF_SK_STORAGE_H
struct sock;
void bpf_sk_storage_free(struct sock *sk);
extern const struct bpf_func_proto bpf_sk_storage_get_proto;
extern const struct bpf_func_proto bpf_sk_storage_delete_proto;
#endif /* _BPF_SK_STORAGE_H */

5
include/net/sock.h
View File

@ -236,6 +236,8 @@ struct sock_common {
/* public: */ /* public: */
}; };
struct bpf_sk_storage;
/** /**
* struct sock - network layer representation of sockets * struct sock - network layer representation of sockets
* @__sk_common: shared layout with inet_timewait_sock * @__sk_common: shared layout with inet_timewait_sock
@ -510,6 +512,9 @@ struct sock {
#endif #endif
void (*sk_destruct)(struct sock *sk); void (*sk_destruct)(struct sock *sk);
struct sock_reuseport __rcu *sk_reuseport_cb; struct sock_reuseport __rcu *sk_reuseport_cb;
#ifdef CONFIG_BPF_SYSCALL
struct bpf_sk_storage __rcu *sk_bpf_storage;
#endif
struct rcu_head sk_rcu; struct rcu_head sk_rcu;
}; };

44
include/uapi/linux/bpf.h
View File

@ -133,6 +133,7 @@ enum bpf_map_type {
BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE,
BPF_MAP_TYPE_QUEUE, BPF_MAP_TYPE_QUEUE,
BPF_MAP_TYPE_STACK, BPF_MAP_TYPE_STACK,
BPF_MAP_TYPE_SK_STORAGE,
}; };
/* Note that tracing related programs such as /* Note that tracing related programs such as
@ -2630,6 +2631,42 @@ union bpf_attr {
* was provided. * was provided.
* *
* **-ERANGE** if resulting value was out of range. * **-ERANGE** if resulting value was out of range.
*
* void *bpf_sk_storage_get(struct bpf_map *map, struct bpf_sock *sk, void *value, u64 flags)
* Description
* Get a bpf-local-storage from a sk.
*
* Logically, it could be thought of getting the value from
* a *map* with *sk* as the **key**. From this
* perspective, the usage is not much different from
* **bpf_map_lookup_elem(map, &sk)** except this
* helper enforces the key must be a **bpf_fullsock()**
* and the map must be a BPF_MAP_TYPE_SK_STORAGE also.
*
* Underneath, the value is stored locally at *sk* instead of
* the map. The *map* is used as the bpf-local-storage **type**.
* The bpf-local-storage **type** (i.e. the *map*) is searched
* against all bpf-local-storages residing at sk.
*
* An optional *flags* (BPF_SK_STORAGE_GET_F_CREATE) can be
* used such that a new bpf-local-storage will be
* created if one does not exist. *value* can be used
* together with BPF_SK_STORAGE_GET_F_CREATE to specify
* the initial value of a bpf-local-storage. If *value* is
* NULL, the new bpf-local-storage will be zero initialized.
* Return
* A bpf-local-storage pointer is returned on success.
*
* **NULL** if not found or there was an error in adding
* a new bpf-local-storage.
*
* int bpf_sk_storage_delete(struct bpf_map *map, struct bpf_sock *sk)
* Description
* Delete a bpf-local-storage from a sk.
* Return
* 0 on success.
*
* **-ENOENT** if the bpf-local-storage cannot be found.
*/ */
#define __BPF_FUNC_MAPPER(FN) \ #define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \ FN(unspec), \
@ -2738,7 +2775,9 @@ union bpf_attr {
FN(sysctl_get_new_value), \ FN(sysctl_get_new_value), \
FN(sysctl_set_new_value), \ FN(sysctl_set_new_value), \
FN(strtol), \ FN(strtol), \
FN(strtoul), FN(strtoul), \
FN(sk_storage_get), \
FN(sk_storage_delete),
/* integer value in 'imm' field of BPF_CALL instruction selects which helper /* integer value in 'imm' field of BPF_CALL instruction selects which helper
* function eBPF program intends to call * function eBPF program intends to call
@ -2814,6 +2853,9 @@ enum bpf_func_id {
/* BPF_FUNC_sysctl_get_name flags. */ /* BPF_FUNC_sysctl_get_name flags. */
#define BPF_F_SYSCTL_BASE_NAME (1ULL << 0) #define BPF_F_SYSCTL_BASE_NAME (1ULL << 0)
/* BPF_FUNC_sk_storage_get flags */
#define BPF_SK_STORAGE_GET_F_CREATE (1ULL << 0)
/* Mode for BPF_FUNC_skb_adjust_room helper. */ /* Mode for BPF_FUNC_skb_adjust_room helper. */
enum bpf_adj_room_mode { enum bpf_adj_room_mode {
BPF_ADJ_ROOM_NET, BPF_ADJ_ROOM_NET,

3
kernel/bpf/syscall.c
View File

@ -526,7 +526,8 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf,
return -EACCES; return -EACCES;
if (map->map_type != BPF_MAP_TYPE_HASH && if (map->map_type != BPF_MAP_TYPE_HASH &&
map->map_type != BPF_MAP_TYPE_ARRAY && map->map_type != BPF_MAP_TYPE_ARRAY &&
map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE) map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
map->map_type != BPF_MAP_TYPE_SK_STORAGE)
return -ENOTSUPP; return -ENOTSUPP;
if (map->spin_lock_off + sizeof(struct bpf_spin_lock) > if (map->spin_lock_off + sizeof(struct bpf_spin_lock) >
map->value_size) { map->value_size) {

25
kernel/bpf/verifier.c
View File

@ -2543,9 +2543,14 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
if (arg_type == ARG_PTR_TO_MAP_KEY || if (arg_type == ARG_PTR_TO_MAP_KEY ||
arg_type == ARG_PTR_TO_MAP_VALUE || arg_type == ARG_PTR_TO_MAP_VALUE ||
arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE ||
arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) {
expected_type = PTR_TO_STACK; expected_type = PTR_TO_STACK;
if (!type_is_pkt_pointer(type) && type != PTR_TO_MAP_VALUE && if (register_is_null(reg) &&
arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL)
/* final test in check_stack_boundary() */;
else if (!type_is_pkt_pointer(type) &&
type != PTR_TO_MAP_VALUE &&
type != expected_type) type != expected_type)
goto err_type; goto err_type;
} else if (arg_type == ARG_CONST_SIZE || } else if (arg_type == ARG_CONST_SIZE ||
@ -2578,6 +2583,10 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
} }
meta->ref_obj_id = reg->ref_obj_id; meta->ref_obj_id = reg->ref_obj_id;
} }
} else if (arg_type == ARG_PTR_TO_SOCKET) {
expected_type = PTR_TO_SOCKET;
if (type != expected_type)
goto err_type;
} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
if (meta->func_id == BPF_FUNC_spin_lock) { if (meta->func_id == BPF_FUNC_spin_lock) {
if (process_spin_lock(env, regno, true)) if (process_spin_lock(env, regno, true))
@ -2635,6 +2644,8 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
meta->map_ptr->key_size, false, meta->map_ptr->key_size, false,
NULL); NULL);
} else if (arg_type == ARG_PTR_TO_MAP_VALUE || } else if (arg_type == ARG_PTR_TO_MAP_VALUE ||
(arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL &&
!register_is_null(reg)) ||
arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
/* bpf_map_xxx(..., map_ptr, ..., value) call: /* bpf_map_xxx(..., map_ptr, ..., value) call:
* check [value, value + map->value_size) validity * check [value, value + map->value_size) validity
@ -2784,6 +2795,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
func_id != BPF_FUNC_map_push_elem) func_id != BPF_FUNC_map_push_elem)
goto error; goto error;
break; break;
case BPF_MAP_TYPE_SK_STORAGE:
if (func_id != BPF_FUNC_sk_storage_get &&
func_id != BPF_FUNC_sk_storage_delete)
goto error;
break;
default: default:
break; break;
} }
@ -2847,6 +2863,11 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
map->map_type != BPF_MAP_TYPE_STACK) map->map_type != BPF_MAP_TYPE_STACK)
goto error; goto error;
break; break;
case BPF_FUNC_sk_storage_get:
case BPF_FUNC_sk_storage_delete:
if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
goto error;
break;
default: default:
break; break;
} }

2
net/bpf/test_run.c
View File

@ -10,6 +10,7 @@
#include <linux/etherdevice.h> #include <linux/etherdevice.h>
#include <linux/filter.h> #include <linux/filter.h>
#include <linux/sched/signal.h> #include <linux/sched/signal.h>
#include <net/bpf_sk_storage.h>
#include <net/sock.h> #include <net/sock.h>
#include <net/tcp.h> #include <net/tcp.h>
@ -335,6 +336,7 @@ int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr,
sizeof(struct __sk_buff)); sizeof(struct __sk_buff));
out: out:
kfree_skb(skb); kfree_skb(skb);
bpf_sk_storage_free(sk);
kfree(sk); kfree(sk);
kfree(ctx); kfree(ctx);
return ret; return ret;

1
net/core/Makefile
View File

@ -34,3 +34,4 @@ obj-$(CONFIG_HWBM) += hwbm.o
obj-$(CONFIG_NET_DEVLINK) += devlink.o obj-$(CONFIG_NET_DEVLINK) += devlink.o
obj-$(CONFIG_GRO_CELLS) += gro_cells.o obj-$(CONFIG_GRO_CELLS) += gro_cells.o
obj-$(CONFIG_FAILOVER) += failover.o obj-$(CONFIG_FAILOVER) += failover.o
obj-$(CONFIG_BPF_SYSCALL) += bpf_sk_storage.o

804
net/core/bpf_sk_storage.c Normal file
View File

@ -0,0 +1,804 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 Facebook */
#include <linux/rculist.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/types.h>
#include <linux/spinlock.h>
#include <linux/bpf.h>
#include <net/bpf_sk_storage.h>
#include <net/sock.h>
#include <uapi/linux/btf.h>
static atomic_t cache_idx;
struct bucket {
struct hlist_head list;
raw_spinlock_t lock;
};
/* Thp map is not the primary owner of a bpf_sk_storage_elem.
* Instead, the sk->sk_bpf_storage is.
*
* The map (bpf_sk_storage_map) is for two purposes
* 1. Define the size of the "sk local storage". It is
* the map's value_size.
*
* 2. Maintain a list to keep track of all elems such
* that they can be cleaned up during the map destruction.
*
* When a bpf local storage is being looked up for a
* particular sk, the "bpf_map" pointer is actually used
* as the "key" to search in the list of elem in
* sk->sk_bpf_storage.
*
* Hence, consider sk->sk_bpf_storage is the mini-map
* with the "bpf_map" pointer as the searching key.
*/
struct bpf_sk_storage_map {
struct bpf_map map;
/* Lookup elem does not require accessing the map.
*
* Updating/Deleting requires a bucket lock to
* link/unlink the elem from the map. Having
* multiple buckets to improve contention.
*/
struct bucket *buckets;
u32 bucket_log;
u16 elem_size;
u16 cache_idx;
};
struct bpf_sk_storage_data {
/* smap is used as the searching key when looking up
* from sk->sk_bpf_storage.
*
* Put it in the same cacheline as the data to minimize
* the number of cachelines access during the cache hit case.
*/
struct bpf_sk_storage_map __rcu *smap;
u8 data[0] __aligned(8);
};
/* Linked to bpf_sk_storage and bpf_sk_storage_map */
struct bpf_sk_storage_elem {
struct hlist_node map_node; /* Linked to bpf_sk_storage_map */
struct hlist_node snode; /* Linked to bpf_sk_storage */
struct bpf_sk_storage __rcu *sk_storage;
struct rcu_head rcu;
/* 8 bytes hole */
/* The data is stored in aother cacheline to minimize
* the number of cachelines access during a cache hit.
*/
struct bpf_sk_storage_data sdata ____cacheline_aligned;
};
#define SELEM(_SDATA) container_of((_SDATA), struct bpf_sk_storage_elem, sdata)
#define SDATA(_SELEM) (&(_SELEM)->sdata)
#define BPF_SK_STORAGE_CACHE_SIZE 16
struct bpf_sk_storage {
struct bpf_sk_storage_data __rcu *cache[BPF_SK_STORAGE_CACHE_SIZE];
struct hlist_head list; /* List of bpf_sk_storage_elem */
struct sock *sk; /* The sk that owns the the above "list" of
* bpf_sk_storage_elem.
*/
struct rcu_head rcu;
raw_spinlock_t lock; /* Protect adding/removing from the "list" */
};
static struct bucket *select_bucket(struct bpf_sk_storage_map *smap,
struct bpf_sk_storage_elem *selem)
{
return &smap->buckets[hash_ptr(selem, smap->bucket_log)];
}
static int omem_charge(struct sock *sk, unsigned int size)
{
/* same check as in sock_kmalloc() */
if (size <= sysctl_optmem_max &&
atomic_read(&sk->sk_omem_alloc) + size < sysctl_optmem_max) {
atomic_add(size, &sk->sk_omem_alloc);
return 0;
}
return -ENOMEM;
}
static bool selem_linked_to_sk(const struct bpf_sk_storage_elem *selem)
{
return !hlist_unhashed(&selem->snode);
}
static bool selem_linked_to_map(const struct bpf_sk_storage_elem *selem)
{
return !hlist_unhashed(&selem->map_node);
}
static struct bpf_sk_storage_elem *selem_alloc(struct bpf_sk_storage_map *smap,
struct sock *sk, void *value,
bool charge_omem)
{
struct bpf_sk_storage_elem *selem;
if (charge_omem && omem_charge(sk, smap->elem_size))
return NULL;
selem = kzalloc(smap->elem_size, GFP_ATOMIC | __GFP_NOWARN);
if (selem) {
if (value)
memcpy(SDATA(selem)->data, value, smap->map.value_size);
return selem;
}
if (charge_omem)
atomic_sub(smap->elem_size, &sk->sk_omem_alloc);
return NULL;
}
/* sk_storage->lock must be held and selem->sk_storage == sk_storage.
* The caller must ensure selem->smap is still valid to be
* dereferenced for its smap->elem_size and smap->cache_idx.
*/
static bool __selem_unlink_sk(struct bpf_sk_storage *sk_storage,
struct bpf_sk_storage_elem *selem,
bool uncharge_omem)
{
struct bpf_sk_storage_map *smap;
bool free_sk_storage;
struct sock *sk;
smap = rcu_dereference(SDATA(selem)->smap);
sk = sk_storage->sk;
/* All uncharging on sk->sk_omem_alloc must be done first.
* sk may be freed once the last selem is unlinked from sk_storage.
*/
if (uncharge_omem)
atomic_sub(smap->elem_size, &sk->sk_omem_alloc);
free_sk_storage = hlist_is_singular_node(&selem->snode,
&sk_storage->list);
if (free_sk_storage) {
atomic_sub(sizeof(struct bpf_sk_storage), &sk->sk_omem_alloc);
sk_storage->sk = NULL;
/* After this RCU_INIT, sk may be freed and cannot be used */
RCU_INIT_POINTER(sk->sk_bpf_storage, NULL);
/* sk_storage is not freed now. sk_storage->lock is
* still held and raw_spin_unlock_bh(&sk_storage->lock)
* will be done by the caller.
*
* Although the unlock will be done under
* rcu_read_lock(), it is more intutivie to
* read if kfree_rcu(sk_storage, rcu) is done
* after the raw_spin_unlock_bh(&sk_storage->lock).
*
* Hence, a "bool free_sk_storage" is returned
* to the caller which then calls the kfree_rcu()
* after unlock.
*/
}
hlist_del_init_rcu(&selem->snode);
if (rcu_access_pointer(sk_storage->cache[smap->cache_idx]) ==
SDATA(selem))
RCU_INIT_POINTER(sk_storage->cache[smap->cache_idx], NULL);
kfree_rcu(selem, rcu);
return free_sk_storage;
}
static void selem_unlink_sk(struct bpf_sk_storage_elem *selem)
{
struct bpf_sk_storage *sk_storage;
bool free_sk_storage = false;
if (unlikely(!selem_linked_to_sk(selem)))
/* selem has already been unlinked from sk */
return;
sk_storage = rcu_dereference(selem->sk_storage);
raw_spin_lock_bh(&sk_storage->lock);
if (likely(selem_linked_to_sk(selem)))
free_sk_storage = __selem_unlink_sk(sk_storage, selem, true);
raw_spin_unlock_bh(&sk_storage->lock);
if (free_sk_storage)
kfree_rcu(sk_storage, rcu);
}
/* sk_storage->lock must be held and sk_storage->list cannot be empty */
static void __selem_link_sk(struct bpf_sk_storage *sk_storage,
struct bpf_sk_storage_elem *selem)
{
RCU_INIT_POINTER(selem->sk_storage, sk_storage);
hlist_add_head(&selem->snode, &sk_storage->list);
}
static void selem_unlink_map(struct bpf_sk_storage_elem *selem)
{
struct bpf_sk_storage_map *smap;
struct bucket *b;
if (unlikely(!selem_linked_to_map(selem)))
/* selem has already be unlinked from smap */
return;
smap = rcu_dereference(SDATA(selem)->smap);
b = select_bucket(smap, selem);
raw_spin_lock_bh(&b->lock);
if (likely(selem_linked_to_map(selem)))
hlist_del_init_rcu(&selem->map_node);
raw_spin_unlock_bh(&b->lock);
}
static void selem_link_map(struct bpf_sk_storage_map *smap,
struct bpf_sk_storage_elem *selem)
{
struct bucket *b = select_bucket(smap, selem);
raw_spin_lock_bh(&b->lock);
RCU_INIT_POINTER(SDATA(selem)->smap, smap);
hlist_add_head_rcu(&selem->map_node, &b->list);
raw_spin_unlock_bh(&b->lock);
}
static void selem_unlink(struct bpf_sk_storage_elem *selem)
{
/* Always unlink from map before unlinking from sk_storage
* because selem will be freed after successfully unlinked from
* the sk_storage.
*/
selem_unlink_map(selem);
selem_unlink_sk(selem);
}
static struct bpf_sk_storage_data *
__sk_storage_lookup(struct bpf_sk_storage *sk_storage,
struct bpf_sk_storage_map *smap,
bool cacheit_lockit)
{
struct bpf_sk_storage_data *sdata;
struct bpf_sk_storage_elem *selem;
/* Fast path (cache hit) */
sdata = rcu_dereference(sk_storage->cache[smap->cache_idx]);
if (sdata && rcu_access_pointer(sdata->smap) == smap)
return sdata;
/* Slow path (cache miss) */
hlist_for_each_entry_rcu(selem, &sk_storage->list, snode)
if (rcu_access_pointer(SDATA(selem)->smap) == smap)
break;
if (!selem)
return NULL;
sdata = SDATA(selem);
if (cacheit_lockit) {
/* spinlock is needed to avoid racing with the
* parallel delete. Otherwise, publishing an already
* deleted sdata to the cache will become a use-after-free
* problem in the next __sk_storage_lookup().
*/
raw_spin_lock_bh(&sk_storage->lock);
if (selem_linked_to_sk(selem))
rcu_assign_pointer(sk_storage->cache[smap->cache_idx],
sdata);
raw_spin_unlock_bh(&sk_storage->lock);
}
return sdata;
}
static struct bpf_sk_storage_data *
sk_storage_lookup(struct sock *sk, struct bpf_map *map, bool cacheit_lockit)
{
struct bpf_sk_storage *sk_storage;
struct bpf_sk_storage_map *smap;
sk_storage = rcu_dereference(sk->sk_bpf_storage);
if (!sk_storage)
return NULL;
smap = (struct bpf_sk_storage_map *)map;
return __sk_storage_lookup(sk_storage, smap, cacheit_lockit);
}
static int check_flags(const struct bpf_sk_storage_data *old_sdata,
u64 map_flags)
{
if (old_sdata && (map_flags & ~BPF_F_LOCK) == BPF_NOEXIST)
/* elem already exists */
return -EEXIST;
if (!old_sdata && (map_flags & ~BPF_F_LOCK) == BPF_EXIST)
/* elem doesn't exist, cannot update it */
return -ENOENT;
return 0;
}
static int sk_storage_alloc(struct sock *sk,
struct bpf_sk_storage_map *smap,
struct bpf_sk_storage_elem *first_selem)
{
struct bpf_sk_storage *prev_sk_storage, *sk_storage;
int err;
err = omem_charge(sk, sizeof(*sk_storage));
if (err)
return err;
sk_storage = kzalloc(sizeof(*sk_storage), GFP_ATOMIC | __GFP_NOWARN);
if (!sk_storage) {
err = -ENOMEM;
goto uncharge;
}
INIT_HLIST_HEAD(&sk_storage->list);
raw_spin_lock_init(&sk_storage->lock);
sk_storage->sk = sk;
__selem_link_sk(sk_storage, first_selem);
selem_link_map(smap, first_selem);
/* Publish sk_storage to sk. sk->sk_lock cannot be acquired.
* Hence, atomic ops is used to set sk->sk_bpf_storage
* from NULL to the newly allocated sk_storage ptr.
*
* From now on, the sk->sk_bpf_storage pointer is protected
* by the sk_storage->lock. Hence, when freeing
* the sk->sk_bpf_storage, the sk_storage->lock must
* be held before setting sk->sk_bpf_storage to NULL.
*/
prev_sk_storage = cmpxchg((struct bpf_sk_storage **)&sk->sk_bpf_storage,
NULL, sk_storage);
if (unlikely(prev_sk_storage)) {
selem_unlink_map(first_selem);
err = -EAGAIN;
goto uncharge;
/* Note that even first_selem was linked to smap's
* bucket->list, first_selem can be freed immediately
* (instead of kfree_rcu) because
* bpf_sk_storage_map_free() does a
* synchronize_rcu() before walking the bucket->list.
* Hence, no one is accessing selem from the
* bucket->list under rcu_read_lock().
*/
}
return 0;
uncharge:
kfree(sk_storage);
atomic_sub(sizeof(*sk_storage), &sk->sk_omem_alloc);
return err;
}
/* sk cannot be going away because it is linking new elem
* to sk->sk_bpf_storage. (i.e. sk->sk_refcnt cannot be 0).
* Otherwise, it will become a leak (and other memory issues
* during map destruction).
*/
static struct bpf_sk_storage_data *sk_storage_update(struct sock *sk,
struct bpf_map *map,
void *value,
u64 map_flags)
{
struct bpf_sk_storage_data *old_sdata = NULL;
struct bpf_sk_storage_elem *selem;
struct bpf_sk_storage *sk_storage;
struct bpf_sk_storage_map *smap;
int err;
/* BPF_EXIST and BPF_NOEXIST cannot be both set */
if (unlikely((map_flags & ~BPF_F_LOCK) > BPF_EXIST) ||
/* BPF_F_LOCK can only be used in a value with spin_lock */
unlikely((map_flags & BPF_F_LOCK) && !map_value_has_spin_lock(map)))
return ERR_PTR(-EINVAL);
smap = (struct bpf_sk_storage_map *)map;
sk_storage = rcu_dereference(sk->sk_bpf_storage);
if (!sk_storage || hlist_empty(&sk_storage->list)) {
/* Very first elem for this sk */
err = check_flags(NULL, map_flags);
if (err)
return ERR_PTR(err);
selem = selem_alloc(smap, sk, value, true);
if (!selem)
return ERR_PTR(-ENOMEM);
err = sk_storage_alloc(sk, smap, selem);
if (err) {
kfree(selem);
atomic_sub(smap->elem_size, &sk->sk_omem_alloc);
return ERR_PTR(err);
}
return SDATA(selem);
}
if ((map_flags & BPF_F_LOCK) && !(map_flags & BPF_NOEXIST)) {
/* Hoping to find an old_sdata to do inline update
* such that it can avoid taking the sk_storage->lock
* and changing the lists.
*/
old_sdata = __sk_storage_lookup(sk_storage, smap, false);
err = check_flags(old_sdata, map_flags);
if (err)
return ERR_PTR(err);
if (old_sdata && selem_linked_to_sk(SELEM(old_sdata))) {
copy_map_value_locked(map, old_sdata->data,
value, false);
return old_sdata;
}
}
raw_spin_lock_bh(&sk_storage->lock);
/* Recheck sk_storage->list under sk_storage->lock */
if (unlikely(hlist_empty(&sk_storage->list))) {
/* A parallel del is happening and sk_storage is going
* away. It has just been checked before, so very
* unlikely. Return instead of retry to keep things
* simple.
*/
err = -EAGAIN;
goto unlock_err;
}
old_sdata = __sk_storage_lookup(sk_storage, smap, false);
err = check_flags(old_sdata, map_flags);
if (err)
goto unlock_err;
if (old_sdata && (map_flags & BPF_F_LOCK)) {
copy_map_value_locked(map, old_sdata->data, value, false);
selem = SELEM(old_sdata);
goto unlock;
}
/* sk_storage->lock is held. Hence, we are sure
* we can unlink and uncharge the old_sdata successfully
* later. Hence, instead of charging the new selem now
* and then uncharge the old selem later (which may cause
* a potential but unnecessary charge failure), avoid taking
* a charge at all here (the "!old_sdata" check) and the
* old_sdata will not be uncharged later during __selem_unlink_sk().
*/
selem = selem_alloc(smap, sk, value, !old_sdata);
if (!selem) {
err = -ENOMEM;
goto unlock_err;
}
/* First, link the new selem to the map */
selem_link_map(smap, selem);
/* Second, link (and publish) the new selem to sk_storage */
__selem_link_sk(sk_storage, selem);
/* Third, remove old selem, SELEM(old_sdata) */
if (old_sdata) {
selem_unlink_map(SELEM(old_sdata));
__selem_unlink_sk(sk_storage, SELEM(old_sdata), false);
}
unlock:
raw_spin_unlock_bh(&sk_storage->lock);
return SDATA(selem);
unlock_err:
raw_spin_unlock_bh(&sk_storage->lock);
return ERR_PTR(err);
}
static int sk_storage_delete(struct sock *sk, struct bpf_map *map)
{
struct bpf_sk_storage_data *sdata;
sdata = sk_storage_lookup(sk, map, false);
if (!sdata)
return -ENOENT;
selem_unlink(SELEM(sdata));
return 0;
}
/* Called by __sk_destruct() */
void bpf_sk_storage_free(struct sock *sk)
{
struct bpf_sk_storage_elem *selem;
struct bpf_sk_storage *sk_storage;
bool free_sk_storage = false;
struct hlist_node *n;
rcu_read_lock();
sk_storage = rcu_dereference(sk->sk_bpf_storage);
if (!sk_storage) {
rcu_read_unlock();
return;
}
/* Netiher the bpf_prog nor the bpf-map's syscall
* could be modifying the sk_storage->list now.
* Thus, no elem can be added-to or deleted-from the
* sk_storage->list by the bpf_prog or by the bpf-map's syscall.
*
* It is racing with bpf_sk_storage_map_free() alone
* when unlinking elem from the sk_storage->list and
* the map's bucket->list.
*/
raw_spin_lock_bh(&sk_storage->lock);
hlist_for_each_entry_safe(selem, n, &sk_storage->list, snode) {
/* Always unlink from map before unlinking from
* sk_storage.
*/
selem_unlink_map(selem);
free_sk_storage = __selem_unlink_sk(sk_storage, selem, true);
}
raw_spin_unlock_bh(&sk_storage->lock);
rcu_read_unlock();
if (free_sk_storage)
kfree_rcu(sk_storage, rcu);
}
static void bpf_sk_storage_map_free(struct bpf_map *map)
{
struct bpf_sk_storage_elem *selem;
struct bpf_sk_storage_map *smap;
struct bucket *b;
unsigned int i;
smap = (struct bpf_sk_storage_map *)map;
synchronize_rcu();
/* bpf prog and the userspace can no longer access this map
* now. No new selem (of this map) can be added
* to the sk->sk_bpf_storage or to the map bucket's list.
*
* The elem of this map can be cleaned up here
* or
* by bpf_sk_storage_free() during __sk_destruct().
*/
for (i = 0; i < (1U << smap->bucket_log); i++) {
b = &smap->buckets[i];
rcu_read_lock();
/* No one is adding to b->list now */
while ((selem = hlist_entry_safe(rcu_dereference_raw(hlist_first_rcu(&b->list)),
struct bpf_sk_storage_elem,
map_node))) {
selem_unlink(selem);
cond_resched_rcu();
}
rcu_read_unlock();
}
/* bpf_sk_storage_free() may still need to access the map.
* e.g. bpf_sk_storage_free() has unlinked selem from the map
* which then made the above while((selem = ...)) loop
* exited immediately.
*
* However, the bpf_sk_storage_free() still needs to access
* the smap->elem_size to do the uncharging in
* __selem_unlink_sk().
*
* Hence, wait another rcu grace period for the
* bpf_sk_storage_free() to finish.
*/
synchronize_rcu();
kvfree(smap->buckets);
kfree(map);
}
static int bpf_sk_storage_map_alloc_check(union bpf_attr *attr)
{
if (attr->map_flags != BPF_F_NO_PREALLOC || attr->max_entries ||
attr->key_size != sizeof(int) || !attr->value_size ||
/* Enforce BTF for userspace sk dumping */
!attr->btf_key_type_id || !attr->btf_value_type_id)
return -EINVAL;
if (!capable(CAP_SYS_ADMIN))
return -EPERM;
if (attr->value_size >= KMALLOC_MAX_SIZE -
MAX_BPF_STACK - sizeof(struct bpf_sk_storage_elem) ||
/* U16_MAX is much more than enough for sk local storage
* considering a tcp_sock is ~2k.
*/
attr->value_size > U16_MAX - sizeof(struct bpf_sk_storage_elem))
return -E2BIG;
return 0;
}
static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
{
struct bpf_sk_storage_map *smap;
unsigned int i;
u32 nbuckets;
u64 cost;
smap = kzalloc(sizeof(*smap), GFP_USER | __GFP_NOWARN);
if (!smap)
return ERR_PTR(-ENOMEM);
bpf_map_init_from_attr(&smap->map, attr);
smap->bucket_log = ilog2(roundup_pow_of_two(num_possible_cpus()));
nbuckets = 1U << smap->bucket_log;
smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets,
GFP_USER | __GFP_NOWARN);
if (!smap->buckets) {
kfree(smap);
return ERR_PTR(-ENOMEM);
}
cost = sizeof(*smap->buckets) * nbuckets + sizeof(*smap);
for (i = 0; i < nbuckets; i++) {
INIT_HLIST_HEAD(&smap->buckets[i].list);
raw_spin_lock_init(&smap->buckets[i].lock);
}
smap->elem_size = sizeof(struct bpf_sk_storage_elem) + attr->value_size;
smap->cache_idx = (unsigned int)atomic_inc_return(&cache_idx) %
BPF_SK_STORAGE_CACHE_SIZE;
smap->map.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
return &smap->map;
}
static int notsupp_get_next_key(struct bpf_map *map, void *key,
void *next_key)
{
return -ENOTSUPP;
}
static int bpf_sk_storage_map_check_btf(const struct bpf_map *map,
const struct btf *btf,
const struct btf_type *key_type,
const struct btf_type *value_type)
{
u32 int_data;
if (BTF_INFO_KIND(key_type->info) != BTF_KIND_INT)
return -EINVAL;
int_data = *(u32 *)(key_type + 1);
if (BTF_INT_BITS(int_data) != 32 || BTF_INT_OFFSET(int_data))
return -EINVAL;
return 0;
}
static void *bpf_fd_sk_storage_lookup_elem(struct bpf_map *map, void *key)
{
struct bpf_sk_storage_data *sdata;
struct socket *sock;
int fd, err;
fd = *(int *)key;
sock = sockfd_lookup(fd, &err);
if (sock) {
sdata = sk_storage_lookup(sock->sk, map, true);
sockfd_put(sock);
return sdata ? sdata->data : NULL;
}
return ERR_PTR(err);
}
static int bpf_fd_sk_storage_update_elem(struct bpf_map *map, void *key,
void *value, u64 map_flags)
{
struct bpf_sk_storage_data *sdata;
struct socket *sock;
int fd, err;
fd = *(int *)key;
sock = sockfd_lookup(fd, &err);
if (sock) {
sdata = sk_storage_update(sock->sk, map, value, map_flags);
sockfd_put(sock);
return IS_ERR(sdata) ? PTR_ERR(sdata) : 0;
}
return err;
}
static int bpf_fd_sk_storage_delete_elem(struct bpf_map *map, void *key)
{
struct socket *sock;
int fd, err;
fd = *(int *)key;
sock = sockfd_lookup(fd, &err);
if (sock) {
err = sk_storage_delete(sock->sk, map);
sockfd_put(sock);
return err;
}
return err;
}
BPF_CALL_4(bpf_sk_storage_get, struct bpf_map *, map, struct sock *, sk,
void *, value, u64, flags)
{
struct bpf_sk_storage_data *sdata;
if (flags > BPF_SK_STORAGE_GET_F_CREATE)
return (unsigned long)NULL;
sdata = sk_storage_lookup(sk, map, true);
if (sdata)
return (unsigned long)sdata->data;
if (flags == BPF_SK_STORAGE_GET_F_CREATE &&
/* Cannot add new elem to a going away sk.
* Otherwise, the new elem may become a leak
* (and also other memory issues during map
* destruction).
*/
refcount_inc_not_zero(&sk->sk_refcnt)) {
sdata = sk_storage_update(sk, map, value, BPF_NOEXIST);
/* sk must be a fullsock (guaranteed by verifier),
* so sock_gen_put() is unnecessary.
*/
sock_put(sk);
return IS_ERR(sdata) ?
(unsigned long)NULL : (unsigned long)sdata->data;
}
return (unsigned long)NULL;
}
BPF_CALL_2(bpf_sk_storage_delete, struct bpf_map *, map, struct sock *, sk)
{
if (refcount_inc_not_zero(&sk->sk_refcnt)) {
int err;
err = sk_storage_delete(sk, map);
sock_put(sk);
return err;
}
return -ENOENT;
}
const struct bpf_map_ops sk_storage_map_ops = {
.map_alloc_check = bpf_sk_storage_map_alloc_check,
.map_alloc = bpf_sk_storage_map_alloc,
.map_free = bpf_sk_storage_map_free,
.map_get_next_key = notsupp_get_next_key,
.map_lookup_elem = bpf_fd_sk_storage_lookup_elem,
.map_update_elem = bpf_fd_sk_storage_update_elem,
.map_delete_elem = bpf_fd_sk_storage_delete_elem,
.map_check_btf = bpf_sk_storage_map_check_btf,
};
const struct bpf_func_proto bpf_sk_storage_get_proto = {
.func = bpf_sk_storage_get,
.gpl_only = false,
.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_SOCKET,
.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
.arg4_type = ARG_ANYTHING,
};
const struct bpf_func_proto bpf_sk_storage_delete_proto = {
.func = bpf_sk_storage_delete,
.gpl_only = false,
.ret_type = RET_INTEGER,
.arg1_type = ARG_CONST_MAP_PTR,
.arg2_type = ARG_PTR_TO_SOCKET,
};

12
net/core/filter.c
View File

@ -75,6 +75,7 @@
#include <net/seg6_local.h> #include <net/seg6_local.h>
#include <net/lwtunnel.h> #include <net/lwtunnel.h>
#include <net/ipv6_stubs.h> #include <net/ipv6_stubs.h>
#include <net/bpf_sk_storage.h>
/** /**
* sk_filter_trim_cap - run a packet through a socket filter * sk_filter_trim_cap - run a packet through a socket filter
@ -5903,6 +5904,9 @@ sk_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
} }
} }
const struct bpf_func_proto bpf_sk_storage_get_proto __weak;
const struct bpf_func_proto bpf_sk_storage_delete_proto __weak;
static const struct bpf_func_proto * static const struct bpf_func_proto *
cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{ {
@ -5911,6 +5915,10 @@ cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
return &bpf_get_local_storage_proto; return &bpf_get_local_storage_proto;
case BPF_FUNC_sk_fullsock: case BPF_FUNC_sk_fullsock:
return &bpf_sk_fullsock_proto; return &bpf_sk_fullsock_proto;
case BPF_FUNC_sk_storage_get:
return &bpf_sk_storage_get_proto;
case BPF_FUNC_sk_storage_delete:
return &bpf_sk_storage_delete_proto;
#ifdef CONFIG_INET #ifdef CONFIG_INET
case BPF_FUNC_tcp_sock: case BPF_FUNC_tcp_sock:
return &bpf_tcp_sock_proto; return &bpf_tcp_sock_proto;
@ -5992,6 +6000,10 @@ tc_cls_act_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
return &bpf_skb_fib_lookup_proto; return &bpf_skb_fib_lookup_proto;
case BPF_FUNC_sk_fullsock: case BPF_FUNC_sk_fullsock:
return &bpf_sk_fullsock_proto; return &bpf_sk_fullsock_proto;
case BPF_FUNC_sk_storage_get:
return &bpf_sk_storage_get_proto;
case BPF_FUNC_sk_storage_delete:
return &bpf_sk_storage_delete_proto;
#ifdef CONFIG_XFRM #ifdef CONFIG_XFRM
case BPF_FUNC_skb_get_xfrm_state: case BPF_FUNC_skb_get_xfrm_state:
return &bpf_skb_get_xfrm_state_proto; return &bpf_skb_get_xfrm_state_proto;

5
net/core/sock.c
View File

@ -137,6 +137,7 @@
#include <linux/filter.h> #include <linux/filter.h>
#include <net/sock_reuseport.h> #include <net/sock_reuseport.h>
#include <net/bpf_sk_storage.h>
#include <trace/events/sock.h> #include <trace/events/sock.h>
@ -1709,6 +1710,10 @@ static void __sk_destruct(struct rcu_head *head)
sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP); sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
#ifdef CONFIG_BPF_SYSCALL
bpf_sk_storage_free(sk);
#endif
if (atomic_read(&sk->sk_omem_alloc)) if (atomic_read(&sk->sk_omem_alloc))
pr_debug("%s: optmem leakage (%d bytes) detected\n", pr_debug("%s: optmem leakage (%d bytes) detected\n",
__func__, atomic_read(&sk->sk_omem_alloc)); __func__, atomic_read(&sk->sk_omem_alloc));

1
tools/bpf/bpftool/map.c
View File

@ -46,6 +46,7 @@ const char * const map_type_name[] = {
[BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage",
[BPF_MAP_TYPE_QUEUE] = "queue", [BPF_MAP_TYPE_QUEUE] = "queue",
[BPF_MAP_TYPE_STACK] = "stack", [BPF_MAP_TYPE_STACK] = "stack",
[BPF_MAP_TYPE_SK_STORAGE] = "sk_storage",
}; };
const size_t map_type_name_size = ARRAY_SIZE(map_type_name); const size_t map_type_name_size = ARRAY_SIZE(map_type_name);

44
tools/include/uapi/linux/bpf.h
View File

@ -133,6 +133,7 @@ enum bpf_map_type {
BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE, BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE,
BPF_MAP_TYPE_QUEUE, BPF_MAP_TYPE_QUEUE,
BPF_MAP_TYPE_STACK, BPF_MAP_TYPE_STACK,
BPF_MAP_TYPE_SK_STORAGE,
}; };
/* Note that tracing related programs such as /* Note that tracing related programs such as
@ -2630,6 +2631,42 @@ union bpf_attr {
* was provided. * was provided.
* *
* **-ERANGE** if resulting value was out of range. * **-ERANGE** if resulting value was out of range.
*
* void *bpf_sk_storage_get(struct bpf_map *map, struct bpf_sock *sk, void *value, u64 flags)
* Description
* Get a bpf-local-storage from a sk.
*
* Logically, it could be thought of getting the value from
* a *map* with *sk* as the **key**. From this
* perspective, the usage is not much different from
* **bpf_map_lookup_elem(map, &sk)** except this
* helper enforces the key must be a **bpf_fullsock()**
* and the map must be a BPF_MAP_TYPE_SK_STORAGE also.
*
* Underneath, the value is stored locally at *sk* instead of
* the map. The *map* is used as the bpf-local-storage **type**.
* The bpf-local-storage **type** (i.e. the *map*) is searched
* against all bpf-local-storages residing at sk.
*
* An optional *flags* (BPF_SK_STORAGE_GET_F_CREATE) can be
* used such that a new bpf-local-storage will be
* created if one does not exist. *value* can be used
* together with BPF_SK_STORAGE_GET_F_CREATE to specify
* the initial value of a bpf-local-storage. If *value* is
* NULL, the new bpf-local-storage will be zero initialized.
* Return
* A bpf-local-storage pointer is returned on success.
*
* **NULL** if not found or there was an error in adding
* a new bpf-local-storage.
*
* int bpf_sk_storage_delete(struct bpf_map *map, struct bpf_sock *sk)
* Description
* Delete a bpf-local-storage from a sk.
* Return
* 0 on success.
*
* **-ENOENT** if the bpf-local-storage cannot be found.
*/ */
#define __BPF_FUNC_MAPPER(FN) \ #define __BPF_FUNC_MAPPER(FN) \
FN(unspec), \ FN(unspec), \
@ -2738,7 +2775,9 @@ union bpf_attr {
FN(sysctl_get_new_value), \ FN(sysctl_get_new_value), \
FN(sysctl_set_new_value), \ FN(sysctl_set_new_value), \
FN(strtol), \ FN(strtol), \
FN(strtoul), FN(strtoul), \
FN(sk_storage_get), \
FN(sk_storage_delete),
/* integer value in 'imm' field of BPF_CALL instruction selects which helper /* integer value in 'imm' field of BPF_CALL instruction selects which helper
* function eBPF program intends to call * function eBPF program intends to call
@ -2814,6 +2853,9 @@ enum bpf_func_id {
/* BPF_FUNC_sysctl_get_name flags. */ /* BPF_FUNC_sysctl_get_name flags. */
#define BPF_F_SYSCTL_BASE_NAME (1ULL << 0) #define BPF_F_SYSCTL_BASE_NAME (1ULL << 0)
/* BPF_FUNC_sk_storage_get flags */
#define BPF_SK_STORAGE_GET_F_CREATE (1ULL << 0)
/* Mode for BPF_FUNC_skb_adjust_room helper. */ /* Mode for BPF_FUNC_skb_adjust_room helper. */
enum bpf_adj_room_mode { enum bpf_adj_room_mode {
BPF_ADJ_ROOM_NET, BPF_ADJ_ROOM_NET,

74
tools/lib/bpf/libbpf_probes.c
View File

@ -9,6 +9,7 @@
#include <net/if.h> #include <net/if.h>
#include <sys/utsname.h> #include <sys/utsname.h>
#include <linux/btf.h>
#include <linux/filter.h> #include <linux/filter.h>
#include <linux/kernel.h> #include <linux/kernel.h>
@ -131,11 +132,65 @@ bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex)
return errno != EINVAL && errno != EOPNOTSUPP; return errno != EINVAL && errno != EOPNOTSUPP;
} }
static int load_btf(void)
{
#define BTF_INFO_ENC(kind, kind_flag, vlen) \
((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN))
#define BTF_TYPE_ENC(name, info, size_or_type) \
(name), (info), (size_or_type)
#define BTF_INT_ENC(encoding, bits_offset, nr_bits) \
((encoding) << 24 | (bits_offset) << 16 | (nr_bits))
#define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \
BTF_INT_ENC(encoding, bits_offset, bits)
#define BTF_MEMBER_ENC(name, type, bits_offset) \
(name), (type), (bits_offset)
const char btf_str_sec[] = "\0bpf_spin_lock\0val\0cnt\0l";
/* struct bpf_spin_lock {
* int val;
* };
* struct val {
* int cnt;
* struct bpf_spin_lock l;
* };
*/
__u32 btf_raw_types[] = {
/* int */
BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
/* struct bpf_spin_lock */ /* [2] */
BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 4),
BTF_MEMBER_ENC(15, 1, 0), /* int val; */
/* struct val */ /* [3] */
BTF_TYPE_ENC(15, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
BTF_MEMBER_ENC(19, 1, 0), /* int cnt; */
BTF_MEMBER_ENC(23, 2, 32),/* struct bpf_spin_lock l; */
};
struct btf_header btf_hdr = {
.magic = BTF_MAGIC,
.version = BTF_VERSION,
.hdr_len = sizeof(struct btf_header),
.type_len = sizeof(btf_raw_types),
.str_off = sizeof(btf_raw_types),
.str_len = sizeof(btf_str_sec),
};
__u8 raw_btf[sizeof(struct btf_header) + sizeof(btf_raw_types) +
sizeof(btf_str_sec)];
memcpy(raw_btf, &btf_hdr, sizeof(btf_hdr));
memcpy(raw_btf + sizeof(btf_hdr), btf_raw_types, sizeof(btf_raw_types));
memcpy(raw_btf + sizeof(btf_hdr) + sizeof(btf_raw_types),
btf_str_sec, sizeof(btf_str_sec));
return bpf_load_btf(raw_btf, sizeof(raw_btf), 0, 0, 0);
}
bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex)
{ {
int key_size, value_size, max_entries, map_flags; int key_size, value_size, max_entries, map_flags;
__u32 btf_key_type_id = 0, btf_value_type_id = 0;
struct bpf_create_map_attr attr = {}; struct bpf_create_map_attr attr = {};
int fd = -1, fd_inner; int fd = -1, btf_fd = -1, fd_inner;
key_size = sizeof(__u32); key_size = sizeof(__u32);
value_size = sizeof(__u32); value_size = sizeof(__u32);
@ -161,6 +216,16 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex)
case BPF_MAP_TYPE_STACK: case BPF_MAP_TYPE_STACK:
key_size = 0; key_size = 0;
break; break;
case BPF_MAP_TYPE_SK_STORAGE:
btf_key_type_id = 1;
btf_value_type_id = 3;
value_size = 8;
max_entries = 0;
map_flags = BPF_F_NO_PREALLOC;
btf_fd = load_btf();
if (btf_fd < 0)
return false;
break;
case BPF_MAP_TYPE_UNSPEC: case BPF_MAP_TYPE_UNSPEC:
case BPF_MAP_TYPE_HASH: case BPF_MAP_TYPE_HASH:
case BPF_MAP_TYPE_ARRAY: case BPF_MAP_TYPE_ARRAY:
@ -206,11 +271,18 @@ bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex)
attr.max_entries = max_entries; attr.max_entries = max_entries;
attr.map_flags = map_flags; attr.map_flags = map_flags;
attr.map_ifindex = ifindex; attr.map_ifindex = ifindex;
if (btf_fd >= 0) {
attr.btf_fd = btf_fd;
attr.btf_key_type_id = btf_key_type_id;
attr.btf_value_type_id = btf_value_type_id;
}
fd = bpf_create_map_xattr(&attr); fd = bpf_create_map_xattr(&attr);
} }
if (fd >= 0) if (fd >= 0)
close(fd); close(fd);
if (btf_fd >= 0)
close(btf_fd);
return fd >= 0; return fd >= 0;
} }

25
tools/testing/selftests/bpf/Makefile
View File

@ -74,6 +74,8 @@ all: $(TEST_CUSTOM_PROGS)
$(OUTPUT)/urandom_read: $(OUTPUT)/%: %.c $(OUTPUT)/urandom_read: $(OUTPUT)/%: %.c
$(CC) -o $@ $< -Wl,--build-id $(CC) -o $@ $< -Wl,--build-id
$(OUTPUT)/test_maps: map_tests/*.c
BPFOBJ := $(OUTPUT)/libbpf.a BPFOBJ := $(OUTPUT)/libbpf.a
$(TEST_GEN_PROGS): $(BPFOBJ) $(TEST_GEN_PROGS): $(BPFOBJ)
@ -232,6 +234,27 @@ $(PROG_TESTS_H): $(PROG_TESTS_DIR) $(PROG_TESTS_FILES)
echo '#endif' \ echo '#endif' \
) > $(PROG_TESTS_H)) ) > $(PROG_TESTS_H))
TEST_MAPS_CFLAGS := -I. -I$(OUTPUT)
MAP_TESTS_DIR = $(OUTPUT)/map_tests
$(MAP_TESTS_DIR):
mkdir -p $@
MAP_TESTS_H := $(MAP_TESTS_DIR)/tests.h
test_maps.c: $(MAP_TESTS_H)
$(OUTPUT)/test_maps: CFLAGS += $(TEST_MAPS_CFLAGS)
MAP_TESTS_FILES := $(wildcard map_tests/*.c)
$(MAP_TESTS_H): $(MAP_TESTS_DIR) $(MAP_TESTS_FILES)
$(shell ( cd map_tests/; \
echo '/* Generated header, do not edit */'; \
echo '#ifdef DECLARE'; \
ls *.c 2> /dev/null | \
sed -e 's@\([^\.]*\)\.c@extern void test_\1(void);@'; \
echo '#endif'; \
echo '#ifdef CALL'; \
ls *.c 2> /dev/null | \
sed -e 's@\([^\.]*\)\.c@test_\1();@'; \
echo '#endif' \
) > $(MAP_TESTS_H))
VERIFIER_TESTS_H := $(OUTPUT)/verifier/tests.h VERIFIER_TESTS_H := $(OUTPUT)/verifier/tests.h
test_verifier.c: $(VERIFIER_TESTS_H) test_verifier.c: $(VERIFIER_TESTS_H)
$(OUTPUT)/test_verifier: CFLAGS += $(TEST_VERIFIER_CFLAGS) $(OUTPUT)/test_verifier: CFLAGS += $(TEST_VERIFIER_CFLAGS)
@ -251,4 +274,4 @@ $(OUTPUT)/verifier/tests.h: $(VERIFIER_TESTS_DIR) $(VERIFIER_TEST_FILES)
) > $(VERIFIER_TESTS_H)) ) > $(VERIFIER_TESTS_H))
EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(ALU32_BUILD_DIR) \ EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(ALU32_BUILD_DIR) \
$(VERIFIER_TESTS_H) $(PROG_TESTS_H) $(VERIFIER_TESTS_H) $(PROG_TESTS_H) $(MAP_TESTS_H)

5
tools/testing/selftests/bpf/bpf_helpers.h
View File

@ -211,6 +211,11 @@ static int (*bpf_strtol)(const char *buf, unsigned long long buf_len,
static int (*bpf_strtoul)(const char *buf, unsigned long long buf_len, static int (*bpf_strtoul)(const char *buf, unsigned long long buf_len,
unsigned long long flags, unsigned long *res) = unsigned long long flags, unsigned long *res) =
(void *) BPF_FUNC_strtoul; (void *) BPF_FUNC_strtoul;
static void *(*bpf_sk_storage_get)(void *map, struct bpf_sock *sk,
void *value, __u64 flags) =
(void *) BPF_FUNC_sk_storage_get;
static int (*bpf_sk_storage_delete)(void *map, struct bpf_sock *sk) =
(void *)BPF_FUNC_sk_storage_delete;
/* llvm builtin functions that eBPF C program may use to /* llvm builtin functions that eBPF C program may use to
* emit BPF_LD_ABS and BPF_LD_IND instructions * emit BPF_LD_ABS and BPF_LD_IND instructions

629
tools/testing/selftests/bpf/map_tests/sk_storage_map.c Normal file
View File

@ -0,0 +1,629 @@
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 Facebook */
#include <linux/compiler.h>
#include <linux/err.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <linux/btf.h>
#include <unistd.h>
#include <signal.h>
#include <errno.h>
#include <string.h>
#include <pthread.h>
#include <bpf/bpf.h>
#include <bpf/libbpf.h>
#include <test_btf.h>
#include <test_maps.h>
static struct bpf_create_map_attr xattr = {
.name = "sk_storage_map",
.map_type = BPF_MAP_TYPE_SK_STORAGE,
.map_flags = BPF_F_NO_PREALLOC,
.max_entries = 0,
.key_size = 4,
.value_size = 8,
.btf_key_type_id = 1,
.btf_value_type_id = 3,
.btf_fd = -1,
};
static unsigned int nr_sk_threads_done;
static unsigned int nr_sk_threads_err;
static unsigned int nr_sk_per_thread = 4096;
static unsigned int nr_sk_threads = 4;
static int sk_storage_map = -1;
static unsigned int stop;
static int runtime_s = 5;
static bool is_stopped(void)
{
return READ_ONCE(stop);
}
static unsigned int threads_err(void)
{
return READ_ONCE(nr_sk_threads_err);
}
static void notify_thread_err(void)
{
__sync_add_and_fetch(&nr_sk_threads_err, 1);
}
static bool wait_for_threads_err(void)
{
while (!is_stopped() && !threads_err())
usleep(500);
return !is_stopped();
}
static unsigned int threads_done(void)
{
return READ_ONCE(nr_sk_threads_done);
}
static void notify_thread_done(void)
{
__sync_add_and_fetch(&nr_sk_threads_done, 1);
}
static void notify_thread_redo(void)
{
__sync_sub_and_fetch(&nr_sk_threads_done, 1);
}
static bool wait_for_threads_done(void)
{
while (threads_done() != nr_sk_threads && !is_stopped() &&
!threads_err())
usleep(50);
return !is_stopped() && !threads_err();
}
static bool wait_for_threads_redo(void)
{
while (threads_done() && !is_stopped() && !threads_err())
usleep(50);
return !is_stopped() && !threads_err();
}
static bool wait_for_map(void)
{
while (READ_ONCE(sk_storage_map) == -1 && !is_stopped())
usleep(50);
return !is_stopped();
}
static bool wait_for_map_close(void)
{
while (READ_ONCE(sk_storage_map) != -1 && !is_stopped())
;
return !is_stopped();
}
static int load_btf(void)
{
const char btf_str_sec[] = "\0bpf_spin_lock\0val\0cnt\0l";
__u32 btf_raw_types[] = {
/* int */
BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
/* struct bpf_spin_lock */ /* [2] */
BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 4),
BTF_MEMBER_ENC(15, 1, 0), /* int val; */
/* struct val */ /* [3] */
BTF_TYPE_ENC(15, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
BTF_MEMBER_ENC(19, 1, 0), /* int cnt; */
BTF_MEMBER_ENC(23, 2, 32),/* struct bpf_spin_lock l; */
};
struct btf_header btf_hdr = {
.magic = BTF_MAGIC,
.version = BTF_VERSION,
.hdr_len = sizeof(struct btf_header),
.type_len = sizeof(btf_raw_types),
.str_off = sizeof(btf_raw_types),
.str_len = sizeof(btf_str_sec),
};
__u8 raw_btf[sizeof(struct btf_header) + sizeof(btf_raw_types) +
sizeof(btf_str_sec)];
memcpy(raw_btf, &btf_hdr, sizeof(btf_hdr));
memcpy(raw_btf + sizeof(btf_hdr), btf_raw_types, sizeof(btf_raw_types));
memcpy(raw_btf + sizeof(btf_hdr) + sizeof(btf_raw_types),
btf_str_sec, sizeof(btf_str_sec));
return bpf_load_btf(raw_btf, sizeof(raw_btf), 0, 0, 0);
}
static int create_sk_storage_map(void)
{
int btf_fd, map_fd;
btf_fd = load_btf();
CHECK(btf_fd == -1, "bpf_load_btf", "btf_fd:%d errno:%d\n",
btf_fd, errno);
xattr.btf_fd = btf_fd;
map_fd = bpf_create_map_xattr(&xattr);
xattr.btf_fd = -1;
close(btf_fd);
CHECK(map_fd == -1,
"bpf_create_map_xattr()", "errno:%d\n", errno);
return map_fd;
}
static void *insert_close_thread(void *arg)
{
struct {
int cnt;
int lock;
} value = { .cnt = 0xeB9F, .lock = 0, };
int i, map_fd, err, *sk_fds;
sk_fds = malloc(sizeof(*sk_fds) * nr_sk_per_thread);
if (!sk_fds) {
notify_thread_err();
return ERR_PTR(-ENOMEM);
}
for (i = 0; i < nr_sk_per_thread; i++)
sk_fds[i] = -1;
while (!is_stopped()) {
if (!wait_for_map())
goto close_all;
map_fd = READ_ONCE(sk_storage_map);
for (i = 0; i < nr_sk_per_thread && !is_stopped(); i++) {
sk_fds[i] = socket(AF_INET6, SOCK_STREAM, 0);
if (sk_fds[i] == -1) {
err = -errno;
fprintf(stderr, "socket(): errno:%d\n", errno);
goto errout;
}
err = bpf_map_update_elem(map_fd, &sk_fds[i], &value,
BPF_NOEXIST);
if (err) {
err = -errno;
fprintf(stderr,
"bpf_map_update_elem(): errno:%d\n",
errno);
goto errout;
}
}
notify_thread_done();
wait_for_map_close();
close_all:
for (i = 0; i < nr_sk_per_thread; i++) {
close(sk_fds[i]);
sk_fds[i] = -1;
}
notify_thread_redo();
}
free(sk_fds);
return NULL;
errout:
for (i = 0; i < nr_sk_per_thread && sk_fds[i] != -1; i++)
close(sk_fds[i]);
free(sk_fds);
notify_thread_err();
return ERR_PTR(err);
}
static int do_sk_storage_map_stress_free(void)
{
int i, map_fd = -1, err = 0, nr_threads_created = 0;
pthread_t *sk_thread_ids;
void *thread_ret;
sk_thread_ids = malloc(sizeof(pthread_t) * nr_sk_threads);
if (!sk_thread_ids) {
fprintf(stderr, "malloc(sk_threads): NULL\n");
return -ENOMEM;
}
for (i = 0; i < nr_sk_threads; i++) {
err = pthread_create(&sk_thread_ids[i], NULL,
insert_close_thread, NULL);
if (err) {
err = -errno;
goto done;
}
nr_threads_created++;
}
while (!is_stopped()) {
map_fd = create_sk_storage_map();
WRITE_ONCE(sk_storage_map, map_fd);
if (!wait_for_threads_done())
break;
WRITE_ONCE(sk_storage_map, -1);
close(map_fd);
map_fd = -1;
if (!wait_for_threads_redo())
break;
}
done:
WRITE_ONCE(stop, 1);
for (i = 0; i < nr_threads_created; i++) {
pthread_join(sk_thread_ids[i], &thread_ret);
if (IS_ERR(thread_ret) && !err) {
err = PTR_ERR(thread_ret);
fprintf(stderr, "threads#%u: err:%d\n", i, err);
}
}
free(sk_thread_ids);
if (map_fd != -1)
close(map_fd);
return err;
}
static void *update_thread(void *arg)
{
struct {
int cnt;
int lock;
} value = { .cnt = 0xeB9F, .lock = 0, };
int map_fd = READ_ONCE(sk_storage_map);
int sk_fd = *(int *)arg;
int err = 0; /* Suppress compiler false alarm */
while (!is_stopped()) {
err = bpf_map_update_elem(map_fd, &sk_fd, &value, 0);
if (err && errno != EAGAIN) {
err = -errno;
fprintf(stderr, "bpf_map_update_elem: %d %d\n",
err, errno);
break;
}
}
if (!is_stopped()) {
notify_thread_err();
return ERR_PTR(err);
}
return NULL;
}
static void *delete_thread(void *arg)
{
int map_fd = READ_ONCE(sk_storage_map);
int sk_fd = *(int *)arg;
int err = 0; /* Suppress compiler false alarm */
while (!is_stopped()) {
err = bpf_map_delete_elem(map_fd, &sk_fd);
if (err && errno != ENOENT) {
err = -errno;
fprintf(stderr, "bpf_map_delete_elem: %d %d\n",
err, errno);
break;
}
}
if (!is_stopped()) {
notify_thread_err();
return ERR_PTR(err);
}
return NULL;
}
static int do_sk_storage_map_stress_change(void)
{
int i, sk_fd, map_fd = -1, err = 0, nr_threads_created = 0;
pthread_t *sk_thread_ids;
void *thread_ret;
sk_thread_ids = malloc(sizeof(pthread_t) * nr_sk_threads);
if (!sk_thread_ids) {
fprintf(stderr, "malloc(sk_threads): NULL\n");
return -ENOMEM;
}
sk_fd = socket(AF_INET6, SOCK_STREAM, 0);
if (sk_fd == -1) {
err = -errno;
goto done;
}
map_fd = create_sk_storage_map();
WRITE_ONCE(sk_storage_map, map_fd);
for (i = 0; i < nr_sk_threads; i++) {
if (i & 0x1)
err = pthread_create(&sk_thread_ids[i], NULL,
update_thread, &sk_fd);
else
err = pthread_create(&sk_thread_ids[i], NULL,
delete_thread, &sk_fd);
if (err) {
err = -errno;
goto done;
}
nr_threads_created++;
}
wait_for_threads_err();
done:
WRITE_ONCE(stop, 1);
for (i = 0; i < nr_threads_created; i++) {
pthread_join(sk_thread_ids[i], &thread_ret);
if (IS_ERR(thread_ret) && !err) {
err = PTR_ERR(thread_ret);
fprintf(stderr, "threads#%u: err:%d\n", i, err);
}
}
free(sk_thread_ids);
if (sk_fd != -1)
close(sk_fd);
close(map_fd);
return err;
}
static void stop_handler(int signum)
{
if (signum != SIGALRM)
printf("stopping...\n");
WRITE_ONCE(stop, 1);
}
#define BPF_SK_STORAGE_MAP_TEST_NR_THREADS "BPF_SK_STORAGE_MAP_TEST_NR_THREADS"
#define BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD "BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD"
#define BPF_SK_STORAGE_MAP_TEST_RUNTIME_S "BPF_SK_STORAGE_MAP_TEST_RUNTIME_S"
#define BPF_SK_STORAGE_MAP_TEST_NAME "BPF_SK_STORAGE_MAP_TEST_NAME"
static void test_sk_storage_map_stress_free(void)
{
struct rlimit rlim_old, rlim_new = {};
int err;
getrlimit(RLIMIT_NOFILE, &rlim_old);
signal(SIGTERM, stop_handler);
signal(SIGINT, stop_handler);
if (runtime_s > 0) {
signal(SIGALRM, stop_handler);
alarm(runtime_s);
}
if (rlim_old.rlim_cur < nr_sk_threads * nr_sk_per_thread) {
rlim_new.rlim_cur = nr_sk_threads * nr_sk_per_thread + 128;
rlim_new.rlim_max = rlim_new.rlim_cur + 128;
err = setrlimit(RLIMIT_NOFILE, &rlim_new);
CHECK(err, "setrlimit(RLIMIT_NOFILE)", "rlim_new:%lu errno:%d",
rlim_new.rlim_cur, errno);
}
err = do_sk_storage_map_stress_free();
signal(SIGTERM, SIG_DFL);
signal(SIGINT, SIG_DFL);
if (runtime_s > 0) {
signal(SIGALRM, SIG_DFL);
alarm(0);
}
if (rlim_new.rlim_cur)
setrlimit(RLIMIT_NOFILE, &rlim_old);
CHECK(err, "test_sk_storage_map_stress_free", "err:%d\n", err);
}
static void test_sk_storage_map_stress_change(void)
{
int err;
signal(SIGTERM, stop_handler);
signal(SIGINT, stop_handler);
if (runtime_s > 0) {
signal(SIGALRM, stop_handler);
alarm(runtime_s);
}
err = do_sk_storage_map_stress_change();
signal(SIGTERM, SIG_DFL);
signal(SIGINT, SIG_DFL);
if (runtime_s > 0) {
signal(SIGALRM, SIG_DFL);
alarm(0);
}
CHECK(err, "test_sk_storage_map_stress_change", "err:%d\n", err);
}
static void test_sk_storage_map_basic(void)
{
struct {
int cnt;
int lock;
} value = { .cnt = 0xeB9f, .lock = 0, }, lookup_value;
struct bpf_create_map_attr bad_xattr;
int btf_fd, map_fd, sk_fd, err;
btf_fd = load_btf();
CHECK(btf_fd == -1, "bpf_load_btf", "btf_fd:%d errno:%d\n",
btf_fd, errno);
xattr.btf_fd = btf_fd;
sk_fd = socket(AF_INET6, SOCK_STREAM, 0);
CHECK(sk_fd == -1, "socket()", "sk_fd:%d errno:%d\n",
sk_fd, errno);
map_fd = bpf_create_map_xattr(&xattr);
CHECK(map_fd == -1, "bpf_create_map_xattr(good_xattr)",
"map_fd:%d errno:%d\n", map_fd, errno);
/* Add new elem */
memcpy(&lookup_value, &value, sizeof(value));
err = bpf_map_update_elem(map_fd, &sk_fd, &value,
BPF_NOEXIST | BPF_F_LOCK);
CHECK(err, "bpf_map_update_elem(BPF_NOEXIST|BPF_F_LOCK)",
"err:%d errno:%d\n", err, errno);
err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
BPF_F_LOCK);
CHECK(err || lookup_value.cnt != value.cnt,
"bpf_map_lookup_elem_flags(BPF_F_LOCK)",
"err:%d errno:%d cnt:%x(%x)\n",
err, errno, lookup_value.cnt, value.cnt);
/* Bump the cnt and update with BPF_EXIST | BPF_F_LOCK */
value.cnt += 1;
err = bpf_map_update_elem(map_fd, &sk_fd, &value,
BPF_EXIST | BPF_F_LOCK);
CHECK(err, "bpf_map_update_elem(BPF_EXIST|BPF_F_LOCK)",
"err:%d errno:%d\n", err, errno);
err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
BPF_F_LOCK);
CHECK(err || lookup_value.cnt != value.cnt,
"bpf_map_lookup_elem_flags(BPF_F_LOCK)",
"err:%d errno:%d cnt:%x(%x)\n",
err, errno, lookup_value.cnt, value.cnt);
/* Bump the cnt and update with BPF_EXIST */
value.cnt += 1;
err = bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_EXIST);
CHECK(err, "bpf_map_update_elem(BPF_EXIST)",
"err:%d errno:%d\n", err, errno);
err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
BPF_F_LOCK);
CHECK(err || lookup_value.cnt != value.cnt,
"bpf_map_lookup_elem_flags(BPF_F_LOCK)",
"err:%d errno:%d cnt:%x(%x)\n",
err, errno, lookup_value.cnt, value.cnt);
/* Update with BPF_NOEXIST */
value.cnt += 1;
err = bpf_map_update_elem(map_fd, &sk_fd, &value,
BPF_NOEXIST | BPF_F_LOCK);
CHECK(!err || errno != EEXIST,
"bpf_map_update_elem(BPF_NOEXIST|BPF_F_LOCK)",
"err:%d errno:%d\n", err, errno);
err = bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_NOEXIST);
CHECK(!err || errno != EEXIST, "bpf_map_update_elem(BPF_NOEXIST)",
"err:%d errno:%d\n", err, errno);
value.cnt -= 1;
err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
BPF_F_LOCK);
CHECK(err || lookup_value.cnt != value.cnt,
"bpf_map_lookup_elem_flags(BPF_F_LOCK)",
"err:%d errno:%d cnt:%x(%x)\n",
err, errno, lookup_value.cnt, value.cnt);
/* Bump the cnt again and update with map_flags == 0 */
value.cnt += 1;
err = bpf_map_update_elem(map_fd, &sk_fd, &value, 0);
CHECK(err, "bpf_map_update_elem()", "err:%d errno:%d\n",
err, errno);
err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
BPF_F_LOCK);
CHECK(err || lookup_value.cnt != value.cnt,
"bpf_map_lookup_elem_flags(BPF_F_LOCK)",
"err:%d errno:%d cnt:%x(%x)\n",
err, errno, lookup_value.cnt, value.cnt);
/* Test delete elem */
err = bpf_map_delete_elem(map_fd, &sk_fd);
CHECK(err, "bpf_map_delete_elem()", "err:%d errno:%d\n",
err, errno);
err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
BPF_F_LOCK);
CHECK(!err || errno != ENOENT,
"bpf_map_lookup_elem_flags(BPF_F_LOCK)",
"err:%d errno:%d\n", err, errno);
err = bpf_map_delete_elem(map_fd, &sk_fd);
CHECK(!err || errno != ENOENT, "bpf_map_delete_elem()",
"err:%d errno:%d\n", err, errno);
memcpy(&bad_xattr, &xattr, sizeof(xattr));
bad_xattr.btf_key_type_id = 0;
err = bpf_create_map_xattr(&bad_xattr);
CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",
"err:%d errno:%d\n", err, errno);
memcpy(&bad_xattr, &xattr, sizeof(xattr));
bad_xattr.btf_key_type_id = 3;
err = bpf_create_map_xattr(&bad_xattr);
CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",
"err:%d errno:%d\n", err, errno);
memcpy(&bad_xattr, &xattr, sizeof(xattr));
bad_xattr.max_entries = 1;
err = bpf_create_map_xattr(&bad_xattr);
CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",
"err:%d errno:%d\n", err, errno);
memcpy(&bad_xattr, &xattr, sizeof(xattr));
bad_xattr.map_flags = 0;
err = bpf_create_map_xattr(&bad_xattr);
CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",
"err:%d errno:%d\n", err, errno);
xattr.btf_fd = -1;
close(btf_fd);
close(map_fd);
close(sk_fd);
}
void test_sk_storage_map(void)
{
const char *test_name, *env_opt;
bool test_ran = false;
test_name = getenv(BPF_SK_STORAGE_MAP_TEST_NAME);
env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_NR_THREADS);
if (env_opt)
nr_sk_threads = atoi(env_opt);
env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD);
if (env_opt)
nr_sk_per_thread = atoi(env_opt);
env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_RUNTIME_S);
if (env_opt)
runtime_s = atoi(env_opt);
if (!test_name || !strcmp(test_name, "basic")) {
test_sk_storage_map_basic();
test_ran = true;
}
if (!test_name || !strcmp(test_name, "stress_free")) {
test_sk_storage_map_stress_free();
test_ran = true;
}
if (!test_name || !strcmp(test_name, "stress_change")) {
test_sk_storage_map_stress_change();
test_ran = true;
}
if (test_ran)
printf("%s:PASS\n", __func__);
else
CHECK(1, "Invalid test_name", "%s\n", test_name);
}

53
tools/testing/selftests/bpf/progs/test_sock_fields_kern.c
View File

@ -55,6 +55,31 @@ struct bpf_map_def SEC("maps") linum_map = {
.max_entries = __NR_BPF_LINUM_ARRAY_IDX, .max_entries = __NR_BPF_LINUM_ARRAY_IDX,
}; };
struct bpf_spinlock_cnt {
struct bpf_spin_lock lock;
__u32 cnt;
};
struct bpf_map_def SEC("maps") sk_pkt_out_cnt = {
.type = BPF_MAP_TYPE_SK_STORAGE,
.key_size = sizeof(int),
.value_size = sizeof(struct bpf_spinlock_cnt),
.max_entries = 0,
.map_flags = BPF_F_NO_PREALLOC,
};
BPF_ANNOTATE_KV_PAIR(sk_pkt_out_cnt, int, struct bpf_spinlock_cnt);
struct bpf_map_def SEC("maps") sk_pkt_out_cnt10 = {
.type = BPF_MAP_TYPE_SK_STORAGE,
.key_size = sizeof(int),
.value_size = sizeof(struct bpf_spinlock_cnt),
.max_entries = 0,
.map_flags = BPF_F_NO_PREALLOC,
};
BPF_ANNOTATE_KV_PAIR(sk_pkt_out_cnt10, int, struct bpf_spinlock_cnt);
static bool is_loopback6(__u32 *a6) static bool is_loopback6(__u32 *a6)
{ {
return !a6[0] && !a6[1] && !a6[2] && a6[3] == bpf_htonl(1); return !a6[0] && !a6[1] && !a6[2] && a6[3] == bpf_htonl(1);
@ -120,7 +145,9 @@ static void tpcpy(struct bpf_tcp_sock *dst,
SEC("cgroup_skb/egress") SEC("cgroup_skb/egress")
int egress_read_sock_fields(struct __sk_buff *skb) int egress_read_sock_fields(struct __sk_buff *skb)
{ {
struct bpf_spinlock_cnt cli_cnt_init = { .lock = 0, .cnt = 0xeB9F };
__u32 srv_idx = ADDR_SRV_IDX, cli_idx = ADDR_CLI_IDX, result_idx; __u32 srv_idx = ADDR_SRV_IDX, cli_idx = ADDR_CLI_IDX, result_idx;
struct bpf_spinlock_cnt *pkt_out_cnt, *pkt_out_cnt10;
struct sockaddr_in6 *srv_sa6, *cli_sa6; struct sockaddr_in6 *srv_sa6, *cli_sa6;
struct bpf_tcp_sock *tp, *tp_ret; struct bpf_tcp_sock *tp, *tp_ret;
struct bpf_sock *sk, *sk_ret; struct bpf_sock *sk, *sk_ret;
@ -161,6 +188,32 @@ int egress_read_sock_fields(struct __sk_buff *skb)
skcpy(sk_ret, sk); skcpy(sk_ret, sk);
tpcpy(tp_ret, tp); tpcpy(tp_ret, tp);
if (result_idx == EGRESS_SRV_IDX) {
/* The userspace has created it for srv sk */
pkt_out_cnt = bpf_sk_storage_get(&sk_pkt_out_cnt, sk, 0, 0);
pkt_out_cnt10 = bpf_sk_storage_get(&sk_pkt_out_cnt10, sk,
0, 0);
} else {
pkt_out_cnt = bpf_sk_storage_get(&sk_pkt_out_cnt, sk,
&cli_cnt_init,
BPF_SK_STORAGE_GET_F_CREATE);
pkt_out_cnt10 = bpf_sk_storage_get(&sk_pkt_out_cnt10,
sk, &cli_cnt_init,
BPF_SK_STORAGE_GET_F_CREATE);
}
if (!pkt_out_cnt || !pkt_out_cnt10)
RETURN;
/* Even both cnt and cnt10 have lock defined in their BTF,
* intentionally one cnt takes lock while one does not
* as a test for the spinlock support in BPF_MAP_TYPE_SK_STORAGE.
*/
pkt_out_cnt->cnt += 1;
bpf_spin_lock(&pkt_out_cnt10->lock);
pkt_out_cnt10->cnt += 10;
bpf_spin_unlock(&pkt_out_cnt10->lock);
RETURN; RETURN;
} }

63
tools/testing/selftests/bpf/test_btf.c
View File

@ -24,6 +24,7 @@
#include "bpf_rlimit.h" #include "bpf_rlimit.h"
#include "bpf_util.h" #include "bpf_util.h"
#include "test_btf.h"
#define MAX_INSNS 512 #define MAX_INSNS 512
#define MAX_SUBPROGS 16 #define MAX_SUBPROGS 16
@ -58,68 +59,6 @@ static int __base_pr(enum libbpf_print_level level __attribute__((unused)),
return vfprintf(stderr, format, args); return vfprintf(stderr, format, args);
} }
#define BTF_INFO_ENC(kind, kind_flag, vlen) \
((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN))
#define BTF_TYPE_ENC(name, info, size_or_type) \
(name), (info), (size_or_type)
#define BTF_INT_ENC(encoding, bits_offset, nr_bits) \
((encoding) << 24 | (bits_offset) << 16 | (nr_bits))
#define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \
BTF_INT_ENC(encoding, bits_offset, bits)
#define BTF_FWD_ENC(name, kind_flag) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FWD, kind_flag, 0), 0)
#define BTF_ARRAY_ENC(type, index_type, nr_elems) \
(type), (index_type), (nr_elems)
#define BTF_TYPE_ARRAY_ENC(type, index_type, nr_elems) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_ARRAY, 0, 0), 0), \
BTF_ARRAY_ENC(type, index_type, nr_elems)
#define BTF_STRUCT_ENC(name, nr_elems, sz) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, nr_elems), sz)
#define BTF_UNION_ENC(name, nr_elems, sz) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_UNION, 0, nr_elems), sz)
#define BTF_VAR_ENC(name, type, linkage) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), type), (linkage)
#define BTF_VAR_SECINFO_ENC(type, offset, size) \
(type), (offset), (size)
#define BTF_MEMBER_ENC(name, type, bits_offset) \
(name), (type), (bits_offset)
#define BTF_ENUM_ENC(name, val) (name), (val)
#define BTF_MEMBER_OFFSET(bitfield_size, bits_offset) \
((bitfield_size) << 24 | (bits_offset))
#define BTF_TYPEDEF_ENC(name, type) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0), type)
#define BTF_PTR_ENC(type) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), type)
#define BTF_CONST_ENC(type) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), type)
#define BTF_VOLATILE_ENC(type) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_VOLATILE, 0, 0), type)
#define BTF_RESTRICT_ENC(type) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_RESTRICT, 0, 0), type)
#define BTF_FUNC_PROTO_ENC(ret_type, nargs) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, nargs), ret_type)
#define BTF_FUNC_PROTO_ARG_ENC(name, type) \
(name), (type)
#define BTF_FUNC_ENC(name, func_proto) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), func_proto)
#define BTF_END_RAW 0xdeadbeef #define BTF_END_RAW 0xdeadbeef
#define NAME_TBD 0xdeadb33f #define NAME_TBD 0xdeadb33f

69
tools/testing/selftests/bpf/test_btf.h Normal file
View File

@ -0,0 +1,69 @@
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2019 Facebook */
#ifndef _TEST_BTF_H
#define _TEST_BTF_H
#define BTF_INFO_ENC(kind, kind_flag, vlen) \
((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN))
#define BTF_TYPE_ENC(name, info, size_or_type) \
(name), (info), (size_or_type)
#define BTF_INT_ENC(encoding, bits_offset, nr_bits) \
((encoding) << 24 | (bits_offset) << 16 | (nr_bits))
#define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \
BTF_INT_ENC(encoding, bits_offset, bits)
#define BTF_FWD_ENC(name, kind_flag) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FWD, kind_flag, 0), 0)
#define BTF_ARRAY_ENC(type, index_type, nr_elems) \
(type), (index_type), (nr_elems)
#define BTF_TYPE_ARRAY_ENC(type, index_type, nr_elems) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_ARRAY, 0, 0), 0), \
BTF_ARRAY_ENC(type, index_type, nr_elems)
#define BTF_STRUCT_ENC(name, nr_elems, sz) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, nr_elems), sz)
#define BTF_UNION_ENC(name, nr_elems, sz) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_UNION, 0, nr_elems), sz)
#define BTF_VAR_ENC(name, type, linkage) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), type), (linkage)
#define BTF_VAR_SECINFO_ENC(type, offset, size) \
(type), (offset), (size)
#define BTF_MEMBER_ENC(name, type, bits_offset) \
(name), (type), (bits_offset)
#define BTF_ENUM_ENC(name, val) (name), (val)
#define BTF_MEMBER_OFFSET(bitfield_size, bits_offset) \
((bitfield_size) << 24 | (bits_offset))
#define BTF_TYPEDEF_ENC(name, type) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0), type)
#define BTF_PTR_ENC(type) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), type)
#define BTF_CONST_ENC(type) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_CONST, 0, 0), type)
#define BTF_VOLATILE_ENC(type) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_VOLATILE, 0, 0), type)
#define BTF_RESTRICT_ENC(type) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_RESTRICT, 0, 0), type)
#define BTF_FUNC_PROTO_ENC(ret_type, nargs) \
BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, nargs), ret_type)
#define BTF_FUNC_PROTO_ARG_ENC(name, type) \
(name), (type)
#define BTF_FUNC_ENC(name, func_proto) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), func_proto)
#endif /* _TEST_BTF_H */

18
tools/testing/selftests/bpf/test_maps.c
View File

@ -27,6 +27,7 @@
#include "bpf_util.h" #include "bpf_util.h"
#include "bpf_rlimit.h" #include "bpf_rlimit.h"
#include "test_maps.h"
#ifndef ENOTSUPP #ifndef ENOTSUPP
#define ENOTSUPP 524 #define ENOTSUPP 524
@ -36,15 +37,6 @@ static int skips;
static int map_flags; static int map_flags;
#define CHECK(condition, tag, format...) ({ \
int __ret = !!(condition); \
if (__ret) { \
printf("%s(%d):FAIL:%s ", __func__, __LINE__, tag); \
printf(format); \
exit(-1); \
} \
})
static void test_hashmap(unsigned int task, void *data) static void test_hashmap(unsigned int task, void *data)
{ {
long long key, next_key, first_key, value; long long key, next_key, first_key, value;
@ -1703,6 +1695,10 @@ static void run_all_tests(void)
test_map_in_map(); test_map_in_map();
} }
#define DECLARE
#include <map_tests/tests.h>
#undef DECLARE
int main(void) int main(void)
{ {
srand(time(NULL)); srand(time(NULL));
@ -1713,6 +1709,10 @@ int main(void)
map_flags = BPF_F_NO_PREALLOC; map_flags = BPF_F_NO_PREALLOC;
run_all_tests(); run_all_tests();
#define CALL
#include <map_tests/tests.h>
#undef CALL
printf("test_maps: OK, %d SKIPPED\n", skips); printf("test_maps: OK, %d SKIPPED\n", skips);
return 0; return 0;
} }

17
tools/testing/selftests/bpf/test_maps.h Normal file
View File

@ -0,0 +1,17 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _TEST_MAPS_H
#define _TEST_MAPS_H
#include <stdio.h>
#include <stdlib.h>
#define CHECK(condition, tag, format...) ({ \
int __ret = !!(condition); \
if (__ret) { \
printf("%s(%d):FAIL:%s ", __func__, __LINE__, tag); \
printf(format); \
exit(-1); \
} \
})
#endif

101
tools/testing/selftests/bpf/test_sock_fields.c
View File

@ -35,6 +35,11 @@ enum bpf_linum_array_idx {
__NR_BPF_LINUM_ARRAY_IDX, __NR_BPF_LINUM_ARRAY_IDX,
}; };
struct bpf_spinlock_cnt {
struct bpf_spin_lock lock;
__u32 cnt;
};
#define CHECK(condition, tag, format...) ({ \ #define CHECK(condition, tag, format...) ({ \
int __ret = !!(condition); \ int __ret = !!(condition); \
if (__ret) { \ if (__ret) { \
@ -50,6 +55,8 @@ enum bpf_linum_array_idx {
#define DATA_LEN sizeof(DATA) #define DATA_LEN sizeof(DATA)
static struct sockaddr_in6 srv_sa6, cli_sa6; static struct sockaddr_in6 srv_sa6, cli_sa6;
static int sk_pkt_out_cnt10_fd;
static int sk_pkt_out_cnt_fd;
static int linum_map_fd; static int linum_map_fd;
static int addr_map_fd; static int addr_map_fd;
static int tp_map_fd; static int tp_map_fd;
@ -220,28 +227,90 @@ static void check_result(void)
"Unexpected listen_tp", "Check listen_tp output. ingress_linum:%u", "Unexpected listen_tp", "Check listen_tp output. ingress_linum:%u",
ingress_linum); ingress_linum);
CHECK(srv_tp.data_segs_out != 1 || CHECK(srv_tp.data_segs_out != 2 ||
srv_tp.data_segs_in || srv_tp.data_segs_in ||
srv_tp.snd_cwnd != 10 || srv_tp.snd_cwnd != 10 ||
srv_tp.total_retrans || srv_tp.total_retrans ||
srv_tp.bytes_acked != DATA_LEN, srv_tp.bytes_acked != 2 * DATA_LEN,
"Unexpected srv_tp", "Check srv_tp output. egress_linum:%u", "Unexpected srv_tp", "Check srv_tp output. egress_linum:%u",
egress_linum); egress_linum);
CHECK(cli_tp.data_segs_out || CHECK(cli_tp.data_segs_out ||
cli_tp.data_segs_in != 1 || cli_tp.data_segs_in != 2 ||
cli_tp.snd_cwnd != 10 || cli_tp.snd_cwnd != 10 ||
cli_tp.total_retrans || cli_tp.total_retrans ||
cli_tp.bytes_received != DATA_LEN, cli_tp.bytes_received != 2 * DATA_LEN,
"Unexpected cli_tp", "Check cli_tp output. egress_linum:%u", "Unexpected cli_tp", "Check cli_tp output. egress_linum:%u",
egress_linum); egress_linum);
} }
static void check_sk_pkt_out_cnt(int accept_fd, int cli_fd)
{
struct bpf_spinlock_cnt pkt_out_cnt = {}, pkt_out_cnt10 = {};
int err;
pkt_out_cnt.cnt = ~0;
pkt_out_cnt10.cnt = ~0;
err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &accept_fd, &pkt_out_cnt);
if (!err)
err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &accept_fd,
&pkt_out_cnt10);
/* The bpf prog only counts for fullsock and
* passive conneciton did not become fullsock until 3WHS
* had been finished.
* The bpf prog only counted two data packet out but we
* specially init accept_fd's pkt_out_cnt by 2 in
* init_sk_storage(). Hence, 4 here.
*/
CHECK(err || pkt_out_cnt.cnt != 4 || pkt_out_cnt10.cnt != 40,
"bpf_map_lookup_elem(sk_pkt_out_cnt, &accept_fd)",
"err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u",
err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt);
pkt_out_cnt.cnt = ~0;
pkt_out_cnt10.cnt = ~0;
err = bpf_map_lookup_elem(sk_pkt_out_cnt_fd, &cli_fd, &pkt_out_cnt);
if (!err)
err = bpf_map_lookup_elem(sk_pkt_out_cnt10_fd, &cli_fd,
&pkt_out_cnt10);
/* Active connection is fullsock from the beginning.
* 1 SYN and 1 ACK during 3WHS
* 2 Acks on data packet.
*
* The bpf_prog initialized it to 0xeB9F.
*/
CHECK(err || pkt_out_cnt.cnt != 0xeB9F + 4 ||
pkt_out_cnt10.cnt != 0xeB9F + 40,
"bpf_map_lookup_elem(sk_pkt_out_cnt, &cli_fd)",
"err:%d errno:%d pkt_out_cnt:%u pkt_out_cnt10:%u",
err, errno, pkt_out_cnt.cnt, pkt_out_cnt10.cnt);
}
static void init_sk_storage(int sk_fd, __u32 pkt_out_cnt)
{
struct bpf_spinlock_cnt scnt = {};
int err;
scnt.cnt = pkt_out_cnt;
err = bpf_map_update_elem(sk_pkt_out_cnt_fd, &sk_fd, &scnt,
BPF_NOEXIST);
CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt_fd)",
"err:%d errno:%d", err, errno);
scnt.cnt *= 10;
err = bpf_map_update_elem(sk_pkt_out_cnt10_fd, &sk_fd, &scnt,
BPF_NOEXIST);
CHECK(err, "bpf_map_update_elem(sk_pkt_out_cnt10_fd)",
"err:%d errno:%d", err, errno);
}
static void test(void) static void test(void)
{ {
int listen_fd, cli_fd, accept_fd, epfd, err; int listen_fd, cli_fd, accept_fd, epfd, err;
struct epoll_event ev; struct epoll_event ev;
socklen_t addrlen; socklen_t addrlen;
int i;
addrlen = sizeof(struct sockaddr_in6); addrlen = sizeof(struct sockaddr_in6);
ev.events = EPOLLIN; ev.events = EPOLLIN;
@ -308,17 +377,20 @@ static void test(void)
accept_fd, errno); accept_fd, errno);
close(listen_fd); close(listen_fd);
ev.data.fd = cli_fd;
err = epoll_ctl(epfd, EPOLL_CTL_ADD, cli_fd, &ev);
CHECK(err, "epoll_ctl(EPOLL_CTL_ADD, cli_fd)", "err:%d errno:%d",
err, errno);
init_sk_storage(accept_fd, 2);
for (i = 0; i < 2; i++) {
/* Send some data from accept_fd to cli_fd */ /* Send some data from accept_fd to cli_fd */
err = send(accept_fd, DATA, DATA_LEN, 0); err = send(accept_fd, DATA, DATA_LEN, 0);
CHECK(err != DATA_LEN, "send(accept_fd)", "err:%d errno:%d", CHECK(err != DATA_LEN, "send(accept_fd)", "err:%d errno:%d",
err, errno); err, errno);
/* Have some timeout in recv(cli_fd). Just in case. */ /* Have some timeout in recv(cli_fd). Just in case. */
ev.data.fd = cli_fd;
err = epoll_ctl(epfd, EPOLL_CTL_ADD, cli_fd, &ev);
CHECK(err, "epoll_ctl(EPOLL_CTL_ADD, cli_fd)", "err:%d errno:%d",
err, errno);
err = epoll_wait(epfd, &ev, 1, 1000); err = epoll_wait(epfd, &ev, 1, 1000);
CHECK(err != 1 || ev.data.fd != cli_fd, CHECK(err != 1 || ev.data.fd != cli_fd,
"epoll_wait(cli_fd)", "err:%d errno:%d ev.data.fd:%d cli_fd:%d", "epoll_wait(cli_fd)", "err:%d errno:%d ev.data.fd:%d cli_fd:%d",
@ -326,6 +398,9 @@ static void test(void)
err = recv(cli_fd, NULL, 0, MSG_TRUNC); err = recv(cli_fd, NULL, 0, MSG_TRUNC);
CHECK(err, "recv(cli_fd)", "err:%d errno:%d", err, errno); CHECK(err, "recv(cli_fd)", "err:%d errno:%d", err, errno);
}
check_sk_pkt_out_cnt(accept_fd, cli_fd);
close(epfd); close(epfd);
close(accept_fd); close(accept_fd);
@ -395,6 +470,14 @@ int main(int argc, char **argv)
CHECK(!map, "cannot find linum_map", "(null)"); CHECK(!map, "cannot find linum_map", "(null)");
linum_map_fd = bpf_map__fd(map); linum_map_fd = bpf_map__fd(map);
map = bpf_object__find_map_by_name(obj, "sk_pkt_out_cnt");
CHECK(!map, "cannot find sk_pkt_out_cnt", "(null)");
sk_pkt_out_cnt_fd = bpf_map__fd(map);
map = bpf_object__find_map_by_name(obj, "sk_pkt_out_cnt10");
CHECK(!map, "cannot find sk_pkt_out_cnt10", "(null)");
sk_pkt_out_cnt10_fd = bpf_map__fd(map);
test(); test();
bpf_object__close(obj); bpf_object__close(obj);

55
tools/testing/selftests/bpf/test_verifier.c
View File

@ -47,12 +47,13 @@
#include "bpf_rlimit.h" #include "bpf_rlimit.h"
#include "bpf_rand.h" #include "bpf_rand.h"
#include "bpf_util.h" #include "bpf_util.h"
#include "test_btf.h"
#include "../../../include/linux/filter.h" #include "../../../include/linux/filter.h"
#define MAX_INSNS BPF_MAXINSNS #define MAX_INSNS BPF_MAXINSNS
#define MAX_TEST_INSNS 1000000 #define MAX_TEST_INSNS 1000000
#define MAX_FIXUPS 8 #define MAX_FIXUPS 8
#define MAX_NR_MAPS 17 #define MAX_NR_MAPS 18
#define MAX_TEST_RUNS 8 #define MAX_TEST_RUNS 8
#define POINTER_VALUE 0xcafe4all #define POINTER_VALUE 0xcafe4all
#define TEST_DATA_LEN 64 #define TEST_DATA_LEN 64
@ -85,6 +86,7 @@ struct bpf_test {
int fixup_map_array_ro[MAX_FIXUPS]; int fixup_map_array_ro[MAX_FIXUPS];
int fixup_map_array_wo[MAX_FIXUPS]; int fixup_map_array_wo[MAX_FIXUPS];
int fixup_map_array_small[MAX_FIXUPS]; int fixup_map_array_small[MAX_FIXUPS];
int fixup_sk_storage_map[MAX_FIXUPS];
const char *errstr; const char *errstr;
const char *errstr_unpriv; const char *errstr_unpriv;
uint32_t retval, retval_unpriv, insn_processed; uint32_t retval, retval_unpriv, insn_processed;
@ -497,24 +499,6 @@ static int create_cgroup_storage(bool percpu)
return fd; return fd;
} }
#define BTF_INFO_ENC(kind, kind_flag, vlen) \
((!!(kind_flag) << 31) | ((kind) << 24) | ((vlen) & BTF_MAX_VLEN))
#define BTF_TYPE_ENC(name, info, size_or_type) \
(name), (info), (size_or_type)
#define BTF_INT_ENC(encoding, bits_offset, nr_bits) \
((encoding) << 24 | (bits_offset) << 16 | (nr_bits))
#define BTF_TYPE_INT_ENC(name, encoding, bits_offset, bits, sz) \
BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_INT, 0, 0), sz), \
BTF_INT_ENC(encoding, bits_offset, bits)
#define BTF_MEMBER_ENC(name, type, bits_offset) \
(name), (type), (bits_offset)
struct btf_raw_data {
__u32 raw_types[64];
const char *str_sec;
__u32 str_sec_size;
};
/* struct bpf_spin_lock { /* struct bpf_spin_lock {
* int val; * int val;
* }; * };
@ -589,6 +573,31 @@ static int create_map_spin_lock(void)
return fd; return fd;
} }
static int create_sk_storage_map(void)
{
struct bpf_create_map_attr attr = {
.name = "test_map",
.map_type = BPF_MAP_TYPE_SK_STORAGE,
.key_size = 4,
.value_size = 8,
.max_entries = 0,
.map_flags = BPF_F_NO_PREALLOC,
.btf_key_type_id = 1,
.btf_value_type_id = 3,
};
int fd, btf_fd;
btf_fd = load_btf();
if (btf_fd < 0)
return -1;
attr.btf_fd = btf_fd;
fd = bpf_create_map_xattr(&attr);
close(attr.btf_fd);
if (fd < 0)
printf("Failed to create sk_storage_map\n");
return fd;
}
static char bpf_vlog[UINT_MAX >> 8]; static char bpf_vlog[UINT_MAX >> 8];
static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type, static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type,
@ -611,6 +620,7 @@ static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type,
int *fixup_map_array_ro = test->fixup_map_array_ro; int *fixup_map_array_ro = test->fixup_map_array_ro;
int *fixup_map_array_wo = test->fixup_map_array_wo; int *fixup_map_array_wo = test->fixup_map_array_wo;
int *fixup_map_array_small = test->fixup_map_array_small; int *fixup_map_array_small = test->fixup_map_array_small;
int *fixup_sk_storage_map = test->fixup_sk_storage_map;
if (test->fill_helper) { if (test->fill_helper) {
test->fill_insns = calloc(MAX_TEST_INSNS, sizeof(struct bpf_insn)); test->fill_insns = calloc(MAX_TEST_INSNS, sizeof(struct bpf_insn));
@ -765,6 +775,13 @@ static void do_test_fixup(struct bpf_test *test, enum bpf_prog_type prog_type,
fixup_map_array_small++; fixup_map_array_small++;
} while (*fixup_map_array_small); } while (*fixup_map_array_small);
} }
if (*fixup_sk_storage_map) {
map_fds[17] = create_sk_storage_map();
do {
prog[*fixup_sk_storage_map].imm = map_fds[17];
fixup_sk_storage_map++;
} while (*fixup_sk_storage_map);
}
} }
static int set_admin(bool admin) static int set_admin(bool admin)

116
tools/testing/selftests/bpf/verifier/sock.c
View File

@ -382,3 +382,119 @@
.result = REJECT, .result = REJECT,
.errstr = "reference has not been acquired before", .errstr = "reference has not been acquired before",
}, },
{
"sk_storage_get(map, skb->sk, NULL, 0): value == NULL",
.insns = {
BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, offsetof(struct __sk_buff, sk)),
BPF_JMP_IMM(BPF_JNE, BPF_REG_1, 0, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
BPF_EMIT_CALL(BPF_FUNC_sk_fullsock),
BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
BPF_MOV64_IMM(BPF_REG_4, 0),
BPF_MOV64_IMM(BPF_REG_3, 0),
BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
BPF_LD_MAP_FD(BPF_REG_1, 0),
BPF_EMIT_CALL(BPF_FUNC_sk_storage_get),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.fixup_sk_storage_map = { 11 },
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
},
{
"sk_storage_get(map, skb->sk, 1, 1): value == 1",
.insns = {
BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, offsetof(struct __sk_buff, sk)),
BPF_JMP_IMM(BPF_JNE, BPF_REG_1, 0, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
BPF_EMIT_CALL(BPF_FUNC_sk_fullsock),
BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
BPF_MOV64_IMM(BPF_REG_4, 1),
BPF_MOV64_IMM(BPF_REG_3, 1),
BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
BPF_LD_MAP_FD(BPF_REG_1, 0),
BPF_EMIT_CALL(BPF_FUNC_sk_storage_get),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.fixup_sk_storage_map = { 11 },
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = REJECT,
.errstr = "R3 type=inv expected=fp",
},
{
"sk_storage_get(map, skb->sk, &stack_value, 1): stack_value",
.insns = {
BPF_MOV64_IMM(BPF_REG_2, 0),
BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_2, -8),
BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, offsetof(struct __sk_buff, sk)),
BPF_JMP_IMM(BPF_JNE, BPF_REG_1, 0, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
BPF_EMIT_CALL(BPF_FUNC_sk_fullsock),
BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
BPF_MOV64_IMM(BPF_REG_4, 1),
BPF_MOV64_REG(BPF_REG_3, BPF_REG_10),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, -8),
BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
BPF_LD_MAP_FD(BPF_REG_1, 0),
BPF_EMIT_CALL(BPF_FUNC_sk_storage_get),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.fixup_sk_storage_map = { 14 },
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = ACCEPT,
},
{
"sk_storage_get(map, skb->sk, &stack_value, 1): partially init stack_value",
.insns = {
BPF_MOV64_IMM(BPF_REG_2, 0),
BPF_STX_MEM(BPF_W, BPF_REG_10, BPF_REG_2, -8),
BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, offsetof(struct __sk_buff, sk)),
BPF_JMP_IMM(BPF_JNE, BPF_REG_1, 0, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
BPF_EMIT_CALL(BPF_FUNC_sk_fullsock),
BPF_JMP_IMM(BPF_JNE, BPF_REG_0, 0, 2),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
BPF_MOV64_IMM(BPF_REG_4, 1),
BPF_MOV64_REG(BPF_REG_3, BPF_REG_10),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_3, -8),
BPF_MOV64_REG(BPF_REG_2, BPF_REG_0),
BPF_LD_MAP_FD(BPF_REG_1, 0),
BPF_EMIT_CALL(BPF_FUNC_sk_storage_get),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.fixup_sk_storage_map = { 14 },
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = REJECT,
.errstr = "invalid indirect read from stack",
},
{
"bpf_map_lookup_elem(smap, &key)",
.insns = {
BPF_ST_MEM(BPF_W, BPF_REG_10, -4, 0),
BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),
BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4),
BPF_LD_MAP_FD(BPF_REG_1, 0),
BPF_EMIT_CALL(BPF_FUNC_map_lookup_elem),
BPF_MOV64_IMM(BPF_REG_0, 0),
BPF_EXIT_INSN(),
},
.fixup_sk_storage_map = { 3 },
.prog_type = BPF_PROG_TYPE_SCHED_CLS,
.result = REJECT,
.errstr = "cannot pass map_type 24 into func bpf_map_lookup_elem",
},