p71924506
/
linux_old1
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'bpf-offload-sharing' 

Jakub Kicinski says:

====================
This patchset adds support for sharing BPF objects within one ASIC.
This will allow us to reuse of the same program on multiple ports of
a device leading to better code store utilization.  It also enables
sharing maps between programs attached to different ports of a device.

v2:
 - rename bpf_offload_match() to bpf_offload_prog_map_match();
 - add split patches 7 into 5, 7 and 8.
====================

Acked-by: Alexei Starovoitov <ast@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
This commit is contained in:
Daniel Borkmann 2018-07-18 15:10:35 +02:00
parent c23e014a4b 7736b6ed66
commit 8ae71e76cf
14 changed files with 547 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
drivers/net/ethernet/netronome/nfp/bpf/main.c
View File

@ -404,6 +404,20 @@ static int nfp_bpf_parse_capabilities(struct nfp_app *app)
return -EINVAL;
}
static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev)
{
struct nfp_app_bpf *bpf = app->priv;
return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev);
}
static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev)
{
struct nfp_app_bpf *bpf = app->priv;
bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev);
}
static int nfp_bpf_init(struct nfp_app *app)
{
struct nfp_app_bpf *bpf;
@ -427,6 +441,11 @@ static int nfp_bpf_init(struct nfp_app *app)
if (err)
goto err_free_neutral_maps;
bpf->bpf_dev = bpf_offload_dev_create();
err = PTR_ERR_OR_ZERO(bpf->bpf_dev);
if (err)
goto err_free_neutral_maps;
return 0;
err_free_neutral_maps:
@ -445,6 +464,7 @@ static void nfp_bpf_clean(struct nfp_app *app)
{
struct nfp_app_bpf *bpf = app->priv;
bpf_offload_dev_destroy(bpf->bpf_dev);
WARN_ON(!skb_queue_empty(&bpf->cmsg_replies));
WARN_ON(!list_empty(&bpf->map_list));
WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
@ -466,6 +486,9 @@ const struct nfp_app_type app_bpf = {
.extra_cap = nfp_bpf_extra_cap,
.ndo_init = nfp_bpf_ndo_init,
.ndo_uninit = nfp_bpf_ndo_uninit,
.vnic_alloc = nfp_bpf_vnic_alloc,
.vnic_free = nfp_bpf_vnic_free,

4
drivers/net/ethernet/netronome/nfp/bpf/main.h
View File

@ -110,6 +110,8 @@ enum pkt_vec {
* struct nfp_app_bpf - bpf app priv structure
* @app: backpointer to the app
*
* @bpf_dev: BPF offload device handle
*
* @tag_allocator: bitmap of control message tags in use
* @tag_alloc_next: next tag bit to allocate
* @tag_alloc_last: next tag bit to be freed
@ -150,6 +152,8 @@ enum pkt_vec {
struct nfp_app_bpf {
struct nfp_app *app;
struct bpf_offload_dev *bpf_dev;
DECLARE_BITMAP(tag_allocator, U16_MAX + 1);
u16 tag_alloc_next;
u16 tag_alloc_last;

10
drivers/net/ethernet/netronome/nfp/bpf/offload.c
View File

@ -566,14 +566,8 @@ int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
{
int err;
if (prog) {
struct bpf_prog_offload *offload = prog->aux->offload;
if (!offload)
return -EINVAL;
if (offload->netdev != nn->dp.netdev)
return -EINVAL;
}
if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev))
return -EINVAL;
if (prog && old_prog) {
u8 cap;

17
drivers/net/ethernet/netronome/nfp/nfp_app.c
View File

@ -86,6 +86,23 @@ const char *nfp_app_mip_name(struct nfp_app *app)
return nfp_mip_name(app->pf->mip);
}
int nfp_app_ndo_init(struct net_device *netdev)
{
struct nfp_app *app = nfp_app_from_netdev(netdev);
if (!app || !app->type->ndo_init)
return 0;
return app->type->ndo_init(app, netdev);
}
void nfp_app_ndo_uninit(struct net_device *netdev)
{
struct nfp_app *app = nfp_app_from_netdev(netdev);
if (app && app->type->ndo_uninit)
app->type->ndo_uninit(app, netdev);
}
u64 *nfp_app_port_get_stats(struct nfp_port *port, u64 *data)
{
if (!port || !port->app || !port->app->type->port_get_stats)

8
drivers/net/ethernet/netronome/nfp/nfp_app.h
View File

@ -78,6 +78,8 @@ extern const struct nfp_app_type app_abm;
* @init: perform basic app checks and init
* @clean: clean app state
* @extra_cap: extra capabilities string
* @ndo_init: vNIC and repr netdev .ndo_init
* @ndo_uninit: vNIC and repr netdev .ndo_unint
* @vnic_alloc: allocate vNICs (assign port types, etc.)
* @vnic_free: free up app's vNIC state
* @vnic_init: vNIC netdev was registered
@ -117,6 +119,9 @@ struct nfp_app_type {
const char *(*extra_cap)(struct nfp_app *app, struct nfp_net *nn);
int (*ndo_init)(struct nfp_app *app, struct net_device *netdev);
void (*ndo_uninit)(struct nfp_app *app, struct net_device *netdev);
int (*vnic_alloc)(struct nfp_app *app, struct nfp_net *nn,
unsigned int id);
void (*vnic_free)(struct nfp_app *app, struct nfp_net *nn);
@ -200,6 +205,9 @@ static inline void nfp_app_clean(struct nfp_app *app)
app->type->clean(app);
}
int nfp_app_ndo_init(struct net_device *netdev);
void nfp_app_ndo_uninit(struct net_device *netdev);
static inline int nfp_app_vnic_alloc(struct nfp_app *app, struct nfp_net *nn,
unsigned int id)
{

2
drivers/net/ethernet/netronome/nfp/nfp_net_common.c
View File

@ -3480,6 +3480,8 @@ static int nfp_net_set_mac_address(struct net_device *netdev, void *addr)
}
const struct net_device_ops nfp_net_netdev_ops = {
.ndo_init = nfp_app_ndo_init,
.ndo_uninit = nfp_app_ndo_uninit,
.ndo_open = nfp_net_netdev_open,
.ndo_stop = nfp_net_netdev_close,
.ndo_start_xmit = nfp_net_tx,

2
drivers/net/ethernet/netronome/nfp/nfp_net_repr.c
View File

@ -262,6 +262,8 @@ static int nfp_repr_open(struct net_device *netdev)
}
const struct net_device_ops nfp_repr_netdev_ops = {
.ndo_init = nfp_app_ndo_init,
.ndo_uninit = nfp_app_ndo_uninit,
.ndo_open = nfp_repr_open,
.ndo_stop = nfp_repr_stop,
.ndo_start_xmit = nfp_repr_xmit,

50
drivers/net/netdevsim/bpf.c
View File

@ -238,8 +238,8 @@ static int nsim_bpf_create_prog(struct netdevsim *ns, struct bpf_prog *prog)
state->state = "verify";
/* Program id is not populated yet when we create the state. */
sprintf(name, "%u", ns->prog_id_gen++);
state->ddir = debugfs_create_dir(name, ns->ddir_bpf_bound_progs);
sprintf(name, "%u", ns->sdev->prog_id_gen++);
state->ddir = debugfs_create_dir(name, ns->sdev->ddir_bpf_bound_progs);
if (IS_ERR_OR_NULL(state->ddir)) {
kfree(state);
return -ENOMEM;
@ -250,7 +250,7 @@ static int nsim_bpf_create_prog(struct netdevsim *ns, struct bpf_prog *prog)
&state->state, &nsim_bpf_string_fops);
debugfs_create_bool("loaded", 0400, state->ddir, &state->is_loaded);
list_add_tail(&state->l, &ns->bpf_bound_progs);
list_add_tail(&state->l, &ns->sdev->bpf_bound_progs);
prog->aux->offload->dev_priv = state;
@ -294,7 +294,7 @@ nsim_setup_prog_hw_checks(struct netdevsim *ns, struct netdev_bpf *bpf)
NSIM_EA(bpf->extack, "xdpoffload of non-bound program");
return -EINVAL;
}
if (bpf->prog->aux->offload->netdev != ns->netdev) {
if (!bpf_offload_dev_match(bpf->prog, ns->netdev)) {
NSIM_EA(bpf->extack, "program bound to different dev");
return -EINVAL;
}
@ -497,7 +497,7 @@ nsim_bpf_map_alloc(struct netdevsim *ns, struct bpf_offloaded_map *offmap)
}
offmap->dev_ops = &nsim_bpf_map_ops;
list_add_tail(&nmap->l, &ns->bpf_bound_maps);
list_add_tail(&nmap->l, &ns->sdev->bpf_bound_maps);
return 0;
@ -582,8 +582,26 @@ int nsim_bpf(struct net_device *dev, struct netdev_bpf *bpf)
int nsim_bpf_init(struct netdevsim *ns)
{
INIT_LIST_HEAD(&ns->bpf_bound_progs);
INIT_LIST_HEAD(&ns->bpf_bound_maps);
int err;
if (ns->sdev->refcnt == 1) {
INIT_LIST_HEAD(&ns->sdev->bpf_bound_progs);
INIT_LIST_HEAD(&ns->sdev->bpf_bound_maps);
ns->sdev->ddir_bpf_bound_progs =
debugfs_create_dir("bpf_bound_progs", ns->sdev->ddir);
if (IS_ERR_OR_NULL(ns->sdev->ddir_bpf_bound_progs))
return -ENOMEM;
ns->sdev->bpf_dev = bpf_offload_dev_create();
err = PTR_ERR_OR_ZERO(ns->sdev->bpf_dev);
if (err)
return err;
}
err = bpf_offload_dev_netdev_register(ns->sdev->bpf_dev, ns->netdev);
if (err)
goto err_destroy_bdev;
debugfs_create_u32("bpf_offloaded_id", 0400, ns->ddir,
&ns->bpf_offloaded_id);
@ -593,10 +611,6 @@ int nsim_bpf_init(struct netdevsim *ns)
&ns->bpf_bind_accept);
debugfs_create_u32("bpf_bind_verifier_delay", 0600, ns->ddir,
&ns->bpf_bind_verifier_delay);
ns->ddir_bpf_bound_progs =
debugfs_create_dir("bpf_bound_progs", ns->ddir);
if (IS_ERR_OR_NULL(ns->ddir_bpf_bound_progs))
return -ENOMEM;
ns->bpf_tc_accept = true;
debugfs_create_bool("bpf_tc_accept", 0600, ns->ddir,
@ -615,13 +629,23 @@ int nsim_bpf_init(struct netdevsim *ns)
&ns->bpf_map_accept);
return 0;
err_destroy_bdev:
if (ns->sdev->refcnt == 1)
bpf_offload_dev_destroy(ns->sdev->bpf_dev);
return err;
}
void nsim_bpf_uninit(struct netdevsim *ns)
{
WARN_ON(!list_empty(&ns->bpf_bound_progs));
WARN_ON(!list_empty(&ns->bpf_bound_maps));
WARN_ON(ns->xdp.prog);
WARN_ON(ns->xdp_hw.prog);
WARN_ON(ns->bpf_offloaded);
bpf_offload_dev_netdev_unregister(ns->sdev->bpf_dev, ns->netdev);
if (ns->sdev->refcnt == 1) {
WARN_ON(!list_empty(&ns->sdev->bpf_bound_progs));
WARN_ON(!list_empty(&ns->sdev->bpf_bound_maps));
bpf_offload_dev_destroy(ns->sdev->bpf_dev);
}
}

103
drivers/net/netdevsim/netdev.c
View File

@ -22,6 +22,7 @@
#include <net/netlink.h>
#include <net/pkt_cls.h>
#include <net/rtnetlink.h>
#include <net/switchdev.h>
#include "netdevsim.h"
@ -144,8 +145,29 @@ static struct device_type nsim_dev_type = {
.release = nsim_dev_release,
};
static int
nsim_port_attr_get(struct net_device *dev, struct switchdev_attr *attr)
{
struct netdevsim *ns = netdev_priv(dev);
switch (attr->id) {
case SWITCHDEV_ATTR_ID_PORT_PARENT_ID:
attr->u.ppid.id_len = sizeof(ns->sdev->switch_id);
memcpy(&attr->u.ppid.id, &ns->sdev->switch_id,
attr->u.ppid.id_len);
return 0;
default:
return -EOPNOTSUPP;
}
}
static const struct switchdev_ops nsim_switchdev_ops = {
.switchdev_port_attr_get = nsim_port_attr_get,
};
static int nsim_init(struct net_device *dev)
{
char sdev_ddir_name[10], sdev_link_name[32];
struct netdevsim *ns = netdev_priv(dev);
int err;
@ -154,9 +176,32 @@ static int nsim_init(struct net_device *dev)
if (IS_ERR_OR_NULL(ns->ddir))
return -ENOMEM;
if (!ns->sdev) {
ns->sdev = kzalloc(sizeof(*ns->sdev), GFP_KERNEL);
if (!ns->sdev) {
err = -ENOMEM;
goto err_debugfs_destroy;
}
ns->sdev->refcnt = 1;
ns->sdev->switch_id = nsim_dev_id;
sprintf(sdev_ddir_name, "%u", ns->sdev->switch_id);
ns->sdev->ddir = debugfs_create_dir(sdev_ddir_name,
nsim_sdev_ddir);
if (IS_ERR_OR_NULL(ns->sdev->ddir)) {
err = PTR_ERR_OR_ZERO(ns->sdev->ddir) ?: -EINVAL;
goto err_sdev_free;
}
} else {
sprintf(sdev_ddir_name, "%u", ns->sdev->switch_id);
ns->sdev->refcnt++;
}
sprintf(sdev_link_name, "../../" DRV_NAME "_sdev/%s", sdev_ddir_name);
debugfs_create_symlink("sdev", ns->ddir, sdev_link_name);
err = nsim_bpf_init(ns);
if (err)
goto err_debugfs_destroy;
goto err_sdev_destroy;
ns->dev.id = nsim_dev_id++;
ns->dev.bus = &nsim_bus;
@ -166,6 +211,7 @@ static int nsim_init(struct net_device *dev)
goto err_bpf_uninit;
SET_NETDEV_DEV(dev, &ns->dev);
SWITCHDEV_SET_OPS(dev, &nsim_switchdev_ops);
err = nsim_devlink_setup(ns);
if (err)
@ -179,6 +225,12 @@ static int nsim_init(struct net_device *dev)
device_unregister(&ns->dev);
err_bpf_uninit:
nsim_bpf_uninit(ns);
err_sdev_destroy:
if (!--ns->sdev->refcnt) {
debugfs_remove_recursive(ns->sdev->ddir);
err_sdev_free:
kfree(ns->sdev);
}
err_debugfs_destroy:
debugfs_remove_recursive(ns->ddir);
return err;
@ -192,6 +244,10 @@ static void nsim_uninit(struct net_device *dev)
nsim_devlink_teardown(ns);
debugfs_remove_recursive(ns->ddir);
nsim_bpf_uninit(ns);
if (!--ns->sdev->refcnt) {
debugfs_remove_recursive(ns->sdev->ddir);
kfree(ns->sdev);
}
}
static void nsim_free(struct net_device *dev)
@ -470,14 +526,48 @@ static int nsim_validate(struct nlattr *tb[], struct nlattr *data[],
return 0;
}
static int nsim_newlink(struct net *src_net, struct net_device *dev,
struct nlattr *tb[], struct nlattr *data[],
struct netlink_ext_ack *extack)
{
struct netdevsim *ns = netdev_priv(dev);
if (tb[IFLA_LINK]) {
struct net_device *joindev;
struct netdevsim *joinns;
joindev = __dev_get_by_index(src_net,
nla_get_u32(tb[IFLA_LINK]));
if (!joindev)
return -ENODEV;
if (joindev->netdev_ops != &nsim_netdev_ops)
return -EINVAL;
joinns = netdev_priv(joindev);
if (!joinns->sdev || !joinns->sdev->refcnt)
return -EINVAL;
ns->sdev = joinns->sdev;
}
return register_netdevice(dev);
}
static void nsim_dellink(struct net_device *dev, struct list_head *head)
{
unregister_netdevice_queue(dev, head);
}
static struct rtnl_link_ops nsim_link_ops __read_mostly = {
.kind = DRV_NAME,
.priv_size = sizeof(struct netdevsim),
.setup = nsim_setup,
.validate = nsim_validate,
.newlink = nsim_newlink,
.dellink = nsim_dellink,
};
struct dentry *nsim_ddir;
struct dentry *nsim_sdev_ddir;
static int __init nsim_module_init(void)
{
@ -487,9 +577,15 @@ static int __init nsim_module_init(void)
if (IS_ERR_OR_NULL(nsim_ddir))
return -ENOMEM;
nsim_sdev_ddir = debugfs_create_dir(DRV_NAME "_sdev", NULL);
if (IS_ERR_OR_NULL(nsim_sdev_ddir)) {
err = -ENOMEM;
goto err_debugfs_destroy;
}
err = bus_register(&nsim_bus);
if (err)
goto err_debugfs_destroy;
goto err_sdir_destroy;
err = nsim_devlink_init();
if (err)
@ -505,6 +601,8 @@ static int __init nsim_module_init(void)
nsim_devlink_exit();
err_unreg_bus:
bus_unregister(&nsim_bus);
err_sdir_destroy:
debugfs_remove_recursive(nsim_sdev_ddir);
err_debugfs_destroy:
debugfs_remove_recursive(nsim_ddir);
return err;
@ -515,6 +613,7 @@ static void __exit nsim_module_exit(void)
rtnl_link_unregister(&nsim_link_ops);
nsim_devlink_exit();
bus_unregister(&nsim_bus);
debugfs_remove_recursive(nsim_sdev_ddir);
debugfs_remove_recursive(nsim_ddir);
}

23
drivers/net/netdevsim/netdevsim.h
View File

@ -27,9 +27,25 @@
#define NSIM_EA(extack, msg) NL_SET_ERR_MSG_MOD((extack), msg)
struct bpf_prog;
struct bpf_offload_dev;
struct dentry;
struct nsim_vf_config;
struct netdevsim_shared_dev {
unsigned int refcnt;
u32 switch_id;
struct dentry *ddir;
struct bpf_offload_dev *bpf_dev;
struct dentry *ddir_bpf_bound_progs;
u32 prog_id_gen;
struct list_head bpf_bound_progs;
struct list_head bpf_bound_maps;
};
#define NSIM_IPSEC_MAX_SA_COUNT 33
#define NSIM_IPSEC_VALID BIT(31)
@ -59,6 +75,7 @@ struct netdevsim {
struct u64_stats_sync syncp;
struct device dev;
struct netdevsim_shared_dev *sdev;
struct dentry *ddir;
@ -71,12 +88,8 @@ struct netdevsim {
struct xdp_attachment_info xdp;
struct xdp_attachment_info xdp_hw;
u32 prog_id_gen;
bool bpf_bind_accept;
u32 bpf_bind_verifier_delay;
struct dentry *ddir_bpf_bound_progs;
struct list_head bpf_bound_progs;
bool bpf_tc_accept;
bool bpf_tc_non_bound_accept;
@ -84,7 +97,6 @@ struct netdevsim {
bool bpf_xdpoffload_accept;
bool bpf_map_accept;
struct list_head bpf_bound_maps;
#if IS_ENABLED(CONFIG_NET_DEVLINK)
struct devlink *devlink;
#endif
@ -92,6 +104,7 @@ struct netdevsim {
};
extern struct dentry *nsim_ddir;
extern struct dentry *nsim_sdev_ddir;
#ifdef CONFIG_BPF_SYSCALL
int nsim_bpf_init(struct netdevsim *ns);

11
include/linux/bpf.h
View File

@ -85,6 +85,7 @@ struct bpf_map {
char name[BPF_OBJ_NAME_LEN];
};
struct bpf_offload_dev;
struct bpf_offloaded_map;
struct bpf_map_dev_ops {
@ -648,7 +649,15 @@ int bpf_map_offload_delete_elem(struct bpf_map *map, void *key);
int bpf_map_offload_get_next_key(struct bpf_map *map,
void *key, void *next_key);
bool bpf_offload_dev_match(struct bpf_prog *prog, struct bpf_map *map);
bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map);
struct bpf_offload_dev *bpf_offload_dev_create(void);
void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev);
int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
struct net_device *netdev);
void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
struct net_device *netdev);
bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev);
#if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL)
int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr);

229
kernel/bpf/offload.c
View File

@ -18,19 +18,43 @@
#include <linux/bug.h>
#include <linux/kdev_t.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/netdevice.h>
#include <linux/printk.h>
#include <linux/proc_ns.h>
#include <linux/rhashtable.h>
#include <linux/rtnetlink.h>
#include <linux/rwsem.h>
/* Protects bpf_prog_offload_devs, bpf_map_offload_devs and offload members
/* Protects offdevs, members of bpf_offload_netdev and offload members
* of all progs.
* RTNL lock cannot be taken when holding this lock.
*/
static DECLARE_RWSEM(bpf_devs_lock);
static LIST_HEAD(bpf_prog_offload_devs);
static LIST_HEAD(bpf_map_offload_devs);
struct bpf_offload_dev {
struct list_head netdevs;
};
struct bpf_offload_netdev {
struct rhash_head l;
struct net_device *netdev;
struct bpf_offload_dev *offdev;
struct list_head progs;
struct list_head maps;
struct list_head offdev_netdevs;
};
static const struct rhashtable_params offdevs_params = {
.nelem_hint = 4,
.key_len = sizeof(struct net_device *),
.key_offset = offsetof(struct bpf_offload_netdev, netdev),
.head_offset = offsetof(struct bpf_offload_netdev, l),
.automatic_shrinking = true,
};
static struct rhashtable offdevs;
static bool offdevs_inited;
static int bpf_dev_offload_check(struct net_device *netdev)
{
@ -41,8 +65,19 @@ static int bpf_dev_offload_check(struct net_device *netdev)
return 0;
}
static struct bpf_offload_netdev *
bpf_offload_find_netdev(struct net_device *netdev)
{
lockdep_assert_held(&bpf_devs_lock);
if (!offdevs_inited)
return NULL;
return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
}
int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr)
{
struct bpf_offload_netdev *ondev;
struct bpf_prog_offload *offload;
int err;
@ -66,12 +101,13 @@ int bpf_prog_offload_init(struct bpf_prog *prog, union bpf_attr *attr)
goto err_maybe_put;
down_write(&bpf_devs_lock);
if (offload->netdev->reg_state != NETREG_REGISTERED) {
ondev = bpf_offload_find_netdev(offload->netdev);
if (!ondev) {
err = -EINVAL;
goto err_unlock;
}
prog->aux->offload = offload;
list_add_tail(&offload->offloads, &bpf_prog_offload_devs);
list_add_tail(&offload->offloads, &ondev->progs);
dev_put(offload->netdev);
up_write(&bpf_devs_lock);
@ -294,6 +330,7 @@ static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap,
struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
{
struct net *net = current->nsproxy->net_ns;
struct bpf_offload_netdev *ondev;
struct bpf_offloaded_map *offmap;
int err;
@ -316,11 +353,17 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
if (err)
goto err_unlock;
ondev = bpf_offload_find_netdev(offmap->netdev);
if (!ondev) {
err = -EINVAL;
goto err_unlock;
}
err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC);
if (err)
goto err_unlock;
list_add_tail(&offmap->offloads, &bpf_map_offload_devs);
list_add_tail(&offmap->offloads, &ondev->maps);
up_write(&bpf_devs_lock);
rtnl_unlock();
@ -468,77 +511,159 @@ int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map)
return 0;
}
bool bpf_offload_dev_match(struct bpf_prog *prog, struct bpf_map *map)
static bool __bpf_offload_dev_match(struct bpf_prog *prog,
struct net_device *netdev)
{
struct bpf_offloaded_map *offmap;
struct bpf_offload_netdev *ondev1, *ondev2;
struct bpf_prog_offload *offload;
bool ret;
if (!bpf_prog_is_dev_bound(prog->aux))
return false;
if (!bpf_map_is_dev_bound(map))
return bpf_map_offload_neutral(map);
offload = prog->aux->offload;
if (!offload)
return false;
if (offload->netdev == netdev)
return true;
ondev1 = bpf_offload_find_netdev(offload->netdev);
ondev2 = bpf_offload_find_netdev(netdev);
return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev;
}
bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev)
{
bool ret;
down_read(&bpf_devs_lock);
offload = prog->aux->offload;
ret = __bpf_offload_dev_match(prog, netdev);
up_read(&bpf_devs_lock);
return ret;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_match);
bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map)
{
struct bpf_offloaded_map *offmap;
bool ret;
if (!bpf_map_is_dev_bound(map))
return bpf_map_offload_neutral(map);
offmap = map_to_offmap(map);
ret = offload && offload->netdev == offmap->netdev;
down_read(&bpf_devs_lock);
ret = __bpf_offload_dev_match(prog, offmap->netdev);
up_read(&bpf_devs_lock);
return ret;
}
static void bpf_offload_orphan_all_progs(struct net_device *netdev)
int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
struct net_device *netdev)
{
struct bpf_prog_offload *offload, *tmp;
struct bpf_offload_netdev *ondev;
int err;
list_for_each_entry_safe(offload, tmp, &bpf_prog_offload_devs, offloads)
if (offload->netdev == netdev)
__bpf_prog_offload_destroy(offload->prog);
ondev = kzalloc(sizeof(*ondev), GFP_KERNEL);
if (!ondev)
return -ENOMEM;
ondev->netdev = netdev;
ondev->offdev = offdev;
INIT_LIST_HEAD(&ondev->progs);
INIT_LIST_HEAD(&ondev->maps);
down_write(&bpf_devs_lock);
err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params);
if (err) {
netdev_warn(netdev, "failed to register for BPF offload\n");
goto err_unlock_free;
}
list_add(&ondev->offdev_netdevs, &offdev->netdevs);
up_write(&bpf_devs_lock);
return 0;
err_unlock_free:
up_write(&bpf_devs_lock);
kfree(ondev);
return err;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register);
static void bpf_offload_orphan_all_maps(struct net_device *netdev)
void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
struct net_device *netdev)
{
struct bpf_offloaded_map *offmap, *tmp;
list_for_each_entry_safe(offmap, tmp, &bpf_map_offload_devs, offloads)
if (offmap->netdev == netdev)
__bpf_map_offload_destroy(offmap);
}
static int bpf_offload_notification(struct notifier_block *notifier,
ulong event, void *ptr)
{
struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
struct bpf_offload_netdev *ondev, *altdev;
struct bpf_offloaded_map *offmap, *mtmp;
struct bpf_prog_offload *offload, *ptmp;
ASSERT_RTNL();
switch (event) {
case NETDEV_UNREGISTER:
/* ignore namespace changes */
if (netdev->reg_state != NETREG_UNREGISTERING)
break;
down_write(&bpf_devs_lock);
ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
if (WARN_ON(!ondev))
goto unlock;
down_write(&bpf_devs_lock);
bpf_offload_orphan_all_progs(netdev);
bpf_offload_orphan_all_maps(netdev);
up_write(&bpf_devs_lock);
break;
default:
break;
WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params));
list_del(&ondev->offdev_netdevs);
/* Try to move the objects to another netdev of the device */
altdev = list_first_entry_or_null(&offdev->netdevs,
struct bpf_offload_netdev,
offdev_netdevs);
if (altdev) {
list_for_each_entry(offload, &ondev->progs, offloads)
offload->netdev = altdev->netdev;
list_splice_init(&ondev->progs, &altdev->progs);
list_for_each_entry(offmap, &ondev->maps, offloads)
offmap->netdev = altdev->netdev;
list_splice_init(&ondev->maps, &altdev->maps);
} else {
list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads)
__bpf_prog_offload_destroy(offload->prog);
list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads)
__bpf_map_offload_destroy(offmap);
}
return NOTIFY_OK;
WARN_ON(!list_empty(&ondev->progs));
WARN_ON(!list_empty(&ondev->maps));
kfree(ondev);
unlock:
up_write(&bpf_devs_lock);
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister);
static struct notifier_block bpf_offload_notifier = {
.notifier_call = bpf_offload_notification,
};
static int __init bpf_offload_init(void)
struct bpf_offload_dev *bpf_offload_dev_create(void)
{
register_netdevice_notifier(&bpf_offload_notifier);
return 0;
}
struct bpf_offload_dev *offdev;
int err;
subsys_initcall(bpf_offload_init);
down_write(&bpf_devs_lock);
if (!offdevs_inited) {
err = rhashtable_init(&offdevs, &offdevs_params);
if (err)
return ERR_PTR(err);
offdevs_inited = true;
}
up_write(&bpf_devs_lock);
offdev = kzalloc(sizeof(*offdev), GFP_KERNEL);
if (!offdev)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD(&offdev->netdevs);
return offdev;
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_create);
void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev)
{
WARN_ON(!list_empty(&offdev->netdevs));
kfree(offdev);
}
EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy);

2
kernel/bpf/verifier.c
View File

@ -5054,7 +5054,7 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env,
}
if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
!bpf_offload_dev_match(prog, map)) {
!bpf_offload_prog_map_match(prog, map)) {
verbose(env, "offload device mismatch between prog and map\n");
return -EINVAL;
}

151
tools/testing/selftests/bpf/test_offload.py
View File

@ -158,8 +158,9 @@ def tool(name, args, flags, JSON=True, ns="", fail=True, include_stderr=False):
else:
return ret, out
def bpftool(args, JSON=True, ns="", fail=True):
return tool("bpftool", args, {"json":"-p"}, JSON=JSON, ns=ns, fail=fail)
def bpftool(args, JSON=True, ns="", fail=True, include_stderr=False):
return tool("bpftool", args, {"json":"-p"}, JSON=JSON, ns=ns,
fail=fail, include_stderr=include_stderr)
def bpftool_prog_list(expected=None, ns=""):
_, progs = bpftool("prog show", JSON=True, ns=ns, fail=True)
@ -201,6 +202,21 @@ def bpftool_map_list_wait(expected=0, n_retry=20):
time.sleep(0.05)
raise Exception("Time out waiting for map counts to stabilize want %d, have %d" % (expected, nmaps))
def bpftool_prog_load(sample, file_name, maps=[], prog_type="xdp", dev=None,
fail=True, include_stderr=False):
args = "prog load %s %s" % (os.path.join(bpf_test_dir, sample), file_name)
if prog_type is not None:
args += " type " + prog_type
if dev is not None:
args += " dev " + dev
if len(maps):
args += " map " + " map ".join(maps)
res = bpftool(args, fail=fail, include_stderr=include_stderr)
if res[0] == 0:
files.append(file_name)
return res
def ip(args, force=False, JSON=True, ns="", fail=True, include_stderr=False):
if force:
args = "-force " + args
@ -307,21 +323,25 @@ class NetdevSim:
Class for netdevsim netdevice and its attributes.
"""
def __init__(self):
def __init__(self, link=None):
self.link = link
self.dev = self._netdevsim_create()
devs.append(self)
self.ns = ""
self.dfs_dir = '/sys/kernel/debug/netdevsim/%s' % (self.dev['ifname'])
self.sdev_dir = self.dfs_dir + '/sdev/'
self.dfs_refresh()
def __getitem__(self, key):
return self.dev[key]
def _netdevsim_create(self):
link = "" if self.link is None else "link " + self.link.dev['ifname']
_, old = ip("link show")
ip("link add sim%d type netdevsim")
ip("link add sim%d {link} type netdevsim".format(link=link))
_, new = ip("link show")
for dev in new:
@ -345,12 +365,12 @@ class NetdevSim:
return data.strip()
def dfs_num_bound_progs(self):
path = os.path.join(self.dfs_dir, "bpf_bound_progs")
path = os.path.join(self.sdev_dir, "bpf_bound_progs")
_, progs = cmd('ls %s' % (path))
return len(progs.split())
def dfs_get_bound_progs(self, expected):
progs = DebugfsDir(os.path.join(self.dfs_dir, "bpf_bound_progs"))
progs = DebugfsDir(os.path.join(self.sdev_dir, "bpf_bound_progs"))
if expected is not None:
if len(progs) != expected:
fail(True, "%d BPF programs bound, expected %d" %
@ -847,6 +867,25 @@ try:
sim.set_mtu(1500)
sim.wait_for_flush()
start_test("Test non-offload XDP attaching to HW...")
bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/nooffload")
nooffload = bpf_pinned("/sys/fs/bpf/nooffload")
ret, _, err = sim.set_xdp(nooffload, "offload",
fail=False, include_stderr=True)
fail(ret == 0, "attached non-offloaded XDP program to HW")
check_extack_nsim(err, "xdpoffload of non-bound program.", args)
rm("/sys/fs/bpf/nooffload")
start_test("Test offload XDP attaching to drv...")
bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload",
dev=sim['ifname'])
offload = bpf_pinned("/sys/fs/bpf/offload")
ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True)
fail(ret == 0, "attached offloaded XDP program to drv")
check_extack(err, "using device-bound program without HW_MODE flag is not supported.", args)
rm("/sys/fs/bpf/offload")
sim.wait_for_flush()
start_test("Test XDP offload...")
_, _, err = sim.set_xdp(obj, "offload", verbose=True, include_stderr=True)
ipl = sim.ip_link_show(xdp=True)
@ -1140,6 +1179,106 @@ try:
fail(ret == 0,
"netdevsim didn't refuse to create a map with offload disabled")
sim.remove()
start_test("Test multi-dev ASIC program reuse...")
simA = NetdevSim()
simB1 = NetdevSim()
simB2 = NetdevSim(link=simB1)
simB3 = NetdevSim(link=simB1)
sims = (simA, simB1, simB2, simB3)
simB = (simB1, simB2, simB3)
bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA",
dev=simA['ifname'])
progA = bpf_pinned("/sys/fs/bpf/nsimA")
bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB",
dev=simB1['ifname'])
progB = bpf_pinned("/sys/fs/bpf/nsimB")
simA.set_xdp(progA, "offload", JSON=False)
for d in simB:
d.set_xdp(progB, "offload", JSON=False)
start_test("Test multi-dev ASIC cross-dev replace...")
ret, _ = simA.set_xdp(progB, "offload", force=True, JSON=False, fail=False)
fail(ret == 0, "cross-ASIC program allowed")
for d in simB:
ret, _ = d.set_xdp(progA, "offload", force=True, JSON=False, fail=False)
fail(ret == 0, "cross-ASIC program allowed")
start_test("Test multi-dev ASIC cross-dev install...")
for d in sims:
d.unset_xdp("offload")
ret, _, err = simA.set_xdp(progB, "offload", force=True, JSON=False,
fail=False, include_stderr=True)
fail(ret == 0, "cross-ASIC program allowed")
check_extack_nsim(err, "program bound to different dev.", args)
for d in simB:
ret, _, err = d.set_xdp(progA, "offload", force=True, JSON=False,
fail=False, include_stderr=True)
fail(ret == 0, "cross-ASIC program allowed")
check_extack_nsim(err, "program bound to different dev.", args)
start_test("Test multi-dev ASIC cross-dev map reuse...")
mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0]
mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0]
ret, _ = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
dev=simB3['ifname'],
maps=["idx 0 id %d" % (mapB)],
fail=False)
fail(ret != 0, "couldn't reuse a map on the same ASIC")
rm("/sys/fs/bpf/nsimB_")
ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA_",
dev=simA['ifname'],
maps=["idx 0 id %d" % (mapB)],
fail=False, include_stderr=True)
fail(ret == 0, "could reuse a map on a different ASIC")
fail(err.count("offload device mismatch between prog and map") == 0,
"error message missing for cross-ASIC map")
ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
dev=simB1['ifname'],
maps=["idx 0 id %d" % (mapA)],
fail=False, include_stderr=True)
fail(ret == 0, "could reuse a map on a different ASIC")
fail(err.count("offload device mismatch between prog and map") == 0,
"error message missing for cross-ASIC map")
start_test("Test multi-dev ASIC cross-dev destruction...")
bpftool_prog_list_wait(expected=2)
simA.remove()
bpftool_prog_list_wait(expected=1)
ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
fail(ifnameB != simB1['ifname'], "program not bound to originial device")
simB1.remove()
bpftool_prog_list_wait(expected=1)
start_test("Test multi-dev ASIC cross-dev destruction - move...")
ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
fail(ifnameB not in (simB2['ifname'], simB3['ifname']),
"program not bound to remaining devices")
simB2.remove()
ifnameB = bpftool("prog show %s" % (progB))[1]["dev"]["ifname"]
fail(ifnameB != simB3['ifname'], "program not bound to remaining device")
simB3.remove()
bpftool_prog_list_wait(expected=0)
start_test("Test multi-dev ASIC cross-dev destruction - orphaned...")
ret, out = bpftool("prog show %s" % (progB), fail=False)
fail(ret == 0, "got information about orphaned program")
fail("error" not in out, "no error reported for get info on orphaned")
fail(out["error"] != "can't get prog info: No such device",
"wrong error for get info on orphaned")
print("%s: OK" % (os.path.basename(__file__)))
finally: