From b592843c6723a850be70bf9618578082f3b73851 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 7 Nov 2018 17:33:34 -0800 Subject: [PATCH 1/7] net: sched: add an offload dump helper Qdisc dump operation of offload-capable qdiscs performs a few extra steps which are identical among all the qdiscs. Add a helper to share this code. Signed-off-by: Jakub Kicinski Reviewed-by: John Hurley Signed-off-by: David S. Miller --- include/net/sch_generic.h | 12 ++++++++++++ net/sched/sch_api.c | 21 +++++++++++++++++++++ net/sched/sch_prio.c | 16 +--------------- net/sched/sch_red.c | 17 +---------------- 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 4d736427a4cb..af55c1c4edb1 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -579,6 +579,18 @@ void qdisc_put(struct Qdisc *qdisc); void qdisc_put_unlocked(struct Qdisc *qdisc); void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, unsigned int n, unsigned int len); +#ifdef CONFIG_NET_SCHED +int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, + void *type_data); +#else +static inline int +qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, + void *type_data) +{ + q->flags &= ~TCQ_F_OFFLOADED; + return 0; +} +#endif struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, struct netlink_ext_ack *extack); diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index ca3b0f46de53..e534825d3d3a 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -810,6 +810,27 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, unsigned int n, } EXPORT_SYMBOL(qdisc_tree_reduce_backlog); +int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type, + void *type_data) +{ + struct net_device *dev = qdisc_dev(sch); + int err; + + sch->flags &= ~TCQ_F_OFFLOADED; + if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) + return 0; + + err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data); + if (err == -EOPNOTSUPP) + return 0; + + if (!err) + sch->flags |= TCQ_F_OFFLOADED; + + return err; +} +EXPORT_SYMBOL(qdisc_offload_dump_helper); + static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, u32 portid, u32 seq, u16 flags, int event) { diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index f8af98621179..4bdd04c30ead 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -251,7 +251,6 @@ static int prio_init(struct Qdisc *sch, struct nlattr *opt, static int prio_dump_offload(struct Qdisc *sch) { - struct net_device *dev = qdisc_dev(sch); struct tc_prio_qopt_offload hw_stats = { .command = TC_PRIO_STATS, .handle = sch->handle, @@ -263,21 +262,8 @@ static int prio_dump_offload(struct Qdisc *sch) }, }, }; - int err; - sch->flags &= ~TCQ_F_OFFLOADED; - if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) - return 0; - - err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_PRIO, - &hw_stats); - if (err == -EOPNOTSUPP) - return 0; - - if (!err) - sch->flags |= TCQ_F_OFFLOADED; - - return err; + return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_PRIO, &hw_stats); } static int prio_dump(struct Qdisc *sch, struct sk_buff *skb) diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 3ce6c0a2c493..d5e441194397 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -281,7 +281,6 @@ static int red_init(struct Qdisc *sch, struct nlattr *opt, static int red_dump_offload_stats(struct Qdisc *sch, struct tc_red_qopt *opt) { - struct net_device *dev = qdisc_dev(sch); struct tc_red_qopt_offload hw_stats = { .command = TC_RED_STATS, .handle = sch->handle, @@ -291,22 +290,8 @@ static int red_dump_offload_stats(struct Qdisc *sch, struct tc_red_qopt *opt) .stats.qstats = &sch->qstats, }, }; - int err; - sch->flags &= ~TCQ_F_OFFLOADED; - - if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) - return 0; - - err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_RED, - &hw_stats); - if (err == -EOPNOTSUPP) - return 0; - - if (!err) - sch->flags |= TCQ_F_OFFLOADED; - - return err; + return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_RED, &hw_stats); } static int red_dump(struct Qdisc *sch, struct sk_buff *skb) From dad54c0fab31c2ed813cb54bb024c65d6467d0b9 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 7 Nov 2018 17:33:35 -0800 Subject: [PATCH 2/7] net: sched: red: remove unnecessary red_dump_offload_stats parameter Offload dump helper does not use opt parameter, remove it. Signed-off-by: Jakub Kicinski Reviewed-by: John Hurley Signed-off-by: David S. Miller --- net/sched/sch_red.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index d5e441194397..2bf1d2fabc48 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -279,7 +279,7 @@ static int red_init(struct Qdisc *sch, struct nlattr *opt, return red_change(sch, opt, extack); } -static int red_dump_offload_stats(struct Qdisc *sch, struct tc_red_qopt *opt) +static int red_dump_offload_stats(struct Qdisc *sch) { struct tc_red_qopt_offload hw_stats = { .command = TC_RED_STATS, @@ -309,7 +309,7 @@ static int red_dump(struct Qdisc *sch, struct sk_buff *skb) }; int err; - err = red_dump_offload_stats(sch, &opt); + err = red_dump_offload_stats(sch); if (err) goto nla_put_failure; From 58f8927399eac8f2ac3c7289a2856b8b5516ba30 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 7 Nov 2018 17:33:36 -0800 Subject: [PATCH 3/7] net: sched: set TCQ_F_OFFLOADED flag for MQ PRIO and RED mark the qdisc with TCQ_F_OFFLOADED upon successful offload, make MQ do the same. The consistency will help with consistent graft callback behaviour. Signed-off-by: Jakub Kicinski Reviewed-by: John Hurley Signed-off-by: David S. Miller --- net/sched/sch_mq.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c index f20f3a0f8424..1db5c1bf6ddd 100644 --- a/net/sched/sch_mq.c +++ b/net/sched/sch_mq.c @@ -38,9 +38,8 @@ static int mq_offload(struct Qdisc *sch, enum tc_mq_command cmd) return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_MQ, &opt); } -static void mq_offload_stats(struct Qdisc *sch) +static int mq_offload_stats(struct Qdisc *sch) { - struct net_device *dev = qdisc_dev(sch); struct tc_mq_qopt_offload opt = { .command = TC_MQ_STATS, .handle = sch->handle, @@ -50,8 +49,7 @@ static void mq_offload_stats(struct Qdisc *sch) }, }; - if (tc_can_offload(dev) && dev->netdev_ops->ndo_setup_tc) - dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_MQ, &opt); + return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_MQ, &opt); } static void mq_destroy(struct Qdisc *sch) @@ -171,9 +169,8 @@ static int mq_dump(struct Qdisc *sch, struct sk_buff *skb) spin_unlock_bh(qdisc_lock(qdisc)); } - mq_offload_stats(sch); - return 0; + return mq_offload_stats(sch); } static struct netdev_queue *mq_queue_get(struct Qdisc *sch, unsigned long cl) From bfaee9113f30abfa1f77ecb5e4a6f53a9d4c690c Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 7 Nov 2018 17:33:37 -0800 Subject: [PATCH 4/7] net: sched: add an offload graft helper Qdisc graft operation of offload-capable qdiscs performs a few extra steps which are identical among all the qdiscs. Add a helper to share this code. Signed-off-by: Jakub Kicinski Reviewed-by: John Hurley Signed-off-by: David S. Miller --- include/net/sch_generic.h | 12 ++++++++++++ net/sched/sch_api.c | 29 +++++++++++++++++++++++++++++ net/sched/sch_prio.c | 27 +++------------------------ 3 files changed, 44 insertions(+), 24 deletions(-) diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index af55c1c4edb1..a8dd1fc141b6 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -582,6 +582,10 @@ void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, unsigned int n, #ifdef CONFIG_NET_SCHED int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, void *type_data); +void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, + struct Qdisc *new, struct Qdisc *old, + enum tc_setup_type type, void *type_data, + struct netlink_ext_ack *extack); #else static inline int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, @@ -590,6 +594,14 @@ qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, q->flags &= ~TCQ_F_OFFLOADED; return 0; } + +static inline void +qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, + struct Qdisc *new, struct Qdisc *old, + enum tc_setup_type type, void *type_data, + struct netlink_ext_ack *extack) +{ +} #endif struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, const struct Qdisc_ops *ops, diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index e534825d3d3a..4b3af41cc1d7 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -831,6 +831,35 @@ int qdisc_offload_dump_helper(struct Qdisc *sch, enum tc_setup_type type, } EXPORT_SYMBOL(qdisc_offload_dump_helper); +void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, + struct Qdisc *new, struct Qdisc *old, + enum tc_setup_type type, void *type_data, + struct netlink_ext_ack *extack) +{ + bool any_qdisc_is_offloaded; + int err; + + if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc) + return; + + err = dev->netdev_ops->ndo_setup_tc(dev, type, type_data); + + /* Don't report error if the graft is part of destroy operation. */ + if (!err || !new || new == &noop_qdisc) + return; + + /* Don't report error if the parent, the old child and the new + * one are not offloaded. + */ + any_qdisc_is_offloaded = new->flags & TCQ_F_OFFLOADED; + any_qdisc_is_offloaded |= sch && sch->flags & TCQ_F_OFFLOADED; + any_qdisc_is_offloaded |= old && old->flags & TCQ_F_OFFLOADED; + + if (any_qdisc_is_offloaded) + NL_SET_ERR_MSG(extack, "Offloading graft operation failed."); +} +EXPORT_SYMBOL(qdisc_offload_graft_helper); + static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid, u32 portid, u32 seq, u16 flags, int event) { diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index 4bdd04c30ead..63a90c5055ee 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -295,43 +295,22 @@ static int prio_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new, { struct prio_sched_data *q = qdisc_priv(sch); struct tc_prio_qopt_offload graft_offload; - struct net_device *dev = qdisc_dev(sch); unsigned long band = arg - 1; - bool any_qdisc_is_offloaded; - int err; if (new == NULL) new = &noop_qdisc; *old = qdisc_replace(sch, new, &q->queues[band]); - if (!tc_can_offload(dev)) - return 0; - graft_offload.handle = sch->handle; graft_offload.parent = sch->parent; graft_offload.graft_params.band = band; graft_offload.graft_params.child_handle = new->handle; graft_offload.command = TC_PRIO_GRAFT; - err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_PRIO, - &graft_offload); - - /* Don't report error if the graft is part of destroy operation. */ - if (err && new != &noop_qdisc) { - /* Don't report error if the parent, the old child and the new - * one are not offloaded. - */ - any_qdisc_is_offloaded = sch->flags & TCQ_F_OFFLOADED; - any_qdisc_is_offloaded |= new->flags & TCQ_F_OFFLOADED; - if (*old) - any_qdisc_is_offloaded |= (*old)->flags & - TCQ_F_OFFLOADED; - - if (any_qdisc_is_offloaded) - NL_SET_ERR_MSG(extack, "Offloading graft operation failed."); - } - + qdisc_offload_graft_helper(qdisc_dev(sch), sch, new, *old, + TC_SETUP_QDISC_PRIO, &graft_offload, + extack); return 0; } From 9da93ece59f4a3e1544dfa2aa53e91f9e724abc6 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 7 Nov 2018 17:33:38 -0800 Subject: [PATCH 5/7] net: sched: refactor grafting Qdiscs with a parent The code for grafting Qdiscs when there is a parent has two needless indentation levels, and breaks the "keep the success path unindented" guideline. Refactor. Signed-off-by: Jakub Kicinski Reviewed-by: John Hurley Signed-off-by: David S. Miller --- net/sched/sch_api.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c index 4b3af41cc1d7..f55bc50cd0a9 100644 --- a/net/sched/sch_api.c +++ b/net/sched/sch_api.c @@ -1007,7 +1007,6 @@ static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, { struct Qdisc *q = old; struct net *net = dev_net(dev); - int err = 0; if (parent == NULL) { unsigned int i, num_q, ingress; @@ -1062,28 +1061,29 @@ static int qdisc_graft(struct net_device *dev, struct Qdisc *parent, dev_activate(dev); } else { const struct Qdisc_class_ops *cops = parent->ops->cl_ops; + unsigned long cl; + int err; /* Only support running class lockless if parent is lockless */ if (new && (new->flags & TCQ_F_NOLOCK) && parent && !(parent->flags & TCQ_F_NOLOCK)) new->flags &= ~TCQ_F_NOLOCK; - err = -EOPNOTSUPP; - if (cops && cops->graft) { - unsigned long cl = cops->find(parent, classid); + if (!cops || !cops->graft) + return -EOPNOTSUPP; - if (cl) { - err = cops->graft(parent, cl, new, &old, - extack); - } else { - NL_SET_ERR_MSG(extack, "Specified class not found"); - err = -ENOENT; - } + cl = cops->find(parent, classid); + if (!cl) { + NL_SET_ERR_MSG(extack, "Specified class not found"); + return -ENOENT; } - if (!err) - notify_and_destroy(net, skb, n, classid, old, new); + + err = cops->graft(parent, cl, new, &old, extack); + if (err) + return err; + notify_and_destroy(net, skb, n, classid, old, new); } - return err; + return 0; } static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca, From 0c8d13ac96070000da33f394f45e9c19638483c5 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 7 Nov 2018 17:33:39 -0800 Subject: [PATCH 6/7] net: sched: red: delay destroying child qdisc on replace Move destroying of the old child qdisc outside of the sch_tree_lock() section. This should improve the software qdisc replace but is even more important for offloads. Firstly calling offloads under a spin lock is best avoided. Secondly the destroy event of existing child would have been sent to the offload device before the replace, causing confusion. Signed-off-by: Jakub Kicinski Reviewed-by: John Hurley Signed-off-by: David S. Miller --- net/sched/sch_red.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c index 2bf1d2fabc48..7682f7a618a1 100644 --- a/net/sched/sch_red.c +++ b/net/sched/sch_red.c @@ -193,10 +193,10 @@ static const struct nla_policy red_policy[TCA_RED_MAX + 1] = { static int red_change(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) { + struct Qdisc *old_child = NULL, *child = NULL; struct red_sched_data *q = qdisc_priv(sch); struct nlattr *tb[TCA_RED_MAX + 1]; struct tc_red_qopt *ctl; - struct Qdisc *child = NULL; int err; u32 max_P; @@ -233,7 +233,7 @@ static int red_change(struct Qdisc *sch, struct nlattr *opt, if (child) { qdisc_tree_reduce_backlog(q->qdisc, q->qdisc->q.qlen, q->qdisc->qstats.backlog); - qdisc_put(q->qdisc); + old_child = q->qdisc; q->qdisc = child; } @@ -252,7 +252,11 @@ static int red_change(struct Qdisc *sch, struct nlattr *opt, red_start_of_idle_period(&q->vars); sch_tree_unlock(sch); + red_offload(sch, true); + + if (old_child) + qdisc_put(old_child); return 0; } From 7b8e0b6e659983154c8d7e756cdb833d89a3d4d7 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Wed, 7 Nov 2018 17:33:40 -0800 Subject: [PATCH 7/7] net: sched: prio: delay destroying child qdiscs on change Move destroying of the old child qdiscs outside of the sch_tree_lock() section. This should improve the software qdisc replace but is even more important for offloads. Calling offloads under a spin lock is best avoided, and child's destroy would be called under sch_tree_lock(). Signed-off-by: Jakub Kicinski Reviewed-by: John Hurley Signed-off-by: David S. Miller --- net/sched/sch_prio.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c index 63a90c5055ee..cdf68706e40f 100644 --- a/net/sched/sch_prio.c +++ b/net/sched/sch_prio.c @@ -220,7 +220,6 @@ static int prio_tune(struct Qdisc *sch, struct nlattr *opt, qdisc_tree_reduce_backlog(child, child->q.qlen, child->qstats.backlog); - qdisc_put(child); } for (i = oldbands; i < q->bands; i++) { @@ -230,6 +229,9 @@ static int prio_tune(struct Qdisc *sch, struct nlattr *opt, } sch_tree_unlock(sch); + + for (i = q->bands; i < oldbands; i++) + qdisc_put(q->queues[i]); return 0; }