openkylin
/
linux
mirror of https://gitee.com/openkylin/linux.git
9
0
Fork 0
Code Issues Projects Releases Wiki Activity

net: sched: pfifo_fast use skb_array 

This converts the pfifo_fast qdisc to use the skb_array data structure
and set the lockless qdisc bit. pfifo_fast is the first qdisc to support
the lockless bit that can be a child of a qdisc requiring locking. So
we add logic to clear the lock bit on initialization in these cases when
the qdisc graft operation occurs.

This also removes the logic used to pick the next band to dequeue from
and instead just checks a per priority array for packets from top priority
to lowest. This might need to be a bit more clever but seems to work
for now.

Signed-off-by: John Fastabend <john.fastabend@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
John Fastabend 2017-12-07 09:58:19 -08:00 committed by David S. Miller
parent 4a86a4cf68
commit c5ad119fb6
2 changed files with 91 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
net/sched/sch_api.c
View File

@ -955,6 +955,11 @@ static int qdisc_graft(struct net_device *dev, struct Qdisc *parent,
} else { } else {
const struct Qdisc_class_ops *cops = parent->ops->cl_ops; const struct Qdisc_class_ops *cops = parent->ops->cl_ops;
/* Only support running class lockless if parent is lockless */
if (new && (new->flags & TCQ_F_NOLOCK) &&
parent && !(parent->flags & TCQ_F_NOLOCK))
new->flags &= ~TCQ_F_NOLOCK;
err = -EOPNOTSUPP; err = -EOPNOTSUPP;
if (cops && cops->graft) { if (cops && cops->graft) {
unsigned long cl = cops->find(parent, classid); unsigned long cl = cops->find(parent, classid);

138
net/sched/sch_generic.c
View File

@ -26,6 +26,7 @@
#include <linux/list.h> #include <linux/list.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/if_vlan.h> #include <linux/if_vlan.h>
#include <linux/skb_array.h>
#include <net/sch_generic.h> #include <net/sch_generic.h>
#include <net/pkt_sched.h> #include <net/pkt_sched.h>
#include <net/dst.h> #include <net/dst.h>
@ -578,93 +579,93 @@ static const u8 prio2band[TC_PRIO_MAX + 1] = {
/* /*
* Private data for a pfifo_fast scheduler containing: * Private data for a pfifo_fast scheduler containing:
* - queues for the three band * - rings for priority bands
* - bitmap indicating which of the bands contain skbs
*/ */
struct pfifo_fast_priv { struct pfifo_fast_priv {
u32 bitmap; struct skb_array q[PFIFO_FAST_BANDS];
struct qdisc_skb_head q[PFIFO_FAST_BANDS];
}; };
/* static inline struct skb_array *band2list(struct pfifo_fast_priv *priv,
* Convert a bitmap to the first band number where an skb is queued, where: int band)
* bitmap=0 means there are no skbs on any band.
* bitmap=1 means there is an skb on band 0.
* bitmap=7 means there are skbs on all 3 bands, etc.
*/
static const int bitmap2band[] = {-1, 0, 1, 0, 2, 0, 1, 0};
static inline struct qdisc_skb_head *band2list(struct pfifo_fast_priv *priv,
int band)
{ {
return priv->q + band; return &priv->q[band];
} }
static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc, static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
struct sk_buff **to_free) struct sk_buff **to_free)
{ {
if (qdisc->q.qlen < qdisc_dev(qdisc)->tx_queue_len) { int band = prio2band[skb->priority & TC_PRIO_MAX];
int band = prio2band[skb->priority & TC_PRIO_MAX]; struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
struct pfifo_fast_priv *priv = qdisc_priv(qdisc); struct skb_array *q = band2list(priv, band);
struct qdisc_skb_head *list = band2list(priv, band); int err;
priv->bitmap |= (1 << band); err = skb_array_produce(q, skb);
qdisc->q.qlen++;
return __qdisc_enqueue_tail(skb, qdisc, list);
}
return qdisc_drop(skb, qdisc, to_free); if (unlikely(err))
return qdisc_drop_cpu(skb, qdisc, to_free);
qdisc_qstats_cpu_qlen_inc(qdisc);
qdisc_qstats_cpu_backlog_inc(qdisc, skb);
return NET_XMIT_SUCCESS;
} }
static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc) static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
{ {
struct pfifo_fast_priv *priv = qdisc_priv(qdisc); struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
int band = bitmap2band[priv->bitmap]; struct sk_buff *skb = NULL;
int band;
if (likely(band >= 0)) { for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
struct qdisc_skb_head *qh = band2list(priv, band); struct skb_array *q = band2list(priv, band);
struct sk_buff *skb = __qdisc_dequeue_head(qh);
if (likely(skb != NULL)) { if (__skb_array_empty(q))
qdisc_qstats_backlog_dec(qdisc, skb); continue;
qdisc_bstats_update(qdisc, skb);
}
qdisc->q.qlen--; skb = skb_array_consume_bh(q);
if (qh->qlen == 0) }
priv->bitmap &= ~(1 << band); if (likely(skb)) {
qdisc_qstats_cpu_backlog_dec(qdisc, skb);
return skb; qdisc_bstats_cpu_update(qdisc, skb);
qdisc_qstats_cpu_qlen_dec(qdisc);
} }
return NULL; return skb;
} }
static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc) static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
{ {
struct pfifo_fast_priv *priv = qdisc_priv(qdisc); struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
int band = bitmap2band[priv->bitmap]; struct sk_buff *skb = NULL;
int band;
if (band >= 0) { for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
struct qdisc_skb_head *qh = band2list(priv, band); struct skb_array *q = band2list(priv, band);
return qh->head; skb = __skb_array_peek(q);
} }
return NULL; return skb;
} }
static void pfifo_fast_reset(struct Qdisc *qdisc) static void pfifo_fast_reset(struct Qdisc *qdisc)
{ {
int prio; int i, band;
struct pfifo_fast_priv *priv = qdisc_priv(qdisc); struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) for (band = 0; band < PFIFO_FAST_BANDS; band++) {
__qdisc_reset_queue(band2list(priv, prio)); struct skb_array *q = band2list(priv, band);
struct sk_buff *skb;
priv->bitmap = 0; while ((skb = skb_array_consume_bh(q)) != NULL)
qdisc->qstats.backlog = 0; kfree_skb(skb);
qdisc->q.qlen = 0; }
for_each_possible_cpu(i) {
struct gnet_stats_queue *q = per_cpu_ptr(qdisc->cpu_qstats, i);
q->backlog = 0;
q->qlen = 0;
}
} }
static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb) static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
@ -682,17 +683,48 @@ static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt) static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt)
{ {
int prio; unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
struct pfifo_fast_priv *priv = qdisc_priv(qdisc); struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
int prio;
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) /* guard against zero length rings */
qdisc_skb_head_init(band2list(priv, prio)); if (!qlen)
return -EINVAL;
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
struct skb_array *q = band2list(priv, prio);
int err;
err = skb_array_init(q, qlen, GFP_KERNEL);
if (err)
return -ENOMEM;
}
/* Can by-pass the queue discipline */ /* Can by-pass the queue discipline */
qdisc->flags |= TCQ_F_CAN_BYPASS; qdisc->flags |= TCQ_F_CAN_BYPASS;
return 0; return 0;
} }
static void pfifo_fast_destroy(struct Qdisc *sch)
{
struct pfifo_fast_priv *priv = qdisc_priv(sch);
int prio;
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
struct skb_array *q = band2list(priv, prio);
/* NULL ring is possible if destroy path is due to a failed
* skb_array_init() in pfifo_fast_init() case.
*/
if (!&q->ring.queue)
continue;
/* Destroy ring but no need to kfree_skb because a call to
* pfifo_fast_reset() has already done that work.
*/
ptr_ring_cleanup(&q->ring, NULL);
}
}
struct Qdisc_ops pfifo_fast_ops __read_mostly = { struct Qdisc_ops pfifo_fast_ops __read_mostly = {
.id = "pfifo_fast", .id = "pfifo_fast",
.priv_size = sizeof(struct pfifo_fast_priv), .priv_size = sizeof(struct pfifo_fast_priv),
@ -700,9 +732,11 @@ struct Qdisc_ops pfifo_fast_ops __read_mostly = {
.dequeue = pfifo_fast_dequeue, .dequeue = pfifo_fast_dequeue,
.peek = pfifo_fast_peek, .peek = pfifo_fast_peek,
.init = pfifo_fast_init, .init = pfifo_fast_init,
.destroy = pfifo_fast_destroy,
.reset = pfifo_fast_reset, .reset = pfifo_fast_reset,
.dump = pfifo_fast_dump, .dump = pfifo_fast_dump,
.owner = THIS_MODULE, .owner = THIS_MODULE,
.static_flags = TCQ_F_NOLOCK | TCQ_F_CPUSTATS,
}; };
EXPORT_SYMBOL(pfifo_fast_ops); EXPORT_SYMBOL(pfifo_fast_ops);