2011-11-28 13:22:18 +08:00
|
|
|
#include <linux/skbuff.h>
|
2012-01-25 05:03:33 +08:00
|
|
|
#include <linux/export.h>
|
2011-11-28 13:22:18 +08:00
|
|
|
#include <linux/ip.h>
|
|
|
|
#include <linux/ipv6.h>
|
|
|
|
#include <linux/if_vlan.h>
|
|
|
|
#include <net/ip.h>
|
2012-07-18 16:11:12 +08:00
|
|
|
#include <net/ipv6.h>
|
2013-03-19 14:39:30 +08:00
|
|
|
#include <linux/igmp.h>
|
|
|
|
#include <linux/icmp.h>
|
|
|
|
#include <linux/sctp.h>
|
|
|
|
#include <linux/dccp.h>
|
2011-11-28 13:22:18 +08:00
|
|
|
#include <linux/if_tunnel.h>
|
|
|
|
#include <linux/if_pppox.h>
|
|
|
|
#include <linux/ppp_defs.h>
|
|
|
|
#include <net/flow_keys.h>
|
2014-09-06 07:20:26 +08:00
|
|
|
#include <scsi/fc/fc_fcoe.h>
|
2011-11-28 13:22:18 +08:00
|
|
|
|
flow_dissector: use a 64bit load/store
Le lundi 28 novembre 2011 à 19:06 -0500, David Miller a écrit :
> From: Dimitris Michailidis <dm@chelsio.com>
> Date: Mon, 28 Nov 2011 08:25:39 -0800
>
> >> +bool skb_flow_dissect(const struct sk_buff *skb, struct flow_keys
> >> *flow)
> >> +{
> >> + int poff, nhoff = skb_network_offset(skb);
> >> + u8 ip_proto;
> >> + u16 proto = skb->protocol;
> >
> > __be16 instead of u16 for proto?
>
> I'll take care of this when I apply these patches.
( CC trimmed )
Thanks David !
Here is a small patch to use one 64bit load/store on x86_64 instead of
two 32bit load/stores.
[PATCH net-next] flow_dissector: use a 64bit load/store
gcc compiler is smart enough to use a single load/store if we
memcpy(dptr, sptr, 8) on x86_64, regardless of
CONFIG_CC_OPTIMIZE_FOR_SIZE
In IP header, daddr immediately follows saddr, this wont change in the
future. We only need to make sure our flow_keys (src,dst) fields wont
break the rule.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2011-11-29 04:30:35 +08:00
|
|
|
/* copy saddr & daddr, possibly using 64bit load/store
|
|
|
|
* Equivalent to : flow->src = iph->saddr;
|
|
|
|
* flow->dst = iph->daddr;
|
|
|
|
*/
|
|
|
|
static void iph_to_flow_copy_addrs(struct flow_keys *flow, const struct iphdr *iph)
|
|
|
|
{
|
|
|
|
BUILD_BUG_ON(offsetof(typeof(*flow), dst) !=
|
|
|
|
offsetof(typeof(*flow), src) + sizeof(flow->src));
|
|
|
|
memcpy(&flow->src, &iph->saddr, sizeof(flow->src) + sizeof(flow->dst));
|
|
|
|
}
|
2011-11-28 13:22:18 +08:00
|
|
|
|
2013-10-02 19:39:24 +08:00
|
|
|
/**
|
2014-08-26 08:03:46 +08:00
|
|
|
* __skb_flow_get_ports - extract the upper layer ports and return them
|
|
|
|
* @skb: sk_buff to extract the ports from
|
2013-10-02 19:39:24 +08:00
|
|
|
* @thoff: transport header offset
|
|
|
|
* @ip_proto: protocol for which to get port offset
|
2014-08-26 08:03:46 +08:00
|
|
|
* @data: raw buffer pointer to the packet, if NULL use skb->data
|
|
|
|
* @hlen: packet header length, if @data is NULL use skb_headlen(skb)
|
2013-10-02 19:39:24 +08:00
|
|
|
*
|
|
|
|
* The function will try to retrieve the ports at offset thoff + poff where poff
|
|
|
|
* is the protocol port offset returned from proto_ports_offset
|
|
|
|
*/
|
2014-08-24 03:13:41 +08:00
|
|
|
__be32 __skb_flow_get_ports(const struct sk_buff *skb, int thoff, u8 ip_proto,
|
|
|
|
void *data, int hlen)
|
2013-10-02 19:39:24 +08:00
|
|
|
{
|
|
|
|
int poff = proto_ports_offset(ip_proto);
|
|
|
|
|
2014-08-24 03:13:41 +08:00
|
|
|
if (!data) {
|
|
|
|
data = skb->data;
|
|
|
|
hlen = skb_headlen(skb);
|
|
|
|
}
|
|
|
|
|
2013-10-02 19:39:24 +08:00
|
|
|
if (poff >= 0) {
|
|
|
|
__be32 *ports, _ports;
|
|
|
|
|
2014-08-24 03:13:41 +08:00
|
|
|
ports = __skb_header_pointer(skb, thoff + poff,
|
|
|
|
sizeof(_ports), data, hlen, &_ports);
|
2013-10-02 19:39:24 +08:00
|
|
|
if (ports)
|
|
|
|
return *ports;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
2014-08-24 03:13:41 +08:00
|
|
|
EXPORT_SYMBOL(__skb_flow_get_ports);
|
2013-10-02 19:39:24 +08:00
|
|
|
|
2014-08-26 08:03:47 +08:00
|
|
|
/**
|
|
|
|
* __skb_flow_dissect - extract the flow_keys struct and return it
|
|
|
|
* @skb: sk_buff to extract the flow from, can be NULL if the rest are specified
|
|
|
|
* @data: raw buffer pointer to the packet, if NULL use skb->data
|
|
|
|
* @proto: protocol for which to get the flow, if @data is NULL use skb->protocol
|
|
|
|
* @nhoff: network header offset, if @data is NULL use skb_network_offset(skb)
|
|
|
|
* @hlen: packet header length, if @data is NULL use skb_headlen(skb)
|
|
|
|
*
|
|
|
|
* The function will try to retrieve the struct flow_keys from either the skbuff
|
|
|
|
* or a raw buffer specified by the rest parameters
|
|
|
|
*/
|
|
|
|
bool __skb_flow_dissect(const struct sk_buff *skb, struct flow_keys *flow,
|
|
|
|
void *data, __be16 proto, int nhoff, int hlen)
|
2011-11-28 13:22:18 +08:00
|
|
|
{
|
|
|
|
u8 ip_proto;
|
|
|
|
|
2014-08-24 03:13:41 +08:00
|
|
|
if (!data) {
|
|
|
|
data = skb->data;
|
2014-08-26 08:03:47 +08:00
|
|
|
proto = skb->protocol;
|
|
|
|
nhoff = skb_network_offset(skb);
|
2014-08-24 03:13:41 +08:00
|
|
|
hlen = skb_headlen(skb);
|
|
|
|
}
|
|
|
|
|
2011-11-28 13:22:18 +08:00
|
|
|
memset(flow, 0, sizeof(*flow));
|
|
|
|
|
|
|
|
again:
|
|
|
|
switch (proto) {
|
2014-03-13 01:04:17 +08:00
|
|
|
case htons(ETH_P_IP): {
|
2011-11-28 13:22:18 +08:00
|
|
|
const struct iphdr *iph;
|
|
|
|
struct iphdr _iph;
|
|
|
|
ip:
|
2014-08-24 03:13:41 +08:00
|
|
|
iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
|
2013-11-01 15:01:10 +08:00
|
|
|
if (!iph || iph->ihl < 5)
|
2011-11-28 13:22:18 +08:00
|
|
|
return false;
|
2013-11-08 00:37:28 +08:00
|
|
|
nhoff += iph->ihl * 4;
|
2011-11-28 13:22:18 +08:00
|
|
|
|
2013-11-08 00:37:28 +08:00
|
|
|
ip_proto = iph->protocol;
|
2011-11-28 13:22:18 +08:00
|
|
|
if (ip_is_fragment(iph))
|
|
|
|
ip_proto = 0;
|
2013-11-08 00:37:28 +08:00
|
|
|
|
2014-10-11 03:09:12 +08:00
|
|
|
/* skip the address processing if skb is NULL. The assumption
|
|
|
|
* here is that if there is no skb we are not looking for flow
|
|
|
|
* info but lengths and protocols.
|
|
|
|
*/
|
|
|
|
if (!skb)
|
|
|
|
break;
|
|
|
|
|
flow_dissector: use a 64bit load/store
Le lundi 28 novembre 2011 à 19:06 -0500, David Miller a écrit :
> From: Dimitris Michailidis <dm@chelsio.com>
> Date: Mon, 28 Nov 2011 08:25:39 -0800
>
> >> +bool skb_flow_dissect(const struct sk_buff *skb, struct flow_keys
> >> *flow)
> >> +{
> >> + int poff, nhoff = skb_network_offset(skb);
> >> + u8 ip_proto;
> >> + u16 proto = skb->protocol;
> >
> > __be16 instead of u16 for proto?
>
> I'll take care of this when I apply these patches.
( CC trimmed )
Thanks David !
Here is a small patch to use one 64bit load/store on x86_64 instead of
two 32bit load/stores.
[PATCH net-next] flow_dissector: use a 64bit load/store
gcc compiler is smart enough to use a single load/store if we
memcpy(dptr, sptr, 8) on x86_64, regardless of
CONFIG_CC_OPTIMIZE_FOR_SIZE
In IP header, daddr immediately follows saddr, this wont change in the
future. We only need to make sure our flow_keys (src,dst) fields wont
break the rule.
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2011-11-29 04:30:35 +08:00
|
|
|
iph_to_flow_copy_addrs(flow, iph);
|
2011-11-28 13:22:18 +08:00
|
|
|
break;
|
|
|
|
}
|
2014-03-13 01:04:17 +08:00
|
|
|
case htons(ETH_P_IPV6): {
|
2011-11-28 13:22:18 +08:00
|
|
|
const struct ipv6hdr *iph;
|
|
|
|
struct ipv6hdr _iph;
|
2014-07-02 12:33:01 +08:00
|
|
|
__be32 flow_label;
|
|
|
|
|
2011-11-28 13:22:18 +08:00
|
|
|
ipv6:
|
2014-08-24 03:13:41 +08:00
|
|
|
iph = __skb_header_pointer(skb, nhoff, sizeof(_iph), data, hlen, &_iph);
|
2011-11-28 13:22:18 +08:00
|
|
|
if (!iph)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
ip_proto = iph->nexthdr;
|
|
|
|
nhoff += sizeof(struct ipv6hdr);
|
2014-07-02 12:33:01 +08:00
|
|
|
|
2014-10-11 03:09:12 +08:00
|
|
|
/* see comment above in IPv4 section */
|
2014-09-06 07:20:26 +08:00
|
|
|
if (!skb)
|
|
|
|
break;
|
|
|
|
|
2014-10-11 03:09:12 +08:00
|
|
|
flow->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
|
|
|
|
flow->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
|
|
|
|
|
2014-07-02 12:33:01 +08:00
|
|
|
flow_label = ip6_flowlabel(iph);
|
|
|
|
if (flow_label) {
|
|
|
|
/* Awesome, IPv6 packet has a flow label so we can
|
|
|
|
* use that to represent the ports without any
|
|
|
|
* further dissection.
|
|
|
|
*/
|
|
|
|
flow->n_proto = proto;
|
|
|
|
flow->ip_proto = ip_proto;
|
|
|
|
flow->ports = flow_label;
|
|
|
|
flow->thoff = (u16)nhoff;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2011-11-28 13:22:18 +08:00
|
|
|
break;
|
|
|
|
}
|
2014-03-13 01:04:17 +08:00
|
|
|
case htons(ETH_P_8021AD):
|
|
|
|
case htons(ETH_P_8021Q): {
|
2011-11-28 13:22:18 +08:00
|
|
|
const struct vlan_hdr *vlan;
|
|
|
|
struct vlan_hdr _vlan;
|
|
|
|
|
2014-08-24 03:13:41 +08:00
|
|
|
vlan = __skb_header_pointer(skb, nhoff, sizeof(_vlan), data, hlen, &_vlan);
|
2011-11-28 13:22:18 +08:00
|
|
|
if (!vlan)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
proto = vlan->h_vlan_encapsulated_proto;
|
|
|
|
nhoff += sizeof(*vlan);
|
|
|
|
goto again;
|
|
|
|
}
|
2014-03-13 01:04:17 +08:00
|
|
|
case htons(ETH_P_PPP_SES): {
|
2011-11-28 13:22:18 +08:00
|
|
|
struct {
|
|
|
|
struct pppoe_hdr hdr;
|
|
|
|
__be16 proto;
|
|
|
|
} *hdr, _hdr;
|
2014-08-24 03:13:41 +08:00
|
|
|
hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
|
2011-11-28 13:22:18 +08:00
|
|
|
if (!hdr)
|
|
|
|
return false;
|
|
|
|
proto = hdr->proto;
|
|
|
|
nhoff += PPPOE_SES_HLEN;
|
|
|
|
switch (proto) {
|
2014-03-13 01:04:17 +08:00
|
|
|
case htons(PPP_IP):
|
2011-11-28 13:22:18 +08:00
|
|
|
goto ip;
|
2014-03-13 01:04:17 +08:00
|
|
|
case htons(PPP_IPV6):
|
2011-11-28 13:22:18 +08:00
|
|
|
goto ipv6;
|
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
2015-01-23 00:10:32 +08:00
|
|
|
case htons(ETH_P_TIPC): {
|
|
|
|
struct {
|
|
|
|
__be32 pre[3];
|
|
|
|
__be32 srcnode;
|
|
|
|
} *hdr, _hdr;
|
|
|
|
hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
|
|
|
|
if (!hdr)
|
|
|
|
return false;
|
|
|
|
flow->src = hdr->srcnode;
|
|
|
|
flow->dst = 0;
|
|
|
|
flow->n_proto = proto;
|
|
|
|
flow->thoff = (u16)nhoff;
|
|
|
|
return true;
|
|
|
|
}
|
2014-09-06 07:20:26 +08:00
|
|
|
case htons(ETH_P_FCOE):
|
|
|
|
flow->thoff = (u16)(nhoff + FCOE_HEADER_LEN);
|
|
|
|
/* fall through */
|
2011-11-28 13:22:18 +08:00
|
|
|
default:
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
switch (ip_proto) {
|
|
|
|
case IPPROTO_GRE: {
|
|
|
|
struct gre_hdr {
|
|
|
|
__be16 flags;
|
|
|
|
__be16 proto;
|
|
|
|
} *hdr, _hdr;
|
|
|
|
|
2014-08-24 03:13:41 +08:00
|
|
|
hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
|
2011-11-28 13:22:18 +08:00
|
|
|
if (!hdr)
|
|
|
|
return false;
|
|
|
|
/*
|
|
|
|
* Only look inside GRE if version zero and no
|
|
|
|
* routing
|
|
|
|
*/
|
|
|
|
if (!(hdr->flags & (GRE_VERSION|GRE_ROUTING))) {
|
|
|
|
proto = hdr->proto;
|
|
|
|
nhoff += 4;
|
|
|
|
if (hdr->flags & GRE_CSUM)
|
|
|
|
nhoff += 4;
|
|
|
|
if (hdr->flags & GRE_KEY)
|
|
|
|
nhoff += 4;
|
|
|
|
if (hdr->flags & GRE_SEQ)
|
|
|
|
nhoff += 4;
|
2013-03-11 14:52:28 +08:00
|
|
|
if (proto == htons(ETH_P_TEB)) {
|
|
|
|
const struct ethhdr *eth;
|
|
|
|
struct ethhdr _eth;
|
|
|
|
|
2014-08-24 03:13:41 +08:00
|
|
|
eth = __skb_header_pointer(skb, nhoff,
|
|
|
|
sizeof(_eth),
|
|
|
|
data, hlen, &_eth);
|
2013-03-11 14:52:28 +08:00
|
|
|
if (!eth)
|
|
|
|
return false;
|
|
|
|
proto = eth->h_proto;
|
|
|
|
nhoff += sizeof(*eth);
|
|
|
|
}
|
2011-11-28 13:22:18 +08:00
|
|
|
goto again;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case IPPROTO_IPIP:
|
2013-07-30 02:07:36 +08:00
|
|
|
proto = htons(ETH_P_IP);
|
|
|
|
goto ip;
|
2013-07-30 02:07:42 +08:00
|
|
|
case IPPROTO_IPV6:
|
|
|
|
proto = htons(ETH_P_IPV6);
|
|
|
|
goto ipv6;
|
2011-11-28 13:22:18 +08:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2014-06-23 18:37:58 +08:00
|
|
|
flow->n_proto = proto;
|
2011-11-28 13:22:18 +08:00
|
|
|
flow->ip_proto = ip_proto;
|
2013-03-19 14:39:29 +08:00
|
|
|
flow->thoff = (u16) nhoff;
|
|
|
|
|
2014-10-11 03:09:12 +08:00
|
|
|
/* unless skb is set we don't need to record port info */
|
|
|
|
if (skb)
|
|
|
|
flow->ports = __skb_flow_get_ports(skb, nhoff, ip_proto,
|
|
|
|
data, hlen);
|
|
|
|
|
2011-11-28 13:22:18 +08:00
|
|
|
return true;
|
|
|
|
}
|
2014-08-24 03:13:41 +08:00
|
|
|
EXPORT_SYMBOL(__skb_flow_dissect);
|
2013-01-21 08:39:24 +08:00
|
|
|
|
|
|
|
static u32 hashrnd __read_mostly;
|
2013-10-24 02:06:00 +08:00
|
|
|
static __always_inline void __flow_hash_secret_init(void)
|
|
|
|
{
|
|
|
|
net_get_random_once(&hashrnd, sizeof(hashrnd));
|
|
|
|
}
|
|
|
|
|
2015-05-02 02:30:12 +08:00
|
|
|
static __always_inline u32 __flow_hash_3words(u32 a, u32 b, u32 c, u32 keyval)
|
2013-10-24 02:06:00 +08:00
|
|
|
{
|
2015-05-02 02:30:12 +08:00
|
|
|
return jhash_3words(a, b, c, keyval);
|
2013-10-24 02:06:00 +08:00
|
|
|
}
|
|
|
|
|
2015-05-02 02:30:12 +08:00
|
|
|
static inline u32 __flow_hash_from_keys(struct flow_keys *keys, u32 keyval)
|
2014-07-02 12:32:05 +08:00
|
|
|
{
|
|
|
|
u32 hash;
|
|
|
|
|
|
|
|
/* get a consistent hash (same value on both flow directions) */
|
|
|
|
if (((__force u32)keys->dst < (__force u32)keys->src) ||
|
|
|
|
(((__force u32)keys->dst == (__force u32)keys->src) &&
|
|
|
|
((__force u16)keys->port16[1] < (__force u16)keys->port16[0]))) {
|
|
|
|
swap(keys->dst, keys->src);
|
|
|
|
swap(keys->port16[0], keys->port16[1]);
|
|
|
|
}
|
|
|
|
|
|
|
|
hash = __flow_hash_3words((__force u32)keys->dst,
|
|
|
|
(__force u32)keys->src,
|
2015-05-02 02:30:12 +08:00
|
|
|
(__force u32)keys->ports,
|
|
|
|
keyval);
|
2014-07-02 12:32:05 +08:00
|
|
|
if (!hash)
|
|
|
|
hash = 1;
|
|
|
|
|
|
|
|
return hash;
|
|
|
|
}
|
|
|
|
|
|
|
|
u32 flow_hash_from_keys(struct flow_keys *keys)
|
|
|
|
{
|
2015-05-02 02:30:12 +08:00
|
|
|
__flow_hash_secret_init();
|
|
|
|
return __flow_hash_from_keys(keys, hashrnd);
|
2014-07-02 12:32:05 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(flow_hash_from_keys);
|
|
|
|
|
2015-05-02 02:30:12 +08:00
|
|
|
static inline u32 ___skb_get_hash(const struct sk_buff *skb,
|
|
|
|
struct flow_keys *keys, u32 keyval)
|
|
|
|
{
|
|
|
|
if (!skb_flow_dissect(skb, keys))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return __flow_hash_from_keys(keys, keyval);
|
|
|
|
}
|
|
|
|
|
2015-05-02 02:30:17 +08:00
|
|
|
struct _flow_keys_digest_data {
|
|
|
|
__be16 n_proto;
|
|
|
|
u8 ip_proto;
|
|
|
|
u8 padding;
|
|
|
|
__be32 ports;
|
|
|
|
__be32 src;
|
|
|
|
__be32 dst;
|
|
|
|
};
|
|
|
|
|
|
|
|
void make_flow_keys_digest(struct flow_keys_digest *digest,
|
|
|
|
const struct flow_keys *flow)
|
|
|
|
{
|
|
|
|
struct _flow_keys_digest_data *data =
|
|
|
|
(struct _flow_keys_digest_data *)digest;
|
|
|
|
|
|
|
|
BUILD_BUG_ON(sizeof(*data) > sizeof(*digest));
|
|
|
|
|
|
|
|
memset(digest, 0, sizeof(*digest));
|
|
|
|
|
|
|
|
data->n_proto = flow->n_proto;
|
|
|
|
data->ip_proto = flow->ip_proto;
|
|
|
|
data->ports = flow->ports;
|
|
|
|
data->src = flow->src;
|
|
|
|
data->dst = flow->dst;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(make_flow_keys_digest);
|
|
|
|
|
2013-01-21 08:39:24 +08:00
|
|
|
/*
|
2013-12-16 14:12:06 +08:00
|
|
|
* __skb_get_hash: calculate a flow hash based on src/dst addresses
|
2014-03-25 06:34:47 +08:00
|
|
|
* and src/dst port numbers. Sets hash in skb to non-zero hash value
|
|
|
|
* on success, zero indicates no valid hash. Also, sets l4_hash in skb
|
2013-01-21 08:39:24 +08:00
|
|
|
* if hash is a canonical 4-tuple hash over transport ports.
|
|
|
|
*/
|
2013-12-16 14:12:06 +08:00
|
|
|
void __skb_get_hash(struct sk_buff *skb)
|
2013-01-21 08:39:24 +08:00
|
|
|
{
|
|
|
|
struct flow_keys keys;
|
2015-05-02 02:30:12 +08:00
|
|
|
u32 hash;
|
2013-01-21 08:39:24 +08:00
|
|
|
|
2015-05-02 02:30:12 +08:00
|
|
|
__flow_hash_secret_init();
|
|
|
|
|
|
|
|
hash = ___skb_get_hash(skb, &keys, hashrnd);
|
|
|
|
if (!hash)
|
2013-01-21 08:39:24 +08:00
|
|
|
return;
|
|
|
|
|
|
|
|
if (keys.ports)
|
2014-03-25 06:34:47 +08:00
|
|
|
skb->l4_hash = 1;
|
2013-01-21 08:39:24 +08:00
|
|
|
|
2014-07-02 12:33:17 +08:00
|
|
|
skb->sw_hash = 1;
|
|
|
|
|
2015-05-02 02:30:12 +08:00
|
|
|
skb->hash = hash;
|
2013-01-21 08:39:24 +08:00
|
|
|
}
|
2013-12-16 14:12:06 +08:00
|
|
|
EXPORT_SYMBOL(__skb_get_hash);
|
2013-01-21 08:39:24 +08:00
|
|
|
|
2015-05-02 02:30:12 +08:00
|
|
|
__u32 skb_get_hash_perturb(const struct sk_buff *skb, u32 perturb)
|
|
|
|
{
|
|
|
|
struct flow_keys keys;
|
|
|
|
|
|
|
|
return ___skb_get_hash(skb, &keys, perturb);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(skb_get_hash_perturb);
|
|
|
|
|
2013-01-21 08:39:24 +08:00
|
|
|
/*
|
|
|
|
* Returns a Tx hash based on the given packet descriptor a Tx queues' number
|
|
|
|
* to be used as a distribution range.
|
|
|
|
*/
|
2014-07-02 12:32:27 +08:00
|
|
|
u16 __skb_tx_hash(const struct net_device *dev, struct sk_buff *skb,
|
2013-01-21 08:39:24 +08:00
|
|
|
unsigned int num_tx_queues)
|
|
|
|
{
|
|
|
|
u32 hash;
|
|
|
|
u16 qoffset = 0;
|
|
|
|
u16 qcount = num_tx_queues;
|
|
|
|
|
|
|
|
if (skb_rx_queue_recorded(skb)) {
|
|
|
|
hash = skb_get_rx_queue(skb);
|
|
|
|
while (unlikely(hash >= num_tx_queues))
|
|
|
|
hash -= num_tx_queues;
|
|
|
|
return hash;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (dev->num_tc) {
|
|
|
|
u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
|
|
|
|
qoffset = dev->tc_to_txq[tc].offset;
|
|
|
|
qcount = dev->tc_to_txq[tc].count;
|
|
|
|
}
|
|
|
|
|
2014-08-24 02:58:54 +08:00
|
|
|
return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
|
2013-01-21 08:39:24 +08:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(__skb_tx_hash);
|
|
|
|
|
2014-09-06 07:20:26 +08:00
|
|
|
u32 __skb_get_poff(const struct sk_buff *skb, void *data,
|
|
|
|
const struct flow_keys *keys, int hlen)
|
2013-03-19 14:39:30 +08:00
|
|
|
{
|
2014-09-06 07:20:26 +08:00
|
|
|
u32 poff = keys->thoff;
|
2013-03-19 14:39:30 +08:00
|
|
|
|
2014-09-06 07:20:26 +08:00
|
|
|
switch (keys->ip_proto) {
|
2013-03-19 14:39:30 +08:00
|
|
|
case IPPROTO_TCP: {
|
2014-10-11 03:09:12 +08:00
|
|
|
/* access doff as u8 to avoid unaligned access */
|
|
|
|
const u8 *doff;
|
|
|
|
u8 _doff;
|
2013-03-19 14:39:30 +08:00
|
|
|
|
2014-10-11 03:09:12 +08:00
|
|
|
doff = __skb_header_pointer(skb, poff + 12, sizeof(_doff),
|
|
|
|
data, hlen, &_doff);
|
|
|
|
if (!doff)
|
2013-03-19 14:39:30 +08:00
|
|
|
return poff;
|
|
|
|
|
2014-10-11 03:09:12 +08:00
|
|
|
poff += max_t(u32, sizeof(struct tcphdr), (*doff & 0xF0) >> 2);
|
2013-03-19 14:39:30 +08:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
case IPPROTO_UDP:
|
|
|
|
case IPPROTO_UDPLITE:
|
|
|
|
poff += sizeof(struct udphdr);
|
|
|
|
break;
|
|
|
|
/* For the rest, we do not really care about header
|
|
|
|
* extensions at this point for now.
|
|
|
|
*/
|
|
|
|
case IPPROTO_ICMP:
|
|
|
|
poff += sizeof(struct icmphdr);
|
|
|
|
break;
|
|
|
|
case IPPROTO_ICMPV6:
|
|
|
|
poff += sizeof(struct icmp6hdr);
|
|
|
|
break;
|
|
|
|
case IPPROTO_IGMP:
|
|
|
|
poff += sizeof(struct igmphdr);
|
|
|
|
break;
|
|
|
|
case IPPROTO_DCCP:
|
|
|
|
poff += sizeof(struct dccp_hdr);
|
|
|
|
break;
|
|
|
|
case IPPROTO_SCTP:
|
|
|
|
poff += sizeof(struct sctphdr);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return poff;
|
|
|
|
}
|
|
|
|
|
2014-09-06 07:20:26 +08:00
|
|
|
/* skb_get_poff() returns the offset to the payload as far as it could
|
|
|
|
* be dissected. The main user is currently BPF, so that we can dynamically
|
|
|
|
* truncate packets without needing to push actual payload to the user
|
|
|
|
* space and can analyze headers only, instead.
|
|
|
|
*/
|
|
|
|
u32 skb_get_poff(const struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct flow_keys keys;
|
|
|
|
|
|
|
|
if (!skb_flow_dissect(skb, &keys))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return __skb_get_poff(skb, skb->data, &keys, skb_headlen(skb));
|
|
|
|
}
|
|
|
|
|
2013-01-21 08:39:24 +08:00
|
|
|
static inline int get_xps_queue(struct net_device *dev, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_XPS
|
|
|
|
struct xps_dev_maps *dev_maps;
|
|
|
|
struct xps_map *map;
|
|
|
|
int queue_index = -1;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
dev_maps = rcu_dereference(dev->xps_maps);
|
|
|
|
if (dev_maps) {
|
|
|
|
map = rcu_dereference(
|
xps: fix xps for stacked devices
A typical qdisc setup is the following :
bond0 : bonding device, using HTB hierarchy
eth1/eth2 : slaves, multiqueue NIC, using MQ + FQ qdisc
XPS allows to spread packets on specific tx queues, based on the cpu
doing the send.
Problem is that dequeues from bond0 qdisc can happen on random cpus,
due to the fact that qdisc_run() can dequeue a batch of packets.
CPUA -> queue packet P1 on bond0 qdisc, P1->ooo_okay=1
CPUA -> queue packet P2 on bond0 qdisc, P2->ooo_okay=0
CPUB -> dequeue packet P1 from bond0
enqueue packet on eth1/eth2
CPUC -> dequeue packet P2 from bond0
enqueue packet on eth1/eth2 using sk cache (ooo_okay is 0)
get_xps_queue() then might select wrong queue for P1, since current cpu
might be different than CPUA.
P2 might be sent on the old queue (stored in sk->sk_tx_queue_mapping),
if CPUC runs a bit faster (or CPUB spins a bit on qdisc lock)
Effect of this bug is TCP reorders, and more generally not optimal
TX queue placement. (A victim bulk flow can be migrated to the wrong TX
queue for a while)
To fix this, we have to record sender cpu number the first time
dev_queue_xmit() is called for one tx skb.
We can union napi_id (used on receive path) and sender_cpu,
granted we clear sender_cpu in skb_scrub_packet() (credit to Willem for
this union idea)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Willem de Bruijn <willemb@google.com>
Cc: Nandita Dukkipati <nanditad@google.com>
Cc: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-02-04 15:48:24 +08:00
|
|
|
dev_maps->cpu_map[skb->sender_cpu - 1]);
|
2013-01-21 08:39:24 +08:00
|
|
|
if (map) {
|
|
|
|
if (map->len == 1)
|
|
|
|
queue_index = map->queues[0];
|
2014-07-02 12:32:27 +08:00
|
|
|
else
|
2014-08-24 02:58:54 +08:00
|
|
|
queue_index = map->queues[reciprocal_scale(skb_get_hash(skb),
|
|
|
|
map->len)];
|
2013-01-21 08:39:24 +08:00
|
|
|
if (unlikely(queue_index >= dev->real_num_tx_queues))
|
|
|
|
queue_index = -1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return queue_index;
|
|
|
|
#else
|
|
|
|
return -1;
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2014-02-16 22:55:20 +08:00
|
|
|
static u16 __netdev_pick_tx(struct net_device *dev, struct sk_buff *skb)
|
2013-01-21 08:39:24 +08:00
|
|
|
{
|
|
|
|
struct sock *sk = skb->sk;
|
|
|
|
int queue_index = sk_tx_queue_get(sk);
|
|
|
|
|
|
|
|
if (queue_index < 0 || skb->ooo_okay ||
|
|
|
|
queue_index >= dev->real_num_tx_queues) {
|
|
|
|
int new_index = get_xps_queue(dev, skb);
|
|
|
|
if (new_index < 0)
|
|
|
|
new_index = skb_tx_hash(dev, skb);
|
|
|
|
|
2013-08-29 09:10:43 +08:00
|
|
|
if (queue_index != new_index && sk &&
|
|
|
|
rcu_access_pointer(sk->sk_dst_cache))
|
2013-09-08 03:02:57 +08:00
|
|
|
sk_tx_queue_set(sk, new_index);
|
2013-01-21 08:39:24 +08:00
|
|
|
|
|
|
|
queue_index = new_index;
|
|
|
|
}
|
|
|
|
|
|
|
|
return queue_index;
|
|
|
|
}
|
|
|
|
|
|
|
|
struct netdev_queue *netdev_pick_tx(struct net_device *dev,
|
2014-01-10 16:18:26 +08:00
|
|
|
struct sk_buff *skb,
|
|
|
|
void *accel_priv)
|
2013-01-21 08:39:24 +08:00
|
|
|
{
|
|
|
|
int queue_index = 0;
|
|
|
|
|
xps: fix xps for stacked devices
A typical qdisc setup is the following :
bond0 : bonding device, using HTB hierarchy
eth1/eth2 : slaves, multiqueue NIC, using MQ + FQ qdisc
XPS allows to spread packets on specific tx queues, based on the cpu
doing the send.
Problem is that dequeues from bond0 qdisc can happen on random cpus,
due to the fact that qdisc_run() can dequeue a batch of packets.
CPUA -> queue packet P1 on bond0 qdisc, P1->ooo_okay=1
CPUA -> queue packet P2 on bond0 qdisc, P2->ooo_okay=0
CPUB -> dequeue packet P1 from bond0
enqueue packet on eth1/eth2
CPUC -> dequeue packet P2 from bond0
enqueue packet on eth1/eth2 using sk cache (ooo_okay is 0)
get_xps_queue() then might select wrong queue for P1, since current cpu
might be different than CPUA.
P2 might be sent on the old queue (stored in sk->sk_tx_queue_mapping),
if CPUC runs a bit faster (or CPUB spins a bit on qdisc lock)
Effect of this bug is TCP reorders, and more generally not optimal
TX queue placement. (A victim bulk flow can be migrated to the wrong TX
queue for a while)
To fix this, we have to record sender cpu number the first time
dev_queue_xmit() is called for one tx skb.
We can union napi_id (used on receive path) and sender_cpu,
granted we clear sender_cpu in skb_scrub_packet() (credit to Willem for
this union idea)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: Willem de Bruijn <willemb@google.com>
Cc: Nandita Dukkipati <nanditad@google.com>
Cc: Yuchung Cheng <ycheng@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2015-02-04 15:48:24 +08:00
|
|
|
#ifdef CONFIG_XPS
|
|
|
|
if (skb->sender_cpu == 0)
|
|
|
|
skb->sender_cpu = raw_smp_processor_id() + 1;
|
|
|
|
#endif
|
|
|
|
|
2013-01-21 08:39:24 +08:00
|
|
|
if (dev->real_num_tx_queues != 1) {
|
|
|
|
const struct net_device_ops *ops = dev->netdev_ops;
|
|
|
|
if (ops->ndo_select_queue)
|
2014-02-16 22:55:20 +08:00
|
|
|
queue_index = ops->ndo_select_queue(dev, skb, accel_priv,
|
|
|
|
__netdev_pick_tx);
|
2013-01-21 08:39:24 +08:00
|
|
|
else
|
|
|
|
queue_index = __netdev_pick_tx(dev, skb);
|
2014-01-10 16:18:26 +08:00
|
|
|
|
|
|
|
if (!accel_priv)
|
2014-02-16 22:55:21 +08:00
|
|
|
queue_index = netdev_cap_txqueue(dev, queue_index);
|
2013-01-21 08:39:24 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
skb_set_queue_mapping(skb, queue_index);
|
|
|
|
return netdev_get_tx_queue(dev, queue_index);
|
|
|
|
}
|