udp: Add GRO functions to UDP socket
This patch adds GRO functions (gro_receive and gro_complete) to UDP sockets. udp_gro_receive is changed to perform socket lookup on a packet. If a socket is found the related GRO functions are called. This features obsoletes using UDP offload infrastructure for GRO (udp_offload). This has the advantage of not being limited to provide offload on a per port basis, GRO is now applied to whatever individual UDP sockets are bound to. This also allows the possbility of "application defined GRO"-- that is we can attach something like a BPF program to a UDP socket to perfrom GRO on an application layer protocol. Signed-off-by: Tom Herbert <tom@herbertland.com> Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Tom Herbert
David S. Miller
parent
63058308cd
commit
a6024562ff
|
|
@ -71,6 +71,14 @@ struct udp_sock {
|
|||
*/
|
||||
int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
|
||||
void (*encap_destroy)(struct sock *sk);
|
||||
|
||||
/* GRO functions for UDP socket */
|
||||
struct sk_buff ** (*gro_receive)(struct sock *sk,
|
||||
struct sk_buff **head,
|
||||
struct sk_buff *skb);
|
||||
int (*gro_complete)(struct sock *sk,
|
||||
struct sk_buff *skb,
|
||||
int nhoff);
|
||||
};
|
||||
|
||||
static inline struct udp_sock *udp_sk(const struct sock *sk)
|
||||
|
|
|
|||
|
|
@ -167,9 +167,12 @@ static inline void udp_csum_pull_header(struct sk_buff *skb)
|
|||
UDP_SKB_CB(skb)->cscov -= sizeof(struct udphdr);
|
||||
}
|
||||
|
||||
typedef struct sock *(*udp_lookup_t)(struct sk_buff *skb, __be16 sport,
|
||||
__be16 dport);
|
||||
|
||||
struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
|
||||
struct udphdr *uh);
|
||||
int udp_gro_complete(struct sk_buff *skb, int nhoff);
|
||||
struct udphdr *uh, udp_lookup_t lookup);
|
||||
int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
|
||||
|
||||
static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,
|
|||
|
||||
return segs;
|
||||
}
|
||||
EXPORT_SYMBOL(skb_udp_tunnel_segment);
|
||||
|
||||
static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,
|
||||
netdev_features_t features)
|
||||
|
|
@ -304,13 +305,13 @@ void udp_del_offload(struct udp_offload *uo)
|
|||
EXPORT_SYMBOL(udp_del_offload);
|
||||
|
||||
struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
|
||||
struct udphdr *uh)
|
||||
struct udphdr *uh, udp_lookup_t lookup)
|
||||
{
|
||||
struct udp_offload_priv *uo_priv;
|
||||
struct sk_buff *p, **pp = NULL;
|
||||
struct udphdr *uh2;
|
||||
unsigned int off = skb_gro_offset(skb);
|
||||
int flush = 1;
|
||||
struct sock *sk;
|
||||
|
||||
if (NAPI_GRO_CB(skb)->encap_mark ||
|
||||
(skb->ip_summed != CHECKSUM_PARTIAL &&
|
||||
|
|
@ -322,13 +323,11 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
|
|||
NAPI_GRO_CB(skb)->encap_mark = 1;
|
||||
|
||||
rcu_read_lock();
|
||||
uo_priv = rcu_dereference(udp_offload_base);
|
||||
for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) {
|
||||
if (net_eq(read_pnet(&uo_priv->net), dev_net(skb->dev)) &&
|
||||
uo_priv->offload->port == uh->dest &&
|
||||
uo_priv->offload->callbacks.gro_receive)
|
||||
goto unflush;
|
||||
}
|
||||
sk = (*lookup)(skb, uh->source, uh->dest);
|
||||
|
||||
if (sk && udp_sk(sk)->gro_receive)
|
||||
goto unflush;
|
||||
|
||||
goto out_unlock;
|
||||
|
||||
unflush:
|
||||
|
|
@ -352,9 +351,7 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
|
|||
|
||||
skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */
|
||||
skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));
|
||||
NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto;
|
||||
pp = uo_priv->offload->callbacks.gro_receive(head, skb,
|
||||
uo_priv->offload);
|
||||
pp = udp_sk(sk)->gro_receive(sk, head, skb);
|
||||
|
||||
out_unlock:
|
||||
rcu_read_unlock();
|
||||
|
|
@ -362,6 +359,7 @@ struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
|
|||
NAPI_GRO_CB(skb)->flush |= flush;
|
||||
return pp;
|
||||
}
|
||||
EXPORT_SYMBOL(udp_gro_receive);
|
||||
|
||||
static struct sk_buff **udp4_gro_receive(struct sk_buff **head,
|
||||
struct sk_buff *skb)
|
||||
|
|
@ -383,39 +381,28 @@ static struct sk_buff **udp4_gro_receive(struct sk_buff **head,
|
|||
inet_gro_compute_pseudo);
|
||||
skip:
|
||||
NAPI_GRO_CB(skb)->is_ipv6 = 0;
|
||||
return udp_gro_receive(head, skb, uh);
|
||||
return udp_gro_receive(head, skb, uh, udp4_lib_lookup_skb);
|
||||
|
||||
flush:
|
||||
NAPI_GRO_CB(skb)->flush = 1;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int udp_gro_complete(struct sk_buff *skb, int nhoff)
|
||||
int udp_gro_complete(struct sk_buff *skb, int nhoff,
|
||||
udp_lookup_t lookup)
|
||||
{
|
||||
struct udp_offload_priv *uo_priv;
|
||||
__be16 newlen = htons(skb->len - nhoff);
|
||||
struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
|
||||
int err = -ENOSYS;
|
||||
struct sock *sk;
|
||||
|
||||
uh->len = newlen;
|
||||
|
||||
rcu_read_lock();
|
||||
|
||||
uo_priv = rcu_dereference(udp_offload_base);
|
||||
for (; uo_priv != NULL; uo_priv = rcu_dereference(uo_priv->next)) {
|
||||
if (net_eq(read_pnet(&uo_priv->net), dev_net(skb->dev)) &&
|
||||
uo_priv->offload->port == uh->dest &&
|
||||
uo_priv->offload->callbacks.gro_complete)
|
||||
break;
|
||||
}
|
||||
|
||||
if (uo_priv) {
|
||||
NAPI_GRO_CB(skb)->proto = uo_priv->offload->ipproto;
|
||||
err = uo_priv->offload->callbacks.gro_complete(skb,
|
||||
nhoff + sizeof(struct udphdr),
|
||||
uo_priv->offload);
|
||||
}
|
||||
|
||||
sk = (*lookup)(skb, uh->source, uh->dest);
|
||||
if (sk && udp_sk(sk)->gro_complete)
|
||||
err = udp_sk(sk)->gro_complete(sk, skb,
|
||||
nhoff + sizeof(struct udphdr));
|
||||
rcu_read_unlock();
|
||||
|
||||
if (skb->remcsum_offload)
|
||||
|
|
@ -426,6 +413,7 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff)
|
|||
|
||||
return err;
|
||||
}
|
||||
EXPORT_SYMBOL(udp_gro_complete);
|
||||
|
||||
static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
|
||||
{
|
||||
|
|
@ -440,7 +428,7 @@ static int udp4_gro_complete(struct sk_buff *skb, int nhoff)
|
|||
skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
|
||||
}
|
||||
|
||||
return udp_gro_complete(skb, nhoff);
|
||||
return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);
|
||||
}
|
||||
|
||||
static const struct net_offload udpv4_offload = {
|
||||
|
|
|
|||
|
|
@ -8,9 +8,10 @@ ipv6-objs := af_inet6.o anycast.o ip6_output.o ip6_input.o addrconf.o \
|
|||
addrlabel.o \
|
||||
route.o ip6_fib.o ipv6_sockglue.o ndisc.o udp.o udplite.o \
|
||||
raw.o icmp.o mcast.o reassembly.o tcp_ipv6.o ping.o \
|
||||
exthdrs.o datagram.o ip6_flowlabel.o inet6_connection_sock.o
|
||||
exthdrs.o datagram.o ip6_flowlabel.o inet6_connection_sock.o \
|
||||
udp_offload.o
|
||||
|
||||
ipv6-offload := ip6_offload.o tcpv6_offload.o udp_offload.o exthdrs_offload.o
|
||||
ipv6-offload := ip6_offload.o tcpv6_offload.o exthdrs_offload.o
|
||||
|
||||
ipv6-$(CONFIG_SYSCTL) = sysctl_net_ipv6.o
|
||||
ipv6-$(CONFIG_IPV6_MROUTE) += ip6mr.o
|
||||
|
|
|
|||
|
|
@ -64,6 +64,8 @@
|
|||
#include <asm/uaccess.h>
|
||||
#include <linux/mroute6.h>
|
||||
|
||||
#include "ip6_offload.h"
|
||||
|
||||
MODULE_AUTHOR("Cast of dozens");
|
||||
MODULE_DESCRIPTION("IPv6 protocol stack for Linux");
|
||||
MODULE_LICENSE("GPL");
|
||||
|
|
@ -959,6 +961,10 @@ static int __init inet6_init(void)
|
|||
if (err)
|
||||
goto udplitev6_fail;
|
||||
|
||||
err = udpv6_offload_init();
|
||||
if (err)
|
||||
goto udpv6_offload_fail;
|
||||
|
||||
err = tcpv6_init();
|
||||
if (err)
|
||||
goto tcpv6_fail;
|
||||
|
|
@ -988,6 +994,8 @@ static int __init inet6_init(void)
|
|||
ipv6_packet_fail:
|
||||
tcpv6_exit();
|
||||
tcpv6_fail:
|
||||
udpv6_offload_exit();
|
||||
udpv6_offload_fail:
|
||||
udplitev6_exit();
|
||||
udplitev6_fail:
|
||||
udpv6_exit();
|
||||
|
|
|
|||
|
|
@ -325,8 +325,6 @@ static int __init ipv6_offload_init(void)
|
|||
|
||||
if (tcpv6_offload_init() < 0)
|
||||
pr_crit("%s: Cannot add TCP protocol offload\n", __func__);
|
||||
if (udp_offload_init() < 0)
|
||||
pr_crit("%s: Cannot add UDP protocol offload\n", __func__);
|
||||
if (ipv6_exthdrs_offload_init() < 0)
|
||||
pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,8 @@
|
|||
#define __ip6_offload_h
|
||||
|
||||
int ipv6_exthdrs_offload_init(void);
|
||||
int udp_offload_init(void);
|
||||
int udpv6_offload_init(void);
|
||||
int udpv6_offload_exit(void);
|
||||
int tcpv6_offload_init(void);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ static struct sk_buff **udp6_gro_receive(struct sk_buff **head,
|
|||
|
||||
skip:
|
||||
NAPI_GRO_CB(skb)->is_ipv6 = 1;
|
||||
return udp_gro_receive(head, skb, uh);
|
||||
return udp_gro_receive(head, skb, uh, udp6_lib_lookup_skb);
|
||||
|
||||
flush:
|
||||
NAPI_GRO_CB(skb)->flush = 1;
|
||||
|
|
@ -173,7 +173,7 @@ static int udp6_gro_complete(struct sk_buff *skb, int nhoff)
|
|||
skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL;
|
||||
}
|
||||
|
||||
return udp_gro_complete(skb, nhoff);
|
||||
return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);
|
||||
}
|
||||
|
||||
static const struct net_offload udpv6_offload = {
|
||||
|
|
@ -184,7 +184,12 @@ static const struct net_offload udpv6_offload = {
|
|||
},
|
||||
};
|
||||
|
||||
int __init udp_offload_init(void)
|
||||
int udpv6_offload_init(void)
|
||||
{
|
||||
return inet6_add_offload(&udpv6_offload, IPPROTO_UDP);
|
||||
}
|
||||
|
||||
int udpv6_offload_exit(void)
|
||||
{
|
||||
return inet6_del_offload(&udpv6_offload, IPPROTO_UDP);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue