Skip to content

Commit d4546c2

Browse files
davem330davem330
committed
net: Convert GRO SKB handling to list_head.
Manage pending per-NAPI GRO packets via list_head. Return an SKB pointer from the GRO receive handlers. When GRO receive handlers return non-NULL, it means that this SKB needs to be completed at this time and removed from the NAPI queue. Several operations are greatly simplified by this transformation, especially timing out the oldest SKB in the list when gro_count exceeds MAX_GRO_SKBS, and napi_gro_flush() which walks the queue in reverse order. Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent 9ff3b40 commit d4546c2

File tree

24 files changed

+133
-141
lines changed

24 files changed

+133
-141
lines changed

‎drivers/net/geneve.c‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -418,11 +418,12 @@ static int geneve_hlen(struct genevehdr *gh)
418418
return sizeof(*gh) + gh->opt_len * 4;
419419
}
420420

421-
static struct sk_buff **geneve_gro_receive(struct sock *sk,
422-
struct sk_buff **head,
423-
struct sk_buff *skb)
421+
static struct sk_buff *geneve_gro_receive(struct sock *sk,
422+
struct list_head *head,
423+
struct sk_buff *skb)
424424
{
425-
struct sk_buff *p, **pp = NULL;
425+
struct sk_buff *pp = NULL;
426+
struct sk_buff *p;
426427
struct genevehdr *gh, *gh2;
427428
unsigned int hlen, gh_len, off_gnv;
428429
const struct packet_offload *ptype;
@@ -449,7 +450,7 @@ static struct sk_buff **geneve_gro_receive(struct sock *sk,
449450
goto out;
450451
}
451452

452-
for (p = *head; p; p = p->next) {
453+
list_for_each_entry(p, head, list) {
453454
if (!NAPI_GRO_CB(p)->same_flow)
454455
continue;
455456

‎drivers/net/vxlan.c‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -568,11 +568,12 @@ static struct vxlanhdr *vxlan_gro_remcsum(struct sk_buff *skb,
568568
return vh;
569569
}
570570

571-
static struct sk_buff **vxlan_gro_receive(struct sock *sk,
572-
struct sk_buff **head,
573-
struct sk_buff *skb)
571+
static struct sk_buff *vxlan_gro_receive(struct sock *sk,
572+
struct list_head *head,
573+
struct sk_buff *skb)
574574
{
575-
struct sk_buff *p, **pp = NULL;
575+
struct sk_buff *pp = NULL;
576+
struct sk_buff *p;
576577
struct vxlanhdr *vh, *vh2;
577578
unsigned int hlen, off_vx;
578579
int flush = 1;
@@ -607,7 +608,7 @@ static struct sk_buff **vxlan_gro_receive(struct sock *sk,
607608

608609
skb_gro_pull(skb, sizeof(struct vxlanhdr)); /* pull vxlan header */
609610

610-
for (p = *head; p; p = p->next) {
611+
list_for_each_entry(p, head, list) {
611612
if (!NAPI_GRO_CB(p)->same_flow)
612613
continue;
613614

‎include/linux/etherdevice.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,7 @@ struct net_device *devm_alloc_etherdev_mqs(struct device *dev, int sizeof_priv,
5959
unsigned int rxqs);
6060
#define devm_alloc_etherdev(dev, sizeof_priv) devm_alloc_etherdev_mqs(dev, sizeof_priv, 1, 1)
6161

62-
struct sk_buff **eth_gro_receive(struct sk_buff **head,
63-
struct sk_buff *skb);
62+
struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb);
6463
int eth_gro_complete(struct sk_buff *skb, int nhoff);
6564

6665
/* Reserved Ethernet Addresses per IEEE 802.1Q */

‎include/linux/netdevice.h‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,7 @@ struct napi_struct {
322322
int poll_owner;
323323
#endif
324324
struct net_device *dev;
325-
struct sk_buff *gro_list;
325+
struct list_head gro_list;
326326
struct sk_buff *skb;
327327
struct hrtimer timer;
328328
struct list_head dev_list;
@@ -2255,10 +2255,10 @@ static inline int gro_recursion_inc_test(struct sk_buff *skb)
22552255
return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;
22562256
}
22572257

2258-
typedef struct sk_buff **(*gro_receive_t)(struct sk_buff **, struct sk_buff *);
2259-
static inline struct sk_buff **call_gro_receive(gro_receive_t cb,
2260-
struct sk_buff **head,
2261-
struct sk_buff *skb)
2258+
typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *);
2259+
static inline struct sk_buff *call_gro_receive(gro_receive_t cb,
2260+
struct list_head *head,
2261+
struct sk_buff *skb)
22622262
{
22632263
if (unlikely(gro_recursion_inc_test(skb))) {
22642264
NAPI_GRO_CB(skb)->flush |= 1;
@@ -2268,12 +2268,12 @@ static inline struct sk_buff **call_gro_receive(gro_receive_t cb,
22682268
return cb(head, skb);
22692269
}
22702270

2271-
typedef struct sk_buff **(*gro_receive_sk_t)(struct sock *, struct sk_buff **,
2272-
struct sk_buff *);
2273-
static inline struct sk_buff **call_gro_receive_sk(gro_receive_sk_t cb,
2274-
struct sock *sk,
2275-
struct sk_buff **head,
2276-
struct sk_buff *skb)
2271+
typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *,
2272+
struct sk_buff *);
2273+
static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb,
2274+
struct sock *sk,
2275+
struct list_head *head,
2276+
struct sk_buff *skb)
22772277
{
22782278
if (unlikely(gro_recursion_inc_test(skb))) {
22792279
NAPI_GRO_CB(skb)->flush |= 1;
@@ -2299,8 +2299,8 @@ struct packet_type {
22992299
struct offload_callbacks {
23002300
struct sk_buff *(*gso_segment)(struct sk_buff *skb,
23012301
netdev_features_t features);
2302-
struct sk_buff **(*gro_receive)(struct sk_buff **head,
2303-
struct sk_buff *skb);
2302+
struct sk_buff *(*gro_receive)(struct list_head *head,
2303+
struct sk_buff *skb);
23042304
int (*gro_complete)(struct sk_buff *skb, int nhoff);
23052305
};
23062306

@@ -2568,7 +2568,7 @@ struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
25682568
struct net_device *dev_get_by_napi_id(unsigned int napi_id);
25692569
int netdev_get_name(struct net *net, char *name, int ifindex);
25702570
int dev_restart(struct net_device *dev);
2571-
int skb_gro_receive(struct sk_buff **head, struct sk_buff *skb);
2571+
int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
25722572

25732573
static inline unsigned int skb_gro_offset(const struct sk_buff *skb)
25742574
{
@@ -2784,13 +2784,13 @@ static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
27842784
}
27852785

27862786
#ifdef CONFIG_XFRM_OFFLOAD
2787-
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2787+
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
27882788
{
27892789
if (PTR_ERR(pp) != -EINPROGRESS)
27902790
NAPI_GRO_CB(skb)->flush |= flush;
27912791
}
27922792
#else
2793-
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff **pp, int flush)
2793+
static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush)
27942794
{
27952795
NAPI_GRO_CB(skb)->flush |= flush;
27962796
}

‎include/linux/skbuff.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,8 @@ struct sk_buff {
677677
int ip_defrag_offset;
678678
};
679679
};
680-
struct rb_node rbnode; /* used in netem & tcp stack */
680+
struct rb_node rbnode; /* used in netem & tcp stack */
681+
struct list_head list;
681682
};
682683
struct sock *sk;
683684

‎include/linux/udp.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ struct udp_sock {
7474
void (*encap_destroy)(struct sock *sk);
7575

7676
/* GRO functions for UDP socket */
77-
struct sk_buff ** (*gro_receive)(struct sock *sk,
78-
struct sk_buff **head,
77+
struct sk_buff * (*gro_receive)(struct sock *sk,
78+
struct list_head *head,
7979
struct sk_buff *skb);
8080
int (*gro_complete)(struct sock *sk,
8181
struct sk_buff *skb,

‎include/net/inet_common.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ int inet_ctl_sock_create(struct sock **sk, unsigned short family,
4343
int inet_recv_error(struct sock *sk, struct msghdr *msg, int len,
4444
int *addr_len);
4545

46-
struct sk_buff **inet_gro_receive(struct sk_buff **head, struct sk_buff *skb);
46+
struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb);
4747
int inet_gro_complete(struct sk_buff *skb, int nhoff);
4848
struct sk_buff *inet_gso_segment(struct sk_buff *skb,
4949
netdev_features_t features);

‎include/net/tcp.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1788,7 +1788,7 @@ void tcp_v4_destroy_sock(struct sock *sk);
17881788

17891789
struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
17901790
netdev_features_t features);
1791-
struct sk_buff **tcp_gro_receive(struct sk_buff **head, struct sk_buff *skb);
1791+
struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
17921792
int tcp_gro_complete(struct sk_buff *skb);
17931793

17941794
void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr);

‎include/net/udp.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,8 @@ static inline void udp_csum_pull_header(struct sk_buff *skb)
170170
typedef struct sock *(*udp_lookup_t)(struct sk_buff *skb, __be16 sport,
171171
__be16 dport);
172172

173-
struct sk_buff **udp_gro_receive(struct sk_buff **head, struct sk_buff *skb,
174-
struct udphdr *uh, udp_lookup_t lookup);
173+
struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,
174+
struct udphdr *uh, udp_lookup_t lookup);
175175
int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup);
176176

177177
struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,

‎include/net/udp_tunnel.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,9 +65,9 @@ static inline int udp_sock_create(struct net *net,
6565

6666
typedef int (*udp_tunnel_encap_rcv_t)(struct sock *sk, struct sk_buff *skb);
6767
typedef void (*udp_tunnel_encap_destroy_t)(struct sock *sk);
68-
typedef struct sk_buff **(*udp_tunnel_gro_receive_t)(struct sock *sk,
69-
struct sk_buff **head,
70-
struct sk_buff *skb);
68+
typedef struct sk_buff *(*udp_tunnel_gro_receive_t)(struct sock *sk,
69+
struct list_head *head,
70+
struct sk_buff *skb);
7171
typedef int (*udp_tunnel_gro_complete_t)(struct sock *sk, struct sk_buff *skb,
7272
int nhoff);
7373

0 commit comments

Comments
 (0)