2 * IPV4 GSO/GRO offload support
3 * Linux INET implementation
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version
8 * 2 of the License, or (at your option) any later version.
13 #include <linux/skbuff.h>
14 #include <linux/init.h>
15 #include <net/protocol.h>
18 static int gre_gso_send_check(struct sk_buff
*skb
)
20 if (!skb
->encapsulation
)
25 static struct sk_buff
*gre_gso_segment(struct sk_buff
*skb
,
26 netdev_features_t features
)
28 struct sk_buff
*segs
= ERR_PTR(-EINVAL
);
29 netdev_features_t enc_features
;
31 struct gre_base_hdr
*greh
;
32 u16 mac_offset
= skb
->mac_header
;
33 int mac_len
= skb
->mac_len
;
34 __be16 protocol
= skb
->protocol
;
38 if (unlikely(skb_shinfo(skb
)->gso_type
&
48 if (unlikely(!pskb_may_pull(skb
, sizeof(*greh
))))
51 greh
= (struct gre_base_hdr
*)skb_transport_header(skb
);
53 ghl
= skb_inner_network_header(skb
) - skb_transport_header(skb
);
54 if (unlikely(ghl
< sizeof(*greh
)))
57 csum
= !!(greh
->flags
& GRE_CSUM
);
59 if (unlikely(!pskb_may_pull(skb
, ghl
)))
62 /* setup inner skb. */
63 skb
->protocol
= greh
->protocol
;
64 skb
->encapsulation
= 0;
67 skb_reset_mac_header(skb
);
68 skb_set_network_header(skb
, skb_inner_network_offset(skb
));
69 skb
->mac_len
= skb_inner_network_offset(skb
);
71 /* segment inner packet. */
72 enc_features
= skb
->dev
->hw_enc_features
& netif_skb_features(skb
);
73 segs
= skb_mac_gso_segment(skb
, enc_features
);
74 if (!segs
|| IS_ERR(segs
)) {
75 skb_gso_error_unwind(skb
, protocol
, ghl
, mac_offset
, mac_len
);
80 tnl_hlen
= skb_tnl_header_len(skb
);
86 if (skb_has_shared_frag(skb
)) {
89 err
= __skb_linearize(skb
);
97 greh
= (struct gre_base_hdr
*)(skb
->data
);
98 pcsum
= (__be32
*)(greh
+ 1);
100 *(__sum16
*)pcsum
= csum_fold(skb_checksum(skb
, 0, skb
->len
, 0));
102 __skb_push(skb
, tnl_hlen
- ghl
);
104 skb_reset_inner_headers(skb
);
105 skb
->encapsulation
= 1;
107 skb_reset_mac_header(skb
);
108 skb_set_network_header(skb
, mac_len
);
109 skb
->mac_len
= mac_len
;
110 skb
->protocol
= protocol
;
111 } while ((skb
= skb
->next
));
116 /* Compute the whole skb csum in s/w and store it, then verify GRO csum
117 * starting from gro_offset.
119 static __sum16
gro_skb_checksum(struct sk_buff
*skb
)
123 skb
->csum
= skb_checksum(skb
, 0, skb
->len
, 0);
124 NAPI_GRO_CB(skb
)->csum
= csum_sub(skb
->csum
,
125 csum_partial(skb
->data
, skb_gro_offset(skb
), 0));
126 sum
= csum_fold(NAPI_GRO_CB(skb
)->csum
);
127 if (unlikely(skb
->ip_summed
== CHECKSUM_COMPLETE
)) {
129 netdev_rx_csum_fault(skb
->dev
);
131 skb
->ip_summed
= CHECKSUM_COMPLETE
;
136 static struct sk_buff
**gre_gro_receive(struct sk_buff
**head
,
139 struct sk_buff
**pp
= NULL
;
141 const struct gre_base_hdr
*greh
;
142 unsigned int hlen
, grehlen
;
145 struct packet_offload
*ptype
;
148 off
= skb_gro_offset(skb
);
149 hlen
= off
+ sizeof(*greh
);
150 greh
= skb_gro_header_fast(skb
, off
);
151 if (skb_gro_header_hard(skb
, hlen
)) {
152 greh
= skb_gro_header_slow(skb
, hlen
, off
);
157 /* Only support version 0 and K (key), C (csum) flags. Note that
158 * although the support for the S (seq#) flag can be added easily
159 * for GRO, this is problematic for GSO hence can not be enabled
160 * here because a GRO pkt may end up in the forwarding path, thus
161 * requiring GSO support to break it up correctly.
163 if ((greh
->flags
& ~(GRE_KEY
|GRE_CSUM
)) != 0)
166 type
= greh
->protocol
;
169 ptype
= gro_find_receive_by_type(type
);
173 grehlen
= GRE_HEADER_SECTION
;
175 if (greh
->flags
& GRE_KEY
)
176 grehlen
+= GRE_HEADER_SECTION
;
178 if (greh
->flags
& GRE_CSUM
)
179 grehlen
+= GRE_HEADER_SECTION
;
181 hlen
= off
+ grehlen
;
182 if (skb_gro_header_hard(skb
, hlen
)) {
183 greh
= skb_gro_header_slow(skb
, hlen
, off
);
187 if (greh
->flags
& GRE_CSUM
) { /* Need to verify GRE csum first */
190 if (skb
->ip_summed
== CHECKSUM_COMPLETE
)
191 csum
= csum_fold(NAPI_GRO_CB(skb
)->csum
);
192 /* Don't trust csum error calculated/reported by h/w */
193 if (skb
->ip_summed
== CHECKSUM_NONE
|| csum
!= 0)
194 csum
= gro_skb_checksum(skb
);
196 /* GRE CSUM is the 1's complement of the 1's complement sum
197 * of the GRE hdr plus payload so it should add up to 0xffff
198 * (and 0 after csum_fold()) just like the IPv4 hdr csum.
205 for (p
= *head
; p
; p
= p
->next
) {
206 const struct gre_base_hdr
*greh2
;
208 if (!NAPI_GRO_CB(p
)->same_flow
)
211 /* The following checks are needed to ensure only pkts
212 * from the same tunnel are considered for aggregation.
213 * The criteria for "the same tunnel" includes:
214 * 1) same version (we only support version 0 here)
215 * 2) same protocol (we only support ETH_P_IP for now)
216 * 3) same set of flags
217 * 4) same key if the key field is present.
219 greh2
= (struct gre_base_hdr
*)(p
->data
+ off
);
221 if (greh2
->flags
!= greh
->flags
||
222 greh2
->protocol
!= greh
->protocol
) {
223 NAPI_GRO_CB(p
)->same_flow
= 0;
226 if (greh
->flags
& GRE_KEY
) {
228 if (*(__be32
*)(greh2
+1) != *(__be32
*)(greh
+1)) {
229 NAPI_GRO_CB(p
)->same_flow
= 0;
235 skb_gro_pull(skb
, grehlen
);
237 /* Adjusted NAPI_GRO_CB(skb)->csum after skb_gro_pull()*/
238 skb_gro_postpull_rcsum(skb
, greh
, grehlen
);
240 pp
= ptype
->callbacks
.gro_receive(head
, skb
);
245 NAPI_GRO_CB(skb
)->flush
|= flush
;
250 static int gre_gro_complete(struct sk_buff
*skb
, int nhoff
)
252 struct gre_base_hdr
*greh
= (struct gre_base_hdr
*)(skb
->data
+ nhoff
);
253 struct packet_offload
*ptype
;
254 unsigned int grehlen
= sizeof(*greh
);
258 type
= greh
->protocol
;
259 if (greh
->flags
& GRE_KEY
)
260 grehlen
+= GRE_HEADER_SECTION
;
262 if (greh
->flags
& GRE_CSUM
)
263 grehlen
+= GRE_HEADER_SECTION
;
266 ptype
= gro_find_complete_by_type(type
);
268 err
= ptype
->callbacks
.gro_complete(skb
, nhoff
+ grehlen
);
274 static const struct net_offload gre_offload
= {
276 .gso_send_check
= gre_gso_send_check
,
277 .gso_segment
= gre_gso_segment
,
278 .gro_receive
= gre_gro_receive
,
279 .gro_complete
= gre_gro_complete
,
283 static int __init
gre_offload_init(void)
285 return inet_add_offload(&gre_offload
, IPPROTO_GRE
);
287 device_initcall(gre_offload_init
);