?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/mm: Add TLB purge to free pmd/pte page interfaces
[linux/fpc-iii.git] / net / ipv6 / ip6_output.c
blobea14466cdca8bbbce5d12ade52642bb1106a928f
1 /*
2 * IPv6 output functions
3 * Linux INET6 implementation
4 *
5 * Authors:
6 * Pedro Roque <roque@di.fc.ul.pt>
7 *
8 * Based on linux/net/ipv4/ip_output.c
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 *
15 * Changes:
16 * A.N.Kuznetsov : airthmetics in fragmentation.
17 * extension headers are implemented.
18 * route changes now work.
19 * ip6_forward does not confuse sniffers.
20 * etc.
21 *
22 * H. von Brand : Added missing #include <linux/string.h>
23 * Imran Patel : frag id should be in NBO
24 * Kazunori MIYAZAWA @USAGI
25 * : add ip6_append_data and related functions
26 * for datagram xmit
27 */
28
29 #include <linux/errno.h>
30 #include <linux/kernel.h>
31 #include <linux/string.h>
32 #include <linux/socket.h>
33 #include <linux/net.h>
34 #include <linux/netdevice.h>
35 #include <linux/if_arp.h>
36 #include <linux/in6.h>
37 #include <linux/tcp.h>
38 #include <linux/route.h>
39 #include <linux/module.h>
40 #include <linux/slab.h>
41
42 #include <linux/netfilter.h>
43 #include <linux/netfilter_ipv6.h>
44
45 #include <net/sock.h>
46 #include <net/snmp.h>
47
48 #include <net/ipv6.h>
49 #include <net/ndisc.h>
50 #include <net/protocol.h>
51 #include <net/ip6_route.h>
52 #include <net/addrconf.h>
53 #include <net/rawv6.h>
54 #include <net/icmp.h>
55 #include <net/xfrm.h>
56 #include <net/checksum.h>
57 #include <linux/mroute6.h>
58 #include <net/l3mdev.h>
59 #include <net/lwtunnel.h>
60
61 static int ip6_finish_output2(struct net *net, struct sock *sk, struct sk_buff *skb)
62 {
63 struct dst_entry *dst = skb_dst(skb);
64 struct net_device *dev = dst->dev;
65 struct neighbour *neigh;
66 struct in6_addr *nexthop;
67 int ret;
68
69 skb->protocol = htons(ETH_P_IPV6);
70 skb->dev = dev;
71
72 if (ipv6_addr_is_multicast(&ipv6_hdr(skb)->daddr)) {
73 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
74
75 if (!(dev->flags & IFF_LOOPBACK) && sk_mc_loop(sk) &&
76 ((mroute6_socket(net, skb) &&
77 !(IP6CB(skb)->flags & IP6SKB_FORWARDED)) ||
78 ipv6_chk_mcast_addr(dev, &ipv6_hdr(skb)->daddr,
79 &ipv6_hdr(skb)->saddr))) {
80 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
81
82 /* Do not check for IFF_ALLMULTI; multicast routing
83 is not supported in any case.
84 */
85 if (newskb)
86 NF_HOOK(NFPROTO_IPV6, NF_INET_POST_ROUTING,
87 net, sk, newskb, NULL, newskb->dev,
88 dev_loopback_xmit);
89
90 if (ipv6_hdr(skb)->hop_limit == 0) {
91 IP6_INC_STATS(net, idev,
92 IPSTATS_MIB_OUTDISCARDS);
93 kfree_skb(skb);
94 return 0;
95 }
96 }
97
98 IP6_UPD_PO_STATS(net, idev, IPSTATS_MIB_OUTMCAST, skb->len);
99
100 if (IPV6_ADDR_MC_SCOPE(&ipv6_hdr(skb)->daddr) <=
101 IPV6_ADDR_SCOPE_NODELOCAL &&
102 !(dev->flags & IFF_LOOPBACK)) {
103 kfree_skb(skb);
104 return 0;
105 }
106 }
107
108 if (lwtunnel_xmit_redirect(dst->lwtstate)) {
109 int res = lwtunnel_xmit(skb);
110
111 if (res < 0 || res == LWTUNNEL_XMIT_DONE)
112 return res;
113 }
114
115 rcu_read_lock_bh();
116 nexthop = rt6_nexthop((struct rt6_info *)dst, &ipv6_hdr(skb)->daddr);
117 neigh = __ipv6_neigh_lookup_noref(dst->dev, nexthop);
118 if (unlikely(!neigh))
119 neigh = __neigh_create(&nd_tbl, nexthop, dst->dev, false);
120 if (!IS_ERR(neigh)) {
121 ret = dst_neigh_output(dst, neigh, skb);
122 rcu_read_unlock_bh();
123 return ret;
124 }
125 rcu_read_unlock_bh();
126
127 IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTNOROUTES);
128 kfree_skb(skb);
129 return -EINVAL;
130 }
131
132 static int ip6_finish_output(struct net *net, struct sock *sk, struct sk_buff *skb)
133 {
134 if ((skb->len > ip6_skb_dst_mtu(skb) && !skb_is_gso(skb)) ||
135 dst_allfrag(skb_dst(skb)) ||
136 (IP6CB(skb)->frag_max_size && skb->len > IP6CB(skb)->frag_max_size))
137 return ip6_fragment(net, sk, skb, ip6_finish_output2);
138 else
139 return ip6_finish_output2(net, sk, skb);
140 }
141
142 int ip6_output(struct net *net, struct sock *sk, struct sk_buff *skb)
143 {
144 struct net_device *dev = skb_dst(skb)->dev;
145 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
146
147 if (unlikely(idev->cnf.disable_ipv6)) {
148 IP6_INC_STATS(net, idev, IPSTATS_MIB_OUTDISCARDS);
149 kfree_skb(skb);
150 return 0;
151 }
152
153 return NF_HOOK_COND(NFPROTO_IPV6, NF_INET_POST_ROUTING,
154 net, sk, skb, NULL, dev,
155 ip6_finish_output,
156 !(IP6CB(skb)->flags & IP6SKB_REROUTED));
157 }
158
159 bool ip6_autoflowlabel(struct net *net, const struct ipv6_pinfo *np)
160 {
161 if (!np->autoflowlabel_set)
162 return ip6_default_np_autolabel(net);
163 else
164 return np->autoflowlabel;
165 }
166
167 /*
168 * xmit an sk_buff (used by TCP, SCTP and DCCP)
169 * Note : socket lock is not held for SYNACK packets, but might be modified
170 * by calls to skb_set_owner_w() and ipv6_local_error(),
171 * which are using proper atomic operations or spinlocks.
172 */
173 int ip6_xmit(const struct sock *sk, struct sk_buff *skb, struct flowi6 *fl6,
174 __u32 mark, struct ipv6_txoptions *opt, int tclass)
175 {
176 struct net *net = sock_net(sk);
177 const struct ipv6_pinfo *np = inet6_sk(sk);
178 struct in6_addr *first_hop = &fl6->daddr;
179 struct dst_entry *dst = skb_dst(skb);
180 struct ipv6hdr *hdr;
181 u8 proto = fl6->flowi6_proto;
182 int seg_len = skb->len;
183 int hlimit = -1;
184 u32 mtu;
185
186 if (opt) {
187 unsigned int head_room;
188
189 /* First: exthdrs may take lots of space (~8K for now)
190 MAX_HEADER is not enough.
191 */
192 head_room = opt->opt_nflen + opt->opt_flen;
193 seg_len += head_room;
194 head_room += sizeof(struct ipv6hdr) + LL_RESERVED_SPACE(dst->dev);
195
196 if (skb_headroom(skb) < head_room) {
197 struct sk_buff *skb2 = skb_realloc_headroom(skb, head_room);
198 if (!skb2) {
199 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
200 IPSTATS_MIB_OUTDISCARDS);
201 kfree_skb(skb);
202 return -ENOBUFS;
203 }
204 consume_skb(skb);
205 skb = skb2;
206 /* skb_set_owner_w() changes sk->sk_wmem_alloc atomically,
207 * it is safe to call in our context (socket lock not held)
208 */
209 skb_set_owner_w(skb, (struct sock *)sk);
210 }
211 if (opt->opt_flen)
212 ipv6_push_frag_opts(skb, opt, &proto);
213 if (opt->opt_nflen)
214 ipv6_push_nfrag_opts(skb, opt, &proto, &first_hop);
215 }
216
217 skb_push(skb, sizeof(struct ipv6hdr));
218 skb_reset_network_header(skb);
219 hdr = ipv6_hdr(skb);
220
221 /*
222 * Fill in the IPv6 header
223 */
224 if (np)
225 hlimit = np->hop_limit;
226 if (hlimit < 0)
227 hlimit = ip6_dst_hoplimit(dst);
228
229 ip6_flow_hdr(hdr, tclass, ip6_make_flowlabel(net, skb, fl6->flowlabel,
230 ip6_autoflowlabel(net, np), fl6));
231
232 hdr->payload_len = htons(seg_len);
233 hdr->nexthdr = proto;
234 hdr->hop_limit = hlimit;
235
236 hdr->saddr = fl6->saddr;
237 hdr->daddr = *first_hop;
238
239 skb->protocol = htons(ETH_P_IPV6);
240 skb->priority = sk->sk_priority;
241 skb->mark = mark;
242
243 mtu = dst_mtu(dst);
244 if ((skb->len <= mtu) || skb->ignore_df || skb_is_gso(skb)) {
245 IP6_UPD_PO_STATS(net, ip6_dst_idev(skb_dst(skb)),
246 IPSTATS_MIB_OUT, skb->len);
247
248 /* if egress device is enslaved to an L3 master device pass the
249 * skb to its handler for processing
250 */
251 skb = l3mdev_ip6_out((struct sock *)sk, skb);
252 if (unlikely(!skb))
253 return 0;
254
255 /* hooks should never assume socket lock is held.
256 * we promote our socket to non const
257 */
258 return NF_HOOK(NFPROTO_IPV6, NF_INET_LOCAL_OUT,
259 net, (struct sock *)sk, skb, NULL, dst->dev,
260 dst_output);
261 }
262
263 skb->dev = dst->dev;
264 /* ipv6_local_error() does not require socket lock,
265 * we promote our socket to non const
266 */
267 ipv6_local_error((struct sock *)sk, EMSGSIZE, fl6, mtu);
268
269 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)), IPSTATS_MIB_FRAGFAILS);
270 kfree_skb(skb);
271 return -EMSGSIZE;
272 }
273 EXPORT_SYMBOL(ip6_xmit);
274
275 static int ip6_call_ra_chain(struct sk_buff *skb, int sel)
276 {
277 struct ip6_ra_chain *ra;
278 struct sock *last = NULL;
279
280 read_lock(&ip6_ra_lock);
281 for (ra = ip6_ra_chain; ra; ra = ra->next) {
282 struct sock *sk = ra->sk;
283 if (sk && ra->sel == sel &&
284 (!sk->sk_bound_dev_if ||
285 sk->sk_bound_dev_if == skb->dev->ifindex)) {
286 if (last) {
287 struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
288 if (skb2)
289 rawv6_rcv(last, skb2);
290 }
291 last = sk;
292 }
293 }
294
295 if (last) {
296 rawv6_rcv(last, skb);
297 read_unlock(&ip6_ra_lock);
298 return 1;
299 }
300 read_unlock(&ip6_ra_lock);
301 return 0;
302 }
303
304 static int ip6_forward_proxy_check(struct sk_buff *skb)
305 {
306 struct ipv6hdr *hdr = ipv6_hdr(skb);
307 u8 nexthdr = hdr->nexthdr;
308 __be16 frag_off;
309 int offset;
310
311 if (ipv6_ext_hdr(nexthdr)) {
312 offset = ipv6_skip_exthdr(skb, sizeof(*hdr), &nexthdr, &frag_off);
313 if (offset < 0)
314 return 0;
315 } else
316 offset = sizeof(struct ipv6hdr);
317
318 if (nexthdr == IPPROTO_ICMPV6) {
319 struct icmp6hdr *icmp6;
320
321 if (!pskb_may_pull(skb, (skb_network_header(skb) +
322 offset + 1 - skb->data)))
323 return 0;
324
325 icmp6 = (struct icmp6hdr *)(skb_network_header(skb) + offset);
326
327 switch (icmp6->icmp6_type) {
328 case NDISC_ROUTER_SOLICITATION:
329 case NDISC_ROUTER_ADVERTISEMENT:
330 case NDISC_NEIGHBOUR_SOLICITATION:
331 case NDISC_NEIGHBOUR_ADVERTISEMENT:
332 case NDISC_REDIRECT:
333 /* For reaction involving unicast neighbor discovery
334 * message destined to the proxied address, pass it to
335 * input function.
336 */
337 return 1;
338 default:
339 break;
340 }
341 }
342
343 /*
344 * The proxying router can't forward traffic sent to a link-local
345 * address, so signal the sender and discard the packet. This
346 * behavior is clarified by the MIPv6 specification.
347 */
348 if (ipv6_addr_type(&hdr->daddr) & IPV6_ADDR_LINKLOCAL) {
349 dst_link_failure(skb);
350 return -1;
351 }
352
353 return 0;
354 }
355
356 static inline int ip6_forward_finish(struct net *net, struct sock *sk,
357 struct sk_buff *skb)
358 {
359 struct dst_entry *dst = skb_dst(skb);
360
361 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTFORWDATAGRAMS);
362 __IP6_ADD_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_OUTOCTETS, skb->len);
363
364 return dst_output(net, sk, skb);
365 }
366
367 static unsigned int ip6_dst_mtu_forward(const struct dst_entry *dst)
368 {
369 unsigned int mtu;
370 struct inet6_dev *idev;
371
372 if (dst_metric_locked(dst, RTAX_MTU)) {
373 mtu = dst_metric_raw(dst, RTAX_MTU);
374 if (mtu)
375 return mtu;
376 }
377
378 mtu = IPV6_MIN_MTU;
379 rcu_read_lock();
380 idev = __in6_dev_get(dst->dev);
381 if (idev)
382 mtu = idev->cnf.mtu6;
383 rcu_read_unlock();
384
385 return mtu;
386 }
387
388 static bool ip6_pkt_too_big(const struct sk_buff *skb, unsigned int mtu)
389 {
390 if (skb->len <= mtu)
391 return false;
392
393 /* ipv6 conntrack defrag sets max_frag_size + ignore_df */
394 if (IP6CB(skb)->frag_max_size && IP6CB(skb)->frag_max_size > mtu)
395 return true;
396
397 if (skb->ignore_df)
398 return false;
399
400 if (skb_is_gso(skb) && skb_gso_validate_mtu(skb, mtu))
401 return false;
402
403 return true;
404 }
405
406 int ip6_forward(struct sk_buff *skb)
407 {
408 struct dst_entry *dst = skb_dst(skb);
409 struct ipv6hdr *hdr = ipv6_hdr(skb);
410 struct inet6_skb_parm *opt = IP6CB(skb);
411 struct net *net = dev_net(dst->dev);
412 u32 mtu;
413
414 if (net->ipv6.devconf_all->forwarding == 0)
415 goto error;
416
417 if (skb->pkt_type != PACKET_HOST)
418 goto drop;
419
420 if (unlikely(skb->sk))
421 goto drop;
422
423 if (skb_warn_if_lro(skb))
424 goto drop;
425
426 if (!xfrm6_policy_check(NULL, XFRM_POLICY_FWD, skb)) {
427 __IP6_INC_STATS(net, ip6_dst_idev(dst),
428 IPSTATS_MIB_INDISCARDS);
429 goto drop;
430 }
431
432 skb_forward_csum(skb);
433
434 /*
435 * We DO NOT make any processing on
436 * RA packets, pushing them to user level AS IS
437 * without ane WARRANTY that application will be able
438 * to interpret them. The reason is that we
439 * cannot make anything clever here.
440 *
441 * We are not end-node, so that if packet contains
442 * AH/ESP, we cannot make anything.
443 * Defragmentation also would be mistake, RA packets
444 * cannot be fragmented, because there is no warranty
445 * that different fragments will go along one path. --ANK
446 */
447 if (unlikely(opt->flags & IP6SKB_ROUTERALERT)) {
448 if (ip6_call_ra_chain(skb, ntohs(opt->ra)))
449 return 0;
450 }
451
452 /*
453 * check and decrement ttl
454 */
455 if (hdr->hop_limit <= 1) {
456 /* Force OUTPUT device used as source address */
457 skb->dev = dst->dev;
458 icmpv6_send(skb, ICMPV6_TIME_EXCEED, ICMPV6_EXC_HOPLIMIT, 0);
459 __IP6_INC_STATS(net, ip6_dst_idev(dst),
460 IPSTATS_MIB_INHDRERRORS);
461
462 kfree_skb(skb);
463 return -ETIMEDOUT;
464 }
465
466 /* XXX: idev->cnf.proxy_ndp? */
467 if (net->ipv6.devconf_all->proxy_ndp &&
468 pneigh_lookup(&nd_tbl, net, &hdr->daddr, skb->dev, 0)) {
469 int proxied = ip6_forward_proxy_check(skb);
470 if (proxied > 0)
471 return ip6_input(skb);
472 else if (proxied < 0) {
473 __IP6_INC_STATS(net, ip6_dst_idev(dst),
474 IPSTATS_MIB_INDISCARDS);
475 goto drop;
476 }
477 }
478
479 if (!xfrm6_route_forward(skb)) {
480 __IP6_INC_STATS(net, ip6_dst_idev(dst),
481 IPSTATS_MIB_INDISCARDS);
482 goto drop;
483 }
484 dst = skb_dst(skb);
485
486 /* IPv6 specs say nothing about it, but it is clear that we cannot
487 send redirects to source routed frames.
488 We don't send redirects to frames decapsulated from IPsec.
489 */
490 if (IP6CB(skb)->iif == dst->dev->ifindex &&
491 opt->srcrt == 0 && !skb_sec_path(skb)) {
492 struct in6_addr *target = NULL;
493 struct inet_peer *peer;
494 struct rt6_info *rt;
495
496 /*
497 * incoming and outgoing devices are the same
498 * send a redirect.
499 */
500
501 rt = (struct rt6_info *) dst;
502 if (rt->rt6i_flags & RTF_GATEWAY)
503 target = &rt->rt6i_gateway;
504 else
505 target = &hdr->daddr;
506
507 peer = inet_getpeer_v6(net->ipv6.peers, &hdr->daddr, 1);
508
509 /* Limit redirects both by destination (here)
510 and by source (inside ndisc_send_redirect)
511 */
512 if (inet_peer_xrlim_allow(peer, 1*HZ))
513 ndisc_send_redirect(skb, target);
514 if (peer)
515 inet_putpeer(peer);
516 } else {
517 int addrtype = ipv6_addr_type(&hdr->saddr);
518
519 /* This check is security critical. */
520 if (addrtype == IPV6_ADDR_ANY ||
521 addrtype & (IPV6_ADDR_MULTICAST | IPV6_ADDR_LOOPBACK))
522 goto error;
523 if (addrtype & IPV6_ADDR_LINKLOCAL) {
524 icmpv6_send(skb, ICMPV6_DEST_UNREACH,
525 ICMPV6_NOT_NEIGHBOUR, 0);
526 goto error;
527 }
528 }
529
530 mtu = ip6_dst_mtu_forward(dst);
531 if (mtu < IPV6_MIN_MTU)
532 mtu = IPV6_MIN_MTU;
533
534 if (ip6_pkt_too_big(skb, mtu)) {
535 /* Again, force OUTPUT device used as source address */
536 skb->dev = dst->dev;
537 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
538 __IP6_INC_STATS(net, ip6_dst_idev(dst),
539 IPSTATS_MIB_INTOOBIGERRORS);
540 __IP6_INC_STATS(net, ip6_dst_idev(dst),
541 IPSTATS_MIB_FRAGFAILS);
542 kfree_skb(skb);
543 return -EMSGSIZE;
544 }
545
546 if (skb_cow(skb, dst->dev->hard_header_len)) {
547 __IP6_INC_STATS(net, ip6_dst_idev(dst),
548 IPSTATS_MIB_OUTDISCARDS);
549 goto drop;
550 }
551
552 hdr = ipv6_hdr(skb);
553
554 /* Mangling hops number delayed to point after skb COW */
555
556 hdr->hop_limit--;
557
558 return NF_HOOK(NFPROTO_IPV6, NF_INET_FORWARD,
559 net, NULL, skb, skb->dev, dst->dev,
560 ip6_forward_finish);
561
562 error:
563 __IP6_INC_STATS(net, ip6_dst_idev(dst), IPSTATS_MIB_INADDRERRORS);
564 drop:
565 kfree_skb(skb);
566 return -EINVAL;
567 }
568
569 static void ip6_copy_metadata(struct sk_buff *to, struct sk_buff *from)
570 {
571 to->pkt_type = from->pkt_type;
572 to->priority = from->priority;
573 to->protocol = from->protocol;
574 skb_dst_drop(to);
575 skb_dst_set(to, dst_clone(skb_dst(from)));
576 to->dev = from->dev;
577 to->mark = from->mark;
578
579 skb_copy_hash(to, from);
580
581 #ifdef CONFIG_NET_SCHED
582 to->tc_index = from->tc_index;
583 #endif
584 nf_copy(to, from);
585 skb_copy_secmark(to, from);
586 }
587
588 int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
589 int (*output)(struct net *, struct sock *, struct sk_buff *))
590 {
591 struct sk_buff *frag;
592 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
593 struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ?
594 inet6_sk(skb->sk) : NULL;
595 struct ipv6hdr *tmp_hdr;
596 struct frag_hdr *fh;
597 unsigned int mtu, hlen, left, len;
598 int hroom, troom;
599 __be32 frag_id;
600 int ptr, offset = 0, err = 0;
601 u8 *prevhdr, nexthdr = 0;
602
603 err = ip6_find_1stfragopt(skb, &prevhdr);
604 if (err < 0)
605 goto fail;
606 hlen = err;
607 nexthdr = *prevhdr;
608
609 mtu = ip6_skb_dst_mtu(skb);
610
611 /* We must not fragment if the socket is set to force MTU discovery
612 * or if the skb it not generated by a local socket.
613 */
614 if (unlikely(!skb->ignore_df && skb->len > mtu))
615 goto fail_toobig;
616
617 if (IP6CB(skb)->frag_max_size) {
618 if (IP6CB(skb)->frag_max_size > mtu)
619 goto fail_toobig;
620
621 /* don't send fragments larger than what we received */
622 mtu = IP6CB(skb)->frag_max_size;
623 if (mtu < IPV6_MIN_MTU)
624 mtu = IPV6_MIN_MTU;
625 }
626
627 if (np && np->frag_size < mtu) {
628 if (np->frag_size)
629 mtu = np->frag_size;
630 }
631 if (mtu < hlen + sizeof(struct frag_hdr) + 8)
632 goto fail_toobig;
633 mtu -= hlen + sizeof(struct frag_hdr);
634
635 frag_id = ipv6_select_ident(net, &ipv6_hdr(skb)->daddr,
636 &ipv6_hdr(skb)->saddr);
637
638 if (skb->ip_summed == CHECKSUM_PARTIAL &&
639 (err = skb_checksum_help(skb)))
640 goto fail;
641
642 hroom = LL_RESERVED_SPACE(rt->dst.dev);
643 if (skb_has_frag_list(skb)) {
644 int first_len = skb_pagelen(skb);
645 struct sk_buff *frag2;
646
647 if (first_len - hlen > mtu ||
648 ((first_len - hlen) & 7) ||
649 skb_cloned(skb) ||
650 skb_headroom(skb) < (hroom + sizeof(struct frag_hdr)))
651 goto slow_path;
652
653 skb_walk_frags(skb, frag) {
654 /* Correct geometry. */
655 if (frag->len > mtu ||
656 ((frag->len & 7) && frag->next) ||
657 skb_headroom(frag) < (hlen + hroom + sizeof(struct frag_hdr)))
658 goto slow_path_clean;
659
660 /* Partially cloned skb? */
661 if (skb_shared(frag))
662 goto slow_path_clean;
663
664 BUG_ON(frag->sk);
665 if (skb->sk) {
666 frag->sk = skb->sk;
667 frag->destructor = sock_wfree;
668 }
669 skb->truesize -= frag->truesize;
670 }
671
672 err = 0;
673 offset = 0;
674 /* BUILD HEADER */
675
676 *prevhdr = NEXTHDR_FRAGMENT;
677 tmp_hdr = kmemdup(skb_network_header(skb), hlen, GFP_ATOMIC);
678 if (!tmp_hdr) {
679 err = -ENOMEM;
680 goto fail;
681 }
682 frag = skb_shinfo(skb)->frag_list;
683 skb_frag_list_init(skb);
684
685 __skb_pull(skb, hlen);
686 fh = (struct frag_hdr *)__skb_push(skb, sizeof(struct frag_hdr));
687 __skb_push(skb, hlen);
688 skb_reset_network_header(skb);
689 memcpy(skb_network_header(skb), tmp_hdr, hlen);
690
691 fh->nexthdr = nexthdr;
692 fh->reserved = 0;
693 fh->frag_off = htons(IP6_MF);
694 fh->identification = frag_id;
695
696 first_len = skb_pagelen(skb);
697 skb->data_len = first_len - skb_headlen(skb);
698 skb->len = first_len;
699 ipv6_hdr(skb)->payload_len = htons(first_len -
700 sizeof(struct ipv6hdr));
701
702 dst_hold(&rt->dst);
703
704 for (;;) {
705 /* Prepare header of the next frame,
706 * before previous one went down. */
707 if (frag) {
708 frag->ip_summed = CHECKSUM_NONE;
709 skb_reset_transport_header(frag);
710 fh = (struct frag_hdr *)__skb_push(frag, sizeof(struct frag_hdr));
711 __skb_push(frag, hlen);
712 skb_reset_network_header(frag);
713 memcpy(skb_network_header(frag), tmp_hdr,
714 hlen);
715 offset += skb->len - hlen - sizeof(struct frag_hdr);
716 fh->nexthdr = nexthdr;
717 fh->reserved = 0;
718 fh->frag_off = htons(offset);
719 if (frag->next)
720 fh->frag_off |= htons(IP6_MF);
721 fh->identification = frag_id;
722 ipv6_hdr(frag)->payload_len =
723 htons(frag->len -
724 sizeof(struct ipv6hdr));
725 ip6_copy_metadata(frag, skb);
726 }
727
728 err = output(net, sk, skb);
729 if (!err)
730 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
731 IPSTATS_MIB_FRAGCREATES);
732
733 if (err || !frag)
734 break;
735
736 skb = frag;
737 frag = skb->next;
738 skb->next = NULL;
739 }
740
741 kfree(tmp_hdr);
742
743 if (err == 0) {
744 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
745 IPSTATS_MIB_FRAGOKS);
746 ip6_rt_put(rt);
747 return 0;
748 }
749
750 kfree_skb_list(frag);
751
752 IP6_INC_STATS(net, ip6_dst_idev(&rt->dst),
753 IPSTATS_MIB_FRAGFAILS);
754 ip6_rt_put(rt);
755 return err;
756
757 slow_path_clean:
758 skb_walk_frags(skb, frag2) {
759 if (frag2 == frag)
760 break;
761 frag2->sk = NULL;
762 frag2->destructor = NULL;
763 skb->truesize += frag2->truesize;
764 }
765 }
766
767 slow_path:
768 left = skb->len - hlen; /* Space per frame */
769 ptr = hlen; /* Where to start from */
770
771 /*
772 * Fragment the datagram.
773 */
774
775 troom = rt->dst.dev->needed_tailroom;
776
777 /*
778 * Keep copying data until we run out.
779 */
780 while (left > 0) {
781 u8 *fragnexthdr_offset;
782
783 len = left;
784 /* IF: it doesn't fit, use 'mtu' - the data space left */
785 if (len > mtu)
786 len = mtu;
787 /* IF: we are not sending up to and including the packet end
788 then align the next start on an eight byte boundary */
789 if (len < left) {
790 len &= ~7;
791 }
792
793 /* Allocate buffer */
794 frag = alloc_skb(len + hlen + sizeof(struct frag_hdr) +
795 hroom + troom, GFP_ATOMIC);
796 if (!frag) {
797 err = -ENOMEM;
798 goto fail;
799 }
800
801 /*
802 * Set up data on packet
803 */
804
805 ip6_copy_metadata(frag, skb);
806 skb_reserve(frag, hroom);
807 skb_put(frag, len + hlen + sizeof(struct frag_hdr));
808 skb_reset_network_header(frag);
809 fh = (struct frag_hdr *)(skb_network_header(frag) + hlen);
810 frag->transport_header = (frag->network_header + hlen +
811 sizeof(struct frag_hdr));
812
813 /*
814 * Charge the memory for the fragment to any owner
815 * it might possess
816 */
817 if (skb->sk)
818 skb_set_owner_w(frag, skb->sk);
819
820 /*
821 * Copy the packet header into the new buffer.
822 */
823 skb_copy_from_linear_data(skb, skb_network_header(frag), hlen);
824
825 fragnexthdr_offset = skb_network_header(frag);
826 fragnexthdr_offset += prevhdr - skb_network_header(skb);
827 *fragnexthdr_offset = NEXTHDR_FRAGMENT;
828
829 /*
830 * Build fragment header.
831 */
832 fh->nexthdr = nexthdr;
833 fh->reserved = 0;
834 fh->identification = frag_id;
835
836 /*
837 * Copy a block of the IP datagram.
838 */
839 BUG_ON(skb_copy_bits(skb, ptr, skb_transport_header(frag),
840 len));
841 left -= len;
842
843 fh->frag_off = htons(offset);
844 if (left > 0)
845 fh->frag_off |= htons(IP6_MF);
846 ipv6_hdr(frag)->payload_len = htons(frag->len -
847 sizeof(struct ipv6hdr));
848
849 ptr += len;
850 offset += len;
851
852 /*
853 * Put this fragment into the sending queue.
854 */
855 err = output(net, sk, frag);
856 if (err)
857 goto fail;
858
859 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
860 IPSTATS_MIB_FRAGCREATES);
861 }
862 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
863 IPSTATS_MIB_FRAGOKS);
864 consume_skb(skb);
865 return err;
866
867 fail_toobig:
868 if (skb->sk && dst_allfrag(skb_dst(skb)))
869 sk_nocaps_add(skb->sk, NETIF_F_GSO_MASK);
870
871 skb->dev = skb_dst(skb)->dev;
872 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
873 err = -EMSGSIZE;
874
875 fail:
876 IP6_INC_STATS(net, ip6_dst_idev(skb_dst(skb)),
877 IPSTATS_MIB_FRAGFAILS);
878 kfree_skb(skb);
879 return err;
880 }
881
882 static inline int ip6_rt_check(const struct rt6key *rt_key,
883 const struct in6_addr *fl_addr,
884 const struct in6_addr *addr_cache)
885 {
886 return (rt_key->plen != 128 || !ipv6_addr_equal(fl_addr, &rt_key->addr)) &&
887 (!addr_cache || !ipv6_addr_equal(fl_addr, addr_cache));
888 }
889
890 static struct dst_entry *ip6_sk_dst_check(struct sock *sk,
891 struct dst_entry *dst,
892 const struct flowi6 *fl6)
893 {
894 struct ipv6_pinfo *np = inet6_sk(sk);
895 struct rt6_info *rt;
896
897 if (!dst)
898 goto out;
899
900 if (dst->ops->family != AF_INET6) {
901 dst_release(dst);
902 return NULL;
903 }
904
905 rt = (struct rt6_info *)dst;
906 /* Yes, checking route validity in not connected
907 * case is not very simple. Take into account,
908 * that we do not support routing by source, TOS,
909 * and MSG_DONTROUTE --ANK (980726)
910 *
911 * 1. ip6_rt_check(): If route was host route,
912 * check that cached destination is current.
913 * If it is network route, we still may
914 * check its validity using saved pointer
915 * to the last used address: daddr_cache.
916 * We do not want to save whole address now,
917 * (because main consumer of this service
918 * is tcp, which has not this problem),
919 * so that the last trick works only on connected
920 * sockets.
921 * 2. oif also should be the same.
922 */
923 if (ip6_rt_check(&rt->rt6i_dst, &fl6->daddr, np->daddr_cache) ||
924 #ifdef CONFIG_IPV6_SUBTREES
925 ip6_rt_check(&rt->rt6i_src, &fl6->saddr, np->saddr_cache) ||
926 #endif
927 (!(fl6->flowi6_flags & FLOWI_FLAG_SKIP_NH_OIF) &&
928 (fl6->flowi6_oif && fl6->flowi6_oif != dst->dev->ifindex))) {
929 dst_release(dst);
930 dst = NULL;
931 }
932
933 out:
934 return dst;
935 }
936
937 static int ip6_dst_lookup_tail(struct net *net, const struct sock *sk,
938 struct dst_entry **dst, struct flowi6 *fl6)
939 {
940 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
941 struct neighbour *n;
942 struct rt6_info *rt;
943 #endif
944 int err;
945 int flags = 0;
946
947 /* The correct way to handle this would be to do
948 * ip6_route_get_saddr, and then ip6_route_output; however,
949 * the route-specific preferred source forces the
950 * ip6_route_output call _before_ ip6_route_get_saddr.
951 *
952 * In source specific routing (no src=any default route),
953 * ip6_route_output will fail given src=any saddr, though, so
954 * that's why we try it again later.
955 */
956 if (ipv6_addr_any(&fl6->saddr) && (!*dst || !(*dst)->error)) {
957 struct rt6_info *rt;
958 bool had_dst = *dst != NULL;
959
960 if (!had_dst)
961 *dst = ip6_route_output(net, sk, fl6);
962 rt = (*dst)->error ? NULL : (struct rt6_info *)*dst;
963 err = ip6_route_get_saddr(net, rt, &fl6->daddr,
964 sk ? inet6_sk(sk)->srcprefs : 0,
965 &fl6->saddr);
966 if (err)
967 goto out_err_release;
968
969 /* If we had an erroneous initial result, pretend it
970 * never existed and let the SA-enabled version take
971 * over.
972 */
973 if (!had_dst && (*dst)->error) {
974 dst_release(*dst);
975 *dst = NULL;
976 }
977
978 if (fl6->flowi6_oif)
979 flags |= RT6_LOOKUP_F_IFACE;
980 }
981
982 if (!*dst)
983 *dst = ip6_route_output_flags(net, sk, fl6, flags);
984
985 err = (*dst)->error;
986 if (err)
987 goto out_err_release;
988
989 #ifdef CONFIG_IPV6_OPTIMISTIC_DAD
990 /*
991 * Here if the dst entry we've looked up
992 * has a neighbour entry that is in the INCOMPLETE
993 * state and the src address from the flow is
994 * marked as OPTIMISTIC, we release the found
995 * dst entry and replace it instead with the
996 * dst entry of the nexthop router
997 */
998 rt = (struct rt6_info *) *dst;
999 rcu_read_lock_bh();
1000 n = __ipv6_neigh_lookup_noref(rt->dst.dev,
1001 rt6_nexthop(rt, &fl6->daddr));
1002 err = n && !(n->nud_state & NUD_VALID) ? -EINVAL : 0;
1003 rcu_read_unlock_bh();
1004
1005 if (err) {
1006 struct inet6_ifaddr *ifp;
1007 struct flowi6 fl_gw6;
1008 int redirect;
1009
1010 ifp = ipv6_get_ifaddr(net, &fl6->saddr,
1011 (*dst)->dev, 1);
1012
1013 redirect = (ifp && ifp->flags & IFA_F_OPTIMISTIC);
1014 if (ifp)
1015 in6_ifa_put(ifp);
1016
1017 if (redirect) {
1018 /*
1019 * We need to get the dst entry for the
1020 * default router instead
1021 */
1022 dst_release(*dst);
1023 memcpy(&fl_gw6, fl6, sizeof(struct flowi6));
1024 memset(&fl_gw6.daddr, 0, sizeof(struct in6_addr));
1025 *dst = ip6_route_output(net, sk, &fl_gw6);
1026 err = (*dst)->error;
1027 if (err)
1028 goto out_err_release;
1029 }
1030 }
1031 #endif
1032 if (ipv6_addr_v4mapped(&fl6->saddr) &&
1033 !(ipv6_addr_v4mapped(&fl6->daddr) || ipv6_addr_any(&fl6->daddr))) {
1034 err = -EAFNOSUPPORT;
1035 goto out_err_release;
1036 }
1037
1038 return 0;
1039
1040 out_err_release:
1041 dst_release(*dst);
1042 *dst = NULL;
1043
1044 if (err == -ENETUNREACH)
1045 IP6_INC_STATS(net, NULL, IPSTATS_MIB_OUTNOROUTES);
1046 return err;
1047 }
1048
1049 /**
1050 * ip6_dst_lookup - perform route lookup on flow
1051 * @sk: socket which provides route info
1052 * @dst: pointer to dst_entry * for result
1053 * @fl6: flow to lookup
1054 *
1055 * This function performs a route lookup on the given flow.
1056 *
1057 * It returns zero on success, or a standard errno code on error.
1058 */
1059 int ip6_dst_lookup(struct net *net, struct sock *sk, struct dst_entry **dst,
1060 struct flowi6 *fl6)
1061 {
1062 *dst = NULL;
1063 return ip6_dst_lookup_tail(net, sk, dst, fl6);
1064 }
1065 EXPORT_SYMBOL_GPL(ip6_dst_lookup);
1066
1067 /**
1068 * ip6_dst_lookup_flow - perform route lookup on flow with ipsec
1069 * @sk: socket which provides route info
1070 * @fl6: flow to lookup
1071 * @final_dst: final destination address for ipsec lookup
1072 *
1073 * This function performs a route lookup on the given flow.
1074 *
1075 * It returns a valid dst pointer on success, or a pointer encoded
1076 * error code.
1077 */
1078 struct dst_entry *ip6_dst_lookup_flow(const struct sock *sk, struct flowi6 *fl6,
1079 const struct in6_addr *final_dst)
1080 {
1081 struct dst_entry *dst = NULL;
1082 int err;
1083
1084 err = ip6_dst_lookup_tail(sock_net(sk), sk, &dst, fl6);
1085 if (err)
1086 return ERR_PTR(err);
1087 if (final_dst)
1088 fl6->daddr = *final_dst;
1089
1090 return xfrm_lookup_route(sock_net(sk), dst, flowi6_to_flowi(fl6), sk, 0);
1091 }
1092 EXPORT_SYMBOL_GPL(ip6_dst_lookup_flow);
1093
1094 /**
1095 * ip6_sk_dst_lookup_flow - perform socket cached route lookup on flow
1096 * @sk: socket which provides the dst cache and route info
1097 * @fl6: flow to lookup
1098 * @final_dst: final destination address for ipsec lookup
1099 *
1100 * This function performs a route lookup on the given flow with the
1101 * possibility of using the cached route in the socket if it is valid.
1102 * It will take the socket dst lock when operating on the dst cache.
1103 * As a result, this function can only be used in process context.
1104 *
1105 * It returns a valid dst pointer on success, or a pointer encoded
1106 * error code.
1107 */
1108 struct dst_entry *ip6_sk_dst_lookup_flow(struct sock *sk, struct flowi6 *fl6,
1109 const struct in6_addr *final_dst)
1110 {
1111 struct dst_entry *dst = sk_dst_check(sk, inet6_sk(sk)->dst_cookie);
1112
1113 dst = ip6_sk_dst_check(sk, dst, fl6);
1114 if (!dst)
1115 dst = ip6_dst_lookup_flow(sk, fl6, final_dst);
1116
1117 return dst;
1118 }
1119 EXPORT_SYMBOL_GPL(ip6_sk_dst_lookup_flow);
1120
1121 static inline int ip6_ufo_append_data(struct sock *sk,
1122 struct sk_buff_head *queue,
1123 int getfrag(void *from, char *to, int offset, int len,
1124 int odd, struct sk_buff *skb),
1125 void *from, int length, int hh_len, int fragheaderlen,
1126 int exthdrlen, int transhdrlen, int mtu,
1127 unsigned int flags, const struct flowi6 *fl6)
1128
1129 {
1130 struct sk_buff *skb;
1131 int err;
1132
1133 /* There is support for UDP large send offload by network
1134 * device, so create one single skb packet containing complete
1135 * udp datagram
1136 */
1137 skb = skb_peek_tail(queue);
1138 if (!skb) {
1139 skb = sock_alloc_send_skb(sk,
1140 hh_len + fragheaderlen + transhdrlen + 20,
1141 (flags & MSG_DONTWAIT), &err);
1142 if (!skb)
1143 return err;
1144
1145 /* reserve space for Hardware header */
1146 skb_reserve(skb, hh_len);
1147
1148 /* create space for UDP/IP header */
1149 skb_put(skb, fragheaderlen + transhdrlen);
1150
1151 /* initialize network header pointer */
1152 skb_set_network_header(skb, exthdrlen);
1153
1154 /* initialize protocol header pointer */
1155 skb->transport_header = skb->network_header + fragheaderlen;
1156
1157 skb->protocol = htons(ETH_P_IPV6);
1158 skb->csum = 0;
1159
1160 __skb_queue_tail(queue, skb);
1161 } else if (skb_is_gso(skb)) {
1162 goto append;
1163 }
1164
1165 skb->ip_summed = CHECKSUM_PARTIAL;
1166 /* Specify the length of each IPv6 datagram fragment.
1167 * It has to be a multiple of 8.
1168 */
1169 skb_shinfo(skb)->gso_size = (mtu - fragheaderlen -
1170 sizeof(struct frag_hdr)) & ~7;
1171 skb_shinfo(skb)->gso_type = SKB_GSO_UDP;
1172 skb_shinfo(skb)->ip6_frag_id = ipv6_select_ident(sock_net(sk),
1173 &fl6->daddr,
1174 &fl6->saddr);
1175
1176 append:
1177 return skb_append_datato_frags(sk, skb, getfrag, from,
1178 (length - transhdrlen));
1179 }
1180
1181 static inline struct ipv6_opt_hdr *ip6_opt_dup(struct ipv6_opt_hdr *src,
1182 gfp_t gfp)
1183 {
1184 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1185 }
1186
1187 static inline struct ipv6_rt_hdr *ip6_rthdr_dup(struct ipv6_rt_hdr *src,
1188 gfp_t gfp)
1189 {
1190 return src ? kmemdup(src, (src->hdrlen + 1) * 8, gfp) : NULL;
1191 }
1192
1193 static void ip6_append_data_mtu(unsigned int *mtu,
1194 int *maxfraglen,
1195 unsigned int fragheaderlen,
1196 struct sk_buff *skb,
1197 struct rt6_info *rt,
1198 unsigned int orig_mtu)
1199 {
1200 if (!(rt->dst.flags & DST_XFRM_TUNNEL)) {
1201 if (!skb) {
1202 /* first fragment, reserve header_len */
1203 *mtu = orig_mtu - rt->dst.header_len;
1204
1205 } else {
1206 /*
1207 * this fragment is not first, the headers
1208 * space is regarded as data space.
1209 */
1210 *mtu = orig_mtu;
1211 }
1212 *maxfraglen = ((*mtu - fragheaderlen) & ~7)
1213 + fragheaderlen - sizeof(struct frag_hdr);
1214 }
1215 }
1216
1217 static int ip6_setup_cork(struct sock *sk, struct inet_cork_full *cork,
1218 struct inet6_cork *v6_cork, struct ipcm6_cookie *ipc6,
1219 struct rt6_info *rt, struct flowi6 *fl6)
1220 {
1221 struct ipv6_pinfo *np = inet6_sk(sk);
1222 unsigned int mtu;
1223 struct ipv6_txoptions *opt = ipc6->opt;
1224
1225 /*
1226 * setup for corking
1227 */
1228 if (opt) {
1229 if (WARN_ON(v6_cork->opt))
1230 return -EINVAL;
1231
1232 v6_cork->opt = kzalloc(sizeof(*opt), sk->sk_allocation);
1233 if (unlikely(!v6_cork->opt))
1234 return -ENOBUFS;
1235
1236 v6_cork->opt->tot_len = sizeof(*opt);
1237 v6_cork->opt->opt_flen = opt->opt_flen;
1238 v6_cork->opt->opt_nflen = opt->opt_nflen;
1239
1240 v6_cork->opt->dst0opt = ip6_opt_dup(opt->dst0opt,
1241 sk->sk_allocation);
1242 if (opt->dst0opt && !v6_cork->opt->dst0opt)
1243 return -ENOBUFS;
1244
1245 v6_cork->opt->dst1opt = ip6_opt_dup(opt->dst1opt,
1246 sk->sk_allocation);
1247 if (opt->dst1opt && !v6_cork->opt->dst1opt)
1248 return -ENOBUFS;
1249
1250 v6_cork->opt->hopopt = ip6_opt_dup(opt->hopopt,
1251 sk->sk_allocation);
1252 if (opt->hopopt && !v6_cork->opt->hopopt)
1253 return -ENOBUFS;
1254
1255 v6_cork->opt->srcrt = ip6_rthdr_dup(opt->srcrt,
1256 sk->sk_allocation);
1257 if (opt->srcrt && !v6_cork->opt->srcrt)
1258 return -ENOBUFS;
1259
1260 /* need source address above miyazawa*/
1261 }
1262 dst_hold(&rt->dst);
1263 cork->base.dst = &rt->dst;
1264 cork->fl.u.ip6 = *fl6;
1265 v6_cork->hop_limit = ipc6->hlimit;
1266 v6_cork->tclass = ipc6->tclass;
1267 if (rt->dst.flags & DST_XFRM_TUNNEL)
1268 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ?
1269 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(&rt->dst);
1270 else
1271 mtu = np->pmtudisc >= IPV6_PMTUDISC_PROBE ?
1272 READ_ONCE(rt->dst.dev->mtu) : dst_mtu(rt->dst.path);
1273 if (np->frag_size < mtu) {
1274 if (np->frag_size)
1275 mtu = np->frag_size;
1276 }
1277 if (mtu < IPV6_MIN_MTU)
1278 return -EINVAL;
1279 cork->base.fragsize = mtu;
1280 if (dst_allfrag(rt->dst.path))
1281 cork->base.flags |= IPCORK_ALLFRAG;
1282 cork->base.length = 0;
1283
1284 return 0;
1285 }
1286
1287 static int __ip6_append_data(struct sock *sk,
1288 struct flowi6 *fl6,
1289 struct sk_buff_head *queue,
1290 struct inet_cork *cork,
1291 struct inet6_cork *v6_cork,
1292 struct page_frag *pfrag,
1293 int getfrag(void *from, char *to, int offset,
1294 int len, int odd, struct sk_buff *skb),
1295 void *from, int length, int transhdrlen,
1296 unsigned int flags, struct ipcm6_cookie *ipc6,
1297 const struct sockcm_cookie *sockc)
1298 {
1299 struct sk_buff *skb, *skb_prev = NULL;
1300 unsigned int maxfraglen, fragheaderlen, mtu, orig_mtu, pmtu;
1301 int exthdrlen = 0;
1302 int dst_exthdrlen = 0;
1303 int hh_len;
1304 int copy;
1305 int err;
1306 int offset = 0;
1307 __u8 tx_flags = 0;
1308 u32 tskey = 0;
1309 struct rt6_info *rt = (struct rt6_info *)cork->dst;
1310 struct ipv6_txoptions *opt = v6_cork->opt;
1311 int csummode = CHECKSUM_NONE;
1312 unsigned int maxnonfragsize, headersize;
1313
1314 skb = skb_peek_tail(queue);
1315 if (!skb) {
1316 exthdrlen = opt ? opt->opt_flen : 0;
1317 dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
1318 }
1319
1320 mtu = cork->fragsize;
1321 orig_mtu = mtu;
1322
1323 hh_len = LL_RESERVED_SPACE(rt->dst.dev);
1324
1325 fragheaderlen = sizeof(struct ipv6hdr) + rt->rt6i_nfheader_len +
1326 (opt ? opt->opt_nflen : 0);
1327 maxfraglen = ((mtu - fragheaderlen) & ~7) + fragheaderlen -
1328 sizeof(struct frag_hdr);
1329
1330 headersize = sizeof(struct ipv6hdr) +
1331 (opt ? opt->opt_flen + opt->opt_nflen : 0) +
1332 (dst_allfrag(&rt->dst) ?
1333 sizeof(struct frag_hdr) : 0) +
1334 rt->rt6i_nfheader_len;
1335
1336 /* as per RFC 7112 section 5, the entire IPv6 Header Chain must fit
1337 * the first fragment
1338 */
1339 if (headersize + transhdrlen > mtu)
1340 goto emsgsize;
1341
1342 if (cork->length + length > mtu - headersize && ipc6->dontfrag &&
1343 (sk->sk_protocol == IPPROTO_UDP ||
1344 sk->sk_protocol == IPPROTO_RAW)) {
1345 ipv6_local_rxpmtu(sk, fl6, mtu - headersize +
1346 sizeof(struct ipv6hdr));
1347 goto emsgsize;
1348 }
1349
1350 if (ip6_sk_ignore_df(sk))
1351 maxnonfragsize = sizeof(struct ipv6hdr) + IPV6_MAXPLEN;
1352 else
1353 maxnonfragsize = mtu;
1354
1355 if (cork->length + length > maxnonfragsize - headersize) {
1356 emsgsize:
1357 pmtu = max_t(int, mtu - headersize + sizeof(struct ipv6hdr), 0);
1358 ipv6_local_error(sk, EMSGSIZE, fl6, pmtu);
1359 return -EMSGSIZE;
1360 }
1361
1362 /* CHECKSUM_PARTIAL only with no extension headers and when
1363 * we are not going to fragment
1364 */
1365 if (transhdrlen && sk->sk_protocol == IPPROTO_UDP &&
1366 headersize == sizeof(struct ipv6hdr) &&
1367 length < mtu - headersize &&
1368 !(flags & MSG_MORE) &&
1369 rt->dst.dev->features & (NETIF_F_IPV6_CSUM | NETIF_F_HW_CSUM))
1370 csummode = CHECKSUM_PARTIAL;
1371
1372 if (sk->sk_type == SOCK_DGRAM || sk->sk_type == SOCK_RAW) {
1373 sock_tx_timestamp(sk, sockc->tsflags, &tx_flags);
1374 if (tx_flags & SKBTX_ANY_SW_TSTAMP &&
1375 sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
1376 tskey = sk->sk_tskey++;
1377 }
1378
1379 /*
1380 * Let's try using as much space as possible.
1381 * Use MTU if total length of the message fits into the MTU.
1382 * Otherwise, we need to reserve fragment header and
1383 * fragment alignment (= 8-15 octects, in total).
1384 *
1385 * Note that we may need to "move" the data from the tail of
1386 * of the buffer to the new fragment when we split
1387 * the message.
1388 *
1389 * FIXME: It may be fragmented into multiple chunks
1390 * at once if non-fragmentable extension headers
1391 * are too large.
1392 * --yoshfuji
1393 */
1394
1395 cork->length += length;
1396 if ((skb && skb_is_gso(skb)) ||
1397 (((length + fragheaderlen) > mtu) &&
1398 (skb_queue_len(queue) <= 1) &&
1399 (sk->sk_protocol == IPPROTO_UDP) &&
1400 (rt->dst.dev->features & NETIF_F_UFO) && !rt->dst.header_len &&
1401 (sk->sk_type == SOCK_DGRAM) && !udp_get_no_check6_tx(sk))) {
1402 err = ip6_ufo_append_data(sk, queue, getfrag, from, length,
1403 hh_len, fragheaderlen, exthdrlen,
1404 transhdrlen, mtu, flags, fl6);
1405 if (err)
1406 goto error;
1407 return 0;
1408 }
1409
1410 if (!skb)
1411 goto alloc_new_skb;
1412
1413 while (length > 0) {
1414 /* Check if the remaining data fits into current packet. */
1415 copy = (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - skb->len;
1416 if (copy < length)
1417 copy = maxfraglen - skb->len;
1418
1419 if (copy <= 0) {
1420 char *data;
1421 unsigned int datalen;
1422 unsigned int fraglen;
1423 unsigned int fraggap;
1424 unsigned int alloclen;
1425 alloc_new_skb:
1426 /* There's no room in the current skb */
1427 if (skb)
1428 fraggap = skb->len - maxfraglen;
1429 else
1430 fraggap = 0;
1431 /* update mtu and maxfraglen if necessary */
1432 if (!skb || !skb_prev)
1433 ip6_append_data_mtu(&mtu, &maxfraglen,
1434 fragheaderlen, skb, rt,
1435 orig_mtu);
1436
1437 skb_prev = skb;
1438
1439 /*
1440 * If remaining data exceeds the mtu,
1441 * we know we need more fragment(s).
1442 */
1443 datalen = length + fraggap;
1444
1445 if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
1446 datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
1447 if ((flags & MSG_MORE) &&
1448 !(rt->dst.dev->features&NETIF_F_SG))
1449 alloclen = mtu;
1450 else
1451 alloclen = datalen + fragheaderlen;
1452
1453 alloclen += dst_exthdrlen;
1454
1455 if (datalen != length + fraggap) {
1456 /*
1457 * this is not the last fragment, the trailer
1458 * space is regarded as data space.
1459 */
1460 datalen += rt->dst.trailer_len;
1461 }
1462
1463 alloclen += rt->dst.trailer_len;
1464 fraglen = datalen + fragheaderlen;
1465
1466 /*
1467 * We just reserve space for fragment header.
1468 * Note: this may be overallocation if the message
1469 * (without MSG_MORE) fits into the MTU.
1470 */
1471 alloclen += sizeof(struct frag_hdr);
1472
1473 copy = datalen - transhdrlen - fraggap;
1474 if (copy < 0) {
1475 err = -EINVAL;
1476 goto error;
1477 }
1478 if (transhdrlen) {
1479 skb = sock_alloc_send_skb(sk,
1480 alloclen + hh_len,
1481 (flags & MSG_DONTWAIT), &err);
1482 } else {
1483 skb = NULL;
1484 if (atomic_read(&sk->sk_wmem_alloc) <=
1485 2 * sk->sk_sndbuf)
1486 skb = sock_wmalloc(sk,
1487 alloclen + hh_len, 1,
1488 sk->sk_allocation);
1489 if (unlikely(!skb))
1490 err = -ENOBUFS;
1491 }
1492 if (!skb)
1493 goto error;
1494 /*
1495 * Fill in the control structures
1496 */
1497 skb->protocol = htons(ETH_P_IPV6);
1498 skb->ip_summed = csummode;
1499 skb->csum = 0;
1500 /* reserve for fragmentation and ipsec header */
1501 skb_reserve(skb, hh_len + sizeof(struct frag_hdr) +
1502 dst_exthdrlen);
1503
1504 /* Only the initial fragment is time stamped */
1505 skb_shinfo(skb)->tx_flags = tx_flags;
1506 tx_flags = 0;
1507 skb_shinfo(skb)->tskey = tskey;
1508 tskey = 0;
1509
1510 /*
1511 * Find where to start putting bytes
1512 */
1513 data = skb_put(skb, fraglen);
1514 skb_set_network_header(skb, exthdrlen);
1515 data += fragheaderlen;
1516 skb->transport_header = (skb->network_header +
1517 fragheaderlen);
1518 if (fraggap) {
1519 skb->csum = skb_copy_and_csum_bits(
1520 skb_prev, maxfraglen,
1521 data + transhdrlen, fraggap, 0);
1522 skb_prev->csum = csum_sub(skb_prev->csum,
1523 skb->csum);
1524 data += fraggap;
1525 pskb_trim_unique(skb_prev, maxfraglen);
1526 }
1527 if (copy > 0 &&
1528 getfrag(from, data + transhdrlen, offset,
1529 copy, fraggap, skb) < 0) {
1530 err = -EFAULT;
1531 kfree_skb(skb);
1532 goto error;
1533 }
1534
1535 offset += copy;
1536 length -= datalen - fraggap;
1537 transhdrlen = 0;
1538 exthdrlen = 0;
1539 dst_exthdrlen = 0;
1540
1541 /*
1542 * Put the packet on the pending queue
1543 */
1544 __skb_queue_tail(queue, skb);
1545 continue;
1546 }
1547
1548 if (copy > length)
1549 copy = length;
1550
1551 if (!(rt->dst.dev->features&NETIF_F_SG) &&
1552 skb_tailroom(skb) >= copy) {
1553 unsigned int off;
1554
1555 off = skb->len;
1556 if (getfrag(from, skb_put(skb, copy),
1557 offset, copy, off, skb) < 0) {
1558 __skb_trim(skb, off);
1559 err = -EFAULT;
1560 goto error;
1561 }
1562 } else {
1563 int i = skb_shinfo(skb)->nr_frags;
1564
1565 err = -ENOMEM;
1566 if (!sk_page_frag_refill(sk, pfrag))
1567 goto error;
1568
1569 if (!skb_can_coalesce(skb, i, pfrag->page,
1570 pfrag->offset)) {
1571 err = -EMSGSIZE;
1572 if (i == MAX_SKB_FRAGS)
1573 goto error;
1574
1575 __skb_fill_page_desc(skb, i, pfrag->page,
1576 pfrag->offset, 0);
1577 skb_shinfo(skb)->nr_frags = ++i;
1578 get_page(pfrag->page);
1579 }
1580 copy = min_t(int, copy, pfrag->size - pfrag->offset);
1581 if (getfrag(from,
1582 page_address(pfrag->page) + pfrag->offset,
1583 offset, copy, skb->len, skb) < 0)
1584 goto error_efault;
1585
1586 pfrag->offset += copy;
1587 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1588 skb->len += copy;
1589 skb->data_len += copy;
1590 skb->truesize += copy;
1591 atomic_add(copy, &sk->sk_wmem_alloc);
1592 }
1593 offset += copy;
1594 length -= copy;
1595 }
1596
1597 return 0;
1598
1599 error_efault:
1600 err = -EFAULT;
1601 error:
1602 cork->length -= length;
1603 IP6_INC_STATS(sock_net(sk), rt->rt6i_idev, IPSTATS_MIB_OUTDISCARDS);
1604 return err;
1605 }
1606
1607 int ip6_append_data(struct sock *sk,
1608 int getfrag(void *from, char *to, int offset, int len,
1609 int odd, struct sk_buff *skb),
1610 void *from, int length, int transhdrlen,
1611 struct ipcm6_cookie *ipc6, struct flowi6 *fl6,
1612 struct rt6_info *rt, unsigned int flags,
1613 const struct sockcm_cookie *sockc)
1614 {
1615 struct inet_sock *inet = inet_sk(sk);
1616 struct ipv6_pinfo *np = inet6_sk(sk);
1617 int exthdrlen;
1618 int err;
1619
1620 if (flags&MSG_PROBE)
1621 return 0;
1622 if (skb_queue_empty(&sk->sk_write_queue)) {
1623 /*
1624 * setup for corking
1625 */
1626 err = ip6_setup_cork(sk, &inet->cork, &np->cork,
1627 ipc6, rt, fl6);
1628 if (err)
1629 return err;
1630
1631 exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
1632 length += exthdrlen;
1633 transhdrlen += exthdrlen;
1634 } else {
1635 fl6 = &inet->cork.fl.u.ip6;
1636 transhdrlen = 0;
1637 }
1638
1639 return __ip6_append_data(sk, fl6, &sk->sk_write_queue, &inet->cork.base,
1640 &np->cork, sk_page_frag(sk), getfrag,
1641 from, length, transhdrlen, flags, ipc6, sockc);
1642 }
1643 EXPORT_SYMBOL_GPL(ip6_append_data);
1644
1645 static void ip6_cork_release(struct inet_cork_full *cork,
1646 struct inet6_cork *v6_cork)
1647 {
1648 if (v6_cork->opt) {
1649 kfree(v6_cork->opt->dst0opt);
1650 kfree(v6_cork->opt->dst1opt);
1651 kfree(v6_cork->opt->hopopt);
1652 kfree(v6_cork->opt->srcrt);
1653 kfree(v6_cork->opt);
1654 v6_cork->opt = NULL;
1655 }
1656
1657 if (cork->base.dst) {
1658 dst_release(cork->base.dst);
1659 cork->base.dst = NULL;
1660 cork->base.flags &= ~IPCORK_ALLFRAG;
1661 }
1662 memset(&cork->fl, 0, sizeof(cork->fl));
1663 }
1664
1665 struct sk_buff *__ip6_make_skb(struct sock *sk,
1666 struct sk_buff_head *queue,
1667 struct inet_cork_full *cork,
1668 struct inet6_cork *v6_cork)
1669 {
1670 struct sk_buff *skb, *tmp_skb;
1671 struct sk_buff **tail_skb;
1672 struct in6_addr final_dst_buf, *final_dst = &final_dst_buf;
1673 struct ipv6_pinfo *np = inet6_sk(sk);
1674 struct net *net = sock_net(sk);
1675 struct ipv6hdr *hdr;
1676 struct ipv6_txoptions *opt = v6_cork->opt;
1677 struct rt6_info *rt = (struct rt6_info *)cork->base.dst;
1678 struct flowi6 *fl6 = &cork->fl.u.ip6;
1679 unsigned char proto = fl6->flowi6_proto;
1680
1681 skb = __skb_dequeue(queue);
1682 if (!skb)
1683 goto out;
1684 tail_skb = &(skb_shinfo(skb)->frag_list);
1685
1686 /* move skb->data to ip header from ext header */
1687 if (skb->data < skb_network_header(skb))
1688 __skb_pull(skb, skb_network_offset(skb));
1689 while ((tmp_skb = __skb_dequeue(queue)) != NULL) {
1690 __skb_pull(tmp_skb, skb_network_header_len(skb));
1691 *tail_skb = tmp_skb;
1692 tail_skb = &(tmp_skb->next);
1693 skb->len += tmp_skb->len;
1694 skb->data_len += tmp_skb->len;
1695 skb->truesize += tmp_skb->truesize;
1696 tmp_skb->destructor = NULL;
1697 tmp_skb->sk = NULL;
1698 }
1699
1700 /* Allow local fragmentation. */
1701 skb->ignore_df = ip6_sk_ignore_df(sk);
1702
1703 *final_dst = fl6->daddr;
1704 __skb_pull(skb, skb_network_header_len(skb));
1705 if (opt && opt->opt_flen)
1706 ipv6_push_frag_opts(skb, opt, &proto);
1707 if (opt && opt->opt_nflen)
1708 ipv6_push_nfrag_opts(skb, opt, &proto, &final_dst);
1709
1710 skb_push(skb, sizeof(struct ipv6hdr));
1711 skb_reset_network_header(skb);
1712 hdr = ipv6_hdr(skb);
1713
1714 ip6_flow_hdr(hdr, v6_cork->tclass,
1715 ip6_make_flowlabel(net, skb, fl6->flowlabel,
1716 ip6_autoflowlabel(net, np), fl6));
1717 hdr->hop_limit = v6_cork->hop_limit;
1718 hdr->nexthdr = proto;
1719 hdr->saddr = fl6->saddr;
1720 hdr->daddr = *final_dst;
1721
1722 skb->priority = sk->sk_priority;
1723 skb->mark = sk->sk_mark;
1724
1725 skb_dst_set(skb, dst_clone(&rt->dst));
1726 IP6_UPD_PO_STATS(net, rt->rt6i_idev, IPSTATS_MIB_OUT, skb->len);
1727 if (proto == IPPROTO_ICMPV6) {
1728 struct inet6_dev *idev = ip6_dst_idev(skb_dst(skb));
1729
1730 ICMP6MSGOUT_INC_STATS(net, idev, icmp6_hdr(skb)->icmp6_type);
1731 ICMP6_INC_STATS(net, idev, ICMP6_MIB_OUTMSGS);
1732 }
1733
1734 ip6_cork_release(cork, v6_cork);
1735 out:
1736 return skb;
1737 }
1738
1739 int ip6_send_skb(struct sk_buff *skb)
1740 {
1741 struct net *net = sock_net(skb->sk);
1742 struct rt6_info *rt = (struct rt6_info *)skb_dst(skb);
1743 int err;
1744
1745 err = ip6_local_out(net, skb->sk, skb);
1746 if (err) {
1747 if (err > 0)
1748 err = net_xmit_errno(err);
1749 if (err)
1750 IP6_INC_STATS(net, rt->rt6i_idev,
1751 IPSTATS_MIB_OUTDISCARDS);
1752 }
1753
1754 return err;
1755 }
1756
1757 int ip6_push_pending_frames(struct sock *sk)
1758 {
1759 struct sk_buff *skb;
1760
1761 skb = ip6_finish_skb(sk);
1762 if (!skb)
1763 return 0;
1764
1765 return ip6_send_skb(skb);
1766 }
1767 EXPORT_SYMBOL_GPL(ip6_push_pending_frames);
1768
1769 static void __ip6_flush_pending_frames(struct sock *sk,
1770 struct sk_buff_head *queue,
1771 struct inet_cork_full *cork,
1772 struct inet6_cork *v6_cork)
1773 {
1774 struct sk_buff *skb;
1775
1776 while ((skb = __skb_dequeue_tail(queue)) != NULL) {
1777 if (skb_dst(skb))
1778 IP6_INC_STATS(sock_net(sk), ip6_dst_idev(skb_dst(skb)),
1779 IPSTATS_MIB_OUTDISCARDS);
1780 kfree_skb(skb);
1781 }
1782
1783 ip6_cork_release(cork, v6_cork);
1784 }
1785
1786 void ip6_flush_pending_frames(struct sock *sk)
1787 {
1788 __ip6_flush_pending_frames(sk, &sk->sk_write_queue,
1789 &inet_sk(sk)->cork, &inet6_sk(sk)->cork);
1790 }
1791 EXPORT_SYMBOL_GPL(ip6_flush_pending_frames);
1792
1793 struct sk_buff *ip6_make_skb(struct sock *sk,
1794 int getfrag(void *from, char *to, int offset,
1795 int len, int odd, struct sk_buff *skb),
1796 void *from, int length, int transhdrlen,
1797 struct ipcm6_cookie *ipc6, struct flowi6 *fl6,
1798 struct rt6_info *rt, unsigned int flags,
1799 const struct sockcm_cookie *sockc)
1800 {
1801 struct inet_cork_full cork;
1802 struct inet6_cork v6_cork;
1803 struct sk_buff_head queue;
1804 int exthdrlen = (ipc6->opt ? ipc6->opt->opt_flen : 0);
1805 int err;
1806
1807 if (flags & MSG_PROBE)
1808 return NULL;
1809
1810 __skb_queue_head_init(&queue);
1811
1812 cork.base.flags = 0;
1813 cork.base.addr = 0;
1814 cork.base.opt = NULL;
1815 cork.base.dst = NULL;
1816 v6_cork.opt = NULL;
1817 err = ip6_setup_cork(sk, &cork, &v6_cork, ipc6, rt, fl6);
1818 if (err) {
1819 ip6_cork_release(&cork, &v6_cork);
1820 return ERR_PTR(err);
1821 }
1822 if (ipc6->dontfrag < 0)
1823 ipc6->dontfrag = inet6_sk(sk)->dontfrag;
1824
1825 err = __ip6_append_data(sk, fl6, &queue, &cork.base, &v6_cork,
1826 &current->task_frag, getfrag, from,
1827 length + exthdrlen, transhdrlen + exthdrlen,
1828 flags, ipc6, sockc);
1829 if (err) {
1830 __ip6_flush_pending_frames(sk, &queue, &cork, &v6_cork);
1831 return ERR_PTR(err);
1832 }
1833
1834 return __ip6_make_skb(sk, &queue, &cork, &v6_cork);
1835 }
Linux with added FPC-III support