net/ipv4/ip_tunnel.c

   1 /*
   2  * Copyright (c) 2013 Nicira, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of version 2 of the GNU General Public
   6  * License as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16  * 02110-1301, USA
  17  */
  18
  19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21 #include <linux/capability.h>
  22 #include <linux/module.h>
  23 #include <linux/types.h>
  24 #include <linux/kernel.h>
  25 #include <linux/slab.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/skbuff.h>
  28 #include <linux/netdevice.h>
  29 #include <linux/in.h>
  30 #include <linux/tcp.h>
  31 #include <linux/udp.h>
  32 #include <linux/if_arp.h>
  33 #include <linux/mroute.h>
  34 #include <linux/init.h>
  35 #include <linux/in6.h>
  36 #include <linux/inetdevice.h>
  37 #include <linux/igmp.h>
  38 #include <linux/netfilter_ipv4.h>
  39 #include <linux/etherdevice.h>
  40 #include <linux/if_ether.h>
  41 #include <linux/if_vlan.h>
  42 #include <linux/rculist.h>
  43 #include <linux/err.h>
  44
  45 #include <net/sock.h>
  46 #include <net/ip.h>
  47 #include <net/icmp.h>
  48 #include <net/protocol.h>
  49 #include <net/ip_tunnels.h>
  50 #include <net/arp.h>
  51 #include <net/checksum.h>
  52 #include <net/dsfield.h>
  53 #include <net/inet_ecn.h>
  54 #include <net/xfrm.h>
  55 #include <net/net_namespace.h>
  56 #include <net/netns/generic.h>
  57 #include <net/rtnetlink.h>
  58 #include <net/udp.h>
  59
  60 #if IS_ENABLED(CONFIG_IPV6)
  61 #include <net/ipv6.h>
  62 #include <net/ip6_fib.h>
  63 #include <net/ip6_route.h>
  64 #endif
  65
  66 static unsigned int ip_tunnel_hash(__be32 key, __be32 remote)
  67 {
  68         return hash_32((__force u32)key ^ (__force u32)remote,
  69                          IP_TNL_HASH_BITS);
  70 }
  71
  72 static bool ip_tunnel_key_match(const struct ip_tunnel_parm *p,
  73                                 __be16 flags, __be32 key)
  74 {
  75         if (p->i_flags & TUNNEL_KEY) {
  76                 if (flags & TUNNEL_KEY)
  77                         return key == p->i_key;
  78                 else
  79                         /* key expected, none present */
  80                         return false;
  81         } else
  82                 return !(flags & TUNNEL_KEY);
  83 }
  84
  85 /* Fallback tunnel: no source, no destination, no key, no options
  86
  87    Tunnel hash table:
  88    We require exact key match i.e. if a key is present in packet
  89    it will match only tunnel with the same key; if it is not present,
  90    it will match only keyless tunnel.
  91
  92    All keysless packets, if not matched configured keyless tunnels
  93    will match fallback tunnel.
  94    Given src, dst and key, find appropriate for input tunnel.
  95 */
  96 struct ip_tunnel *ip_tunnel_lookup(struct ip_tunnel_net *itn,
  97                                    int link, __be16 flags,
  98                                    __be32 remote, __be32 local,
  99                                    __be32 key)
 100 {
 101         unsigned int hash;
 102         struct ip_tunnel *t, *cand = NULL;
 103         struct hlist_head *head;
 104
 105         hash = ip_tunnel_hash(key, remote);
 106         head = &itn->tunnels[hash];
 107
 108         hlist_for_each_entry_rcu(t, head, hash_node) {
 109                 if (local != t->parms.iph.saddr ||
 110                     remote != t->parms.iph.daddr ||
 111                     !(t->dev->flags & IFF_UP))
 112                         continue;
 113
 114                 if (!ip_tunnel_key_match(&t->parms, flags, key))
 115                         continue;
 116
 117                 if (t->parms.link == link)
 118                         return t;
 119                 else
 120                         cand = t;
 121         }
 122
 123         hlist_for_each_entry_rcu(t, head, hash_node) {
 124                 if (remote != t->parms.iph.daddr ||
 125                     t->parms.iph.saddr != 0 ||
 126                     !(t->dev->flags & IFF_UP))
 127                         continue;
 128
 129                 if (!ip_tunnel_key_match(&t->parms, flags, key))
 130                         continue;
 131
 132                 if (t->parms.link == link)
 133                         return t;
 134                 else if (!cand)
 135                         cand = t;
 136         }
 137
 138         hash = ip_tunnel_hash(key, 0);
 139         head = &itn->tunnels[hash];
 140
 141         hlist_for_each_entry_rcu(t, head, hash_node) {
 142                 if ((local != t->parms.iph.saddr || t->parms.iph.daddr != 0) &&
 143                     (local != t->parms.iph.daddr || !ipv4_is_multicast(local)))
 144                         continue;
 145
 146                 if (!(t->dev->flags & IFF_UP))
 147                         continue;
 148
 149                 if (!ip_tunnel_key_match(&t->parms, flags, key))
 150                         continue;
 151
 152                 if (t->parms.link == link)
 153                         return t;
 154                 else if (!cand)
 155                         cand = t;
 156         }
 157
 158         if (flags & TUNNEL_NO_KEY)
 159                 goto skip_key_lookup;
 160
 161         hlist_for_each_entry_rcu(t, head, hash_node) {
 162                 if (t->parms.i_key != key ||
 163                     t->parms.iph.saddr != 0 ||
 164                     t->parms.iph.daddr != 0 ||
 165                     !(t->dev->flags & IFF_UP))
 166                         continue;
 167
 168                 if (t->parms.link == link)
 169                         return t;
 170                 else if (!cand)
 171                         cand = t;
 172         }
 173
 174 skip_key_lookup:
 175         if (cand)
 176                 return cand;
 177
 178         t = rcu_dereference(itn->collect_md_tun);
 179         if (t)
 180                 return t;
 181
 182         if (itn->fb_tunnel_dev && itn->fb_tunnel_dev->flags & IFF_UP)
 183                 return netdev_priv(itn->fb_tunnel_dev);
 184
 185         return NULL;
 186 }
 187 EXPORT_SYMBOL_GPL(ip_tunnel_lookup);
 188
 189 static struct hlist_head *ip_bucket(struct ip_tunnel_net *itn,
 190                                     struct ip_tunnel_parm *parms)
 191 {
 192         unsigned int h;
 193         __be32 remote;
 194         __be32 i_key = parms->i_key;
 195
 196         if (parms->iph.daddr && !ipv4_is_multicast(parms->iph.daddr))
 197                 remote = parms->iph.daddr;
 198         else
 199                 remote = 0;
 200
 201         if (!(parms->i_flags & TUNNEL_KEY) && (parms->i_flags & VTI_ISVTI))
 202                 i_key = 0;
 203
 204         h = ip_tunnel_hash(i_key, remote);
 205         return &itn->tunnels[h];
 206 }
 207
 208 static void ip_tunnel_add(struct ip_tunnel_net *itn, struct ip_tunnel *t)
 209 {
 210         struct hlist_head *head = ip_bucket(itn, &t->parms);
 211
 212         if (t->collect_md)
 213                 rcu_assign_pointer(itn->collect_md_tun, t);
 214         hlist_add_head_rcu(&t->hash_node, head);
 215 }
 216
 217 static void ip_tunnel_del(struct ip_tunnel_net *itn, struct ip_tunnel *t)
 218 {
 219         if (t->collect_md)
 220                 rcu_assign_pointer(itn->collect_md_tun, NULL);
 221         hlist_del_init_rcu(&t->hash_node);
 222 }
 223
 224 static struct ip_tunnel *ip_tunnel_find(struct ip_tunnel_net *itn,
 225                                         struct ip_tunnel_parm *parms,
 226                                         int type)
 227 {
 228         __be32 remote = parms->iph.daddr;
 229         __be32 local = parms->iph.saddr;
 230         __be32 key = parms->i_key;
 231         __be16 flags = parms->i_flags;
 232         int link = parms->link;
 233         struct ip_tunnel *t = NULL;
 234         struct hlist_head *head = ip_bucket(itn, parms);
 235
 236         hlist_for_each_entry_rcu(t, head, hash_node) {
 237                 if (local == t->parms.iph.saddr &&
 238                     remote == t->parms.iph.daddr &&
 239                     link == t->parms.link &&
 240                     type == t->dev->type &&
 241                     ip_tunnel_key_match(&t->parms, flags, key))
 242                         break;
 243         }
 244         return t;
 245 }
 246
 247 static struct net_device *__ip_tunnel_create(struct net *net,
 248                                              const struct rtnl_link_ops *ops,
 249                                              struct ip_tunnel_parm *parms)
 250 {
 251         int err;
 252         struct ip_tunnel *tunnel;
 253         struct net_device *dev;
 254         char name[IFNAMSIZ];
 255
 256         err = -E2BIG;
 257         if (parms->name[0]) {
 258                 if (!dev_valid_name(parms->name))
 259                         goto failed;
 260                 strlcpy(name, parms->name, IFNAMSIZ);
 261         } else {
 262                 if (strlen(ops->kind) > (IFNAMSIZ - 3))
 263                         goto failed;
 264                 strlcpy(name, ops->kind, IFNAMSIZ);
 265                 strncat(name, "%d", 2);
 266         }
 267
 268         ASSERT_RTNL();
 269         dev = alloc_netdev(ops->priv_size, name, NET_NAME_UNKNOWN, ops->setup);
 270         if (!dev) {
 271                 err = -ENOMEM;
 272                 goto failed;
 273         }
 274         dev_net_set(dev, net);
 275
 276         dev->rtnl_link_ops = ops;
 277
 278         tunnel = netdev_priv(dev);
 279         tunnel->parms = *parms;
 280         tunnel->net = net;
 281
 282         err = register_netdevice(dev);
 283         if (err)
 284                 goto failed_free;
 285
 286         return dev;
 287
 288 failed_free:
 289         free_netdev(dev);
 290 failed:
 291         return ERR_PTR(err);
 292 }
 293
 294 static inline void init_tunnel_flow(struct flowi4 *fl4,
 295                                     int proto,
 296                                     __be32 daddr, __be32 saddr,
 297                                     __be32 key, __u8 tos, int oif)
 298 {
 299         memset(fl4, 0, sizeof(*fl4));
 300         fl4->flowi4_oif = oif;
 301         fl4->daddr = daddr;
 302         fl4->saddr = saddr;
 303         fl4->flowi4_tos = tos;
 304         fl4->flowi4_proto = proto;
 305         fl4->fl4_gre_key = key;
 306 }
 307
 308 static int ip_tunnel_bind_dev(struct net_device *dev)
 309 {
 310         struct net_device *tdev = NULL;
 311         struct ip_tunnel *tunnel = netdev_priv(dev);
 312         const struct iphdr *iph;
 313         int hlen = LL_MAX_HEADER;
 314         int mtu = ETH_DATA_LEN;
 315         int t_hlen = tunnel->hlen + sizeof(struct iphdr);
 316
 317         iph = &tunnel->parms.iph;
 318
 319         /* Guess output device to choose reasonable mtu and needed_headroom */
 320         if (iph->daddr) {
 321                 struct flowi4 fl4;
 322                 struct rtable *rt;
 323
 324                 init_tunnel_flow(&fl4, iph->protocol, iph->daddr,
 325                                  iph->saddr, tunnel->parms.o_key,
 326                                  RT_TOS(iph->tos), tunnel->parms.link);
 327                 rt = ip_route_output_key(tunnel->net, &fl4);
 328
 329                 if (!IS_ERR(rt)) {
 330                         tdev = rt->dst.dev;
 331                         ip_rt_put(rt);
 332                 }
 333                 if (dev->type != ARPHRD_ETHER)
 334                         dev->flags |= IFF_POINTOPOINT;
 335
 336                 dst_cache_reset(&tunnel->dst_cache);
 337         }
 338
 339         if (!tdev && tunnel->parms.link)
 340                 tdev = __dev_get_by_index(tunnel->net, tunnel->parms.link);
 341
 342         if (tdev) {
 343                 hlen = tdev->hard_header_len + tdev->needed_headroom;
 344                 mtu = tdev->mtu;
 345         }
 346
 347         dev->needed_headroom = t_hlen + hlen;
 348         mtu -= (dev->hard_header_len + t_hlen);
 349
 350         if (mtu < IPV4_MIN_MTU)
 351                 mtu = IPV4_MIN_MTU;
 352
 353         return mtu;
 354 }
 355
 356 static struct ip_tunnel *ip_tunnel_create(struct net *net,
 357                                           struct ip_tunnel_net *itn,
 358                                           struct ip_tunnel_parm *parms)
 359 {
 360         struct ip_tunnel *nt;
 361         struct net_device *dev;
 362
 363         BUG_ON(!itn->fb_tunnel_dev);
 364         dev = __ip_tunnel_create(net, itn->fb_tunnel_dev->rtnl_link_ops, parms);
 365         if (IS_ERR(dev))
 366                 return ERR_CAST(dev);
 367
 368         dev->mtu = ip_tunnel_bind_dev(dev);
 369
 370         nt = netdev_priv(dev);
 371         ip_tunnel_add(itn, nt);
 372         return nt;
 373 }
 374
 375 int ip_tunnel_rcv(struct ip_tunnel *tunnel, struct sk_buff *skb,
 376                   const struct tnl_ptk_info *tpi, struct metadata_dst *tun_dst,
 377                   bool log_ecn_error)
 378 {
 379         struct pcpu_sw_netstats *tstats;
 380         const struct iphdr *iph = ip_hdr(skb);
 381         int err;
 382
 383 #ifdef CONFIG_NET_IPGRE_BROADCAST
 384         if (ipv4_is_multicast(iph->daddr)) {
 385                 tunnel->dev->stats.multicast++;
 386                 skb->pkt_type = PACKET_BROADCAST;
 387         }
 388 #endif
 389
 390         if ((!(tpi->flags&TUNNEL_CSUM) &&  (tunnel->parms.i_flags&TUNNEL_CSUM)) ||
 391              ((tpi->flags&TUNNEL_CSUM) && !(tunnel->parms.i_flags&TUNNEL_CSUM))) {
 392                 tunnel->dev->stats.rx_crc_errors++;
 393                 tunnel->dev->stats.rx_errors++;
 394                 goto drop;
 395         }
 396
 397         if (tunnel->parms.i_flags&TUNNEL_SEQ) {
 398                 if (!(tpi->flags&TUNNEL_SEQ) ||
 399                     (tunnel->i_seqno && (s32)(ntohl(tpi->seq) - tunnel->i_seqno) < 0)) {
 400                         tunnel->dev->stats.rx_fifo_errors++;
 401                         tunnel->dev->stats.rx_errors++;
 402                         goto drop;
 403                 }
 404                 tunnel->i_seqno = ntohl(tpi->seq) + 1;
 405         }
 406
 407         skb_reset_network_header(skb);
 408
 409         err = IP_ECN_decapsulate(iph, skb);
 410         if (unlikely(err)) {
 411                 if (log_ecn_error)
 412                         net_info_ratelimited("non-ECT from %pI4 with TOS=%#x\n",
 413                                         &iph->saddr, iph->tos);
 414                 if (err > 1) {
 415                         ++tunnel->dev->stats.rx_frame_errors;
 416                         ++tunnel->dev->stats.rx_errors;
 417                         goto drop;
 418                 }
 419         }
 420
 421         tstats = this_cpu_ptr(tunnel->dev->tstats);
 422         u64_stats_update_begin(&tstats->syncp);
 423         tstats->rx_packets++;
 424         tstats->rx_bytes += skb->len;
 425         u64_stats_update_end(&tstats->syncp);
 426
 427         skb_scrub_packet(skb, !net_eq(tunnel->net, dev_net(tunnel->dev)));
 428
 429         if (tunnel->dev->type == ARPHRD_ETHER) {
 430                 skb->protocol = eth_type_trans(skb, tunnel->dev);
 431                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
 432         } else {
 433                 skb->dev = tunnel->dev;
 434         }
 435
 436         if (tun_dst)
 437                 skb_dst_set(skb, (struct dst_entry *)tun_dst);
 438
 439         gro_cells_receive(&tunnel->gro_cells, skb);
 440         return 0;
 441
 442 drop:
 443         kfree_skb(skb);
 444         return 0;
 445 }
 446 EXPORT_SYMBOL_GPL(ip_tunnel_rcv);
 447
 448 static int ip_encap_hlen(struct ip_tunnel_encap *e)
 449 {
 450         const struct ip_tunnel_encap_ops *ops;
 451         int hlen = -EINVAL;
 452
 453         if (e->type == TUNNEL_ENCAP_NONE)
 454                 return 0;
 455
 456         if (e->type >= MAX_IPTUN_ENCAP_OPS)
 457                 return -EINVAL;
 458
 459         rcu_read_lock();
 460         ops = rcu_dereference(iptun_encaps[e->type]);
 461         if (likely(ops && ops->encap_hlen))
 462                 hlen = ops->encap_hlen(e);
 463         rcu_read_unlock();
 464
 465         return hlen;
 466 }
 467
 468 const struct ip_tunnel_encap_ops __rcu *
 469                 iptun_encaps[MAX_IPTUN_ENCAP_OPS] __read_mostly;
 470
 471 int ip_tunnel_encap_add_ops(const struct ip_tunnel_encap_ops *ops,
 472                             unsigned int num)
 473 {
 474         if (num >= MAX_IPTUN_ENCAP_OPS)
 475                 return -ERANGE;
 476
 477         return !cmpxchg((const struct ip_tunnel_encap_ops **)
 478                         &iptun_encaps[num],
 479                         NULL, ops) ? 0 : -1;
 480 }
 481 EXPORT_SYMBOL(ip_tunnel_encap_add_ops);
 482
 483 int ip_tunnel_encap_del_ops(const struct ip_tunnel_encap_ops *ops,
 484                             unsigned int num)
 485 {
 486         int ret;
 487
 488         if (num >= MAX_IPTUN_ENCAP_OPS)
 489                 return -ERANGE;
 490
 491         ret = (cmpxchg((const struct ip_tunnel_encap_ops **)
 492                        &iptun_encaps[num],
 493                        ops, NULL) == ops) ? 0 : -1;
 494
 495         synchronize_net();
 496
 497         return ret;
 498 }
 499 EXPORT_SYMBOL(ip_tunnel_encap_del_ops);
 500
 501 int ip_tunnel_encap_setup(struct ip_tunnel *t,
 502                           struct ip_tunnel_encap *ipencap)
 503 {
 504         int hlen;
 505
 506         memset(&t->encap, 0, sizeof(t->encap));
 507
 508         hlen = ip_encap_hlen(ipencap);
 509         if (hlen < 0)
 510                 return hlen;
 511
 512         t->encap.type = ipencap->type;
 513         t->encap.sport = ipencap->sport;
 514         t->encap.dport = ipencap->dport;
 515         t->encap.flags = ipencap->flags;
 516
 517         t->encap_hlen = hlen;
 518         t->hlen = t->encap_hlen + t->tun_hlen;
 519
 520         return 0;
 521 }
 522 EXPORT_SYMBOL_GPL(ip_tunnel_encap_setup);
 523
 524 int ip_tunnel_encap(struct sk_buff *skb, struct ip_tunnel *t,
 525                     u8 *protocol, struct flowi4 *fl4)
 526 {
 527         const struct ip_tunnel_encap_ops *ops;
 528         int ret = -EINVAL;
 529
 530         if (t->encap.type == TUNNEL_ENCAP_NONE)
 531                 return 0;
 532
 533         if (t->encap.type >= MAX_IPTUN_ENCAP_OPS)
 534                 return -EINVAL;
 535
 536         rcu_read_lock();
 537         ops = rcu_dereference(iptun_encaps[t->encap.type]);
 538         if (likely(ops && ops->build_header))
 539                 ret = ops->build_header(skb, &t->encap, protocol, fl4);
 540         rcu_read_unlock();
 541
 542         return ret;
 543 }
 544 EXPORT_SYMBOL(ip_tunnel_encap);
 545
 546 static int tnl_update_pmtu(struct net_device *dev, struct sk_buff *skb,
 547                             struct rtable *rt, __be16 df,
 548                             const struct iphdr *inner_iph)
 549 {
 550         struct ip_tunnel *tunnel = netdev_priv(dev);
 551         int pkt_size = skb->len - tunnel->hlen - dev->hard_header_len;
 552         int mtu;
 553
 554         if (df)
 555                 mtu = dst_mtu(&rt->dst) - dev->hard_header_len
 556                                         - sizeof(struct iphdr) - tunnel->hlen;
 557         else
 558                 mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
 559
 560         if (skb_dst(skb))
 561                 skb_dst(skb)->ops->update_pmtu(skb_dst(skb), NULL, skb, mtu);
 562
 563         if (skb->protocol == htons(ETH_P_IP)) {
 564                 if (!skb_is_gso(skb) &&
 565                     (inner_iph->frag_off & htons(IP_DF)) &&
 566                     mtu < pkt_size) {
 567                         memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
 568                         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu));
 569                         return -E2BIG;
 570                 }
 571         }
 572 #if IS_ENABLED(CONFIG_IPV6)
 573         else if (skb->protocol == htons(ETH_P_IPV6)) {
 574                 struct rt6_info *rt6 = (struct rt6_info *)skb_dst(skb);
 575
 576                 if (rt6 && mtu < dst_mtu(skb_dst(skb)) &&
 577                            mtu >= IPV6_MIN_MTU) {
 578                         if ((tunnel->parms.iph.daddr &&
 579                             !ipv4_is_multicast(tunnel->parms.iph.daddr)) ||
 580                             rt6->rt6i_dst.plen == 128) {
 581                                 rt6->rt6i_flags |= RTF_MODIFIED;
 582                                 dst_metric_set(skb_dst(skb), RTAX_MTU, mtu);
 583                         }
 584                 }
 585
 586                 if (!skb_is_gso(skb) && mtu >= IPV6_MIN_MTU &&
 587                                         mtu < pkt_size) {
 588                         icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
 589                         return -E2BIG;
 590                 }
 591         }
 592 #endif
 593         return 0;
 594 }
 595
 596 void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
 597                     const struct iphdr *tnl_params, u8 protocol)
 598 {
 599         struct ip_tunnel *tunnel = netdev_priv(dev);
 600         const struct iphdr *inner_iph;
 601         struct flowi4 fl4;
 602         u8     tos, ttl;
 603         __be16 df;
 604         struct rtable *rt;              /* Route to the other host */
 605         unsigned int max_headroom;      /* The extra header space needed */
 606         __be32 dst;
 607         int err;
 608         bool connected;
 609
 610         inner_iph = (const struct iphdr *)skb_inner_network_header(skb);
 611         connected = (tunnel->parms.iph.daddr != 0);
 612
 613         memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
 614
 615         dst = tnl_params->daddr;
 616         if (dst == 0) {
 617                 /* NBMA tunnel */
 618
 619                 if (!skb_dst(skb)) {
 620                         dev->stats.tx_fifo_errors++;
 621                         goto tx_error;
 622                 }
 623
 624                 if (skb->protocol == htons(ETH_P_IP)) {
 625                         rt = skb_rtable(skb);
 626                         dst = rt_nexthop(rt, inner_iph->daddr);
 627                 }
 628 #if IS_ENABLED(CONFIG_IPV6)
 629                 else if (skb->protocol == htons(ETH_P_IPV6)) {
 630                         const struct in6_addr *addr6;
 631                         struct neighbour *neigh;
 632                         bool do_tx_error_icmp;
 633                         int addr_type;
 634
 635                         neigh = dst_neigh_lookup(skb_dst(skb),
 636                                                  &ipv6_hdr(skb)->daddr);
 637                         if (!neigh)
 638                                 goto tx_error;
 639
 640                         addr6 = (const struct in6_addr *)&neigh->primary_key;
 641                         addr_type = ipv6_addr_type(addr6);
 642
 643                         if (addr_type == IPV6_ADDR_ANY) {
 644                                 addr6 = &ipv6_hdr(skb)->daddr;
 645                                 addr_type = ipv6_addr_type(addr6);
 646                         }
 647
 648                         if ((addr_type & IPV6_ADDR_COMPATv4) == 0)
 649                                 do_tx_error_icmp = true;
 650                         else {
 651                                 do_tx_error_icmp = false;
 652                                 dst = addr6->s6_addr32[3];
 653                         }
 654                         neigh_release(neigh);
 655                         if (do_tx_error_icmp)
 656                                 goto tx_error_icmp;
 657                 }
 658 #endif
 659                 else
 660                         goto tx_error;
 661
 662                 connected = false;
 663         }
 664
 665         tos = tnl_params->tos;
 666         if (tos & 0x1) {
 667                 tos &= ~0x1;
 668                 if (skb->protocol == htons(ETH_P_IP)) {
 669                         tos = inner_iph->tos;
 670                         connected = false;
 671                 } else if (skb->protocol == htons(ETH_P_IPV6)) {
 672                         tos = ipv6_get_dsfield((const struct ipv6hdr *)inner_iph);
 673                         connected = false;
 674                 }
 675         }
 676
 677         init_tunnel_flow(&fl4, protocol, dst, tnl_params->saddr,
 678                          tunnel->parms.o_key, RT_TOS(tos), tunnel->parms.link);
 679
 680         if (ip_tunnel_encap(skb, tunnel, &protocol, &fl4) < 0)
 681                 goto tx_error;
 682
 683         rt = connected ? dst_cache_get_ip4(&tunnel->dst_cache, &fl4.saddr) :
 684                          NULL;
 685
 686         if (!rt) {
 687                 rt = ip_route_output_key(tunnel->net, &fl4);
 688
 689                 if (IS_ERR(rt)) {
 690                         dev->stats.tx_carrier_errors++;
 691                         goto tx_error;
 692                 }
 693                 if (connected)
 694                         dst_cache_set_ip4(&tunnel->dst_cache, &rt->dst,
 695                                           fl4.saddr);
 696         }
 697
 698         if (rt->dst.dev == dev) {
 699                 ip_rt_put(rt);
 700                 dev->stats.collisions++;
 701                 goto tx_error;
 702         }
 703
 704         if (tnl_update_pmtu(dev, skb, rt, tnl_params->frag_off, inner_iph)) {
 705                 ip_rt_put(rt);
 706                 goto tx_error;
 707         }
 708
 709         if (tunnel->err_count > 0) {
 710                 if (time_before(jiffies,
 711                                 tunnel->err_time + IPTUNNEL_ERR_TIMEO)) {
 712                         tunnel->err_count--;
 713
 714                         dst_link_failure(skb);
 715                 } else
 716                         tunnel->err_count = 0;
 717         }
 718
 719         tos = ip_tunnel_ecn_encap(tos, inner_iph, skb);
 720         ttl = tnl_params->ttl;
 721         if (ttl == 0) {
 722                 if (skb->protocol == htons(ETH_P_IP))
 723                         ttl = inner_iph->ttl;
 724 #if IS_ENABLED(CONFIG_IPV6)
 725                 else if (skb->protocol == htons(ETH_P_IPV6))
 726                         ttl = ((const struct ipv6hdr *)inner_iph)->hop_limit;
 727 #endif
 728                 else
 729                         ttl = ip4_dst_hoplimit(&rt->dst);
 730         }
 731
 732         df = tnl_params->frag_off;
 733         if (skb->protocol == htons(ETH_P_IP))
 734                 df |= (inner_iph->frag_off&htons(IP_DF));
 735
 736         max_headroom = LL_RESERVED_SPACE(rt->dst.dev) + sizeof(struct iphdr)
 737                         + rt->dst.header_len + ip_encap_hlen(&tunnel->encap);
 738         if (max_headroom > dev->needed_headroom)
 739                 dev->needed_headroom = max_headroom;
 740
 741         if (skb_cow_head(skb, dev->needed_headroom)) {
 742                 ip_rt_put(rt);
 743                 dev->stats.tx_dropped++;
 744                 kfree_skb(skb);
 745                 return;
 746         }
 747
 748         err = iptunnel_xmit(NULL, rt, skb, fl4.saddr, fl4.daddr, protocol,
 749                             tos, ttl, df, !net_eq(tunnel->net, dev_net(dev)));
 750         iptunnel_xmit_stats(err, &dev->stats, dev->tstats);
 751
 752         return;
 753
 754 #if IS_ENABLED(CONFIG_IPV6)
 755 tx_error_icmp:
 756         dst_link_failure(skb);
 757 #endif
 758 tx_error:
 759         dev->stats.tx_errors++;
 760         kfree_skb(skb);
 761 }
 762 EXPORT_SYMBOL_GPL(ip_tunnel_xmit);
 763
 764 static void ip_tunnel_update(struct ip_tunnel_net *itn,
 765                              struct ip_tunnel *t,
 766                              struct net_device *dev,
 767                              struct ip_tunnel_parm *p,
 768                              bool set_mtu)
 769 {
 770         ip_tunnel_del(itn, t);
 771         t->parms.iph.saddr = p->iph.saddr;
 772         t->parms.iph.daddr = p->iph.daddr;
 773         t->parms.i_key = p->i_key;
 774         t->parms.o_key = p->o_key;
 775         if (dev->type != ARPHRD_ETHER) {
 776                 memcpy(dev->dev_addr, &p->iph.saddr, 4);
 777                 memcpy(dev->broadcast, &p->iph.daddr, 4);
 778         }
 779         ip_tunnel_add(itn, t);
 780
 781         t->parms.iph.ttl = p->iph.ttl;
 782         t->parms.iph.tos = p->iph.tos;
 783         t->parms.iph.frag_off = p->iph.frag_off;
 784
 785         if (t->parms.link != p->link) {
 786                 int mtu;
 787
 788                 t->parms.link = p->link;
 789                 mtu = ip_tunnel_bind_dev(dev);
 790                 if (set_mtu)
 791                         dev->mtu = mtu;
 792         }
 793         dst_cache_reset(&t->dst_cache);
 794         netdev_state_change(dev);
 795 }
 796
 797 int ip_tunnel_ioctl(struct net_device *dev, struct ip_tunnel_parm *p, int cmd)
 798 {
 799         int err = 0;
 800         struct ip_tunnel *t = netdev_priv(dev);
 801         struct net *net = t->net;
 802         struct ip_tunnel_net *itn = net_generic(net, t->ip_tnl_net_id);
 803
 804         BUG_ON(!itn->fb_tunnel_dev);
 805         switch (cmd) {
 806         case SIOCGETTUNNEL:
 807                 if (dev == itn->fb_tunnel_dev) {
 808                         t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
 809                         if (!t)
 810                                 t = netdev_priv(dev);
 811                 }
 812                 memcpy(p, &t->parms, sizeof(*p));
 813                 break;
 814
 815         case SIOCADDTUNNEL:
 816         case SIOCCHGTUNNEL:
 817                 err = -EPERM;
 818                 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
 819                         goto done;
 820                 if (p->iph.ttl)
 821                         p->iph.frag_off |= htons(IP_DF);
 822                 if (!(p->i_flags & VTI_ISVTI)) {
 823                         if (!(p->i_flags & TUNNEL_KEY))
 824                                 p->i_key = 0;
 825                         if (!(p->o_flags & TUNNEL_KEY))
 826                                 p->o_key = 0;
 827                 }
 828
 829                 t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
 830
 831                 if (cmd == SIOCADDTUNNEL) {
 832                         if (!t) {
 833                                 t = ip_tunnel_create(net, itn, p);
 834                                 err = PTR_ERR_OR_ZERO(t);
 835                                 break;
 836                         }
 837
 838                         err = -EEXIST;
 839                         break;
 840                 }
 841                 if (dev != itn->fb_tunnel_dev && cmd == SIOCCHGTUNNEL) {
 842                         if (t) {
 843                                 if (t->dev != dev) {
 844                                         err = -EEXIST;
 845                                         break;
 846                                 }
 847                         } else {
 848                                 unsigned int nflags = 0;
 849
 850                                 if (ipv4_is_multicast(p->iph.daddr))
 851                                         nflags = IFF_BROADCAST;
 852                                 else if (p->iph.daddr)
 853                                         nflags = IFF_POINTOPOINT;
 854
 855                                 if ((dev->flags^nflags)&(IFF_POINTOPOINT|IFF_BROADCAST)) {
 856                                         err = -EINVAL;
 857                                         break;
 858                                 }
 859
 860                                 t = netdev_priv(dev);
 861                         }
 862                 }
 863
 864                 if (t) {
 865                         err = 0;
 866                         ip_tunnel_update(itn, t, dev, p, true);
 867                 } else {
 868                         err = -ENOENT;
 869                 }
 870                 break;
 871
 872         case SIOCDELTUNNEL:
 873                 err = -EPERM;
 874                 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
 875                         goto done;
 876
 877                 if (dev == itn->fb_tunnel_dev) {
 878                         err = -ENOENT;
 879                         t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
 880                         if (!t)
 881                                 goto done;
 882                         err = -EPERM;
 883                         if (t == netdev_priv(itn->fb_tunnel_dev))
 884                                 goto done;
 885                         dev = t->dev;
 886                 }
 887                 unregister_netdevice(dev);
 888                 err = 0;
 889                 break;
 890
 891         default:
 892                 err = -EINVAL;
 893         }
 894
 895 done:
 896         return err;
 897 }
 898 EXPORT_SYMBOL_GPL(ip_tunnel_ioctl);
 899
 900 int __ip_tunnel_change_mtu(struct net_device *dev, int new_mtu, bool strict)
 901 {
 902         struct ip_tunnel *tunnel = netdev_priv(dev);
 903         int t_hlen = tunnel->hlen + sizeof(struct iphdr);
 904         int max_mtu = 0xFFF8 - dev->hard_header_len - t_hlen;
 905
 906         if (new_mtu < 68)
 907                 return -EINVAL;
 908
 909         if (new_mtu > max_mtu) {
 910                 if (strict)
 911                         return -EINVAL;
 912
 913                 new_mtu = max_mtu;
 914         }
 915
 916         dev->mtu = new_mtu;
 917         return 0;
 918 }
 919 EXPORT_SYMBOL_GPL(__ip_tunnel_change_mtu);
 920
 921 int ip_tunnel_change_mtu(struct net_device *dev, int new_mtu)
 922 {
 923         return __ip_tunnel_change_mtu(dev, new_mtu, true);
 924 }
 925 EXPORT_SYMBOL_GPL(ip_tunnel_change_mtu);
 926
 927 static void ip_tunnel_dev_free(struct net_device *dev)
 928 {
 929         struct ip_tunnel *tunnel = netdev_priv(dev);
 930
 931         gro_cells_destroy(&tunnel->gro_cells);
 932         dst_cache_destroy(&tunnel->dst_cache);
 933         free_percpu(dev->tstats);
 934         free_netdev(dev);
 935 }
 936
 937 void ip_tunnel_dellink(struct net_device *dev, struct list_head *head)
 938 {
 939         struct ip_tunnel *tunnel = netdev_priv(dev);
 940         struct ip_tunnel_net *itn;
 941
 942         itn = net_generic(tunnel->net, tunnel->ip_tnl_net_id);
 943
 944         if (itn->fb_tunnel_dev != dev) {
 945                 ip_tunnel_del(itn, netdev_priv(dev));
 946                 unregister_netdevice_queue(dev, head);
 947         }
 948 }
 949 EXPORT_SYMBOL_GPL(ip_tunnel_dellink);
 950
 951 struct net *ip_tunnel_get_link_net(const struct net_device *dev)
 952 {
 953         struct ip_tunnel *tunnel = netdev_priv(dev);
 954
 955         return tunnel->net;
 956 }
 957 EXPORT_SYMBOL(ip_tunnel_get_link_net);
 958
 959 int ip_tunnel_get_iflink(const struct net_device *dev)
 960 {
 961         struct ip_tunnel *tunnel = netdev_priv(dev);
 962
 963         return tunnel->parms.link;
 964 }
 965 EXPORT_SYMBOL(ip_tunnel_get_iflink);
 966
 967 int ip_tunnel_init_net(struct net *net, int ip_tnl_net_id,
 968                                   struct rtnl_link_ops *ops, char *devname)
 969 {
 970         struct ip_tunnel_net *itn = net_generic(net, ip_tnl_net_id);
 971         struct ip_tunnel_parm parms;
 972         unsigned int i;
 973
 974         for (i = 0; i < IP_TNL_HASH_SIZE; i++)
 975                 INIT_HLIST_HEAD(&itn->tunnels[i]);
 976
 977         if (!ops) {
 978                 itn->fb_tunnel_dev = NULL;
 979                 return 0;
 980         }
 981
 982         memset(&parms, 0, sizeof(parms));
 983         if (devname)
 984                 strlcpy(parms.name, devname, IFNAMSIZ);
 985
 986         rtnl_lock();
 987         itn->fb_tunnel_dev = __ip_tunnel_create(net, ops, &parms);
 988         /* FB netdevice is special: we have one, and only one per netns.
 989          * Allowing to move it to another netns is clearly unsafe.
 990          */
 991         if (!IS_ERR(itn->fb_tunnel_dev)) {
 992                 itn->fb_tunnel_dev->features |= NETIF_F_NETNS_LOCAL;
 993                 itn->fb_tunnel_dev->mtu = ip_tunnel_bind_dev(itn->fb_tunnel_dev);
 994                 ip_tunnel_add(itn, netdev_priv(itn->fb_tunnel_dev));
 995         }
 996         rtnl_unlock();
 997
 998         return PTR_ERR_OR_ZERO(itn->fb_tunnel_dev);
 999 }
1000 EXPORT_SYMBOL_GPL(ip_tunnel_init_net);
1001
1002 static void ip_tunnel_destroy(struct ip_tunnel_net *itn, struct list_head *head,
1003                               struct rtnl_link_ops *ops)
1004 {
1005         struct net *net = dev_net(itn->fb_tunnel_dev);
1006         struct net_device *dev, *aux;
1007         int h;
1008
1009         for_each_netdev_safe(net, dev, aux)
1010                 if (dev->rtnl_link_ops == ops)
1011                         unregister_netdevice_queue(dev, head);
1012
1013         for (h = 0; h < IP_TNL_HASH_SIZE; h++) {
1014                 struct ip_tunnel *t;
1015                 struct hlist_node *n;
1016                 struct hlist_head *thead = &itn->tunnels[h];
1017
1018                 hlist_for_each_entry_safe(t, n, thead, hash_node)
1019                         /* If dev is in the same netns, it has already
1020                          * been added to the list by the previous loop.
1021                          */
1022                         if (!net_eq(dev_net(t->dev), net))
1023                                 unregister_netdevice_queue(t->dev, head);
1024         }
1025 }
1026
1027 void ip_tunnel_delete_net(struct ip_tunnel_net *itn, struct rtnl_link_ops *ops)
1028 {
1029         LIST_HEAD(list);
1030
1031         rtnl_lock();
1032         ip_tunnel_destroy(itn, &list, ops);
1033         unregister_netdevice_many(&list);
1034         rtnl_unlock();
1035 }
1036 EXPORT_SYMBOL_GPL(ip_tunnel_delete_net);
1037
1038 int ip_tunnel_newlink(struct net_device *dev, struct nlattr *tb[],
1039                       struct ip_tunnel_parm *p)
1040 {
1041         struct ip_tunnel *nt;
1042         struct net *net = dev_net(dev);
1043         struct ip_tunnel_net *itn;
1044         int mtu;
1045         int err;
1046
1047         nt = netdev_priv(dev);
1048         itn = net_generic(net, nt->ip_tnl_net_id);
1049
1050         if (nt->collect_md) {
1051                 if (rtnl_dereference(itn->collect_md_tun))
1052                         return -EEXIST;
1053         } else {
1054                 if (ip_tunnel_find(itn, p, dev->type))
1055                         return -EEXIST;
1056         }
1057
1058         nt->net = net;
1059         nt->parms = *p;
1060         err = register_netdevice(dev);
1061         if (err)
1062                 goto out;
1063
1064         if (dev->type == ARPHRD_ETHER && !tb[IFLA_ADDRESS])
1065                 eth_hw_addr_random(dev);
1066
1067         mtu = ip_tunnel_bind_dev(dev);
1068         if (!tb[IFLA_MTU])
1069                 dev->mtu = mtu;
1070
1071         ip_tunnel_add(itn, nt);
1072 out:
1073         return err;
1074 }
1075 EXPORT_SYMBOL_GPL(ip_tunnel_newlink);
1076
1077 int ip_tunnel_changelink(struct net_device *dev, struct nlattr *tb[],
1078                          struct ip_tunnel_parm *p)
1079 {
1080         struct ip_tunnel *t;
1081         struct ip_tunnel *tunnel = netdev_priv(dev);
1082         struct net *net = tunnel->net;
1083         struct ip_tunnel_net *itn = net_generic(net, tunnel->ip_tnl_net_id);
1084
1085         if (dev == itn->fb_tunnel_dev)
1086                 return -EINVAL;
1087
1088         t = ip_tunnel_find(itn, p, dev->type);
1089
1090         if (t) {
1091                 if (t->dev != dev)
1092                         return -EEXIST;
1093         } else {
1094                 t = tunnel;
1095
1096                 if (dev->type != ARPHRD_ETHER) {
1097                         unsigned int nflags = 0;
1098
1099                         if (ipv4_is_multicast(p->iph.daddr))
1100                                 nflags = IFF_BROADCAST;
1101                         else if (p->iph.daddr)
1102                                 nflags = IFF_POINTOPOINT;
1103
1104                         if ((dev->flags ^ nflags) &
1105                             (IFF_POINTOPOINT | IFF_BROADCAST))
1106                                 return -EINVAL;
1107                 }
1108         }
1109
1110         ip_tunnel_update(itn, t, dev, p, !tb[IFLA_MTU]);
1111         return 0;
1112 }
1113 EXPORT_SYMBOL_GPL(ip_tunnel_changelink);
1114
1115 int ip_tunnel_init(struct net_device *dev)
1116 {
1117         struct ip_tunnel *tunnel = netdev_priv(dev);
1118         struct iphdr *iph = &tunnel->parms.iph;
1119         int err;
1120
1121         dev->destructor = ip_tunnel_dev_free;
1122         dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
1123         if (!dev->tstats)
1124                 return -ENOMEM;
1125
1126         err = dst_cache_init(&tunnel->dst_cache, GFP_KERNEL);
1127         if (err) {
1128                 free_percpu(dev->tstats);
1129                 return err;
1130         }
1131
1132         err = gro_cells_init(&tunnel->gro_cells, dev);
1133         if (err) {
1134                 dst_cache_destroy(&tunnel->dst_cache);
1135                 free_percpu(dev->tstats);
1136                 return err;
1137         }
1138
1139         tunnel->dev = dev;
1140         tunnel->net = dev_net(dev);
1141         strcpy(tunnel->parms.name, dev->name);
1142         iph->version            = 4;
1143         iph->ihl                = 5;
1144
1145         if (tunnel->collect_md) {
1146                 dev->features |= NETIF_F_NETNS_LOCAL;
1147                 netif_keep_dst(dev);
1148         }
1149         return 0;
1150 }
1151 EXPORT_SYMBOL_GPL(ip_tunnel_init);
1152
1153 void ip_tunnel_uninit(struct net_device *dev)
1154 {
1155         struct ip_tunnel *tunnel = netdev_priv(dev);
1156         struct net *net = tunnel->net;
1157         struct ip_tunnel_net *itn;
1158
1159         itn = net_generic(net, tunnel->ip_tnl_net_id);
1160         /* fb_tunnel_dev will be unregisted in net-exit call. */
1161         if (itn->fb_tunnel_dev != dev)
1162                 ip_tunnel_del(itn, netdev_priv(dev));
1163
1164         dst_cache_reset(&tunnel->dst_cache);
1165 }
1166 EXPORT_SYMBOL_GPL(ip_tunnel_uninit);
1167
1168 /* Do least required initialization, rest of init is done in tunnel_init call */
1169 void ip_tunnel_setup(struct net_device *dev, int net_id)
1170 {
1171         struct ip_tunnel *tunnel = netdev_priv(dev);
1172         tunnel->ip_tnl_net_id = net_id;
1173 }
1174 EXPORT_SYMBOL_GPL(ip_tunnel_setup);
1175
1176 MODULE_LICENSE("GPL");