net/ipv4/ip_tunnel.c

   1 /*
   2  * Copyright (c) 2013 Nicira, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of version 2 of the GNU General Public
   6  * License as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * along with this program; if not, write to the Free Software
  15  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  16  * 02110-1301, USA
  17  */
  18
  19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  20
  21 #include <linux/capability.h>
  22 #include <linux/module.h>
  23 #include <linux/types.h>
  24 #include <linux/kernel.h>
  25 #include <linux/slab.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/skbuff.h>
  28 #include <linux/netdevice.h>
  29 #include <linux/in.h>
  30 #include <linux/tcp.h>
  31 #include <linux/udp.h>
  32 #include <linux/if_arp.h>
  33 #include <linux/mroute.h>
  34 #include <linux/init.h>
  35 #include <linux/in6.h>
  36 #include <linux/inetdevice.h>
  37 #include <linux/igmp.h>
  38 #include <linux/netfilter_ipv4.h>
  39 #include <linux/etherdevice.h>
  40 #include <linux/if_ether.h>
  41 #include <linux/if_vlan.h>
  42 #include <linux/rculist.h>
  43
  44 #include <net/sock.h>
  45 #include <net/ip.h>
  46 #include <net/icmp.h>
  47 #include <net/protocol.h>
  48 #include <net/ip_tunnels.h>
  49 #include <net/arp.h>
  50 #include <net/checksum.h>
  51 #include <net/dsfield.h>
  52 #include <net/inet_ecn.h>
  53 #include <net/xfrm.h>
  54 #include <net/net_namespace.h>
  55 #include <net/netns/generic.h>
  56 #include <net/rtnetlink.h>
  57
  58 #if IS_ENABLED(CONFIG_IPV6)
  59 #include <net/ipv6.h>
  60 #include <net/ip6_fib.h>
  61 #include <net/ip6_route.h>
  62 #endif
  63
  64 static unsigned int ip_tunnel_hash(struct ip_tunnel_net *itn,
  65                                    __be32 key, __be32 remote)
  66 {
  67         return hash_32((__force u32)key ^ (__force u32)remote,
  68                          IP_TNL_HASH_BITS);
  69 }
  70
  71 /* Often modified stats are per cpu, other are shared (netdev->stats) */
  72 struct rtnl_link_stats64 *ip_tunnel_get_stats64(struct net_device *dev,
  73                                                 struct rtnl_link_stats64 *tot)
  74 {
  75         int i;
  76
  77         for_each_possible_cpu(i) {
  78                 const struct pcpu_tstats *tstats = per_cpu_ptr(dev->tstats, i);
  79                 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
  80                 unsigned int start;
  81
  82                 do {
  83                         start = u64_stats_fetch_begin_bh(&tstats->syncp);
  84                         rx_packets = tstats->rx_packets;
  85                         tx_packets = tstats->tx_packets;
  86                         rx_bytes = tstats->rx_bytes;
  87                         tx_bytes = tstats->tx_bytes;
  88                 } while (u64_stats_fetch_retry_bh(&tstats->syncp, start));
  89
  90                 tot->rx_packets += rx_packets;
  91                 tot->tx_packets += tx_packets;
  92                 tot->rx_bytes   += rx_bytes;
  93                 tot->tx_bytes   += tx_bytes;
  94         }
  95
  96         tot->multicast = dev->stats.multicast;
  97
  98         tot->rx_crc_errors = dev->stats.rx_crc_errors;
  99         tot->rx_fifo_errors = dev->stats.rx_fifo_errors;
 100         tot->rx_length_errors = dev->stats.rx_length_errors;
 101         tot->rx_frame_errors = dev->stats.rx_frame_errors;
 102         tot->rx_errors = dev->stats.rx_errors;
 103
 104         tot->tx_fifo_errors = dev->stats.tx_fifo_errors;
 105         tot->tx_carrier_errors = dev->stats.tx_carrier_errors;
 106         tot->tx_dropped = dev->stats.tx_dropped;
 107         tot->tx_aborted_errors = dev->stats.tx_aborted_errors;
 108         tot->tx_errors = dev->stats.tx_errors;
 109
 110         tot->collisions  = dev->stats.collisions;
 111
 112         return tot;
 113 }
 114 EXPORT_SYMBOL_GPL(ip_tunnel_get_stats64);
 115
 116 static bool ip_tunnel_key_match(const struct ip_tunnel_parm *p,
 117                                 __be16 flags, __be32 key)
 118 {
 119         if (p->i_flags & TUNNEL_KEY) {
 120                 if (flags & TUNNEL_KEY)
 121                         return key == p->i_key;
 122                 else
 123                         /* key expected, none present */
 124                         return false;
 125         } else
 126                 return !(flags & TUNNEL_KEY);
 127 }
 128
 129 /* Fallback tunnel: no source, no destination, no key, no options
 130
 131    Tunnel hash table:
 132    We require exact key match i.e. if a key is present in packet
 133    it will match only tunnel with the same key; if it is not present,
 134    it will match only keyless tunnel.
 135
 136    All keysless packets, if not matched configured keyless tunnels
 137    will match fallback tunnel.
 138    Given src, dst and key, find appropriate for input tunnel.
 139 */
 140 struct ip_tunnel *ip_tunnel_lookup(struct ip_tunnel_net *itn,
 141                                    int link, __be16 flags,
 142                                    __be32 remote, __be32 local,
 143                                    __be32 key)
 144 {
 145         unsigned int hash;
 146         struct ip_tunnel *t, *cand = NULL;
 147         struct hlist_head *head;
 148
 149         hash = ip_tunnel_hash(itn, key, remote);
 150         head = &itn->tunnels[hash];
 151
 152         hlist_for_each_entry_rcu(t, head, hash_node) {
 153                 if (local != t->parms.iph.saddr ||
 154                     remote != t->parms.iph.daddr ||
 155                     !(t->dev->flags & IFF_UP))
 156                         continue;
 157
 158                 if (!ip_tunnel_key_match(&t->parms, flags, key))
 159                         continue;
 160
 161                 if (t->parms.link == link)
 162                         return t;
 163                 else
 164                         cand = t;
 165         }
 166
 167         hlist_for_each_entry_rcu(t, head, hash_node) {
 168                 if (remote != t->parms.iph.daddr ||
 169                     t->parms.iph.saddr != 0 ||
 170                     !(t->dev->flags & IFF_UP))
 171                         continue;
 172
 173                 if (!ip_tunnel_key_match(&t->parms, flags, key))
 174                         continue;
 175
 176                 if (t->parms.link == link)
 177                         return t;
 178                 else if (!cand)
 179                         cand = t;
 180         }
 181
 182         hash = ip_tunnel_hash(itn, key, 0);
 183         head = &itn->tunnels[hash];
 184
 185         hlist_for_each_entry_rcu(t, head, hash_node) {
 186                 if ((local != t->parms.iph.saddr || t->parms.iph.daddr != 0) &&
 187                     (local != t->parms.iph.daddr || !ipv4_is_multicast(local)))
 188                         continue;
 189
 190                 if (!(t->dev->flags & IFF_UP))
 191                         continue;
 192
 193                 if (!ip_tunnel_key_match(&t->parms, flags, key))
 194                         continue;
 195
 196                 if (t->parms.link == link)
 197                         return t;
 198                 else if (!cand)
 199                         cand = t;
 200         }
 201
 202         if (flags & TUNNEL_NO_KEY)
 203                 goto skip_key_lookup;
 204
 205         hlist_for_each_entry_rcu(t, head, hash_node) {
 206                 if (t->parms.i_key != key ||
 207                     t->parms.iph.saddr != 0 ||
 208                     t->parms.iph.daddr != 0 ||
 209                     !(t->dev->flags & IFF_UP))
 210                         continue;
 211
 212                 if (t->parms.link == link)
 213                         return t;
 214                 else if (!cand)
 215                         cand = t;
 216         }
 217
 218 skip_key_lookup:
 219         if (cand)
 220                 return cand;
 221
 222         if (itn->fb_tunnel_dev && itn->fb_tunnel_dev->flags & IFF_UP)
 223                 return netdev_priv(itn->fb_tunnel_dev);
 224
 225
 226         return NULL;
 227 }
 228 EXPORT_SYMBOL_GPL(ip_tunnel_lookup);
 229
 230 static struct hlist_head *ip_bucket(struct ip_tunnel_net *itn,
 231                                     struct ip_tunnel_parm *parms)
 232 {
 233         unsigned int h;
 234         __be32 remote;
 235
 236         if (parms->iph.daddr && !ipv4_is_multicast(parms->iph.daddr))
 237                 remote = parms->iph.daddr;
 238         else
 239                 remote = 0;
 240
 241         h = ip_tunnel_hash(itn, parms->i_key, remote);
 242         return &itn->tunnels[h];
 243 }
 244
 245 static void ip_tunnel_add(struct ip_tunnel_net *itn, struct ip_tunnel *t)
 246 {
 247         struct hlist_head *head = ip_bucket(itn, &t->parms);
 248
 249         hlist_add_head_rcu(&t->hash_node, head);
 250 }
 251
 252 static void ip_tunnel_del(struct ip_tunnel *t)
 253 {
 254         hlist_del_init_rcu(&t->hash_node);
 255 }
 256
 257 static struct ip_tunnel *ip_tunnel_find(struct ip_tunnel_net *itn,
 258                                         struct ip_tunnel_parm *parms,
 259                                         int type)
 260 {
 261         __be32 remote = parms->iph.daddr;
 262         __be32 local = parms->iph.saddr;
 263         __be32 key = parms->i_key;
 264         int link = parms->link;
 265         struct ip_tunnel *t = NULL;
 266         struct hlist_head *head = ip_bucket(itn, parms);
 267
 268         hlist_for_each_entry_rcu(t, head, hash_node) {
 269                 if (local == t->parms.iph.saddr &&
 270                     remote == t->parms.iph.daddr &&
 271                     key == t->parms.i_key &&
 272                     link == t->parms.link &&
 273                     type == t->dev->type)
 274                         break;
 275         }
 276         return t;
 277 }
 278
 279 static struct net_device *__ip_tunnel_create(struct net *net,
 280                                              const struct rtnl_link_ops *ops,
 281                                              struct ip_tunnel_parm *parms)
 282 {
 283         int err;
 284         struct ip_tunnel *tunnel;
 285         struct net_device *dev;
 286         char name[IFNAMSIZ];
 287
 288         if (parms->name[0])
 289                 strlcpy(name, parms->name, IFNAMSIZ);
 290         else {
 291                 if (strlen(ops->kind) > (IFNAMSIZ - 3)) {
 292                         err = -E2BIG;
 293                         goto failed;
 294                 }
 295                 strlcpy(name, ops->kind, IFNAMSIZ);
 296                 strncat(name, "%d", 2);
 297         }
 298
 299         ASSERT_RTNL();
 300         dev = alloc_netdev(ops->priv_size, name, ops->setup);
 301         if (!dev) {
 302                 err = -ENOMEM;
 303                 goto failed;
 304         }
 305         dev_net_set(dev, net);
 306
 307         dev->rtnl_link_ops = ops;
 308
 309         tunnel = netdev_priv(dev);
 310         tunnel->parms = *parms;
 311         tunnel->net = net;
 312
 313         err = register_netdevice(dev);
 314         if (err)
 315                 goto failed_free;
 316
 317         return dev;
 318
 319 failed_free:
 320         free_netdev(dev);
 321 failed:
 322         return ERR_PTR(err);
 323 }
 324
 325 static inline struct rtable *ip_route_output_tunnel(struct net *net,
 326                                                     struct flowi4 *fl4,
 327                                                     int proto,
 328                                                     __be32 daddr, __be32 saddr,
 329                                                     __be32 key, __u8 tos, int oif)
 330 {
 331         memset(fl4, 0, sizeof(*fl4));
 332         fl4->flowi4_oif = oif;
 333         fl4->daddr = daddr;
 334         fl4->saddr = saddr;
 335         fl4->flowi4_tos = tos;
 336         fl4->flowi4_proto = proto;
 337         fl4->fl4_gre_key = key;
 338         return ip_route_output_key(net, fl4);
 339 }
 340
 341 static int ip_tunnel_bind_dev(struct net_device *dev)
 342 {
 343         struct net_device *tdev = NULL;
 344         struct ip_tunnel *tunnel = netdev_priv(dev);
 345         const struct iphdr *iph;
 346         int hlen = LL_MAX_HEADER;
 347         int mtu = ETH_DATA_LEN;
 348         int t_hlen = tunnel->hlen + sizeof(struct iphdr);
 349
 350         iph = &tunnel->parms.iph;
 351
 352         /* Guess output device to choose reasonable mtu and needed_headroom */
 353         if (iph->daddr) {
 354                 struct flowi4 fl4;
 355                 struct rtable *rt;
 356
 357                 rt = ip_route_output_tunnel(tunnel->net, &fl4,
 358                                             tunnel->parms.iph.protocol,
 359                                             iph->daddr, iph->saddr,
 360                                             tunnel->parms.o_key,
 361                                             RT_TOS(iph->tos),
 362                                             tunnel->parms.link);
 363                 if (!IS_ERR(rt)) {
 364                         tdev = rt->dst.dev;
 365                         ip_rt_put(rt);
 366                 }
 367                 if (dev->type != ARPHRD_ETHER)
 368                         dev->flags |= IFF_POINTOPOINT;
 369         }
 370
 371         if (!tdev && tunnel->parms.link)
 372                 tdev = __dev_get_by_index(tunnel->net, tunnel->parms.link);
 373
 374         if (tdev) {
 375                 hlen = tdev->hard_header_len + tdev->needed_headroom;
 376                 mtu = tdev->mtu;
 377         }
 378         dev->iflink = tunnel->parms.link;
 379
 380         dev->needed_headroom = t_hlen + hlen;
 381         mtu -= (dev->hard_header_len + t_hlen);
 382
 383         if (mtu < 68)
 384                 mtu = 68;
 385
 386         return mtu;
 387 }
 388
 389 static struct ip_tunnel *ip_tunnel_create(struct net *net,
 390                                           struct ip_tunnel_net *itn,
 391                                           struct ip_tunnel_parm *parms)
 392 {
 393         struct ip_tunnel *nt, *fbt;
 394         struct net_device *dev;
 395
 396         BUG_ON(!itn->fb_tunnel_dev);
 397         fbt = netdev_priv(itn->fb_tunnel_dev);
 398         dev = __ip_tunnel_create(net, itn->fb_tunnel_dev->rtnl_link_ops, parms);
 399         if (IS_ERR(dev))
 400                 return NULL;
 401
 402         dev->mtu = ip_tunnel_bind_dev(dev);
 403
 404         nt = netdev_priv(dev);
 405         ip_tunnel_add(itn, nt);
 406         return nt;
 407 }
 408
 409 int ip_tunnel_rcv(struct ip_tunnel *tunnel, struct sk_buff *skb,
 410                   const struct tnl_ptk_info *tpi, bool log_ecn_error)
 411 {
 412         struct pcpu_tstats *tstats;
 413         const struct iphdr *iph = ip_hdr(skb);
 414         int err;
 415
 416 #ifdef CONFIG_NET_IPGRE_BROADCAST
 417         if (ipv4_is_multicast(iph->daddr)) {
 418                 tunnel->dev->stats.multicast++;
 419                 skb->pkt_type = PACKET_BROADCAST;
 420         }
 421 #endif
 422
 423         if ((!(tpi->flags&TUNNEL_CSUM) &&  (tunnel->parms.i_flags&TUNNEL_CSUM)) ||
 424              ((tpi->flags&TUNNEL_CSUM) && !(tunnel->parms.i_flags&TUNNEL_CSUM))) {
 425                 tunnel->dev->stats.rx_crc_errors++;
 426                 tunnel->dev->stats.rx_errors++;
 427                 goto drop;
 428         }
 429
 430         if (tunnel->parms.i_flags&TUNNEL_SEQ) {
 431                 if (!(tpi->flags&TUNNEL_SEQ) ||
 432                     (tunnel->i_seqno && (s32)(ntohl(tpi->seq) - tunnel->i_seqno) < 0)) {
 433                         tunnel->dev->stats.rx_fifo_errors++;
 434                         tunnel->dev->stats.rx_errors++;
 435                         goto drop;
 436                 }
 437                 tunnel->i_seqno = ntohl(tpi->seq) + 1;
 438         }
 439
 440         skb_reset_network_header(skb);
 441
 442         err = IP_ECN_decapsulate(iph, skb);
 443         if (unlikely(err)) {
 444                 if (log_ecn_error)
 445                         net_info_ratelimited("non-ECT from %pI4 with TOS=%#x\n",
 446                                         &iph->saddr, iph->tos);
 447                 if (err > 1) {
 448                         ++tunnel->dev->stats.rx_frame_errors;
 449                         ++tunnel->dev->stats.rx_errors;
 450                         goto drop;
 451                 }
 452         }
 453
 454         tstats = this_cpu_ptr(tunnel->dev->tstats);
 455         u64_stats_update_begin(&tstats->syncp);
 456         tstats->rx_packets++;
 457         tstats->rx_bytes += skb->len;
 458         u64_stats_update_end(&tstats->syncp);
 459
 460         skb_scrub_packet(skb, !net_eq(tunnel->net, dev_net(tunnel->dev)));
 461
 462         if (tunnel->dev->type == ARPHRD_ETHER) {
 463                 skb->protocol = eth_type_trans(skb, tunnel->dev);
 464                 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
 465         } else {
 466                 skb->dev = tunnel->dev;
 467         }
 468
 469         gro_cells_receive(&tunnel->gro_cells, skb);
 470         return 0;
 471
 472 drop:
 473         kfree_skb(skb);
 474         return 0;
 475 }
 476 EXPORT_SYMBOL_GPL(ip_tunnel_rcv);
 477
 478 static int tnl_update_pmtu(struct net_device *dev, struct sk_buff *skb,
 479                             struct rtable *rt, __be16 df,
 480                             const struct iphdr *inner_iph)
 481 {
 482         struct ip_tunnel *tunnel = netdev_priv(dev);
 483         int pkt_size = skb->len - tunnel->hlen - dev->hard_header_len;
 484         int mtu;
 485
 486         if (df)
 487                 mtu = dst_mtu(&rt->dst) - dev->hard_header_len
 488                                         - sizeof(struct iphdr) - tunnel->hlen;
 489         else
 490                 mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
 491
 492         if (skb_dst(skb))
 493                 skb_dst(skb)->ops->update_pmtu(skb_dst(skb), NULL, skb, mtu);
 494
 495         if (skb->protocol == htons(ETH_P_IP)) {
 496                 if (!skb_is_gso(skb) &&
 497                     (inner_iph->frag_off & htons(IP_DF)) &&
 498                     mtu < pkt_size) {
 499                         memset(IPCB(skb), 0, sizeof(*IPCB(skb)));
 500                         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_FRAG_NEEDED, htonl(mtu));
 501                         return -E2BIG;
 502                 }
 503         }
 504 #if IS_ENABLED(CONFIG_IPV6)
 505         else if (skb->protocol == htons(ETH_P_IPV6)) {
 506                 struct rt6_info *rt6 = (struct rt6_info *)skb_dst(skb);
 507
 508                 if (rt6 && mtu < dst_mtu(skb_dst(skb)) &&
 509                            mtu >= IPV6_MIN_MTU) {
 510                         if ((tunnel->parms.iph.daddr &&
 511                             !ipv4_is_multicast(tunnel->parms.iph.daddr)) ||
 512                             rt6->rt6i_dst.plen == 128) {
 513                                 rt6->rt6i_flags |= RTF_MODIFIED;
 514                                 dst_metric_set(skb_dst(skb), RTAX_MTU, mtu);
 515                         }
 516                 }
 517
 518                 if (!skb_is_gso(skb) && mtu >= IPV6_MIN_MTU &&
 519                                         mtu < pkt_size) {
 520                         icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu);
 521                         return -E2BIG;
 522                 }
 523         }
 524 #endif
 525         return 0;
 526 }
 527
 528 void ip_tunnel_xmit(struct sk_buff *skb, struct net_device *dev,
 529                     const struct iphdr *tnl_params, const u8 protocol)
 530 {
 531         struct ip_tunnel *tunnel = netdev_priv(dev);
 532         const struct iphdr *inner_iph;
 533         struct flowi4 fl4;
 534         u8     tos, ttl;
 535         __be16 df;
 536         struct rtable *rt;              /* Route to the other host */
 537         unsigned int max_headroom;      /* The extra header space needed */
 538         __be32 dst;
 539         int err;
 540
 541         inner_iph = (const struct iphdr *)skb_inner_network_header(skb);
 542
 543         memset(&(IPCB(skb)->opt), 0, sizeof(IPCB(skb)->opt));
 544
 545         dst = tnl_params->daddr;
 546         if (dst == 0) {
 547                 /* NBMA tunnel */
 548
 549                 if (skb_dst(skb) == NULL) {
 550                         dev->stats.tx_fifo_errors++;
 551                         goto tx_error;
 552                 }
 553
 554                 if (skb->protocol == htons(ETH_P_IP)) {
 555                         rt = skb_rtable(skb);
 556                         dst = rt_nexthop(rt, inner_iph->daddr);
 557                 }
 558 #if IS_ENABLED(CONFIG_IPV6)
 559                 else if (skb->protocol == htons(ETH_P_IPV6)) {
 560                         const struct in6_addr *addr6;
 561                         struct neighbour *neigh;
 562                         bool do_tx_error_icmp;
 563                         int addr_type;
 564
 565                         neigh = dst_neigh_lookup(skb_dst(skb),
 566                                                  &ipv6_hdr(skb)->daddr);
 567                         if (neigh == NULL)
 568                                 goto tx_error;
 569
 570                         addr6 = (const struct in6_addr *)&neigh->primary_key;
 571                         addr_type = ipv6_addr_type(addr6);
 572
 573                         if (addr_type == IPV6_ADDR_ANY) {
 574                                 addr6 = &ipv6_hdr(skb)->daddr;
 575                                 addr_type = ipv6_addr_type(addr6);
 576                         }
 577
 578                         if ((addr_type & IPV6_ADDR_COMPATv4) == 0)
 579                                 do_tx_error_icmp = true;
 580                         else {
 581                                 do_tx_error_icmp = false;
 582                                 dst = addr6->s6_addr32[3];
 583                         }
 584                         neigh_release(neigh);
 585                         if (do_tx_error_icmp)
 586                                 goto tx_error_icmp;
 587                 }
 588 #endif
 589                 else
 590                         goto tx_error;
 591         }
 592
 593         tos = tnl_params->tos;
 594         if (tos & 0x1) {
 595                 tos &= ~0x1;
 596                 if (skb->protocol == htons(ETH_P_IP))
 597                         tos = inner_iph->tos;
 598                 else if (skb->protocol == htons(ETH_P_IPV6))
 599                         tos = ipv6_get_dsfield((const struct ipv6hdr *)inner_iph);
 600         }
 601
 602         rt = ip_route_output_tunnel(tunnel->net, &fl4,
 603                                     protocol,
 604                                     dst, tnl_params->saddr,
 605                                     tunnel->parms.o_key,
 606                                     RT_TOS(tos),
 607                                     tunnel->parms.link);
 608         if (IS_ERR(rt)) {
 609                 dev->stats.tx_carrier_errors++;
 610                 goto tx_error;
 611         }
 612         if (rt->dst.dev == dev) {
 613                 ip_rt_put(rt);
 614                 dev->stats.collisions++;
 615                 goto tx_error;
 616         }
 617
 618         if (tnl_update_pmtu(dev, skb, rt, tnl_params->frag_off, inner_iph)) {
 619                 ip_rt_put(rt);
 620                 goto tx_error;
 621         }
 622
 623         if (tunnel->err_count > 0) {
 624                 if (time_before(jiffies,
 625                                 tunnel->err_time + IPTUNNEL_ERR_TIMEO)) {
 626                         tunnel->err_count--;
 627
 628                         dst_link_failure(skb);
 629                 } else
 630                         tunnel->err_count = 0;
 631         }
 632
 633         tos = ip_tunnel_ecn_encap(tos, inner_iph, skb);
 634         ttl = tnl_params->ttl;
 635         if (ttl == 0) {
 636                 if (skb->protocol == htons(ETH_P_IP))
 637                         ttl = inner_iph->ttl;
 638 #if IS_ENABLED(CONFIG_IPV6)
 639                 else if (skb->protocol == htons(ETH_P_IPV6))
 640                         ttl = ((const struct ipv6hdr *)inner_iph)->hop_limit;
 641 #endif
 642                 else
 643                         ttl = ip4_dst_hoplimit(&rt->dst);
 644         }
 645
 646         df = tnl_params->frag_off;
 647         if (skb->protocol == htons(ETH_P_IP))
 648                 df |= (inner_iph->frag_off&htons(IP_DF));
 649
 650         max_headroom = LL_RESERVED_SPACE(rt->dst.dev) + sizeof(struct iphdr)
 651                         + rt->dst.header_len;
 652         if (max_headroom > dev->needed_headroom)
 653                 dev->needed_headroom = max_headroom;
 654
 655         if (skb_cow_head(skb, dev->needed_headroom)) {
 656                 dev->stats.tx_dropped++;
 657                 dev_kfree_skb(skb);
 658                 return;
 659         }
 660
 661         err = iptunnel_xmit(rt, skb, fl4.saddr, fl4.daddr, protocol,
 662                             tos, ttl, df, !net_eq(tunnel->net, dev_net(dev)));
 663         iptunnel_xmit_stats(err, &dev->stats, dev->tstats);
 664
 665         return;
 666
 667 #if IS_ENABLED(CONFIG_IPV6)
 668 tx_error_icmp:
 669         dst_link_failure(skb);
 670 #endif
 671 tx_error:
 672         dev->stats.tx_errors++;
 673         dev_kfree_skb(skb);
 674 }
 675 EXPORT_SYMBOL_GPL(ip_tunnel_xmit);
 676
 677 static void ip_tunnel_update(struct ip_tunnel_net *itn,
 678                              struct ip_tunnel *t,
 679                              struct net_device *dev,
 680                              struct ip_tunnel_parm *p,
 681                              bool set_mtu)
 682 {
 683         ip_tunnel_del(t);
 684         t->parms.iph.saddr = p->iph.saddr;
 685         t->parms.iph.daddr = p->iph.daddr;
 686         t->parms.i_key = p->i_key;
 687         t->parms.o_key = p->o_key;
 688         if (dev->type != ARPHRD_ETHER) {
 689                 memcpy(dev->dev_addr, &p->iph.saddr, 4);
 690                 memcpy(dev->broadcast, &p->iph.daddr, 4);
 691         }
 692         ip_tunnel_add(itn, t);
 693
 694         t->parms.iph.ttl = p->iph.ttl;
 695         t->parms.iph.tos = p->iph.tos;
 696         t->parms.iph.frag_off = p->iph.frag_off;
 697
 698         if (t->parms.link != p->link) {
 699                 int mtu;
 700
 701                 t->parms.link = p->link;
 702                 mtu = ip_tunnel_bind_dev(dev);
 703                 if (set_mtu)
 704                         dev->mtu = mtu;
 705         }
 706         netdev_state_change(dev);
 707 }
 708
 709 int ip_tunnel_ioctl(struct net_device *dev, struct ip_tunnel_parm *p, int cmd)
 710 {
 711         int err = 0;
 712         struct ip_tunnel *t;
 713         struct net *net = dev_net(dev);
 714         struct ip_tunnel *tunnel = netdev_priv(dev);
 715         struct ip_tunnel_net *itn = net_generic(net, tunnel->ip_tnl_net_id);
 716
 717         BUG_ON(!itn->fb_tunnel_dev);
 718         switch (cmd) {
 719         case SIOCGETTUNNEL:
 720                 t = NULL;
 721                 if (dev == itn->fb_tunnel_dev)
 722                         t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
 723                 if (t == NULL)
 724                         t = netdev_priv(dev);
 725                 memcpy(p, &t->parms, sizeof(*p));
 726                 break;
 727
 728         case SIOCADDTUNNEL:
 729         case SIOCCHGTUNNEL:
 730                 err = -EPERM;
 731                 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
 732                         goto done;
 733                 if (p->iph.ttl)
 734                         p->iph.frag_off |= htons(IP_DF);
 735                 if (!(p->i_flags&TUNNEL_KEY))
 736                         p->i_key = 0;
 737                 if (!(p->o_flags&TUNNEL_KEY))
 738                         p->o_key = 0;
 739
 740                 t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
 741
 742                 if (!t && (cmd == SIOCADDTUNNEL))
 743                         t = ip_tunnel_create(net, itn, p);
 744
 745                 if (dev != itn->fb_tunnel_dev && cmd == SIOCCHGTUNNEL) {
 746                         if (t != NULL) {
 747                                 if (t->dev != dev) {
 748                                         err = -EEXIST;
 749                                         break;
 750                                 }
 751                         } else {
 752                                 unsigned int nflags = 0;
 753
 754                                 if (ipv4_is_multicast(p->iph.daddr))
 755                                         nflags = IFF_BROADCAST;
 756                                 else if (p->iph.daddr)
 757                                         nflags = IFF_POINTOPOINT;
 758
 759                                 if ((dev->flags^nflags)&(IFF_POINTOPOINT|IFF_BROADCAST)) {
 760                                         err = -EINVAL;
 761                                         break;
 762                                 }
 763
 764                                 t = netdev_priv(dev);
 765                         }
 766                 }
 767
 768                 if (t) {
 769                         err = 0;
 770                         ip_tunnel_update(itn, t, dev, p, true);
 771                 } else
 772                         err = (cmd == SIOCADDTUNNEL ? -ENOBUFS : -ENOENT);
 773                 break;
 774
 775         case SIOCDELTUNNEL:
 776                 err = -EPERM;
 777                 if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
 778                         goto done;
 779
 780                 if (dev == itn->fb_tunnel_dev) {
 781                         err = -ENOENT;
 782                         t = ip_tunnel_find(itn, p, itn->fb_tunnel_dev->type);
 783                         if (t == NULL)
 784                                 goto done;
 785                         err = -EPERM;
 786                         if (t == netdev_priv(itn->fb_tunnel_dev))
 787                                 goto done;
 788                         dev = t->dev;
 789                 }
 790                 unregister_netdevice(dev);
 791                 err = 0;
 792                 break;
 793
 794         default:
 795                 err = -EINVAL;
 796         }
 797
 798 done:
 799         return err;
 800 }
 801 EXPORT_SYMBOL_GPL(ip_tunnel_ioctl);
 802
 803 int ip_tunnel_change_mtu(struct net_device *dev, int new_mtu)
 804 {
 805         struct ip_tunnel *tunnel = netdev_priv(dev);
 806         int t_hlen = tunnel->hlen + sizeof(struct iphdr);
 807
 808         if (new_mtu < 68 ||
 809             new_mtu > 0xFFF8 - dev->hard_header_len - t_hlen)
 810                 return -EINVAL;
 811         dev->mtu = new_mtu;
 812         return 0;
 813 }
 814 EXPORT_SYMBOL_GPL(ip_tunnel_change_mtu);
 815
 816 static void ip_tunnel_dev_free(struct net_device *dev)
 817 {
 818         struct ip_tunnel *tunnel = netdev_priv(dev);
 819
 820         gro_cells_destroy(&tunnel->gro_cells);
 821         free_percpu(dev->tstats);
 822         free_netdev(dev);
 823 }
 824
 825 void ip_tunnel_dellink(struct net_device *dev, struct list_head *head)
 826 {
 827         struct ip_tunnel *tunnel = netdev_priv(dev);
 828         struct ip_tunnel_net *itn;
 829
 830         itn = net_generic(tunnel->net, tunnel->ip_tnl_net_id);
 831
 832         if (itn->fb_tunnel_dev != dev) {
 833                 ip_tunnel_del(netdev_priv(dev));
 834                 unregister_netdevice_queue(dev, head);
 835         }
 836 }
 837 EXPORT_SYMBOL_GPL(ip_tunnel_dellink);
 838
 839 int ip_tunnel_init_net(struct net *net, int ip_tnl_net_id,
 840                                   struct rtnl_link_ops *ops, char *devname)
 841 {
 842         struct ip_tunnel_net *itn = net_generic(net, ip_tnl_net_id);
 843         struct ip_tunnel_parm parms;
 844         unsigned int i;
 845
 846         for (i = 0; i < IP_TNL_HASH_SIZE; i++)
 847                 INIT_HLIST_HEAD(&itn->tunnels[i]);
 848
 849         if (!ops) {
 850                 itn->fb_tunnel_dev = NULL;
 851                 return 0;
 852         }
 853
 854         memset(&parms, 0, sizeof(parms));
 855         if (devname)
 856                 strlcpy(parms.name, devname, IFNAMSIZ);
 857
 858         rtnl_lock();
 859         itn->fb_tunnel_dev = __ip_tunnel_create(net, ops, &parms);
 860         /* FB netdevice is special: we have one, and only one per netns.
 861          * Allowing to move it to another netns is clearly unsafe.
 862          */
 863         if (!IS_ERR(itn->fb_tunnel_dev)) {
 864                 itn->fb_tunnel_dev->features |= NETIF_F_NETNS_LOCAL;
 865                 itn->fb_tunnel_dev->mtu = ip_tunnel_bind_dev(itn->fb_tunnel_dev);
 866                 ip_tunnel_add(itn, netdev_priv(itn->fb_tunnel_dev));
 867         }
 868         rtnl_unlock();
 869
 870         return PTR_RET(itn->fb_tunnel_dev);
 871 }
 872 EXPORT_SYMBOL_GPL(ip_tunnel_init_net);
 873
 874 static void ip_tunnel_destroy(struct ip_tunnel_net *itn, struct list_head *head,
 875                               struct rtnl_link_ops *ops)
 876 {
 877         struct net *net = dev_net(itn->fb_tunnel_dev);
 878         struct net_device *dev, *aux;
 879         int h;
 880
 881         for_each_netdev_safe(net, dev, aux)
 882                 if (dev->rtnl_link_ops == ops)
 883                         unregister_netdevice_queue(dev, head);
 884
 885         for (h = 0; h < IP_TNL_HASH_SIZE; h++) {
 886                 struct ip_tunnel *t;
 887                 struct hlist_node *n;
 888                 struct hlist_head *thead = &itn->tunnels[h];
 889
 890                 hlist_for_each_entry_safe(t, n, thead, hash_node)
 891                         /* If dev is in the same netns, it has already
 892                          * been added to the list by the previous loop.
 893                          */
 894                         if (!net_eq(dev_net(t->dev), net))
 895                                 unregister_netdevice_queue(t->dev, head);
 896         }
 897 }
 898
 899 void ip_tunnel_delete_net(struct ip_tunnel_net *itn, struct rtnl_link_ops *ops)
 900 {
 901         LIST_HEAD(list);
 902
 903         rtnl_lock();
 904         ip_tunnel_destroy(itn, &list, ops);
 905         unregister_netdevice_many(&list);
 906         rtnl_unlock();
 907 }
 908 EXPORT_SYMBOL_GPL(ip_tunnel_delete_net);
 909
 910 int ip_tunnel_newlink(struct net_device *dev, struct nlattr *tb[],
 911                       struct ip_tunnel_parm *p)
 912 {
 913         struct ip_tunnel *nt;
 914         struct net *net = dev_net(dev);
 915         struct ip_tunnel_net *itn;
 916         int mtu;
 917         int err;
 918
 919         nt = netdev_priv(dev);
 920         itn = net_generic(net, nt->ip_tnl_net_id);
 921
 922         if (ip_tunnel_find(itn, p, dev->type))
 923                 return -EEXIST;
 924
 925         nt->net = net;
 926         nt->parms = *p;
 927         err = register_netdevice(dev);
 928         if (err)
 929                 goto out;
 930
 931         if (dev->type == ARPHRD_ETHER && !tb[IFLA_ADDRESS])
 932                 eth_hw_addr_random(dev);
 933
 934         mtu = ip_tunnel_bind_dev(dev);
 935         if (!tb[IFLA_MTU])
 936                 dev->mtu = mtu;
 937
 938         ip_tunnel_add(itn, nt);
 939
 940 out:
 941         return err;
 942 }
 943 EXPORT_SYMBOL_GPL(ip_tunnel_newlink);
 944
 945 int ip_tunnel_changelink(struct net_device *dev, struct nlattr *tb[],
 946                          struct ip_tunnel_parm *p)
 947 {
 948         struct ip_tunnel *t;
 949         struct ip_tunnel *tunnel = netdev_priv(dev);
 950         struct net *net = tunnel->net;
 951         struct ip_tunnel_net *itn = net_generic(net, tunnel->ip_tnl_net_id);
 952
 953         if (dev == itn->fb_tunnel_dev)
 954                 return -EINVAL;
 955
 956         t = ip_tunnel_find(itn, p, dev->type);
 957
 958         if (t) {
 959                 if (t->dev != dev)
 960                         return -EEXIST;
 961         } else {
 962                 t = tunnel;
 963
 964                 if (dev->type != ARPHRD_ETHER) {
 965                         unsigned int nflags = 0;
 966
 967                         if (ipv4_is_multicast(p->iph.daddr))
 968                                 nflags = IFF_BROADCAST;
 969                         else if (p->iph.daddr)
 970                                 nflags = IFF_POINTOPOINT;
 971
 972                         if ((dev->flags ^ nflags) &
 973                             (IFF_POINTOPOINT | IFF_BROADCAST))
 974                                 return -EINVAL;
 975                 }
 976         }
 977
 978         ip_tunnel_update(itn, t, dev, p, !tb[IFLA_MTU]);
 979         return 0;
 980 }
 981 EXPORT_SYMBOL_GPL(ip_tunnel_changelink);
 982
 983 int ip_tunnel_init(struct net_device *dev)
 984 {
 985         struct ip_tunnel *tunnel = netdev_priv(dev);
 986         struct iphdr *iph = &tunnel->parms.iph;
 987         int err;
 988
 989         dev->destructor = ip_tunnel_dev_free;
 990         dev->tstats = alloc_percpu(struct pcpu_tstats);
 991         if (!dev->tstats)
 992                 return -ENOMEM;
 993
 994         err = gro_cells_init(&tunnel->gro_cells, dev);
 995         if (err) {
 996                 free_percpu(dev->tstats);
 997                 return err;
 998         }
 999
1000         tunnel->dev = dev;
1001         tunnel->net = dev_net(dev);
1002         strcpy(tunnel->parms.name, dev->name);
1003         iph->version            = 4;
1004         iph->ihl                = 5;
1005
1006         return 0;
1007 }
1008 EXPORT_SYMBOL_GPL(ip_tunnel_init);
1009
1010 void ip_tunnel_uninit(struct net_device *dev)
1011 {
1012         struct ip_tunnel *tunnel = netdev_priv(dev);
1013         struct net *net = tunnel->net;
1014         struct ip_tunnel_net *itn;
1015
1016         itn = net_generic(net, tunnel->ip_tnl_net_id);
1017         /* fb_tunnel_dev will be unregisted in net-exit call. */
1018         if (itn->fb_tunnel_dev != dev)
1019                 ip_tunnel_del(netdev_priv(dev));
1020 }
1021 EXPORT_SYMBOL_GPL(ip_tunnel_uninit);
1022
1023 /* Do least required initialization, rest of init is done in tunnel_init call */
1024 void ip_tunnel_setup(struct net_device *dev, int net_id)
1025 {
1026         struct ip_tunnel *tunnel = netdev_priv(dev);
1027         tunnel->ip_tnl_net_id = net_id;
1028 }
1029 EXPORT_SYMBOL_GPL(ip_tunnel_setup);
1030
1031 MODULE_LICENSE("GPL");