net/xfrm/xfrm_device.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * xfrm_device.c - IPsec device offloading code.
   4  *
   5  * Copyright (c) 2015 secunet Security Networks AG
   6  *
   7  * Author:
   8  * Steffen Klassert <steffen.klassert@secunet.com>
   9  */
  10
  11 #include <linux/errno.h>
  12 #include <linux/module.h>
  13 #include <linux/netdevice.h>
  14 #include <linux/skbuff.h>
  15 #include <linux/slab.h>
  16 #include <linux/spinlock.h>
  17 #include <net/dst.h>
  18 #include <net/xfrm.h>
  19 #include <linux/notifier.h>
  20
  21 #ifdef CONFIG_XFRM_OFFLOAD
  22 static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb,
  23                                   unsigned int hsize)
  24 {
  25         struct xfrm_offload *xo = xfrm_offload(skb);
  26
  27         skb_reset_mac_len(skb);
  28         if (xo->flags & XFRM_GSO_SEGMENT)
  29                 skb->transport_header -= x->props.header_len;
  30
  31         pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);
  32 }
  33
  34 static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb,
  35                                     unsigned int hsize)
  36
  37 {
  38         struct xfrm_offload *xo = xfrm_offload(skb);
  39
  40         if (xo->flags & XFRM_GSO_SEGMENT)
  41                 skb->transport_header = skb->network_header + hsize;
  42
  43         skb_reset_mac_len(skb);
  44         pskb_pull(skb, skb->mac_len + x->props.header_len);
  45 }
  46
  47 /* Adjust pointers into the packet when IPsec is done at layer2 */
  48 static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb)
  49 {
  50         switch (x->outer_mode.encap) {
  51         case XFRM_MODE_TUNNEL:
  52                 if (x->outer_mode.family == AF_INET)
  53                         return __xfrm_mode_tunnel_prep(x, skb,
  54                                                        sizeof(struct iphdr));
  55                 if (x->outer_mode.family == AF_INET6)
  56                         return __xfrm_mode_tunnel_prep(x, skb,
  57                                                        sizeof(struct ipv6hdr));
  58                 break;
  59         case XFRM_MODE_TRANSPORT:
  60                 if (x->outer_mode.family == AF_INET)
  61                         return __xfrm_transport_prep(x, skb,
  62                                                      sizeof(struct iphdr));
  63                 if (x->outer_mode.family == AF_INET6)
  64                         return __xfrm_transport_prep(x, skb,
  65                                                      sizeof(struct ipv6hdr));
  66                 break;
  67         case XFRM_MODE_ROUTEOPTIMIZATION:
  68         case XFRM_MODE_IN_TRIGGER:
  69         case XFRM_MODE_BEET:
  70                 break;
  71         }
  72 }
  73
  74 struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
  75 {
  76         int err;
  77         unsigned long flags;
  78         struct xfrm_state *x;
  79         struct sk_buff *skb2;
  80         struct softnet_data *sd;
  81         netdev_features_t esp_features = features;
  82         struct xfrm_offload *xo = xfrm_offload(skb);
  83         struct sec_path *sp;
  84
  85         if (!xo || (xo->flags & XFRM_XMIT))
  86                 return skb;
  87
  88         if (!(features & NETIF_F_HW_ESP))
  89                 esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);
  90
  91         sp = skb_sec_path(skb);
  92         x = sp->xvec[sp->len - 1];
  93         if (xo->flags & XFRM_GRO || x->xso.flags & XFRM_OFFLOAD_INBOUND)
  94                 return skb;
  95
  96         local_irq_save(flags);
  97         sd = this_cpu_ptr(&softnet_data);
  98         err = !skb_queue_empty(&sd->xfrm_backlog);
  99         local_irq_restore(flags);
 100
 101         if (err) {
 102                 *again = true;
 103                 return skb;
 104         }
 105
 106         xo->flags |= XFRM_XMIT;
 107
 108         if (skb_is_gso(skb)) {
 109                 struct net_device *dev = skb->dev;
 110
 111                 if (unlikely(x->xso.dev != dev)) {
 112                         struct sk_buff *segs;
 113
 114                         /* Packet got rerouted, fixup features and segment it. */
 115                         esp_features = esp_features & ~(NETIF_F_HW_ESP
 116                                                         | NETIF_F_GSO_ESP);
 117
 118                         segs = skb_gso_segment(skb, esp_features);
 119                         if (IS_ERR(segs)) {
 120                                 kfree_skb(skb);
 121                                 atomic_long_inc(&dev->tx_dropped);
 122                                 return NULL;
 123                         } else {
 124                                 consume_skb(skb);
 125                                 skb = segs;
 126                         }
 127                 }
 128         }
 129
 130         if (!skb->next) {
 131                 esp_features |= skb->dev->gso_partial_features;
 132                 xfrm_outer_mode_prep(x, skb);
 133
 134                 xo->flags |= XFRM_DEV_RESUME;
 135
 136                 err = x->type_offload->xmit(x, skb, esp_features);
 137                 if (err) {
 138                         if (err == -EINPROGRESS)
 139                                 return NULL;
 140
 141                         XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
 142                         kfree_skb(skb);
 143                         return NULL;
 144                 }
 145
 146                 skb_push(skb, skb->data - skb_mac_header(skb));
 147
 148                 return skb;
 149         }
 150
 151         skb2 = skb;
 152
 153         do {
 154                 struct sk_buff *nskb = skb2->next;
 155
 156                 esp_features |= skb->dev->gso_partial_features;
 157                 skb_mark_not_on_list(skb2);
 158
 159                 xo = xfrm_offload(skb2);
 160                 xo->flags |= XFRM_DEV_RESUME;
 161
 162                 xfrm_outer_mode_prep(x, skb2);
 163
 164                 err = x->type_offload->xmit(x, skb2, esp_features);
 165                 if (!err) {
 166                         skb2->next = nskb;
 167                 } else if (err != -EINPROGRESS) {
 168                         XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
 169                         skb2->next = nskb;
 170                         kfree_skb_list(skb2);
 171                         return NULL;
 172                 } else {
 173                         if (skb == skb2)
 174                                 skb = nskb;
 175
 176                         if (!skb)
 177                                 return NULL;
 178
 179                         goto skip_push;
 180                 }
 181
 182                 skb_push(skb2, skb2->data - skb_mac_header(skb2));
 183
 184 skip_push:
 185                 skb2 = nskb;
 186         } while (skb2);
 187
 188         return skb;
 189 }
 190 EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
 191
 192 int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
 193                        struct xfrm_user_offload *xuo)
 194 {
 195         int err;
 196         struct dst_entry *dst;
 197         struct net_device *dev;
 198         struct xfrm_state_offload *xso = &x->xso;
 199         xfrm_address_t *saddr;
 200         xfrm_address_t *daddr;
 201
 202         if (!x->type_offload)
 203                 return -EINVAL;
 204
 205         /* We don't yet support UDP encapsulation and TFC padding. */
 206         if (x->encap || x->tfcpad)
 207                 return -EINVAL;
 208
 209         dev = dev_get_by_index(net, xuo->ifindex);
 210         if (!dev) {
 211                 if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
 212                         saddr = &x->props.saddr;
 213                         daddr = &x->id.daddr;
 214                 } else {
 215                         saddr = &x->id.daddr;
 216                         daddr = &x->props.saddr;
 217                 }
 218
 219                 dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr,
 220                                         x->props.family,
 221                                         xfrm_smark_get(0, x));
 222                 if (IS_ERR(dst))
 223                         return 0;
 224
 225                 dev = dst->dev;
 226
 227                 dev_hold(dev);
 228                 dst_release(dst);
 229         }
 230
 231         if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
 232                 xso->dev = NULL;
 233                 dev_put(dev);
 234                 return 0;
 235         }
 236
 237         if (x->props.flags & XFRM_STATE_ESN &&
 238             !dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
 239                 xso->dev = NULL;
 240                 dev_put(dev);
 241                 return -EINVAL;
 242         }
 243
 244         xso->dev = dev;
 245         xso->num_exthdrs = 1;
 246         xso->flags = xuo->flags;
 247
 248         err = dev->xfrmdev_ops->xdo_dev_state_add(x);
 249         if (err) {
 250                 xso->num_exthdrs = 0;
 251                 xso->flags = 0;
 252                 xso->dev = NULL;
 253                 dev_put(dev);
 254
 255                 if (err != -EOPNOTSUPP)
 256                         return err;
 257         }
 258
 259         return 0;
 260 }
 261 EXPORT_SYMBOL_GPL(xfrm_dev_state_add);
 262
 263 bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
 264 {
 265         int mtu;
 266         struct dst_entry *dst = skb_dst(skb);
 267         struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
 268         struct net_device *dev = x->xso.dev;
 269
 270         if (!x->type_offload || x->encap)
 271                 return false;
 272
 273         if ((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
 274             (!xdst->child->xfrm)) {
 275                 mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
 276                 if (skb->len <= mtu)
 277                         goto ok;
 278
 279                 if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
 280                         goto ok;
 281         }
 282
 283         return false;
 284
 285 ok:
 286         if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)
 287                 return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
 288
 289         return true;
 290 }
 291 EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);
 292
 293 void xfrm_dev_resume(struct sk_buff *skb)
 294 {
 295         struct net_device *dev = skb->dev;
 296         int ret = NETDEV_TX_BUSY;
 297         struct netdev_queue *txq;
 298         struct softnet_data *sd;
 299         unsigned long flags;
 300
 301         rcu_read_lock();
 302         txq = netdev_core_pick_tx(dev, skb, NULL);
 303
 304         HARD_TX_LOCK(dev, txq, smp_processor_id());
 305         if (!netif_xmit_frozen_or_stopped(txq))
 306                 skb = dev_hard_start_xmit(skb, dev, txq, &ret);
 307         HARD_TX_UNLOCK(dev, txq);
 308
 309         if (!dev_xmit_complete(ret)) {
 310                 local_irq_save(flags);
 311                 sd = this_cpu_ptr(&softnet_data);
 312                 skb_queue_tail(&sd->xfrm_backlog, skb);
 313                 raise_softirq_irqoff(NET_TX_SOFTIRQ);
 314                 local_irq_restore(flags);
 315         }
 316         rcu_read_unlock();
 317 }
 318 EXPORT_SYMBOL_GPL(xfrm_dev_resume);
 319
 320 void xfrm_dev_backlog(struct softnet_data *sd)
 321 {
 322         struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;
 323         struct sk_buff_head list;
 324         struct sk_buff *skb;
 325
 326         if (skb_queue_empty(xfrm_backlog))
 327                 return;
 328
 329         __skb_queue_head_init(&list);
 330
 331         spin_lock(&xfrm_backlog->lock);
 332         skb_queue_splice_init(xfrm_backlog, &list);
 333         spin_unlock(&xfrm_backlog->lock);
 334
 335         while (!skb_queue_empty(&list)) {
 336                 skb = __skb_dequeue(&list);
 337                 xfrm_dev_resume(skb);
 338         }
 339
 340 }
 341 #endif
 342
 343 static int xfrm_api_check(struct net_device *dev)
 344 {
 345 #ifdef CONFIG_XFRM_OFFLOAD
 346         if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
 347             !(dev->features & NETIF_F_HW_ESP))
 348                 return NOTIFY_BAD;
 349
 350         if ((dev->features & NETIF_F_HW_ESP) &&
 351             (!(dev->xfrmdev_ops &&
 352                dev->xfrmdev_ops->xdo_dev_state_add &&
 353                dev->xfrmdev_ops->xdo_dev_state_delete)))
 354                 return NOTIFY_BAD;
 355 #else
 356         if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))
 357                 return NOTIFY_BAD;
 358 #endif
 359
 360         return NOTIFY_DONE;
 361 }
 362
 363 static int xfrm_dev_register(struct net_device *dev)
 364 {
 365         return xfrm_api_check(dev);
 366 }
 367
 368 static int xfrm_dev_feat_change(struct net_device *dev)
 369 {
 370         return xfrm_api_check(dev);
 371 }
 372
 373 static int xfrm_dev_down(struct net_device *dev)
 374 {
 375         if (dev->features & NETIF_F_HW_ESP)
 376                 xfrm_dev_state_flush(dev_net(dev), dev, true);
 377
 378         return NOTIFY_DONE;
 379 }
 380
 381 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
 382 {
 383         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
 384
 385         switch (event) {
 386         case NETDEV_REGISTER:
 387                 return xfrm_dev_register(dev);
 388
 389         case NETDEV_FEAT_CHANGE:
 390                 return xfrm_dev_feat_change(dev);
 391
 392         case NETDEV_DOWN:
 393         case NETDEV_UNREGISTER:
 394                 return xfrm_dev_down(dev);
 395         }
 396         return NOTIFY_DONE;
 397 }
 398
 399 static struct notifier_block xfrm_dev_notifier = {
 400         .notifier_call  = xfrm_dev_event,
 401 };
 402
 403 void __init xfrm_dev_init(void)
 404 {
 405         register_netdevice_notifier(&xfrm_dev_notifier);
 406 }