net/ipv6/ila/ila_xlat.c

   1 #include <linux/jhash.h>
   2 #include <linux/netfilter.h>
   3 #include <linux/rcupdate.h>
   4 #include <linux/rhashtable.h>
   5 #include <linux/vmalloc.h>
   6 #include <net/genetlink.h>
   7 #include <net/ila.h>
   8 #include <net/netns/generic.h>
   9 #include <uapi/linux/genetlink.h>
  10 #include "ila.h"
  11
  12 struct ila_xlat_params {
  13         struct ila_params ip;
  14         int ifindex;
  15 };
  16
  17 struct ila_map {
  18         struct ila_xlat_params xp;
  19         struct rhash_head node;
  20         struct ila_map __rcu *next;
  21         struct rcu_head rcu;
  22 };
  23
  24 static unsigned int ila_net_id;
  25
  26 struct ila_net {
  27         struct rhashtable rhash_table;
  28         spinlock_t *locks; /* Bucket locks for entry manipulation */
  29         unsigned int locks_mask;
  30         bool hooks_registered;
  31 };
  32
  33 #define LOCKS_PER_CPU 10
  34
  35 static int alloc_ila_locks(struct ila_net *ilan)
  36 {
  37         unsigned int i, size;
  38         unsigned int nr_pcpus = num_possible_cpus();
  39
  40         nr_pcpus = min_t(unsigned int, nr_pcpus, 32UL);
  41         size = roundup_pow_of_two(nr_pcpus * LOCKS_PER_CPU);
  42
  43         if (sizeof(spinlock_t) != 0) {
  44                 ilan->locks = kvmalloc(size * sizeof(spinlock_t), GFP_KERNEL);
  45                 if (!ilan->locks)
  46                         return -ENOMEM;
  47                 for (i = 0; i < size; i++)
  48                         spin_lock_init(&ilan->locks[i]);
  49         }
  50         ilan->locks_mask = size - 1;
  51
  52         return 0;
  53 }
  54
  55 static u32 hashrnd __read_mostly;
  56 static __always_inline void __ila_hash_secret_init(void)
  57 {
  58         net_get_random_once(&hashrnd, sizeof(hashrnd));
  59 }
  60
  61 static inline u32 ila_locator_hash(struct ila_locator loc)
  62 {
  63         u32 *v = (u32 *)loc.v32;
  64
  65         return jhash_2words(v[0], v[1], hashrnd);
  66 }
  67
  68 static inline spinlock_t *ila_get_lock(struct ila_net *ilan,
  69                                        struct ila_locator loc)
  70 {
  71         return &ilan->locks[ila_locator_hash(loc) & ilan->locks_mask];
  72 }
  73
  74 static inline int ila_cmp_wildcards(struct ila_map *ila,
  75                                     struct ila_addr *iaddr, int ifindex)
  76 {
  77         return (ila->xp.ifindex && ila->xp.ifindex != ifindex);
  78 }
  79
  80 static inline int ila_cmp_params(struct ila_map *ila,
  81                                  struct ila_xlat_params *xp)
  82 {
  83         return (ila->xp.ifindex != xp->ifindex);
  84 }
  85
  86 static int ila_cmpfn(struct rhashtable_compare_arg *arg,
  87                      const void *obj)
  88 {
  89         const struct ila_map *ila = obj;
  90
  91         return (ila->xp.ip.locator_match.v64 != *(__be64 *)arg->key);
  92 }
  93
  94 static inline int ila_order(struct ila_map *ila)
  95 {
  96         int score = 0;
  97
  98         if (ila->xp.ifindex)
  99                 score += 1 << 1;
 100
 101         return score;
 102 }
 103
 104 static const struct rhashtable_params rht_params = {
 105         .nelem_hint = 1024,
 106         .head_offset = offsetof(struct ila_map, node),
 107         .key_offset = offsetof(struct ila_map, xp.ip.locator_match),
 108         .key_len = sizeof(u64), /* identifier */
 109         .max_size = 1048576,
 110         .min_size = 256,
 111         .automatic_shrinking = true,
 112         .obj_cmpfn = ila_cmpfn,
 113 };
 114
 115 static struct genl_family ila_nl_family;
 116
 117 static const struct nla_policy ila_nl_policy[ILA_ATTR_MAX + 1] = {
 118         [ILA_ATTR_LOCATOR] = { .type = NLA_U64, },
 119         [ILA_ATTR_LOCATOR_MATCH] = { .type = NLA_U64, },
 120         [ILA_ATTR_IFINDEX] = { .type = NLA_U32, },
 121         [ILA_ATTR_CSUM_MODE] = { .type = NLA_U8, },
 122 };
 123
 124 static int parse_nl_config(struct genl_info *info,
 125                            struct ila_xlat_params *xp)
 126 {
 127         memset(xp, 0, sizeof(*xp));
 128
 129         if (info->attrs[ILA_ATTR_LOCATOR])
 130                 xp->ip.locator.v64 = (__force __be64)nla_get_u64(
 131                         info->attrs[ILA_ATTR_LOCATOR]);
 132
 133         if (info->attrs[ILA_ATTR_LOCATOR_MATCH])
 134                 xp->ip.locator_match.v64 = (__force __be64)nla_get_u64(
 135                         info->attrs[ILA_ATTR_LOCATOR_MATCH]);
 136
 137         if (info->attrs[ILA_ATTR_CSUM_MODE])
 138                 xp->ip.csum_mode = nla_get_u8(info->attrs[ILA_ATTR_CSUM_MODE]);
 139
 140         if (info->attrs[ILA_ATTR_IFINDEX])
 141                 xp->ifindex = nla_get_s32(info->attrs[ILA_ATTR_IFINDEX]);
 142
 143         return 0;
 144 }
 145
 146 /* Must be called with rcu readlock */
 147 static inline struct ila_map *ila_lookup_wildcards(struct ila_addr *iaddr,
 148                                                    int ifindex,
 149                                                    struct ila_net *ilan)
 150 {
 151         struct ila_map *ila;
 152
 153         ila = rhashtable_lookup_fast(&ilan->rhash_table, &iaddr->loc,
 154                                      rht_params);
 155         while (ila) {
 156                 if (!ila_cmp_wildcards(ila, iaddr, ifindex))
 157                         return ila;
 158                 ila = rcu_access_pointer(ila->next);
 159         }
 160
 161         return NULL;
 162 }
 163
 164 /* Must be called with rcu readlock */
 165 static inline struct ila_map *ila_lookup_by_params(struct ila_xlat_params *xp,
 166                                                    struct ila_net *ilan)
 167 {
 168         struct ila_map *ila;
 169
 170         ila = rhashtable_lookup_fast(&ilan->rhash_table,
 171                                      &xp->ip.locator_match,
 172                                      rht_params);
 173         while (ila) {
 174                 if (!ila_cmp_params(ila, xp))
 175                         return ila;
 176                 ila = rcu_access_pointer(ila->next);
 177         }
 178
 179         return NULL;
 180 }
 181
 182 static inline void ila_release(struct ila_map *ila)
 183 {
 184         kfree_rcu(ila, rcu);
 185 }
 186
 187 static void ila_free_cb(void *ptr, void *arg)
 188 {
 189         struct ila_map *ila = (struct ila_map *)ptr, *next;
 190
 191         /* Assume rcu_readlock held */
 192         while (ila) {
 193                 next = rcu_access_pointer(ila->next);
 194                 ila_release(ila);
 195                 ila = next;
 196         }
 197 }
 198
 199 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral);
 200
 201 static unsigned int
 202 ila_nf_input(void *priv,
 203              struct sk_buff *skb,
 204              const struct nf_hook_state *state)
 205 {
 206         ila_xlat_addr(skb, false);
 207         return NF_ACCEPT;
 208 }
 209
 210 static struct nf_hook_ops ila_nf_hook_ops[] __read_mostly = {
 211         {
 212                 .hook = ila_nf_input,
 213                 .pf = NFPROTO_IPV6,
 214                 .hooknum = NF_INET_PRE_ROUTING,
 215                 .priority = -1,
 216         },
 217 };
 218
 219 static int ila_add_mapping(struct net *net, struct ila_xlat_params *xp)
 220 {
 221         struct ila_net *ilan = net_generic(net, ila_net_id);
 222         struct ila_map *ila, *head;
 223         spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
 224         int err = 0, order;
 225
 226         if (!ilan->hooks_registered) {
 227                 /* We defer registering net hooks in the namespace until the
 228                  * first mapping is added.
 229                  */
 230                 err = nf_register_net_hooks(net, ila_nf_hook_ops,
 231                                             ARRAY_SIZE(ila_nf_hook_ops));
 232                 if (err)
 233                         return err;
 234
 235                 ilan->hooks_registered = true;
 236         }
 237
 238         ila = kzalloc(sizeof(*ila), GFP_KERNEL);
 239         if (!ila)
 240                 return -ENOMEM;
 241
 242         ila_init_saved_csum(&xp->ip);
 243
 244         ila->xp = *xp;
 245
 246         order = ila_order(ila);
 247
 248         spin_lock(lock);
 249
 250         head = rhashtable_lookup_fast(&ilan->rhash_table,
 251                                       &xp->ip.locator_match,
 252                                       rht_params);
 253         if (!head) {
 254                 /* New entry for the rhash_table */
 255                 err = rhashtable_lookup_insert_fast(&ilan->rhash_table,
 256                                                     &ila->node, rht_params);
 257         } else {
 258                 struct ila_map *tila = head, *prev = NULL;
 259
 260                 do {
 261                         if (!ila_cmp_params(tila, xp)) {
 262                                 err = -EEXIST;
 263                                 goto out;
 264                         }
 265
 266                         if (order > ila_order(tila))
 267                                 break;
 268
 269                         prev = tila;
 270                         tila = rcu_dereference_protected(tila->next,
 271                                 lockdep_is_held(lock));
 272                 } while (tila);
 273
 274                 if (prev) {
 275                         /* Insert in sub list of head */
 276                         RCU_INIT_POINTER(ila->next, tila);
 277                         rcu_assign_pointer(prev->next, ila);
 278                 } else {
 279                         /* Make this ila new head */
 280                         RCU_INIT_POINTER(ila->next, head);
 281                         err = rhashtable_replace_fast(&ilan->rhash_table,
 282                                                       &head->node,
 283                                                       &ila->node, rht_params);
 284                         if (err)
 285                                 goto out;
 286                 }
 287         }
 288
 289 out:
 290         spin_unlock(lock);
 291
 292         if (err)
 293                 kfree(ila);
 294
 295         return err;
 296 }
 297
 298 static int ila_del_mapping(struct net *net, struct ila_xlat_params *xp)
 299 {
 300         struct ila_net *ilan = net_generic(net, ila_net_id);
 301         struct ila_map *ila, *head, *prev;
 302         spinlock_t *lock = ila_get_lock(ilan, xp->ip.locator_match);
 303         int err = -ENOENT;
 304
 305         spin_lock(lock);
 306
 307         head = rhashtable_lookup_fast(&ilan->rhash_table,
 308                                       &xp->ip.locator_match, rht_params);
 309         ila = head;
 310
 311         prev = NULL;
 312
 313         while (ila) {
 314                 if (ila_cmp_params(ila, xp)) {
 315                         prev = ila;
 316                         ila = rcu_dereference_protected(ila->next,
 317                                                         lockdep_is_held(lock));
 318                         continue;
 319                 }
 320
 321                 err = 0;
 322
 323                 if (prev) {
 324                         /* Not head, just delete from list */
 325                         rcu_assign_pointer(prev->next, ila->next);
 326                 } else {
 327                         /* It is the head. If there is something in the
 328                          * sublist we need to make a new head.
 329                          */
 330                         head = rcu_dereference_protected(ila->next,
 331                                                          lockdep_is_held(lock));
 332                         if (head) {
 333                                 /* Put first entry in the sublist into the
 334                                  * table
 335                                  */
 336                                 err = rhashtable_replace_fast(
 337                                         &ilan->rhash_table, &ila->node,
 338                                         &head->node, rht_params);
 339                                 if (err)
 340                                         goto out;
 341                         } else {
 342                                 /* Entry no longer used */
 343                                 err = rhashtable_remove_fast(&ilan->rhash_table,
 344                                                              &ila->node,
 345                                                              rht_params);
 346                         }
 347                 }
 348
 349                 ila_release(ila);
 350
 351                 break;
 352         }
 353
 354 out:
 355         spin_unlock(lock);
 356
 357         return err;
 358 }
 359
 360 static int ila_nl_cmd_add_mapping(struct sk_buff *skb, struct genl_info *info)
 361 {
 362         struct net *net = genl_info_net(info);
 363         struct ila_xlat_params p;
 364         int err;
 365
 366         err = parse_nl_config(info, &p);
 367         if (err)
 368                 return err;
 369
 370         return ila_add_mapping(net, &p);
 371 }
 372
 373 static int ila_nl_cmd_del_mapping(struct sk_buff *skb, struct genl_info *info)
 374 {
 375         struct net *net = genl_info_net(info);
 376         struct ila_xlat_params xp;
 377         int err;
 378
 379         err = parse_nl_config(info, &xp);
 380         if (err)
 381                 return err;
 382
 383         ila_del_mapping(net, &xp);
 384
 385         return 0;
 386 }
 387
 388 static int ila_fill_info(struct ila_map *ila, struct sk_buff *msg)
 389 {
 390         if (nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR,
 391                               (__force u64)ila->xp.ip.locator.v64,
 392                               ILA_ATTR_PAD) ||
 393             nla_put_u64_64bit(msg, ILA_ATTR_LOCATOR_MATCH,
 394                               (__force u64)ila->xp.ip.locator_match.v64,
 395                               ILA_ATTR_PAD) ||
 396             nla_put_s32(msg, ILA_ATTR_IFINDEX, ila->xp.ifindex) ||
 397             nla_put_u32(msg, ILA_ATTR_CSUM_MODE, ila->xp.ip.csum_mode))
 398                 return -1;
 399
 400         return 0;
 401 }
 402
 403 static int ila_dump_info(struct ila_map *ila,
 404                          u32 portid, u32 seq, u32 flags,
 405                          struct sk_buff *skb, u8 cmd)
 406 {
 407         void *hdr;
 408
 409         hdr = genlmsg_put(skb, portid, seq, &ila_nl_family, flags, cmd);
 410         if (!hdr)
 411                 return -ENOMEM;
 412
 413         if (ila_fill_info(ila, skb) < 0)
 414                 goto nla_put_failure;
 415
 416         genlmsg_end(skb, hdr);
 417         return 0;
 418
 419 nla_put_failure:
 420         genlmsg_cancel(skb, hdr);
 421         return -EMSGSIZE;
 422 }
 423
 424 static int ila_nl_cmd_get_mapping(struct sk_buff *skb, struct genl_info *info)
 425 {
 426         struct net *net = genl_info_net(info);
 427         struct ila_net *ilan = net_generic(net, ila_net_id);
 428         struct sk_buff *msg;
 429         struct ila_xlat_params xp;
 430         struct ila_map *ila;
 431         int ret;
 432
 433         ret = parse_nl_config(info, &xp);
 434         if (ret)
 435                 return ret;
 436
 437         msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
 438         if (!msg)
 439                 return -ENOMEM;
 440
 441         rcu_read_lock();
 442
 443         ila = ila_lookup_by_params(&xp, ilan);
 444         if (ila) {
 445                 ret = ila_dump_info(ila,
 446                                     info->snd_portid,
 447                                     info->snd_seq, 0, msg,
 448                                     info->genlhdr->cmd);
 449         }
 450
 451         rcu_read_unlock();
 452
 453         if (ret < 0)
 454                 goto out_free;
 455
 456         return genlmsg_reply(msg, info);
 457
 458 out_free:
 459         nlmsg_free(msg);
 460         return ret;
 461 }
 462
 463 struct ila_dump_iter {
 464         struct rhashtable_iter rhiter;
 465 };
 466
 467 static int ila_nl_dump_start(struct netlink_callback *cb)
 468 {
 469         struct net *net = sock_net(cb->skb->sk);
 470         struct ila_net *ilan = net_generic(net, ila_net_id);
 471         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
 472
 473         if (!iter) {
 474                 iter = kmalloc(sizeof(*iter), GFP_KERNEL);
 475                 if (!iter)
 476                         return -ENOMEM;
 477
 478                 cb->args[0] = (long)iter;
 479         }
 480
 481         return rhashtable_walk_init(&ilan->rhash_table, &iter->rhiter,
 482                                     GFP_KERNEL);
 483 }
 484
 485 static int ila_nl_dump_done(struct netlink_callback *cb)
 486 {
 487         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
 488
 489         rhashtable_walk_exit(&iter->rhiter);
 490
 491         kfree(iter);
 492
 493         return 0;
 494 }
 495
 496 static int ila_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
 497 {
 498         struct ila_dump_iter *iter = (struct ila_dump_iter *)cb->args[0];
 499         struct rhashtable_iter *rhiter = &iter->rhiter;
 500         struct ila_map *ila;
 501         int ret;
 502
 503         ret = rhashtable_walk_start(rhiter);
 504         if (ret && ret != -EAGAIN)
 505                 goto done;
 506
 507         for (;;) {
 508                 ila = rhashtable_walk_next(rhiter);
 509
 510                 if (IS_ERR(ila)) {
 511                         if (PTR_ERR(ila) == -EAGAIN)
 512                                 continue;
 513                         ret = PTR_ERR(ila);
 514                         goto done;
 515                 } else if (!ila) {
 516                         break;
 517                 }
 518
 519                 while (ila) {
 520                         ret =  ila_dump_info(ila, NETLINK_CB(cb->skb).portid,
 521                                              cb->nlh->nlmsg_seq, NLM_F_MULTI,
 522                                              skb, ILA_CMD_GET);
 523                         if (ret)
 524                                 goto done;
 525
 526                         ila = rcu_access_pointer(ila->next);
 527                 }
 528         }
 529
 530         ret = skb->len;
 531
 532 done:
 533         rhashtable_walk_stop(rhiter);
 534         return ret;
 535 }
 536
 537 static const struct genl_ops ila_nl_ops[] = {
 538         {
 539                 .cmd = ILA_CMD_ADD,
 540                 .doit = ila_nl_cmd_add_mapping,
 541                 .policy = ila_nl_policy,
 542                 .flags = GENL_ADMIN_PERM,
 543         },
 544         {
 545                 .cmd = ILA_CMD_DEL,
 546                 .doit = ila_nl_cmd_del_mapping,
 547                 .policy = ila_nl_policy,
 548                 .flags = GENL_ADMIN_PERM,
 549         },
 550         {
 551                 .cmd = ILA_CMD_GET,
 552                 .doit = ila_nl_cmd_get_mapping,
 553                 .start = ila_nl_dump_start,
 554                 .dumpit = ila_nl_dump,
 555                 .done = ila_nl_dump_done,
 556                 .policy = ila_nl_policy,
 557         },
 558 };
 559
 560 static struct genl_family ila_nl_family __ro_after_init = {
 561         .hdrsize        = 0,
 562         .name           = ILA_GENL_NAME,
 563         .version        = ILA_GENL_VERSION,
 564         .maxattr        = ILA_ATTR_MAX,
 565         .netnsok        = true,
 566         .parallel_ops   = true,
 567         .module         = THIS_MODULE,
 568         .ops            = ila_nl_ops,
 569         .n_ops          = ARRAY_SIZE(ila_nl_ops),
 570 };
 571
 572 #define ILA_HASH_TABLE_SIZE 1024
 573
 574 static __net_init int ila_init_net(struct net *net)
 575 {
 576         int err;
 577         struct ila_net *ilan = net_generic(net, ila_net_id);
 578
 579         err = alloc_ila_locks(ilan);
 580         if (err)
 581                 return err;
 582
 583         rhashtable_init(&ilan->rhash_table, &rht_params);
 584
 585         return 0;
 586 }
 587
 588 static __net_exit void ila_exit_net(struct net *net)
 589 {
 590         struct ila_net *ilan = net_generic(net, ila_net_id);
 591
 592         rhashtable_free_and_destroy(&ilan->rhash_table, ila_free_cb, NULL);
 593
 594         kvfree(ilan->locks);
 595
 596         if (ilan->hooks_registered)
 597                 nf_unregister_net_hooks(net, ila_nf_hook_ops,
 598                                         ARRAY_SIZE(ila_nf_hook_ops));
 599 }
 600
 601 static struct pernet_operations ila_net_ops = {
 602         .init = ila_init_net,
 603         .exit = ila_exit_net,
 604         .id   = &ila_net_id,
 605         .size = sizeof(struct ila_net),
 606 };
 607
 608 static int ila_xlat_addr(struct sk_buff *skb, bool set_csum_neutral)
 609 {
 610         struct ila_map *ila;
 611         struct ipv6hdr *ip6h = ipv6_hdr(skb);
 612         struct net *net = dev_net(skb->dev);
 613         struct ila_net *ilan = net_generic(net, ila_net_id);
 614         struct ila_addr *iaddr = ila_a2i(&ip6h->daddr);
 615
 616         /* Assumes skb contains a valid IPv6 header that is pulled */
 617
 618         if (!ila_addr_is_ila(iaddr)) {
 619                 /* Type indicates this is not an ILA address */
 620                 return 0;
 621         }
 622
 623         rcu_read_lock();
 624
 625         ila = ila_lookup_wildcards(iaddr, skb->dev->ifindex, ilan);
 626         if (ila)
 627                 ila_update_ipv6_locator(skb, &ila->xp.ip, set_csum_neutral);
 628
 629         rcu_read_unlock();
 630
 631         return 0;
 632 }
 633
 634 int __init ila_xlat_init(void)
 635 {
 636         int ret;
 637
 638         ret = register_pernet_device(&ila_net_ops);
 639         if (ret)
 640                 goto exit;
 641
 642         ret = genl_register_family(&ila_nl_family);
 643         if (ret < 0)
 644                 goto unregister;
 645
 646         return 0;
 647
 648 unregister:
 649         unregister_pernet_device(&ila_net_ops);
 650 exit:
 651         return ret;
 652 }
 653
 654 void ila_xlat_fini(void)
 655 {
 656         genl_unregister_family(&ila_nl_family);
 657         unregister_pernet_device(&ila_net_ops);
 658 }