1 #include <linux/jhash.h>
2 #include <linux/netfilter.h>
3 #include <linux/rcupdate.h>
4 #include <linux/rhashtable.h>
5 #include <linux/vmalloc.h>
6 #include <net/genetlink.h>
8 #include <net/netns/generic.h>
9 #include <uapi/linux/genetlink.h>
12 struct ila_xlat_params
{
18 struct ila_xlat_params xp
;
19 struct rhash_head node
;
20 struct ila_map __rcu
*next
;
24 static unsigned int ila_net_id
;
27 struct rhashtable rhash_table
;
28 spinlock_t
*locks
; /* Bucket locks for entry manipulation */
29 unsigned int locks_mask
;
30 bool hooks_registered
;
33 #define LOCKS_PER_CPU 10
35 static int alloc_ila_locks(struct ila_net
*ilan
)
38 unsigned int nr_pcpus
= num_possible_cpus();
40 nr_pcpus
= min_t(unsigned int, nr_pcpus
, 32UL);
41 size
= roundup_pow_of_two(nr_pcpus
* LOCKS_PER_CPU
);
43 if (sizeof(spinlock_t
) != 0) {
45 if (size
* sizeof(spinlock_t
) > PAGE_SIZE
)
46 ilan
->locks
= vmalloc(size
* sizeof(spinlock_t
));
49 ilan
->locks
= kmalloc_array(size
, sizeof(spinlock_t
),
53 for (i
= 0; i
< size
; i
++)
54 spin_lock_init(&ilan
->locks
[i
]);
56 ilan
->locks_mask
= size
- 1;
61 static u32 hashrnd __read_mostly
;
62 static __always_inline
void __ila_hash_secret_init(void)
64 net_get_random_once(&hashrnd
, sizeof(hashrnd
));
67 static inline u32
ila_locator_hash(struct ila_locator loc
)
69 u32
*v
= (u32
*)loc
.v32
;
71 return jhash_2words(v
[0], v
[1], hashrnd
);
74 static inline spinlock_t
*ila_get_lock(struct ila_net
*ilan
,
75 struct ila_locator loc
)
77 return &ilan
->locks
[ila_locator_hash(loc
) & ilan
->locks_mask
];
80 static inline int ila_cmp_wildcards(struct ila_map
*ila
,
81 struct ila_addr
*iaddr
, int ifindex
)
83 return (ila
->xp
.ifindex
&& ila
->xp
.ifindex
!= ifindex
);
86 static inline int ila_cmp_params(struct ila_map
*ila
,
87 struct ila_xlat_params
*xp
)
89 return (ila
->xp
.ifindex
!= xp
->ifindex
);
92 static int ila_cmpfn(struct rhashtable_compare_arg
*arg
,
95 const struct ila_map
*ila
= obj
;
97 return (ila
->xp
.ip
.locator_match
.v64
!= *(__be64
*)arg
->key
);
100 static inline int ila_order(struct ila_map
*ila
)
110 static const struct rhashtable_params rht_params
= {
112 .head_offset
= offsetof(struct ila_map
, node
),
113 .key_offset
= offsetof(struct ila_map
, xp
.ip
.locator_match
),
114 .key_len
= sizeof(u64
), /* identifier */
117 .automatic_shrinking
= true,
118 .obj_cmpfn
= ila_cmpfn
,
121 static struct genl_family ila_nl_family
= {
122 .id
= GENL_ID_GENERATE
,
124 .name
= ILA_GENL_NAME
,
125 .version
= ILA_GENL_VERSION
,
126 .maxattr
= ILA_ATTR_MAX
,
128 .parallel_ops
= true,
131 static struct nla_policy ila_nl_policy
[ILA_ATTR_MAX
+ 1] = {
132 [ILA_ATTR_LOCATOR
] = { .type
= NLA_U64
, },
133 [ILA_ATTR_LOCATOR_MATCH
] = { .type
= NLA_U64
, },
134 [ILA_ATTR_IFINDEX
] = { .type
= NLA_U32
, },
135 [ILA_ATTR_CSUM_MODE
] = { .type
= NLA_U8
, },
138 static int parse_nl_config(struct genl_info
*info
,
139 struct ila_xlat_params
*xp
)
141 memset(xp
, 0, sizeof(*xp
));
143 if (info
->attrs
[ILA_ATTR_LOCATOR
])
144 xp
->ip
.locator
.v64
= (__force __be64
)nla_get_u64(
145 info
->attrs
[ILA_ATTR_LOCATOR
]);
147 if (info
->attrs
[ILA_ATTR_LOCATOR_MATCH
])
148 xp
->ip
.locator_match
.v64
= (__force __be64
)nla_get_u64(
149 info
->attrs
[ILA_ATTR_LOCATOR_MATCH
]);
151 if (info
->attrs
[ILA_ATTR_CSUM_MODE
])
152 xp
->ip
.csum_mode
= nla_get_u8(info
->attrs
[ILA_ATTR_CSUM_MODE
]);
154 if (info
->attrs
[ILA_ATTR_IFINDEX
])
155 xp
->ifindex
= nla_get_s32(info
->attrs
[ILA_ATTR_IFINDEX
]);
160 /* Must be called with rcu readlock */
161 static inline struct ila_map
*ila_lookup_wildcards(struct ila_addr
*iaddr
,
163 struct ila_net
*ilan
)
167 ila
= rhashtable_lookup_fast(&ilan
->rhash_table
, &iaddr
->loc
,
170 if (!ila_cmp_wildcards(ila
, iaddr
, ifindex
))
172 ila
= rcu_access_pointer(ila
->next
);
178 /* Must be called with rcu readlock */
179 static inline struct ila_map
*ila_lookup_by_params(struct ila_xlat_params
*xp
,
180 struct ila_net
*ilan
)
184 ila
= rhashtable_lookup_fast(&ilan
->rhash_table
,
185 &xp
->ip
.locator_match
,
188 if (!ila_cmp_params(ila
, xp
))
190 ila
= rcu_access_pointer(ila
->next
);
196 static inline void ila_release(struct ila_map
*ila
)
201 static void ila_free_cb(void *ptr
, void *arg
)
203 struct ila_map
*ila
= (struct ila_map
*)ptr
, *next
;
205 /* Assume rcu_readlock held */
207 next
= rcu_access_pointer(ila
->next
);
213 static int ila_xlat_addr(struct sk_buff
*skb
, bool set_csum_neutral
);
216 ila_nf_input(void *priv
,
218 const struct nf_hook_state
*state
)
220 ila_xlat_addr(skb
, false);
224 static struct nf_hook_ops ila_nf_hook_ops
[] __read_mostly
= {
226 .hook
= ila_nf_input
,
228 .hooknum
= NF_INET_PRE_ROUTING
,
233 static int ila_add_mapping(struct net
*net
, struct ila_xlat_params
*xp
)
235 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
236 struct ila_map
*ila
, *head
;
237 spinlock_t
*lock
= ila_get_lock(ilan
, xp
->ip
.locator_match
);
240 if (!ilan
->hooks_registered
) {
241 /* We defer registering net hooks in the namespace until the
242 * first mapping is added.
244 err
= nf_register_net_hooks(net
, ila_nf_hook_ops
,
245 ARRAY_SIZE(ila_nf_hook_ops
));
249 ilan
->hooks_registered
= true;
252 ila
= kzalloc(sizeof(*ila
), GFP_KERNEL
);
256 ila_init_saved_csum(&xp
->ip
);
260 order
= ila_order(ila
);
264 head
= rhashtable_lookup_fast(&ilan
->rhash_table
,
265 &xp
->ip
.locator_match
,
268 /* New entry for the rhash_table */
269 err
= rhashtable_lookup_insert_fast(&ilan
->rhash_table
,
270 &ila
->node
, rht_params
);
272 struct ila_map
*tila
= head
, *prev
= NULL
;
275 if (!ila_cmp_params(tila
, xp
)) {
280 if (order
> ila_order(tila
))
284 tila
= rcu_dereference_protected(tila
->next
,
285 lockdep_is_held(lock
));
289 /* Insert in sub list of head */
290 RCU_INIT_POINTER(ila
->next
, tila
);
291 rcu_assign_pointer(prev
->next
, ila
);
293 /* Make this ila new head */
294 RCU_INIT_POINTER(ila
->next
, head
);
295 err
= rhashtable_replace_fast(&ilan
->rhash_table
,
297 &ila
->node
, rht_params
);
312 static int ila_del_mapping(struct net
*net
, struct ila_xlat_params
*xp
)
314 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
315 struct ila_map
*ila
, *head
, *prev
;
316 spinlock_t
*lock
= ila_get_lock(ilan
, xp
->ip
.locator_match
);
321 head
= rhashtable_lookup_fast(&ilan
->rhash_table
,
322 &xp
->ip
.locator_match
, rht_params
);
328 if (ila_cmp_params(ila
, xp
)) {
330 ila
= rcu_dereference_protected(ila
->next
,
331 lockdep_is_held(lock
));
338 /* Not head, just delete from list */
339 rcu_assign_pointer(prev
->next
, ila
->next
);
341 /* It is the head. If there is something in the
342 * sublist we need to make a new head.
344 head
= rcu_dereference_protected(ila
->next
,
345 lockdep_is_held(lock
));
347 /* Put first entry in the sublist into the
350 err
= rhashtable_replace_fast(
351 &ilan
->rhash_table
, &ila
->node
,
352 &head
->node
, rht_params
);
356 /* Entry no longer used */
357 err
= rhashtable_remove_fast(&ilan
->rhash_table
,
374 static int ila_nl_cmd_add_mapping(struct sk_buff
*skb
, struct genl_info
*info
)
376 struct net
*net
= genl_info_net(info
);
377 struct ila_xlat_params p
;
380 err
= parse_nl_config(info
, &p
);
384 return ila_add_mapping(net
, &p
);
387 static int ila_nl_cmd_del_mapping(struct sk_buff
*skb
, struct genl_info
*info
)
389 struct net
*net
= genl_info_net(info
);
390 struct ila_xlat_params xp
;
393 err
= parse_nl_config(info
, &xp
);
397 ila_del_mapping(net
, &xp
);
402 static int ila_fill_info(struct ila_map
*ila
, struct sk_buff
*msg
)
404 if (nla_put_u64_64bit(msg
, ILA_ATTR_LOCATOR
,
405 (__force u64
)ila
->xp
.ip
.locator
.v64
,
407 nla_put_u64_64bit(msg
, ILA_ATTR_LOCATOR_MATCH
,
408 (__force u64
)ila
->xp
.ip
.locator_match
.v64
,
410 nla_put_s32(msg
, ILA_ATTR_IFINDEX
, ila
->xp
.ifindex
) ||
411 nla_put_u32(msg
, ILA_ATTR_CSUM_MODE
, ila
->xp
.ip
.csum_mode
))
417 static int ila_dump_info(struct ila_map
*ila
,
418 u32 portid
, u32 seq
, u32 flags
,
419 struct sk_buff
*skb
, u8 cmd
)
423 hdr
= genlmsg_put(skb
, portid
, seq
, &ila_nl_family
, flags
, cmd
);
427 if (ila_fill_info(ila
, skb
) < 0)
428 goto nla_put_failure
;
430 genlmsg_end(skb
, hdr
);
434 genlmsg_cancel(skb
, hdr
);
438 static int ila_nl_cmd_get_mapping(struct sk_buff
*skb
, struct genl_info
*info
)
440 struct net
*net
= genl_info_net(info
);
441 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
443 struct ila_xlat_params xp
;
447 ret
= parse_nl_config(info
, &xp
);
451 msg
= nlmsg_new(NLMSG_DEFAULT_SIZE
, GFP_KERNEL
);
457 ila
= ila_lookup_by_params(&xp
, ilan
);
459 ret
= ila_dump_info(ila
,
461 info
->snd_seq
, 0, msg
,
470 return genlmsg_reply(msg
, info
);
477 struct ila_dump_iter
{
478 struct rhashtable_iter rhiter
;
481 static int ila_nl_dump_start(struct netlink_callback
*cb
)
483 struct net
*net
= sock_net(cb
->skb
->sk
);
484 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
485 struct ila_dump_iter
*iter
= (struct ila_dump_iter
*)cb
->args
;
487 return rhashtable_walk_init(&ilan
->rhash_table
, &iter
->rhiter
,
491 static int ila_nl_dump_done(struct netlink_callback
*cb
)
493 struct ila_dump_iter
*iter
= (struct ila_dump_iter
*)cb
->args
;
495 rhashtable_walk_exit(&iter
->rhiter
);
500 static int ila_nl_dump(struct sk_buff
*skb
, struct netlink_callback
*cb
)
502 struct ila_dump_iter
*iter
= (struct ila_dump_iter
*)cb
->args
;
503 struct rhashtable_iter
*rhiter
= &iter
->rhiter
;
507 ret
= rhashtable_walk_start(rhiter
);
508 if (ret
&& ret
!= -EAGAIN
)
512 ila
= rhashtable_walk_next(rhiter
);
515 if (PTR_ERR(ila
) == -EAGAIN
)
524 ret
= ila_dump_info(ila
, NETLINK_CB(cb
->skb
).portid
,
525 cb
->nlh
->nlmsg_seq
, NLM_F_MULTI
,
530 ila
= rcu_access_pointer(ila
->next
);
537 rhashtable_walk_stop(rhiter
);
541 static const struct genl_ops ila_nl_ops
[] = {
544 .doit
= ila_nl_cmd_add_mapping
,
545 .policy
= ila_nl_policy
,
546 .flags
= GENL_ADMIN_PERM
,
550 .doit
= ila_nl_cmd_del_mapping
,
551 .policy
= ila_nl_policy
,
552 .flags
= GENL_ADMIN_PERM
,
556 .doit
= ila_nl_cmd_get_mapping
,
557 .start
= ila_nl_dump_start
,
558 .dumpit
= ila_nl_dump
,
559 .done
= ila_nl_dump_done
,
560 .policy
= ila_nl_policy
,
564 #define ILA_HASH_TABLE_SIZE 1024
566 static __net_init
int ila_init_net(struct net
*net
)
569 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
571 err
= alloc_ila_locks(ilan
);
575 rhashtable_init(&ilan
->rhash_table
, &rht_params
);
580 static __net_exit
void ila_exit_net(struct net
*net
)
582 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
584 rhashtable_free_and_destroy(&ilan
->rhash_table
, ila_free_cb
, NULL
);
588 if (ilan
->hooks_registered
)
589 nf_unregister_net_hooks(net
, ila_nf_hook_ops
,
590 ARRAY_SIZE(ila_nf_hook_ops
));
593 static struct pernet_operations ila_net_ops
= {
594 .init
= ila_init_net
,
595 .exit
= ila_exit_net
,
597 .size
= sizeof(struct ila_net
),
600 static int ila_xlat_addr(struct sk_buff
*skb
, bool set_csum_neutral
)
603 struct ipv6hdr
*ip6h
= ipv6_hdr(skb
);
604 struct net
*net
= dev_net(skb
->dev
);
605 struct ila_net
*ilan
= net_generic(net
, ila_net_id
);
606 struct ila_addr
*iaddr
= ila_a2i(&ip6h
->daddr
);
608 /* Assumes skb contains a valid IPv6 header that is pulled */
610 if (!ila_addr_is_ila(iaddr
)) {
611 /* Type indicates this is not an ILA address */
617 ila
= ila_lookup_wildcards(iaddr
, skb
->dev
->ifindex
, ilan
);
619 ila_update_ipv6_locator(skb
, &ila
->xp
.ip
, set_csum_neutral
);
626 int ila_xlat_init(void)
630 ret
= register_pernet_device(&ila_net_ops
);
634 ret
= genl_register_family_with_ops(&ila_nl_family
,
642 unregister_pernet_device(&ila_net_ops
);
647 void ila_xlat_fini(void)
649 genl_unregister_family(&ila_nl_family
);
650 unregister_pernet_device(&ila_net_ops
);