1 // SPDX-License-Identifier: GPL-2.0-only
3 * net/sched/act_sample.c - Packet sampling tc action
4 * Copyright (c) 2017 Yotam Gigi <yotamg@mellanox.com>
7 #include <linux/types.h>
8 #include <linux/kernel.h>
9 #include <linux/string.h>
10 #include <linux/errno.h>
11 #include <linux/skbuff.h>
12 #include <linux/rtnetlink.h>
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/gfp.h>
16 #include <net/net_namespace.h>
17 #include <net/netlink.h>
18 #include <net/pkt_sched.h>
19 #include <linux/tc_act/tc_sample.h>
20 #include <net/tc_act/tc_sample.h>
21 #include <net/psample.h>
22 #include <net/pkt_cls.h>
23 #include <net/tc_wrapper.h>
25 #include <linux/if_arp.h>
27 static struct tc_action_ops act_sample_ops
;
29 static const struct nla_policy sample_policy
[TCA_SAMPLE_MAX
+ 1] = {
30 [TCA_SAMPLE_PARMS
] = { .len
= sizeof(struct tc_sample
) },
31 [TCA_SAMPLE_RATE
] = { .type
= NLA_U32
},
32 [TCA_SAMPLE_TRUNC_SIZE
] = { .type
= NLA_U32
},
33 [TCA_SAMPLE_PSAMPLE_GROUP
] = { .type
= NLA_U32
},
36 static int tcf_sample_init(struct net
*net
, struct nlattr
*nla
,
37 struct nlattr
*est
, struct tc_action
**a
,
39 u32 flags
, struct netlink_ext_ack
*extack
)
41 struct tc_action_net
*tn
= net_generic(net
, act_sample_ops
.net_id
);
42 bool bind
= flags
& TCA_ACT_FLAGS_BIND
;
43 struct nlattr
*tb
[TCA_SAMPLE_MAX
+ 1];
44 struct psample_group
*psample_group
;
45 u32 psample_group_num
, rate
, index
;
46 struct tcf_chain
*goto_ch
= NULL
;
47 struct tc_sample
*parm
;
54 ret
= nla_parse_nested_deprecated(tb
, TCA_SAMPLE_MAX
, nla
,
59 if (!tb
[TCA_SAMPLE_PARMS
])
62 parm
= nla_data(tb
[TCA_SAMPLE_PARMS
]);
64 err
= tcf_idr_check_alloc(tn
, &index
, a
, bind
);
72 ret
= tcf_idr_create(tn
, index
, est
, a
,
73 &act_sample_ops
, bind
, true, flags
);
75 tcf_idr_cleanup(tn
, index
);
79 } else if (!(flags
& TCA_ACT_FLAGS_REPLACE
)) {
80 tcf_idr_release(*a
, bind
);
84 if (!tb
[TCA_SAMPLE_RATE
] || !tb
[TCA_SAMPLE_PSAMPLE_GROUP
]) {
85 NL_SET_ERR_MSG(extack
, "sample rate and group are required");
90 err
= tcf_action_check_ctrlact(parm
->action
, tp
, &goto_ch
, extack
);
94 rate
= nla_get_u32(tb
[TCA_SAMPLE_RATE
]);
96 NL_SET_ERR_MSG(extack
, "invalid sample rate");
100 psample_group_num
= nla_get_u32(tb
[TCA_SAMPLE_PSAMPLE_GROUP
]);
101 psample_group
= psample_group_get(net
, psample_group_num
);
102 if (!psample_group
) {
109 spin_lock_bh(&s
->tcf_lock
);
110 goto_ch
= tcf_action_set_ctrlact(*a
, parm
->action
, goto_ch
);
112 s
->psample_group_num
= psample_group_num
;
113 psample_group
= rcu_replace_pointer(s
->psample_group
, psample_group
,
114 lockdep_is_held(&s
->tcf_lock
));
116 if (tb
[TCA_SAMPLE_TRUNC_SIZE
]) {
118 s
->trunc_size
= nla_get_u32(tb
[TCA_SAMPLE_TRUNC_SIZE
]);
120 spin_unlock_bh(&s
->tcf_lock
);
123 psample_group_put(psample_group
);
125 tcf_chain_put_by_act(goto_ch
);
130 tcf_chain_put_by_act(goto_ch
);
132 tcf_idr_release(*a
, bind
);
136 static void tcf_sample_cleanup(struct tc_action
*a
)
138 struct tcf_sample
*s
= to_sample(a
);
139 struct psample_group
*psample_group
;
141 /* last reference to action, no need to lock */
142 psample_group
= rcu_dereference_protected(s
->psample_group
, 1);
143 RCU_INIT_POINTER(s
->psample_group
, NULL
);
145 psample_group_put(psample_group
);
148 static bool tcf_sample_dev_ok_push(struct net_device
*dev
)
164 TC_INDIRECT_SCOPE
int tcf_sample_act(struct sk_buff
*skb
,
165 const struct tc_action
*a
,
166 struct tcf_result
*res
)
168 struct tcf_sample
*s
= to_sample(a
);
169 struct psample_group
*psample_group
;
170 u8 cookie_data
[TC_COOKIE_MAX_SIZE
];
171 struct psample_metadata md
= {};
172 struct tc_cookie
*user_cookie
;
175 tcf_lastuse_update(&s
->tcf_tm
);
176 bstats_update(this_cpu_ptr(s
->common
.cpu_bstats
), skb
);
177 retval
= READ_ONCE(s
->tcf_action
);
179 psample_group
= rcu_dereference_bh(s
->psample_group
);
181 /* randomly sample packets according to rate */
182 if (psample_group
&& (get_random_u32_below(s
->rate
) == 0)) {
183 if (!skb_at_tc_ingress(skb
)) {
184 md
.in_ifindex
= skb
->skb_iif
;
185 md
.out_ifindex
= skb
->dev
->ifindex
;
187 md
.in_ifindex
= skb
->dev
->ifindex
;
190 /* on ingress, the mac header gets popped, so push it back */
191 if (skb_at_tc_ingress(skb
) && tcf_sample_dev_ok_push(skb
->dev
))
192 skb_push(skb
, skb
->mac_len
);
195 user_cookie
= rcu_dereference(a
->user_cookie
);
197 memcpy(cookie_data
, user_cookie
->data
,
199 md
.user_cookie
= cookie_data
;
200 md
.user_cookie_len
= user_cookie
->len
;
204 md
.trunc_size
= s
->truncate
? s
->trunc_size
: skb
->len
;
205 psample_sample_packet(psample_group
, skb
, s
->rate
, &md
);
207 if (skb_at_tc_ingress(skb
) && tcf_sample_dev_ok_push(skb
->dev
))
208 skb_pull(skb
, skb
->mac_len
);
214 static void tcf_sample_stats_update(struct tc_action
*a
, u64 bytes
, u64 packets
,
215 u64 drops
, u64 lastuse
, bool hw
)
217 struct tcf_sample
*s
= to_sample(a
);
218 struct tcf_t
*tm
= &s
->tcf_tm
;
220 tcf_action_update_stats(a
, bytes
, packets
, drops
, hw
);
221 tm
->lastuse
= max_t(u64
, tm
->lastuse
, lastuse
);
224 static int tcf_sample_dump(struct sk_buff
*skb
, struct tc_action
*a
,
227 unsigned char *b
= skb_tail_pointer(skb
);
228 struct tcf_sample
*s
= to_sample(a
);
229 struct tc_sample opt
= {
230 .index
= s
->tcf_index
,
231 .refcnt
= refcount_read(&s
->tcf_refcnt
) - ref
,
232 .bindcnt
= atomic_read(&s
->tcf_bindcnt
) - bind
,
236 spin_lock_bh(&s
->tcf_lock
);
237 opt
.action
= s
->tcf_action
;
238 if (nla_put(skb
, TCA_SAMPLE_PARMS
, sizeof(opt
), &opt
))
239 goto nla_put_failure
;
241 tcf_tm_dump(&t
, &s
->tcf_tm
);
242 if (nla_put_64bit(skb
, TCA_SAMPLE_TM
, sizeof(t
), &t
, TCA_SAMPLE_PAD
))
243 goto nla_put_failure
;
245 if (nla_put_u32(skb
, TCA_SAMPLE_RATE
, s
->rate
))
246 goto nla_put_failure
;
249 if (nla_put_u32(skb
, TCA_SAMPLE_TRUNC_SIZE
, s
->trunc_size
))
250 goto nla_put_failure
;
252 if (nla_put_u32(skb
, TCA_SAMPLE_PSAMPLE_GROUP
, s
->psample_group_num
))
253 goto nla_put_failure
;
254 spin_unlock_bh(&s
->tcf_lock
);
259 spin_unlock_bh(&s
->tcf_lock
);
264 static void tcf_psample_group_put(void *priv
)
266 struct psample_group
*group
= priv
;
268 psample_group_put(group
);
271 static struct psample_group
*
272 tcf_sample_get_group(const struct tc_action
*a
,
273 tc_action_priv_destructor
*destructor
)
275 struct tcf_sample
*s
= to_sample(a
);
276 struct psample_group
*group
;
278 group
= rcu_dereference_protected(s
->psample_group
,
279 lockdep_is_held(&s
->tcf_lock
));
281 psample_group_take(group
);
282 *destructor
= tcf_psample_group_put
;
288 static void tcf_offload_sample_get_group(struct flow_action_entry
*entry
,
289 const struct tc_action
*act
)
291 entry
->sample
.psample_group
=
292 act
->ops
->get_psample_group(act
, &entry
->destructor
);
293 entry
->destructor_priv
= entry
->sample
.psample_group
;
296 static int tcf_sample_offload_act_setup(struct tc_action
*act
, void *entry_data
,
297 u32
*index_inc
, bool bind
,
298 struct netlink_ext_ack
*extack
)
301 struct flow_action_entry
*entry
= entry_data
;
303 entry
->id
= FLOW_ACTION_SAMPLE
;
304 entry
->sample
.trunc_size
= tcf_sample_trunc_size(act
);
305 entry
->sample
.truncate
= tcf_sample_truncate(act
);
306 entry
->sample
.rate
= tcf_sample_rate(act
);
307 tcf_offload_sample_get_group(entry
, act
);
310 struct flow_offload_action
*fl_action
= entry_data
;
312 fl_action
->id
= FLOW_ACTION_SAMPLE
;
318 static struct tc_action_ops act_sample_ops
= {
321 .owner
= THIS_MODULE
,
322 .act
= tcf_sample_act
,
323 .stats_update
= tcf_sample_stats_update
,
324 .dump
= tcf_sample_dump
,
325 .init
= tcf_sample_init
,
326 .cleanup
= tcf_sample_cleanup
,
327 .get_psample_group
= tcf_sample_get_group
,
328 .offload_act_setup
= tcf_sample_offload_act_setup
,
329 .size
= sizeof(struct tcf_sample
),
331 MODULE_ALIAS_NET_ACT("sample");
333 static __net_init
int sample_init_net(struct net
*net
)
335 struct tc_action_net
*tn
= net_generic(net
, act_sample_ops
.net_id
);
337 return tc_action_net_init(net
, tn
, &act_sample_ops
);
340 static void __net_exit
sample_exit_net(struct list_head
*net_list
)
342 tc_action_net_exit(net_list
, act_sample_ops
.net_id
);
345 static struct pernet_operations sample_net_ops
= {
346 .init
= sample_init_net
,
347 .exit_batch
= sample_exit_net
,
348 .id
= &act_sample_ops
.net_id
,
349 .size
= sizeof(struct tc_action_net
),
352 static int __init
sample_init_module(void)
354 return tcf_register_action(&act_sample_ops
, &sample_net_ops
);
357 static void __exit
sample_cleanup_module(void)
359 tcf_unregister_action(&act_sample_ops
, &sample_net_ops
);
362 module_init(sample_init_module
);
363 module_exit(sample_cleanup_module
);
365 MODULE_AUTHOR("Yotam Gigi <yotam.gi@gmail.com>");
366 MODULE_DESCRIPTION("Packet sampling action");
367 MODULE_LICENSE("GPL v2");