1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * net/sched/act_pedit.c Generic packet editor
5 * Authors: Jamal Hadi Salim (2002-4)
8 #include <linux/types.h>
9 #include <linux/kernel.h>
10 #include <linux/string.h>
11 #include <linux/errno.h>
12 #include <linux/skbuff.h>
13 #include <linux/rtnetlink.h>
14 #include <linux/module.h>
15 #include <linux/init.h>
17 #include <linux/ipv6.h>
18 #include <linux/slab.h>
20 #include <net/netlink.h>
21 #include <net/pkt_sched.h>
22 #include <linux/tc_act/tc_pedit.h>
23 #include <net/tc_act/tc_pedit.h>
24 #include <uapi/linux/tc_act/tc_pedit.h>
25 #include <net/pkt_cls.h>
26 #include <net/tc_wrapper.h>
28 static struct tc_action_ops act_pedit_ops
;
30 static const struct nla_policy pedit_policy
[TCA_PEDIT_MAX
+ 1] = {
31 [TCA_PEDIT_PARMS
] = { .len
= sizeof(struct tc_pedit
) },
32 [TCA_PEDIT_PARMS_EX
] = { .len
= sizeof(struct tc_pedit
) },
33 [TCA_PEDIT_KEYS_EX
] = { .type
= NLA_NESTED
},
36 static const struct nla_policy pedit_key_ex_policy
[TCA_PEDIT_KEY_EX_MAX
+ 1] = {
37 [TCA_PEDIT_KEY_EX_HTYPE
] =
38 NLA_POLICY_MAX(NLA_U16
, TCA_PEDIT_HDR_TYPE_MAX
),
39 [TCA_PEDIT_KEY_EX_CMD
] = NLA_POLICY_MAX(NLA_U16
, TCA_PEDIT_CMD_MAX
),
42 static struct tcf_pedit_key_ex
*tcf_pedit_keys_ex_parse(struct nlattr
*nla
,
43 u8 n
, struct netlink_ext_ack
*extack
)
45 struct tcf_pedit_key_ex
*keys_ex
;
46 struct tcf_pedit_key_ex
*k
;
47 const struct nlattr
*ka
;
54 keys_ex
= kcalloc(n
, sizeof(*k
), GFP_KERNEL
);
56 return ERR_PTR(-ENOMEM
);
60 nla_for_each_nested(ka
, nla
, rem
) {
61 struct nlattr
*tb
[TCA_PEDIT_KEY_EX_MAX
+ 1];
64 NL_SET_ERR_MSG_MOD(extack
, "Can't parse more extended keys than requested");
70 if (nla_type(ka
) != TCA_PEDIT_KEY_EX
) {
71 NL_SET_ERR_MSG_ATTR(extack
, ka
, "Unknown attribute, expected extended key");
76 err
= nla_parse_nested_deprecated(tb
, TCA_PEDIT_KEY_EX_MAX
,
77 ka
, pedit_key_ex_policy
,
82 if (NL_REQ_ATTR_CHECK(extack
, nla
, tb
, TCA_PEDIT_KEY_EX_HTYPE
)) {
83 NL_SET_ERR_MSG(extack
, "Missing required attribute");
88 if (NL_REQ_ATTR_CHECK(extack
, nla
, tb
, TCA_PEDIT_KEY_EX_CMD
)) {
89 NL_SET_ERR_MSG(extack
, "Missing required attribute");
94 k
->htype
= nla_get_u16(tb
[TCA_PEDIT_KEY_EX_HTYPE
]);
95 k
->cmd
= nla_get_u16(tb
[TCA_PEDIT_KEY_EX_CMD
]);
101 NL_SET_ERR_MSG_MOD(extack
, "Not enough extended keys to parse");
113 static int tcf_pedit_key_ex_dump(struct sk_buff
*skb
,
114 struct tcf_pedit_key_ex
*keys_ex
, int n
)
116 struct nlattr
*keys_start
= nla_nest_start_noflag(skb
,
122 struct nlattr
*key_start
;
124 key_start
= nla_nest_start_noflag(skb
, TCA_PEDIT_KEY_EX
);
128 if (nla_put_u16(skb
, TCA_PEDIT_KEY_EX_HTYPE
, keys_ex
->htype
) ||
129 nla_put_u16(skb
, TCA_PEDIT_KEY_EX_CMD
, keys_ex
->cmd
))
132 nla_nest_end(skb
, key_start
);
137 nla_nest_end(skb
, keys_start
);
141 nla_nest_cancel(skb
, keys_start
);
145 static void tcf_pedit_cleanup_rcu(struct rcu_head
*head
)
147 struct tcf_pedit_parms
*parms
=
148 container_of(head
, struct tcf_pedit_parms
, rcu
);
150 kfree(parms
->tcfp_keys_ex
);
151 kfree(parms
->tcfp_keys
);
156 static int tcf_pedit_init(struct net
*net
, struct nlattr
*nla
,
157 struct nlattr
*est
, struct tc_action
**a
,
158 struct tcf_proto
*tp
, u32 flags
,
159 struct netlink_ext_ack
*extack
)
161 struct tc_action_net
*tn
= net_generic(net
, act_pedit_ops
.net_id
);
162 bool bind
= flags
& TCA_ACT_FLAGS_BIND
;
163 struct tcf_chain
*goto_ch
= NULL
;
164 struct tcf_pedit_parms
*oparms
, *nparms
;
165 struct nlattr
*tb
[TCA_PEDIT_MAX
+ 1];
166 struct tc_pedit
*parm
;
167 struct nlattr
*pattr
;
174 NL_SET_ERR_MSG_MOD(extack
, "Pedit requires attributes to be passed");
178 err
= nla_parse_nested_deprecated(tb
, TCA_PEDIT_MAX
, nla
,
183 pattr
= tb
[TCA_PEDIT_PARMS
];
185 pattr
= tb
[TCA_PEDIT_PARMS_EX
];
187 NL_SET_ERR_MSG_MOD(extack
, "Missing required TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute");
191 parm
= nla_data(pattr
);
194 err
= tcf_idr_check_alloc(tn
, &index
, a
, bind
);
196 ret
= tcf_idr_create_from_flags(tn
, index
, est
, a
,
197 &act_pedit_ops
, bind
, flags
);
199 tcf_idr_cleanup(tn
, index
);
203 } else if (err
> 0) {
206 if (!(flags
& TCA_ACT_FLAGS_REPLACE
)) {
215 NL_SET_ERR_MSG_MOD(extack
, "Pedit requires keys to be passed");
219 ksize
= parm
->nkeys
* sizeof(struct tc_pedit_key
);
220 if (nla_len(pattr
) < sizeof(*parm
) + ksize
) {
221 NL_SET_ERR_MSG_ATTR(extack
, pattr
, "Length of TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute is invalid");
226 nparms
= kzalloc(sizeof(*nparms
), GFP_KERNEL
);
232 nparms
->tcfp_keys_ex
=
233 tcf_pedit_keys_ex_parse(tb
[TCA_PEDIT_KEYS_EX
], parm
->nkeys
, extack
);
234 if (IS_ERR(nparms
->tcfp_keys_ex
)) {
235 ret
= PTR_ERR(nparms
->tcfp_keys_ex
);
239 err
= tcf_action_check_ctrlact(parm
->action
, tp
, &goto_ch
, extack
);
245 nparms
->tcfp_off_max_hint
= 0;
246 nparms
->tcfp_flags
= parm
->flags
;
247 nparms
->tcfp_nkeys
= parm
->nkeys
;
249 nparms
->tcfp_keys
= kmemdup(parm
->keys
, ksize
, GFP_KERNEL
);
250 if (!nparms
->tcfp_keys
) {
255 for (i
= 0; i
< nparms
->tcfp_nkeys
; ++i
) {
256 u32 offmask
= nparms
->tcfp_keys
[i
].offmask
;
257 u32 cur
= nparms
->tcfp_keys
[i
].off
;
259 /* The AT option can be added to static offsets in the datapath */
260 if (!offmask
&& cur
% 4) {
261 NL_SET_ERR_MSG_MOD(extack
, "Offsets must be on 32bit boundaries");
266 /* sanitize the shift value for any later use */
267 nparms
->tcfp_keys
[i
].shift
= min_t(size_t,
268 BITS_PER_TYPE(int) - 1,
269 nparms
->tcfp_keys
[i
].shift
);
271 /* The AT option can read a single byte, we can bound the actual
272 * value with uchar max.
274 cur
+= (0xff & offmask
) >> nparms
->tcfp_keys
[i
].shift
;
276 /* Each key touches 4 bytes starting from the computed offset */
277 nparms
->tcfp_off_max_hint
=
278 max(nparms
->tcfp_off_max_hint
, cur
+ 4);
283 spin_lock_bh(&p
->tcf_lock
);
284 goto_ch
= tcf_action_set_ctrlact(*a
, parm
->action
, goto_ch
);
285 oparms
= rcu_replace_pointer(p
->parms
, nparms
, 1);
286 spin_unlock_bh(&p
->tcf_lock
);
289 call_rcu(&oparms
->rcu
, tcf_pedit_cleanup_rcu
);
292 tcf_chain_put_by_act(goto_ch
);
297 kfree(nparms
->tcfp_keys
);
300 tcf_chain_put_by_act(goto_ch
);
302 kfree(nparms
->tcfp_keys_ex
);
306 tcf_idr_release(*a
, bind
);
310 static void tcf_pedit_cleanup(struct tc_action
*a
)
312 struct tcf_pedit
*p
= to_pedit(a
);
313 struct tcf_pedit_parms
*parms
;
315 parms
= rcu_dereference_protected(p
->parms
, 1);
318 call_rcu(&parms
->rcu
, tcf_pedit_cleanup_rcu
);
321 static bool offset_valid(struct sk_buff
*skb
, int offset
)
323 if (offset
> 0 && offset
> skb
->len
)
326 if (offset
< 0 && -offset
> skb_headroom(skb
))
332 static int pedit_l4_skb_offset(struct sk_buff
*skb
, int *hoffset
, const int header_type
)
334 const int noff
= skb_network_offset(skb
);
338 switch (skb
->protocol
) {
339 case htons(ETH_P_IP
): {
340 const struct iphdr
*iph
= skb_header_pointer(skb
, noff
, sizeof(_iph
), &_iph
);
344 *hoffset
= noff
+ iph
->ihl
* 4;
348 case htons(ETH_P_IPV6
):
349 ret
= ipv6_find_hdr(skb
, hoffset
, header_type
, NULL
, NULL
) == header_type
? 0 : -EINVAL
;
356 static int pedit_skb_hdr_offset(struct sk_buff
*skb
,
357 enum pedit_header_type htype
, int *hoffset
)
360 /* 'htype' is validated in the netlink parsing */
362 case TCA_PEDIT_KEY_EX_HDR_TYPE_ETH
:
363 if (skb_mac_header_was_set(skb
)) {
364 *hoffset
= skb_mac_offset(skb
);
368 case TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK
:
369 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4
:
370 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6
:
371 *hoffset
= skb_network_offset(skb
);
374 case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP
:
375 ret
= pedit_l4_skb_offset(skb
, hoffset
, IPPROTO_TCP
);
377 case TCA_PEDIT_KEY_EX_HDR_TYPE_UDP
:
378 ret
= pedit_l4_skb_offset(skb
, hoffset
, IPPROTO_UDP
);
386 TC_INDIRECT_SCOPE
int tcf_pedit_act(struct sk_buff
*skb
,
387 const struct tc_action
*a
,
388 struct tcf_result
*res
)
390 enum pedit_header_type htype
= TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK
;
391 enum pedit_cmd cmd
= TCA_PEDIT_KEY_EX_CMD_SET
;
392 struct tcf_pedit
*p
= to_pedit(a
);
393 struct tcf_pedit_key_ex
*tkey_ex
;
394 struct tcf_pedit_parms
*parms
;
395 struct tc_pedit_key
*tkey
;
399 parms
= rcu_dereference_bh(p
->parms
);
401 max_offset
= (skb_transport_header_was_set(skb
) ?
402 skb_transport_offset(skb
) :
403 skb_network_offset(skb
)) +
404 parms
->tcfp_off_max_hint
;
405 if (skb_ensure_writable(skb
, min(skb
->len
, max_offset
)))
408 tcf_lastuse_update(&p
->tcf_tm
);
409 tcf_action_update_bstats(&p
->common
, skb
);
411 tkey
= parms
->tcfp_keys
;
412 tkey_ex
= parms
->tcfp_keys_ex
;
414 for (i
= parms
->tcfp_nkeys
; i
> 0; i
--, tkey
++) {
415 int offset
= tkey
->off
;
422 htype
= tkey_ex
->htype
;
428 rc
= pedit_skb_hdr_offset(skb
, htype
, &hoffset
);
430 pr_info_ratelimited("tc action pedit unable to extract header offset for header type (0x%x)\n", htype
);
437 if (!offset_valid(skb
, hoffset
+ tkey
->at
)) {
438 pr_info_ratelimited("tc action pedit 'at' offset %d out of bounds\n",
442 d
= skb_header_pointer(skb
, hoffset
+ tkey
->at
,
447 offset
+= (*d
& tkey
->offmask
) >> tkey
->shift
;
449 pr_info_ratelimited("tc action pedit offset must be on 32 bit boundaries\n");
454 if (!offset_valid(skb
, hoffset
+ offset
)) {
455 pr_info_ratelimited("tc action pedit offset %d out of bounds\n", hoffset
+ offset
);
459 ptr
= skb_header_pointer(skb
, hoffset
+ offset
,
460 sizeof(hdata
), &hdata
);
463 /* just do it, baby */
465 case TCA_PEDIT_KEY_EX_CMD_SET
:
468 case TCA_PEDIT_KEY_EX_CMD_ADD
:
469 val
= (*ptr
+ tkey
->val
) & ~tkey
->mask
;
472 pr_info_ratelimited("tc action pedit bad command (%d)\n", cmd
);
476 *ptr
= ((*ptr
& tkey
->mask
) ^ val
);
478 skb_store_bits(skb
, hoffset
+ offset
, ptr
, 4);
484 tcf_action_inc_overlimit_qstats(&p
->common
);
486 return p
->tcf_action
;
489 static void tcf_pedit_stats_update(struct tc_action
*a
, u64 bytes
, u64 packets
,
490 u64 drops
, u64 lastuse
, bool hw
)
492 struct tcf_pedit
*d
= to_pedit(a
);
493 struct tcf_t
*tm
= &d
->tcf_tm
;
495 tcf_action_update_stats(a
, bytes
, packets
, drops
, hw
);
496 tm
->lastuse
= max_t(u64
, tm
->lastuse
, lastuse
);
499 static int tcf_pedit_dump(struct sk_buff
*skb
, struct tc_action
*a
,
502 unsigned char *b
= skb_tail_pointer(skb
);
503 struct tcf_pedit
*p
= to_pedit(a
);
504 struct tcf_pedit_parms
*parms
;
505 struct tc_pedit
*opt
;
509 spin_lock_bh(&p
->tcf_lock
);
510 parms
= rcu_dereference_protected(p
->parms
, 1);
511 s
= struct_size(opt
, keys
, parms
->tcfp_nkeys
);
513 opt
= kzalloc(s
, GFP_ATOMIC
);
514 if (unlikely(!opt
)) {
515 spin_unlock_bh(&p
->tcf_lock
);
518 opt
->nkeys
= parms
->tcfp_nkeys
;
520 memcpy(opt
->keys
, parms
->tcfp_keys
,
521 flex_array_size(opt
, keys
, parms
->tcfp_nkeys
));
522 opt
->index
= p
->tcf_index
;
523 opt
->flags
= parms
->tcfp_flags
;
524 opt
->action
= p
->tcf_action
;
525 opt
->refcnt
= refcount_read(&p
->tcf_refcnt
) - ref
;
526 opt
->bindcnt
= atomic_read(&p
->tcf_bindcnt
) - bind
;
528 if (parms
->tcfp_keys_ex
) {
529 if (tcf_pedit_key_ex_dump(skb
, parms
->tcfp_keys_ex
,
531 goto nla_put_failure
;
533 if (nla_put(skb
, TCA_PEDIT_PARMS_EX
, s
, opt
))
534 goto nla_put_failure
;
536 if (nla_put(skb
, TCA_PEDIT_PARMS
, s
, opt
))
537 goto nla_put_failure
;
540 tcf_tm_dump(&t
, &p
->tcf_tm
);
541 if (nla_put_64bit(skb
, TCA_PEDIT_TM
, sizeof(t
), &t
, TCA_PEDIT_PAD
))
542 goto nla_put_failure
;
543 spin_unlock_bh(&p
->tcf_lock
);
549 spin_unlock_bh(&p
->tcf_lock
);
555 static int tcf_pedit_offload_act_setup(struct tc_action
*act
, void *entry_data
,
556 u32
*index_inc
, bool bind
,
557 struct netlink_ext_ack
*extack
)
560 struct flow_action_entry
*entry
= entry_data
;
563 for (k
= 0; k
< tcf_pedit_nkeys(act
); k
++) {
564 switch (tcf_pedit_cmd(act
, k
)) {
565 case TCA_PEDIT_KEY_EX_CMD_SET
:
566 entry
->id
= FLOW_ACTION_MANGLE
;
568 case TCA_PEDIT_KEY_EX_CMD_ADD
:
569 entry
->id
= FLOW_ACTION_ADD
;
572 NL_SET_ERR_MSG_MOD(extack
, "Unsupported pedit command offload");
575 entry
->mangle
.htype
= tcf_pedit_htype(act
, k
);
576 entry
->mangle
.mask
= tcf_pedit_mask(act
, k
);
577 entry
->mangle
.val
= tcf_pedit_val(act
, k
);
578 entry
->mangle
.offset
= tcf_pedit_offset(act
, k
);
579 entry
->hw_stats
= tc_act_hw_stats(act
->hw_stats
);
584 struct flow_offload_action
*fl_action
= entry_data
;
585 u32 cmd
= tcf_pedit_cmd(act
, 0);
589 case TCA_PEDIT_KEY_EX_CMD_SET
:
590 fl_action
->id
= FLOW_ACTION_MANGLE
;
592 case TCA_PEDIT_KEY_EX_CMD_ADD
:
593 fl_action
->id
= FLOW_ACTION_ADD
;
596 NL_SET_ERR_MSG_MOD(extack
, "Unsupported pedit command offload");
600 for (k
= 1; k
< tcf_pedit_nkeys(act
); k
++) {
601 if (cmd
!= tcf_pedit_cmd(act
, k
)) {
602 NL_SET_ERR_MSG_MOD(extack
, "Unsupported pedit command offload");
611 static struct tc_action_ops act_pedit_ops
= {
614 .owner
= THIS_MODULE
,
615 .act
= tcf_pedit_act
,
616 .stats_update
= tcf_pedit_stats_update
,
617 .dump
= tcf_pedit_dump
,
618 .cleanup
= tcf_pedit_cleanup
,
619 .init
= tcf_pedit_init
,
620 .offload_act_setup
= tcf_pedit_offload_act_setup
,
621 .size
= sizeof(struct tcf_pedit
),
623 MODULE_ALIAS_NET_ACT("pedit");
625 static __net_init
int pedit_init_net(struct net
*net
)
627 struct tc_action_net
*tn
= net_generic(net
, act_pedit_ops
.net_id
);
629 return tc_action_net_init(net
, tn
, &act_pedit_ops
);
632 static void __net_exit
pedit_exit_net(struct list_head
*net_list
)
634 tc_action_net_exit(net_list
, act_pedit_ops
.net_id
);
637 static struct pernet_operations pedit_net_ops
= {
638 .init
= pedit_init_net
,
639 .exit_batch
= pedit_exit_net
,
640 .id
= &act_pedit_ops
.net_id
,
641 .size
= sizeof(struct tc_action_net
),
644 MODULE_AUTHOR("Jamal Hadi Salim(2002-4)");
645 MODULE_DESCRIPTION("Generic Packet Editor actions");
646 MODULE_LICENSE("GPL");
648 static int __init
pedit_init_module(void)
650 return tcf_register_action(&act_pedit_ops
, &pedit_net_ops
);
653 static void __exit
pedit_cleanup_module(void)
655 tcf_unregister_action(&act_pedit_ops
, &pedit_net_ops
);
658 module_init(pedit_init_module
);
659 module_exit(pedit_cleanup_module
);