?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
staging: erofs: integrate decompression inplace
[linux/fpc-iii.git] / net / openvswitch / actions.c
blob151518dbabad743e5506101cf4925af59678332d
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2007-2017 Nicira, Inc.
4 */
5
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8 #include <linux/skbuff.h>
9 #include <linux/in.h>
10 #include <linux/ip.h>
11 #include <linux/openvswitch.h>
12 #include <linux/netfilter_ipv6.h>
13 #include <linux/sctp.h>
14 #include <linux/tcp.h>
15 #include <linux/udp.h>
16 #include <linux/in6.h>
17 #include <linux/if_arp.h>
18 #include <linux/if_vlan.h>
19
20 #include <net/dst.h>
21 #include <net/ip.h>
22 #include <net/ipv6.h>
23 #include <net/ip6_fib.h>
24 #include <net/checksum.h>
25 #include <net/dsfield.h>
26 #include <net/mpls.h>
27 #include <net/sctp/checksum.h>
28
29 #include "datapath.h"
30 #include "flow.h"
31 #include "conntrack.h"
32 #include "vport.h"
33 #include "flow_netlink.h"
34
35 struct deferred_action {
36 struct sk_buff *skb;
37 const struct nlattr *actions;
38 int actions_len;
39
40 /* Store pkt_key clone when creating deferred action. */
41 struct sw_flow_key pkt_key;
42 };
43
44 #define MAX_L2_LEN (VLAN_ETH_HLEN + 3 * MPLS_HLEN)
45 struct ovs_frag_data {
46 unsigned long dst;
47 struct vport *vport;
48 struct ovs_skb_cb cb;
49 __be16 inner_protocol;
50 u16 network_offset; /* valid only for MPLS */
51 u16 vlan_tci;
52 __be16 vlan_proto;
53 unsigned int l2_len;
54 u8 mac_proto;
55 u8 l2_data[MAX_L2_LEN];
56 };
57
58 static DEFINE_PER_CPU(struct ovs_frag_data, ovs_frag_data_storage);
59
60 #define DEFERRED_ACTION_FIFO_SIZE 10
61 #define OVS_RECURSION_LIMIT 5
62 #define OVS_DEFERRED_ACTION_THRESHOLD (OVS_RECURSION_LIMIT - 2)
63 struct action_fifo {
64 int head;
65 int tail;
66 /* Deferred action fifo queue storage. */
67 struct deferred_action fifo[DEFERRED_ACTION_FIFO_SIZE];
68 };
69
70 struct action_flow_keys {
71 struct sw_flow_key key[OVS_DEFERRED_ACTION_THRESHOLD];
72 };
73
74 static struct action_fifo __percpu *action_fifos;
75 static struct action_flow_keys __percpu *flow_keys;
76 static DEFINE_PER_CPU(int, exec_actions_level);
77
78 /* Make a clone of the 'key', using the pre-allocated percpu 'flow_keys'
79 * space. Return NULL if out of key spaces.
80 */
81 static struct sw_flow_key *clone_key(const struct sw_flow_key *key_)
82 {
83 struct action_flow_keys *keys = this_cpu_ptr(flow_keys);
84 int level = this_cpu_read(exec_actions_level);
85 struct sw_flow_key *key = NULL;
86
87 if (level <= OVS_DEFERRED_ACTION_THRESHOLD) {
88 key = &keys->key[level - 1];
89 *key = *key_;
90 }
91
92 return key;
93 }
94
95 static void action_fifo_init(struct action_fifo *fifo)
96 {
97 fifo->head = 0;
98 fifo->tail = 0;
99 }
100
101 static bool action_fifo_is_empty(const struct action_fifo *fifo)
102 {
103 return (fifo->head == fifo->tail);
104 }
105
106 static struct deferred_action *action_fifo_get(struct action_fifo *fifo)
107 {
108 if (action_fifo_is_empty(fifo))
109 return NULL;
110
111 return &fifo->fifo[fifo->tail++];
112 }
113
114 static struct deferred_action *action_fifo_put(struct action_fifo *fifo)
115 {
116 if (fifo->head >= DEFERRED_ACTION_FIFO_SIZE - 1)
117 return NULL;
118
119 return &fifo->fifo[fifo->head++];
120 }
121
122 /* Return true if fifo is not full */
123 static struct deferred_action *add_deferred_actions(struct sk_buff *skb,
124 const struct sw_flow_key *key,
125 const struct nlattr *actions,
126 const int actions_len)
127 {
128 struct action_fifo *fifo;
129 struct deferred_action *da;
130
131 fifo = this_cpu_ptr(action_fifos);
132 da = action_fifo_put(fifo);
133 if (da) {
134 da->skb = skb;
135 da->actions = actions;
136 da->actions_len = actions_len;
137 da->pkt_key = *key;
138 }
139
140 return da;
141 }
142
143 static void invalidate_flow_key(struct sw_flow_key *key)
144 {
145 key->mac_proto |= SW_FLOW_KEY_INVALID;
146 }
147
148 static bool is_flow_key_valid(const struct sw_flow_key *key)
149 {
150 return !(key->mac_proto & SW_FLOW_KEY_INVALID);
151 }
152
153 static int clone_execute(struct datapath *dp, struct sk_buff *skb,
154 struct sw_flow_key *key,
155 u32 recirc_id,
156 const struct nlattr *actions, int len,
157 bool last, bool clone_flow_key);
158
159 static int do_execute_actions(struct datapath *dp, struct sk_buff *skb,
160 struct sw_flow_key *key,
161 const struct nlattr *attr, int len);
162
163 static void update_ethertype(struct sk_buff *skb, struct ethhdr *hdr,
164 __be16 ethertype)
165 {
166 if (skb->ip_summed == CHECKSUM_COMPLETE) {
167 __be16 diff[] = { ~(hdr->h_proto), ethertype };
168
169 skb->csum = ~csum_partial((char *)diff, sizeof(diff),
170 ~skb->csum);
171 }
172
173 hdr->h_proto = ethertype;
174 }
175
176 static int push_mpls(struct sk_buff *skb, struct sw_flow_key *key,
177 const struct ovs_action_push_mpls *mpls)
178 {
179 struct mpls_shim_hdr *new_mpls_lse;
180
181 /* Networking stack do not allow simultaneous Tunnel and MPLS GSO. */
182 if (skb->encapsulation)
183 return -ENOTSUPP;
184
185 if (skb_cow_head(skb, MPLS_HLEN) < 0)
186 return -ENOMEM;
187
188 if (!skb->inner_protocol) {
189 skb_set_inner_network_header(skb, skb->mac_len);
190 skb_set_inner_protocol(skb, skb->protocol);
191 }
192
193 skb_push(skb, MPLS_HLEN);
194 memmove(skb_mac_header(skb) - MPLS_HLEN, skb_mac_header(skb),
195 skb->mac_len);
196 skb_reset_mac_header(skb);
197 skb_set_network_header(skb, skb->mac_len);
198
199 new_mpls_lse = mpls_hdr(skb);
200 new_mpls_lse->label_stack_entry = mpls->mpls_lse;
201
202 skb_postpush_rcsum(skb, new_mpls_lse, MPLS_HLEN);
203
204 if (ovs_key_mac_proto(key) == MAC_PROTO_ETHERNET)
205 update_ethertype(skb, eth_hdr(skb), mpls->mpls_ethertype);
206 skb->protocol = mpls->mpls_ethertype;
207
208 invalidate_flow_key(key);
209 return 0;
210 }
211
212 static int pop_mpls(struct sk_buff *skb, struct sw_flow_key *key,
213 const __be16 ethertype)
214 {
215 int err;
216
217 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
218 if (unlikely(err))
219 return err;
220
221 skb_postpull_rcsum(skb, mpls_hdr(skb), MPLS_HLEN);
222
223 memmove(skb_mac_header(skb) + MPLS_HLEN, skb_mac_header(skb),
224 skb->mac_len);
225
226 __skb_pull(skb, MPLS_HLEN);
227 skb_reset_mac_header(skb);
228 skb_set_network_header(skb, skb->mac_len);
229
230 if (ovs_key_mac_proto(key) == MAC_PROTO_ETHERNET) {
231 struct ethhdr *hdr;
232
233 /* mpls_hdr() is used to locate the ethertype field correctly in the
234 * presence of VLAN tags.
235 */
236 hdr = (struct ethhdr *)((void *)mpls_hdr(skb) - ETH_HLEN);
237 update_ethertype(skb, hdr, ethertype);
238 }
239 if (eth_p_mpls(skb->protocol))
240 skb->protocol = ethertype;
241
242 invalidate_flow_key(key);
243 return 0;
244 }
245
246 static int set_mpls(struct sk_buff *skb, struct sw_flow_key *flow_key,
247 const __be32 *mpls_lse, const __be32 *mask)
248 {
249 struct mpls_shim_hdr *stack;
250 __be32 lse;
251 int err;
252
253 err = skb_ensure_writable(skb, skb->mac_len + MPLS_HLEN);
254 if (unlikely(err))
255 return err;
256
257 stack = mpls_hdr(skb);
258 lse = OVS_MASKED(stack->label_stack_entry, *mpls_lse, *mask);
259 if (skb->ip_summed == CHECKSUM_COMPLETE) {
260 __be32 diff[] = { ~(stack->label_stack_entry), lse };
261
262 skb->csum = ~csum_partial((char *)diff, sizeof(diff),
263 ~skb->csum);
264 }
265
266 stack->label_stack_entry = lse;
267 flow_key->mpls.top_lse = lse;
268 return 0;
269 }
270
271 static int pop_vlan(struct sk_buff *skb, struct sw_flow_key *key)
272 {
273 int err;
274
275 err = skb_vlan_pop(skb);
276 if (skb_vlan_tag_present(skb)) {
277 invalidate_flow_key(key);
278 } else {
279 key->eth.vlan.tci = 0;
280 key->eth.vlan.tpid = 0;
281 }
282 return err;
283 }
284
285 static int push_vlan(struct sk_buff *skb, struct sw_flow_key *key,
286 const struct ovs_action_push_vlan *vlan)
287 {
288 if (skb_vlan_tag_present(skb)) {
289 invalidate_flow_key(key);
290 } else {
291 key->eth.vlan.tci = vlan->vlan_tci;
292 key->eth.vlan.tpid = vlan->vlan_tpid;
293 }
294 return skb_vlan_push(skb, vlan->vlan_tpid,
295 ntohs(vlan->vlan_tci) & ~VLAN_CFI_MASK);
296 }
297
298 /* 'src' is already properly masked. */
299 static void ether_addr_copy_masked(u8 *dst_, const u8 *src_, const u8 *mask_)
300 {
301 u16 *dst = (u16 *)dst_;
302 const u16 *src = (const u16 *)src_;
303 const u16 *mask = (const u16 *)mask_;
304
305 OVS_SET_MASKED(dst[0], src[0], mask[0]);
306 OVS_SET_MASKED(dst[1], src[1], mask[1]);
307 OVS_SET_MASKED(dst[2], src[2], mask[2]);
308 }
309
310 static int set_eth_addr(struct sk_buff *skb, struct sw_flow_key *flow_key,
311 const struct ovs_key_ethernet *key,
312 const struct ovs_key_ethernet *mask)
313 {
314 int err;
315
316 err = skb_ensure_writable(skb, ETH_HLEN);
317 if (unlikely(err))
318 return err;
319
320 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_ALEN * 2);
321
322 ether_addr_copy_masked(eth_hdr(skb)->h_source, key->eth_src,
323 mask->eth_src);
324 ether_addr_copy_masked(eth_hdr(skb)->h_dest, key->eth_dst,
325 mask->eth_dst);
326
327 skb_postpush_rcsum(skb, eth_hdr(skb), ETH_ALEN * 2);
328
329 ether_addr_copy(flow_key->eth.src, eth_hdr(skb)->h_source);
330 ether_addr_copy(flow_key->eth.dst, eth_hdr(skb)->h_dest);
331 return 0;
332 }
333
334 /* pop_eth does not support VLAN packets as this action is never called
335 * for them.
336 */
337 static int pop_eth(struct sk_buff *skb, struct sw_flow_key *key)
338 {
339 skb_pull_rcsum(skb, ETH_HLEN);
340 skb_reset_mac_header(skb);
341 skb_reset_mac_len(skb);
342
343 /* safe right before invalidate_flow_key */
344 key->mac_proto = MAC_PROTO_NONE;
345 invalidate_flow_key(key);
346 return 0;
347 }
348
349 static int push_eth(struct sk_buff *skb, struct sw_flow_key *key,
350 const struct ovs_action_push_eth *ethh)
351 {
352 struct ethhdr *hdr;
353
354 /* Add the new Ethernet header */
355 if (skb_cow_head(skb, ETH_HLEN) < 0)
356 return -ENOMEM;
357
358 skb_push(skb, ETH_HLEN);
359 skb_reset_mac_header(skb);
360 skb_reset_mac_len(skb);
361
362 hdr = eth_hdr(skb);
363 ether_addr_copy(hdr->h_source, ethh->addresses.eth_src);
364 ether_addr_copy(hdr->h_dest, ethh->addresses.eth_dst);
365 hdr->h_proto = skb->protocol;
366
367 skb_postpush_rcsum(skb, hdr, ETH_HLEN);
368
369 /* safe right before invalidate_flow_key */
370 key->mac_proto = MAC_PROTO_ETHERNET;
371 invalidate_flow_key(key);
372 return 0;
373 }
374
375 static int push_nsh(struct sk_buff *skb, struct sw_flow_key *key,
376 const struct nshhdr *nh)
377 {
378 int err;
379
380 err = nsh_push(skb, nh);
381 if (err)
382 return err;
383
384 /* safe right before invalidate_flow_key */
385 key->mac_proto = MAC_PROTO_NONE;
386 invalidate_flow_key(key);
387 return 0;
388 }
389
390 static int pop_nsh(struct sk_buff *skb, struct sw_flow_key *key)
391 {
392 int err;
393
394 err = nsh_pop(skb);
395 if (err)
396 return err;
397
398 /* safe right before invalidate_flow_key */
399 if (skb->protocol == htons(ETH_P_TEB))
400 key->mac_proto = MAC_PROTO_ETHERNET;
401 else
402 key->mac_proto = MAC_PROTO_NONE;
403 invalidate_flow_key(key);
404 return 0;
405 }
406
407 static void update_ip_l4_checksum(struct sk_buff *skb, struct iphdr *nh,
408 __be32 addr, __be32 new_addr)
409 {
410 int transport_len = skb->len - skb_transport_offset(skb);
411
412 if (nh->frag_off & htons(IP_OFFSET))
413 return;
414
415 if (nh->protocol == IPPROTO_TCP) {
416 if (likely(transport_len >= sizeof(struct tcphdr)))
417 inet_proto_csum_replace4(&tcp_hdr(skb)->check, skb,
418 addr, new_addr, true);
419 } else if (nh->protocol == IPPROTO_UDP) {
420 if (likely(transport_len >= sizeof(struct udphdr))) {
421 struct udphdr *uh = udp_hdr(skb);
422
423 if (uh->check || skb->ip_summed == CHECKSUM_PARTIAL) {
424 inet_proto_csum_replace4(&uh->check, skb,
425 addr, new_addr, true);
426 if (!uh->check)
427 uh->check = CSUM_MANGLED_0;
428 }
429 }
430 }
431 }
432
433 static void set_ip_addr(struct sk_buff *skb, struct iphdr *nh,
434 __be32 *addr, __be32 new_addr)
435 {
436 update_ip_l4_checksum(skb, nh, *addr, new_addr);
437 csum_replace4(&nh->check, *addr, new_addr);
438 skb_clear_hash(skb);
439 *addr = new_addr;
440 }
441
442 static void update_ipv6_checksum(struct sk_buff *skb, u8 l4_proto,
443 __be32 addr[4], const __be32 new_addr[4])
444 {
445 int transport_len = skb->len - skb_transport_offset(skb);
446
447 if (l4_proto == NEXTHDR_TCP) {
448 if (likely(transport_len >= sizeof(struct tcphdr)))
449 inet_proto_csum_replace16(&tcp_hdr(skb)->check, skb,
450 addr, new_addr, true);
451 } else if (l4_proto == NEXTHDR_UDP) {
452 if (likely(transport_len >= sizeof(struct udphdr))) {
453 struct udphdr *uh = udp_hdr(skb);
454
455 if (uh->check || skb->ip_summed == CHECKSUM_PARTIAL) {
456 inet_proto_csum_replace16(&uh->check, skb,
457 addr, new_addr, true);
458 if (!uh->check)
459 uh->check = CSUM_MANGLED_0;
460 }
461 }
462 } else if (l4_proto == NEXTHDR_ICMP) {
463 if (likely(transport_len >= sizeof(struct icmp6hdr)))
464 inet_proto_csum_replace16(&icmp6_hdr(skb)->icmp6_cksum,
465 skb, addr, new_addr, true);
466 }
467 }
468
469 static void mask_ipv6_addr(const __be32 old[4], const __be32 addr[4],
470 const __be32 mask[4], __be32 masked[4])
471 {
472 masked[0] = OVS_MASKED(old[0], addr[0], mask[0]);
473 masked[1] = OVS_MASKED(old[1], addr[1], mask[1]);
474 masked[2] = OVS_MASKED(old[2], addr[2], mask[2]);
475 masked[3] = OVS_MASKED(old[3], addr[3], mask[3]);
476 }
477
478 static void set_ipv6_addr(struct sk_buff *skb, u8 l4_proto,
479 __be32 addr[4], const __be32 new_addr[4],
480 bool recalculate_csum)
481 {
482 if (recalculate_csum)
483 update_ipv6_checksum(skb, l4_proto, addr, new_addr);
484
485 skb_clear_hash(skb);
486 memcpy(addr, new_addr, sizeof(__be32[4]));
487 }
488
489 static void set_ipv6_fl(struct ipv6hdr *nh, u32 fl, u32 mask)
490 {
491 /* Bits 21-24 are always unmasked, so this retains their values. */
492 OVS_SET_MASKED(nh->flow_lbl[0], (u8)(fl >> 16), (u8)(mask >> 16));
493 OVS_SET_MASKED(nh->flow_lbl[1], (u8)(fl >> 8), (u8)(mask >> 8));
494 OVS_SET_MASKED(nh->flow_lbl[2], (u8)fl, (u8)mask);
495 }
496
497 static void set_ip_ttl(struct sk_buff *skb, struct iphdr *nh, u8 new_ttl,
498 u8 mask)
499 {
500 new_ttl = OVS_MASKED(nh->ttl, new_ttl, mask);
501
502 csum_replace2(&nh->check, htons(nh->ttl << 8), htons(new_ttl << 8));
503 nh->ttl = new_ttl;
504 }
505
506 static int set_ipv4(struct sk_buff *skb, struct sw_flow_key *flow_key,
507 const struct ovs_key_ipv4 *key,
508 const struct ovs_key_ipv4 *mask)
509 {
510 struct iphdr *nh;
511 __be32 new_addr;
512 int err;
513
514 err = skb_ensure_writable(skb, skb_network_offset(skb) +
515 sizeof(struct iphdr));
516 if (unlikely(err))
517 return err;
518
519 nh = ip_hdr(skb);
520
521 /* Setting an IP addresses is typically only a side effect of
522 * matching on them in the current userspace implementation, so it
523 * makes sense to check if the value actually changed.
524 */
525 if (mask->ipv4_src) {
526 new_addr = OVS_MASKED(nh->saddr, key->ipv4_src, mask->ipv4_src);
527
528 if (unlikely(new_addr != nh->saddr)) {
529 set_ip_addr(skb, nh, &nh->saddr, new_addr);
530 flow_key->ipv4.addr.src = new_addr;
531 }
532 }
533 if (mask->ipv4_dst) {
534 new_addr = OVS_MASKED(nh->daddr, key->ipv4_dst, mask->ipv4_dst);
535
536 if (unlikely(new_addr != nh->daddr)) {
537 set_ip_addr(skb, nh, &nh->daddr, new_addr);
538 flow_key->ipv4.addr.dst = new_addr;
539 }
540 }
541 if (mask->ipv4_tos) {
542 ipv4_change_dsfield(nh, ~mask->ipv4_tos, key->ipv4_tos);
543 flow_key->ip.tos = nh->tos;
544 }
545 if (mask->ipv4_ttl) {
546 set_ip_ttl(skb, nh, key->ipv4_ttl, mask->ipv4_ttl);
547 flow_key->ip.ttl = nh->ttl;
548 }
549
550 return 0;
551 }
552
553 static bool is_ipv6_mask_nonzero(const __be32 addr[4])
554 {
555 return !!(addr[0] | addr[1] | addr[2] | addr[3]);
556 }
557
558 static int set_ipv6(struct sk_buff *skb, struct sw_flow_key *flow_key,
559 const struct ovs_key_ipv6 *key,
560 const struct ovs_key_ipv6 *mask)
561 {
562 struct ipv6hdr *nh;
563 int err;
564
565 err = skb_ensure_writable(skb, skb_network_offset(skb) +
566 sizeof(struct ipv6hdr));
567 if (unlikely(err))
568 return err;
569
570 nh = ipv6_hdr(skb);
571
572 /* Setting an IP addresses is typically only a side effect of
573 * matching on them in the current userspace implementation, so it
574 * makes sense to check if the value actually changed.
575 */
576 if (is_ipv6_mask_nonzero(mask->ipv6_src)) {
577 __be32 *saddr = (__be32 *)&nh->saddr;
578 __be32 masked[4];
579
580 mask_ipv6_addr(saddr, key->ipv6_src, mask->ipv6_src, masked);
581
582 if (unlikely(memcmp(saddr, masked, sizeof(masked)))) {
583 set_ipv6_addr(skb, flow_key->ip.proto, saddr, masked,
584 true);
585 memcpy(&flow_key->ipv6.addr.src, masked,
586 sizeof(flow_key->ipv6.addr.src));
587 }
588 }
589 if (is_ipv6_mask_nonzero(mask->ipv6_dst)) {
590 unsigned int offset = 0;
591 int flags = IP6_FH_F_SKIP_RH;
592 bool recalc_csum = true;
593 __be32 *daddr = (__be32 *)&nh->daddr;
594 __be32 masked[4];
595
596 mask_ipv6_addr(daddr, key->ipv6_dst, mask->ipv6_dst, masked);
597
598 if (unlikely(memcmp(daddr, masked, sizeof(masked)))) {
599 if (ipv6_ext_hdr(nh->nexthdr))
600 recalc_csum = (ipv6_find_hdr(skb, &offset,
601 NEXTHDR_ROUTING,
602 NULL, &flags)
603 != NEXTHDR_ROUTING);
604
605 set_ipv6_addr(skb, flow_key->ip.proto, daddr, masked,
606 recalc_csum);
607 memcpy(&flow_key->ipv6.addr.dst, masked,
608 sizeof(flow_key->ipv6.addr.dst));
609 }
610 }
611 if (mask->ipv6_tclass) {
612 ipv6_change_dsfield(nh, ~mask->ipv6_tclass, key->ipv6_tclass);
613 flow_key->ip.tos = ipv6_get_dsfield(nh);
614 }
615 if (mask->ipv6_label) {
616 set_ipv6_fl(nh, ntohl(key->ipv6_label),
617 ntohl(mask->ipv6_label));
618 flow_key->ipv6.label =
619 *(__be32 *)nh & htonl(IPV6_FLOWINFO_FLOWLABEL);
620 }
621 if (mask->ipv6_hlimit) {
622 OVS_SET_MASKED(nh->hop_limit, key->ipv6_hlimit,
623 mask->ipv6_hlimit);
624 flow_key->ip.ttl = nh->hop_limit;
625 }
626 return 0;
627 }
628
629 static int set_nsh(struct sk_buff *skb, struct sw_flow_key *flow_key,
630 const struct nlattr *a)
631 {
632 struct nshhdr *nh;
633 size_t length;
634 int err;
635 u8 flags;
636 u8 ttl;
637 int i;
638
639 struct ovs_key_nsh key;
640 struct ovs_key_nsh mask;
641
642 err = nsh_key_from_nlattr(a, &key, &mask);
643 if (err)
644 return err;
645
646 /* Make sure the NSH base header is there */
647 if (!pskb_may_pull(skb, skb_network_offset(skb) + NSH_BASE_HDR_LEN))
648 return -ENOMEM;
649
650 nh = nsh_hdr(skb);
651 length = nsh_hdr_len(nh);
652
653 /* Make sure the whole NSH header is there */
654 err = skb_ensure_writable(skb, skb_network_offset(skb) +
655 length);
656 if (unlikely(err))
657 return err;
658
659 nh = nsh_hdr(skb);
660 skb_postpull_rcsum(skb, nh, length);
661 flags = nsh_get_flags(nh);
662 flags = OVS_MASKED(flags, key.base.flags, mask.base.flags);
663 flow_key->nsh.base.flags = flags;
664 ttl = nsh_get_ttl(nh);
665 ttl = OVS_MASKED(ttl, key.base.ttl, mask.base.ttl);
666 flow_key->nsh.base.ttl = ttl;
667 nsh_set_flags_and_ttl(nh, flags, ttl);
668 nh->path_hdr = OVS_MASKED(nh->path_hdr, key.base.path_hdr,
669 mask.base.path_hdr);
670 flow_key->nsh.base.path_hdr = nh->path_hdr;
671 switch (nh->mdtype) {
672 case NSH_M_TYPE1:
673 for (i = 0; i < NSH_MD1_CONTEXT_SIZE; i++) {
674 nh->md1.context[i] =
675 OVS_MASKED(nh->md1.context[i], key.context[i],
676 mask.context[i]);
677 }
678 memcpy(flow_key->nsh.context, nh->md1.context,
679 sizeof(nh->md1.context));
680 break;
681 case NSH_M_TYPE2:
682 memset(flow_key->nsh.context, 0,
683 sizeof(flow_key->nsh.context));
684 break;
685 default:
686 return -EINVAL;
687 }
688 skb_postpush_rcsum(skb, nh, length);
689 return 0;
690 }
691
692 /* Must follow skb_ensure_writable() since that can move the skb data. */
693 static void set_tp_port(struct sk_buff *skb, __be16 *port,
694 __be16 new_port, __sum16 *check)
695 {
696 inet_proto_csum_replace2(check, skb, *port, new_port, false);
697 *port = new_port;
698 }
699
700 static int set_udp(struct sk_buff *skb, struct sw_flow_key *flow_key,
701 const struct ovs_key_udp *key,
702 const struct ovs_key_udp *mask)
703 {
704 struct udphdr *uh;
705 __be16 src, dst;
706 int err;
707
708 err = skb_ensure_writable(skb, skb_transport_offset(skb) +
709 sizeof(struct udphdr));
710 if (unlikely(err))
711 return err;
712
713 uh = udp_hdr(skb);
714 /* Either of the masks is non-zero, so do not bother checking them. */
715 src = OVS_MASKED(uh->source, key->udp_src, mask->udp_src);
716 dst = OVS_MASKED(uh->dest, key->udp_dst, mask->udp_dst);
717
718 if (uh->check && skb->ip_summed != CHECKSUM_PARTIAL) {
719 if (likely(src != uh->source)) {
720 set_tp_port(skb, &uh->source, src, &uh->check);
721 flow_key->tp.src = src;
722 }
723 if (likely(dst != uh->dest)) {
724 set_tp_port(skb, &uh->dest, dst, &uh->check);
725 flow_key->tp.dst = dst;
726 }
727
728 if (unlikely(!uh->check))
729 uh->check = CSUM_MANGLED_0;
730 } else {
731 uh->source = src;
732 uh->dest = dst;
733 flow_key->tp.src = src;
734 flow_key->tp.dst = dst;
735 }
736
737 skb_clear_hash(skb);
738
739 return 0;
740 }
741
742 static int set_tcp(struct sk_buff *skb, struct sw_flow_key *flow_key,
743 const struct ovs_key_tcp *key,
744 const struct ovs_key_tcp *mask)
745 {
746 struct tcphdr *th;
747 __be16 src, dst;
748 int err;
749
750 err = skb_ensure_writable(skb, skb_transport_offset(skb) +
751 sizeof(struct tcphdr));
752 if (unlikely(err))
753 return err;
754
755 th = tcp_hdr(skb);
756 src = OVS_MASKED(th->source, key->tcp_src, mask->tcp_src);
757 if (likely(src != th->source)) {
758 set_tp_port(skb, &th->source, src, &th->check);
759 flow_key->tp.src = src;
760 }
761 dst = OVS_MASKED(th->dest, key->tcp_dst, mask->tcp_dst);
762 if (likely(dst != th->dest)) {
763 set_tp_port(skb, &th->dest, dst, &th->check);
764 flow_key->tp.dst = dst;
765 }
766 skb_clear_hash(skb);
767
768 return 0;
769 }
770
771 static int set_sctp(struct sk_buff *skb, struct sw_flow_key *flow_key,
772 const struct ovs_key_sctp *key,
773 const struct ovs_key_sctp *mask)
774 {
775 unsigned int sctphoff = skb_transport_offset(skb);
776 struct sctphdr *sh;
777 __le32 old_correct_csum, new_csum, old_csum;
778 int err;
779
780 err = skb_ensure_writable(skb, sctphoff + sizeof(struct sctphdr));
781 if (unlikely(err))
782 return err;
783
784 sh = sctp_hdr(skb);
785 old_csum = sh->checksum;
786 old_correct_csum = sctp_compute_cksum(skb, sctphoff);
787
788 sh->source = OVS_MASKED(sh->source, key->sctp_src, mask->sctp_src);
789 sh->dest = OVS_MASKED(sh->dest, key->sctp_dst, mask->sctp_dst);
790
791 new_csum = sctp_compute_cksum(skb, sctphoff);
792
793 /* Carry any checksum errors through. */
794 sh->checksum = old_csum ^ old_correct_csum ^ new_csum;
795
796 skb_clear_hash(skb);
797 flow_key->tp.src = sh->source;
798 flow_key->tp.dst = sh->dest;
799
800 return 0;
801 }
802
803 static int ovs_vport_output(struct net *net, struct sock *sk, struct sk_buff *skb)
804 {
805 struct ovs_frag_data *data = this_cpu_ptr(&ovs_frag_data_storage);
806 struct vport *vport = data->vport;
807
808 if (skb_cow_head(skb, data->l2_len) < 0) {
809 kfree_skb(skb);
810 return -ENOMEM;
811 }
812
813 __skb_dst_copy(skb, data->dst);
814 *OVS_CB(skb) = data->cb;
815 skb->inner_protocol = data->inner_protocol;
816 if (data->vlan_tci & VLAN_CFI_MASK)
817 __vlan_hwaccel_put_tag(skb, data->vlan_proto, data->vlan_tci & ~VLAN_CFI_MASK);
818 else
819 __vlan_hwaccel_clear_tag(skb);
820
821 /* Reconstruct the MAC header. */
822 skb_push(skb, data->l2_len);
823 memcpy(skb->data, &data->l2_data, data->l2_len);
824 skb_postpush_rcsum(skb, skb->data, data->l2_len);
825 skb_reset_mac_header(skb);
826
827 if (eth_p_mpls(skb->protocol)) {
828 skb->inner_network_header = skb->network_header;
829 skb_set_network_header(skb, data->network_offset);
830 skb_reset_mac_len(skb);
831 }
832
833 ovs_vport_send(vport, skb, data->mac_proto);
834 return 0;
835 }
836
837 static unsigned int
838 ovs_dst_get_mtu(const struct dst_entry *dst)
839 {
840 return dst->dev->mtu;
841 }
842
843 static struct dst_ops ovs_dst_ops = {
844 .family = AF_UNSPEC,
845 .mtu = ovs_dst_get_mtu,
846 };
847
848 /* prepare_frag() is called once per (larger-than-MTU) frame; its inverse is
849 * ovs_vport_output(), which is called once per fragmented packet.
850 */
851 static void prepare_frag(struct vport *vport, struct sk_buff *skb,
852 u16 orig_network_offset, u8 mac_proto)
853 {
854 unsigned int hlen = skb_network_offset(skb);
855 struct ovs_frag_data *data;
856
857 data = this_cpu_ptr(&ovs_frag_data_storage);
858 data->dst = skb->_skb_refdst;
859 data->vport = vport;
860 data->cb = *OVS_CB(skb);
861 data->inner_protocol = skb->inner_protocol;
862 data->network_offset = orig_network_offset;
863 if (skb_vlan_tag_present(skb))
864 data->vlan_tci = skb_vlan_tag_get(skb) | VLAN_CFI_MASK;
865 else
866 data->vlan_tci = 0;
867 data->vlan_proto = skb->vlan_proto;
868 data->mac_proto = mac_proto;
869 data->l2_len = hlen;
870 memcpy(&data->l2_data, skb->data, hlen);
871
872 memset(IPCB(skb), 0, sizeof(struct inet_skb_parm));
873 skb_pull(skb, hlen);
874 }
875
876 static void ovs_fragment(struct net *net, struct vport *vport,
877 struct sk_buff *skb, u16 mru,
878 struct sw_flow_key *key)
879 {
880 u16 orig_network_offset = 0;
881
882 if (eth_p_mpls(skb->protocol)) {
883 orig_network_offset = skb_network_offset(skb);
884 skb->network_header = skb->inner_network_header;
885 }
886
887 if (skb_network_offset(skb) > MAX_L2_LEN) {
888 OVS_NLERR(1, "L2 header too long to fragment");
889 goto err;
890 }
891
892 if (key->eth.type == htons(ETH_P_IP)) {
893 struct dst_entry ovs_dst;
894 unsigned long orig_dst;
895
896 prepare_frag(vport, skb, orig_network_offset,
897 ovs_key_mac_proto(key));
898 dst_init(&ovs_dst, &ovs_dst_ops, NULL, 1,
899 DST_OBSOLETE_NONE, DST_NOCOUNT);
900 ovs_dst.dev = vport->dev;
901
902 orig_dst = skb->_skb_refdst;
903 skb_dst_set_noref(skb, &ovs_dst);
904 IPCB(skb)->frag_max_size = mru;
905
906 ip_do_fragment(net, skb->sk, skb, ovs_vport_output);
907 refdst_drop(orig_dst);
908 } else if (key->eth.type == htons(ETH_P_IPV6)) {
909 const struct nf_ipv6_ops *v6ops = nf_get_ipv6_ops();
910 unsigned long orig_dst;
911 struct rt6_info ovs_rt;
912
913 if (!v6ops)
914 goto err;
915
916 prepare_frag(vport, skb, orig_network_offset,
917 ovs_key_mac_proto(key));
918 memset(&ovs_rt, 0, sizeof(ovs_rt));
919 dst_init(&ovs_rt.dst, &ovs_dst_ops, NULL, 1,
920 DST_OBSOLETE_NONE, DST_NOCOUNT);
921 ovs_rt.dst.dev = vport->dev;
922
923 orig_dst = skb->_skb_refdst;
924 skb_dst_set_noref(skb, &ovs_rt.dst);
925 IP6CB(skb)->frag_max_size = mru;
926
927 v6ops->fragment(net, skb->sk, skb, ovs_vport_output);
928 refdst_drop(orig_dst);
929 } else {
930 WARN_ONCE(1, "Failed fragment ->%s: eth=%04x, MRU=%d, MTU=%d.",
931 ovs_vport_name(vport), ntohs(key->eth.type), mru,
932 vport->dev->mtu);
933 goto err;
934 }
935
936 return;
937 err:
938 kfree_skb(skb);
939 }
940
941 static void do_output(struct datapath *dp, struct sk_buff *skb, int out_port,
942 struct sw_flow_key *key)
943 {
944 struct vport *vport = ovs_vport_rcu(dp, out_port);
945
946 if (likely(vport)) {
947 u16 mru = OVS_CB(skb)->mru;
948 u32 cutlen = OVS_CB(skb)->cutlen;
949
950 if (unlikely(cutlen > 0)) {
951 if (skb->len - cutlen > ovs_mac_header_len(key))
952 pskb_trim(skb, skb->len - cutlen);
953 else
954 pskb_trim(skb, ovs_mac_header_len(key));
955 }
956
957 if (likely(!mru ||
958 (skb->len <= mru + vport->dev->hard_header_len))) {
959 ovs_vport_send(vport, skb, ovs_key_mac_proto(key));
960 } else if (mru <= vport->dev->mtu) {
961 struct net *net = read_pnet(&dp->net);
962
963 ovs_fragment(net, vport, skb, mru, key);
964 } else {
965 kfree_skb(skb);
966 }
967 } else {
968 kfree_skb(skb);
969 }
970 }
971
972 static int output_userspace(struct datapath *dp, struct sk_buff *skb,
973 struct sw_flow_key *key, const struct nlattr *attr,
974 const struct nlattr *actions, int actions_len,
975 uint32_t cutlen)
976 {
977 struct dp_upcall_info upcall;
978 const struct nlattr *a;
979 int rem;
980
981 memset(&upcall, 0, sizeof(upcall));
982 upcall.cmd = OVS_PACKET_CMD_ACTION;
983 upcall.mru = OVS_CB(skb)->mru;
984
985 for (a = nla_data(attr), rem = nla_len(attr); rem > 0;
986 a = nla_next(a, &rem)) {
987 switch (nla_type(a)) {
988 case OVS_USERSPACE_ATTR_USERDATA:
989 upcall.userdata = a;
990 break;
991
992 case OVS_USERSPACE_ATTR_PID:
993 upcall.portid = nla_get_u32(a);
994 break;
995
996 case OVS_USERSPACE_ATTR_EGRESS_TUN_PORT: {
997 /* Get out tunnel info. */
998 struct vport *vport;
999
1000 vport = ovs_vport_rcu(dp, nla_get_u32(a));
1001 if (vport) {
1002 int err;
1003
1004 err = dev_fill_metadata_dst(vport->dev, skb);
1005 if (!err)
1006 upcall.egress_tun_info = skb_tunnel_info(skb);
1007 }
1008
1009 break;
1010 }
1011
1012 case OVS_USERSPACE_ATTR_ACTIONS: {
1013 /* Include actions. */
1014 upcall.actions = actions;
1015 upcall.actions_len = actions_len;
1016 break;
1017 }
1018
1019 } /* End of switch. */
1020 }
1021
1022 return ovs_dp_upcall(dp, skb, key, &upcall, cutlen);
1023 }
1024
1025 /* When 'last' is true, sample() should always consume the 'skb'.
1026 * Otherwise, sample() should keep 'skb' intact regardless what
1027 * actions are executed within sample().
1028 */
1029 static int sample(struct datapath *dp, struct sk_buff *skb,
1030 struct sw_flow_key *key, const struct nlattr *attr,
1031 bool last)
1032 {
1033 struct nlattr *actions;
1034 struct nlattr *sample_arg;
1035 int rem = nla_len(attr);
1036 const struct sample_arg *arg;
1037 bool clone_flow_key;
1038
1039 /* The first action is always 'OVS_SAMPLE_ATTR_ARG'. */
1040 sample_arg = nla_data(attr);
1041 arg = nla_data(sample_arg);
1042 actions = nla_next(sample_arg, &rem);
1043
1044 if ((arg->probability != U32_MAX) &&
1045 (!arg->probability || prandom_u32() > arg->probability)) {
1046 if (last)
1047 consume_skb(skb);
1048 return 0;
1049 }
1050
1051 clone_flow_key = !arg->exec;
1052 return clone_execute(dp, skb, key, 0, actions, rem, last,
1053 clone_flow_key);
1054 }
1055
1056 /* When 'last' is true, clone() should always consume the 'skb'.
1057 * Otherwise, clone() should keep 'skb' intact regardless what
1058 * actions are executed within clone().
1059 */
1060 static int clone(struct datapath *dp, struct sk_buff *skb,
1061 struct sw_flow_key *key, const struct nlattr *attr,
1062 bool last)
1063 {
1064 struct nlattr *actions;
1065 struct nlattr *clone_arg;
1066 int rem = nla_len(attr);
1067 bool dont_clone_flow_key;
1068
1069 /* The first action is always 'OVS_CLONE_ATTR_ARG'. */
1070 clone_arg = nla_data(attr);
1071 dont_clone_flow_key = nla_get_u32(clone_arg);
1072 actions = nla_next(clone_arg, &rem);
1073
1074 return clone_execute(dp, skb, key, 0, actions, rem, last,
1075 !dont_clone_flow_key);
1076 }
1077
1078 static void execute_hash(struct sk_buff *skb, struct sw_flow_key *key,
1079 const struct nlattr *attr)
1080 {
1081 struct ovs_action_hash *hash_act = nla_data(attr);
1082 u32 hash = 0;
1083
1084 /* OVS_HASH_ALG_L4 is the only possible hash algorithm. */
1085 hash = skb_get_hash(skb);
1086 hash = jhash_1word(hash, hash_act->hash_basis);
1087 if (!hash)
1088 hash = 0x1;
1089
1090 key->ovs_flow_hash = hash;
1091 }
1092
1093 static int execute_set_action(struct sk_buff *skb,
1094 struct sw_flow_key *flow_key,
1095 const struct nlattr *a)
1096 {
1097 /* Only tunnel set execution is supported without a mask. */
1098 if (nla_type(a) == OVS_KEY_ATTR_TUNNEL_INFO) {
1099 struct ovs_tunnel_info *tun = nla_data(a);
1100
1101 skb_dst_drop(skb);
1102 dst_hold((struct dst_entry *)tun->tun_dst);
1103 skb_dst_set(skb, (struct dst_entry *)tun->tun_dst);
1104 return 0;
1105 }
1106
1107 return -EINVAL;
1108 }
1109
1110 /* Mask is at the midpoint of the data. */
1111 #define get_mask(a, type) ((const type)nla_data(a) + 1)
1112
1113 static int execute_masked_set_action(struct sk_buff *skb,
1114 struct sw_flow_key *flow_key,
1115 const struct nlattr *a)
1116 {
1117 int err = 0;
1118
1119 switch (nla_type(a)) {
1120 case OVS_KEY_ATTR_PRIORITY:
1121 OVS_SET_MASKED(skb->priority, nla_get_u32(a),
1122 *get_mask(a, u32 *));
1123 flow_key->phy.priority = skb->priority;
1124 break;
1125
1126 case OVS_KEY_ATTR_SKB_MARK:
1127 OVS_SET_MASKED(skb->mark, nla_get_u32(a), *get_mask(a, u32 *));
1128 flow_key->phy.skb_mark = skb->mark;
1129 break;
1130
1131 case OVS_KEY_ATTR_TUNNEL_INFO:
1132 /* Masked data not supported for tunnel. */
1133 err = -EINVAL;
1134 break;
1135
1136 case OVS_KEY_ATTR_ETHERNET:
1137 err = set_eth_addr(skb, flow_key, nla_data(a),
1138 get_mask(a, struct ovs_key_ethernet *));
1139 break;
1140
1141 case OVS_KEY_ATTR_NSH:
1142 err = set_nsh(skb, flow_key, a);
1143 break;
1144
1145 case OVS_KEY_ATTR_IPV4:
1146 err = set_ipv4(skb, flow_key, nla_data(a),
1147 get_mask(a, struct ovs_key_ipv4 *));
1148 break;
1149
1150 case OVS_KEY_ATTR_IPV6:
1151 err = set_ipv6(skb, flow_key, nla_data(a),
1152 get_mask(a, struct ovs_key_ipv6 *));
1153 break;
1154
1155 case OVS_KEY_ATTR_TCP:
1156 err = set_tcp(skb, flow_key, nla_data(a),
1157 get_mask(a, struct ovs_key_tcp *));
1158 break;
1159
1160 case OVS_KEY_ATTR_UDP:
1161 err = set_udp(skb, flow_key, nla_data(a),
1162 get_mask(a, struct ovs_key_udp *));
1163 break;
1164
1165 case OVS_KEY_ATTR_SCTP:
1166 err = set_sctp(skb, flow_key, nla_data(a),
1167 get_mask(a, struct ovs_key_sctp *));
1168 break;
1169
1170 case OVS_KEY_ATTR_MPLS:
1171 err = set_mpls(skb, flow_key, nla_data(a), get_mask(a,
1172 __be32 *));
1173 break;
1174
1175 case OVS_KEY_ATTR_CT_STATE:
1176 case OVS_KEY_ATTR_CT_ZONE:
1177 case OVS_KEY_ATTR_CT_MARK:
1178 case OVS_KEY_ATTR_CT_LABELS:
1179 case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4:
1180 case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6:
1181 err = -EINVAL;
1182 break;
1183 }
1184
1185 return err;
1186 }
1187
1188 static int execute_recirc(struct datapath *dp, struct sk_buff *skb,
1189 struct sw_flow_key *key,
1190 const struct nlattr *a, bool last)
1191 {
1192 u32 recirc_id;
1193
1194 if (!is_flow_key_valid(key)) {
1195 int err;
1196
1197 err = ovs_flow_key_update(skb, key);
1198 if (err)
1199 return err;
1200 }
1201 BUG_ON(!is_flow_key_valid(key));
1202
1203 recirc_id = nla_get_u32(a);
1204 return clone_execute(dp, skb, key, recirc_id, NULL, 0, last, true);
1205 }
1206
1207 static int execute_check_pkt_len(struct datapath *dp, struct sk_buff *skb,
1208 struct sw_flow_key *key,
1209 const struct nlattr *attr, bool last)
1210 {
1211 const struct nlattr *actions, *cpl_arg;
1212 const struct check_pkt_len_arg *arg;
1213 int rem = nla_len(attr);
1214 bool clone_flow_key;
1215
1216 /* The first netlink attribute in 'attr' is always
1217 * 'OVS_CHECK_PKT_LEN_ATTR_ARG'.
1218 */
1219 cpl_arg = nla_data(attr);
1220 arg = nla_data(cpl_arg);
1221
1222 if (skb->len <= arg->pkt_len) {
1223 /* Second netlink attribute in 'attr' is always
1224 * 'OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL'.
1225 */
1226 actions = nla_next(cpl_arg, &rem);
1227 clone_flow_key = !arg->exec_for_lesser_equal;
1228 } else {
1229 /* Third netlink attribute in 'attr' is always
1230 * 'OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER'.
1231 */
1232 actions = nla_next(cpl_arg, &rem);
1233 actions = nla_next(actions, &rem);
1234 clone_flow_key = !arg->exec_for_greater;
1235 }
1236
1237 return clone_execute(dp, skb, key, 0, nla_data(actions),
1238 nla_len(actions), last, clone_flow_key);
1239 }
1240
1241 /* Execute a list of actions against 'skb'. */
1242 static int do_execute_actions(struct datapath *dp, struct sk_buff *skb,
1243 struct sw_flow_key *key,
1244 const struct nlattr *attr, int len)
1245 {
1246 const struct nlattr *a;
1247 int rem;
1248
1249 for (a = attr, rem = len; rem > 0;
1250 a = nla_next(a, &rem)) {
1251 int err = 0;
1252
1253 switch (nla_type(a)) {
1254 case OVS_ACTION_ATTR_OUTPUT: {
1255 int port = nla_get_u32(a);
1256 struct sk_buff *clone;
1257
1258 /* Every output action needs a separate clone
1259 * of 'skb', In case the output action is the
1260 * last action, cloning can be avoided.
1261 */
1262 if (nla_is_last(a, rem)) {
1263 do_output(dp, skb, port, key);
1264 /* 'skb' has been used for output.
1265 */
1266 return 0;
1267 }
1268
1269 clone = skb_clone(skb, GFP_ATOMIC);
1270 if (clone)
1271 do_output(dp, clone, port, key);
1272 OVS_CB(skb)->cutlen = 0;
1273 break;
1274 }
1275
1276 case OVS_ACTION_ATTR_TRUNC: {
1277 struct ovs_action_trunc *trunc = nla_data(a);
1278
1279 if (skb->len > trunc->max_len)
1280 OVS_CB(skb)->cutlen = skb->len - trunc->max_len;
1281 break;
1282 }
1283
1284 case OVS_ACTION_ATTR_USERSPACE:
1285 output_userspace(dp, skb, key, a, attr,
1286 len, OVS_CB(skb)->cutlen);
1287 OVS_CB(skb)->cutlen = 0;
1288 break;
1289
1290 case OVS_ACTION_ATTR_HASH:
1291 execute_hash(skb, key, a);
1292 break;
1293
1294 case OVS_ACTION_ATTR_PUSH_MPLS:
1295 err = push_mpls(skb, key, nla_data(a));
1296 break;
1297
1298 case OVS_ACTION_ATTR_POP_MPLS:
1299 err = pop_mpls(skb, key, nla_get_be16(a));
1300 break;
1301
1302 case OVS_ACTION_ATTR_PUSH_VLAN:
1303 err = push_vlan(skb, key, nla_data(a));
1304 break;
1305
1306 case OVS_ACTION_ATTR_POP_VLAN:
1307 err = pop_vlan(skb, key);
1308 break;
1309
1310 case OVS_ACTION_ATTR_RECIRC: {
1311 bool last = nla_is_last(a, rem);
1312
1313 err = execute_recirc(dp, skb, key, a, last);
1314 if (last) {
1315 /* If this is the last action, the skb has
1316 * been consumed or freed.
1317 * Return immediately.
1318 */
1319 return err;
1320 }
1321 break;
1322 }
1323
1324 case OVS_ACTION_ATTR_SET:
1325 err = execute_set_action(skb, key, nla_data(a));
1326 break;
1327
1328 case OVS_ACTION_ATTR_SET_MASKED:
1329 case OVS_ACTION_ATTR_SET_TO_MASKED:
1330 err = execute_masked_set_action(skb, key, nla_data(a));
1331 break;
1332
1333 case OVS_ACTION_ATTR_SAMPLE: {
1334 bool last = nla_is_last(a, rem);
1335
1336 err = sample(dp, skb, key, a, last);
1337 if (last)
1338 return err;
1339
1340 break;
1341 }
1342
1343 case OVS_ACTION_ATTR_CT:
1344 if (!is_flow_key_valid(key)) {
1345 err = ovs_flow_key_update(skb, key);
1346 if (err)
1347 return err;
1348 }
1349
1350 err = ovs_ct_execute(ovs_dp_get_net(dp), skb, key,
1351 nla_data(a));
1352
1353 /* Hide stolen IP fragments from user space. */
1354 if (err)
1355 return err == -EINPROGRESS ? 0 : err;
1356 break;
1357
1358 case OVS_ACTION_ATTR_CT_CLEAR:
1359 err = ovs_ct_clear(skb, key);
1360 break;
1361
1362 case OVS_ACTION_ATTR_PUSH_ETH:
1363 err = push_eth(skb, key, nla_data(a));
1364 break;
1365
1366 case OVS_ACTION_ATTR_POP_ETH:
1367 err = pop_eth(skb, key);
1368 break;
1369
1370 case OVS_ACTION_ATTR_PUSH_NSH: {
1371 u8 buffer[NSH_HDR_MAX_LEN];
1372 struct nshhdr *nh = (struct nshhdr *)buffer;
1373
1374 err = nsh_hdr_from_nlattr(nla_data(a), nh,
1375 NSH_HDR_MAX_LEN);
1376 if (unlikely(err))
1377 break;
1378 err = push_nsh(skb, key, nh);
1379 break;
1380 }
1381
1382 case OVS_ACTION_ATTR_POP_NSH:
1383 err = pop_nsh(skb, key);
1384 break;
1385
1386 case OVS_ACTION_ATTR_METER:
1387 if (ovs_meter_execute(dp, skb, key, nla_get_u32(a))) {
1388 consume_skb(skb);
1389 return 0;
1390 }
1391 break;
1392
1393 case OVS_ACTION_ATTR_CLONE: {
1394 bool last = nla_is_last(a, rem);
1395
1396 err = clone(dp, skb, key, a, last);
1397 if (last)
1398 return err;
1399
1400 break;
1401 }
1402
1403 case OVS_ACTION_ATTR_CHECK_PKT_LEN: {
1404 bool last = nla_is_last(a, rem);
1405
1406 err = execute_check_pkt_len(dp, skb, key, a, last);
1407 if (last)
1408 return err;
1409
1410 break;
1411 }
1412 }
1413
1414 if (unlikely(err)) {
1415 kfree_skb(skb);
1416 return err;
1417 }
1418 }
1419
1420 consume_skb(skb);
1421 return 0;
1422 }
1423
1424 /* Execute the actions on the clone of the packet. The effect of the
1425 * execution does not affect the original 'skb' nor the original 'key'.
1426 *
1427 * The execution may be deferred in case the actions can not be executed
1428 * immediately.
1429 */
1430 static int clone_execute(struct datapath *dp, struct sk_buff *skb,
1431 struct sw_flow_key *key, u32 recirc_id,
1432 const struct nlattr *actions, int len,
1433 bool last, bool clone_flow_key)
1434 {
1435 struct deferred_action *da;
1436 struct sw_flow_key *clone;
1437
1438 skb = last ? skb : skb_clone(skb, GFP_ATOMIC);
1439 if (!skb) {
1440 /* Out of memory, skip this action.
1441 */
1442 return 0;
1443 }
1444
1445 /* When clone_flow_key is false, the 'key' will not be change
1446 * by the actions, then the 'key' can be used directly.
1447 * Otherwise, try to clone key from the next recursion level of
1448 * 'flow_keys'. If clone is successful, execute the actions
1449 * without deferring.
1450 */
1451 clone = clone_flow_key ? clone_key(key) : key;
1452 if (clone) {
1453 int err = 0;
1454
1455 if (actions) { /* Sample action */
1456 if (clone_flow_key)
1457 __this_cpu_inc(exec_actions_level);
1458
1459 err = do_execute_actions(dp, skb, clone,
1460 actions, len);
1461
1462 if (clone_flow_key)
1463 __this_cpu_dec(exec_actions_level);
1464 } else { /* Recirc action */
1465 clone->recirc_id = recirc_id;
1466 ovs_dp_process_packet(skb, clone);
1467 }
1468 return err;
1469 }
1470
1471 /* Out of 'flow_keys' space. Defer actions */
1472 da = add_deferred_actions(skb, key, actions, len);
1473 if (da) {
1474 if (!actions) { /* Recirc action */
1475 key = &da->pkt_key;
1476 key->recirc_id = recirc_id;
1477 }
1478 } else {
1479 /* Out of per CPU action FIFO space. Drop the 'skb' and
1480 * log an error.
1481 */
1482 kfree_skb(skb);
1483
1484 if (net_ratelimit()) {
1485 if (actions) { /* Sample action */
1486 pr_warn("%s: deferred action limit reached, drop sample action\n",
1487 ovs_dp_name(dp));
1488 } else { /* Recirc action */
1489 pr_warn("%s: deferred action limit reached, drop recirc action\n",
1490 ovs_dp_name(dp));
1491 }
1492 }
1493 }
1494 return 0;
1495 }
1496
1497 static void process_deferred_actions(struct datapath *dp)
1498 {
1499 struct action_fifo *fifo = this_cpu_ptr(action_fifos);
1500
1501 /* Do not touch the FIFO in case there is no deferred actions. */
1502 if (action_fifo_is_empty(fifo))
1503 return;
1504
1505 /* Finishing executing all deferred actions. */
1506 do {
1507 struct deferred_action *da = action_fifo_get(fifo);
1508 struct sk_buff *skb = da->skb;
1509 struct sw_flow_key *key = &da->pkt_key;
1510 const struct nlattr *actions = da->actions;
1511 int actions_len = da->actions_len;
1512
1513 if (actions)
1514 do_execute_actions(dp, skb, key, actions, actions_len);
1515 else
1516 ovs_dp_process_packet(skb, key);
1517 } while (!action_fifo_is_empty(fifo));
1518
1519 /* Reset FIFO for the next packet. */
1520 action_fifo_init(fifo);
1521 }
1522
1523 /* Execute a list of actions against 'skb'. */
1524 int ovs_execute_actions(struct datapath *dp, struct sk_buff *skb,
1525 const struct sw_flow_actions *acts,
1526 struct sw_flow_key *key)
1527 {
1528 int err, level;
1529
1530 level = __this_cpu_inc_return(exec_actions_level);
1531 if (unlikely(level > OVS_RECURSION_LIMIT)) {
1532 net_crit_ratelimited("ovs: recursion limit reached on datapath %s, probable configuration error\n",
1533 ovs_dp_name(dp));
1534 kfree_skb(skb);
1535 err = -ENETDOWN;
1536 goto out;
1537 }
1538
1539 OVS_CB(skb)->acts_origlen = acts->orig_len;
1540 err = do_execute_actions(dp, skb, key,
1541 acts->actions, acts->actions_len);
1542
1543 if (level == 1)
1544 process_deferred_actions(dp);
1545
1546 out:
1547 __this_cpu_dec(exec_actions_level);
1548 return err;
1549 }
1550
1551 int action_fifos_init(void)
1552 {
1553 action_fifos = alloc_percpu(struct action_fifo);
1554 if (!action_fifos)
1555 return -ENOMEM;
1556
1557 flow_keys = alloc_percpu(struct action_flow_keys);
1558 if (!flow_keys) {
1559 free_percpu(action_fifos);
1560 return -ENOMEM;
1561 }
1562
1563 return 0;
1564 }
1565
1566 void action_fifos_exit(void)
1567 {
1568 free_percpu(action_fifos);
1569 free_percpu(flow_keys);
1570 }
Linux with added FPC-III support