?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
code style scripts/checkpatch.pl (linux-3.9-rc1) formatting
[linux-2.6.34.14-moxart.git] / net / packet / af_packet.c
blob4096a66f6379e30d5e84cd0abddb3b4ea6ea8c79
1 /*
2 * INET An implementation of the TCP/IP protocol suite for the LINUX
3 * operating system. INET is implemented using the BSD Socket
4 * interface as the means of communication with the user level.
5 *
6 * PACKET - implements raw packet sockets.
7 *
8 * Authors: Ross Biro
9 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
10 * Alan Cox, <gw4pts@gw4pts.ampr.org>
11 *
12 * Fixes:
13 * Alan Cox : verify_area() now used correctly
14 * Alan Cox : new skbuff lists, look ma no backlogs!
15 * Alan Cox : tidied skbuff lists.
16 * Alan Cox : Now uses generic datagram routines I
17 * added. Also fixed the peek/read crash
18 * from all old Linux datagram code.
19 * Alan Cox : Uses the improved datagram code.
20 * Alan Cox : Added NULL's for socket options.
21 * Alan Cox : Re-commented the code.
22 * Alan Cox : Use new kernel side addressing
23 * Rob Janssen : Correct MTU usage.
24 * Dave Platt : Counter leaks caused by incorrect
25 * interrupt locking and some slightly
26 * dubious gcc output. Can you read
27 * compiler: it said _VOLATILE_
28 * Richard Kooijman : Timestamp fixes.
29 * Alan Cox : New buffers. Use sk->mac.raw.
30 * Alan Cox : sendmsg/recvmsg support.
31 * Alan Cox : Protocol setting support
32 * Alexey Kuznetsov : Untied from IPv4 stack.
33 * Cyrus Durgin : Fixed kerneld for kmod.
34 * Michal Ostrowski : Module initialization cleanup.
35 * Ulises Alonso : Frame number limit removal and
36 * packet_set_ring memory leak.
37 * Eric Biederman : Allow for > 8 byte hardware addresses.
38 * The convention is that longer addresses
39 * will simply extend the hardware address
40 * byte arrays at the end of sockaddr_ll
41 * and packet_mreq.
42 * Johann Baudy : Added TX RING.
43 *
44 * This program is free software; you can redistribute it and/or
45 * modify it under the terms of the GNU General Public License
46 * as published by the Free Software Foundation; either version
47 * 2 of the License, or (at your option) any later version.
48 *
49 */
50
51 #include <linux/types.h>
52 #include <linux/mm.h>
53 #include <linux/capability.h>
54 #include <linux/fcntl.h>
55 #include <linux/socket.h>
56 #include <linux/in.h>
57 #include <linux/inet.h>
58 #include <linux/netdevice.h>
59 #include <linux/if_packet.h>
60 #include <linux/wireless.h>
61 #include <linux/kernel.h>
62 #include <linux/kmod.h>
63 #include <linux/slab.h>
64 #include <net/net_namespace.h>
65 #include <net/ip.h>
66 #include <net/protocol.h>
67 #include <linux/skbuff.h>
68 #include <net/sock.h>
69 #include <linux/errno.h>
70 #include <linux/timer.h>
71 #include <asm/system.h>
72 #include <asm/uaccess.h>
73 #include <asm/ioctls.h>
74 #include <asm/page.h>
75 #include <asm/cacheflush.h>
76 #include <asm/io.h>
77 #include <linux/proc_fs.h>
78 #include <linux/seq_file.h>
79 #include <linux/poll.h>
80 #include <linux/module.h>
81 #include <linux/init.h>
82 #include <linux/mutex.h>
83 #include <linux/if_vlan.h>
84 #include <linux/virtio_net.h>
85
86 #ifdef CONFIG_INET
87 #include <net/inet_common.h>
88 #endif
89
90 /*
91 Assumptions:
92 - if device has no dev->hard_header routine, it adds and removes ll header
93 inside itself. In this case ll header is invisible outside of device,
94 but higher levels still should reserve dev->hard_header_len.
95 Some devices are enough clever to reallocate skb, when header
96 will not fit to reserved space (tunnel), another ones are silly
97 (PPP).
98 - packet socket receives packets with pulled ll header,
99 so that SOCK_RAW should push it back.
100
101 On receive:
102 -----------
103
104 Incoming, dev->hard_header!=NULL
105 mac_header -> ll header
106 data -> data
107
108 Outgoing, dev->hard_header!=NULL
109 mac_header -> ll header
110 data -> ll header
111
112 Incoming, dev->hard_header==NULL
113 mac_header -> UNKNOWN position. It is very likely, that it points to ll
114 header. PPP makes it, that is wrong, because introduce
115 assymetry between rx and tx paths.
116 data -> data
117
118 Outgoing, dev->hard_header==NULL
119 mac_header -> data. ll header is still not built!
120 data -> data
121
122 Resume
123 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
124
125
126 On transmit:
127 ------------
128
129 dev->hard_header != NULL
130 mac_header -> ll header
131 data -> ll header
132
133 dev->hard_header == NULL (ll header is added by device, we cannot control it)
134 mac_header -> data
135 data -> data
136
137 We should set nh.raw on output to correct posistion,
138 packet classifier depends on it.
139 */
140
141 /* Private packet socket structures. */
142
143 struct packet_mclist {
144 struct packet_mclist *next;
145 int ifindex;
146 int count;
147 unsigned short type;
148 unsigned short alen;
149 unsigned char addr[MAX_ADDR_LEN];
150 };
151 /* identical to struct packet_mreq except it has
152 * a longer address field.
153 */
154 struct packet_mreq_max {
155 int mr_ifindex;
156 unsigned short mr_type;
157 unsigned short mr_alen;
158 unsigned char mr_address[MAX_ADDR_LEN];
159 };
160
161 static int packet_set_ring(struct sock *sk, struct tpacket_req *req,
162 int closing, int tx_ring);
163
164 struct packet_ring_buffer {
165 char **pg_vec;
166 unsigned int head;
167 unsigned int frames_per_block;
168 unsigned int frame_size;
169 unsigned int frame_max;
170
171 unsigned int pg_vec_order;
172 unsigned int pg_vec_pages;
173 unsigned int pg_vec_len;
174
175 atomic_t pending;
176 };
177
178 struct packet_sock;
179 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg);
180
181 static void packet_flush_mclist(struct sock *sk);
182
183 struct packet_sock {
184 /* struct sock has to be the first member of packet_sock */
185 struct sock sk;
186 struct tpacket_stats stats;
187 struct packet_ring_buffer rx_ring;
188 struct packet_ring_buffer tx_ring;
189 int copy_thresh;
190 spinlock_t bind_lock;
191 struct mutex pg_vec_lock;
192 unsigned int running:1, /* prot_hook is attached*/
193 auxdata:1,
194 origdev:1,
195 has_vnet_hdr:1;
196 int ifindex; /* bound device */
197 __be16 num;
198 struct packet_mclist *mclist;
199 atomic_t mapped;
200 enum tpacket_versions tp_version;
201 unsigned int tp_hdrlen;
202 unsigned int tp_reserve;
203 unsigned int tp_loss:1;
204 struct packet_type prot_hook ____cacheline_aligned_in_smp;
205 };
206
207 struct packet_skb_cb {
208 unsigned int origlen;
209 union {
210 struct sockaddr_pkt pkt;
211 struct sockaddr_ll ll;
212 } sa;
213 };
214
215 #define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
216
217 static void __packet_set_status(struct packet_sock *po, void *frame, int status)
218 {
219 union {
220 struct tpacket_hdr *h1;
221 struct tpacket2_hdr *h2;
222 void *raw;
223 } h;
224
225 h.raw = frame;
226 switch (po->tp_version) {
227 case TPACKET_V1:
228 h.h1->tp_status = status;
229 flush_dcache_page(virt_to_page(&h.h1->tp_status));
230 break;
231 case TPACKET_V2:
232 h.h2->tp_status = status;
233 flush_dcache_page(virt_to_page(&h.h2->tp_status));
234 break;
235 default:
236 pr_err("TPACKET version not supported\n");
237 BUG();
238 }
239
240 smp_wmb();
241 }
242
243 static int __packet_get_status(struct packet_sock *po, void *frame)
244 {
245 union {
246 struct tpacket_hdr *h1;
247 struct tpacket2_hdr *h2;
248 void *raw;
249 } h;
250
251 smp_rmb();
252
253 h.raw = frame;
254 switch (po->tp_version) {
255 case TPACKET_V1:
256 flush_dcache_page(virt_to_page(&h.h1->tp_status));
257 return h.h1->tp_status;
258 case TPACKET_V2:
259 flush_dcache_page(virt_to_page(&h.h2->tp_status));
260 return h.h2->tp_status;
261 default:
262 pr_err("TPACKET version not supported\n");
263 BUG();
264 return 0;
265 }
266 }
267
268 static void *packet_lookup_frame(struct packet_sock *po,
269 struct packet_ring_buffer *rb,
270 unsigned int position,
271 int status)
272 {
273 unsigned int pg_vec_pos, frame_offset;
274 union {
275 struct tpacket_hdr *h1;
276 struct tpacket2_hdr *h2;
277 void *raw;
278 } h;
279
280 pg_vec_pos = position / rb->frames_per_block;
281 frame_offset = position % rb->frames_per_block;
282
283 h.raw = rb->pg_vec[pg_vec_pos] + (frame_offset * rb->frame_size);
284
285 if (status != __packet_get_status(po, h.raw))
286 return NULL;
287
288 return h.raw;
289 }
290
291 static inline void *packet_current_frame(struct packet_sock *po,
292 struct packet_ring_buffer *rb,
293 int status)
294 {
295 return packet_lookup_frame(po, rb, rb->head, status);
296 }
297
298 static inline void *packet_previous_frame(struct packet_sock *po,
299 struct packet_ring_buffer *rb,
300 int status)
301 {
302 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max;
303 return packet_lookup_frame(po, rb, previous, status);
304 }
305
306 static inline void packet_increment_head(struct packet_ring_buffer *buff)
307 {
308 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0;
309 }
310
311 static inline struct packet_sock *pkt_sk(struct sock *sk)
312 {
313 return (struct packet_sock *)sk;
314 }
315
316 static void packet_sock_destruct(struct sock *sk)
317 {
318 WARN_ON(atomic_read(&sk->sk_rmem_alloc));
319 WARN_ON(atomic_read(&sk->sk_wmem_alloc));
320
321 if (!sock_flag(sk, SOCK_DEAD)) {
322 pr_err("Attempt to release alive packet socket: %p\n", sk);
323 return;
324 }
325
326 sk_refcnt_debug_dec(sk);
327 }
328
329
330 static const struct proto_ops packet_ops;
331
332 static const struct proto_ops packet_ops_spkt;
333
334 static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
335 struct packet_type *pt, struct net_device *orig_dev)
336 {
337 struct sock *sk;
338 struct sockaddr_pkt *spkt;
339
340 /*
341 * When we registered the protocol we saved the socket in the data
342 * field for just this event.
343 */
344
345 sk = pt->af_packet_priv;
346
347 /*
348 * Yank back the headers [hope the device set this
349 * right or kerboom...]
350 *
351 * Incoming packets have ll header pulled,
352 * push it back.
353 *
354 * For outgoing ones skb->data == skb_mac_header(skb)
355 * so that this procedure is noop.
356 */
357
358 if (skb->pkt_type == PACKET_LOOPBACK)
359 goto out;
360
361 if (!net_eq(dev_net(dev), sock_net(sk)))
362 goto out;
363
364 skb = skb_share_check(skb, GFP_ATOMIC);
365 if (skb == NULL)
366 goto oom;
367
368 /* drop any routing info */
369 skb_dst_drop(skb);
370
371 /* drop conntrack reference */
372 nf_reset(skb);
373
374 spkt = &PACKET_SKB_CB(skb)->sa.pkt;
375
376 skb_push(skb, skb->data - skb_mac_header(skb));
377
378 /*
379 * The SOCK_PACKET socket receives _all_ frames.
380 */
381
382 spkt->spkt_family = dev->type;
383 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
384 spkt->spkt_protocol = skb->protocol;
385
386 /*
387 * Charge the memory to the socket. This is done specifically
388 * to prevent sockets using all the memory up.
389 */
390
391 if (sock_queue_rcv_skb(sk, skb) == 0)
392 return 0;
393
394 out:
395 kfree_skb(skb);
396 oom:
397 return 0;
398 }
399
400
401 /*
402 * Output a raw packet to a device layer. This bypasses all the other
403 * protocol layers and you must therefore supply it with a complete frame
404 */
405
406 static int packet_sendmsg_spkt(struct kiocb *iocb, struct socket *sock,
407 struct msghdr *msg, size_t len)
408 {
409 struct sock *sk = sock->sk;
410 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name;
411 struct sk_buff *skb = NULL;
412 struct net_device *dev;
413 __be16 proto = 0;
414 int err;
415
416 /*
417 * Get and verify the address.
418 */
419
420 if (saddr) {
421 if (msg->msg_namelen < sizeof(struct sockaddr))
422 return -EINVAL;
423 if (msg->msg_namelen == sizeof(struct sockaddr_pkt))
424 proto = saddr->spkt_protocol;
425 } else
426 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */
427
428 /*
429 * Find the device first to size check it
430 */
431
432 saddr->spkt_device[13] = 0;
433 retry:
434 rcu_read_lock();
435 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device);
436 err = -ENODEV;
437 if (dev == NULL)
438 goto out_unlock;
439
440 err = -ENETDOWN;
441 if (!(dev->flags & IFF_UP))
442 goto out_unlock;
443
444 /*
445 * You may not queue a frame bigger than the mtu. This is the lowest level
446 * raw protocol and you must do your own fragmentation at this level.
447 */
448
449 err = -EMSGSIZE;
450 if (len > dev->mtu + dev->hard_header_len)
451 goto out_unlock;
452
453 if (!skb) {
454 size_t reserved = LL_RESERVED_SPACE(dev);
455 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0;
456
457 rcu_read_unlock();
458 skb = sock_wmalloc(sk, len + reserved, 0, GFP_KERNEL);
459 if (skb == NULL)
460 return -ENOBUFS;
461 /* FIXME: Save some space for broken drivers that write a hard
462 * header at transmission time by themselves. PPP is the notable
463 * one here. This should really be fixed at the driver level.
464 */
465 skb_reserve(skb, reserved);
466 skb_reset_network_header(skb);
467
468 /* Try to align data part correctly */
469 if (hhlen) {
470 skb->data -= hhlen;
471 skb->tail -= hhlen;
472 if (len < hhlen)
473 skb_reset_network_header(skb);
474 }
475 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len);
476 if (err)
477 goto out_free;
478 goto retry;
479 }
480
481
482 skb->protocol = proto;
483 skb->dev = dev;
484 skb->priority = sk->sk_priority;
485 skb->mark = sk->sk_mark;
486
487 dev_queue_xmit(skb);
488 rcu_read_unlock();
489 return len;
490
491 out_unlock:
492 rcu_read_unlock();
493 out_free:
494 kfree_skb(skb);
495 return err;
496 }
497
498 static inline unsigned int run_filter(struct sk_buff *skb, struct sock *sk,
499 unsigned int res)
500 {
501 struct sk_filter *filter;
502
503 rcu_read_lock_bh();
504 filter = rcu_dereference_bh(sk->sk_filter);
505 if (filter != NULL)
506 res = sk_run_filter(skb, filter->insns, filter->len);
507 rcu_read_unlock_bh();
508
509 return res;
510 }
511
512 /*
513 This function makes lazy skb cloning in hope that most of packets
514 are discarded by BPF.
515
516 Note tricky part: we DO mangle shared skb! skb->data, skb->len
517 and skb->cb are mangled. It works because (and until) packets
518 falling here are owned by current CPU. Output packets are cloned
519 by dev_queue_xmit_nit(), input packets are processed by net_bh
520 sequencially, so that if we return skb to original state on exit,
521 we will not harm anyone.
522 */
523
524 static int packet_rcv(struct sk_buff *skb, struct net_device *dev,
525 struct packet_type *pt, struct net_device *orig_dev)
526 {
527 struct sock *sk;
528 struct sockaddr_ll *sll;
529 struct packet_sock *po;
530 u8 *skb_head = skb->data;
531 int skb_len = skb->len;
532 unsigned int snaplen, res;
533
534 if (skb->pkt_type == PACKET_LOOPBACK)
535 goto drop;
536
537 sk = pt->af_packet_priv;
538 po = pkt_sk(sk);
539
540 if (!net_eq(dev_net(dev), sock_net(sk)))
541 goto drop;
542
543 skb->dev = dev;
544
545 if (dev->header_ops) {
546 /* The device has an explicit notion of ll header,
547 exported to higher levels.
548
549 Otherwise, the device hides datails of it frame
550 structure, so that corresponding packet head
551 never delivered to user.
552 */
553 if (sk->sk_type != SOCK_DGRAM)
554 skb_push(skb, skb->data - skb_mac_header(skb));
555 else if (skb->pkt_type == PACKET_OUTGOING) {
556 /* Special case: outgoing packets have ll header at head */
557 skb_pull(skb, skb_network_offset(skb));
558 }
559 }
560
561 snaplen = skb->len;
562
563 res = run_filter(skb, sk, snaplen);
564 if (!res)
565 goto drop_n_restore;
566 if (snaplen > res)
567 snaplen = res;
568
569 if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
570 (unsigned)sk->sk_rcvbuf)
571 goto drop_n_acct;
572
573 if (skb_shared(skb)) {
574 struct sk_buff *nskb = skb_clone(skb, GFP_ATOMIC);
575 if (nskb == NULL)
576 goto drop_n_acct;
577
578 if (skb_head != skb->data) {
579 skb->data = skb_head;
580 skb->len = skb_len;
581 }
582 kfree_skb(skb);
583 skb = nskb;
584 }
585
586 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 >
587 sizeof(skb->cb));
588
589 sll = &PACKET_SKB_CB(skb)->sa.ll;
590 sll->sll_family = AF_PACKET;
591 sll->sll_hatype = dev->type;
592 sll->sll_protocol = skb->protocol;
593 sll->sll_pkttype = skb->pkt_type;
594 if (unlikely(po->origdev))
595 sll->sll_ifindex = orig_dev->ifindex;
596 else
597 sll->sll_ifindex = dev->ifindex;
598
599 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
600
601 PACKET_SKB_CB(skb)->origlen = skb->len;
602
603 if (pskb_trim(skb, snaplen))
604 goto drop_n_acct;
605
606 skb_set_owner_r(skb, sk);
607 skb->dev = NULL;
608 skb_dst_drop(skb);
609
610 /* drop conntrack reference */
611 nf_reset(skb);
612
613 spin_lock(&sk->sk_receive_queue.lock);
614 po->stats.tp_packets++;
615 skb->dropcount = atomic_read(&sk->sk_drops);
616 __skb_queue_tail(&sk->sk_receive_queue, skb);
617 spin_unlock(&sk->sk_receive_queue.lock);
618 sk->sk_data_ready(sk, skb->len);
619 return 0;
620
621 drop_n_acct:
622 po->stats.tp_drops = atomic_inc_return(&sk->sk_drops);
623
624 drop_n_restore:
625 if (skb_head != skb->data && skb_shared(skb)) {
626 skb->data = skb_head;
627 skb->len = skb_len;
628 }
629 drop:
630 consume_skb(skb);
631 return 0;
632 }
633
634 static int tpacket_rcv(struct sk_buff *skb, struct net_device *dev,
635 struct packet_type *pt, struct net_device *orig_dev)
636 {
637 struct sock *sk;
638 struct packet_sock *po;
639 struct sockaddr_ll *sll;
640 union {
641 struct tpacket_hdr *h1;
642 struct tpacket2_hdr *h2;
643 void *raw;
644 } h;
645 u8 *skb_head = skb->data;
646 int skb_len = skb->len;
647 unsigned int snaplen, res;
648 unsigned long status = TP_STATUS_LOSING|TP_STATUS_USER;
649 unsigned short macoff, netoff, hdrlen;
650 struct sk_buff *copy_skb = NULL;
651 struct timeval tv;
652 struct timespec ts;
653
654 if (skb->pkt_type == PACKET_LOOPBACK)
655 goto drop;
656
657 sk = pt->af_packet_priv;
658 po = pkt_sk(sk);
659
660 if (!net_eq(dev_net(dev), sock_net(sk)))
661 goto drop;
662
663 if (dev->header_ops) {
664 if (sk->sk_type != SOCK_DGRAM)
665 skb_push(skb, skb->data - skb_mac_header(skb));
666 else if (skb->pkt_type == PACKET_OUTGOING) {
667 /* Special case: outgoing packets have ll header at head */
668 skb_pull(skb, skb_network_offset(skb));
669 }
670 }
671
672 if (skb->ip_summed == CHECKSUM_PARTIAL)
673 status |= TP_STATUS_CSUMNOTREADY;
674
675 snaplen = skb->len;
676
677 res = run_filter(skb, sk, snaplen);
678 if (!res)
679 goto drop_n_restore;
680 if (snaplen > res)
681 snaplen = res;
682
683 if (sk->sk_type == SOCK_DGRAM) {
684 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 +
685 po->tp_reserve;
686 } else {
687 unsigned maclen = skb_network_offset(skb);
688 netoff = TPACKET_ALIGN(po->tp_hdrlen +
689 (maclen < 16 ? 16 : maclen)) +
690 po->tp_reserve;
691 macoff = netoff - maclen;
692 }
693
694 if (macoff + snaplen > po->rx_ring.frame_size) {
695 if (po->copy_thresh &&
696 atomic_read(&sk->sk_rmem_alloc) + skb->truesize <
697 (unsigned)sk->sk_rcvbuf) {
698 if (skb_shared(skb)) {
699 copy_skb = skb_clone(skb, GFP_ATOMIC);
700 } else {
701 copy_skb = skb_get(skb);
702 skb_head = skb->data;
703 }
704 if (copy_skb)
705 skb_set_owner_r(copy_skb, sk);
706 }
707 snaplen = po->rx_ring.frame_size - macoff;
708 if ((int)snaplen < 0)
709 snaplen = 0;
710 }
711
712 spin_lock(&sk->sk_receive_queue.lock);
713 h.raw = packet_current_frame(po, &po->rx_ring, TP_STATUS_KERNEL);
714 if (!h.raw)
715 goto ring_is_full;
716 packet_increment_head(&po->rx_ring);
717 po->stats.tp_packets++;
718 if (copy_skb) {
719 status |= TP_STATUS_COPY;
720 __skb_queue_tail(&sk->sk_receive_queue, copy_skb);
721 }
722 if (!po->stats.tp_drops)
723 status &= ~TP_STATUS_LOSING;
724 spin_unlock(&sk->sk_receive_queue.lock);
725
726 skb_copy_bits(skb, 0, h.raw + macoff, snaplen);
727
728 switch (po->tp_version) {
729 case TPACKET_V1:
730 h.h1->tp_len = skb->len;
731 h.h1->tp_snaplen = snaplen;
732 h.h1->tp_mac = macoff;
733 h.h1->tp_net = netoff;
734 if (skb->tstamp.tv64)
735 tv = ktime_to_timeval(skb->tstamp);
736 else
737 do_gettimeofday(&tv);
738 h.h1->tp_sec = tv.tv_sec;
739 h.h1->tp_usec = tv.tv_usec;
740 hdrlen = sizeof(*h.h1);
741 break;
742 case TPACKET_V2:
743 h.h2->tp_len = skb->len;
744 h.h2->tp_snaplen = snaplen;
745 h.h2->tp_mac = macoff;
746 h.h2->tp_net = netoff;
747 if (skb->tstamp.tv64)
748 ts = ktime_to_timespec(skb->tstamp);
749 else
750 getnstimeofday(&ts);
751 h.h2->tp_sec = ts.tv_sec;
752 h.h2->tp_nsec = ts.tv_nsec;
753 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb);
754 h.h2->tp_padding = 0;
755 hdrlen = sizeof(*h.h2);
756 break;
757 default:
758 BUG();
759 }
760
761 sll = h.raw + TPACKET_ALIGN(hdrlen);
762 sll->sll_halen = dev_parse_header(skb, sll->sll_addr);
763 sll->sll_family = AF_PACKET;
764 sll->sll_hatype = dev->type;
765 sll->sll_protocol = skb->protocol;
766 sll->sll_pkttype = skb->pkt_type;
767 if (unlikely(po->origdev))
768 sll->sll_ifindex = orig_dev->ifindex;
769 else
770 sll->sll_ifindex = dev->ifindex;
771
772 __packet_set_status(po, h.raw, status);
773 smp_mb();
774 {
775 struct page *p_start, *p_end;
776 u8 *h_end = h.raw + macoff + snaplen - 1;
777
778 p_start = virt_to_page(h.raw);
779 p_end = virt_to_page(h_end);
780 while (p_start <= p_end) {
781 flush_dcache_page(p_start);
782 p_start++;
783 }
784 }
785
786 sk->sk_data_ready(sk, 0);
787
788 drop_n_restore:
789 if (skb_head != skb->data && skb_shared(skb)) {
790 skb->data = skb_head;
791 skb->len = skb_len;
792 }
793 drop:
794 kfree_skb(skb);
795 return 0;
796
797 ring_is_full:
798 po->stats.tp_drops++;
799 spin_unlock(&sk->sk_receive_queue.lock);
800
801 sk->sk_data_ready(sk, 0);
802 kfree_skb(copy_skb);
803 goto drop_n_restore;
804 }
805
806 static void tpacket_destruct_skb(struct sk_buff *skb)
807 {
808 struct packet_sock *po = pkt_sk(skb->sk);
809 void *ph;
810
811 BUG_ON(skb == NULL);
812
813 if (likely(po->tx_ring.pg_vec)) {
814 ph = skb_shinfo(skb)->destructor_arg;
815 BUG_ON(__packet_get_status(po, ph) != TP_STATUS_SENDING);
816 BUG_ON(atomic_read(&po->tx_ring.pending) == 0);
817 atomic_dec(&po->tx_ring.pending);
818 __packet_set_status(po, ph, TP_STATUS_AVAILABLE);
819 }
820
821 sock_wfree(skb);
822 }
823
824 static int tpacket_fill_skb(struct packet_sock *po, struct sk_buff *skb,
825 void *frame, struct net_device *dev, int size_max,
826 __be16 proto, unsigned char *addr)
827 {
828 union {
829 struct tpacket_hdr *h1;
830 struct tpacket2_hdr *h2;
831 void *raw;
832 } ph;
833 int to_write, offset, len, tp_len, nr_frags, len_max;
834 struct socket *sock = po->sk.sk_socket;
835 struct page *page;
836 void *data;
837 int err;
838
839 ph.raw = frame;
840
841 skb->protocol = proto;
842 skb->dev = dev;
843 skb->priority = po->sk.sk_priority;
844 skb->mark = po->sk.sk_mark;
845 skb_shinfo(skb)->destructor_arg = ph.raw;
846
847 switch (po->tp_version) {
848 case TPACKET_V2:
849 tp_len = ph.h2->tp_len;
850 break;
851 default:
852 tp_len = ph.h1->tp_len;
853 break;
854 }
855 if (unlikely(tp_len > size_max)) {
856 pr_err("packet size is too long (%d > %d)\n", tp_len, size_max);
857 return -EMSGSIZE;
858 }
859
860 skb_reserve(skb, LL_RESERVED_SPACE(dev));
861 skb_reset_network_header(skb);
862
863 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll);
864 to_write = tp_len;
865
866 if (sock->type == SOCK_DGRAM) {
867 err = dev_hard_header(skb, dev, ntohs(proto), addr,
868 NULL, tp_len);
869 if (unlikely(err < 0))
870 return -EINVAL;
871 } else if (dev->hard_header_len) {
872 /* net device doesn't like empty head */
873 if (unlikely(tp_len <= dev->hard_header_len)) {
874 pr_err("packet size is too short (%d < %d)\n",
875 tp_len, dev->hard_header_len);
876 return -EINVAL;
877 }
878
879 skb_push(skb, dev->hard_header_len);
880 err = skb_store_bits(skb, 0, data,
881 dev->hard_header_len);
882 if (unlikely(err))
883 return err;
884
885 data += dev->hard_header_len;
886 to_write -= dev->hard_header_len;
887 }
888
889 err = -EFAULT;
890 page = virt_to_page(data);
891 offset = offset_in_page(data);
892 len_max = PAGE_SIZE - offset;
893 len = ((to_write > len_max) ? len_max : to_write);
894
895 skb->data_len = to_write;
896 skb->len += to_write;
897 skb->truesize += to_write;
898 atomic_add(to_write, &po->sk.sk_wmem_alloc);
899
900 while (likely(to_write)) {
901 nr_frags = skb_shinfo(skb)->nr_frags;
902
903 if (unlikely(nr_frags >= MAX_SKB_FRAGS)) {
904 pr_err("Packet exceed the number of skb frags(%lu)\n",
905 MAX_SKB_FRAGS);
906 return -EFAULT;
907 }
908
909 flush_dcache_page(page);
910 get_page(page);
911 skb_fill_page_desc(skb,
912 nr_frags,
913 page++, offset, len);
914 to_write -= len;
915 offset = 0;
916 len_max = PAGE_SIZE;
917 len = ((to_write > len_max) ? len_max : to_write);
918 }
919
920 return tp_len;
921 }
922
923 static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
924 {
925 struct socket *sock;
926 struct sk_buff *skb;
927 struct net_device *dev;
928 __be16 proto;
929 int ifindex, err, reserve = 0;
930 void *ph;
931 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
932 int tp_len, size_max;
933 unsigned char *addr;
934 int len_sum = 0;
935 int status = 0;
936
937 sock = po->sk.sk_socket;
938
939 mutex_lock(&po->pg_vec_lock);
940
941 err = -EBUSY;
942 if (saddr == NULL) {
943 ifindex = po->ifindex;
944 proto = po->num;
945 addr = NULL;
946 } else {
947 err = -EINVAL;
948 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
949 goto out;
950 if (msg->msg_namelen < (saddr->sll_halen
951 + offsetof(struct sockaddr_ll,
952 sll_addr)))
953 goto out;
954 ifindex = saddr->sll_ifindex;
955 proto = saddr->sll_protocol;
956 addr = saddr->sll_addr;
957 }
958
959 dev = dev_get_by_index(sock_net(&po->sk), ifindex);
960 err = -ENXIO;
961 if (unlikely(dev == NULL))
962 goto out;
963
964 reserve = dev->hard_header_len;
965
966 err = -ENETDOWN;
967 if (unlikely(!(dev->flags & IFF_UP)))
968 goto out_put;
969
970 size_max = po->tx_ring.frame_size
971 - (po->tp_hdrlen - sizeof(struct sockaddr_ll));
972
973 if (size_max > dev->mtu + reserve)
974 size_max = dev->mtu + reserve;
975
976 do {
977 ph = packet_current_frame(po, &po->tx_ring,
978 TP_STATUS_SEND_REQUEST);
979
980 if (unlikely(ph == NULL)) {
981 schedule();
982 continue;
983 }
984
985 status = TP_STATUS_SEND_REQUEST;
986 skb = sock_alloc_send_skb(&po->sk,
987 LL_ALLOCATED_SPACE(dev)
988 + sizeof(struct sockaddr_ll),
989 0, &err);
990
991 if (unlikely(skb == NULL))
992 goto out_status;
993
994 tp_len = tpacket_fill_skb(po, skb, ph, dev, size_max, proto,
995 addr);
996
997 if (unlikely(tp_len < 0)) {
998 if (po->tp_loss) {
999 __packet_set_status(po, ph,
1000 TP_STATUS_AVAILABLE);
1001 packet_increment_head(&po->tx_ring);
1002 kfree_skb(skb);
1003 continue;
1004 } else {
1005 status = TP_STATUS_WRONG_FORMAT;
1006 err = tp_len;
1007 goto out_status;
1008 }
1009 }
1010
1011 skb->destructor = tpacket_destruct_skb;
1012 __packet_set_status(po, ph, TP_STATUS_SENDING);
1013 atomic_inc(&po->tx_ring.pending);
1014
1015 status = TP_STATUS_SEND_REQUEST;
1016 err = dev_queue_xmit(skb);
1017 if (unlikely(err > 0)) {
1018 err = net_xmit_errno(err);
1019 if (err && __packet_get_status(po, ph) ==
1020 TP_STATUS_AVAILABLE) {
1021 /* skb was destructed already */
1022 skb = NULL;
1023 goto out_status;
1024 }
1025 /*
1026 * skb was dropped but not destructed yet;
1027 * let's treat it like congestion or err < 0
1028 */
1029 err = 0;
1030 }
1031 packet_increment_head(&po->tx_ring);
1032 len_sum += tp_len;
1033 } while (likely((ph != NULL) ||
1034 ((!(msg->msg_flags & MSG_DONTWAIT)) &&
1035 (atomic_read(&po->tx_ring.pending))))
1036 );
1037
1038 err = len_sum;
1039 goto out_put;
1040
1041 out_status:
1042 __packet_set_status(po, ph, status);
1043 kfree_skb(skb);
1044 out_put:
1045 dev_put(dev);
1046 out:
1047 mutex_unlock(&po->pg_vec_lock);
1048 return err;
1049 }
1050
1051 static inline struct sk_buff *packet_alloc_skb(struct sock *sk, size_t prepad,
1052 size_t reserve, size_t len,
1053 size_t linear, int noblock,
1054 int *err)
1055 {
1056 struct sk_buff *skb;
1057
1058 /* Under a page? Don't bother with paged skb. */
1059 if (prepad + len < PAGE_SIZE || !linear)
1060 linear = len;
1061
1062 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
1063 err);
1064 if (!skb)
1065 return NULL;
1066
1067 skb_reserve(skb, reserve);
1068 skb_put(skb, linear);
1069 skb->data_len = len - linear;
1070 skb->len += len - linear;
1071
1072 return skb;
1073 }
1074
1075 static int packet_snd(struct socket *sock,
1076 struct msghdr *msg, size_t len)
1077 {
1078 struct sock *sk = sock->sk;
1079 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name;
1080 struct sk_buff *skb;
1081 struct net_device *dev;
1082 __be16 proto;
1083 unsigned char *addr;
1084 int ifindex, err, reserve = 0;
1085 struct virtio_net_hdr vnet_hdr = { 0 };
1086 int offset = 0;
1087 int vnet_hdr_len;
1088 struct packet_sock *po = pkt_sk(sk);
1089 unsigned short gso_type = 0;
1090
1091 /*
1092 * Get and verify the address.
1093 */
1094
1095 if (saddr == NULL) {
1096 ifindex = po->ifindex;
1097 proto = po->num;
1098 addr = NULL;
1099 } else {
1100 err = -EINVAL;
1101 if (msg->msg_namelen < sizeof(struct sockaddr_ll))
1102 goto out;
1103 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr)))
1104 goto out;
1105 ifindex = saddr->sll_ifindex;
1106 proto = saddr->sll_protocol;
1107 addr = saddr->sll_addr;
1108 }
1109
1110
1111 dev = dev_get_by_index(sock_net(sk), ifindex);
1112 err = -ENXIO;
1113 if (dev == NULL)
1114 goto out_unlock;
1115 if (sock->type == SOCK_RAW)
1116 reserve = dev->hard_header_len;
1117
1118 err = -ENETDOWN;
1119 if (!(dev->flags & IFF_UP))
1120 goto out_unlock;
1121
1122 if (po->has_vnet_hdr) {
1123 vnet_hdr_len = sizeof(vnet_hdr);
1124
1125 err = -EINVAL;
1126 if (len < vnet_hdr_len)
1127 goto out_unlock;
1128
1129 len -= vnet_hdr_len;
1130
1131 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov,
1132 vnet_hdr_len);
1133 if (err < 0)
1134 goto out_unlock;
1135
1136 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) &&
1137 (vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 >
1138 vnet_hdr.hdr_len))
1139 vnet_hdr.hdr_len = vnet_hdr.csum_start +
1140 vnet_hdr.csum_offset + 2;
1141
1142 err = -EINVAL;
1143 if (vnet_hdr.hdr_len > len)
1144 goto out_unlock;
1145
1146 if (vnet_hdr.gso_type != VIRTIO_NET_HDR_GSO_NONE) {
1147 switch (vnet_hdr.gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
1148 case VIRTIO_NET_HDR_GSO_TCPV4:
1149 gso_type = SKB_GSO_TCPV4;
1150 break;
1151 case VIRTIO_NET_HDR_GSO_TCPV6:
1152 gso_type = SKB_GSO_TCPV6;
1153 break;
1154 case VIRTIO_NET_HDR_GSO_UDP:
1155 gso_type = SKB_GSO_UDP;
1156 break;
1157 default:
1158 goto out_unlock;
1159 }
1160
1161 if (vnet_hdr.gso_type & VIRTIO_NET_HDR_GSO_ECN)
1162 gso_type |= SKB_GSO_TCP_ECN;
1163
1164 if (vnet_hdr.gso_size == 0)
1165 goto out_unlock;
1166
1167 }
1168 }
1169
1170 err = -EMSGSIZE;
1171 if (!gso_type && (len > dev->mtu+reserve))
1172 goto out_unlock;
1173
1174 err = -ENOBUFS;
1175 skb = packet_alloc_skb(sk, LL_ALLOCATED_SPACE(dev),
1176 LL_RESERVED_SPACE(dev), len, vnet_hdr.hdr_len,
1177 msg->msg_flags & MSG_DONTWAIT, &err);
1178 if (skb == NULL)
1179 goto out_unlock;
1180
1181 skb_set_network_header(skb, reserve);
1182
1183 err = -EINVAL;
1184 if (sock->type == SOCK_DGRAM &&
1185 (offset = dev_hard_header(skb, dev, ntohs(proto), addr, NULL, len)) < 0)
1186 goto out_free;
1187
1188 /* Returns -EFAULT on error */
1189 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len);
1190 if (err)
1191 goto out_free;
1192
1193 skb->protocol = proto;
1194 skb->dev = dev;
1195 skb->priority = sk->sk_priority;
1196 skb->mark = sk->sk_mark;
1197
1198 if (po->has_vnet_hdr) {
1199 if (vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
1200 if (!skb_partial_csum_set(skb, vnet_hdr.csum_start,
1201 vnet_hdr.csum_offset)) {
1202 err = -EINVAL;
1203 goto out_free;
1204 }
1205 }
1206
1207 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size;
1208 skb_shinfo(skb)->gso_type = gso_type;
1209
1210 /* Header must be checked, and gso_segs computed. */
1211 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY;
1212 skb_shinfo(skb)->gso_segs = 0;
1213
1214 len += vnet_hdr_len;
1215 }
1216
1217 /*
1218 * Now send it
1219 */
1220
1221 err = dev_queue_xmit(skb);
1222 if (err > 0 && (err = net_xmit_errno(err)) != 0)
1223 goto out_unlock;
1224
1225 dev_put(dev);
1226
1227 return len;
1228
1229 out_free:
1230 kfree_skb(skb);
1231 out_unlock:
1232 if (dev)
1233 dev_put(dev);
1234 out:
1235 return err;
1236 }
1237
1238 static int packet_sendmsg(struct kiocb *iocb, struct socket *sock,
1239 struct msghdr *msg, size_t len)
1240 {
1241 struct sock *sk = sock->sk;
1242 struct packet_sock *po = pkt_sk(sk);
1243 if (po->tx_ring.pg_vec)
1244 return tpacket_snd(po, msg);
1245 else
1246 return packet_snd(sock, msg, len);
1247 }
1248
1249 /*
1250 * Close a PACKET socket. This is fairly simple. We immediately go
1251 * to 'closed' state and remove our protocol entry in the device list.
1252 */
1253
1254 static int packet_release(struct socket *sock)
1255 {
1256 struct sock *sk = sock->sk;
1257 struct packet_sock *po;
1258 struct net *net;
1259 struct tpacket_req req;
1260
1261 if (!sk)
1262 return 0;
1263
1264 net = sock_net(sk);
1265 po = pkt_sk(sk);
1266
1267 spin_lock_bh(&net->packet.sklist_lock);
1268 sk_del_node_init_rcu(sk);
1269 sock_prot_inuse_add(net, sk->sk_prot, -1);
1270 spin_unlock_bh(&net->packet.sklist_lock);
1271
1272 spin_lock(&po->bind_lock);
1273 if (po->running) {
1274 /*
1275 * Remove from protocol table
1276 */
1277 po->running = 0;
1278 po->num = 0;
1279 __dev_remove_pack(&po->prot_hook);
1280 __sock_put(sk);
1281 }
1282 spin_unlock(&po->bind_lock);
1283
1284 packet_flush_mclist(sk);
1285
1286 memset(&req, 0, sizeof(req));
1287
1288 if (po->rx_ring.pg_vec)
1289 packet_set_ring(sk, &req, 1, 0);
1290
1291 if (po->tx_ring.pg_vec)
1292 packet_set_ring(sk, &req, 1, 1);
1293
1294 synchronize_net();
1295 /*
1296 * Now the socket is dead. No more input will appear.
1297 */
1298 sock_orphan(sk);
1299 sock->sk = NULL;
1300
1301 /* Purge queues */
1302
1303 skb_queue_purge(&sk->sk_receive_queue);
1304 sk_refcnt_debug_release(sk);
1305
1306 sock_put(sk);
1307 return 0;
1308 }
1309
1310 /*
1311 * Attach a packet hook.
1312 */
1313
1314 static int packet_do_bind(struct sock *sk, struct net_device *dev, __be16 protocol)
1315 {
1316 struct packet_sock *po = pkt_sk(sk);
1317 /*
1318 * Detach an existing hook if present.
1319 */
1320
1321 lock_sock(sk);
1322
1323 spin_lock(&po->bind_lock);
1324 if (po->running) {
1325 __sock_put(sk);
1326 po->running = 0;
1327 po->num = 0;
1328 spin_unlock(&po->bind_lock);
1329 dev_remove_pack(&po->prot_hook);
1330 spin_lock(&po->bind_lock);
1331 }
1332
1333 po->num = protocol;
1334 po->prot_hook.type = protocol;
1335 po->prot_hook.dev = dev;
1336
1337 po->ifindex = dev ? dev->ifindex : 0;
1338
1339 if (protocol == 0)
1340 goto out_unlock;
1341
1342 if (!dev || (dev->flags & IFF_UP)) {
1343 dev_add_pack(&po->prot_hook);
1344 sock_hold(sk);
1345 po->running = 1;
1346 } else {
1347 sk->sk_err = ENETDOWN;
1348 if (!sock_flag(sk, SOCK_DEAD))
1349 sk->sk_error_report(sk);
1350 }
1351
1352 out_unlock:
1353 spin_unlock(&po->bind_lock);
1354 release_sock(sk);
1355 return 0;
1356 }
1357
1358 /*
1359 * Bind a packet socket to a device
1360 */
1361
1362 static int packet_bind_spkt(struct socket *sock, struct sockaddr *uaddr,
1363 int addr_len)
1364 {
1365 struct sock *sk = sock->sk;
1366 char name[15];
1367 struct net_device *dev;
1368 int err = -ENODEV;
1369
1370 /*
1371 * Check legality
1372 */
1373
1374 if (addr_len != sizeof(struct sockaddr))
1375 return -EINVAL;
1376 strlcpy(name, uaddr->sa_data, sizeof(name));
1377
1378 dev = dev_get_by_name(sock_net(sk), name);
1379 if (dev) {
1380 err = packet_do_bind(sk, dev, pkt_sk(sk)->num);
1381 dev_put(dev);
1382 }
1383 return err;
1384 }
1385
1386 static int packet_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
1387 {
1388 struct sockaddr_ll *sll = (struct sockaddr_ll *)uaddr;
1389 struct sock *sk = sock->sk;
1390 struct net_device *dev = NULL;
1391 int err;
1392
1393
1394 /*
1395 * Check legality
1396 */
1397
1398 if (addr_len < sizeof(struct sockaddr_ll))
1399 return -EINVAL;
1400 if (sll->sll_family != AF_PACKET)
1401 return -EINVAL;
1402
1403 if (sll->sll_ifindex) {
1404 err = -ENODEV;
1405 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex);
1406 if (dev == NULL)
1407 goto out;
1408 }
1409 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num);
1410 if (dev)
1411 dev_put(dev);
1412
1413 out:
1414 return err;
1415 }
1416
1417 static struct proto packet_proto = {
1418 .name = "PACKET",
1419 .owner = THIS_MODULE,
1420 .obj_size = sizeof(struct packet_sock),
1421 };
1422
1423 /*
1424 * Create a packet of type SOCK_PACKET.
1425 */
1426
1427 static int packet_create(struct net *net, struct socket *sock, int protocol,
1428 int kern)
1429 {
1430 struct sock *sk;
1431 struct packet_sock *po;
1432 __be16 proto = (__force __be16)protocol; /* weird, but documented */
1433 int err;
1434
1435 if (!capable(CAP_NET_RAW))
1436 return -EPERM;
1437 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW &&
1438 sock->type != SOCK_PACKET)
1439 return -ESOCKTNOSUPPORT;
1440
1441 sock->state = SS_UNCONNECTED;
1442
1443 err = -ENOBUFS;
1444 sk = sk_alloc(net, PF_PACKET, GFP_KERNEL, &packet_proto);
1445 if (sk == NULL)
1446 goto out;
1447
1448 sock->ops = &packet_ops;
1449 if (sock->type == SOCK_PACKET)
1450 sock->ops = &packet_ops_spkt;
1451
1452 sock_init_data(sock, sk);
1453
1454 po = pkt_sk(sk);
1455 sk->sk_family = PF_PACKET;
1456 po->num = proto;
1457
1458 sk->sk_destruct = packet_sock_destruct;
1459 sk_refcnt_debug_inc(sk);
1460
1461 /*
1462 * Attach a protocol block
1463 */
1464
1465 spin_lock_init(&po->bind_lock);
1466 mutex_init(&po->pg_vec_lock);
1467 po->prot_hook.func = packet_rcv;
1468
1469 if (sock->type == SOCK_PACKET)
1470 po->prot_hook.func = packet_rcv_spkt;
1471
1472 po->prot_hook.af_packet_priv = sk;
1473
1474 if (proto) {
1475 po->prot_hook.type = proto;
1476 dev_add_pack(&po->prot_hook);
1477 sock_hold(sk);
1478 po->running = 1;
1479 }
1480
1481 spin_lock_bh(&net->packet.sklist_lock);
1482 sk_add_node_rcu(sk, &net->packet.sklist);
1483 sock_prot_inuse_add(net, &packet_proto, 1);
1484 spin_unlock_bh(&net->packet.sklist_lock);
1485
1486 return 0;
1487 out:
1488 return err;
1489 }
1490
1491 /*
1492 * Pull a packet from our receive queue and hand it to the user.
1493 * If necessary we block.
1494 */
1495
1496 static int packet_recvmsg(struct kiocb *iocb, struct socket *sock,
1497 struct msghdr *msg, size_t len, int flags)
1498 {
1499 struct sock *sk = sock->sk;
1500 struct sk_buff *skb;
1501 int copied, err;
1502 struct sockaddr_ll *sll;
1503 int vnet_hdr_len = 0;
1504
1505 err = -EINVAL;
1506 if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT))
1507 goto out;
1508
1509 #if 0
1510 /* What error should we return now? EUNATTACH? */
1511 if (pkt_sk(sk)->ifindex < 0)
1512 return -ENODEV;
1513 #endif
1514
1515 /*
1516 * Call the generic datagram receiver. This handles all sorts
1517 * of horrible races and re-entrancy so we can forget about it
1518 * in the protocol layers.
1519 *
1520 * Now it will return ENETDOWN, if device have just gone down,
1521 * but then it will block.
1522 */
1523
1524 skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);
1525
1526 /*
1527 * An error occurred so return it. Because skb_recv_datagram()
1528 * handles the blocking we don't see and worry about blocking
1529 * retries.
1530 */
1531
1532 if (skb == NULL)
1533 goto out;
1534
1535 if (pkt_sk(sk)->has_vnet_hdr) {
1536 struct virtio_net_hdr vnet_hdr = { 0 };
1537
1538 err = -EINVAL;
1539 vnet_hdr_len = sizeof(vnet_hdr);
1540 if ((len -= vnet_hdr_len) < 0)
1541 goto out_free;
1542
1543 if (skb_is_gso(skb)) {
1544 struct skb_shared_info *sinfo = skb_shinfo(skb);
1545
1546 /* This is a hint as to how much should be linear. */
1547 vnet_hdr.hdr_len = skb_headlen(skb);
1548 vnet_hdr.gso_size = sinfo->gso_size;
1549 if (sinfo->gso_type & SKB_GSO_TCPV4)
1550 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1551 else if (sinfo->gso_type & SKB_GSO_TCPV6)
1552 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1553 else if (sinfo->gso_type & SKB_GSO_UDP)
1554 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_UDP;
1555 else if (sinfo->gso_type & SKB_GSO_FCOE)
1556 goto out_free;
1557 else
1558 BUG();
1559 if (sinfo->gso_type & SKB_GSO_TCP_ECN)
1560 vnet_hdr.gso_type |= VIRTIO_NET_HDR_GSO_ECN;
1561 } else
1562 vnet_hdr.gso_type = VIRTIO_NET_HDR_GSO_NONE;
1563
1564 if (skb->ip_summed == CHECKSUM_PARTIAL) {
1565 vnet_hdr.flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
1566 vnet_hdr.csum_start = skb->csum_start -
1567 skb_headroom(skb);
1568 vnet_hdr.csum_offset = skb->csum_offset;
1569 } /* else everything is zero */
1570
1571 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr,
1572 vnet_hdr_len);
1573 if (err < 0)
1574 goto out_free;
1575 }
1576
1577 /*
1578 * If the address length field is there to be filled in, we fill
1579 * it in now.
1580 */
1581
1582 sll = &PACKET_SKB_CB(skb)->sa.ll;
1583 if (sock->type == SOCK_PACKET)
1584 msg->msg_namelen = sizeof(struct sockaddr_pkt);
1585 else
1586 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr);
1587
1588 /*
1589 * You lose any data beyond the buffer you gave. If it worries a
1590 * user program they can ask the device for its MTU anyway.
1591 */
1592
1593 copied = skb->len;
1594 if (copied > len) {
1595 copied = len;
1596 msg->msg_flags |= MSG_TRUNC;
1597 }
1598
1599 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied);
1600 if (err)
1601 goto out_free;
1602
1603 sock_recv_ts_and_drops(msg, sk, skb);
1604
1605 if (msg->msg_name)
1606 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa,
1607 msg->msg_namelen);
1608
1609 if (pkt_sk(sk)->auxdata) {
1610 struct tpacket_auxdata aux;
1611
1612 aux.tp_status = TP_STATUS_USER;
1613 if (skb->ip_summed == CHECKSUM_PARTIAL)
1614 aux.tp_status |= TP_STATUS_CSUMNOTREADY;
1615 aux.tp_len = PACKET_SKB_CB(skb)->origlen;
1616 aux.tp_snaplen = skb->len;
1617 aux.tp_mac = 0;
1618 aux.tp_net = skb_network_offset(skb);
1619 aux.tp_vlan_tci = vlan_tx_tag_get(skb);
1620
1621 aux.tp_padding = 0;
1622 put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
1623 }
1624
1625 /*
1626 * Free or return the buffer as appropriate. Again this
1627 * hides all the races and re-entrancy issues from us.
1628 */
1629 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);
1630
1631 out_free:
1632 skb_free_datagram(sk, skb);
1633 out:
1634 return err;
1635 }
1636
1637 static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
1638 int *uaddr_len, int peer)
1639 {
1640 struct net_device *dev;
1641 struct sock *sk = sock->sk;
1642
1643 if (peer)
1644 return -EOPNOTSUPP;
1645
1646 uaddr->sa_family = AF_PACKET;
1647 rcu_read_lock();
1648 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex);
1649 if (dev)
1650 strncpy(uaddr->sa_data, dev->name, 14);
1651 else
1652 memset(uaddr->sa_data, 0, 14);
1653 rcu_read_unlock();
1654 *uaddr_len = sizeof(*uaddr);
1655
1656 return 0;
1657 }
1658
1659 static int packet_getname(struct socket *sock, struct sockaddr *uaddr,
1660 int *uaddr_len, int peer)
1661 {
1662 struct net_device *dev;
1663 struct sock *sk = sock->sk;
1664 struct packet_sock *po = pkt_sk(sk);
1665 DECLARE_SOCKADDR(struct sockaddr_ll *, sll, uaddr);
1666
1667 if (peer)
1668 return -EOPNOTSUPP;
1669
1670 sll->sll_family = AF_PACKET;
1671 sll->sll_ifindex = po->ifindex;
1672 sll->sll_protocol = po->num;
1673 sll->sll_pkttype = 0;
1674 rcu_read_lock();
1675 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex);
1676 if (dev) {
1677 sll->sll_hatype = dev->type;
1678 sll->sll_halen = dev->addr_len;
1679 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len);
1680 } else {
1681 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */
1682 sll->sll_halen = 0;
1683 }
1684 rcu_read_unlock();
1685 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen;
1686
1687 return 0;
1688 }
1689
1690 static int packet_dev_mc(struct net_device *dev, struct packet_mclist *i,
1691 int what)
1692 {
1693 switch (i->type) {
1694 case PACKET_MR_MULTICAST:
1695 if (i->alen != dev->addr_len)
1696 return -EINVAL;
1697 if (what > 0)
1698 return dev_mc_add(dev, i->addr, i->alen, 0);
1699 else
1700 return dev_mc_delete(dev, i->addr, i->alen, 0);
1701 break;
1702 case PACKET_MR_PROMISC:
1703 return dev_set_promiscuity(dev, what);
1704 break;
1705 case PACKET_MR_ALLMULTI:
1706 return dev_set_allmulti(dev, what);
1707 break;
1708 case PACKET_MR_UNICAST:
1709 if (i->alen != dev->addr_len)
1710 return -EINVAL;
1711 if (what > 0)
1712 return dev_unicast_add(dev, i->addr);
1713 else
1714 return dev_unicast_delete(dev, i->addr);
1715 break;
1716 default:
1717 break;
1718 }
1719 return 0;
1720 }
1721
1722 static void packet_dev_mclist(struct net_device *dev, struct packet_mclist *i, int what)
1723 {
1724 for ( ; i; i = i->next) {
1725 if (i->ifindex == dev->ifindex)
1726 packet_dev_mc(dev, i, what);
1727 }
1728 }
1729
1730 static int packet_mc_add(struct sock *sk, struct packet_mreq_max *mreq)
1731 {
1732 struct packet_sock *po = pkt_sk(sk);
1733 struct packet_mclist *ml, *i;
1734 struct net_device *dev;
1735 int err;
1736
1737 rtnl_lock();
1738
1739 err = -ENODEV;
1740 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex);
1741 if (!dev)
1742 goto done;
1743
1744 err = -EINVAL;
1745 if (mreq->mr_alen > dev->addr_len)
1746 goto done;
1747
1748 err = -ENOBUFS;
1749 i = kmalloc(sizeof(*i), GFP_KERNEL);
1750 if (i == NULL)
1751 goto done;
1752
1753 err = 0;
1754 for (ml = po->mclist; ml; ml = ml->next) {
1755 if (ml->ifindex == mreq->mr_ifindex &&
1756 ml->type == mreq->mr_type &&
1757 ml->alen == mreq->mr_alen &&
1758 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
1759 ml->count++;
1760 /* Free the new element ... */
1761 kfree(i);
1762 goto done;
1763 }
1764 }
1765
1766 i->type = mreq->mr_type;
1767 i->ifindex = mreq->mr_ifindex;
1768 i->alen = mreq->mr_alen;
1769 memcpy(i->addr, mreq->mr_address, i->alen);
1770 i->count = 1;
1771 i->next = po->mclist;
1772 po->mclist = i;
1773 err = packet_dev_mc(dev, i, 1);
1774 if (err) {
1775 po->mclist = i->next;
1776 kfree(i);
1777 }
1778
1779 done:
1780 rtnl_unlock();
1781 return err;
1782 }
1783
1784 static int packet_mc_drop(struct sock *sk, struct packet_mreq_max *mreq)
1785 {
1786 struct packet_mclist *ml, **mlp;
1787
1788 rtnl_lock();
1789
1790 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) {
1791 if (ml->ifindex == mreq->mr_ifindex &&
1792 ml->type == mreq->mr_type &&
1793 ml->alen == mreq->mr_alen &&
1794 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) {
1795 if (--ml->count == 0) {
1796 struct net_device *dev;
1797 *mlp = ml->next;
1798 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
1799 if (dev)
1800 packet_dev_mc(dev, ml, -1);
1801 kfree(ml);
1802 }
1803 rtnl_unlock();
1804 return 0;
1805 }
1806 }
1807 rtnl_unlock();
1808 return -EADDRNOTAVAIL;
1809 }
1810
1811 static void packet_flush_mclist(struct sock *sk)
1812 {
1813 struct packet_sock *po = pkt_sk(sk);
1814 struct packet_mclist *ml;
1815
1816 if (!po->mclist)
1817 return;
1818
1819 rtnl_lock();
1820 while ((ml = po->mclist) != NULL) {
1821 struct net_device *dev;
1822
1823 po->mclist = ml->next;
1824 dev = __dev_get_by_index(sock_net(sk), ml->ifindex);
1825 if (dev != NULL)
1826 packet_dev_mc(dev, ml, -1);
1827 kfree(ml);
1828 }
1829 rtnl_unlock();
1830 }
1831
1832 static int
1833 packet_setsockopt(struct socket *sock, int level, int optname, char __user *optval, unsigned int optlen)
1834 {
1835 struct sock *sk = sock->sk;
1836 struct packet_sock *po = pkt_sk(sk);
1837 int ret;
1838
1839 if (level != SOL_PACKET)
1840 return -ENOPROTOOPT;
1841
1842 switch (optname) {
1843 case PACKET_ADD_MEMBERSHIP:
1844 case PACKET_DROP_MEMBERSHIP:
1845 {
1846 struct packet_mreq_max mreq;
1847 int len = optlen;
1848 memset(&mreq, 0, sizeof(mreq));
1849 if (len < sizeof(struct packet_mreq))
1850 return -EINVAL;
1851 if (len > sizeof(mreq))
1852 len = sizeof(mreq);
1853 if (copy_from_user(&mreq, optval, len))
1854 return -EFAULT;
1855 if (len < (mreq.mr_alen + offsetof(struct packet_mreq, mr_address)))
1856 return -EINVAL;
1857 if (optname == PACKET_ADD_MEMBERSHIP)
1858 ret = packet_mc_add(sk, &mreq);
1859 else
1860 ret = packet_mc_drop(sk, &mreq);
1861 return ret;
1862 }
1863
1864 case PACKET_RX_RING:
1865 case PACKET_TX_RING:
1866 {
1867 struct tpacket_req req;
1868
1869 if (optlen < sizeof(req))
1870 return -EINVAL;
1871 if (pkt_sk(sk)->has_vnet_hdr)
1872 return -EINVAL;
1873 if (copy_from_user(&req, optval, sizeof(req)))
1874 return -EFAULT;
1875 return packet_set_ring(sk, &req, 0, optname == PACKET_TX_RING);
1876 }
1877 case PACKET_COPY_THRESH:
1878 {
1879 int val;
1880
1881 if (optlen != sizeof(val))
1882 return -EINVAL;
1883 if (copy_from_user(&val, optval, sizeof(val)))
1884 return -EFAULT;
1885
1886 pkt_sk(sk)->copy_thresh = val;
1887 return 0;
1888 }
1889 case PACKET_VERSION:
1890 {
1891 int val;
1892
1893 if (optlen != sizeof(val))
1894 return -EINVAL;
1895 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1896 return -EBUSY;
1897 if (copy_from_user(&val, optval, sizeof(val)))
1898 return -EFAULT;
1899 switch (val) {
1900 case TPACKET_V1:
1901 case TPACKET_V2:
1902 po->tp_version = val;
1903 return 0;
1904 default:
1905 return -EINVAL;
1906 }
1907 }
1908 case PACKET_RESERVE:
1909 {
1910 unsigned int val;
1911
1912 if (optlen != sizeof(val))
1913 return -EINVAL;
1914 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1915 return -EBUSY;
1916 if (copy_from_user(&val, optval, sizeof(val)))
1917 return -EFAULT;
1918 po->tp_reserve = val;
1919 return 0;
1920 }
1921 case PACKET_LOSS:
1922 {
1923 unsigned int val;
1924
1925 if (optlen != sizeof(val))
1926 return -EINVAL;
1927 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1928 return -EBUSY;
1929 if (copy_from_user(&val, optval, sizeof(val)))
1930 return -EFAULT;
1931 po->tp_loss = !!val;
1932 return 0;
1933 }
1934 case PACKET_AUXDATA:
1935 {
1936 int val;
1937
1938 if (optlen < sizeof(val))
1939 return -EINVAL;
1940 if (copy_from_user(&val, optval, sizeof(val)))
1941 return -EFAULT;
1942
1943 po->auxdata = !!val;
1944 return 0;
1945 }
1946 case PACKET_ORIGDEV:
1947 {
1948 int val;
1949
1950 if (optlen < sizeof(val))
1951 return -EINVAL;
1952 if (copy_from_user(&val, optval, sizeof(val)))
1953 return -EFAULT;
1954
1955 po->origdev = !!val;
1956 return 0;
1957 }
1958 case PACKET_VNET_HDR:
1959 {
1960 int val;
1961
1962 if (sock->type != SOCK_RAW)
1963 return -EINVAL;
1964 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec)
1965 return -EBUSY;
1966 if (optlen < sizeof(val))
1967 return -EINVAL;
1968 if (copy_from_user(&val, optval, sizeof(val)))
1969 return -EFAULT;
1970
1971 po->has_vnet_hdr = !!val;
1972 return 0;
1973 }
1974 default:
1975 return -ENOPROTOOPT;
1976 }
1977 }
1978
1979 static int packet_getsockopt(struct socket *sock, int level, int optname,
1980 char __user *optval, int __user *optlen)
1981 {
1982 int len;
1983 int val;
1984 struct sock *sk = sock->sk;
1985 struct packet_sock *po = pkt_sk(sk);
1986 void *data;
1987 struct tpacket_stats st;
1988
1989 if (level != SOL_PACKET)
1990 return -ENOPROTOOPT;
1991
1992 if (get_user(len, optlen))
1993 return -EFAULT;
1994
1995 if (len < 0)
1996 return -EINVAL;
1997
1998 switch (optname) {
1999 case PACKET_STATISTICS:
2000 if (len > sizeof(struct tpacket_stats))
2001 len = sizeof(struct tpacket_stats);
2002 spin_lock_bh(&sk->sk_receive_queue.lock);
2003 st = po->stats;
2004 memset(&po->stats, 0, sizeof(st));
2005 spin_unlock_bh(&sk->sk_receive_queue.lock);
2006 st.tp_packets += st.tp_drops;
2007
2008 data = &st;
2009 break;
2010 case PACKET_AUXDATA:
2011 if (len > sizeof(int))
2012 len = sizeof(int);
2013 val = po->auxdata;
2014
2015 data = &val;
2016 break;
2017 case PACKET_ORIGDEV:
2018 if (len > sizeof(int))
2019 len = sizeof(int);
2020 val = po->origdev;
2021
2022 data = &val;
2023 break;
2024 case PACKET_VNET_HDR:
2025 if (len > sizeof(int))
2026 len = sizeof(int);
2027 val = po->has_vnet_hdr;
2028
2029 data = &val;
2030 break;
2031 case PACKET_VERSION:
2032 if (len > sizeof(int))
2033 len = sizeof(int);
2034 val = po->tp_version;
2035 data = &val;
2036 break;
2037 case PACKET_HDRLEN:
2038 if (len > sizeof(int))
2039 len = sizeof(int);
2040 if (copy_from_user(&val, optval, len))
2041 return -EFAULT;
2042 switch (val) {
2043 case TPACKET_V1:
2044 val = sizeof(struct tpacket_hdr);
2045 break;
2046 case TPACKET_V2:
2047 val = sizeof(struct tpacket2_hdr);
2048 break;
2049 default:
2050 return -EINVAL;
2051 }
2052 data = &val;
2053 break;
2054 case PACKET_RESERVE:
2055 if (len > sizeof(unsigned int))
2056 len = sizeof(unsigned int);
2057 val = po->tp_reserve;
2058 data = &val;
2059 break;
2060 case PACKET_LOSS:
2061 if (len > sizeof(unsigned int))
2062 len = sizeof(unsigned int);
2063 val = po->tp_loss;
2064 data = &val;
2065 break;
2066 default:
2067 return -ENOPROTOOPT;
2068 }
2069
2070 if (put_user(len, optlen))
2071 return -EFAULT;
2072 if (copy_to_user(optval, data, len))
2073 return -EFAULT;
2074 return 0;
2075 }
2076
2077
2078 static int packet_notifier(struct notifier_block *this, unsigned long msg, void *data)
2079 {
2080 struct sock *sk;
2081 struct hlist_node *node;
2082 struct net_device *dev = data;
2083 struct net *net = dev_net(dev);
2084
2085 rcu_read_lock();
2086 sk_for_each_rcu(sk, node, &net->packet.sklist) {
2087 struct packet_sock *po = pkt_sk(sk);
2088
2089 switch (msg) {
2090 case NETDEV_UNREGISTER:
2091 if (po->mclist)
2092 packet_dev_mclist(dev, po->mclist, -1);
2093 /* fallthrough */
2094
2095 case NETDEV_DOWN:
2096 if (dev->ifindex == po->ifindex) {
2097 spin_lock(&po->bind_lock);
2098 if (po->running) {
2099 __dev_remove_pack(&po->prot_hook);
2100 __sock_put(sk);
2101 po->running = 0;
2102 sk->sk_err = ENETDOWN;
2103 if (!sock_flag(sk, SOCK_DEAD))
2104 sk->sk_error_report(sk);
2105 }
2106 if (msg == NETDEV_UNREGISTER) {
2107 po->ifindex = -1;
2108 po->prot_hook.dev = NULL;
2109 }
2110 spin_unlock(&po->bind_lock);
2111 }
2112 break;
2113 case NETDEV_UP:
2114 if (dev->ifindex == po->ifindex) {
2115 spin_lock(&po->bind_lock);
2116 if (po->num && !po->running) {
2117 dev_add_pack(&po->prot_hook);
2118 sock_hold(sk);
2119 po->running = 1;
2120 }
2121 spin_unlock(&po->bind_lock);
2122 }
2123 break;
2124 }
2125 }
2126 rcu_read_unlock();
2127 return NOTIFY_DONE;
2128 }
2129
2130
2131 static int packet_ioctl(struct socket *sock, unsigned int cmd,
2132 unsigned long arg)
2133 {
2134 struct sock *sk = sock->sk;
2135
2136 switch (cmd) {
2137 case SIOCOUTQ:
2138 {
2139 int amount = sk_wmem_alloc_get(sk);
2140
2141 return put_user(amount, (int __user *)arg);
2142 }
2143 case SIOCINQ:
2144 {
2145 struct sk_buff *skb;
2146 int amount = 0;
2147
2148 spin_lock_bh(&sk->sk_receive_queue.lock);
2149 skb = skb_peek(&sk->sk_receive_queue);
2150 if (skb)
2151 amount = skb->len;
2152 spin_unlock_bh(&sk->sk_receive_queue.lock);
2153 return put_user(amount, (int __user *)arg);
2154 }
2155 case SIOCGSTAMP:
2156 return sock_get_timestamp(sk, (struct timeval __user *)arg);
2157 case SIOCGSTAMPNS:
2158 return sock_get_timestampns(sk, (struct timespec __user *)arg);
2159
2160 #ifdef CONFIG_INET
2161 case SIOCADDRT:
2162 case SIOCDELRT:
2163 case SIOCDARP:
2164 case SIOCGARP:
2165 case SIOCSARP:
2166 case SIOCGIFADDR:
2167 case SIOCSIFADDR:
2168 case SIOCGIFBRDADDR:
2169 case SIOCSIFBRDADDR:
2170 case SIOCGIFNETMASK:
2171 case SIOCSIFNETMASK:
2172 case SIOCGIFDSTADDR:
2173 case SIOCSIFDSTADDR:
2174 case SIOCSIFFLAGS:
2175 return inet_dgram_ops.ioctl(sock, cmd, arg);
2176 #endif
2177
2178 default:
2179 return -ENOIOCTLCMD;
2180 }
2181 return 0;
2182 }
2183
2184 static unsigned int packet_poll(struct file *file, struct socket *sock,
2185 poll_table *wait)
2186 {
2187 struct sock *sk = sock->sk;
2188 struct packet_sock *po = pkt_sk(sk);
2189 unsigned int mask = datagram_poll(file, sock, wait);
2190
2191 spin_lock_bh(&sk->sk_receive_queue.lock);
2192 if (po->rx_ring.pg_vec) {
2193 if (!packet_previous_frame(po, &po->rx_ring, TP_STATUS_KERNEL))
2194 mask |= POLLIN | POLLRDNORM;
2195 }
2196 spin_unlock_bh(&sk->sk_receive_queue.lock);
2197 spin_lock_bh(&sk->sk_write_queue.lock);
2198 if (po->tx_ring.pg_vec) {
2199 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE))
2200 mask |= POLLOUT | POLLWRNORM;
2201 }
2202 spin_unlock_bh(&sk->sk_write_queue.lock);
2203 return mask;
2204 }
2205
2206
2207 /* Dirty? Well, I still did not learn better way to account
2208 * for user mmaps.
2209 */
2210
2211 static void packet_mm_open(struct vm_area_struct *vma)
2212 {
2213 struct file *file = vma->vm_file;
2214 struct socket *sock = file->private_data;
2215 struct sock *sk = sock->sk;
2216
2217 if (sk)
2218 atomic_inc(&pkt_sk(sk)->mapped);
2219 }
2220
2221 static void packet_mm_close(struct vm_area_struct *vma)
2222 {
2223 struct file *file = vma->vm_file;
2224 struct socket *sock = file->private_data;
2225 struct sock *sk = sock->sk;
2226
2227 if (sk)
2228 atomic_dec(&pkt_sk(sk)->mapped);
2229 }
2230
2231 static const struct vm_operations_struct packet_mmap_ops = {
2232 .open = packet_mm_open,
2233 .close = packet_mm_close,
2234 };
2235
2236 static void free_pg_vec(char **pg_vec, unsigned int order, unsigned int len)
2237 {
2238 int i;
2239
2240 for (i = 0; i < len; i++) {
2241 if (likely(pg_vec[i]))
2242 free_pages((unsigned long) pg_vec[i], order);
2243 }
2244 kfree(pg_vec);
2245 }
2246
2247 static inline char *alloc_one_pg_vec_page(unsigned long order)
2248 {
2249 gfp_t gfp_flags = GFP_KERNEL | __GFP_COMP | __GFP_ZERO | __GFP_NOWARN;
2250
2251 return (char *) __get_free_pages(gfp_flags, order);
2252 }
2253
2254 static char **alloc_pg_vec(struct tpacket_req *req, int order)
2255 {
2256 unsigned int block_nr = req->tp_block_nr;
2257 char **pg_vec;
2258 int i;
2259
2260 pg_vec = kzalloc(block_nr * sizeof(char *), GFP_KERNEL);
2261 if (unlikely(!pg_vec))
2262 goto out;
2263
2264 for (i = 0; i < block_nr; i++) {
2265 pg_vec[i] = alloc_one_pg_vec_page(order);
2266 if (unlikely(!pg_vec[i]))
2267 goto out_free_pgvec;
2268 }
2269
2270 out:
2271 return pg_vec;
2272
2273 out_free_pgvec:
2274 free_pg_vec(pg_vec, order, block_nr);
2275 pg_vec = NULL;
2276 goto out;
2277 }
2278
2279 static int packet_set_ring(struct sock *sk, struct tpacket_req *req,
2280 int closing, int tx_ring)
2281 {
2282 char **pg_vec = NULL;
2283 struct packet_sock *po = pkt_sk(sk);
2284 int was_running, order = 0;
2285 struct packet_ring_buffer *rb;
2286 struct sk_buff_head *rb_queue;
2287 __be16 num;
2288 int err;
2289
2290 rb = tx_ring ? &po->tx_ring : &po->rx_ring;
2291 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue;
2292
2293 err = -EBUSY;
2294 if (!closing) {
2295 if (atomic_read(&po->mapped))
2296 goto out;
2297 if (atomic_read(&rb->pending))
2298 goto out;
2299 }
2300
2301 if (req->tp_block_nr) {
2302 /* Sanity tests and some calculations */
2303 err = -EBUSY;
2304 if (unlikely(rb->pg_vec))
2305 goto out;
2306
2307 switch (po->tp_version) {
2308 case TPACKET_V1:
2309 po->tp_hdrlen = TPACKET_HDRLEN;
2310 break;
2311 case TPACKET_V2:
2312 po->tp_hdrlen = TPACKET2_HDRLEN;
2313 break;
2314 }
2315
2316 err = -EINVAL;
2317 if (unlikely((int)req->tp_block_size <= 0))
2318 goto out;
2319 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1)))
2320 goto out;
2321 if (unlikely(req->tp_frame_size < po->tp_hdrlen +
2322 po->tp_reserve))
2323 goto out;
2324 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1)))
2325 goto out;
2326
2327 rb->frames_per_block = req->tp_block_size/req->tp_frame_size;
2328 if (unlikely(rb->frames_per_block <= 0))
2329 goto out;
2330 if (unlikely((rb->frames_per_block * req->tp_block_nr) !=
2331 req->tp_frame_nr))
2332 goto out;
2333
2334 err = -ENOMEM;
2335 order = get_order(req->tp_block_size);
2336 pg_vec = alloc_pg_vec(req, order);
2337 if (unlikely(!pg_vec))
2338 goto out;
2339 }
2340 /* Done */
2341 else {
2342 err = -EINVAL;
2343 if (unlikely(req->tp_frame_nr))
2344 goto out;
2345 }
2346
2347 lock_sock(sk);
2348
2349 /* Detach socket from network */
2350 spin_lock(&po->bind_lock);
2351 was_running = po->running;
2352 num = po->num;
2353 if (was_running) {
2354 __dev_remove_pack(&po->prot_hook);
2355 po->num = 0;
2356 po->running = 0;
2357 __sock_put(sk);
2358 }
2359 spin_unlock(&po->bind_lock);
2360
2361 synchronize_net();
2362
2363 err = -EBUSY;
2364 mutex_lock(&po->pg_vec_lock);
2365 if (closing || atomic_read(&po->mapped) == 0) {
2366 err = 0;
2367 #define XC(a, b) ({ __typeof__ ((a)) __t; __t = (a); (a) = (b); __t; })
2368 spin_lock_bh(&rb_queue->lock);
2369 pg_vec = XC(rb->pg_vec, pg_vec);
2370 rb->frame_max = (req->tp_frame_nr - 1);
2371 rb->head = 0;
2372 rb->frame_size = req->tp_frame_size;
2373 spin_unlock_bh(&rb_queue->lock);
2374
2375 order = XC(rb->pg_vec_order, order);
2376 req->tp_block_nr = XC(rb->pg_vec_len, req->tp_block_nr);
2377
2378 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE;
2379 po->prot_hook.func = (po->rx_ring.pg_vec) ?
2380 tpacket_rcv : packet_rcv;
2381 skb_queue_purge(rb_queue);
2382 #undef XC
2383 if (atomic_read(&po->mapped))
2384 pr_err("packet_mmap: vma is busy: %d\n",
2385 atomic_read(&po->mapped));
2386 }
2387 mutex_unlock(&po->pg_vec_lock);
2388
2389 spin_lock(&po->bind_lock);
2390 if (was_running && !po->running) {
2391 sock_hold(sk);
2392 po->running = 1;
2393 po->num = num;
2394 dev_add_pack(&po->prot_hook);
2395 }
2396 spin_unlock(&po->bind_lock);
2397
2398 release_sock(sk);
2399
2400 if (pg_vec)
2401 free_pg_vec(pg_vec, order, req->tp_block_nr);
2402 out:
2403 return err;
2404 }
2405
2406 static int packet_mmap(struct file *file, struct socket *sock,
2407 struct vm_area_struct *vma)
2408 {
2409 struct sock *sk = sock->sk;
2410 struct packet_sock *po = pkt_sk(sk);
2411 unsigned long size, expected_size;
2412 struct packet_ring_buffer *rb;
2413 unsigned long start;
2414 int err = -EINVAL;
2415 int i;
2416
2417 if (vma->vm_pgoff)
2418 return -EINVAL;
2419
2420 mutex_lock(&po->pg_vec_lock);
2421
2422 expected_size = 0;
2423 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
2424 if (rb->pg_vec) {
2425 expected_size += rb->pg_vec_len
2426 * rb->pg_vec_pages
2427 * PAGE_SIZE;
2428 }
2429 }
2430
2431 if (expected_size == 0)
2432 goto out;
2433
2434 size = vma->vm_end - vma->vm_start;
2435 if (size != expected_size)
2436 goto out;
2437
2438 start = vma->vm_start;
2439 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) {
2440 if (rb->pg_vec == NULL)
2441 continue;
2442
2443 for (i = 0; i < rb->pg_vec_len; i++) {
2444 struct page *page = virt_to_page(rb->pg_vec[i]);
2445 int pg_num;
2446
2447 for (pg_num = 0; pg_num < rb->pg_vec_pages;
2448 pg_num++, page++) {
2449 err = vm_insert_page(vma, start, page);
2450 if (unlikely(err))
2451 goto out;
2452 start += PAGE_SIZE;
2453 }
2454 }
2455 }
2456
2457 atomic_inc(&po->mapped);
2458 vma->vm_ops = &packet_mmap_ops;
2459 err = 0;
2460
2461 out:
2462 mutex_unlock(&po->pg_vec_lock);
2463 return err;
2464 }
2465
2466 static const struct proto_ops packet_ops_spkt = {
2467 .family = PF_PACKET,
2468 .owner = THIS_MODULE,
2469 .release = packet_release,
2470 .bind = packet_bind_spkt,
2471 .connect = sock_no_connect,
2472 .socketpair = sock_no_socketpair,
2473 .accept = sock_no_accept,
2474 .getname = packet_getname_spkt,
2475 .poll = datagram_poll,
2476 .ioctl = packet_ioctl,
2477 .listen = sock_no_listen,
2478 .shutdown = sock_no_shutdown,
2479 .setsockopt = sock_no_setsockopt,
2480 .getsockopt = sock_no_getsockopt,
2481 .sendmsg = packet_sendmsg_spkt,
2482 .recvmsg = packet_recvmsg,
2483 .mmap = sock_no_mmap,
2484 .sendpage = sock_no_sendpage,
2485 };
2486
2487 static const struct proto_ops packet_ops = {
2488 .family = PF_PACKET,
2489 .owner = THIS_MODULE,
2490 .release = packet_release,
2491 .bind = packet_bind,
2492 .connect = sock_no_connect,
2493 .socketpair = sock_no_socketpair,
2494 .accept = sock_no_accept,
2495 .getname = packet_getname,
2496 .poll = packet_poll,
2497 .ioctl = packet_ioctl,
2498 .listen = sock_no_listen,
2499 .shutdown = sock_no_shutdown,
2500 .setsockopt = packet_setsockopt,
2501 .getsockopt = packet_getsockopt,
2502 .sendmsg = packet_sendmsg,
2503 .recvmsg = packet_recvmsg,
2504 .mmap = packet_mmap,
2505 .sendpage = sock_no_sendpage,
2506 };
2507
2508 static const struct net_proto_family packet_family_ops = {
2509 .family = PF_PACKET,
2510 .create = packet_create,
2511 .owner = THIS_MODULE,
2512 };
2513
2514 static struct notifier_block packet_netdev_notifier = {
2515 .notifier_call = packet_notifier,
2516 };
2517
2518 #ifdef CONFIG_PROC_FS
2519
2520 static void *packet_seq_start(struct seq_file *seq, loff_t *pos)
2521 __acquires(RCU)
2522 {
2523 struct net *net = seq_file_net(seq);
2524
2525 rcu_read_lock();
2526 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos);
2527 }
2528
2529 static void *packet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2530 {
2531 struct net *net = seq_file_net(seq);
2532 return seq_hlist_next_rcu(v, &net->packet.sklist, pos);
2533 }
2534
2535 static void packet_seq_stop(struct seq_file *seq, void *v)
2536 __releases(RCU)
2537 {
2538 rcu_read_unlock();
2539 }
2540
2541 static int packet_seq_show(struct seq_file *seq, void *v)
2542 {
2543 if (v == SEQ_START_TOKEN)
2544 seq_puts(seq, "sk RefCnt Type Proto Iface R Rmem User Inode\n");
2545 else {
2546 struct sock *s = sk_entry(v);
2547 const struct packet_sock *po = pkt_sk(s);
2548
2549 seq_printf(seq,
2550 "%p %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n",
2551 s,
2552 atomic_read(&s->sk_refcnt),
2553 s->sk_type,
2554 ntohs(po->num),
2555 po->ifindex,
2556 po->running,
2557 atomic_read(&s->sk_rmem_alloc),
2558 sock_i_uid(s),
2559 sock_i_ino(s));
2560 }
2561
2562 return 0;
2563 }
2564
2565 static const struct seq_operations packet_seq_ops = {
2566 .start = packet_seq_start,
2567 .next = packet_seq_next,
2568 .stop = packet_seq_stop,
2569 .show = packet_seq_show,
2570 };
2571
2572 static int packet_seq_open(struct inode *inode, struct file *file)
2573 {
2574 return seq_open_net(inode, file, &packet_seq_ops,
2575 sizeof(struct seq_net_private));
2576 }
2577
2578 static const struct file_operations packet_seq_fops = {
2579 .owner = THIS_MODULE,
2580 .open = packet_seq_open,
2581 .read = seq_read,
2582 .llseek = seq_lseek,
2583 .release = seq_release_net,
2584 };
2585
2586 #endif
2587
2588 static int __net_init packet_net_init(struct net *net)
2589 {
2590 spin_lock_init(&net->packet.sklist_lock);
2591 INIT_HLIST_HEAD(&net->packet.sklist);
2592
2593 if (!proc_net_fops_create(net, "packet", 0, &packet_seq_fops))
2594 return -ENOMEM;
2595
2596 return 0;
2597 }
2598
2599 static void __net_exit packet_net_exit(struct net *net)
2600 {
2601 proc_net_remove(net, "packet");
2602 }
2603
2604 static struct pernet_operations packet_net_ops = {
2605 .init = packet_net_init,
2606 .exit = packet_net_exit,
2607 };
2608
2609
2610 static void __exit packet_exit(void)
2611 {
2612 unregister_netdevice_notifier(&packet_netdev_notifier);
2613 unregister_pernet_subsys(&packet_net_ops);
2614 sock_unregister(PF_PACKET);
2615 proto_unregister(&packet_proto);
2616 }
2617
2618 static int __init packet_init(void)
2619 {
2620 int rc = proto_register(&packet_proto, 0);
2621
2622 if (rc != 0)
2623 goto out;
2624
2625 sock_register(&packet_family_ops);
2626 register_pernet_subsys(&packet_net_ops);
2627 register_netdevice_notifier(&packet_netdev_notifier);
2628 out:
2629 return rc;
2630 }
2631
2632 module_init(packet_init);
2633 module_exit(packet_exit);
2634 MODULE_LICENSE("GPL");
2635 MODULE_ALIAS_NETPROTO(PF_PACKET);
Linux ARM platform port for MOXA ART SoC