?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf tools: Don't clone maps from parent when synthesizing forks
[linux/fpc-iii.git] / drivers / net / hyperv / netvsc_drv.c
blobcf36e7ff31919850ee829819f3e02d39bb99d794
1 /*
2 * Copyright (c) 2009, Microsoft Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, see <http://www.gnu.org/licenses/>.
15 *
16 * Authors:
17 * Haiyang Zhang <haiyangz@microsoft.com>
18 * Hank Janssen <hjanssen@microsoft.com>
19 */
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21
22 #include <linux/init.h>
23 #include <linux/atomic.h>
24 #include <linux/module.h>
25 #include <linux/highmem.h>
26 #include <linux/device.h>
27 #include <linux/io.h>
28 #include <linux/delay.h>
29 #include <linux/netdevice.h>
30 #include <linux/inetdevice.h>
31 #include <linux/etherdevice.h>
32 #include <linux/pci.h>
33 #include <linux/skbuff.h>
34 #include <linux/if_vlan.h>
35 #include <linux/in.h>
36 #include <linux/slab.h>
37 #include <linux/rtnetlink.h>
38 #include <linux/netpoll.h>
39
40 #include <net/arp.h>
41 #include <net/route.h>
42 #include <net/sock.h>
43 #include <net/pkt_sched.h>
44 #include <net/checksum.h>
45 #include <net/ip6_checksum.h>
46
47 #include "hyperv_net.h"
48
49 #define RING_SIZE_MIN 64
50 #define RETRY_US_LO 5000
51 #define RETRY_US_HI 10000
52 #define RETRY_MAX 2000 /* >10 sec */
53
54 #define LINKCHANGE_INT (2 * HZ)
55 #define VF_TAKEOVER_INT (HZ / 10)
56
57 static unsigned int ring_size __ro_after_init = 128;
58 module_param(ring_size, uint, 0444);
59 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)");
60 unsigned int netvsc_ring_bytes __ro_after_init;
61
62 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE |
63 NETIF_MSG_LINK | NETIF_MSG_IFUP |
64 NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR |
65 NETIF_MSG_TX_ERR;
66
67 static int debug = -1;
68 module_param(debug, int, 0444);
69 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
70
71 static LIST_HEAD(netvsc_dev_list);
72
73 static void netvsc_change_rx_flags(struct net_device *net, int change)
74 {
75 struct net_device_context *ndev_ctx = netdev_priv(net);
76 struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
77 int inc;
78
79 if (!vf_netdev)
80 return;
81
82 if (change & IFF_PROMISC) {
83 inc = (net->flags & IFF_PROMISC) ? 1 : -1;
84 dev_set_promiscuity(vf_netdev, inc);
85 }
86
87 if (change & IFF_ALLMULTI) {
88 inc = (net->flags & IFF_ALLMULTI) ? 1 : -1;
89 dev_set_allmulti(vf_netdev, inc);
90 }
91 }
92
93 static void netvsc_set_rx_mode(struct net_device *net)
94 {
95 struct net_device_context *ndev_ctx = netdev_priv(net);
96 struct net_device *vf_netdev;
97 struct netvsc_device *nvdev;
98
99 rcu_read_lock();
100 vf_netdev = rcu_dereference(ndev_ctx->vf_netdev);
101 if (vf_netdev) {
102 dev_uc_sync(vf_netdev, net);
103 dev_mc_sync(vf_netdev, net);
104 }
105
106 nvdev = rcu_dereference(ndev_ctx->nvdev);
107 if (nvdev)
108 rndis_filter_update(nvdev);
109 rcu_read_unlock();
110 }
111
112 static int netvsc_open(struct net_device *net)
113 {
114 struct net_device_context *ndev_ctx = netdev_priv(net);
115 struct net_device *vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
116 struct netvsc_device *nvdev = rtnl_dereference(ndev_ctx->nvdev);
117 struct rndis_device *rdev;
118 int ret = 0;
119
120 netif_carrier_off(net);
121
122 /* Open up the device */
123 ret = rndis_filter_open(nvdev);
124 if (ret != 0) {
125 netdev_err(net, "unable to open device (ret %d).\n", ret);
126 return ret;
127 }
128
129 rdev = nvdev->extension;
130 if (!rdev->link_state) {
131 netif_carrier_on(net);
132 netif_tx_wake_all_queues(net);
133 }
134
135 if (vf_netdev) {
136 /* Setting synthetic device up transparently sets
137 * slave as up. If open fails, then slave will be
138 * still be offline (and not used).
139 */
140 ret = dev_open(vf_netdev);
141 if (ret)
142 netdev_warn(net,
143 "unable to open slave: %s: %d\n",
144 vf_netdev->name, ret);
145 }
146 return 0;
147 }
148
149 static int netvsc_wait_until_empty(struct netvsc_device *nvdev)
150 {
151 unsigned int retry = 0;
152 int i;
153
154 /* Ensure pending bytes in ring are read */
155 for (;;) {
156 u32 aread = 0;
157
158 for (i = 0; i < nvdev->num_chn; i++) {
159 struct vmbus_channel *chn
160 = nvdev->chan_table[i].channel;
161
162 if (!chn)
163 continue;
164
165 /* make sure receive not running now */
166 napi_synchronize(&nvdev->chan_table[i].napi);
167
168 aread = hv_get_bytes_to_read(&chn->inbound);
169 if (aread)
170 break;
171
172 aread = hv_get_bytes_to_read(&chn->outbound);
173 if (aread)
174 break;
175 }
176
177 if (aread == 0)
178 return 0;
179
180 if (++retry > RETRY_MAX)
181 return -ETIMEDOUT;
182
183 usleep_range(RETRY_US_LO, RETRY_US_HI);
184 }
185 }
186
187 static int netvsc_close(struct net_device *net)
188 {
189 struct net_device_context *net_device_ctx = netdev_priv(net);
190 struct net_device *vf_netdev
191 = rtnl_dereference(net_device_ctx->vf_netdev);
192 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
193 int ret;
194
195 netif_tx_disable(net);
196
197 /* No need to close rndis filter if it is removed already */
198 if (!nvdev)
199 return 0;
200
201 ret = rndis_filter_close(nvdev);
202 if (ret != 0) {
203 netdev_err(net, "unable to close device (ret %d).\n", ret);
204 return ret;
205 }
206
207 ret = netvsc_wait_until_empty(nvdev);
208 if (ret)
209 netdev_err(net, "Ring buffer not empty after closing rndis\n");
210
211 if (vf_netdev)
212 dev_close(vf_netdev);
213
214 return ret;
215 }
216
217 static inline void *init_ppi_data(struct rndis_message *msg,
218 u32 ppi_size, u32 pkt_type)
219 {
220 struct rndis_packet *rndis_pkt = &msg->msg.pkt;
221 struct rndis_per_packet_info *ppi;
222
223 rndis_pkt->data_offset += ppi_size;
224 ppi = (void *)rndis_pkt + rndis_pkt->per_pkt_info_offset
225 + rndis_pkt->per_pkt_info_len;
226
227 ppi->size = ppi_size;
228 ppi->type = pkt_type;
229 ppi->internal = 0;
230 ppi->ppi_offset = sizeof(struct rndis_per_packet_info);
231
232 rndis_pkt->per_pkt_info_len += ppi_size;
233
234 return ppi + 1;
235 }
236
237 /* Azure hosts don't support non-TCP port numbers in hashing for fragmented
238 * packets. We can use ethtool to change UDP hash level when necessary.
239 */
240 static inline u32 netvsc_get_hash(
241 struct sk_buff *skb,
242 const struct net_device_context *ndc)
243 {
244 struct flow_keys flow;
245 u32 hash, pkt_proto = 0;
246 static u32 hashrnd __read_mostly;
247
248 net_get_random_once(&hashrnd, sizeof(hashrnd));
249
250 if (!skb_flow_dissect_flow_keys(skb, &flow, 0))
251 return 0;
252
253 switch (flow.basic.ip_proto) {
254 case IPPROTO_TCP:
255 if (flow.basic.n_proto == htons(ETH_P_IP))
256 pkt_proto = HV_TCP4_L4HASH;
257 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
258 pkt_proto = HV_TCP6_L4HASH;
259
260 break;
261
262 case IPPROTO_UDP:
263 if (flow.basic.n_proto == htons(ETH_P_IP))
264 pkt_proto = HV_UDP4_L4HASH;
265 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
266 pkt_proto = HV_UDP6_L4HASH;
267
268 break;
269 }
270
271 if (pkt_proto & ndc->l4_hash) {
272 return skb_get_hash(skb);
273 } else {
274 if (flow.basic.n_proto == htons(ETH_P_IP))
275 hash = jhash2((u32 *)&flow.addrs.v4addrs, 2, hashrnd);
276 else if (flow.basic.n_proto == htons(ETH_P_IPV6))
277 hash = jhash2((u32 *)&flow.addrs.v6addrs, 8, hashrnd);
278 else
279 hash = 0;
280
281 skb_set_hash(skb, hash, PKT_HASH_TYPE_L3);
282 }
283
284 return hash;
285 }
286
287 static inline int netvsc_get_tx_queue(struct net_device *ndev,
288 struct sk_buff *skb, int old_idx)
289 {
290 const struct net_device_context *ndc = netdev_priv(ndev);
291 struct sock *sk = skb->sk;
292 int q_idx;
293
294 q_idx = ndc->tx_table[netvsc_get_hash(skb, ndc) &
295 (VRSS_SEND_TAB_SIZE - 1)];
296
297 /* If queue index changed record the new value */
298 if (q_idx != old_idx &&
299 sk && sk_fullsock(sk) && rcu_access_pointer(sk->sk_dst_cache))
300 sk_tx_queue_set(sk, q_idx);
301
302 return q_idx;
303 }
304
305 /*
306 * Select queue for transmit.
307 *
308 * If a valid queue has already been assigned, then use that.
309 * Otherwise compute tx queue based on hash and the send table.
310 *
311 * This is basically similar to default (__netdev_pick_tx) with the added step
312 * of using the host send_table when no other queue has been assigned.
313 *
314 * TODO support XPS - but get_xps_queue not exported
315 */
316 static u16 netvsc_pick_tx(struct net_device *ndev, struct sk_buff *skb)
317 {
318 int q_idx = sk_tx_queue_get(skb->sk);
319
320 if (q_idx < 0 || skb->ooo_okay || q_idx >= ndev->real_num_tx_queues) {
321 /* If forwarding a packet, we use the recorded queue when
322 * available for better cache locality.
323 */
324 if (skb_rx_queue_recorded(skb))
325 q_idx = skb_get_rx_queue(skb);
326 else
327 q_idx = netvsc_get_tx_queue(ndev, skb, q_idx);
328 }
329
330 return q_idx;
331 }
332
333 static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb,
334 struct net_device *sb_dev,
335 select_queue_fallback_t fallback)
336 {
337 struct net_device_context *ndc = netdev_priv(ndev);
338 struct net_device *vf_netdev;
339 u16 txq;
340
341 rcu_read_lock();
342 vf_netdev = rcu_dereference(ndc->vf_netdev);
343 if (vf_netdev) {
344 const struct net_device_ops *vf_ops = vf_netdev->netdev_ops;
345
346 if (vf_ops->ndo_select_queue)
347 txq = vf_ops->ndo_select_queue(vf_netdev, skb,
348 sb_dev, fallback);
349 else
350 txq = fallback(vf_netdev, skb, NULL);
351
352 /* Record the queue selected by VF so that it can be
353 * used for common case where VF has more queues than
354 * the synthetic device.
355 */
356 qdisc_skb_cb(skb)->slave_dev_queue_mapping = txq;
357 } else {
358 txq = netvsc_pick_tx(ndev, skb);
359 }
360 rcu_read_unlock();
361
362 while (unlikely(txq >= ndev->real_num_tx_queues))
363 txq -= ndev->real_num_tx_queues;
364
365 return txq;
366 }
367
368 static u32 fill_pg_buf(struct page *page, u32 offset, u32 len,
369 struct hv_page_buffer *pb)
370 {
371 int j = 0;
372
373 /* Deal with compund pages by ignoring unused part
374 * of the page.
375 */
376 page += (offset >> PAGE_SHIFT);
377 offset &= ~PAGE_MASK;
378
379 while (len > 0) {
380 unsigned long bytes;
381
382 bytes = PAGE_SIZE - offset;
383 if (bytes > len)
384 bytes = len;
385 pb[j].pfn = page_to_pfn(page);
386 pb[j].offset = offset;
387 pb[j].len = bytes;
388
389 offset += bytes;
390 len -= bytes;
391
392 if (offset == PAGE_SIZE && len) {
393 page++;
394 offset = 0;
395 j++;
396 }
397 }
398
399 return j + 1;
400 }
401
402 static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb,
403 struct hv_netvsc_packet *packet,
404 struct hv_page_buffer *pb)
405 {
406 u32 slots_used = 0;
407 char *data = skb->data;
408 int frags = skb_shinfo(skb)->nr_frags;
409 int i;
410
411 /* The packet is laid out thus:
412 * 1. hdr: RNDIS header and PPI
413 * 2. skb linear data
414 * 3. skb fragment data
415 */
416 slots_used += fill_pg_buf(virt_to_page(hdr),
417 offset_in_page(hdr),
418 len, &pb[slots_used]);
419
420 packet->rmsg_size = len;
421 packet->rmsg_pgcnt = slots_used;
422
423 slots_used += fill_pg_buf(virt_to_page(data),
424 offset_in_page(data),
425 skb_headlen(skb), &pb[slots_used]);
426
427 for (i = 0; i < frags; i++) {
428 skb_frag_t *frag = skb_shinfo(skb)->frags + i;
429
430 slots_used += fill_pg_buf(skb_frag_page(frag),
431 frag->page_offset,
432 skb_frag_size(frag), &pb[slots_used]);
433 }
434 return slots_used;
435 }
436
437 static int count_skb_frag_slots(struct sk_buff *skb)
438 {
439 int i, frags = skb_shinfo(skb)->nr_frags;
440 int pages = 0;
441
442 for (i = 0; i < frags; i++) {
443 skb_frag_t *frag = skb_shinfo(skb)->frags + i;
444 unsigned long size = skb_frag_size(frag);
445 unsigned long offset = frag->page_offset;
446
447 /* Skip unused frames from start of page */
448 offset &= ~PAGE_MASK;
449 pages += PFN_UP(offset + size);
450 }
451 return pages;
452 }
453
454 static int netvsc_get_slots(struct sk_buff *skb)
455 {
456 char *data = skb->data;
457 unsigned int offset = offset_in_page(data);
458 unsigned int len = skb_headlen(skb);
459 int slots;
460 int frag_slots;
461
462 slots = DIV_ROUND_UP(offset + len, PAGE_SIZE);
463 frag_slots = count_skb_frag_slots(skb);
464 return slots + frag_slots;
465 }
466
467 static u32 net_checksum_info(struct sk_buff *skb)
468 {
469 if (skb->protocol == htons(ETH_P_IP)) {
470 struct iphdr *ip = ip_hdr(skb);
471
472 if (ip->protocol == IPPROTO_TCP)
473 return TRANSPORT_INFO_IPV4_TCP;
474 else if (ip->protocol == IPPROTO_UDP)
475 return TRANSPORT_INFO_IPV4_UDP;
476 } else {
477 struct ipv6hdr *ip6 = ipv6_hdr(skb);
478
479 if (ip6->nexthdr == IPPROTO_TCP)
480 return TRANSPORT_INFO_IPV6_TCP;
481 else if (ip6->nexthdr == IPPROTO_UDP)
482 return TRANSPORT_INFO_IPV6_UDP;
483 }
484
485 return TRANSPORT_INFO_NOT_IP;
486 }
487
488 /* Send skb on the slave VF device. */
489 static int netvsc_vf_xmit(struct net_device *net, struct net_device *vf_netdev,
490 struct sk_buff *skb)
491 {
492 struct net_device_context *ndev_ctx = netdev_priv(net);
493 unsigned int len = skb->len;
494 int rc;
495
496 skb->dev = vf_netdev;
497 skb->queue_mapping = qdisc_skb_cb(skb)->slave_dev_queue_mapping;
498
499 rc = dev_queue_xmit(skb);
500 if (likely(rc == NET_XMIT_SUCCESS || rc == NET_XMIT_CN)) {
501 struct netvsc_vf_pcpu_stats *pcpu_stats
502 = this_cpu_ptr(ndev_ctx->vf_stats);
503
504 u64_stats_update_begin(&pcpu_stats->syncp);
505 pcpu_stats->tx_packets++;
506 pcpu_stats->tx_bytes += len;
507 u64_stats_update_end(&pcpu_stats->syncp);
508 } else {
509 this_cpu_inc(ndev_ctx->vf_stats->tx_dropped);
510 }
511
512 return rc;
513 }
514
515 static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net)
516 {
517 struct net_device_context *net_device_ctx = netdev_priv(net);
518 struct hv_netvsc_packet *packet = NULL;
519 int ret;
520 unsigned int num_data_pgs;
521 struct rndis_message *rndis_msg;
522 struct net_device *vf_netdev;
523 u32 rndis_msg_size;
524 u32 hash;
525 struct hv_page_buffer pb[MAX_PAGE_BUFFER_COUNT];
526
527 /* if VF is present and up then redirect packets
528 * already called with rcu_read_lock_bh
529 */
530 vf_netdev = rcu_dereference_bh(net_device_ctx->vf_netdev);
531 if (vf_netdev && netif_running(vf_netdev) &&
532 !netpoll_tx_running(net))
533 return netvsc_vf_xmit(net, vf_netdev, skb);
534
535 /* We will atmost need two pages to describe the rndis
536 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number
537 * of pages in a single packet. If skb is scattered around
538 * more pages we try linearizing it.
539 */
540
541 num_data_pgs = netvsc_get_slots(skb) + 2;
542
543 if (unlikely(num_data_pgs > MAX_PAGE_BUFFER_COUNT)) {
544 ++net_device_ctx->eth_stats.tx_scattered;
545
546 if (skb_linearize(skb))
547 goto no_memory;
548
549 num_data_pgs = netvsc_get_slots(skb) + 2;
550 if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) {
551 ++net_device_ctx->eth_stats.tx_too_big;
552 goto drop;
553 }
554 }
555
556 /*
557 * Place the rndis header in the skb head room and
558 * the skb->cb will be used for hv_netvsc_packet
559 * structure.
560 */
561 ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE);
562 if (ret)
563 goto no_memory;
564
565 /* Use the skb control buffer for building up the packet */
566 BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) >
567 FIELD_SIZEOF(struct sk_buff, cb));
568 packet = (struct hv_netvsc_packet *)skb->cb;
569
570 packet->q_idx = skb_get_queue_mapping(skb);
571
572 packet->total_data_buflen = skb->len;
573 packet->total_bytes = skb->len;
574 packet->total_packets = 1;
575
576 rndis_msg = (struct rndis_message *)skb->head;
577
578 /* Add the rndis header */
579 rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET;
580 rndis_msg->msg_len = packet->total_data_buflen;
581
582 rndis_msg->msg.pkt = (struct rndis_packet) {
583 .data_offset = sizeof(struct rndis_packet),
584 .data_len = packet->total_data_buflen,
585 .per_pkt_info_offset = sizeof(struct rndis_packet),
586 };
587
588 rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet);
589
590 hash = skb_get_hash_raw(skb);
591 if (hash != 0 && net->real_num_tx_queues > 1) {
592 u32 *hash_info;
593
594 rndis_msg_size += NDIS_HASH_PPI_SIZE;
595 hash_info = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE,
596 NBL_HASH_VALUE);
597 *hash_info = hash;
598 }
599
600 if (skb_vlan_tag_present(skb)) {
601 struct ndis_pkt_8021q_info *vlan;
602
603 rndis_msg_size += NDIS_VLAN_PPI_SIZE;
604 vlan = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE,
605 IEEE_8021Q_INFO);
606
607 vlan->value = 0;
608 vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK;
609 vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >>
610 VLAN_PRIO_SHIFT;
611 }
612
613 if (skb_is_gso(skb)) {
614 struct ndis_tcp_lso_info *lso_info;
615
616 rndis_msg_size += NDIS_LSO_PPI_SIZE;
617 lso_info = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE,
618 TCP_LARGESEND_PKTINFO);
619
620 lso_info->value = 0;
621 lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE;
622 if (skb->protocol == htons(ETH_P_IP)) {
623 lso_info->lso_v2_transmit.ip_version =
624 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4;
625 ip_hdr(skb)->tot_len = 0;
626 ip_hdr(skb)->check = 0;
627 tcp_hdr(skb)->check =
628 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
629 ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
630 } else {
631 lso_info->lso_v2_transmit.ip_version =
632 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6;
633 ipv6_hdr(skb)->payload_len = 0;
634 tcp_hdr(skb)->check =
635 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
636 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
637 }
638 lso_info->lso_v2_transmit.tcp_header_offset = skb_transport_offset(skb);
639 lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size;
640 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
641 if (net_checksum_info(skb) & net_device_ctx->tx_checksum_mask) {
642 struct ndis_tcp_ip_checksum_info *csum_info;
643
644 rndis_msg_size += NDIS_CSUM_PPI_SIZE;
645 csum_info = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE,
646 TCPIP_CHKSUM_PKTINFO);
647
648 csum_info->value = 0;
649 csum_info->transmit.tcp_header_offset = skb_transport_offset(skb);
650
651 if (skb->protocol == htons(ETH_P_IP)) {
652 csum_info->transmit.is_ipv4 = 1;
653
654 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
655 csum_info->transmit.tcp_checksum = 1;
656 else
657 csum_info->transmit.udp_checksum = 1;
658 } else {
659 csum_info->transmit.is_ipv6 = 1;
660
661 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
662 csum_info->transmit.tcp_checksum = 1;
663 else
664 csum_info->transmit.udp_checksum = 1;
665 }
666 } else {
667 /* Can't do offload of this type of checksum */
668 if (skb_checksum_help(skb))
669 goto drop;
670 }
671 }
672
673 /* Start filling in the page buffers with the rndis hdr */
674 rndis_msg->msg_len += rndis_msg_size;
675 packet->total_data_buflen = rndis_msg->msg_len;
676 packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size,
677 skb, packet, pb);
678
679 /* timestamp packet in software */
680 skb_tx_timestamp(skb);
681
682 ret = netvsc_send(net, packet, rndis_msg, pb, skb);
683 if (likely(ret == 0))
684 return NETDEV_TX_OK;
685
686 if (ret == -EAGAIN) {
687 ++net_device_ctx->eth_stats.tx_busy;
688 return NETDEV_TX_BUSY;
689 }
690
691 if (ret == -ENOSPC)
692 ++net_device_ctx->eth_stats.tx_no_space;
693
694 drop:
695 dev_kfree_skb_any(skb);
696 net->stats.tx_dropped++;
697
698 return NETDEV_TX_OK;
699
700 no_memory:
701 ++net_device_ctx->eth_stats.tx_no_memory;
702 goto drop;
703 }
704
705 /*
706 * netvsc_linkstatus_callback - Link up/down notification
707 */
708 void netvsc_linkstatus_callback(struct net_device *net,
709 struct rndis_message *resp)
710 {
711 struct rndis_indicate_status *indicate = &resp->msg.indicate_status;
712 struct net_device_context *ndev_ctx = netdev_priv(net);
713 struct netvsc_reconfig *event;
714 unsigned long flags;
715
716 /* Update the physical link speed when changing to another vSwitch */
717 if (indicate->status == RNDIS_STATUS_LINK_SPEED_CHANGE) {
718 u32 speed;
719
720 speed = *(u32 *)((void *)indicate
721 + indicate->status_buf_offset) / 10000;
722 ndev_ctx->speed = speed;
723 return;
724 }
725
726 /* Handle these link change statuses below */
727 if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE &&
728 indicate->status != RNDIS_STATUS_MEDIA_CONNECT &&
729 indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT)
730 return;
731
732 if (net->reg_state != NETREG_REGISTERED)
733 return;
734
735 event = kzalloc(sizeof(*event), GFP_ATOMIC);
736 if (!event)
737 return;
738 event->event = indicate->status;
739
740 spin_lock_irqsave(&ndev_ctx->lock, flags);
741 list_add_tail(&event->list, &ndev_ctx->reconfig_events);
742 spin_unlock_irqrestore(&ndev_ctx->lock, flags);
743
744 schedule_delayed_work(&ndev_ctx->dwork, 0);
745 }
746
747 static struct sk_buff *netvsc_alloc_recv_skb(struct net_device *net,
748 struct netvsc_channel *nvchan)
749 {
750 struct napi_struct *napi = &nvchan->napi;
751 const struct ndis_pkt_8021q_info *vlan = nvchan->rsc.vlan;
752 const struct ndis_tcp_ip_checksum_info *csum_info =
753 nvchan->rsc.csum_info;
754 struct sk_buff *skb;
755 int i;
756
757 skb = napi_alloc_skb(napi, nvchan->rsc.pktlen);
758 if (!skb)
759 return skb;
760
761 /*
762 * Copy to skb. This copy is needed here since the memory pointed by
763 * hv_netvsc_packet cannot be deallocated
764 */
765 for (i = 0; i < nvchan->rsc.cnt; i++)
766 skb_put_data(skb, nvchan->rsc.data[i], nvchan->rsc.len[i]);
767
768 skb->protocol = eth_type_trans(skb, net);
769
770 /* skb is already created with CHECKSUM_NONE */
771 skb_checksum_none_assert(skb);
772
773 /*
774 * In Linux, the IP checksum is always checked.
775 * Do L4 checksum offload if enabled and present.
776 */
777 if (csum_info && (net->features & NETIF_F_RXCSUM)) {
778 if (csum_info->receive.tcp_checksum_succeeded ||
779 csum_info->receive.udp_checksum_succeeded)
780 skb->ip_summed = CHECKSUM_UNNECESSARY;
781 }
782
783 if (vlan) {
784 u16 vlan_tci = vlan->vlanid | (vlan->pri << VLAN_PRIO_SHIFT);
785
786 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
787 vlan_tci);
788 }
789
790 return skb;
791 }
792
793 /*
794 * netvsc_recv_callback - Callback when we receive a packet from the
795 * "wire" on the specified device.
796 */
797 int netvsc_recv_callback(struct net_device *net,
798 struct netvsc_device *net_device,
799 struct netvsc_channel *nvchan)
800 {
801 struct net_device_context *net_device_ctx = netdev_priv(net);
802 struct vmbus_channel *channel = nvchan->channel;
803 u16 q_idx = channel->offermsg.offer.sub_channel_index;
804 struct sk_buff *skb;
805 struct netvsc_stats *rx_stats;
806
807 if (net->reg_state != NETREG_REGISTERED)
808 return NVSP_STAT_FAIL;
809
810 /* Allocate a skb - TODO direct I/O to pages? */
811 skb = netvsc_alloc_recv_skb(net, nvchan);
812
813 if (unlikely(!skb)) {
814 ++net_device_ctx->eth_stats.rx_no_memory;
815 rcu_read_unlock();
816 return NVSP_STAT_FAIL;
817 }
818
819 skb_record_rx_queue(skb, q_idx);
820
821 /*
822 * Even if injecting the packet, record the statistics
823 * on the synthetic device because modifying the VF device
824 * statistics will not work correctly.
825 */
826 rx_stats = &nvchan->rx_stats;
827 u64_stats_update_begin(&rx_stats->syncp);
828 rx_stats->packets++;
829 rx_stats->bytes += nvchan->rsc.pktlen;
830
831 if (skb->pkt_type == PACKET_BROADCAST)
832 ++rx_stats->broadcast;
833 else if (skb->pkt_type == PACKET_MULTICAST)
834 ++rx_stats->multicast;
835 u64_stats_update_end(&rx_stats->syncp);
836
837 napi_gro_receive(&nvchan->napi, skb);
838 return NVSP_STAT_SUCCESS;
839 }
840
841 static void netvsc_get_drvinfo(struct net_device *net,
842 struct ethtool_drvinfo *info)
843 {
844 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
845 strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
846 }
847
848 static void netvsc_get_channels(struct net_device *net,
849 struct ethtool_channels *channel)
850 {
851 struct net_device_context *net_device_ctx = netdev_priv(net);
852 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
853
854 if (nvdev) {
855 channel->max_combined = nvdev->max_chn;
856 channel->combined_count = nvdev->num_chn;
857 }
858 }
859
860 static int netvsc_detach(struct net_device *ndev,
861 struct netvsc_device *nvdev)
862 {
863 struct net_device_context *ndev_ctx = netdev_priv(ndev);
864 struct hv_device *hdev = ndev_ctx->device_ctx;
865 int ret;
866
867 /* Don't try continuing to try and setup sub channels */
868 if (cancel_work_sync(&nvdev->subchan_work))
869 nvdev->num_chn = 1;
870
871 /* If device was up (receiving) then shutdown */
872 if (netif_running(ndev)) {
873 netif_tx_disable(ndev);
874
875 ret = rndis_filter_close(nvdev);
876 if (ret) {
877 netdev_err(ndev,
878 "unable to close device (ret %d).\n", ret);
879 return ret;
880 }
881
882 ret = netvsc_wait_until_empty(nvdev);
883 if (ret) {
884 netdev_err(ndev,
885 "Ring buffer not empty after closing rndis\n");
886 return ret;
887 }
888 }
889
890 netif_device_detach(ndev);
891
892 rndis_filter_device_remove(hdev, nvdev);
893
894 return 0;
895 }
896
897 static int netvsc_attach(struct net_device *ndev,
898 struct netvsc_device_info *dev_info)
899 {
900 struct net_device_context *ndev_ctx = netdev_priv(ndev);
901 struct hv_device *hdev = ndev_ctx->device_ctx;
902 struct netvsc_device *nvdev;
903 struct rndis_device *rdev;
904 int ret;
905
906 nvdev = rndis_filter_device_add(hdev, dev_info);
907 if (IS_ERR(nvdev))
908 return PTR_ERR(nvdev);
909
910 if (nvdev->num_chn > 1) {
911 ret = rndis_set_subchannel(ndev, nvdev);
912
913 /* if unavailable, just proceed with one queue */
914 if (ret) {
915 nvdev->max_chn = 1;
916 nvdev->num_chn = 1;
917 }
918 }
919
920 /* In any case device is now ready */
921 netif_device_attach(ndev);
922
923 /* Note: enable and attach happen when sub-channels setup */
924 netif_carrier_off(ndev);
925
926 if (netif_running(ndev)) {
927 ret = rndis_filter_open(nvdev);
928 if (ret)
929 return ret;
930
931 rdev = nvdev->extension;
932 if (!rdev->link_state)
933 netif_carrier_on(ndev);
934 }
935
936 return 0;
937 }
938
939 static int netvsc_set_channels(struct net_device *net,
940 struct ethtool_channels *channels)
941 {
942 struct net_device_context *net_device_ctx = netdev_priv(net);
943 struct netvsc_device *nvdev = rtnl_dereference(net_device_ctx->nvdev);
944 unsigned int orig, count = channels->combined_count;
945 struct netvsc_device_info device_info;
946 int ret;
947
948 /* We do not support separate count for rx, tx, or other */
949 if (count == 0 ||
950 channels->rx_count || channels->tx_count || channels->other_count)
951 return -EINVAL;
952
953 if (!nvdev || nvdev->destroy)
954 return -ENODEV;
955
956 if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5)
957 return -EINVAL;
958
959 if (count > nvdev->max_chn)
960 return -EINVAL;
961
962 orig = nvdev->num_chn;
963
964 memset(&device_info, 0, sizeof(device_info));
965 device_info.num_chn = count;
966 device_info.send_sections = nvdev->send_section_cnt;
967 device_info.send_section_size = nvdev->send_section_size;
968 device_info.recv_sections = nvdev->recv_section_cnt;
969 device_info.recv_section_size = nvdev->recv_section_size;
970
971 ret = netvsc_detach(net, nvdev);
972 if (ret)
973 return ret;
974
975 ret = netvsc_attach(net, &device_info);
976 if (ret) {
977 device_info.num_chn = orig;
978 if (netvsc_attach(net, &device_info))
979 netdev_err(net, "restoring channel setting failed\n");
980 }
981
982 return ret;
983 }
984
985 static bool
986 netvsc_validate_ethtool_ss_cmd(const struct ethtool_link_ksettings *cmd)
987 {
988 struct ethtool_link_ksettings diff1 = *cmd;
989 struct ethtool_link_ksettings diff2 = {};
990
991 diff1.base.speed = 0;
992 diff1.base.duplex = 0;
993 /* advertising and cmd are usually set */
994 ethtool_link_ksettings_zero_link_mode(&diff1, advertising);
995 diff1.base.cmd = 0;
996 /* We set port to PORT_OTHER */
997 diff2.base.port = PORT_OTHER;
998
999 return !memcmp(&diff1, &diff2, sizeof(diff1));
1000 }
1001
1002 static void netvsc_init_settings(struct net_device *dev)
1003 {
1004 struct net_device_context *ndc = netdev_priv(dev);
1005
1006 ndc->l4_hash = HV_DEFAULT_L4HASH;
1007
1008 ndc->speed = SPEED_UNKNOWN;
1009 ndc->duplex = DUPLEX_FULL;
1010
1011 dev->features = NETIF_F_LRO;
1012 }
1013
1014 static int netvsc_get_link_ksettings(struct net_device *dev,
1015 struct ethtool_link_ksettings *cmd)
1016 {
1017 struct net_device_context *ndc = netdev_priv(dev);
1018
1019 cmd->base.speed = ndc->speed;
1020 cmd->base.duplex = ndc->duplex;
1021 cmd->base.port = PORT_OTHER;
1022
1023 return 0;
1024 }
1025
1026 static int netvsc_set_link_ksettings(struct net_device *dev,
1027 const struct ethtool_link_ksettings *cmd)
1028 {
1029 struct net_device_context *ndc = netdev_priv(dev);
1030 u32 speed;
1031
1032 speed = cmd->base.speed;
1033 if (!ethtool_validate_speed(speed) ||
1034 !ethtool_validate_duplex(cmd->base.duplex) ||
1035 !netvsc_validate_ethtool_ss_cmd(cmd))
1036 return -EINVAL;
1037
1038 ndc->speed = speed;
1039 ndc->duplex = cmd->base.duplex;
1040
1041 return 0;
1042 }
1043
1044 static int netvsc_change_mtu(struct net_device *ndev, int mtu)
1045 {
1046 struct net_device_context *ndevctx = netdev_priv(ndev);
1047 struct net_device *vf_netdev = rtnl_dereference(ndevctx->vf_netdev);
1048 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1049 int orig_mtu = ndev->mtu;
1050 struct netvsc_device_info device_info;
1051 int ret = 0;
1052
1053 if (!nvdev || nvdev->destroy)
1054 return -ENODEV;
1055
1056 /* Change MTU of underlying VF netdev first. */
1057 if (vf_netdev) {
1058 ret = dev_set_mtu(vf_netdev, mtu);
1059 if (ret)
1060 return ret;
1061 }
1062
1063 memset(&device_info, 0, sizeof(device_info));
1064 device_info.num_chn = nvdev->num_chn;
1065 device_info.send_sections = nvdev->send_section_cnt;
1066 device_info.send_section_size = nvdev->send_section_size;
1067 device_info.recv_sections = nvdev->recv_section_cnt;
1068 device_info.recv_section_size = nvdev->recv_section_size;
1069
1070 ret = netvsc_detach(ndev, nvdev);
1071 if (ret)
1072 goto rollback_vf;
1073
1074 ndev->mtu = mtu;
1075
1076 ret = netvsc_attach(ndev, &device_info);
1077 if (ret)
1078 goto rollback;
1079
1080 return 0;
1081
1082 rollback:
1083 /* Attempt rollback to original MTU */
1084 ndev->mtu = orig_mtu;
1085
1086 if (netvsc_attach(ndev, &device_info))
1087 netdev_err(ndev, "restoring mtu failed\n");
1088 rollback_vf:
1089 if (vf_netdev)
1090 dev_set_mtu(vf_netdev, orig_mtu);
1091
1092 return ret;
1093 }
1094
1095 static void netvsc_get_vf_stats(struct net_device *net,
1096 struct netvsc_vf_pcpu_stats *tot)
1097 {
1098 struct net_device_context *ndev_ctx = netdev_priv(net);
1099 int i;
1100
1101 memset(tot, 0, sizeof(*tot));
1102
1103 for_each_possible_cpu(i) {
1104 const struct netvsc_vf_pcpu_stats *stats
1105 = per_cpu_ptr(ndev_ctx->vf_stats, i);
1106 u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
1107 unsigned int start;
1108
1109 do {
1110 start = u64_stats_fetch_begin_irq(&stats->syncp);
1111 rx_packets = stats->rx_packets;
1112 tx_packets = stats->tx_packets;
1113 rx_bytes = stats->rx_bytes;
1114 tx_bytes = stats->tx_bytes;
1115 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1116
1117 tot->rx_packets += rx_packets;
1118 tot->tx_packets += tx_packets;
1119 tot->rx_bytes += rx_bytes;
1120 tot->tx_bytes += tx_bytes;
1121 tot->tx_dropped += stats->tx_dropped;
1122 }
1123 }
1124
1125 static void netvsc_get_pcpu_stats(struct net_device *net,
1126 struct netvsc_ethtool_pcpu_stats *pcpu_tot)
1127 {
1128 struct net_device_context *ndev_ctx = netdev_priv(net);
1129 struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev);
1130 int i;
1131
1132 /* fetch percpu stats of vf */
1133 for_each_possible_cpu(i) {
1134 const struct netvsc_vf_pcpu_stats *stats =
1135 per_cpu_ptr(ndev_ctx->vf_stats, i);
1136 struct netvsc_ethtool_pcpu_stats *this_tot = &pcpu_tot[i];
1137 unsigned int start;
1138
1139 do {
1140 start = u64_stats_fetch_begin_irq(&stats->syncp);
1141 this_tot->vf_rx_packets = stats->rx_packets;
1142 this_tot->vf_tx_packets = stats->tx_packets;
1143 this_tot->vf_rx_bytes = stats->rx_bytes;
1144 this_tot->vf_tx_bytes = stats->tx_bytes;
1145 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1146 this_tot->rx_packets = this_tot->vf_rx_packets;
1147 this_tot->tx_packets = this_tot->vf_tx_packets;
1148 this_tot->rx_bytes = this_tot->vf_rx_bytes;
1149 this_tot->tx_bytes = this_tot->vf_tx_bytes;
1150 }
1151
1152 /* fetch percpu stats of netvsc */
1153 for (i = 0; i < nvdev->num_chn; i++) {
1154 const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
1155 const struct netvsc_stats *stats;
1156 struct netvsc_ethtool_pcpu_stats *this_tot =
1157 &pcpu_tot[nvchan->channel->target_cpu];
1158 u64 packets, bytes;
1159 unsigned int start;
1160
1161 stats = &nvchan->tx_stats;
1162 do {
1163 start = u64_stats_fetch_begin_irq(&stats->syncp);
1164 packets = stats->packets;
1165 bytes = stats->bytes;
1166 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1167
1168 this_tot->tx_bytes += bytes;
1169 this_tot->tx_packets += packets;
1170
1171 stats = &nvchan->rx_stats;
1172 do {
1173 start = u64_stats_fetch_begin_irq(&stats->syncp);
1174 packets = stats->packets;
1175 bytes = stats->bytes;
1176 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1177
1178 this_tot->rx_bytes += bytes;
1179 this_tot->rx_packets += packets;
1180 }
1181 }
1182
1183 static void netvsc_get_stats64(struct net_device *net,
1184 struct rtnl_link_stats64 *t)
1185 {
1186 struct net_device_context *ndev_ctx = netdev_priv(net);
1187 struct netvsc_device *nvdev = rcu_dereference_rtnl(ndev_ctx->nvdev);
1188 struct netvsc_vf_pcpu_stats vf_tot;
1189 int i;
1190
1191 if (!nvdev)
1192 return;
1193
1194 netdev_stats_to_stats64(t, &net->stats);
1195
1196 netvsc_get_vf_stats(net, &vf_tot);
1197 t->rx_packets += vf_tot.rx_packets;
1198 t->tx_packets += vf_tot.tx_packets;
1199 t->rx_bytes += vf_tot.rx_bytes;
1200 t->tx_bytes += vf_tot.tx_bytes;
1201 t->tx_dropped += vf_tot.tx_dropped;
1202
1203 for (i = 0; i < nvdev->num_chn; i++) {
1204 const struct netvsc_channel *nvchan = &nvdev->chan_table[i];
1205 const struct netvsc_stats *stats;
1206 u64 packets, bytes, multicast;
1207 unsigned int start;
1208
1209 stats = &nvchan->tx_stats;
1210 do {
1211 start = u64_stats_fetch_begin_irq(&stats->syncp);
1212 packets = stats->packets;
1213 bytes = stats->bytes;
1214 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1215
1216 t->tx_bytes += bytes;
1217 t->tx_packets += packets;
1218
1219 stats = &nvchan->rx_stats;
1220 do {
1221 start = u64_stats_fetch_begin_irq(&stats->syncp);
1222 packets = stats->packets;
1223 bytes = stats->bytes;
1224 multicast = stats->multicast + stats->broadcast;
1225 } while (u64_stats_fetch_retry_irq(&stats->syncp, start));
1226
1227 t->rx_bytes += bytes;
1228 t->rx_packets += packets;
1229 t->multicast += multicast;
1230 }
1231 }
1232
1233 static int netvsc_set_mac_addr(struct net_device *ndev, void *p)
1234 {
1235 struct net_device_context *ndc = netdev_priv(ndev);
1236 struct net_device *vf_netdev = rtnl_dereference(ndc->vf_netdev);
1237 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1238 struct sockaddr *addr = p;
1239 int err;
1240
1241 err = eth_prepare_mac_addr_change(ndev, p);
1242 if (err)
1243 return err;
1244
1245 if (!nvdev)
1246 return -ENODEV;
1247
1248 if (vf_netdev) {
1249 err = dev_set_mac_address(vf_netdev, addr);
1250 if (err)
1251 return err;
1252 }
1253
1254 err = rndis_filter_set_device_mac(nvdev, addr->sa_data);
1255 if (!err) {
1256 eth_commit_mac_addr_change(ndev, p);
1257 } else if (vf_netdev) {
1258 /* rollback change on VF */
1259 memcpy(addr->sa_data, ndev->dev_addr, ETH_ALEN);
1260 dev_set_mac_address(vf_netdev, addr);
1261 }
1262
1263 return err;
1264 }
1265
1266 static const struct {
1267 char name[ETH_GSTRING_LEN];
1268 u16 offset;
1269 } netvsc_stats[] = {
1270 { "tx_scattered", offsetof(struct netvsc_ethtool_stats, tx_scattered) },
1271 { "tx_no_memory", offsetof(struct netvsc_ethtool_stats, tx_no_memory) },
1272 { "tx_no_space", offsetof(struct netvsc_ethtool_stats, tx_no_space) },
1273 { "tx_too_big", offsetof(struct netvsc_ethtool_stats, tx_too_big) },
1274 { "tx_busy", offsetof(struct netvsc_ethtool_stats, tx_busy) },
1275 { "tx_send_full", offsetof(struct netvsc_ethtool_stats, tx_send_full) },
1276 { "rx_comp_busy", offsetof(struct netvsc_ethtool_stats, rx_comp_busy) },
1277 { "rx_no_memory", offsetof(struct netvsc_ethtool_stats, rx_no_memory) },
1278 { "stop_queue", offsetof(struct netvsc_ethtool_stats, stop_queue) },
1279 { "wake_queue", offsetof(struct netvsc_ethtool_stats, wake_queue) },
1280 }, pcpu_stats[] = {
1281 { "cpu%u_rx_packets",
1282 offsetof(struct netvsc_ethtool_pcpu_stats, rx_packets) },
1283 { "cpu%u_rx_bytes",
1284 offsetof(struct netvsc_ethtool_pcpu_stats, rx_bytes) },
1285 { "cpu%u_tx_packets",
1286 offsetof(struct netvsc_ethtool_pcpu_stats, tx_packets) },
1287 { "cpu%u_tx_bytes",
1288 offsetof(struct netvsc_ethtool_pcpu_stats, tx_bytes) },
1289 { "cpu%u_vf_rx_packets",
1290 offsetof(struct netvsc_ethtool_pcpu_stats, vf_rx_packets) },
1291 { "cpu%u_vf_rx_bytes",
1292 offsetof(struct netvsc_ethtool_pcpu_stats, vf_rx_bytes) },
1293 { "cpu%u_vf_tx_packets",
1294 offsetof(struct netvsc_ethtool_pcpu_stats, vf_tx_packets) },
1295 { "cpu%u_vf_tx_bytes",
1296 offsetof(struct netvsc_ethtool_pcpu_stats, vf_tx_bytes) },
1297 }, vf_stats[] = {
1298 { "vf_rx_packets", offsetof(struct netvsc_vf_pcpu_stats, rx_packets) },
1299 { "vf_rx_bytes", offsetof(struct netvsc_vf_pcpu_stats, rx_bytes) },
1300 { "vf_tx_packets", offsetof(struct netvsc_vf_pcpu_stats, tx_packets) },
1301 { "vf_tx_bytes", offsetof(struct netvsc_vf_pcpu_stats, tx_bytes) },
1302 { "vf_tx_dropped", offsetof(struct netvsc_vf_pcpu_stats, tx_dropped) },
1303 };
1304
1305 #define NETVSC_GLOBAL_STATS_LEN ARRAY_SIZE(netvsc_stats)
1306 #define NETVSC_VF_STATS_LEN ARRAY_SIZE(vf_stats)
1307
1308 /* statistics per queue (rx/tx packets/bytes) */
1309 #define NETVSC_PCPU_STATS_LEN (num_present_cpus() * ARRAY_SIZE(pcpu_stats))
1310
1311 /* 4 statistics per queue (rx/tx packets/bytes) */
1312 #define NETVSC_QUEUE_STATS_LEN(dev) ((dev)->num_chn * 4)
1313
1314 static int netvsc_get_sset_count(struct net_device *dev, int string_set)
1315 {
1316 struct net_device_context *ndc = netdev_priv(dev);
1317 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1318
1319 if (!nvdev)
1320 return -ENODEV;
1321
1322 switch (string_set) {
1323 case ETH_SS_STATS:
1324 return NETVSC_GLOBAL_STATS_LEN
1325 + NETVSC_VF_STATS_LEN
1326 + NETVSC_QUEUE_STATS_LEN(nvdev)
1327 + NETVSC_PCPU_STATS_LEN;
1328 default:
1329 return -EINVAL;
1330 }
1331 }
1332
1333 static void netvsc_get_ethtool_stats(struct net_device *dev,
1334 struct ethtool_stats *stats, u64 *data)
1335 {
1336 struct net_device_context *ndc = netdev_priv(dev);
1337 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1338 const void *nds = &ndc->eth_stats;
1339 const struct netvsc_stats *qstats;
1340 struct netvsc_vf_pcpu_stats sum;
1341 struct netvsc_ethtool_pcpu_stats *pcpu_sum;
1342 unsigned int start;
1343 u64 packets, bytes;
1344 int i, j, cpu;
1345
1346 if (!nvdev)
1347 return;
1348
1349 for (i = 0; i < NETVSC_GLOBAL_STATS_LEN; i++)
1350 data[i] = *(unsigned long *)(nds + netvsc_stats[i].offset);
1351
1352 netvsc_get_vf_stats(dev, &sum);
1353 for (j = 0; j < NETVSC_VF_STATS_LEN; j++)
1354 data[i++] = *(u64 *)((void *)&sum + vf_stats[j].offset);
1355
1356 for (j = 0; j < nvdev->num_chn; j++) {
1357 qstats = &nvdev->chan_table[j].tx_stats;
1358
1359 do {
1360 start = u64_stats_fetch_begin_irq(&qstats->syncp);
1361 packets = qstats->packets;
1362 bytes = qstats->bytes;
1363 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1364 data[i++] = packets;
1365 data[i++] = bytes;
1366
1367 qstats = &nvdev->chan_table[j].rx_stats;
1368 do {
1369 start = u64_stats_fetch_begin_irq(&qstats->syncp);
1370 packets = qstats->packets;
1371 bytes = qstats->bytes;
1372 } while (u64_stats_fetch_retry_irq(&qstats->syncp, start));
1373 data[i++] = packets;
1374 data[i++] = bytes;
1375 }
1376
1377 pcpu_sum = kvmalloc_array(num_possible_cpus(),
1378 sizeof(struct netvsc_ethtool_pcpu_stats),
1379 GFP_KERNEL);
1380 netvsc_get_pcpu_stats(dev, pcpu_sum);
1381 for_each_present_cpu(cpu) {
1382 struct netvsc_ethtool_pcpu_stats *this_sum = &pcpu_sum[cpu];
1383
1384 for (j = 0; j < ARRAY_SIZE(pcpu_stats); j++)
1385 data[i++] = *(u64 *)((void *)this_sum
1386 + pcpu_stats[j].offset);
1387 }
1388 kvfree(pcpu_sum);
1389 }
1390
1391 static void netvsc_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1392 {
1393 struct net_device_context *ndc = netdev_priv(dev);
1394 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1395 u8 *p = data;
1396 int i, cpu;
1397
1398 if (!nvdev)
1399 return;
1400
1401 switch (stringset) {
1402 case ETH_SS_STATS:
1403 for (i = 0; i < ARRAY_SIZE(netvsc_stats); i++) {
1404 memcpy(p, netvsc_stats[i].name, ETH_GSTRING_LEN);
1405 p += ETH_GSTRING_LEN;
1406 }
1407
1408 for (i = 0; i < ARRAY_SIZE(vf_stats); i++) {
1409 memcpy(p, vf_stats[i].name, ETH_GSTRING_LEN);
1410 p += ETH_GSTRING_LEN;
1411 }
1412
1413 for (i = 0; i < nvdev->num_chn; i++) {
1414 sprintf(p, "tx_queue_%u_packets", i);
1415 p += ETH_GSTRING_LEN;
1416 sprintf(p, "tx_queue_%u_bytes", i);
1417 p += ETH_GSTRING_LEN;
1418 sprintf(p, "rx_queue_%u_packets", i);
1419 p += ETH_GSTRING_LEN;
1420 sprintf(p, "rx_queue_%u_bytes", i);
1421 p += ETH_GSTRING_LEN;
1422 }
1423
1424 for_each_present_cpu(cpu) {
1425 for (i = 0; i < ARRAY_SIZE(pcpu_stats); i++) {
1426 sprintf(p, pcpu_stats[i].name, cpu);
1427 p += ETH_GSTRING_LEN;
1428 }
1429 }
1430
1431 break;
1432 }
1433 }
1434
1435 static int
1436 netvsc_get_rss_hash_opts(struct net_device_context *ndc,
1437 struct ethtool_rxnfc *info)
1438 {
1439 const u32 l4_flag = RXH_L4_B_0_1 | RXH_L4_B_2_3;
1440
1441 info->data = RXH_IP_SRC | RXH_IP_DST;
1442
1443 switch (info->flow_type) {
1444 case TCP_V4_FLOW:
1445 if (ndc->l4_hash & HV_TCP4_L4HASH)
1446 info->data |= l4_flag;
1447
1448 break;
1449
1450 case TCP_V6_FLOW:
1451 if (ndc->l4_hash & HV_TCP6_L4HASH)
1452 info->data |= l4_flag;
1453
1454 break;
1455
1456 case UDP_V4_FLOW:
1457 if (ndc->l4_hash & HV_UDP4_L4HASH)
1458 info->data |= l4_flag;
1459
1460 break;
1461
1462 case UDP_V6_FLOW:
1463 if (ndc->l4_hash & HV_UDP6_L4HASH)
1464 info->data |= l4_flag;
1465
1466 break;
1467
1468 case IPV4_FLOW:
1469 case IPV6_FLOW:
1470 break;
1471 default:
1472 info->data = 0;
1473 break;
1474 }
1475
1476 return 0;
1477 }
1478
1479 static int
1480 netvsc_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1481 u32 *rules)
1482 {
1483 struct net_device_context *ndc = netdev_priv(dev);
1484 struct netvsc_device *nvdev = rtnl_dereference(ndc->nvdev);
1485
1486 if (!nvdev)
1487 return -ENODEV;
1488
1489 switch (info->cmd) {
1490 case ETHTOOL_GRXRINGS:
1491 info->data = nvdev->num_chn;
1492 return 0;
1493
1494 case ETHTOOL_GRXFH:
1495 return netvsc_get_rss_hash_opts(ndc, info);
1496 }
1497 return -EOPNOTSUPP;
1498 }
1499
1500 static int netvsc_set_rss_hash_opts(struct net_device_context *ndc,
1501 struct ethtool_rxnfc *info)
1502 {
1503 if (info->data == (RXH_IP_SRC | RXH_IP_DST |
1504 RXH_L4_B_0_1 | RXH_L4_B_2_3)) {
1505 switch (info->flow_type) {
1506 case TCP_V4_FLOW:
1507 ndc->l4_hash |= HV_TCP4_L4HASH;
1508 break;
1509
1510 case TCP_V6_FLOW:
1511 ndc->l4_hash |= HV_TCP6_L4HASH;
1512 break;
1513
1514 case UDP_V4_FLOW:
1515 ndc->l4_hash |= HV_UDP4_L4HASH;
1516 break;
1517
1518 case UDP_V6_FLOW:
1519 ndc->l4_hash |= HV_UDP6_L4HASH;
1520 break;
1521
1522 default:
1523 return -EOPNOTSUPP;
1524 }
1525
1526 return 0;
1527 }
1528
1529 if (info->data == (RXH_IP_SRC | RXH_IP_DST)) {
1530 switch (info->flow_type) {
1531 case TCP_V4_FLOW:
1532 ndc->l4_hash &= ~HV_TCP4_L4HASH;
1533 break;
1534
1535 case TCP_V6_FLOW:
1536 ndc->l4_hash &= ~HV_TCP6_L4HASH;
1537 break;
1538
1539 case UDP_V4_FLOW:
1540 ndc->l4_hash &= ~HV_UDP4_L4HASH;
1541 break;
1542
1543 case UDP_V6_FLOW:
1544 ndc->l4_hash &= ~HV_UDP6_L4HASH;
1545 break;
1546
1547 default:
1548 return -EOPNOTSUPP;
1549 }
1550
1551 return 0;
1552 }
1553
1554 return -EOPNOTSUPP;
1555 }
1556
1557 static int
1558 netvsc_set_rxnfc(struct net_device *ndev, struct ethtool_rxnfc *info)
1559 {
1560 struct net_device_context *ndc = netdev_priv(ndev);
1561
1562 if (info->cmd == ETHTOOL_SRXFH)
1563 return netvsc_set_rss_hash_opts(ndc, info);
1564
1565 return -EOPNOTSUPP;
1566 }
1567
1568 static u32 netvsc_get_rxfh_key_size(struct net_device *dev)
1569 {
1570 return NETVSC_HASH_KEYLEN;
1571 }
1572
1573 static u32 netvsc_rss_indir_size(struct net_device *dev)
1574 {
1575 return ITAB_NUM;
1576 }
1577
1578 static int netvsc_get_rxfh(struct net_device *dev, u32 *indir, u8 *key,
1579 u8 *hfunc)
1580 {
1581 struct net_device_context *ndc = netdev_priv(dev);
1582 struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1583 struct rndis_device *rndis_dev;
1584 int i;
1585
1586 if (!ndev)
1587 return -ENODEV;
1588
1589 if (hfunc)
1590 *hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */
1591
1592 rndis_dev = ndev->extension;
1593 if (indir) {
1594 for (i = 0; i < ITAB_NUM; i++)
1595 indir[i] = rndis_dev->rx_table[i];
1596 }
1597
1598 if (key)
1599 memcpy(key, rndis_dev->rss_key, NETVSC_HASH_KEYLEN);
1600
1601 return 0;
1602 }
1603
1604 static int netvsc_set_rxfh(struct net_device *dev, const u32 *indir,
1605 const u8 *key, const u8 hfunc)
1606 {
1607 struct net_device_context *ndc = netdev_priv(dev);
1608 struct netvsc_device *ndev = rtnl_dereference(ndc->nvdev);
1609 struct rndis_device *rndis_dev;
1610 int i;
1611
1612 if (!ndev)
1613 return -ENODEV;
1614
1615 if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
1616 return -EOPNOTSUPP;
1617
1618 rndis_dev = ndev->extension;
1619 if (indir) {
1620 for (i = 0; i < ITAB_NUM; i++)
1621 if (indir[i] >= ndev->num_chn)
1622 return -EINVAL;
1623
1624 for (i = 0; i < ITAB_NUM; i++)
1625 rndis_dev->rx_table[i] = indir[i];
1626 }
1627
1628 if (!key) {
1629 if (!indir)
1630 return 0;
1631
1632 key = rndis_dev->rss_key;
1633 }
1634
1635 return rndis_filter_set_rss_param(rndis_dev, key);
1636 }
1637
1638 /* Hyper-V RNDIS protocol does not have ring in the HW sense.
1639 * It does have pre-allocated receive area which is divided into sections.
1640 */
1641 static void __netvsc_get_ringparam(struct netvsc_device *nvdev,
1642 struct ethtool_ringparam *ring)
1643 {
1644 u32 max_buf_size;
1645
1646 ring->rx_pending = nvdev->recv_section_cnt;
1647 ring->tx_pending = nvdev->send_section_cnt;
1648
1649 if (nvdev->nvsp_version <= NVSP_PROTOCOL_VERSION_2)
1650 max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY;
1651 else
1652 max_buf_size = NETVSC_RECEIVE_BUFFER_SIZE;
1653
1654 ring->rx_max_pending = max_buf_size / nvdev->recv_section_size;
1655 ring->tx_max_pending = NETVSC_SEND_BUFFER_SIZE
1656 / nvdev->send_section_size;
1657 }
1658
1659 static void netvsc_get_ringparam(struct net_device *ndev,
1660 struct ethtool_ringparam *ring)
1661 {
1662 struct net_device_context *ndevctx = netdev_priv(ndev);
1663 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1664
1665 if (!nvdev)
1666 return;
1667
1668 __netvsc_get_ringparam(nvdev, ring);
1669 }
1670
1671 static int netvsc_set_ringparam(struct net_device *ndev,
1672 struct ethtool_ringparam *ring)
1673 {
1674 struct net_device_context *ndevctx = netdev_priv(ndev);
1675 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1676 struct netvsc_device_info device_info;
1677 struct ethtool_ringparam orig;
1678 u32 new_tx, new_rx;
1679 int ret = 0;
1680
1681 if (!nvdev || nvdev->destroy)
1682 return -ENODEV;
1683
1684 memset(&orig, 0, sizeof(orig));
1685 __netvsc_get_ringparam(nvdev, &orig);
1686
1687 new_tx = clamp_t(u32, ring->tx_pending,
1688 NETVSC_MIN_TX_SECTIONS, orig.tx_max_pending);
1689 new_rx = clamp_t(u32, ring->rx_pending,
1690 NETVSC_MIN_RX_SECTIONS, orig.rx_max_pending);
1691
1692 if (new_tx == orig.tx_pending &&
1693 new_rx == orig.rx_pending)
1694 return 0; /* no change */
1695
1696 memset(&device_info, 0, sizeof(device_info));
1697 device_info.num_chn = nvdev->num_chn;
1698 device_info.send_sections = new_tx;
1699 device_info.send_section_size = nvdev->send_section_size;
1700 device_info.recv_sections = new_rx;
1701 device_info.recv_section_size = nvdev->recv_section_size;
1702
1703 ret = netvsc_detach(ndev, nvdev);
1704 if (ret)
1705 return ret;
1706
1707 ret = netvsc_attach(ndev, &device_info);
1708 if (ret) {
1709 device_info.send_sections = orig.tx_pending;
1710 device_info.recv_sections = orig.rx_pending;
1711
1712 if (netvsc_attach(ndev, &device_info))
1713 netdev_err(ndev, "restoring ringparam failed");
1714 }
1715
1716 return ret;
1717 }
1718
1719 static int netvsc_set_features(struct net_device *ndev,
1720 netdev_features_t features)
1721 {
1722 netdev_features_t change = features ^ ndev->features;
1723 struct net_device_context *ndevctx = netdev_priv(ndev);
1724 struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev);
1725 struct ndis_offload_params offloads;
1726
1727 if (!nvdev || nvdev->destroy)
1728 return -ENODEV;
1729
1730 if (!(change & NETIF_F_LRO))
1731 return 0;
1732
1733 memset(&offloads, 0, sizeof(struct ndis_offload_params));
1734
1735 if (features & NETIF_F_LRO) {
1736 offloads.rsc_ip_v4 = NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED;
1737 offloads.rsc_ip_v6 = NDIS_OFFLOAD_PARAMETERS_RSC_ENABLED;
1738 } else {
1739 offloads.rsc_ip_v4 = NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED;
1740 offloads.rsc_ip_v6 = NDIS_OFFLOAD_PARAMETERS_RSC_DISABLED;
1741 }
1742
1743 return rndis_filter_set_offload_params(ndev, nvdev, &offloads);
1744 }
1745
1746 static u32 netvsc_get_msglevel(struct net_device *ndev)
1747 {
1748 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1749
1750 return ndev_ctx->msg_enable;
1751 }
1752
1753 static void netvsc_set_msglevel(struct net_device *ndev, u32 val)
1754 {
1755 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1756
1757 ndev_ctx->msg_enable = val;
1758 }
1759
1760 static const struct ethtool_ops ethtool_ops = {
1761 .get_drvinfo = netvsc_get_drvinfo,
1762 .get_msglevel = netvsc_get_msglevel,
1763 .set_msglevel = netvsc_set_msglevel,
1764 .get_link = ethtool_op_get_link,
1765 .get_ethtool_stats = netvsc_get_ethtool_stats,
1766 .get_sset_count = netvsc_get_sset_count,
1767 .get_strings = netvsc_get_strings,
1768 .get_channels = netvsc_get_channels,
1769 .set_channels = netvsc_set_channels,
1770 .get_ts_info = ethtool_op_get_ts_info,
1771 .get_rxnfc = netvsc_get_rxnfc,
1772 .set_rxnfc = netvsc_set_rxnfc,
1773 .get_rxfh_key_size = netvsc_get_rxfh_key_size,
1774 .get_rxfh_indir_size = netvsc_rss_indir_size,
1775 .get_rxfh = netvsc_get_rxfh,
1776 .set_rxfh = netvsc_set_rxfh,
1777 .get_link_ksettings = netvsc_get_link_ksettings,
1778 .set_link_ksettings = netvsc_set_link_ksettings,
1779 .get_ringparam = netvsc_get_ringparam,
1780 .set_ringparam = netvsc_set_ringparam,
1781 };
1782
1783 static const struct net_device_ops device_ops = {
1784 .ndo_open = netvsc_open,
1785 .ndo_stop = netvsc_close,
1786 .ndo_start_xmit = netvsc_start_xmit,
1787 .ndo_change_rx_flags = netvsc_change_rx_flags,
1788 .ndo_set_rx_mode = netvsc_set_rx_mode,
1789 .ndo_set_features = netvsc_set_features,
1790 .ndo_change_mtu = netvsc_change_mtu,
1791 .ndo_validate_addr = eth_validate_addr,
1792 .ndo_set_mac_address = netvsc_set_mac_addr,
1793 .ndo_select_queue = netvsc_select_queue,
1794 .ndo_get_stats64 = netvsc_get_stats64,
1795 };
1796
1797 /*
1798 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link
1799 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is
1800 * present send GARP packet to network peers with netif_notify_peers().
1801 */
1802 static void netvsc_link_change(struct work_struct *w)
1803 {
1804 struct net_device_context *ndev_ctx =
1805 container_of(w, struct net_device_context, dwork.work);
1806 struct hv_device *device_obj = ndev_ctx->device_ctx;
1807 struct net_device *net = hv_get_drvdata(device_obj);
1808 struct netvsc_device *net_device;
1809 struct rndis_device *rdev;
1810 struct netvsc_reconfig *event = NULL;
1811 bool notify = false, reschedule = false;
1812 unsigned long flags, next_reconfig, delay;
1813
1814 /* if changes are happening, comeback later */
1815 if (!rtnl_trylock()) {
1816 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1817 return;
1818 }
1819
1820 net_device = rtnl_dereference(ndev_ctx->nvdev);
1821 if (!net_device)
1822 goto out_unlock;
1823
1824 rdev = net_device->extension;
1825
1826 next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT;
1827 if (time_is_after_jiffies(next_reconfig)) {
1828 /* link_watch only sends one notification with current state
1829 * per second, avoid doing reconfig more frequently. Handle
1830 * wrap around.
1831 */
1832 delay = next_reconfig - jiffies;
1833 delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT;
1834 schedule_delayed_work(&ndev_ctx->dwork, delay);
1835 goto out_unlock;
1836 }
1837 ndev_ctx->last_reconfig = jiffies;
1838
1839 spin_lock_irqsave(&ndev_ctx->lock, flags);
1840 if (!list_empty(&ndev_ctx->reconfig_events)) {
1841 event = list_first_entry(&ndev_ctx->reconfig_events,
1842 struct netvsc_reconfig, list);
1843 list_del(&event->list);
1844 reschedule = !list_empty(&ndev_ctx->reconfig_events);
1845 }
1846 spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1847
1848 if (!event)
1849 goto out_unlock;
1850
1851 switch (event->event) {
1852 /* Only the following events are possible due to the check in
1853 * netvsc_linkstatus_callback()
1854 */
1855 case RNDIS_STATUS_MEDIA_CONNECT:
1856 if (rdev->link_state) {
1857 rdev->link_state = false;
1858 netif_carrier_on(net);
1859 netif_tx_wake_all_queues(net);
1860 } else {
1861 notify = true;
1862 }
1863 kfree(event);
1864 break;
1865 case RNDIS_STATUS_MEDIA_DISCONNECT:
1866 if (!rdev->link_state) {
1867 rdev->link_state = true;
1868 netif_carrier_off(net);
1869 netif_tx_stop_all_queues(net);
1870 }
1871 kfree(event);
1872 break;
1873 case RNDIS_STATUS_NETWORK_CHANGE:
1874 /* Only makes sense if carrier is present */
1875 if (!rdev->link_state) {
1876 rdev->link_state = true;
1877 netif_carrier_off(net);
1878 netif_tx_stop_all_queues(net);
1879 event->event = RNDIS_STATUS_MEDIA_CONNECT;
1880 spin_lock_irqsave(&ndev_ctx->lock, flags);
1881 list_add(&event->list, &ndev_ctx->reconfig_events);
1882 spin_unlock_irqrestore(&ndev_ctx->lock, flags);
1883 reschedule = true;
1884 }
1885 break;
1886 }
1887
1888 rtnl_unlock();
1889
1890 if (notify)
1891 netdev_notify_peers(net);
1892
1893 /* link_watch only sends one notification with current state per
1894 * second, handle next reconfig event in 2 seconds.
1895 */
1896 if (reschedule)
1897 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT);
1898
1899 return;
1900
1901 out_unlock:
1902 rtnl_unlock();
1903 }
1904
1905 static struct net_device *get_netvsc_byref(struct net_device *vf_netdev)
1906 {
1907 struct net_device_context *net_device_ctx;
1908 struct net_device *dev;
1909
1910 dev = netdev_master_upper_dev_get(vf_netdev);
1911 if (!dev || dev->netdev_ops != &device_ops)
1912 return NULL; /* not a netvsc device */
1913
1914 net_device_ctx = netdev_priv(dev);
1915 if (!rtnl_dereference(net_device_ctx->nvdev))
1916 return NULL; /* device is removed */
1917
1918 return dev;
1919 }
1920
1921 /* Called when VF is injecting data into network stack.
1922 * Change the associated network device from VF to netvsc.
1923 * note: already called with rcu_read_lock
1924 */
1925 static rx_handler_result_t netvsc_vf_handle_frame(struct sk_buff **pskb)
1926 {
1927 struct sk_buff *skb = *pskb;
1928 struct net_device *ndev = rcu_dereference(skb->dev->rx_handler_data);
1929 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1930 struct netvsc_vf_pcpu_stats *pcpu_stats
1931 = this_cpu_ptr(ndev_ctx->vf_stats);
1932
1933 skb->dev = ndev;
1934
1935 u64_stats_update_begin(&pcpu_stats->syncp);
1936 pcpu_stats->rx_packets++;
1937 pcpu_stats->rx_bytes += skb->len;
1938 u64_stats_update_end(&pcpu_stats->syncp);
1939
1940 return RX_HANDLER_ANOTHER;
1941 }
1942
1943 static int netvsc_vf_join(struct net_device *vf_netdev,
1944 struct net_device *ndev)
1945 {
1946 struct net_device_context *ndev_ctx = netdev_priv(ndev);
1947 int ret;
1948
1949 ret = netdev_rx_handler_register(vf_netdev,
1950 netvsc_vf_handle_frame, ndev);
1951 if (ret != 0) {
1952 netdev_err(vf_netdev,
1953 "can not register netvsc VF receive handler (err = %d)\n",
1954 ret);
1955 goto rx_handler_failed;
1956 }
1957
1958 ret = netdev_master_upper_dev_link(vf_netdev, ndev,
1959 NULL, NULL, NULL);
1960 if (ret != 0) {
1961 netdev_err(vf_netdev,
1962 "can not set master device %s (err = %d)\n",
1963 ndev->name, ret);
1964 goto upper_link_failed;
1965 }
1966
1967 /* set slave flag before open to prevent IPv6 addrconf */
1968 vf_netdev->flags |= IFF_SLAVE;
1969
1970 schedule_delayed_work(&ndev_ctx->vf_takeover, VF_TAKEOVER_INT);
1971
1972 call_netdevice_notifiers(NETDEV_JOIN, vf_netdev);
1973
1974 netdev_info(vf_netdev, "joined to %s\n", ndev->name);
1975 return 0;
1976
1977 upper_link_failed:
1978 netdev_rx_handler_unregister(vf_netdev);
1979 rx_handler_failed:
1980 return ret;
1981 }
1982
1983 static void __netvsc_vf_setup(struct net_device *ndev,
1984 struct net_device *vf_netdev)
1985 {
1986 int ret;
1987
1988 /* Align MTU of VF with master */
1989 ret = dev_set_mtu(vf_netdev, ndev->mtu);
1990 if (ret)
1991 netdev_warn(vf_netdev,
1992 "unable to change mtu to %u\n", ndev->mtu);
1993
1994 /* set multicast etc flags on VF */
1995 dev_change_flags(vf_netdev, ndev->flags | IFF_SLAVE);
1996
1997 /* sync address list from ndev to VF */
1998 netif_addr_lock_bh(ndev);
1999 dev_uc_sync(vf_netdev, ndev);
2000 dev_mc_sync(vf_netdev, ndev);
2001 netif_addr_unlock_bh(ndev);
2002
2003 if (netif_running(ndev)) {
2004 ret = dev_open(vf_netdev);
2005 if (ret)
2006 netdev_warn(vf_netdev,
2007 "unable to open: %d\n", ret);
2008 }
2009 }
2010
2011 /* Setup VF as slave of the synthetic device.
2012 * Runs in workqueue to avoid recursion in netlink callbacks.
2013 */
2014 static void netvsc_vf_setup(struct work_struct *w)
2015 {
2016 struct net_device_context *ndev_ctx
2017 = container_of(w, struct net_device_context, vf_takeover.work);
2018 struct net_device *ndev = hv_get_drvdata(ndev_ctx->device_ctx);
2019 struct net_device *vf_netdev;
2020
2021 if (!rtnl_trylock()) {
2022 schedule_delayed_work(&ndev_ctx->vf_takeover, 0);
2023 return;
2024 }
2025
2026 vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
2027 if (vf_netdev)
2028 __netvsc_vf_setup(ndev, vf_netdev);
2029
2030 rtnl_unlock();
2031 }
2032
2033 /* Find netvsc by VF serial number.
2034 * The PCI hyperv controller records the serial number as the slot kobj name.
2035 */
2036 static struct net_device *get_netvsc_byslot(const struct net_device *vf_netdev)
2037 {
2038 struct device *parent = vf_netdev->dev.parent;
2039 struct net_device_context *ndev_ctx;
2040 struct pci_dev *pdev;
2041 u32 serial;
2042
2043 if (!parent || !dev_is_pci(parent))
2044 return NULL; /* not a PCI device */
2045
2046 pdev = to_pci_dev(parent);
2047 if (!pdev->slot) {
2048 netdev_notice(vf_netdev, "no PCI slot information\n");
2049 return NULL;
2050 }
2051
2052 if (kstrtou32(pci_slot_name(pdev->slot), 10, &serial)) {
2053 netdev_notice(vf_netdev, "Invalid vf serial:%s\n",
2054 pci_slot_name(pdev->slot));
2055 return NULL;
2056 }
2057
2058 list_for_each_entry(ndev_ctx, &netvsc_dev_list, list) {
2059 if (!ndev_ctx->vf_alloc)
2060 continue;
2061
2062 if (ndev_ctx->vf_serial == serial)
2063 return hv_get_drvdata(ndev_ctx->device_ctx);
2064 }
2065
2066 netdev_notice(vf_netdev,
2067 "no netdev found for vf serial:%u\n", serial);
2068 return NULL;
2069 }
2070
2071 static int netvsc_register_vf(struct net_device *vf_netdev)
2072 {
2073 struct net_device_context *net_device_ctx;
2074 struct netvsc_device *netvsc_dev;
2075 struct net_device *ndev;
2076 int ret;
2077
2078 if (vf_netdev->addr_len != ETH_ALEN)
2079 return NOTIFY_DONE;
2080
2081 ndev = get_netvsc_byslot(vf_netdev);
2082 if (!ndev)
2083 return NOTIFY_DONE;
2084
2085 net_device_ctx = netdev_priv(ndev);
2086 netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
2087 if (!netvsc_dev || rtnl_dereference(net_device_ctx->vf_netdev))
2088 return NOTIFY_DONE;
2089
2090 /* if syntihetic interface is a different namespace,
2091 * then move the VF to that namespace; join will be
2092 * done again in that context.
2093 */
2094 if (!net_eq(dev_net(ndev), dev_net(vf_netdev))) {
2095 ret = dev_change_net_namespace(vf_netdev,
2096 dev_net(ndev), "eth%d");
2097 if (ret)
2098 netdev_err(vf_netdev,
2099 "could not move to same namespace as %s: %d\n",
2100 ndev->name, ret);
2101 else
2102 netdev_info(vf_netdev,
2103 "VF moved to namespace with: %s\n",
2104 ndev->name);
2105 return NOTIFY_DONE;
2106 }
2107
2108 netdev_info(ndev, "VF registering: %s\n", vf_netdev->name);
2109
2110 if (netvsc_vf_join(vf_netdev, ndev) != 0)
2111 return NOTIFY_DONE;
2112
2113 dev_hold(vf_netdev);
2114 rcu_assign_pointer(net_device_ctx->vf_netdev, vf_netdev);
2115 return NOTIFY_OK;
2116 }
2117
2118 /* VF up/down change detected, schedule to change data path */
2119 static int netvsc_vf_changed(struct net_device *vf_netdev)
2120 {
2121 struct net_device_context *net_device_ctx;
2122 struct netvsc_device *netvsc_dev;
2123 struct net_device *ndev;
2124 bool vf_is_up = netif_running(vf_netdev);
2125
2126 ndev = get_netvsc_byref(vf_netdev);
2127 if (!ndev)
2128 return NOTIFY_DONE;
2129
2130 net_device_ctx = netdev_priv(ndev);
2131 netvsc_dev = rtnl_dereference(net_device_ctx->nvdev);
2132 if (!netvsc_dev)
2133 return NOTIFY_DONE;
2134
2135 netvsc_switch_datapath(ndev, vf_is_up);
2136 netdev_info(ndev, "Data path switched %s VF: %s\n",
2137 vf_is_up ? "to" : "from", vf_netdev->name);
2138
2139 return NOTIFY_OK;
2140 }
2141
2142 static int netvsc_unregister_vf(struct net_device *vf_netdev)
2143 {
2144 struct net_device *ndev;
2145 struct net_device_context *net_device_ctx;
2146
2147 ndev = get_netvsc_byref(vf_netdev);
2148 if (!ndev)
2149 return NOTIFY_DONE;
2150
2151 net_device_ctx = netdev_priv(ndev);
2152 cancel_delayed_work_sync(&net_device_ctx->vf_takeover);
2153
2154 netdev_info(ndev, "VF unregistering: %s\n", vf_netdev->name);
2155
2156 netdev_rx_handler_unregister(vf_netdev);
2157 netdev_upper_dev_unlink(vf_netdev, ndev);
2158 RCU_INIT_POINTER(net_device_ctx->vf_netdev, NULL);
2159 dev_put(vf_netdev);
2160
2161 return NOTIFY_OK;
2162 }
2163
2164 static int netvsc_probe(struct hv_device *dev,
2165 const struct hv_vmbus_device_id *dev_id)
2166 {
2167 struct net_device *net = NULL;
2168 struct net_device_context *net_device_ctx;
2169 struct netvsc_device_info device_info;
2170 struct netvsc_device *nvdev;
2171 int ret = -ENOMEM;
2172
2173 net = alloc_etherdev_mq(sizeof(struct net_device_context),
2174 VRSS_CHANNEL_MAX);
2175 if (!net)
2176 goto no_net;
2177
2178 netif_carrier_off(net);
2179
2180 netvsc_init_settings(net);
2181
2182 net_device_ctx = netdev_priv(net);
2183 net_device_ctx->device_ctx = dev;
2184 net_device_ctx->msg_enable = netif_msg_init(debug, default_msg);
2185 if (netif_msg_probe(net_device_ctx))
2186 netdev_dbg(net, "netvsc msg_enable: %d\n",
2187 net_device_ctx->msg_enable);
2188
2189 hv_set_drvdata(dev, net);
2190
2191 INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change);
2192
2193 spin_lock_init(&net_device_ctx->lock);
2194 INIT_LIST_HEAD(&net_device_ctx->reconfig_events);
2195 INIT_DELAYED_WORK(&net_device_ctx->vf_takeover, netvsc_vf_setup);
2196
2197 net_device_ctx->vf_stats
2198 = netdev_alloc_pcpu_stats(struct netvsc_vf_pcpu_stats);
2199 if (!net_device_ctx->vf_stats)
2200 goto no_stats;
2201
2202 net->netdev_ops = &device_ops;
2203 net->ethtool_ops = &ethtool_ops;
2204 SET_NETDEV_DEV(net, &dev->device);
2205
2206 /* We always need headroom for rndis header */
2207 net->needed_headroom = RNDIS_AND_PPI_SIZE;
2208
2209 /* Initialize the number of queues to be 1, we may change it if more
2210 * channels are offered later.
2211 */
2212 netif_set_real_num_tx_queues(net, 1);
2213 netif_set_real_num_rx_queues(net, 1);
2214
2215 /* Notify the netvsc driver of the new device */
2216 memset(&device_info, 0, sizeof(device_info));
2217 device_info.num_chn = VRSS_CHANNEL_DEFAULT;
2218 device_info.send_sections = NETVSC_DEFAULT_TX;
2219 device_info.send_section_size = NETVSC_SEND_SECTION_SIZE;
2220 device_info.recv_sections = NETVSC_DEFAULT_RX;
2221 device_info.recv_section_size = NETVSC_RECV_SECTION_SIZE;
2222
2223 nvdev = rndis_filter_device_add(dev, &device_info);
2224 if (IS_ERR(nvdev)) {
2225 ret = PTR_ERR(nvdev);
2226 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret);
2227 goto rndis_failed;
2228 }
2229
2230 memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN);
2231
2232 /* We must get rtnl lock before scheduling nvdev->subchan_work,
2233 * otherwise netvsc_subchan_work() can get rtnl lock first and wait
2234 * all subchannels to show up, but that may not happen because
2235 * netvsc_probe() can't get rtnl lock and as a result vmbus_onoffer()
2236 * -> ... -> device_add() -> ... -> __device_attach() can't get
2237 * the device lock, so all the subchannels can't be processed --
2238 * finally netvsc_subchan_work() hangs for ever.
2239 */
2240 rtnl_lock();
2241
2242 if (nvdev->num_chn > 1)
2243 schedule_work(&nvdev->subchan_work);
2244
2245 /* hw_features computed in rndis_netdev_set_hwcaps() */
2246 net->features = net->hw_features |
2247 NETIF_F_HIGHDMA | NETIF_F_SG |
2248 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
2249 net->vlan_features = net->features;
2250
2251 netdev_lockdep_set_classes(net);
2252
2253 /* MTU range: 68 - 1500 or 65521 */
2254 net->min_mtu = NETVSC_MTU_MIN;
2255 if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2)
2256 net->max_mtu = NETVSC_MTU - ETH_HLEN;
2257 else
2258 net->max_mtu = ETH_DATA_LEN;
2259
2260 ret = register_netdevice(net);
2261 if (ret != 0) {
2262 pr_err("Unable to register netdev.\n");
2263 goto register_failed;
2264 }
2265
2266 list_add(&net_device_ctx->list, &netvsc_dev_list);
2267 rtnl_unlock();
2268 return 0;
2269
2270 register_failed:
2271 rtnl_unlock();
2272 rndis_filter_device_remove(dev, nvdev);
2273 rndis_failed:
2274 free_percpu(net_device_ctx->vf_stats);
2275 no_stats:
2276 hv_set_drvdata(dev, NULL);
2277 free_netdev(net);
2278 no_net:
2279 return ret;
2280 }
2281
2282 static int netvsc_remove(struct hv_device *dev)
2283 {
2284 struct net_device_context *ndev_ctx;
2285 struct net_device *vf_netdev, *net;
2286 struct netvsc_device *nvdev;
2287
2288 net = hv_get_drvdata(dev);
2289 if (net == NULL) {
2290 dev_err(&dev->device, "No net device to remove\n");
2291 return 0;
2292 }
2293
2294 ndev_ctx = netdev_priv(net);
2295
2296 cancel_delayed_work_sync(&ndev_ctx->dwork);
2297
2298 rtnl_lock();
2299 nvdev = rtnl_dereference(ndev_ctx->nvdev);
2300 if (nvdev)
2301 cancel_work_sync(&nvdev->subchan_work);
2302
2303 /*
2304 * Call to the vsc driver to let it know that the device is being
2305 * removed. Also blocks mtu and channel changes.
2306 */
2307 vf_netdev = rtnl_dereference(ndev_ctx->vf_netdev);
2308 if (vf_netdev)
2309 netvsc_unregister_vf(vf_netdev);
2310
2311 if (nvdev)
2312 rndis_filter_device_remove(dev, nvdev);
2313
2314 unregister_netdevice(net);
2315 list_del(&ndev_ctx->list);
2316
2317 rtnl_unlock();
2318
2319 hv_set_drvdata(dev, NULL);
2320
2321 free_percpu(ndev_ctx->vf_stats);
2322 free_netdev(net);
2323 return 0;
2324 }
2325
2326 static const struct hv_vmbus_device_id id_table[] = {
2327 /* Network guid */
2328 { HV_NIC_GUID, },
2329 { },
2330 };
2331
2332 MODULE_DEVICE_TABLE(vmbus, id_table);
2333
2334 /* The one and only one */
2335 static struct hv_driver netvsc_drv = {
2336 .name = KBUILD_MODNAME,
2337 .id_table = id_table,
2338 .probe = netvsc_probe,
2339 .remove = netvsc_remove,
2340 .driver = {
2341 .probe_type = PROBE_PREFER_ASYNCHRONOUS,
2342 },
2343 };
2344
2345 /*
2346 * On Hyper-V, every VF interface is matched with a corresponding
2347 * synthetic interface. The synthetic interface is presented first
2348 * to the guest. When the corresponding VF instance is registered,
2349 * we will take care of switching the data path.
2350 */
2351 static int netvsc_netdev_event(struct notifier_block *this,
2352 unsigned long event, void *ptr)
2353 {
2354 struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
2355
2356 /* Skip our own events */
2357 if (event_dev->netdev_ops == &device_ops)
2358 return NOTIFY_DONE;
2359
2360 /* Avoid non-Ethernet type devices */
2361 if (event_dev->type != ARPHRD_ETHER)
2362 return NOTIFY_DONE;
2363
2364 /* Avoid Vlan dev with same MAC registering as VF */
2365 if (is_vlan_dev(event_dev))
2366 return NOTIFY_DONE;
2367
2368 /* Avoid Bonding master dev with same MAC registering as VF */
2369 if ((event_dev->priv_flags & IFF_BONDING) &&
2370 (event_dev->flags & IFF_MASTER))
2371 return NOTIFY_DONE;
2372
2373 switch (event) {
2374 case NETDEV_REGISTER:
2375 return netvsc_register_vf(event_dev);
2376 case NETDEV_UNREGISTER:
2377 return netvsc_unregister_vf(event_dev);
2378 case NETDEV_UP:
2379 case NETDEV_DOWN:
2380 return netvsc_vf_changed(event_dev);
2381 default:
2382 return NOTIFY_DONE;
2383 }
2384 }
2385
2386 static struct notifier_block netvsc_netdev_notifier = {
2387 .notifier_call = netvsc_netdev_event,
2388 };
2389
2390 static void __exit netvsc_drv_exit(void)
2391 {
2392 unregister_netdevice_notifier(&netvsc_netdev_notifier);
2393 vmbus_driver_unregister(&netvsc_drv);
2394 }
2395
2396 static int __init netvsc_drv_init(void)
2397 {
2398 int ret;
2399
2400 if (ring_size < RING_SIZE_MIN) {
2401 ring_size = RING_SIZE_MIN;
2402 pr_info("Increased ring_size to %u (min allowed)\n",
2403 ring_size);
2404 }
2405 netvsc_ring_bytes = ring_size * PAGE_SIZE;
2406
2407 ret = vmbus_driver_register(&netvsc_drv);
2408 if (ret)
2409 return ret;
2410
2411 register_netdevice_notifier(&netvsc_netdev_notifier);
2412 return 0;
2413 }
2414
2415 MODULE_LICENSE("GPL");
2416 MODULE_DESCRIPTION("Microsoft Hyper-V network driver");
2417
2418 module_init(netvsc_drv_init);
2419 module_exit(netvsc_drv_exit);
Linux with added FPC-III support