4 * Copyright(c) 2017-2018 Intel Corporation.
5 * Copyright(c) 2020 Red Hat, Inc.
7 * This work is licensed under the terms of the GNU GPL, version 2 or later.
8 * See the COPYING file in the top-level directory.
12 #include "qemu/osdep.h"
14 #include "hw/virtio/virtio-net.h"
15 #include "net/vhost_net.h"
16 #include "net/vhost-vdpa.h"
17 #include "hw/virtio/vhost-vdpa.h"
18 #include "qemu/config-file.h"
19 #include "qemu/error-report.h"
21 #include "qemu/memalign.h"
22 #include "qemu/option.h"
23 #include "qapi/error.h"
24 #include <linux/vhost.h>
25 #include <sys/ioctl.h>
27 #include "standard-headers/linux/virtio_net.h"
28 #include "monitor/monitor.h"
29 #include "migration/misc.h"
30 #include "hw/virtio/vhost.h"
33 /* Todo:need to add the multiqueue support here */
34 typedef struct VhostVDPAState
{
36 struct vhost_vdpa vhost_vdpa
;
37 NotifierWithReturn migration_state
;
38 VHostNetState
*vhost_net
;
40 /* Control commands shadow buffers */
41 void *cvq_cmd_out_buffer
;
42 virtio_net_ctrl_ack
*status
;
44 /* The device always have SVQ enabled */
47 /* The device can isolate CVQ in its own ASID */
54 * The array is sorted alphabetically in ascending order,
55 * with the exception of VHOST_INVALID_FEATURE_BIT,
56 * which should always be the last entry.
58 const int vdpa_feature_bits
[] = {
60 VIRTIO_F_IOMMU_PLATFORM
,
61 VIRTIO_F_NOTIFY_ON_EMPTY
,
66 VIRTIO_F_NOTIFICATION_DATA
,
68 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
,
69 VIRTIO_NET_F_CTRL_MAC_ADDR
,
71 VIRTIO_NET_F_CTRL_RX_EXTRA
,
72 VIRTIO_NET_F_CTRL_VLAN
,
75 VIRTIO_NET_F_GUEST_CSUM
,
76 VIRTIO_NET_F_GUEST_ECN
,
77 VIRTIO_NET_F_GUEST_TSO4
,
78 VIRTIO_NET_F_GUEST_TSO6
,
79 VIRTIO_NET_F_GUEST_UFO
,
80 VIRTIO_NET_F_GUEST_USO4
,
81 VIRTIO_NET_F_GUEST_USO6
,
82 VIRTIO_NET_F_HASH_REPORT
,
83 VIRTIO_NET_F_HOST_ECN
,
84 VIRTIO_NET_F_HOST_TSO4
,
85 VIRTIO_NET_F_HOST_TSO6
,
86 VIRTIO_NET_F_HOST_UFO
,
87 VIRTIO_NET_F_HOST_USO
,
89 VIRTIO_NET_F_MRG_RXBUF
,
94 VIRTIO_RING_F_EVENT_IDX
,
95 VIRTIO_RING_F_INDIRECT_DESC
,
97 /* VHOST_INVALID_FEATURE_BIT should always be the last entry */
98 VHOST_INVALID_FEATURE_BIT
101 /** Supported device specific feature bits with SVQ */
102 static const uint64_t vdpa_svq_device_features
=
103 BIT_ULL(VIRTIO_NET_F_CSUM
) |
104 BIT_ULL(VIRTIO_NET_F_GUEST_CSUM
) |
105 BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
) |
106 BIT_ULL(VIRTIO_NET_F_MTU
) |
107 BIT_ULL(VIRTIO_NET_F_MAC
) |
108 BIT_ULL(VIRTIO_NET_F_GUEST_TSO4
) |
109 BIT_ULL(VIRTIO_NET_F_GUEST_TSO6
) |
110 BIT_ULL(VIRTIO_NET_F_GUEST_ECN
) |
111 BIT_ULL(VIRTIO_NET_F_GUEST_UFO
) |
112 BIT_ULL(VIRTIO_NET_F_HOST_TSO4
) |
113 BIT_ULL(VIRTIO_NET_F_HOST_TSO6
) |
114 BIT_ULL(VIRTIO_NET_F_HOST_ECN
) |
115 BIT_ULL(VIRTIO_NET_F_HOST_UFO
) |
116 BIT_ULL(VIRTIO_NET_F_MRG_RXBUF
) |
117 BIT_ULL(VIRTIO_NET_F_STATUS
) |
118 BIT_ULL(VIRTIO_NET_F_CTRL_VQ
) |
119 BIT_ULL(VIRTIO_NET_F_CTRL_RX
) |
120 BIT_ULL(VIRTIO_NET_F_CTRL_VLAN
) |
121 BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA
) |
122 BIT_ULL(VIRTIO_NET_F_MQ
) |
123 BIT_ULL(VIRTIO_F_ANY_LAYOUT
) |
124 BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR
) |
125 /* VHOST_F_LOG_ALL is exposed by SVQ */
126 BIT_ULL(VHOST_F_LOG_ALL
) |
127 BIT_ULL(VIRTIO_NET_F_HASH_REPORT
) |
128 BIT_ULL(VIRTIO_NET_F_RSS
) |
129 BIT_ULL(VIRTIO_NET_F_RSC_EXT
) |
130 BIT_ULL(VIRTIO_NET_F_STANDBY
) |
131 BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX
);
133 #define VHOST_VDPA_NET_CVQ_ASID 1
135 VHostNetState
*vhost_vdpa_get_vhost_net(NetClientState
*nc
)
137 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
138 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
142 static size_t vhost_vdpa_net_cvq_cmd_len(void)
145 * MAC_TABLE_SET is the ctrl command that produces the longer out buffer.
146 * In buffer is always 1 byte, so it should fit here
148 return sizeof(struct virtio_net_ctrl_hdr
) +
149 2 * sizeof(struct virtio_net_ctrl_mac
) +
150 MAC_TABLE_ENTRIES
* ETH_ALEN
;
153 static size_t vhost_vdpa_net_cvq_cmd_page_len(void)
155 return ROUND_UP(vhost_vdpa_net_cvq_cmd_len(), qemu_real_host_page_size());
158 static bool vhost_vdpa_net_valid_svq_features(uint64_t features
, Error
**errp
)
160 uint64_t invalid_dev_features
=
161 features
& ~vdpa_svq_device_features
&
162 /* Transport are all accepted at this point */
163 ~MAKE_64BIT_MASK(VIRTIO_TRANSPORT_F_START
,
164 VIRTIO_TRANSPORT_F_END
- VIRTIO_TRANSPORT_F_START
);
166 if (invalid_dev_features
) {
167 error_setg(errp
, "vdpa svq does not work with features 0x%" PRIx64
,
168 invalid_dev_features
);
172 return vhost_svq_valid_features(features
, errp
);
175 static int vhost_vdpa_net_check_device_id(struct vhost_net
*net
)
179 struct vhost_dev
*hdev
;
181 hdev
= (struct vhost_dev
*)&net
->dev
;
182 ret
= hdev
->vhost_ops
->vhost_get_device_id(hdev
, &device_id
);
183 if (device_id
!= VIRTIO_ID_NET
) {
189 static int vhost_vdpa_add(NetClientState
*ncs
, void *be
,
190 int queue_pair_index
, int nvqs
)
192 VhostNetOptions options
;
193 struct vhost_net
*net
= NULL
;
197 options
.backend_type
= VHOST_BACKEND_TYPE_VDPA
;
198 assert(ncs
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
199 s
= DO_UPCAST(VhostVDPAState
, nc
, ncs
);
200 options
.net_backend
= ncs
;
202 options
.busyloop_timeout
= 0;
205 net
= vhost_net_init(&options
);
207 error_report("failed to init vhost_net for queue");
211 ret
= vhost_vdpa_net_check_device_id(net
);
217 vhost_net_cleanup(net
);
223 static void vhost_vdpa_cleanup(NetClientState
*nc
)
225 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
228 * If a peer NIC is attached, do not cleanup anything.
229 * Cleanup will happen as a part of qemu_cleanup() -> net_cleanup()
230 * when the guest is shutting down.
232 if (nc
->peer
&& nc
->peer
->info
->type
== NET_CLIENT_DRIVER_NIC
) {
235 munmap(s
->cvq_cmd_out_buffer
, vhost_vdpa_net_cvq_cmd_page_len());
236 munmap(s
->status
, vhost_vdpa_net_cvq_cmd_page_len());
238 vhost_net_cleanup(s
->vhost_net
);
239 g_free(s
->vhost_net
);
242 if (s
->vhost_vdpa
.index
!= 0) {
245 qemu_close(s
->vhost_vdpa
.shared
->device_fd
);
246 g_free(s
->vhost_vdpa
.shared
);
249 /** Dummy SetSteeringEBPF to support RSS for vhost-vdpa backend */
250 static bool vhost_vdpa_set_steering_ebpf(NetClientState
*nc
, int prog_fd
)
255 static bool vhost_vdpa_has_vnet_hdr(NetClientState
*nc
)
257 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
262 static bool vhost_vdpa_has_ufo(NetClientState
*nc
)
264 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
265 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
266 uint64_t features
= 0;
267 features
|= (1ULL << VIRTIO_NET_F_HOST_UFO
);
268 features
= vhost_net_get_features(s
->vhost_net
, features
);
269 return !!(features
& (1ULL << VIRTIO_NET_F_HOST_UFO
));
273 static bool vhost_vdpa_check_peer_type(NetClientState
*nc
, ObjectClass
*oc
,
276 const char *driver
= object_class_get_name(oc
);
278 if (!g_str_has_prefix(driver
, "virtio-net-")) {
279 error_setg(errp
, "vhost-vdpa requires frontend driver virtio-net-*");
286 /** Dummy receive in case qemu falls back to userland tap networking */
287 static ssize_t
vhost_vdpa_receive(NetClientState
*nc
, const uint8_t *buf
,
294 /** From any vdpa net client, get the netclient of the i-th queue pair */
295 static VhostVDPAState
*vhost_vdpa_net_get_nc_vdpa(VhostVDPAState
*s
, int i
)
297 NICState
*nic
= qemu_get_nic(s
->nc
.peer
);
298 NetClientState
*nc_i
= qemu_get_peer(nic
->ncs
, i
);
300 return DO_UPCAST(VhostVDPAState
, nc
, nc_i
);
303 static VhostVDPAState
*vhost_vdpa_net_first_nc_vdpa(VhostVDPAState
*s
)
305 return vhost_vdpa_net_get_nc_vdpa(s
, 0);
308 static void vhost_vdpa_net_log_global_enable(VhostVDPAState
*s
, bool enable
)
310 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
313 int data_queue_pairs
, cvq
, r
;
315 /* We are only called on the first data vqs and only if x-svq is not set */
316 if (s
->vhost_vdpa
.shadow_vqs_enabled
== enable
) {
321 n
= VIRTIO_NET(vdev
);
322 if (!n
->vhost_started
) {
326 data_queue_pairs
= n
->multiqueue
? n
->max_queue_pairs
: 1;
327 cvq
= virtio_vdev_has_feature(vdev
, VIRTIO_NET_F_CTRL_VQ
) ?
328 n
->max_ncs
- n
->max_queue_pairs
: 0;
329 v
->shared
->svq_switching
= enable
?
330 SVQ_TSTATE_ENABLING
: SVQ_TSTATE_DISABLING
;
332 * TODO: vhost_net_stop does suspend, get_base and reset. We can be smarter
333 * in the future and resume the device if read-only operations between
334 * suspend and reset goes wrong.
336 vhost_net_stop(vdev
, n
->nic
->ncs
, data_queue_pairs
, cvq
);
338 /* Start will check migration setup_or_active to configure or not SVQ */
339 r
= vhost_net_start(vdev
, n
->nic
->ncs
, data_queue_pairs
, cvq
);
340 if (unlikely(r
< 0)) {
341 error_report("unable to start vhost net: %s(%d)", g_strerror(-r
), -r
);
343 v
->shared
->svq_switching
= SVQ_TSTATE_DONE
;
346 static int vdpa_net_migration_state_notifier(NotifierWithReturn
*notifier
,
347 MigrationEvent
*e
, Error
**errp
)
349 VhostVDPAState
*s
= container_of(notifier
, VhostVDPAState
, migration_state
);
351 if (e
->type
== MIG_EVENT_PRECOPY_SETUP
) {
352 vhost_vdpa_net_log_global_enable(s
, true);
353 } else if (e
->type
== MIG_EVENT_PRECOPY_FAILED
) {
354 vhost_vdpa_net_log_global_enable(s
, false);
359 static void vhost_vdpa_net_data_start_first(VhostVDPAState
*s
)
361 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
363 migration_add_notifier(&s
->migration_state
,
364 vdpa_net_migration_state_notifier
);
365 if (v
->shadow_vqs_enabled
) {
366 v
->shared
->iova_tree
= vhost_iova_tree_new(v
->shared
->iova_range
.first
,
367 v
->shared
->iova_range
.last
);
371 static int vhost_vdpa_net_data_start(NetClientState
*nc
)
373 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
374 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
376 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
379 migration_is_setup_or_active()) {
380 v
->shadow_vqs_enabled
= true;
382 v
->shadow_vqs_enabled
= false;
386 v
->shared
->shadow_data
= v
->shadow_vqs_enabled
;
387 vhost_vdpa_net_data_start_first(s
);
394 static int vhost_vdpa_net_data_load(NetClientState
*nc
)
396 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
397 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
398 bool has_cvq
= v
->dev
->vq_index_end
% 2;
404 for (int i
= 0; i
< v
->dev
->nvqs
; ++i
) {
405 int ret
= vhost_vdpa_set_vring_ready(v
, i
+ v
->dev
->vq_index
);
413 static void vhost_vdpa_net_client_stop(NetClientState
*nc
)
415 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
416 struct vhost_dev
*dev
;
418 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
420 if (s
->vhost_vdpa
.index
== 0) {
421 migration_remove_notifier(&s
->migration_state
);
424 dev
= s
->vhost_vdpa
.dev
;
425 if (dev
->vq_index
+ dev
->nvqs
== dev
->vq_index_end
) {
426 g_clear_pointer(&s
->vhost_vdpa
.shared
->iova_tree
,
427 vhost_iova_tree_delete
);
431 static NetClientInfo net_vhost_vdpa_info
= {
432 .type
= NET_CLIENT_DRIVER_VHOST_VDPA
,
433 .size
= sizeof(VhostVDPAState
),
434 .receive
= vhost_vdpa_receive
,
435 .start
= vhost_vdpa_net_data_start
,
436 .load
= vhost_vdpa_net_data_load
,
437 .stop
= vhost_vdpa_net_client_stop
,
438 .cleanup
= vhost_vdpa_cleanup
,
439 .has_vnet_hdr
= vhost_vdpa_has_vnet_hdr
,
440 .has_ufo
= vhost_vdpa_has_ufo
,
441 .check_peer_type
= vhost_vdpa_check_peer_type
,
442 .set_steering_ebpf
= vhost_vdpa_set_steering_ebpf
,
445 static int64_t vhost_vdpa_get_vring_group(int device_fd
, unsigned vq_index
,
448 struct vhost_vring_state state
= {
451 int r
= ioctl(device_fd
, VHOST_VDPA_GET_VRING_GROUP
, &state
);
453 if (unlikely(r
< 0)) {
455 error_setg_errno(errp
, errno
, "Cannot get VQ %u group", vq_index
);
462 static int vhost_vdpa_set_address_space_id(struct vhost_vdpa
*v
,
466 struct vhost_vring_state asid
= {
472 trace_vhost_vdpa_set_address_space_id(v
, vq_group
, asid_num
);
474 r
= ioctl(v
->shared
->device_fd
, VHOST_VDPA_SET_GROUP_ASID
, &asid
);
475 if (unlikely(r
< 0)) {
476 error_report("Can't set vq group %u asid %u, errno=%d (%s)",
477 asid
.index
, asid
.num
, errno
, g_strerror(errno
));
482 static void vhost_vdpa_cvq_unmap_buf(struct vhost_vdpa
*v
, void *addr
)
484 VhostIOVATree
*tree
= v
->shared
->iova_tree
;
487 * No need to specify size or to look for more translations since
488 * this contiguous chunk was allocated by us.
490 .translated_addr
= (hwaddr
)(uintptr_t)addr
,
492 const DMAMap
*map
= vhost_iova_tree_find_iova(tree
, &needle
);
495 if (unlikely(!map
)) {
496 error_report("Cannot locate expected map");
500 r
= vhost_vdpa_dma_unmap(v
->shared
, v
->address_space_id
, map
->iova
,
502 if (unlikely(r
!= 0)) {
503 error_report("Device cannot unmap: %s(%d)", g_strerror(r
), r
);
506 vhost_iova_tree_remove(tree
, *map
);
509 /** Map CVQ buffer. */
510 static int vhost_vdpa_cvq_map_buf(struct vhost_vdpa
*v
, void *buf
, size_t size
,
516 map
.translated_addr
= (hwaddr
)(uintptr_t)buf
;
518 map
.perm
= write
? IOMMU_RW
: IOMMU_RO
,
519 r
= vhost_iova_tree_map_alloc(v
->shared
->iova_tree
, &map
);
520 if (unlikely(r
!= IOVA_OK
)) {
521 error_report("Cannot map injected element");
525 r
= vhost_vdpa_dma_map(v
->shared
, v
->address_space_id
, map
.iova
,
526 vhost_vdpa_net_cvq_cmd_page_len(), buf
, !write
);
527 if (unlikely(r
< 0)) {
534 vhost_iova_tree_remove(v
->shared
->iova_tree
, map
);
538 static int vhost_vdpa_net_cvq_start(NetClientState
*nc
)
540 VhostVDPAState
*s
, *s0
;
541 struct vhost_vdpa
*v
;
546 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
548 s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
551 s0
= vhost_vdpa_net_first_nc_vdpa(s
);
552 v
->shadow_vqs_enabled
= s0
->vhost_vdpa
.shadow_vqs_enabled
;
553 s
->vhost_vdpa
.address_space_id
= VHOST_VDPA_GUEST_PA_ASID
;
555 if (v
->shared
->shadow_data
) {
556 /* SVQ is already configured for all virtqueues */
561 * If we early return in these cases SVQ will not be enabled. The migration
562 * will be blocked as long as vhost-vdpa backends will not offer _F_LOG.
564 if (!vhost_vdpa_net_valid_svq_features(v
->dev
->features
, NULL
)) {
568 if (!s
->cvq_isolated
) {
572 cvq_group
= vhost_vdpa_get_vring_group(v
->shared
->device_fd
,
573 v
->dev
->vq_index_end
- 1,
575 if (unlikely(cvq_group
< 0)) {
576 error_report_err(err
);
580 r
= vhost_vdpa_set_address_space_id(v
, cvq_group
, VHOST_VDPA_NET_CVQ_ASID
);
581 if (unlikely(r
< 0)) {
585 v
->shadow_vqs_enabled
= true;
586 s
->vhost_vdpa
.address_space_id
= VHOST_VDPA_NET_CVQ_ASID
;
589 if (!s
->vhost_vdpa
.shadow_vqs_enabled
) {
594 * If other vhost_vdpa already have an iova_tree, reuse it for simplicity,
595 * whether CVQ shares ASID with guest or not, because:
596 * - Memory listener need access to guest's memory addresses allocated in
598 * - There should be plenty of IOVA address space for both ASID not to
599 * worry about collisions between them. Guest's translations are still
600 * validated with virtio virtqueue_pop so there is no risk for the guest
601 * to access memory that it shouldn't.
603 * To allocate a iova tree per ASID is doable but it complicates the code
604 * and it is not worth it for the moment.
606 if (!v
->shared
->iova_tree
) {
607 v
->shared
->iova_tree
= vhost_iova_tree_new(v
->shared
->iova_range
.first
,
608 v
->shared
->iova_range
.last
);
611 r
= vhost_vdpa_cvq_map_buf(&s
->vhost_vdpa
, s
->cvq_cmd_out_buffer
,
612 vhost_vdpa_net_cvq_cmd_page_len(), false);
613 if (unlikely(r
< 0)) {
617 r
= vhost_vdpa_cvq_map_buf(&s
->vhost_vdpa
, s
->status
,
618 vhost_vdpa_net_cvq_cmd_page_len(), true);
619 if (unlikely(r
< 0)) {
620 vhost_vdpa_cvq_unmap_buf(&s
->vhost_vdpa
, s
->cvq_cmd_out_buffer
);
626 static void vhost_vdpa_net_cvq_stop(NetClientState
*nc
)
628 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
630 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
632 if (s
->vhost_vdpa
.shadow_vqs_enabled
) {
633 vhost_vdpa_cvq_unmap_buf(&s
->vhost_vdpa
, s
->cvq_cmd_out_buffer
);
634 vhost_vdpa_cvq_unmap_buf(&s
->vhost_vdpa
, s
->status
);
637 vhost_vdpa_net_client_stop(nc
);
640 static ssize_t
vhost_vdpa_net_cvq_add(VhostVDPAState
*s
,
641 const struct iovec
*out_sg
, size_t out_num
,
642 const struct iovec
*in_sg
, size_t in_num
)
644 VhostShadowVirtqueue
*svq
= g_ptr_array_index(s
->vhost_vdpa
.shadow_vqs
, 0);
647 r
= vhost_svq_add(svq
, out_sg
, out_num
, in_sg
, in_num
, NULL
);
648 if (unlikely(r
!= 0)) {
649 if (unlikely(r
== -ENOSPC
)) {
650 qemu_log_mask(LOG_GUEST_ERROR
, "%s: No space on device queue\n",
659 * Convenience wrapper to poll SVQ for multiple control commands.
661 * Caller should hold the BQL when invoking this function, and should take
662 * the answer before SVQ pulls by itself when BQL is released.
664 static ssize_t
vhost_vdpa_net_svq_poll(VhostVDPAState
*s
, size_t cmds_in_flight
)
666 VhostShadowVirtqueue
*svq
= g_ptr_array_index(s
->vhost_vdpa
.shadow_vqs
, 0);
667 return vhost_svq_poll(svq
, cmds_in_flight
);
670 static void vhost_vdpa_net_load_cursor_reset(VhostVDPAState
*s
,
671 struct iovec
*out_cursor
,
672 struct iovec
*in_cursor
)
674 /* reset the cursor of the output buffer for the device */
675 out_cursor
->iov_base
= s
->cvq_cmd_out_buffer
;
676 out_cursor
->iov_len
= vhost_vdpa_net_cvq_cmd_page_len();
678 /* reset the cursor of the in buffer for the device */
679 in_cursor
->iov_base
= s
->status
;
680 in_cursor
->iov_len
= vhost_vdpa_net_cvq_cmd_page_len();
684 * Poll SVQ for multiple pending control commands and check the device's ack.
686 * Caller should hold the BQL when invoking this function.
688 * @s: The VhostVDPAState
689 * @len: The length of the pending status shadow buffer
691 static ssize_t
vhost_vdpa_net_svq_flush(VhostVDPAState
*s
, size_t len
)
693 /* device uses a one-byte length ack for each control command */
694 ssize_t dev_written
= vhost_vdpa_net_svq_poll(s
, len
);
695 if (unlikely(dev_written
!= len
)) {
699 /* check the device's ack */
700 for (int i
= 0; i
< len
; ++i
) {
701 if (s
->status
[i
] != VIRTIO_NET_OK
) {
708 static ssize_t
vhost_vdpa_net_load_cmd(VhostVDPAState
*s
,
709 struct iovec
*out_cursor
,
710 struct iovec
*in_cursor
, uint8_t class,
711 uint8_t cmd
, const struct iovec
*data_sg
,
714 const struct virtio_net_ctrl_hdr ctrl
= {
718 size_t data_size
= iov_size(data_sg
, data_num
), cmd_size
;
719 struct iovec out
, in
;
721 unsigned dummy_cursor_iov_cnt
;
722 VhostShadowVirtqueue
*svq
= g_ptr_array_index(s
->vhost_vdpa
.shadow_vqs
, 0);
724 assert(data_size
< vhost_vdpa_net_cvq_cmd_page_len() - sizeof(ctrl
));
725 cmd_size
= sizeof(ctrl
) + data_size
;
726 trace_vhost_vdpa_net_load_cmd(s
, class, cmd
, data_num
, data_size
);
727 if (vhost_svq_available_slots(svq
) < 2 ||
728 iov_size(out_cursor
, 1) < cmd_size
) {
730 * It is time to flush all pending control commands if SVQ is full
731 * or control commands shadow buffers are full.
733 * We can poll here since we've had BQL from the time
734 * we sent the descriptor.
736 r
= vhost_vdpa_net_svq_flush(s
, in_cursor
->iov_base
-
738 if (unlikely(r
< 0)) {
742 vhost_vdpa_net_load_cursor_reset(s
, out_cursor
, in_cursor
);
745 /* pack the CVQ command header */
746 iov_from_buf(out_cursor
, 1, 0, &ctrl
, sizeof(ctrl
));
747 /* pack the CVQ command command-specific-data */
748 iov_to_buf(data_sg
, data_num
, 0,
749 out_cursor
->iov_base
+ sizeof(ctrl
), data_size
);
751 /* extract the required buffer from the cursor for output */
752 iov_copy(&out
, 1, out_cursor
, 1, 0, cmd_size
);
753 /* extract the required buffer from the cursor for input */
754 iov_copy(&in
, 1, in_cursor
, 1, 0, sizeof(*s
->status
));
756 r
= vhost_vdpa_net_cvq_add(s
, &out
, 1, &in
, 1);
757 if (unlikely(r
< 0)) {
758 trace_vhost_vdpa_net_load_cmd_retval(s
, class, cmd
, r
);
762 /* iterate the cursors */
763 dummy_cursor_iov_cnt
= 1;
764 iov_discard_front(&out_cursor
, &dummy_cursor_iov_cnt
, cmd_size
);
765 dummy_cursor_iov_cnt
= 1;
766 iov_discard_front(&in_cursor
, &dummy_cursor_iov_cnt
, sizeof(*s
->status
));
771 static int vhost_vdpa_net_load_mac(VhostVDPAState
*s
, const VirtIONet
*n
,
772 struct iovec
*out_cursor
,
773 struct iovec
*in_cursor
)
775 if (virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_CTRL_MAC_ADDR
)) {
776 const struct iovec data
= {
777 .iov_base
= (void *)n
->mac
,
778 .iov_len
= sizeof(n
->mac
),
780 ssize_t r
= vhost_vdpa_net_load_cmd(s
, out_cursor
, in_cursor
,
782 VIRTIO_NET_CTRL_MAC_ADDR_SET
,
784 if (unlikely(r
< 0)) {
790 * According to VirtIO standard, "The device MUST have an
791 * empty MAC filtering table on reset.".
793 * Therefore, there is no need to send this CVQ command if the
794 * driver also sets an empty MAC filter table, which aligns with
795 * the device's defaults.
797 * Note that the device's defaults can mismatch the driver's
798 * configuration only at live migration.
800 if (!virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_CTRL_RX
) ||
801 n
->mac_table
.in_use
== 0) {
805 uint32_t uni_entries
= n
->mac_table
.first_multi
,
806 uni_macs_size
= uni_entries
* ETH_ALEN
,
807 mul_entries
= n
->mac_table
.in_use
- uni_entries
,
808 mul_macs_size
= mul_entries
* ETH_ALEN
;
809 struct virtio_net_ctrl_mac uni
= {
810 .entries
= cpu_to_le32(uni_entries
),
812 struct virtio_net_ctrl_mac mul
= {
813 .entries
= cpu_to_le32(mul_entries
),
815 const struct iovec data
[] = {
818 .iov_len
= sizeof(uni
),
820 .iov_base
= n
->mac_table
.macs
,
821 .iov_len
= uni_macs_size
,
824 .iov_len
= sizeof(mul
),
826 .iov_base
= &n
->mac_table
.macs
[uni_macs_size
],
827 .iov_len
= mul_macs_size
,
830 ssize_t r
= vhost_vdpa_net_load_cmd(s
, out_cursor
, in_cursor
,
832 VIRTIO_NET_CTRL_MAC_TABLE_SET
,
833 data
, ARRAY_SIZE(data
));
834 if (unlikely(r
< 0)) {
841 static int vhost_vdpa_net_load_rss(VhostVDPAState
*s
, const VirtIONet
*n
,
842 struct iovec
*out_cursor
,
843 struct iovec
*in_cursor
, bool do_rss
)
845 struct virtio_net_rss_config cfg
= {};
847 g_autofree
uint16_t *table
= NULL
;
850 * According to VirtIO standard, "Initially the device has all hash
851 * types disabled and reports only VIRTIO_NET_HASH_REPORT_NONE.".
853 * Therefore, there is no need to send this CVQ command if the
854 * driver disables the all hash types, which aligns with
855 * the device's defaults.
857 * Note that the device's defaults can mismatch the driver's
858 * configuration only at live migration.
860 if (!n
->rss_data
.enabled
||
861 n
->rss_data
.hash_types
== VIRTIO_NET_HASH_REPORT_NONE
) {
865 table
= g_malloc_n(n
->rss_data
.indirections_len
,
866 sizeof(n
->rss_data
.indirections_table
[0]));
867 cfg
.hash_types
= cpu_to_le32(n
->rss_data
.hash_types
);
871 * According to VirtIO standard, "Number of entries in indirection_table
872 * is (indirection_table_mask + 1)".
874 cfg
.indirection_table_mask
= cpu_to_le16(n
->rss_data
.indirections_len
-
876 cfg
.unclassified_queue
= cpu_to_le16(n
->rss_data
.default_queue
);
877 for (int i
= 0; i
< n
->rss_data
.indirections_len
; ++i
) {
878 table
[i
] = cpu_to_le16(n
->rss_data
.indirections_table
[i
]);
880 cfg
.max_tx_vq
= cpu_to_le16(n
->curr_queue_pairs
);
883 * According to VirtIO standard, "Field reserved MUST contain zeroes.
884 * It is defined to make the structure to match the layout of
885 * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
887 * Therefore, we need to zero the fields in
888 * struct virtio_net_rss_config, which corresponds to the
889 * `reserved` field in struct virtio_net_hash_config.
891 * Note that all other fields are zeroed at their definitions,
892 * except for the `indirection_table` field, where the actual data
893 * is stored in the `table` variable to ensure compatibility
894 * with RSS case. Therefore, we need to zero the `table` variable here.
900 * Considering that virtio_net_handle_rss() currently does not restore
901 * the hash key length parsed from the CVQ command sent from the guest
902 * into n->rss_data and uses the maximum key length in other code, so
903 * we also employ the maximum key length here.
905 cfg
.hash_key_length
= sizeof(n
->rss_data
.key
);
907 const struct iovec data
[] = {
910 .iov_len
= offsetof(struct virtio_net_rss_config
,
914 .iov_len
= n
->rss_data
.indirections_len
*
915 sizeof(n
->rss_data
.indirections_table
[0]),
917 .iov_base
= &cfg
.max_tx_vq
,
918 .iov_len
= offsetof(struct virtio_net_rss_config
, hash_key_data
) -
919 offsetof(struct virtio_net_rss_config
, max_tx_vq
),
921 .iov_base
= (void *)n
->rss_data
.key
,
922 .iov_len
= sizeof(n
->rss_data
.key
),
926 r
= vhost_vdpa_net_load_cmd(s
, out_cursor
, in_cursor
,
928 do_rss
? VIRTIO_NET_CTRL_MQ_RSS_CONFIG
:
929 VIRTIO_NET_CTRL_MQ_HASH_CONFIG
,
930 data
, ARRAY_SIZE(data
));
931 if (unlikely(r
< 0)) {
938 static int vhost_vdpa_net_load_mq(VhostVDPAState
*s
,
940 struct iovec
*out_cursor
,
941 struct iovec
*in_cursor
)
943 struct virtio_net_ctrl_mq mq
;
946 if (!virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_MQ
)) {
950 trace_vhost_vdpa_net_load_mq(s
, n
->curr_queue_pairs
);
952 mq
.virtqueue_pairs
= cpu_to_le16(n
->curr_queue_pairs
);
953 const struct iovec data
= {
955 .iov_len
= sizeof(mq
),
957 r
= vhost_vdpa_net_load_cmd(s
, out_cursor
, in_cursor
,
959 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
,
961 if (unlikely(r
< 0)) {
965 if (virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_RSS
)) {
966 /* load the receive-side scaling state */
967 r
= vhost_vdpa_net_load_rss(s
, n
, out_cursor
, in_cursor
, true);
968 if (unlikely(r
< 0)) {
971 } else if (virtio_vdev_has_feature(&n
->parent_obj
,
972 VIRTIO_NET_F_HASH_REPORT
)) {
973 /* load the hash calculation state */
974 r
= vhost_vdpa_net_load_rss(s
, n
, out_cursor
, in_cursor
, false);
975 if (unlikely(r
< 0)) {
983 static int vhost_vdpa_net_load_offloads(VhostVDPAState
*s
,
985 struct iovec
*out_cursor
,
986 struct iovec
*in_cursor
)
991 if (!virtio_vdev_has_feature(&n
->parent_obj
,
992 VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
)) {
996 if (n
->curr_guest_offloads
== virtio_net_supported_guest_offloads(n
)) {
998 * According to VirtIO standard, "Upon feature negotiation
999 * corresponding offload gets enabled to preserve
1000 * backward compatibility.".
1002 * Therefore, there is no need to send this CVQ command if the
1003 * driver also enables all supported offloads, which aligns with
1004 * the device's defaults.
1006 * Note that the device's defaults can mismatch the driver's
1007 * configuration only at live migration.
1012 offloads
= cpu_to_le64(n
->curr_guest_offloads
);
1013 const struct iovec data
= {
1014 .iov_base
= &offloads
,
1015 .iov_len
= sizeof(offloads
),
1017 r
= vhost_vdpa_net_load_cmd(s
, out_cursor
, in_cursor
,
1018 VIRTIO_NET_CTRL_GUEST_OFFLOADS
,
1019 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET
,
1021 if (unlikely(r
< 0)) {
1028 static int vhost_vdpa_net_load_rx_mode(VhostVDPAState
*s
,
1029 struct iovec
*out_cursor
,
1030 struct iovec
*in_cursor
,
1034 const struct iovec data
= {
1036 .iov_len
= sizeof(on
),
1040 r
= vhost_vdpa_net_load_cmd(s
, out_cursor
, in_cursor
,
1041 VIRTIO_NET_CTRL_RX
, cmd
, &data
, 1);
1042 if (unlikely(r
< 0)) {
1049 static int vhost_vdpa_net_load_rx(VhostVDPAState
*s
,
1051 struct iovec
*out_cursor
,
1052 struct iovec
*in_cursor
)
1056 if (!virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_CTRL_RX
)) {
1061 * According to virtio_net_reset(), device turns promiscuous mode
1064 * Additionally, according to VirtIO standard, "Since there are
1065 * no guarantees, it can use a hash filter or silently switch to
1066 * allmulti or promiscuous mode if it is given too many addresses.".
1067 * QEMU marks `n->mac_table.uni_overflow` if guest sets too many
1068 * non-multicast MAC addresses, indicating that promiscuous mode
1069 * should be enabled.
1071 * Therefore, QEMU should only send this CVQ command if the
1072 * `n->mac_table.uni_overflow` is not marked and `n->promisc` is off,
1073 * which sets promiscuous mode on, different from the device's defaults.
1075 * Note that the device's defaults can mismatch the driver's
1076 * configuration only at live migration.
1078 if (!n
->mac_table
.uni_overflow
&& !n
->promisc
) {
1079 r
= vhost_vdpa_net_load_rx_mode(s
, out_cursor
, in_cursor
,
1080 VIRTIO_NET_CTRL_RX_PROMISC
, 0);
1081 if (unlikely(r
< 0)) {
1087 * According to virtio_net_reset(), device turns all-multicast mode
1090 * According to VirtIO standard, "Since there are no guarantees,
1091 * it can use a hash filter or silently switch to allmulti or
1092 * promiscuous mode if it is given too many addresses.". QEMU marks
1093 * `n->mac_table.multi_overflow` if guest sets too many
1094 * non-multicast MAC addresses.
1096 * Therefore, QEMU should only send this CVQ command if the
1097 * `n->mac_table.multi_overflow` is marked or `n->allmulti` is on,
1098 * which sets all-multicast mode on, different from the device's defaults.
1100 * Note that the device's defaults can mismatch the driver's
1101 * configuration only at live migration.
1103 if (n
->mac_table
.multi_overflow
|| n
->allmulti
) {
1104 r
= vhost_vdpa_net_load_rx_mode(s
, out_cursor
, in_cursor
,
1105 VIRTIO_NET_CTRL_RX_ALLMULTI
, 1);
1106 if (unlikely(r
< 0)) {
1111 if (!virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_CTRL_RX_EXTRA
)) {
1116 * According to virtio_net_reset(), device turns all-unicast mode
1119 * Therefore, QEMU should only send this CVQ command if the driver
1120 * sets all-unicast mode on, different from the device's defaults.
1122 * Note that the device's defaults can mismatch the driver's
1123 * configuration only at live migration.
1126 r
= vhost_vdpa_net_load_rx_mode(s
, out_cursor
, in_cursor
,
1127 VIRTIO_NET_CTRL_RX_ALLUNI
, 1);
1134 * According to virtio_net_reset(), device turns non-multicast mode
1137 * Therefore, QEMU should only send this CVQ command if the driver
1138 * sets non-multicast mode on, different from the device's defaults.
1140 * Note that the device's defaults can mismatch the driver's
1141 * configuration only at live migration.
1144 r
= vhost_vdpa_net_load_rx_mode(s
, out_cursor
, in_cursor
,
1145 VIRTIO_NET_CTRL_RX_NOMULTI
, 1);
1152 * According to virtio_net_reset(), device turns non-unicast mode
1155 * Therefore, QEMU should only send this CVQ command if the driver
1156 * sets non-unicast mode on, different from the device's defaults.
1158 * Note that the device's defaults can mismatch the driver's
1159 * configuration only at live migration.
1162 r
= vhost_vdpa_net_load_rx_mode(s
, out_cursor
, in_cursor
,
1163 VIRTIO_NET_CTRL_RX_NOUNI
, 1);
1170 * According to virtio_net_reset(), device turns non-broadcast mode
1173 * Therefore, QEMU should only send this CVQ command if the driver
1174 * sets non-broadcast mode on, different from the device's defaults.
1176 * Note that the device's defaults can mismatch the driver's
1177 * configuration only at live migration.
1180 r
= vhost_vdpa_net_load_rx_mode(s
, out_cursor
, in_cursor
,
1181 VIRTIO_NET_CTRL_RX_NOBCAST
, 1);
1190 static int vhost_vdpa_net_load_single_vlan(VhostVDPAState
*s
,
1192 struct iovec
*out_cursor
,
1193 struct iovec
*in_cursor
,
1196 const struct iovec data
= {
1198 .iov_len
= sizeof(vid
),
1200 ssize_t r
= vhost_vdpa_net_load_cmd(s
, out_cursor
, in_cursor
,
1201 VIRTIO_NET_CTRL_VLAN
,
1202 VIRTIO_NET_CTRL_VLAN_ADD
,
1204 if (unlikely(r
< 0)) {
1211 static int vhost_vdpa_net_load_vlan(VhostVDPAState
*s
,
1213 struct iovec
*out_cursor
,
1214 struct iovec
*in_cursor
)
1218 if (!virtio_vdev_has_feature(&n
->parent_obj
, VIRTIO_NET_F_CTRL_VLAN
)) {
1222 for (int i
= 0; i
< MAX_VLAN
>> 5; i
++) {
1223 for (int j
= 0; n
->vlans
[i
] && j
<= 0x1f; j
++) {
1224 if (n
->vlans
[i
] & (1U << j
)) {
1225 r
= vhost_vdpa_net_load_single_vlan(s
, n
, out_cursor
,
1226 in_cursor
, (i
<< 5) + j
);
1227 if (unlikely(r
!= 0)) {
1237 static int vhost_vdpa_net_cvq_load(NetClientState
*nc
)
1239 VhostVDPAState
*s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
1240 struct vhost_vdpa
*v
= &s
->vhost_vdpa
;
1243 struct iovec out_cursor
, in_cursor
;
1245 assert(nc
->info
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
1247 r
= vhost_vdpa_set_vring_ready(v
, v
->dev
->vq_index
);
1248 if (unlikely(r
< 0)) {
1252 if (v
->shadow_vqs_enabled
) {
1253 n
= VIRTIO_NET(v
->dev
->vdev
);
1254 vhost_vdpa_net_load_cursor_reset(s
, &out_cursor
, &in_cursor
);
1255 r
= vhost_vdpa_net_load_mac(s
, n
, &out_cursor
, &in_cursor
);
1256 if (unlikely(r
< 0)) {
1259 r
= vhost_vdpa_net_load_mq(s
, n
, &out_cursor
, &in_cursor
);
1263 r
= vhost_vdpa_net_load_offloads(s
, n
, &out_cursor
, &in_cursor
);
1267 r
= vhost_vdpa_net_load_rx(s
, n
, &out_cursor
, &in_cursor
);
1271 r
= vhost_vdpa_net_load_vlan(s
, n
, &out_cursor
, &in_cursor
);
1277 * We need to poll and check all pending device's used buffers.
1279 * We can poll here since we've had BQL from the time
1280 * we sent the descriptor.
1282 r
= vhost_vdpa_net_svq_flush(s
, in_cursor
.iov_base
- (void *)s
->status
);
1288 for (int i
= 0; i
< v
->dev
->vq_index
; ++i
) {
1289 r
= vhost_vdpa_set_vring_ready(v
, i
);
1290 if (unlikely(r
< 0)) {
1298 static NetClientInfo net_vhost_vdpa_cvq_info
= {
1299 .type
= NET_CLIENT_DRIVER_VHOST_VDPA
,
1300 .size
= sizeof(VhostVDPAState
),
1301 .receive
= vhost_vdpa_receive
,
1302 .start
= vhost_vdpa_net_cvq_start
,
1303 .load
= vhost_vdpa_net_cvq_load
,
1304 .stop
= vhost_vdpa_net_cvq_stop
,
1305 .cleanup
= vhost_vdpa_cleanup
,
1306 .has_vnet_hdr
= vhost_vdpa_has_vnet_hdr
,
1307 .has_ufo
= vhost_vdpa_has_ufo
,
1308 .check_peer_type
= vhost_vdpa_check_peer_type
,
1309 .set_steering_ebpf
= vhost_vdpa_set_steering_ebpf
,
1313 * Forward the excessive VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command to
1316 * Considering that QEMU cannot send the entire filter table to the
1317 * vdpa device, it should send the VIRTIO_NET_CTRL_RX_PROMISC CVQ
1318 * command to enable promiscuous mode to receive all packets,
1319 * according to VirtIO standard, "Since there are no guarantees,
1320 * it can use a hash filter or silently switch to allmulti or
1321 * promiscuous mode if it is given too many addresses.".
1323 * Since QEMU ignores MAC addresses beyond `MAC_TABLE_ENTRIES` and
1324 * marks `n->mac_table.x_overflow` accordingly, it should have
1325 * the same effect on the device model to receive
1326 * (`MAC_TABLE_ENTRIES` + 1) or more non-multicast MAC addresses.
1327 * The same applies to multicast MAC addresses.
1329 * Therefore, QEMU can provide the device model with a fake
1330 * VIRTIO_NET_CTRL_MAC_TABLE_SET command with (`MAC_TABLE_ENTRIES` + 1)
1331 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1) multicast
1332 * MAC addresses. This ensures that the device model marks
1333 * `n->mac_table.uni_overflow` and `n->mac_table.multi_overflow`,
1334 * allowing all packets to be received, which aligns with the
1335 * state of the vdpa device.
1337 static int vhost_vdpa_net_excessive_mac_filter_cvq_add(VhostVDPAState
*s
,
1338 VirtQueueElement
*elem
,
1340 const struct iovec
*in
)
1342 struct virtio_net_ctrl_mac mac_data
, *mac_ptr
;
1343 struct virtio_net_ctrl_hdr
*hdr_ptr
;
1348 /* parse the non-multicast MAC address entries from CVQ command */
1349 cursor
= sizeof(*hdr_ptr
);
1350 r
= iov_to_buf(elem
->out_sg
, elem
->out_num
, cursor
,
1351 &mac_data
, sizeof(mac_data
));
1352 if (unlikely(r
!= sizeof(mac_data
))) {
1354 * If the CVQ command is invalid, we should simulate the vdpa device
1355 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1357 *s
->status
= VIRTIO_NET_ERR
;
1358 return sizeof(*s
->status
);
1360 cursor
+= sizeof(mac_data
) + le32_to_cpu(mac_data
.entries
) * ETH_ALEN
;
1362 /* parse the multicast MAC address entries from CVQ command */
1363 r
= iov_to_buf(elem
->out_sg
, elem
->out_num
, cursor
,
1364 &mac_data
, sizeof(mac_data
));
1365 if (r
!= sizeof(mac_data
)) {
1367 * If the CVQ command is invalid, we should simulate the vdpa device
1368 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1370 *s
->status
= VIRTIO_NET_ERR
;
1371 return sizeof(*s
->status
);
1373 cursor
+= sizeof(mac_data
) + le32_to_cpu(mac_data
.entries
) * ETH_ALEN
;
1375 /* validate the CVQ command */
1376 if (iov_size(elem
->out_sg
, elem
->out_num
) != cursor
) {
1378 * If the CVQ command is invalid, we should simulate the vdpa device
1379 * to reject the VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1381 *s
->status
= VIRTIO_NET_ERR
;
1382 return sizeof(*s
->status
);
1386 * According to VirtIO standard, "Since there are no guarantees,
1387 * it can use a hash filter or silently switch to allmulti or
1388 * promiscuous mode if it is given too many addresses.".
1390 * Therefore, considering that QEMU is unable to send the entire
1391 * filter table to the vdpa device, it should send the
1392 * VIRTIO_NET_CTRL_RX_PROMISC CVQ command to enable promiscuous mode
1394 hdr_ptr
= out
->iov_base
;
1395 out
->iov_len
= sizeof(*hdr_ptr
) + sizeof(on
);
1397 hdr_ptr
->class = VIRTIO_NET_CTRL_RX
;
1398 hdr_ptr
->cmd
= VIRTIO_NET_CTRL_RX_PROMISC
;
1399 iov_from_buf(out
, 1, sizeof(*hdr_ptr
), &on
, sizeof(on
));
1400 r
= vhost_vdpa_net_cvq_add(s
, out
, 1, in
, 1);
1401 if (unlikely(r
< 0)) {
1406 * We can poll here since we've had BQL from the time
1407 * we sent the descriptor.
1409 r
= vhost_vdpa_net_svq_poll(s
, 1);
1410 if (unlikely(r
< sizeof(*s
->status
))) {
1413 if (*s
->status
!= VIRTIO_NET_OK
) {
1414 return sizeof(*s
->status
);
1418 * QEMU should also send a fake VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ
1419 * command to the device model, including (`MAC_TABLE_ENTRIES` + 1)
1420 * non-multicast MAC addresses and (`MAC_TABLE_ENTRIES` + 1)
1421 * multicast MAC addresses.
1423 * By doing so, the device model can mark `n->mac_table.uni_overflow`
1424 * and `n->mac_table.multi_overflow`, enabling all packets to be
1425 * received, which aligns with the state of the vdpa device.
1428 uint32_t fake_uni_entries
= MAC_TABLE_ENTRIES
+ 1,
1429 fake_mul_entries
= MAC_TABLE_ENTRIES
+ 1,
1430 fake_cvq_size
= sizeof(struct virtio_net_ctrl_hdr
) +
1431 sizeof(mac_data
) + fake_uni_entries
* ETH_ALEN
+
1432 sizeof(mac_data
) + fake_mul_entries
* ETH_ALEN
;
1434 assert(fake_cvq_size
< vhost_vdpa_net_cvq_cmd_page_len());
1435 out
->iov_len
= fake_cvq_size
;
1437 /* pack the header for fake CVQ command */
1438 hdr_ptr
= out
->iov_base
+ cursor
;
1439 hdr_ptr
->class = VIRTIO_NET_CTRL_MAC
;
1440 hdr_ptr
->cmd
= VIRTIO_NET_CTRL_MAC_TABLE_SET
;
1441 cursor
+= sizeof(*hdr_ptr
);
1444 * Pack the non-multicast MAC addresses part for fake CVQ command.
1446 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1447 * addresses provided in CVQ command. Therefore, only the entries
1448 * field need to be prepared in the CVQ command.
1450 mac_ptr
= out
->iov_base
+ cursor
;
1451 mac_ptr
->entries
= cpu_to_le32(fake_uni_entries
);
1452 cursor
+= sizeof(*mac_ptr
) + fake_uni_entries
* ETH_ALEN
;
1455 * Pack the multicast MAC addresses part for fake CVQ command.
1457 * According to virtio_net_handle_mac(), QEMU doesn't verify the MAC
1458 * addresses provided in CVQ command. Therefore, only the entries
1459 * field need to be prepared in the CVQ command.
1461 mac_ptr
= out
->iov_base
+ cursor
;
1462 mac_ptr
->entries
= cpu_to_le32(fake_mul_entries
);
1465 * Simulating QEMU poll a vdpa device used buffer
1466 * for VIRTIO_NET_CTRL_MAC_TABLE_SET CVQ command
1468 return sizeof(*s
->status
);
1472 * Validate and copy control virtqueue commands.
1474 * Following QEMU guidelines, we offer a copy of the buffers to the device to
1475 * prevent TOCTOU bugs.
1477 static int vhost_vdpa_net_handle_ctrl_avail(VhostShadowVirtqueue
*svq
,
1478 VirtQueueElement
*elem
,
1481 VhostVDPAState
*s
= opaque
;
1483 const struct virtio_net_ctrl_hdr
*ctrl
;
1484 virtio_net_ctrl_ack status
= VIRTIO_NET_ERR
;
1485 /* Out buffer sent to both the vdpa device and the device model */
1486 struct iovec out
= {
1487 .iov_base
= s
->cvq_cmd_out_buffer
,
1489 /* in buffer used for device model */
1490 const struct iovec model_in
= {
1491 .iov_base
= &status
,
1492 .iov_len
= sizeof(status
),
1494 /* in buffer used for vdpa device */
1495 const struct iovec vdpa_in
= {
1496 .iov_base
= s
->status
,
1497 .iov_len
= sizeof(*s
->status
),
1499 ssize_t dev_written
= -EINVAL
;
1501 out
.iov_len
= iov_to_buf(elem
->out_sg
, elem
->out_num
, 0,
1502 s
->cvq_cmd_out_buffer
,
1503 vhost_vdpa_net_cvq_cmd_page_len());
1505 ctrl
= s
->cvq_cmd_out_buffer
;
1506 if (ctrl
->class == VIRTIO_NET_CTRL_ANNOUNCE
) {
1508 * Guest announce capability is emulated by qemu, so don't forward to
1511 dev_written
= sizeof(status
);
1512 *s
->status
= VIRTIO_NET_OK
;
1513 } else if (unlikely(ctrl
->class == VIRTIO_NET_CTRL_MAC
&&
1514 ctrl
->cmd
== VIRTIO_NET_CTRL_MAC_TABLE_SET
&&
1515 iov_size(elem
->out_sg
, elem
->out_num
) > out
.iov_len
)) {
1517 * Due to the size limitation of the out buffer sent to the vdpa device,
1518 * which is determined by vhost_vdpa_net_cvq_cmd_page_len(), excessive
1519 * MAC addresses set by the driver for the filter table can cause
1520 * truncation of the CVQ command in QEMU. As a result, the vdpa device
1521 * rejects the flawed CVQ command.
1523 * Therefore, QEMU must handle this situation instead of sending
1524 * the CVQ command directly.
1526 dev_written
= vhost_vdpa_net_excessive_mac_filter_cvq_add(s
, elem
,
1528 if (unlikely(dev_written
< 0)) {
1533 r
= vhost_vdpa_net_cvq_add(s
, &out
, 1, &vdpa_in
, 1);
1534 if (unlikely(r
< 0)) {
1540 * We can poll here since we've had BQL from the time
1541 * we sent the descriptor.
1543 dev_written
= vhost_vdpa_net_svq_poll(s
, 1);
1546 if (unlikely(dev_written
< sizeof(status
))) {
1547 error_report("Insufficient written data (%zu)", dev_written
);
1551 if (*s
->status
!= VIRTIO_NET_OK
) {
1555 status
= VIRTIO_NET_ERR
;
1556 virtio_net_handle_ctrl_iov(svq
->vdev
, &model_in
, 1, &out
, 1);
1557 if (status
!= VIRTIO_NET_OK
) {
1558 error_report("Bad CVQ processing in model");
1562 in_len
= iov_from_buf(elem
->in_sg
, elem
->in_num
, 0, &status
,
1564 if (unlikely(in_len
< sizeof(status
))) {
1565 error_report("Bad device CVQ written length");
1567 vhost_svq_push_elem(svq
, elem
, MIN(in_len
, sizeof(status
)));
1569 * `elem` belongs to vhost_vdpa_net_handle_ctrl_avail() only when
1570 * the function successfully forwards the CVQ command, indicated
1571 * by a non-negative value of `dev_written`. Otherwise, it still
1573 * This function should only free the `elem` when it owns.
1575 if (dev_written
>= 0) {
1578 return dev_written
< 0 ? dev_written
: 0;
1581 static const VhostShadowVirtqueueOps vhost_vdpa_net_svq_ops
= {
1582 .avail_handler
= vhost_vdpa_net_handle_ctrl_avail
,
1586 * Probe if CVQ is isolated
1588 * @device_fd The vdpa device fd
1589 * @features Features offered by the device.
1590 * @cvq_index The control vq pair index
1592 * Returns <0 in case of failure, 0 if false and 1 if true.
1594 static int vhost_vdpa_probe_cvq_isolation(int device_fd
, uint64_t features
,
1595 int cvq_index
, Error
**errp
)
1598 uint64_t backend_features
;
1600 uint8_t status
= VIRTIO_CONFIG_S_ACKNOWLEDGE
|
1601 VIRTIO_CONFIG_S_DRIVER
;
1604 r
= ioctl(device_fd
, VHOST_GET_BACKEND_FEATURES
, &backend_features
);
1605 if (unlikely(r
< 0)) {
1606 error_setg_errno(errp
, errno
, "Cannot get vdpa backend_features");
1610 if (!(backend_features
& BIT_ULL(VHOST_BACKEND_F_IOTLB_ASID
))) {
1614 r
= ioctl(device_fd
, VHOST_VDPA_SET_STATUS
, &status
);
1616 error_setg_errno(errp
, -r
, "Cannot set device status");
1620 r
= ioctl(device_fd
, VHOST_SET_FEATURES
, &features
);
1622 error_setg_errno(errp
, -r
, "Cannot set features");
1626 status
|= VIRTIO_CONFIG_S_FEATURES_OK
;
1627 r
= ioctl(device_fd
, VHOST_VDPA_SET_STATUS
, &status
);
1629 error_setg_errno(errp
, -r
, "Cannot set device status");
1633 cvq_group
= vhost_vdpa_get_vring_group(device_fd
, cvq_index
, errp
);
1634 if (unlikely(cvq_group
< 0)) {
1635 if (cvq_group
!= -ENOTSUP
) {
1641 * The kernel report VHOST_BACKEND_F_IOTLB_ASID if the vdpa frontend
1642 * support ASID even if the parent driver does not. The CVQ cannot be
1643 * isolated in this case.
1651 for (int i
= 0; i
< cvq_index
; ++i
) {
1652 int64_t group
= vhost_vdpa_get_vring_group(device_fd
, i
, errp
);
1653 if (unlikely(group
< 0)) {
1658 if (group
== (int64_t)cvq_group
) {
1668 ioctl(device_fd
, VHOST_VDPA_SET_STATUS
, &status
);
1672 static NetClientState
*net_vhost_vdpa_init(NetClientState
*peer
,
1676 int queue_pair_index
,
1680 struct vhost_vdpa_iova_range iova_range
,
1682 VhostVDPAShared
*shared
,
1685 NetClientState
*nc
= NULL
;
1689 int cvq_isolated
= 0;
1692 nc
= qemu_new_net_client(&net_vhost_vdpa_info
, peer
, device
,
1695 cvq_isolated
= vhost_vdpa_probe_cvq_isolation(vdpa_device_fd
, features
,
1696 queue_pair_index
* 2,
1698 if (unlikely(cvq_isolated
< 0)) {
1702 nc
= qemu_new_net_control_client(&net_vhost_vdpa_cvq_info
, peer
,
1705 qemu_set_info_str(nc
, TYPE_VHOST_VDPA
);
1706 s
= DO_UPCAST(VhostVDPAState
, nc
, nc
);
1708 s
->vhost_vdpa
.index
= queue_pair_index
;
1709 s
->always_svq
= svq
;
1710 s
->migration_state
.notify
= NULL
;
1711 s
->vhost_vdpa
.shadow_vqs_enabled
= svq
;
1712 if (queue_pair_index
== 0) {
1713 vhost_vdpa_net_valid_svq_features(features
,
1714 &s
->vhost_vdpa
.migration_blocker
);
1715 s
->vhost_vdpa
.shared
= g_new0(VhostVDPAShared
, 1);
1716 s
->vhost_vdpa
.shared
->device_fd
= vdpa_device_fd
;
1717 s
->vhost_vdpa
.shared
->iova_range
= iova_range
;
1718 s
->vhost_vdpa
.shared
->shadow_data
= svq
;
1719 } else if (!is_datapath
) {
1720 s
->cvq_cmd_out_buffer
= mmap(NULL
, vhost_vdpa_net_cvq_cmd_page_len(),
1721 PROT_READ
| PROT_WRITE
,
1722 MAP_SHARED
| MAP_ANONYMOUS
, -1, 0);
1723 s
->status
= mmap(NULL
, vhost_vdpa_net_cvq_cmd_page_len(),
1724 PROT_READ
| PROT_WRITE
, MAP_SHARED
| MAP_ANONYMOUS
,
1727 s
->vhost_vdpa
.shadow_vq_ops
= &vhost_vdpa_net_svq_ops
;
1728 s
->vhost_vdpa
.shadow_vq_ops_opaque
= s
;
1729 s
->cvq_isolated
= cvq_isolated
;
1731 if (queue_pair_index
!= 0) {
1732 s
->vhost_vdpa
.shared
= shared
;
1735 ret
= vhost_vdpa_add(nc
, (void *)&s
->vhost_vdpa
, queue_pair_index
, nvqs
);
1737 qemu_del_net_client(nc
);
1744 static int vhost_vdpa_get_features(int fd
, uint64_t *features
, Error
**errp
)
1746 int ret
= ioctl(fd
, VHOST_GET_FEATURES
, features
);
1747 if (unlikely(ret
< 0)) {
1748 error_setg_errno(errp
, errno
,
1749 "Fail to query features from vhost-vDPA device");
1754 static int vhost_vdpa_get_max_queue_pairs(int fd
, uint64_t features
,
1755 int *has_cvq
, Error
**errp
)
1757 unsigned long config_size
= offsetof(struct vhost_vdpa_config
, buf
);
1758 g_autofree
struct vhost_vdpa_config
*config
= NULL
;
1759 __virtio16
*max_queue_pairs
;
1762 if (features
& (1 << VIRTIO_NET_F_CTRL_VQ
)) {
1768 if (features
& (1 << VIRTIO_NET_F_MQ
)) {
1769 config
= g_malloc0(config_size
+ sizeof(*max_queue_pairs
));
1770 config
->off
= offsetof(struct virtio_net_config
, max_virtqueue_pairs
);
1771 config
->len
= sizeof(*max_queue_pairs
);
1773 ret
= ioctl(fd
, VHOST_VDPA_GET_CONFIG
, config
);
1775 error_setg(errp
, "Fail to get config from vhost-vDPA device");
1779 max_queue_pairs
= (__virtio16
*)&config
->buf
;
1781 return lduw_le_p(max_queue_pairs
);
1787 int net_init_vhost_vdpa(const Netdev
*netdev
, const char *name
,
1788 NetClientState
*peer
, Error
**errp
)
1791 const NetdevVhostVDPAOptions
*opts
;
1794 g_autofree NetClientState
**ncs
= NULL
;
1795 struct vhost_vdpa_iova_range iova_range
;
1797 int queue_pairs
, r
, i
= 0, has_cvq
= 0;
1799 assert(netdev
->type
== NET_CLIENT_DRIVER_VHOST_VDPA
);
1800 opts
= &netdev
->u
.vhost_vdpa
;
1801 if (!opts
->vhostdev
&& !opts
->vhostfd
) {
1803 "vhost-vdpa: neither vhostdev= nor vhostfd= was specified");
1807 if (opts
->vhostdev
&& opts
->vhostfd
) {
1809 "vhost-vdpa: vhostdev= and vhostfd= are mutually exclusive");
1813 if (opts
->vhostdev
) {
1814 vdpa_device_fd
= qemu_open(opts
->vhostdev
, O_RDWR
, errp
);
1815 if (vdpa_device_fd
== -1) {
1820 vdpa_device_fd
= monitor_fd_param(monitor_cur(), opts
->vhostfd
, errp
);
1821 if (vdpa_device_fd
== -1) {
1822 error_prepend(errp
, "vhost-vdpa: unable to parse vhostfd: ");
1827 r
= vhost_vdpa_get_features(vdpa_device_fd
, &features
, errp
);
1828 if (unlikely(r
< 0)) {
1832 queue_pairs
= vhost_vdpa_get_max_queue_pairs(vdpa_device_fd
, features
,
1834 if (queue_pairs
< 0) {
1835 qemu_close(vdpa_device_fd
);
1839 r
= vhost_vdpa_get_iova_range(vdpa_device_fd
, &iova_range
);
1840 if (unlikely(r
< 0)) {
1841 error_setg(errp
, "vhost-vdpa: get iova range failed: %s",
1846 if (opts
->x_svq
&& !vhost_vdpa_net_valid_svq_features(features
, errp
)) {
1850 ncs
= g_malloc0(sizeof(*ncs
) * queue_pairs
);
1852 for (i
= 0; i
< queue_pairs
; i
++) {
1853 VhostVDPAShared
*shared
= NULL
;
1856 shared
= DO_UPCAST(VhostVDPAState
, nc
, ncs
[0])->vhost_vdpa
.shared
;
1858 ncs
[i
] = net_vhost_vdpa_init(peer
, TYPE_VHOST_VDPA
, name
,
1859 vdpa_device_fd
, i
, 2, true, opts
->x_svq
,
1860 iova_range
, features
, shared
, errp
);
1866 VhostVDPAState
*s0
= DO_UPCAST(VhostVDPAState
, nc
, ncs
[0]);
1867 VhostVDPAShared
*shared
= s0
->vhost_vdpa
.shared
;
1869 nc
= net_vhost_vdpa_init(peer
, TYPE_VHOST_VDPA
, name
,
1870 vdpa_device_fd
, i
, 1, false,
1871 opts
->x_svq
, iova_range
, features
, shared
,
1881 for (i
--; i
>= 0; i
--) {
1882 qemu_del_net_client(ncs
[i
]);
1886 qemu_close(vdpa_device_fd
);