?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'remotes/dgilbert-gitlab/tags/pull-migration-20210726a...
[qemu/armbru.git] / hw / virtio / vhost-user.c
blob29ea2b4fceb21933c5b0e1882701cab9058b9411
1 /*
2 * vhost-user
3 *
4 * Copyright (c) 2013 Virtual Open Systems Sarl.
5 *
6 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7 * See the COPYING file in the top-level directory.
8 *
9 */
10
11 #include "qemu/osdep.h"
12 #include "qapi/error.h"
13 #include "hw/virtio/vhost.h"
14 #include "hw/virtio/vhost-user.h"
15 #include "hw/virtio/vhost-backend.h"
16 #include "hw/virtio/virtio.h"
17 #include "hw/virtio/virtio-net.h"
18 #include "chardev/char-fe.h"
19 #include "io/channel-socket.h"
20 #include "sysemu/kvm.h"
21 #include "qemu/error-report.h"
22 #include "qemu/main-loop.h"
23 #include "qemu/sockets.h"
24 #include "sysemu/cryptodev.h"
25 #include "migration/migration.h"
26 #include "migration/postcopy-ram.h"
27 #include "trace.h"
28
29 #include <sys/ioctl.h>
30 #include <sys/socket.h>
31 #include <sys/un.h>
32
33 #include "standard-headers/linux/vhost_types.h"
34
35 #ifdef CONFIG_LINUX
36 #include <linux/userfaultfd.h>
37 #endif
38
39 #define VHOST_MEMORY_BASELINE_NREGIONS 8
40 #define VHOST_USER_F_PROTOCOL_FEATURES 30
41 #define VHOST_USER_SLAVE_MAX_FDS 8
42
43 /*
44 * Set maximum number of RAM slots supported to
45 * the maximum number supported by the target
46 * hardware plaform.
47 */
48 #if defined(TARGET_X86) || defined(TARGET_X86_64) || \
49 defined(TARGET_ARM) || defined(TARGET_ARM_64)
50 #include "hw/acpi/acpi.h"
51 #define VHOST_USER_MAX_RAM_SLOTS ACPI_MAX_RAM_SLOTS
52
53 #elif defined(TARGET_PPC) || defined(TARGET_PPC_64)
54 #include "hw/ppc/spapr.h"
55 #define VHOST_USER_MAX_RAM_SLOTS SPAPR_MAX_RAM_SLOTS
56
57 #else
58 #define VHOST_USER_MAX_RAM_SLOTS 512
59 #endif
60
61 /*
62 * Maximum size of virtio device config space
63 */
64 #define VHOST_USER_MAX_CONFIG_SIZE 256
65
66 enum VhostUserProtocolFeature {
67 VHOST_USER_PROTOCOL_F_MQ = 0,
68 VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
69 VHOST_USER_PROTOCOL_F_RARP = 2,
70 VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
71 VHOST_USER_PROTOCOL_F_NET_MTU = 4,
72 VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
73 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
74 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
75 VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
76 VHOST_USER_PROTOCOL_F_CONFIG = 9,
77 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
78 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
79 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12,
80 VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
81 /* Feature 14 reserved for VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS. */
82 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS = 15,
83 VHOST_USER_PROTOCOL_F_MAX
84 };
85
86 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
87
88 typedef enum VhostUserRequest {
89 VHOST_USER_NONE = 0,
90 VHOST_USER_GET_FEATURES = 1,
91 VHOST_USER_SET_FEATURES = 2,
92 VHOST_USER_SET_OWNER = 3,
93 VHOST_USER_RESET_OWNER = 4,
94 VHOST_USER_SET_MEM_TABLE = 5,
95 VHOST_USER_SET_LOG_BASE = 6,
96 VHOST_USER_SET_LOG_FD = 7,
97 VHOST_USER_SET_VRING_NUM = 8,
98 VHOST_USER_SET_VRING_ADDR = 9,
99 VHOST_USER_SET_VRING_BASE = 10,
100 VHOST_USER_GET_VRING_BASE = 11,
101 VHOST_USER_SET_VRING_KICK = 12,
102 VHOST_USER_SET_VRING_CALL = 13,
103 VHOST_USER_SET_VRING_ERR = 14,
104 VHOST_USER_GET_PROTOCOL_FEATURES = 15,
105 VHOST_USER_SET_PROTOCOL_FEATURES = 16,
106 VHOST_USER_GET_QUEUE_NUM = 17,
107 VHOST_USER_SET_VRING_ENABLE = 18,
108 VHOST_USER_SEND_RARP = 19,
109 VHOST_USER_NET_SET_MTU = 20,
110 VHOST_USER_SET_SLAVE_REQ_FD = 21,
111 VHOST_USER_IOTLB_MSG = 22,
112 VHOST_USER_SET_VRING_ENDIAN = 23,
113 VHOST_USER_GET_CONFIG = 24,
114 VHOST_USER_SET_CONFIG = 25,
115 VHOST_USER_CREATE_CRYPTO_SESSION = 26,
116 VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
117 VHOST_USER_POSTCOPY_ADVISE = 28,
118 VHOST_USER_POSTCOPY_LISTEN = 29,
119 VHOST_USER_POSTCOPY_END = 30,
120 VHOST_USER_GET_INFLIGHT_FD = 31,
121 VHOST_USER_SET_INFLIGHT_FD = 32,
122 VHOST_USER_GPU_SET_SOCKET = 33,
123 VHOST_USER_RESET_DEVICE = 34,
124 /* Message number 35 reserved for VHOST_USER_VRING_KICK. */
125 VHOST_USER_GET_MAX_MEM_SLOTS = 36,
126 VHOST_USER_ADD_MEM_REG = 37,
127 VHOST_USER_REM_MEM_REG = 38,
128 VHOST_USER_MAX
129 } VhostUserRequest;
130
131 typedef enum VhostUserSlaveRequest {
132 VHOST_USER_SLAVE_NONE = 0,
133 VHOST_USER_SLAVE_IOTLB_MSG = 1,
134 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
135 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
136 VHOST_USER_SLAVE_MAX
137 } VhostUserSlaveRequest;
138
139 typedef struct VhostUserMemoryRegion {
140 uint64_t guest_phys_addr;
141 uint64_t memory_size;
142 uint64_t userspace_addr;
143 uint64_t mmap_offset;
144 } VhostUserMemoryRegion;
145
146 typedef struct VhostUserMemory {
147 uint32_t nregions;
148 uint32_t padding;
149 VhostUserMemoryRegion regions[VHOST_MEMORY_BASELINE_NREGIONS];
150 } VhostUserMemory;
151
152 typedef struct VhostUserMemRegMsg {
153 uint64_t padding;
154 VhostUserMemoryRegion region;
155 } VhostUserMemRegMsg;
156
157 typedef struct VhostUserLog {
158 uint64_t mmap_size;
159 uint64_t mmap_offset;
160 } VhostUserLog;
161
162 typedef struct VhostUserConfig {
163 uint32_t offset;
164 uint32_t size;
165 uint32_t flags;
166 uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
167 } VhostUserConfig;
168
169 #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
170 #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
171
172 typedef struct VhostUserCryptoSession {
173 /* session id for success, -1 on errors */
174 int64_t session_id;
175 CryptoDevBackendSymSessionInfo session_setup_data;
176 uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
177 uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
178 } VhostUserCryptoSession;
179
180 static VhostUserConfig c __attribute__ ((unused));
181 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
182 + sizeof(c.size) \
183 + sizeof(c.flags))
184
185 typedef struct VhostUserVringArea {
186 uint64_t u64;
187 uint64_t size;
188 uint64_t offset;
189 } VhostUserVringArea;
190
191 typedef struct VhostUserInflight {
192 uint64_t mmap_size;
193 uint64_t mmap_offset;
194 uint16_t num_queues;
195 uint16_t queue_size;
196 } VhostUserInflight;
197
198 typedef struct {
199 VhostUserRequest request;
200
201 #define VHOST_USER_VERSION_MASK (0x3)
202 #define VHOST_USER_REPLY_MASK (0x1<<2)
203 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
204 uint32_t flags;
205 uint32_t size; /* the following payload size */
206 } QEMU_PACKED VhostUserHeader;
207
208 typedef union {
209 #define VHOST_USER_VRING_IDX_MASK (0xff)
210 #define VHOST_USER_VRING_NOFD_MASK (0x1<<8)
211 uint64_t u64;
212 struct vhost_vring_state state;
213 struct vhost_vring_addr addr;
214 VhostUserMemory memory;
215 VhostUserMemRegMsg mem_reg;
216 VhostUserLog log;
217 struct vhost_iotlb_msg iotlb;
218 VhostUserConfig config;
219 VhostUserCryptoSession session;
220 VhostUserVringArea area;
221 VhostUserInflight inflight;
222 } VhostUserPayload;
223
224 typedef struct VhostUserMsg {
225 VhostUserHeader hdr;
226 VhostUserPayload payload;
227 } QEMU_PACKED VhostUserMsg;
228
229 static VhostUserMsg m __attribute__ ((unused));
230 #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
231
232 #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
233
234 /* The version of the protocol we support */
235 #define VHOST_USER_VERSION (0x1)
236
237 struct vhost_user {
238 struct vhost_dev *dev;
239 /* Shared between vhost devs of the same virtio device */
240 VhostUserState *user;
241 QIOChannel *slave_ioc;
242 GSource *slave_src;
243 NotifierWithReturn postcopy_notifier;
244 struct PostCopyFD postcopy_fd;
245 uint64_t postcopy_client_bases[VHOST_USER_MAX_RAM_SLOTS];
246 /* Length of the region_rb and region_rb_offset arrays */
247 size_t region_rb_len;
248 /* RAMBlock associated with a given region */
249 RAMBlock **region_rb;
250 /* The offset from the start of the RAMBlock to the start of the
251 * vhost region.
252 */
253 ram_addr_t *region_rb_offset;
254
255 /* True once we've entered postcopy_listen */
256 bool postcopy_listen;
257
258 /* Our current regions */
259 int num_shadow_regions;
260 struct vhost_memory_region shadow_regions[VHOST_USER_MAX_RAM_SLOTS];
261 };
262
263 struct scrub_regions {
264 struct vhost_memory_region *region;
265 int reg_idx;
266 int fd_idx;
267 };
268
269 static bool ioeventfd_enabled(void)
270 {
271 return !kvm_enabled() || kvm_eventfds_enabled();
272 }
273
274 static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg)
275 {
276 struct vhost_user *u = dev->opaque;
277 CharBackend *chr = u->user->chr;
278 uint8_t *p = (uint8_t *) msg;
279 int r, size = VHOST_USER_HDR_SIZE;
280
281 r = qemu_chr_fe_read_all(chr, p, size);
282 if (r != size) {
283 error_report("Failed to read msg header. Read %d instead of %d."
284 " Original request %d.", r, size, msg->hdr.request);
285 return -1;
286 }
287
288 /* validate received flags */
289 if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
290 error_report("Failed to read msg header."
291 " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
292 VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
293 return -1;
294 }
295
296 return 0;
297 }
298
299 struct vhost_user_read_cb_data {
300 struct vhost_dev *dev;
301 VhostUserMsg *msg;
302 GMainLoop *loop;
303 int ret;
304 };
305
306 static gboolean vhost_user_read_cb(GIOChannel *source, GIOCondition condition,
307 gpointer opaque)
308 {
309 struct vhost_user_read_cb_data *data = opaque;
310 struct vhost_dev *dev = data->dev;
311 VhostUserMsg *msg = data->msg;
312 struct vhost_user *u = dev->opaque;
313 CharBackend *chr = u->user->chr;
314 uint8_t *p = (uint8_t *) msg;
315 int r, size;
316
317 if (vhost_user_read_header(dev, msg) < 0) {
318 data->ret = -1;
319 goto end;
320 }
321
322 /* validate message size is sane */
323 if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
324 error_report("Failed to read msg header."
325 " Size %d exceeds the maximum %zu.", msg->hdr.size,
326 VHOST_USER_PAYLOAD_SIZE);
327 data->ret = -1;
328 goto end;
329 }
330
331 if (msg->hdr.size) {
332 p += VHOST_USER_HDR_SIZE;
333 size = msg->hdr.size;
334 r = qemu_chr_fe_read_all(chr, p, size);
335 if (r != size) {
336 error_report("Failed to read msg payload."
337 " Read %d instead of %d.", r, msg->hdr.size);
338 data->ret = -1;
339 goto end;
340 }
341 }
342
343 end:
344 g_main_loop_quit(data->loop);
345 return G_SOURCE_REMOVE;
346 }
347
348 static gboolean slave_read(QIOChannel *ioc, GIOCondition condition,
349 gpointer opaque);
350
351 /*
352 * This updates the read handler to use a new event loop context.
353 * Event sources are removed from the previous context : this ensures
354 * that events detected in the previous context are purged. They will
355 * be re-detected and processed in the new context.
356 */
357 static void slave_update_read_handler(struct vhost_dev *dev,
358 GMainContext *ctxt)
359 {
360 struct vhost_user *u = dev->opaque;
361
362 if (!u->slave_ioc) {
363 return;
364 }
365
366 if (u->slave_src) {
367 g_source_destroy(u->slave_src);
368 g_source_unref(u->slave_src);
369 }
370
371 u->slave_src = qio_channel_add_watch_source(u->slave_ioc,
372 G_IO_IN | G_IO_HUP,
373 slave_read, dev, NULL,
374 ctxt);
375 }
376
377 static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
378 {
379 struct vhost_user *u = dev->opaque;
380 CharBackend *chr = u->user->chr;
381 GMainContext *prev_ctxt = chr->chr->gcontext;
382 GMainContext *ctxt = g_main_context_new();
383 GMainLoop *loop = g_main_loop_new(ctxt, FALSE);
384 struct vhost_user_read_cb_data data = {
385 .dev = dev,
386 .loop = loop,
387 .msg = msg,
388 .ret = 0
389 };
390
391 /*
392 * We want to be able to monitor the slave channel fd while waiting
393 * for chr I/O. This requires an event loop, but we can't nest the
394 * one to which chr is currently attached : its fd handlers might not
395 * be prepared for re-entrancy. So we create a new one and switch chr
396 * to use it.
397 */
398 slave_update_read_handler(dev, ctxt);
399 qemu_chr_be_update_read_handlers(chr->chr, ctxt);
400 qemu_chr_fe_add_watch(chr, G_IO_IN | G_IO_HUP, vhost_user_read_cb, &data);
401
402 g_main_loop_run(loop);
403
404 /*
405 * Restore the previous event loop context. This also destroys/recreates
406 * event sources : this guarantees that all pending events in the original
407 * context that have been processed by the nested loop are purged.
408 */
409 qemu_chr_be_update_read_handlers(chr->chr, prev_ctxt);
410 slave_update_read_handler(dev, NULL);
411
412 g_main_loop_unref(loop);
413 g_main_context_unref(ctxt);
414
415 return data.ret;
416 }
417
418 static int process_message_reply(struct vhost_dev *dev,
419 const VhostUserMsg *msg)
420 {
421 VhostUserMsg msg_reply;
422
423 if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
424 return 0;
425 }
426
427 if (vhost_user_read(dev, &msg_reply) < 0) {
428 return -1;
429 }
430
431 if (msg_reply.hdr.request != msg->hdr.request) {
432 error_report("Received unexpected msg type."
433 "Expected %d received %d",
434 msg->hdr.request, msg_reply.hdr.request);
435 return -1;
436 }
437
438 return msg_reply.payload.u64 ? -1 : 0;
439 }
440
441 static bool vhost_user_one_time_request(VhostUserRequest request)
442 {
443 switch (request) {
444 case VHOST_USER_SET_OWNER:
445 case VHOST_USER_RESET_OWNER:
446 case VHOST_USER_SET_MEM_TABLE:
447 case VHOST_USER_GET_QUEUE_NUM:
448 case VHOST_USER_NET_SET_MTU:
449 return true;
450 default:
451 return false;
452 }
453 }
454
455 /* most non-init callers ignore the error */
456 static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
457 int *fds, int fd_num)
458 {
459 struct vhost_user *u = dev->opaque;
460 CharBackend *chr = u->user->chr;
461 int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
462
463 /*
464 * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
465 * we just need send it once in the first time. For later such
466 * request, we just ignore it.
467 */
468 if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) {
469 msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
470 return 0;
471 }
472
473 if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
474 error_report("Failed to set msg fds.");
475 return -1;
476 }
477
478 ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
479 if (ret != size) {
480 error_report("Failed to write msg."
481 " Wrote %d instead of %d.", ret, size);
482 return -1;
483 }
484
485 return 0;
486 }
487
488 int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd)
489 {
490 VhostUserMsg msg = {
491 .hdr.request = VHOST_USER_GPU_SET_SOCKET,
492 .hdr.flags = VHOST_USER_VERSION,
493 };
494
495 return vhost_user_write(dev, &msg, &fd, 1);
496 }
497
498 static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
499 struct vhost_log *log)
500 {
501 int fds[VHOST_USER_MAX_RAM_SLOTS];
502 size_t fd_num = 0;
503 bool shmfd = virtio_has_feature(dev->protocol_features,
504 VHOST_USER_PROTOCOL_F_LOG_SHMFD);
505 VhostUserMsg msg = {
506 .hdr.request = VHOST_USER_SET_LOG_BASE,
507 .hdr.flags = VHOST_USER_VERSION,
508 .payload.log.mmap_size = log->size * sizeof(*(log->log)),
509 .payload.log.mmap_offset = 0,
510 .hdr.size = sizeof(msg.payload.log),
511 };
512
513 if (shmfd && log->fd != -1) {
514 fds[fd_num++] = log->fd;
515 }
516
517 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
518 return -1;
519 }
520
521 if (shmfd) {
522 msg.hdr.size = 0;
523 if (vhost_user_read(dev, &msg) < 0) {
524 return -1;
525 }
526
527 if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
528 error_report("Received unexpected msg type. "
529 "Expected %d received %d",
530 VHOST_USER_SET_LOG_BASE, msg.hdr.request);
531 return -1;
532 }
533 }
534
535 return 0;
536 }
537
538 static MemoryRegion *vhost_user_get_mr_data(uint64_t addr, ram_addr_t *offset,
539 int *fd)
540 {
541 MemoryRegion *mr;
542
543 assert((uintptr_t)addr == addr);
544 mr = memory_region_from_host((void *)(uintptr_t)addr, offset);
545 *fd = memory_region_get_fd(mr);
546
547 return mr;
548 }
549
550 static void vhost_user_fill_msg_region(VhostUserMemoryRegion *dst,
551 struct vhost_memory_region *src,
552 uint64_t mmap_offset)
553 {
554 assert(src != NULL && dst != NULL);
555 dst->userspace_addr = src->userspace_addr;
556 dst->memory_size = src->memory_size;
557 dst->guest_phys_addr = src->guest_phys_addr;
558 dst->mmap_offset = mmap_offset;
559 }
560
561 static int vhost_user_fill_set_mem_table_msg(struct vhost_user *u,
562 struct vhost_dev *dev,
563 VhostUserMsg *msg,
564 int *fds, size_t *fd_num,
565 bool track_ramblocks)
566 {
567 int i, fd;
568 ram_addr_t offset;
569 MemoryRegion *mr;
570 struct vhost_memory_region *reg;
571 VhostUserMemoryRegion region_buffer;
572
573 msg->hdr.request = VHOST_USER_SET_MEM_TABLE;
574
575 for (i = 0; i < dev->mem->nregions; ++i) {
576 reg = dev->mem->regions + i;
577
578 mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
579 if (fd > 0) {
580 if (track_ramblocks) {
581 assert(*fd_num < VHOST_MEMORY_BASELINE_NREGIONS);
582 trace_vhost_user_set_mem_table_withfd(*fd_num, mr->name,
583 reg->memory_size,
584 reg->guest_phys_addr,
585 reg->userspace_addr,
586 offset);
587 u->region_rb_offset[i] = offset;
588 u->region_rb[i] = mr->ram_block;
589 } else if (*fd_num == VHOST_MEMORY_BASELINE_NREGIONS) {
590 error_report("Failed preparing vhost-user memory table msg");
591 return -1;
592 }
593 vhost_user_fill_msg_region(&region_buffer, reg, offset);
594 msg->payload.memory.regions[*fd_num] = region_buffer;
595 fds[(*fd_num)++] = fd;
596 } else if (track_ramblocks) {
597 u->region_rb_offset[i] = 0;
598 u->region_rb[i] = NULL;
599 }
600 }
601
602 msg->payload.memory.nregions = *fd_num;
603
604 if (!*fd_num) {
605 error_report("Failed initializing vhost-user memory map, "
606 "consider using -object memory-backend-file share=on");
607 return -1;
608 }
609
610 msg->hdr.size = sizeof(msg->payload.memory.nregions);
611 msg->hdr.size += sizeof(msg->payload.memory.padding);
612 msg->hdr.size += *fd_num * sizeof(VhostUserMemoryRegion);
613
614 return 1;
615 }
616
617 static inline bool reg_equal(struct vhost_memory_region *shadow_reg,
618 struct vhost_memory_region *vdev_reg)
619 {
620 return shadow_reg->guest_phys_addr == vdev_reg->guest_phys_addr &&
621 shadow_reg->userspace_addr == vdev_reg->userspace_addr &&
622 shadow_reg->memory_size == vdev_reg->memory_size;
623 }
624
625 static void scrub_shadow_regions(struct vhost_dev *dev,
626 struct scrub_regions *add_reg,
627 int *nr_add_reg,
628 struct scrub_regions *rem_reg,
629 int *nr_rem_reg, uint64_t *shadow_pcb,
630 bool track_ramblocks)
631 {
632 struct vhost_user *u = dev->opaque;
633 bool found[VHOST_USER_MAX_RAM_SLOTS] = {};
634 struct vhost_memory_region *reg, *shadow_reg;
635 int i, j, fd, add_idx = 0, rm_idx = 0, fd_num = 0;
636 ram_addr_t offset;
637 MemoryRegion *mr;
638 bool matching;
639
640 /*
641 * Find memory regions present in our shadow state which are not in
642 * the device's current memory state.
643 *
644 * Mark regions in both the shadow and device state as "found".
645 */
646 for (i = 0; i < u->num_shadow_regions; i++) {
647 shadow_reg = &u->shadow_regions[i];
648 matching = false;
649
650 for (j = 0; j < dev->mem->nregions; j++) {
651 reg = &dev->mem->regions[j];
652
653 mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
654
655 if (reg_equal(shadow_reg, reg)) {
656 matching = true;
657 found[j] = true;
658 if (track_ramblocks) {
659 /*
660 * Reset postcopy client bases, region_rb, and
661 * region_rb_offset in case regions are removed.
662 */
663 if (fd > 0) {
664 u->region_rb_offset[j] = offset;
665 u->region_rb[j] = mr->ram_block;
666 shadow_pcb[j] = u->postcopy_client_bases[i];
667 } else {
668 u->region_rb_offset[j] = 0;
669 u->region_rb[j] = NULL;
670 }
671 }
672 break;
673 }
674 }
675
676 /*
677 * If the region was not found in the current device memory state
678 * create an entry for it in the removed list.
679 */
680 if (!matching) {
681 rem_reg[rm_idx].region = shadow_reg;
682 rem_reg[rm_idx++].reg_idx = i;
683 }
684 }
685
686 /*
687 * For regions not marked "found", create entries in the added list.
688 *
689 * Note their indexes in the device memory state and the indexes of their
690 * file descriptors.
691 */
692 for (i = 0; i < dev->mem->nregions; i++) {
693 reg = &dev->mem->regions[i];
694 vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
695 if (fd > 0) {
696 ++fd_num;
697 }
698
699 /*
700 * If the region was in both the shadow and device state we don't
701 * need to send a VHOST_USER_ADD_MEM_REG message for it.
702 */
703 if (found[i]) {
704 continue;
705 }
706
707 add_reg[add_idx].region = reg;
708 add_reg[add_idx].reg_idx = i;
709 add_reg[add_idx++].fd_idx = fd_num;
710 }
711 *nr_rem_reg = rm_idx;
712 *nr_add_reg = add_idx;
713
714 return;
715 }
716
717 static int send_remove_regions(struct vhost_dev *dev,
718 struct scrub_regions *remove_reg,
719 int nr_rem_reg, VhostUserMsg *msg,
720 bool reply_supported)
721 {
722 struct vhost_user *u = dev->opaque;
723 struct vhost_memory_region *shadow_reg;
724 int i, fd, shadow_reg_idx, ret;
725 ram_addr_t offset;
726 VhostUserMemoryRegion region_buffer;
727
728 /*
729 * The regions in remove_reg appear in the same order they do in the
730 * shadow table. Therefore we can minimize memory copies by iterating
731 * through remove_reg backwards.
732 */
733 for (i = nr_rem_reg - 1; i >= 0; i--) {
734 shadow_reg = remove_reg[i].region;
735 shadow_reg_idx = remove_reg[i].reg_idx;
736
737 vhost_user_get_mr_data(shadow_reg->userspace_addr, &offset, &fd);
738
739 if (fd > 0) {
740 msg->hdr.request = VHOST_USER_REM_MEM_REG;
741 vhost_user_fill_msg_region(&region_buffer, shadow_reg, 0);
742 msg->payload.mem_reg.region = region_buffer;
743
744 if (vhost_user_write(dev, msg, &fd, 1) < 0) {
745 return -1;
746 }
747
748 if (reply_supported) {
749 ret = process_message_reply(dev, msg);
750 if (ret) {
751 return ret;
752 }
753 }
754 }
755
756 /*
757 * At this point we know the backend has unmapped the region. It is now
758 * safe to remove it from the shadow table.
759 */
760 memmove(&u->shadow_regions[shadow_reg_idx],
761 &u->shadow_regions[shadow_reg_idx + 1],
762 sizeof(struct vhost_memory_region) *
763 (u->num_shadow_regions - shadow_reg_idx - 1));
764 u->num_shadow_regions--;
765 }
766
767 return 0;
768 }
769
770 static int send_add_regions(struct vhost_dev *dev,
771 struct scrub_regions *add_reg, int nr_add_reg,
772 VhostUserMsg *msg, uint64_t *shadow_pcb,
773 bool reply_supported, bool track_ramblocks)
774 {
775 struct vhost_user *u = dev->opaque;
776 int i, fd, ret, reg_idx, reg_fd_idx;
777 struct vhost_memory_region *reg;
778 MemoryRegion *mr;
779 ram_addr_t offset;
780 VhostUserMsg msg_reply;
781 VhostUserMemoryRegion region_buffer;
782
783 for (i = 0; i < nr_add_reg; i++) {
784 reg = add_reg[i].region;
785 reg_idx = add_reg[i].reg_idx;
786 reg_fd_idx = add_reg[i].fd_idx;
787
788 mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
789
790 if (fd > 0) {
791 if (track_ramblocks) {
792 trace_vhost_user_set_mem_table_withfd(reg_fd_idx, mr->name,
793 reg->memory_size,
794 reg->guest_phys_addr,
795 reg->userspace_addr,
796 offset);
797 u->region_rb_offset[reg_idx] = offset;
798 u->region_rb[reg_idx] = mr->ram_block;
799 }
800 msg->hdr.request = VHOST_USER_ADD_MEM_REG;
801 vhost_user_fill_msg_region(&region_buffer, reg, offset);
802 msg->payload.mem_reg.region = region_buffer;
803
804 if (vhost_user_write(dev, msg, &fd, 1) < 0) {
805 return -1;
806 }
807
808 if (track_ramblocks) {
809 uint64_t reply_gpa;
810
811 if (vhost_user_read(dev, &msg_reply) < 0) {
812 return -1;
813 }
814
815 reply_gpa = msg_reply.payload.mem_reg.region.guest_phys_addr;
816
817 if (msg_reply.hdr.request != VHOST_USER_ADD_MEM_REG) {
818 error_report("%s: Received unexpected msg type."
819 "Expected %d received %d", __func__,
820 VHOST_USER_ADD_MEM_REG,
821 msg_reply.hdr.request);
822 return -1;
823 }
824
825 /*
826 * We're using the same structure, just reusing one of the
827 * fields, so it should be the same size.
828 */
829 if (msg_reply.hdr.size != msg->hdr.size) {
830 error_report("%s: Unexpected size for postcopy reply "
831 "%d vs %d", __func__, msg_reply.hdr.size,
832 msg->hdr.size);
833 return -1;
834 }
835
836 /* Get the postcopy client base from the backend's reply. */
837 if (reply_gpa == dev->mem->regions[reg_idx].guest_phys_addr) {
838 shadow_pcb[reg_idx] =
839 msg_reply.payload.mem_reg.region.userspace_addr;
840 trace_vhost_user_set_mem_table_postcopy(
841 msg_reply.payload.mem_reg.region.userspace_addr,
842 msg->payload.mem_reg.region.userspace_addr,
843 reg_fd_idx, reg_idx);
844 } else {
845 error_report("%s: invalid postcopy reply for region. "
846 "Got guest physical address %" PRIX64 ", expected "
847 "%" PRIX64, __func__, reply_gpa,
848 dev->mem->regions[reg_idx].guest_phys_addr);
849 return -1;
850 }
851 } else if (reply_supported) {
852 ret = process_message_reply(dev, msg);
853 if (ret) {
854 return ret;
855 }
856 }
857 } else if (track_ramblocks) {
858 u->region_rb_offset[reg_idx] = 0;
859 u->region_rb[reg_idx] = NULL;
860 }
861
862 /*
863 * At this point, we know the backend has mapped in the new
864 * region, if the region has a valid file descriptor.
865 *
866 * The region should now be added to the shadow table.
867 */
868 u->shadow_regions[u->num_shadow_regions].guest_phys_addr =
869 reg->guest_phys_addr;
870 u->shadow_regions[u->num_shadow_regions].userspace_addr =
871 reg->userspace_addr;
872 u->shadow_regions[u->num_shadow_regions].memory_size =
873 reg->memory_size;
874 u->num_shadow_regions++;
875 }
876
877 return 0;
878 }
879
880 static int vhost_user_add_remove_regions(struct vhost_dev *dev,
881 VhostUserMsg *msg,
882 bool reply_supported,
883 bool track_ramblocks)
884 {
885 struct vhost_user *u = dev->opaque;
886 struct scrub_regions add_reg[VHOST_USER_MAX_RAM_SLOTS];
887 struct scrub_regions rem_reg[VHOST_USER_MAX_RAM_SLOTS];
888 uint64_t shadow_pcb[VHOST_USER_MAX_RAM_SLOTS] = {};
889 int nr_add_reg, nr_rem_reg;
890
891 msg->hdr.size = sizeof(msg->payload.mem_reg);
892
893 /* Find the regions which need to be removed or added. */
894 scrub_shadow_regions(dev, add_reg, &nr_add_reg, rem_reg, &nr_rem_reg,
895 shadow_pcb, track_ramblocks);
896
897 if (nr_rem_reg && send_remove_regions(dev, rem_reg, nr_rem_reg, msg,
898 reply_supported) < 0)
899 {
900 goto err;
901 }
902
903 if (nr_add_reg && send_add_regions(dev, add_reg, nr_add_reg, msg,
904 shadow_pcb, reply_supported, track_ramblocks) < 0)
905 {
906 goto err;
907 }
908
909 if (track_ramblocks) {
910 memcpy(u->postcopy_client_bases, shadow_pcb,
911 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
912 /*
913 * Now we've registered this with the postcopy code, we ack to the
914 * client, because now we're in the position to be able to deal with
915 * any faults it generates.
916 */
917 /* TODO: Use this for failure cases as well with a bad value. */
918 msg->hdr.size = sizeof(msg->payload.u64);
919 msg->payload.u64 = 0; /* OK */
920
921 if (vhost_user_write(dev, msg, NULL, 0) < 0) {
922 return -1;
923 }
924 }
925
926 return 0;
927
928 err:
929 if (track_ramblocks) {
930 memcpy(u->postcopy_client_bases, shadow_pcb,
931 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
932 }
933
934 return -1;
935 }
936
937 static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
938 struct vhost_memory *mem,
939 bool reply_supported,
940 bool config_mem_slots)
941 {
942 struct vhost_user *u = dev->opaque;
943 int fds[VHOST_MEMORY_BASELINE_NREGIONS];
944 size_t fd_num = 0;
945 VhostUserMsg msg_reply;
946 int region_i, msg_i;
947
948 VhostUserMsg msg = {
949 .hdr.flags = VHOST_USER_VERSION,
950 };
951
952 if (u->region_rb_len < dev->mem->nregions) {
953 u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
954 u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
955 dev->mem->nregions);
956 memset(&(u->region_rb[u->region_rb_len]), '\0',
957 sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len));
958 memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
959 sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
960 u->region_rb_len = dev->mem->nregions;
961 }
962
963 if (config_mem_slots) {
964 if (vhost_user_add_remove_regions(dev, &msg, reply_supported,
965 true) < 0) {
966 return -1;
967 }
968 } else {
969 if (vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
970 true) < 0) {
971 return -1;
972 }
973
974 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
975 return -1;
976 }
977
978 if (vhost_user_read(dev, &msg_reply) < 0) {
979 return -1;
980 }
981
982 if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
983 error_report("%s: Received unexpected msg type."
984 "Expected %d received %d", __func__,
985 VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
986 return -1;
987 }
988
989 /*
990 * We're using the same structure, just reusing one of the
991 * fields, so it should be the same size.
992 */
993 if (msg_reply.hdr.size != msg.hdr.size) {
994 error_report("%s: Unexpected size for postcopy reply "
995 "%d vs %d", __func__, msg_reply.hdr.size,
996 msg.hdr.size);
997 return -1;
998 }
999
1000 memset(u->postcopy_client_bases, 0,
1001 sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
1002
1003 /*
1004 * They're in the same order as the regions that were sent
1005 * but some of the regions were skipped (above) if they
1006 * didn't have fd's
1007 */
1008 for (msg_i = 0, region_i = 0;
1009 region_i < dev->mem->nregions;
1010 region_i++) {
1011 if (msg_i < fd_num &&
1012 msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
1013 dev->mem->regions[region_i].guest_phys_addr) {
1014 u->postcopy_client_bases[region_i] =
1015 msg_reply.payload.memory.regions[msg_i].userspace_addr;
1016 trace_vhost_user_set_mem_table_postcopy(
1017 msg_reply.payload.memory.regions[msg_i].userspace_addr,
1018 msg.payload.memory.regions[msg_i].userspace_addr,
1019 msg_i, region_i);
1020 msg_i++;
1021 }
1022 }
1023 if (msg_i != fd_num) {
1024 error_report("%s: postcopy reply not fully consumed "
1025 "%d vs %zd",
1026 __func__, msg_i, fd_num);
1027 return -1;
1028 }
1029
1030 /*
1031 * Now we've registered this with the postcopy code, we ack to the
1032 * client, because now we're in the position to be able to deal
1033 * with any faults it generates.
1034 */
1035 /* TODO: Use this for failure cases as well with a bad value. */
1036 msg.hdr.size = sizeof(msg.payload.u64);
1037 msg.payload.u64 = 0; /* OK */
1038 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1039 return -1;
1040 }
1041 }
1042
1043 return 0;
1044 }
1045
1046 static int vhost_user_set_mem_table(struct vhost_dev *dev,
1047 struct vhost_memory *mem)
1048 {
1049 struct vhost_user *u = dev->opaque;
1050 int fds[VHOST_MEMORY_BASELINE_NREGIONS];
1051 size_t fd_num = 0;
1052 bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
1053 bool reply_supported = virtio_has_feature(dev->protocol_features,
1054 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1055 bool config_mem_slots =
1056 virtio_has_feature(dev->protocol_features,
1057 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
1058
1059 if (do_postcopy) {
1060 /*
1061 * Postcopy has enough differences that it's best done in it's own
1062 * version
1063 */
1064 return vhost_user_set_mem_table_postcopy(dev, mem, reply_supported,
1065 config_mem_slots);
1066 }
1067
1068 VhostUserMsg msg = {
1069 .hdr.flags = VHOST_USER_VERSION,
1070 };
1071
1072 if (reply_supported) {
1073 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1074 }
1075
1076 if (config_mem_slots) {
1077 if (vhost_user_add_remove_regions(dev, &msg, reply_supported,
1078 false) < 0) {
1079 return -1;
1080 }
1081 } else {
1082 if (vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
1083 false) < 0) {
1084 return -1;
1085 }
1086 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
1087 return -1;
1088 }
1089
1090 if (reply_supported) {
1091 return process_message_reply(dev, &msg);
1092 }
1093 }
1094
1095 return 0;
1096 }
1097
1098 static int vhost_user_set_vring_addr(struct vhost_dev *dev,
1099 struct vhost_vring_addr *addr)
1100 {
1101 VhostUserMsg msg = {
1102 .hdr.request = VHOST_USER_SET_VRING_ADDR,
1103 .hdr.flags = VHOST_USER_VERSION,
1104 .payload.addr = *addr,
1105 .hdr.size = sizeof(msg.payload.addr),
1106 };
1107
1108 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1109 return -1;
1110 }
1111
1112 return 0;
1113 }
1114
1115 static int vhost_user_set_vring_endian(struct vhost_dev *dev,
1116 struct vhost_vring_state *ring)
1117 {
1118 bool cross_endian = virtio_has_feature(dev->protocol_features,
1119 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
1120 VhostUserMsg msg = {
1121 .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
1122 .hdr.flags = VHOST_USER_VERSION,
1123 .payload.state = *ring,
1124 .hdr.size = sizeof(msg.payload.state),
1125 };
1126
1127 if (!cross_endian) {
1128 error_report("vhost-user trying to send unhandled ioctl");
1129 return -1;
1130 }
1131
1132 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1133 return -1;
1134 }
1135
1136 return 0;
1137 }
1138
1139 static int vhost_set_vring(struct vhost_dev *dev,
1140 unsigned long int request,
1141 struct vhost_vring_state *ring)
1142 {
1143 VhostUserMsg msg = {
1144 .hdr.request = request,
1145 .hdr.flags = VHOST_USER_VERSION,
1146 .payload.state = *ring,
1147 .hdr.size = sizeof(msg.payload.state),
1148 };
1149
1150 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1151 return -1;
1152 }
1153
1154 return 0;
1155 }
1156
1157 static int vhost_user_set_vring_num(struct vhost_dev *dev,
1158 struct vhost_vring_state *ring)
1159 {
1160 return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
1161 }
1162
1163 static void vhost_user_host_notifier_restore(struct vhost_dev *dev,
1164 int queue_idx)
1165 {
1166 struct vhost_user *u = dev->opaque;
1167 VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
1168 VirtIODevice *vdev = dev->vdev;
1169
1170 if (n->addr && !n->set) {
1171 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true);
1172 n->set = true;
1173 }
1174 }
1175
1176 static void vhost_user_host_notifier_remove(struct vhost_dev *dev,
1177 int queue_idx)
1178 {
1179 struct vhost_user *u = dev->opaque;
1180 VhostUserHostNotifier *n = &u->user->notifier[queue_idx];
1181 VirtIODevice *vdev = dev->vdev;
1182
1183 if (n->addr && n->set) {
1184 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
1185 n->set = false;
1186 }
1187 }
1188
1189 static int vhost_user_set_vring_base(struct vhost_dev *dev,
1190 struct vhost_vring_state *ring)
1191 {
1192 vhost_user_host_notifier_restore(dev, ring->index);
1193
1194 return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
1195 }
1196
1197 static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
1198 {
1199 int i;
1200
1201 if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1202 return -1;
1203 }
1204
1205 for (i = 0; i < dev->nvqs; ++i) {
1206 struct vhost_vring_state state = {
1207 .index = dev->vq_index + i,
1208 .num = enable,
1209 };
1210
1211 vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
1212 }
1213
1214 return 0;
1215 }
1216
1217 static int vhost_user_get_vring_base(struct vhost_dev *dev,
1218 struct vhost_vring_state *ring)
1219 {
1220 VhostUserMsg msg = {
1221 .hdr.request = VHOST_USER_GET_VRING_BASE,
1222 .hdr.flags = VHOST_USER_VERSION,
1223 .payload.state = *ring,
1224 .hdr.size = sizeof(msg.payload.state),
1225 };
1226
1227 vhost_user_host_notifier_remove(dev, ring->index);
1228
1229 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1230 return -1;
1231 }
1232
1233 if (vhost_user_read(dev, &msg) < 0) {
1234 return -1;
1235 }
1236
1237 if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
1238 error_report("Received unexpected msg type. Expected %d received %d",
1239 VHOST_USER_GET_VRING_BASE, msg.hdr.request);
1240 return -1;
1241 }
1242
1243 if (msg.hdr.size != sizeof(msg.payload.state)) {
1244 error_report("Received bad msg size.");
1245 return -1;
1246 }
1247
1248 *ring = msg.payload.state;
1249
1250 return 0;
1251 }
1252
1253 static int vhost_set_vring_file(struct vhost_dev *dev,
1254 VhostUserRequest request,
1255 struct vhost_vring_file *file)
1256 {
1257 int fds[VHOST_USER_MAX_RAM_SLOTS];
1258 size_t fd_num = 0;
1259 VhostUserMsg msg = {
1260 .hdr.request = request,
1261 .hdr.flags = VHOST_USER_VERSION,
1262 .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
1263 .hdr.size = sizeof(msg.payload.u64),
1264 };
1265
1266 if (ioeventfd_enabled() && file->fd > 0) {
1267 fds[fd_num++] = file->fd;
1268 } else {
1269 msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
1270 }
1271
1272 if (vhost_user_write(dev, &msg, fds, fd_num) < 0) {
1273 return -1;
1274 }
1275
1276 return 0;
1277 }
1278
1279 static int vhost_user_set_vring_kick(struct vhost_dev *dev,
1280 struct vhost_vring_file *file)
1281 {
1282 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
1283 }
1284
1285 static int vhost_user_set_vring_call(struct vhost_dev *dev,
1286 struct vhost_vring_file *file)
1287 {
1288 return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
1289 }
1290
1291 static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64)
1292 {
1293 VhostUserMsg msg = {
1294 .hdr.request = request,
1295 .hdr.flags = VHOST_USER_VERSION,
1296 .payload.u64 = u64,
1297 .hdr.size = sizeof(msg.payload.u64),
1298 };
1299
1300 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1301 return -1;
1302 }
1303
1304 return 0;
1305 }
1306
1307 static int vhost_user_set_features(struct vhost_dev *dev,
1308 uint64_t features)
1309 {
1310 return vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES, features);
1311 }
1312
1313 static int vhost_user_set_protocol_features(struct vhost_dev *dev,
1314 uint64_t features)
1315 {
1316 return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features);
1317 }
1318
1319 static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
1320 {
1321 VhostUserMsg msg = {
1322 .hdr.request = request,
1323 .hdr.flags = VHOST_USER_VERSION,
1324 };
1325
1326 if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
1327 return 0;
1328 }
1329
1330 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1331 return -1;
1332 }
1333
1334 if (vhost_user_read(dev, &msg) < 0) {
1335 return -1;
1336 }
1337
1338 if (msg.hdr.request != request) {
1339 error_report("Received unexpected msg type. Expected %d received %d",
1340 request, msg.hdr.request);
1341 return -1;
1342 }
1343
1344 if (msg.hdr.size != sizeof(msg.payload.u64)) {
1345 error_report("Received bad msg size.");
1346 return -1;
1347 }
1348
1349 *u64 = msg.payload.u64;
1350
1351 return 0;
1352 }
1353
1354 static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
1355 {
1356 if (vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features) < 0) {
1357 return -EPROTO;
1358 }
1359
1360 return 0;
1361 }
1362
1363 static int vhost_user_set_owner(struct vhost_dev *dev)
1364 {
1365 VhostUserMsg msg = {
1366 .hdr.request = VHOST_USER_SET_OWNER,
1367 .hdr.flags = VHOST_USER_VERSION,
1368 };
1369
1370 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1371 return -EPROTO;
1372 }
1373
1374 return 0;
1375 }
1376
1377 static int vhost_user_get_max_memslots(struct vhost_dev *dev,
1378 uint64_t *max_memslots)
1379 {
1380 uint64_t backend_max_memslots;
1381 int err;
1382
1383 err = vhost_user_get_u64(dev, VHOST_USER_GET_MAX_MEM_SLOTS,
1384 &backend_max_memslots);
1385 if (err < 0) {
1386 return err;
1387 }
1388
1389 *max_memslots = backend_max_memslots;
1390
1391 return 0;
1392 }
1393
1394 static int vhost_user_reset_device(struct vhost_dev *dev)
1395 {
1396 VhostUserMsg msg = {
1397 .hdr.flags = VHOST_USER_VERSION,
1398 };
1399
1400 msg.hdr.request = virtio_has_feature(dev->protocol_features,
1401 VHOST_USER_PROTOCOL_F_RESET_DEVICE)
1402 ? VHOST_USER_RESET_DEVICE
1403 : VHOST_USER_RESET_OWNER;
1404
1405 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1406 return -1;
1407 }
1408
1409 return 0;
1410 }
1411
1412 static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
1413 {
1414 int ret = -1;
1415
1416 if (!dev->config_ops) {
1417 return -1;
1418 }
1419
1420 if (dev->config_ops->vhost_dev_config_notifier) {
1421 ret = dev->config_ops->vhost_dev_config_notifier(dev);
1422 }
1423
1424 return ret;
1425 }
1426
1427 static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
1428 VhostUserVringArea *area,
1429 int fd)
1430 {
1431 int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
1432 size_t page_size = qemu_real_host_page_size;
1433 struct vhost_user *u = dev->opaque;
1434 VhostUserState *user = u->user;
1435 VirtIODevice *vdev = dev->vdev;
1436 VhostUserHostNotifier *n;
1437 void *addr;
1438 char *name;
1439
1440 if (!virtio_has_feature(dev->protocol_features,
1441 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
1442 vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
1443 return -1;
1444 }
1445
1446 n = &user->notifier[queue_idx];
1447
1448 if (n->addr) {
1449 virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, false);
1450 object_unparent(OBJECT(&n->mr));
1451 munmap(n->addr, page_size);
1452 n->addr = NULL;
1453 }
1454
1455 if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
1456 return 0;
1457 }
1458
1459 /* Sanity check. */
1460 if (area->size != page_size) {
1461 return -1;
1462 }
1463
1464 addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
1465 fd, area->offset);
1466 if (addr == MAP_FAILED) {
1467 return -1;
1468 }
1469
1470 name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
1471 user, queue_idx);
1472 memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
1473 page_size, addr);
1474 g_free(name);
1475
1476 if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
1477 munmap(addr, page_size);
1478 return -1;
1479 }
1480
1481 n->addr = addr;
1482 n->set = true;
1483
1484 return 0;
1485 }
1486
1487 static void close_slave_channel(struct vhost_user *u)
1488 {
1489 g_source_destroy(u->slave_src);
1490 g_source_unref(u->slave_src);
1491 u->slave_src = NULL;
1492 object_unref(OBJECT(u->slave_ioc));
1493 u->slave_ioc = NULL;
1494 }
1495
1496 static gboolean slave_read(QIOChannel *ioc, GIOCondition condition,
1497 gpointer opaque)
1498 {
1499 struct vhost_dev *dev = opaque;
1500 struct vhost_user *u = dev->opaque;
1501 VhostUserHeader hdr = { 0, };
1502 VhostUserPayload payload = { 0, };
1503 Error *local_err = NULL;
1504 gboolean rc = G_SOURCE_CONTINUE;
1505 int ret = 0;
1506 struct iovec iov;
1507 g_autofree int *fd = NULL;
1508 size_t fdsize = 0;
1509 int i;
1510
1511 /* Read header */
1512 iov.iov_base = &hdr;
1513 iov.iov_len = VHOST_USER_HDR_SIZE;
1514
1515 if (qio_channel_readv_full_all(ioc, &iov, 1, &fd, &fdsize, &local_err)) {
1516 error_report_err(local_err);
1517 goto err;
1518 }
1519
1520 if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
1521 error_report("Failed to read msg header."
1522 " Size %d exceeds the maximum %zu.", hdr.size,
1523 VHOST_USER_PAYLOAD_SIZE);
1524 goto err;
1525 }
1526
1527 /* Read payload */
1528 if (qio_channel_read_all(ioc, (char *) &payload, hdr.size, &local_err)) {
1529 error_report_err(local_err);
1530 goto err;
1531 }
1532
1533 switch (hdr.request) {
1534 case VHOST_USER_SLAVE_IOTLB_MSG:
1535 ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
1536 break;
1537 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
1538 ret = vhost_user_slave_handle_config_change(dev);
1539 break;
1540 case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
1541 ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
1542 fd ? fd[0] : -1);
1543 break;
1544 default:
1545 error_report("Received unexpected msg type: %d.", hdr.request);
1546 ret = -EINVAL;
1547 }
1548
1549 /*
1550 * REPLY_ACK feature handling. Other reply types has to be managed
1551 * directly in their request handlers.
1552 */
1553 if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
1554 struct iovec iovec[2];
1555
1556
1557 hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
1558 hdr.flags |= VHOST_USER_REPLY_MASK;
1559
1560 payload.u64 = !!ret;
1561 hdr.size = sizeof(payload.u64);
1562
1563 iovec[0].iov_base = &hdr;
1564 iovec[0].iov_len = VHOST_USER_HDR_SIZE;
1565 iovec[1].iov_base = &payload;
1566 iovec[1].iov_len = hdr.size;
1567
1568 if (qio_channel_writev_all(ioc, iovec, ARRAY_SIZE(iovec), &local_err)) {
1569 error_report_err(local_err);
1570 goto err;
1571 }
1572 }
1573
1574 goto fdcleanup;
1575
1576 err:
1577 close_slave_channel(u);
1578 rc = G_SOURCE_REMOVE;
1579
1580 fdcleanup:
1581 if (fd) {
1582 for (i = 0; i < fdsize; i++) {
1583 close(fd[i]);
1584 }
1585 }
1586 return rc;
1587 }
1588
1589 static int vhost_setup_slave_channel(struct vhost_dev *dev)
1590 {
1591 VhostUserMsg msg = {
1592 .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD,
1593 .hdr.flags = VHOST_USER_VERSION,
1594 };
1595 struct vhost_user *u = dev->opaque;
1596 int sv[2], ret = 0;
1597 bool reply_supported = virtio_has_feature(dev->protocol_features,
1598 VHOST_USER_PROTOCOL_F_REPLY_ACK);
1599 Error *local_err = NULL;
1600 QIOChannel *ioc;
1601
1602 if (!virtio_has_feature(dev->protocol_features,
1603 VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
1604 return 0;
1605 }
1606
1607 if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
1608 error_report("socketpair() failed");
1609 return -1;
1610 }
1611
1612 ioc = QIO_CHANNEL(qio_channel_socket_new_fd(sv[0], &local_err));
1613 if (!ioc) {
1614 error_report_err(local_err);
1615 return -1;
1616 }
1617 u->slave_ioc = ioc;
1618 slave_update_read_handler(dev, NULL);
1619
1620 if (reply_supported) {
1621 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
1622 }
1623
1624 ret = vhost_user_write(dev, &msg, &sv[1], 1);
1625 if (ret) {
1626 goto out;
1627 }
1628
1629 if (reply_supported) {
1630 ret = process_message_reply(dev, &msg);
1631 }
1632
1633 out:
1634 close(sv[1]);
1635 if (ret) {
1636 close_slave_channel(u);
1637 }
1638
1639 return ret;
1640 }
1641
1642 #ifdef CONFIG_LINUX
1643 /*
1644 * Called back from the postcopy fault thread when a fault is received on our
1645 * ufd.
1646 * TODO: This is Linux specific
1647 */
1648 static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
1649 void *ufd)
1650 {
1651 struct vhost_dev *dev = pcfd->data;
1652 struct vhost_user *u = dev->opaque;
1653 struct uffd_msg *msg = ufd;
1654 uint64_t faultaddr = msg->arg.pagefault.address;
1655 RAMBlock *rb = NULL;
1656 uint64_t rb_offset;
1657 int i;
1658
1659 trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
1660 dev->mem->nregions);
1661 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1662 trace_vhost_user_postcopy_fault_handler_loop(i,
1663 u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
1664 if (faultaddr >= u->postcopy_client_bases[i]) {
1665 /* Ofset of the fault address in the vhost region */
1666 uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
1667 if (region_offset < dev->mem->regions[i].memory_size) {
1668 rb_offset = region_offset + u->region_rb_offset[i];
1669 trace_vhost_user_postcopy_fault_handler_found(i,
1670 region_offset, rb_offset);
1671 rb = u->region_rb[i];
1672 return postcopy_request_shared_page(pcfd, rb, faultaddr,
1673 rb_offset);
1674 }
1675 }
1676 }
1677 error_report("%s: Failed to find region for fault %" PRIx64,
1678 __func__, faultaddr);
1679 return -1;
1680 }
1681
1682 static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
1683 uint64_t offset)
1684 {
1685 struct vhost_dev *dev = pcfd->data;
1686 struct vhost_user *u = dev->opaque;
1687 int i;
1688
1689 trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
1690
1691 if (!u) {
1692 return 0;
1693 }
1694 /* Translate the offset into an address in the clients address space */
1695 for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
1696 if (u->region_rb[i] == rb &&
1697 offset >= u->region_rb_offset[i] &&
1698 offset < (u->region_rb_offset[i] +
1699 dev->mem->regions[i].memory_size)) {
1700 uint64_t client_addr = (offset - u->region_rb_offset[i]) +
1701 u->postcopy_client_bases[i];
1702 trace_vhost_user_postcopy_waker_found(client_addr);
1703 return postcopy_wake_shared(pcfd, client_addr, rb);
1704 }
1705 }
1706
1707 trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
1708 return 0;
1709 }
1710 #endif
1711
1712 /*
1713 * Called at the start of an inbound postcopy on reception of the
1714 * 'advise' command.
1715 */
1716 static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
1717 {
1718 #ifdef CONFIG_LINUX
1719 struct vhost_user *u = dev->opaque;
1720 CharBackend *chr = u->user->chr;
1721 int ufd;
1722 VhostUserMsg msg = {
1723 .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
1724 .hdr.flags = VHOST_USER_VERSION,
1725 };
1726
1727 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1728 error_setg(errp, "Failed to send postcopy_advise to vhost");
1729 return -1;
1730 }
1731
1732 if (vhost_user_read(dev, &msg) < 0) {
1733 error_setg(errp, "Failed to get postcopy_advise reply from vhost");
1734 return -1;
1735 }
1736
1737 if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
1738 error_setg(errp, "Unexpected msg type. Expected %d received %d",
1739 VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
1740 return -1;
1741 }
1742
1743 if (msg.hdr.size) {
1744 error_setg(errp, "Received bad msg size.");
1745 return -1;
1746 }
1747 ufd = qemu_chr_fe_get_msgfd(chr);
1748 if (ufd < 0) {
1749 error_setg(errp, "%s: Failed to get ufd", __func__);
1750 return -1;
1751 }
1752 qemu_set_nonblock(ufd);
1753
1754 /* register ufd with userfault thread */
1755 u->postcopy_fd.fd = ufd;
1756 u->postcopy_fd.data = dev;
1757 u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
1758 u->postcopy_fd.waker = vhost_user_postcopy_waker;
1759 u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */
1760 postcopy_register_shared_ufd(&u->postcopy_fd);
1761 return 0;
1762 #else
1763 error_setg(errp, "Postcopy not supported on non-Linux systems");
1764 return -1;
1765 #endif
1766 }
1767
1768 /*
1769 * Called at the switch to postcopy on reception of the 'listen' command.
1770 */
1771 static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp)
1772 {
1773 struct vhost_user *u = dev->opaque;
1774 int ret;
1775 VhostUserMsg msg = {
1776 .hdr.request = VHOST_USER_POSTCOPY_LISTEN,
1777 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1778 };
1779 u->postcopy_listen = true;
1780 trace_vhost_user_postcopy_listen();
1781 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1782 error_setg(errp, "Failed to send postcopy_listen to vhost");
1783 return -1;
1784 }
1785
1786 ret = process_message_reply(dev, &msg);
1787 if (ret) {
1788 error_setg(errp, "Failed to receive reply to postcopy_listen");
1789 return ret;
1790 }
1791
1792 return 0;
1793 }
1794
1795 /*
1796 * Called at the end of postcopy
1797 */
1798 static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp)
1799 {
1800 VhostUserMsg msg = {
1801 .hdr.request = VHOST_USER_POSTCOPY_END,
1802 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
1803 };
1804 int ret;
1805 struct vhost_user *u = dev->opaque;
1806
1807 trace_vhost_user_postcopy_end_entry();
1808 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
1809 error_setg(errp, "Failed to send postcopy_end to vhost");
1810 return -1;
1811 }
1812
1813 ret = process_message_reply(dev, &msg);
1814 if (ret) {
1815 error_setg(errp, "Failed to receive reply to postcopy_end");
1816 return ret;
1817 }
1818 postcopy_unregister_shared_ufd(&u->postcopy_fd);
1819 close(u->postcopy_fd.fd);
1820 u->postcopy_fd.handler = NULL;
1821
1822 trace_vhost_user_postcopy_end_exit();
1823
1824 return 0;
1825 }
1826
1827 static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
1828 void *opaque)
1829 {
1830 struct PostcopyNotifyData *pnd = opaque;
1831 struct vhost_user *u = container_of(notifier, struct vhost_user,
1832 postcopy_notifier);
1833 struct vhost_dev *dev = u->dev;
1834
1835 switch (pnd->reason) {
1836 case POSTCOPY_NOTIFY_PROBE:
1837 if (!virtio_has_feature(dev->protocol_features,
1838 VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
1839 /* TODO: Get the device name into this error somehow */
1840 error_setg(pnd->errp,
1841 "vhost-user backend not capable of postcopy");
1842 return -ENOENT;
1843 }
1844 break;
1845
1846 case POSTCOPY_NOTIFY_INBOUND_ADVISE:
1847 return vhost_user_postcopy_advise(dev, pnd->errp);
1848
1849 case POSTCOPY_NOTIFY_INBOUND_LISTEN:
1850 return vhost_user_postcopy_listen(dev, pnd->errp);
1851
1852 case POSTCOPY_NOTIFY_INBOUND_END:
1853 return vhost_user_postcopy_end(dev, pnd->errp);
1854
1855 default:
1856 /* We ignore notifications we don't know */
1857 break;
1858 }
1859
1860 return 0;
1861 }
1862
1863 static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque,
1864 Error **errp)
1865 {
1866 uint64_t features, protocol_features, ram_slots;
1867 struct vhost_user *u;
1868 int err;
1869
1870 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1871
1872 u = g_new0(struct vhost_user, 1);
1873 u->user = opaque;
1874 u->dev = dev;
1875 dev->opaque = u;
1876
1877 err = vhost_user_get_features(dev, &features);
1878 if (err < 0) {
1879 return err;
1880 }
1881
1882 if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
1883 dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
1884
1885 err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
1886 &protocol_features);
1887 if (err < 0) {
1888 return -EPROTO;
1889 }
1890
1891 dev->protocol_features =
1892 protocol_features & VHOST_USER_PROTOCOL_FEATURE_MASK;
1893
1894 if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) {
1895 /* Don't acknowledge CONFIG feature if device doesn't support it */
1896 dev->protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
1897 } else if (!(protocol_features &
1898 (1ULL << VHOST_USER_PROTOCOL_F_CONFIG))) {
1899 error_setg(errp, "Device expects VHOST_USER_PROTOCOL_F_CONFIG "
1900 "but backend does not support it.");
1901 return -EINVAL;
1902 }
1903
1904 err = vhost_user_set_protocol_features(dev, dev->protocol_features);
1905 if (err < 0) {
1906 return -EPROTO;
1907 }
1908
1909 /* query the max queues we support if backend supports Multiple Queue */
1910 if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
1911 err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
1912 &dev->max_queues);
1913 if (err < 0) {
1914 return -EPROTO;
1915 }
1916 } else {
1917 dev->max_queues = 1;
1918 }
1919
1920 if (dev->num_queues && dev->max_queues < dev->num_queues) {
1921 error_setg(errp, "The maximum number of queues supported by the "
1922 "backend is %" PRIu64, dev->max_queues);
1923 return -EINVAL;
1924 }
1925
1926 if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
1927 !(virtio_has_feature(dev->protocol_features,
1928 VHOST_USER_PROTOCOL_F_SLAVE_REQ) &&
1929 virtio_has_feature(dev->protocol_features,
1930 VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
1931 error_setg(errp, "IOMMU support requires reply-ack and "
1932 "slave-req protocol features.");
1933 return -EINVAL;
1934 }
1935
1936 /* get max memory regions if backend supports configurable RAM slots */
1937 if (!virtio_has_feature(dev->protocol_features,
1938 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS)) {
1939 u->user->memory_slots = VHOST_MEMORY_BASELINE_NREGIONS;
1940 } else {
1941 err = vhost_user_get_max_memslots(dev, &ram_slots);
1942 if (err < 0) {
1943 return -EPROTO;
1944 }
1945
1946 if (ram_slots < u->user->memory_slots) {
1947 error_setg(errp, "The backend specified a max ram slots limit "
1948 "of %" PRIu64", when the prior validated limit was "
1949 "%d. This limit should never decrease.", ram_slots,
1950 u->user->memory_slots);
1951 return -EINVAL;
1952 }
1953
1954 u->user->memory_slots = MIN(ram_slots, VHOST_USER_MAX_RAM_SLOTS);
1955 }
1956 }
1957
1958 if (dev->migration_blocker == NULL &&
1959 !virtio_has_feature(dev->protocol_features,
1960 VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
1961 error_setg(&dev->migration_blocker,
1962 "Migration disabled: vhost-user backend lacks "
1963 "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
1964 }
1965
1966 if (dev->vq_index == 0) {
1967 err = vhost_setup_slave_channel(dev);
1968 if (err < 0) {
1969 return -EPROTO;
1970 }
1971 }
1972
1973 u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
1974 postcopy_add_notifier(&u->postcopy_notifier);
1975
1976 return 0;
1977 }
1978
1979 static int vhost_user_backend_cleanup(struct vhost_dev *dev)
1980 {
1981 struct vhost_user *u;
1982
1983 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
1984
1985 u = dev->opaque;
1986 if (u->postcopy_notifier.notify) {
1987 postcopy_remove_notifier(&u->postcopy_notifier);
1988 u->postcopy_notifier.notify = NULL;
1989 }
1990 u->postcopy_listen = false;
1991 if (u->postcopy_fd.handler) {
1992 postcopy_unregister_shared_ufd(&u->postcopy_fd);
1993 close(u->postcopy_fd.fd);
1994 u->postcopy_fd.handler = NULL;
1995 }
1996 if (u->slave_ioc) {
1997 close_slave_channel(u);
1998 }
1999 g_free(u->region_rb);
2000 u->region_rb = NULL;
2001 g_free(u->region_rb_offset);
2002 u->region_rb_offset = NULL;
2003 u->region_rb_len = 0;
2004 g_free(u);
2005 dev->opaque = 0;
2006
2007 return 0;
2008 }
2009
2010 static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
2011 {
2012 assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
2013
2014 return idx;
2015 }
2016
2017 static int vhost_user_memslots_limit(struct vhost_dev *dev)
2018 {
2019 struct vhost_user *u = dev->opaque;
2020
2021 return u->user->memory_slots;
2022 }
2023
2024 static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
2025 {
2026 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2027
2028 return virtio_has_feature(dev->protocol_features,
2029 VHOST_USER_PROTOCOL_F_LOG_SHMFD);
2030 }
2031
2032 static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
2033 {
2034 VhostUserMsg msg = { };
2035
2036 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2037
2038 /* If guest supports GUEST_ANNOUNCE do nothing */
2039 if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
2040 return 0;
2041 }
2042
2043 /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
2044 if (virtio_has_feature(dev->protocol_features,
2045 VHOST_USER_PROTOCOL_F_RARP)) {
2046 msg.hdr.request = VHOST_USER_SEND_RARP;
2047 msg.hdr.flags = VHOST_USER_VERSION;
2048 memcpy((char *)&msg.payload.u64, mac_addr, 6);
2049 msg.hdr.size = sizeof(msg.payload.u64);
2050
2051 return vhost_user_write(dev, &msg, NULL, 0);
2052 }
2053 return -1;
2054 }
2055
2056 static bool vhost_user_can_merge(struct vhost_dev *dev,
2057 uint64_t start1, uint64_t size1,
2058 uint64_t start2, uint64_t size2)
2059 {
2060 ram_addr_t offset;
2061 int mfd, rfd;
2062
2063 (void)vhost_user_get_mr_data(start1, &offset, &mfd);
2064 (void)vhost_user_get_mr_data(start2, &offset, &rfd);
2065
2066 return mfd == rfd;
2067 }
2068
2069 static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
2070 {
2071 VhostUserMsg msg;
2072 bool reply_supported = virtio_has_feature(dev->protocol_features,
2073 VHOST_USER_PROTOCOL_F_REPLY_ACK);
2074
2075 if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
2076 return 0;
2077 }
2078
2079 msg.hdr.request = VHOST_USER_NET_SET_MTU;
2080 msg.payload.u64 = mtu;
2081 msg.hdr.size = sizeof(msg.payload.u64);
2082 msg.hdr.flags = VHOST_USER_VERSION;
2083 if (reply_supported) {
2084 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2085 }
2086
2087 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2088 return -1;
2089 }
2090
2091 /* If reply_ack supported, slave has to ack specified MTU is valid */
2092 if (reply_supported) {
2093 return process_message_reply(dev, &msg);
2094 }
2095
2096 return 0;
2097 }
2098
2099 static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
2100 struct vhost_iotlb_msg *imsg)
2101 {
2102 VhostUserMsg msg = {
2103 .hdr.request = VHOST_USER_IOTLB_MSG,
2104 .hdr.size = sizeof(msg.payload.iotlb),
2105 .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
2106 .payload.iotlb = *imsg,
2107 };
2108
2109 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2110 return -EFAULT;
2111 }
2112
2113 return process_message_reply(dev, &msg);
2114 }
2115
2116
2117 static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
2118 {
2119 /* No-op as the receive channel is not dedicated to IOTLB messages. */
2120 }
2121
2122 static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
2123 uint32_t config_len, Error **errp)
2124 {
2125 VhostUserMsg msg = {
2126 .hdr.request = VHOST_USER_GET_CONFIG,
2127 .hdr.flags = VHOST_USER_VERSION,
2128 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
2129 };
2130
2131 if (!virtio_has_feature(dev->protocol_features,
2132 VHOST_USER_PROTOCOL_F_CONFIG)) {
2133 error_setg(errp, "VHOST_USER_PROTOCOL_F_CONFIG not supported");
2134 return -EINVAL;
2135 }
2136
2137 assert(config_len <= VHOST_USER_MAX_CONFIG_SIZE);
2138
2139 msg.payload.config.offset = 0;
2140 msg.payload.config.size = config_len;
2141 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2142 return -EPROTO;
2143 }
2144
2145 if (vhost_user_read(dev, &msg) < 0) {
2146 return -EPROTO;
2147 }
2148
2149 if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
2150 error_setg(errp,
2151 "Received unexpected msg type. Expected %d received %d",
2152 VHOST_USER_GET_CONFIG, msg.hdr.request);
2153 return -EINVAL;
2154 }
2155
2156 if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
2157 error_setg(errp, "Received bad msg size.");
2158 return -EINVAL;
2159 }
2160
2161 memcpy(config, msg.payload.config.region, config_len);
2162
2163 return 0;
2164 }
2165
2166 static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
2167 uint32_t offset, uint32_t size, uint32_t flags)
2168 {
2169 uint8_t *p;
2170 bool reply_supported = virtio_has_feature(dev->protocol_features,
2171 VHOST_USER_PROTOCOL_F_REPLY_ACK);
2172
2173 VhostUserMsg msg = {
2174 .hdr.request = VHOST_USER_SET_CONFIG,
2175 .hdr.flags = VHOST_USER_VERSION,
2176 .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
2177 };
2178
2179 if (!virtio_has_feature(dev->protocol_features,
2180 VHOST_USER_PROTOCOL_F_CONFIG)) {
2181 return -1;
2182 }
2183
2184 if (reply_supported) {
2185 msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
2186 }
2187
2188 if (size > VHOST_USER_MAX_CONFIG_SIZE) {
2189 return -1;
2190 }
2191
2192 msg.payload.config.offset = offset,
2193 msg.payload.config.size = size,
2194 msg.payload.config.flags = flags,
2195 p = msg.payload.config.region;
2196 memcpy(p, data, size);
2197
2198 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2199 return -1;
2200 }
2201
2202 if (reply_supported) {
2203 return process_message_reply(dev, &msg);
2204 }
2205
2206 return 0;
2207 }
2208
2209 static int vhost_user_crypto_create_session(struct vhost_dev *dev,
2210 void *session_info,
2211 uint64_t *session_id)
2212 {
2213 bool crypto_session = virtio_has_feature(dev->protocol_features,
2214 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2215 CryptoDevBackendSymSessionInfo *sess_info = session_info;
2216 VhostUserMsg msg = {
2217 .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
2218 .hdr.flags = VHOST_USER_VERSION,
2219 .hdr.size = sizeof(msg.payload.session),
2220 };
2221
2222 assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
2223
2224 if (!crypto_session) {
2225 error_report("vhost-user trying to send unhandled ioctl");
2226 return -1;
2227 }
2228
2229 memcpy(&msg.payload.session.session_setup_data, sess_info,
2230 sizeof(CryptoDevBackendSymSessionInfo));
2231 if (sess_info->key_len) {
2232 memcpy(&msg.payload.session.key, sess_info->cipher_key,
2233 sess_info->key_len);
2234 }
2235 if (sess_info->auth_key_len > 0) {
2236 memcpy(&msg.payload.session.auth_key, sess_info->auth_key,
2237 sess_info->auth_key_len);
2238 }
2239 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2240 error_report("vhost_user_write() return -1, create session failed");
2241 return -1;
2242 }
2243
2244 if (vhost_user_read(dev, &msg) < 0) {
2245 error_report("vhost_user_read() return -1, create session failed");
2246 return -1;
2247 }
2248
2249 if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
2250 error_report("Received unexpected msg type. Expected %d received %d",
2251 VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
2252 return -1;
2253 }
2254
2255 if (msg.hdr.size != sizeof(msg.payload.session)) {
2256 error_report("Received bad msg size.");
2257 return -1;
2258 }
2259
2260 if (msg.payload.session.session_id < 0) {
2261 error_report("Bad session id: %" PRId64 "",
2262 msg.payload.session.session_id);
2263 return -1;
2264 }
2265 *session_id = msg.payload.session.session_id;
2266
2267 return 0;
2268 }
2269
2270 static int
2271 vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
2272 {
2273 bool crypto_session = virtio_has_feature(dev->protocol_features,
2274 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
2275 VhostUserMsg msg = {
2276 .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
2277 .hdr.flags = VHOST_USER_VERSION,
2278 .hdr.size = sizeof(msg.payload.u64),
2279 };
2280 msg.payload.u64 = session_id;
2281
2282 if (!crypto_session) {
2283 error_report("vhost-user trying to send unhandled ioctl");
2284 return -1;
2285 }
2286
2287 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2288 error_report("vhost_user_write() return -1, close session failed");
2289 return -1;
2290 }
2291
2292 return 0;
2293 }
2294
2295 static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
2296 MemoryRegionSection *section)
2297 {
2298 bool result;
2299
2300 result = memory_region_get_fd(section->mr) >= 0;
2301
2302 return result;
2303 }
2304
2305 static int vhost_user_get_inflight_fd(struct vhost_dev *dev,
2306 uint16_t queue_size,
2307 struct vhost_inflight *inflight)
2308 {
2309 void *addr;
2310 int fd;
2311 struct vhost_user *u = dev->opaque;
2312 CharBackend *chr = u->user->chr;
2313 VhostUserMsg msg = {
2314 .hdr.request = VHOST_USER_GET_INFLIGHT_FD,
2315 .hdr.flags = VHOST_USER_VERSION,
2316 .payload.inflight.num_queues = dev->nvqs,
2317 .payload.inflight.queue_size = queue_size,
2318 .hdr.size = sizeof(msg.payload.inflight),
2319 };
2320
2321 if (!virtio_has_feature(dev->protocol_features,
2322 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2323 return 0;
2324 }
2325
2326 if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
2327 return -1;
2328 }
2329
2330 if (vhost_user_read(dev, &msg) < 0) {
2331 return -1;
2332 }
2333
2334 if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) {
2335 error_report("Received unexpected msg type. "
2336 "Expected %d received %d",
2337 VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request);
2338 return -1;
2339 }
2340
2341 if (msg.hdr.size != sizeof(msg.payload.inflight)) {
2342 error_report("Received bad msg size.");
2343 return -1;
2344 }
2345
2346 if (!msg.payload.inflight.mmap_size) {
2347 return 0;
2348 }
2349
2350 fd = qemu_chr_fe_get_msgfd(chr);
2351 if (fd < 0) {
2352 error_report("Failed to get mem fd");
2353 return -1;
2354 }
2355
2356 addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ | PROT_WRITE,
2357 MAP_SHARED, fd, msg.payload.inflight.mmap_offset);
2358
2359 if (addr == MAP_FAILED) {
2360 error_report("Failed to mmap mem fd");
2361 close(fd);
2362 return -1;
2363 }
2364
2365 inflight->addr = addr;
2366 inflight->fd = fd;
2367 inflight->size = msg.payload.inflight.mmap_size;
2368 inflight->offset = msg.payload.inflight.mmap_offset;
2369 inflight->queue_size = queue_size;
2370
2371 return 0;
2372 }
2373
2374 static int vhost_user_set_inflight_fd(struct vhost_dev *dev,
2375 struct vhost_inflight *inflight)
2376 {
2377 VhostUserMsg msg = {
2378 .hdr.request = VHOST_USER_SET_INFLIGHT_FD,
2379 .hdr.flags = VHOST_USER_VERSION,
2380 .payload.inflight.mmap_size = inflight->size,
2381 .payload.inflight.mmap_offset = inflight->offset,
2382 .payload.inflight.num_queues = dev->nvqs,
2383 .payload.inflight.queue_size = inflight->queue_size,
2384 .hdr.size = sizeof(msg.payload.inflight),
2385 };
2386
2387 if (!virtio_has_feature(dev->protocol_features,
2388 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
2389 return 0;
2390 }
2391
2392 if (vhost_user_write(dev, &msg, &inflight->fd, 1) < 0) {
2393 return -1;
2394 }
2395
2396 return 0;
2397 }
2398
2399 bool vhost_user_init(VhostUserState *user, CharBackend *chr, Error **errp)
2400 {
2401 if (user->chr) {
2402 error_setg(errp, "Cannot initialize vhost-user state");
2403 return false;
2404 }
2405 user->chr = chr;
2406 user->memory_slots = 0;
2407 return true;
2408 }
2409
2410 void vhost_user_cleanup(VhostUserState *user)
2411 {
2412 int i;
2413
2414 if (!user->chr) {
2415 return;
2416 }
2417
2418 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2419 if (user->notifier[i].addr) {
2420 object_unparent(OBJECT(&user->notifier[i].mr));
2421 munmap(user->notifier[i].addr, qemu_real_host_page_size);
2422 user->notifier[i].addr = NULL;
2423 }
2424 }
2425 user->chr = NULL;
2426 }
2427
2428 const VhostOps user_ops = {
2429 .backend_type = VHOST_BACKEND_TYPE_USER,
2430 .vhost_backend_init = vhost_user_backend_init,
2431 .vhost_backend_cleanup = vhost_user_backend_cleanup,
2432 .vhost_backend_memslots_limit = vhost_user_memslots_limit,
2433 .vhost_set_log_base = vhost_user_set_log_base,
2434 .vhost_set_mem_table = vhost_user_set_mem_table,
2435 .vhost_set_vring_addr = vhost_user_set_vring_addr,
2436 .vhost_set_vring_endian = vhost_user_set_vring_endian,
2437 .vhost_set_vring_num = vhost_user_set_vring_num,
2438 .vhost_set_vring_base = vhost_user_set_vring_base,
2439 .vhost_get_vring_base = vhost_user_get_vring_base,
2440 .vhost_set_vring_kick = vhost_user_set_vring_kick,
2441 .vhost_set_vring_call = vhost_user_set_vring_call,
2442 .vhost_set_features = vhost_user_set_features,
2443 .vhost_get_features = vhost_user_get_features,
2444 .vhost_set_owner = vhost_user_set_owner,
2445 .vhost_reset_device = vhost_user_reset_device,
2446 .vhost_get_vq_index = vhost_user_get_vq_index,
2447 .vhost_set_vring_enable = vhost_user_set_vring_enable,
2448 .vhost_requires_shm_log = vhost_user_requires_shm_log,
2449 .vhost_migration_done = vhost_user_migration_done,
2450 .vhost_backend_can_merge = vhost_user_can_merge,
2451 .vhost_net_set_mtu = vhost_user_net_set_mtu,
2452 .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
2453 .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
2454 .vhost_get_config = vhost_user_get_config,
2455 .vhost_set_config = vhost_user_set_config,
2456 .vhost_crypto_create_session = vhost_user_crypto_create_session,
2457 .vhost_crypto_close_session = vhost_user_crypto_close_session,
2458 .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
2459 .vhost_get_inflight_fd = vhost_user_get_inflight_fd,
2460 .vhost_set_inflight_fd = vhost_user_set_inflight_fd,
2461 };
armbru's QEMU hackery