4 * Copyright (c) 2016 Red Hat, Inc.
7 * Victor Kaplansky <victork@redhat.com>
8 * Marc-André Lureau <mlureau@redhat.com>
10 * This work is licensed under the terms of the GNU GPL, version 2 or
11 * later. See the COPYING file in the top-level directory.
14 #ifndef LIBVHOST_USER_H
15 #define LIBVHOST_USER_H
21 #include <linux/vhost.h>
23 #include "standard-headers/linux/virtio_ring.h"
25 /* Based on qemu/hw/virtio/vhost-user.c */
26 #define VHOST_USER_F_PROTOCOL_FEATURES 30
27 #define VHOST_LOG_PAGE 4096
29 #define VIRTQUEUE_MAX_SIZE 1024
31 #define VHOST_MEMORY_BASELINE_NREGIONS 8
34 * Set a reasonable maximum number of ram slots, which will be supported by
37 #define VHOST_USER_MAX_RAM_SLOTS 32
39 typedef enum VhostSetConfigType
{
40 VHOST_SET_CONFIG_TYPE_MASTER
= 0,
41 VHOST_SET_CONFIG_TYPE_MIGRATION
= 1,
45 * Maximum size of virtio device config space
47 #define VHOST_USER_MAX_CONFIG_SIZE 256
49 enum VhostUserProtocolFeature
{
50 VHOST_USER_PROTOCOL_F_MQ
= 0,
51 VHOST_USER_PROTOCOL_F_LOG_SHMFD
= 1,
52 VHOST_USER_PROTOCOL_F_RARP
= 2,
53 VHOST_USER_PROTOCOL_F_REPLY_ACK
= 3,
54 VHOST_USER_PROTOCOL_F_NET_MTU
= 4,
55 VHOST_USER_PROTOCOL_F_SLAVE_REQ
= 5,
56 VHOST_USER_PROTOCOL_F_CROSS_ENDIAN
= 6,
57 VHOST_USER_PROTOCOL_F_CRYPTO_SESSION
= 7,
58 VHOST_USER_PROTOCOL_F_PAGEFAULT
= 8,
59 VHOST_USER_PROTOCOL_F_CONFIG
= 9,
60 VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD
= 10,
61 VHOST_USER_PROTOCOL_F_HOST_NOTIFIER
= 11,
62 VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD
= 12,
63 VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS
= 14,
64 VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS
= 15,
66 VHOST_USER_PROTOCOL_F_MAX
69 #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
71 typedef enum VhostUserRequest
{
73 VHOST_USER_GET_FEATURES
= 1,
74 VHOST_USER_SET_FEATURES
= 2,
75 VHOST_USER_SET_OWNER
= 3,
76 VHOST_USER_RESET_OWNER
= 4,
77 VHOST_USER_SET_MEM_TABLE
= 5,
78 VHOST_USER_SET_LOG_BASE
= 6,
79 VHOST_USER_SET_LOG_FD
= 7,
80 VHOST_USER_SET_VRING_NUM
= 8,
81 VHOST_USER_SET_VRING_ADDR
= 9,
82 VHOST_USER_SET_VRING_BASE
= 10,
83 VHOST_USER_GET_VRING_BASE
= 11,
84 VHOST_USER_SET_VRING_KICK
= 12,
85 VHOST_USER_SET_VRING_CALL
= 13,
86 VHOST_USER_SET_VRING_ERR
= 14,
87 VHOST_USER_GET_PROTOCOL_FEATURES
= 15,
88 VHOST_USER_SET_PROTOCOL_FEATURES
= 16,
89 VHOST_USER_GET_QUEUE_NUM
= 17,
90 VHOST_USER_SET_VRING_ENABLE
= 18,
91 VHOST_USER_SEND_RARP
= 19,
92 VHOST_USER_NET_SET_MTU
= 20,
93 VHOST_USER_SET_SLAVE_REQ_FD
= 21,
94 VHOST_USER_IOTLB_MSG
= 22,
95 VHOST_USER_SET_VRING_ENDIAN
= 23,
96 VHOST_USER_GET_CONFIG
= 24,
97 VHOST_USER_SET_CONFIG
= 25,
98 VHOST_USER_CREATE_CRYPTO_SESSION
= 26,
99 VHOST_USER_CLOSE_CRYPTO_SESSION
= 27,
100 VHOST_USER_POSTCOPY_ADVISE
= 28,
101 VHOST_USER_POSTCOPY_LISTEN
= 29,
102 VHOST_USER_POSTCOPY_END
= 30,
103 VHOST_USER_GET_INFLIGHT_FD
= 31,
104 VHOST_USER_SET_INFLIGHT_FD
= 32,
105 VHOST_USER_GPU_SET_SOCKET
= 33,
106 VHOST_USER_VRING_KICK
= 35,
107 VHOST_USER_GET_MAX_MEM_SLOTS
= 36,
108 VHOST_USER_ADD_MEM_REG
= 37,
109 VHOST_USER_REM_MEM_REG
= 38,
113 typedef enum VhostUserSlaveRequest
{
114 VHOST_USER_SLAVE_NONE
= 0,
115 VHOST_USER_SLAVE_IOTLB_MSG
= 1,
116 VHOST_USER_SLAVE_CONFIG_CHANGE_MSG
= 2,
117 VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG
= 3,
118 VHOST_USER_SLAVE_VRING_CALL
= 4,
119 VHOST_USER_SLAVE_VRING_ERR
= 5,
121 } VhostUserSlaveRequest
;
123 typedef struct VhostUserMemoryRegion
{
124 uint64_t guest_phys_addr
;
125 uint64_t memory_size
;
126 uint64_t userspace_addr
;
127 uint64_t mmap_offset
;
128 } VhostUserMemoryRegion
;
130 typedef struct VhostUserMemory
{
133 VhostUserMemoryRegion regions
[VHOST_MEMORY_BASELINE_NREGIONS
];
136 typedef struct VhostUserMemRegMsg
{
138 VhostUserMemoryRegion region
;
139 } VhostUserMemRegMsg
;
141 typedef struct VhostUserLog
{
143 uint64_t mmap_offset
;
146 typedef struct VhostUserConfig
{
150 uint8_t region
[VHOST_USER_MAX_CONFIG_SIZE
];
153 static VhostUserConfig c
__attribute__ ((unused
));
154 #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
158 typedef struct VhostUserVringArea
{
162 } VhostUserVringArea
;
164 typedef struct VhostUserInflight
{
166 uint64_t mmap_offset
;
171 #if defined(_WIN32) && (defined(__x86_64__) || defined(__i386__))
172 # define VU_PACKED __attribute__((gcc_struct, packed))
174 # define VU_PACKED __attribute__((packed))
177 typedef struct VhostUserMsg
{
180 #define VHOST_USER_VERSION_MASK (0x3)
181 #define VHOST_USER_REPLY_MASK (0x1 << 2)
182 #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
184 uint32_t size
; /* the following payload size */
187 #define VHOST_USER_VRING_IDX_MASK (0xff)
188 #define VHOST_USER_VRING_NOFD_MASK (0x1 << 8)
190 struct vhost_vring_state state
;
191 struct vhost_vring_addr addr
;
192 VhostUserMemory memory
;
193 VhostUserMemRegMsg memreg
;
195 VhostUserConfig config
;
196 VhostUserVringArea area
;
197 VhostUserInflight inflight
;
200 int fds
[VHOST_MEMORY_BASELINE_NREGIONS
];
203 } VU_PACKED VhostUserMsg
;
205 typedef struct VuDevRegion
{
206 /* Guest Physical address. */
208 /* Memory region size. */
210 /* QEMU virtual address (userspace). */
212 /* Starting offset in our mmaped space. */
213 uint64_t mmap_offset
;
214 /* Start address of mmaped space. */
218 typedef struct VuDev VuDev
;
220 typedef uint64_t (*vu_get_features_cb
) (VuDev
*dev
);
221 typedef void (*vu_set_features_cb
) (VuDev
*dev
, uint64_t features
);
222 typedef int (*vu_process_msg_cb
) (VuDev
*dev
, VhostUserMsg
*vmsg
,
224 typedef void (*vu_queue_set_started_cb
) (VuDev
*dev
, int qidx
, bool started
);
225 typedef bool (*vu_queue_is_processed_in_order_cb
) (VuDev
*dev
, int qidx
);
226 typedef int (*vu_get_config_cb
) (VuDev
*dev
, uint8_t *config
, uint32_t len
);
227 typedef int (*vu_set_config_cb
) (VuDev
*dev
, const uint8_t *data
,
228 uint32_t offset
, uint32_t size
,
231 typedef struct VuDevIface
{
232 /* called by VHOST_USER_GET_FEATURES to get the features bitmask */
233 vu_get_features_cb get_features
;
234 /* enable vhost implementation features */
235 vu_set_features_cb set_features
;
236 /* get the protocol feature bitmask from the underlying vhost
238 vu_get_features_cb get_protocol_features
;
239 /* enable protocol features in the underlying vhost implementation. */
240 vu_set_features_cb set_protocol_features
;
241 /* process_msg is called for each vhost-user message received */
242 /* skip libvhost-user processing if return value != 0 */
243 vu_process_msg_cb process_msg
;
244 /* tells when queues can be processed */
245 vu_queue_set_started_cb queue_set_started
;
247 * If the queue is processed in order, in which case it will be
248 * resumed to vring.used->idx. This can help to support resuming
249 * on unmanaged exit/crash.
251 vu_queue_is_processed_in_order_cb queue_is_processed_in_order
;
252 /* get the config space of the device */
253 vu_get_config_cb get_config
;
254 /* set the config space of the device */
255 vu_set_config_cb set_config
;
258 typedef void (*vu_queue_handler_cb
) (VuDev
*dev
, int qidx
);
260 typedef struct VuRing
{
262 struct vring_desc
*desc
;
263 struct vring_avail
*avail
;
264 struct vring_used
*used
;
265 uint64_t log_guest_addr
;
269 typedef struct VuDescStateSplit
{
270 /* Indicate whether this descriptor is inflight or not.
271 * Only available for head-descriptor. */
277 /* Maintain a list for the last batch of used descriptors.
278 * Only available when batching is used for submitting */
281 /* Used to preserve the order of fetching available descriptors.
282 * Only available for head-descriptor. */
286 typedef struct VuVirtqInflight
{
287 /* The feature flags of this region. Now it's initialized to 0. */
290 /* The version of this region. It's 1 currently.
291 * Zero value indicates a vm reset happened. */
294 /* The size of VuDescStateSplit array. It's equal to the virtqueue
295 * size. Slave could get it from queue size field of VhostUserInflight. */
298 /* The head of list that track the last batch of used descriptors. */
299 uint16_t last_batch_head
;
301 /* Storing the idx value of used ring */
304 /* Used to track the state of each descriptor in descriptor table */
305 VuDescStateSplit desc
[];
308 typedef struct VuVirtqInflightDesc
{
311 } VuVirtqInflightDesc
;
313 typedef struct VuVirtq
{
316 VuVirtqInflight
*inflight
;
318 VuVirtqInflightDesc
*resubmit_list
;
320 uint16_t resubmit_num
;
324 /* Next head to pop */
325 uint16_t last_avail_idx
;
327 /* Last avail_idx read from VQ. */
328 uint16_t shadow_avail_idx
;
332 /* Last used index value we have signalled on */
333 uint16_t signalled_used
;
335 /* Last used index value we have signalled on */
336 bool signalled_used_valid
;
338 /* Notification enabled? */
343 vu_queue_handler_cb handler
;
351 /* Guest addresses of our ring */
352 struct vhost_vring_addr vra
;
355 enum VuWatchCondtion
{
356 VU_WATCH_IN
= POLLIN
,
357 VU_WATCH_OUT
= POLLOUT
,
358 VU_WATCH_PRI
= POLLPRI
,
359 VU_WATCH_ERR
= POLLERR
,
360 VU_WATCH_HUP
= POLLHUP
,
363 typedef void (*vu_panic_cb
) (VuDev
*dev
, const char *err
);
364 typedef void (*vu_watch_cb
) (VuDev
*dev
, int condition
, void *data
);
365 typedef void (*vu_set_watch_cb
) (VuDev
*dev
, int fd
, int condition
,
366 vu_watch_cb cb
, void *data
);
367 typedef void (*vu_remove_watch_cb
) (VuDev
*dev
, int fd
);
369 typedef struct VuDevInflightInfo
{
378 VuDevRegion regions
[VHOST_USER_MAX_RAM_SLOTS
];
380 VuDevInflightInfo inflight_info
;
382 /* Must be held while using slave_fd */
383 pthread_mutex_t slave_mutex
;
388 uint64_t protocol_features
;
392 /* @set_watch: add or update the given fd to the watch set,
393 * call cb when condition is met */
394 vu_set_watch_cb set_watch
;
396 /* @remove_watch: remove the given fd from the watch set */
397 vu_remove_watch_cb remove_watch
;
399 /* @panic: encountered an unrecoverable error, you may try to
402 const VuDevIface
*iface
;
406 bool postcopy_listening
;
409 typedef struct VuVirtqElement
{
411 unsigned int out_num
;
414 struct iovec
*out_sg
;
419 * @dev: a VuDev context
420 * @max_queues: maximum number of virtqueues
421 * @socket: the socket connected to vhost-user master
422 * @panic: a panic callback
423 * @set_watch: a set_watch callback
424 * @remove_watch: a remove_watch callback
425 * @iface: a VuDevIface structure with vhost-user device callbacks
427 * Intializes a VuDev vhost-user context.
429 * Returns: true on success, false on failure.
431 bool vu_init(VuDev
*dev
,
435 vu_set_watch_cb set_watch
,
436 vu_remove_watch_cb remove_watch
,
437 const VuDevIface
*iface
);
442 * @dev: a VuDev context
444 * Cleans up the VuDev context
446 void vu_deinit(VuDev
*dev
);
450 * @dev: a VuDev context
452 * Process one vhost-user message.
454 * Returns: TRUE on success, FALSE on failure.
456 bool vu_dispatch(VuDev
*dev
);
460 * @dev: a VuDev context
461 * @plen: guest memory size
462 * @guest_addr: guest address
464 * Translate a guest address to a pointer. Returns NULL on failure.
466 void *vu_gpa_to_va(VuDev
*dev
, uint64_t *plen
, uint64_t guest_addr
);
470 * @dev: a VuDev context
473 * Returns the queue number @qidx.
475 VuVirtq
*vu_get_queue(VuDev
*dev
, int qidx
);
478 * vu_set_queue_handler:
479 * @dev: a VuDev context
480 * @vq: a VuVirtq queue
481 * @handler: the queue handler callback
483 * Set the queue handler. This function may be called several times
484 * for the same queue. If called with NULL @handler, the handler is
487 void vu_set_queue_handler(VuDev
*dev
, VuVirtq
*vq
,
488 vu_queue_handler_cb handler
);
491 * vu_set_queue_host_notifier:
492 * @dev: a VuDev context
493 * @vq: a VuVirtq queue
494 * @fd: a file descriptor
495 * @size: host page size
496 * @offset: notifier offset in @fd file
498 * Set queue's host notifier. This function may be called several
499 * times for the same queue. If called with -1 @fd, the notifier
502 bool vu_set_queue_host_notifier(VuDev
*dev
, VuVirtq
*vq
, int fd
,
503 int size
, int offset
);
506 * vu_queue_set_notification:
507 * @dev: a VuDev context
508 * @vq: a VuVirtq queue
511 * Set whether the queue notifies (via event index or interrupt)
513 void vu_queue_set_notification(VuDev
*dev
, VuVirtq
*vq
, int enable
);
517 * @dev: a VuDev context
518 * @vq: a VuVirtq queue
520 * Returns: whether the queue is enabled.
522 bool vu_queue_enabled(VuDev
*dev
, VuVirtq
*vq
);
526 * @dev: a VuDev context
527 * @vq: a VuVirtq queue
529 * Returns: whether the queue is started.
531 bool vu_queue_started(const VuDev
*dev
, const VuVirtq
*vq
);
535 * @dev: a VuDev context
536 * @vq: a VuVirtq queue
538 * Returns: true if the queue is empty or not ready.
540 bool vu_queue_empty(VuDev
*dev
, VuVirtq
*vq
);
544 * @dev: a VuDev context
545 * @vq: a VuVirtq queue
547 * Request to notify the queue via callfd (skipped if unnecessary)
549 void vu_queue_notify(VuDev
*dev
, VuVirtq
*vq
);
552 * vu_queue_notify_sync:
553 * @dev: a VuDev context
554 * @vq: a VuVirtq queue
556 * Request to notify the queue via callfd (skipped if unnecessary)
557 * or sync message if possible.
559 void vu_queue_notify_sync(VuDev
*dev
, VuVirtq
*vq
);
563 * @dev: a VuDev context
564 * @vq: a VuVirtq queue
565 * @sz: the size of struct to return (must be >= VuVirtqElement)
567 * Returns: a VuVirtqElement filled from the queue or NULL. The
568 * returned element must be free()-d by the caller.
570 void *vu_queue_pop(VuDev
*dev
, VuVirtq
*vq
, size_t sz
);
575 * @dev: a VuDev context
576 * @vq: a VuVirtq queue
577 * @elem: The #VuVirtqElement
578 * @len: number of bytes written
580 * Pretend the most recent element wasn't popped from the virtqueue. The next
581 * call to vu_queue_pop() will refetch the element.
583 void vu_queue_unpop(VuDev
*dev
, VuVirtq
*vq
, VuVirtqElement
*elem
,
588 * @dev: a VuDev context
589 * @vq: a VuVirtq queue
590 * @num: number of elements to push back
592 * Pretend that elements weren't popped from the virtqueue. The next
593 * virtqueue_pop() will refetch the oldest element.
595 * Returns: true on success, false if @num is greater than the number of in use
598 bool vu_queue_rewind(VuDev
*dev
, VuVirtq
*vq
, unsigned int num
);
602 * @dev: a VuDev context
603 * @vq: a VuVirtq queue
604 * @elem: a VuVirtqElement
605 * @len: length in bytes to write
606 * @idx: optional offset for the used ring index (0 in general)
608 * Fill the used ring with @elem element.
610 void vu_queue_fill(VuDev
*dev
, VuVirtq
*vq
,
611 const VuVirtqElement
*elem
,
612 unsigned int len
, unsigned int idx
);
616 * @dev: a VuDev context
617 * @vq: a VuVirtq queue
618 * @elem: a VuVirtqElement
619 * @len: length in bytes to write
621 * Helper that combines vu_queue_fill() with a vu_queue_flush().
623 void vu_queue_push(VuDev
*dev
, VuVirtq
*vq
,
624 const VuVirtqElement
*elem
, unsigned int len
);
628 * @dev: a VuDev context
629 * @vq: a VuVirtq queue
630 * @num: number of elements to flush
632 * Mark the last number of elements as done (used.idx is updated by
635 void vu_queue_flush(VuDev
*dev
, VuVirtq
*vq
, unsigned int num
);
638 * vu_queue_get_avail_bytes:
639 * @dev: a VuDev context
640 * @vq: a VuVirtq queue
641 * @in_bytes: in bytes
642 * @out_bytes: out bytes
643 * @max_in_bytes: stop counting after max_in_bytes
644 * @max_out_bytes: stop counting after max_out_bytes
646 * Count the number of available bytes, up to max_in_bytes/max_out_bytes.
648 void vu_queue_get_avail_bytes(VuDev
*vdev
, VuVirtq
*vq
, unsigned int *in_bytes
,
649 unsigned int *out_bytes
,
650 unsigned max_in_bytes
, unsigned max_out_bytes
);
653 * vu_queue_avail_bytes:
654 * @dev: a VuDev context
655 * @vq: a VuVirtq queue
656 * @in_bytes: expected in bytes
657 * @out_bytes: expected out bytes
659 * Returns: true if in_bytes <= in_total && out_bytes <= out_total
661 bool vu_queue_avail_bytes(VuDev
*dev
, VuVirtq
*vq
, unsigned int in_bytes
,
662 unsigned int out_bytes
);
664 #endif /* LIBVHOST_USER_H */