?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
aspeed: Fix realize error API violation
[qemu/armbru.git] / hw / virtio / virtio.c
blobcc9c9dc1621eec3ea98182106e970cff656e32ad
1 /*
2 * Virtio Support
3 *
4 * Copyright IBM, Corp. 2007
5 *
6 * Authors:
7 * Anthony Liguori <aliguori@us.ibm.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 */
13
14 #include "qemu/osdep.h"
15 #include "qapi/error.h"
16 #include "cpu.h"
17 #include "trace.h"
18 #include "exec/address-spaces.h"
19 #include "qemu/error-report.h"
20 #include "qemu/main-loop.h"
21 #include "qemu/module.h"
22 #include "hw/virtio/virtio.h"
23 #include "migration/qemu-file-types.h"
24 #include "qemu/atomic.h"
25 #include "hw/virtio/virtio-bus.h"
26 #include "hw/qdev-properties.h"
27 #include "hw/virtio/virtio-access.h"
28 #include "sysemu/dma.h"
29 #include "sysemu/runstate.h"
30
31 /*
32 * The alignment to use between consumer and producer parts of vring.
33 * x86 pagesize again. This is the default, used by transports like PCI
34 * which don't provide a means for the guest to tell the host the alignment.
35 */
36 #define VIRTIO_PCI_VRING_ALIGN 4096
37
38 typedef struct VRingDesc
39 {
40 uint64_t addr;
41 uint32_t len;
42 uint16_t flags;
43 uint16_t next;
44 } VRingDesc;
45
46 typedef struct VRingPackedDesc {
47 uint64_t addr;
48 uint32_t len;
49 uint16_t id;
50 uint16_t flags;
51 } VRingPackedDesc;
52
53 typedef struct VRingAvail
54 {
55 uint16_t flags;
56 uint16_t idx;
57 uint16_t ring[];
58 } VRingAvail;
59
60 typedef struct VRingUsedElem
61 {
62 uint32_t id;
63 uint32_t len;
64 } VRingUsedElem;
65
66 typedef struct VRingUsed
67 {
68 uint16_t flags;
69 uint16_t idx;
70 VRingUsedElem ring[];
71 } VRingUsed;
72
73 typedef struct VRingMemoryRegionCaches {
74 struct rcu_head rcu;
75 MemoryRegionCache desc;
76 MemoryRegionCache avail;
77 MemoryRegionCache used;
78 } VRingMemoryRegionCaches;
79
80 typedef struct VRing
81 {
82 unsigned int num;
83 unsigned int num_default;
84 unsigned int align;
85 hwaddr desc;
86 hwaddr avail;
87 hwaddr used;
88 VRingMemoryRegionCaches *caches;
89 } VRing;
90
91 typedef struct VRingPackedDescEvent {
92 uint16_t off_wrap;
93 uint16_t flags;
94 } VRingPackedDescEvent ;
95
96 struct VirtQueue
97 {
98 VRing vring;
99 VirtQueueElement *used_elems;
100
101 /* Next head to pop */
102 uint16_t last_avail_idx;
103 bool last_avail_wrap_counter;
104
105 /* Last avail_idx read from VQ. */
106 uint16_t shadow_avail_idx;
107 bool shadow_avail_wrap_counter;
108
109 uint16_t used_idx;
110 bool used_wrap_counter;
111
112 /* Last used index value we have signalled on */
113 uint16_t signalled_used;
114
115 /* Last used index value we have signalled on */
116 bool signalled_used_valid;
117
118 /* Notification enabled? */
119 bool notification;
120
121 uint16_t queue_index;
122
123 unsigned int inuse;
124
125 uint16_t vector;
126 VirtIOHandleOutput handle_output;
127 VirtIOHandleAIOOutput handle_aio_output;
128 VirtIODevice *vdev;
129 EventNotifier guest_notifier;
130 EventNotifier host_notifier;
131 bool host_notifier_enabled;
132 QLIST_ENTRY(VirtQueue) node;
133 };
134
135 static void virtio_free_region_cache(VRingMemoryRegionCaches *caches)
136 {
137 if (!caches) {
138 return;
139 }
140
141 address_space_cache_destroy(&caches->desc);
142 address_space_cache_destroy(&caches->avail);
143 address_space_cache_destroy(&caches->used);
144 g_free(caches);
145 }
146
147 static void virtio_virtqueue_reset_region_cache(struct VirtQueue *vq)
148 {
149 VRingMemoryRegionCaches *caches;
150
151 caches = atomic_read(&vq->vring.caches);
152 atomic_rcu_set(&vq->vring.caches, NULL);
153 if (caches) {
154 call_rcu(caches, virtio_free_region_cache, rcu);
155 }
156 }
157
158 static void virtio_init_region_cache(VirtIODevice *vdev, int n)
159 {
160 VirtQueue *vq = &vdev->vq[n];
161 VRingMemoryRegionCaches *old = vq->vring.caches;
162 VRingMemoryRegionCaches *new = NULL;
163 hwaddr addr, size;
164 int64_t len;
165 bool packed;
166
167
168 addr = vq->vring.desc;
169 if (!addr) {
170 goto out_no_cache;
171 }
172 new = g_new0(VRingMemoryRegionCaches, 1);
173 size = virtio_queue_get_desc_size(vdev, n);
174 packed = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
175 true : false;
176 len = address_space_cache_init(&new->desc, vdev->dma_as,
177 addr, size, packed);
178 if (len < size) {
179 virtio_error(vdev, "Cannot map desc");
180 goto err_desc;
181 }
182
183 size = virtio_queue_get_used_size(vdev, n);
184 len = address_space_cache_init(&new->used, vdev->dma_as,
185 vq->vring.used, size, true);
186 if (len < size) {
187 virtio_error(vdev, "Cannot map used");
188 goto err_used;
189 }
190
191 size = virtio_queue_get_avail_size(vdev, n);
192 len = address_space_cache_init(&new->avail, vdev->dma_as,
193 vq->vring.avail, size, false);
194 if (len < size) {
195 virtio_error(vdev, "Cannot map avail");
196 goto err_avail;
197 }
198
199 atomic_rcu_set(&vq->vring.caches, new);
200 if (old) {
201 call_rcu(old, virtio_free_region_cache, rcu);
202 }
203 return;
204
205 err_avail:
206 address_space_cache_destroy(&new->avail);
207 err_used:
208 address_space_cache_destroy(&new->used);
209 err_desc:
210 address_space_cache_destroy(&new->desc);
211 out_no_cache:
212 g_free(new);
213 virtio_virtqueue_reset_region_cache(vq);
214 }
215
216 /* virt queue functions */
217 void virtio_queue_update_rings(VirtIODevice *vdev, int n)
218 {
219 VRing *vring = &vdev->vq[n].vring;
220
221 if (!vring->num || !vring->desc || !vring->align) {
222 /* not yet setup -> nothing to do */
223 return;
224 }
225 vring->avail = vring->desc + vring->num * sizeof(VRingDesc);
226 vring->used = vring_align(vring->avail +
227 offsetof(VRingAvail, ring[vring->num]),
228 vring->align);
229 virtio_init_region_cache(vdev, n);
230 }
231
232 /* Called within rcu_read_lock(). */
233 static void vring_split_desc_read(VirtIODevice *vdev, VRingDesc *desc,
234 MemoryRegionCache *cache, int i)
235 {
236 address_space_read_cached(cache, i * sizeof(VRingDesc),
237 desc, sizeof(VRingDesc));
238 virtio_tswap64s(vdev, &desc->addr);
239 virtio_tswap32s(vdev, &desc->len);
240 virtio_tswap16s(vdev, &desc->flags);
241 virtio_tswap16s(vdev, &desc->next);
242 }
243
244 static void vring_packed_event_read(VirtIODevice *vdev,
245 MemoryRegionCache *cache,
246 VRingPackedDescEvent *e)
247 {
248 hwaddr off_off = offsetof(VRingPackedDescEvent, off_wrap);
249 hwaddr off_flags = offsetof(VRingPackedDescEvent, flags);
250
251 address_space_read_cached(cache, off_flags, &e->flags,
252 sizeof(e->flags));
253 /* Make sure flags is seen before off_wrap */
254 smp_rmb();
255 address_space_read_cached(cache, off_off, &e->off_wrap,
256 sizeof(e->off_wrap));
257 virtio_tswap16s(vdev, &e->off_wrap);
258 virtio_tswap16s(vdev, &e->flags);
259 }
260
261 static void vring_packed_off_wrap_write(VirtIODevice *vdev,
262 MemoryRegionCache *cache,
263 uint16_t off_wrap)
264 {
265 hwaddr off = offsetof(VRingPackedDescEvent, off_wrap);
266
267 virtio_tswap16s(vdev, &off_wrap);
268 address_space_write_cached(cache, off, &off_wrap, sizeof(off_wrap));
269 address_space_cache_invalidate(cache, off, sizeof(off_wrap));
270 }
271
272 static void vring_packed_flags_write(VirtIODevice *vdev,
273 MemoryRegionCache *cache, uint16_t flags)
274 {
275 hwaddr off = offsetof(VRingPackedDescEvent, flags);
276
277 virtio_tswap16s(vdev, &flags);
278 address_space_write_cached(cache, off, &flags, sizeof(flags));
279 address_space_cache_invalidate(cache, off, sizeof(flags));
280 }
281
282 /* Called within rcu_read_lock(). */
283 static VRingMemoryRegionCaches *vring_get_region_caches(struct VirtQueue *vq)
284 {
285 return atomic_rcu_read(&vq->vring.caches);
286 }
287
288 /* Called within rcu_read_lock(). */
289 static inline uint16_t vring_avail_flags(VirtQueue *vq)
290 {
291 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
292 hwaddr pa = offsetof(VRingAvail, flags);
293
294 if (!caches) {
295 return 0;
296 }
297
298 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
299 }
300
301 /* Called within rcu_read_lock(). */
302 static inline uint16_t vring_avail_idx(VirtQueue *vq)
303 {
304 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
305 hwaddr pa = offsetof(VRingAvail, idx);
306
307 if (!caches) {
308 return 0;
309 }
310
311 vq->shadow_avail_idx = virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
312 return vq->shadow_avail_idx;
313 }
314
315 /* Called within rcu_read_lock(). */
316 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
317 {
318 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
319 hwaddr pa = offsetof(VRingAvail, ring[i]);
320
321 if (!caches) {
322 return 0;
323 }
324
325 return virtio_lduw_phys_cached(vq->vdev, &caches->avail, pa);
326 }
327
328 /* Called within rcu_read_lock(). */
329 static inline uint16_t vring_get_used_event(VirtQueue *vq)
330 {
331 return vring_avail_ring(vq, vq->vring.num);
332 }
333
334 /* Called within rcu_read_lock(). */
335 static inline void vring_used_write(VirtQueue *vq, VRingUsedElem *uelem,
336 int i)
337 {
338 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
339 hwaddr pa = offsetof(VRingUsed, ring[i]);
340
341 if (!caches) {
342 return;
343 }
344
345 virtio_tswap32s(vq->vdev, &uelem->id);
346 virtio_tswap32s(vq->vdev, &uelem->len);
347 address_space_write_cached(&caches->used, pa, uelem, sizeof(VRingUsedElem));
348 address_space_cache_invalidate(&caches->used, pa, sizeof(VRingUsedElem));
349 }
350
351 /* Called within rcu_read_lock(). */
352 static uint16_t vring_used_idx(VirtQueue *vq)
353 {
354 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
355 hwaddr pa = offsetof(VRingUsed, idx);
356
357 if (!caches) {
358 return 0;
359 }
360
361 return virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
362 }
363
364 /* Called within rcu_read_lock(). */
365 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
366 {
367 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
368 hwaddr pa = offsetof(VRingUsed, idx);
369
370 if (caches) {
371 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
372 address_space_cache_invalidate(&caches->used, pa, sizeof(val));
373 }
374
375 vq->used_idx = val;
376 }
377
378 /* Called within rcu_read_lock(). */
379 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
380 {
381 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
382 VirtIODevice *vdev = vq->vdev;
383 hwaddr pa = offsetof(VRingUsed, flags);
384 uint16_t flags;
385
386 if (!caches) {
387 return;
388 }
389
390 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
391 virtio_stw_phys_cached(vdev, &caches->used, pa, flags | mask);
392 address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
393 }
394
395 /* Called within rcu_read_lock(). */
396 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
397 {
398 VRingMemoryRegionCaches *caches = vring_get_region_caches(vq);
399 VirtIODevice *vdev = vq->vdev;
400 hwaddr pa = offsetof(VRingUsed, flags);
401 uint16_t flags;
402
403 if (!caches) {
404 return;
405 }
406
407 flags = virtio_lduw_phys_cached(vq->vdev, &caches->used, pa);
408 virtio_stw_phys_cached(vdev, &caches->used, pa, flags & ~mask);
409 address_space_cache_invalidate(&caches->used, pa, sizeof(flags));
410 }
411
412 /* Called within rcu_read_lock(). */
413 static inline void vring_set_avail_event(VirtQueue *vq, uint16_t val)
414 {
415 VRingMemoryRegionCaches *caches;
416 hwaddr pa;
417 if (!vq->notification) {
418 return;
419 }
420
421 caches = vring_get_region_caches(vq);
422 if (!caches) {
423 return;
424 }
425
426 pa = offsetof(VRingUsed, ring[vq->vring.num]);
427 virtio_stw_phys_cached(vq->vdev, &caches->used, pa, val);
428 address_space_cache_invalidate(&caches->used, pa, sizeof(val));
429 }
430
431 static void virtio_queue_split_set_notification(VirtQueue *vq, int enable)
432 {
433 RCU_READ_LOCK_GUARD();
434
435 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
436 vring_set_avail_event(vq, vring_avail_idx(vq));
437 } else if (enable) {
438 vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY);
439 } else {
440 vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY);
441 }
442 if (enable) {
443 /* Expose avail event/used flags before caller checks the avail idx. */
444 smp_mb();
445 }
446 }
447
448 static void virtio_queue_packed_set_notification(VirtQueue *vq, int enable)
449 {
450 uint16_t off_wrap;
451 VRingPackedDescEvent e;
452 VRingMemoryRegionCaches *caches;
453
454 RCU_READ_LOCK_GUARD();
455 caches = vring_get_region_caches(vq);
456 if (!caches) {
457 return;
458 }
459
460 vring_packed_event_read(vq->vdev, &caches->used, &e);
461
462 if (!enable) {
463 e.flags = VRING_PACKED_EVENT_FLAG_DISABLE;
464 } else if (virtio_vdev_has_feature(vq->vdev, VIRTIO_RING_F_EVENT_IDX)) {
465 off_wrap = vq->shadow_avail_idx | vq->shadow_avail_wrap_counter << 15;
466 vring_packed_off_wrap_write(vq->vdev, &caches->used, off_wrap);
467 /* Make sure off_wrap is wrote before flags */
468 smp_wmb();
469 e.flags = VRING_PACKED_EVENT_FLAG_DESC;
470 } else {
471 e.flags = VRING_PACKED_EVENT_FLAG_ENABLE;
472 }
473
474 vring_packed_flags_write(vq->vdev, &caches->used, e.flags);
475 if (enable) {
476 /* Expose avail event/used flags before caller checks the avail idx. */
477 smp_mb();
478 }
479 }
480
481 bool virtio_queue_get_notification(VirtQueue *vq)
482 {
483 return vq->notification;
484 }
485
486 void virtio_queue_set_notification(VirtQueue *vq, int enable)
487 {
488 vq->notification = enable;
489
490 if (!vq->vring.desc) {
491 return;
492 }
493
494 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
495 virtio_queue_packed_set_notification(vq, enable);
496 } else {
497 virtio_queue_split_set_notification(vq, enable);
498 }
499 }
500
501 int virtio_queue_ready(VirtQueue *vq)
502 {
503 return vq->vring.avail != 0;
504 }
505
506 static void vring_packed_desc_read_flags(VirtIODevice *vdev,
507 uint16_t *flags,
508 MemoryRegionCache *cache,
509 int i)
510 {
511 address_space_read_cached(cache,
512 i * sizeof(VRingPackedDesc) +
513 offsetof(VRingPackedDesc, flags),
514 flags, sizeof(*flags));
515 virtio_tswap16s(vdev, flags);
516 }
517
518 static void vring_packed_desc_read(VirtIODevice *vdev,
519 VRingPackedDesc *desc,
520 MemoryRegionCache *cache,
521 int i, bool strict_order)
522 {
523 hwaddr off = i * sizeof(VRingPackedDesc);
524
525 vring_packed_desc_read_flags(vdev, &desc->flags, cache, i);
526
527 if (strict_order) {
528 /* Make sure flags is read before the rest fields. */
529 smp_rmb();
530 }
531
532 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, addr),
533 &desc->addr, sizeof(desc->addr));
534 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, id),
535 &desc->id, sizeof(desc->id));
536 address_space_read_cached(cache, off + offsetof(VRingPackedDesc, len),
537 &desc->len, sizeof(desc->len));
538 virtio_tswap64s(vdev, &desc->addr);
539 virtio_tswap16s(vdev, &desc->id);
540 virtio_tswap32s(vdev, &desc->len);
541 }
542
543 static void vring_packed_desc_write_data(VirtIODevice *vdev,
544 VRingPackedDesc *desc,
545 MemoryRegionCache *cache,
546 int i)
547 {
548 hwaddr off_id = i * sizeof(VRingPackedDesc) +
549 offsetof(VRingPackedDesc, id);
550 hwaddr off_len = i * sizeof(VRingPackedDesc) +
551 offsetof(VRingPackedDesc, len);
552
553 virtio_tswap32s(vdev, &desc->len);
554 virtio_tswap16s(vdev, &desc->id);
555 address_space_write_cached(cache, off_id, &desc->id, sizeof(desc->id));
556 address_space_cache_invalidate(cache, off_id, sizeof(desc->id));
557 address_space_write_cached(cache, off_len, &desc->len, sizeof(desc->len));
558 address_space_cache_invalidate(cache, off_len, sizeof(desc->len));
559 }
560
561 static void vring_packed_desc_write_flags(VirtIODevice *vdev,
562 VRingPackedDesc *desc,
563 MemoryRegionCache *cache,
564 int i)
565 {
566 hwaddr off = i * sizeof(VRingPackedDesc) + offsetof(VRingPackedDesc, flags);
567
568 virtio_tswap16s(vdev, &desc->flags);
569 address_space_write_cached(cache, off, &desc->flags, sizeof(desc->flags));
570 address_space_cache_invalidate(cache, off, sizeof(desc->flags));
571 }
572
573 static void vring_packed_desc_write(VirtIODevice *vdev,
574 VRingPackedDesc *desc,
575 MemoryRegionCache *cache,
576 int i, bool strict_order)
577 {
578 vring_packed_desc_write_data(vdev, desc, cache, i);
579 if (strict_order) {
580 /* Make sure data is wrote before flags. */
581 smp_wmb();
582 }
583 vring_packed_desc_write_flags(vdev, desc, cache, i);
584 }
585
586 static inline bool is_desc_avail(uint16_t flags, bool wrap_counter)
587 {
588 bool avail, used;
589
590 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
591 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
592 return (avail != used) && (avail == wrap_counter);
593 }
594
595 /* Fetch avail_idx from VQ memory only when we really need to know if
596 * guest has added some buffers.
597 * Called within rcu_read_lock(). */
598 static int virtio_queue_empty_rcu(VirtQueue *vq)
599 {
600 if (virtio_device_disabled(vq->vdev)) {
601 return 1;
602 }
603
604 if (unlikely(!vq->vring.avail)) {
605 return 1;
606 }
607
608 if (vq->shadow_avail_idx != vq->last_avail_idx) {
609 return 0;
610 }
611
612 return vring_avail_idx(vq) == vq->last_avail_idx;
613 }
614
615 static int virtio_queue_split_empty(VirtQueue *vq)
616 {
617 bool empty;
618
619 if (virtio_device_disabled(vq->vdev)) {
620 return 1;
621 }
622
623 if (unlikely(!vq->vring.avail)) {
624 return 1;
625 }
626
627 if (vq->shadow_avail_idx != vq->last_avail_idx) {
628 return 0;
629 }
630
631 RCU_READ_LOCK_GUARD();
632 empty = vring_avail_idx(vq) == vq->last_avail_idx;
633 return empty;
634 }
635
636 static int virtio_queue_packed_empty_rcu(VirtQueue *vq)
637 {
638 struct VRingPackedDesc desc;
639 VRingMemoryRegionCaches *cache;
640
641 if (unlikely(!vq->vring.desc)) {
642 return 1;
643 }
644
645 cache = vring_get_region_caches(vq);
646 if (!cache) {
647 return 1;
648 }
649
650 vring_packed_desc_read_flags(vq->vdev, &desc.flags, &cache->desc,
651 vq->last_avail_idx);
652
653 return !is_desc_avail(desc.flags, vq->last_avail_wrap_counter);
654 }
655
656 static int virtio_queue_packed_empty(VirtQueue *vq)
657 {
658 RCU_READ_LOCK_GUARD();
659 return virtio_queue_packed_empty_rcu(vq);
660 }
661
662 int virtio_queue_empty(VirtQueue *vq)
663 {
664 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
665 return virtio_queue_packed_empty(vq);
666 } else {
667 return virtio_queue_split_empty(vq);
668 }
669 }
670
671 static void virtqueue_unmap_sg(VirtQueue *vq, const VirtQueueElement *elem,
672 unsigned int len)
673 {
674 AddressSpace *dma_as = vq->vdev->dma_as;
675 unsigned int offset;
676 int i;
677
678 offset = 0;
679 for (i = 0; i < elem->in_num; i++) {
680 size_t size = MIN(len - offset, elem->in_sg[i].iov_len);
681
682 dma_memory_unmap(dma_as, elem->in_sg[i].iov_base,
683 elem->in_sg[i].iov_len,
684 DMA_DIRECTION_FROM_DEVICE, size);
685
686 offset += size;
687 }
688
689 for (i = 0; i < elem->out_num; i++)
690 dma_memory_unmap(dma_as, elem->out_sg[i].iov_base,
691 elem->out_sg[i].iov_len,
692 DMA_DIRECTION_TO_DEVICE,
693 elem->out_sg[i].iov_len);
694 }
695
696 /* virtqueue_detach_element:
697 * @vq: The #VirtQueue
698 * @elem: The #VirtQueueElement
699 * @len: number of bytes written
700 *
701 * Detach the element from the virtqueue. This function is suitable for device
702 * reset or other situations where a #VirtQueueElement is simply freed and will
703 * not be pushed or discarded.
704 */
705 void virtqueue_detach_element(VirtQueue *vq, const VirtQueueElement *elem,
706 unsigned int len)
707 {
708 vq->inuse -= elem->ndescs;
709 virtqueue_unmap_sg(vq, elem, len);
710 }
711
712 static void virtqueue_split_rewind(VirtQueue *vq, unsigned int num)
713 {
714 vq->last_avail_idx -= num;
715 }
716
717 static void virtqueue_packed_rewind(VirtQueue *vq, unsigned int num)
718 {
719 if (vq->last_avail_idx < num) {
720 vq->last_avail_idx = vq->vring.num + vq->last_avail_idx - num;
721 vq->last_avail_wrap_counter ^= 1;
722 } else {
723 vq->last_avail_idx -= num;
724 }
725 }
726
727 /* virtqueue_unpop:
728 * @vq: The #VirtQueue
729 * @elem: The #VirtQueueElement
730 * @len: number of bytes written
731 *
732 * Pretend the most recent element wasn't popped from the virtqueue. The next
733 * call to virtqueue_pop() will refetch the element.
734 */
735 void virtqueue_unpop(VirtQueue *vq, const VirtQueueElement *elem,
736 unsigned int len)
737 {
738
739 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
740 virtqueue_packed_rewind(vq, 1);
741 } else {
742 virtqueue_split_rewind(vq, 1);
743 }
744
745 virtqueue_detach_element(vq, elem, len);
746 }
747
748 /* virtqueue_rewind:
749 * @vq: The #VirtQueue
750 * @num: Number of elements to push back
751 *
752 * Pretend that elements weren't popped from the virtqueue. The next
753 * virtqueue_pop() will refetch the oldest element.
754 *
755 * Use virtqueue_unpop() instead if you have a VirtQueueElement.
756 *
757 * Returns: true on success, false if @num is greater than the number of in use
758 * elements.
759 */
760 bool virtqueue_rewind(VirtQueue *vq, unsigned int num)
761 {
762 if (num > vq->inuse) {
763 return false;
764 }
765
766 vq->inuse -= num;
767 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
768 virtqueue_packed_rewind(vq, num);
769 } else {
770 virtqueue_split_rewind(vq, num);
771 }
772 return true;
773 }
774
775 static void virtqueue_split_fill(VirtQueue *vq, const VirtQueueElement *elem,
776 unsigned int len, unsigned int idx)
777 {
778 VRingUsedElem uelem;
779
780 if (unlikely(!vq->vring.used)) {
781 return;
782 }
783
784 idx = (idx + vq->used_idx) % vq->vring.num;
785
786 uelem.id = elem->index;
787 uelem.len = len;
788 vring_used_write(vq, &uelem, idx);
789 }
790
791 static void virtqueue_packed_fill(VirtQueue *vq, const VirtQueueElement *elem,
792 unsigned int len, unsigned int idx)
793 {
794 vq->used_elems[idx].index = elem->index;
795 vq->used_elems[idx].len = len;
796 vq->used_elems[idx].ndescs = elem->ndescs;
797 }
798
799 static void virtqueue_packed_fill_desc(VirtQueue *vq,
800 const VirtQueueElement *elem,
801 unsigned int idx,
802 bool strict_order)
803 {
804 uint16_t head;
805 VRingMemoryRegionCaches *caches;
806 VRingPackedDesc desc = {
807 .id = elem->index,
808 .len = elem->len,
809 };
810 bool wrap_counter = vq->used_wrap_counter;
811
812 if (unlikely(!vq->vring.desc)) {
813 return;
814 }
815
816 head = vq->used_idx + idx;
817 if (head >= vq->vring.num) {
818 head -= vq->vring.num;
819 wrap_counter ^= 1;
820 }
821 if (wrap_counter) {
822 desc.flags |= (1 << VRING_PACKED_DESC_F_AVAIL);
823 desc.flags |= (1 << VRING_PACKED_DESC_F_USED);
824 } else {
825 desc.flags &= ~(1 << VRING_PACKED_DESC_F_AVAIL);
826 desc.flags &= ~(1 << VRING_PACKED_DESC_F_USED);
827 }
828
829 caches = vring_get_region_caches(vq);
830 if (!caches) {
831 return;
832 }
833
834 vring_packed_desc_write(vq->vdev, &desc, &caches->desc, head, strict_order);
835 }
836
837 /* Called within rcu_read_lock(). */
838 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
839 unsigned int len, unsigned int idx)
840 {
841 trace_virtqueue_fill(vq, elem, len, idx);
842
843 virtqueue_unmap_sg(vq, elem, len);
844
845 if (virtio_device_disabled(vq->vdev)) {
846 return;
847 }
848
849 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
850 virtqueue_packed_fill(vq, elem, len, idx);
851 } else {
852 virtqueue_split_fill(vq, elem, len, idx);
853 }
854 }
855
856 /* Called within rcu_read_lock(). */
857 static void virtqueue_split_flush(VirtQueue *vq, unsigned int count)
858 {
859 uint16_t old, new;
860
861 if (unlikely(!vq->vring.used)) {
862 return;
863 }
864
865 /* Make sure buffer is written before we update index. */
866 smp_wmb();
867 trace_virtqueue_flush(vq, count);
868 old = vq->used_idx;
869 new = old + count;
870 vring_used_idx_set(vq, new);
871 vq->inuse -= count;
872 if (unlikely((int16_t)(new - vq->signalled_used) < (uint16_t)(new - old)))
873 vq->signalled_used_valid = false;
874 }
875
876 static void virtqueue_packed_flush(VirtQueue *vq, unsigned int count)
877 {
878 unsigned int i, ndescs = 0;
879
880 if (unlikely(!vq->vring.desc)) {
881 return;
882 }
883
884 for (i = 1; i < count; i++) {
885 virtqueue_packed_fill_desc(vq, &vq->used_elems[i], i, false);
886 ndescs += vq->used_elems[i].ndescs;
887 }
888 virtqueue_packed_fill_desc(vq, &vq->used_elems[0], 0, true);
889 ndescs += vq->used_elems[0].ndescs;
890
891 vq->inuse -= ndescs;
892 vq->used_idx += ndescs;
893 if (vq->used_idx >= vq->vring.num) {
894 vq->used_idx -= vq->vring.num;
895 vq->used_wrap_counter ^= 1;
896 }
897 }
898
899 void virtqueue_flush(VirtQueue *vq, unsigned int count)
900 {
901 if (virtio_device_disabled(vq->vdev)) {
902 vq->inuse -= count;
903 return;
904 }
905
906 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
907 virtqueue_packed_flush(vq, count);
908 } else {
909 virtqueue_split_flush(vq, count);
910 }
911 }
912
913 void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem,
914 unsigned int len)
915 {
916 RCU_READ_LOCK_GUARD();
917 virtqueue_fill(vq, elem, len, 0);
918 virtqueue_flush(vq, 1);
919 }
920
921 /* Called within rcu_read_lock(). */
922 static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx)
923 {
924 uint16_t num_heads = vring_avail_idx(vq) - idx;
925
926 /* Check it isn't doing very strange things with descriptor numbers. */
927 if (num_heads > vq->vring.num) {
928 virtio_error(vq->vdev, "Guest moved used index from %u to %u",
929 idx, vq->shadow_avail_idx);
930 return -EINVAL;
931 }
932 /* On success, callers read a descriptor at vq->last_avail_idx.
933 * Make sure descriptor read does not bypass avail index read. */
934 if (num_heads) {
935 smp_rmb();
936 }
937
938 return num_heads;
939 }
940
941 /* Called within rcu_read_lock(). */
942 static bool virtqueue_get_head(VirtQueue *vq, unsigned int idx,
943 unsigned int *head)
944 {
945 /* Grab the next descriptor number they're advertising, and increment
946 * the index we've seen. */
947 *head = vring_avail_ring(vq, idx % vq->vring.num);
948
949 /* If their number is silly, that's a fatal mistake. */
950 if (*head >= vq->vring.num) {
951 virtio_error(vq->vdev, "Guest says index %u is available", *head);
952 return false;
953 }
954
955 return true;
956 }
957
958 enum {
959 VIRTQUEUE_READ_DESC_ERROR = -1,
960 VIRTQUEUE_READ_DESC_DONE = 0, /* end of chain */
961 VIRTQUEUE_READ_DESC_MORE = 1, /* more buffers in chain */
962 };
963
964 static int virtqueue_split_read_next_desc(VirtIODevice *vdev, VRingDesc *desc,
965 MemoryRegionCache *desc_cache,
966 unsigned int max, unsigned int *next)
967 {
968 /* If this descriptor says it doesn't chain, we're done. */
969 if (!(desc->flags & VRING_DESC_F_NEXT)) {
970 return VIRTQUEUE_READ_DESC_DONE;
971 }
972
973 /* Check they're not leading us off end of descriptors. */
974 *next = desc->next;
975 /* Make sure compiler knows to grab that: we don't want it changing! */
976 smp_wmb();
977
978 if (*next >= max) {
979 virtio_error(vdev, "Desc next is %u", *next);
980 return VIRTQUEUE_READ_DESC_ERROR;
981 }
982
983 vring_split_desc_read(vdev, desc, desc_cache, *next);
984 return VIRTQUEUE_READ_DESC_MORE;
985 }
986
987 static void virtqueue_split_get_avail_bytes(VirtQueue *vq,
988 unsigned int *in_bytes, unsigned int *out_bytes,
989 unsigned max_in_bytes, unsigned max_out_bytes)
990 {
991 VirtIODevice *vdev = vq->vdev;
992 unsigned int max, idx;
993 unsigned int total_bufs, in_total, out_total;
994 VRingMemoryRegionCaches *caches;
995 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
996 int64_t len = 0;
997 int rc;
998
999 RCU_READ_LOCK_GUARD();
1000
1001 idx = vq->last_avail_idx;
1002 total_bufs = in_total = out_total = 0;
1003
1004 max = vq->vring.num;
1005 caches = vring_get_region_caches(vq);
1006 if (!caches) {
1007 goto err;
1008 }
1009
1010 while ((rc = virtqueue_num_heads(vq, idx)) > 0) {
1011 MemoryRegionCache *desc_cache = &caches->desc;
1012 unsigned int num_bufs;
1013 VRingDesc desc;
1014 unsigned int i;
1015
1016 num_bufs = total_bufs;
1017
1018 if (!virtqueue_get_head(vq, idx++, &i)) {
1019 goto err;
1020 }
1021
1022 vring_split_desc_read(vdev, &desc, desc_cache, i);
1023
1024 if (desc.flags & VRING_DESC_F_INDIRECT) {
1025 if (!desc.len || (desc.len % sizeof(VRingDesc))) {
1026 virtio_error(vdev, "Invalid size for indirect buffer table");
1027 goto err;
1028 }
1029
1030 /* If we've got too many, that implies a descriptor loop. */
1031 if (num_bufs >= max) {
1032 virtio_error(vdev, "Looped descriptor");
1033 goto err;
1034 }
1035
1036 /* loop over the indirect descriptor table */
1037 len = address_space_cache_init(&indirect_desc_cache,
1038 vdev->dma_as,
1039 desc.addr, desc.len, false);
1040 desc_cache = &indirect_desc_cache;
1041 if (len < desc.len) {
1042 virtio_error(vdev, "Cannot map indirect buffer");
1043 goto err;
1044 }
1045
1046 max = desc.len / sizeof(VRingDesc);
1047 num_bufs = i = 0;
1048 vring_split_desc_read(vdev, &desc, desc_cache, i);
1049 }
1050
1051 do {
1052 /* If we've got too many, that implies a descriptor loop. */
1053 if (++num_bufs > max) {
1054 virtio_error(vdev, "Looped descriptor");
1055 goto err;
1056 }
1057
1058 if (desc.flags & VRING_DESC_F_WRITE) {
1059 in_total += desc.len;
1060 } else {
1061 out_total += desc.len;
1062 }
1063 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1064 goto done;
1065 }
1066
1067 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i);
1068 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1069
1070 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1071 goto err;
1072 }
1073
1074 if (desc_cache == &indirect_desc_cache) {
1075 address_space_cache_destroy(&indirect_desc_cache);
1076 total_bufs++;
1077 } else {
1078 total_bufs = num_bufs;
1079 }
1080 }
1081
1082 if (rc < 0) {
1083 goto err;
1084 }
1085
1086 done:
1087 address_space_cache_destroy(&indirect_desc_cache);
1088 if (in_bytes) {
1089 *in_bytes = in_total;
1090 }
1091 if (out_bytes) {
1092 *out_bytes = out_total;
1093 }
1094 return;
1095
1096 err:
1097 in_total = out_total = 0;
1098 goto done;
1099 }
1100
1101 static int virtqueue_packed_read_next_desc(VirtQueue *vq,
1102 VRingPackedDesc *desc,
1103 MemoryRegionCache
1104 *desc_cache,
1105 unsigned int max,
1106 unsigned int *next,
1107 bool indirect)
1108 {
1109 /* If this descriptor says it doesn't chain, we're done. */
1110 if (!indirect && !(desc->flags & VRING_DESC_F_NEXT)) {
1111 return VIRTQUEUE_READ_DESC_DONE;
1112 }
1113
1114 ++*next;
1115 if (*next == max) {
1116 if (indirect) {
1117 return VIRTQUEUE_READ_DESC_DONE;
1118 } else {
1119 (*next) -= vq->vring.num;
1120 }
1121 }
1122
1123 vring_packed_desc_read(vq->vdev, desc, desc_cache, *next, false);
1124 return VIRTQUEUE_READ_DESC_MORE;
1125 }
1126
1127 static void virtqueue_packed_get_avail_bytes(VirtQueue *vq,
1128 unsigned int *in_bytes,
1129 unsigned int *out_bytes,
1130 unsigned max_in_bytes,
1131 unsigned max_out_bytes)
1132 {
1133 VirtIODevice *vdev = vq->vdev;
1134 unsigned int max, idx;
1135 unsigned int total_bufs, in_total, out_total;
1136 MemoryRegionCache *desc_cache;
1137 VRingMemoryRegionCaches *caches;
1138 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
1139 int64_t len = 0;
1140 VRingPackedDesc desc;
1141 bool wrap_counter;
1142
1143 RCU_READ_LOCK_GUARD();
1144 idx = vq->last_avail_idx;
1145 wrap_counter = vq->last_avail_wrap_counter;
1146 total_bufs = in_total = out_total = 0;
1147
1148 max = vq->vring.num;
1149 caches = vring_get_region_caches(vq);
1150 if (!caches) {
1151 goto err;
1152 }
1153
1154 for (;;) {
1155 unsigned int num_bufs = total_bufs;
1156 unsigned int i = idx;
1157 int rc;
1158
1159 desc_cache = &caches->desc;
1160 vring_packed_desc_read(vdev, &desc, desc_cache, idx, true);
1161 if (!is_desc_avail(desc.flags, wrap_counter)) {
1162 break;
1163 }
1164
1165 if (desc.flags & VRING_DESC_F_INDIRECT) {
1166 if (desc.len % sizeof(VRingPackedDesc)) {
1167 virtio_error(vdev, "Invalid size for indirect buffer table");
1168 goto err;
1169 }
1170
1171 /* If we've got too many, that implies a descriptor loop. */
1172 if (num_bufs >= max) {
1173 virtio_error(vdev, "Looped descriptor");
1174 goto err;
1175 }
1176
1177 /* loop over the indirect descriptor table */
1178 len = address_space_cache_init(&indirect_desc_cache,
1179 vdev->dma_as,
1180 desc.addr, desc.len, false);
1181 desc_cache = &indirect_desc_cache;
1182 if (len < desc.len) {
1183 virtio_error(vdev, "Cannot map indirect buffer");
1184 goto err;
1185 }
1186
1187 max = desc.len / sizeof(VRingPackedDesc);
1188 num_bufs = i = 0;
1189 vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1190 }
1191
1192 do {
1193 /* If we've got too many, that implies a descriptor loop. */
1194 if (++num_bufs > max) {
1195 virtio_error(vdev, "Looped descriptor");
1196 goto err;
1197 }
1198
1199 if (desc.flags & VRING_DESC_F_WRITE) {
1200 in_total += desc.len;
1201 } else {
1202 out_total += desc.len;
1203 }
1204 if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
1205 goto done;
1206 }
1207
1208 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max,
1209 &i, desc_cache ==
1210 &indirect_desc_cache);
1211 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1212
1213 if (desc_cache == &indirect_desc_cache) {
1214 address_space_cache_destroy(&indirect_desc_cache);
1215 total_bufs++;
1216 idx++;
1217 } else {
1218 idx += num_bufs - total_bufs;
1219 total_bufs = num_bufs;
1220 }
1221
1222 if (idx >= vq->vring.num) {
1223 idx -= vq->vring.num;
1224 wrap_counter ^= 1;
1225 }
1226 }
1227
1228 /* Record the index and wrap counter for a kick we want */
1229 vq->shadow_avail_idx = idx;
1230 vq->shadow_avail_wrap_counter = wrap_counter;
1231 done:
1232 address_space_cache_destroy(&indirect_desc_cache);
1233 if (in_bytes) {
1234 *in_bytes = in_total;
1235 }
1236 if (out_bytes) {
1237 *out_bytes = out_total;
1238 }
1239 return;
1240
1241 err:
1242 in_total = out_total = 0;
1243 goto done;
1244 }
1245
1246 void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
1247 unsigned int *out_bytes,
1248 unsigned max_in_bytes, unsigned max_out_bytes)
1249 {
1250 uint16_t desc_size;
1251 VRingMemoryRegionCaches *caches;
1252
1253 if (unlikely(!vq->vring.desc)) {
1254 goto err;
1255 }
1256
1257 caches = vring_get_region_caches(vq);
1258 if (!caches) {
1259 goto err;
1260 }
1261
1262 desc_size = virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED) ?
1263 sizeof(VRingPackedDesc) : sizeof(VRingDesc);
1264 if (caches->desc.len < vq->vring.num * desc_size) {
1265 virtio_error(vq->vdev, "Cannot map descriptor ring");
1266 goto err;
1267 }
1268
1269 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1270 virtqueue_packed_get_avail_bytes(vq, in_bytes, out_bytes,
1271 max_in_bytes, max_out_bytes);
1272 } else {
1273 virtqueue_split_get_avail_bytes(vq, in_bytes, out_bytes,
1274 max_in_bytes, max_out_bytes);
1275 }
1276
1277 return;
1278 err:
1279 if (in_bytes) {
1280 *in_bytes = 0;
1281 }
1282 if (out_bytes) {
1283 *out_bytes = 0;
1284 }
1285 }
1286
1287 int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
1288 unsigned int out_bytes)
1289 {
1290 unsigned int in_total, out_total;
1291
1292 virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
1293 return in_bytes <= in_total && out_bytes <= out_total;
1294 }
1295
1296 static bool virtqueue_map_desc(VirtIODevice *vdev, unsigned int *p_num_sg,
1297 hwaddr *addr, struct iovec *iov,
1298 unsigned int max_num_sg, bool is_write,
1299 hwaddr pa, size_t sz)
1300 {
1301 bool ok = false;
1302 unsigned num_sg = *p_num_sg;
1303 assert(num_sg <= max_num_sg);
1304
1305 if (!sz) {
1306 virtio_error(vdev, "virtio: zero sized buffers are not allowed");
1307 goto out;
1308 }
1309
1310 while (sz) {
1311 hwaddr len = sz;
1312
1313 if (num_sg == max_num_sg) {
1314 virtio_error(vdev, "virtio: too many write descriptors in "
1315 "indirect table");
1316 goto out;
1317 }
1318
1319 iov[num_sg].iov_base = dma_memory_map(vdev->dma_as, pa, &len,
1320 is_write ?
1321 DMA_DIRECTION_FROM_DEVICE :
1322 DMA_DIRECTION_TO_DEVICE);
1323 if (!iov[num_sg].iov_base) {
1324 virtio_error(vdev, "virtio: bogus descriptor or out of resources");
1325 goto out;
1326 }
1327
1328 iov[num_sg].iov_len = len;
1329 addr[num_sg] = pa;
1330
1331 sz -= len;
1332 pa += len;
1333 num_sg++;
1334 }
1335 ok = true;
1336
1337 out:
1338 *p_num_sg = num_sg;
1339 return ok;
1340 }
1341
1342 /* Only used by error code paths before we have a VirtQueueElement (therefore
1343 * virtqueue_unmap_sg() can't be used). Assumes buffers weren't written to
1344 * yet.
1345 */
1346 static void virtqueue_undo_map_desc(unsigned int out_num, unsigned int in_num,
1347 struct iovec *iov)
1348 {
1349 unsigned int i;
1350
1351 for (i = 0; i < out_num + in_num; i++) {
1352 int is_write = i >= out_num;
1353
1354 cpu_physical_memory_unmap(iov->iov_base, iov->iov_len, is_write, 0);
1355 iov++;
1356 }
1357 }
1358
1359 static void virtqueue_map_iovec(VirtIODevice *vdev, struct iovec *sg,
1360 hwaddr *addr, unsigned int num_sg,
1361 bool is_write)
1362 {
1363 unsigned int i;
1364 hwaddr len;
1365
1366 for (i = 0; i < num_sg; i++) {
1367 len = sg[i].iov_len;
1368 sg[i].iov_base = dma_memory_map(vdev->dma_as,
1369 addr[i], &len, is_write ?
1370 DMA_DIRECTION_FROM_DEVICE :
1371 DMA_DIRECTION_TO_DEVICE);
1372 if (!sg[i].iov_base) {
1373 error_report("virtio: error trying to map MMIO memory");
1374 exit(1);
1375 }
1376 if (len != sg[i].iov_len) {
1377 error_report("virtio: unexpected memory split");
1378 exit(1);
1379 }
1380 }
1381 }
1382
1383 void virtqueue_map(VirtIODevice *vdev, VirtQueueElement *elem)
1384 {
1385 virtqueue_map_iovec(vdev, elem->in_sg, elem->in_addr, elem->in_num, true);
1386 virtqueue_map_iovec(vdev, elem->out_sg, elem->out_addr, elem->out_num,
1387 false);
1388 }
1389
1390 static void *virtqueue_alloc_element(size_t sz, unsigned out_num, unsigned in_num)
1391 {
1392 VirtQueueElement *elem;
1393 size_t in_addr_ofs = QEMU_ALIGN_UP(sz, __alignof__(elem->in_addr[0]));
1394 size_t out_addr_ofs = in_addr_ofs + in_num * sizeof(elem->in_addr[0]);
1395 size_t out_addr_end = out_addr_ofs + out_num * sizeof(elem->out_addr[0]);
1396 size_t in_sg_ofs = QEMU_ALIGN_UP(out_addr_end, __alignof__(elem->in_sg[0]));
1397 size_t out_sg_ofs = in_sg_ofs + in_num * sizeof(elem->in_sg[0]);
1398 size_t out_sg_end = out_sg_ofs + out_num * sizeof(elem->out_sg[0]);
1399
1400 assert(sz >= sizeof(VirtQueueElement));
1401 elem = g_malloc(out_sg_end);
1402 trace_virtqueue_alloc_element(elem, sz, in_num, out_num);
1403 elem->out_num = out_num;
1404 elem->in_num = in_num;
1405 elem->in_addr = (void *)elem + in_addr_ofs;
1406 elem->out_addr = (void *)elem + out_addr_ofs;
1407 elem->in_sg = (void *)elem + in_sg_ofs;
1408 elem->out_sg = (void *)elem + out_sg_ofs;
1409 return elem;
1410 }
1411
1412 static void *virtqueue_split_pop(VirtQueue *vq, size_t sz)
1413 {
1414 unsigned int i, head, max;
1415 VRingMemoryRegionCaches *caches;
1416 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
1417 MemoryRegionCache *desc_cache;
1418 int64_t len;
1419 VirtIODevice *vdev = vq->vdev;
1420 VirtQueueElement *elem = NULL;
1421 unsigned out_num, in_num, elem_entries;
1422 hwaddr addr[VIRTQUEUE_MAX_SIZE];
1423 struct iovec iov[VIRTQUEUE_MAX_SIZE];
1424 VRingDesc desc;
1425 int rc;
1426
1427 RCU_READ_LOCK_GUARD();
1428 if (virtio_queue_empty_rcu(vq)) {
1429 goto done;
1430 }
1431 /* Needed after virtio_queue_empty(), see comment in
1432 * virtqueue_num_heads(). */
1433 smp_rmb();
1434
1435 /* When we start there are none of either input nor output. */
1436 out_num = in_num = elem_entries = 0;
1437
1438 max = vq->vring.num;
1439
1440 if (vq->inuse >= vq->vring.num) {
1441 virtio_error(vdev, "Virtqueue size exceeded");
1442 goto done;
1443 }
1444
1445 if (!virtqueue_get_head(vq, vq->last_avail_idx++, &head)) {
1446 goto done;
1447 }
1448
1449 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
1450 vring_set_avail_event(vq, vq->last_avail_idx);
1451 }
1452
1453 i = head;
1454
1455 caches = vring_get_region_caches(vq);
1456 if (!caches) {
1457 virtio_error(vdev, "Region caches not initialized");
1458 goto done;
1459 }
1460
1461 if (caches->desc.len < max * sizeof(VRingDesc)) {
1462 virtio_error(vdev, "Cannot map descriptor ring");
1463 goto done;
1464 }
1465
1466 desc_cache = &caches->desc;
1467 vring_split_desc_read(vdev, &desc, desc_cache, i);
1468 if (desc.flags & VRING_DESC_F_INDIRECT) {
1469 if (!desc.len || (desc.len % sizeof(VRingDesc))) {
1470 virtio_error(vdev, "Invalid size for indirect buffer table");
1471 goto done;
1472 }
1473
1474 /* loop over the indirect descriptor table */
1475 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1476 desc.addr, desc.len, false);
1477 desc_cache = &indirect_desc_cache;
1478 if (len < desc.len) {
1479 virtio_error(vdev, "Cannot map indirect buffer");
1480 goto done;
1481 }
1482
1483 max = desc.len / sizeof(VRingDesc);
1484 i = 0;
1485 vring_split_desc_read(vdev, &desc, desc_cache, i);
1486 }
1487
1488 /* Collect all the descriptors */
1489 do {
1490 bool map_ok;
1491
1492 if (desc.flags & VRING_DESC_F_WRITE) {
1493 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1494 iov + out_num,
1495 VIRTQUEUE_MAX_SIZE - out_num, true,
1496 desc.addr, desc.len);
1497 } else {
1498 if (in_num) {
1499 virtio_error(vdev, "Incorrect order for descriptors");
1500 goto err_undo_map;
1501 }
1502 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1503 VIRTQUEUE_MAX_SIZE, false,
1504 desc.addr, desc.len);
1505 }
1506 if (!map_ok) {
1507 goto err_undo_map;
1508 }
1509
1510 /* If we've got too many, that implies a descriptor loop. */
1511 if (++elem_entries > max) {
1512 virtio_error(vdev, "Looped descriptor");
1513 goto err_undo_map;
1514 }
1515
1516 rc = virtqueue_split_read_next_desc(vdev, &desc, desc_cache, max, &i);
1517 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1518
1519 if (rc == VIRTQUEUE_READ_DESC_ERROR) {
1520 goto err_undo_map;
1521 }
1522
1523 /* Now copy what we have collected and mapped */
1524 elem = virtqueue_alloc_element(sz, out_num, in_num);
1525 elem->index = head;
1526 elem->ndescs = 1;
1527 for (i = 0; i < out_num; i++) {
1528 elem->out_addr[i] = addr[i];
1529 elem->out_sg[i] = iov[i];
1530 }
1531 for (i = 0; i < in_num; i++) {
1532 elem->in_addr[i] = addr[out_num + i];
1533 elem->in_sg[i] = iov[out_num + i];
1534 }
1535
1536 vq->inuse++;
1537
1538 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
1539 done:
1540 address_space_cache_destroy(&indirect_desc_cache);
1541
1542 return elem;
1543
1544 err_undo_map:
1545 virtqueue_undo_map_desc(out_num, in_num, iov);
1546 goto done;
1547 }
1548
1549 static void *virtqueue_packed_pop(VirtQueue *vq, size_t sz)
1550 {
1551 unsigned int i, max;
1552 VRingMemoryRegionCaches *caches;
1553 MemoryRegionCache indirect_desc_cache = MEMORY_REGION_CACHE_INVALID;
1554 MemoryRegionCache *desc_cache;
1555 int64_t len;
1556 VirtIODevice *vdev = vq->vdev;
1557 VirtQueueElement *elem = NULL;
1558 unsigned out_num, in_num, elem_entries;
1559 hwaddr addr[VIRTQUEUE_MAX_SIZE];
1560 struct iovec iov[VIRTQUEUE_MAX_SIZE];
1561 VRingPackedDesc desc;
1562 uint16_t id;
1563 int rc;
1564
1565 RCU_READ_LOCK_GUARD();
1566 if (virtio_queue_packed_empty_rcu(vq)) {
1567 goto done;
1568 }
1569
1570 /* When we start there are none of either input nor output. */
1571 out_num = in_num = elem_entries = 0;
1572
1573 max = vq->vring.num;
1574
1575 if (vq->inuse >= vq->vring.num) {
1576 virtio_error(vdev, "Virtqueue size exceeded");
1577 goto done;
1578 }
1579
1580 i = vq->last_avail_idx;
1581
1582 caches = vring_get_region_caches(vq);
1583 if (!caches) {
1584 virtio_error(vdev, "Region caches not initialized");
1585 goto done;
1586 }
1587
1588 if (caches->desc.len < max * sizeof(VRingDesc)) {
1589 virtio_error(vdev, "Cannot map descriptor ring");
1590 goto done;
1591 }
1592
1593 desc_cache = &caches->desc;
1594 vring_packed_desc_read(vdev, &desc, desc_cache, i, true);
1595 id = desc.id;
1596 if (desc.flags & VRING_DESC_F_INDIRECT) {
1597 if (desc.len % sizeof(VRingPackedDesc)) {
1598 virtio_error(vdev, "Invalid size for indirect buffer table");
1599 goto done;
1600 }
1601
1602 /* loop over the indirect descriptor table */
1603 len = address_space_cache_init(&indirect_desc_cache, vdev->dma_as,
1604 desc.addr, desc.len, false);
1605 desc_cache = &indirect_desc_cache;
1606 if (len < desc.len) {
1607 virtio_error(vdev, "Cannot map indirect buffer");
1608 goto done;
1609 }
1610
1611 max = desc.len / sizeof(VRingPackedDesc);
1612 i = 0;
1613 vring_packed_desc_read(vdev, &desc, desc_cache, i, false);
1614 }
1615
1616 /* Collect all the descriptors */
1617 do {
1618 bool map_ok;
1619
1620 if (desc.flags & VRING_DESC_F_WRITE) {
1621 map_ok = virtqueue_map_desc(vdev, &in_num, addr + out_num,
1622 iov + out_num,
1623 VIRTQUEUE_MAX_SIZE - out_num, true,
1624 desc.addr, desc.len);
1625 } else {
1626 if (in_num) {
1627 virtio_error(vdev, "Incorrect order for descriptors");
1628 goto err_undo_map;
1629 }
1630 map_ok = virtqueue_map_desc(vdev, &out_num, addr, iov,
1631 VIRTQUEUE_MAX_SIZE, false,
1632 desc.addr, desc.len);
1633 }
1634 if (!map_ok) {
1635 goto err_undo_map;
1636 }
1637
1638 /* If we've got too many, that implies a descriptor loop. */
1639 if (++elem_entries > max) {
1640 virtio_error(vdev, "Looped descriptor");
1641 goto err_undo_map;
1642 }
1643
1644 rc = virtqueue_packed_read_next_desc(vq, &desc, desc_cache, max, &i,
1645 desc_cache ==
1646 &indirect_desc_cache);
1647 } while (rc == VIRTQUEUE_READ_DESC_MORE);
1648
1649 /* Now copy what we have collected and mapped */
1650 elem = virtqueue_alloc_element(sz, out_num, in_num);
1651 for (i = 0; i < out_num; i++) {
1652 elem->out_addr[i] = addr[i];
1653 elem->out_sg[i] = iov[i];
1654 }
1655 for (i = 0; i < in_num; i++) {
1656 elem->in_addr[i] = addr[out_num + i];
1657 elem->in_sg[i] = iov[out_num + i];
1658 }
1659
1660 elem->index = id;
1661 elem->ndescs = (desc_cache == &indirect_desc_cache) ? 1 : elem_entries;
1662 vq->last_avail_idx += elem->ndescs;
1663 vq->inuse += elem->ndescs;
1664
1665 if (vq->last_avail_idx >= vq->vring.num) {
1666 vq->last_avail_idx -= vq->vring.num;
1667 vq->last_avail_wrap_counter ^= 1;
1668 }
1669
1670 vq->shadow_avail_idx = vq->last_avail_idx;
1671 vq->shadow_avail_wrap_counter = vq->last_avail_wrap_counter;
1672
1673 trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num);
1674 done:
1675 address_space_cache_destroy(&indirect_desc_cache);
1676
1677 return elem;
1678
1679 err_undo_map:
1680 virtqueue_undo_map_desc(out_num, in_num, iov);
1681 goto done;
1682 }
1683
1684 void *virtqueue_pop(VirtQueue *vq, size_t sz)
1685 {
1686 if (virtio_device_disabled(vq->vdev)) {
1687 return NULL;
1688 }
1689
1690 if (virtio_vdev_has_feature(vq->vdev, VIRTIO_F_RING_PACKED)) {
1691 return virtqueue_packed_pop(vq, sz);
1692 } else {
1693 return virtqueue_split_pop(vq, sz);
1694 }
1695 }
1696
1697 static unsigned int virtqueue_packed_drop_all(VirtQueue *vq)
1698 {
1699 VRingMemoryRegionCaches *caches;
1700 MemoryRegionCache *desc_cache;
1701 unsigned int dropped = 0;
1702 VirtQueueElement elem = {};
1703 VirtIODevice *vdev = vq->vdev;
1704 VRingPackedDesc desc;
1705
1706 caches = vring_get_region_caches(vq);
1707 if (!caches) {
1708 return 0;
1709 }
1710
1711 desc_cache = &caches->desc;
1712
1713 virtio_queue_set_notification(vq, 0);
1714
1715 while (vq->inuse < vq->vring.num) {
1716 unsigned int idx = vq->last_avail_idx;
1717 /*
1718 * works similar to virtqueue_pop but does not map buffers
1719 * and does not allocate any memory.
1720 */
1721 vring_packed_desc_read(vdev, &desc, desc_cache,
1722 vq->last_avail_idx , true);
1723 if (!is_desc_avail(desc.flags, vq->last_avail_wrap_counter)) {
1724 break;
1725 }
1726 elem.index = desc.id;
1727 elem.ndescs = 1;
1728 while (virtqueue_packed_read_next_desc(vq, &desc, desc_cache,
1729 vq->vring.num, &idx, false)) {
1730 ++elem.ndescs;
1731 }
1732 /*
1733 * immediately push the element, nothing to unmap
1734 * as both in_num and out_num are set to 0.
1735 */
1736 virtqueue_push(vq, &elem, 0);
1737 dropped++;
1738 vq->last_avail_idx += elem.ndescs;
1739 if (vq->last_avail_idx >= vq->vring.num) {
1740 vq->last_avail_idx -= vq->vring.num;
1741 vq->last_avail_wrap_counter ^= 1;
1742 }
1743 }
1744
1745 return dropped;
1746 }
1747
1748 static unsigned int virtqueue_split_drop_all(VirtQueue *vq)
1749 {
1750 unsigned int dropped = 0;
1751 VirtQueueElement elem = {};
1752 VirtIODevice *vdev = vq->vdev;
1753 bool fEventIdx = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1754
1755 while (!virtio_queue_empty(vq) && vq->inuse < vq->vring.num) {
1756 /* works similar to virtqueue_pop but does not map buffers
1757 * and does not allocate any memory */
1758 smp_rmb();
1759 if (!virtqueue_get_head(vq, vq->last_avail_idx, &elem.index)) {
1760 break;
1761 }
1762 vq->inuse++;
1763 vq->last_avail_idx++;
1764 if (fEventIdx) {
1765 vring_set_avail_event(vq, vq->last_avail_idx);
1766 }
1767 /* immediately push the element, nothing to unmap
1768 * as both in_num and out_num are set to 0 */
1769 virtqueue_push(vq, &elem, 0);
1770 dropped++;
1771 }
1772
1773 return dropped;
1774 }
1775
1776 /* virtqueue_drop_all:
1777 * @vq: The #VirtQueue
1778 * Drops all queued buffers and indicates them to the guest
1779 * as if they are done. Useful when buffers can not be
1780 * processed but must be returned to the guest.
1781 */
1782 unsigned int virtqueue_drop_all(VirtQueue *vq)
1783 {
1784 struct VirtIODevice *vdev = vq->vdev;
1785
1786 if (virtio_device_disabled(vq->vdev)) {
1787 return 0;
1788 }
1789
1790 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
1791 return virtqueue_packed_drop_all(vq);
1792 } else {
1793 return virtqueue_split_drop_all(vq);
1794 }
1795 }
1796
1797 /* Reading and writing a structure directly to QEMUFile is *awful*, but
1798 * it is what QEMU has always done by mistake. We can change it sooner
1799 * or later by bumping the version number of the affected vm states.
1800 * In the meanwhile, since the in-memory layout of VirtQueueElement
1801 * has changed, we need to marshal to and from the layout that was
1802 * used before the change.
1803 */
1804 typedef struct VirtQueueElementOld {
1805 unsigned int index;
1806 unsigned int out_num;
1807 unsigned int in_num;
1808 hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
1809 hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
1810 struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
1811 struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
1812 } VirtQueueElementOld;
1813
1814 void *qemu_get_virtqueue_element(VirtIODevice *vdev, QEMUFile *f, size_t sz)
1815 {
1816 VirtQueueElement *elem;
1817 VirtQueueElementOld data;
1818 int i;
1819
1820 qemu_get_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1821
1822 /* TODO: teach all callers that this can fail, and return failure instead
1823 * of asserting here.
1824 * This is just one thing (there are probably more) that must be
1825 * fixed before we can allow NDEBUG compilation.
1826 */
1827 assert(ARRAY_SIZE(data.in_addr) >= data.in_num);
1828 assert(ARRAY_SIZE(data.out_addr) >= data.out_num);
1829
1830 elem = virtqueue_alloc_element(sz, data.out_num, data.in_num);
1831 elem->index = data.index;
1832
1833 for (i = 0; i < elem->in_num; i++) {
1834 elem->in_addr[i] = data.in_addr[i];
1835 }
1836
1837 for (i = 0; i < elem->out_num; i++) {
1838 elem->out_addr[i] = data.out_addr[i];
1839 }
1840
1841 for (i = 0; i < elem->in_num; i++) {
1842 /* Base is overwritten by virtqueue_map. */
1843 elem->in_sg[i].iov_base = 0;
1844 elem->in_sg[i].iov_len = data.in_sg[i].iov_len;
1845 }
1846
1847 for (i = 0; i < elem->out_num; i++) {
1848 /* Base is overwritten by virtqueue_map. */
1849 elem->out_sg[i].iov_base = 0;
1850 elem->out_sg[i].iov_len = data.out_sg[i].iov_len;
1851 }
1852
1853 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
1854 qemu_get_be32s(f, &elem->ndescs);
1855 }
1856
1857 virtqueue_map(vdev, elem);
1858 return elem;
1859 }
1860
1861 void qemu_put_virtqueue_element(VirtIODevice *vdev, QEMUFile *f,
1862 VirtQueueElement *elem)
1863 {
1864 VirtQueueElementOld data;
1865 int i;
1866
1867 memset(&data, 0, sizeof(data));
1868 data.index = elem->index;
1869 data.in_num = elem->in_num;
1870 data.out_num = elem->out_num;
1871
1872 for (i = 0; i < elem->in_num; i++) {
1873 data.in_addr[i] = elem->in_addr[i];
1874 }
1875
1876 for (i = 0; i < elem->out_num; i++) {
1877 data.out_addr[i] = elem->out_addr[i];
1878 }
1879
1880 for (i = 0; i < elem->in_num; i++) {
1881 /* Base is overwritten by virtqueue_map when loading. Do not
1882 * save it, as it would leak the QEMU address space layout. */
1883 data.in_sg[i].iov_len = elem->in_sg[i].iov_len;
1884 }
1885
1886 for (i = 0; i < elem->out_num; i++) {
1887 /* Do not save iov_base as above. */
1888 data.out_sg[i].iov_len = elem->out_sg[i].iov_len;
1889 }
1890
1891 if (virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
1892 qemu_put_be32s(f, &elem->ndescs);
1893 }
1894
1895 qemu_put_buffer(f, (uint8_t *)&data, sizeof(VirtQueueElementOld));
1896 }
1897
1898 /* virtio device */
1899 static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector)
1900 {
1901 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
1902 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
1903
1904 if (virtio_device_disabled(vdev)) {
1905 return;
1906 }
1907
1908 if (k->notify) {
1909 k->notify(qbus->parent, vector);
1910 }
1911 }
1912
1913 void virtio_update_irq(VirtIODevice *vdev)
1914 {
1915 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
1916 }
1917
1918 static int virtio_validate_features(VirtIODevice *vdev)
1919 {
1920 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1921
1922 if (virtio_host_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM) &&
1923 !virtio_vdev_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM)) {
1924 return -EFAULT;
1925 }
1926
1927 if (k->validate_features) {
1928 return k->validate_features(vdev);
1929 } else {
1930 return 0;
1931 }
1932 }
1933
1934 int virtio_set_status(VirtIODevice *vdev, uint8_t val)
1935 {
1936 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1937 trace_virtio_set_status(vdev, val);
1938
1939 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
1940 if (!(vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) &&
1941 val & VIRTIO_CONFIG_S_FEATURES_OK) {
1942 int ret = virtio_validate_features(vdev);
1943
1944 if (ret) {
1945 return ret;
1946 }
1947 }
1948 }
1949
1950 if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) !=
1951 (val & VIRTIO_CONFIG_S_DRIVER_OK)) {
1952 virtio_set_started(vdev, val & VIRTIO_CONFIG_S_DRIVER_OK);
1953 }
1954
1955 if (k->set_status) {
1956 k->set_status(vdev, val);
1957 }
1958 vdev->status = val;
1959
1960 return 0;
1961 }
1962
1963 static enum virtio_device_endian virtio_default_endian(void)
1964 {
1965 if (target_words_bigendian()) {
1966 return VIRTIO_DEVICE_ENDIAN_BIG;
1967 } else {
1968 return VIRTIO_DEVICE_ENDIAN_LITTLE;
1969 }
1970 }
1971
1972 static enum virtio_device_endian virtio_current_cpu_endian(void)
1973 {
1974 CPUClass *cc = CPU_GET_CLASS(current_cpu);
1975
1976 if (cc->virtio_is_big_endian(current_cpu)) {
1977 return VIRTIO_DEVICE_ENDIAN_BIG;
1978 } else {
1979 return VIRTIO_DEVICE_ENDIAN_LITTLE;
1980 }
1981 }
1982
1983 void virtio_reset(void *opaque)
1984 {
1985 VirtIODevice *vdev = opaque;
1986 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
1987 int i;
1988
1989 virtio_set_status(vdev, 0);
1990 if (current_cpu) {
1991 /* Guest initiated reset */
1992 vdev->device_endian = virtio_current_cpu_endian();
1993 } else {
1994 /* System reset */
1995 vdev->device_endian = virtio_default_endian();
1996 }
1997
1998 if (k->reset) {
1999 k->reset(vdev);
2000 }
2001
2002 vdev->start_on_kick = false;
2003 vdev->started = false;
2004 vdev->broken = false;
2005 vdev->guest_features = 0;
2006 vdev->queue_sel = 0;
2007 vdev->status = 0;
2008 vdev->disabled = false;
2009 atomic_set(&vdev->isr, 0);
2010 vdev->config_vector = VIRTIO_NO_VECTOR;
2011 virtio_notify_vector(vdev, vdev->config_vector);
2012
2013 for(i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2014 vdev->vq[i].vring.desc = 0;
2015 vdev->vq[i].vring.avail = 0;
2016 vdev->vq[i].vring.used = 0;
2017 vdev->vq[i].last_avail_idx = 0;
2018 vdev->vq[i].shadow_avail_idx = 0;
2019 vdev->vq[i].used_idx = 0;
2020 vdev->vq[i].last_avail_wrap_counter = true;
2021 vdev->vq[i].shadow_avail_wrap_counter = true;
2022 vdev->vq[i].used_wrap_counter = true;
2023 virtio_queue_set_vector(vdev, i, VIRTIO_NO_VECTOR);
2024 vdev->vq[i].signalled_used = 0;
2025 vdev->vq[i].signalled_used_valid = false;
2026 vdev->vq[i].notification = true;
2027 vdev->vq[i].vring.num = vdev->vq[i].vring.num_default;
2028 vdev->vq[i].inuse = 0;
2029 virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
2030 }
2031 }
2032
2033 uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr)
2034 {
2035 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2036 uint8_t val;
2037
2038 if (addr + sizeof(val) > vdev->config_len) {
2039 return (uint32_t)-1;
2040 }
2041
2042 k->get_config(vdev, vdev->config);
2043
2044 val = ldub_p(vdev->config + addr);
2045 return val;
2046 }
2047
2048 uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr)
2049 {
2050 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2051 uint16_t val;
2052
2053 if (addr + sizeof(val) > vdev->config_len) {
2054 return (uint32_t)-1;
2055 }
2056
2057 k->get_config(vdev, vdev->config);
2058
2059 val = lduw_p(vdev->config + addr);
2060 return val;
2061 }
2062
2063 uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr)
2064 {
2065 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2066 uint32_t val;
2067
2068 if (addr + sizeof(val) > vdev->config_len) {
2069 return (uint32_t)-1;
2070 }
2071
2072 k->get_config(vdev, vdev->config);
2073
2074 val = ldl_p(vdev->config + addr);
2075 return val;
2076 }
2077
2078 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data)
2079 {
2080 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2081 uint8_t val = data;
2082
2083 if (addr + sizeof(val) > vdev->config_len) {
2084 return;
2085 }
2086
2087 stb_p(vdev->config + addr, val);
2088
2089 if (k->set_config) {
2090 k->set_config(vdev, vdev->config);
2091 }
2092 }
2093
2094 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data)
2095 {
2096 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2097 uint16_t val = data;
2098
2099 if (addr + sizeof(val) > vdev->config_len) {
2100 return;
2101 }
2102
2103 stw_p(vdev->config + addr, val);
2104
2105 if (k->set_config) {
2106 k->set_config(vdev, vdev->config);
2107 }
2108 }
2109
2110 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
2111 {
2112 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2113 uint32_t val = data;
2114
2115 if (addr + sizeof(val) > vdev->config_len) {
2116 return;
2117 }
2118
2119 stl_p(vdev->config + addr, val);
2120
2121 if (k->set_config) {
2122 k->set_config(vdev, vdev->config);
2123 }
2124 }
2125
2126 uint32_t virtio_config_modern_readb(VirtIODevice *vdev, uint32_t addr)
2127 {
2128 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2129 uint8_t val;
2130
2131 if (addr + sizeof(val) > vdev->config_len) {
2132 return (uint32_t)-1;
2133 }
2134
2135 k->get_config(vdev, vdev->config);
2136
2137 val = ldub_p(vdev->config + addr);
2138 return val;
2139 }
2140
2141 uint32_t virtio_config_modern_readw(VirtIODevice *vdev, uint32_t addr)
2142 {
2143 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2144 uint16_t val;
2145
2146 if (addr + sizeof(val) > vdev->config_len) {
2147 return (uint32_t)-1;
2148 }
2149
2150 k->get_config(vdev, vdev->config);
2151
2152 val = lduw_le_p(vdev->config + addr);
2153 return val;
2154 }
2155
2156 uint32_t virtio_config_modern_readl(VirtIODevice *vdev, uint32_t addr)
2157 {
2158 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2159 uint32_t val;
2160
2161 if (addr + sizeof(val) > vdev->config_len) {
2162 return (uint32_t)-1;
2163 }
2164
2165 k->get_config(vdev, vdev->config);
2166
2167 val = ldl_le_p(vdev->config + addr);
2168 return val;
2169 }
2170
2171 void virtio_config_modern_writeb(VirtIODevice *vdev,
2172 uint32_t addr, uint32_t data)
2173 {
2174 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2175 uint8_t val = data;
2176
2177 if (addr + sizeof(val) > vdev->config_len) {
2178 return;
2179 }
2180
2181 stb_p(vdev->config + addr, val);
2182
2183 if (k->set_config) {
2184 k->set_config(vdev, vdev->config);
2185 }
2186 }
2187
2188 void virtio_config_modern_writew(VirtIODevice *vdev,
2189 uint32_t addr, uint32_t data)
2190 {
2191 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2192 uint16_t val = data;
2193
2194 if (addr + sizeof(val) > vdev->config_len) {
2195 return;
2196 }
2197
2198 stw_le_p(vdev->config + addr, val);
2199
2200 if (k->set_config) {
2201 k->set_config(vdev, vdev->config);
2202 }
2203 }
2204
2205 void virtio_config_modern_writel(VirtIODevice *vdev,
2206 uint32_t addr, uint32_t data)
2207 {
2208 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2209 uint32_t val = data;
2210
2211 if (addr + sizeof(val) > vdev->config_len) {
2212 return;
2213 }
2214
2215 stl_le_p(vdev->config + addr, val);
2216
2217 if (k->set_config) {
2218 k->set_config(vdev, vdev->config);
2219 }
2220 }
2221
2222 void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
2223 {
2224 if (!vdev->vq[n].vring.num) {
2225 return;
2226 }
2227 vdev->vq[n].vring.desc = addr;
2228 virtio_queue_update_rings(vdev, n);
2229 }
2230
2231 hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
2232 {
2233 return vdev->vq[n].vring.desc;
2234 }
2235
2236 void virtio_queue_set_rings(VirtIODevice *vdev, int n, hwaddr desc,
2237 hwaddr avail, hwaddr used)
2238 {
2239 if (!vdev->vq[n].vring.num) {
2240 return;
2241 }
2242 vdev->vq[n].vring.desc = desc;
2243 vdev->vq[n].vring.avail = avail;
2244 vdev->vq[n].vring.used = used;
2245 virtio_init_region_cache(vdev, n);
2246 }
2247
2248 void virtio_queue_set_num(VirtIODevice *vdev, int n, int num)
2249 {
2250 /* Don't allow guest to flip queue between existent and
2251 * nonexistent states, or to set it to an invalid size.
2252 */
2253 if (!!num != !!vdev->vq[n].vring.num ||
2254 num > VIRTQUEUE_MAX_SIZE ||
2255 num < 0) {
2256 return;
2257 }
2258 vdev->vq[n].vring.num = num;
2259 }
2260
2261 VirtQueue *virtio_vector_first_queue(VirtIODevice *vdev, uint16_t vector)
2262 {
2263 return QLIST_FIRST(&vdev->vector_queues[vector]);
2264 }
2265
2266 VirtQueue *virtio_vector_next_queue(VirtQueue *vq)
2267 {
2268 return QLIST_NEXT(vq, node);
2269 }
2270
2271 int virtio_queue_get_num(VirtIODevice *vdev, int n)
2272 {
2273 return vdev->vq[n].vring.num;
2274 }
2275
2276 int virtio_queue_get_max_num(VirtIODevice *vdev, int n)
2277 {
2278 return vdev->vq[n].vring.num_default;
2279 }
2280
2281 int virtio_get_num_queues(VirtIODevice *vdev)
2282 {
2283 int i;
2284
2285 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2286 if (!virtio_queue_get_num(vdev, i)) {
2287 break;
2288 }
2289 }
2290
2291 return i;
2292 }
2293
2294 void virtio_queue_set_align(VirtIODevice *vdev, int n, int align)
2295 {
2296 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2297 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2298
2299 /* virtio-1 compliant devices cannot change the alignment */
2300 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2301 error_report("tried to modify queue alignment for virtio-1 device");
2302 return;
2303 }
2304 /* Check that the transport told us it was going to do this
2305 * (so a buggy transport will immediately assert rather than
2306 * silently failing to migrate this state)
2307 */
2308 assert(k->has_variable_vring_alignment);
2309
2310 if (align) {
2311 vdev->vq[n].vring.align = align;
2312 virtio_queue_update_rings(vdev, n);
2313 }
2314 }
2315
2316 static bool virtio_queue_notify_aio_vq(VirtQueue *vq)
2317 {
2318 bool ret = false;
2319
2320 if (vq->vring.desc && vq->handle_aio_output) {
2321 VirtIODevice *vdev = vq->vdev;
2322
2323 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2324 ret = vq->handle_aio_output(vdev, vq);
2325
2326 if (unlikely(vdev->start_on_kick)) {
2327 virtio_set_started(vdev, true);
2328 }
2329 }
2330
2331 return ret;
2332 }
2333
2334 static void virtio_queue_notify_vq(VirtQueue *vq)
2335 {
2336 if (vq->vring.desc && vq->handle_output) {
2337 VirtIODevice *vdev = vq->vdev;
2338
2339 if (unlikely(vdev->broken)) {
2340 return;
2341 }
2342
2343 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2344 vq->handle_output(vdev, vq);
2345
2346 if (unlikely(vdev->start_on_kick)) {
2347 virtio_set_started(vdev, true);
2348 }
2349 }
2350 }
2351
2352 void virtio_queue_notify(VirtIODevice *vdev, int n)
2353 {
2354 VirtQueue *vq = &vdev->vq[n];
2355
2356 if (unlikely(!vq->vring.desc || vdev->broken)) {
2357 return;
2358 }
2359
2360 trace_virtio_queue_notify(vdev, vq - vdev->vq, vq);
2361 if (vq->host_notifier_enabled) {
2362 event_notifier_set(&vq->host_notifier);
2363 } else if (vq->handle_output) {
2364 vq->handle_output(vdev, vq);
2365
2366 if (unlikely(vdev->start_on_kick)) {
2367 virtio_set_started(vdev, true);
2368 }
2369 }
2370 }
2371
2372 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n)
2373 {
2374 return n < VIRTIO_QUEUE_MAX ? vdev->vq[n].vector :
2375 VIRTIO_NO_VECTOR;
2376 }
2377
2378 void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector)
2379 {
2380 VirtQueue *vq = &vdev->vq[n];
2381
2382 if (n < VIRTIO_QUEUE_MAX) {
2383 if (vdev->vector_queues &&
2384 vdev->vq[n].vector != VIRTIO_NO_VECTOR) {
2385 QLIST_REMOVE(vq, node);
2386 }
2387 vdev->vq[n].vector = vector;
2388 if (vdev->vector_queues &&
2389 vector != VIRTIO_NO_VECTOR) {
2390 QLIST_INSERT_HEAD(&vdev->vector_queues[vector], vq, node);
2391 }
2392 }
2393 }
2394
2395 VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size,
2396 VirtIOHandleOutput handle_output)
2397 {
2398 int i;
2399
2400 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2401 if (vdev->vq[i].vring.num == 0)
2402 break;
2403 }
2404
2405 if (i == VIRTIO_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE)
2406 abort();
2407
2408 vdev->vq[i].vring.num = queue_size;
2409 vdev->vq[i].vring.num_default = queue_size;
2410 vdev->vq[i].vring.align = VIRTIO_PCI_VRING_ALIGN;
2411 vdev->vq[i].handle_output = handle_output;
2412 vdev->vq[i].handle_aio_output = NULL;
2413 vdev->vq[i].used_elems = g_malloc0(sizeof(VirtQueueElement) *
2414 queue_size);
2415
2416 return &vdev->vq[i];
2417 }
2418
2419 void virtio_delete_queue(VirtQueue *vq)
2420 {
2421 vq->vring.num = 0;
2422 vq->vring.num_default = 0;
2423 vq->handle_output = NULL;
2424 vq->handle_aio_output = NULL;
2425 g_free(vq->used_elems);
2426 vq->used_elems = NULL;
2427 virtio_virtqueue_reset_region_cache(vq);
2428 }
2429
2430 void virtio_del_queue(VirtIODevice *vdev, int n)
2431 {
2432 if (n < 0 || n >= VIRTIO_QUEUE_MAX) {
2433 abort();
2434 }
2435
2436 virtio_delete_queue(&vdev->vq[n]);
2437 }
2438
2439 static void virtio_set_isr(VirtIODevice *vdev, int value)
2440 {
2441 uint8_t old = atomic_read(&vdev->isr);
2442
2443 /* Do not write ISR if it does not change, so that its cacheline remains
2444 * shared in the common case where the guest does not read it.
2445 */
2446 if ((old & value) != value) {
2447 atomic_or(&vdev->isr, value);
2448 }
2449 }
2450
2451 static bool virtio_split_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2452 {
2453 uint16_t old, new;
2454 bool v;
2455 /* We need to expose used array entries before checking used event. */
2456 smp_mb();
2457 /* Always notify when queue is empty (when feature acknowledge) */
2458 if (virtio_vdev_has_feature(vdev, VIRTIO_F_NOTIFY_ON_EMPTY) &&
2459 !vq->inuse && virtio_queue_empty(vq)) {
2460 return true;
2461 }
2462
2463 if (!virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
2464 return !(vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT);
2465 }
2466
2467 v = vq->signalled_used_valid;
2468 vq->signalled_used_valid = true;
2469 old = vq->signalled_used;
2470 new = vq->signalled_used = vq->used_idx;
2471 return !v || vring_need_event(vring_get_used_event(vq), new, old);
2472 }
2473
2474 static bool vring_packed_need_event(VirtQueue *vq, bool wrap,
2475 uint16_t off_wrap, uint16_t new,
2476 uint16_t old)
2477 {
2478 int off = off_wrap & ~(1 << 15);
2479
2480 if (wrap != off_wrap >> 15) {
2481 off -= vq->vring.num;
2482 }
2483
2484 return vring_need_event(off, new, old);
2485 }
2486
2487 static bool virtio_packed_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2488 {
2489 VRingPackedDescEvent e;
2490 uint16_t old, new;
2491 bool v;
2492 VRingMemoryRegionCaches *caches;
2493
2494 caches = vring_get_region_caches(vq);
2495 if (!caches) {
2496 return false;
2497 }
2498
2499 vring_packed_event_read(vdev, &caches->avail, &e);
2500
2501 old = vq->signalled_used;
2502 new = vq->signalled_used = vq->used_idx;
2503 v = vq->signalled_used_valid;
2504 vq->signalled_used_valid = true;
2505
2506 if (e.flags == VRING_PACKED_EVENT_FLAG_DISABLE) {
2507 return false;
2508 } else if (e.flags == VRING_PACKED_EVENT_FLAG_ENABLE) {
2509 return true;
2510 }
2511
2512 return !v || vring_packed_need_event(vq, vq->used_wrap_counter,
2513 e.off_wrap, new, old);
2514 }
2515
2516 /* Called within rcu_read_lock(). */
2517 static bool virtio_should_notify(VirtIODevice *vdev, VirtQueue *vq)
2518 {
2519 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
2520 return virtio_packed_should_notify(vdev, vq);
2521 } else {
2522 return virtio_split_should_notify(vdev, vq);
2523 }
2524 }
2525
2526 void virtio_notify_irqfd(VirtIODevice *vdev, VirtQueue *vq)
2527 {
2528 WITH_RCU_READ_LOCK_GUARD() {
2529 if (!virtio_should_notify(vdev, vq)) {
2530 return;
2531 }
2532 }
2533
2534 trace_virtio_notify_irqfd(vdev, vq);
2535
2536 /*
2537 * virtio spec 1.0 says ISR bit 0 should be ignored with MSI, but
2538 * windows drivers included in virtio-win 1.8.0 (circa 2015) are
2539 * incorrectly polling this bit during crashdump and hibernation
2540 * in MSI mode, causing a hang if this bit is never updated.
2541 * Recent releases of Windows do not really shut down, but rather
2542 * log out and hibernate to make the next startup faster. Hence,
2543 * this manifested as a more serious hang during shutdown with
2544 *
2545 * Next driver release from 2016 fixed this problem, so working around it
2546 * is not a must, but it's easy to do so let's do it here.
2547 *
2548 * Note: it's safe to update ISR from any thread as it was switched
2549 * to an atomic operation.
2550 */
2551 virtio_set_isr(vq->vdev, 0x1);
2552 event_notifier_set(&vq->guest_notifier);
2553 }
2554
2555 static void virtio_irq(VirtQueue *vq)
2556 {
2557 virtio_set_isr(vq->vdev, 0x1);
2558 virtio_notify_vector(vq->vdev, vq->vector);
2559 }
2560
2561 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq)
2562 {
2563 WITH_RCU_READ_LOCK_GUARD() {
2564 if (!virtio_should_notify(vdev, vq)) {
2565 return;
2566 }
2567 }
2568
2569 trace_virtio_notify(vdev, vq);
2570 virtio_irq(vq);
2571 }
2572
2573 void virtio_notify_config(VirtIODevice *vdev)
2574 {
2575 if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))
2576 return;
2577
2578 virtio_set_isr(vdev, 0x3);
2579 vdev->generation++;
2580 virtio_notify_vector(vdev, vdev->config_vector);
2581 }
2582
2583 static bool virtio_device_endian_needed(void *opaque)
2584 {
2585 VirtIODevice *vdev = opaque;
2586
2587 assert(vdev->device_endian != VIRTIO_DEVICE_ENDIAN_UNKNOWN);
2588 if (!virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2589 return vdev->device_endian != virtio_default_endian();
2590 }
2591 /* Devices conforming to VIRTIO 1.0 or later are always LE. */
2592 return vdev->device_endian != VIRTIO_DEVICE_ENDIAN_LITTLE;
2593 }
2594
2595 static bool virtio_64bit_features_needed(void *opaque)
2596 {
2597 VirtIODevice *vdev = opaque;
2598
2599 return (vdev->host_features >> 32) != 0;
2600 }
2601
2602 static bool virtio_virtqueue_needed(void *opaque)
2603 {
2604 VirtIODevice *vdev = opaque;
2605
2606 return virtio_host_has_feature(vdev, VIRTIO_F_VERSION_1);
2607 }
2608
2609 static bool virtio_packed_virtqueue_needed(void *opaque)
2610 {
2611 VirtIODevice *vdev = opaque;
2612
2613 return virtio_host_has_feature(vdev, VIRTIO_F_RING_PACKED);
2614 }
2615
2616 static bool virtio_ringsize_needed(void *opaque)
2617 {
2618 VirtIODevice *vdev = opaque;
2619 int i;
2620
2621 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2622 if (vdev->vq[i].vring.num != vdev->vq[i].vring.num_default) {
2623 return true;
2624 }
2625 }
2626 return false;
2627 }
2628
2629 static bool virtio_extra_state_needed(void *opaque)
2630 {
2631 VirtIODevice *vdev = opaque;
2632 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2633 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2634
2635 return k->has_extra_state &&
2636 k->has_extra_state(qbus->parent);
2637 }
2638
2639 static bool virtio_broken_needed(void *opaque)
2640 {
2641 VirtIODevice *vdev = opaque;
2642
2643 return vdev->broken;
2644 }
2645
2646 static bool virtio_started_needed(void *opaque)
2647 {
2648 VirtIODevice *vdev = opaque;
2649
2650 return vdev->started;
2651 }
2652
2653 static bool virtio_disabled_needed(void *opaque)
2654 {
2655 VirtIODevice *vdev = opaque;
2656
2657 return vdev->disabled;
2658 }
2659
2660 static const VMStateDescription vmstate_virtqueue = {
2661 .name = "virtqueue_state",
2662 .version_id = 1,
2663 .minimum_version_id = 1,
2664 .fields = (VMStateField[]) {
2665 VMSTATE_UINT64(vring.avail, struct VirtQueue),
2666 VMSTATE_UINT64(vring.used, struct VirtQueue),
2667 VMSTATE_END_OF_LIST()
2668 }
2669 };
2670
2671 static const VMStateDescription vmstate_packed_virtqueue = {
2672 .name = "packed_virtqueue_state",
2673 .version_id = 1,
2674 .minimum_version_id = 1,
2675 .fields = (VMStateField[]) {
2676 VMSTATE_UINT16(last_avail_idx, struct VirtQueue),
2677 VMSTATE_BOOL(last_avail_wrap_counter, struct VirtQueue),
2678 VMSTATE_UINT16(used_idx, struct VirtQueue),
2679 VMSTATE_BOOL(used_wrap_counter, struct VirtQueue),
2680 VMSTATE_UINT32(inuse, struct VirtQueue),
2681 VMSTATE_END_OF_LIST()
2682 }
2683 };
2684
2685 static const VMStateDescription vmstate_virtio_virtqueues = {
2686 .name = "virtio/virtqueues",
2687 .version_id = 1,
2688 .minimum_version_id = 1,
2689 .needed = &virtio_virtqueue_needed,
2690 .fields = (VMStateField[]) {
2691 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2692 VIRTIO_QUEUE_MAX, 0, vmstate_virtqueue, VirtQueue),
2693 VMSTATE_END_OF_LIST()
2694 }
2695 };
2696
2697 static const VMStateDescription vmstate_virtio_packed_virtqueues = {
2698 .name = "virtio/packed_virtqueues",
2699 .version_id = 1,
2700 .minimum_version_id = 1,
2701 .needed = &virtio_packed_virtqueue_needed,
2702 .fields = (VMStateField[]) {
2703 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2704 VIRTIO_QUEUE_MAX, 0, vmstate_packed_virtqueue, VirtQueue),
2705 VMSTATE_END_OF_LIST()
2706 }
2707 };
2708
2709 static const VMStateDescription vmstate_ringsize = {
2710 .name = "ringsize_state",
2711 .version_id = 1,
2712 .minimum_version_id = 1,
2713 .fields = (VMStateField[]) {
2714 VMSTATE_UINT32(vring.num_default, struct VirtQueue),
2715 VMSTATE_END_OF_LIST()
2716 }
2717 };
2718
2719 static const VMStateDescription vmstate_virtio_ringsize = {
2720 .name = "virtio/ringsize",
2721 .version_id = 1,
2722 .minimum_version_id = 1,
2723 .needed = &virtio_ringsize_needed,
2724 .fields = (VMStateField[]) {
2725 VMSTATE_STRUCT_VARRAY_POINTER_KNOWN(vq, struct VirtIODevice,
2726 VIRTIO_QUEUE_MAX, 0, vmstate_ringsize, VirtQueue),
2727 VMSTATE_END_OF_LIST()
2728 }
2729 };
2730
2731 static int get_extra_state(QEMUFile *f, void *pv, size_t size,
2732 const VMStateField *field)
2733 {
2734 VirtIODevice *vdev = pv;
2735 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2736 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2737
2738 if (!k->load_extra_state) {
2739 return -1;
2740 } else {
2741 return k->load_extra_state(qbus->parent, f);
2742 }
2743 }
2744
2745 static int put_extra_state(QEMUFile *f, void *pv, size_t size,
2746 const VMStateField *field, QJSON *vmdesc)
2747 {
2748 VirtIODevice *vdev = pv;
2749 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2750 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2751
2752 k->save_extra_state(qbus->parent, f);
2753 return 0;
2754 }
2755
2756 static const VMStateInfo vmstate_info_extra_state = {
2757 .name = "virtqueue_extra_state",
2758 .get = get_extra_state,
2759 .put = put_extra_state,
2760 };
2761
2762 static const VMStateDescription vmstate_virtio_extra_state = {
2763 .name = "virtio/extra_state",
2764 .version_id = 1,
2765 .minimum_version_id = 1,
2766 .needed = &virtio_extra_state_needed,
2767 .fields = (VMStateField[]) {
2768 {
2769 .name = "extra_state",
2770 .version_id = 0,
2771 .field_exists = NULL,
2772 .size = 0,
2773 .info = &vmstate_info_extra_state,
2774 .flags = VMS_SINGLE,
2775 .offset = 0,
2776 },
2777 VMSTATE_END_OF_LIST()
2778 }
2779 };
2780
2781 static const VMStateDescription vmstate_virtio_device_endian = {
2782 .name = "virtio/device_endian",
2783 .version_id = 1,
2784 .minimum_version_id = 1,
2785 .needed = &virtio_device_endian_needed,
2786 .fields = (VMStateField[]) {
2787 VMSTATE_UINT8(device_endian, VirtIODevice),
2788 VMSTATE_END_OF_LIST()
2789 }
2790 };
2791
2792 static const VMStateDescription vmstate_virtio_64bit_features = {
2793 .name = "virtio/64bit_features",
2794 .version_id = 1,
2795 .minimum_version_id = 1,
2796 .needed = &virtio_64bit_features_needed,
2797 .fields = (VMStateField[]) {
2798 VMSTATE_UINT64(guest_features, VirtIODevice),
2799 VMSTATE_END_OF_LIST()
2800 }
2801 };
2802
2803 static const VMStateDescription vmstate_virtio_broken = {
2804 .name = "virtio/broken",
2805 .version_id = 1,
2806 .minimum_version_id = 1,
2807 .needed = &virtio_broken_needed,
2808 .fields = (VMStateField[]) {
2809 VMSTATE_BOOL(broken, VirtIODevice),
2810 VMSTATE_END_OF_LIST()
2811 }
2812 };
2813
2814 static const VMStateDescription vmstate_virtio_started = {
2815 .name = "virtio/started",
2816 .version_id = 1,
2817 .minimum_version_id = 1,
2818 .needed = &virtio_started_needed,
2819 .fields = (VMStateField[]) {
2820 VMSTATE_BOOL(started, VirtIODevice),
2821 VMSTATE_END_OF_LIST()
2822 }
2823 };
2824
2825 static const VMStateDescription vmstate_virtio_disabled = {
2826 .name = "virtio/disabled",
2827 .version_id = 1,
2828 .minimum_version_id = 1,
2829 .needed = &virtio_disabled_needed,
2830 .fields = (VMStateField[]) {
2831 VMSTATE_BOOL(disabled, VirtIODevice),
2832 VMSTATE_END_OF_LIST()
2833 }
2834 };
2835
2836 static const VMStateDescription vmstate_virtio = {
2837 .name = "virtio",
2838 .version_id = 1,
2839 .minimum_version_id = 1,
2840 .minimum_version_id_old = 1,
2841 .fields = (VMStateField[]) {
2842 VMSTATE_END_OF_LIST()
2843 },
2844 .subsections = (const VMStateDescription*[]) {
2845 &vmstate_virtio_device_endian,
2846 &vmstate_virtio_64bit_features,
2847 &vmstate_virtio_virtqueues,
2848 &vmstate_virtio_ringsize,
2849 &vmstate_virtio_broken,
2850 &vmstate_virtio_extra_state,
2851 &vmstate_virtio_started,
2852 &vmstate_virtio_packed_virtqueues,
2853 &vmstate_virtio_disabled,
2854 NULL
2855 }
2856 };
2857
2858 int virtio_save(VirtIODevice *vdev, QEMUFile *f)
2859 {
2860 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
2861 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
2862 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
2863 uint32_t guest_features_lo = (vdev->guest_features & 0xffffffff);
2864 int i;
2865
2866 if (k->save_config) {
2867 k->save_config(qbus->parent, f);
2868 }
2869
2870 qemu_put_8s(f, &vdev->status);
2871 qemu_put_8s(f, &vdev->isr);
2872 qemu_put_be16s(f, &vdev->queue_sel);
2873 qemu_put_be32s(f, &guest_features_lo);
2874 qemu_put_be32(f, vdev->config_len);
2875 qemu_put_buffer(f, vdev->config, vdev->config_len);
2876
2877 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2878 if (vdev->vq[i].vring.num == 0)
2879 break;
2880 }
2881
2882 qemu_put_be32(f, i);
2883
2884 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2885 if (vdev->vq[i].vring.num == 0)
2886 break;
2887
2888 qemu_put_be32(f, vdev->vq[i].vring.num);
2889 if (k->has_variable_vring_alignment) {
2890 qemu_put_be32(f, vdev->vq[i].vring.align);
2891 }
2892 /*
2893 * Save desc now, the rest of the ring addresses are saved in
2894 * subsections for VIRTIO-1 devices.
2895 */
2896 qemu_put_be64(f, vdev->vq[i].vring.desc);
2897 qemu_put_be16s(f, &vdev->vq[i].last_avail_idx);
2898 if (k->save_queue) {
2899 k->save_queue(qbus->parent, i, f);
2900 }
2901 }
2902
2903 if (vdc->save != NULL) {
2904 vdc->save(vdev, f);
2905 }
2906
2907 if (vdc->vmsd) {
2908 int ret = vmstate_save_state(f, vdc->vmsd, vdev, NULL);
2909 if (ret) {
2910 return ret;
2911 }
2912 }
2913
2914 /* Subsections */
2915 return vmstate_save_state(f, &vmstate_virtio, vdev, NULL);
2916 }
2917
2918 /* A wrapper for use as a VMState .put function */
2919 static int virtio_device_put(QEMUFile *f, void *opaque, size_t size,
2920 const VMStateField *field, QJSON *vmdesc)
2921 {
2922 return virtio_save(VIRTIO_DEVICE(opaque), f);
2923 }
2924
2925 /* A wrapper for use as a VMState .get function */
2926 static int virtio_device_get(QEMUFile *f, void *opaque, size_t size,
2927 const VMStateField *field)
2928 {
2929 VirtIODevice *vdev = VIRTIO_DEVICE(opaque);
2930 DeviceClass *dc = DEVICE_CLASS(VIRTIO_DEVICE_GET_CLASS(vdev));
2931
2932 return virtio_load(vdev, f, dc->vmsd->version_id);
2933 }
2934
2935 const VMStateInfo virtio_vmstate_info = {
2936 .name = "virtio",
2937 .get = virtio_device_get,
2938 .put = virtio_device_put,
2939 };
2940
2941 static int virtio_set_features_nocheck(VirtIODevice *vdev, uint64_t val)
2942 {
2943 VirtioDeviceClass *k = VIRTIO_DEVICE_GET_CLASS(vdev);
2944 bool bad = (val & ~(vdev->host_features)) != 0;
2945
2946 val &= vdev->host_features;
2947 if (k->set_features) {
2948 k->set_features(vdev, val);
2949 }
2950 vdev->guest_features = val;
2951 return bad ? -1 : 0;
2952 }
2953
2954 int virtio_set_features(VirtIODevice *vdev, uint64_t val)
2955 {
2956 int ret;
2957 /*
2958 * The driver must not attempt to set features after feature negotiation
2959 * has finished.
2960 */
2961 if (vdev->status & VIRTIO_CONFIG_S_FEATURES_OK) {
2962 return -EINVAL;
2963 }
2964 ret = virtio_set_features_nocheck(vdev, val);
2965 if (!ret) {
2966 if (virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX)) {
2967 /* VIRTIO_RING_F_EVENT_IDX changes the size of the caches. */
2968 int i;
2969 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
2970 if (vdev->vq[i].vring.num != 0) {
2971 virtio_init_region_cache(vdev, i);
2972 }
2973 }
2974 }
2975
2976 if (!virtio_device_started(vdev, vdev->status) &&
2977 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
2978 vdev->start_on_kick = true;
2979 }
2980 }
2981 return ret;
2982 }
2983
2984 size_t virtio_feature_get_config_size(VirtIOFeature *feature_sizes,
2985 uint64_t host_features)
2986 {
2987 size_t config_size = 0;
2988 int i;
2989
2990 for (i = 0; feature_sizes[i].flags != 0; i++) {
2991 if (host_features & feature_sizes[i].flags) {
2992 config_size = MAX(feature_sizes[i].end, config_size);
2993 }
2994 }
2995
2996 return config_size;
2997 }
2998
2999 int virtio_load(VirtIODevice *vdev, QEMUFile *f, int version_id)
3000 {
3001 int i, ret;
3002 int32_t config_len;
3003 uint32_t num;
3004 uint32_t features;
3005 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3006 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3007 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(vdev);
3008
3009 /*
3010 * We poison the endianness to ensure it does not get used before
3011 * subsections have been loaded.
3012 */
3013 vdev->device_endian = VIRTIO_DEVICE_ENDIAN_UNKNOWN;
3014
3015 if (k->load_config) {
3016 ret = k->load_config(qbus->parent, f);
3017 if (ret)
3018 return ret;
3019 }
3020
3021 qemu_get_8s(f, &vdev->status);
3022 qemu_get_8s(f, &vdev->isr);
3023 qemu_get_be16s(f, &vdev->queue_sel);
3024 if (vdev->queue_sel >= VIRTIO_QUEUE_MAX) {
3025 return -1;
3026 }
3027 qemu_get_be32s(f, &features);
3028
3029 /*
3030 * Temporarily set guest_features low bits - needed by
3031 * virtio net load code testing for VIRTIO_NET_F_CTRL_GUEST_OFFLOADS
3032 * VIRTIO_NET_F_GUEST_ANNOUNCE and VIRTIO_NET_F_CTRL_VQ.
3033 *
3034 * Note: devices should always test host features in future - don't create
3035 * new dependencies like this.
3036 */
3037 vdev->guest_features = features;
3038
3039 config_len = qemu_get_be32(f);
3040
3041 /*
3042 * There are cases where the incoming config can be bigger or smaller
3043 * than what we have; so load what we have space for, and skip
3044 * any excess that's in the stream.
3045 */
3046 qemu_get_buffer(f, vdev->config, MIN(config_len, vdev->config_len));
3047
3048 while (config_len > vdev->config_len) {
3049 qemu_get_byte(f);
3050 config_len--;
3051 }
3052
3053 num = qemu_get_be32(f);
3054
3055 if (num > VIRTIO_QUEUE_MAX) {
3056 error_report("Invalid number of virtqueues: 0x%x", num);
3057 return -1;
3058 }
3059
3060 for (i = 0; i < num; i++) {
3061 vdev->vq[i].vring.num = qemu_get_be32(f);
3062 if (k->has_variable_vring_alignment) {
3063 vdev->vq[i].vring.align = qemu_get_be32(f);
3064 }
3065 vdev->vq[i].vring.desc = qemu_get_be64(f);
3066 qemu_get_be16s(f, &vdev->vq[i].last_avail_idx);
3067 vdev->vq[i].signalled_used_valid = false;
3068 vdev->vq[i].notification = true;
3069
3070 if (!vdev->vq[i].vring.desc && vdev->vq[i].last_avail_idx) {
3071 error_report("VQ %d address 0x0 "
3072 "inconsistent with Host index 0x%x",
3073 i, vdev->vq[i].last_avail_idx);
3074 return -1;
3075 }
3076 if (k->load_queue) {
3077 ret = k->load_queue(qbus->parent, i, f);
3078 if (ret)
3079 return ret;
3080 }
3081 }
3082
3083 virtio_notify_vector(vdev, VIRTIO_NO_VECTOR);
3084
3085 if (vdc->load != NULL) {
3086 ret = vdc->load(vdev, f, version_id);
3087 if (ret) {
3088 return ret;
3089 }
3090 }
3091
3092 if (vdc->vmsd) {
3093 ret = vmstate_load_state(f, vdc->vmsd, vdev, version_id);
3094 if (ret) {
3095 return ret;
3096 }
3097 }
3098
3099 /* Subsections */
3100 ret = vmstate_load_state(f, &vmstate_virtio, vdev, 1);
3101 if (ret) {
3102 return ret;
3103 }
3104
3105 if (vdev->device_endian == VIRTIO_DEVICE_ENDIAN_UNKNOWN) {
3106 vdev->device_endian = virtio_default_endian();
3107 }
3108
3109 if (virtio_64bit_features_needed(vdev)) {
3110 /*
3111 * Subsection load filled vdev->guest_features. Run them
3112 * through virtio_set_features to sanity-check them against
3113 * host_features.
3114 */
3115 uint64_t features64 = vdev->guest_features;
3116 if (virtio_set_features_nocheck(vdev, features64) < 0) {
3117 error_report("Features 0x%" PRIx64 " unsupported. "
3118 "Allowed features: 0x%" PRIx64,
3119 features64, vdev->host_features);
3120 return -1;
3121 }
3122 } else {
3123 if (virtio_set_features_nocheck(vdev, features) < 0) {
3124 error_report("Features 0x%x unsupported. "
3125 "Allowed features: 0x%" PRIx64,
3126 features, vdev->host_features);
3127 return -1;
3128 }
3129 }
3130
3131 if (!virtio_device_started(vdev, vdev->status) &&
3132 !virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3133 vdev->start_on_kick = true;
3134 }
3135
3136 RCU_READ_LOCK_GUARD();
3137 for (i = 0; i < num; i++) {
3138 if (vdev->vq[i].vring.desc) {
3139 uint16_t nheads;
3140
3141 /*
3142 * VIRTIO-1 devices migrate desc, used, and avail ring addresses so
3143 * only the region cache needs to be set up. Legacy devices need
3144 * to calculate used and avail ring addresses based on the desc
3145 * address.
3146 */
3147 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3148 virtio_init_region_cache(vdev, i);
3149 } else {
3150 virtio_queue_update_rings(vdev, i);
3151 }
3152
3153 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3154 vdev->vq[i].shadow_avail_idx = vdev->vq[i].last_avail_idx;
3155 vdev->vq[i].shadow_avail_wrap_counter =
3156 vdev->vq[i].last_avail_wrap_counter;
3157 continue;
3158 }
3159
3160 nheads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx;
3161 /* Check it isn't doing strange things with descriptor numbers. */
3162 if (nheads > vdev->vq[i].vring.num) {
3163 error_report("VQ %d size 0x%x Guest index 0x%x "
3164 "inconsistent with Host index 0x%x: delta 0x%x",
3165 i, vdev->vq[i].vring.num,
3166 vring_avail_idx(&vdev->vq[i]),
3167 vdev->vq[i].last_avail_idx, nheads);
3168 return -1;
3169 }
3170 vdev->vq[i].used_idx = vring_used_idx(&vdev->vq[i]);
3171 vdev->vq[i].shadow_avail_idx = vring_avail_idx(&vdev->vq[i]);
3172
3173 /*
3174 * Some devices migrate VirtQueueElements that have been popped
3175 * from the avail ring but not yet returned to the used ring.
3176 * Since max ring size < UINT16_MAX it's safe to use modulo
3177 * UINT16_MAX + 1 subtraction.
3178 */
3179 vdev->vq[i].inuse = (uint16_t)(vdev->vq[i].last_avail_idx -
3180 vdev->vq[i].used_idx);
3181 if (vdev->vq[i].inuse > vdev->vq[i].vring.num) {
3182 error_report("VQ %d size 0x%x < last_avail_idx 0x%x - "
3183 "used_idx 0x%x",
3184 i, vdev->vq[i].vring.num,
3185 vdev->vq[i].last_avail_idx,
3186 vdev->vq[i].used_idx);
3187 return -1;
3188 }
3189 }
3190 }
3191
3192 if (vdc->post_load) {
3193 ret = vdc->post_load(vdev);
3194 if (ret) {
3195 return ret;
3196 }
3197 }
3198
3199 return 0;
3200 }
3201
3202 void virtio_cleanup(VirtIODevice *vdev)
3203 {
3204 qemu_del_vm_change_state_handler(vdev->vmstate);
3205 }
3206
3207 static void virtio_vmstate_change(void *opaque, int running, RunState state)
3208 {
3209 VirtIODevice *vdev = opaque;
3210 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3211 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3212 bool backend_run = running && virtio_device_started(vdev, vdev->status);
3213 vdev->vm_running = running;
3214
3215 if (backend_run) {
3216 virtio_set_status(vdev, vdev->status);
3217 }
3218
3219 if (k->vmstate_change) {
3220 k->vmstate_change(qbus->parent, backend_run);
3221 }
3222
3223 if (!backend_run) {
3224 virtio_set_status(vdev, vdev->status);
3225 }
3226 }
3227
3228 void virtio_instance_init_common(Object *proxy_obj, void *data,
3229 size_t vdev_size, const char *vdev_name)
3230 {
3231 DeviceState *vdev = data;
3232
3233 object_initialize_child_with_props(proxy_obj, "virtio-backend", vdev,
3234 vdev_size, vdev_name, &error_abort,
3235 NULL);
3236 qdev_alias_all_properties(vdev, proxy_obj);
3237 }
3238
3239 void virtio_init(VirtIODevice *vdev, const char *name,
3240 uint16_t device_id, size_t config_size)
3241 {
3242 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3243 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3244 int i;
3245 int nvectors = k->query_nvectors ? k->query_nvectors(qbus->parent) : 0;
3246
3247 if (nvectors) {
3248 vdev->vector_queues =
3249 g_malloc0(sizeof(*vdev->vector_queues) * nvectors);
3250 }
3251
3252 vdev->start_on_kick = false;
3253 vdev->started = false;
3254 vdev->device_id = device_id;
3255 vdev->status = 0;
3256 atomic_set(&vdev->isr, 0);
3257 vdev->queue_sel = 0;
3258 vdev->config_vector = VIRTIO_NO_VECTOR;
3259 vdev->vq = g_malloc0(sizeof(VirtQueue) * VIRTIO_QUEUE_MAX);
3260 vdev->vm_running = runstate_is_running();
3261 vdev->broken = false;
3262 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3263 vdev->vq[i].vector = VIRTIO_NO_VECTOR;
3264 vdev->vq[i].vdev = vdev;
3265 vdev->vq[i].queue_index = i;
3266 vdev->vq[i].host_notifier_enabled = false;
3267 }
3268
3269 vdev->name = name;
3270 vdev->config_len = config_size;
3271 if (vdev->config_len) {
3272 vdev->config = g_malloc0(config_size);
3273 } else {
3274 vdev->config = NULL;
3275 }
3276 vdev->vmstate = qdev_add_vm_change_state_handler(DEVICE(vdev),
3277 virtio_vmstate_change, vdev);
3278 vdev->device_endian = virtio_default_endian();
3279 vdev->use_guest_notifier_mask = true;
3280 }
3281
3282 hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
3283 {
3284 return vdev->vq[n].vring.desc;
3285 }
3286
3287 bool virtio_queue_enabled(VirtIODevice *vdev, int n)
3288 {
3289 return virtio_queue_get_desc_addr(vdev, n) != 0;
3290 }
3291
3292 hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
3293 {
3294 return vdev->vq[n].vring.avail;
3295 }
3296
3297 hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
3298 {
3299 return vdev->vq[n].vring.used;
3300 }
3301
3302 hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
3303 {
3304 return sizeof(VRingDesc) * vdev->vq[n].vring.num;
3305 }
3306
3307 hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
3308 {
3309 int s;
3310
3311 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3312 return sizeof(struct VRingPackedDescEvent);
3313 }
3314
3315 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
3316 return offsetof(VRingAvail, ring) +
3317 sizeof(uint16_t) * vdev->vq[n].vring.num + s;
3318 }
3319
3320 hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
3321 {
3322 int s;
3323
3324 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3325 return sizeof(struct VRingPackedDescEvent);
3326 }
3327
3328 s = virtio_vdev_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
3329 return offsetof(VRingUsed, ring) +
3330 sizeof(VRingUsedElem) * vdev->vq[n].vring.num + s;
3331 }
3332
3333 static unsigned int virtio_queue_packed_get_last_avail_idx(VirtIODevice *vdev,
3334 int n)
3335 {
3336 unsigned int avail, used;
3337
3338 avail = vdev->vq[n].last_avail_idx;
3339 avail |= ((uint16_t)vdev->vq[n].last_avail_wrap_counter) << 15;
3340
3341 used = vdev->vq[n].used_idx;
3342 used |= ((uint16_t)vdev->vq[n].used_wrap_counter) << 15;
3343
3344 return avail | used << 16;
3345 }
3346
3347 static uint16_t virtio_queue_split_get_last_avail_idx(VirtIODevice *vdev,
3348 int n)
3349 {
3350 return vdev->vq[n].last_avail_idx;
3351 }
3352
3353 unsigned int virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n)
3354 {
3355 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3356 return virtio_queue_packed_get_last_avail_idx(vdev, n);
3357 } else {
3358 return virtio_queue_split_get_last_avail_idx(vdev, n);
3359 }
3360 }
3361
3362 static void virtio_queue_packed_set_last_avail_idx(VirtIODevice *vdev,
3363 int n, unsigned int idx)
3364 {
3365 struct VirtQueue *vq = &vdev->vq[n];
3366
3367 vq->last_avail_idx = vq->shadow_avail_idx = idx & 0x7fff;
3368 vq->last_avail_wrap_counter =
3369 vq->shadow_avail_wrap_counter = !!(idx & 0x8000);
3370 idx >>= 16;
3371 vq->used_idx = idx & 0x7ffff;
3372 vq->used_wrap_counter = !!(idx & 0x8000);
3373 }
3374
3375 static void virtio_queue_split_set_last_avail_idx(VirtIODevice *vdev,
3376 int n, unsigned int idx)
3377 {
3378 vdev->vq[n].last_avail_idx = idx;
3379 vdev->vq[n].shadow_avail_idx = idx;
3380 }
3381
3382 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n,
3383 unsigned int idx)
3384 {
3385 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3386 virtio_queue_packed_set_last_avail_idx(vdev, n, idx);
3387 } else {
3388 virtio_queue_split_set_last_avail_idx(vdev, n, idx);
3389 }
3390 }
3391
3392 static void virtio_queue_packed_restore_last_avail_idx(VirtIODevice *vdev,
3393 int n)
3394 {
3395 /* We don't have a reference like avail idx in shared memory */
3396 return;
3397 }
3398
3399 static void virtio_queue_split_restore_last_avail_idx(VirtIODevice *vdev,
3400 int n)
3401 {
3402 RCU_READ_LOCK_GUARD();
3403 if (vdev->vq[n].vring.desc) {
3404 vdev->vq[n].last_avail_idx = vring_used_idx(&vdev->vq[n]);
3405 vdev->vq[n].shadow_avail_idx = vdev->vq[n].last_avail_idx;
3406 }
3407 }
3408
3409 void virtio_queue_restore_last_avail_idx(VirtIODevice *vdev, int n)
3410 {
3411 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3412 virtio_queue_packed_restore_last_avail_idx(vdev, n);
3413 } else {
3414 virtio_queue_split_restore_last_avail_idx(vdev, n);
3415 }
3416 }
3417
3418 static void virtio_queue_packed_update_used_idx(VirtIODevice *vdev, int n)
3419 {
3420 /* used idx was updated through set_last_avail_idx() */
3421 return;
3422 }
3423
3424 static void virtio_split_packed_update_used_idx(VirtIODevice *vdev, int n)
3425 {
3426 RCU_READ_LOCK_GUARD();
3427 if (vdev->vq[n].vring.desc) {
3428 vdev->vq[n].used_idx = vring_used_idx(&vdev->vq[n]);
3429 }
3430 }
3431
3432 void virtio_queue_update_used_idx(VirtIODevice *vdev, int n)
3433 {
3434 if (virtio_vdev_has_feature(vdev, VIRTIO_F_RING_PACKED)) {
3435 return virtio_queue_packed_update_used_idx(vdev, n);
3436 } else {
3437 return virtio_split_packed_update_used_idx(vdev, n);
3438 }
3439 }
3440
3441 void virtio_queue_invalidate_signalled_used(VirtIODevice *vdev, int n)
3442 {
3443 vdev->vq[n].signalled_used_valid = false;
3444 }
3445
3446 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n)
3447 {
3448 return vdev->vq + n;
3449 }
3450
3451 uint16_t virtio_get_queue_index(VirtQueue *vq)
3452 {
3453 return vq->queue_index;
3454 }
3455
3456 static void virtio_queue_guest_notifier_read(EventNotifier *n)
3457 {
3458 VirtQueue *vq = container_of(n, VirtQueue, guest_notifier);
3459 if (event_notifier_test_and_clear(n)) {
3460 virtio_irq(vq);
3461 }
3462 }
3463
3464 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
3465 bool with_irqfd)
3466 {
3467 if (assign && !with_irqfd) {
3468 event_notifier_set_handler(&vq->guest_notifier,
3469 virtio_queue_guest_notifier_read);
3470 } else {
3471 event_notifier_set_handler(&vq->guest_notifier, NULL);
3472 }
3473 if (!assign) {
3474 /* Test and clear notifier before closing it,
3475 * in case poll callback didn't have time to run. */
3476 virtio_queue_guest_notifier_read(&vq->guest_notifier);
3477 }
3478 }
3479
3480 EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq)
3481 {
3482 return &vq->guest_notifier;
3483 }
3484
3485 static void virtio_queue_host_notifier_aio_read(EventNotifier *n)
3486 {
3487 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3488 if (event_notifier_test_and_clear(n)) {
3489 virtio_queue_notify_aio_vq(vq);
3490 }
3491 }
3492
3493 static void virtio_queue_host_notifier_aio_poll_begin(EventNotifier *n)
3494 {
3495 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3496
3497 virtio_queue_set_notification(vq, 0);
3498 }
3499
3500 static bool virtio_queue_host_notifier_aio_poll(void *opaque)
3501 {
3502 EventNotifier *n = opaque;
3503 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3504
3505 if (!vq->vring.desc || virtio_queue_empty(vq)) {
3506 return false;
3507 }
3508
3509 return virtio_queue_notify_aio_vq(vq);
3510 }
3511
3512 static void virtio_queue_host_notifier_aio_poll_end(EventNotifier *n)
3513 {
3514 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3515
3516 /* Caller polls once more after this to catch requests that race with us */
3517 virtio_queue_set_notification(vq, 1);
3518 }
3519
3520 void virtio_queue_aio_set_host_notifier_handler(VirtQueue *vq, AioContext *ctx,
3521 VirtIOHandleAIOOutput handle_output)
3522 {
3523 if (handle_output) {
3524 vq->handle_aio_output = handle_output;
3525 aio_set_event_notifier(ctx, &vq->host_notifier, true,
3526 virtio_queue_host_notifier_aio_read,
3527 virtio_queue_host_notifier_aio_poll);
3528 aio_set_event_notifier_poll(ctx, &vq->host_notifier,
3529 virtio_queue_host_notifier_aio_poll_begin,
3530 virtio_queue_host_notifier_aio_poll_end);
3531 } else {
3532 aio_set_event_notifier(ctx, &vq->host_notifier, true, NULL, NULL);
3533 /* Test and clear notifier before after disabling event,
3534 * in case poll callback didn't have time to run. */
3535 virtio_queue_host_notifier_aio_read(&vq->host_notifier);
3536 vq->handle_aio_output = NULL;
3537 }
3538 }
3539
3540 void virtio_queue_host_notifier_read(EventNotifier *n)
3541 {
3542 VirtQueue *vq = container_of(n, VirtQueue, host_notifier);
3543 if (event_notifier_test_and_clear(n)) {
3544 virtio_queue_notify_vq(vq);
3545 }
3546 }
3547
3548 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq)
3549 {
3550 return &vq->host_notifier;
3551 }
3552
3553 void virtio_queue_set_host_notifier_enabled(VirtQueue *vq, bool enabled)
3554 {
3555 vq->host_notifier_enabled = enabled;
3556 }
3557
3558 int virtio_queue_set_host_notifier_mr(VirtIODevice *vdev, int n,
3559 MemoryRegion *mr, bool assign)
3560 {
3561 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3562 VirtioBusClass *k = VIRTIO_BUS_GET_CLASS(qbus);
3563
3564 if (k->set_host_notifier_mr) {
3565 return k->set_host_notifier_mr(qbus->parent, n, mr, assign);
3566 }
3567
3568 return -1;
3569 }
3570
3571 void virtio_device_set_child_bus_name(VirtIODevice *vdev, char *bus_name)
3572 {
3573 g_free(vdev->bus_name);
3574 vdev->bus_name = g_strdup(bus_name);
3575 }
3576
3577 void GCC_FMT_ATTR(2, 3) virtio_error(VirtIODevice *vdev, const char *fmt, ...)
3578 {
3579 va_list ap;
3580
3581 va_start(ap, fmt);
3582 error_vreport(fmt, ap);
3583 va_end(ap);
3584
3585 if (virtio_vdev_has_feature(vdev, VIRTIO_F_VERSION_1)) {
3586 vdev->status = vdev->status | VIRTIO_CONFIG_S_NEEDS_RESET;
3587 virtio_notify_config(vdev);
3588 }
3589
3590 vdev->broken = true;
3591 }
3592
3593 static void virtio_memory_listener_commit(MemoryListener *listener)
3594 {
3595 VirtIODevice *vdev = container_of(listener, VirtIODevice, listener);
3596 int i;
3597
3598 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3599 if (vdev->vq[i].vring.num == 0) {
3600 break;
3601 }
3602 virtio_init_region_cache(vdev, i);
3603 }
3604 }
3605
3606 static void virtio_device_realize(DeviceState *dev, Error **errp)
3607 {
3608 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
3609 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
3610 Error *err = NULL;
3611
3612 /* Devices should either use vmsd or the load/save methods */
3613 assert(!vdc->vmsd || !vdc->load);
3614
3615 if (vdc->realize != NULL) {
3616 vdc->realize(dev, &err);
3617 if (err != NULL) {
3618 error_propagate(errp, err);
3619 return;
3620 }
3621 }
3622
3623 virtio_bus_device_plugged(vdev, &err);
3624 if (err != NULL) {
3625 error_propagate(errp, err);
3626 vdc->unrealize(dev);
3627 return;
3628 }
3629
3630 vdev->listener.commit = virtio_memory_listener_commit;
3631 memory_listener_register(&vdev->listener, vdev->dma_as);
3632 }
3633
3634 static void virtio_device_unrealize(DeviceState *dev)
3635 {
3636 VirtIODevice *vdev = VIRTIO_DEVICE(dev);
3637 VirtioDeviceClass *vdc = VIRTIO_DEVICE_GET_CLASS(dev);
3638
3639 virtio_bus_device_unplugged(vdev);
3640
3641 if (vdc->unrealize != NULL) {
3642 vdc->unrealize(dev);
3643 }
3644
3645 g_free(vdev->bus_name);
3646 vdev->bus_name = NULL;
3647 }
3648
3649 static void virtio_device_free_virtqueues(VirtIODevice *vdev)
3650 {
3651 int i;
3652 if (!vdev->vq) {
3653 return;
3654 }
3655
3656 for (i = 0; i < VIRTIO_QUEUE_MAX; i++) {
3657 if (vdev->vq[i].vring.num == 0) {
3658 break;
3659 }
3660 virtio_virtqueue_reset_region_cache(&vdev->vq[i]);
3661 }
3662 g_free(vdev->vq);
3663 }
3664
3665 static void virtio_device_instance_finalize(Object *obj)
3666 {
3667 VirtIODevice *vdev = VIRTIO_DEVICE(obj);
3668
3669 memory_listener_unregister(&vdev->listener);
3670 virtio_device_free_virtqueues(vdev);
3671
3672 g_free(vdev->config);
3673 g_free(vdev->vector_queues);
3674 }
3675
3676 static Property virtio_properties[] = {
3677 DEFINE_VIRTIO_COMMON_FEATURES(VirtIODevice, host_features),
3678 DEFINE_PROP_BOOL("use-started", VirtIODevice, use_started, true),
3679 DEFINE_PROP_BOOL("use-disabled-flag", VirtIODevice, use_disabled_flag, true),
3680 DEFINE_PROP_END_OF_LIST(),
3681 };
3682
3683 static int virtio_device_start_ioeventfd_impl(VirtIODevice *vdev)
3684 {
3685 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
3686 int i, n, r, err;
3687
3688 memory_region_transaction_begin();
3689 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3690 VirtQueue *vq = &vdev->vq[n];
3691 if (!virtio_queue_get_num(vdev, n)) {
3692 continue;
3693 }
3694 r = virtio_bus_set_host_notifier(qbus, n, true);
3695 if (r < 0) {
3696 err = r;
3697 goto assign_error;
3698 }
3699 event_notifier_set_handler(&vq->host_notifier,
3700 virtio_queue_host_notifier_read);
3701 }
3702
3703 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3704 /* Kick right away to begin processing requests already in vring */
3705 VirtQueue *vq = &vdev->vq[n];
3706 if (!vq->vring.num) {
3707 continue;
3708 }
3709 event_notifier_set(&vq->host_notifier);
3710 }
3711 memory_region_transaction_commit();
3712 return 0;
3713
3714 assign_error:
3715 i = n; /* save n for a second iteration after transaction is committed. */
3716 while (--n >= 0) {
3717 VirtQueue *vq = &vdev->vq[n];
3718 if (!virtio_queue_get_num(vdev, n)) {
3719 continue;
3720 }
3721
3722 event_notifier_set_handler(&vq->host_notifier, NULL);
3723 r = virtio_bus_set_host_notifier(qbus, n, false);
3724 assert(r >= 0);
3725 }
3726 memory_region_transaction_commit();
3727
3728 while (--i >= 0) {
3729 if (!virtio_queue_get_num(vdev, i)) {
3730 continue;
3731 }
3732 virtio_bus_cleanup_host_notifier(qbus, i);
3733 }
3734 return err;
3735 }
3736
3737 int virtio_device_start_ioeventfd(VirtIODevice *vdev)
3738 {
3739 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3740 VirtioBusState *vbus = VIRTIO_BUS(qbus);
3741
3742 return virtio_bus_start_ioeventfd(vbus);
3743 }
3744
3745 static void virtio_device_stop_ioeventfd_impl(VirtIODevice *vdev)
3746 {
3747 VirtioBusState *qbus = VIRTIO_BUS(qdev_get_parent_bus(DEVICE(vdev)));
3748 int n, r;
3749
3750 memory_region_transaction_begin();
3751 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3752 VirtQueue *vq = &vdev->vq[n];
3753
3754 if (!virtio_queue_get_num(vdev, n)) {
3755 continue;
3756 }
3757 event_notifier_set_handler(&vq->host_notifier, NULL);
3758 r = virtio_bus_set_host_notifier(qbus, n, false);
3759 assert(r >= 0);
3760 }
3761 memory_region_transaction_commit();
3762
3763 for (n = 0; n < VIRTIO_QUEUE_MAX; n++) {
3764 if (!virtio_queue_get_num(vdev, n)) {
3765 continue;
3766 }
3767 virtio_bus_cleanup_host_notifier(qbus, n);
3768 }
3769 }
3770
3771 int virtio_device_grab_ioeventfd(VirtIODevice *vdev)
3772 {
3773 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3774 VirtioBusState *vbus = VIRTIO_BUS(qbus);
3775
3776 return virtio_bus_grab_ioeventfd(vbus);
3777 }
3778
3779 void virtio_device_release_ioeventfd(VirtIODevice *vdev)
3780 {
3781 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3782 VirtioBusState *vbus = VIRTIO_BUS(qbus);
3783
3784 virtio_bus_release_ioeventfd(vbus);
3785 }
3786
3787 static void virtio_device_class_init(ObjectClass *klass, void *data)
3788 {
3789 /* Set the default value here. */
3790 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
3791 DeviceClass *dc = DEVICE_CLASS(klass);
3792
3793 dc->realize = virtio_device_realize;
3794 dc->unrealize = virtio_device_unrealize;
3795 dc->bus_type = TYPE_VIRTIO_BUS;
3796 device_class_set_props(dc, virtio_properties);
3797 vdc->start_ioeventfd = virtio_device_start_ioeventfd_impl;
3798 vdc->stop_ioeventfd = virtio_device_stop_ioeventfd_impl;
3799
3800 vdc->legacy_features |= VIRTIO_LEGACY_FEATURES;
3801 }
3802
3803 bool virtio_device_ioeventfd_enabled(VirtIODevice *vdev)
3804 {
3805 BusState *qbus = qdev_get_parent_bus(DEVICE(vdev));
3806 VirtioBusState *vbus = VIRTIO_BUS(qbus);
3807
3808 return virtio_bus_ioeventfd_enabled(vbus);
3809 }
3810
3811 static const TypeInfo virtio_device_info = {
3812 .name = TYPE_VIRTIO_DEVICE,
3813 .parent = TYPE_DEVICE,
3814 .instance_size = sizeof(VirtIODevice),
3815 .class_init = virtio_device_class_init,
3816 .instance_finalize = virtio_device_instance_finalize,
3817 .abstract = true,
3818 .class_size = sizeof(VirtioDeviceClass),
3819 };
3820
3821 static void virtio_register_types(void)
3822 {
3823 type_register_static(&virtio_device_info);
3824 }
3825
3826 type_init(virtio_register_types)
armbru's QEMU hackery