?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
perf python: Do not force closing original perf descriptor in evlist.get_pollfd()
[linux/fpc-iii.git] / drivers / vhost / vhost.c
blob6b98d8e3a5bf8247784303ce890a990fb8ec1259
1 /* Copyright (C) 2009 Red Hat, Inc.
2 * Copyright (C) 2006 Rusty Russell IBM Corporation
3 *
4 * Author: Michael S. Tsirkin <mst@redhat.com>
5 *
6 * Inspiration, some code, and most witty comments come from
7 * Documentation/virtual/lguest/lguest.c, by Rusty Russell
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2.
10 *
11 * Generic code for virtio server in host kernel.
12 */
13
14 #include <linux/eventfd.h>
15 #include <linux/vhost.h>
16 #include <linux/uio.h>
17 #include <linux/mm.h>
18 #include <linux/mmu_context.h>
19 #include <linux/miscdevice.h>
20 #include <linux/mutex.h>
21 #include <linux/poll.h>
22 #include <linux/file.h>
23 #include <linux/highmem.h>
24 #include <linux/slab.h>
25 #include <linux/vmalloc.h>
26 #include <linux/kthread.h>
27 #include <linux/cgroup.h>
28 #include <linux/module.h>
29 #include <linux/sort.h>
30 #include <linux/sched/mm.h>
31 #include <linux/sched/signal.h>
32 #include <linux/interval_tree_generic.h>
33 #include <linux/nospec.h>
34
35 #include "vhost.h"
36
37 static ushort max_mem_regions = 64;
38 module_param(max_mem_regions, ushort, 0444);
39 MODULE_PARM_DESC(max_mem_regions,
40 "Maximum number of memory regions in memory map. (default: 64)");
41 static int max_iotlb_entries = 2048;
42 module_param(max_iotlb_entries, int, 0444);
43 MODULE_PARM_DESC(max_iotlb_entries,
44 "Maximum number of iotlb entries. (default: 2048)");
45
46 enum {
47 VHOST_MEMORY_F_LOG = 0x1,
48 };
49
50 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num])
51 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num])
52
53 INTERVAL_TREE_DEFINE(struct vhost_umem_node,
54 rb, __u64, __subtree_last,
55 START, LAST, static inline, vhost_umem_interval_tree);
56
57 #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY
58 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
59 {
60 vq->user_be = !virtio_legacy_is_little_endian();
61 }
62
63 static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq)
64 {
65 vq->user_be = true;
66 }
67
68 static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq)
69 {
70 vq->user_be = false;
71 }
72
73 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
74 {
75 struct vhost_vring_state s;
76
77 if (vq->private_data)
78 return -EBUSY;
79
80 if (copy_from_user(&s, argp, sizeof(s)))
81 return -EFAULT;
82
83 if (s.num != VHOST_VRING_LITTLE_ENDIAN &&
84 s.num != VHOST_VRING_BIG_ENDIAN)
85 return -EINVAL;
86
87 if (s.num == VHOST_VRING_BIG_ENDIAN)
88 vhost_enable_cross_endian_big(vq);
89 else
90 vhost_enable_cross_endian_little(vq);
91
92 return 0;
93 }
94
95 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
96 int __user *argp)
97 {
98 struct vhost_vring_state s = {
99 .index = idx,
100 .num = vq->user_be
101 };
102
103 if (copy_to_user(argp, &s, sizeof(s)))
104 return -EFAULT;
105
106 return 0;
107 }
108
109 static void vhost_init_is_le(struct vhost_virtqueue *vq)
110 {
111 /* Note for legacy virtio: user_be is initialized at reset time
112 * according to the host endianness. If userspace does not set an
113 * explicit endianness, the default behavior is native endian, as
114 * expected by legacy virtio.
115 */
116 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be;
117 }
118 #else
119 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq)
120 {
121 }
122
123 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp)
124 {
125 return -ENOIOCTLCMD;
126 }
127
128 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx,
129 int __user *argp)
130 {
131 return -ENOIOCTLCMD;
132 }
133
134 static void vhost_init_is_le(struct vhost_virtqueue *vq)
135 {
136 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1)
137 || virtio_legacy_is_little_endian();
138 }
139 #endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */
140
141 static void vhost_reset_is_le(struct vhost_virtqueue *vq)
142 {
143 vhost_init_is_le(vq);
144 }
145
146 struct vhost_flush_struct {
147 struct vhost_work work;
148 struct completion wait_event;
149 };
150
151 static void vhost_flush_work(struct vhost_work *work)
152 {
153 struct vhost_flush_struct *s;
154
155 s = container_of(work, struct vhost_flush_struct, work);
156 complete(&s->wait_event);
157 }
158
159 static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh,
160 poll_table *pt)
161 {
162 struct vhost_poll *poll;
163
164 poll = container_of(pt, struct vhost_poll, table);
165 poll->wqh = wqh;
166 add_wait_queue(wqh, &poll->wait);
167 }
168
169 static int vhost_poll_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync,
170 void *key)
171 {
172 struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait);
173
174 if (!(key_to_poll(key) & poll->mask))
175 return 0;
176
177 vhost_poll_queue(poll);
178 return 0;
179 }
180
181 void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn)
182 {
183 clear_bit(VHOST_WORK_QUEUED, &work->flags);
184 work->fn = fn;
185 }
186 EXPORT_SYMBOL_GPL(vhost_work_init);
187
188 /* Init poll structure */
189 void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn,
190 __poll_t mask, struct vhost_dev *dev)
191 {
192 init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup);
193 init_poll_funcptr(&poll->table, vhost_poll_func);
194 poll->mask = mask;
195 poll->dev = dev;
196 poll->wqh = NULL;
197
198 vhost_work_init(&poll->work, fn);
199 }
200 EXPORT_SYMBOL_GPL(vhost_poll_init);
201
202 /* Start polling a file. We add ourselves to file's wait queue. The caller must
203 * keep a reference to a file until after vhost_poll_stop is called. */
204 int vhost_poll_start(struct vhost_poll *poll, struct file *file)
205 {
206 __poll_t mask;
207 int ret = 0;
208
209 if (poll->wqh)
210 return 0;
211
212 mask = vfs_poll(file, &poll->table);
213 if (mask)
214 vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask));
215 if (mask & EPOLLERR) {
216 vhost_poll_stop(poll);
217 ret = -EINVAL;
218 }
219
220 return ret;
221 }
222 EXPORT_SYMBOL_GPL(vhost_poll_start);
223
224 /* Stop polling a file. After this function returns, it becomes safe to drop the
225 * file reference. You must also flush afterwards. */
226 void vhost_poll_stop(struct vhost_poll *poll)
227 {
228 if (poll->wqh) {
229 remove_wait_queue(poll->wqh, &poll->wait);
230 poll->wqh = NULL;
231 }
232 }
233 EXPORT_SYMBOL_GPL(vhost_poll_stop);
234
235 void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work)
236 {
237 struct vhost_flush_struct flush;
238
239 if (dev->worker) {
240 init_completion(&flush.wait_event);
241 vhost_work_init(&flush.work, vhost_flush_work);
242
243 vhost_work_queue(dev, &flush.work);
244 wait_for_completion(&flush.wait_event);
245 }
246 }
247 EXPORT_SYMBOL_GPL(vhost_work_flush);
248
249 /* Flush any work that has been scheduled. When calling this, don't hold any
250 * locks that are also used by the callback. */
251 void vhost_poll_flush(struct vhost_poll *poll)
252 {
253 vhost_work_flush(poll->dev, &poll->work);
254 }
255 EXPORT_SYMBOL_GPL(vhost_poll_flush);
256
257 void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work)
258 {
259 if (!dev->worker)
260 return;
261
262 if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) {
263 /* We can only add the work to the list after we're
264 * sure it was not in the list.
265 * test_and_set_bit() implies a memory barrier.
266 */
267 llist_add(&work->node, &dev->work_list);
268 wake_up_process(dev->worker);
269 }
270 }
271 EXPORT_SYMBOL_GPL(vhost_work_queue);
272
273 /* A lockless hint for busy polling code to exit the loop */
274 bool vhost_has_work(struct vhost_dev *dev)
275 {
276 return !llist_empty(&dev->work_list);
277 }
278 EXPORT_SYMBOL_GPL(vhost_has_work);
279
280 void vhost_poll_queue(struct vhost_poll *poll)
281 {
282 vhost_work_queue(poll->dev, &poll->work);
283 }
284 EXPORT_SYMBOL_GPL(vhost_poll_queue);
285
286 static void __vhost_vq_meta_reset(struct vhost_virtqueue *vq)
287 {
288 int j;
289
290 for (j = 0; j < VHOST_NUM_ADDRS; j++)
291 vq->meta_iotlb[j] = NULL;
292 }
293
294 static void vhost_vq_meta_reset(struct vhost_dev *d)
295 {
296 int i;
297
298 for (i = 0; i < d->nvqs; ++i) {
299 mutex_lock(&d->vqs[i]->mutex);
300 __vhost_vq_meta_reset(d->vqs[i]);
301 mutex_unlock(&d->vqs[i]->mutex);
302 }
303 }
304
305 static void vhost_vq_reset(struct vhost_dev *dev,
306 struct vhost_virtqueue *vq)
307 {
308 vq->num = 1;
309 vq->desc = NULL;
310 vq->avail = NULL;
311 vq->used = NULL;
312 vq->last_avail_idx = 0;
313 vq->avail_idx = 0;
314 vq->last_used_idx = 0;
315 vq->signalled_used = 0;
316 vq->signalled_used_valid = false;
317 vq->used_flags = 0;
318 vq->log_used = false;
319 vq->log_addr = -1ull;
320 vq->private_data = NULL;
321 vq->acked_features = 0;
322 vq->acked_backend_features = 0;
323 vq->log_base = NULL;
324 vq->error_ctx = NULL;
325 vq->kick = NULL;
326 vq->call_ctx = NULL;
327 vq->log_ctx = NULL;
328 vhost_reset_is_le(vq);
329 vhost_disable_cross_endian(vq);
330 vq->busyloop_timeout = 0;
331 vq->umem = NULL;
332 vq->iotlb = NULL;
333 __vhost_vq_meta_reset(vq);
334 }
335
336 static int vhost_worker(void *data)
337 {
338 struct vhost_dev *dev = data;
339 struct vhost_work *work, *work_next;
340 struct llist_node *node;
341 mm_segment_t oldfs = get_fs();
342
343 set_fs(USER_DS);
344 use_mm(dev->mm);
345
346 for (;;) {
347 /* mb paired w/ kthread_stop */
348 set_current_state(TASK_INTERRUPTIBLE);
349
350 if (kthread_should_stop()) {
351 __set_current_state(TASK_RUNNING);
352 break;
353 }
354
355 node = llist_del_all(&dev->work_list);
356 if (!node)
357 schedule();
358
359 node = llist_reverse_order(node);
360 /* make sure flag is seen after deletion */
361 smp_wmb();
362 llist_for_each_entry_safe(work, work_next, node, node) {
363 clear_bit(VHOST_WORK_QUEUED, &work->flags);
364 __set_current_state(TASK_RUNNING);
365 work->fn(work);
366 if (need_resched())
367 schedule();
368 }
369 }
370 unuse_mm(dev->mm);
371 set_fs(oldfs);
372 return 0;
373 }
374
375 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq)
376 {
377 kfree(vq->indirect);
378 vq->indirect = NULL;
379 kfree(vq->log);
380 vq->log = NULL;
381 kfree(vq->heads);
382 vq->heads = NULL;
383 }
384
385 /* Helper to allocate iovec buffers for all vqs. */
386 static long vhost_dev_alloc_iovecs(struct vhost_dev *dev)
387 {
388 struct vhost_virtqueue *vq;
389 int i;
390
391 for (i = 0; i < dev->nvqs; ++i) {
392 vq = dev->vqs[i];
393 vq->indirect = kmalloc_array(UIO_MAXIOV,
394 sizeof(*vq->indirect),
395 GFP_KERNEL);
396 vq->log = kmalloc_array(UIO_MAXIOV, sizeof(*vq->log),
397 GFP_KERNEL);
398 vq->heads = kmalloc_array(UIO_MAXIOV, sizeof(*vq->heads),
399 GFP_KERNEL);
400 if (!vq->indirect || !vq->log || !vq->heads)
401 goto err_nomem;
402 }
403 return 0;
404
405 err_nomem:
406 for (; i >= 0; --i)
407 vhost_vq_free_iovecs(dev->vqs[i]);
408 return -ENOMEM;
409 }
410
411 static void vhost_dev_free_iovecs(struct vhost_dev *dev)
412 {
413 int i;
414
415 for (i = 0; i < dev->nvqs; ++i)
416 vhost_vq_free_iovecs(dev->vqs[i]);
417 }
418
419 void vhost_dev_init(struct vhost_dev *dev,
420 struct vhost_virtqueue **vqs, int nvqs)
421 {
422 struct vhost_virtqueue *vq;
423 int i;
424
425 dev->vqs = vqs;
426 dev->nvqs = nvqs;
427 mutex_init(&dev->mutex);
428 dev->log_ctx = NULL;
429 dev->umem = NULL;
430 dev->iotlb = NULL;
431 dev->mm = NULL;
432 dev->worker = NULL;
433 init_llist_head(&dev->work_list);
434 init_waitqueue_head(&dev->wait);
435 INIT_LIST_HEAD(&dev->read_list);
436 INIT_LIST_HEAD(&dev->pending_list);
437 spin_lock_init(&dev->iotlb_lock);
438
439
440 for (i = 0; i < dev->nvqs; ++i) {
441 vq = dev->vqs[i];
442 vq->log = NULL;
443 vq->indirect = NULL;
444 vq->heads = NULL;
445 vq->dev = dev;
446 mutex_init(&vq->mutex);
447 vhost_vq_reset(dev, vq);
448 if (vq->handle_kick)
449 vhost_poll_init(&vq->poll, vq->handle_kick,
450 EPOLLIN, dev);
451 }
452 }
453 EXPORT_SYMBOL_GPL(vhost_dev_init);
454
455 /* Caller should have device mutex */
456 long vhost_dev_check_owner(struct vhost_dev *dev)
457 {
458 /* Are you the owner? If not, I don't think you mean to do that */
459 return dev->mm == current->mm ? 0 : -EPERM;
460 }
461 EXPORT_SYMBOL_GPL(vhost_dev_check_owner);
462
463 struct vhost_attach_cgroups_struct {
464 struct vhost_work work;
465 struct task_struct *owner;
466 int ret;
467 };
468
469 static void vhost_attach_cgroups_work(struct vhost_work *work)
470 {
471 struct vhost_attach_cgroups_struct *s;
472
473 s = container_of(work, struct vhost_attach_cgroups_struct, work);
474 s->ret = cgroup_attach_task_all(s->owner, current);
475 }
476
477 static int vhost_attach_cgroups(struct vhost_dev *dev)
478 {
479 struct vhost_attach_cgroups_struct attach;
480
481 attach.owner = current;
482 vhost_work_init(&attach.work, vhost_attach_cgroups_work);
483 vhost_work_queue(dev, &attach.work);
484 vhost_work_flush(dev, &attach.work);
485 return attach.ret;
486 }
487
488 /* Caller should have device mutex */
489 bool vhost_dev_has_owner(struct vhost_dev *dev)
490 {
491 return dev->mm;
492 }
493 EXPORT_SYMBOL_GPL(vhost_dev_has_owner);
494
495 /* Caller should have device mutex */
496 long vhost_dev_set_owner(struct vhost_dev *dev)
497 {
498 struct task_struct *worker;
499 int err;
500
501 /* Is there an owner already? */
502 if (vhost_dev_has_owner(dev)) {
503 err = -EBUSY;
504 goto err_mm;
505 }
506
507 /* No owner, become one */
508 dev->mm = get_task_mm(current);
509 worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid);
510 if (IS_ERR(worker)) {
511 err = PTR_ERR(worker);
512 goto err_worker;
513 }
514
515 dev->worker = worker;
516 wake_up_process(worker); /* avoid contributing to loadavg */
517
518 err = vhost_attach_cgroups(dev);
519 if (err)
520 goto err_cgroup;
521
522 err = vhost_dev_alloc_iovecs(dev);
523 if (err)
524 goto err_cgroup;
525
526 return 0;
527 err_cgroup:
528 kthread_stop(worker);
529 dev->worker = NULL;
530 err_worker:
531 if (dev->mm)
532 mmput(dev->mm);
533 dev->mm = NULL;
534 err_mm:
535 return err;
536 }
537 EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
538
539 struct vhost_umem *vhost_dev_reset_owner_prepare(void)
540 {
541 return kvzalloc(sizeof(struct vhost_umem), GFP_KERNEL);
542 }
543 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare);
544
545 /* Caller should have device mutex */
546 void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem)
547 {
548 int i;
549
550 vhost_dev_cleanup(dev);
551
552 /* Restore memory to default empty mapping. */
553 INIT_LIST_HEAD(&umem->umem_list);
554 dev->umem = umem;
555 /* We don't need VQ locks below since vhost_dev_cleanup makes sure
556 * VQs aren't running.
557 */
558 for (i = 0; i < dev->nvqs; ++i)
559 dev->vqs[i]->umem = umem;
560 }
561 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner);
562
563 void vhost_dev_stop(struct vhost_dev *dev)
564 {
565 int i;
566
567 for (i = 0; i < dev->nvqs; ++i) {
568 if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) {
569 vhost_poll_stop(&dev->vqs[i]->poll);
570 vhost_poll_flush(&dev->vqs[i]->poll);
571 }
572 }
573 }
574 EXPORT_SYMBOL_GPL(vhost_dev_stop);
575
576 static void vhost_umem_free(struct vhost_umem *umem,
577 struct vhost_umem_node *node)
578 {
579 vhost_umem_interval_tree_remove(node, &umem->umem_tree);
580 list_del(&node->link);
581 kfree(node);
582 umem->numem--;
583 }
584
585 static void vhost_umem_clean(struct vhost_umem *umem)
586 {
587 struct vhost_umem_node *node, *tmp;
588
589 if (!umem)
590 return;
591
592 list_for_each_entry_safe(node, tmp, &umem->umem_list, link)
593 vhost_umem_free(umem, node);
594
595 kvfree(umem);
596 }
597
598 static void vhost_clear_msg(struct vhost_dev *dev)
599 {
600 struct vhost_msg_node *node, *n;
601
602 spin_lock(&dev->iotlb_lock);
603
604 list_for_each_entry_safe(node, n, &dev->read_list, node) {
605 list_del(&node->node);
606 kfree(node);
607 }
608
609 list_for_each_entry_safe(node, n, &dev->pending_list, node) {
610 list_del(&node->node);
611 kfree(node);
612 }
613
614 spin_unlock(&dev->iotlb_lock);
615 }
616
617 void vhost_dev_cleanup(struct vhost_dev *dev)
618 {
619 int i;
620
621 for (i = 0; i < dev->nvqs; ++i) {
622 if (dev->vqs[i]->error_ctx)
623 eventfd_ctx_put(dev->vqs[i]->error_ctx);
624 if (dev->vqs[i]->kick)
625 fput(dev->vqs[i]->kick);
626 if (dev->vqs[i]->call_ctx)
627 eventfd_ctx_put(dev->vqs[i]->call_ctx);
628 vhost_vq_reset(dev, dev->vqs[i]);
629 }
630 vhost_dev_free_iovecs(dev);
631 if (dev->log_ctx)
632 eventfd_ctx_put(dev->log_ctx);
633 dev->log_ctx = NULL;
634 /* No one will access memory at this point */
635 vhost_umem_clean(dev->umem);
636 dev->umem = NULL;
637 vhost_umem_clean(dev->iotlb);
638 dev->iotlb = NULL;
639 vhost_clear_msg(dev);
640 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
641 WARN_ON(!llist_empty(&dev->work_list));
642 if (dev->worker) {
643 kthread_stop(dev->worker);
644 dev->worker = NULL;
645 }
646 if (dev->mm)
647 mmput(dev->mm);
648 dev->mm = NULL;
649 }
650 EXPORT_SYMBOL_GPL(vhost_dev_cleanup);
651
652 static bool log_access_ok(void __user *log_base, u64 addr, unsigned long sz)
653 {
654 u64 a = addr / VHOST_PAGE_SIZE / 8;
655
656 /* Make sure 64 bit math will not overflow. */
657 if (a > ULONG_MAX - (unsigned long)log_base ||
658 a + (unsigned long)log_base > ULONG_MAX)
659 return false;
660
661 return access_ok(VERIFY_WRITE, log_base + a,
662 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8);
663 }
664
665 static bool vhost_overflow(u64 uaddr, u64 size)
666 {
667 /* Make sure 64 bit math will not overflow. */
668 return uaddr > ULONG_MAX || size > ULONG_MAX || uaddr > ULONG_MAX - size;
669 }
670
671 /* Caller should have vq mutex and device mutex. */
672 static bool vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem,
673 int log_all)
674 {
675 struct vhost_umem_node *node;
676
677 if (!umem)
678 return false;
679
680 list_for_each_entry(node, &umem->umem_list, link) {
681 unsigned long a = node->userspace_addr;
682
683 if (vhost_overflow(node->userspace_addr, node->size))
684 return false;
685
686
687 if (!access_ok(VERIFY_WRITE, (void __user *)a,
688 node->size))
689 return false;
690 else if (log_all && !log_access_ok(log_base,
691 node->start,
692 node->size))
693 return false;
694 }
695 return true;
696 }
697
698 static inline void __user *vhost_vq_meta_fetch(struct vhost_virtqueue *vq,
699 u64 addr, unsigned int size,
700 int type)
701 {
702 const struct vhost_umem_node *node = vq->meta_iotlb[type];
703
704 if (!node)
705 return NULL;
706
707 return (void *)(uintptr_t)(node->userspace_addr + addr - node->start);
708 }
709
710 /* Can we switch to this memory table? */
711 /* Caller should have device mutex but not vq mutex */
712 static bool memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem,
713 int log_all)
714 {
715 int i;
716
717 for (i = 0; i < d->nvqs; ++i) {
718 bool ok;
719 bool log;
720
721 mutex_lock(&d->vqs[i]->mutex);
722 log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL);
723 /* If ring is inactive, will check when it's enabled. */
724 if (d->vqs[i]->private_data)
725 ok = vq_memory_access_ok(d->vqs[i]->log_base,
726 umem, log);
727 else
728 ok = true;
729 mutex_unlock(&d->vqs[i]->mutex);
730 if (!ok)
731 return false;
732 }
733 return true;
734 }
735
736 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
737 struct iovec iov[], int iov_size, int access);
738
739 static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to,
740 const void *from, unsigned size)
741 {
742 int ret;
743
744 if (!vq->iotlb)
745 return __copy_to_user(to, from, size);
746 else {
747 /* This function should be called after iotlb
748 * prefetch, which means we're sure that all vq
749 * could be access through iotlb. So -EAGAIN should
750 * not happen in this case.
751 */
752 struct iov_iter t;
753 void __user *uaddr = vhost_vq_meta_fetch(vq,
754 (u64)(uintptr_t)to, size,
755 VHOST_ADDR_USED);
756
757 if (uaddr)
758 return __copy_to_user(uaddr, from, size);
759
760 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov,
761 ARRAY_SIZE(vq->iotlb_iov),
762 VHOST_ACCESS_WO);
763 if (ret < 0)
764 goto out;
765 iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size);
766 ret = copy_to_iter(from, size, &t);
767 if (ret == size)
768 ret = 0;
769 }
770 out:
771 return ret;
772 }
773
774 static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to,
775 void __user *from, unsigned size)
776 {
777 int ret;
778
779 if (!vq->iotlb)
780 return __copy_from_user(to, from, size);
781 else {
782 /* This function should be called after iotlb
783 * prefetch, which means we're sure that vq
784 * could be access through iotlb. So -EAGAIN should
785 * not happen in this case.
786 */
787 void __user *uaddr = vhost_vq_meta_fetch(vq,
788 (u64)(uintptr_t)from, size,
789 VHOST_ADDR_DESC);
790 struct iov_iter f;
791
792 if (uaddr)
793 return __copy_from_user(to, uaddr, size);
794
795 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov,
796 ARRAY_SIZE(vq->iotlb_iov),
797 VHOST_ACCESS_RO);
798 if (ret < 0) {
799 vq_err(vq, "IOTLB translation failure: uaddr "
800 "%p size 0x%llx\n", from,
801 (unsigned long long) size);
802 goto out;
803 }
804 iov_iter_init(&f, READ, vq->iotlb_iov, ret, size);
805 ret = copy_from_iter(to, size, &f);
806 if (ret == size)
807 ret = 0;
808 }
809
810 out:
811 return ret;
812 }
813
814 static void __user *__vhost_get_user_slow(struct vhost_virtqueue *vq,
815 void __user *addr, unsigned int size,
816 int type)
817 {
818 int ret;
819
820 ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov,
821 ARRAY_SIZE(vq->iotlb_iov),
822 VHOST_ACCESS_RO);
823 if (ret < 0) {
824 vq_err(vq, "IOTLB translation failure: uaddr "
825 "%p size 0x%llx\n", addr,
826 (unsigned long long) size);
827 return NULL;
828 }
829
830 if (ret != 1 || vq->iotlb_iov[0].iov_len != size) {
831 vq_err(vq, "Non atomic userspace memory access: uaddr "
832 "%p size 0x%llx\n", addr,
833 (unsigned long long) size);
834 return NULL;
835 }
836
837 return vq->iotlb_iov[0].iov_base;
838 }
839
840 /* This function should be called after iotlb
841 * prefetch, which means we're sure that vq
842 * could be access through iotlb. So -EAGAIN should
843 * not happen in this case.
844 */
845 static inline void __user *__vhost_get_user(struct vhost_virtqueue *vq,
846 void *addr, unsigned int size,
847 int type)
848 {
849 void __user *uaddr = vhost_vq_meta_fetch(vq,
850 (u64)(uintptr_t)addr, size, type);
851 if (uaddr)
852 return uaddr;
853
854 return __vhost_get_user_slow(vq, addr, size, type);
855 }
856
857 #define vhost_put_user(vq, x, ptr) \
858 ({ \
859 int ret = -EFAULT; \
860 if (!vq->iotlb) { \
861 ret = __put_user(x, ptr); \
862 } else { \
863 __typeof__(ptr) to = \
864 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
865 sizeof(*ptr), VHOST_ADDR_USED); \
866 if (to != NULL) \
867 ret = __put_user(x, to); \
868 else \
869 ret = -EFAULT; \
870 } \
871 ret; \
872 })
873
874 #define vhost_get_user(vq, x, ptr, type) \
875 ({ \
876 int ret; \
877 if (!vq->iotlb) { \
878 ret = __get_user(x, ptr); \
879 } else { \
880 __typeof__(ptr) from = \
881 (__typeof__(ptr)) __vhost_get_user(vq, ptr, \
882 sizeof(*ptr), \
883 type); \
884 if (from != NULL) \
885 ret = __get_user(x, from); \
886 else \
887 ret = -EFAULT; \
888 } \
889 ret; \
890 })
891
892 #define vhost_get_avail(vq, x, ptr) \
893 vhost_get_user(vq, x, ptr, VHOST_ADDR_AVAIL)
894
895 #define vhost_get_used(vq, x, ptr) \
896 vhost_get_user(vq, x, ptr, VHOST_ADDR_USED)
897
898 static int vhost_new_umem_range(struct vhost_umem *umem,
899 u64 start, u64 size, u64 end,
900 u64 userspace_addr, int perm)
901 {
902 struct vhost_umem_node *tmp, *node = kmalloc(sizeof(*node), GFP_ATOMIC);
903
904 if (!node)
905 return -ENOMEM;
906
907 if (umem->numem == max_iotlb_entries) {
908 tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link);
909 vhost_umem_free(umem, tmp);
910 }
911
912 node->start = start;
913 node->size = size;
914 node->last = end;
915 node->userspace_addr = userspace_addr;
916 node->perm = perm;
917 INIT_LIST_HEAD(&node->link);
918 list_add_tail(&node->link, &umem->umem_list);
919 vhost_umem_interval_tree_insert(node, &umem->umem_tree);
920 umem->numem++;
921
922 return 0;
923 }
924
925 static void vhost_del_umem_range(struct vhost_umem *umem,
926 u64 start, u64 end)
927 {
928 struct vhost_umem_node *node;
929
930 while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
931 start, end)))
932 vhost_umem_free(umem, node);
933 }
934
935 static void vhost_iotlb_notify_vq(struct vhost_dev *d,
936 struct vhost_iotlb_msg *msg)
937 {
938 struct vhost_msg_node *node, *n;
939
940 spin_lock(&d->iotlb_lock);
941
942 list_for_each_entry_safe(node, n, &d->pending_list, node) {
943 struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb;
944 if (msg->iova <= vq_msg->iova &&
945 msg->iova + msg->size - 1 >= vq_msg->iova &&
946 vq_msg->type == VHOST_IOTLB_MISS) {
947 vhost_poll_queue(&node->vq->poll);
948 list_del(&node->node);
949 kfree(node);
950 }
951 }
952
953 spin_unlock(&d->iotlb_lock);
954 }
955
956 static bool umem_access_ok(u64 uaddr, u64 size, int access)
957 {
958 unsigned long a = uaddr;
959
960 /* Make sure 64 bit math will not overflow. */
961 if (vhost_overflow(uaddr, size))
962 return false;
963
964 if ((access & VHOST_ACCESS_RO) &&
965 !access_ok(VERIFY_READ, (void __user *)a, size))
966 return false;
967 if ((access & VHOST_ACCESS_WO) &&
968 !access_ok(VERIFY_WRITE, (void __user *)a, size))
969 return false;
970 return true;
971 }
972
973 static int vhost_process_iotlb_msg(struct vhost_dev *dev,
974 struct vhost_iotlb_msg *msg)
975 {
976 int ret = 0;
977
978 mutex_lock(&dev->mutex);
979 switch (msg->type) {
980 case VHOST_IOTLB_UPDATE:
981 if (!dev->iotlb) {
982 ret = -EFAULT;
983 break;
984 }
985 if (!umem_access_ok(msg->uaddr, msg->size, msg->perm)) {
986 ret = -EFAULT;
987 break;
988 }
989 vhost_vq_meta_reset(dev);
990 if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size,
991 msg->iova + msg->size - 1,
992 msg->uaddr, msg->perm)) {
993 ret = -ENOMEM;
994 break;
995 }
996 vhost_iotlb_notify_vq(dev, msg);
997 break;
998 case VHOST_IOTLB_INVALIDATE:
999 if (!dev->iotlb) {
1000 ret = -EFAULT;
1001 break;
1002 }
1003 vhost_vq_meta_reset(dev);
1004 vhost_del_umem_range(dev->iotlb, msg->iova,
1005 msg->iova + msg->size - 1);
1006 break;
1007 default:
1008 ret = -EINVAL;
1009 break;
1010 }
1011
1012 mutex_unlock(&dev->mutex);
1013
1014 return ret;
1015 }
1016 ssize_t vhost_chr_write_iter(struct vhost_dev *dev,
1017 struct iov_iter *from)
1018 {
1019 struct vhost_iotlb_msg msg;
1020 size_t offset;
1021 int type, ret;
1022
1023 ret = copy_from_iter(&type, sizeof(type), from);
1024 if (ret != sizeof(type))
1025 goto done;
1026
1027 switch (type) {
1028 case VHOST_IOTLB_MSG:
1029 /* There maybe a hole after type for V1 message type,
1030 * so skip it here.
1031 */
1032 offset = offsetof(struct vhost_msg, iotlb) - sizeof(int);
1033 break;
1034 case VHOST_IOTLB_MSG_V2:
1035 offset = sizeof(__u32);
1036 break;
1037 default:
1038 ret = -EINVAL;
1039 goto done;
1040 }
1041
1042 iov_iter_advance(from, offset);
1043 ret = copy_from_iter(&msg, sizeof(msg), from);
1044 if (ret != sizeof(msg))
1045 goto done;
1046 if (vhost_process_iotlb_msg(dev, &msg)) {
1047 ret = -EFAULT;
1048 goto done;
1049 }
1050
1051 ret = (type == VHOST_IOTLB_MSG) ? sizeof(struct vhost_msg) :
1052 sizeof(struct vhost_msg_v2);
1053 done:
1054 return ret;
1055 }
1056 EXPORT_SYMBOL(vhost_chr_write_iter);
1057
1058 __poll_t vhost_chr_poll(struct file *file, struct vhost_dev *dev,
1059 poll_table *wait)
1060 {
1061 __poll_t mask = 0;
1062
1063 poll_wait(file, &dev->wait, wait);
1064
1065 if (!list_empty(&dev->read_list))
1066 mask |= EPOLLIN | EPOLLRDNORM;
1067
1068 return mask;
1069 }
1070 EXPORT_SYMBOL(vhost_chr_poll);
1071
1072 ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to,
1073 int noblock)
1074 {
1075 DEFINE_WAIT(wait);
1076 struct vhost_msg_node *node;
1077 ssize_t ret = 0;
1078 unsigned size = sizeof(struct vhost_msg);
1079
1080 if (iov_iter_count(to) < size)
1081 return 0;
1082
1083 while (1) {
1084 if (!noblock)
1085 prepare_to_wait(&dev->wait, &wait,
1086 TASK_INTERRUPTIBLE);
1087
1088 node = vhost_dequeue_msg(dev, &dev->read_list);
1089 if (node)
1090 break;
1091 if (noblock) {
1092 ret = -EAGAIN;
1093 break;
1094 }
1095 if (signal_pending(current)) {
1096 ret = -ERESTARTSYS;
1097 break;
1098 }
1099 if (!dev->iotlb) {
1100 ret = -EBADFD;
1101 break;
1102 }
1103
1104 schedule();
1105 }
1106
1107 if (!noblock)
1108 finish_wait(&dev->wait, &wait);
1109
1110 if (node) {
1111 struct vhost_iotlb_msg *msg;
1112 void *start = &node->msg;
1113
1114 switch (node->msg.type) {
1115 case VHOST_IOTLB_MSG:
1116 size = sizeof(node->msg);
1117 msg = &node->msg.iotlb;
1118 break;
1119 case VHOST_IOTLB_MSG_V2:
1120 size = sizeof(node->msg_v2);
1121 msg = &node->msg_v2.iotlb;
1122 break;
1123 default:
1124 BUG();
1125 break;
1126 }
1127
1128 ret = copy_to_iter(start, size, to);
1129 if (ret != size || msg->type != VHOST_IOTLB_MISS) {
1130 kfree(node);
1131 return ret;
1132 }
1133 vhost_enqueue_msg(dev, &dev->pending_list, node);
1134 }
1135
1136 return ret;
1137 }
1138 EXPORT_SYMBOL_GPL(vhost_chr_read_iter);
1139
1140 static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access)
1141 {
1142 struct vhost_dev *dev = vq->dev;
1143 struct vhost_msg_node *node;
1144 struct vhost_iotlb_msg *msg;
1145 bool v2 = vhost_backend_has_feature(vq, VHOST_BACKEND_F_IOTLB_MSG_V2);
1146
1147 node = vhost_new_msg(vq, v2 ? VHOST_IOTLB_MSG_V2 : VHOST_IOTLB_MSG);
1148 if (!node)
1149 return -ENOMEM;
1150
1151 if (v2) {
1152 node->msg_v2.type = VHOST_IOTLB_MSG_V2;
1153 msg = &node->msg_v2.iotlb;
1154 } else {
1155 msg = &node->msg.iotlb;
1156 }
1157
1158 msg->type = VHOST_IOTLB_MISS;
1159 msg->iova = iova;
1160 msg->perm = access;
1161
1162 vhost_enqueue_msg(dev, &dev->read_list, node);
1163
1164 return 0;
1165 }
1166
1167 static bool vq_access_ok(struct vhost_virtqueue *vq, unsigned int num,
1168 struct vring_desc __user *desc,
1169 struct vring_avail __user *avail,
1170 struct vring_used __user *used)
1171
1172 {
1173 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
1174
1175 return access_ok(VERIFY_READ, desc, num * sizeof *desc) &&
1176 access_ok(VERIFY_READ, avail,
1177 sizeof *avail + num * sizeof *avail->ring + s) &&
1178 access_ok(VERIFY_WRITE, used,
1179 sizeof *used + num * sizeof *used->ring + s);
1180 }
1181
1182 static void vhost_vq_meta_update(struct vhost_virtqueue *vq,
1183 const struct vhost_umem_node *node,
1184 int type)
1185 {
1186 int access = (type == VHOST_ADDR_USED) ?
1187 VHOST_ACCESS_WO : VHOST_ACCESS_RO;
1188
1189 if (likely(node->perm & access))
1190 vq->meta_iotlb[type] = node;
1191 }
1192
1193 static bool iotlb_access_ok(struct vhost_virtqueue *vq,
1194 int access, u64 addr, u64 len, int type)
1195 {
1196 const struct vhost_umem_node *node;
1197 struct vhost_umem *umem = vq->iotlb;
1198 u64 s = 0, size, orig_addr = addr, last = addr + len - 1;
1199
1200 if (vhost_vq_meta_fetch(vq, addr, len, type))
1201 return true;
1202
1203 while (len > s) {
1204 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1205 addr,
1206 last);
1207 if (node == NULL || node->start > addr) {
1208 vhost_iotlb_miss(vq, addr, access);
1209 return false;
1210 } else if (!(node->perm & access)) {
1211 /* Report the possible access violation by
1212 * request another translation from userspace.
1213 */
1214 return false;
1215 }
1216
1217 size = node->size - addr + node->start;
1218
1219 if (orig_addr == addr && size >= len)
1220 vhost_vq_meta_update(vq, node, type);
1221
1222 s += size;
1223 addr += size;
1224 }
1225
1226 return true;
1227 }
1228
1229 int vq_iotlb_prefetch(struct vhost_virtqueue *vq)
1230 {
1231 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
1232 unsigned int num = vq->num;
1233
1234 if (!vq->iotlb)
1235 return 1;
1236
1237 return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc,
1238 num * sizeof(*vq->desc), VHOST_ADDR_DESC) &&
1239 iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail,
1240 sizeof *vq->avail +
1241 num * sizeof(*vq->avail->ring) + s,
1242 VHOST_ADDR_AVAIL) &&
1243 iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used,
1244 sizeof *vq->used +
1245 num * sizeof(*vq->used->ring) + s,
1246 VHOST_ADDR_USED);
1247 }
1248 EXPORT_SYMBOL_GPL(vq_iotlb_prefetch);
1249
1250 /* Can we log writes? */
1251 /* Caller should have device mutex but not vq mutex */
1252 bool vhost_log_access_ok(struct vhost_dev *dev)
1253 {
1254 return memory_access_ok(dev, dev->umem, 1);
1255 }
1256 EXPORT_SYMBOL_GPL(vhost_log_access_ok);
1257
1258 /* Verify access for write logging. */
1259 /* Caller should have vq mutex and device mutex */
1260 static bool vq_log_access_ok(struct vhost_virtqueue *vq,
1261 void __user *log_base)
1262 {
1263 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0;
1264
1265 return vq_memory_access_ok(log_base, vq->umem,
1266 vhost_has_feature(vq, VHOST_F_LOG_ALL)) &&
1267 (!vq->log_used || log_access_ok(log_base, vq->log_addr,
1268 sizeof *vq->used +
1269 vq->num * sizeof *vq->used->ring + s));
1270 }
1271
1272 /* Can we start vq? */
1273 /* Caller should have vq mutex and device mutex */
1274 bool vhost_vq_access_ok(struct vhost_virtqueue *vq)
1275 {
1276 if (!vq_log_access_ok(vq, vq->log_base))
1277 return false;
1278
1279 /* Access validation occurs at prefetch time with IOTLB */
1280 if (vq->iotlb)
1281 return true;
1282
1283 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used);
1284 }
1285 EXPORT_SYMBOL_GPL(vhost_vq_access_ok);
1286
1287 static struct vhost_umem *vhost_umem_alloc(void)
1288 {
1289 struct vhost_umem *umem = kvzalloc(sizeof(*umem), GFP_KERNEL);
1290
1291 if (!umem)
1292 return NULL;
1293
1294 umem->umem_tree = RB_ROOT_CACHED;
1295 umem->numem = 0;
1296 INIT_LIST_HEAD(&umem->umem_list);
1297
1298 return umem;
1299 }
1300
1301 static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
1302 {
1303 struct vhost_memory mem, *newmem;
1304 struct vhost_memory_region *region;
1305 struct vhost_umem *newumem, *oldumem;
1306 unsigned long size = offsetof(struct vhost_memory, regions);
1307 int i;
1308
1309 if (copy_from_user(&mem, m, size))
1310 return -EFAULT;
1311 if (mem.padding)
1312 return -EOPNOTSUPP;
1313 if (mem.nregions > max_mem_regions)
1314 return -E2BIG;
1315 newmem = kvzalloc(struct_size(newmem, regions, mem.nregions),
1316 GFP_KERNEL);
1317 if (!newmem)
1318 return -ENOMEM;
1319
1320 memcpy(newmem, &mem, size);
1321 if (copy_from_user(newmem->regions, m->regions,
1322 mem.nregions * sizeof *m->regions)) {
1323 kvfree(newmem);
1324 return -EFAULT;
1325 }
1326
1327 newumem = vhost_umem_alloc();
1328 if (!newumem) {
1329 kvfree(newmem);
1330 return -ENOMEM;
1331 }
1332
1333 for (region = newmem->regions;
1334 region < newmem->regions + mem.nregions;
1335 region++) {
1336 if (vhost_new_umem_range(newumem,
1337 region->guest_phys_addr,
1338 region->memory_size,
1339 region->guest_phys_addr +
1340 region->memory_size - 1,
1341 region->userspace_addr,
1342 VHOST_ACCESS_RW))
1343 goto err;
1344 }
1345
1346 if (!memory_access_ok(d, newumem, 0))
1347 goto err;
1348
1349 oldumem = d->umem;
1350 d->umem = newumem;
1351
1352 /* All memory accesses are done under some VQ mutex. */
1353 for (i = 0; i < d->nvqs; ++i) {
1354 mutex_lock(&d->vqs[i]->mutex);
1355 d->vqs[i]->umem = newumem;
1356 mutex_unlock(&d->vqs[i]->mutex);
1357 }
1358
1359 kvfree(newmem);
1360 vhost_umem_clean(oldumem);
1361 return 0;
1362
1363 err:
1364 vhost_umem_clean(newumem);
1365 kvfree(newmem);
1366 return -EFAULT;
1367 }
1368
1369 long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
1370 {
1371 struct file *eventfp, *filep = NULL;
1372 bool pollstart = false, pollstop = false;
1373 struct eventfd_ctx *ctx = NULL;
1374 u32 __user *idxp = argp;
1375 struct vhost_virtqueue *vq;
1376 struct vhost_vring_state s;
1377 struct vhost_vring_file f;
1378 struct vhost_vring_addr a;
1379 u32 idx;
1380 long r;
1381
1382 r = get_user(idx, idxp);
1383 if (r < 0)
1384 return r;
1385 if (idx >= d->nvqs)
1386 return -ENOBUFS;
1387
1388 idx = array_index_nospec(idx, d->nvqs);
1389 vq = d->vqs[idx];
1390
1391 mutex_lock(&vq->mutex);
1392
1393 switch (ioctl) {
1394 case VHOST_SET_VRING_NUM:
1395 /* Resizing ring with an active backend?
1396 * You don't want to do that. */
1397 if (vq->private_data) {
1398 r = -EBUSY;
1399 break;
1400 }
1401 if (copy_from_user(&s, argp, sizeof s)) {
1402 r = -EFAULT;
1403 break;
1404 }
1405 if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) {
1406 r = -EINVAL;
1407 break;
1408 }
1409 vq->num = s.num;
1410 break;
1411 case VHOST_SET_VRING_BASE:
1412 /* Moving base with an active backend?
1413 * You don't want to do that. */
1414 if (vq->private_data) {
1415 r = -EBUSY;
1416 break;
1417 }
1418 if (copy_from_user(&s, argp, sizeof s)) {
1419 r = -EFAULT;
1420 break;
1421 }
1422 if (s.num > 0xffff) {
1423 r = -EINVAL;
1424 break;
1425 }
1426 vq->last_avail_idx = s.num;
1427 /* Forget the cached index value. */
1428 vq->avail_idx = vq->last_avail_idx;
1429 break;
1430 case VHOST_GET_VRING_BASE:
1431 s.index = idx;
1432 s.num = vq->last_avail_idx;
1433 if (copy_to_user(argp, &s, sizeof s))
1434 r = -EFAULT;
1435 break;
1436 case VHOST_SET_VRING_ADDR:
1437 if (copy_from_user(&a, argp, sizeof a)) {
1438 r = -EFAULT;
1439 break;
1440 }
1441 if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) {
1442 r = -EOPNOTSUPP;
1443 break;
1444 }
1445 /* For 32bit, verify that the top 32bits of the user
1446 data are set to zero. */
1447 if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr ||
1448 (u64)(unsigned long)a.used_user_addr != a.used_user_addr ||
1449 (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) {
1450 r = -EFAULT;
1451 break;
1452 }
1453
1454 /* Make sure it's safe to cast pointers to vring types. */
1455 BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE);
1456 BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE);
1457 if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) ||
1458 (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) ||
1459 (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) {
1460 r = -EINVAL;
1461 break;
1462 }
1463
1464 /* We only verify access here if backend is configured.
1465 * If it is not, we don't as size might not have been setup.
1466 * We will verify when backend is configured. */
1467 if (vq->private_data) {
1468 if (!vq_access_ok(vq, vq->num,
1469 (void __user *)(unsigned long)a.desc_user_addr,
1470 (void __user *)(unsigned long)a.avail_user_addr,
1471 (void __user *)(unsigned long)a.used_user_addr)) {
1472 r = -EINVAL;
1473 break;
1474 }
1475
1476 /* Also validate log access for used ring if enabled. */
1477 if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) &&
1478 !log_access_ok(vq->log_base, a.log_guest_addr,
1479 sizeof *vq->used +
1480 vq->num * sizeof *vq->used->ring)) {
1481 r = -EINVAL;
1482 break;
1483 }
1484 }
1485
1486 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG));
1487 vq->desc = (void __user *)(unsigned long)a.desc_user_addr;
1488 vq->avail = (void __user *)(unsigned long)a.avail_user_addr;
1489 vq->log_addr = a.log_guest_addr;
1490 vq->used = (void __user *)(unsigned long)a.used_user_addr;
1491 break;
1492 case VHOST_SET_VRING_KICK:
1493 if (copy_from_user(&f, argp, sizeof f)) {
1494 r = -EFAULT;
1495 break;
1496 }
1497 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd);
1498 if (IS_ERR(eventfp)) {
1499 r = PTR_ERR(eventfp);
1500 break;
1501 }
1502 if (eventfp != vq->kick) {
1503 pollstop = (filep = vq->kick) != NULL;
1504 pollstart = (vq->kick = eventfp) != NULL;
1505 } else
1506 filep = eventfp;
1507 break;
1508 case VHOST_SET_VRING_CALL:
1509 if (copy_from_user(&f, argp, sizeof f)) {
1510 r = -EFAULT;
1511 break;
1512 }
1513 ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd);
1514 if (IS_ERR(ctx)) {
1515 r = PTR_ERR(ctx);
1516 break;
1517 }
1518 swap(ctx, vq->call_ctx);
1519 break;
1520 case VHOST_SET_VRING_ERR:
1521 if (copy_from_user(&f, argp, sizeof f)) {
1522 r = -EFAULT;
1523 break;
1524 }
1525 ctx = f.fd == -1 ? NULL : eventfd_ctx_fdget(f.fd);
1526 if (IS_ERR(ctx)) {
1527 r = PTR_ERR(ctx);
1528 break;
1529 }
1530 swap(ctx, vq->error_ctx);
1531 break;
1532 case VHOST_SET_VRING_ENDIAN:
1533 r = vhost_set_vring_endian(vq, argp);
1534 break;
1535 case VHOST_GET_VRING_ENDIAN:
1536 r = vhost_get_vring_endian(vq, idx, argp);
1537 break;
1538 case VHOST_SET_VRING_BUSYLOOP_TIMEOUT:
1539 if (copy_from_user(&s, argp, sizeof(s))) {
1540 r = -EFAULT;
1541 break;
1542 }
1543 vq->busyloop_timeout = s.num;
1544 break;
1545 case VHOST_GET_VRING_BUSYLOOP_TIMEOUT:
1546 s.index = idx;
1547 s.num = vq->busyloop_timeout;
1548 if (copy_to_user(argp, &s, sizeof(s)))
1549 r = -EFAULT;
1550 break;
1551 default:
1552 r = -ENOIOCTLCMD;
1553 }
1554
1555 if (pollstop && vq->handle_kick)
1556 vhost_poll_stop(&vq->poll);
1557
1558 if (!IS_ERR_OR_NULL(ctx))
1559 eventfd_ctx_put(ctx);
1560 if (filep)
1561 fput(filep);
1562
1563 if (pollstart && vq->handle_kick)
1564 r = vhost_poll_start(&vq->poll, vq->kick);
1565
1566 mutex_unlock(&vq->mutex);
1567
1568 if (pollstop && vq->handle_kick)
1569 vhost_poll_flush(&vq->poll);
1570 return r;
1571 }
1572 EXPORT_SYMBOL_GPL(vhost_vring_ioctl);
1573
1574 int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled)
1575 {
1576 struct vhost_umem *niotlb, *oiotlb;
1577 int i;
1578
1579 niotlb = vhost_umem_alloc();
1580 if (!niotlb)
1581 return -ENOMEM;
1582
1583 oiotlb = d->iotlb;
1584 d->iotlb = niotlb;
1585
1586 for (i = 0; i < d->nvqs; ++i) {
1587 struct vhost_virtqueue *vq = d->vqs[i];
1588
1589 mutex_lock(&vq->mutex);
1590 vq->iotlb = niotlb;
1591 __vhost_vq_meta_reset(vq);
1592 mutex_unlock(&vq->mutex);
1593 }
1594
1595 vhost_umem_clean(oiotlb);
1596
1597 return 0;
1598 }
1599 EXPORT_SYMBOL_GPL(vhost_init_device_iotlb);
1600
1601 /* Caller must have device mutex */
1602 long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp)
1603 {
1604 struct eventfd_ctx *ctx;
1605 u64 p;
1606 long r;
1607 int i, fd;
1608
1609 /* If you are not the owner, you can become one */
1610 if (ioctl == VHOST_SET_OWNER) {
1611 r = vhost_dev_set_owner(d);
1612 goto done;
1613 }
1614
1615 /* You must be the owner to do anything else */
1616 r = vhost_dev_check_owner(d);
1617 if (r)
1618 goto done;
1619
1620 switch (ioctl) {
1621 case VHOST_SET_MEM_TABLE:
1622 r = vhost_set_memory(d, argp);
1623 break;
1624 case VHOST_SET_LOG_BASE:
1625 if (copy_from_user(&p, argp, sizeof p)) {
1626 r = -EFAULT;
1627 break;
1628 }
1629 if ((u64)(unsigned long)p != p) {
1630 r = -EFAULT;
1631 break;
1632 }
1633 for (i = 0; i < d->nvqs; ++i) {
1634 struct vhost_virtqueue *vq;
1635 void __user *base = (void __user *)(unsigned long)p;
1636 vq = d->vqs[i];
1637 mutex_lock(&vq->mutex);
1638 /* If ring is inactive, will check when it's enabled. */
1639 if (vq->private_data && !vq_log_access_ok(vq, base))
1640 r = -EFAULT;
1641 else
1642 vq->log_base = base;
1643 mutex_unlock(&vq->mutex);
1644 }
1645 break;
1646 case VHOST_SET_LOG_FD:
1647 r = get_user(fd, (int __user *)argp);
1648 if (r < 0)
1649 break;
1650 ctx = fd == -1 ? NULL : eventfd_ctx_fdget(fd);
1651 if (IS_ERR(ctx)) {
1652 r = PTR_ERR(ctx);
1653 break;
1654 }
1655 swap(ctx, d->log_ctx);
1656 for (i = 0; i < d->nvqs; ++i) {
1657 mutex_lock(&d->vqs[i]->mutex);
1658 d->vqs[i]->log_ctx = d->log_ctx;
1659 mutex_unlock(&d->vqs[i]->mutex);
1660 }
1661 if (ctx)
1662 eventfd_ctx_put(ctx);
1663 break;
1664 default:
1665 r = -ENOIOCTLCMD;
1666 break;
1667 }
1668 done:
1669 return r;
1670 }
1671 EXPORT_SYMBOL_GPL(vhost_dev_ioctl);
1672
1673 /* TODO: This is really inefficient. We need something like get_user()
1674 * (instruction directly accesses the data, with an exception table entry
1675 * returning -EFAULT). See Documentation/x86/exception-tables.txt.
1676 */
1677 static int set_bit_to_user(int nr, void __user *addr)
1678 {
1679 unsigned long log = (unsigned long)addr;
1680 struct page *page;
1681 void *base;
1682 int bit = nr + (log % PAGE_SIZE) * 8;
1683 int r;
1684
1685 r = get_user_pages_fast(log, 1, 1, &page);
1686 if (r < 0)
1687 return r;
1688 BUG_ON(r != 1);
1689 base = kmap_atomic(page);
1690 set_bit(bit, base);
1691 kunmap_atomic(base);
1692 set_page_dirty_lock(page);
1693 put_page(page);
1694 return 0;
1695 }
1696
1697 static int log_write(void __user *log_base,
1698 u64 write_address, u64 write_length)
1699 {
1700 u64 write_page = write_address / VHOST_PAGE_SIZE;
1701 int r;
1702
1703 if (!write_length)
1704 return 0;
1705 write_length += write_address % VHOST_PAGE_SIZE;
1706 for (;;) {
1707 u64 base = (u64)(unsigned long)log_base;
1708 u64 log = base + write_page / 8;
1709 int bit = write_page % 8;
1710 if ((u64)(unsigned long)log != log)
1711 return -EFAULT;
1712 r = set_bit_to_user(bit, (void __user *)(unsigned long)log);
1713 if (r < 0)
1714 return r;
1715 if (write_length <= VHOST_PAGE_SIZE)
1716 break;
1717 write_length -= VHOST_PAGE_SIZE;
1718 write_page += 1;
1719 }
1720 return r;
1721 }
1722
1723 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
1724 unsigned int log_num, u64 len)
1725 {
1726 int i, r;
1727
1728 /* Make sure data written is seen before log. */
1729 smp_wmb();
1730 for (i = 0; i < log_num; ++i) {
1731 u64 l = min(log[i].len, len);
1732 r = log_write(vq->log_base, log[i].addr, l);
1733 if (r < 0)
1734 return r;
1735 len -= l;
1736 if (!len) {
1737 if (vq->log_ctx)
1738 eventfd_signal(vq->log_ctx, 1);
1739 return 0;
1740 }
1741 }
1742 /* Length written exceeds what we have stored. This is a bug. */
1743 BUG();
1744 return 0;
1745 }
1746 EXPORT_SYMBOL_GPL(vhost_log_write);
1747
1748 static int vhost_update_used_flags(struct vhost_virtqueue *vq)
1749 {
1750 void __user *used;
1751 if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags),
1752 &vq->used->flags) < 0)
1753 return -EFAULT;
1754 if (unlikely(vq->log_used)) {
1755 /* Make sure the flag is seen before log. */
1756 smp_wmb();
1757 /* Log used flag write. */
1758 used = &vq->used->flags;
1759 log_write(vq->log_base, vq->log_addr +
1760 (used - (void __user *)vq->used),
1761 sizeof vq->used->flags);
1762 if (vq->log_ctx)
1763 eventfd_signal(vq->log_ctx, 1);
1764 }
1765 return 0;
1766 }
1767
1768 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event)
1769 {
1770 if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx),
1771 vhost_avail_event(vq)))
1772 return -EFAULT;
1773 if (unlikely(vq->log_used)) {
1774 void __user *used;
1775 /* Make sure the event is seen before log. */
1776 smp_wmb();
1777 /* Log avail event write */
1778 used = vhost_avail_event(vq);
1779 log_write(vq->log_base, vq->log_addr +
1780 (used - (void __user *)vq->used),
1781 sizeof *vhost_avail_event(vq));
1782 if (vq->log_ctx)
1783 eventfd_signal(vq->log_ctx, 1);
1784 }
1785 return 0;
1786 }
1787
1788 int vhost_vq_init_access(struct vhost_virtqueue *vq)
1789 {
1790 __virtio16 last_used_idx;
1791 int r;
1792 bool is_le = vq->is_le;
1793
1794 if (!vq->private_data)
1795 return 0;
1796
1797 vhost_init_is_le(vq);
1798
1799 r = vhost_update_used_flags(vq);
1800 if (r)
1801 goto err;
1802 vq->signalled_used_valid = false;
1803 if (!vq->iotlb &&
1804 !access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx)) {
1805 r = -EFAULT;
1806 goto err;
1807 }
1808 r = vhost_get_used(vq, last_used_idx, &vq->used->idx);
1809 if (r) {
1810 vq_err(vq, "Can't access used idx at %p\n",
1811 &vq->used->idx);
1812 goto err;
1813 }
1814 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx);
1815 return 0;
1816
1817 err:
1818 vq->is_le = is_le;
1819 return r;
1820 }
1821 EXPORT_SYMBOL_GPL(vhost_vq_init_access);
1822
1823 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len,
1824 struct iovec iov[], int iov_size, int access)
1825 {
1826 const struct vhost_umem_node *node;
1827 struct vhost_dev *dev = vq->dev;
1828 struct vhost_umem *umem = dev->iotlb ? dev->iotlb : dev->umem;
1829 struct iovec *_iov;
1830 u64 s = 0;
1831 int ret = 0;
1832
1833 while ((u64)len > s) {
1834 u64 size;
1835 if (unlikely(ret >= iov_size)) {
1836 ret = -ENOBUFS;
1837 break;
1838 }
1839
1840 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree,
1841 addr, addr + len - 1);
1842 if (node == NULL || node->start > addr) {
1843 if (umem != dev->iotlb) {
1844 ret = -EFAULT;
1845 break;
1846 }
1847 ret = -EAGAIN;
1848 break;
1849 } else if (!(node->perm & access)) {
1850 ret = -EPERM;
1851 break;
1852 }
1853
1854 _iov = iov + ret;
1855 size = node->size - addr + node->start;
1856 _iov->iov_len = min((u64)len - s, size);
1857 _iov->iov_base = (void __user *)(unsigned long)
1858 (node->userspace_addr + addr - node->start);
1859 s += size;
1860 addr += size;
1861 ++ret;
1862 }
1863
1864 if (ret == -EAGAIN)
1865 vhost_iotlb_miss(vq, addr, access);
1866 return ret;
1867 }
1868
1869 /* Each buffer in the virtqueues is actually a chain of descriptors. This
1870 * function returns the next descriptor in the chain,
1871 * or -1U if we're at the end. */
1872 static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc)
1873 {
1874 unsigned int next;
1875
1876 /* If this descriptor says it doesn't chain, we're done. */
1877 if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT)))
1878 return -1U;
1879
1880 /* Check they're not leading us off end of descriptors. */
1881 next = vhost16_to_cpu(vq, READ_ONCE(desc->next));
1882 return next;
1883 }
1884
1885 static int get_indirect(struct vhost_virtqueue *vq,
1886 struct iovec iov[], unsigned int iov_size,
1887 unsigned int *out_num, unsigned int *in_num,
1888 struct vhost_log *log, unsigned int *log_num,
1889 struct vring_desc *indirect)
1890 {
1891 struct vring_desc desc;
1892 unsigned int i = 0, count, found = 0;
1893 u32 len = vhost32_to_cpu(vq, indirect->len);
1894 struct iov_iter from;
1895 int ret, access;
1896
1897 /* Sanity check */
1898 if (unlikely(len % sizeof desc)) {
1899 vq_err(vq, "Invalid length in indirect descriptor: "
1900 "len 0x%llx not multiple of 0x%zx\n",
1901 (unsigned long long)len,
1902 sizeof desc);
1903 return -EINVAL;
1904 }
1905
1906 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect,
1907 UIO_MAXIOV, VHOST_ACCESS_RO);
1908 if (unlikely(ret < 0)) {
1909 if (ret != -EAGAIN)
1910 vq_err(vq, "Translation failure %d in indirect.\n", ret);
1911 return ret;
1912 }
1913 iov_iter_init(&from, READ, vq->indirect, ret, len);
1914
1915 /* We will use the result as an address to read from, so most
1916 * architectures only need a compiler barrier here. */
1917 read_barrier_depends();
1918
1919 count = len / sizeof desc;
1920 /* Buffers are chained via a 16 bit next field, so
1921 * we can have at most 2^16 of these. */
1922 if (unlikely(count > USHRT_MAX + 1)) {
1923 vq_err(vq, "Indirect buffer length too big: %d\n",
1924 indirect->len);
1925 return -E2BIG;
1926 }
1927
1928 do {
1929 unsigned iov_count = *in_num + *out_num;
1930 if (unlikely(++found > count)) {
1931 vq_err(vq, "Loop detected: last one at %u "
1932 "indirect size %u\n",
1933 i, count);
1934 return -EINVAL;
1935 }
1936 if (unlikely(!copy_from_iter_full(&desc, sizeof(desc), &from))) {
1937 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n",
1938 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
1939 return -EINVAL;
1940 }
1941 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) {
1942 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n",
1943 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc);
1944 return -EINVAL;
1945 }
1946
1947 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
1948 access = VHOST_ACCESS_WO;
1949 else
1950 access = VHOST_ACCESS_RO;
1951
1952 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
1953 vhost32_to_cpu(vq, desc.len), iov + iov_count,
1954 iov_size - iov_count, access);
1955 if (unlikely(ret < 0)) {
1956 if (ret != -EAGAIN)
1957 vq_err(vq, "Translation failure %d indirect idx %d\n",
1958 ret, i);
1959 return ret;
1960 }
1961 /* If this is an input descriptor, increment that count. */
1962 if (access == VHOST_ACCESS_WO) {
1963 *in_num += ret;
1964 if (unlikely(log)) {
1965 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
1966 log[*log_num].len = vhost32_to_cpu(vq, desc.len);
1967 ++*log_num;
1968 }
1969 } else {
1970 /* If it's an output descriptor, they're all supposed
1971 * to come before any input descriptors. */
1972 if (unlikely(*in_num)) {
1973 vq_err(vq, "Indirect descriptor "
1974 "has out after in: idx %d\n", i);
1975 return -EINVAL;
1976 }
1977 *out_num += ret;
1978 }
1979 } while ((i = next_desc(vq, &desc)) != -1);
1980 return 0;
1981 }
1982
1983 /* This looks in the virtqueue and for the first available buffer, and converts
1984 * it to an iovec for convenient access. Since descriptors consist of some
1985 * number of output then some number of input descriptors, it's actually two
1986 * iovecs, but we pack them into one and note how many of each there were.
1987 *
1988 * This function returns the descriptor number found, or vq->num (which is
1989 * never a valid descriptor number) if none was found. A negative code is
1990 * returned on error. */
1991 int vhost_get_vq_desc(struct vhost_virtqueue *vq,
1992 struct iovec iov[], unsigned int iov_size,
1993 unsigned int *out_num, unsigned int *in_num,
1994 struct vhost_log *log, unsigned int *log_num)
1995 {
1996 struct vring_desc desc;
1997 unsigned int i, head, found = 0;
1998 u16 last_avail_idx;
1999 __virtio16 avail_idx;
2000 __virtio16 ring_head;
2001 int ret, access;
2002
2003 /* Check it isn't doing very strange things with descriptor numbers. */
2004 last_avail_idx = vq->last_avail_idx;
2005
2006 if (vq->avail_idx == vq->last_avail_idx) {
2007 if (unlikely(vhost_get_avail(vq, avail_idx, &vq->avail->idx))) {
2008 vq_err(vq, "Failed to access avail idx at %p\n",
2009 &vq->avail->idx);
2010 return -EFAULT;
2011 }
2012 vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
2013
2014 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
2015 vq_err(vq, "Guest moved used index from %u to %u",
2016 last_avail_idx, vq->avail_idx);
2017 return -EFAULT;
2018 }
2019
2020 /* If there's nothing new since last we looked, return
2021 * invalid.
2022 */
2023 if (vq->avail_idx == last_avail_idx)
2024 return vq->num;
2025
2026 /* Only get avail ring entries after they have been
2027 * exposed by guest.
2028 */
2029 smp_rmb();
2030 }
2031
2032 /* Grab the next descriptor number they're advertising, and increment
2033 * the index we've seen. */
2034 if (unlikely(vhost_get_avail(vq, ring_head,
2035 &vq->avail->ring[last_avail_idx & (vq->num - 1)]))) {
2036 vq_err(vq, "Failed to read head: idx %d address %p\n",
2037 last_avail_idx,
2038 &vq->avail->ring[last_avail_idx % vq->num]);
2039 return -EFAULT;
2040 }
2041
2042 head = vhost16_to_cpu(vq, ring_head);
2043
2044 /* If their number is silly, that's an error. */
2045 if (unlikely(head >= vq->num)) {
2046 vq_err(vq, "Guest says index %u > %u is available",
2047 head, vq->num);
2048 return -EINVAL;
2049 }
2050
2051 /* When we start there are none of either input nor output. */
2052 *out_num = *in_num = 0;
2053 if (unlikely(log))
2054 *log_num = 0;
2055
2056 i = head;
2057 do {
2058 unsigned iov_count = *in_num + *out_num;
2059 if (unlikely(i >= vq->num)) {
2060 vq_err(vq, "Desc index is %u > %u, head = %u",
2061 i, vq->num, head);
2062 return -EINVAL;
2063 }
2064 if (unlikely(++found > vq->num)) {
2065 vq_err(vq, "Loop detected: last one at %u "
2066 "vq size %u head %u\n",
2067 i, vq->num, head);
2068 return -EINVAL;
2069 }
2070 ret = vhost_copy_from_user(vq, &desc, vq->desc + i,
2071 sizeof desc);
2072 if (unlikely(ret)) {
2073 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n",
2074 i, vq->desc + i);
2075 return -EFAULT;
2076 }
2077 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) {
2078 ret = get_indirect(vq, iov, iov_size,
2079 out_num, in_num,
2080 log, log_num, &desc);
2081 if (unlikely(ret < 0)) {
2082 if (ret != -EAGAIN)
2083 vq_err(vq, "Failure detected "
2084 "in indirect descriptor at idx %d\n", i);
2085 return ret;
2086 }
2087 continue;
2088 }
2089
2090 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE))
2091 access = VHOST_ACCESS_WO;
2092 else
2093 access = VHOST_ACCESS_RO;
2094 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr),
2095 vhost32_to_cpu(vq, desc.len), iov + iov_count,
2096 iov_size - iov_count, access);
2097 if (unlikely(ret < 0)) {
2098 if (ret != -EAGAIN)
2099 vq_err(vq, "Translation failure %d descriptor idx %d\n",
2100 ret, i);
2101 return ret;
2102 }
2103 if (access == VHOST_ACCESS_WO) {
2104 /* If this is an input descriptor,
2105 * increment that count. */
2106 *in_num += ret;
2107 if (unlikely(log)) {
2108 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr);
2109 log[*log_num].len = vhost32_to_cpu(vq, desc.len);
2110 ++*log_num;
2111 }
2112 } else {
2113 /* If it's an output descriptor, they're all supposed
2114 * to come before any input descriptors. */
2115 if (unlikely(*in_num)) {
2116 vq_err(vq, "Descriptor has out after in: "
2117 "idx %d\n", i);
2118 return -EINVAL;
2119 }
2120 *out_num += ret;
2121 }
2122 } while ((i = next_desc(vq, &desc)) != -1);
2123
2124 /* On success, increment avail index. */
2125 vq->last_avail_idx++;
2126
2127 /* Assume notifications from guest are disabled at this point,
2128 * if they aren't we would need to update avail_event index. */
2129 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
2130 return head;
2131 }
2132 EXPORT_SYMBOL_GPL(vhost_get_vq_desc);
2133
2134 /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */
2135 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n)
2136 {
2137 vq->last_avail_idx -= n;
2138 }
2139 EXPORT_SYMBOL_GPL(vhost_discard_vq_desc);
2140
2141 /* After we've used one of their buffers, we tell them about it. We'll then
2142 * want to notify the guest, using eventfd. */
2143 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len)
2144 {
2145 struct vring_used_elem heads = {
2146 cpu_to_vhost32(vq, head),
2147 cpu_to_vhost32(vq, len)
2148 };
2149
2150 return vhost_add_used_n(vq, &heads, 1);
2151 }
2152 EXPORT_SYMBOL_GPL(vhost_add_used);
2153
2154 static int __vhost_add_used_n(struct vhost_virtqueue *vq,
2155 struct vring_used_elem *heads,
2156 unsigned count)
2157 {
2158 struct vring_used_elem __user *used;
2159 u16 old, new;
2160 int start;
2161
2162 start = vq->last_used_idx & (vq->num - 1);
2163 used = vq->used->ring + start;
2164 if (count == 1) {
2165 if (vhost_put_user(vq, heads[0].id, &used->id)) {
2166 vq_err(vq, "Failed to write used id");
2167 return -EFAULT;
2168 }
2169 if (vhost_put_user(vq, heads[0].len, &used->len)) {
2170 vq_err(vq, "Failed to write used len");
2171 return -EFAULT;
2172 }
2173 } else if (vhost_copy_to_user(vq, used, heads, count * sizeof *used)) {
2174 vq_err(vq, "Failed to write used");
2175 return -EFAULT;
2176 }
2177 if (unlikely(vq->log_used)) {
2178 /* Make sure data is seen before log. */
2179 smp_wmb();
2180 /* Log used ring entry write. */
2181 log_write(vq->log_base,
2182 vq->log_addr +
2183 ((void __user *)used - (void __user *)vq->used),
2184 count * sizeof *used);
2185 }
2186 old = vq->last_used_idx;
2187 new = (vq->last_used_idx += count);
2188 /* If the driver never bothers to signal in a very long while,
2189 * used index might wrap around. If that happens, invalidate
2190 * signalled_used index we stored. TODO: make sure driver
2191 * signals at least once in 2^16 and remove this. */
2192 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old)))
2193 vq->signalled_used_valid = false;
2194 return 0;
2195 }
2196
2197 /* After we've used one of their buffers, we tell them about it. We'll then
2198 * want to notify the guest, using eventfd. */
2199 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads,
2200 unsigned count)
2201 {
2202 int start, n, r;
2203
2204 start = vq->last_used_idx & (vq->num - 1);
2205 n = vq->num - start;
2206 if (n < count) {
2207 r = __vhost_add_used_n(vq, heads, n);
2208 if (r < 0)
2209 return r;
2210 heads += n;
2211 count -= n;
2212 }
2213 r = __vhost_add_used_n(vq, heads, count);
2214
2215 /* Make sure buffer is written before we update index. */
2216 smp_wmb();
2217 if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx),
2218 &vq->used->idx)) {
2219 vq_err(vq, "Failed to increment used idx");
2220 return -EFAULT;
2221 }
2222 if (unlikely(vq->log_used)) {
2223 /* Log used index update. */
2224 log_write(vq->log_base,
2225 vq->log_addr + offsetof(struct vring_used, idx),
2226 sizeof vq->used->idx);
2227 if (vq->log_ctx)
2228 eventfd_signal(vq->log_ctx, 1);
2229 }
2230 return r;
2231 }
2232 EXPORT_SYMBOL_GPL(vhost_add_used_n);
2233
2234 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2235 {
2236 __u16 old, new;
2237 __virtio16 event;
2238 bool v;
2239 /* Flush out used index updates. This is paired
2240 * with the barrier that the Guest executes when enabling
2241 * interrupts. */
2242 smp_mb();
2243
2244 if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) &&
2245 unlikely(vq->avail_idx == vq->last_avail_idx))
2246 return true;
2247
2248 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
2249 __virtio16 flags;
2250 if (vhost_get_avail(vq, flags, &vq->avail->flags)) {
2251 vq_err(vq, "Failed to get flags");
2252 return true;
2253 }
2254 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT));
2255 }
2256 old = vq->signalled_used;
2257 v = vq->signalled_used_valid;
2258 new = vq->signalled_used = vq->last_used_idx;
2259 vq->signalled_used_valid = true;
2260
2261 if (unlikely(!v))
2262 return true;
2263
2264 if (vhost_get_avail(vq, event, vhost_used_event(vq))) {
2265 vq_err(vq, "Failed to get used event idx");
2266 return true;
2267 }
2268 return vring_need_event(vhost16_to_cpu(vq, event), new, old);
2269 }
2270
2271 /* This actually signals the guest, using eventfd. */
2272 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2273 {
2274 /* Signal the Guest tell them we used something up. */
2275 if (vq->call_ctx && vhost_notify(dev, vq))
2276 eventfd_signal(vq->call_ctx, 1);
2277 }
2278 EXPORT_SYMBOL_GPL(vhost_signal);
2279
2280 /* And here's the combo meal deal. Supersize me! */
2281 void vhost_add_used_and_signal(struct vhost_dev *dev,
2282 struct vhost_virtqueue *vq,
2283 unsigned int head, int len)
2284 {
2285 vhost_add_used(vq, head, len);
2286 vhost_signal(dev, vq);
2287 }
2288 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal);
2289
2290 /* multi-buffer version of vhost_add_used_and_signal */
2291 void vhost_add_used_and_signal_n(struct vhost_dev *dev,
2292 struct vhost_virtqueue *vq,
2293 struct vring_used_elem *heads, unsigned count)
2294 {
2295 vhost_add_used_n(vq, heads, count);
2296 vhost_signal(dev, vq);
2297 }
2298 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n);
2299
2300 /* return true if we're sure that avaiable ring is empty */
2301 bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2302 {
2303 __virtio16 avail_idx;
2304 int r;
2305
2306 if (vq->avail_idx != vq->last_avail_idx)
2307 return false;
2308
2309 r = vhost_get_avail(vq, avail_idx, &vq->avail->idx);
2310 if (unlikely(r))
2311 return false;
2312 vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
2313
2314 return vq->avail_idx == vq->last_avail_idx;
2315 }
2316 EXPORT_SYMBOL_GPL(vhost_vq_avail_empty);
2317
2318 /* OK, now we need to know about added descriptors. */
2319 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2320 {
2321 __virtio16 avail_idx;
2322 int r;
2323
2324 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
2325 return false;
2326 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY;
2327 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
2328 r = vhost_update_used_flags(vq);
2329 if (r) {
2330 vq_err(vq, "Failed to enable notification at %p: %d\n",
2331 &vq->used->flags, r);
2332 return false;
2333 }
2334 } else {
2335 r = vhost_update_avail_event(vq, vq->avail_idx);
2336 if (r) {
2337 vq_err(vq, "Failed to update avail event index at %p: %d\n",
2338 vhost_avail_event(vq), r);
2339 return false;
2340 }
2341 }
2342 /* They could have slipped one in as we were doing that: make
2343 * sure it's written, then check again. */
2344 smp_mb();
2345 r = vhost_get_avail(vq, avail_idx, &vq->avail->idx);
2346 if (r) {
2347 vq_err(vq, "Failed to check avail idx at %p: %d\n",
2348 &vq->avail->idx, r);
2349 return false;
2350 }
2351
2352 return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx;
2353 }
2354 EXPORT_SYMBOL_GPL(vhost_enable_notify);
2355
2356 /* We don't need to be notified again. */
2357 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2358 {
2359 int r;
2360
2361 if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
2362 return;
2363 vq->used_flags |= VRING_USED_F_NO_NOTIFY;
2364 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) {
2365 r = vhost_update_used_flags(vq);
2366 if (r)
2367 vq_err(vq, "Failed to enable notification at %p: %d\n",
2368 &vq->used->flags, r);
2369 }
2370 }
2371 EXPORT_SYMBOL_GPL(vhost_disable_notify);
2372
2373 /* Create a new message. */
2374 struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type)
2375 {
2376 struct vhost_msg_node *node = kmalloc(sizeof *node, GFP_KERNEL);
2377 if (!node)
2378 return NULL;
2379
2380 /* Make sure all padding within the structure is initialized. */
2381 memset(&node->msg, 0, sizeof node->msg);
2382 node->vq = vq;
2383 node->msg.type = type;
2384 return node;
2385 }
2386 EXPORT_SYMBOL_GPL(vhost_new_msg);
2387
2388 void vhost_enqueue_msg(struct vhost_dev *dev, struct list_head *head,
2389 struct vhost_msg_node *node)
2390 {
2391 spin_lock(&dev->iotlb_lock);
2392 list_add_tail(&node->node, head);
2393 spin_unlock(&dev->iotlb_lock);
2394
2395 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM);
2396 }
2397 EXPORT_SYMBOL_GPL(vhost_enqueue_msg);
2398
2399 struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev,
2400 struct list_head *head)
2401 {
2402 struct vhost_msg_node *node = NULL;
2403
2404 spin_lock(&dev->iotlb_lock);
2405 if (!list_empty(head)) {
2406 node = list_first_entry(head, struct vhost_msg_node,
2407 node);
2408 list_del(&node->node);
2409 }
2410 spin_unlock(&dev->iotlb_lock);
2411
2412 return node;
2413 }
2414 EXPORT_SYMBOL_GPL(vhost_dequeue_msg);
2415
2416
2417 static int __init vhost_init(void)
2418 {
2419 return 0;
2420 }
2421
2422 static void __exit vhost_exit(void)
2423 {
2424 }
2425
2426 module_init(vhost_init);
2427 module_exit(vhost_exit);
2428
2429 MODULE_VERSION("0.0.1");
2430 MODULE_LICENSE("GPL v2");
2431 MODULE_AUTHOR("Michael S. Tsirkin");
2432 MODULE_DESCRIPTION("Host kernel accelerator for virtio");
Linux with added FPC-III support