?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
gpio: rcar: Fix runtime PM imbalance on error
[linux/fpc-iii.git] / fs / fuse / virtio_fs.c
blobbade74768903336bbc7bd2ccc90298c953e94f32
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * virtio-fs: Virtio Filesystem
4 * Copyright (C) 2018 Red Hat, Inc.
5 */
6
7 #include <linux/fs.h>
8 #include <linux/module.h>
9 #include <linux/virtio.h>
10 #include <linux/virtio_fs.h>
11 #include <linux/delay.h>
12 #include <linux/fs_context.h>
13 #include <linux/highmem.h>
14 #include "fuse_i.h"
15
16 /* List of virtio-fs device instances and a lock for the list. Also provides
17 * mutual exclusion in device removal and mounting path
18 */
19 static DEFINE_MUTEX(virtio_fs_mutex);
20 static LIST_HEAD(virtio_fs_instances);
21
22 enum {
23 VQ_HIPRIO,
24 VQ_REQUEST
25 };
26
27 /* Per-virtqueue state */
28 struct virtio_fs_vq {
29 spinlock_t lock;
30 struct virtqueue *vq; /* protected by ->lock */
31 struct work_struct done_work;
32 struct list_head queued_reqs;
33 struct list_head end_reqs; /* End these requests */
34 struct delayed_work dispatch_work;
35 struct fuse_dev *fud;
36 bool connected;
37 long in_flight;
38 struct completion in_flight_zero; /* No inflight requests */
39 char name[24];
40 } ____cacheline_aligned_in_smp;
41
42 /* A virtio-fs device instance */
43 struct virtio_fs {
44 struct kref refcount;
45 struct list_head list; /* on virtio_fs_instances */
46 char *tag;
47 struct virtio_fs_vq *vqs;
48 unsigned int nvqs; /* number of virtqueues */
49 unsigned int num_request_queues; /* number of request queues */
50 };
51
52 struct virtio_fs_forget_req {
53 struct fuse_in_header ih;
54 struct fuse_forget_in arg;
55 };
56
57 struct virtio_fs_forget {
58 /* This request can be temporarily queued on virt queue */
59 struct list_head list;
60 struct virtio_fs_forget_req req;
61 };
62
63 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
64 struct fuse_req *req, bool in_flight);
65
66 static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq)
67 {
68 struct virtio_fs *fs = vq->vdev->priv;
69
70 return &fs->vqs[vq->index];
71 }
72
73 static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq)
74 {
75 return &vq_to_fsvq(vq)->fud->pq;
76 }
77
78 /* Should be called with fsvq->lock held. */
79 static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq)
80 {
81 fsvq->in_flight++;
82 }
83
84 /* Should be called with fsvq->lock held. */
85 static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq)
86 {
87 WARN_ON(fsvq->in_flight <= 0);
88 fsvq->in_flight--;
89 if (!fsvq->in_flight)
90 complete(&fsvq->in_flight_zero);
91 }
92
93 static void release_virtio_fs_obj(struct kref *ref)
94 {
95 struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount);
96
97 kfree(vfs->vqs);
98 kfree(vfs);
99 }
100
101 /* Make sure virtiofs_mutex is held */
102 static void virtio_fs_put(struct virtio_fs *fs)
103 {
104 kref_put(&fs->refcount, release_virtio_fs_obj);
105 }
106
107 static void virtio_fs_fiq_release(struct fuse_iqueue *fiq)
108 {
109 struct virtio_fs *vfs = fiq->priv;
110
111 mutex_lock(&virtio_fs_mutex);
112 virtio_fs_put(vfs);
113 mutex_unlock(&virtio_fs_mutex);
114 }
115
116 static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)
117 {
118 WARN_ON(fsvq->in_flight < 0);
119
120 /* Wait for in flight requests to finish.*/
121 spin_lock(&fsvq->lock);
122 if (fsvq->in_flight) {
123 /* We are holding virtio_fs_mutex. There should not be any
124 * waiters waiting for completion.
125 */
126 reinit_completion(&fsvq->in_flight_zero);
127 spin_unlock(&fsvq->lock);
128 wait_for_completion(&fsvq->in_flight_zero);
129 } else {
130 spin_unlock(&fsvq->lock);
131 }
132
133 flush_work(&fsvq->done_work);
134 flush_delayed_work(&fsvq->dispatch_work);
135 }
136
137 static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs)
138 {
139 struct virtio_fs_vq *fsvq;
140 int i;
141
142 for (i = 0; i < fs->nvqs; i++) {
143 fsvq = &fs->vqs[i];
144 virtio_fs_drain_queue(fsvq);
145 }
146 }
147
148 static void virtio_fs_drain_all_queues(struct virtio_fs *fs)
149 {
150 /* Provides mutual exclusion between ->remove and ->kill_sb
151 * paths. We don't want both of these draining queue at the
152 * same time. Current completion logic reinits completion
153 * and that means there should not be any other thread
154 * doing reinit or waiting for completion already.
155 */
156 mutex_lock(&virtio_fs_mutex);
157 virtio_fs_drain_all_queues_locked(fs);
158 mutex_unlock(&virtio_fs_mutex);
159 }
160
161 static void virtio_fs_start_all_queues(struct virtio_fs *fs)
162 {
163 struct virtio_fs_vq *fsvq;
164 int i;
165
166 for (i = 0; i < fs->nvqs; i++) {
167 fsvq = &fs->vqs[i];
168 spin_lock(&fsvq->lock);
169 fsvq->connected = true;
170 spin_unlock(&fsvq->lock);
171 }
172 }
173
174 /* Add a new instance to the list or return -EEXIST if tag name exists*/
175 static int virtio_fs_add_instance(struct virtio_fs *fs)
176 {
177 struct virtio_fs *fs2;
178 bool duplicate = false;
179
180 mutex_lock(&virtio_fs_mutex);
181
182 list_for_each_entry(fs2, &virtio_fs_instances, list) {
183 if (strcmp(fs->tag, fs2->tag) == 0)
184 duplicate = true;
185 }
186
187 if (!duplicate)
188 list_add_tail(&fs->list, &virtio_fs_instances);
189
190 mutex_unlock(&virtio_fs_mutex);
191
192 if (duplicate)
193 return -EEXIST;
194 return 0;
195 }
196
197 /* Return the virtio_fs with a given tag, or NULL */
198 static struct virtio_fs *virtio_fs_find_instance(const char *tag)
199 {
200 struct virtio_fs *fs;
201
202 mutex_lock(&virtio_fs_mutex);
203
204 list_for_each_entry(fs, &virtio_fs_instances, list) {
205 if (strcmp(fs->tag, tag) == 0) {
206 kref_get(&fs->refcount);
207 goto found;
208 }
209 }
210
211 fs = NULL; /* not found */
212
213 found:
214 mutex_unlock(&virtio_fs_mutex);
215
216 return fs;
217 }
218
219 static void virtio_fs_free_devs(struct virtio_fs *fs)
220 {
221 unsigned int i;
222
223 for (i = 0; i < fs->nvqs; i++) {
224 struct virtio_fs_vq *fsvq = &fs->vqs[i];
225
226 if (!fsvq->fud)
227 continue;
228
229 fuse_dev_free(fsvq->fud);
230 fsvq->fud = NULL;
231 }
232 }
233
234 /* Read filesystem name from virtio config into fs->tag (must kfree()). */
235 static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs)
236 {
237 char tag_buf[sizeof_field(struct virtio_fs_config, tag)];
238 char *end;
239 size_t len;
240
241 virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag),
242 &tag_buf, sizeof(tag_buf));
243 end = memchr(tag_buf, '\0', sizeof(tag_buf));
244 if (end == tag_buf)
245 return -EINVAL; /* empty tag */
246 if (!end)
247 end = &tag_buf[sizeof(tag_buf)];
248
249 len = end - tag_buf;
250 fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL);
251 if (!fs->tag)
252 return -ENOMEM;
253 memcpy(fs->tag, tag_buf, len);
254 fs->tag[len] = '\0';
255 return 0;
256 }
257
258 /* Work function for hiprio completion */
259 static void virtio_fs_hiprio_done_work(struct work_struct *work)
260 {
261 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
262 done_work);
263 struct virtqueue *vq = fsvq->vq;
264
265 /* Free completed FUSE_FORGET requests */
266 spin_lock(&fsvq->lock);
267 do {
268 unsigned int len;
269 void *req;
270
271 virtqueue_disable_cb(vq);
272
273 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
274 kfree(req);
275 dec_in_flight_req(fsvq);
276 }
277 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
278 spin_unlock(&fsvq->lock);
279 }
280
281 static void virtio_fs_request_dispatch_work(struct work_struct *work)
282 {
283 struct fuse_req *req;
284 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
285 dispatch_work.work);
286 struct fuse_conn *fc = fsvq->fud->fc;
287 int ret;
288
289 pr_debug("virtio-fs: worker %s called.\n", __func__);
290 while (1) {
291 spin_lock(&fsvq->lock);
292 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req,
293 list);
294 if (!req) {
295 spin_unlock(&fsvq->lock);
296 break;
297 }
298
299 list_del_init(&req->list);
300 spin_unlock(&fsvq->lock);
301 fuse_request_end(fc, req);
302 }
303
304 /* Dispatch pending requests */
305 while (1) {
306 spin_lock(&fsvq->lock);
307 req = list_first_entry_or_null(&fsvq->queued_reqs,
308 struct fuse_req, list);
309 if (!req) {
310 spin_unlock(&fsvq->lock);
311 return;
312 }
313 list_del_init(&req->list);
314 spin_unlock(&fsvq->lock);
315
316 ret = virtio_fs_enqueue_req(fsvq, req, true);
317 if (ret < 0) {
318 if (ret == -ENOMEM || ret == -ENOSPC) {
319 spin_lock(&fsvq->lock);
320 list_add_tail(&req->list, &fsvq->queued_reqs);
321 schedule_delayed_work(&fsvq->dispatch_work,
322 msecs_to_jiffies(1));
323 spin_unlock(&fsvq->lock);
324 return;
325 }
326 req->out.h.error = ret;
327 spin_lock(&fsvq->lock);
328 dec_in_flight_req(fsvq);
329 spin_unlock(&fsvq->lock);
330 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n",
331 ret);
332 fuse_request_end(fc, req);
333 }
334 }
335 }
336
337 /*
338 * Returns 1 if queue is full and sender should wait a bit before sending
339 * next request, 0 otherwise.
340 */
341 static int send_forget_request(struct virtio_fs_vq *fsvq,
342 struct virtio_fs_forget *forget,
343 bool in_flight)
344 {
345 struct scatterlist sg;
346 struct virtqueue *vq;
347 int ret = 0;
348 bool notify;
349 struct virtio_fs_forget_req *req = &forget->req;
350
351 spin_lock(&fsvq->lock);
352 if (!fsvq->connected) {
353 if (in_flight)
354 dec_in_flight_req(fsvq);
355 kfree(forget);
356 goto out;
357 }
358
359 sg_init_one(&sg, req, sizeof(*req));
360 vq = fsvq->vq;
361 dev_dbg(&vq->vdev->dev, "%s\n", __func__);
362
363 ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC);
364 if (ret < 0) {
365 if (ret == -ENOMEM || ret == -ENOSPC) {
366 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
367 ret);
368 list_add_tail(&forget->list, &fsvq->queued_reqs);
369 schedule_delayed_work(&fsvq->dispatch_work,
370 msecs_to_jiffies(1));
371 if (!in_flight)
372 inc_in_flight_req(fsvq);
373 /* Queue is full */
374 ret = 1;
375 } else {
376 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
377 ret);
378 kfree(forget);
379 if (in_flight)
380 dec_in_flight_req(fsvq);
381 }
382 goto out;
383 }
384
385 if (!in_flight)
386 inc_in_flight_req(fsvq);
387 notify = virtqueue_kick_prepare(vq);
388 spin_unlock(&fsvq->lock);
389
390 if (notify)
391 virtqueue_notify(vq);
392 return ret;
393 out:
394 spin_unlock(&fsvq->lock);
395 return ret;
396 }
397
398 static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
399 {
400 struct virtio_fs_forget *forget;
401 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
402 dispatch_work.work);
403 pr_debug("virtio-fs: worker %s called.\n", __func__);
404 while (1) {
405 spin_lock(&fsvq->lock);
406 forget = list_first_entry_or_null(&fsvq->queued_reqs,
407 struct virtio_fs_forget, list);
408 if (!forget) {
409 spin_unlock(&fsvq->lock);
410 return;
411 }
412
413 list_del(&forget->list);
414 spin_unlock(&fsvq->lock);
415 if (send_forget_request(fsvq, forget, true))
416 return;
417 }
418 }
419
420 /* Allocate and copy args into req->argbuf */
421 static int copy_args_to_argbuf(struct fuse_req *req)
422 {
423 struct fuse_args *args = req->args;
424 unsigned int offset = 0;
425 unsigned int num_in;
426 unsigned int num_out;
427 unsigned int len;
428 unsigned int i;
429
430 num_in = args->in_numargs - args->in_pages;
431 num_out = args->out_numargs - args->out_pages;
432 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) +
433 fuse_len_args(num_out, args->out_args);
434
435 req->argbuf = kmalloc(len, GFP_ATOMIC);
436 if (!req->argbuf)
437 return -ENOMEM;
438
439 for (i = 0; i < num_in; i++) {
440 memcpy(req->argbuf + offset,
441 args->in_args[i].value,
442 args->in_args[i].size);
443 offset += args->in_args[i].size;
444 }
445
446 return 0;
447 }
448
449 /* Copy args out of and free req->argbuf */
450 static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req)
451 {
452 unsigned int remaining;
453 unsigned int offset;
454 unsigned int num_in;
455 unsigned int num_out;
456 unsigned int i;
457
458 remaining = req->out.h.len - sizeof(req->out.h);
459 num_in = args->in_numargs - args->in_pages;
460 num_out = args->out_numargs - args->out_pages;
461 offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args);
462
463 for (i = 0; i < num_out; i++) {
464 unsigned int argsize = args->out_args[i].size;
465
466 if (args->out_argvar &&
467 i == args->out_numargs - 1 &&
468 argsize > remaining) {
469 argsize = remaining;
470 }
471
472 memcpy(args->out_args[i].value, req->argbuf + offset, argsize);
473 offset += argsize;
474
475 if (i != args->out_numargs - 1)
476 remaining -= argsize;
477 }
478
479 /* Store the actual size of the variable-length arg */
480 if (args->out_argvar)
481 args->out_args[args->out_numargs - 1].size = remaining;
482
483 kfree(req->argbuf);
484 req->argbuf = NULL;
485 }
486
487 /* Work function for request completion */
488 static void virtio_fs_requests_done_work(struct work_struct *work)
489 {
490 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
491 done_work);
492 struct fuse_pqueue *fpq = &fsvq->fud->pq;
493 struct fuse_conn *fc = fsvq->fud->fc;
494 struct virtqueue *vq = fsvq->vq;
495 struct fuse_req *req;
496 struct fuse_args_pages *ap;
497 struct fuse_req *next;
498 struct fuse_args *args;
499 unsigned int len, i, thislen;
500 struct page *page;
501 LIST_HEAD(reqs);
502
503 /* Collect completed requests off the virtqueue */
504 spin_lock(&fsvq->lock);
505 do {
506 virtqueue_disable_cb(vq);
507
508 while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
509 spin_lock(&fpq->lock);
510 list_move_tail(&req->list, &reqs);
511 spin_unlock(&fpq->lock);
512 }
513 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
514 spin_unlock(&fsvq->lock);
515
516 /* End requests */
517 list_for_each_entry_safe(req, next, &reqs, list) {
518 /*
519 * TODO verify that server properly follows FUSE protocol
520 * (oh.uniq, oh.len)
521 */
522 args = req->args;
523 copy_args_from_argbuf(args, req);
524
525 if (args->out_pages && args->page_zeroing) {
526 len = args->out_args[args->out_numargs - 1].size;
527 ap = container_of(args, typeof(*ap), args);
528 for (i = 0; i < ap->num_pages; i++) {
529 thislen = ap->descs[i].length;
530 if (len < thislen) {
531 WARN_ON(ap->descs[i].offset);
532 page = ap->pages[i];
533 zero_user_segment(page, len, thislen);
534 len = 0;
535 } else {
536 len -= thislen;
537 }
538 }
539 }
540
541 spin_lock(&fpq->lock);
542 clear_bit(FR_SENT, &req->flags);
543 list_del_init(&req->list);
544 spin_unlock(&fpq->lock);
545
546 fuse_request_end(fc, req);
547 spin_lock(&fsvq->lock);
548 dec_in_flight_req(fsvq);
549 spin_unlock(&fsvq->lock);
550 }
551 }
552
553 /* Virtqueue interrupt handler */
554 static void virtio_fs_vq_done(struct virtqueue *vq)
555 {
556 struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);
557
558 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);
559
560 schedule_work(&fsvq->done_work);
561 }
562
563 /* Initialize virtqueues */
564 static int virtio_fs_setup_vqs(struct virtio_device *vdev,
565 struct virtio_fs *fs)
566 {
567 struct virtqueue **vqs;
568 vq_callback_t **callbacks;
569 const char **names;
570 unsigned int i;
571 int ret = 0;
572
573 virtio_cread(vdev, struct virtio_fs_config, num_request_queues,
574 &fs->num_request_queues);
575 if (fs->num_request_queues == 0)
576 return -EINVAL;
577
578 fs->nvqs = 1 + fs->num_request_queues;
579 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL);
580 if (!fs->vqs)
581 return -ENOMEM;
582
583 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
584 callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
585 GFP_KERNEL);
586 names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
587 if (!vqs || !callbacks || !names) {
588 ret = -ENOMEM;
589 goto out;
590 }
591
592 callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
593 snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name),
594 "hiprio");
595 names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
596 INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work);
597 INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs);
598 INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs);
599 INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work,
600 virtio_fs_hiprio_dispatch_work);
601 init_completion(&fs->vqs[VQ_HIPRIO].in_flight_zero);
602 spin_lock_init(&fs->vqs[VQ_HIPRIO].lock);
603
604 /* Initialize the requests virtqueues */
605 for (i = VQ_REQUEST; i < fs->nvqs; i++) {
606 spin_lock_init(&fs->vqs[i].lock);
607 INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work);
608 INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work,
609 virtio_fs_request_dispatch_work);
610 INIT_LIST_HEAD(&fs->vqs[i].queued_reqs);
611 INIT_LIST_HEAD(&fs->vqs[i].end_reqs);
612 init_completion(&fs->vqs[i].in_flight_zero);
613 snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name),
614 "requests.%u", i - VQ_REQUEST);
615 callbacks[i] = virtio_fs_vq_done;
616 names[i] = fs->vqs[i].name;
617 }
618
619 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL);
620 if (ret < 0)
621 goto out;
622
623 for (i = 0; i < fs->nvqs; i++)
624 fs->vqs[i].vq = vqs[i];
625
626 virtio_fs_start_all_queues(fs);
627 out:
628 kfree(names);
629 kfree(callbacks);
630 kfree(vqs);
631 if (ret)
632 kfree(fs->vqs);
633 return ret;
634 }
635
636 /* Free virtqueues (device must already be reset) */
637 static void virtio_fs_cleanup_vqs(struct virtio_device *vdev,
638 struct virtio_fs *fs)
639 {
640 vdev->config->del_vqs(vdev);
641 }
642
643 static int virtio_fs_probe(struct virtio_device *vdev)
644 {
645 struct virtio_fs *fs;
646 int ret;
647
648 fs = kzalloc(sizeof(*fs), GFP_KERNEL);
649 if (!fs)
650 return -ENOMEM;
651 kref_init(&fs->refcount);
652 vdev->priv = fs;
653
654 ret = virtio_fs_read_tag(vdev, fs);
655 if (ret < 0)
656 goto out;
657
658 ret = virtio_fs_setup_vqs(vdev, fs);
659 if (ret < 0)
660 goto out;
661
662 /* TODO vq affinity */
663
664 /* Bring the device online in case the filesystem is mounted and
665 * requests need to be sent before we return.
666 */
667 virtio_device_ready(vdev);
668
669 ret = virtio_fs_add_instance(fs);
670 if (ret < 0)
671 goto out_vqs;
672
673 return 0;
674
675 out_vqs:
676 vdev->config->reset(vdev);
677 virtio_fs_cleanup_vqs(vdev, fs);
678
679 out:
680 vdev->priv = NULL;
681 kfree(fs);
682 return ret;
683 }
684
685 static void virtio_fs_stop_all_queues(struct virtio_fs *fs)
686 {
687 struct virtio_fs_vq *fsvq;
688 int i;
689
690 for (i = 0; i < fs->nvqs; i++) {
691 fsvq = &fs->vqs[i];
692 spin_lock(&fsvq->lock);
693 fsvq->connected = false;
694 spin_unlock(&fsvq->lock);
695 }
696 }
697
698 static void virtio_fs_remove(struct virtio_device *vdev)
699 {
700 struct virtio_fs *fs = vdev->priv;
701
702 mutex_lock(&virtio_fs_mutex);
703 /* This device is going away. No one should get new reference */
704 list_del_init(&fs->list);
705 virtio_fs_stop_all_queues(fs);
706 virtio_fs_drain_all_queues_locked(fs);
707 vdev->config->reset(vdev);
708 virtio_fs_cleanup_vqs(vdev, fs);
709
710 vdev->priv = NULL;
711 /* Put device reference on virtio_fs object */
712 virtio_fs_put(fs);
713 mutex_unlock(&virtio_fs_mutex);
714 }
715
716 #ifdef CONFIG_PM_SLEEP
717 static int virtio_fs_freeze(struct virtio_device *vdev)
718 {
719 /* TODO need to save state here */
720 pr_warn("virtio-fs: suspend/resume not yet supported\n");
721 return -EOPNOTSUPP;
722 }
723
724 static int virtio_fs_restore(struct virtio_device *vdev)
725 {
726 /* TODO need to restore state here */
727 return 0;
728 }
729 #endif /* CONFIG_PM_SLEEP */
730
731 static const struct virtio_device_id id_table[] = {
732 { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID },
733 {},
734 };
735
736 static const unsigned int feature_table[] = {};
737
738 static struct virtio_driver virtio_fs_driver = {
739 .driver.name = KBUILD_MODNAME,
740 .driver.owner = THIS_MODULE,
741 .id_table = id_table,
742 .feature_table = feature_table,
743 .feature_table_size = ARRAY_SIZE(feature_table),
744 .probe = virtio_fs_probe,
745 .remove = virtio_fs_remove,
746 #ifdef CONFIG_PM_SLEEP
747 .freeze = virtio_fs_freeze,
748 .restore = virtio_fs_restore,
749 #endif
750 };
751
752 static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq)
753 __releases(fiq->lock)
754 {
755 struct fuse_forget_link *link;
756 struct virtio_fs_forget *forget;
757 struct virtio_fs_forget_req *req;
758 struct virtio_fs *fs;
759 struct virtio_fs_vq *fsvq;
760 u64 unique;
761
762 link = fuse_dequeue_forget(fiq, 1, NULL);
763 unique = fuse_get_unique(fiq);
764
765 fs = fiq->priv;
766 fsvq = &fs->vqs[VQ_HIPRIO];
767 spin_unlock(&fiq->lock);
768
769 /* Allocate a buffer for the request */
770 forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL);
771 req = &forget->req;
772
773 req->ih = (struct fuse_in_header){
774 .opcode = FUSE_FORGET,
775 .nodeid = link->forget_one.nodeid,
776 .unique = unique,
777 .len = sizeof(*req),
778 };
779 req->arg = (struct fuse_forget_in){
780 .nlookup = link->forget_one.nlookup,
781 };
782
783 send_forget_request(fsvq, forget, false);
784 kfree(link);
785 }
786
787 static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq)
788 __releases(fiq->lock)
789 {
790 /*
791 * TODO interrupts.
792 *
793 * Normal fs operations on a local filesystems aren't interruptible.
794 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW)
795 * with shared lock between host and guest.
796 */
797 spin_unlock(&fiq->lock);
798 }
799
800 /* Return the number of scatter-gather list elements required */
801 static unsigned int sg_count_fuse_req(struct fuse_req *req)
802 {
803 struct fuse_args *args = req->args;
804 struct fuse_args_pages *ap = container_of(args, typeof(*ap), args);
805 unsigned int total_sgs = 1 /* fuse_in_header */;
806
807 if (args->in_numargs - args->in_pages)
808 total_sgs += 1;
809
810 if (args->in_pages)
811 total_sgs += ap->num_pages;
812
813 if (!test_bit(FR_ISREPLY, &req->flags))
814 return total_sgs;
815
816 total_sgs += 1 /* fuse_out_header */;
817
818 if (args->out_numargs - args->out_pages)
819 total_sgs += 1;
820
821 if (args->out_pages)
822 total_sgs += ap->num_pages;
823
824 return total_sgs;
825 }
826
827 /* Add pages to scatter-gather list and return number of elements used */
828 static unsigned int sg_init_fuse_pages(struct scatterlist *sg,
829 struct page **pages,
830 struct fuse_page_desc *page_descs,
831 unsigned int num_pages,
832 unsigned int total_len)
833 {
834 unsigned int i;
835 unsigned int this_len;
836
837 for (i = 0; i < num_pages && total_len; i++) {
838 sg_init_table(&sg[i], 1);
839 this_len = min(page_descs[i].length, total_len);
840 sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset);
841 total_len -= this_len;
842 }
843
844 return i;
845 }
846
847 /* Add args to scatter-gather list and return number of elements used */
848 static unsigned int sg_init_fuse_args(struct scatterlist *sg,
849 struct fuse_req *req,
850 struct fuse_arg *args,
851 unsigned int numargs,
852 bool argpages,
853 void *argbuf,
854 unsigned int *len_used)
855 {
856 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
857 unsigned int total_sgs = 0;
858 unsigned int len;
859
860 len = fuse_len_args(numargs - argpages, args);
861 if (len)
862 sg_init_one(&sg[total_sgs++], argbuf, len);
863
864 if (argpages)
865 total_sgs += sg_init_fuse_pages(&sg[total_sgs],
866 ap->pages, ap->descs,
867 ap->num_pages,
868 args[numargs - 1].size);
869
870 if (len_used)
871 *len_used = len;
872
873 return total_sgs;
874 }
875
876 /* Add a request to a virtqueue and kick the device */
877 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
878 struct fuse_req *req, bool in_flight)
879 {
880 /* requests need at least 4 elements */
881 struct scatterlist *stack_sgs[6];
882 struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)];
883 struct scatterlist **sgs = stack_sgs;
884 struct scatterlist *sg = stack_sg;
885 struct virtqueue *vq;
886 struct fuse_args *args = req->args;
887 unsigned int argbuf_used = 0;
888 unsigned int out_sgs = 0;
889 unsigned int in_sgs = 0;
890 unsigned int total_sgs;
891 unsigned int i;
892 int ret;
893 bool notify;
894 struct fuse_pqueue *fpq;
895
896 /* Does the sglist fit on the stack? */
897 total_sgs = sg_count_fuse_req(req);
898 if (total_sgs > ARRAY_SIZE(stack_sgs)) {
899 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC);
900 sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC);
901 if (!sgs || !sg) {
902 ret = -ENOMEM;
903 goto out;
904 }
905 }
906
907 /* Use a bounce buffer since stack args cannot be mapped */
908 ret = copy_args_to_argbuf(req);
909 if (ret < 0)
910 goto out;
911
912 /* Request elements */
913 sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h));
914 out_sgs += sg_init_fuse_args(&sg[out_sgs], req,
915 (struct fuse_arg *)args->in_args,
916 args->in_numargs, args->in_pages,
917 req->argbuf, &argbuf_used);
918
919 /* Reply elements */
920 if (test_bit(FR_ISREPLY, &req->flags)) {
921 sg_init_one(&sg[out_sgs + in_sgs++],
922 &req->out.h, sizeof(req->out.h));
923 in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req,
924 args->out_args, args->out_numargs,
925 args->out_pages,
926 req->argbuf + argbuf_used, NULL);
927 }
928
929 WARN_ON(out_sgs + in_sgs != total_sgs);
930
931 for (i = 0; i < total_sgs; i++)
932 sgs[i] = &sg[i];
933
934 spin_lock(&fsvq->lock);
935
936 if (!fsvq->connected) {
937 spin_unlock(&fsvq->lock);
938 ret = -ENOTCONN;
939 goto out;
940 }
941
942 vq = fsvq->vq;
943 ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC);
944 if (ret < 0) {
945 spin_unlock(&fsvq->lock);
946 goto out;
947 }
948
949 /* Request successfully sent. */
950 fpq = &fsvq->fud->pq;
951 spin_lock(&fpq->lock);
952 list_add_tail(&req->list, fpq->processing);
953 spin_unlock(&fpq->lock);
954 set_bit(FR_SENT, &req->flags);
955 /* matches barrier in request_wait_answer() */
956 smp_mb__after_atomic();
957
958 if (!in_flight)
959 inc_in_flight_req(fsvq);
960 notify = virtqueue_kick_prepare(vq);
961
962 spin_unlock(&fsvq->lock);
963
964 if (notify)
965 virtqueue_notify(vq);
966
967 out:
968 if (ret < 0 && req->argbuf) {
969 kfree(req->argbuf);
970 req->argbuf = NULL;
971 }
972 if (sgs != stack_sgs) {
973 kfree(sgs);
974 kfree(sg);
975 }
976
977 return ret;
978 }
979
980 static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq)
981 __releases(fiq->lock)
982 {
983 unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */
984 struct virtio_fs *fs;
985 struct fuse_req *req;
986 struct virtio_fs_vq *fsvq;
987 int ret;
988
989 WARN_ON(list_empty(&fiq->pending));
990 req = list_last_entry(&fiq->pending, struct fuse_req, list);
991 clear_bit(FR_PENDING, &req->flags);
992 list_del_init(&req->list);
993 WARN_ON(!list_empty(&fiq->pending));
994 spin_unlock(&fiq->lock);
995
996 fs = fiq->priv;
997
998 pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n",
999 __func__, req->in.h.opcode, req->in.h.unique,
1000 req->in.h.nodeid, req->in.h.len,
1001 fuse_len_args(req->args->out_numargs, req->args->out_args));
1002
1003 fsvq = &fs->vqs[queue_id];
1004 ret = virtio_fs_enqueue_req(fsvq, req, false);
1005 if (ret < 0) {
1006 if (ret == -ENOMEM || ret == -ENOSPC) {
1007 /*
1008 * Virtqueue full. Retry submission from worker
1009 * context as we might be holding fc->bg_lock.
1010 */
1011 spin_lock(&fsvq->lock);
1012 list_add_tail(&req->list, &fsvq->queued_reqs);
1013 inc_in_flight_req(fsvq);
1014 schedule_delayed_work(&fsvq->dispatch_work,
1015 msecs_to_jiffies(1));
1016 spin_unlock(&fsvq->lock);
1017 return;
1018 }
1019 req->out.h.error = ret;
1020 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret);
1021
1022 /* Can't end request in submission context. Use a worker */
1023 spin_lock(&fsvq->lock);
1024 list_add_tail(&req->list, &fsvq->end_reqs);
1025 schedule_delayed_work(&fsvq->dispatch_work, 0);
1026 spin_unlock(&fsvq->lock);
1027 return;
1028 }
1029 }
1030
1031 static const struct fuse_iqueue_ops virtio_fs_fiq_ops = {
1032 .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock,
1033 .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock,
1034 .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock,
1035 .release = virtio_fs_fiq_release,
1036 };
1037
1038 static int virtio_fs_fill_super(struct super_block *sb)
1039 {
1040 struct fuse_conn *fc = get_fuse_conn_super(sb);
1041 struct virtio_fs *fs = fc->iq.priv;
1042 unsigned int i;
1043 int err;
1044 struct fuse_fs_context ctx = {
1045 .rootmode = S_IFDIR,
1046 .default_permissions = 1,
1047 .allow_other = 1,
1048 .max_read = UINT_MAX,
1049 .blksize = 512,
1050 .destroy = true,
1051 .no_control = true,
1052 .no_force_umount = true,
1053 .no_mount_options = true,
1054 };
1055
1056 mutex_lock(&virtio_fs_mutex);
1057
1058 /* After holding mutex, make sure virtiofs device is still there.
1059 * Though we are holding a reference to it, drive ->remove might
1060 * still have cleaned up virtual queues. In that case bail out.
1061 */
1062 err = -EINVAL;
1063 if (list_empty(&fs->list)) {
1064 pr_info("virtio-fs: tag <%s> not found\n", fs->tag);
1065 goto err;
1066 }
1067
1068 err = -ENOMEM;
1069 /* Allocate fuse_dev for hiprio and notification queues */
1070 for (i = 0; i < VQ_REQUEST; i++) {
1071 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1072
1073 fsvq->fud = fuse_dev_alloc();
1074 if (!fsvq->fud)
1075 goto err_free_fuse_devs;
1076 }
1077
1078 ctx.fudptr = (void **)&fs->vqs[VQ_REQUEST].fud;
1079 err = fuse_fill_super_common(sb, &ctx);
1080 if (err < 0)
1081 goto err_free_fuse_devs;
1082
1083 fc = fs->vqs[VQ_REQUEST].fud->fc;
1084
1085 for (i = 0; i < fs->nvqs; i++) {
1086 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1087
1088 if (i == VQ_REQUEST)
1089 continue; /* already initialized */
1090 fuse_dev_install(fsvq->fud, fc);
1091 }
1092
1093 /* Previous unmount will stop all queues. Start these again */
1094 virtio_fs_start_all_queues(fs);
1095 fuse_send_init(fc);
1096 mutex_unlock(&virtio_fs_mutex);
1097 return 0;
1098
1099 err_free_fuse_devs:
1100 virtio_fs_free_devs(fs);
1101 err:
1102 mutex_unlock(&virtio_fs_mutex);
1103 return err;
1104 }
1105
1106 static void virtio_kill_sb(struct super_block *sb)
1107 {
1108 struct fuse_conn *fc = get_fuse_conn_super(sb);
1109 struct virtio_fs *vfs;
1110 struct virtio_fs_vq *fsvq;
1111
1112 /* If mount failed, we can still be called without any fc */
1113 if (!fc)
1114 return fuse_kill_sb_anon(sb);
1115
1116 vfs = fc->iq.priv;
1117 fsvq = &vfs->vqs[VQ_HIPRIO];
1118
1119 /* Stop forget queue. Soon destroy will be sent */
1120 spin_lock(&fsvq->lock);
1121 fsvq->connected = false;
1122 spin_unlock(&fsvq->lock);
1123 virtio_fs_drain_all_queues(vfs);
1124
1125 fuse_kill_sb_anon(sb);
1126
1127 /* fuse_kill_sb_anon() must have sent destroy. Stop all queues
1128 * and drain one more time and free fuse devices. Freeing fuse
1129 * devices will drop their reference on fuse_conn and that in
1130 * turn will drop its reference on virtio_fs object.
1131 */
1132 virtio_fs_stop_all_queues(vfs);
1133 virtio_fs_drain_all_queues(vfs);
1134 virtio_fs_free_devs(vfs);
1135 }
1136
1137 static int virtio_fs_test_super(struct super_block *sb,
1138 struct fs_context *fsc)
1139 {
1140 struct fuse_conn *fc = fsc->s_fs_info;
1141
1142 return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv;
1143 }
1144
1145 static int virtio_fs_set_super(struct super_block *sb,
1146 struct fs_context *fsc)
1147 {
1148 int err;
1149
1150 err = get_anon_bdev(&sb->s_dev);
1151 if (!err)
1152 fuse_conn_get(fsc->s_fs_info);
1153
1154 return err;
1155 }
1156
1157 static int virtio_fs_get_tree(struct fs_context *fsc)
1158 {
1159 struct virtio_fs *fs;
1160 struct super_block *sb;
1161 struct fuse_conn *fc;
1162 int err;
1163
1164 /* This gets a reference on virtio_fs object. This ptr gets installed
1165 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
1166 * to drop the reference to this object.
1167 */
1168 fs = virtio_fs_find_instance(fsc->source);
1169 if (!fs) {
1170 pr_info("virtio-fs: tag <%s> not found\n", fsc->source);
1171 return -EINVAL;
1172 }
1173
1174 fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL);
1175 if (!fc) {
1176 mutex_lock(&virtio_fs_mutex);
1177 virtio_fs_put(fs);
1178 mutex_unlock(&virtio_fs_mutex);
1179 return -ENOMEM;
1180 }
1181
1182 fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops,
1183 fs);
1184 fc->release = fuse_free_conn;
1185 fc->delete_stale = true;
1186
1187 fsc->s_fs_info = fc;
1188 sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super);
1189 fuse_conn_put(fc);
1190 if (IS_ERR(sb))
1191 return PTR_ERR(sb);
1192
1193 if (!sb->s_root) {
1194 err = virtio_fs_fill_super(sb);
1195 if (err) {
1196 deactivate_locked_super(sb);
1197 return err;
1198 }
1199
1200 sb->s_flags |= SB_ACTIVE;
1201 }
1202
1203 WARN_ON(fsc->root);
1204 fsc->root = dget(sb->s_root);
1205 return 0;
1206 }
1207
1208 static const struct fs_context_operations virtio_fs_context_ops = {
1209 .get_tree = virtio_fs_get_tree,
1210 };
1211
1212 static int virtio_fs_init_fs_context(struct fs_context *fsc)
1213 {
1214 fsc->ops = &virtio_fs_context_ops;
1215 return 0;
1216 }
1217
1218 static struct file_system_type virtio_fs_type = {
1219 .owner = THIS_MODULE,
1220 .name = "virtiofs",
1221 .init_fs_context = virtio_fs_init_fs_context,
1222 .kill_sb = virtio_kill_sb,
1223 };
1224
1225 static int __init virtio_fs_init(void)
1226 {
1227 int ret;
1228
1229 ret = register_virtio_driver(&virtio_fs_driver);
1230 if (ret < 0)
1231 return ret;
1232
1233 ret = register_filesystem(&virtio_fs_type);
1234 if (ret < 0) {
1235 unregister_virtio_driver(&virtio_fs_driver);
1236 return ret;
1237 }
1238
1239 return 0;
1240 }
1241 module_init(virtio_fs_init);
1242
1243 static void __exit virtio_fs_exit(void)
1244 {
1245 unregister_filesystem(&virtio_fs_type);
1246 unregister_virtio_driver(&virtio_fs_driver);
1247 }
1248 module_exit(virtio_fs_exit);
1249
1250 MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>");
1251 MODULE_DESCRIPTION("Virtio Filesystem");
1252 MODULE_LICENSE("GPL");
1253 MODULE_ALIAS_FS(KBUILD_MODNAME);
1254 MODULE_DEVICE_TABLE(virtio, id_table);
Linux with added FPC-III support