1 // SPDX-License-Identifier: GPL-2.0-only
3 * The Virtio 9p transport driver
5 * This is a block based transport driver based on the lguest block driver
8 * Copyright (C) 2007, 2008 Eric Van Hensbergen, IBM Corporation
10 * Based on virtio console driver
11 * Copyright (C) 2006, 2007 Rusty Russell, IBM Corporation
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
17 #include <linux/module.h>
18 #include <linux/net.h>
19 #include <linux/ipv6.h>
20 #include <linux/errno.h>
21 #include <linux/kernel.h>
23 #include <linux/uaccess.h>
24 #include <linux/inet.h>
25 #include <linux/idr.h>
26 #include <linux/file.h>
27 #include <linux/highmem.h>
28 #include <linux/slab.h>
29 #include <net/9p/9p.h>
30 #include <linux/parser.h>
31 #include <net/9p/client.h>
32 #include <net/9p/transport.h>
33 #include <linux/scatterlist.h>
34 #include <linux/swap.h>
35 #include <linux/virtio.h>
36 #include <linux/virtio_9p.h>
37 #include "trans_common.h"
39 #define VIRTQUEUE_NUM 128
41 /* a single mutex to manage channel initialization and attachment */
42 static DEFINE_MUTEX(virtio_9p_lock
);
43 static DECLARE_WAIT_QUEUE_HEAD(vp_wq
);
44 static atomic_t vp_pinned
= ATOMIC_INIT(0);
47 * struct virtio_chan - per-instance transport information
48 * @inuse: whether the channel is in use
49 * @lock: protects multiple elements within this structure
50 * @client: client instance
51 * @vdev: virtio dev associated with this channel
52 * @vq: virtio queue associated with this channel
53 * @ring_bufs_avail: flag to indicate there is some available in the ring buf
54 * @vc_wq: wait queue for waiting for thing to be added to ring buf
55 * @p9_max_pages: maximum number of pinned pages
56 * @sg: scatter gather list which is used to pack a request (protected?)
57 * @chan_list: linked list of channels
59 * We keep all per-channel information in a structure.
60 * This structure is allocated within the devices dev->mem space.
61 * A pointer to the structure will get put in the transport private.
70 struct p9_client
*client
;
71 struct virtio_device
*vdev
;
74 wait_queue_head_t
*vc_wq
;
75 /* This is global limit. Since we don't have a global structure,
76 * will be placing it in each channel.
78 unsigned long p9_max_pages
;
79 /* Scatterlist: can be too big for stack. */
80 struct scatterlist sg
[VIRTQUEUE_NUM
];
82 * @tag: name to identify a mount null terminated
86 struct list_head chan_list
;
89 static struct list_head virtio_chan_list
;
91 /* How many bytes left in this page. */
92 static unsigned int rest_of_page(void *data
)
94 return PAGE_SIZE
- offset_in_page(data
);
98 * p9_virtio_close - reclaim resources of a channel
99 * @client: client instance
101 * This reclaims a channel by freeing its resources and
102 * reseting its inuse flag.
106 static void p9_virtio_close(struct p9_client
*client
)
108 struct virtio_chan
*chan
= client
->trans
;
110 mutex_lock(&virtio_9p_lock
);
113 mutex_unlock(&virtio_9p_lock
);
117 * req_done - callback which signals activity from the server
118 * @vq: virtio queue activity was received on
120 * This notifies us that the server has triggered some activity
121 * on the virtio channel - most likely a response to request we
122 * sent. Figure out which requests now have responses and wake up
125 * Bugs: could do with some additional sanity checking, but appears to work.
129 static void req_done(struct virtqueue
*vq
)
131 struct virtio_chan
*chan
= vq
->vdev
->priv
;
133 struct p9_req_t
*req
;
134 bool need_wakeup
= false;
137 p9_debug(P9_DEBUG_TRANS
, ": request done\n");
139 spin_lock_irqsave(&chan
->lock
, flags
);
140 while ((req
= virtqueue_get_buf(chan
->vq
, &len
)) != NULL
) {
141 if (!chan
->ring_bufs_avail
) {
142 chan
->ring_bufs_avail
= 1;
148 p9_client_cb(chan
->client
, req
, REQ_STATUS_RCVD
);
151 spin_unlock_irqrestore(&chan
->lock
, flags
);
152 /* Wakeup if anyone waiting for VirtIO ring space. */
154 wake_up(chan
->vc_wq
);
158 * pack_sg_list - pack a scatter gather list from a linear buffer
159 * @sg: scatter/gather list to pack into
160 * @start: which segment of the sg_list to start at
161 * @limit: maximum segment to pack data to
162 * @data: data to pack into scatter/gather list
163 * @count: amount of data to pack into the scatter/gather list
165 * sg_lists have multiple segments of various sizes. This will pack
166 * arbitrary data into an existing scatter gather list, segmenting the
167 * data as necessary within constraints.
171 static int pack_sg_list(struct scatterlist
*sg
, int start
,
172 int limit
, char *data
, int count
)
178 s
= rest_of_page(data
);
181 BUG_ON(index
>= limit
);
182 /* Make sure we don't terminate early. */
183 sg_unmark_end(&sg
[index
]);
184 sg_set_buf(&sg
[index
++], data
, s
);
189 sg_mark_end(&sg
[index
- 1]);
193 /* We don't currently allow canceling of virtio requests */
194 static int p9_virtio_cancel(struct p9_client
*client
, struct p9_req_t
*req
)
199 /* Reply won't come, so drop req ref */
200 static int p9_virtio_cancelled(struct p9_client
*client
, struct p9_req_t
*req
)
207 * pack_sg_list_p - Just like pack_sg_list. Instead of taking a buffer,
208 * this takes a list of pages.
209 * @sg: scatter/gather list to pack into
210 * @start: which segment of the sg_list to start at
211 * @limit: maximum number of pages in sg list.
212 * @pdata: a list of pages to add into sg.
213 * @nr_pages: number of pages to pack into the scatter/gather list
214 * @offs: amount of data in the beginning of first page _not_ to pack
215 * @count: amount of data to pack into the scatter/gather list
218 pack_sg_list_p(struct scatterlist
*sg
, int start
, int limit
,
219 struct page
**pdata
, int nr_pages
, size_t offs
, int count
)
225 BUG_ON(nr_pages
> (limit
- start
));
227 * if the first page doesn't start at
228 * page boundary find the offset
231 s
= PAGE_SIZE
- data_off
;
234 BUG_ON(index
>= limit
);
235 /* Make sure we don't terminate early. */
236 sg_unmark_end(&sg
[index
]);
237 sg_set_page(&sg
[index
++], pdata
[i
++], s
, data_off
);
244 sg_mark_end(&sg
[index
- 1]);
245 return index
- start
;
249 * p9_virtio_request - issue a request
250 * @client: client instance issuing the request
251 * @req: request to be issued
256 p9_virtio_request(struct p9_client
*client
, struct p9_req_t
*req
)
259 int in
, out
, out_sgs
, in_sgs
;
261 struct virtio_chan
*chan
= client
->trans
;
262 struct scatterlist
*sgs
[2];
264 p9_debug(P9_DEBUG_TRANS
, "9p debug: virtio request\n");
266 req
->status
= REQ_STATUS_SENT
;
268 spin_lock_irqsave(&chan
->lock
, flags
);
270 out_sgs
= in_sgs
= 0;
271 /* Handle out VirtIO ring buffers */
272 out
= pack_sg_list(chan
->sg
, 0,
273 VIRTQUEUE_NUM
, req
->tc
.sdata
, req
->tc
.size
);
275 sgs
[out_sgs
++] = chan
->sg
;
277 in
= pack_sg_list(chan
->sg
, out
,
278 VIRTQUEUE_NUM
, req
->rc
.sdata
, req
->rc
.capacity
);
280 sgs
[out_sgs
+ in_sgs
++] = chan
->sg
+ out
;
282 err
= virtqueue_add_sgs(chan
->vq
, sgs
, out_sgs
, in_sgs
, req
,
285 if (err
== -ENOSPC
) {
286 chan
->ring_bufs_avail
= 0;
287 spin_unlock_irqrestore(&chan
->lock
, flags
);
288 err
= wait_event_killable(*chan
->vc_wq
,
289 chan
->ring_bufs_avail
);
290 if (err
== -ERESTARTSYS
)
293 p9_debug(P9_DEBUG_TRANS
, "Retry virtio request\n");
296 spin_unlock_irqrestore(&chan
->lock
, flags
);
297 p9_debug(P9_DEBUG_TRANS
,
298 "virtio rpc add_sgs returned failure\n");
302 virtqueue_kick(chan
->vq
);
303 spin_unlock_irqrestore(&chan
->lock
, flags
);
305 p9_debug(P9_DEBUG_TRANS
, "virtio request kicked\n");
309 static int p9_get_mapped_pages(struct virtio_chan
*chan
,
310 struct page
***pages
,
311 struct iov_iter
*data
,
319 if (!iov_iter_count(data
))
322 if (!iov_iter_is_kvec(data
)) {
325 * We allow only p9_max_pages pinned. We wait for the
326 * Other zc request to finish here
328 if (atomic_read(&vp_pinned
) >= chan
->p9_max_pages
) {
329 err
= wait_event_killable(vp_wq
,
330 (atomic_read(&vp_pinned
) < chan
->p9_max_pages
));
331 if (err
== -ERESTARTSYS
)
334 n
= iov_iter_get_pages_alloc(data
, pages
, count
, offs
);
338 nr_pages
= DIV_ROUND_UP(n
+ *offs
, PAGE_SIZE
);
339 atomic_add(nr_pages
, &vp_pinned
);
342 /* kernel buffer, no need to pin pages */
347 /* we'd already checked that it's non-empty */
349 len
= iov_iter_single_seg_count(data
);
351 p
= data
->kvec
->iov_base
+ data
->iov_offset
;
354 iov_iter_advance(data
, 0);
359 nr_pages
= DIV_ROUND_UP((unsigned long)p
+ len
, PAGE_SIZE
) -
360 (unsigned long)p
/ PAGE_SIZE
;
362 *pages
= kmalloc_array(nr_pages
, sizeof(struct page
*),
368 p
-= (*offs
= offset_in_page(p
));
369 for (index
= 0; index
< nr_pages
; index
++) {
370 if (is_vmalloc_addr(p
))
371 (*pages
)[index
] = vmalloc_to_page(p
);
373 (*pages
)[index
] = kmap_to_page(p
);
381 * p9_virtio_zc_request - issue a zero copy request
382 * @client: client instance issuing the request
383 * @req: request to be issued
384 * @uidata: user buffer that should be used for zero copy read
385 * @uodata: user buffer that should be used for zero copy write
386 * @inlen: read buffer size
387 * @outlen: write buffer size
388 * @in_hdr_len: reader header size, This is the size of response protocol data
392 p9_virtio_zc_request(struct p9_client
*client
, struct p9_req_t
*req
,
393 struct iov_iter
*uidata
, struct iov_iter
*uodata
,
394 int inlen
, int outlen
, int in_hdr_len
)
396 int in
, out
, err
, out_sgs
, in_sgs
;
398 int in_nr_pages
= 0, out_nr_pages
= 0;
399 struct page
**in_pages
= NULL
, **out_pages
= NULL
;
400 struct virtio_chan
*chan
= client
->trans
;
401 struct scatterlist
*sgs
[4];
406 p9_debug(P9_DEBUG_TRANS
, "virtio request\n");
410 int n
= p9_get_mapped_pages(chan
, &out_pages
, uodata
,
411 outlen
, &offs
, &need_drop
);
416 out_nr_pages
= DIV_ROUND_UP(n
+ offs
, PAGE_SIZE
);
418 __le32 v
= cpu_to_le32(n
);
419 memcpy(&req
->tc
.sdata
[req
->tc
.size
- 4], &v
, 4);
422 /* The size field of the message must include the length of the
423 * header and the length of the data. We didn't actually know
424 * the length of the data until this point so add it in now.
426 sz
= cpu_to_le32(req
->tc
.size
+ outlen
);
427 memcpy(&req
->tc
.sdata
[0], &sz
, sizeof(sz
));
429 int n
= p9_get_mapped_pages(chan
, &in_pages
, uidata
,
430 inlen
, &offs
, &need_drop
);
435 in_nr_pages
= DIV_ROUND_UP(n
+ offs
, PAGE_SIZE
);
437 __le32 v
= cpu_to_le32(n
);
438 memcpy(&req
->tc
.sdata
[req
->tc
.size
- 4], &v
, 4);
442 req
->status
= REQ_STATUS_SENT
;
444 spin_lock_irqsave(&chan
->lock
, flags
);
446 out_sgs
= in_sgs
= 0;
449 out
= pack_sg_list(chan
->sg
, 0,
450 VIRTQUEUE_NUM
, req
->tc
.sdata
, req
->tc
.size
);
453 sgs
[out_sgs
++] = chan
->sg
;
456 sgs
[out_sgs
++] = chan
->sg
+ out
;
457 out
+= pack_sg_list_p(chan
->sg
, out
, VIRTQUEUE_NUM
,
458 out_pages
, out_nr_pages
, offs
, outlen
);
462 * Take care of in data
463 * For example TREAD have 11.
464 * 11 is the read/write header = PDU Header(7) + IO Size (4).
465 * Arrange in such a way that server places header in the
466 * alloced memory and payload onto the user buffer.
468 in
= pack_sg_list(chan
->sg
, out
,
469 VIRTQUEUE_NUM
, req
->rc
.sdata
, in_hdr_len
);
471 sgs
[out_sgs
+ in_sgs
++] = chan
->sg
+ out
;
474 sgs
[out_sgs
+ in_sgs
++] = chan
->sg
+ out
+ in
;
475 in
+= pack_sg_list_p(chan
->sg
, out
+ in
, VIRTQUEUE_NUM
,
476 in_pages
, in_nr_pages
, offs
, inlen
);
479 BUG_ON(out_sgs
+ in_sgs
> ARRAY_SIZE(sgs
));
480 err
= virtqueue_add_sgs(chan
->vq
, sgs
, out_sgs
, in_sgs
, req
,
483 if (err
== -ENOSPC
) {
484 chan
->ring_bufs_avail
= 0;
485 spin_unlock_irqrestore(&chan
->lock
, flags
);
486 err
= wait_event_killable(*chan
->vc_wq
,
487 chan
->ring_bufs_avail
);
488 if (err
== -ERESTARTSYS
)
491 p9_debug(P9_DEBUG_TRANS
, "Retry virtio request\n");
492 goto req_retry_pinned
;
494 spin_unlock_irqrestore(&chan
->lock
, flags
);
495 p9_debug(P9_DEBUG_TRANS
,
496 "virtio rpc add_sgs returned failure\n");
501 virtqueue_kick(chan
->vq
);
502 spin_unlock_irqrestore(&chan
->lock
, flags
);
504 p9_debug(P9_DEBUG_TRANS
, "virtio request kicked\n");
505 err
= wait_event_killable(req
->wq
, req
->status
>= REQ_STATUS_RCVD
);
507 * Non kernel buffers are pinned, unpin them
512 p9_release_pages(in_pages
, in_nr_pages
);
513 atomic_sub(in_nr_pages
, &vp_pinned
);
516 p9_release_pages(out_pages
, out_nr_pages
);
517 atomic_sub(out_nr_pages
, &vp_pinned
);
519 /* wakeup anybody waiting for slots to pin pages */
525 /* reply won't come */
531 static ssize_t
p9_mount_tag_show(struct device
*dev
,
532 struct device_attribute
*attr
, char *buf
)
534 struct virtio_chan
*chan
;
535 struct virtio_device
*vdev
;
538 vdev
= dev_to_virtio(dev
);
540 tag_len
= strlen(chan
->tag
);
542 memcpy(buf
, chan
->tag
, tag_len
+ 1);
547 static DEVICE_ATTR(mount_tag
, 0444, p9_mount_tag_show
, NULL
);
550 * p9_virtio_probe - probe for existence of 9P virtio channels
551 * @vdev: virtio device to probe
553 * This probes for existing virtio channels.
557 static int p9_virtio_probe(struct virtio_device
*vdev
)
562 struct virtio_chan
*chan
;
564 if (!vdev
->config
->get
) {
565 dev_err(&vdev
->dev
, "%s failure: config access disabled\n",
570 chan
= kmalloc(sizeof(struct virtio_chan
), GFP_KERNEL
);
572 pr_err("Failed to allocate virtio 9P channel\n");
579 /* We expect one virtqueue, for requests. */
580 chan
->vq
= virtio_find_single_vq(vdev
, req_done
, "requests");
581 if (IS_ERR(chan
->vq
)) {
582 err
= PTR_ERR(chan
->vq
);
585 chan
->vq
->vdev
->priv
= chan
;
586 spin_lock_init(&chan
->lock
);
588 sg_init_table(chan
->sg
, VIRTQUEUE_NUM
);
591 if (virtio_has_feature(vdev
, VIRTIO_9P_MOUNT_TAG
)) {
592 virtio_cread(vdev
, struct virtio_9p_config
, tag_len
, &tag_len
);
597 tag
= kzalloc(tag_len
+ 1, GFP_KERNEL
);
603 virtio_cread_bytes(vdev
, offsetof(struct virtio_9p_config
, tag
),
606 err
= sysfs_create_file(&(vdev
->dev
.kobj
), &dev_attr_mount_tag
.attr
);
610 chan
->vc_wq
= kmalloc(sizeof(wait_queue_head_t
), GFP_KERNEL
);
615 init_waitqueue_head(chan
->vc_wq
);
616 chan
->ring_bufs_avail
= 1;
617 /* Ceiling limit to avoid denial of service attacks */
618 chan
->p9_max_pages
= nr_free_buffer_pages()/4;
620 virtio_device_ready(vdev
);
622 mutex_lock(&virtio_9p_lock
);
623 list_add_tail(&chan
->chan_list
, &virtio_chan_list
);
624 mutex_unlock(&virtio_9p_lock
);
626 /* Let udev rules use the new mount_tag attribute. */
627 kobject_uevent(&(vdev
->dev
.kobj
), KOBJ_CHANGE
);
634 vdev
->config
->del_vqs(vdev
);
643 * p9_virtio_create - allocate a new virtio channel
644 * @client: client instance invoking this transport
645 * @devname: string identifying the channel to connect to (unused)
646 * @args: args passed from sys_mount() for per-transport options (unused)
648 * This sets up a transport channel for 9p communication. Right now
649 * we only match the first available channel, but eventually we couldlook up
650 * alternate channels by matching devname versus a virtio_config entry.
651 * We use a simple reference count mechanism to ensure that only a single
652 * mount has a channel open at a time.
657 p9_virtio_create(struct p9_client
*client
, const char *devname
, char *args
)
659 struct virtio_chan
*chan
;
666 mutex_lock(&virtio_9p_lock
);
667 list_for_each_entry(chan
, &virtio_chan_list
, chan_list
) {
668 if (!strcmp(devname
, chan
->tag
)) {
677 mutex_unlock(&virtio_9p_lock
);
680 pr_err("no channels available for device %s\n", devname
);
684 client
->trans
= (void *)chan
;
685 client
->status
= Connected
;
686 chan
->client
= client
;
692 * p9_virtio_remove - clean up resources associated with a virtio device
693 * @vdev: virtio device to remove
697 static void p9_virtio_remove(struct virtio_device
*vdev
)
699 struct virtio_chan
*chan
= vdev
->priv
;
700 unsigned long warning_time
;
702 mutex_lock(&virtio_9p_lock
);
704 /* Remove self from list so we don't get new users. */
705 list_del(&chan
->chan_list
);
706 warning_time
= jiffies
;
708 /* Wait for existing users to close. */
709 while (chan
->inuse
) {
710 mutex_unlock(&virtio_9p_lock
);
712 if (time_after(jiffies
, warning_time
+ 10 * HZ
)) {
713 dev_emerg(&vdev
->dev
,
714 "p9_virtio_remove: waiting for device in use.\n");
715 warning_time
= jiffies
;
717 mutex_lock(&virtio_9p_lock
);
720 mutex_unlock(&virtio_9p_lock
);
722 vdev
->config
->reset(vdev
);
723 vdev
->config
->del_vqs(vdev
);
725 sysfs_remove_file(&(vdev
->dev
.kobj
), &dev_attr_mount_tag
.attr
);
726 kobject_uevent(&(vdev
->dev
.kobj
), KOBJ_CHANGE
);
733 static struct virtio_device_id id_table
[] = {
734 { VIRTIO_ID_9P
, VIRTIO_DEV_ANY_ID
},
738 static unsigned int features
[] = {
742 /* The standard "struct lguest_driver": */
743 static struct virtio_driver p9_virtio_drv
= {
744 .feature_table
= features
,
745 .feature_table_size
= ARRAY_SIZE(features
),
746 .driver
.name
= KBUILD_MODNAME
,
747 .driver
.owner
= THIS_MODULE
,
748 .id_table
= id_table
,
749 .probe
= p9_virtio_probe
,
750 .remove
= p9_virtio_remove
,
753 static struct p9_trans_module p9_virtio_trans
= {
755 .create
= p9_virtio_create
,
756 .close
= p9_virtio_close
,
757 .request
= p9_virtio_request
,
758 .zc_request
= p9_virtio_zc_request
,
759 .cancel
= p9_virtio_cancel
,
760 .cancelled
= p9_virtio_cancelled
,
762 * We leave one entry for input and one entry for response
763 * headers. We also skip one more entry to accomodate, address
764 * that are not at page boundary, that can result in an extra
767 .maxsize
= PAGE_SIZE
* (VIRTQUEUE_NUM
- 3),
769 .owner
= THIS_MODULE
,
772 /* The standard init function */
773 static int __init
p9_virtio_init(void)
777 INIT_LIST_HEAD(&virtio_chan_list
);
779 v9fs_register_trans(&p9_virtio_trans
);
780 rc
= register_virtio_driver(&p9_virtio_drv
);
782 v9fs_unregister_trans(&p9_virtio_trans
);
787 static void __exit
p9_virtio_cleanup(void)
789 unregister_virtio_driver(&p9_virtio_drv
);
790 v9fs_unregister_trans(&p9_virtio_trans
);
793 module_init(p9_virtio_init
);
794 module_exit(p9_virtio_cleanup
);
796 MODULE_DEVICE_TABLE(virtio
, id_table
);
797 MODULE_AUTHOR("Eric Van Hensbergen <ericvh@gmail.com>");
798 MODULE_DESCRIPTION("Virtio 9p Transport");
799 MODULE_LICENSE("GPL");