2 * The Virtio 9p transport driver
4 * This is a block based transport driver based on the lguest block driver
7 * Copyright (C) 2007, 2008 Eric Van Hensbergen, IBM Corporation
9 * Based on virtio console driver
10 * Copyright (C) 2006, 2007 Rusty Russell, IBM Corporation
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2
14 * as published by the Free Software Foundation.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to:
23 * Free Software Foundation
24 * 51 Franklin Street, Fifth Floor
25 * Boston, MA 02111-1301 USA
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
32 #include <linux/module.h>
33 #include <linux/net.h>
34 #include <linux/ipv6.h>
35 #include <linux/errno.h>
36 #include <linux/kernel.h>
38 #include <linux/uaccess.h>
39 #include <linux/inet.h>
40 #include <linux/idr.h>
41 #include <linux/file.h>
42 #include <linux/highmem.h>
43 #include <linux/slab.h>
44 #include <net/9p/9p.h>
45 #include <linux/parser.h>
46 #include <net/9p/client.h>
47 #include <net/9p/transport.h>
48 #include <linux/scatterlist.h>
49 #include <linux/swap.h>
50 #include <linux/virtio.h>
51 #include <linux/virtio_9p.h>
52 #include "trans_common.h"
54 #define VIRTQUEUE_NUM 128
56 /* a single mutex to manage channel initialization and attachment */
57 static DEFINE_MUTEX(virtio_9p_lock
);
58 static DECLARE_WAIT_QUEUE_HEAD(vp_wq
);
59 static atomic_t vp_pinned
= ATOMIC_INIT(0);
62 * struct virtio_chan - per-instance transport information
63 * @inuse: whether the channel is in use
64 * @lock: protects multiple elements within this structure
65 * @client: client instance
66 * @vdev: virtio dev associated with this channel
67 * @vq: virtio queue associated with this channel
68 * @sg: scatter gather list which is used to pack a request (protected?)
70 * We keep all per-channel information in a structure.
71 * This structure is allocated within the devices dev->mem space.
72 * A pointer to the structure will get put in the transport private.
81 struct p9_client
*client
;
82 struct virtio_device
*vdev
;
85 wait_queue_head_t
*vc_wq
;
86 /* This is global limit. Since we don't have a global structure,
87 * will be placing it in each channel.
89 unsigned long p9_max_pages
;
90 /* Scatterlist: can be too big for stack. */
91 struct scatterlist sg
[VIRTQUEUE_NUM
];
95 * tag name to identify a mount Non-null terminated
99 struct list_head chan_list
;
102 static struct list_head virtio_chan_list
;
104 /* How many bytes left in this page. */
105 static unsigned int rest_of_page(void *data
)
107 return PAGE_SIZE
- offset_in_page(data
);
111 * p9_virtio_close - reclaim resources of a channel
112 * @client: client instance
114 * This reclaims a channel by freeing its resources and
115 * reseting its inuse flag.
119 static void p9_virtio_close(struct p9_client
*client
)
121 struct virtio_chan
*chan
= client
->trans
;
123 mutex_lock(&virtio_9p_lock
);
126 mutex_unlock(&virtio_9p_lock
);
130 * req_done - callback which signals activity from the server
131 * @vq: virtio queue activity was received on
133 * This notifies us that the server has triggered some activity
134 * on the virtio channel - most likely a response to request we
135 * sent. Figure out which requests now have responses and wake up
138 * Bugs: could do with some additional sanity checking, but appears to work.
142 static void req_done(struct virtqueue
*vq
)
144 struct virtio_chan
*chan
= vq
->vdev
->priv
;
146 struct p9_req_t
*req
;
149 p9_debug(P9_DEBUG_TRANS
, ": request done\n");
152 spin_lock_irqsave(&chan
->lock
, flags
);
153 req
= virtqueue_get_buf(chan
->vq
, &len
);
155 spin_unlock_irqrestore(&chan
->lock
, flags
);
158 chan
->ring_bufs_avail
= 1;
159 spin_unlock_irqrestore(&chan
->lock
, flags
);
160 /* Wakeup if anyone waiting for VirtIO ring space. */
161 wake_up(chan
->vc_wq
);
163 p9_client_cb(chan
->client
, req
, REQ_STATUS_RCVD
);
168 * pack_sg_list - pack a scatter gather list from a linear buffer
169 * @sg: scatter/gather list to pack into
170 * @start: which segment of the sg_list to start at
171 * @limit: maximum segment to pack data to
172 * @data: data to pack into scatter/gather list
173 * @count: amount of data to pack into the scatter/gather list
175 * sg_lists have multiple segments of various sizes. This will pack
176 * arbitrary data into an existing scatter gather list, segmenting the
177 * data as necessary within constraints.
181 static int pack_sg_list(struct scatterlist
*sg
, int start
,
182 int limit
, char *data
, int count
)
188 s
= rest_of_page(data
);
191 BUG_ON(index
>= limit
);
192 /* Make sure we don't terminate early. */
193 sg_unmark_end(&sg
[index
]);
194 sg_set_buf(&sg
[index
++], data
, s
);
199 sg_mark_end(&sg
[index
- 1]);
203 /* We don't currently allow canceling of virtio requests */
204 static int p9_virtio_cancel(struct p9_client
*client
, struct p9_req_t
*req
)
210 * pack_sg_list_p - Just like pack_sg_list. Instead of taking a buffer,
211 * this takes a list of pages.
212 * @sg: scatter/gather list to pack into
213 * @start: which segment of the sg_list to start at
214 * @pdata: a list of pages to add into sg.
215 * @nr_pages: number of pages to pack into the scatter/gather list
216 * @offs: amount of data in the beginning of first page _not_ to pack
217 * @count: amount of data to pack into the scatter/gather list
220 pack_sg_list_p(struct scatterlist
*sg
, int start
, int limit
,
221 struct page
**pdata
, int nr_pages
, size_t offs
, int count
)
227 BUG_ON(nr_pages
> (limit
- start
));
229 * if the first page doesn't start at
230 * page boundary find the offset
233 s
= PAGE_SIZE
- data_off
;
236 BUG_ON(index
>= limit
);
237 /* Make sure we don't terminate early. */
238 sg_unmark_end(&sg
[index
]);
239 sg_set_page(&sg
[index
++], pdata
[i
++], s
, data_off
);
246 sg_mark_end(&sg
[index
- 1]);
247 return index
- start
;
251 * p9_virtio_request - issue a request
252 * @client: client instance issuing the request
253 * @req: request to be issued
258 p9_virtio_request(struct p9_client
*client
, struct p9_req_t
*req
)
261 int in
, out
, out_sgs
, in_sgs
;
263 struct virtio_chan
*chan
= client
->trans
;
264 struct scatterlist
*sgs
[2];
266 p9_debug(P9_DEBUG_TRANS
, "9p debug: virtio request\n");
268 req
->status
= REQ_STATUS_SENT
;
270 spin_lock_irqsave(&chan
->lock
, flags
);
272 out_sgs
= in_sgs
= 0;
273 /* Handle out VirtIO ring buffers */
274 out
= pack_sg_list(chan
->sg
, 0,
275 VIRTQUEUE_NUM
, req
->tc
->sdata
, req
->tc
->size
);
277 sgs
[out_sgs
++] = chan
->sg
;
279 in
= pack_sg_list(chan
->sg
, out
,
280 VIRTQUEUE_NUM
, req
->rc
->sdata
, req
->rc
->capacity
);
282 sgs
[out_sgs
+ in_sgs
++] = chan
->sg
+ out
;
284 err
= virtqueue_add_sgs(chan
->vq
, sgs
, out_sgs
, in_sgs
, req
,
287 if (err
== -ENOSPC
) {
288 chan
->ring_bufs_avail
= 0;
289 spin_unlock_irqrestore(&chan
->lock
, flags
);
290 err
= wait_event_killable(*chan
->vc_wq
,
291 chan
->ring_bufs_avail
);
292 if (err
== -ERESTARTSYS
)
295 p9_debug(P9_DEBUG_TRANS
, "Retry virtio request\n");
298 spin_unlock_irqrestore(&chan
->lock
, flags
);
299 p9_debug(P9_DEBUG_TRANS
,
300 "virtio rpc add_sgs returned failure\n");
304 virtqueue_kick(chan
->vq
);
305 spin_unlock_irqrestore(&chan
->lock
, flags
);
307 p9_debug(P9_DEBUG_TRANS
, "virtio request kicked\n");
311 static int p9_get_mapped_pages(struct virtio_chan
*chan
,
312 struct page
***pages
,
313 struct iov_iter
*data
,
321 if (!iov_iter_count(data
))
324 if (!(data
->type
& ITER_KVEC
)) {
327 * We allow only p9_max_pages pinned. We wait for the
328 * Other zc request to finish here
330 if (atomic_read(&vp_pinned
) >= chan
->p9_max_pages
) {
331 err
= wait_event_killable(vp_wq
,
332 (atomic_read(&vp_pinned
) < chan
->p9_max_pages
));
333 if (err
== -ERESTARTSYS
)
336 n
= iov_iter_get_pages_alloc(data
, pages
, count
, offs
);
340 nr_pages
= DIV_ROUND_UP(n
+ *offs
, PAGE_SIZE
);
341 atomic_add(nr_pages
, &vp_pinned
);
344 /* kernel buffer, no need to pin pages */
349 /* we'd already checked that it's non-empty */
351 len
= iov_iter_single_seg_count(data
);
353 p
= data
->kvec
->iov_base
+ data
->iov_offset
;
356 iov_iter_advance(data
, 0);
361 nr_pages
= DIV_ROUND_UP((unsigned long)p
+ len
, PAGE_SIZE
) -
362 (unsigned long)p
/ PAGE_SIZE
;
364 *pages
= kmalloc_array(nr_pages
, sizeof(struct page
*),
370 p
-= (*offs
= offset_in_page(p
));
371 for (index
= 0; index
< nr_pages
; index
++) {
372 if (is_vmalloc_addr(p
))
373 (*pages
)[index
] = vmalloc_to_page(p
);
375 (*pages
)[index
] = kmap_to_page(p
);
383 * p9_virtio_zc_request - issue a zero copy request
384 * @client: client instance issuing the request
385 * @req: request to be issued
386 * @uidata: user bffer that should be ued for zero copy read
387 * @uodata: user buffer that shoud be user for zero copy write
388 * @inlen: read buffer size
389 * @outlen: write buffer size
390 * @in_hdr_len: reader header size, This is the size of response protocol data
394 p9_virtio_zc_request(struct p9_client
*client
, struct p9_req_t
*req
,
395 struct iov_iter
*uidata
, struct iov_iter
*uodata
,
396 int inlen
, int outlen
, int in_hdr_len
)
398 int in
, out
, err
, out_sgs
, in_sgs
;
400 int in_nr_pages
= 0, out_nr_pages
= 0;
401 struct page
**in_pages
= NULL
, **out_pages
= NULL
;
402 struct virtio_chan
*chan
= client
->trans
;
403 struct scatterlist
*sgs
[4];
407 p9_debug(P9_DEBUG_TRANS
, "virtio request\n");
411 int n
= p9_get_mapped_pages(chan
, &out_pages
, uodata
,
412 outlen
, &offs
, &need_drop
);
415 out_nr_pages
= DIV_ROUND_UP(n
+ offs
, PAGE_SIZE
);
417 __le32 v
= cpu_to_le32(n
);
418 memcpy(&req
->tc
->sdata
[req
->tc
->size
- 4], &v
, 4);
421 /* The size field of the message must include the length of the
422 * header and the length of the data. We didn't actually know
423 * the length of the data until this point so add it in now.
425 sz
= cpu_to_le32(req
->tc
->size
+ outlen
);
426 memcpy(&req
->tc
->sdata
[0], &sz
, sizeof(sz
));
428 int n
= p9_get_mapped_pages(chan
, &in_pages
, uidata
,
429 inlen
, &offs
, &need_drop
);
432 in_nr_pages
= DIV_ROUND_UP(n
+ offs
, PAGE_SIZE
);
434 __le32 v
= cpu_to_le32(n
);
435 memcpy(&req
->tc
->sdata
[req
->tc
->size
- 4], &v
, 4);
439 req
->status
= REQ_STATUS_SENT
;
441 spin_lock_irqsave(&chan
->lock
, flags
);
443 out_sgs
= in_sgs
= 0;
446 out
= pack_sg_list(chan
->sg
, 0,
447 VIRTQUEUE_NUM
, req
->tc
->sdata
, req
->tc
->size
);
450 sgs
[out_sgs
++] = chan
->sg
;
453 sgs
[out_sgs
++] = chan
->sg
+ out
;
454 out
+= pack_sg_list_p(chan
->sg
, out
, VIRTQUEUE_NUM
,
455 out_pages
, out_nr_pages
, offs
, outlen
);
459 * Take care of in data
460 * For example TREAD have 11.
461 * 11 is the read/write header = PDU Header(7) + IO Size (4).
462 * Arrange in such a way that server places header in the
463 * alloced memory and payload onto the user buffer.
465 in
= pack_sg_list(chan
->sg
, out
,
466 VIRTQUEUE_NUM
, req
->rc
->sdata
, in_hdr_len
);
468 sgs
[out_sgs
+ in_sgs
++] = chan
->sg
+ out
;
471 sgs
[out_sgs
+ in_sgs
++] = chan
->sg
+ out
+ in
;
472 in
+= pack_sg_list_p(chan
->sg
, out
+ in
, VIRTQUEUE_NUM
,
473 in_pages
, in_nr_pages
, offs
, inlen
);
476 BUG_ON(out_sgs
+ in_sgs
> ARRAY_SIZE(sgs
));
477 err
= virtqueue_add_sgs(chan
->vq
, sgs
, out_sgs
, in_sgs
, req
,
480 if (err
== -ENOSPC
) {
481 chan
->ring_bufs_avail
= 0;
482 spin_unlock_irqrestore(&chan
->lock
, flags
);
483 err
= wait_event_killable(*chan
->vc_wq
,
484 chan
->ring_bufs_avail
);
485 if (err
== -ERESTARTSYS
)
488 p9_debug(P9_DEBUG_TRANS
, "Retry virtio request\n");
489 goto req_retry_pinned
;
491 spin_unlock_irqrestore(&chan
->lock
, flags
);
492 p9_debug(P9_DEBUG_TRANS
,
493 "virtio rpc add_sgs returned failure\n");
498 virtqueue_kick(chan
->vq
);
499 spin_unlock_irqrestore(&chan
->lock
, flags
);
500 p9_debug(P9_DEBUG_TRANS
, "virtio request kicked\n");
501 err
= wait_event_killable(*req
->wq
, req
->status
>= REQ_STATUS_RCVD
);
503 * Non kernel buffers are pinned, unpin them
508 p9_release_pages(in_pages
, in_nr_pages
);
509 atomic_sub(in_nr_pages
, &vp_pinned
);
512 p9_release_pages(out_pages
, out_nr_pages
);
513 atomic_sub(out_nr_pages
, &vp_pinned
);
515 /* wakeup anybody waiting for slots to pin pages */
523 static ssize_t
p9_mount_tag_show(struct device
*dev
,
524 struct device_attribute
*attr
, char *buf
)
526 struct virtio_chan
*chan
;
527 struct virtio_device
*vdev
;
529 vdev
= dev_to_virtio(dev
);
532 memcpy(buf
, chan
->tag
, chan
->tag_len
);
533 buf
[chan
->tag_len
] = 0;
535 return chan
->tag_len
+ 1;
538 static DEVICE_ATTR(mount_tag
, 0444, p9_mount_tag_show
, NULL
);
541 * p9_virtio_probe - probe for existence of 9P virtio channels
542 * @vdev: virtio device to probe
544 * This probes for existing virtio channels.
548 static int p9_virtio_probe(struct virtio_device
*vdev
)
553 struct virtio_chan
*chan
;
555 if (!vdev
->config
->get
) {
556 dev_err(&vdev
->dev
, "%s failure: config access disabled\n",
561 chan
= kmalloc(sizeof(struct virtio_chan
), GFP_KERNEL
);
563 pr_err("Failed to allocate virtio 9P channel\n");
570 /* We expect one virtqueue, for requests. */
571 chan
->vq
= virtio_find_single_vq(vdev
, req_done
, "requests");
572 if (IS_ERR(chan
->vq
)) {
573 err
= PTR_ERR(chan
->vq
);
576 chan
->vq
->vdev
->priv
= chan
;
577 spin_lock_init(&chan
->lock
);
579 sg_init_table(chan
->sg
, VIRTQUEUE_NUM
);
582 if (virtio_has_feature(vdev
, VIRTIO_9P_MOUNT_TAG
)) {
583 virtio_cread(vdev
, struct virtio_9p_config
, tag_len
, &tag_len
);
588 tag
= kmalloc(tag_len
, GFP_KERNEL
);
594 virtio_cread_bytes(vdev
, offsetof(struct virtio_9p_config
, tag
),
597 chan
->tag_len
= tag_len
;
598 err
= sysfs_create_file(&(vdev
->dev
.kobj
), &dev_attr_mount_tag
.attr
);
602 chan
->vc_wq
= kmalloc(sizeof(wait_queue_head_t
), GFP_KERNEL
);
607 init_waitqueue_head(chan
->vc_wq
);
608 chan
->ring_bufs_avail
= 1;
609 /* Ceiling limit to avoid denial of service attacks */
610 chan
->p9_max_pages
= nr_free_buffer_pages()/4;
612 virtio_device_ready(vdev
);
614 mutex_lock(&virtio_9p_lock
);
615 list_add_tail(&chan
->chan_list
, &virtio_chan_list
);
616 mutex_unlock(&virtio_9p_lock
);
618 /* Let udev rules use the new mount_tag attribute. */
619 kobject_uevent(&(vdev
->dev
.kobj
), KOBJ_CHANGE
);
626 vdev
->config
->del_vqs(vdev
);
635 * p9_virtio_create - allocate a new virtio channel
636 * @client: client instance invoking this transport
637 * @devname: string identifying the channel to connect to (unused)
638 * @args: args passed from sys_mount() for per-transport options (unused)
640 * This sets up a transport channel for 9p communication. Right now
641 * we only match the first available channel, but eventually we couldlook up
642 * alternate channels by matching devname versus a virtio_config entry.
643 * We use a simple reference count mechanism to ensure that only a single
644 * mount has a channel open at a time.
649 p9_virtio_create(struct p9_client
*client
, const char *devname
, char *args
)
651 struct virtio_chan
*chan
;
658 mutex_lock(&virtio_9p_lock
);
659 list_for_each_entry(chan
, &virtio_chan_list
, chan_list
) {
660 if (!strncmp(devname
, chan
->tag
, chan
->tag_len
) &&
661 strlen(devname
) == chan
->tag_len
) {
670 mutex_unlock(&virtio_9p_lock
);
673 pr_err("no channels available for device %s\n", devname
);
677 client
->trans
= (void *)chan
;
678 client
->status
= Connected
;
679 chan
->client
= client
;
685 * p9_virtio_remove - clean up resources associated with a virtio device
686 * @vdev: virtio device to remove
690 static void p9_virtio_remove(struct virtio_device
*vdev
)
692 struct virtio_chan
*chan
= vdev
->priv
;
693 unsigned long warning_time
;
695 mutex_lock(&virtio_9p_lock
);
697 /* Remove self from list so we don't get new users. */
698 list_del(&chan
->chan_list
);
699 warning_time
= jiffies
;
701 /* Wait for existing users to close. */
702 while (chan
->inuse
) {
703 mutex_unlock(&virtio_9p_lock
);
705 if (time_after(jiffies
, warning_time
+ 10 * HZ
)) {
706 dev_emerg(&vdev
->dev
,
707 "p9_virtio_remove: waiting for device in use.\n");
708 warning_time
= jiffies
;
710 mutex_lock(&virtio_9p_lock
);
713 mutex_unlock(&virtio_9p_lock
);
715 vdev
->config
->reset(vdev
);
716 vdev
->config
->del_vqs(vdev
);
718 sysfs_remove_file(&(vdev
->dev
.kobj
), &dev_attr_mount_tag
.attr
);
719 kobject_uevent(&(vdev
->dev
.kobj
), KOBJ_CHANGE
);
726 static struct virtio_device_id id_table
[] = {
727 { VIRTIO_ID_9P
, VIRTIO_DEV_ANY_ID
},
731 static unsigned int features
[] = {
735 /* The standard "struct lguest_driver": */
736 static struct virtio_driver p9_virtio_drv
= {
737 .feature_table
= features
,
738 .feature_table_size
= ARRAY_SIZE(features
),
739 .driver
.name
= KBUILD_MODNAME
,
740 .driver
.owner
= THIS_MODULE
,
741 .id_table
= id_table
,
742 .probe
= p9_virtio_probe
,
743 .remove
= p9_virtio_remove
,
746 static struct p9_trans_module p9_virtio_trans
= {
748 .create
= p9_virtio_create
,
749 .close
= p9_virtio_close
,
750 .request
= p9_virtio_request
,
751 .zc_request
= p9_virtio_zc_request
,
752 .cancel
= p9_virtio_cancel
,
754 * We leave one entry for input and one entry for response
755 * headers. We also skip one more entry to accomodate, address
756 * that are not at page boundary, that can result in an extra
759 .maxsize
= PAGE_SIZE
* (VIRTQUEUE_NUM
- 3),
761 .owner
= THIS_MODULE
,
764 /* The standard init function */
765 static int __init
p9_virtio_init(void)
767 INIT_LIST_HEAD(&virtio_chan_list
);
769 v9fs_register_trans(&p9_virtio_trans
);
770 return register_virtio_driver(&p9_virtio_drv
);
773 static void __exit
p9_virtio_cleanup(void)
775 unregister_virtio_driver(&p9_virtio_drv
);
776 v9fs_unregister_trans(&p9_virtio_trans
);
779 module_init(p9_virtio_init
);
780 module_exit(p9_virtio_cleanup
);
782 MODULE_DEVICE_TABLE(virtio
, id_table
);
783 MODULE_AUTHOR("Eric Van Hensbergen <ericvh@gmail.com>");
784 MODULE_DESCRIPTION("Virtio 9p Transport");
785 MODULE_LICENSE("GPL");