2 * Virtio SCSI HBA driver
4 * Copyright IBM Corp. 2010
5 * Copyright Red Hat, Inc. 2011
8 * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>
9 * Paolo Bonzini <pbonzini@redhat.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
16 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18 #include <linux/module.h>
19 #include <linux/slab.h>
20 #include <linux/mempool.h>
21 #include <linux/virtio.h>
22 #include <linux/virtio_ids.h>
23 #include <linux/virtio_config.h>
24 #include <linux/virtio_scsi.h>
25 #include <linux/cpu.h>
26 #include <linux/blkdev.h>
27 #include <scsi/scsi_host.h>
28 #include <scsi/scsi_device.h>
29 #include <scsi/scsi_cmnd.h>
30 #include <scsi/scsi_tcq.h>
31 #include <scsi/scsi_devinfo.h>
32 #include <linux/seqlock.h>
34 #define VIRTIO_SCSI_MEMPOOL_SZ 64
35 #define VIRTIO_SCSI_EVENT_LEN 8
36 #define VIRTIO_SCSI_VQ_BASE 2
38 /* Command queue element */
39 struct virtio_scsi_cmd
{
41 struct completion
*comp
;
43 struct virtio_scsi_cmd_req cmd
;
44 struct virtio_scsi_cmd_req_pi cmd_pi
;
45 struct virtio_scsi_ctrl_tmf_req tmf
;
46 struct virtio_scsi_ctrl_an_req an
;
49 struct virtio_scsi_cmd_resp cmd
;
50 struct virtio_scsi_ctrl_tmf_resp tmf
;
51 struct virtio_scsi_ctrl_an_resp an
;
52 struct virtio_scsi_event evt
;
54 } ____cacheline_aligned_in_smp
;
56 struct virtio_scsi_event_node
{
57 struct virtio_scsi
*vscsi
;
58 struct virtio_scsi_event event
;
59 struct work_struct work
;
62 struct virtio_scsi_vq
{
70 * Per-target queue state.
72 * This struct holds the data needed by the queue steering policy. When a
73 * target is sent multiple requests, we need to drive them to the same queue so
74 * that FIFO processing order is kept. However, if a target was idle, we can
75 * choose a queue arbitrarily. In this case the queue is chosen according to
76 * the current VCPU, so the driver expects the number of request queues to be
77 * equal to the number of VCPUs. This makes it easy and fast to select the
78 * queue, and also lets the driver optimize the IRQ affinity for the virtqueues
79 * (each virtqueue's affinity is set to the CPU that "owns" the queue).
81 * tgt_seq is held to serialize reading and writing req_vq.
83 * Decrements of reqs are never concurrent with writes of req_vq: before the
84 * decrement reqs will be != 0; after the decrement the virtqueue completion
85 * routine will not use the req_vq so it can be changed by a new request.
86 * Thus they can happen outside the tgt_seq, provided of course we make reqs
89 struct virtio_scsi_target_state
{
92 /* Count of outstanding requests. */
95 /* Currently active virtqueue for requests sent to this target. */
96 struct virtio_scsi_vq
*req_vq
;
99 /* Driver instance state */
101 struct virtio_device
*vdev
;
103 /* Get some buffers ready for event vq */
104 struct virtio_scsi_event_node event_list
[VIRTIO_SCSI_EVENT_LEN
];
108 /* If the affinity hint is set for virtqueues */
109 bool affinity_hint_set
;
111 /* CPU hotplug notifier */
112 struct notifier_block nb
;
114 /* Protected by event_vq lock */
117 struct virtio_scsi_vq ctrl_vq
;
118 struct virtio_scsi_vq event_vq
;
119 struct virtio_scsi_vq req_vqs
[];
122 static struct kmem_cache
*virtscsi_cmd_cache
;
123 static mempool_t
*virtscsi_cmd_pool
;
125 static inline struct Scsi_Host
*virtio_scsi_host(struct virtio_device
*vdev
)
130 static void virtscsi_compute_resid(struct scsi_cmnd
*sc
, u32 resid
)
135 if (!scsi_bidi_cmnd(sc
)) {
136 scsi_set_resid(sc
, resid
);
140 scsi_in(sc
)->resid
= min(resid
, scsi_in(sc
)->length
);
141 scsi_out(sc
)->resid
= resid
- scsi_in(sc
)->resid
;
145 * virtscsi_complete_cmd - finish a scsi_cmd and invoke scsi_done
147 * Called with vq_lock held.
149 static void virtscsi_complete_cmd(struct virtio_scsi
*vscsi
, void *buf
)
151 struct virtio_scsi_cmd
*cmd
= buf
;
152 struct scsi_cmnd
*sc
= cmd
->sc
;
153 struct virtio_scsi_cmd_resp
*resp
= &cmd
->resp
.cmd
;
154 struct virtio_scsi_target_state
*tgt
=
155 scsi_target(sc
->device
)->hostdata
;
157 dev_dbg(&sc
->device
->sdev_gendev
,
158 "cmd %p response %u status %#02x sense_len %u\n",
159 sc
, resp
->response
, resp
->status
, resp
->sense_len
);
161 sc
->result
= resp
->status
;
162 virtscsi_compute_resid(sc
, virtio32_to_cpu(vscsi
->vdev
, resp
->resid
));
163 switch (resp
->response
) {
164 case VIRTIO_SCSI_S_OK
:
165 set_host_byte(sc
, DID_OK
);
167 case VIRTIO_SCSI_S_OVERRUN
:
168 set_host_byte(sc
, DID_ERROR
);
170 case VIRTIO_SCSI_S_ABORTED
:
171 set_host_byte(sc
, DID_ABORT
);
173 case VIRTIO_SCSI_S_BAD_TARGET
:
174 set_host_byte(sc
, DID_BAD_TARGET
);
176 case VIRTIO_SCSI_S_RESET
:
177 set_host_byte(sc
, DID_RESET
);
179 case VIRTIO_SCSI_S_BUSY
:
180 set_host_byte(sc
, DID_BUS_BUSY
);
182 case VIRTIO_SCSI_S_TRANSPORT_FAILURE
:
183 set_host_byte(sc
, DID_TRANSPORT_DISRUPTED
);
185 case VIRTIO_SCSI_S_TARGET_FAILURE
:
186 set_host_byte(sc
, DID_TARGET_FAILURE
);
188 case VIRTIO_SCSI_S_NEXUS_FAILURE
:
189 set_host_byte(sc
, DID_NEXUS_FAILURE
);
192 scmd_printk(KERN_WARNING
, sc
, "Unknown response %d",
195 case VIRTIO_SCSI_S_FAILURE
:
196 set_host_byte(sc
, DID_ERROR
);
200 WARN_ON(virtio32_to_cpu(vscsi
->vdev
, resp
->sense_len
) >
201 VIRTIO_SCSI_SENSE_SIZE
);
202 if (sc
->sense_buffer
) {
203 memcpy(sc
->sense_buffer
, resp
->sense
,
205 virtio32_to_cpu(vscsi
->vdev
, resp
->sense_len
),
206 VIRTIO_SCSI_SENSE_SIZE
));
208 set_driver_byte(sc
, DRIVER_SENSE
);
213 atomic_dec(&tgt
->reqs
);
216 static void virtscsi_vq_done(struct virtio_scsi
*vscsi
,
217 struct virtio_scsi_vq
*virtscsi_vq
,
218 void (*fn
)(struct virtio_scsi
*vscsi
, void *buf
))
223 struct virtqueue
*vq
= virtscsi_vq
->vq
;
225 spin_lock_irqsave(&virtscsi_vq
->vq_lock
, flags
);
227 virtqueue_disable_cb(vq
);
228 while ((buf
= virtqueue_get_buf(vq
, &len
)) != NULL
)
231 if (unlikely(virtqueue_is_broken(vq
)))
233 } while (!virtqueue_enable_cb(vq
));
234 spin_unlock_irqrestore(&virtscsi_vq
->vq_lock
, flags
);
237 static void virtscsi_req_done(struct virtqueue
*vq
)
239 struct Scsi_Host
*sh
= virtio_scsi_host(vq
->vdev
);
240 struct virtio_scsi
*vscsi
= shost_priv(sh
);
241 int index
= vq
->index
- VIRTIO_SCSI_VQ_BASE
;
242 struct virtio_scsi_vq
*req_vq
= &vscsi
->req_vqs
[index
];
244 virtscsi_vq_done(vscsi
, req_vq
, virtscsi_complete_cmd
);
247 static void virtscsi_poll_requests(struct virtio_scsi
*vscsi
)
251 num_vqs
= vscsi
->num_queues
;
252 for (i
= 0; i
< num_vqs
; i
++)
253 virtscsi_vq_done(vscsi
, &vscsi
->req_vqs
[i
],
254 virtscsi_complete_cmd
);
257 static void virtscsi_complete_free(struct virtio_scsi
*vscsi
, void *buf
)
259 struct virtio_scsi_cmd
*cmd
= buf
;
262 complete_all(cmd
->comp
);
265 static void virtscsi_ctrl_done(struct virtqueue
*vq
)
267 struct Scsi_Host
*sh
= virtio_scsi_host(vq
->vdev
);
268 struct virtio_scsi
*vscsi
= shost_priv(sh
);
270 virtscsi_vq_done(vscsi
, &vscsi
->ctrl_vq
, virtscsi_complete_free
);
273 static void virtscsi_handle_event(struct work_struct
*work
);
275 static int virtscsi_kick_event(struct virtio_scsi
*vscsi
,
276 struct virtio_scsi_event_node
*event_node
)
279 struct scatterlist sg
;
282 INIT_WORK(&event_node
->work
, virtscsi_handle_event
);
283 sg_init_one(&sg
, &event_node
->event
, sizeof(struct virtio_scsi_event
));
285 spin_lock_irqsave(&vscsi
->event_vq
.vq_lock
, flags
);
287 err
= virtqueue_add_inbuf(vscsi
->event_vq
.vq
, &sg
, 1, event_node
,
290 virtqueue_kick(vscsi
->event_vq
.vq
);
292 spin_unlock_irqrestore(&vscsi
->event_vq
.vq_lock
, flags
);
297 static int virtscsi_kick_event_all(struct virtio_scsi
*vscsi
)
301 for (i
= 0; i
< VIRTIO_SCSI_EVENT_LEN
; i
++) {
302 vscsi
->event_list
[i
].vscsi
= vscsi
;
303 virtscsi_kick_event(vscsi
, &vscsi
->event_list
[i
]);
309 static void virtscsi_cancel_event_work(struct virtio_scsi
*vscsi
)
313 /* Stop scheduling work before calling cancel_work_sync. */
314 spin_lock_irq(&vscsi
->event_vq
.vq_lock
);
315 vscsi
->stop_events
= true;
316 spin_unlock_irq(&vscsi
->event_vq
.vq_lock
);
318 for (i
= 0; i
< VIRTIO_SCSI_EVENT_LEN
; i
++)
319 cancel_work_sync(&vscsi
->event_list
[i
].work
);
322 static void virtscsi_handle_transport_reset(struct virtio_scsi
*vscsi
,
323 struct virtio_scsi_event
*event
)
325 struct scsi_device
*sdev
;
326 struct Scsi_Host
*shost
= virtio_scsi_host(vscsi
->vdev
);
327 unsigned int target
= event
->lun
[1];
328 unsigned int lun
= (event
->lun
[2] << 8) | event
->lun
[3];
330 switch (virtio32_to_cpu(vscsi
->vdev
, event
->reason
)) {
331 case VIRTIO_SCSI_EVT_RESET_RESCAN
:
332 scsi_add_device(shost
, 0, target
, lun
);
334 case VIRTIO_SCSI_EVT_RESET_REMOVED
:
335 sdev
= scsi_device_lookup(shost
, 0, target
, lun
);
337 scsi_remove_device(sdev
);
338 scsi_device_put(sdev
);
340 pr_err("SCSI device %d 0 %d %d not found\n",
341 shost
->host_no
, target
, lun
);
345 pr_info("Unsupport virtio scsi event reason %x\n", event
->reason
);
349 static void virtscsi_handle_param_change(struct virtio_scsi
*vscsi
,
350 struct virtio_scsi_event
*event
)
352 struct scsi_device
*sdev
;
353 struct Scsi_Host
*shost
= virtio_scsi_host(vscsi
->vdev
);
354 unsigned int target
= event
->lun
[1];
355 unsigned int lun
= (event
->lun
[2] << 8) | event
->lun
[3];
356 u8 asc
= virtio32_to_cpu(vscsi
->vdev
, event
->reason
) & 255;
357 u8 ascq
= virtio32_to_cpu(vscsi
->vdev
, event
->reason
) >> 8;
359 sdev
= scsi_device_lookup(shost
, 0, target
, lun
);
361 pr_err("SCSI device %d 0 %d %d not found\n",
362 shost
->host_no
, target
, lun
);
366 /* Handle "Parameters changed", "Mode parameters changed", and
367 "Capacity data has changed". */
368 if (asc
== 0x2a && (ascq
== 0x00 || ascq
== 0x01 || ascq
== 0x09))
369 scsi_rescan_device(&sdev
->sdev_gendev
);
371 scsi_device_put(sdev
);
374 static void virtscsi_handle_event(struct work_struct
*work
)
376 struct virtio_scsi_event_node
*event_node
=
377 container_of(work
, struct virtio_scsi_event_node
, work
);
378 struct virtio_scsi
*vscsi
= event_node
->vscsi
;
379 struct virtio_scsi_event
*event
= &event_node
->event
;
382 cpu_to_virtio32(vscsi
->vdev
, VIRTIO_SCSI_T_EVENTS_MISSED
)) {
383 event
->event
&= ~cpu_to_virtio32(vscsi
->vdev
,
384 VIRTIO_SCSI_T_EVENTS_MISSED
);
385 scsi_scan_host(virtio_scsi_host(vscsi
->vdev
));
388 switch (virtio32_to_cpu(vscsi
->vdev
, event
->event
)) {
389 case VIRTIO_SCSI_T_NO_EVENT
:
391 case VIRTIO_SCSI_T_TRANSPORT_RESET
:
392 virtscsi_handle_transport_reset(vscsi
, event
);
394 case VIRTIO_SCSI_T_PARAM_CHANGE
:
395 virtscsi_handle_param_change(vscsi
, event
);
398 pr_err("Unsupport virtio scsi event %x\n", event
->event
);
400 virtscsi_kick_event(vscsi
, event_node
);
403 static void virtscsi_complete_event(struct virtio_scsi
*vscsi
, void *buf
)
405 struct virtio_scsi_event_node
*event_node
= buf
;
407 if (!vscsi
->stop_events
)
408 queue_work(system_freezable_wq
, &event_node
->work
);
411 static void virtscsi_event_done(struct virtqueue
*vq
)
413 struct Scsi_Host
*sh
= virtio_scsi_host(vq
->vdev
);
414 struct virtio_scsi
*vscsi
= shost_priv(sh
);
416 virtscsi_vq_done(vscsi
, &vscsi
->event_vq
, virtscsi_complete_event
);
420 * virtscsi_add_cmd - add a virtio_scsi_cmd to a virtqueue
421 * @vq : the struct virtqueue we're talking about
422 * @cmd : command structure
423 * @req_size : size of the request buffer
424 * @resp_size : size of the response buffer
426 static int virtscsi_add_cmd(struct virtqueue
*vq
,
427 struct virtio_scsi_cmd
*cmd
,
428 size_t req_size
, size_t resp_size
)
430 struct scsi_cmnd
*sc
= cmd
->sc
;
431 struct scatterlist
*sgs
[6], req
, resp
;
432 struct sg_table
*out
, *in
;
433 unsigned out_num
= 0, in_num
= 0;
437 if (sc
&& sc
->sc_data_direction
!= DMA_NONE
) {
438 if (sc
->sc_data_direction
!= DMA_FROM_DEVICE
)
439 out
= &scsi_out(sc
)->table
;
440 if (sc
->sc_data_direction
!= DMA_TO_DEVICE
)
441 in
= &scsi_in(sc
)->table
;
444 /* Request header. */
445 sg_init_one(&req
, &cmd
->req
, req_size
);
446 sgs
[out_num
++] = &req
;
448 /* Data-out buffer. */
450 /* Place WRITE protection SGLs before Data OUT payload */
451 if (scsi_prot_sg_count(sc
))
452 sgs
[out_num
++] = scsi_prot_sglist(sc
);
453 sgs
[out_num
++] = out
->sgl
;
456 /* Response header. */
457 sg_init_one(&resp
, &cmd
->resp
, resp_size
);
458 sgs
[out_num
+ in_num
++] = &resp
;
462 /* Place READ protection SGLs before Data IN payload */
463 if (scsi_prot_sg_count(sc
))
464 sgs
[out_num
+ in_num
++] = scsi_prot_sglist(sc
);
465 sgs
[out_num
+ in_num
++] = in
->sgl
;
468 return virtqueue_add_sgs(vq
, sgs
, out_num
, in_num
, cmd
, GFP_ATOMIC
);
471 static int virtscsi_kick_cmd(struct virtio_scsi_vq
*vq
,
472 struct virtio_scsi_cmd
*cmd
,
473 size_t req_size
, size_t resp_size
)
477 bool needs_kick
= false;
479 spin_lock_irqsave(&vq
->vq_lock
, flags
);
480 err
= virtscsi_add_cmd(vq
->vq
, cmd
, req_size
, resp_size
);
482 needs_kick
= virtqueue_kick_prepare(vq
->vq
);
484 spin_unlock_irqrestore(&vq
->vq_lock
, flags
);
487 virtqueue_notify(vq
->vq
);
491 static void virtio_scsi_init_hdr(struct virtio_device
*vdev
,
492 struct virtio_scsi_cmd_req
*cmd
,
493 struct scsi_cmnd
*sc
)
496 cmd
->lun
[1] = sc
->device
->id
;
497 cmd
->lun
[2] = (sc
->device
->lun
>> 8) | 0x40;
498 cmd
->lun
[3] = sc
->device
->lun
& 0xff;
499 cmd
->tag
= cpu_to_virtio64(vdev
, (unsigned long)sc
);
500 cmd
->task_attr
= VIRTIO_SCSI_S_SIMPLE
;
505 #ifdef CONFIG_BLK_DEV_INTEGRITY
506 static void virtio_scsi_init_hdr_pi(struct virtio_device
*vdev
,
507 struct virtio_scsi_cmd_req_pi
*cmd_pi
,
508 struct scsi_cmnd
*sc
)
510 struct request
*rq
= sc
->request
;
511 struct blk_integrity
*bi
;
513 virtio_scsi_init_hdr(vdev
, (struct virtio_scsi_cmd_req
*)cmd_pi
, sc
);
515 if (!rq
|| !scsi_prot_sg_count(sc
))
518 bi
= blk_get_integrity(rq
->rq_disk
);
520 if (sc
->sc_data_direction
== DMA_TO_DEVICE
)
521 cmd_pi
->pi_bytesout
= cpu_to_virtio32(vdev
,
524 else if (sc
->sc_data_direction
== DMA_FROM_DEVICE
)
525 cmd_pi
->pi_bytesin
= cpu_to_virtio32(vdev
,
531 static int virtscsi_queuecommand(struct virtio_scsi
*vscsi
,
532 struct virtio_scsi_vq
*req_vq
,
533 struct scsi_cmnd
*sc
)
535 struct Scsi_Host
*shost
= virtio_scsi_host(vscsi
->vdev
);
536 struct virtio_scsi_cmd
*cmd
= scsi_cmd_priv(sc
);
541 BUG_ON(scsi_sg_count(sc
) > shost
->sg_tablesize
);
543 /* TODO: check feature bit and fail if unsupported? */
544 BUG_ON(sc
->sc_data_direction
== DMA_BIDIRECTIONAL
);
546 dev_dbg(&sc
->device
->sdev_gendev
,
547 "cmd %p CDB: %#02x\n", sc
, sc
->cmnd
[0]);
549 memset(cmd
, 0, sizeof(*cmd
));
552 BUG_ON(sc
->cmd_len
> VIRTIO_SCSI_CDB_SIZE
);
554 #ifdef CONFIG_BLK_DEV_INTEGRITY
555 if (virtio_has_feature(vscsi
->vdev
, VIRTIO_SCSI_F_T10_PI
)) {
556 virtio_scsi_init_hdr_pi(vscsi
->vdev
, &cmd
->req
.cmd_pi
, sc
);
557 memcpy(cmd
->req
.cmd_pi
.cdb
, sc
->cmnd
, sc
->cmd_len
);
558 req_size
= sizeof(cmd
->req
.cmd_pi
);
562 virtio_scsi_init_hdr(vscsi
->vdev
, &cmd
->req
.cmd
, sc
);
563 memcpy(cmd
->req
.cmd
.cdb
, sc
->cmnd
, sc
->cmd_len
);
564 req_size
= sizeof(cmd
->req
.cmd
);
567 ret
= virtscsi_kick_cmd(req_vq
, cmd
, req_size
, sizeof(cmd
->resp
.cmd
));
569 cmd
->resp
.cmd
.response
= VIRTIO_SCSI_S_BAD_TARGET
;
570 spin_lock_irqsave(&req_vq
->vq_lock
, flags
);
571 virtscsi_complete_cmd(vscsi
, cmd
);
572 spin_unlock_irqrestore(&req_vq
->vq_lock
, flags
);
573 } else if (ret
!= 0) {
574 return SCSI_MLQUEUE_HOST_BUSY
;
579 static int virtscsi_queuecommand_single(struct Scsi_Host
*sh
,
580 struct scsi_cmnd
*sc
)
582 struct virtio_scsi
*vscsi
= shost_priv(sh
);
583 struct virtio_scsi_target_state
*tgt
=
584 scsi_target(sc
->device
)->hostdata
;
586 atomic_inc(&tgt
->reqs
);
587 return virtscsi_queuecommand(vscsi
, &vscsi
->req_vqs
[0], sc
);
590 static struct virtio_scsi_vq
*virtscsi_pick_vq_mq(struct virtio_scsi
*vscsi
,
591 struct scsi_cmnd
*sc
)
593 u32 tag
= blk_mq_unique_tag(sc
->request
);
594 u16 hwq
= blk_mq_unique_tag_to_hwq(tag
);
596 return &vscsi
->req_vqs
[hwq
];
599 static struct virtio_scsi_vq
*virtscsi_pick_vq(struct virtio_scsi
*vscsi
,
600 struct virtio_scsi_target_state
*tgt
)
602 struct virtio_scsi_vq
*vq
;
606 local_irq_save(flags
);
607 if (atomic_inc_return(&tgt
->reqs
) > 1) {
611 seq
= read_seqcount_begin(&tgt
->tgt_seq
);
613 } while (read_seqcount_retry(&tgt
->tgt_seq
, seq
));
615 /* no writes can be concurrent because of atomic_t */
616 write_seqcount_begin(&tgt
->tgt_seq
);
618 /* keep previous req_vq if a reader just arrived */
619 if (unlikely(atomic_read(&tgt
->reqs
) > 1)) {
624 queue_num
= smp_processor_id();
625 while (unlikely(queue_num
>= vscsi
->num_queues
))
626 queue_num
-= vscsi
->num_queues
;
627 tgt
->req_vq
= vq
= &vscsi
->req_vqs
[queue_num
];
629 write_seqcount_end(&tgt
->tgt_seq
);
631 local_irq_restore(flags
);
636 static int virtscsi_queuecommand_multi(struct Scsi_Host
*sh
,
637 struct scsi_cmnd
*sc
)
639 struct virtio_scsi
*vscsi
= shost_priv(sh
);
640 struct virtio_scsi_target_state
*tgt
=
641 scsi_target(sc
->device
)->hostdata
;
642 struct virtio_scsi_vq
*req_vq
;
644 if (shost_use_blk_mq(sh
))
645 req_vq
= virtscsi_pick_vq_mq(vscsi
, sc
);
647 req_vq
= virtscsi_pick_vq(vscsi
, tgt
);
649 return virtscsi_queuecommand(vscsi
, req_vq
, sc
);
652 static int virtscsi_tmf(struct virtio_scsi
*vscsi
, struct virtio_scsi_cmd
*cmd
)
654 DECLARE_COMPLETION_ONSTACK(comp
);
658 if (virtscsi_kick_cmd(&vscsi
->ctrl_vq
, cmd
,
659 sizeof cmd
->req
.tmf
, sizeof cmd
->resp
.tmf
) < 0)
662 wait_for_completion(&comp
);
663 if (cmd
->resp
.tmf
.response
== VIRTIO_SCSI_S_OK
||
664 cmd
->resp
.tmf
.response
== VIRTIO_SCSI_S_FUNCTION_SUCCEEDED
)
668 * The spec guarantees that all requests related to the TMF have
669 * been completed, but the callback might not have run yet if
670 * we're using independent interrupts (e.g. MSI). Poll the
673 * In the abort case, sc->scsi_done will do nothing, because
674 * the block layer must have detected a timeout and as a result
675 * REQ_ATOM_COMPLETE has been set.
677 virtscsi_poll_requests(vscsi
);
680 mempool_free(cmd
, virtscsi_cmd_pool
);
684 static int virtscsi_device_reset(struct scsi_cmnd
*sc
)
686 struct virtio_scsi
*vscsi
= shost_priv(sc
->device
->host
);
687 struct virtio_scsi_cmd
*cmd
;
689 sdev_printk(KERN_INFO
, sc
->device
, "device reset\n");
690 cmd
= mempool_alloc(virtscsi_cmd_pool
, GFP_NOIO
);
694 memset(cmd
, 0, sizeof(*cmd
));
696 cmd
->req
.tmf
= (struct virtio_scsi_ctrl_tmf_req
){
697 .type
= VIRTIO_SCSI_T_TMF
,
698 .subtype
= cpu_to_virtio32(vscsi
->vdev
,
699 VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET
),
701 .lun
[1] = sc
->device
->id
,
702 .lun
[2] = (sc
->device
->lun
>> 8) | 0x40,
703 .lun
[3] = sc
->device
->lun
& 0xff,
705 return virtscsi_tmf(vscsi
, cmd
);
708 static int virtscsi_device_alloc(struct scsi_device
*sdevice
)
711 * Passed through SCSI targets (e.g. with qemu's 'scsi-block')
712 * may have transfer limits which come from the host SCSI
713 * controller or something on the host side other than the
716 * To make this work properly, the hypervisor can adjust the
717 * target's VPD information to advertise these limits. But
718 * for that to work, the guest has to look at the VPD pages,
719 * which we won't do by default if it is an SPC-2 device, even
720 * if it does actually support it.
722 * So, set the blist to always try to read the VPD pages.
724 sdevice
->sdev_bflags
= BLIST_TRY_VPD_PAGES
;
731 * virtscsi_change_queue_depth() - Change a virtscsi target's queue depth
732 * @sdev: Virtscsi target whose queue depth to change
733 * @qdepth: New queue depth
735 static int virtscsi_change_queue_depth(struct scsi_device
*sdev
, int qdepth
)
737 struct Scsi_Host
*shost
= sdev
->host
;
738 int max_depth
= shost
->cmd_per_lun
;
740 return scsi_change_queue_depth(sdev
, min(max_depth
, qdepth
));
743 static int virtscsi_abort(struct scsi_cmnd
*sc
)
745 struct virtio_scsi
*vscsi
= shost_priv(sc
->device
->host
);
746 struct virtio_scsi_cmd
*cmd
;
748 scmd_printk(KERN_INFO
, sc
, "abort\n");
749 cmd
= mempool_alloc(virtscsi_cmd_pool
, GFP_NOIO
);
753 memset(cmd
, 0, sizeof(*cmd
));
755 cmd
->req
.tmf
= (struct virtio_scsi_ctrl_tmf_req
){
756 .type
= VIRTIO_SCSI_T_TMF
,
757 .subtype
= VIRTIO_SCSI_T_TMF_ABORT_TASK
,
759 .lun
[1] = sc
->device
->id
,
760 .lun
[2] = (sc
->device
->lun
>> 8) | 0x40,
761 .lun
[3] = sc
->device
->lun
& 0xff,
762 .tag
= cpu_to_virtio64(vscsi
->vdev
, (unsigned long)sc
),
764 return virtscsi_tmf(vscsi
, cmd
);
767 static int virtscsi_target_alloc(struct scsi_target
*starget
)
769 struct Scsi_Host
*sh
= dev_to_shost(starget
->dev
.parent
);
770 struct virtio_scsi
*vscsi
= shost_priv(sh
);
772 struct virtio_scsi_target_state
*tgt
=
773 kmalloc(sizeof(*tgt
), GFP_KERNEL
);
777 seqcount_init(&tgt
->tgt_seq
);
778 atomic_set(&tgt
->reqs
, 0);
779 tgt
->req_vq
= &vscsi
->req_vqs
[0];
781 starget
->hostdata
= tgt
;
785 static void virtscsi_target_destroy(struct scsi_target
*starget
)
787 struct virtio_scsi_target_state
*tgt
= starget
->hostdata
;
791 static struct scsi_host_template virtscsi_host_template_single
= {
792 .module
= THIS_MODULE
,
793 .name
= "Virtio SCSI HBA",
794 .proc_name
= "virtio_scsi",
796 .cmd_size
= sizeof(struct virtio_scsi_cmd
),
797 .queuecommand
= virtscsi_queuecommand_single
,
798 .change_queue_depth
= virtscsi_change_queue_depth
,
799 .eh_abort_handler
= virtscsi_abort
,
800 .eh_device_reset_handler
= virtscsi_device_reset
,
801 .slave_alloc
= virtscsi_device_alloc
,
804 .dma_boundary
= UINT_MAX
,
805 .use_clustering
= ENABLE_CLUSTERING
,
806 .target_alloc
= virtscsi_target_alloc
,
807 .target_destroy
= virtscsi_target_destroy
,
808 .track_queue_depth
= 1,
811 static struct scsi_host_template virtscsi_host_template_multi
= {
812 .module
= THIS_MODULE
,
813 .name
= "Virtio SCSI HBA",
814 .proc_name
= "virtio_scsi",
816 .cmd_size
= sizeof(struct virtio_scsi_cmd
),
817 .queuecommand
= virtscsi_queuecommand_multi
,
818 .change_queue_depth
= virtscsi_change_queue_depth
,
819 .eh_abort_handler
= virtscsi_abort
,
820 .eh_device_reset_handler
= virtscsi_device_reset
,
822 .slave_alloc
= virtscsi_device_alloc
,
824 .dma_boundary
= UINT_MAX
,
825 .use_clustering
= ENABLE_CLUSTERING
,
826 .target_alloc
= virtscsi_target_alloc
,
827 .target_destroy
= virtscsi_target_destroy
,
828 .track_queue_depth
= 1,
831 #define virtscsi_config_get(vdev, fld) \
833 typeof(((struct virtio_scsi_config *)0)->fld) __val; \
834 virtio_cread(vdev, struct virtio_scsi_config, fld, &__val); \
838 #define virtscsi_config_set(vdev, fld, val) \
840 typeof(((struct virtio_scsi_config *)0)->fld) __val = (val); \
841 virtio_cwrite(vdev, struct virtio_scsi_config, fld, &__val); \
844 static void __virtscsi_set_affinity(struct virtio_scsi
*vscsi
, bool affinity
)
849 /* In multiqueue mode, when the number of cpu is equal
850 * to the number of request queues, we let the qeueues
851 * to be private to one cpu by setting the affinity hint
852 * to eliminate the contention.
854 if ((vscsi
->num_queues
== 1 ||
855 vscsi
->num_queues
!= num_online_cpus()) && affinity
) {
856 if (vscsi
->affinity_hint_set
)
864 for_each_online_cpu(cpu
) {
865 virtqueue_set_affinity(vscsi
->req_vqs
[i
].vq
, cpu
);
869 vscsi
->affinity_hint_set
= true;
871 for (i
= 0; i
< vscsi
->num_queues
; i
++) {
872 if (!vscsi
->req_vqs
[i
].vq
)
875 virtqueue_set_affinity(vscsi
->req_vqs
[i
].vq
, -1);
878 vscsi
->affinity_hint_set
= false;
882 static void virtscsi_set_affinity(struct virtio_scsi
*vscsi
, bool affinity
)
885 __virtscsi_set_affinity(vscsi
, affinity
);
889 static int virtscsi_cpu_callback(struct notifier_block
*nfb
,
890 unsigned long action
, void *hcpu
)
892 struct virtio_scsi
*vscsi
= container_of(nfb
, struct virtio_scsi
, nb
);
895 case CPU_ONLINE_FROZEN
:
897 case CPU_DEAD_FROZEN
:
898 __virtscsi_set_affinity(vscsi
, true);
906 static void virtscsi_init_vq(struct virtio_scsi_vq
*virtscsi_vq
,
907 struct virtqueue
*vq
)
909 spin_lock_init(&virtscsi_vq
->vq_lock
);
910 virtscsi_vq
->vq
= vq
;
913 static void virtscsi_remove_vqs(struct virtio_device
*vdev
)
915 struct Scsi_Host
*sh
= virtio_scsi_host(vdev
);
916 struct virtio_scsi
*vscsi
= shost_priv(sh
);
918 virtscsi_set_affinity(vscsi
, false);
920 /* Stop all the virtqueues. */
921 vdev
->config
->reset(vdev
);
923 vdev
->config
->del_vqs(vdev
);
926 static int virtscsi_init(struct virtio_device
*vdev
,
927 struct virtio_scsi
*vscsi
)
932 vq_callback_t
**callbacks
;
934 struct virtqueue
**vqs
;
936 num_vqs
= vscsi
->num_queues
+ VIRTIO_SCSI_VQ_BASE
;
937 vqs
= kmalloc(num_vqs
* sizeof(struct virtqueue
*), GFP_KERNEL
);
938 callbacks
= kmalloc(num_vqs
* sizeof(vq_callback_t
*), GFP_KERNEL
);
939 names
= kmalloc(num_vqs
* sizeof(char *), GFP_KERNEL
);
941 if (!callbacks
|| !vqs
|| !names
) {
946 callbacks
[0] = virtscsi_ctrl_done
;
947 callbacks
[1] = virtscsi_event_done
;
948 names
[0] = "control";
950 for (i
= VIRTIO_SCSI_VQ_BASE
; i
< num_vqs
; i
++) {
951 callbacks
[i
] = virtscsi_req_done
;
952 names
[i
] = "request";
955 /* Discover virtqueues and write information to configuration. */
956 err
= vdev
->config
->find_vqs(vdev
, num_vqs
, vqs
, callbacks
, names
);
960 virtscsi_init_vq(&vscsi
->ctrl_vq
, vqs
[0]);
961 virtscsi_init_vq(&vscsi
->event_vq
, vqs
[1]);
962 for (i
= VIRTIO_SCSI_VQ_BASE
; i
< num_vqs
; i
++)
963 virtscsi_init_vq(&vscsi
->req_vqs
[i
- VIRTIO_SCSI_VQ_BASE
],
966 virtscsi_set_affinity(vscsi
, true);
968 virtscsi_config_set(vdev
, cdb_size
, VIRTIO_SCSI_CDB_SIZE
);
969 virtscsi_config_set(vdev
, sense_size
, VIRTIO_SCSI_SENSE_SIZE
);
978 virtscsi_remove_vqs(vdev
);
982 static int virtscsi_probe(struct virtio_device
*vdev
)
984 struct Scsi_Host
*shost
;
985 struct virtio_scsi
*vscsi
;
987 u32 sg_elems
, num_targets
;
990 struct scsi_host_template
*hostt
;
992 if (!vdev
->config
->get
) {
993 dev_err(&vdev
->dev
, "%s failure: config access disabled\n",
998 /* We need to know how many queues before we allocate. */
999 num_queues
= virtscsi_config_get(vdev
, num_queues
) ? : 1;
1001 num_targets
= virtscsi_config_get(vdev
, max_target
) + 1;
1003 if (num_queues
== 1)
1004 hostt
= &virtscsi_host_template_single
;
1006 hostt
= &virtscsi_host_template_multi
;
1008 shost
= scsi_host_alloc(hostt
,
1009 sizeof(*vscsi
) + sizeof(vscsi
->req_vqs
[0]) * num_queues
);
1013 sg_elems
= virtscsi_config_get(vdev
, seg_max
) ?: 1;
1014 shost
->sg_tablesize
= sg_elems
;
1015 vscsi
= shost_priv(shost
);
1017 vscsi
->num_queues
= num_queues
;
1020 err
= virtscsi_init(vdev
, vscsi
);
1022 goto virtscsi_init_failed
;
1024 vscsi
->nb
.notifier_call
= &virtscsi_cpu_callback
;
1025 err
= register_hotcpu_notifier(&vscsi
->nb
);
1027 pr_err("registering cpu notifier failed\n");
1028 goto scsi_add_host_failed
;
1031 cmd_per_lun
= virtscsi_config_get(vdev
, cmd_per_lun
) ?: 1;
1032 shost
->cmd_per_lun
= min_t(u32
, cmd_per_lun
, shost
->can_queue
);
1033 shost
->max_sectors
= virtscsi_config_get(vdev
, max_sectors
) ?: 0xFFFF;
1035 /* LUNs > 256 are reported with format 1, so they go in the range
1038 shost
->max_lun
= virtscsi_config_get(vdev
, max_lun
) + 1 + 0x4000;
1039 shost
->max_id
= num_targets
;
1040 shost
->max_channel
= 0;
1041 shost
->max_cmd_len
= VIRTIO_SCSI_CDB_SIZE
;
1042 shost
->nr_hw_queues
= num_queues
;
1044 #ifdef CONFIG_BLK_DEV_INTEGRITY
1045 if (virtio_has_feature(vdev
, VIRTIO_SCSI_F_T10_PI
)) {
1048 host_prot
= SHOST_DIF_TYPE1_PROTECTION
| SHOST_DIF_TYPE2_PROTECTION
|
1049 SHOST_DIF_TYPE3_PROTECTION
| SHOST_DIX_TYPE1_PROTECTION
|
1050 SHOST_DIX_TYPE2_PROTECTION
| SHOST_DIX_TYPE3_PROTECTION
;
1052 scsi_host_set_prot(shost
, host_prot
);
1053 scsi_host_set_guard(shost
, SHOST_DIX_GUARD_CRC
);
1057 err
= scsi_add_host(shost
, &vdev
->dev
);
1059 goto scsi_add_host_failed
;
1061 virtio_device_ready(vdev
);
1063 if (virtio_has_feature(vdev
, VIRTIO_SCSI_F_HOTPLUG
))
1064 virtscsi_kick_event_all(vscsi
);
1066 scsi_scan_host(shost
);
1069 scsi_add_host_failed
:
1070 vdev
->config
->del_vqs(vdev
);
1071 virtscsi_init_failed
:
1072 scsi_host_put(shost
);
1076 static void virtscsi_remove(struct virtio_device
*vdev
)
1078 struct Scsi_Host
*shost
= virtio_scsi_host(vdev
);
1079 struct virtio_scsi
*vscsi
= shost_priv(shost
);
1081 if (virtio_has_feature(vdev
, VIRTIO_SCSI_F_HOTPLUG
))
1082 virtscsi_cancel_event_work(vscsi
);
1084 scsi_remove_host(shost
);
1086 unregister_hotcpu_notifier(&vscsi
->nb
);
1088 virtscsi_remove_vqs(vdev
);
1089 scsi_host_put(shost
);
1092 #ifdef CONFIG_PM_SLEEP
1093 static int virtscsi_freeze(struct virtio_device
*vdev
)
1095 struct Scsi_Host
*sh
= virtio_scsi_host(vdev
);
1096 struct virtio_scsi
*vscsi
= shost_priv(sh
);
1098 unregister_hotcpu_notifier(&vscsi
->nb
);
1099 virtscsi_remove_vqs(vdev
);
1103 static int virtscsi_restore(struct virtio_device
*vdev
)
1105 struct Scsi_Host
*sh
= virtio_scsi_host(vdev
);
1106 struct virtio_scsi
*vscsi
= shost_priv(sh
);
1109 err
= virtscsi_init(vdev
, vscsi
);
1113 err
= register_hotcpu_notifier(&vscsi
->nb
);
1115 vdev
->config
->del_vqs(vdev
);
1119 virtio_device_ready(vdev
);
1121 if (virtio_has_feature(vdev
, VIRTIO_SCSI_F_HOTPLUG
))
1122 virtscsi_kick_event_all(vscsi
);
1128 static struct virtio_device_id id_table
[] = {
1129 { VIRTIO_ID_SCSI
, VIRTIO_DEV_ANY_ID
},
1133 static unsigned int features
[] = {
1134 VIRTIO_SCSI_F_HOTPLUG
,
1135 VIRTIO_SCSI_F_CHANGE
,
1136 #ifdef CONFIG_BLK_DEV_INTEGRITY
1137 VIRTIO_SCSI_F_T10_PI
,
1141 static struct virtio_driver virtio_scsi_driver
= {
1142 .feature_table
= features
,
1143 .feature_table_size
= ARRAY_SIZE(features
),
1144 .driver
.name
= KBUILD_MODNAME
,
1145 .driver
.owner
= THIS_MODULE
,
1146 .id_table
= id_table
,
1147 .probe
= virtscsi_probe
,
1148 #ifdef CONFIG_PM_SLEEP
1149 .freeze
= virtscsi_freeze
,
1150 .restore
= virtscsi_restore
,
1152 .remove
= virtscsi_remove
,
1155 static int __init
init(void)
1159 virtscsi_cmd_cache
= KMEM_CACHE(virtio_scsi_cmd
, 0);
1160 if (!virtscsi_cmd_cache
) {
1161 pr_err("kmem_cache_create() for virtscsi_cmd_cache failed\n");
1167 mempool_create_slab_pool(VIRTIO_SCSI_MEMPOOL_SZ
,
1168 virtscsi_cmd_cache
);
1169 if (!virtscsi_cmd_pool
) {
1170 pr_err("mempool_create() for virtscsi_cmd_pool failed\n");
1173 ret
= register_virtio_driver(&virtio_scsi_driver
);
1180 if (virtscsi_cmd_pool
) {
1181 mempool_destroy(virtscsi_cmd_pool
);
1182 virtscsi_cmd_pool
= NULL
;
1184 if (virtscsi_cmd_cache
) {
1185 kmem_cache_destroy(virtscsi_cmd_cache
);
1186 virtscsi_cmd_cache
= NULL
;
1191 static void __exit
fini(void)
1193 unregister_virtio_driver(&virtio_scsi_driver
);
1194 mempool_destroy(virtscsi_cmd_pool
);
1195 kmem_cache_destroy(virtscsi_cmd_cache
);
1200 MODULE_DEVICE_TABLE(virtio
, id_table
);
1201 MODULE_DESCRIPTION("Virtio SCSI HBA driver");
1202 MODULE_LICENSE("GPL");