2 * Driver for sTec s1120 PCIe SSDs. sTec was acquired in 2013 by HGST and HGST
3 * was acquired by Western Digital in 2012.
5 * Copyright 2012 sTec, Inc.
6 * Copyright (c) 2017 Western Digital Corporation or its affiliates.
8 * This file is part of the Linux kernel, and is made available under
9 * the terms of the GNU General Public License version 2.
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/pci.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/blkdev.h>
19 #include <linux/blk-mq.h>
20 #include <linux/sched.h>
21 #include <linux/interrupt.h>
22 #include <linux/compiler.h>
23 #include <linux/workqueue.h>
24 #include <linux/delay.h>
25 #include <linux/time.h>
26 #include <linux/hdreg.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/completion.h>
29 #include <linux/scatterlist.h>
30 #include <linux/version.h>
31 #include <linux/err.h>
32 #include <linux/aer.h>
33 #include <linux/wait.h>
34 #include <linux/stringify.h>
35 #include <scsi/scsi.h>
38 #include <linux/uaccess.h>
39 #include <asm/unaligned.h>
41 #include "skd_s1120.h"
43 static int skd_dbg_level
;
44 static int skd_isr_comp_limit
= 4;
46 #define SKD_ASSERT(expr) \
48 if (unlikely(!(expr))) { \
49 pr_err("Assertion failed! %s,%s,%s,line=%d\n", \
50 # expr, __FILE__, __func__, __LINE__); \
54 #define DRV_NAME "skd"
55 #define PFX DRV_NAME ": "
57 MODULE_LICENSE("GPL");
59 MODULE_DESCRIPTION("STEC s1120 PCIe SSD block driver");
61 #define PCI_VENDOR_ID_STEC 0x1B39
62 #define PCI_DEVICE_ID_S1120 0x0001
64 #define SKD_FUA_NV (1 << 1)
65 #define SKD_MINORS_PER_DEVICE 16
67 #define SKD_MAX_QUEUE_DEPTH 200u
69 #define SKD_PAUSE_TIMEOUT (5 * 1000)
71 #define SKD_N_FITMSG_BYTES (512u)
72 #define SKD_MAX_REQ_PER_MSG 14
74 #define SKD_N_SPECIAL_FITMSG_BYTES (128u)
76 /* SG elements are 32 bytes, so we can make this 4096 and still be under the
77 * 128KB limit. That allows 4096*4K = 16M xfer size
79 #define SKD_N_SG_PER_REQ_DEFAULT 256u
81 #define SKD_N_COMPLETION_ENTRY 256u
82 #define SKD_N_READ_CAP_BYTES (8u)
84 #define SKD_N_INTERNAL_BYTES (512u)
86 #define SKD_SKCOMP_SIZE \
87 ((sizeof(struct fit_completion_entry_v1) + \
88 sizeof(struct fit_comp_error_info)) * SKD_N_COMPLETION_ENTRY)
90 /* 5 bits of uniqifier, 0xF800 */
91 #define SKD_ID_TABLE_MASK (3u << 8u)
92 #define SKD_ID_RW_REQUEST (0u << 8u)
93 #define SKD_ID_INTERNAL (1u << 8u)
94 #define SKD_ID_FIT_MSG (3u << 8u)
95 #define SKD_ID_SLOT_MASK 0x00FFu
96 #define SKD_ID_SLOT_AND_TABLE_MASK 0x03FFu
98 #define SKD_N_MAX_SECTORS 2048u
100 #define SKD_MAX_RETRIES 2u
102 #define SKD_TIMER_SECONDS(seconds) (seconds)
103 #define SKD_TIMER_MINUTES(minutes) ((minutes) * (60))
105 #define INQ_STD_NBYTES 36
107 enum skd_drvr_state
{
111 SKD_DRVR_STATE_STARTING
,
112 SKD_DRVR_STATE_ONLINE
,
113 SKD_DRVR_STATE_PAUSING
,
114 SKD_DRVR_STATE_PAUSED
,
115 SKD_DRVR_STATE_RESTARTING
,
116 SKD_DRVR_STATE_RESUMING
,
117 SKD_DRVR_STATE_STOPPING
,
118 SKD_DRVR_STATE_FAULT
,
119 SKD_DRVR_STATE_DISAPPEARED
,
120 SKD_DRVR_STATE_PROTOCOL_MISMATCH
,
121 SKD_DRVR_STATE_BUSY_ERASE
,
122 SKD_DRVR_STATE_BUSY_SANITIZE
,
123 SKD_DRVR_STATE_BUSY_IMMINENT
,
124 SKD_DRVR_STATE_WAIT_BOOT
,
125 SKD_DRVR_STATE_SYNCING
,
128 #define SKD_WAIT_BOOT_TIMO SKD_TIMER_SECONDS(90u)
129 #define SKD_STARTING_TIMO SKD_TIMER_SECONDS(8u)
130 #define SKD_RESTARTING_TIMO SKD_TIMER_MINUTES(4u)
131 #define SKD_BUSY_TIMO SKD_TIMER_MINUTES(20u)
132 #define SKD_STARTED_BUSY_TIMO SKD_TIMER_SECONDS(60u)
133 #define SKD_START_WAIT_SECONDS 90u
139 SKD_REQ_STATE_COMPLETED
,
140 SKD_REQ_STATE_TIMEOUT
,
143 enum skd_check_status_action
{
144 SKD_CHECK_STATUS_REPORT_GOOD
,
145 SKD_CHECK_STATUS_REPORT_SMART_ALERT
,
146 SKD_CHECK_STATUS_REQUEUE_REQUEST
,
147 SKD_CHECK_STATUS_REPORT_ERROR
,
148 SKD_CHECK_STATUS_BUSY_IMMINENT
,
152 struct fit_msg_hdr fmh
;
153 struct skd_scsi_request scsi
[SKD_MAX_REQ_PER_MSG
];
156 struct skd_fitmsg_context
{
161 struct skd_msg_buf
*msg_buf
;
162 dma_addr_t mb_dma_address
;
165 struct skd_request_context
{
166 enum skd_req_state state
;
173 enum dma_data_direction data_dir
;
174 struct scatterlist
*sg
;
178 struct fit_sg_descriptor
*sksg_list
;
179 dma_addr_t sksg_dma_address
;
181 struct fit_completion_entry_v1 completion
;
183 struct fit_comp_error_info err_info
;
188 struct skd_special_context
{
189 struct skd_request_context req
;
192 dma_addr_t db_dma_address
;
194 struct skd_msg_buf
*msg_buf
;
195 dma_addr_t mb_dma_address
;
198 typedef enum skd_irq_type
{
204 #define SKD_MAX_BARS 2
207 void __iomem
*mem_map
[SKD_MAX_BARS
];
208 resource_size_t mem_phys
[SKD_MAX_BARS
];
209 u32 mem_size
[SKD_MAX_BARS
];
211 struct skd_msix_entry
*msix_entries
;
213 struct pci_dev
*pdev
;
214 int pcie_error_reporting_is_enabled
;
217 struct gendisk
*disk
;
218 struct blk_mq_tag_set tag_set
;
219 struct request_queue
*queue
;
220 struct skd_fitmsg_context
*skmsg
;
221 struct device
*class_dev
;
229 enum skd_drvr_state state
;
232 u32 cur_max_queue_depth
;
233 u32 queue_low_water_mark
;
234 u32 dev_max_queue_depth
;
236 u32 num_fitmsg_context
;
239 struct skd_fitmsg_context
*skmsg_table
;
241 struct skd_special_context internal_skspcl
;
242 u32 read_cap_blocksize
;
243 u32 read_cap_last_lba
;
244 int read_cap_is_valid
;
245 int inquiry_is_valid
;
246 u8 inq_serial_num
[13]; /*12 chars plus null term */
250 struct kmem_cache
*msgbuf_cache
;
251 struct kmem_cache
*sglist_cache
;
252 struct kmem_cache
*databuf_cache
;
253 struct fit_completion_entry_v1
*skcomp_table
;
254 struct fit_comp_error_info
*skerr_table
;
255 dma_addr_t cq_dma_address
;
257 wait_queue_head_t waitq
;
259 struct timer_list timer
;
269 u32 connect_time_stamp
;
271 #define SKD_MAX_CONNECT_RETRIES 16
276 struct work_struct start_queue
;
277 struct work_struct completion_worker
;
280 #define SKD_WRITEL(DEV, VAL, OFF) skd_reg_write32(DEV, VAL, OFF)
281 #define SKD_READL(DEV, OFF) skd_reg_read32(DEV, OFF)
282 #define SKD_WRITEQ(DEV, VAL, OFF) skd_reg_write64(DEV, VAL, OFF)
284 static inline u32
skd_reg_read32(struct skd_device
*skdev
, u32 offset
)
286 u32 val
= readl(skdev
->mem_map
[1] + offset
);
288 if (unlikely(skdev
->dbg_level
>= 2))
289 dev_dbg(&skdev
->pdev
->dev
, "offset %x = %x\n", offset
, val
);
293 static inline void skd_reg_write32(struct skd_device
*skdev
, u32 val
,
296 writel(val
, skdev
->mem_map
[1] + offset
);
297 if (unlikely(skdev
->dbg_level
>= 2))
298 dev_dbg(&skdev
->pdev
->dev
, "offset %x = %x\n", offset
, val
);
301 static inline void skd_reg_write64(struct skd_device
*skdev
, u64 val
,
304 writeq(val
, skdev
->mem_map
[1] + offset
);
305 if (unlikely(skdev
->dbg_level
>= 2))
306 dev_dbg(&skdev
->pdev
->dev
, "offset %x = %016llx\n", offset
,
311 #define SKD_IRQ_DEFAULT SKD_IRQ_MSIX
312 static int skd_isr_type
= SKD_IRQ_DEFAULT
;
314 module_param(skd_isr_type
, int, 0444);
315 MODULE_PARM_DESC(skd_isr_type
, "Interrupt type capability."
316 " (0==legacy, 1==MSI, 2==MSI-X, default==1)");
318 #define SKD_MAX_REQ_PER_MSG_DEFAULT 1
319 static int skd_max_req_per_msg
= SKD_MAX_REQ_PER_MSG_DEFAULT
;
321 module_param(skd_max_req_per_msg
, int, 0444);
322 MODULE_PARM_DESC(skd_max_req_per_msg
,
323 "Maximum SCSI requests packed in a single message."
324 " (1-" __stringify(SKD_MAX_REQ_PER_MSG
) ", default==1)");
326 #define SKD_MAX_QUEUE_DEPTH_DEFAULT 64
327 #define SKD_MAX_QUEUE_DEPTH_DEFAULT_STR "64"
328 static int skd_max_queue_depth
= SKD_MAX_QUEUE_DEPTH_DEFAULT
;
330 module_param(skd_max_queue_depth
, int, 0444);
331 MODULE_PARM_DESC(skd_max_queue_depth
,
332 "Maximum SCSI requests issued to s1120."
333 " (1-200, default==" SKD_MAX_QUEUE_DEPTH_DEFAULT_STR
")");
335 static int skd_sgs_per_request
= SKD_N_SG_PER_REQ_DEFAULT
;
336 module_param(skd_sgs_per_request
, int, 0444);
337 MODULE_PARM_DESC(skd_sgs_per_request
,
338 "Maximum SG elements per block request."
339 " (1-4096, default==256)");
341 static int skd_max_pass_thru
= 1;
342 module_param(skd_max_pass_thru
, int, 0444);
343 MODULE_PARM_DESC(skd_max_pass_thru
,
344 "Maximum SCSI pass-thru at a time. IGNORED");
346 module_param(skd_dbg_level
, int, 0444);
347 MODULE_PARM_DESC(skd_dbg_level
, "s1120 debug level (0,1,2)");
349 module_param(skd_isr_comp_limit
, int, 0444);
350 MODULE_PARM_DESC(skd_isr_comp_limit
, "s1120 isr comp limit (0=none) default=4");
352 /* Major device number dynamically assigned. */
353 static u32 skd_major
;
355 static void skd_destruct(struct skd_device
*skdev
);
356 static const struct block_device_operations skd_blockdev_ops
;
357 static void skd_send_fitmsg(struct skd_device
*skdev
,
358 struct skd_fitmsg_context
*skmsg
);
359 static void skd_send_special_fitmsg(struct skd_device
*skdev
,
360 struct skd_special_context
*skspcl
);
361 static bool skd_preop_sg_list(struct skd_device
*skdev
,
362 struct skd_request_context
*skreq
);
363 static void skd_postop_sg_list(struct skd_device
*skdev
,
364 struct skd_request_context
*skreq
);
366 static void skd_restart_device(struct skd_device
*skdev
);
367 static int skd_quiesce_dev(struct skd_device
*skdev
);
368 static int skd_unquiesce_dev(struct skd_device
*skdev
);
369 static void skd_disable_interrupts(struct skd_device
*skdev
);
370 static void skd_isr_fwstate(struct skd_device
*skdev
);
371 static void skd_recover_requests(struct skd_device
*skdev
);
372 static void skd_soft_reset(struct skd_device
*skdev
);
374 const char *skd_drive_state_to_str(int state
);
375 const char *skd_skdev_state_to_str(enum skd_drvr_state state
);
376 static void skd_log_skdev(struct skd_device
*skdev
, const char *event
);
377 static void skd_log_skreq(struct skd_device
*skdev
,
378 struct skd_request_context
*skreq
, const char *event
);
381 *****************************************************************************
382 * READ/WRITE REQUESTS
383 *****************************************************************************
385 static void skd_inc_in_flight(struct request
*rq
, void *data
, bool reserved
)
392 static int skd_in_flight(struct skd_device
*skdev
)
396 blk_mq_tagset_busy_iter(&skdev
->tag_set
, skd_inc_in_flight
, &count
);
402 skd_prep_rw_cdb(struct skd_scsi_request
*scsi_req
,
403 int data_dir
, unsigned lba
,
406 if (data_dir
== READ
)
407 scsi_req
->cdb
[0] = READ_10
;
409 scsi_req
->cdb
[0] = WRITE_10
;
411 scsi_req
->cdb
[1] = 0;
412 scsi_req
->cdb
[2] = (lba
& 0xff000000) >> 24;
413 scsi_req
->cdb
[3] = (lba
& 0xff0000) >> 16;
414 scsi_req
->cdb
[4] = (lba
& 0xff00) >> 8;
415 scsi_req
->cdb
[5] = (lba
& 0xff);
416 scsi_req
->cdb
[6] = 0;
417 scsi_req
->cdb
[7] = (count
& 0xff00) >> 8;
418 scsi_req
->cdb
[8] = count
& 0xff;
419 scsi_req
->cdb
[9] = 0;
423 skd_prep_zerosize_flush_cdb(struct skd_scsi_request
*scsi_req
,
424 struct skd_request_context
*skreq
)
426 skreq
->flush_cmd
= 1;
428 scsi_req
->cdb
[0] = SYNCHRONIZE_CACHE
;
429 scsi_req
->cdb
[1] = 0;
430 scsi_req
->cdb
[2] = 0;
431 scsi_req
->cdb
[3] = 0;
432 scsi_req
->cdb
[4] = 0;
433 scsi_req
->cdb
[5] = 0;
434 scsi_req
->cdb
[6] = 0;
435 scsi_req
->cdb
[7] = 0;
436 scsi_req
->cdb
[8] = 0;
437 scsi_req
->cdb
[9] = 0;
441 * Return true if and only if all pending requests should be failed.
443 static bool skd_fail_all(struct request_queue
*q
)
445 struct skd_device
*skdev
= q
->queuedata
;
447 SKD_ASSERT(skdev
->state
!= SKD_DRVR_STATE_ONLINE
);
449 skd_log_skdev(skdev
, "req_not_online");
450 switch (skdev
->state
) {
451 case SKD_DRVR_STATE_PAUSING
:
452 case SKD_DRVR_STATE_PAUSED
:
453 case SKD_DRVR_STATE_STARTING
:
454 case SKD_DRVR_STATE_RESTARTING
:
455 case SKD_DRVR_STATE_WAIT_BOOT
:
456 /* In case of starting, we haven't started the queue,
457 * so we can't get here... but requests are
458 * possibly hanging out waiting for us because we
459 * reported the dev/skd0 already. They'll wait
460 * forever if connect doesn't complete.
461 * What to do??? delay dev/skd0 ??
463 case SKD_DRVR_STATE_BUSY
:
464 case SKD_DRVR_STATE_BUSY_IMMINENT
:
465 case SKD_DRVR_STATE_BUSY_ERASE
:
468 case SKD_DRVR_STATE_BUSY_SANITIZE
:
469 case SKD_DRVR_STATE_STOPPING
:
470 case SKD_DRVR_STATE_SYNCING
:
471 case SKD_DRVR_STATE_FAULT
:
472 case SKD_DRVR_STATE_DISAPPEARED
:
478 static blk_status_t
skd_mq_queue_rq(struct blk_mq_hw_ctx
*hctx
,
479 const struct blk_mq_queue_data
*mqd
)
481 struct request
*const req
= mqd
->rq
;
482 struct request_queue
*const q
= req
->q
;
483 struct skd_device
*skdev
= q
->queuedata
;
484 struct skd_fitmsg_context
*skmsg
;
485 struct fit_msg_hdr
*fmh
;
486 const u32 tag
= blk_mq_unique_tag(req
);
487 struct skd_request_context
*const skreq
= blk_mq_rq_to_pdu(req
);
488 struct skd_scsi_request
*scsi_req
;
489 unsigned long flags
= 0;
490 const u32 lba
= blk_rq_pos(req
);
491 const u32 count
= blk_rq_sectors(req
);
492 const int data_dir
= rq_data_dir(req
);
494 if (unlikely(skdev
->state
!= SKD_DRVR_STATE_ONLINE
))
495 return skd_fail_all(q
) ? BLK_STS_IOERR
: BLK_STS_RESOURCE
;
497 blk_mq_start_request(req
);
499 WARN_ONCE(tag
>= skd_max_queue_depth
, "%#x > %#x (nr_requests = %lu)\n",
500 tag
, skd_max_queue_depth
, q
->nr_requests
);
502 SKD_ASSERT(skreq
->state
== SKD_REQ_STATE_IDLE
);
504 dev_dbg(&skdev
->pdev
->dev
,
505 "new req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req
, lba
,
506 lba
, count
, count
, data_dir
);
508 skreq
->id
= tag
+ SKD_ID_RW_REQUEST
;
509 skreq
->flush_cmd
= 0;
511 skreq
->sg_byte_count
= 0;
513 skreq
->fitmsg_id
= 0;
515 skreq
->data_dir
= data_dir
== READ
? DMA_FROM_DEVICE
: DMA_TO_DEVICE
;
517 if (req
->bio
&& !skd_preop_sg_list(skdev
, skreq
)) {
518 dev_dbg(&skdev
->pdev
->dev
, "error Out\n");
519 skreq
->status
= BLK_STS_RESOURCE
;
520 blk_mq_complete_request(req
);
524 dma_sync_single_for_device(&skdev
->pdev
->dev
, skreq
->sksg_dma_address
,
526 sizeof(struct fit_sg_descriptor
),
529 /* Either a FIT msg is in progress or we have to start one. */
530 if (skd_max_req_per_msg
== 1) {
533 spin_lock_irqsave(&skdev
->lock
, flags
);
534 skmsg
= skdev
->skmsg
;
537 skmsg
= &skdev
->skmsg_table
[tag
];
538 skdev
->skmsg
= skmsg
;
540 /* Initialize the FIT msg header */
541 fmh
= &skmsg
->msg_buf
->fmh
;
542 memset(fmh
, 0, sizeof(*fmh
));
543 fmh
->protocol_id
= FIT_PROTOCOL_ID_SOFIT
;
544 skmsg
->length
= sizeof(*fmh
);
546 fmh
= &skmsg
->msg_buf
->fmh
;
549 skreq
->fitmsg_id
= skmsg
->id
;
551 scsi_req
= &skmsg
->msg_buf
->scsi
[fmh
->num_protocol_cmds_coalesced
];
552 memset(scsi_req
, 0, sizeof(*scsi_req
));
554 scsi_req
->hdr
.tag
= skreq
->id
;
555 scsi_req
->hdr
.sg_list_dma_address
=
556 cpu_to_be64(skreq
->sksg_dma_address
);
558 if (req_op(req
) == REQ_OP_FLUSH
) {
559 skd_prep_zerosize_flush_cdb(scsi_req
, skreq
);
560 SKD_ASSERT(skreq
->flush_cmd
== 1);
562 skd_prep_rw_cdb(scsi_req
, data_dir
, lba
, count
);
565 if (req
->cmd_flags
& REQ_FUA
)
566 scsi_req
->cdb
[1] |= SKD_FUA_NV
;
568 scsi_req
->hdr
.sg_list_len_bytes
= cpu_to_be32(skreq
->sg_byte_count
);
570 /* Complete resource allocations. */
571 skreq
->state
= SKD_REQ_STATE_BUSY
;
573 skmsg
->length
+= sizeof(struct skd_scsi_request
);
574 fmh
->num_protocol_cmds_coalesced
++;
576 dev_dbg(&skdev
->pdev
->dev
, "req=0x%x busy=%d\n", skreq
->id
,
577 skd_in_flight(skdev
));
580 * If the FIT msg buffer is full send it.
582 if (skd_max_req_per_msg
== 1) {
583 skd_send_fitmsg(skdev
, skmsg
);
586 fmh
->num_protocol_cmds_coalesced
>= skd_max_req_per_msg
) {
587 skd_send_fitmsg(skdev
, skmsg
);
590 spin_unlock_irqrestore(&skdev
->lock
, flags
);
596 static enum blk_eh_timer_return
skd_timed_out(struct request
*req
,
599 struct skd_device
*skdev
= req
->q
->queuedata
;
601 dev_err(&skdev
->pdev
->dev
, "request with tag %#x timed out\n",
602 blk_mq_unique_tag(req
));
604 return BLK_EH_RESET_TIMER
;
607 static void skd_complete_rq(struct request
*req
)
609 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(req
);
611 blk_mq_end_request(req
, skreq
->status
);
614 static bool skd_preop_sg_list(struct skd_device
*skdev
,
615 struct skd_request_context
*skreq
)
617 struct request
*req
= blk_mq_rq_from_pdu(skreq
);
618 struct scatterlist
*sgl
= &skreq
->sg
[0], *sg
;
622 skreq
->sg_byte_count
= 0;
624 WARN_ON_ONCE(skreq
->data_dir
!= DMA_TO_DEVICE
&&
625 skreq
->data_dir
!= DMA_FROM_DEVICE
);
627 n_sg
= blk_rq_map_sg(skdev
->queue
, req
, sgl
);
632 * Map scatterlist to PCI bus addresses.
633 * Note PCI might change the number of entries.
635 n_sg
= pci_map_sg(skdev
->pdev
, sgl
, n_sg
, skreq
->data_dir
);
639 SKD_ASSERT(n_sg
<= skdev
->sgs_per_request
);
643 for_each_sg(sgl
, sg
, n_sg
, i
) {
644 struct fit_sg_descriptor
*sgd
= &skreq
->sksg_list
[i
];
645 u32 cnt
= sg_dma_len(sg
);
646 uint64_t dma_addr
= sg_dma_address(sg
);
648 sgd
->control
= FIT_SGD_CONTROL_NOT_LAST
;
649 sgd
->byte_count
= cnt
;
650 skreq
->sg_byte_count
+= cnt
;
651 sgd
->host_side_addr
= dma_addr
;
652 sgd
->dev_side_addr
= 0;
655 skreq
->sksg_list
[n_sg
- 1].next_desc_ptr
= 0LL;
656 skreq
->sksg_list
[n_sg
- 1].control
= FIT_SGD_CONTROL_LAST
;
658 if (unlikely(skdev
->dbg_level
> 1)) {
659 dev_dbg(&skdev
->pdev
->dev
,
660 "skreq=%x sksg_list=%p sksg_dma=%llx\n",
661 skreq
->id
, skreq
->sksg_list
, skreq
->sksg_dma_address
);
662 for (i
= 0; i
< n_sg
; i
++) {
663 struct fit_sg_descriptor
*sgd
= &skreq
->sksg_list
[i
];
665 dev_dbg(&skdev
->pdev
->dev
,
666 " sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
667 i
, sgd
->byte_count
, sgd
->control
,
668 sgd
->host_side_addr
, sgd
->next_desc_ptr
);
675 static void skd_postop_sg_list(struct skd_device
*skdev
,
676 struct skd_request_context
*skreq
)
679 * restore the next ptr for next IO request so we
680 * don't have to set it every time.
682 skreq
->sksg_list
[skreq
->n_sg
- 1].next_desc_ptr
=
683 skreq
->sksg_dma_address
+
684 ((skreq
->n_sg
) * sizeof(struct fit_sg_descriptor
));
685 pci_unmap_sg(skdev
->pdev
, &skreq
->sg
[0], skreq
->n_sg
, skreq
->data_dir
);
689 *****************************************************************************
691 *****************************************************************************
694 static void skd_timer_tick_not_online(struct skd_device
*skdev
);
696 static void skd_start_queue(struct work_struct
*work
)
698 struct skd_device
*skdev
= container_of(work
, typeof(*skdev
),
702 * Although it is safe to call blk_start_queue() from interrupt
703 * context, blk_mq_start_hw_queues() must not be called from
706 blk_mq_start_hw_queues(skdev
->queue
);
709 static void skd_timer_tick(struct timer_list
*t
)
711 struct skd_device
*skdev
= from_timer(skdev
, t
, timer
);
712 unsigned long reqflags
;
715 if (skdev
->state
== SKD_DRVR_STATE_FAULT
)
716 /* The driver has declared fault, and we want it to
717 * stay that way until driver is reloaded.
721 spin_lock_irqsave(&skdev
->lock
, reqflags
);
723 state
= SKD_READL(skdev
, FIT_STATUS
);
724 state
&= FIT_SR_DRIVE_STATE_MASK
;
725 if (state
!= skdev
->drive_state
)
726 skd_isr_fwstate(skdev
);
728 if (skdev
->state
!= SKD_DRVR_STATE_ONLINE
)
729 skd_timer_tick_not_online(skdev
);
731 mod_timer(&skdev
->timer
, (jiffies
+ HZ
));
733 spin_unlock_irqrestore(&skdev
->lock
, reqflags
);
736 static void skd_timer_tick_not_online(struct skd_device
*skdev
)
738 switch (skdev
->state
) {
739 case SKD_DRVR_STATE_IDLE
:
740 case SKD_DRVR_STATE_LOAD
:
742 case SKD_DRVR_STATE_BUSY_SANITIZE
:
743 dev_dbg(&skdev
->pdev
->dev
,
744 "drive busy sanitize[%x], driver[%x]\n",
745 skdev
->drive_state
, skdev
->state
);
746 /* If we've been in sanitize for 3 seconds, we figure we're not
747 * going to get anymore completions, so recover requests now
749 if (skdev
->timer_countdown
> 0) {
750 skdev
->timer_countdown
--;
753 skd_recover_requests(skdev
);
756 case SKD_DRVR_STATE_BUSY
:
757 case SKD_DRVR_STATE_BUSY_IMMINENT
:
758 case SKD_DRVR_STATE_BUSY_ERASE
:
759 dev_dbg(&skdev
->pdev
->dev
, "busy[%x], countdown=%d\n",
760 skdev
->state
, skdev
->timer_countdown
);
761 if (skdev
->timer_countdown
> 0) {
762 skdev
->timer_countdown
--;
765 dev_dbg(&skdev
->pdev
->dev
,
766 "busy[%x], timedout=%d, restarting device.",
767 skdev
->state
, skdev
->timer_countdown
);
768 skd_restart_device(skdev
);
771 case SKD_DRVR_STATE_WAIT_BOOT
:
772 case SKD_DRVR_STATE_STARTING
:
773 if (skdev
->timer_countdown
> 0) {
774 skdev
->timer_countdown
--;
777 /* For now, we fault the drive. Could attempt resets to
778 * revcover at some point. */
779 skdev
->state
= SKD_DRVR_STATE_FAULT
;
781 dev_err(&skdev
->pdev
->dev
, "DriveFault Connect Timeout (%x)\n",
784 /*start the queue so we can respond with error to requests */
785 /* wakeup anyone waiting for startup complete */
786 schedule_work(&skdev
->start_queue
);
787 skdev
->gendisk_on
= -1;
788 wake_up_interruptible(&skdev
->waitq
);
791 case SKD_DRVR_STATE_ONLINE
:
792 /* shouldn't get here. */
795 case SKD_DRVR_STATE_PAUSING
:
796 case SKD_DRVR_STATE_PAUSED
:
799 case SKD_DRVR_STATE_RESTARTING
:
800 if (skdev
->timer_countdown
> 0) {
801 skdev
->timer_countdown
--;
804 /* For now, we fault the drive. Could attempt resets to
805 * revcover at some point. */
806 skdev
->state
= SKD_DRVR_STATE_FAULT
;
807 dev_err(&skdev
->pdev
->dev
,
808 "DriveFault Reconnect Timeout (%x)\n",
812 * Recovering does two things:
813 * 1. completes IO with error
814 * 2. reclaims dma resources
815 * When is it safe to recover requests?
816 * - if the drive state is faulted
817 * - if the state is still soft reset after out timeout
818 * - if the drive registers are dead (state = FF)
819 * If it is "unsafe", we still need to recover, so we will
820 * disable pci bus mastering and disable our interrupts.
823 if ((skdev
->drive_state
== FIT_SR_DRIVE_SOFT_RESET
) ||
824 (skdev
->drive_state
== FIT_SR_DRIVE_FAULT
) ||
825 (skdev
->drive_state
== FIT_SR_DRIVE_STATE_MASK
))
826 /* It never came out of soft reset. Try to
827 * recover the requests and then let them
828 * fail. This is to mitigate hung processes. */
829 skd_recover_requests(skdev
);
831 dev_err(&skdev
->pdev
->dev
, "Disable BusMaster (%x)\n",
833 pci_disable_device(skdev
->pdev
);
834 skd_disable_interrupts(skdev
);
835 skd_recover_requests(skdev
);
838 /*start the queue so we can respond with error to requests */
839 /* wakeup anyone waiting for startup complete */
840 schedule_work(&skdev
->start_queue
);
841 skdev
->gendisk_on
= -1;
842 wake_up_interruptible(&skdev
->waitq
);
845 case SKD_DRVR_STATE_RESUMING
:
846 case SKD_DRVR_STATE_STOPPING
:
847 case SKD_DRVR_STATE_SYNCING
:
848 case SKD_DRVR_STATE_FAULT
:
849 case SKD_DRVR_STATE_DISAPPEARED
:
855 static int skd_start_timer(struct skd_device
*skdev
)
859 timer_setup(&skdev
->timer
, skd_timer_tick
, 0);
861 rc
= mod_timer(&skdev
->timer
, (jiffies
+ HZ
));
863 dev_err(&skdev
->pdev
->dev
, "failed to start timer %d\n", rc
);
867 static void skd_kill_timer(struct skd_device
*skdev
)
869 del_timer_sync(&skdev
->timer
);
873 *****************************************************************************
874 * INTERNAL REQUESTS -- generated by driver itself
875 *****************************************************************************
878 static int skd_format_internal_skspcl(struct skd_device
*skdev
)
880 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
881 struct fit_sg_descriptor
*sgd
= &skspcl
->req
.sksg_list
[0];
882 struct fit_msg_hdr
*fmh
;
883 uint64_t dma_address
;
884 struct skd_scsi_request
*scsi
;
886 fmh
= &skspcl
->msg_buf
->fmh
;
887 fmh
->protocol_id
= FIT_PROTOCOL_ID_SOFIT
;
888 fmh
->num_protocol_cmds_coalesced
= 1;
890 scsi
= &skspcl
->msg_buf
->scsi
[0];
891 memset(scsi
, 0, sizeof(*scsi
));
892 dma_address
= skspcl
->req
.sksg_dma_address
;
893 scsi
->hdr
.sg_list_dma_address
= cpu_to_be64(dma_address
);
894 skspcl
->req
.n_sg
= 1;
895 sgd
->control
= FIT_SGD_CONTROL_LAST
;
897 sgd
->host_side_addr
= skspcl
->db_dma_address
;
898 sgd
->dev_side_addr
= 0;
899 sgd
->next_desc_ptr
= 0LL;
904 #define WR_BUF_SIZE SKD_N_INTERNAL_BYTES
906 static void skd_send_internal_skspcl(struct skd_device
*skdev
,
907 struct skd_special_context
*skspcl
,
910 struct fit_sg_descriptor
*sgd
= &skspcl
->req
.sksg_list
[0];
911 struct skd_scsi_request
*scsi
;
912 unsigned char *buf
= skspcl
->data_buf
;
915 if (skspcl
->req
.state
!= SKD_REQ_STATE_IDLE
)
917 * A refresh is already in progress.
918 * Just wait for it to finish.
922 skspcl
->req
.state
= SKD_REQ_STATE_BUSY
;
924 scsi
= &skspcl
->msg_buf
->scsi
[0];
925 scsi
->hdr
.tag
= skspcl
->req
.id
;
927 memset(scsi
->cdb
, 0, sizeof(scsi
->cdb
));
930 case TEST_UNIT_READY
:
931 scsi
->cdb
[0] = TEST_UNIT_READY
;
933 scsi
->hdr
.sg_list_len_bytes
= 0;
937 scsi
->cdb
[0] = READ_CAPACITY
;
938 sgd
->byte_count
= SKD_N_READ_CAP_BYTES
;
939 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
943 scsi
->cdb
[0] = INQUIRY
;
944 scsi
->cdb
[1] = 0x01; /* evpd */
945 scsi
->cdb
[2] = 0x80; /* serial number page */
947 sgd
->byte_count
= 16;
948 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
951 case SYNCHRONIZE_CACHE
:
952 scsi
->cdb
[0] = SYNCHRONIZE_CACHE
;
954 scsi
->hdr
.sg_list_len_bytes
= 0;
958 scsi
->cdb
[0] = WRITE_BUFFER
;
960 scsi
->cdb
[7] = (WR_BUF_SIZE
& 0xFF00) >> 8;
961 scsi
->cdb
[8] = WR_BUF_SIZE
& 0xFF;
962 sgd
->byte_count
= WR_BUF_SIZE
;
963 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
964 /* fill incrementing byte pattern */
965 for (i
= 0; i
< sgd
->byte_count
; i
++)
970 scsi
->cdb
[0] = READ_BUFFER
;
972 scsi
->cdb
[7] = (WR_BUF_SIZE
& 0xFF00) >> 8;
973 scsi
->cdb
[8] = WR_BUF_SIZE
& 0xFF;
974 sgd
->byte_count
= WR_BUF_SIZE
;
975 scsi
->hdr
.sg_list_len_bytes
= cpu_to_be32(sgd
->byte_count
);
976 memset(skspcl
->data_buf
, 0, sgd
->byte_count
);
980 SKD_ASSERT("Don't know what to send");
984 skd_send_special_fitmsg(skdev
, skspcl
);
987 static void skd_refresh_device_data(struct skd_device
*skdev
)
989 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
991 skd_send_internal_skspcl(skdev
, skspcl
, TEST_UNIT_READY
);
994 static int skd_chk_read_buf(struct skd_device
*skdev
,
995 struct skd_special_context
*skspcl
)
997 unsigned char *buf
= skspcl
->data_buf
;
1000 /* check for incrementing byte pattern */
1001 for (i
= 0; i
< WR_BUF_SIZE
; i
++)
1002 if (buf
[i
] != (i
& 0xFF))
1008 static void skd_log_check_status(struct skd_device
*skdev
, u8 status
, u8 key
,
1009 u8 code
, u8 qual
, u8 fruc
)
1011 /* If the check condition is of special interest, log a message */
1012 if ((status
== SAM_STAT_CHECK_CONDITION
) && (key
== 0x02)
1013 && (code
== 0x04) && (qual
== 0x06)) {
1014 dev_err(&skdev
->pdev
->dev
,
1015 "*** LOST_WRITE_DATA ERROR *** key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
1016 key
, code
, qual
, fruc
);
1020 static void skd_complete_internal(struct skd_device
*skdev
,
1021 struct fit_completion_entry_v1
*skcomp
,
1022 struct fit_comp_error_info
*skerr
,
1023 struct skd_special_context
*skspcl
)
1025 u8
*buf
= skspcl
->data_buf
;
1028 struct skd_scsi_request
*scsi
= &skspcl
->msg_buf
->scsi
[0];
1030 lockdep_assert_held(&skdev
->lock
);
1032 SKD_ASSERT(skspcl
== &skdev
->internal_skspcl
);
1034 dev_dbg(&skdev
->pdev
->dev
, "complete internal %x\n", scsi
->cdb
[0]);
1036 dma_sync_single_for_cpu(&skdev
->pdev
->dev
,
1037 skspcl
->db_dma_address
,
1038 skspcl
->req
.sksg_list
[0].byte_count
,
1041 skspcl
->req
.completion
= *skcomp
;
1042 skspcl
->req
.state
= SKD_REQ_STATE_IDLE
;
1044 status
= skspcl
->req
.completion
.status
;
1046 skd_log_check_status(skdev
, status
, skerr
->key
, skerr
->code
,
1047 skerr
->qual
, skerr
->fruc
);
1049 switch (scsi
->cdb
[0]) {
1050 case TEST_UNIT_READY
:
1051 if (status
== SAM_STAT_GOOD
)
1052 skd_send_internal_skspcl(skdev
, skspcl
, WRITE_BUFFER
);
1053 else if ((status
== SAM_STAT_CHECK_CONDITION
) &&
1054 (skerr
->key
== MEDIUM_ERROR
))
1055 skd_send_internal_skspcl(skdev
, skspcl
, WRITE_BUFFER
);
1057 if (skdev
->state
== SKD_DRVR_STATE_STOPPING
) {
1058 dev_dbg(&skdev
->pdev
->dev
,
1059 "TUR failed, don't send anymore state 0x%x\n",
1063 dev_dbg(&skdev
->pdev
->dev
,
1064 "**** TUR failed, retry skerr\n");
1065 skd_send_internal_skspcl(skdev
, skspcl
,
1071 if (status
== SAM_STAT_GOOD
)
1072 skd_send_internal_skspcl(skdev
, skspcl
, READ_BUFFER
);
1074 if (skdev
->state
== SKD_DRVR_STATE_STOPPING
) {
1075 dev_dbg(&skdev
->pdev
->dev
,
1076 "write buffer failed, don't send anymore state 0x%x\n",
1080 dev_dbg(&skdev
->pdev
->dev
,
1081 "**** write buffer failed, retry skerr\n");
1082 skd_send_internal_skspcl(skdev
, skspcl
,
1088 if (status
== SAM_STAT_GOOD
) {
1089 if (skd_chk_read_buf(skdev
, skspcl
) == 0)
1090 skd_send_internal_skspcl(skdev
, skspcl
,
1093 dev_err(&skdev
->pdev
->dev
,
1094 "*** W/R Buffer mismatch %d ***\n",
1095 skdev
->connect_retries
);
1096 if (skdev
->connect_retries
<
1097 SKD_MAX_CONNECT_RETRIES
) {
1098 skdev
->connect_retries
++;
1099 skd_soft_reset(skdev
);
1101 dev_err(&skdev
->pdev
->dev
,
1102 "W/R Buffer Connect Error\n");
1108 if (skdev
->state
== SKD_DRVR_STATE_STOPPING
) {
1109 dev_dbg(&skdev
->pdev
->dev
,
1110 "read buffer failed, don't send anymore state 0x%x\n",
1114 dev_dbg(&skdev
->pdev
->dev
,
1115 "**** read buffer failed, retry skerr\n");
1116 skd_send_internal_skspcl(skdev
, skspcl
,
1122 skdev
->read_cap_is_valid
= 0;
1123 if (status
== SAM_STAT_GOOD
) {
1124 skdev
->read_cap_last_lba
=
1125 (buf
[0] << 24) | (buf
[1] << 16) |
1126 (buf
[2] << 8) | buf
[3];
1127 skdev
->read_cap_blocksize
=
1128 (buf
[4] << 24) | (buf
[5] << 16) |
1129 (buf
[6] << 8) | buf
[7];
1131 dev_dbg(&skdev
->pdev
->dev
, "last lba %d, bs %d\n",
1132 skdev
->read_cap_last_lba
,
1133 skdev
->read_cap_blocksize
);
1135 set_capacity(skdev
->disk
, skdev
->read_cap_last_lba
+ 1);
1137 skdev
->read_cap_is_valid
= 1;
1139 skd_send_internal_skspcl(skdev
, skspcl
, INQUIRY
);
1140 } else if ((status
== SAM_STAT_CHECK_CONDITION
) &&
1141 (skerr
->key
== MEDIUM_ERROR
)) {
1142 skdev
->read_cap_last_lba
= ~0;
1143 set_capacity(skdev
->disk
, skdev
->read_cap_last_lba
+ 1);
1144 dev_dbg(&skdev
->pdev
->dev
, "**** MEDIUM ERROR caused READCAP to fail, ignore failure and continue to inquiry\n");
1145 skd_send_internal_skspcl(skdev
, skspcl
, INQUIRY
);
1147 dev_dbg(&skdev
->pdev
->dev
, "**** READCAP failed, retry TUR\n");
1148 skd_send_internal_skspcl(skdev
, skspcl
,
1154 skdev
->inquiry_is_valid
= 0;
1155 if (status
== SAM_STAT_GOOD
) {
1156 skdev
->inquiry_is_valid
= 1;
1158 for (i
= 0; i
< 12; i
++)
1159 skdev
->inq_serial_num
[i
] = buf
[i
+ 4];
1160 skdev
->inq_serial_num
[12] = 0;
1163 if (skd_unquiesce_dev(skdev
) < 0)
1164 dev_dbg(&skdev
->pdev
->dev
, "**** failed, to ONLINE device\n");
1165 /* connection is complete */
1166 skdev
->connect_retries
= 0;
1169 case SYNCHRONIZE_CACHE
:
1170 if (status
== SAM_STAT_GOOD
)
1171 skdev
->sync_done
= 1;
1173 skdev
->sync_done
= -1;
1174 wake_up_interruptible(&skdev
->waitq
);
1178 SKD_ASSERT("we didn't send this");
1183 *****************************************************************************
1185 *****************************************************************************
1188 static void skd_send_fitmsg(struct skd_device
*skdev
,
1189 struct skd_fitmsg_context
*skmsg
)
1193 dev_dbg(&skdev
->pdev
->dev
, "dma address 0x%llx, busy=%d\n",
1194 skmsg
->mb_dma_address
, skd_in_flight(skdev
));
1195 dev_dbg(&skdev
->pdev
->dev
, "msg_buf %p\n", skmsg
->msg_buf
);
1197 qcmd
= skmsg
->mb_dma_address
;
1198 qcmd
|= FIT_QCMD_QID_NORMAL
;
1200 if (unlikely(skdev
->dbg_level
> 1)) {
1201 u8
*bp
= (u8
*)skmsg
->msg_buf
;
1203 for (i
= 0; i
< skmsg
->length
; i
+= 8) {
1204 dev_dbg(&skdev
->pdev
->dev
, "msg[%2d] %8ph\n", i
,
1211 if (skmsg
->length
> 256)
1212 qcmd
|= FIT_QCMD_MSGSIZE_512
;
1213 else if (skmsg
->length
> 128)
1214 qcmd
|= FIT_QCMD_MSGSIZE_256
;
1215 else if (skmsg
->length
> 64)
1216 qcmd
|= FIT_QCMD_MSGSIZE_128
;
1219 * This makes no sense because the FIT msg header is
1220 * 64 bytes. If the msg is only 64 bytes long it has
1223 qcmd
|= FIT_QCMD_MSGSIZE_64
;
1225 dma_sync_single_for_device(&skdev
->pdev
->dev
, skmsg
->mb_dma_address
,
1226 skmsg
->length
, DMA_TO_DEVICE
);
1228 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
1231 SKD_WRITEQ(skdev
, qcmd
, FIT_Q_COMMAND
);
1234 static void skd_send_special_fitmsg(struct skd_device
*skdev
,
1235 struct skd_special_context
*skspcl
)
1239 WARN_ON_ONCE(skspcl
->req
.n_sg
!= 1);
1241 if (unlikely(skdev
->dbg_level
> 1)) {
1242 u8
*bp
= (u8
*)skspcl
->msg_buf
;
1245 for (i
= 0; i
< SKD_N_SPECIAL_FITMSG_BYTES
; i
+= 8) {
1246 dev_dbg(&skdev
->pdev
->dev
, " spcl[%2d] %8ph\n", i
,
1252 dev_dbg(&skdev
->pdev
->dev
,
1253 "skspcl=%p id=%04x sksg_list=%p sksg_dma=%llx\n",
1254 skspcl
, skspcl
->req
.id
, skspcl
->req
.sksg_list
,
1255 skspcl
->req
.sksg_dma_address
);
1256 for (i
= 0; i
< skspcl
->req
.n_sg
; i
++) {
1257 struct fit_sg_descriptor
*sgd
=
1258 &skspcl
->req
.sksg_list
[i
];
1260 dev_dbg(&skdev
->pdev
->dev
,
1261 " sg[%d] count=%u ctrl=0x%x addr=0x%llx next=0x%llx\n",
1262 i
, sgd
->byte_count
, sgd
->control
,
1263 sgd
->host_side_addr
, sgd
->next_desc_ptr
);
1268 * Special FIT msgs are always 128 bytes: a 64-byte FIT hdr
1269 * and one 64-byte SSDI command.
1271 qcmd
= skspcl
->mb_dma_address
;
1272 qcmd
|= FIT_QCMD_QID_NORMAL
+ FIT_QCMD_MSGSIZE_128
;
1274 dma_sync_single_for_device(&skdev
->pdev
->dev
, skspcl
->mb_dma_address
,
1275 SKD_N_SPECIAL_FITMSG_BYTES
, DMA_TO_DEVICE
);
1276 dma_sync_single_for_device(&skdev
->pdev
->dev
,
1277 skspcl
->req
.sksg_dma_address
,
1278 1 * sizeof(struct fit_sg_descriptor
),
1280 dma_sync_single_for_device(&skdev
->pdev
->dev
,
1281 skspcl
->db_dma_address
,
1282 skspcl
->req
.sksg_list
[0].byte_count
,
1285 /* Make sure skd_msg_buf is written before the doorbell is triggered. */
1288 SKD_WRITEQ(skdev
, qcmd
, FIT_Q_COMMAND
);
1292 *****************************************************************************
1294 *****************************************************************************
1297 static void skd_complete_other(struct skd_device
*skdev
,
1298 struct fit_completion_entry_v1
*skcomp
,
1299 struct fit_comp_error_info
*skerr
);
1308 enum skd_check_status_action action
;
1311 static struct sns_info skd_chkstat_table
[] = {
1313 { 0x70, 0x02, RECOVERED_ERROR
, 0, 0, 0x1c,
1314 SKD_CHECK_STATUS_REPORT_GOOD
},
1317 { 0x70, 0x02, NO_SENSE
, 0x0B, 0x00, 0x1E, /* warnings */
1318 SKD_CHECK_STATUS_REPORT_SMART_ALERT
},
1319 { 0x70, 0x02, NO_SENSE
, 0x5D, 0x00, 0x1E, /* thresholds */
1320 SKD_CHECK_STATUS_REPORT_SMART_ALERT
},
1321 { 0x70, 0x02, RECOVERED_ERROR
, 0x0B, 0x01, 0x1F, /* temperature over trigger */
1322 SKD_CHECK_STATUS_REPORT_SMART_ALERT
},
1324 /* Retry (with limits) */
1325 { 0x70, 0x02, 0x0B, 0, 0, 0x1C, /* This one is for DMA ERROR */
1326 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1327 { 0x70, 0x02, 0x06, 0x0B, 0x00, 0x1E, /* warnings */
1328 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1329 { 0x70, 0x02, 0x06, 0x5D, 0x00, 0x1E, /* thresholds */
1330 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1331 { 0x70, 0x02, 0x06, 0x80, 0x30, 0x1F, /* backup power */
1332 SKD_CHECK_STATUS_REQUEUE_REQUEST
},
1334 /* Busy (or about to be) */
1335 { 0x70, 0x02, 0x06, 0x3f, 0x01, 0x1F, /* fw changed */
1336 SKD_CHECK_STATUS_BUSY_IMMINENT
},
1340 * Look up status and sense data to decide how to handle the error
1342 * mask says which fields must match e.g., mask=0x18 means check
1343 * type and stat, ignore key, asc, ascq.
1346 static enum skd_check_status_action
1347 skd_check_status(struct skd_device
*skdev
,
1348 u8 cmp_status
, struct fit_comp_error_info
*skerr
)
1352 dev_err(&skdev
->pdev
->dev
, "key/asc/ascq/fruc %02x/%02x/%02x/%02x\n",
1353 skerr
->key
, skerr
->code
, skerr
->qual
, skerr
->fruc
);
1355 dev_dbg(&skdev
->pdev
->dev
,
1356 "stat: t=%02x stat=%02x k=%02x c=%02x q=%02x fruc=%02x\n",
1357 skerr
->type
, cmp_status
, skerr
->key
, skerr
->code
, skerr
->qual
,
1360 /* Does the info match an entry in the good category? */
1361 for (i
= 0; i
< ARRAY_SIZE(skd_chkstat_table
); i
++) {
1362 struct sns_info
*sns
= &skd_chkstat_table
[i
];
1364 if (sns
->mask
& 0x10)
1365 if (skerr
->type
!= sns
->type
)
1368 if (sns
->mask
& 0x08)
1369 if (cmp_status
!= sns
->stat
)
1372 if (sns
->mask
& 0x04)
1373 if (skerr
->key
!= sns
->key
)
1376 if (sns
->mask
& 0x02)
1377 if (skerr
->code
!= sns
->asc
)
1380 if (sns
->mask
& 0x01)
1381 if (skerr
->qual
!= sns
->ascq
)
1384 if (sns
->action
== SKD_CHECK_STATUS_REPORT_SMART_ALERT
) {
1385 dev_err(&skdev
->pdev
->dev
,
1386 "SMART Alert: sense key/asc/ascq %02x/%02x/%02x\n",
1387 skerr
->key
, skerr
->code
, skerr
->qual
);
1392 /* No other match, so nonzero status means error,
1393 * zero status means good
1396 dev_dbg(&skdev
->pdev
->dev
, "status check: error\n");
1397 return SKD_CHECK_STATUS_REPORT_ERROR
;
1400 dev_dbg(&skdev
->pdev
->dev
, "status check good default\n");
1401 return SKD_CHECK_STATUS_REPORT_GOOD
;
1404 static void skd_resolve_req_exception(struct skd_device
*skdev
,
1405 struct skd_request_context
*skreq
,
1406 struct request
*req
)
1408 u8 cmp_status
= skreq
->completion
.status
;
1410 switch (skd_check_status(skdev
, cmp_status
, &skreq
->err_info
)) {
1411 case SKD_CHECK_STATUS_REPORT_GOOD
:
1412 case SKD_CHECK_STATUS_REPORT_SMART_ALERT
:
1413 skreq
->status
= BLK_STS_OK
;
1414 blk_mq_complete_request(req
);
1417 case SKD_CHECK_STATUS_BUSY_IMMINENT
:
1418 skd_log_skreq(skdev
, skreq
, "retry(busy)");
1419 blk_requeue_request(skdev
->queue
, req
);
1420 dev_info(&skdev
->pdev
->dev
, "drive BUSY imminent\n");
1421 skdev
->state
= SKD_DRVR_STATE_BUSY_IMMINENT
;
1422 skdev
->timer_countdown
= SKD_TIMER_MINUTES(20);
1423 skd_quiesce_dev(skdev
);
1426 case SKD_CHECK_STATUS_REQUEUE_REQUEST
:
1427 if ((unsigned long) ++req
->special
< SKD_MAX_RETRIES
) {
1428 skd_log_skreq(skdev
, skreq
, "retry");
1429 blk_requeue_request(skdev
->queue
, req
);
1434 case SKD_CHECK_STATUS_REPORT_ERROR
:
1436 skreq
->status
= BLK_STS_IOERR
;
1437 blk_mq_complete_request(req
);
1442 static void skd_release_skreq(struct skd_device
*skdev
,
1443 struct skd_request_context
*skreq
)
1446 * Reclaim the skd_request_context
1448 skreq
->state
= SKD_REQ_STATE_IDLE
;
1451 static int skd_isr_completion_posted(struct skd_device
*skdev
,
1452 int limit
, int *enqueued
)
1454 struct fit_completion_entry_v1
*skcmp
;
1455 struct fit_comp_error_info
*skerr
;
1460 struct skd_request_context
*skreq
;
1468 lockdep_assert_held(&skdev
->lock
);
1471 SKD_ASSERT(skdev
->skcomp_ix
< SKD_N_COMPLETION_ENTRY
);
1473 skcmp
= &skdev
->skcomp_table
[skdev
->skcomp_ix
];
1474 cmp_cycle
= skcmp
->cycle
;
1475 cmp_cntxt
= skcmp
->tag
;
1476 cmp_status
= skcmp
->status
;
1477 cmp_bytes
= be32_to_cpu(skcmp
->num_returned_bytes
);
1479 skerr
= &skdev
->skerr_table
[skdev
->skcomp_ix
];
1481 dev_dbg(&skdev
->pdev
->dev
,
1482 "cycle=%d ix=%d got cycle=%d cmdctxt=0x%x stat=%d busy=%d rbytes=0x%x proto=%d\n",
1483 skdev
->skcomp_cycle
, skdev
->skcomp_ix
, cmp_cycle
,
1484 cmp_cntxt
, cmp_status
, skd_in_flight(skdev
),
1485 cmp_bytes
, skdev
->proto_ver
);
1487 if (cmp_cycle
!= skdev
->skcomp_cycle
) {
1488 dev_dbg(&skdev
->pdev
->dev
, "end of completions\n");
1492 * Update the completion queue head index and possibly
1493 * the completion cycle count. 8-bit wrap-around.
1496 if (skdev
->skcomp_ix
>= SKD_N_COMPLETION_ENTRY
) {
1497 skdev
->skcomp_ix
= 0;
1498 skdev
->skcomp_cycle
++;
1502 * The command context is a unique 32-bit ID. The low order
1503 * bits help locate the request. The request is usually a
1504 * r/w request (see skd_start() above) or a special request.
1507 tag
= req_id
& SKD_ID_SLOT_AND_TABLE_MASK
;
1509 /* Is this other than a r/w request? */
1510 if (tag
>= skdev
->num_req_context
) {
1512 * This is not a completion for a r/w request.
1514 WARN_ON_ONCE(blk_mq_tag_to_rq(skdev
->tag_set
.tags
[hwq
],
1516 skd_complete_other(skdev
, skcmp
, skerr
);
1520 rq
= blk_mq_tag_to_rq(skdev
->tag_set
.tags
[hwq
], tag
);
1521 if (WARN(!rq
, "No request for tag %#x -> %#x\n", cmp_cntxt
,
1524 skreq
= blk_mq_rq_to_pdu(rq
);
1527 * Make sure the request ID for the slot matches.
1529 if (skreq
->id
!= req_id
) {
1530 dev_err(&skdev
->pdev
->dev
,
1531 "Completion mismatch comp_id=0x%04x skreq=0x%04x new=0x%04x\n",
1532 req_id
, skreq
->id
, cmp_cntxt
);
1537 SKD_ASSERT(skreq
->state
== SKD_REQ_STATE_BUSY
);
1539 skreq
->completion
= *skcmp
;
1540 if (unlikely(cmp_status
== SAM_STAT_CHECK_CONDITION
)) {
1541 skreq
->err_info
= *skerr
;
1542 skd_log_check_status(skdev
, cmp_status
, skerr
->key
,
1543 skerr
->code
, skerr
->qual
,
1546 /* Release DMA resources for the request. */
1547 if (skreq
->n_sg
> 0)
1548 skd_postop_sg_list(skdev
, skreq
);
1550 skd_release_skreq(skdev
, skreq
);
1553 * Capture the outcome and post it back to the native request.
1555 if (likely(cmp_status
== SAM_STAT_GOOD
)) {
1556 skreq
->status
= BLK_STS_OK
;
1557 blk_mq_complete_request(rq
);
1559 skd_resolve_req_exception(skdev
, skreq
, rq
);
1562 /* skd_isr_comp_limit equal zero means no limit */
1564 if (++processed
>= limit
) {
1571 if (skdev
->state
== SKD_DRVR_STATE_PAUSING
&&
1572 skd_in_flight(skdev
) == 0) {
1573 skdev
->state
= SKD_DRVR_STATE_PAUSED
;
1574 wake_up_interruptible(&skdev
->waitq
);
1580 static void skd_complete_other(struct skd_device
*skdev
,
1581 struct fit_completion_entry_v1
*skcomp
,
1582 struct fit_comp_error_info
*skerr
)
1587 struct skd_special_context
*skspcl
;
1589 lockdep_assert_held(&skdev
->lock
);
1591 req_id
= skcomp
->tag
;
1592 req_table
= req_id
& SKD_ID_TABLE_MASK
;
1593 req_slot
= req_id
& SKD_ID_SLOT_MASK
;
1595 dev_dbg(&skdev
->pdev
->dev
, "table=0x%x id=0x%x slot=%d\n", req_table
,
1599 * Based on the request id, determine how to dispatch this completion.
1600 * This swich/case is finding the good cases and forwarding the
1601 * completion entry. Errors are reported below the switch.
1603 switch (req_table
) {
1604 case SKD_ID_RW_REQUEST
:
1606 * The caller, skd_isr_completion_posted() above,
1607 * handles r/w requests. The only way we get here
1608 * is if the req_slot is out of bounds.
1612 case SKD_ID_INTERNAL
:
1613 if (req_slot
== 0) {
1614 skspcl
= &skdev
->internal_skspcl
;
1615 if (skspcl
->req
.id
== req_id
&&
1616 skspcl
->req
.state
== SKD_REQ_STATE_BUSY
) {
1617 skd_complete_internal(skdev
,
1618 skcomp
, skerr
, skspcl
);
1624 case SKD_ID_FIT_MSG
:
1626 * These id's should never appear in a completion record.
1632 * These id's should never appear anywhere;
1638 * If we get here it is a bad or stale id.
1642 static void skd_reset_skcomp(struct skd_device
*skdev
)
1644 memset(skdev
->skcomp_table
, 0, SKD_SKCOMP_SIZE
);
1646 skdev
->skcomp_ix
= 0;
1647 skdev
->skcomp_cycle
= 1;
1651 *****************************************************************************
1653 *****************************************************************************
1655 static void skd_completion_worker(struct work_struct
*work
)
1657 struct skd_device
*skdev
=
1658 container_of(work
, struct skd_device
, completion_worker
);
1659 unsigned long flags
;
1660 int flush_enqueued
= 0;
1662 spin_lock_irqsave(&skdev
->lock
, flags
);
1665 * pass in limit=0, which means no limit..
1666 * process everything in compq
1668 skd_isr_completion_posted(skdev
, 0, &flush_enqueued
);
1669 schedule_work(&skdev
->start_queue
);
1671 spin_unlock_irqrestore(&skdev
->lock
, flags
);
1674 static void skd_isr_msg_from_dev(struct skd_device
*skdev
);
1677 skd_isr(int irq
, void *ptr
)
1679 struct skd_device
*skdev
= ptr
;
1684 int flush_enqueued
= 0;
1686 spin_lock(&skdev
->lock
);
1689 intstat
= SKD_READL(skdev
, FIT_INT_STATUS_HOST
);
1691 ack
= FIT_INT_DEF_MASK
;
1694 dev_dbg(&skdev
->pdev
->dev
, "intstat=0x%x ack=0x%x\n", intstat
,
1697 /* As long as there is an int pending on device, keep
1698 * running loop. When none, get out, but if we've never
1699 * done any processing, call completion handler?
1702 /* No interrupts on device, but run the completion
1706 if (likely (skdev
->state
1707 == SKD_DRVR_STATE_ONLINE
))
1714 SKD_WRITEL(skdev
, ack
, FIT_INT_STATUS_HOST
);
1716 if (likely((skdev
->state
!= SKD_DRVR_STATE_LOAD
) &&
1717 (skdev
->state
!= SKD_DRVR_STATE_STOPPING
))) {
1718 if (intstat
& FIT_ISH_COMPLETION_POSTED
) {
1720 * If we have already deferred completion
1721 * processing, don't bother running it again
1725 skd_isr_completion_posted(skdev
,
1726 skd_isr_comp_limit
, &flush_enqueued
);
1729 if (intstat
& FIT_ISH_FW_STATE_CHANGE
) {
1730 skd_isr_fwstate(skdev
);
1731 if (skdev
->state
== SKD_DRVR_STATE_FAULT
||
1733 SKD_DRVR_STATE_DISAPPEARED
) {
1734 spin_unlock(&skdev
->lock
);
1739 if (intstat
& FIT_ISH_MSG_FROM_DEV
)
1740 skd_isr_msg_from_dev(skdev
);
1744 if (unlikely(flush_enqueued
))
1745 schedule_work(&skdev
->start_queue
);
1748 schedule_work(&skdev
->completion_worker
);
1749 else if (!flush_enqueued
)
1750 schedule_work(&skdev
->start_queue
);
1752 spin_unlock(&skdev
->lock
);
1757 static void skd_drive_fault(struct skd_device
*skdev
)
1759 skdev
->state
= SKD_DRVR_STATE_FAULT
;
1760 dev_err(&skdev
->pdev
->dev
, "Drive FAULT\n");
1763 static void skd_drive_disappeared(struct skd_device
*skdev
)
1765 skdev
->state
= SKD_DRVR_STATE_DISAPPEARED
;
1766 dev_err(&skdev
->pdev
->dev
, "Drive DISAPPEARED\n");
1769 static void skd_isr_fwstate(struct skd_device
*skdev
)
1774 int prev_driver_state
= skdev
->state
;
1776 sense
= SKD_READL(skdev
, FIT_STATUS
);
1777 state
= sense
& FIT_SR_DRIVE_STATE_MASK
;
1779 dev_err(&skdev
->pdev
->dev
, "s1120 state %s(%d)=>%s(%d)\n",
1780 skd_drive_state_to_str(skdev
->drive_state
), skdev
->drive_state
,
1781 skd_drive_state_to_str(state
), state
);
1783 skdev
->drive_state
= state
;
1785 switch (skdev
->drive_state
) {
1786 case FIT_SR_DRIVE_INIT
:
1787 if (skdev
->state
== SKD_DRVR_STATE_PROTOCOL_MISMATCH
) {
1788 skd_disable_interrupts(skdev
);
1791 if (skdev
->state
== SKD_DRVR_STATE_RESTARTING
)
1792 skd_recover_requests(skdev
);
1793 if (skdev
->state
== SKD_DRVR_STATE_WAIT_BOOT
) {
1794 skdev
->timer_countdown
= SKD_STARTING_TIMO
;
1795 skdev
->state
= SKD_DRVR_STATE_STARTING
;
1796 skd_soft_reset(skdev
);
1799 mtd
= FIT_MXD_CONS(FIT_MTD_FITFW_INIT
, 0, 0);
1800 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1801 skdev
->last_mtd
= mtd
;
1804 case FIT_SR_DRIVE_ONLINE
:
1805 skdev
->cur_max_queue_depth
= skd_max_queue_depth
;
1806 if (skdev
->cur_max_queue_depth
> skdev
->dev_max_queue_depth
)
1807 skdev
->cur_max_queue_depth
= skdev
->dev_max_queue_depth
;
1809 skdev
->queue_low_water_mark
=
1810 skdev
->cur_max_queue_depth
* 2 / 3 + 1;
1811 if (skdev
->queue_low_water_mark
< 1)
1812 skdev
->queue_low_water_mark
= 1;
1813 dev_info(&skdev
->pdev
->dev
,
1814 "Queue depth limit=%d dev=%d lowat=%d\n",
1815 skdev
->cur_max_queue_depth
,
1816 skdev
->dev_max_queue_depth
,
1817 skdev
->queue_low_water_mark
);
1819 skd_refresh_device_data(skdev
);
1822 case FIT_SR_DRIVE_BUSY
:
1823 skdev
->state
= SKD_DRVR_STATE_BUSY
;
1824 skdev
->timer_countdown
= SKD_BUSY_TIMO
;
1825 skd_quiesce_dev(skdev
);
1827 case FIT_SR_DRIVE_BUSY_SANITIZE
:
1828 /* set timer for 3 seconds, we'll abort any unfinished
1829 * commands after that expires
1831 skdev
->state
= SKD_DRVR_STATE_BUSY_SANITIZE
;
1832 skdev
->timer_countdown
= SKD_TIMER_SECONDS(3);
1833 schedule_work(&skdev
->start_queue
);
1835 case FIT_SR_DRIVE_BUSY_ERASE
:
1836 skdev
->state
= SKD_DRVR_STATE_BUSY_ERASE
;
1837 skdev
->timer_countdown
= SKD_BUSY_TIMO
;
1839 case FIT_SR_DRIVE_OFFLINE
:
1840 skdev
->state
= SKD_DRVR_STATE_IDLE
;
1842 case FIT_SR_DRIVE_SOFT_RESET
:
1843 switch (skdev
->state
) {
1844 case SKD_DRVR_STATE_STARTING
:
1845 case SKD_DRVR_STATE_RESTARTING
:
1846 /* Expected by a caller of skd_soft_reset() */
1849 skdev
->state
= SKD_DRVR_STATE_RESTARTING
;
1853 case FIT_SR_DRIVE_FW_BOOTING
:
1854 dev_dbg(&skdev
->pdev
->dev
, "ISR FIT_SR_DRIVE_FW_BOOTING\n");
1855 skdev
->state
= SKD_DRVR_STATE_WAIT_BOOT
;
1856 skdev
->timer_countdown
= SKD_WAIT_BOOT_TIMO
;
1859 case FIT_SR_DRIVE_DEGRADED
:
1860 case FIT_SR_PCIE_LINK_DOWN
:
1861 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD
:
1864 case FIT_SR_DRIVE_FAULT
:
1865 skd_drive_fault(skdev
);
1866 skd_recover_requests(skdev
);
1867 schedule_work(&skdev
->start_queue
);
1870 /* PCIe bus returned all Fs? */
1872 dev_info(&skdev
->pdev
->dev
, "state=0x%x sense=0x%x\n", state
,
1874 skd_drive_disappeared(skdev
);
1875 skd_recover_requests(skdev
);
1876 schedule_work(&skdev
->start_queue
);
1880 * Uknown FW State. Wait for a state we recognize.
1884 dev_err(&skdev
->pdev
->dev
, "Driver state %s(%d)=>%s(%d)\n",
1885 skd_skdev_state_to_str(prev_driver_state
), prev_driver_state
,
1886 skd_skdev_state_to_str(skdev
->state
), skdev
->state
);
1889 static void skd_recover_request(struct request
*req
, void *data
, bool reserved
)
1891 struct skd_device
*const skdev
= data
;
1892 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(req
);
1894 if (skreq
->state
!= SKD_REQ_STATE_BUSY
)
1897 skd_log_skreq(skdev
, skreq
, "recover");
1899 /* Release DMA resources for the request. */
1900 if (skreq
->n_sg
> 0)
1901 skd_postop_sg_list(skdev
, skreq
);
1903 skreq
->state
= SKD_REQ_STATE_IDLE
;
1904 skreq
->status
= BLK_STS_IOERR
;
1905 blk_mq_complete_request(req
);
1908 static void skd_recover_requests(struct skd_device
*skdev
)
1910 blk_mq_tagset_busy_iter(&skdev
->tag_set
, skd_recover_request
, skdev
);
1913 static void skd_isr_msg_from_dev(struct skd_device
*skdev
)
1919 mfd
= SKD_READL(skdev
, FIT_MSG_FROM_DEVICE
);
1921 dev_dbg(&skdev
->pdev
->dev
, "mfd=0x%x last_mtd=0x%x\n", mfd
,
1924 /* ignore any mtd that is an ack for something we didn't send */
1925 if (FIT_MXD_TYPE(mfd
) != FIT_MXD_TYPE(skdev
->last_mtd
))
1928 switch (FIT_MXD_TYPE(mfd
)) {
1929 case FIT_MTD_FITFW_INIT
:
1930 skdev
->proto_ver
= FIT_PROTOCOL_MAJOR_VER(mfd
);
1932 if (skdev
->proto_ver
!= FIT_PROTOCOL_VERSION_1
) {
1933 dev_err(&skdev
->pdev
->dev
, "protocol mismatch\n");
1934 dev_err(&skdev
->pdev
->dev
, " got=%d support=%d\n",
1935 skdev
->proto_ver
, FIT_PROTOCOL_VERSION_1
);
1936 dev_err(&skdev
->pdev
->dev
, " please upgrade driver\n");
1937 skdev
->state
= SKD_DRVR_STATE_PROTOCOL_MISMATCH
;
1938 skd_soft_reset(skdev
);
1941 mtd
= FIT_MXD_CONS(FIT_MTD_GET_CMDQ_DEPTH
, 0, 0);
1942 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1943 skdev
->last_mtd
= mtd
;
1946 case FIT_MTD_GET_CMDQ_DEPTH
:
1947 skdev
->dev_max_queue_depth
= FIT_MXD_DATA(mfd
);
1948 mtd
= FIT_MXD_CONS(FIT_MTD_SET_COMPQ_DEPTH
, 0,
1949 SKD_N_COMPLETION_ENTRY
);
1950 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1951 skdev
->last_mtd
= mtd
;
1954 case FIT_MTD_SET_COMPQ_DEPTH
:
1955 SKD_WRITEQ(skdev
, skdev
->cq_dma_address
, FIT_MSG_TO_DEVICE_ARG
);
1956 mtd
= FIT_MXD_CONS(FIT_MTD_SET_COMPQ_ADDR
, 0, 0);
1957 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1958 skdev
->last_mtd
= mtd
;
1961 case FIT_MTD_SET_COMPQ_ADDR
:
1962 skd_reset_skcomp(skdev
);
1963 mtd
= FIT_MXD_CONS(FIT_MTD_CMD_LOG_HOST_ID
, 0, skdev
->devno
);
1964 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1965 skdev
->last_mtd
= mtd
;
1968 case FIT_MTD_CMD_LOG_HOST_ID
:
1969 /* hardware interface overflows in y2106 */
1970 skdev
->connect_time_stamp
= (u32
)ktime_get_real_seconds();
1971 data
= skdev
->connect_time_stamp
& 0xFFFF;
1972 mtd
= FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_LO
, 0, data
);
1973 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1974 skdev
->last_mtd
= mtd
;
1977 case FIT_MTD_CMD_LOG_TIME_STAMP_LO
:
1978 skdev
->drive_jiffies
= FIT_MXD_DATA(mfd
);
1979 data
= (skdev
->connect_time_stamp
>> 16) & 0xFFFF;
1980 mtd
= FIT_MXD_CONS(FIT_MTD_CMD_LOG_TIME_STAMP_HI
, 0, data
);
1981 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1982 skdev
->last_mtd
= mtd
;
1985 case FIT_MTD_CMD_LOG_TIME_STAMP_HI
:
1986 skdev
->drive_jiffies
|= (FIT_MXD_DATA(mfd
) << 16);
1987 mtd
= FIT_MXD_CONS(FIT_MTD_ARM_QUEUE
, 0, 0);
1988 SKD_WRITEL(skdev
, mtd
, FIT_MSG_TO_DEVICE
);
1989 skdev
->last_mtd
= mtd
;
1991 dev_err(&skdev
->pdev
->dev
, "Time sync driver=0x%x device=0x%x\n",
1992 skdev
->connect_time_stamp
, skdev
->drive_jiffies
);
1995 case FIT_MTD_ARM_QUEUE
:
1996 skdev
->last_mtd
= 0;
1998 * State should be, or soon will be, FIT_SR_DRIVE_ONLINE.
2007 static void skd_disable_interrupts(struct skd_device
*skdev
)
2011 sense
= SKD_READL(skdev
, FIT_CONTROL
);
2012 sense
&= ~FIT_CR_ENABLE_INTERRUPTS
;
2013 SKD_WRITEL(skdev
, sense
, FIT_CONTROL
);
2014 dev_dbg(&skdev
->pdev
->dev
, "sense 0x%x\n", sense
);
2016 /* Note that the 1s is written. A 1-bit means
2017 * disable, a 0 means enable.
2019 SKD_WRITEL(skdev
, ~0, FIT_INT_MASK_HOST
);
2022 static void skd_enable_interrupts(struct skd_device
*skdev
)
2026 /* unmask interrupts first */
2027 val
= FIT_ISH_FW_STATE_CHANGE
+
2028 FIT_ISH_COMPLETION_POSTED
+ FIT_ISH_MSG_FROM_DEV
;
2030 /* Note that the compliment of mask is written. A 1-bit means
2031 * disable, a 0 means enable. */
2032 SKD_WRITEL(skdev
, ~val
, FIT_INT_MASK_HOST
);
2033 dev_dbg(&skdev
->pdev
->dev
, "interrupt mask=0x%x\n", ~val
);
2035 val
= SKD_READL(skdev
, FIT_CONTROL
);
2036 val
|= FIT_CR_ENABLE_INTERRUPTS
;
2037 dev_dbg(&skdev
->pdev
->dev
, "control=0x%x\n", val
);
2038 SKD_WRITEL(skdev
, val
, FIT_CONTROL
);
2042 *****************************************************************************
2043 * START, STOP, RESTART, QUIESCE, UNQUIESCE
2044 *****************************************************************************
2047 static void skd_soft_reset(struct skd_device
*skdev
)
2051 val
= SKD_READL(skdev
, FIT_CONTROL
);
2052 val
|= (FIT_CR_SOFT_RESET
);
2053 dev_dbg(&skdev
->pdev
->dev
, "control=0x%x\n", val
);
2054 SKD_WRITEL(skdev
, val
, FIT_CONTROL
);
2057 static void skd_start_device(struct skd_device
*skdev
)
2059 unsigned long flags
;
2063 spin_lock_irqsave(&skdev
->lock
, flags
);
2065 /* ack all ghost interrupts */
2066 SKD_WRITEL(skdev
, FIT_INT_DEF_MASK
, FIT_INT_STATUS_HOST
);
2068 sense
= SKD_READL(skdev
, FIT_STATUS
);
2070 dev_dbg(&skdev
->pdev
->dev
, "initial status=0x%x\n", sense
);
2072 state
= sense
& FIT_SR_DRIVE_STATE_MASK
;
2073 skdev
->drive_state
= state
;
2074 skdev
->last_mtd
= 0;
2076 skdev
->state
= SKD_DRVR_STATE_STARTING
;
2077 skdev
->timer_countdown
= SKD_STARTING_TIMO
;
2079 skd_enable_interrupts(skdev
);
2081 switch (skdev
->drive_state
) {
2082 case FIT_SR_DRIVE_OFFLINE
:
2083 dev_err(&skdev
->pdev
->dev
, "Drive offline...\n");
2086 case FIT_SR_DRIVE_FW_BOOTING
:
2087 dev_dbg(&skdev
->pdev
->dev
, "FIT_SR_DRIVE_FW_BOOTING\n");
2088 skdev
->state
= SKD_DRVR_STATE_WAIT_BOOT
;
2089 skdev
->timer_countdown
= SKD_WAIT_BOOT_TIMO
;
2092 case FIT_SR_DRIVE_BUSY_SANITIZE
:
2093 dev_info(&skdev
->pdev
->dev
, "Start: BUSY_SANITIZE\n");
2094 skdev
->state
= SKD_DRVR_STATE_BUSY_SANITIZE
;
2095 skdev
->timer_countdown
= SKD_STARTED_BUSY_TIMO
;
2098 case FIT_SR_DRIVE_BUSY_ERASE
:
2099 dev_info(&skdev
->pdev
->dev
, "Start: BUSY_ERASE\n");
2100 skdev
->state
= SKD_DRVR_STATE_BUSY_ERASE
;
2101 skdev
->timer_countdown
= SKD_STARTED_BUSY_TIMO
;
2104 case FIT_SR_DRIVE_INIT
:
2105 case FIT_SR_DRIVE_ONLINE
:
2106 skd_soft_reset(skdev
);
2109 case FIT_SR_DRIVE_BUSY
:
2110 dev_err(&skdev
->pdev
->dev
, "Drive Busy...\n");
2111 skdev
->state
= SKD_DRVR_STATE_BUSY
;
2112 skdev
->timer_countdown
= SKD_STARTED_BUSY_TIMO
;
2115 case FIT_SR_DRIVE_SOFT_RESET
:
2116 dev_err(&skdev
->pdev
->dev
, "drive soft reset in prog\n");
2119 case FIT_SR_DRIVE_FAULT
:
2120 /* Fault state is bad...soft reset won't do it...
2121 * Hard reset, maybe, but does it work on device?
2122 * For now, just fault so the system doesn't hang.
2124 skd_drive_fault(skdev
);
2125 /*start the queue so we can respond with error to requests */
2126 dev_dbg(&skdev
->pdev
->dev
, "starting queue\n");
2127 schedule_work(&skdev
->start_queue
);
2128 skdev
->gendisk_on
= -1;
2129 wake_up_interruptible(&skdev
->waitq
);
2133 /* Most likely the device isn't there or isn't responding
2134 * to the BAR1 addresses. */
2135 skd_drive_disappeared(skdev
);
2136 /*start the queue so we can respond with error to requests */
2137 dev_dbg(&skdev
->pdev
->dev
,
2138 "starting queue to error-out reqs\n");
2139 schedule_work(&skdev
->start_queue
);
2140 skdev
->gendisk_on
= -1;
2141 wake_up_interruptible(&skdev
->waitq
);
2145 dev_err(&skdev
->pdev
->dev
, "Start: unknown state %x\n",
2146 skdev
->drive_state
);
2150 state
= SKD_READL(skdev
, FIT_CONTROL
);
2151 dev_dbg(&skdev
->pdev
->dev
, "FIT Control Status=0x%x\n", state
);
2153 state
= SKD_READL(skdev
, FIT_INT_STATUS_HOST
);
2154 dev_dbg(&skdev
->pdev
->dev
, "Intr Status=0x%x\n", state
);
2156 state
= SKD_READL(skdev
, FIT_INT_MASK_HOST
);
2157 dev_dbg(&skdev
->pdev
->dev
, "Intr Mask=0x%x\n", state
);
2159 state
= SKD_READL(skdev
, FIT_MSG_FROM_DEVICE
);
2160 dev_dbg(&skdev
->pdev
->dev
, "Msg from Dev=0x%x\n", state
);
2162 state
= SKD_READL(skdev
, FIT_HW_VERSION
);
2163 dev_dbg(&skdev
->pdev
->dev
, "HW version=0x%x\n", state
);
2165 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2168 static void skd_stop_device(struct skd_device
*skdev
)
2170 unsigned long flags
;
2171 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
2175 spin_lock_irqsave(&skdev
->lock
, flags
);
2177 if (skdev
->state
!= SKD_DRVR_STATE_ONLINE
) {
2178 dev_err(&skdev
->pdev
->dev
, "%s not online no sync\n", __func__
);
2182 if (skspcl
->req
.state
!= SKD_REQ_STATE_IDLE
) {
2183 dev_err(&skdev
->pdev
->dev
, "%s no special\n", __func__
);
2187 skdev
->state
= SKD_DRVR_STATE_SYNCING
;
2188 skdev
->sync_done
= 0;
2190 skd_send_internal_skspcl(skdev
, skspcl
, SYNCHRONIZE_CACHE
);
2192 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2194 wait_event_interruptible_timeout(skdev
->waitq
,
2195 (skdev
->sync_done
), (10 * HZ
));
2197 spin_lock_irqsave(&skdev
->lock
, flags
);
2199 switch (skdev
->sync_done
) {
2201 dev_err(&skdev
->pdev
->dev
, "%s no sync\n", __func__
);
2204 dev_err(&skdev
->pdev
->dev
, "%s sync done\n", __func__
);
2207 dev_err(&skdev
->pdev
->dev
, "%s sync error\n", __func__
);
2211 skdev
->state
= SKD_DRVR_STATE_STOPPING
;
2212 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2214 skd_kill_timer(skdev
);
2216 spin_lock_irqsave(&skdev
->lock
, flags
);
2217 skd_disable_interrupts(skdev
);
2219 /* ensure all ints on device are cleared */
2220 /* soft reset the device to unload with a clean slate */
2221 SKD_WRITEL(skdev
, FIT_INT_DEF_MASK
, FIT_INT_STATUS_HOST
);
2222 SKD_WRITEL(skdev
, FIT_CR_SOFT_RESET
, FIT_CONTROL
);
2224 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2226 /* poll every 100ms, 1 second timeout */
2227 for (i
= 0; i
< 10; i
++) {
2229 SKD_READL(skdev
, FIT_STATUS
) & FIT_SR_DRIVE_STATE_MASK
;
2230 if (dev_state
== FIT_SR_DRIVE_INIT
)
2232 set_current_state(TASK_INTERRUPTIBLE
);
2233 schedule_timeout(msecs_to_jiffies(100));
2236 if (dev_state
!= FIT_SR_DRIVE_INIT
)
2237 dev_err(&skdev
->pdev
->dev
, "%s state error 0x%02x\n", __func__
,
2241 /* assume spinlock is held */
2242 static void skd_restart_device(struct skd_device
*skdev
)
2246 /* ack all ghost interrupts */
2247 SKD_WRITEL(skdev
, FIT_INT_DEF_MASK
, FIT_INT_STATUS_HOST
);
2249 state
= SKD_READL(skdev
, FIT_STATUS
);
2251 dev_dbg(&skdev
->pdev
->dev
, "drive status=0x%x\n", state
);
2253 state
&= FIT_SR_DRIVE_STATE_MASK
;
2254 skdev
->drive_state
= state
;
2255 skdev
->last_mtd
= 0;
2257 skdev
->state
= SKD_DRVR_STATE_RESTARTING
;
2258 skdev
->timer_countdown
= SKD_RESTARTING_TIMO
;
2260 skd_soft_reset(skdev
);
2263 /* assume spinlock is held */
2264 static int skd_quiesce_dev(struct skd_device
*skdev
)
2268 switch (skdev
->state
) {
2269 case SKD_DRVR_STATE_BUSY
:
2270 case SKD_DRVR_STATE_BUSY_IMMINENT
:
2271 dev_dbg(&skdev
->pdev
->dev
, "stopping queue\n");
2272 blk_mq_stop_hw_queues(skdev
->queue
);
2274 case SKD_DRVR_STATE_ONLINE
:
2275 case SKD_DRVR_STATE_STOPPING
:
2276 case SKD_DRVR_STATE_SYNCING
:
2277 case SKD_DRVR_STATE_PAUSING
:
2278 case SKD_DRVR_STATE_PAUSED
:
2279 case SKD_DRVR_STATE_STARTING
:
2280 case SKD_DRVR_STATE_RESTARTING
:
2281 case SKD_DRVR_STATE_RESUMING
:
2284 dev_dbg(&skdev
->pdev
->dev
, "state [%d] not implemented\n",
2290 /* assume spinlock is held */
2291 static int skd_unquiesce_dev(struct skd_device
*skdev
)
2293 int prev_driver_state
= skdev
->state
;
2295 skd_log_skdev(skdev
, "unquiesce");
2296 if (skdev
->state
== SKD_DRVR_STATE_ONLINE
) {
2297 dev_dbg(&skdev
->pdev
->dev
, "**** device already ONLINE\n");
2300 if (skdev
->drive_state
!= FIT_SR_DRIVE_ONLINE
) {
2302 * If there has been an state change to other than
2303 * ONLINE, we will rely on controller state change
2304 * to come back online and restart the queue.
2305 * The BUSY state means that driver is ready to
2306 * continue normal processing but waiting for controller
2307 * to become available.
2309 skdev
->state
= SKD_DRVR_STATE_BUSY
;
2310 dev_dbg(&skdev
->pdev
->dev
, "drive BUSY state\n");
2315 * Drive has just come online, driver is either in startup,
2316 * paused performing a task, or bust waiting for hardware.
2318 switch (skdev
->state
) {
2319 case SKD_DRVR_STATE_PAUSED
:
2320 case SKD_DRVR_STATE_BUSY
:
2321 case SKD_DRVR_STATE_BUSY_IMMINENT
:
2322 case SKD_DRVR_STATE_BUSY_ERASE
:
2323 case SKD_DRVR_STATE_STARTING
:
2324 case SKD_DRVR_STATE_RESTARTING
:
2325 case SKD_DRVR_STATE_FAULT
:
2326 case SKD_DRVR_STATE_IDLE
:
2327 case SKD_DRVR_STATE_LOAD
:
2328 skdev
->state
= SKD_DRVR_STATE_ONLINE
;
2329 dev_err(&skdev
->pdev
->dev
, "Driver state %s(%d)=>%s(%d)\n",
2330 skd_skdev_state_to_str(prev_driver_state
),
2331 prev_driver_state
, skd_skdev_state_to_str(skdev
->state
),
2333 dev_dbg(&skdev
->pdev
->dev
,
2334 "**** device ONLINE...starting block queue\n");
2335 dev_dbg(&skdev
->pdev
->dev
, "starting queue\n");
2336 dev_info(&skdev
->pdev
->dev
, "STEC s1120 ONLINE\n");
2337 schedule_work(&skdev
->start_queue
);
2338 skdev
->gendisk_on
= 1;
2339 wake_up_interruptible(&skdev
->waitq
);
2342 case SKD_DRVR_STATE_DISAPPEARED
:
2344 dev_dbg(&skdev
->pdev
->dev
,
2345 "**** driver state %d, not implemented\n",
2353 *****************************************************************************
2354 * PCIe MSI/MSI-X INTERRUPT HANDLERS
2355 *****************************************************************************
2358 static irqreturn_t
skd_reserved_isr(int irq
, void *skd_host_data
)
2360 struct skd_device
*skdev
= skd_host_data
;
2361 unsigned long flags
;
2363 spin_lock_irqsave(&skdev
->lock
, flags
);
2364 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2365 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2366 dev_err(&skdev
->pdev
->dev
, "MSIX reserved irq %d = 0x%x\n", irq
,
2367 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2368 SKD_WRITEL(skdev
, FIT_INT_RESERVED_MASK
, FIT_INT_STATUS_HOST
);
2369 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2373 static irqreturn_t
skd_statec_isr(int irq
, void *skd_host_data
)
2375 struct skd_device
*skdev
= skd_host_data
;
2376 unsigned long flags
;
2378 spin_lock_irqsave(&skdev
->lock
, flags
);
2379 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2380 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2381 SKD_WRITEL(skdev
, FIT_ISH_FW_STATE_CHANGE
, FIT_INT_STATUS_HOST
);
2382 skd_isr_fwstate(skdev
);
2383 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2387 static irqreturn_t
skd_comp_q(int irq
, void *skd_host_data
)
2389 struct skd_device
*skdev
= skd_host_data
;
2390 unsigned long flags
;
2391 int flush_enqueued
= 0;
2394 spin_lock_irqsave(&skdev
->lock
, flags
);
2395 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2396 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2397 SKD_WRITEL(skdev
, FIT_ISH_COMPLETION_POSTED
, FIT_INT_STATUS_HOST
);
2398 deferred
= skd_isr_completion_posted(skdev
, skd_isr_comp_limit
,
2401 schedule_work(&skdev
->start_queue
);
2404 schedule_work(&skdev
->completion_worker
);
2405 else if (!flush_enqueued
)
2406 schedule_work(&skdev
->start_queue
);
2408 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2413 static irqreturn_t
skd_msg_isr(int irq
, void *skd_host_data
)
2415 struct skd_device
*skdev
= skd_host_data
;
2416 unsigned long flags
;
2418 spin_lock_irqsave(&skdev
->lock
, flags
);
2419 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2420 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2421 SKD_WRITEL(skdev
, FIT_ISH_MSG_FROM_DEV
, FIT_INT_STATUS_HOST
);
2422 skd_isr_msg_from_dev(skdev
);
2423 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2427 static irqreturn_t
skd_qfull_isr(int irq
, void *skd_host_data
)
2429 struct skd_device
*skdev
= skd_host_data
;
2430 unsigned long flags
;
2432 spin_lock_irqsave(&skdev
->lock
, flags
);
2433 dev_dbg(&skdev
->pdev
->dev
, "MSIX = 0x%x\n",
2434 SKD_READL(skdev
, FIT_INT_STATUS_HOST
));
2435 SKD_WRITEL(skdev
, FIT_INT_QUEUE_FULL
, FIT_INT_STATUS_HOST
);
2436 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2441 *****************************************************************************
2442 * PCIe MSI/MSI-X SETUP
2443 *****************************************************************************
2446 struct skd_msix_entry
{
2450 struct skd_init_msix_entry
{
2452 irq_handler_t handler
;
2455 #define SKD_MAX_MSIX_COUNT 13
2456 #define SKD_MIN_MSIX_COUNT 7
2457 #define SKD_BASE_MSIX_IRQ 4
2459 static struct skd_init_msix_entry msix_entries
[SKD_MAX_MSIX_COUNT
] = {
2460 { "(DMA 0)", skd_reserved_isr
},
2461 { "(DMA 1)", skd_reserved_isr
},
2462 { "(DMA 2)", skd_reserved_isr
},
2463 { "(DMA 3)", skd_reserved_isr
},
2464 { "(State Change)", skd_statec_isr
},
2465 { "(COMPL_Q)", skd_comp_q
},
2466 { "(MSG)", skd_msg_isr
},
2467 { "(Reserved)", skd_reserved_isr
},
2468 { "(Reserved)", skd_reserved_isr
},
2469 { "(Queue Full 0)", skd_qfull_isr
},
2470 { "(Queue Full 1)", skd_qfull_isr
},
2471 { "(Queue Full 2)", skd_qfull_isr
},
2472 { "(Queue Full 3)", skd_qfull_isr
},
2475 static int skd_acquire_msix(struct skd_device
*skdev
)
2478 struct pci_dev
*pdev
= skdev
->pdev
;
2480 rc
= pci_alloc_irq_vectors(pdev
, SKD_MAX_MSIX_COUNT
, SKD_MAX_MSIX_COUNT
,
2483 dev_err(&skdev
->pdev
->dev
, "failed to enable MSI-X %d\n", rc
);
2487 skdev
->msix_entries
= kcalloc(SKD_MAX_MSIX_COUNT
,
2488 sizeof(struct skd_msix_entry
), GFP_KERNEL
);
2489 if (!skdev
->msix_entries
) {
2491 dev_err(&skdev
->pdev
->dev
, "msix table allocation error\n");
2495 /* Enable MSI-X vectors for the base queue */
2496 for (i
= 0; i
< SKD_MAX_MSIX_COUNT
; i
++) {
2497 struct skd_msix_entry
*qentry
= &skdev
->msix_entries
[i
];
2499 snprintf(qentry
->isr_name
, sizeof(qentry
->isr_name
),
2500 "%s%d-msix %s", DRV_NAME
, skdev
->devno
,
2501 msix_entries
[i
].name
);
2503 rc
= devm_request_irq(&skdev
->pdev
->dev
,
2504 pci_irq_vector(skdev
->pdev
, i
),
2505 msix_entries
[i
].handler
, 0,
2506 qentry
->isr_name
, skdev
);
2508 dev_err(&skdev
->pdev
->dev
,
2509 "Unable to register(%d) MSI-X handler %d: %s\n",
2510 rc
, i
, qentry
->isr_name
);
2515 dev_dbg(&skdev
->pdev
->dev
, "%d msix irq(s) enabled\n",
2516 SKD_MAX_MSIX_COUNT
);
2521 devm_free_irq(&pdev
->dev
, pci_irq_vector(pdev
, i
), skdev
);
2523 kfree(skdev
->msix_entries
);
2524 skdev
->msix_entries
= NULL
;
2528 static int skd_acquire_irq(struct skd_device
*skdev
)
2530 struct pci_dev
*pdev
= skdev
->pdev
;
2531 unsigned int irq_flag
= PCI_IRQ_LEGACY
;
2534 if (skd_isr_type
== SKD_IRQ_MSIX
) {
2535 rc
= skd_acquire_msix(skdev
);
2539 dev_err(&skdev
->pdev
->dev
,
2540 "failed to enable MSI-X, re-trying with MSI %d\n", rc
);
2543 snprintf(skdev
->isr_name
, sizeof(skdev
->isr_name
), "%s%d", DRV_NAME
,
2546 if (skd_isr_type
!= SKD_IRQ_LEGACY
)
2547 irq_flag
|= PCI_IRQ_MSI
;
2548 rc
= pci_alloc_irq_vectors(pdev
, 1, 1, irq_flag
);
2550 dev_err(&skdev
->pdev
->dev
,
2551 "failed to allocate the MSI interrupt %d\n", rc
);
2555 rc
= devm_request_irq(&pdev
->dev
, pdev
->irq
, skd_isr
,
2556 pdev
->msi_enabled
? 0 : IRQF_SHARED
,
2557 skdev
->isr_name
, skdev
);
2559 pci_free_irq_vectors(pdev
);
2560 dev_err(&skdev
->pdev
->dev
, "failed to allocate interrupt %d\n",
2568 static void skd_release_irq(struct skd_device
*skdev
)
2570 struct pci_dev
*pdev
= skdev
->pdev
;
2572 if (skdev
->msix_entries
) {
2575 for (i
= 0; i
< SKD_MAX_MSIX_COUNT
; i
++) {
2576 devm_free_irq(&pdev
->dev
, pci_irq_vector(pdev
, i
),
2580 kfree(skdev
->msix_entries
);
2581 skdev
->msix_entries
= NULL
;
2583 devm_free_irq(&pdev
->dev
, pdev
->irq
, skdev
);
2586 pci_free_irq_vectors(pdev
);
2590 *****************************************************************************
2592 *****************************************************************************
2595 static void *skd_alloc_dma(struct skd_device
*skdev
, struct kmem_cache
*s
,
2596 dma_addr_t
*dma_handle
, gfp_t gfp
,
2597 enum dma_data_direction dir
)
2599 struct device
*dev
= &skdev
->pdev
->dev
;
2602 buf
= kmem_cache_alloc(s
, gfp
);
2605 *dma_handle
= dma_map_single(dev
, buf
,
2606 kmem_cache_size(s
), dir
);
2607 if (dma_mapping_error(dev
, *dma_handle
)) {
2608 kmem_cache_free(s
, buf
);
2614 static void skd_free_dma(struct skd_device
*skdev
, struct kmem_cache
*s
,
2615 void *vaddr
, dma_addr_t dma_handle
,
2616 enum dma_data_direction dir
)
2621 dma_unmap_single(&skdev
->pdev
->dev
, dma_handle
,
2622 kmem_cache_size(s
), dir
);
2623 kmem_cache_free(s
, vaddr
);
2626 static int skd_cons_skcomp(struct skd_device
*skdev
)
2629 struct fit_completion_entry_v1
*skcomp
;
2631 dev_dbg(&skdev
->pdev
->dev
,
2632 "comp pci_alloc, total bytes %zd entries %d\n",
2633 SKD_SKCOMP_SIZE
, SKD_N_COMPLETION_ENTRY
);
2635 skcomp
= pci_zalloc_consistent(skdev
->pdev
, SKD_SKCOMP_SIZE
,
2636 &skdev
->cq_dma_address
);
2638 if (skcomp
== NULL
) {
2643 skdev
->skcomp_table
= skcomp
;
2644 skdev
->skerr_table
= (struct fit_comp_error_info
*)((char *)skcomp
+
2646 SKD_N_COMPLETION_ENTRY
);
2652 static int skd_cons_skmsg(struct skd_device
*skdev
)
2657 dev_dbg(&skdev
->pdev
->dev
,
2658 "skmsg_table kcalloc, struct %lu, count %u total %lu\n",
2659 sizeof(struct skd_fitmsg_context
), skdev
->num_fitmsg_context
,
2660 sizeof(struct skd_fitmsg_context
) * skdev
->num_fitmsg_context
);
2662 skdev
->skmsg_table
= kcalloc(skdev
->num_fitmsg_context
,
2663 sizeof(struct skd_fitmsg_context
),
2665 if (skdev
->skmsg_table
== NULL
) {
2670 for (i
= 0; i
< skdev
->num_fitmsg_context
; i
++) {
2671 struct skd_fitmsg_context
*skmsg
;
2673 skmsg
= &skdev
->skmsg_table
[i
];
2675 skmsg
->id
= i
+ SKD_ID_FIT_MSG
;
2677 skmsg
->msg_buf
= pci_alloc_consistent(skdev
->pdev
,
2679 &skmsg
->mb_dma_address
);
2681 if (skmsg
->msg_buf
== NULL
) {
2686 WARN(((uintptr_t)skmsg
->msg_buf
| skmsg
->mb_dma_address
) &
2687 (FIT_QCMD_ALIGN
- 1),
2688 "not aligned: msg_buf %p mb_dma_address %#llx\n",
2689 skmsg
->msg_buf
, skmsg
->mb_dma_address
);
2690 memset(skmsg
->msg_buf
, 0, SKD_N_FITMSG_BYTES
);
2697 static struct fit_sg_descriptor
*skd_cons_sg_list(struct skd_device
*skdev
,
2699 dma_addr_t
*ret_dma_addr
)
2701 struct fit_sg_descriptor
*sg_list
;
2703 sg_list
= skd_alloc_dma(skdev
, skdev
->sglist_cache
, ret_dma_addr
,
2704 GFP_DMA
| __GFP_ZERO
, DMA_TO_DEVICE
);
2706 if (sg_list
!= NULL
) {
2707 uint64_t dma_address
= *ret_dma_addr
;
2710 for (i
= 0; i
< n_sg
- 1; i
++) {
2712 ndp_off
= (i
+ 1) * sizeof(struct fit_sg_descriptor
);
2714 sg_list
[i
].next_desc_ptr
= dma_address
+ ndp_off
;
2716 sg_list
[i
].next_desc_ptr
= 0LL;
2722 static void skd_free_sg_list(struct skd_device
*skdev
,
2723 struct fit_sg_descriptor
*sg_list
,
2724 dma_addr_t dma_addr
)
2726 if (WARN_ON_ONCE(!sg_list
))
2729 skd_free_dma(skdev
, skdev
->sglist_cache
, sg_list
, dma_addr
,
2733 static int skd_init_request(struct blk_mq_tag_set
*set
, struct request
*rq
,
2734 unsigned int hctx_idx
, unsigned int numa_node
)
2736 struct skd_device
*skdev
= set
->driver_data
;
2737 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(rq
);
2739 skreq
->state
= SKD_REQ_STATE_IDLE
;
2740 skreq
->sg
= (void *)(skreq
+ 1);
2741 sg_init_table(skreq
->sg
, skd_sgs_per_request
);
2742 skreq
->sksg_list
= skd_cons_sg_list(skdev
, skd_sgs_per_request
,
2743 &skreq
->sksg_dma_address
);
2745 return skreq
->sksg_list
? 0 : -ENOMEM
;
2748 static void skd_exit_request(struct blk_mq_tag_set
*set
, struct request
*rq
,
2749 unsigned int hctx_idx
)
2751 struct skd_device
*skdev
= set
->driver_data
;
2752 struct skd_request_context
*skreq
= blk_mq_rq_to_pdu(rq
);
2754 skd_free_sg_list(skdev
, skreq
->sksg_list
, skreq
->sksg_dma_address
);
2757 static int skd_cons_sksb(struct skd_device
*skdev
)
2760 struct skd_special_context
*skspcl
;
2762 skspcl
= &skdev
->internal_skspcl
;
2764 skspcl
->req
.id
= 0 + SKD_ID_INTERNAL
;
2765 skspcl
->req
.state
= SKD_REQ_STATE_IDLE
;
2767 skspcl
->data_buf
= skd_alloc_dma(skdev
, skdev
->databuf_cache
,
2768 &skspcl
->db_dma_address
,
2769 GFP_DMA
| __GFP_ZERO
,
2771 if (skspcl
->data_buf
== NULL
) {
2776 skspcl
->msg_buf
= skd_alloc_dma(skdev
, skdev
->msgbuf_cache
,
2777 &skspcl
->mb_dma_address
,
2778 GFP_DMA
| __GFP_ZERO
, DMA_TO_DEVICE
);
2779 if (skspcl
->msg_buf
== NULL
) {
2784 skspcl
->req
.sksg_list
= skd_cons_sg_list(skdev
, 1,
2785 &skspcl
->req
.sksg_dma_address
);
2786 if (skspcl
->req
.sksg_list
== NULL
) {
2791 if (!skd_format_internal_skspcl(skdev
)) {
2800 static const struct blk_mq_ops skd_mq_ops
= {
2801 .queue_rq
= skd_mq_queue_rq
,
2802 .complete
= skd_complete_rq
,
2803 .timeout
= skd_timed_out
,
2804 .init_request
= skd_init_request
,
2805 .exit_request
= skd_exit_request
,
2808 static int skd_cons_disk(struct skd_device
*skdev
)
2811 struct gendisk
*disk
;
2812 struct request_queue
*q
;
2813 unsigned long flags
;
2815 disk
= alloc_disk(SKD_MINORS_PER_DEVICE
);
2822 sprintf(disk
->disk_name
, DRV_NAME
"%u", skdev
->devno
);
2824 disk
->major
= skdev
->major
;
2825 disk
->first_minor
= skdev
->devno
* SKD_MINORS_PER_DEVICE
;
2826 disk
->fops
= &skd_blockdev_ops
;
2827 disk
->private_data
= skdev
;
2829 memset(&skdev
->tag_set
, 0, sizeof(skdev
->tag_set
));
2830 skdev
->tag_set
.ops
= &skd_mq_ops
;
2831 skdev
->tag_set
.nr_hw_queues
= 1;
2832 skdev
->tag_set
.queue_depth
= skd_max_queue_depth
;
2833 skdev
->tag_set
.cmd_size
= sizeof(struct skd_request_context
) +
2834 skdev
->sgs_per_request
* sizeof(struct scatterlist
);
2835 skdev
->tag_set
.numa_node
= NUMA_NO_NODE
;
2836 skdev
->tag_set
.flags
= BLK_MQ_F_SHOULD_MERGE
|
2838 BLK_ALLOC_POLICY_TO_MQ_FLAG(BLK_TAG_ALLOC_FIFO
);
2839 skdev
->tag_set
.driver_data
= skdev
;
2840 rc
= blk_mq_alloc_tag_set(&skdev
->tag_set
);
2843 q
= blk_mq_init_queue(&skdev
->tag_set
);
2845 blk_mq_free_tag_set(&skdev
->tag_set
);
2849 q
->queuedata
= skdev
;
2854 blk_queue_write_cache(q
, true, true);
2855 blk_queue_max_segments(q
, skdev
->sgs_per_request
);
2856 blk_queue_max_hw_sectors(q
, SKD_N_MAX_SECTORS
);
2858 /* set optimal I/O size to 8KB */
2859 blk_queue_io_opt(q
, 8192);
2861 queue_flag_set_unlocked(QUEUE_FLAG_NONROT
, q
);
2862 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM
, q
);
2864 blk_queue_rq_timeout(q
, 8 * HZ
);
2866 spin_lock_irqsave(&skdev
->lock
, flags
);
2867 dev_dbg(&skdev
->pdev
->dev
, "stopping queue\n");
2868 blk_mq_stop_hw_queues(skdev
->queue
);
2869 spin_unlock_irqrestore(&skdev
->lock
, flags
);
2875 #define SKD_N_DEV_TABLE 16u
2876 static u32 skd_next_devno
;
2878 static struct skd_device
*skd_construct(struct pci_dev
*pdev
)
2880 struct skd_device
*skdev
;
2881 int blk_major
= skd_major
;
2885 skdev
= kzalloc(sizeof(*skdev
), GFP_KERNEL
);
2888 dev_err(&pdev
->dev
, "memory alloc failure\n");
2892 skdev
->state
= SKD_DRVR_STATE_LOAD
;
2894 skdev
->devno
= skd_next_devno
++;
2895 skdev
->major
= blk_major
;
2896 skdev
->dev_max_queue_depth
= 0;
2898 skdev
->num_req_context
= skd_max_queue_depth
;
2899 skdev
->num_fitmsg_context
= skd_max_queue_depth
;
2900 skdev
->cur_max_queue_depth
= 1;
2901 skdev
->queue_low_water_mark
= 1;
2902 skdev
->proto_ver
= 99;
2903 skdev
->sgs_per_request
= skd_sgs_per_request
;
2904 skdev
->dbg_level
= skd_dbg_level
;
2906 spin_lock_init(&skdev
->lock
);
2908 INIT_WORK(&skdev
->start_queue
, skd_start_queue
);
2909 INIT_WORK(&skdev
->completion_worker
, skd_completion_worker
);
2911 size
= max(SKD_N_FITMSG_BYTES
, SKD_N_SPECIAL_FITMSG_BYTES
);
2912 skdev
->msgbuf_cache
= kmem_cache_create("skd-msgbuf", size
, 0,
2913 SLAB_HWCACHE_ALIGN
, NULL
);
2914 if (!skdev
->msgbuf_cache
)
2916 WARN_ONCE(kmem_cache_size(skdev
->msgbuf_cache
) < size
,
2917 "skd-msgbuf: %d < %zd\n",
2918 kmem_cache_size(skdev
->msgbuf_cache
), size
);
2919 size
= skd_sgs_per_request
* sizeof(struct fit_sg_descriptor
);
2920 skdev
->sglist_cache
= kmem_cache_create("skd-sglist", size
, 0,
2921 SLAB_HWCACHE_ALIGN
, NULL
);
2922 if (!skdev
->sglist_cache
)
2924 WARN_ONCE(kmem_cache_size(skdev
->sglist_cache
) < size
,
2925 "skd-sglist: %d < %zd\n",
2926 kmem_cache_size(skdev
->sglist_cache
), size
);
2927 size
= SKD_N_INTERNAL_BYTES
;
2928 skdev
->databuf_cache
= kmem_cache_create("skd-databuf", size
, 0,
2929 SLAB_HWCACHE_ALIGN
, NULL
);
2930 if (!skdev
->databuf_cache
)
2932 WARN_ONCE(kmem_cache_size(skdev
->databuf_cache
) < size
,
2933 "skd-databuf: %d < %zd\n",
2934 kmem_cache_size(skdev
->databuf_cache
), size
);
2936 dev_dbg(&skdev
->pdev
->dev
, "skcomp\n");
2937 rc
= skd_cons_skcomp(skdev
);
2941 dev_dbg(&skdev
->pdev
->dev
, "skmsg\n");
2942 rc
= skd_cons_skmsg(skdev
);
2946 dev_dbg(&skdev
->pdev
->dev
, "sksb\n");
2947 rc
= skd_cons_sksb(skdev
);
2951 dev_dbg(&skdev
->pdev
->dev
, "disk\n");
2952 rc
= skd_cons_disk(skdev
);
2956 dev_dbg(&skdev
->pdev
->dev
, "VICTORY\n");
2960 dev_dbg(&skdev
->pdev
->dev
, "construct failed\n");
2961 skd_destruct(skdev
);
2966 *****************************************************************************
2968 *****************************************************************************
2971 static void skd_free_skcomp(struct skd_device
*skdev
)
2973 if (skdev
->skcomp_table
)
2974 pci_free_consistent(skdev
->pdev
, SKD_SKCOMP_SIZE
,
2975 skdev
->skcomp_table
, skdev
->cq_dma_address
);
2977 skdev
->skcomp_table
= NULL
;
2978 skdev
->cq_dma_address
= 0;
2981 static void skd_free_skmsg(struct skd_device
*skdev
)
2985 if (skdev
->skmsg_table
== NULL
)
2988 for (i
= 0; i
< skdev
->num_fitmsg_context
; i
++) {
2989 struct skd_fitmsg_context
*skmsg
;
2991 skmsg
= &skdev
->skmsg_table
[i
];
2993 if (skmsg
->msg_buf
!= NULL
) {
2994 pci_free_consistent(skdev
->pdev
, SKD_N_FITMSG_BYTES
,
2996 skmsg
->mb_dma_address
);
2998 skmsg
->msg_buf
= NULL
;
2999 skmsg
->mb_dma_address
= 0;
3002 kfree(skdev
->skmsg_table
);
3003 skdev
->skmsg_table
= NULL
;
3006 static void skd_free_sksb(struct skd_device
*skdev
)
3008 struct skd_special_context
*skspcl
= &skdev
->internal_skspcl
;
3010 skd_free_dma(skdev
, skdev
->databuf_cache
, skspcl
->data_buf
,
3011 skspcl
->db_dma_address
, DMA_BIDIRECTIONAL
);
3013 skspcl
->data_buf
= NULL
;
3014 skspcl
->db_dma_address
= 0;
3016 skd_free_dma(skdev
, skdev
->msgbuf_cache
, skspcl
->msg_buf
,
3017 skspcl
->mb_dma_address
, DMA_TO_DEVICE
);
3019 skspcl
->msg_buf
= NULL
;
3020 skspcl
->mb_dma_address
= 0;
3022 skd_free_sg_list(skdev
, skspcl
->req
.sksg_list
,
3023 skspcl
->req
.sksg_dma_address
);
3025 skspcl
->req
.sksg_list
= NULL
;
3026 skspcl
->req
.sksg_dma_address
= 0;
3029 static void skd_free_disk(struct skd_device
*skdev
)
3031 struct gendisk
*disk
= skdev
->disk
;
3033 if (disk
&& (disk
->flags
& GENHD_FL_UP
))
3037 blk_cleanup_queue(skdev
->queue
);
3038 skdev
->queue
= NULL
;
3043 if (skdev
->tag_set
.tags
)
3044 blk_mq_free_tag_set(&skdev
->tag_set
);
3050 static void skd_destruct(struct skd_device
*skdev
)
3055 cancel_work_sync(&skdev
->start_queue
);
3057 dev_dbg(&skdev
->pdev
->dev
, "disk\n");
3058 skd_free_disk(skdev
);
3060 dev_dbg(&skdev
->pdev
->dev
, "sksb\n");
3061 skd_free_sksb(skdev
);
3063 dev_dbg(&skdev
->pdev
->dev
, "skmsg\n");
3064 skd_free_skmsg(skdev
);
3066 dev_dbg(&skdev
->pdev
->dev
, "skcomp\n");
3067 skd_free_skcomp(skdev
);
3069 kmem_cache_destroy(skdev
->databuf_cache
);
3070 kmem_cache_destroy(skdev
->sglist_cache
);
3071 kmem_cache_destroy(skdev
->msgbuf_cache
);
3073 dev_dbg(&skdev
->pdev
->dev
, "skdev\n");
3078 *****************************************************************************
3079 * BLOCK DEVICE (BDEV) GLUE
3080 *****************************************************************************
3083 static int skd_bdev_getgeo(struct block_device
*bdev
, struct hd_geometry
*geo
)
3085 struct skd_device
*skdev
;
3088 skdev
= bdev
->bd_disk
->private_data
;
3090 dev_dbg(&skdev
->pdev
->dev
, "%s: CMD[%s] getgeo device\n",
3091 bdev
->bd_disk
->disk_name
, current
->comm
);
3093 if (skdev
->read_cap_is_valid
) {
3094 capacity
= get_capacity(skdev
->disk
);
3097 geo
->cylinders
= (capacity
) / (255 * 64);
3104 static int skd_bdev_attach(struct device
*parent
, struct skd_device
*skdev
)
3106 dev_dbg(&skdev
->pdev
->dev
, "add_disk\n");
3107 device_add_disk(parent
, skdev
->disk
);
3111 static const struct block_device_operations skd_blockdev_ops
= {
3112 .owner
= THIS_MODULE
,
3113 .getgeo
= skd_bdev_getgeo
,
3117 *****************************************************************************
3119 *****************************************************************************
3122 static const struct pci_device_id skd_pci_tbl
[] = {
3123 { PCI_VENDOR_ID_STEC
, PCI_DEVICE_ID_S1120
,
3124 PCI_ANY_ID
, PCI_ANY_ID
, 0, 0, },
3125 { 0 } /* terminate list */
3128 MODULE_DEVICE_TABLE(pci
, skd_pci_tbl
);
3130 static char *skd_pci_info(struct skd_device
*skdev
, char *str
)
3134 strcpy(str
, "PCIe (");
3135 pcie_reg
= pci_find_capability(skdev
->pdev
, PCI_CAP_ID_EXP
);
3140 uint16_t pcie_lstat
, lspeed
, lwidth
;
3143 pci_read_config_word(skdev
->pdev
, pcie_reg
, &pcie_lstat
);
3144 lspeed
= pcie_lstat
& (0xF);
3145 lwidth
= (pcie_lstat
& 0x3F0) >> 4;
3148 strcat(str
, "2.5GT/s ");
3149 else if (lspeed
== 2)
3150 strcat(str
, "5.0GT/s ");
3152 strcat(str
, "<unknown> ");
3153 snprintf(lwstr
, sizeof(lwstr
), "%dX)", lwidth
);
3159 static int skd_pci_probe(struct pci_dev
*pdev
, const struct pci_device_id
*ent
)
3164 struct skd_device
*skdev
;
3166 dev_dbg(&pdev
->dev
, "vendor=%04X device=%04x\n", pdev
->vendor
,
3169 rc
= pci_enable_device(pdev
);
3172 rc
= pci_request_regions(pdev
, DRV_NAME
);
3175 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(64));
3177 if (pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64))) {
3178 dev_err(&pdev
->dev
, "consistent DMA mask error %d\n",
3182 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
3184 dev_err(&pdev
->dev
, "DMA mask error %d\n", rc
);
3185 goto err_out_regions
;
3190 rc
= register_blkdev(0, DRV_NAME
);
3192 goto err_out_regions
;
3197 skdev
= skd_construct(pdev
);
3198 if (skdev
== NULL
) {
3200 goto err_out_regions
;
3203 skd_pci_info(skdev
, pci_str
);
3204 dev_info(&pdev
->dev
, "%s 64bit\n", pci_str
);
3206 pci_set_master(pdev
);
3207 rc
= pci_enable_pcie_error_reporting(pdev
);
3210 "bad enable of PCIe error reporting rc=%d\n", rc
);
3211 skdev
->pcie_error_reporting_is_enabled
= 0;
3213 skdev
->pcie_error_reporting_is_enabled
= 1;
3215 pci_set_drvdata(pdev
, skdev
);
3217 for (i
= 0; i
< SKD_MAX_BARS
; i
++) {
3218 skdev
->mem_phys
[i
] = pci_resource_start(pdev
, i
);
3219 skdev
->mem_size
[i
] = (u32
)pci_resource_len(pdev
, i
);
3220 skdev
->mem_map
[i
] = ioremap(skdev
->mem_phys
[i
],
3221 skdev
->mem_size
[i
]);
3222 if (!skdev
->mem_map
[i
]) {
3224 "Unable to map adapter memory!\n");
3226 goto err_out_iounmap
;
3228 dev_dbg(&pdev
->dev
, "mem_map=%p, phyd=%016llx, size=%d\n",
3229 skdev
->mem_map
[i
], (uint64_t)skdev
->mem_phys
[i
],
3230 skdev
->mem_size
[i
]);
3233 rc
= skd_acquire_irq(skdev
);
3235 dev_err(&pdev
->dev
, "interrupt resource error %d\n", rc
);
3236 goto err_out_iounmap
;
3239 rc
= skd_start_timer(skdev
);
3243 init_waitqueue_head(&skdev
->waitq
);
3245 skd_start_device(skdev
);
3247 rc
= wait_event_interruptible_timeout(skdev
->waitq
,
3248 (skdev
->gendisk_on
),
3249 (SKD_START_WAIT_SECONDS
* HZ
));
3250 if (skdev
->gendisk_on
> 0) {
3251 /* device came on-line after reset */
3252 skd_bdev_attach(&pdev
->dev
, skdev
);
3255 /* we timed out, something is wrong with the device,
3256 don't add the disk structure */
3257 dev_err(&pdev
->dev
, "error: waiting for s1120 timed out %d!\n",
3259 /* in case of no error; we timeout with ENXIO */
3268 skd_stop_device(skdev
);
3269 skd_release_irq(skdev
);
3272 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3273 if (skdev
->mem_map
[i
])
3274 iounmap(skdev
->mem_map
[i
]);
3276 if (skdev
->pcie_error_reporting_is_enabled
)
3277 pci_disable_pcie_error_reporting(pdev
);
3279 skd_destruct(skdev
);
3282 pci_release_regions(pdev
);
3285 pci_disable_device(pdev
);
3286 pci_set_drvdata(pdev
, NULL
);
3290 static void skd_pci_remove(struct pci_dev
*pdev
)
3293 struct skd_device
*skdev
;
3295 skdev
= pci_get_drvdata(pdev
);
3297 dev_err(&pdev
->dev
, "no device data for PCI\n");
3300 skd_stop_device(skdev
);
3301 skd_release_irq(skdev
);
3303 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3304 if (skdev
->mem_map
[i
])
3305 iounmap(skdev
->mem_map
[i
]);
3307 if (skdev
->pcie_error_reporting_is_enabled
)
3308 pci_disable_pcie_error_reporting(pdev
);
3310 skd_destruct(skdev
);
3312 pci_release_regions(pdev
);
3313 pci_disable_device(pdev
);
3314 pci_set_drvdata(pdev
, NULL
);
3319 static int skd_pci_suspend(struct pci_dev
*pdev
, pm_message_t state
)
3322 struct skd_device
*skdev
;
3324 skdev
= pci_get_drvdata(pdev
);
3326 dev_err(&pdev
->dev
, "no device data for PCI\n");
3330 skd_stop_device(skdev
);
3332 skd_release_irq(skdev
);
3334 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3335 if (skdev
->mem_map
[i
])
3336 iounmap(skdev
->mem_map
[i
]);
3338 if (skdev
->pcie_error_reporting_is_enabled
)
3339 pci_disable_pcie_error_reporting(pdev
);
3341 pci_release_regions(pdev
);
3342 pci_save_state(pdev
);
3343 pci_disable_device(pdev
);
3344 pci_set_power_state(pdev
, pci_choose_state(pdev
, state
));
3348 static int skd_pci_resume(struct pci_dev
*pdev
)
3352 struct skd_device
*skdev
;
3354 skdev
= pci_get_drvdata(pdev
);
3356 dev_err(&pdev
->dev
, "no device data for PCI\n");
3360 pci_set_power_state(pdev
, PCI_D0
);
3361 pci_enable_wake(pdev
, PCI_D0
, 0);
3362 pci_restore_state(pdev
);
3364 rc
= pci_enable_device(pdev
);
3367 rc
= pci_request_regions(pdev
, DRV_NAME
);
3370 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(64));
3372 if (pci_set_consistent_dma_mask(pdev
, DMA_BIT_MASK(64))) {
3374 dev_err(&pdev
->dev
, "consistent DMA mask error %d\n",
3378 rc
= pci_set_dma_mask(pdev
, DMA_BIT_MASK(32));
3381 dev_err(&pdev
->dev
, "DMA mask error %d\n", rc
);
3382 goto err_out_regions
;
3386 pci_set_master(pdev
);
3387 rc
= pci_enable_pcie_error_reporting(pdev
);
3390 "bad enable of PCIe error reporting rc=%d\n", rc
);
3391 skdev
->pcie_error_reporting_is_enabled
= 0;
3393 skdev
->pcie_error_reporting_is_enabled
= 1;
3395 for (i
= 0; i
< SKD_MAX_BARS
; i
++) {
3397 skdev
->mem_phys
[i
] = pci_resource_start(pdev
, i
);
3398 skdev
->mem_size
[i
] = (u32
)pci_resource_len(pdev
, i
);
3399 skdev
->mem_map
[i
] = ioremap(skdev
->mem_phys
[i
],
3400 skdev
->mem_size
[i
]);
3401 if (!skdev
->mem_map
[i
]) {
3402 dev_err(&pdev
->dev
, "Unable to map adapter memory!\n");
3404 goto err_out_iounmap
;
3406 dev_dbg(&pdev
->dev
, "mem_map=%p, phyd=%016llx, size=%d\n",
3407 skdev
->mem_map
[i
], (uint64_t)skdev
->mem_phys
[i
],
3408 skdev
->mem_size
[i
]);
3410 rc
= skd_acquire_irq(skdev
);
3412 dev_err(&pdev
->dev
, "interrupt resource error %d\n", rc
);
3413 goto err_out_iounmap
;
3416 rc
= skd_start_timer(skdev
);
3420 init_waitqueue_head(&skdev
->waitq
);
3422 skd_start_device(skdev
);
3427 skd_stop_device(skdev
);
3428 skd_release_irq(skdev
);
3431 for (i
= 0; i
< SKD_MAX_BARS
; i
++)
3432 if (skdev
->mem_map
[i
])
3433 iounmap(skdev
->mem_map
[i
]);
3435 if (skdev
->pcie_error_reporting_is_enabled
)
3436 pci_disable_pcie_error_reporting(pdev
);
3439 pci_release_regions(pdev
);
3442 pci_disable_device(pdev
);
3446 static void skd_pci_shutdown(struct pci_dev
*pdev
)
3448 struct skd_device
*skdev
;
3450 dev_err(&pdev
->dev
, "%s called\n", __func__
);
3452 skdev
= pci_get_drvdata(pdev
);
3454 dev_err(&pdev
->dev
, "no device data for PCI\n");
3458 dev_err(&pdev
->dev
, "calling stop\n");
3459 skd_stop_device(skdev
);
3462 static struct pci_driver skd_driver
= {
3464 .id_table
= skd_pci_tbl
,
3465 .probe
= skd_pci_probe
,
3466 .remove
= skd_pci_remove
,
3467 .suspend
= skd_pci_suspend
,
3468 .resume
= skd_pci_resume
,
3469 .shutdown
= skd_pci_shutdown
,
3473 *****************************************************************************
3475 *****************************************************************************
3478 const char *skd_drive_state_to_str(int state
)
3481 case FIT_SR_DRIVE_OFFLINE
:
3483 case FIT_SR_DRIVE_INIT
:
3485 case FIT_SR_DRIVE_ONLINE
:
3487 case FIT_SR_DRIVE_BUSY
:
3489 case FIT_SR_DRIVE_FAULT
:
3491 case FIT_SR_DRIVE_DEGRADED
:
3493 case FIT_SR_PCIE_LINK_DOWN
:
3495 case FIT_SR_DRIVE_SOFT_RESET
:
3496 return "SOFT_RESET";
3497 case FIT_SR_DRIVE_NEED_FW_DOWNLOAD
:
3499 case FIT_SR_DRIVE_INIT_FAULT
:
3500 return "INIT_FAULT";
3501 case FIT_SR_DRIVE_BUSY_SANITIZE
:
3502 return "BUSY_SANITIZE";
3503 case FIT_SR_DRIVE_BUSY_ERASE
:
3504 return "BUSY_ERASE";
3505 case FIT_SR_DRIVE_FW_BOOTING
:
3506 return "FW_BOOTING";
3512 const char *skd_skdev_state_to_str(enum skd_drvr_state state
)
3515 case SKD_DRVR_STATE_LOAD
:
3517 case SKD_DRVR_STATE_IDLE
:
3519 case SKD_DRVR_STATE_BUSY
:
3521 case SKD_DRVR_STATE_STARTING
:
3523 case SKD_DRVR_STATE_ONLINE
:
3525 case SKD_DRVR_STATE_PAUSING
:
3527 case SKD_DRVR_STATE_PAUSED
:
3529 case SKD_DRVR_STATE_RESTARTING
:
3530 return "RESTARTING";
3531 case SKD_DRVR_STATE_RESUMING
:
3533 case SKD_DRVR_STATE_STOPPING
:
3535 case SKD_DRVR_STATE_SYNCING
:
3537 case SKD_DRVR_STATE_FAULT
:
3539 case SKD_DRVR_STATE_DISAPPEARED
:
3540 return "DISAPPEARED";
3541 case SKD_DRVR_STATE_BUSY_ERASE
:
3542 return "BUSY_ERASE";
3543 case SKD_DRVR_STATE_BUSY_SANITIZE
:
3544 return "BUSY_SANITIZE";
3545 case SKD_DRVR_STATE_BUSY_IMMINENT
:
3546 return "BUSY_IMMINENT";
3547 case SKD_DRVR_STATE_WAIT_BOOT
:
3555 static const char *skd_skreq_state_to_str(enum skd_req_state state
)
3558 case SKD_REQ_STATE_IDLE
:
3560 case SKD_REQ_STATE_SETUP
:
3562 case SKD_REQ_STATE_BUSY
:
3564 case SKD_REQ_STATE_COMPLETED
:
3566 case SKD_REQ_STATE_TIMEOUT
:
3573 static void skd_log_skdev(struct skd_device
*skdev
, const char *event
)
3575 dev_dbg(&skdev
->pdev
->dev
, "skdev=%p event='%s'\n", skdev
, event
);
3576 dev_dbg(&skdev
->pdev
->dev
, " drive_state=%s(%d) driver_state=%s(%d)\n",
3577 skd_drive_state_to_str(skdev
->drive_state
), skdev
->drive_state
,
3578 skd_skdev_state_to_str(skdev
->state
), skdev
->state
);
3579 dev_dbg(&skdev
->pdev
->dev
, " busy=%d limit=%d dev=%d lowat=%d\n",
3580 skd_in_flight(skdev
), skdev
->cur_max_queue_depth
,
3581 skdev
->dev_max_queue_depth
, skdev
->queue_low_water_mark
);
3582 dev_dbg(&skdev
->pdev
->dev
, " cycle=%d cycle_ix=%d\n",
3583 skdev
->skcomp_cycle
, skdev
->skcomp_ix
);
3586 static void skd_log_skreq(struct skd_device
*skdev
,
3587 struct skd_request_context
*skreq
, const char *event
)
3589 struct request
*req
= blk_mq_rq_from_pdu(skreq
);
3590 u32 lba
= blk_rq_pos(req
);
3591 u32 count
= blk_rq_sectors(req
);
3593 dev_dbg(&skdev
->pdev
->dev
, "skreq=%p event='%s'\n", skreq
, event
);
3594 dev_dbg(&skdev
->pdev
->dev
, " state=%s(%d) id=0x%04x fitmsg=0x%04x\n",
3595 skd_skreq_state_to_str(skreq
->state
), skreq
->state
, skreq
->id
,
3597 dev_dbg(&skdev
->pdev
->dev
, " sg_dir=%d n_sg=%d\n",
3598 skreq
->data_dir
, skreq
->n_sg
);
3600 dev_dbg(&skdev
->pdev
->dev
,
3601 "req=%p lba=%u(0x%x) count=%u(0x%x) dir=%d\n", req
, lba
, lba
,
3602 count
, count
, (int)rq_data_dir(req
));
3606 *****************************************************************************
3608 *****************************************************************************
3611 static int __init
skd_init(void)
3613 BUILD_BUG_ON(sizeof(struct fit_completion_entry_v1
) != 8);
3614 BUILD_BUG_ON(sizeof(struct fit_comp_error_info
) != 32);
3615 BUILD_BUG_ON(sizeof(struct skd_command_header
) != 16);
3616 BUILD_BUG_ON(sizeof(struct skd_scsi_request
) != 32);
3617 BUILD_BUG_ON(sizeof(struct driver_inquiry_data
) != 44);
3618 BUILD_BUG_ON(offsetof(struct skd_msg_buf
, fmh
) != 0);
3619 BUILD_BUG_ON(offsetof(struct skd_msg_buf
, scsi
) != 64);
3620 BUILD_BUG_ON(sizeof(struct skd_msg_buf
) != SKD_N_FITMSG_BYTES
);
3622 switch (skd_isr_type
) {
3623 case SKD_IRQ_LEGACY
:
3628 pr_err(PFX
"skd_isr_type %d invalid, re-set to %d\n",
3629 skd_isr_type
, SKD_IRQ_DEFAULT
);
3630 skd_isr_type
= SKD_IRQ_DEFAULT
;
3633 if (skd_max_queue_depth
< 1 ||
3634 skd_max_queue_depth
> SKD_MAX_QUEUE_DEPTH
) {
3635 pr_err(PFX
"skd_max_queue_depth %d invalid, re-set to %d\n",
3636 skd_max_queue_depth
, SKD_MAX_QUEUE_DEPTH_DEFAULT
);
3637 skd_max_queue_depth
= SKD_MAX_QUEUE_DEPTH_DEFAULT
;
3640 if (skd_max_req_per_msg
< 1 ||
3641 skd_max_req_per_msg
> SKD_MAX_REQ_PER_MSG
) {
3642 pr_err(PFX
"skd_max_req_per_msg %d invalid, re-set to %d\n",
3643 skd_max_req_per_msg
, SKD_MAX_REQ_PER_MSG_DEFAULT
);
3644 skd_max_req_per_msg
= SKD_MAX_REQ_PER_MSG_DEFAULT
;
3647 if (skd_sgs_per_request
< 1 || skd_sgs_per_request
> 4096) {
3648 pr_err(PFX
"skd_sg_per_request %d invalid, re-set to %d\n",
3649 skd_sgs_per_request
, SKD_N_SG_PER_REQ_DEFAULT
);
3650 skd_sgs_per_request
= SKD_N_SG_PER_REQ_DEFAULT
;
3653 if (skd_dbg_level
< 0 || skd_dbg_level
> 2) {
3654 pr_err(PFX
"skd_dbg_level %d invalid, re-set to %d\n",
3659 if (skd_isr_comp_limit
< 0) {
3660 pr_err(PFX
"skd_isr_comp_limit %d invalid, set to %d\n",
3661 skd_isr_comp_limit
, 0);
3662 skd_isr_comp_limit
= 0;
3665 return pci_register_driver(&skd_driver
);
3668 static void __exit
skd_exit(void)
3670 pci_unregister_driver(&skd_driver
);
3673 unregister_blkdev(skd_major
, DRV_NAME
);
3676 module_init(skd_init
);
3677 module_exit(skd_exit
);