2 * Copyright (C) 2005 - 2011 Emulex
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
10 * Written by: Jayamohan Kallickal (jayamohan.kallickal@emulex.com)
12 * Contact Information:
13 * linux-drivers@emulex.com
17 * Costa Mesa, CA 92626
20 #include <linux/reboot.h>
21 #include <linux/delay.h>
22 #include <linux/slab.h>
23 #include <linux/interrupt.h>
24 #include <linux/blkdev.h>
25 #include <linux/pci.h>
26 #include <linux/string.h>
27 #include <linux/kernel.h>
28 #include <linux/semaphore.h>
29 #include <linux/iscsi_boot_sysfs.h>
31 #include <scsi/libiscsi.h>
32 #include <scsi/scsi_transport_iscsi.h>
33 #include <scsi/scsi_transport.h>
34 #include <scsi/scsi_cmnd.h>
35 #include <scsi/scsi_device.h>
36 #include <scsi/scsi_host.h>
37 #include <scsi/scsi.h>
42 static unsigned int be_iopoll_budget
= 10;
43 static unsigned int be_max_phys_size
= 64;
44 static unsigned int enable_msix
= 1;
45 static unsigned int gcrashmode
= 0;
46 static unsigned int num_hba
= 0;
48 MODULE_DEVICE_TABLE(pci
, beiscsi_pci_id_table
);
49 MODULE_DESCRIPTION(DRV_DESC
" " BUILD_STR
);
50 MODULE_AUTHOR("ServerEngines Corporation");
51 MODULE_LICENSE("GPL");
52 module_param(be_iopoll_budget
, int, 0);
53 module_param(enable_msix
, int, 0);
54 module_param(be_max_phys_size
, uint
, S_IRUGO
);
55 MODULE_PARM_DESC(be_max_phys_size
, "Maximum Size (In Kilobytes) of physically"
56 "contiguous memory that can be allocated."
59 static int beiscsi_slave_configure(struct scsi_device
*sdev
)
61 blk_queue_max_segment_size(sdev
->request_queue
, 65536);
65 static int beiscsi_eh_abort(struct scsi_cmnd
*sc
)
67 struct iscsi_cls_session
*cls_session
;
68 struct iscsi_task
*aborted_task
= (struct iscsi_task
*)sc
->SCp
.ptr
;
69 struct beiscsi_io_task
*aborted_io_task
;
70 struct iscsi_conn
*conn
;
71 struct beiscsi_conn
*beiscsi_conn
;
72 struct beiscsi_hba
*phba
;
73 struct iscsi_session
*session
;
74 struct invalidate_command_table
*inv_tbl
;
75 struct be_dma_mem nonemb_cmd
;
76 unsigned int cid
, tag
, num_invalidate
;
78 cls_session
= starget_to_session(scsi_target(sc
->device
));
79 session
= cls_session
->dd_data
;
81 spin_lock_bh(&session
->lock
);
82 if (!aborted_task
|| !aborted_task
->sc
) {
84 spin_unlock_bh(&session
->lock
);
88 aborted_io_task
= aborted_task
->dd_data
;
89 if (!aborted_io_task
->scsi_cmnd
) {
90 /* raced or invalid command */
91 spin_unlock_bh(&session
->lock
);
94 spin_unlock_bh(&session
->lock
);
95 conn
= aborted_task
->conn
;
96 beiscsi_conn
= conn
->dd_data
;
97 phba
= beiscsi_conn
->phba
;
100 cid
= beiscsi_conn
->beiscsi_conn_cid
;
101 inv_tbl
= phba
->inv_tbl
;
102 memset(inv_tbl
, 0x0, sizeof(*inv_tbl
));
104 inv_tbl
->icd
= aborted_io_task
->psgl_handle
->sgl_index
;
106 nonemb_cmd
.va
= pci_alloc_consistent(phba
->ctrl
.pdev
,
107 sizeof(struct invalidate_commands_params_in
),
109 if (nonemb_cmd
.va
== NULL
) {
111 "Failed to allocate memory for"
112 "mgmt_invalidate_icds\n");
115 nonemb_cmd
.size
= sizeof(struct invalidate_commands_params_in
);
117 tag
= mgmt_invalidate_icds(phba
, inv_tbl
, num_invalidate
,
120 shost_printk(KERN_WARNING
, phba
->shost
,
121 "mgmt_invalidate_icds could not be"
123 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
124 nonemb_cmd
.va
, nonemb_cmd
.dma
);
128 wait_event_interruptible(phba
->ctrl
.mcc_wait
[tag
],
129 phba
->ctrl
.mcc_numtag
[tag
]);
130 free_mcc_tag(&phba
->ctrl
, tag
);
132 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
133 nonemb_cmd
.va
, nonemb_cmd
.dma
);
134 return iscsi_eh_abort(sc
);
137 static int beiscsi_eh_device_reset(struct scsi_cmnd
*sc
)
139 struct iscsi_task
*abrt_task
;
140 struct beiscsi_io_task
*abrt_io_task
;
141 struct iscsi_conn
*conn
;
142 struct beiscsi_conn
*beiscsi_conn
;
143 struct beiscsi_hba
*phba
;
144 struct iscsi_session
*session
;
145 struct iscsi_cls_session
*cls_session
;
146 struct invalidate_command_table
*inv_tbl
;
147 struct be_dma_mem nonemb_cmd
;
148 unsigned int cid
, tag
, i
, num_invalidate
;
151 /* invalidate iocbs */
152 cls_session
= starget_to_session(scsi_target(sc
->device
));
153 session
= cls_session
->dd_data
;
154 spin_lock_bh(&session
->lock
);
155 if (!session
->leadconn
|| session
->state
!= ISCSI_STATE_LOGGED_IN
)
158 conn
= session
->leadconn
;
159 beiscsi_conn
= conn
->dd_data
;
160 phba
= beiscsi_conn
->phba
;
161 cid
= beiscsi_conn
->beiscsi_conn_cid
;
162 inv_tbl
= phba
->inv_tbl
;
163 memset(inv_tbl
, 0x0, sizeof(*inv_tbl
) * BE2_CMDS_PER_CXN
);
165 for (i
= 0; i
< conn
->session
->cmds_max
; i
++) {
166 abrt_task
= conn
->session
->cmds
[i
];
167 abrt_io_task
= abrt_task
->dd_data
;
168 if (!abrt_task
->sc
|| abrt_task
->state
== ISCSI_TASK_FREE
)
171 if (abrt_task
->sc
->device
->lun
!= abrt_task
->sc
->device
->lun
)
175 inv_tbl
->icd
= abrt_io_task
->psgl_handle
->sgl_index
;
179 spin_unlock_bh(&session
->lock
);
180 inv_tbl
= phba
->inv_tbl
;
182 nonemb_cmd
.va
= pci_alloc_consistent(phba
->ctrl
.pdev
,
183 sizeof(struct invalidate_commands_params_in
),
185 if (nonemb_cmd
.va
== NULL
) {
187 "Failed to allocate memory for"
188 "mgmt_invalidate_icds\n");
191 nonemb_cmd
.size
= sizeof(struct invalidate_commands_params_in
);
192 memset(nonemb_cmd
.va
, 0, nonemb_cmd
.size
);
193 tag
= mgmt_invalidate_icds(phba
, inv_tbl
, num_invalidate
,
196 shost_printk(KERN_WARNING
, phba
->shost
,
197 "mgmt_invalidate_icds could not be"
199 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
200 nonemb_cmd
.va
, nonemb_cmd
.dma
);
203 wait_event_interruptible(phba
->ctrl
.mcc_wait
[tag
],
204 phba
->ctrl
.mcc_numtag
[tag
]);
205 free_mcc_tag(&phba
->ctrl
, tag
);
207 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
208 nonemb_cmd
.va
, nonemb_cmd
.dma
);
209 return iscsi_eh_device_reset(sc
);
211 spin_unlock_bh(&session
->lock
);
215 static ssize_t
beiscsi_show_boot_tgt_info(void *data
, int type
, char *buf
)
217 struct beiscsi_hba
*phba
= data
;
218 struct mgmt_session_info
*boot_sess
= &phba
->boot_sess
;
219 struct mgmt_conn_info
*boot_conn
= &boot_sess
->conn_list
[0];
224 case ISCSI_BOOT_TGT_NAME
:
225 rc
= sprintf(buf
, "%.*s\n",
226 (int)strlen(boot_sess
->target_name
),
227 (char *)&boot_sess
->target_name
);
229 case ISCSI_BOOT_TGT_IP_ADDR
:
230 if (boot_conn
->dest_ipaddr
.ip_type
== 0x1)
231 rc
= sprintf(buf
, "%pI4\n",
232 (char *)&boot_conn
->dest_ipaddr
.ip_address
);
234 rc
= sprintf(str
, "%pI6\n",
235 (char *)&boot_conn
->dest_ipaddr
.ip_address
);
237 case ISCSI_BOOT_TGT_PORT
:
238 rc
= sprintf(str
, "%d\n", boot_conn
->dest_port
);
241 case ISCSI_BOOT_TGT_CHAP_NAME
:
242 rc
= sprintf(str
, "%.*s\n",
243 boot_conn
->negotiated_login_options
.auth_data
.chap
.
244 target_chap_name_length
,
245 (char *)&boot_conn
->negotiated_login_options
.
246 auth_data
.chap
.target_chap_name
);
248 case ISCSI_BOOT_TGT_CHAP_SECRET
:
249 rc
= sprintf(str
, "%.*s\n",
250 boot_conn
->negotiated_login_options
.auth_data
.chap
.
251 target_secret_length
,
252 (char *)&boot_conn
->negotiated_login_options
.
253 auth_data
.chap
.target_secret
);
255 case ISCSI_BOOT_TGT_REV_CHAP_NAME
:
256 rc
= sprintf(str
, "%.*s\n",
257 boot_conn
->negotiated_login_options
.auth_data
.chap
.
258 intr_chap_name_length
,
259 (char *)&boot_conn
->negotiated_login_options
.
260 auth_data
.chap
.intr_chap_name
);
262 case ISCSI_BOOT_TGT_REV_CHAP_SECRET
:
263 rc
= sprintf(str
, "%.*s\n",
264 boot_conn
->negotiated_login_options
.auth_data
.chap
.
266 (char *)&boot_conn
->negotiated_login_options
.
267 auth_data
.chap
.intr_secret
);
269 case ISCSI_BOOT_TGT_FLAGS
:
270 rc
= sprintf(str
, "2\n");
272 case ISCSI_BOOT_TGT_NIC_ASSOC
:
273 rc
= sprintf(str
, "0\n");
282 static ssize_t
beiscsi_show_boot_ini_info(void *data
, int type
, char *buf
)
284 struct beiscsi_hba
*phba
= data
;
289 case ISCSI_BOOT_INI_INITIATOR_NAME
:
290 rc
= sprintf(str
, "%s\n", phba
->boot_sess
.initiator_iscsiname
);
299 static ssize_t
beiscsi_show_boot_eth_info(void *data
, int type
, char *buf
)
301 struct beiscsi_hba
*phba
= data
;
306 case ISCSI_BOOT_ETH_FLAGS
:
307 rc
= sprintf(str
, "2\n");
309 case ISCSI_BOOT_ETH_INDEX
:
310 rc
= sprintf(str
, "0\n");
312 case ISCSI_BOOT_ETH_MAC
:
313 rc
= beiscsi_get_macaddr(buf
, phba
);
315 SE_DEBUG(DBG_LVL_1
, "beiscsi_get_macaddr Failed\n");
327 static mode_t
beiscsi_tgt_get_attr_visibility(void *data
, int type
)
332 case ISCSI_BOOT_TGT_NAME
:
333 case ISCSI_BOOT_TGT_IP_ADDR
:
334 case ISCSI_BOOT_TGT_PORT
:
335 case ISCSI_BOOT_TGT_CHAP_NAME
:
336 case ISCSI_BOOT_TGT_CHAP_SECRET
:
337 case ISCSI_BOOT_TGT_REV_CHAP_NAME
:
338 case ISCSI_BOOT_TGT_REV_CHAP_SECRET
:
339 case ISCSI_BOOT_TGT_NIC_ASSOC
:
340 case ISCSI_BOOT_TGT_FLAGS
:
350 static mode_t
beiscsi_ini_get_attr_visibility(void *data
, int type
)
355 case ISCSI_BOOT_INI_INITIATOR_NAME
:
366 static mode_t
beiscsi_eth_get_attr_visibility(void *data
, int type
)
371 case ISCSI_BOOT_ETH_FLAGS
:
372 case ISCSI_BOOT_ETH_MAC
:
373 case ISCSI_BOOT_ETH_INDEX
:
383 /*------------------- PCI Driver operations and data ----------------- */
384 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table
) = {
385 { PCI_DEVICE(BE_VENDOR_ID
, BE_DEVICE_ID1
) },
386 { PCI_DEVICE(BE_VENDOR_ID
, BE_DEVICE_ID2
) },
387 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID1
) },
388 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID2
) },
389 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID3
) },
392 MODULE_DEVICE_TABLE(pci
, beiscsi_pci_id_table
);
394 static struct scsi_host_template beiscsi_sht
= {
395 .module
= THIS_MODULE
,
396 .name
= "ServerEngines 10Gbe open-iscsi Initiator Driver",
397 .proc_name
= DRV_NAME
,
398 .queuecommand
= iscsi_queuecommand
,
399 .change_queue_depth
= iscsi_change_queue_depth
,
400 .slave_configure
= beiscsi_slave_configure
,
401 .target_alloc
= iscsi_target_alloc
,
402 .eh_abort_handler
= beiscsi_eh_abort
,
403 .eh_device_reset_handler
= beiscsi_eh_device_reset
,
404 .eh_target_reset_handler
= iscsi_eh_session_reset
,
405 .sg_tablesize
= BEISCSI_SGLIST_ELEMENTS
,
406 .can_queue
= BE2_IO_DEPTH
,
408 .max_sectors
= BEISCSI_MAX_SECTORS
,
409 .cmd_per_lun
= BEISCSI_CMD_PER_LUN
,
410 .use_clustering
= ENABLE_CLUSTERING
,
413 static struct scsi_transport_template
*beiscsi_scsi_transport
;
415 static struct beiscsi_hba
*beiscsi_hba_alloc(struct pci_dev
*pcidev
)
417 struct beiscsi_hba
*phba
;
418 struct Scsi_Host
*shost
;
420 shost
= iscsi_host_alloc(&beiscsi_sht
, sizeof(*phba
), 0);
422 dev_err(&pcidev
->dev
, "beiscsi_hba_alloc -"
423 "iscsi_host_alloc failed\n");
426 shost
->dma_boundary
= pcidev
->dma_mask
;
427 shost
->max_id
= BE2_MAX_SESSIONS
;
428 shost
->max_channel
= 0;
429 shost
->max_cmd_len
= BEISCSI_MAX_CMD_LEN
;
430 shost
->max_lun
= BEISCSI_NUM_MAX_LUN
;
431 shost
->transportt
= beiscsi_scsi_transport
;
432 phba
= iscsi_host_priv(shost
);
433 memset(phba
, 0, sizeof(*phba
));
435 phba
->pcidev
= pci_dev_get(pcidev
);
436 pci_set_drvdata(pcidev
, phba
);
438 if (iscsi_host_add(shost
, &phba
->pcidev
->dev
))
444 pci_dev_put(phba
->pcidev
);
445 iscsi_host_free(phba
->shost
);
449 static void beiscsi_unmap_pci_function(struct beiscsi_hba
*phba
)
452 iounmap(phba
->csr_va
);
456 iounmap(phba
->db_va
);
460 iounmap(phba
->pci_va
);
465 static int beiscsi_map_pci_bars(struct beiscsi_hba
*phba
,
466 struct pci_dev
*pcidev
)
471 addr
= ioremap_nocache(pci_resource_start(pcidev
, 2),
472 pci_resource_len(pcidev
, 2));
475 phba
->ctrl
.csr
= addr
;
477 phba
->csr_pa
.u
.a64
.address
= pci_resource_start(pcidev
, 2);
479 addr
= ioremap_nocache(pci_resource_start(pcidev
, 4), 128 * 1024);
482 phba
->ctrl
.db
= addr
;
484 phba
->db_pa
.u
.a64
.address
= pci_resource_start(pcidev
, 4);
486 if (phba
->generation
== BE_GEN2
)
491 addr
= ioremap_nocache(pci_resource_start(pcidev
, pcicfg_reg
),
492 pci_resource_len(pcidev
, pcicfg_reg
));
496 phba
->ctrl
.pcicfg
= addr
;
498 phba
->pci_pa
.u
.a64
.address
= pci_resource_start(pcidev
, pcicfg_reg
);
502 beiscsi_unmap_pci_function(phba
);
506 static int beiscsi_enable_pci(struct pci_dev
*pcidev
)
510 ret
= pci_enable_device(pcidev
);
512 dev_err(&pcidev
->dev
, "beiscsi_enable_pci - enable device "
513 "failed. Returning -ENODEV\n");
517 pci_set_master(pcidev
);
518 if (pci_set_consistent_dma_mask(pcidev
, DMA_BIT_MASK(64))) {
519 ret
= pci_set_consistent_dma_mask(pcidev
, DMA_BIT_MASK(32));
521 dev_err(&pcidev
->dev
, "Could not set PCI DMA Mask\n");
522 pci_disable_device(pcidev
);
529 static int be_ctrl_init(struct beiscsi_hba
*phba
, struct pci_dev
*pdev
)
531 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
532 struct be_dma_mem
*mbox_mem_alloc
= &ctrl
->mbox_mem_alloced
;
533 struct be_dma_mem
*mbox_mem_align
= &ctrl
->mbox_mem
;
537 status
= beiscsi_map_pci_bars(phba
, pdev
);
540 mbox_mem_alloc
->size
= sizeof(struct be_mcc_mailbox
) + 16;
541 mbox_mem_alloc
->va
= pci_alloc_consistent(pdev
,
542 mbox_mem_alloc
->size
,
543 &mbox_mem_alloc
->dma
);
544 if (!mbox_mem_alloc
->va
) {
545 beiscsi_unmap_pci_function(phba
);
550 mbox_mem_align
->size
= sizeof(struct be_mcc_mailbox
);
551 mbox_mem_align
->va
= PTR_ALIGN(mbox_mem_alloc
->va
, 16);
552 mbox_mem_align
->dma
= PTR_ALIGN(mbox_mem_alloc
->dma
, 16);
553 memset(mbox_mem_align
->va
, 0, sizeof(struct be_mcc_mailbox
));
554 spin_lock_init(&ctrl
->mbox_lock
);
555 spin_lock_init(&phba
->ctrl
.mcc_lock
);
556 spin_lock_init(&phba
->ctrl
.mcc_cq_lock
);
561 static void beiscsi_get_params(struct beiscsi_hba
*phba
)
563 phba
->params
.ios_per_ctrl
= (phba
->fw_config
.iscsi_icd_count
564 - (phba
->fw_config
.iscsi_cid_count
567 phba
->params
.cxns_per_ctrl
= phba
->fw_config
.iscsi_cid_count
;
568 phba
->params
.asyncpdus_per_ctrl
= phba
->fw_config
.iscsi_cid_count
* 2;
569 phba
->params
.icds_per_ctrl
= phba
->fw_config
.iscsi_icd_count
;
570 phba
->params
.num_sge_per_io
= BE2_SGE
;
571 phba
->params
.defpdu_hdr_sz
= BE2_DEFPDU_HDR_SZ
;
572 phba
->params
.defpdu_data_sz
= BE2_DEFPDU_DATA_SZ
;
573 phba
->params
.eq_timer
= 64;
574 phba
->params
.num_eq_entries
=
575 (((BE2_CMDS_PER_CXN
* 2 + phba
->fw_config
.iscsi_cid_count
* 2
576 + BE2_TMFS
) / 512) + 1) * 512;
577 phba
->params
.num_eq_entries
= (phba
->params
.num_eq_entries
< 1024)
578 ? 1024 : phba
->params
.num_eq_entries
;
579 SE_DEBUG(DBG_LVL_8
, "phba->params.num_eq_entries=%d\n",
580 phba
->params
.num_eq_entries
);
581 phba
->params
.num_cq_entries
=
582 (((BE2_CMDS_PER_CXN
* 2 + phba
->fw_config
.iscsi_cid_count
* 2
583 + BE2_TMFS
) / 512) + 1) * 512;
584 phba
->params
.wrbs_per_cxn
= 256;
587 static void hwi_ring_eq_db(struct beiscsi_hba
*phba
,
588 unsigned int id
, unsigned int clr_interrupt
,
589 unsigned int num_processed
,
590 unsigned char rearm
, unsigned char event
)
593 val
|= id
& DB_EQ_RING_ID_MASK
;
595 val
|= 1 << DB_EQ_REARM_SHIFT
;
597 val
|= 1 << DB_EQ_CLR_SHIFT
;
599 val
|= 1 << DB_EQ_EVNT_SHIFT
;
600 val
|= num_processed
<< DB_EQ_NUM_POPPED_SHIFT
;
601 iowrite32(val
, phba
->db_va
+ DB_EQ_OFFSET
);
605 * be_isr_mcc - The isr routine of the driver.
607 * @dev_id: Pointer to host adapter structure
609 static irqreturn_t
be_isr_mcc(int irq
, void *dev_id
)
611 struct beiscsi_hba
*phba
;
612 struct be_eq_entry
*eqe
= NULL
;
613 struct be_queue_info
*eq
;
614 struct be_queue_info
*mcc
;
615 unsigned int num_eq_processed
;
616 struct be_eq_obj
*pbe_eq
;
622 mcc
= &phba
->ctrl
.mcc_obj
.cq
;
623 eqe
= queue_tail_node(eq
);
625 SE_DEBUG(DBG_LVL_1
, "eqe is NULL\n");
627 num_eq_processed
= 0;
629 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
631 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
633 EQE_RESID_MASK
) >> 16) == mcc
->id
) {
634 spin_lock_irqsave(&phba
->isr_lock
, flags
);
635 phba
->todo_mcc_cq
= 1;
636 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
638 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
640 eqe
= queue_tail_node(eq
);
643 if (phba
->todo_mcc_cq
)
644 queue_work(phba
->wq
, &phba
->work_cqs
);
645 if (num_eq_processed
)
646 hwi_ring_eq_db(phba
, eq
->id
, 1, num_eq_processed
, 1, 1);
652 * be_isr_msix - The isr routine of the driver.
654 * @dev_id: Pointer to host adapter structure
656 static irqreturn_t
be_isr_msix(int irq
, void *dev_id
)
658 struct beiscsi_hba
*phba
;
659 struct be_eq_entry
*eqe
= NULL
;
660 struct be_queue_info
*eq
;
661 struct be_queue_info
*cq
;
662 unsigned int num_eq_processed
;
663 struct be_eq_obj
*pbe_eq
;
669 eqe
= queue_tail_node(eq
);
671 SE_DEBUG(DBG_LVL_1
, "eqe is NULL\n");
674 num_eq_processed
= 0;
675 if (blk_iopoll_enabled
) {
676 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
678 if (!blk_iopoll_sched_prep(&pbe_eq
->iopoll
))
679 blk_iopoll_sched(&pbe_eq
->iopoll
);
681 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
683 eqe
= queue_tail_node(eq
);
686 if (num_eq_processed
)
687 hwi_ring_eq_db(phba
, eq
->id
, 1, num_eq_processed
, 0, 1);
691 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
693 spin_lock_irqsave(&phba
->isr_lock
, flags
);
695 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
696 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
698 eqe
= queue_tail_node(eq
);
702 queue_work(phba
->wq
, &phba
->work_cqs
);
704 if (num_eq_processed
)
705 hwi_ring_eq_db(phba
, eq
->id
, 1, num_eq_processed
, 1, 1);
712 * be_isr - The isr routine of the driver.
714 * @dev_id: Pointer to host adapter structure
716 static irqreturn_t
be_isr(int irq
, void *dev_id
)
718 struct beiscsi_hba
*phba
;
719 struct hwi_controller
*phwi_ctrlr
;
720 struct hwi_context_memory
*phwi_context
;
721 struct be_eq_entry
*eqe
= NULL
;
722 struct be_queue_info
*eq
;
723 struct be_queue_info
*cq
;
724 struct be_queue_info
*mcc
;
725 unsigned long flags
, index
;
726 unsigned int num_mcceq_processed
, num_ioeq_processed
;
727 struct be_ctrl_info
*ctrl
;
728 struct be_eq_obj
*pbe_eq
;
733 isr
= ioread32(ctrl
->csr
+ CEV_ISR0_OFFSET
+
734 (PCI_FUNC(ctrl
->pdev
->devfn
) * CEV_ISR_SIZE
));
738 phwi_ctrlr
= phba
->phwi_ctrlr
;
739 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
740 pbe_eq
= &phwi_context
->be_eq
[0];
742 eq
= &phwi_context
->be_eq
[0].q
;
743 mcc
= &phba
->ctrl
.mcc_obj
.cq
;
745 eqe
= queue_tail_node(eq
);
747 SE_DEBUG(DBG_LVL_1
, "eqe is NULL\n");
749 num_ioeq_processed
= 0;
750 num_mcceq_processed
= 0;
751 if (blk_iopoll_enabled
) {
752 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
754 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
756 EQE_RESID_MASK
) >> 16) == mcc
->id
) {
757 spin_lock_irqsave(&phba
->isr_lock
, flags
);
758 phba
->todo_mcc_cq
= 1;
759 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
760 num_mcceq_processed
++;
762 if (!blk_iopoll_sched_prep(&pbe_eq
->iopoll
))
763 blk_iopoll_sched(&pbe_eq
->iopoll
);
764 num_ioeq_processed
++;
766 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
768 eqe
= queue_tail_node(eq
);
770 if (num_ioeq_processed
|| num_mcceq_processed
) {
771 if (phba
->todo_mcc_cq
)
772 queue_work(phba
->wq
, &phba
->work_cqs
);
774 if ((num_mcceq_processed
) && (!num_ioeq_processed
))
775 hwi_ring_eq_db(phba
, eq
->id
, 0,
776 (num_ioeq_processed
+
777 num_mcceq_processed
) , 1, 1);
779 hwi_ring_eq_db(phba
, eq
->id
, 0,
780 (num_ioeq_processed
+
781 num_mcceq_processed
), 0, 1);
787 cq
= &phwi_context
->be_cq
[0];
788 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
791 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
793 EQE_RESID_MASK
) >> 16) != cq
->id
) {
794 spin_lock_irqsave(&phba
->isr_lock
, flags
);
795 phba
->todo_mcc_cq
= 1;
796 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
798 spin_lock_irqsave(&phba
->isr_lock
, flags
);
800 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
802 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
804 eqe
= queue_tail_node(eq
);
805 num_ioeq_processed
++;
807 if (phba
->todo_cq
|| phba
->todo_mcc_cq
)
808 queue_work(phba
->wq
, &phba
->work_cqs
);
810 if (num_ioeq_processed
) {
811 hwi_ring_eq_db(phba
, eq
->id
, 0,
812 num_ioeq_processed
, 1, 1);
819 static int beiscsi_init_irqs(struct beiscsi_hba
*phba
)
821 struct pci_dev
*pcidev
= phba
->pcidev
;
822 struct hwi_controller
*phwi_ctrlr
;
823 struct hwi_context_memory
*phwi_context
;
824 int ret
, msix_vec
, i
, j
;
826 phwi_ctrlr
= phba
->phwi_ctrlr
;
827 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
829 if (phba
->msix_enabled
) {
830 for (i
= 0; i
< phba
->num_cpus
; i
++) {
831 phba
->msi_name
[i
] = kzalloc(BEISCSI_MSI_NAME
,
833 if (!phba
->msi_name
[i
]) {
838 sprintf(phba
->msi_name
[i
], "beiscsi_%02x_%02x",
839 phba
->shost
->host_no
, i
);
840 msix_vec
= phba
->msix_entries
[i
].vector
;
841 ret
= request_irq(msix_vec
, be_isr_msix
, 0,
843 &phwi_context
->be_eq
[i
]);
845 shost_printk(KERN_ERR
, phba
->shost
,
846 "beiscsi_init_irqs-Failed to"
847 "register msix for i = %d\n", i
);
848 kfree(phba
->msi_name
[i
]);
852 phba
->msi_name
[i
] = kzalloc(BEISCSI_MSI_NAME
, GFP_KERNEL
);
853 if (!phba
->msi_name
[i
]) {
857 sprintf(phba
->msi_name
[i
], "beiscsi_mcc_%02x",
858 phba
->shost
->host_no
);
859 msix_vec
= phba
->msix_entries
[i
].vector
;
860 ret
= request_irq(msix_vec
, be_isr_mcc
, 0, phba
->msi_name
[i
],
861 &phwi_context
->be_eq
[i
]);
863 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_init_irqs-"
864 "Failed to register beiscsi_msix_mcc\n");
865 kfree(phba
->msi_name
[i
]);
870 ret
= request_irq(pcidev
->irq
, be_isr
, IRQF_SHARED
,
873 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_init_irqs-"
874 "Failed to register irq\\n");
880 for (j
= i
- 1; j
>= 0; j
--) {
881 kfree(phba
->msi_name
[j
]);
882 msix_vec
= phba
->msix_entries
[j
].vector
;
883 free_irq(msix_vec
, &phwi_context
->be_eq
[j
]);
888 static void hwi_ring_cq_db(struct beiscsi_hba
*phba
,
889 unsigned int id
, unsigned int num_processed
,
890 unsigned char rearm
, unsigned char event
)
893 val
|= id
& DB_CQ_RING_ID_MASK
;
895 val
|= 1 << DB_CQ_REARM_SHIFT
;
896 val
|= num_processed
<< DB_CQ_NUM_POPPED_SHIFT
;
897 iowrite32(val
, phba
->db_va
+ DB_CQ_OFFSET
);
901 beiscsi_process_async_pdu(struct beiscsi_conn
*beiscsi_conn
,
902 struct beiscsi_hba
*phba
,
904 struct pdu_base
*ppdu
,
905 unsigned long pdu_len
,
906 void *pbuffer
, unsigned long buf_len
)
908 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
909 struct iscsi_session
*session
= conn
->session
;
910 struct iscsi_task
*task
;
911 struct beiscsi_io_task
*io_task
;
912 struct iscsi_hdr
*login_hdr
;
914 switch (ppdu
->dw
[offsetof(struct amap_pdu_base
, opcode
) / 32] &
915 PDUBASE_OPCODE_MASK
) {
916 case ISCSI_OP_NOOP_IN
:
920 case ISCSI_OP_ASYNC_EVENT
:
922 case ISCSI_OP_REJECT
:
924 WARN_ON(!(buf_len
== 48));
925 SE_DEBUG(DBG_LVL_1
, "In ISCSI_OP_REJECT\n");
927 case ISCSI_OP_LOGIN_RSP
:
928 case ISCSI_OP_TEXT_RSP
:
929 task
= conn
->login_task
;
930 io_task
= task
->dd_data
;
931 login_hdr
= (struct iscsi_hdr
*)ppdu
;
932 login_hdr
->itt
= io_task
->libiscsi_itt
;
935 shost_printk(KERN_WARNING
, phba
->shost
,
936 "Unrecognized opcode 0x%x in async msg\n",
938 dw
[offsetof(struct amap_pdu_base
, opcode
) / 32]
939 & PDUBASE_OPCODE_MASK
));
943 spin_lock_bh(&session
->lock
);
944 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)ppdu
, pbuffer
, buf_len
);
945 spin_unlock_bh(&session
->lock
);
949 static struct sgl_handle
*alloc_io_sgl_handle(struct beiscsi_hba
*phba
)
951 struct sgl_handle
*psgl_handle
;
953 if (phba
->io_sgl_hndl_avbl
) {
955 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
956 phba
->io_sgl_alloc_index
);
957 psgl_handle
= phba
->io_sgl_hndl_base
[phba
->
959 phba
->io_sgl_hndl_base
[phba
->io_sgl_alloc_index
] = NULL
;
960 phba
->io_sgl_hndl_avbl
--;
961 if (phba
->io_sgl_alloc_index
== (phba
->params
.
963 phba
->io_sgl_alloc_index
= 0;
965 phba
->io_sgl_alloc_index
++;
972 free_io_sgl_handle(struct beiscsi_hba
*phba
, struct sgl_handle
*psgl_handle
)
974 SE_DEBUG(DBG_LVL_8
, "In free_,io_sgl_free_index=%d\n",
975 phba
->io_sgl_free_index
);
976 if (phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
]) {
978 * this can happen if clean_task is called on a task that
979 * failed in xmit_task or alloc_pdu.
982 "Double Free in IO SGL io_sgl_free_index=%d,"
983 "value there=%p\n", phba
->io_sgl_free_index
,
984 phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
]);
987 phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
] = psgl_handle
;
988 phba
->io_sgl_hndl_avbl
++;
989 if (phba
->io_sgl_free_index
== (phba
->params
.ios_per_ctrl
- 1))
990 phba
->io_sgl_free_index
= 0;
992 phba
->io_sgl_free_index
++;
996 * alloc_wrb_handle - To allocate a wrb handle
997 * @phba: The hba pointer
998 * @cid: The cid to use for allocation
1000 * This happens under session_lock until submission to chip
1002 struct wrb_handle
*alloc_wrb_handle(struct beiscsi_hba
*phba
, unsigned int cid
)
1004 struct hwi_wrb_context
*pwrb_context
;
1005 struct hwi_controller
*phwi_ctrlr
;
1006 struct wrb_handle
*pwrb_handle
, *pwrb_handle_tmp
;
1008 phwi_ctrlr
= phba
->phwi_ctrlr
;
1009 pwrb_context
= &phwi_ctrlr
->wrb_context
[cid
];
1010 if (pwrb_context
->wrb_handles_available
>= 2) {
1011 pwrb_handle
= pwrb_context
->pwrb_handle_base
[
1012 pwrb_context
->alloc_index
];
1013 pwrb_context
->wrb_handles_available
--;
1014 if (pwrb_context
->alloc_index
==
1015 (phba
->params
.wrbs_per_cxn
- 1))
1016 pwrb_context
->alloc_index
= 0;
1018 pwrb_context
->alloc_index
++;
1019 pwrb_handle_tmp
= pwrb_context
->pwrb_handle_base
[
1020 pwrb_context
->alloc_index
];
1021 pwrb_handle
->nxt_wrb_index
= pwrb_handle_tmp
->wrb_index
;
1028 * free_wrb_handle - To free the wrb handle back to pool
1029 * @phba: The hba pointer
1030 * @pwrb_context: The context to free from
1031 * @pwrb_handle: The wrb_handle to free
1033 * This happens under session_lock until submission to chip
1036 free_wrb_handle(struct beiscsi_hba
*phba
, struct hwi_wrb_context
*pwrb_context
,
1037 struct wrb_handle
*pwrb_handle
)
1039 pwrb_context
->pwrb_handle_base
[pwrb_context
->free_index
] = pwrb_handle
;
1040 pwrb_context
->wrb_handles_available
++;
1041 if (pwrb_context
->free_index
== (phba
->params
.wrbs_per_cxn
- 1))
1042 pwrb_context
->free_index
= 0;
1044 pwrb_context
->free_index
++;
1047 "FREE WRB: pwrb_handle=%p free_index=0x%x"
1048 "wrb_handles_available=%d\n",
1049 pwrb_handle
, pwrb_context
->free_index
,
1050 pwrb_context
->wrb_handles_available
);
1053 static struct sgl_handle
*alloc_mgmt_sgl_handle(struct beiscsi_hba
*phba
)
1055 struct sgl_handle
*psgl_handle
;
1057 if (phba
->eh_sgl_hndl_avbl
) {
1058 psgl_handle
= phba
->eh_sgl_hndl_base
[phba
->eh_sgl_alloc_index
];
1059 phba
->eh_sgl_hndl_base
[phba
->eh_sgl_alloc_index
] = NULL
;
1060 SE_DEBUG(DBG_LVL_8
, "mgmt_sgl_alloc_index=%d=0x%x\n",
1061 phba
->eh_sgl_alloc_index
, phba
->eh_sgl_alloc_index
);
1062 phba
->eh_sgl_hndl_avbl
--;
1063 if (phba
->eh_sgl_alloc_index
==
1064 (phba
->params
.icds_per_ctrl
- phba
->params
.ios_per_ctrl
-
1066 phba
->eh_sgl_alloc_index
= 0;
1068 phba
->eh_sgl_alloc_index
++;
1075 free_mgmt_sgl_handle(struct beiscsi_hba
*phba
, struct sgl_handle
*psgl_handle
)
1078 SE_DEBUG(DBG_LVL_8
, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
1079 phba
->eh_sgl_free_index
);
1080 if (phba
->eh_sgl_hndl_base
[phba
->eh_sgl_free_index
]) {
1082 * this can happen if clean_task is called on a task that
1083 * failed in xmit_task or alloc_pdu.
1086 "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
1087 phba
->eh_sgl_free_index
);
1090 phba
->eh_sgl_hndl_base
[phba
->eh_sgl_free_index
] = psgl_handle
;
1091 phba
->eh_sgl_hndl_avbl
++;
1092 if (phba
->eh_sgl_free_index
==
1093 (phba
->params
.icds_per_ctrl
- phba
->params
.ios_per_ctrl
- 1))
1094 phba
->eh_sgl_free_index
= 0;
1096 phba
->eh_sgl_free_index
++;
1100 be_complete_io(struct beiscsi_conn
*beiscsi_conn
,
1101 struct iscsi_task
*task
, struct sol_cqe
*psol
)
1103 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1104 struct be_status_bhs
*sts_bhs
=
1105 (struct be_status_bhs
*)io_task
->cmd_bhs
;
1106 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1107 unsigned int sense_len
;
1108 unsigned char *sense
;
1109 u32 resid
= 0, exp_cmdsn
, max_cmdsn
;
1110 u8 rsp
, status
, flags
;
1113 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
1114 & SOL_EXP_CMD_SN_MASK
);
1115 max_cmdsn
= ((psol
->
1116 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
1117 & SOL_EXP_CMD_SN_MASK
) +
1118 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
1119 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
1120 rsp
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_resp
) / 32]
1121 & SOL_RESP_MASK
) >> 16);
1122 status
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_sts
) / 32]
1123 & SOL_STS_MASK
) >> 8);
1124 flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
1125 & SOL_FLAGS_MASK
) >> 24) | 0x80;
1127 task
->sc
->result
= (DID_OK
<< 16) | status
;
1128 if (rsp
!= ISCSI_STATUS_CMD_COMPLETED
) {
1129 task
->sc
->result
= DID_ERROR
<< 16;
1133 /* bidi not initially supported */
1134 if (flags
& (ISCSI_FLAG_CMD_UNDERFLOW
| ISCSI_FLAG_CMD_OVERFLOW
)) {
1135 resid
= (psol
->dw
[offsetof(struct amap_sol_cqe
, i_res_cnt
) /
1136 32] & SOL_RES_CNT_MASK
);
1138 if (!status
&& (flags
& ISCSI_FLAG_CMD_OVERFLOW
))
1139 task
->sc
->result
= DID_ERROR
<< 16;
1141 if (flags
& ISCSI_FLAG_CMD_UNDERFLOW
) {
1142 scsi_set_resid(task
->sc
, resid
);
1143 if (!status
&& (scsi_bufflen(task
->sc
) - resid
<
1144 task
->sc
->underflow
))
1145 task
->sc
->result
= DID_ERROR
<< 16;
1149 if (status
== SAM_STAT_CHECK_CONDITION
) {
1150 unsigned short *slen
= (unsigned short *)sts_bhs
->sense_info
;
1151 sense
= sts_bhs
->sense_info
+ sizeof(unsigned short);
1152 sense_len
= cpu_to_be16(*slen
);
1153 memcpy(task
->sc
->sense_buffer
, sense
,
1154 min_t(u16
, sense_len
, SCSI_SENSE_BUFFERSIZE
));
1157 if (io_task
->cmd_bhs
->iscsi_hdr
.flags
& ISCSI_FLAG_CMD_READ
) {
1158 if (psol
->dw
[offsetof(struct amap_sol_cqe
, i_res_cnt
) / 32]
1160 conn
->rxdata_octets
+= (psol
->
1161 dw
[offsetof(struct amap_sol_cqe
, i_res_cnt
) / 32]
1162 & SOL_RES_CNT_MASK
);
1165 scsi_dma_unmap(io_task
->scsi_cmnd
);
1166 iscsi_complete_scsi_task(task
, exp_cmdsn
, max_cmdsn
);
1170 be_complete_logout(struct beiscsi_conn
*beiscsi_conn
,
1171 struct iscsi_task
*task
, struct sol_cqe
*psol
)
1173 struct iscsi_logout_rsp
*hdr
;
1174 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1175 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1177 hdr
= (struct iscsi_logout_rsp
*)task
->hdr
;
1178 hdr
->opcode
= ISCSI_OP_LOGOUT_RSP
;
1181 hdr
->flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
1182 & SOL_FLAGS_MASK
) >> 24) | 0x80;
1183 hdr
->response
= (psol
->dw
[offsetof(struct amap_sol_cqe
, i_resp
) /
1184 32] & SOL_RESP_MASK
);
1185 hdr
->exp_cmdsn
= cpu_to_be32(psol
->
1186 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
1187 & SOL_EXP_CMD_SN_MASK
);
1188 hdr
->max_cmdsn
= be32_to_cpu((psol
->
1189 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
1190 & SOL_EXP_CMD_SN_MASK
) +
1191 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
1192 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
1193 hdr
->dlength
[0] = 0;
1194 hdr
->dlength
[1] = 0;
1195 hdr
->dlength
[2] = 0;
1197 hdr
->itt
= io_task
->libiscsi_itt
;
1198 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
1202 be_complete_tmf(struct beiscsi_conn
*beiscsi_conn
,
1203 struct iscsi_task
*task
, struct sol_cqe
*psol
)
1205 struct iscsi_tm_rsp
*hdr
;
1206 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1207 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1209 hdr
= (struct iscsi_tm_rsp
*)task
->hdr
;
1210 hdr
->opcode
= ISCSI_OP_SCSI_TMFUNC_RSP
;
1211 hdr
->flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
1212 & SOL_FLAGS_MASK
) >> 24) | 0x80;
1213 hdr
->response
= (psol
->dw
[offsetof(struct amap_sol_cqe
, i_resp
) /
1214 32] & SOL_RESP_MASK
);
1215 hdr
->exp_cmdsn
= cpu_to_be32(psol
->dw
[offsetof(struct amap_sol_cqe
,
1216 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
);
1217 hdr
->max_cmdsn
= be32_to_cpu((psol
->dw
[offsetof(struct amap_sol_cqe
,
1218 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
) +
1219 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
1220 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
1221 hdr
->itt
= io_task
->libiscsi_itt
;
1222 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
1226 hwi_complete_drvr_msgs(struct beiscsi_conn
*beiscsi_conn
,
1227 struct beiscsi_hba
*phba
, struct sol_cqe
*psol
)
1229 struct hwi_wrb_context
*pwrb_context
;
1230 struct wrb_handle
*pwrb_handle
= NULL
;
1231 struct hwi_controller
*phwi_ctrlr
;
1232 struct iscsi_task
*task
;
1233 struct beiscsi_io_task
*io_task
;
1234 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1235 struct iscsi_session
*session
= conn
->session
;
1237 phwi_ctrlr
= phba
->phwi_ctrlr
;
1238 pwrb_context
= &phwi_ctrlr
->wrb_context
[((psol
->
1239 dw
[offsetof(struct amap_sol_cqe
, cid
) / 32] &
1240 SOL_CID_MASK
) >> 6) -
1241 phba
->fw_config
.iscsi_cid_start
];
1242 pwrb_handle
= pwrb_context
->pwrb_handle_basestd
[((psol
->
1243 dw
[offsetof(struct amap_sol_cqe
, wrb_index
) /
1244 32] & SOL_WRB_INDEX_MASK
) >> 16)];
1245 task
= pwrb_handle
->pio_handle
;
1247 io_task
= task
->dd_data
;
1248 spin_lock(&phba
->mgmt_sgl_lock
);
1249 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
1250 spin_unlock(&phba
->mgmt_sgl_lock
);
1251 spin_lock_bh(&session
->lock
);
1252 free_wrb_handle(phba
, pwrb_context
, pwrb_handle
);
1253 spin_unlock_bh(&session
->lock
);
1257 be_complete_nopin_resp(struct beiscsi_conn
*beiscsi_conn
,
1258 struct iscsi_task
*task
, struct sol_cqe
*psol
)
1260 struct iscsi_nopin
*hdr
;
1261 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1262 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1264 hdr
= (struct iscsi_nopin
*)task
->hdr
;
1265 hdr
->flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
1266 & SOL_FLAGS_MASK
) >> 24) | 0x80;
1267 hdr
->exp_cmdsn
= cpu_to_be32(psol
->dw
[offsetof(struct amap_sol_cqe
,
1268 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
);
1269 hdr
->max_cmdsn
= be32_to_cpu((psol
->dw
[offsetof(struct amap_sol_cqe
,
1270 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
) +
1271 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
1272 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
1273 hdr
->opcode
= ISCSI_OP_NOOP_IN
;
1274 hdr
->itt
= io_task
->libiscsi_itt
;
1275 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
1278 static void hwi_complete_cmd(struct beiscsi_conn
*beiscsi_conn
,
1279 struct beiscsi_hba
*phba
, struct sol_cqe
*psol
)
1281 struct hwi_wrb_context
*pwrb_context
;
1282 struct wrb_handle
*pwrb_handle
;
1283 struct iscsi_wrb
*pwrb
= NULL
;
1284 struct hwi_controller
*phwi_ctrlr
;
1285 struct iscsi_task
*task
;
1287 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1288 struct iscsi_session
*session
= conn
->session
;
1290 phwi_ctrlr
= phba
->phwi_ctrlr
;
1291 pwrb_context
= &phwi_ctrlr
->wrb_context
[((psol
->dw
[offsetof
1292 (struct amap_sol_cqe
, cid
) / 32]
1293 & SOL_CID_MASK
) >> 6) -
1294 phba
->fw_config
.iscsi_cid_start
];
1295 pwrb_handle
= pwrb_context
->pwrb_handle_basestd
[((psol
->
1296 dw
[offsetof(struct amap_sol_cqe
, wrb_index
) /
1297 32] & SOL_WRB_INDEX_MASK
) >> 16)];
1298 task
= pwrb_handle
->pio_handle
;
1299 pwrb
= pwrb_handle
->pwrb
;
1300 type
= (pwrb
->dw
[offsetof(struct amap_iscsi_wrb
, type
) / 32] &
1301 WRB_TYPE_MASK
) >> 28;
1303 spin_lock_bh(&session
->lock
);
1306 case HWH_TYPE_IO_RD
:
1307 if ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) ==
1309 be_complete_nopin_resp(beiscsi_conn
, task
, psol
);
1311 be_complete_io(beiscsi_conn
, task
, psol
);
1314 case HWH_TYPE_LOGOUT
:
1315 if ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGOUT
)
1316 be_complete_logout(beiscsi_conn
, task
, psol
);
1318 be_complete_tmf(beiscsi_conn
, task
, psol
);
1322 case HWH_TYPE_LOGIN
:
1324 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
1325 "- Solicited path\n");
1329 be_complete_nopin_resp(beiscsi_conn
, task
, psol
);
1333 shost_printk(KERN_WARNING
, phba
->shost
,
1334 "In hwi_complete_cmd, unknown type = %d"
1335 "wrb_index 0x%x CID 0x%x\n", type
,
1336 ((psol
->dw
[offsetof(struct amap_iscsi_wrb
,
1337 type
) / 32] & SOL_WRB_INDEX_MASK
) >> 16),
1338 ((psol
->dw
[offsetof(struct amap_sol_cqe
,
1339 cid
) / 32] & SOL_CID_MASK
) >> 6));
1343 spin_unlock_bh(&session
->lock
);
1346 static struct list_head
*hwi_get_async_busy_list(struct hwi_async_pdu_context
1347 *pasync_ctx
, unsigned int is_header
,
1348 unsigned int host_write_ptr
)
1351 return &pasync_ctx
->async_entry
[host_write_ptr
].
1354 return &pasync_ctx
->async_entry
[host_write_ptr
].data_busy_list
;
1357 static struct async_pdu_handle
*
1358 hwi_get_async_handle(struct beiscsi_hba
*phba
,
1359 struct beiscsi_conn
*beiscsi_conn
,
1360 struct hwi_async_pdu_context
*pasync_ctx
,
1361 struct i_t_dpdu_cqe
*pdpdu_cqe
, unsigned int *pcq_index
)
1363 struct be_bus_address phys_addr
;
1364 struct list_head
*pbusy_list
;
1365 struct async_pdu_handle
*pasync_handle
= NULL
;
1367 unsigned char buffer_index
= -1;
1368 unsigned char is_header
= 0;
1370 phys_addr
.u
.a32
.address_lo
=
1371 pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, db_addr_lo
) / 32] -
1372 ((pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, dpl
) / 32]
1373 & PDUCQE_DPL_MASK
) >> 16);
1374 phys_addr
.u
.a32
.address_hi
=
1375 pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, db_addr_hi
) / 32];
1377 phys_addr
.u
.a64
.address
=
1378 *((unsigned long long *)(&phys_addr
.u
.a64
.address
));
1380 switch (pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, code
) / 32]
1381 & PDUCQE_CODE_MASK
) {
1382 case UNSOL_HDR_NOTIFY
:
1385 pbusy_list
= hwi_get_async_busy_list(pasync_ctx
, 1,
1386 (pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1387 index
) / 32] & PDUCQE_INDEX_MASK
));
1389 buffer_len
= (unsigned int)(phys_addr
.u
.a64
.address
-
1390 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
);
1392 buffer_index
= buffer_len
/
1393 pasync_ctx
->async_header
.buffer_size
;
1396 case UNSOL_DATA_NOTIFY
:
1397 pbusy_list
= hwi_get_async_busy_list(pasync_ctx
, 0, (pdpdu_cqe
->
1398 dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1399 index
) / 32] & PDUCQE_INDEX_MASK
));
1400 buffer_len
= (unsigned long)(phys_addr
.u
.a64
.address
-
1401 pasync_ctx
->async_data
.pa_base
.u
.
1403 buffer_index
= buffer_len
/ pasync_ctx
->async_data
.buffer_size
;
1407 shost_printk(KERN_WARNING
, phba
->shost
,
1408 "Unexpected code=%d\n",
1409 pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1410 code
) / 32] & PDUCQE_CODE_MASK
);
1414 WARN_ON(!(buffer_index
<= pasync_ctx
->async_data
.num_entries
));
1415 WARN_ON(list_empty(pbusy_list
));
1416 list_for_each_entry(pasync_handle
, pbusy_list
, link
) {
1417 WARN_ON(pasync_handle
->consumed
);
1418 if (pasync_handle
->index
== buffer_index
)
1422 WARN_ON(!pasync_handle
);
1424 pasync_handle
->cri
= (unsigned short)beiscsi_conn
->beiscsi_conn_cid
-
1425 phba
->fw_config
.iscsi_cid_start
;
1426 pasync_handle
->is_header
= is_header
;
1427 pasync_handle
->buffer_len
= ((pdpdu_cqe
->
1428 dw
[offsetof(struct amap_i_t_dpdu_cqe
, dpl
) / 32]
1429 & PDUCQE_DPL_MASK
) >> 16);
1431 *pcq_index
= (pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1432 index
) / 32] & PDUCQE_INDEX_MASK
);
1433 return pasync_handle
;
1437 hwi_update_async_writables(struct hwi_async_pdu_context
*pasync_ctx
,
1438 unsigned int is_header
, unsigned int cq_index
)
1440 struct list_head
*pbusy_list
;
1441 struct async_pdu_handle
*pasync_handle
;
1442 unsigned int num_entries
, writables
= 0;
1443 unsigned int *pep_read_ptr
, *pwritables
;
1447 pep_read_ptr
= &pasync_ctx
->async_header
.ep_read_ptr
;
1448 pwritables
= &pasync_ctx
->async_header
.writables
;
1449 num_entries
= pasync_ctx
->async_header
.num_entries
;
1451 pep_read_ptr
= &pasync_ctx
->async_data
.ep_read_ptr
;
1452 pwritables
= &pasync_ctx
->async_data
.writables
;
1453 num_entries
= pasync_ctx
->async_data
.num_entries
;
1456 while ((*pep_read_ptr
) != cq_index
) {
1458 *pep_read_ptr
= (*pep_read_ptr
) % num_entries
;
1460 pbusy_list
= hwi_get_async_busy_list(pasync_ctx
, is_header
,
1463 WARN_ON(list_empty(pbusy_list
));
1465 if (!list_empty(pbusy_list
)) {
1466 pasync_handle
= list_entry(pbusy_list
->next
,
1467 struct async_pdu_handle
,
1469 WARN_ON(!pasync_handle
);
1470 pasync_handle
->consumed
= 1;
1478 "Duplicate notification received - index 0x%x!!\n",
1483 *pwritables
= *pwritables
+ writables
;
1487 static unsigned int hwi_free_async_msg(struct beiscsi_hba
*phba
,
1490 struct hwi_controller
*phwi_ctrlr
;
1491 struct hwi_async_pdu_context
*pasync_ctx
;
1492 struct async_pdu_handle
*pasync_handle
, *tmp_handle
;
1493 struct list_head
*plist
;
1496 phwi_ctrlr
= phba
->phwi_ctrlr
;
1497 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1499 plist
= &pasync_ctx
->async_entry
[cri
].wait_queue
.list
;
1501 list_for_each_entry_safe(pasync_handle
, tmp_handle
, plist
, link
) {
1502 list_del(&pasync_handle
->link
);
1505 list_add_tail(&pasync_handle
->link
,
1506 &pasync_ctx
->async_header
.free_list
);
1507 pasync_ctx
->async_header
.free_entries
++;
1510 list_add_tail(&pasync_handle
->link
,
1511 &pasync_ctx
->async_data
.free_list
);
1512 pasync_ctx
->async_data
.free_entries
++;
1517 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[cri
].wait_queue
.list
);
1518 pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
= 0;
1519 pasync_ctx
->async_entry
[cri
].wait_queue
.bytes_received
= 0;
1523 static struct phys_addr
*
1524 hwi_get_ring_address(struct hwi_async_pdu_context
*pasync_ctx
,
1525 unsigned int is_header
, unsigned int host_write_ptr
)
1527 struct phys_addr
*pasync_sge
= NULL
;
1530 pasync_sge
= pasync_ctx
->async_header
.ring_base
;
1532 pasync_sge
= pasync_ctx
->async_data
.ring_base
;
1534 return pasync_sge
+ host_write_ptr
;
1537 static void hwi_post_async_buffers(struct beiscsi_hba
*phba
,
1538 unsigned int is_header
)
1540 struct hwi_controller
*phwi_ctrlr
;
1541 struct hwi_async_pdu_context
*pasync_ctx
;
1542 struct async_pdu_handle
*pasync_handle
;
1543 struct list_head
*pfree_link
, *pbusy_list
;
1544 struct phys_addr
*pasync_sge
;
1545 unsigned int ring_id
, num_entries
;
1546 unsigned int host_write_num
;
1547 unsigned int writables
;
1551 phwi_ctrlr
= phba
->phwi_ctrlr
;
1552 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1555 num_entries
= pasync_ctx
->async_header
.num_entries
;
1556 writables
= min(pasync_ctx
->async_header
.writables
,
1557 pasync_ctx
->async_header
.free_entries
);
1558 pfree_link
= pasync_ctx
->async_header
.free_list
.next
;
1559 host_write_num
= pasync_ctx
->async_header
.host_write_ptr
;
1560 ring_id
= phwi_ctrlr
->default_pdu_hdr
.id
;
1562 num_entries
= pasync_ctx
->async_data
.num_entries
;
1563 writables
= min(pasync_ctx
->async_data
.writables
,
1564 pasync_ctx
->async_data
.free_entries
);
1565 pfree_link
= pasync_ctx
->async_data
.free_list
.next
;
1566 host_write_num
= pasync_ctx
->async_data
.host_write_ptr
;
1567 ring_id
= phwi_ctrlr
->default_pdu_data
.id
;
1570 writables
= (writables
/ 8) * 8;
1572 for (i
= 0; i
< writables
; i
++) {
1574 hwi_get_async_busy_list(pasync_ctx
, is_header
,
1577 list_entry(pfree_link
, struct async_pdu_handle
,
1579 WARN_ON(!pasync_handle
);
1580 pasync_handle
->consumed
= 0;
1582 pfree_link
= pfree_link
->next
;
1584 pasync_sge
= hwi_get_ring_address(pasync_ctx
,
1585 is_header
, host_write_num
);
1587 pasync_sge
->hi
= pasync_handle
->pa
.u
.a32
.address_lo
;
1588 pasync_sge
->lo
= pasync_handle
->pa
.u
.a32
.address_hi
;
1590 list_move(&pasync_handle
->link
, pbusy_list
);
1593 host_write_num
= host_write_num
% num_entries
;
1597 pasync_ctx
->async_header
.host_write_ptr
=
1599 pasync_ctx
->async_header
.free_entries
-= writables
;
1600 pasync_ctx
->async_header
.writables
-= writables
;
1601 pasync_ctx
->async_header
.busy_entries
+= writables
;
1603 pasync_ctx
->async_data
.host_write_ptr
= host_write_num
;
1604 pasync_ctx
->async_data
.free_entries
-= writables
;
1605 pasync_ctx
->async_data
.writables
-= writables
;
1606 pasync_ctx
->async_data
.busy_entries
+= writables
;
1609 doorbell
|= ring_id
& DB_DEF_PDU_RING_ID_MASK
;
1610 doorbell
|= 1 << DB_DEF_PDU_REARM_SHIFT
;
1611 doorbell
|= 0 << DB_DEF_PDU_EVENT_SHIFT
;
1612 doorbell
|= (writables
& DB_DEF_PDU_CQPROC_MASK
)
1613 << DB_DEF_PDU_CQPROC_SHIFT
;
1615 iowrite32(doorbell
, phba
->db_va
+ DB_RXULP0_OFFSET
);
1619 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba
*phba
,
1620 struct beiscsi_conn
*beiscsi_conn
,
1621 struct i_t_dpdu_cqe
*pdpdu_cqe
)
1623 struct hwi_controller
*phwi_ctrlr
;
1624 struct hwi_async_pdu_context
*pasync_ctx
;
1625 struct async_pdu_handle
*pasync_handle
= NULL
;
1626 unsigned int cq_index
= -1;
1628 phwi_ctrlr
= phba
->phwi_ctrlr
;
1629 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1631 pasync_handle
= hwi_get_async_handle(phba
, beiscsi_conn
, pasync_ctx
,
1632 pdpdu_cqe
, &cq_index
);
1633 BUG_ON(pasync_handle
->is_header
!= 0);
1634 if (pasync_handle
->consumed
== 0)
1635 hwi_update_async_writables(pasync_ctx
, pasync_handle
->is_header
,
1638 hwi_free_async_msg(phba
, pasync_handle
->cri
);
1639 hwi_post_async_buffers(phba
, pasync_handle
->is_header
);
1643 hwi_fwd_async_msg(struct beiscsi_conn
*beiscsi_conn
,
1644 struct beiscsi_hba
*phba
,
1645 struct hwi_async_pdu_context
*pasync_ctx
, unsigned short cri
)
1647 struct list_head
*plist
;
1648 struct async_pdu_handle
*pasync_handle
;
1650 unsigned int hdr_len
= 0, buf_len
= 0;
1651 unsigned int status
, index
= 0, offset
= 0;
1652 void *pfirst_buffer
= NULL
;
1653 unsigned int num_buf
= 0;
1655 plist
= &pasync_ctx
->async_entry
[cri
].wait_queue
.list
;
1657 list_for_each_entry(pasync_handle
, plist
, link
) {
1659 phdr
= pasync_handle
->pbuffer
;
1660 hdr_len
= pasync_handle
->buffer_len
;
1662 buf_len
= pasync_handle
->buffer_len
;
1664 pfirst_buffer
= pasync_handle
->pbuffer
;
1667 memcpy(pfirst_buffer
+ offset
,
1668 pasync_handle
->pbuffer
, buf_len
);
1674 status
= beiscsi_process_async_pdu(beiscsi_conn
, phba
,
1675 (beiscsi_conn
->beiscsi_conn_cid
-
1676 phba
->fw_config
.iscsi_cid_start
),
1677 phdr
, hdr_len
, pfirst_buffer
,
1681 hwi_free_async_msg(phba
, cri
);
1686 hwi_gather_async_pdu(struct beiscsi_conn
*beiscsi_conn
,
1687 struct beiscsi_hba
*phba
,
1688 struct async_pdu_handle
*pasync_handle
)
1690 struct hwi_async_pdu_context
*pasync_ctx
;
1691 struct hwi_controller
*phwi_ctrlr
;
1692 unsigned int bytes_needed
= 0, status
= 0;
1693 unsigned short cri
= pasync_handle
->cri
;
1694 struct pdu_base
*ppdu
;
1696 phwi_ctrlr
= phba
->phwi_ctrlr
;
1697 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1699 list_del(&pasync_handle
->link
);
1700 if (pasync_handle
->is_header
) {
1701 pasync_ctx
->async_header
.busy_entries
--;
1702 if (pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
) {
1703 hwi_free_async_msg(phba
, cri
);
1707 pasync_ctx
->async_entry
[cri
].wait_queue
.bytes_received
= 0;
1708 pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
= 1;
1709 pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_len
=
1710 (unsigned short)pasync_handle
->buffer_len
;
1711 list_add_tail(&pasync_handle
->link
,
1712 &pasync_ctx
->async_entry
[cri
].wait_queue
.list
);
1714 ppdu
= pasync_handle
->pbuffer
;
1715 bytes_needed
= ((((ppdu
->dw
[offsetof(struct amap_pdu_base
,
1716 data_len_hi
) / 32] & PDUBASE_DATALENHI_MASK
) << 8) &
1717 0xFFFF0000) | ((be16_to_cpu((ppdu
->
1718 dw
[offsetof(struct amap_pdu_base
, data_len_lo
) / 32]
1719 & PDUBASE_DATALENLO_MASK
) >> 16)) & 0x0000FFFF));
1722 pasync_ctx
->async_entry
[cri
].wait_queue
.bytes_needed
=
1725 if (bytes_needed
== 0)
1726 status
= hwi_fwd_async_msg(beiscsi_conn
, phba
,
1730 pasync_ctx
->async_data
.busy_entries
--;
1731 if (pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
) {
1732 list_add_tail(&pasync_handle
->link
,
1733 &pasync_ctx
->async_entry
[cri
].wait_queue
.
1735 pasync_ctx
->async_entry
[cri
].wait_queue
.
1737 (unsigned short)pasync_handle
->buffer_len
;
1739 if (pasync_ctx
->async_entry
[cri
].wait_queue
.
1741 pasync_ctx
->async_entry
[cri
].wait_queue
.
1743 status
= hwi_fwd_async_msg(beiscsi_conn
, phba
,
1750 static void hwi_process_default_pdu_ring(struct beiscsi_conn
*beiscsi_conn
,
1751 struct beiscsi_hba
*phba
,
1752 struct i_t_dpdu_cqe
*pdpdu_cqe
)
1754 struct hwi_controller
*phwi_ctrlr
;
1755 struct hwi_async_pdu_context
*pasync_ctx
;
1756 struct async_pdu_handle
*pasync_handle
= NULL
;
1757 unsigned int cq_index
= -1;
1759 phwi_ctrlr
= phba
->phwi_ctrlr
;
1760 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1761 pasync_handle
= hwi_get_async_handle(phba
, beiscsi_conn
, pasync_ctx
,
1762 pdpdu_cqe
, &cq_index
);
1764 if (pasync_handle
->consumed
== 0)
1765 hwi_update_async_writables(pasync_ctx
, pasync_handle
->is_header
,
1767 hwi_gather_async_pdu(beiscsi_conn
, phba
, pasync_handle
);
1768 hwi_post_async_buffers(phba
, pasync_handle
->is_header
);
1771 static void beiscsi_process_mcc_isr(struct beiscsi_hba
*phba
)
1773 struct be_queue_info
*mcc_cq
;
1774 struct be_mcc_compl
*mcc_compl
;
1775 unsigned int num_processed
= 0;
1777 mcc_cq
= &phba
->ctrl
.mcc_obj
.cq
;
1778 mcc_compl
= queue_tail_node(mcc_cq
);
1779 mcc_compl
->flags
= le32_to_cpu(mcc_compl
->flags
);
1780 while (mcc_compl
->flags
& CQE_FLAGS_VALID_MASK
) {
1782 if (num_processed
>= 32) {
1783 hwi_ring_cq_db(phba
, mcc_cq
->id
,
1784 num_processed
, 0, 0);
1787 if (mcc_compl
->flags
& CQE_FLAGS_ASYNC_MASK
) {
1788 /* Interpret flags as an async trailer */
1789 if (is_link_state_evt(mcc_compl
->flags
))
1790 /* Interpret compl as a async link evt */
1791 beiscsi_async_link_state_process(phba
,
1792 (struct be_async_event_link_state
*) mcc_compl
);
1795 " Unsupported Async Event, flags"
1796 " = 0x%08x\n", mcc_compl
->flags
);
1797 } else if (mcc_compl
->flags
& CQE_FLAGS_COMPLETED_MASK
) {
1798 be_mcc_compl_process_isr(&phba
->ctrl
, mcc_compl
);
1799 atomic_dec(&phba
->ctrl
.mcc_obj
.q
.used
);
1802 mcc_compl
->flags
= 0;
1803 queue_tail_inc(mcc_cq
);
1804 mcc_compl
= queue_tail_node(mcc_cq
);
1805 mcc_compl
->flags
= le32_to_cpu(mcc_compl
->flags
);
1809 if (num_processed
> 0)
1810 hwi_ring_cq_db(phba
, mcc_cq
->id
, num_processed
, 1, 0);
1814 static unsigned int beiscsi_process_cq(struct be_eq_obj
*pbe_eq
)
1816 struct be_queue_info
*cq
;
1817 struct sol_cqe
*sol
;
1818 struct dmsg_cqe
*dmsg
;
1819 unsigned int num_processed
= 0;
1820 unsigned int tot_nump
= 0;
1821 struct beiscsi_conn
*beiscsi_conn
;
1822 struct beiscsi_endpoint
*beiscsi_ep
;
1823 struct iscsi_endpoint
*ep
;
1824 struct beiscsi_hba
*phba
;
1827 sol
= queue_tail_node(cq
);
1828 phba
= pbe_eq
->phba
;
1830 while (sol
->dw
[offsetof(struct amap_sol_cqe
, valid
) / 32] &
1832 be_dws_le_to_cpu(sol
, sizeof(struct sol_cqe
));
1834 ep
= phba
->ep_array
[(u32
) ((sol
->
1835 dw
[offsetof(struct amap_sol_cqe
, cid
) / 32] &
1836 SOL_CID_MASK
) >> 6) -
1837 phba
->fw_config
.iscsi_cid_start
];
1839 beiscsi_ep
= ep
->dd_data
;
1840 beiscsi_conn
= beiscsi_ep
->conn
;
1842 if (num_processed
>= 32) {
1843 hwi_ring_cq_db(phba
, cq
->id
,
1844 num_processed
, 0, 0);
1845 tot_nump
+= num_processed
;
1849 switch ((u32
) sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1850 32] & CQE_CODE_MASK
) {
1851 case SOL_CMD_COMPLETE
:
1852 hwi_complete_cmd(beiscsi_conn
, phba
, sol
);
1854 case DRIVERMSG_NOTIFY
:
1855 SE_DEBUG(DBG_LVL_8
, "Received DRIVERMSG_NOTIFY\n");
1856 dmsg
= (struct dmsg_cqe
*)sol
;
1857 hwi_complete_drvr_msgs(beiscsi_conn
, phba
, sol
);
1859 case UNSOL_HDR_NOTIFY
:
1860 SE_DEBUG(DBG_LVL_8
, "Received UNSOL_HDR_ NOTIFY\n");
1861 hwi_process_default_pdu_ring(beiscsi_conn
, phba
,
1862 (struct i_t_dpdu_cqe
*)sol
);
1864 case UNSOL_DATA_NOTIFY
:
1865 SE_DEBUG(DBG_LVL_8
, "Received UNSOL_DATA_NOTIFY\n");
1866 hwi_process_default_pdu_ring(beiscsi_conn
, phba
,
1867 (struct i_t_dpdu_cqe
*)sol
);
1869 case CXN_INVALIDATE_INDEX_NOTIFY
:
1870 case CMD_INVALIDATED_NOTIFY
:
1871 case CXN_INVALIDATE_NOTIFY
:
1873 "Ignoring CQ Error notification for cmd/cxn"
1876 case SOL_CMD_KILLED_DATA_DIGEST_ERR
:
1877 case CMD_KILLED_INVALID_STATSN_RCVD
:
1878 case CMD_KILLED_INVALID_R2T_RCVD
:
1879 case CMD_CXN_KILLED_LUN_INVALID
:
1880 case CMD_CXN_KILLED_ICD_INVALID
:
1881 case CMD_CXN_KILLED_ITT_INVALID
:
1882 case CMD_CXN_KILLED_SEQ_OUTOFORDER
:
1883 case CMD_CXN_KILLED_INVALID_DATASN_RCVD
:
1885 "CQ Error notification for cmd.. "
1886 "code %d cid 0x%x\n",
1887 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1888 32] & CQE_CODE_MASK
,
1889 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1890 32] & SOL_CID_MASK
));
1892 case UNSOL_DATA_DIGEST_ERROR_NOTIFY
:
1894 "Digest error on def pdu ring, dropping..\n");
1895 hwi_flush_default_pdu_buffer(phba
, beiscsi_conn
,
1896 (struct i_t_dpdu_cqe
*) sol
);
1898 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL
:
1899 case CXN_KILLED_BURST_LEN_MISMATCH
:
1900 case CXN_KILLED_AHS_RCVD
:
1901 case CXN_KILLED_HDR_DIGEST_ERR
:
1902 case CXN_KILLED_UNKNOWN_HDR
:
1903 case CXN_KILLED_STALE_ITT_TTT_RCVD
:
1904 case CXN_KILLED_INVALID_ITT_TTT_RCVD
:
1905 case CXN_KILLED_TIMED_OUT
:
1906 case CXN_KILLED_FIN_RCVD
:
1907 case CXN_KILLED_BAD_UNSOL_PDU_RCVD
:
1908 case CXN_KILLED_BAD_WRB_INDEX_ERROR
:
1909 case CXN_KILLED_OVER_RUN_RESIDUAL
:
1910 case CXN_KILLED_UNDER_RUN_RESIDUAL
:
1911 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN
:
1912 SE_DEBUG(DBG_LVL_1
, "CQ Error %d, reset CID "
1914 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1915 32] & CQE_CODE_MASK
,
1916 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1917 32] & CQE_CID_MASK
));
1918 iscsi_conn_failure(beiscsi_conn
->conn
,
1919 ISCSI_ERR_CONN_FAILED
);
1921 case CXN_KILLED_RST_SENT
:
1922 case CXN_KILLED_RST_RCVD
:
1923 SE_DEBUG(DBG_LVL_1
, "CQ Error %d, reset"
1924 "received/sent on CID 0x%x...\n",
1925 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1926 32] & CQE_CODE_MASK
,
1927 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1928 32] & CQE_CID_MASK
));
1929 iscsi_conn_failure(beiscsi_conn
->conn
,
1930 ISCSI_ERR_CONN_FAILED
);
1933 SE_DEBUG(DBG_LVL_1
, "CQ Error Invalid code= %d "
1934 "received on CID 0x%x...\n",
1935 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1936 32] & CQE_CODE_MASK
,
1937 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1938 32] & CQE_CID_MASK
));
1942 AMAP_SET_BITS(struct amap_sol_cqe
, valid
, sol
, 0);
1944 sol
= queue_tail_node(cq
);
1948 if (num_processed
> 0) {
1949 tot_nump
+= num_processed
;
1950 hwi_ring_cq_db(phba
, cq
->id
, num_processed
, 1, 0);
1955 void beiscsi_process_all_cqs(struct work_struct
*work
)
1957 unsigned long flags
;
1958 struct hwi_controller
*phwi_ctrlr
;
1959 struct hwi_context_memory
*phwi_context
;
1960 struct be_eq_obj
*pbe_eq
;
1961 struct beiscsi_hba
*phba
=
1962 container_of(work
, struct beiscsi_hba
, work_cqs
);
1964 phwi_ctrlr
= phba
->phwi_ctrlr
;
1965 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
1966 if (phba
->msix_enabled
)
1967 pbe_eq
= &phwi_context
->be_eq
[phba
->num_cpus
];
1969 pbe_eq
= &phwi_context
->be_eq
[0];
1971 if (phba
->todo_mcc_cq
) {
1972 spin_lock_irqsave(&phba
->isr_lock
, flags
);
1973 phba
->todo_mcc_cq
= 0;
1974 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
1975 beiscsi_process_mcc_isr(phba
);
1978 if (phba
->todo_cq
) {
1979 spin_lock_irqsave(&phba
->isr_lock
, flags
);
1981 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
1982 beiscsi_process_cq(pbe_eq
);
1986 static int be_iopoll(struct blk_iopoll
*iop
, int budget
)
1988 static unsigned int ret
;
1989 struct beiscsi_hba
*phba
;
1990 struct be_eq_obj
*pbe_eq
;
1992 pbe_eq
= container_of(iop
, struct be_eq_obj
, iopoll
);
1993 ret
= beiscsi_process_cq(pbe_eq
);
1995 phba
= pbe_eq
->phba
;
1996 blk_iopoll_complete(iop
);
1997 SE_DEBUG(DBG_LVL_8
, "rearm pbe_eq->q.id =%d\n", pbe_eq
->q
.id
);
1998 hwi_ring_eq_db(phba
, pbe_eq
->q
.id
, 0, 0, 1, 1);
2004 hwi_write_sgl(struct iscsi_wrb
*pwrb
, struct scatterlist
*sg
,
2005 unsigned int num_sg
, struct beiscsi_io_task
*io_task
)
2007 struct iscsi_sge
*psgl
;
2008 unsigned int sg_len
, index
;
2009 unsigned int sge_len
= 0;
2010 unsigned long long addr
;
2011 struct scatterlist
*l_sg
;
2012 unsigned int offset
;
2014 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_lo
, pwrb
,
2015 io_task
->bhs_pa
.u
.a32
.address_lo
);
2016 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_hi
, pwrb
,
2017 io_task
->bhs_pa
.u
.a32
.address_hi
);
2020 for (index
= 0; (index
< num_sg
) && (index
< 2); index
++,
2023 sg_len
= sg_dma_len(sg
);
2024 addr
= (u64
) sg_dma_address(sg
);
2025 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_lo
, pwrb
,
2026 ((u32
)(addr
& 0xFFFFFFFF)));
2027 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_hi
, pwrb
,
2028 ((u32
)(addr
>> 32)));
2029 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_len
, pwrb
,
2033 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_r2t_offset
,
2035 sg_len
= sg_dma_len(sg
);
2036 addr
= (u64
) sg_dma_address(sg
);
2037 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_addr_lo
, pwrb
,
2038 ((u32
)(addr
& 0xFFFFFFFF)));
2039 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_addr_hi
, pwrb
,
2040 ((u32
)(addr
>> 32)));
2041 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_len
, pwrb
,
2045 psgl
= (struct iscsi_sge
*)io_task
->psgl_handle
->pfrag
;
2046 memset(psgl
, 0, sizeof(*psgl
) * BE2_SGE
);
2048 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, io_task
->bhs_len
- 2);
2050 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2051 io_task
->bhs_pa
.u
.a32
.address_hi
);
2052 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2053 io_task
->bhs_pa
.u
.a32
.address_lo
);
2056 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
2058 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
,
2060 } else if (num_sg
== 2) {
2061 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
2063 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
,
2066 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
2068 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
,
2075 for (index
= 0; index
< num_sg
; index
++, sg
= sg_next(sg
), psgl
++) {
2076 sg_len
= sg_dma_len(sg
);
2077 addr
= (u64
) sg_dma_address(sg
);
2078 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2079 (addr
& 0xFFFFFFFF));
2080 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2082 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, sg_len
);
2083 AMAP_SET_BITS(struct amap_iscsi_sge
, sge_offset
, psgl
, offset
);
2084 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 0);
2088 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 1);
2091 static void hwi_write_buffer(struct iscsi_wrb
*pwrb
, struct iscsi_task
*task
)
2093 struct iscsi_sge
*psgl
;
2094 unsigned long long addr
;
2095 struct beiscsi_io_task
*io_task
= task
->dd_data
;
2096 struct beiscsi_conn
*beiscsi_conn
= io_task
->conn
;
2097 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
2099 io_task
->bhs_len
= sizeof(struct be_nonio_bhs
) - 2;
2100 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_lo
, pwrb
,
2101 io_task
->bhs_pa
.u
.a32
.address_lo
);
2102 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_hi
, pwrb
,
2103 io_task
->bhs_pa
.u
.a32
.address_hi
);
2106 if (task
->data_count
) {
2107 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 1);
2108 addr
= (u64
) pci_map_single(phba
->pcidev
,
2110 task
->data_count
, 1);
2112 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
2115 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_lo
, pwrb
,
2116 ((u32
)(addr
& 0xFFFFFFFF)));
2117 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_hi
, pwrb
,
2118 ((u32
)(addr
>> 32)));
2119 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_len
, pwrb
,
2122 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
, 1);
2124 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
2128 psgl
= (struct iscsi_sge
*)io_task
->psgl_handle
->pfrag
;
2130 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, io_task
->bhs_len
);
2132 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2133 io_task
->bhs_pa
.u
.a32
.address_hi
);
2134 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2135 io_task
->bhs_pa
.u
.a32
.address_lo
);
2138 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
, 0);
2139 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
, 0);
2140 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, 0);
2141 AMAP_SET_BITS(struct amap_iscsi_sge
, sge_offset
, psgl
, 0);
2142 AMAP_SET_BITS(struct amap_iscsi_sge
, rsvd0
, psgl
, 0);
2143 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 0);
2147 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2148 ((u32
)(addr
& 0xFFFFFFFF)));
2149 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2150 ((u32
)(addr
>> 32)));
2152 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, 0x106);
2154 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 1);
2157 static void beiscsi_find_mem_req(struct beiscsi_hba
*phba
)
2159 unsigned int num_cq_pages
, num_async_pdu_buf_pages
;
2160 unsigned int num_async_pdu_data_pages
, wrb_sz_per_cxn
;
2161 unsigned int num_async_pdu_buf_sgl_pages
, num_async_pdu_data_sgl_pages
;
2163 num_cq_pages
= PAGES_REQUIRED(phba
->params
.num_cq_entries
* \
2164 sizeof(struct sol_cqe
));
2165 num_async_pdu_buf_pages
=
2166 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
2167 phba
->params
.defpdu_hdr_sz
);
2168 num_async_pdu_buf_sgl_pages
=
2169 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
2170 sizeof(struct phys_addr
));
2171 num_async_pdu_data_pages
=
2172 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
2173 phba
->params
.defpdu_data_sz
);
2174 num_async_pdu_data_sgl_pages
=
2175 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
2176 sizeof(struct phys_addr
));
2178 phba
->params
.hwi_ws_sz
= sizeof(struct hwi_controller
);
2180 phba
->mem_req
[ISCSI_MEM_GLOBAL_HEADER
] = 2 *
2181 BE_ISCSI_PDU_HEADER_SIZE
;
2182 phba
->mem_req
[HWI_MEM_ADDN_CONTEXT
] =
2183 sizeof(struct hwi_context_memory
);
2186 phba
->mem_req
[HWI_MEM_WRB
] = sizeof(struct iscsi_wrb
)
2187 * (phba
->params
.wrbs_per_cxn
)
2188 * phba
->params
.cxns_per_ctrl
;
2189 wrb_sz_per_cxn
= sizeof(struct wrb_handle
) *
2190 (phba
->params
.wrbs_per_cxn
);
2191 phba
->mem_req
[HWI_MEM_WRBH
] = roundup_pow_of_two((wrb_sz_per_cxn
) *
2192 phba
->params
.cxns_per_ctrl
);
2194 phba
->mem_req
[HWI_MEM_SGLH
] = sizeof(struct sgl_handle
) *
2195 phba
->params
.icds_per_ctrl
;
2196 phba
->mem_req
[HWI_MEM_SGE
] = sizeof(struct iscsi_sge
) *
2197 phba
->params
.num_sge_per_io
* phba
->params
.icds_per_ctrl
;
2199 phba
->mem_req
[HWI_MEM_ASYNC_HEADER_BUF
] =
2200 num_async_pdu_buf_pages
* PAGE_SIZE
;
2201 phba
->mem_req
[HWI_MEM_ASYNC_DATA_BUF
] =
2202 num_async_pdu_data_pages
* PAGE_SIZE
;
2203 phba
->mem_req
[HWI_MEM_ASYNC_HEADER_RING
] =
2204 num_async_pdu_buf_sgl_pages
* PAGE_SIZE
;
2205 phba
->mem_req
[HWI_MEM_ASYNC_DATA_RING
] =
2206 num_async_pdu_data_sgl_pages
* PAGE_SIZE
;
2207 phba
->mem_req
[HWI_MEM_ASYNC_HEADER_HANDLE
] =
2208 phba
->params
.asyncpdus_per_ctrl
*
2209 sizeof(struct async_pdu_handle
);
2210 phba
->mem_req
[HWI_MEM_ASYNC_DATA_HANDLE
] =
2211 phba
->params
.asyncpdus_per_ctrl
*
2212 sizeof(struct async_pdu_handle
);
2213 phba
->mem_req
[HWI_MEM_ASYNC_PDU_CONTEXT
] =
2214 sizeof(struct hwi_async_pdu_context
) +
2215 (phba
->params
.cxns_per_ctrl
* sizeof(struct hwi_async_entry
));
2218 static int beiscsi_alloc_mem(struct beiscsi_hba
*phba
)
2220 struct be_mem_descriptor
*mem_descr
;
2222 struct mem_array
*mem_arr
, *mem_arr_orig
;
2223 unsigned int i
, j
, alloc_size
, curr_alloc_size
;
2225 phba
->phwi_ctrlr
= kmalloc(phba
->params
.hwi_ws_sz
, GFP_KERNEL
);
2226 if (!phba
->phwi_ctrlr
)
2229 phba
->init_mem
= kcalloc(SE_MEM_MAX
, sizeof(*mem_descr
),
2231 if (!phba
->init_mem
) {
2232 kfree(phba
->phwi_ctrlr
);
2236 mem_arr_orig
= kmalloc(sizeof(*mem_arr_orig
) * BEISCSI_MAX_FRAGS_INIT
,
2238 if (!mem_arr_orig
) {
2239 kfree(phba
->init_mem
);
2240 kfree(phba
->phwi_ctrlr
);
2244 mem_descr
= phba
->init_mem
;
2245 for (i
= 0; i
< SE_MEM_MAX
; i
++) {
2247 mem_arr
= mem_arr_orig
;
2248 alloc_size
= phba
->mem_req
[i
];
2249 memset(mem_arr
, 0, sizeof(struct mem_array
) *
2250 BEISCSI_MAX_FRAGS_INIT
);
2251 curr_alloc_size
= min(be_max_phys_size
* 1024, alloc_size
);
2253 mem_arr
->virtual_address
= pci_alloc_consistent(
2257 if (!mem_arr
->virtual_address
) {
2258 if (curr_alloc_size
<= BE_MIN_MEM_SIZE
)
2260 if (curr_alloc_size
-
2261 rounddown_pow_of_two(curr_alloc_size
))
2262 curr_alloc_size
= rounddown_pow_of_two
2265 curr_alloc_size
= curr_alloc_size
/ 2;
2267 mem_arr
->bus_address
.u
.
2268 a64
.address
= (__u64
) bus_add
;
2269 mem_arr
->size
= curr_alloc_size
;
2270 alloc_size
-= curr_alloc_size
;
2271 curr_alloc_size
= min(be_max_phys_size
*
2276 } while (alloc_size
);
2277 mem_descr
->num_elements
= j
;
2278 mem_descr
->size_in_bytes
= phba
->mem_req
[i
];
2279 mem_descr
->mem_array
= kmalloc(sizeof(*mem_arr
) * j
,
2281 if (!mem_descr
->mem_array
)
2284 memcpy(mem_descr
->mem_array
, mem_arr_orig
,
2285 sizeof(struct mem_array
) * j
);
2288 kfree(mem_arr_orig
);
2291 mem_descr
->num_elements
= j
;
2292 while ((i
) || (j
)) {
2293 for (j
= mem_descr
->num_elements
; j
> 0; j
--) {
2294 pci_free_consistent(phba
->pcidev
,
2295 mem_descr
->mem_array
[j
- 1].size
,
2296 mem_descr
->mem_array
[j
- 1].
2298 (unsigned long)mem_descr
->
2300 bus_address
.u
.a64
.address
);
2304 kfree(mem_descr
->mem_array
);
2308 kfree(mem_arr_orig
);
2309 kfree(phba
->init_mem
);
2310 kfree(phba
->phwi_ctrlr
);
2314 static int beiscsi_get_memory(struct beiscsi_hba
*phba
)
2316 beiscsi_find_mem_req(phba
);
2317 return beiscsi_alloc_mem(phba
);
2320 static void iscsi_init_global_templates(struct beiscsi_hba
*phba
)
2322 struct pdu_data_out
*pdata_out
;
2323 struct pdu_nop_out
*pnop_out
;
2324 struct be_mem_descriptor
*mem_descr
;
2326 mem_descr
= phba
->init_mem
;
2327 mem_descr
+= ISCSI_MEM_GLOBAL_HEADER
;
2329 (struct pdu_data_out
*)mem_descr
->mem_array
[0].virtual_address
;
2330 memset(pdata_out
, 0, BE_ISCSI_PDU_HEADER_SIZE
);
2332 AMAP_SET_BITS(struct amap_pdu_data_out
, opcode
, pdata_out
,
2336 (struct pdu_nop_out
*)((unsigned char *)mem_descr
->mem_array
[0].
2337 virtual_address
+ BE_ISCSI_PDU_HEADER_SIZE
);
2339 memset(pnop_out
, 0, BE_ISCSI_PDU_HEADER_SIZE
);
2340 AMAP_SET_BITS(struct amap_pdu_nop_out
, ttt
, pnop_out
, 0xFFFFFFFF);
2341 AMAP_SET_BITS(struct amap_pdu_nop_out
, f_bit
, pnop_out
, 1);
2342 AMAP_SET_BITS(struct amap_pdu_nop_out
, i_bit
, pnop_out
, 0);
2345 static void beiscsi_init_wrb_handle(struct beiscsi_hba
*phba
)
2347 struct be_mem_descriptor
*mem_descr_wrbh
, *mem_descr_wrb
;
2348 struct wrb_handle
*pwrb_handle
;
2349 struct hwi_controller
*phwi_ctrlr
;
2350 struct hwi_wrb_context
*pwrb_context
;
2351 struct iscsi_wrb
*pwrb
;
2352 unsigned int num_cxn_wrbh
;
2353 unsigned int num_cxn_wrb
, j
, idx
, index
;
2355 mem_descr_wrbh
= phba
->init_mem
;
2356 mem_descr_wrbh
+= HWI_MEM_WRBH
;
2358 mem_descr_wrb
= phba
->init_mem
;
2359 mem_descr_wrb
+= HWI_MEM_WRB
;
2362 pwrb_handle
= mem_descr_wrbh
->mem_array
[idx
].virtual_address
;
2363 num_cxn_wrbh
= ((mem_descr_wrbh
->mem_array
[idx
].size
) /
2364 ((sizeof(struct wrb_handle
)) *
2365 phba
->params
.wrbs_per_cxn
));
2366 phwi_ctrlr
= phba
->phwi_ctrlr
;
2368 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
* 2; index
+= 2) {
2369 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
2370 pwrb_context
->pwrb_handle_base
=
2371 kzalloc(sizeof(struct wrb_handle
*) *
2372 phba
->params
.wrbs_per_cxn
, GFP_KERNEL
);
2373 pwrb_context
->pwrb_handle_basestd
=
2374 kzalloc(sizeof(struct wrb_handle
*) *
2375 phba
->params
.wrbs_per_cxn
, GFP_KERNEL
);
2377 pwrb_context
->alloc_index
= 0;
2378 pwrb_context
->wrb_handles_available
= 0;
2379 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2380 pwrb_context
->pwrb_handle_base
[j
] = pwrb_handle
;
2381 pwrb_context
->pwrb_handle_basestd
[j
] =
2383 pwrb_context
->wrb_handles_available
++;
2384 pwrb_handle
->wrb_index
= j
;
2387 pwrb_context
->free_index
= 0;
2392 mem_descr_wrbh
->mem_array
[idx
].virtual_address
;
2394 ((mem_descr_wrbh
->mem_array
[idx
].size
) /
2395 ((sizeof(struct wrb_handle
)) *
2396 phba
->params
.wrbs_per_cxn
));
2397 pwrb_context
->alloc_index
= 0;
2398 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2399 pwrb_context
->pwrb_handle_base
[j
] = pwrb_handle
;
2400 pwrb_context
->pwrb_handle_basestd
[j
] =
2402 pwrb_context
->wrb_handles_available
++;
2403 pwrb_handle
->wrb_index
= j
;
2406 pwrb_context
->free_index
= 0;
2411 pwrb
= mem_descr_wrb
->mem_array
[idx
].virtual_address
;
2412 num_cxn_wrb
= (mem_descr_wrb
->mem_array
[idx
].size
) /
2413 ((sizeof(struct iscsi_wrb
) *
2414 phba
->params
.wrbs_per_cxn
));
2415 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
* 2; index
+= 2) {
2416 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
2418 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2419 pwrb_handle
= pwrb_context
->pwrb_handle_base
[j
];
2420 pwrb_handle
->pwrb
= pwrb
;
2426 pwrb
= mem_descr_wrb
->mem_array
[idx
].virtual_address
;
2427 num_cxn_wrb
= (mem_descr_wrb
->mem_array
[idx
].size
) /
2428 ((sizeof(struct iscsi_wrb
) *
2429 phba
->params
.wrbs_per_cxn
));
2430 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2431 pwrb_handle
= pwrb_context
->pwrb_handle_base
[j
];
2432 pwrb_handle
->pwrb
= pwrb
;
2440 static void hwi_init_async_pdu_ctx(struct beiscsi_hba
*phba
)
2442 struct hwi_controller
*phwi_ctrlr
;
2443 struct hba_parameters
*p
= &phba
->params
;
2444 struct hwi_async_pdu_context
*pasync_ctx
;
2445 struct async_pdu_handle
*pasync_header_h
, *pasync_data_h
;
2447 struct be_mem_descriptor
*mem_descr
;
2449 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2450 mem_descr
+= HWI_MEM_ASYNC_PDU_CONTEXT
;
2452 phwi_ctrlr
= phba
->phwi_ctrlr
;
2453 phwi_ctrlr
->phwi_ctxt
->pasync_ctx
= (struct hwi_async_pdu_context
*)
2454 mem_descr
->mem_array
[0].virtual_address
;
2455 pasync_ctx
= phwi_ctrlr
->phwi_ctxt
->pasync_ctx
;
2456 memset(pasync_ctx
, 0, sizeof(*pasync_ctx
));
2458 pasync_ctx
->async_header
.num_entries
= p
->asyncpdus_per_ctrl
;
2459 pasync_ctx
->async_header
.buffer_size
= p
->defpdu_hdr_sz
;
2460 pasync_ctx
->async_data
.buffer_size
= p
->defpdu_data_sz
;
2461 pasync_ctx
->async_data
.num_entries
= p
->asyncpdus_per_ctrl
;
2463 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2464 mem_descr
+= HWI_MEM_ASYNC_HEADER_BUF
;
2465 if (mem_descr
->mem_array
[0].virtual_address
) {
2467 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2468 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2470 shost_printk(KERN_WARNING
, phba
->shost
,
2471 "No Virtual address\n");
2473 pasync_ctx
->async_header
.va_base
=
2474 mem_descr
->mem_array
[0].virtual_address
;
2476 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
=
2477 mem_descr
->mem_array
[0].bus_address
.u
.a64
.address
;
2479 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2480 mem_descr
+= HWI_MEM_ASYNC_HEADER_RING
;
2481 if (mem_descr
->mem_array
[0].virtual_address
) {
2483 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2484 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2486 shost_printk(KERN_WARNING
, phba
->shost
,
2487 "No Virtual address\n");
2488 pasync_ctx
->async_header
.ring_base
=
2489 mem_descr
->mem_array
[0].virtual_address
;
2491 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2492 mem_descr
+= HWI_MEM_ASYNC_HEADER_HANDLE
;
2493 if (mem_descr
->mem_array
[0].virtual_address
) {
2495 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2496 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2498 shost_printk(KERN_WARNING
, phba
->shost
,
2499 "No Virtual address\n");
2501 pasync_ctx
->async_header
.handle_base
=
2502 mem_descr
->mem_array
[0].virtual_address
;
2503 pasync_ctx
->async_header
.writables
= 0;
2504 INIT_LIST_HEAD(&pasync_ctx
->async_header
.free_list
);
2506 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2507 mem_descr
+= HWI_MEM_ASYNC_DATA_BUF
;
2508 if (mem_descr
->mem_array
[0].virtual_address
) {
2510 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2511 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2513 shost_printk(KERN_WARNING
, phba
->shost
,
2514 "No Virtual address\n");
2515 pasync_ctx
->async_data
.va_base
=
2516 mem_descr
->mem_array
[0].virtual_address
;
2517 pasync_ctx
->async_data
.pa_base
.u
.a64
.address
=
2518 mem_descr
->mem_array
[0].bus_address
.u
.a64
.address
;
2520 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2521 mem_descr
+= HWI_MEM_ASYNC_DATA_RING
;
2522 if (mem_descr
->mem_array
[0].virtual_address
) {
2524 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2525 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2527 shost_printk(KERN_WARNING
, phba
->shost
,
2528 "No Virtual address\n");
2530 pasync_ctx
->async_data
.ring_base
=
2531 mem_descr
->mem_array
[0].virtual_address
;
2533 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2534 mem_descr
+= HWI_MEM_ASYNC_DATA_HANDLE
;
2535 if (!mem_descr
->mem_array
[0].virtual_address
)
2536 shost_printk(KERN_WARNING
, phba
->shost
,
2537 "No Virtual address\n");
2539 pasync_ctx
->async_data
.handle_base
=
2540 mem_descr
->mem_array
[0].virtual_address
;
2541 pasync_ctx
->async_data
.writables
= 0;
2542 INIT_LIST_HEAD(&pasync_ctx
->async_data
.free_list
);
2545 (struct async_pdu_handle
*)pasync_ctx
->async_header
.handle_base
;
2547 (struct async_pdu_handle
*)pasync_ctx
->async_data
.handle_base
;
2549 for (index
= 0; index
< p
->asyncpdus_per_ctrl
; index
++) {
2550 pasync_header_h
->cri
= -1;
2551 pasync_header_h
->index
= (char)index
;
2552 INIT_LIST_HEAD(&pasync_header_h
->link
);
2553 pasync_header_h
->pbuffer
=
2554 (void *)((unsigned long)
2555 (pasync_ctx
->async_header
.va_base
) +
2556 (p
->defpdu_hdr_sz
* index
));
2558 pasync_header_h
->pa
.u
.a64
.address
=
2559 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
+
2560 (p
->defpdu_hdr_sz
* index
);
2562 list_add_tail(&pasync_header_h
->link
,
2563 &pasync_ctx
->async_header
.free_list
);
2565 pasync_ctx
->async_header
.free_entries
++;
2566 pasync_ctx
->async_header
.writables
++;
2568 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].wait_queue
.list
);
2569 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].
2571 pasync_data_h
->cri
= -1;
2572 pasync_data_h
->index
= (char)index
;
2573 INIT_LIST_HEAD(&pasync_data_h
->link
);
2574 pasync_data_h
->pbuffer
=
2575 (void *)((unsigned long)
2576 (pasync_ctx
->async_data
.va_base
) +
2577 (p
->defpdu_data_sz
* index
));
2579 pasync_data_h
->pa
.u
.a64
.address
=
2580 pasync_ctx
->async_data
.pa_base
.u
.a64
.address
+
2581 (p
->defpdu_data_sz
* index
);
2583 list_add_tail(&pasync_data_h
->link
,
2584 &pasync_ctx
->async_data
.free_list
);
2586 pasync_ctx
->async_data
.free_entries
++;
2587 pasync_ctx
->async_data
.writables
++;
2589 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].data_busy_list
);
2592 pasync_ctx
->async_header
.host_write_ptr
= 0;
2593 pasync_ctx
->async_header
.ep_read_ptr
= -1;
2594 pasync_ctx
->async_data
.host_write_ptr
= 0;
2595 pasync_ctx
->async_data
.ep_read_ptr
= -1;
2599 be_sgl_create_contiguous(void *virtual_address
,
2600 u64 physical_address
, u32 length
,
2601 struct be_dma_mem
*sgl
)
2603 WARN_ON(!virtual_address
);
2604 WARN_ON(!physical_address
);
2605 WARN_ON(!length
> 0);
2608 sgl
->va
= virtual_address
;
2609 sgl
->dma
= (unsigned long)physical_address
;
2615 static void be_sgl_destroy_contiguous(struct be_dma_mem
*sgl
)
2617 memset(sgl
, 0, sizeof(*sgl
));
2621 hwi_build_be_sgl_arr(struct beiscsi_hba
*phba
,
2622 struct mem_array
*pmem
, struct be_dma_mem
*sgl
)
2625 be_sgl_destroy_contiguous(sgl
);
2627 be_sgl_create_contiguous(pmem
->virtual_address
,
2628 pmem
->bus_address
.u
.a64
.address
,
2633 hwi_build_be_sgl_by_offset(struct beiscsi_hba
*phba
,
2634 struct mem_array
*pmem
, struct be_dma_mem
*sgl
)
2637 be_sgl_destroy_contiguous(sgl
);
2639 be_sgl_create_contiguous((unsigned char *)pmem
->virtual_address
,
2640 pmem
->bus_address
.u
.a64
.address
,
2644 static int be_fill_queue(struct be_queue_info
*q
,
2645 u16 len
, u16 entry_size
, void *vaddress
)
2647 struct be_dma_mem
*mem
= &q
->dma_mem
;
2649 memset(q
, 0, sizeof(*q
));
2651 q
->entry_size
= entry_size
;
2652 mem
->size
= len
* entry_size
;
2656 memset(mem
->va
, 0, mem
->size
);
2660 static int beiscsi_create_eqs(struct beiscsi_hba
*phba
,
2661 struct hwi_context_memory
*phwi_context
)
2663 unsigned int i
, num_eq_pages
;
2664 int ret
, eq_for_mcc
;
2665 struct be_queue_info
*eq
;
2666 struct be_dma_mem
*mem
;
2670 num_eq_pages
= PAGES_REQUIRED(phba
->params
.num_eq_entries
* \
2671 sizeof(struct be_eq_entry
));
2673 if (phba
->msix_enabled
)
2677 for (i
= 0; i
< (phba
->num_cpus
+ eq_for_mcc
); i
++) {
2678 eq
= &phwi_context
->be_eq
[i
].q
;
2680 phwi_context
->be_eq
[i
].phba
= phba
;
2681 eq_vaddress
= pci_alloc_consistent(phba
->pcidev
,
2682 num_eq_pages
* PAGE_SIZE
,
2685 goto create_eq_error
;
2687 mem
->va
= eq_vaddress
;
2688 ret
= be_fill_queue(eq
, phba
->params
.num_eq_entries
,
2689 sizeof(struct be_eq_entry
), eq_vaddress
);
2691 shost_printk(KERN_ERR
, phba
->shost
,
2692 "be_fill_queue Failed for EQ\n");
2693 goto create_eq_error
;
2697 ret
= beiscsi_cmd_eq_create(&phba
->ctrl
, eq
,
2698 phwi_context
->cur_eqd
);
2700 shost_printk(KERN_ERR
, phba
->shost
,
2701 "beiscsi_cmd_eq_create"
2703 goto create_eq_error
;
2705 SE_DEBUG(DBG_LVL_8
, "eqid = %d\n", phwi_context
->be_eq
[i
].q
.id
);
2709 for (i
= 0; i
< (phba
->num_cpus
+ 1); i
++) {
2710 eq
= &phwi_context
->be_eq
[i
].q
;
2713 pci_free_consistent(phba
->pcidev
, num_eq_pages
2720 static int beiscsi_create_cqs(struct beiscsi_hba
*phba
,
2721 struct hwi_context_memory
*phwi_context
)
2723 unsigned int i
, num_cq_pages
;
2725 struct be_queue_info
*cq
, *eq
;
2726 struct be_dma_mem
*mem
;
2727 struct be_eq_obj
*pbe_eq
;
2731 num_cq_pages
= PAGES_REQUIRED(phba
->params
.num_cq_entries
* \
2732 sizeof(struct sol_cqe
));
2734 for (i
= 0; i
< phba
->num_cpus
; i
++) {
2735 cq
= &phwi_context
->be_cq
[i
];
2736 eq
= &phwi_context
->be_eq
[i
].q
;
2737 pbe_eq
= &phwi_context
->be_eq
[i
];
2739 pbe_eq
->phba
= phba
;
2741 cq_vaddress
= pci_alloc_consistent(phba
->pcidev
,
2742 num_cq_pages
* PAGE_SIZE
,
2745 goto create_cq_error
;
2746 ret
= be_fill_queue(cq
, phba
->params
.num_cq_entries
,
2747 sizeof(struct sol_cqe
), cq_vaddress
);
2749 shost_printk(KERN_ERR
, phba
->shost
,
2750 "be_fill_queue Failed for ISCSI CQ\n");
2751 goto create_cq_error
;
2755 ret
= beiscsi_cmd_cq_create(&phba
->ctrl
, cq
, eq
, false,
2758 shost_printk(KERN_ERR
, phba
->shost
,
2759 "beiscsi_cmd_eq_create"
2760 "Failed for ISCSI CQ\n");
2761 goto create_cq_error
;
2763 SE_DEBUG(DBG_LVL_8
, "iscsi cq_id is %d for eq_id %d\n",
2765 SE_DEBUG(DBG_LVL_8
, "ISCSI CQ CREATED\n");
2770 for (i
= 0; i
< phba
->num_cpus
; i
++) {
2771 cq
= &phwi_context
->be_cq
[i
];
2774 pci_free_consistent(phba
->pcidev
, num_cq_pages
2783 beiscsi_create_def_hdr(struct beiscsi_hba
*phba
,
2784 struct hwi_context_memory
*phwi_context
,
2785 struct hwi_controller
*phwi_ctrlr
,
2786 unsigned int def_pdu_ring_sz
)
2790 struct be_queue_info
*dq
, *cq
;
2791 struct be_dma_mem
*mem
;
2792 struct be_mem_descriptor
*mem_descr
;
2796 dq
= &phwi_context
->be_def_hdrq
;
2797 cq
= &phwi_context
->be_cq
[0];
2799 mem_descr
= phba
->init_mem
;
2800 mem_descr
+= HWI_MEM_ASYNC_HEADER_RING
;
2801 dq_vaddress
= mem_descr
->mem_array
[idx
].virtual_address
;
2802 ret
= be_fill_queue(dq
, mem_descr
->mem_array
[0].size
/
2803 sizeof(struct phys_addr
),
2804 sizeof(struct phys_addr
), dq_vaddress
);
2806 shost_printk(KERN_ERR
, phba
->shost
,
2807 "be_fill_queue Failed for DEF PDU HDR\n");
2810 mem
->dma
= (unsigned long)mem_descr
->mem_array
[idx
].
2811 bus_address
.u
.a64
.address
;
2812 ret
= be_cmd_create_default_pdu_queue(&phba
->ctrl
, cq
, dq
,
2814 phba
->params
.defpdu_hdr_sz
);
2816 shost_printk(KERN_ERR
, phba
->shost
,
2817 "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2820 phwi_ctrlr
->default_pdu_hdr
.id
= phwi_context
->be_def_hdrq
.id
;
2821 SE_DEBUG(DBG_LVL_8
, "iscsi def pdu id is %d\n",
2822 phwi_context
->be_def_hdrq
.id
);
2823 hwi_post_async_buffers(phba
, 1);
2828 beiscsi_create_def_data(struct beiscsi_hba
*phba
,
2829 struct hwi_context_memory
*phwi_context
,
2830 struct hwi_controller
*phwi_ctrlr
,
2831 unsigned int def_pdu_ring_sz
)
2835 struct be_queue_info
*dataq
, *cq
;
2836 struct be_dma_mem
*mem
;
2837 struct be_mem_descriptor
*mem_descr
;
2841 dataq
= &phwi_context
->be_def_dataq
;
2842 cq
= &phwi_context
->be_cq
[0];
2843 mem
= &dataq
->dma_mem
;
2844 mem_descr
= phba
->init_mem
;
2845 mem_descr
+= HWI_MEM_ASYNC_DATA_RING
;
2846 dq_vaddress
= mem_descr
->mem_array
[idx
].virtual_address
;
2847 ret
= be_fill_queue(dataq
, mem_descr
->mem_array
[0].size
/
2848 sizeof(struct phys_addr
),
2849 sizeof(struct phys_addr
), dq_vaddress
);
2851 shost_printk(KERN_ERR
, phba
->shost
,
2852 "be_fill_queue Failed for DEF PDU DATA\n");
2855 mem
->dma
= (unsigned long)mem_descr
->mem_array
[idx
].
2856 bus_address
.u
.a64
.address
;
2857 ret
= be_cmd_create_default_pdu_queue(&phba
->ctrl
, cq
, dataq
,
2859 phba
->params
.defpdu_data_sz
);
2861 shost_printk(KERN_ERR
, phba
->shost
,
2862 "be_cmd_create_default_pdu_queue Failed"
2863 " for DEF PDU DATA\n");
2866 phwi_ctrlr
->default_pdu_data
.id
= phwi_context
->be_def_dataq
.id
;
2867 SE_DEBUG(DBG_LVL_8
, "iscsi def data id is %d\n",
2868 phwi_context
->be_def_dataq
.id
);
2869 hwi_post_async_buffers(phba
, 0);
2870 SE_DEBUG(DBG_LVL_8
, "DEFAULT PDU DATA RING CREATED\n");
2875 beiscsi_post_pages(struct beiscsi_hba
*phba
)
2877 struct be_mem_descriptor
*mem_descr
;
2878 struct mem_array
*pm_arr
;
2879 unsigned int page_offset
, i
;
2880 struct be_dma_mem sgl
;
2883 mem_descr
= phba
->init_mem
;
2884 mem_descr
+= HWI_MEM_SGE
;
2885 pm_arr
= mem_descr
->mem_array
;
2887 page_offset
= (sizeof(struct iscsi_sge
) * phba
->params
.num_sge_per_io
*
2888 phba
->fw_config
.iscsi_icd_start
) / PAGE_SIZE
;
2889 for (i
= 0; i
< mem_descr
->num_elements
; i
++) {
2890 hwi_build_be_sgl_arr(phba
, pm_arr
, &sgl
);
2891 status
= be_cmd_iscsi_post_sgl_pages(&phba
->ctrl
, &sgl
,
2893 (pm_arr
->size
/ PAGE_SIZE
));
2894 page_offset
+= pm_arr
->size
/ PAGE_SIZE
;
2896 shost_printk(KERN_ERR
, phba
->shost
,
2897 "post sgl failed.\n");
2902 SE_DEBUG(DBG_LVL_8
, "POSTED PAGES\n");
2906 static void be_queue_free(struct beiscsi_hba
*phba
, struct be_queue_info
*q
)
2908 struct be_dma_mem
*mem
= &q
->dma_mem
;
2910 pci_free_consistent(phba
->pcidev
, mem
->size
,
2914 static int be_queue_alloc(struct beiscsi_hba
*phba
, struct be_queue_info
*q
,
2915 u16 len
, u16 entry_size
)
2917 struct be_dma_mem
*mem
= &q
->dma_mem
;
2919 memset(q
, 0, sizeof(*q
));
2921 q
->entry_size
= entry_size
;
2922 mem
->size
= len
* entry_size
;
2923 mem
->va
= pci_alloc_consistent(phba
->pcidev
, mem
->size
, &mem
->dma
);
2926 memset(mem
->va
, 0, mem
->size
);
2931 beiscsi_create_wrb_rings(struct beiscsi_hba
*phba
,
2932 struct hwi_context_memory
*phwi_context
,
2933 struct hwi_controller
*phwi_ctrlr
)
2935 unsigned int wrb_mem_index
, offset
, size
, num_wrb_rings
;
2937 unsigned int idx
, num
, i
;
2938 struct mem_array
*pwrb_arr
;
2940 struct be_dma_mem sgl
;
2941 struct be_mem_descriptor
*mem_descr
;
2945 mem_descr
= phba
->init_mem
;
2946 mem_descr
+= HWI_MEM_WRB
;
2947 pwrb_arr
= kmalloc(sizeof(*pwrb_arr
) * phba
->params
.cxns_per_ctrl
,
2950 shost_printk(KERN_ERR
, phba
->shost
,
2951 "Memory alloc failed in create wrb ring.\n");
2954 wrb_vaddr
= mem_descr
->mem_array
[idx
].virtual_address
;
2955 pa_addr_lo
= mem_descr
->mem_array
[idx
].bus_address
.u
.a64
.address
;
2956 num_wrb_rings
= mem_descr
->mem_array
[idx
].size
/
2957 (phba
->params
.wrbs_per_cxn
* sizeof(struct iscsi_wrb
));
2959 for (num
= 0; num
< phba
->params
.cxns_per_ctrl
; num
++) {
2960 if (num_wrb_rings
) {
2961 pwrb_arr
[num
].virtual_address
= wrb_vaddr
;
2962 pwrb_arr
[num
].bus_address
.u
.a64
.address
= pa_addr_lo
;
2963 pwrb_arr
[num
].size
= phba
->params
.wrbs_per_cxn
*
2964 sizeof(struct iscsi_wrb
);
2965 wrb_vaddr
+= pwrb_arr
[num
].size
;
2966 pa_addr_lo
+= pwrb_arr
[num
].size
;
2970 wrb_vaddr
= mem_descr
->mem_array
[idx
].virtual_address
;
2971 pa_addr_lo
= mem_descr
->mem_array
[idx
].\
2972 bus_address
.u
.a64
.address
;
2973 num_wrb_rings
= mem_descr
->mem_array
[idx
].size
/
2974 (phba
->params
.wrbs_per_cxn
*
2975 sizeof(struct iscsi_wrb
));
2976 pwrb_arr
[num
].virtual_address
= wrb_vaddr
;
2977 pwrb_arr
[num
].bus_address
.u
.a64
.address\
2979 pwrb_arr
[num
].size
= phba
->params
.wrbs_per_cxn
*
2980 sizeof(struct iscsi_wrb
);
2981 wrb_vaddr
+= pwrb_arr
[num
].size
;
2982 pa_addr_lo
+= pwrb_arr
[num
].size
;
2986 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
2991 hwi_build_be_sgl_by_offset(phba
, &pwrb_arr
[i
], &sgl
);
2992 status
= be_cmd_wrbq_create(&phba
->ctrl
, &sgl
,
2993 &phwi_context
->be_wrbq
[i
]);
2995 shost_printk(KERN_ERR
, phba
->shost
,
2996 "wrbq create failed.");
3000 phwi_ctrlr
->wrb_context
[i
* 2].cid
= phwi_context
->be_wrbq
[i
].
3007 static void free_wrb_handles(struct beiscsi_hba
*phba
)
3010 struct hwi_controller
*phwi_ctrlr
;
3011 struct hwi_wrb_context
*pwrb_context
;
3013 phwi_ctrlr
= phba
->phwi_ctrlr
;
3014 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
* 2; index
+= 2) {
3015 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
3016 kfree(pwrb_context
->pwrb_handle_base
);
3017 kfree(pwrb_context
->pwrb_handle_basestd
);
3021 static void be_mcc_queues_destroy(struct beiscsi_hba
*phba
)
3023 struct be_queue_info
*q
;
3024 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3026 q
= &phba
->ctrl
.mcc_obj
.q
;
3028 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_MCCQ
);
3029 be_queue_free(phba
, q
);
3031 q
= &phba
->ctrl
.mcc_obj
.cq
;
3033 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_CQ
);
3034 be_queue_free(phba
, q
);
3037 static void hwi_cleanup(struct beiscsi_hba
*phba
)
3039 struct be_queue_info
*q
;
3040 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3041 struct hwi_controller
*phwi_ctrlr
;
3042 struct hwi_context_memory
*phwi_context
;
3045 phwi_ctrlr
= phba
->phwi_ctrlr
;
3046 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3047 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
3048 q
= &phwi_context
->be_wrbq
[i
];
3050 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_WRBQ
);
3052 free_wrb_handles(phba
);
3054 q
= &phwi_context
->be_def_hdrq
;
3056 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_DPDUQ
);
3058 q
= &phwi_context
->be_def_dataq
;
3060 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_DPDUQ
);
3062 beiscsi_cmd_q_destroy(ctrl
, NULL
, QTYPE_SGL
);
3064 for (i
= 0; i
< (phba
->num_cpus
); i
++) {
3065 q
= &phwi_context
->be_cq
[i
];
3067 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_CQ
);
3069 if (phba
->msix_enabled
)
3073 for (i
= 0; i
< (phba
->num_cpus
+ eq_num
); i
++) {
3074 q
= &phwi_context
->be_eq
[i
].q
;
3076 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_EQ
);
3078 be_mcc_queues_destroy(phba
);
3081 static int be_mcc_queues_create(struct beiscsi_hba
*phba
,
3082 struct hwi_context_memory
*phwi_context
)
3084 struct be_queue_info
*q
, *cq
;
3085 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3087 /* Alloc MCC compl queue */
3088 cq
= &phba
->ctrl
.mcc_obj
.cq
;
3089 if (be_queue_alloc(phba
, cq
, MCC_CQ_LEN
,
3090 sizeof(struct be_mcc_compl
)))
3092 /* Ask BE to create MCC compl queue; */
3093 if (phba
->msix_enabled
) {
3094 if (beiscsi_cmd_cq_create(ctrl
, cq
, &phwi_context
->be_eq
3095 [phba
->num_cpus
].q
, false, true, 0))
3098 if (beiscsi_cmd_cq_create(ctrl
, cq
, &phwi_context
->be_eq
[0].q
,
3103 /* Alloc MCC queue */
3104 q
= &phba
->ctrl
.mcc_obj
.q
;
3105 if (be_queue_alloc(phba
, q
, MCC_Q_LEN
, sizeof(struct be_mcc_wrb
)))
3106 goto mcc_cq_destroy
;
3108 /* Ask BE to create MCC queue */
3109 if (beiscsi_cmd_mccq_create(phba
, q
, cq
))
3115 be_queue_free(phba
, q
);
3117 beiscsi_cmd_q_destroy(ctrl
, cq
, QTYPE_CQ
);
3119 be_queue_free(phba
, cq
);
3124 static int find_num_cpus(void)
3128 num_cpus
= num_online_cpus();
3129 if (num_cpus
>= MAX_CPUS
)
3130 num_cpus
= MAX_CPUS
- 1;
3132 SE_DEBUG(DBG_LVL_8
, "num_cpus = %d\n", num_cpus
);
3136 static int hwi_init_port(struct beiscsi_hba
*phba
)
3138 struct hwi_controller
*phwi_ctrlr
;
3139 struct hwi_context_memory
*phwi_context
;
3140 unsigned int def_pdu_ring_sz
;
3141 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3145 phba
->params
.asyncpdus_per_ctrl
* sizeof(struct phys_addr
);
3146 phwi_ctrlr
= phba
->phwi_ctrlr
;
3147 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3148 phwi_context
->max_eqd
= 0;
3149 phwi_context
->min_eqd
= 0;
3150 phwi_context
->cur_eqd
= 64;
3151 be_cmd_fw_initialize(&phba
->ctrl
);
3153 status
= beiscsi_create_eqs(phba
, phwi_context
);
3155 shost_printk(KERN_ERR
, phba
->shost
, "EQ not created\n");
3159 status
= be_mcc_queues_create(phba
, phwi_context
);
3163 status
= mgmt_check_supported_fw(ctrl
, phba
);
3165 shost_printk(KERN_ERR
, phba
->shost
,
3166 "Unsupported fw version\n");
3170 status
= beiscsi_create_cqs(phba
, phwi_context
);
3172 shost_printk(KERN_ERR
, phba
->shost
, "CQ not created\n");
3176 status
= beiscsi_create_def_hdr(phba
, phwi_context
, phwi_ctrlr
,
3179 shost_printk(KERN_ERR
, phba
->shost
,
3180 "Default Header not created\n");
3184 status
= beiscsi_create_def_data(phba
, phwi_context
,
3185 phwi_ctrlr
, def_pdu_ring_sz
);
3187 shost_printk(KERN_ERR
, phba
->shost
,
3188 "Default Data not created\n");
3192 status
= beiscsi_post_pages(phba
);
3194 shost_printk(KERN_ERR
, phba
->shost
, "Post SGL Pages Failed\n");
3198 status
= beiscsi_create_wrb_rings(phba
, phwi_context
, phwi_ctrlr
);
3200 shost_printk(KERN_ERR
, phba
->shost
,
3201 "WRB Rings not created\n");
3205 SE_DEBUG(DBG_LVL_8
, "hwi_init_port success\n");
3209 shost_printk(KERN_ERR
, phba
->shost
, "hwi_init_port failed");
3214 static int hwi_init_controller(struct beiscsi_hba
*phba
)
3216 struct hwi_controller
*phwi_ctrlr
;
3218 phwi_ctrlr
= phba
->phwi_ctrlr
;
3219 if (1 == phba
->init_mem
[HWI_MEM_ADDN_CONTEXT
].num_elements
) {
3220 phwi_ctrlr
->phwi_ctxt
= (struct hwi_context_memory
*)phba
->
3221 init_mem
[HWI_MEM_ADDN_CONTEXT
].mem_array
[0].virtual_address
;
3222 SE_DEBUG(DBG_LVL_8
, " phwi_ctrlr->phwi_ctxt=%p\n",
3223 phwi_ctrlr
->phwi_ctxt
);
3225 shost_printk(KERN_ERR
, phba
->shost
,
3226 "HWI_MEM_ADDN_CONTEXT is more than one element."
3227 "Failing to load\n");
3231 iscsi_init_global_templates(phba
);
3232 beiscsi_init_wrb_handle(phba
);
3233 hwi_init_async_pdu_ctx(phba
);
3234 if (hwi_init_port(phba
) != 0) {
3235 shost_printk(KERN_ERR
, phba
->shost
,
3236 "hwi_init_controller failed\n");
3242 static void beiscsi_free_mem(struct beiscsi_hba
*phba
)
3244 struct be_mem_descriptor
*mem_descr
;
3247 mem_descr
= phba
->init_mem
;
3250 for (i
= 0; i
< SE_MEM_MAX
; i
++) {
3251 for (j
= mem_descr
->num_elements
; j
> 0; j
--) {
3252 pci_free_consistent(phba
->pcidev
,
3253 mem_descr
->mem_array
[j
- 1].size
,
3254 mem_descr
->mem_array
[j
- 1].virtual_address
,
3255 (unsigned long)mem_descr
->mem_array
[j
- 1].
3256 bus_address
.u
.a64
.address
);
3258 kfree(mem_descr
->mem_array
);
3261 kfree(phba
->init_mem
);
3262 kfree(phba
->phwi_ctrlr
);
3265 static int beiscsi_init_controller(struct beiscsi_hba
*phba
)
3269 ret
= beiscsi_get_memory(phba
);
3271 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe -"
3272 "Failed in beiscsi_alloc_memory\n");
3276 ret
= hwi_init_controller(phba
);
3279 SE_DEBUG(DBG_LVL_8
, "Return success from beiscsi_init_controller");
3283 beiscsi_free_mem(phba
);
3287 static int beiscsi_init_sgl_handle(struct beiscsi_hba
*phba
)
3289 struct be_mem_descriptor
*mem_descr_sglh
, *mem_descr_sg
;
3290 struct sgl_handle
*psgl_handle
;
3291 struct iscsi_sge
*pfrag
;
3292 unsigned int arr_index
, i
, idx
;
3294 phba
->io_sgl_hndl_avbl
= 0;
3295 phba
->eh_sgl_hndl_avbl
= 0;
3297 mem_descr_sglh
= phba
->init_mem
;
3298 mem_descr_sglh
+= HWI_MEM_SGLH
;
3299 if (1 == mem_descr_sglh
->num_elements
) {
3300 phba
->io_sgl_hndl_base
= kzalloc(sizeof(struct sgl_handle
*) *
3301 phba
->params
.ios_per_ctrl
,
3303 if (!phba
->io_sgl_hndl_base
) {
3304 shost_printk(KERN_ERR
, phba
->shost
,
3305 "Mem Alloc Failed. Failing to load\n");
3308 phba
->eh_sgl_hndl_base
= kzalloc(sizeof(struct sgl_handle
*) *
3309 (phba
->params
.icds_per_ctrl
-
3310 phba
->params
.ios_per_ctrl
),
3312 if (!phba
->eh_sgl_hndl_base
) {
3313 kfree(phba
->io_sgl_hndl_base
);
3314 shost_printk(KERN_ERR
, phba
->shost
,
3315 "Mem Alloc Failed. Failing to load\n");
3319 shost_printk(KERN_ERR
, phba
->shost
,
3320 "HWI_MEM_SGLH is more than one element."
3321 "Failing to load\n");
3327 while (idx
< mem_descr_sglh
->num_elements
) {
3328 psgl_handle
= mem_descr_sglh
->mem_array
[idx
].virtual_address
;
3330 for (i
= 0; i
< (mem_descr_sglh
->mem_array
[idx
].size
/
3331 sizeof(struct sgl_handle
)); i
++) {
3332 if (arr_index
< phba
->params
.ios_per_ctrl
) {
3333 phba
->io_sgl_hndl_base
[arr_index
] = psgl_handle
;
3334 phba
->io_sgl_hndl_avbl
++;
3337 phba
->eh_sgl_hndl_base
[arr_index
-
3338 phba
->params
.ios_per_ctrl
] =
3341 phba
->eh_sgl_hndl_avbl
++;
3348 "phba->io_sgl_hndl_avbl=%d"
3349 "phba->eh_sgl_hndl_avbl=%d\n",
3350 phba
->io_sgl_hndl_avbl
,
3351 phba
->eh_sgl_hndl_avbl
);
3352 mem_descr_sg
= phba
->init_mem
;
3353 mem_descr_sg
+= HWI_MEM_SGE
;
3354 SE_DEBUG(DBG_LVL_8
, "\n mem_descr_sg->num_elements=%d\n",
3355 mem_descr_sg
->num_elements
);
3358 while (idx
< mem_descr_sg
->num_elements
) {
3359 pfrag
= mem_descr_sg
->mem_array
[idx
].virtual_address
;
3362 i
< (mem_descr_sg
->mem_array
[idx
].size
) /
3363 (sizeof(struct iscsi_sge
) * phba
->params
.num_sge_per_io
);
3365 if (arr_index
< phba
->params
.ios_per_ctrl
)
3366 psgl_handle
= phba
->io_sgl_hndl_base
[arr_index
];
3368 psgl_handle
= phba
->eh_sgl_hndl_base
[arr_index
-
3369 phba
->params
.ios_per_ctrl
];
3370 psgl_handle
->pfrag
= pfrag
;
3371 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, pfrag
, 0);
3372 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, pfrag
, 0);
3373 pfrag
+= phba
->params
.num_sge_per_io
;
3374 psgl_handle
->sgl_index
=
3375 phba
->fw_config
.iscsi_icd_start
+ arr_index
++;
3379 phba
->io_sgl_free_index
= 0;
3380 phba
->io_sgl_alloc_index
= 0;
3381 phba
->eh_sgl_free_index
= 0;
3382 phba
->eh_sgl_alloc_index
= 0;
3386 static int hba_setup_cid_tbls(struct beiscsi_hba
*phba
)
3390 phba
->cid_array
= kzalloc(sizeof(void *) * phba
->params
.cxns_per_ctrl
,
3392 if (!phba
->cid_array
) {
3393 shost_printk(KERN_ERR
, phba
->shost
,
3394 "Failed to allocate memory in "
3395 "hba_setup_cid_tbls\n");
3398 phba
->ep_array
= kzalloc(sizeof(struct iscsi_endpoint
*) *
3399 phba
->params
.cxns_per_ctrl
* 2, GFP_KERNEL
);
3400 if (!phba
->ep_array
) {
3401 shost_printk(KERN_ERR
, phba
->shost
,
3402 "Failed to allocate memory in "
3403 "hba_setup_cid_tbls\n");
3404 kfree(phba
->cid_array
);
3407 new_cid
= phba
->fw_config
.iscsi_cid_start
;
3408 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
3409 phba
->cid_array
[i
] = new_cid
;
3412 phba
->avlbl_cids
= phba
->params
.cxns_per_ctrl
;
3416 static void hwi_enable_intr(struct beiscsi_hba
*phba
)
3418 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3419 struct hwi_controller
*phwi_ctrlr
;
3420 struct hwi_context_memory
*phwi_context
;
3421 struct be_queue_info
*eq
;
3426 phwi_ctrlr
= phba
->phwi_ctrlr
;
3427 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3429 addr
= (u8 __iomem
*) ((u8 __iomem
*) ctrl
->pcicfg
+
3430 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET
);
3431 reg
= ioread32(addr
);
3433 enabled
= reg
& MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3435 reg
|= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3436 SE_DEBUG(DBG_LVL_8
, "reg =x%08x addr=%p\n", reg
, addr
);
3437 iowrite32(reg
, addr
);
3440 if (!phba
->msix_enabled
) {
3441 eq
= &phwi_context
->be_eq
[0].q
;
3442 SE_DEBUG(DBG_LVL_8
, "eq->id=%d\n", eq
->id
);
3443 hwi_ring_eq_db(phba
, eq
->id
, 0, 0, 1, 1);
3445 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
3446 eq
= &phwi_context
->be_eq
[i
].q
;
3447 SE_DEBUG(DBG_LVL_8
, "eq->id=%d\n", eq
->id
);
3448 hwi_ring_eq_db(phba
, eq
->id
, 0, 0, 1, 1);
3453 static void hwi_disable_intr(struct beiscsi_hba
*phba
)
3455 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3457 u8 __iomem
*addr
= ctrl
->pcicfg
+ PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET
;
3458 u32 reg
= ioread32(addr
);
3460 u32 enabled
= reg
& MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3462 reg
&= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3463 iowrite32(reg
, addr
);
3465 shost_printk(KERN_WARNING
, phba
->shost
,
3466 "In hwi_disable_intr, Already Disabled\n");
3469 static int beiscsi_get_boot_info(struct beiscsi_hba
*phba
)
3471 struct be_cmd_resp_get_boot_target
*boot_resp
;
3472 struct be_cmd_resp_get_session
*session_resp
;
3473 struct be_mcc_wrb
*wrb
;
3474 struct be_dma_mem nonemb_cmd
;
3475 unsigned int tag
, wrb_num
;
3476 unsigned short status
, extd_status
;
3477 struct be_queue_info
*mccq
= &phba
->ctrl
.mcc_obj
.q
;
3480 tag
= beiscsi_get_boot_target(phba
);
3482 SE_DEBUG(DBG_LVL_1
, "be_cmd_get_mac_addr Failed\n");
3485 wait_event_interruptible(phba
->ctrl
.mcc_wait
[tag
],
3486 phba
->ctrl
.mcc_numtag
[tag
]);
3488 wrb_num
= (phba
->ctrl
.mcc_numtag
[tag
] & 0x00FF0000) >> 16;
3489 extd_status
= (phba
->ctrl
.mcc_numtag
[tag
] & 0x0000FF00) >> 8;
3490 status
= phba
->ctrl
.mcc_numtag
[tag
] & 0x000000FF;
3491 if (status
|| extd_status
) {
3492 SE_DEBUG(DBG_LVL_1
, "be_cmd_get_mac_addr Failed"
3493 " status = %d extd_status = %d\n",
3494 status
, extd_status
);
3495 free_mcc_tag(&phba
->ctrl
, tag
);
3498 wrb
= queue_get_wrb(mccq
, wrb_num
);
3499 free_mcc_tag(&phba
->ctrl
, tag
);
3500 boot_resp
= embedded_payload(wrb
);
3502 if (boot_resp
->boot_session_handle
< 0) {
3503 shost_printk(KERN_INFO
, phba
->shost
, "No Boot Session.\n");
3507 nonemb_cmd
.va
= pci_alloc_consistent(phba
->ctrl
.pdev
,
3508 sizeof(*session_resp
),
3510 if (nonemb_cmd
.va
== NULL
) {
3512 "Failed to allocate memory for"
3513 "beiscsi_get_session_info\n");
3517 memset(nonemb_cmd
.va
, 0, sizeof(*session_resp
));
3518 tag
= beiscsi_get_session_info(phba
,
3519 boot_resp
->boot_session_handle
, &nonemb_cmd
);
3521 SE_DEBUG(DBG_LVL_1
, "beiscsi_get_session_info"
3525 wait_event_interruptible(phba
->ctrl
.mcc_wait
[tag
],
3526 phba
->ctrl
.mcc_numtag
[tag
]);
3528 wrb_num
= (phba
->ctrl
.mcc_numtag
[tag
] & 0x00FF0000) >> 16;
3529 extd_status
= (phba
->ctrl
.mcc_numtag
[tag
] & 0x0000FF00) >> 8;
3530 status
= phba
->ctrl
.mcc_numtag
[tag
] & 0x000000FF;
3531 if (status
|| extd_status
) {
3532 SE_DEBUG(DBG_LVL_1
, "beiscsi_get_session_info Failed"
3533 " status = %d extd_status = %d\n",
3534 status
, extd_status
);
3535 free_mcc_tag(&phba
->ctrl
, tag
);
3538 wrb
= queue_get_wrb(mccq
, wrb_num
);
3539 free_mcc_tag(&phba
->ctrl
, tag
);
3540 session_resp
= nonemb_cmd
.va
;
3542 memcpy(&phba
->boot_sess
, &session_resp
->session_info
,
3543 sizeof(struct mgmt_session_info
));
3547 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
3548 nonemb_cmd
.va
, nonemb_cmd
.dma
);
3552 static void beiscsi_boot_release(void *data
)
3554 struct beiscsi_hba
*phba
= data
;
3556 scsi_host_put(phba
->shost
);
3559 static int beiscsi_setup_boot_info(struct beiscsi_hba
*phba
)
3561 struct iscsi_boot_kobj
*boot_kobj
;
3563 /* get boot info using mgmt cmd */
3564 if (beiscsi_get_boot_info(phba
))
3565 /* Try to see if we can carry on without this */
3568 phba
->boot_kset
= iscsi_boot_create_host_kset(phba
->shost
->host_no
);
3569 if (!phba
->boot_kset
)
3572 /* get a ref because the show function will ref the phba */
3573 if (!scsi_host_get(phba
->shost
))
3575 boot_kobj
= iscsi_boot_create_target(phba
->boot_kset
, 0, phba
,
3576 beiscsi_show_boot_tgt_info
,
3577 beiscsi_tgt_get_attr_visibility
,
3578 beiscsi_boot_release
);
3582 if (!scsi_host_get(phba
->shost
))
3584 boot_kobj
= iscsi_boot_create_initiator(phba
->boot_kset
, 0, phba
,
3585 beiscsi_show_boot_ini_info
,
3586 beiscsi_ini_get_attr_visibility
,
3587 beiscsi_boot_release
);
3591 if (!scsi_host_get(phba
->shost
))
3593 boot_kobj
= iscsi_boot_create_ethernet(phba
->boot_kset
, 0, phba
,
3594 beiscsi_show_boot_eth_info
,
3595 beiscsi_eth_get_attr_visibility
,
3596 beiscsi_boot_release
);
3602 scsi_host_put(phba
->shost
);
3604 iscsi_boot_destroy_kset(phba
->boot_kset
);
3608 static int beiscsi_init_port(struct beiscsi_hba
*phba
)
3612 ret
= beiscsi_init_controller(phba
);
3614 shost_printk(KERN_ERR
, phba
->shost
,
3615 "beiscsi_dev_probe - Failed in"
3616 "beiscsi_init_controller\n");
3619 ret
= beiscsi_init_sgl_handle(phba
);
3621 shost_printk(KERN_ERR
, phba
->shost
,
3622 "beiscsi_dev_probe - Failed in"
3623 "beiscsi_init_sgl_handle\n");
3624 goto do_cleanup_ctrlr
;
3627 if (hba_setup_cid_tbls(phba
)) {
3628 shost_printk(KERN_ERR
, phba
->shost
,
3629 "Failed in hba_setup_cid_tbls\n");
3630 kfree(phba
->io_sgl_hndl_base
);
3631 kfree(phba
->eh_sgl_hndl_base
);
3632 goto do_cleanup_ctrlr
;
3642 static void hwi_purge_eq(struct beiscsi_hba
*phba
)
3644 struct hwi_controller
*phwi_ctrlr
;
3645 struct hwi_context_memory
*phwi_context
;
3646 struct be_queue_info
*eq
;
3647 struct be_eq_entry
*eqe
= NULL
;
3649 unsigned int num_processed
;
3651 phwi_ctrlr
= phba
->phwi_ctrlr
;
3652 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3653 if (phba
->msix_enabled
)
3658 for (i
= 0; i
< (phba
->num_cpus
+ eq_msix
); i
++) {
3659 eq
= &phwi_context
->be_eq
[i
].q
;
3660 eqe
= queue_tail_node(eq
);
3662 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
3664 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
3666 eqe
= queue_tail_node(eq
);
3671 hwi_ring_eq_db(phba
, eq
->id
, 1, num_processed
, 1, 1);
3675 static void beiscsi_clean_port(struct beiscsi_hba
*phba
)
3679 mgmt_status
= mgmt_epfw_cleanup(phba
, CMD_CONNECTION_CHUTE_0
);
3681 shost_printk(KERN_WARNING
, phba
->shost
,
3682 "mgmt_epfw_cleanup FAILED\n");
3686 kfree(phba
->io_sgl_hndl_base
);
3687 kfree(phba
->eh_sgl_hndl_base
);
3688 kfree(phba
->cid_array
);
3689 kfree(phba
->ep_array
);
3693 beiscsi_offload_connection(struct beiscsi_conn
*beiscsi_conn
,
3694 struct beiscsi_offload_params
*params
)
3696 struct wrb_handle
*pwrb_handle
;
3697 struct iscsi_target_context_update_wrb
*pwrb
= NULL
;
3698 struct be_mem_descriptor
*mem_descr
;
3699 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3703 * We can always use 0 here because it is reserved by libiscsi for
3704 * login/startup related tasks.
3706 pwrb_handle
= alloc_wrb_handle(phba
, (beiscsi_conn
->beiscsi_conn_cid
-
3707 phba
->fw_config
.iscsi_cid_start
));
3708 pwrb
= (struct iscsi_target_context_update_wrb
*)pwrb_handle
->pwrb
;
3709 memset(pwrb
, 0, sizeof(*pwrb
));
3710 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3711 max_burst_length
, pwrb
, params
->dw
[offsetof
3712 (struct amap_beiscsi_offload_params
,
3713 max_burst_length
) / 32]);
3714 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3715 max_send_data_segment_length
, pwrb
,
3716 params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3717 max_send_data_segment_length
) / 32]);
3718 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3721 params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3722 first_burst_length
) / 32]);
3724 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, erl
, pwrb
,
3725 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3726 erl
) / 32] & OFFLD_PARAMS_ERL
));
3727 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, dde
, pwrb
,
3728 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3729 dde
) / 32] & OFFLD_PARAMS_DDE
) >> 2);
3730 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, hde
, pwrb
,
3731 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3732 hde
) / 32] & OFFLD_PARAMS_HDE
) >> 3);
3733 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, ir2t
, pwrb
,
3734 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3735 ir2t
) / 32] & OFFLD_PARAMS_IR2T
) >> 4);
3736 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, imd
, pwrb
,
3737 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3738 imd
) / 32] & OFFLD_PARAMS_IMD
) >> 5);
3739 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, stat_sn
,
3741 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3742 exp_statsn
) / 32] + 1));
3743 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, type
, pwrb
,
3745 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, wrb_idx
,
3746 pwrb
, pwrb_handle
->wrb_index
);
3747 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, ptr2nextwrb
,
3748 pwrb
, pwrb_handle
->nxt_wrb_index
);
3749 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3750 session_state
, pwrb
, 0);
3751 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, compltonack
,
3753 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, notpredblq
,
3755 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, mode
, pwrb
,
3758 mem_descr
= phba
->init_mem
;
3759 mem_descr
+= ISCSI_MEM_GLOBAL_HEADER
;
3761 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3762 pad_buffer_addr_hi
, pwrb
,
3763 mem_descr
->mem_array
[0].bus_address
.u
.a32
.address_hi
);
3764 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3765 pad_buffer_addr_lo
, pwrb
,
3766 mem_descr
->mem_array
[0].bus_address
.u
.a32
.address_lo
);
3768 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_target_context_update_wrb
));
3770 doorbell
|= beiscsi_conn
->beiscsi_conn_cid
& DB_WRB_POST_CID_MASK
;
3771 doorbell
|= (pwrb_handle
->wrb_index
& DB_DEF_PDU_WRB_INDEX_MASK
)
3772 << DB_DEF_PDU_WRB_INDEX_SHIFT
;
3773 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
3775 iowrite32(doorbell
, phba
->db_va
+ DB_TXULP0_OFFSET
);
3778 static void beiscsi_parse_pdu(struct iscsi_conn
*conn
, itt_t itt
,
3779 int *index
, int *age
)
3783 *age
= conn
->session
->age
;
3787 * beiscsi_alloc_pdu - allocates pdu and related resources
3788 * @task: libiscsi task
3789 * @opcode: opcode of pdu for task
3791 * This is called with the session lock held. It will allocate
3792 * the wrb and sgl if needed for the command. And it will prep
3793 * the pdu's itt. beiscsi_parse_pdu will later translate
3794 * the pdu itt to the libiscsi task itt.
3796 static int beiscsi_alloc_pdu(struct iscsi_task
*task
, uint8_t opcode
)
3798 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3799 struct iscsi_conn
*conn
= task
->conn
;
3800 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3801 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3802 struct hwi_wrb_context
*pwrb_context
;
3803 struct hwi_controller
*phwi_ctrlr
;
3805 struct beiscsi_session
*beiscsi_sess
= beiscsi_conn
->beiscsi_sess
;
3808 io_task
->cmd_bhs
= pci_pool_alloc(beiscsi_sess
->bhs_pool
,
3809 GFP_ATOMIC
, &paddr
);
3810 if (!io_task
->cmd_bhs
)
3812 io_task
->bhs_pa
.u
.a64
.address
= paddr
;
3813 io_task
->libiscsi_itt
= (itt_t
)task
->itt
;
3814 io_task
->conn
= beiscsi_conn
;
3816 task
->hdr
= (struct iscsi_hdr
*)&io_task
->cmd_bhs
->iscsi_hdr
;
3817 task
->hdr_max
= sizeof(struct be_cmd_bhs
);
3818 io_task
->psgl_handle
= NULL
;
3819 io_task
->psgl_handle
= NULL
;
3822 spin_lock(&phba
->io_sgl_lock
);
3823 io_task
->psgl_handle
= alloc_io_sgl_handle(phba
);
3824 spin_unlock(&phba
->io_sgl_lock
);
3825 if (!io_task
->psgl_handle
)
3827 io_task
->pwrb_handle
= alloc_wrb_handle(phba
,
3828 beiscsi_conn
->beiscsi_conn_cid
-
3829 phba
->fw_config
.iscsi_cid_start
);
3830 if (!io_task
->pwrb_handle
)
3833 io_task
->scsi_cmnd
= NULL
;
3834 if ((opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGIN
) {
3835 if (!beiscsi_conn
->login_in_progress
) {
3836 spin_lock(&phba
->mgmt_sgl_lock
);
3837 io_task
->psgl_handle
= (struct sgl_handle
*)
3838 alloc_mgmt_sgl_handle(phba
);
3839 spin_unlock(&phba
->mgmt_sgl_lock
);
3840 if (!io_task
->psgl_handle
)
3843 beiscsi_conn
->login_in_progress
= 1;
3844 beiscsi_conn
->plogin_sgl_handle
=
3845 io_task
->psgl_handle
;
3846 io_task
->pwrb_handle
=
3847 alloc_wrb_handle(phba
,
3848 beiscsi_conn
->beiscsi_conn_cid
-
3849 phba
->fw_config
.iscsi_cid_start
);
3850 if (!io_task
->pwrb_handle
)
3852 beiscsi_conn
->plogin_wrb_handle
=
3853 io_task
->pwrb_handle
;
3856 io_task
->psgl_handle
=
3857 beiscsi_conn
->plogin_sgl_handle
;
3858 io_task
->pwrb_handle
=
3859 beiscsi_conn
->plogin_wrb_handle
;
3862 spin_lock(&phba
->mgmt_sgl_lock
);
3863 io_task
->psgl_handle
= alloc_mgmt_sgl_handle(phba
);
3864 spin_unlock(&phba
->mgmt_sgl_lock
);
3865 if (!io_task
->psgl_handle
)
3867 io_task
->pwrb_handle
=
3868 alloc_wrb_handle(phba
,
3869 beiscsi_conn
->beiscsi_conn_cid
-
3870 phba
->fw_config
.iscsi_cid_start
);
3871 if (!io_task
->pwrb_handle
)
3872 goto free_mgmt_hndls
;
3876 itt
= (itt_t
) cpu_to_be32(((unsigned int)io_task
->pwrb_handle
->
3877 wrb_index
<< 16) | (unsigned int)
3878 (io_task
->psgl_handle
->sgl_index
));
3879 io_task
->pwrb_handle
->pio_handle
= task
;
3881 io_task
->cmd_bhs
->iscsi_hdr
.itt
= itt
;
3885 spin_lock(&phba
->io_sgl_lock
);
3886 free_io_sgl_handle(phba
, io_task
->psgl_handle
);
3887 spin_unlock(&phba
->io_sgl_lock
);
3890 spin_lock(&phba
->mgmt_sgl_lock
);
3891 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
3892 spin_unlock(&phba
->mgmt_sgl_lock
);
3894 phwi_ctrlr
= phba
->phwi_ctrlr
;
3895 pwrb_context
= &phwi_ctrlr
->wrb_context
[
3896 beiscsi_conn
->beiscsi_conn_cid
-
3897 phba
->fw_config
.iscsi_cid_start
];
3898 if (io_task
->pwrb_handle
)
3899 free_wrb_handle(phba
, pwrb_context
, io_task
->pwrb_handle
);
3900 io_task
->pwrb_handle
= NULL
;
3901 pci_pool_free(beiscsi_sess
->bhs_pool
, io_task
->cmd_bhs
,
3902 io_task
->bhs_pa
.u
.a64
.address
);
3903 SE_DEBUG(DBG_LVL_1
, "Alloc of SGL_ICD Failed\n");
3907 static void beiscsi_cleanup_task(struct iscsi_task
*task
)
3909 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3910 struct iscsi_conn
*conn
= task
->conn
;
3911 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3912 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3913 struct beiscsi_session
*beiscsi_sess
= beiscsi_conn
->beiscsi_sess
;
3914 struct hwi_wrb_context
*pwrb_context
;
3915 struct hwi_controller
*phwi_ctrlr
;
3917 phwi_ctrlr
= phba
->phwi_ctrlr
;
3918 pwrb_context
= &phwi_ctrlr
->wrb_context
[beiscsi_conn
->beiscsi_conn_cid
3919 - phba
->fw_config
.iscsi_cid_start
];
3920 if (io_task
->pwrb_handle
) {
3921 free_wrb_handle(phba
, pwrb_context
, io_task
->pwrb_handle
);
3922 io_task
->pwrb_handle
= NULL
;
3925 if (io_task
->cmd_bhs
) {
3926 pci_pool_free(beiscsi_sess
->bhs_pool
, io_task
->cmd_bhs
,
3927 io_task
->bhs_pa
.u
.a64
.address
);
3931 if (io_task
->psgl_handle
) {
3932 spin_lock(&phba
->io_sgl_lock
);
3933 free_io_sgl_handle(phba
, io_task
->psgl_handle
);
3934 spin_unlock(&phba
->io_sgl_lock
);
3935 io_task
->psgl_handle
= NULL
;
3939 ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGIN
))
3941 if (io_task
->psgl_handle
) {
3942 spin_lock(&phba
->mgmt_sgl_lock
);
3943 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
3944 spin_unlock(&phba
->mgmt_sgl_lock
);
3945 io_task
->psgl_handle
= NULL
;
3950 static int beiscsi_iotask(struct iscsi_task
*task
, struct scatterlist
*sg
,
3951 unsigned int num_sg
, unsigned int xferlen
,
3952 unsigned int writedir
)
3955 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3956 struct iscsi_conn
*conn
= task
->conn
;
3957 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3958 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3959 struct iscsi_wrb
*pwrb
= NULL
;
3960 unsigned int doorbell
= 0;
3962 pwrb
= io_task
->pwrb_handle
->pwrb
;
3963 io_task
->cmd_bhs
->iscsi_hdr
.exp_statsn
= 0;
3964 io_task
->bhs_len
= sizeof(struct be_cmd_bhs
);
3967 memset(&io_task
->cmd_bhs
->iscsi_data_pdu
, 0, 48);
3968 AMAP_SET_BITS(struct amap_pdu_data_out
, itt
,
3969 &io_task
->cmd_bhs
->iscsi_data_pdu
,
3970 (unsigned int)io_task
->cmd_bhs
->iscsi_hdr
.itt
);
3971 AMAP_SET_BITS(struct amap_pdu_data_out
, opcode
,
3972 &io_task
->cmd_bhs
->iscsi_data_pdu
,
3973 ISCSI_OPCODE_SCSI_DATA_OUT
);
3974 AMAP_SET_BITS(struct amap_pdu_data_out
, final_bit
,
3975 &io_task
->cmd_bhs
->iscsi_data_pdu
, 1);
3976 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3978 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 1);
3980 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3982 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
3984 memcpy(&io_task
->cmd_bhs
->iscsi_data_pdu
.
3985 dw
[offsetof(struct amap_pdu_data_out
, lun
) / 32],
3986 &io_task
->cmd_bhs
->iscsi_hdr
.lun
, sizeof(struct scsi_lun
));
3988 AMAP_SET_BITS(struct amap_iscsi_wrb
, lun
, pwrb
,
3989 cpu_to_be16(*(unsigned short *)&io_task
->cmd_bhs
->iscsi_hdr
.lun
));
3990 AMAP_SET_BITS(struct amap_iscsi_wrb
, r2t_exp_dtl
, pwrb
, xferlen
);
3991 AMAP_SET_BITS(struct amap_iscsi_wrb
, wrb_idx
, pwrb
,
3992 io_task
->pwrb_handle
->wrb_index
);
3993 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
,
3994 be32_to_cpu(task
->cmdsn
));
3995 AMAP_SET_BITS(struct amap_iscsi_wrb
, sgl_icd_idx
, pwrb
,
3996 io_task
->psgl_handle
->sgl_index
);
3998 hwi_write_sgl(pwrb
, sg
, num_sg
, io_task
);
4000 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
, pwrb
,
4001 io_task
->pwrb_handle
->nxt_wrb_index
);
4002 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_wrb
));
4004 doorbell
|= beiscsi_conn
->beiscsi_conn_cid
& DB_WRB_POST_CID_MASK
;
4005 doorbell
|= (io_task
->pwrb_handle
->wrb_index
&
4006 DB_DEF_PDU_WRB_INDEX_MASK
) << DB_DEF_PDU_WRB_INDEX_SHIFT
;
4007 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
4009 iowrite32(doorbell
, phba
->db_va
+ DB_TXULP0_OFFSET
);
4013 static int beiscsi_mtask(struct iscsi_task
*task
)
4015 struct beiscsi_io_task
*io_task
= task
->dd_data
;
4016 struct iscsi_conn
*conn
= task
->conn
;
4017 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
4018 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
4019 struct iscsi_wrb
*pwrb
= NULL
;
4020 unsigned int doorbell
= 0;
4023 cid
= beiscsi_conn
->beiscsi_conn_cid
;
4024 pwrb
= io_task
->pwrb_handle
->pwrb
;
4025 memset(pwrb
, 0, sizeof(*pwrb
));
4026 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
,
4027 be32_to_cpu(task
->cmdsn
));
4028 AMAP_SET_BITS(struct amap_iscsi_wrb
, wrb_idx
, pwrb
,
4029 io_task
->pwrb_handle
->wrb_index
);
4030 AMAP_SET_BITS(struct amap_iscsi_wrb
, sgl_icd_idx
, pwrb
,
4031 io_task
->psgl_handle
->sgl_index
);
4033 switch (task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) {
4034 case ISCSI_OP_LOGIN
:
4035 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4037 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4038 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
, 1);
4039 hwi_write_buffer(pwrb
, task
);
4041 case ISCSI_OP_NOOP_OUT
:
4042 if (task
->hdr
->ttt
!= ISCSI_RESERVED_TAG
) {
4043 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4045 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
,
4047 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4049 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4051 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 1);
4053 hwi_write_buffer(pwrb
, task
);
4056 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4058 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4059 hwi_write_buffer(pwrb
, task
);
4061 case ISCSI_OP_SCSI_TMFUNC
:
4062 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4064 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4065 hwi_write_buffer(pwrb
, task
);
4067 case ISCSI_OP_LOGOUT
:
4068 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4069 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4071 hwi_write_buffer(pwrb
, task
);
4075 SE_DEBUG(DBG_LVL_1
, "opcode =%d Not supported\n",
4076 task
->hdr
->opcode
& ISCSI_OPCODE_MASK
);
4080 AMAP_SET_BITS(struct amap_iscsi_wrb
, r2t_exp_dtl
, pwrb
,
4082 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
, pwrb
,
4083 io_task
->pwrb_handle
->nxt_wrb_index
);
4084 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_wrb
));
4086 doorbell
|= cid
& DB_WRB_POST_CID_MASK
;
4087 doorbell
|= (io_task
->pwrb_handle
->wrb_index
&
4088 DB_DEF_PDU_WRB_INDEX_MASK
) << DB_DEF_PDU_WRB_INDEX_SHIFT
;
4089 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
4090 iowrite32(doorbell
, phba
->db_va
+ DB_TXULP0_OFFSET
);
4094 static int beiscsi_task_xmit(struct iscsi_task
*task
)
4096 struct beiscsi_io_task
*io_task
= task
->dd_data
;
4097 struct scsi_cmnd
*sc
= task
->sc
;
4098 struct scatterlist
*sg
;
4100 unsigned int writedir
= 0, xferlen
= 0;
4103 return beiscsi_mtask(task
);
4105 io_task
->scsi_cmnd
= sc
;
4106 num_sg
= scsi_dma_map(sc
);
4108 SE_DEBUG(DBG_LVL_1
, " scsi_dma_map Failed\n")
4111 xferlen
= scsi_bufflen(sc
);
4112 sg
= scsi_sglist(sc
);
4113 if (sc
->sc_data_direction
== DMA_TO_DEVICE
) {
4115 SE_DEBUG(DBG_LVL_4
, "task->imm_count=0x%08x\n",
4119 return beiscsi_iotask(task
, sg
, num_sg
, xferlen
, writedir
);
4122 static void beiscsi_quiesce(struct beiscsi_hba
*phba
)
4124 struct hwi_controller
*phwi_ctrlr
;
4125 struct hwi_context_memory
*phwi_context
;
4126 struct be_eq_obj
*pbe_eq
;
4127 unsigned int i
, msix_vec
;
4128 u8
*real_offset
= 0;
4131 phwi_ctrlr
= phba
->phwi_ctrlr
;
4132 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
4133 hwi_disable_intr(phba
);
4134 if (phba
->msix_enabled
) {
4135 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
4136 msix_vec
= phba
->msix_entries
[i
].vector
;
4137 free_irq(msix_vec
, &phwi_context
->be_eq
[i
]);
4138 kfree(phba
->msi_name
[i
]);
4141 if (phba
->pcidev
->irq
)
4142 free_irq(phba
->pcidev
->irq
, phba
);
4143 pci_disable_msix(phba
->pcidev
);
4144 destroy_workqueue(phba
->wq
);
4145 if (blk_iopoll_enabled
)
4146 for (i
= 0; i
< phba
->num_cpus
; i
++) {
4147 pbe_eq
= &phwi_context
->be_eq
[i
];
4148 blk_iopoll_disable(&pbe_eq
->iopoll
);
4151 beiscsi_clean_port(phba
);
4152 beiscsi_free_mem(phba
);
4153 real_offset
= (u8
*)phba
->csr_va
+ MPU_EP_SEMAPHORE
;
4155 value
= readl((void *)real_offset
);
4157 if (value
& 0x00010000) {
4158 value
&= 0xfffeffff;
4159 writel(value
, (void *)real_offset
);
4161 beiscsi_unmap_pci_function(phba
);
4162 pci_free_consistent(phba
->pcidev
,
4163 phba
->ctrl
.mbox_mem_alloced
.size
,
4164 phba
->ctrl
.mbox_mem_alloced
.va
,
4165 phba
->ctrl
.mbox_mem_alloced
.dma
);
4168 static void beiscsi_remove(struct pci_dev
*pcidev
)
4171 struct beiscsi_hba
*phba
= NULL
;
4173 phba
= pci_get_drvdata(pcidev
);
4175 dev_err(&pcidev
->dev
, "beiscsi_remove called with no phba\n");
4179 beiscsi_quiesce(phba
);
4180 iscsi_boot_destroy_kset(phba
->boot_kset
);
4181 iscsi_host_remove(phba
->shost
);
4182 pci_dev_put(phba
->pcidev
);
4183 iscsi_host_free(phba
->shost
);
4184 pci_disable_device(pcidev
);
4187 static void beiscsi_shutdown(struct pci_dev
*pcidev
)
4190 struct beiscsi_hba
*phba
= NULL
;
4192 phba
= (struct beiscsi_hba
*)pci_get_drvdata(pcidev
);
4194 dev_err(&pcidev
->dev
, "beiscsi_shutdown called with no phba\n");
4198 beiscsi_quiesce(phba
);
4199 pci_disable_device(pcidev
);
4202 static void beiscsi_msix_enable(struct beiscsi_hba
*phba
)
4206 for (i
= 0; i
<= phba
->num_cpus
; i
++)
4207 phba
->msix_entries
[i
].entry
= i
;
4209 status
= pci_enable_msix(phba
->pcidev
, phba
->msix_entries
,
4210 (phba
->num_cpus
+ 1));
4212 phba
->msix_enabled
= true;
4217 static int __devinit
beiscsi_dev_probe(struct pci_dev
*pcidev
,
4218 const struct pci_device_id
*id
)
4220 struct beiscsi_hba
*phba
= NULL
;
4221 struct hwi_controller
*phwi_ctrlr
;
4222 struct hwi_context_memory
*phwi_context
;
4223 struct be_eq_obj
*pbe_eq
;
4224 int ret
, num_cpus
, i
;
4225 u8
*real_offset
= 0;
4228 ret
= beiscsi_enable_pci(pcidev
);
4230 dev_err(&pcidev
->dev
, "beiscsi_dev_probe-"
4231 " Failed to enable pci device\n");
4235 phba
= beiscsi_hba_alloc(pcidev
);
4237 dev_err(&pcidev
->dev
, "beiscsi_dev_probe-"
4238 " Failed in beiscsi_hba_alloc\n");
4242 switch (pcidev
->device
) {
4246 phba
->generation
= BE_GEN2
;
4250 phba
->generation
= BE_GEN3
;
4253 phba
->generation
= 0;
4257 num_cpus
= find_num_cpus();
4260 phba
->num_cpus
= num_cpus
;
4261 SE_DEBUG(DBG_LVL_8
, "num_cpus = %d\n", phba
->num_cpus
);
4264 beiscsi_msix_enable(phba
);
4265 ret
= be_ctrl_init(phba
, pcidev
);
4267 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
4268 "Failed in be_ctrl_init\n");
4273 real_offset
= (u8
*)phba
->csr_va
+ MPU_EP_SEMAPHORE
;
4274 value
= readl((void *)real_offset
);
4275 if (value
& 0x00010000) {
4277 shost_printk(KERN_ERR
, phba
->shost
,
4278 "Loading Driver in crashdump mode\n");
4279 ret
= beiscsi_pci_soft_reset(phba
);
4281 shost_printk(KERN_ERR
, phba
->shost
,
4282 "Reset Failed. Aborting Crashdump\n");
4285 ret
= be_chk_reset_complete(phba
);
4287 shost_printk(KERN_ERR
, phba
->shost
,
4288 "Failed to get out of reset."
4289 "Aborting Crashdump\n");
4293 value
|= 0x00010000;
4294 writel(value
, (void *)real_offset
);
4299 spin_lock_init(&phba
->io_sgl_lock
);
4300 spin_lock_init(&phba
->mgmt_sgl_lock
);
4301 spin_lock_init(&phba
->isr_lock
);
4302 ret
= mgmt_get_fw_config(&phba
->ctrl
, phba
);
4304 shost_printk(KERN_ERR
, phba
->shost
,
4305 "Error getting fw config\n");
4308 phba
->shost
->max_id
= phba
->fw_config
.iscsi_cid_count
;
4309 beiscsi_get_params(phba
);
4310 phba
->shost
->can_queue
= phba
->params
.ios_per_ctrl
;
4311 ret
= beiscsi_init_port(phba
);
4313 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
4314 "Failed in beiscsi_init_port\n");
4318 for (i
= 0; i
< MAX_MCC_CMD
; i
++) {
4319 init_waitqueue_head(&phba
->ctrl
.mcc_wait
[i
+ 1]);
4320 phba
->ctrl
.mcc_tag
[i
] = i
+ 1;
4321 phba
->ctrl
.mcc_numtag
[i
+ 1] = 0;
4322 phba
->ctrl
.mcc_tag_available
++;
4325 phba
->ctrl
.mcc_alloc_index
= phba
->ctrl
.mcc_free_index
= 0;
4327 snprintf(phba
->wq_name
, sizeof(phba
->wq_name
), "beiscsi_q_irq%u",
4328 phba
->shost
->host_no
);
4329 phba
->wq
= alloc_workqueue(phba
->wq_name
, WQ_MEM_RECLAIM
, 1);
4331 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
4332 "Failed to allocate work queue\n");
4336 INIT_WORK(&phba
->work_cqs
, beiscsi_process_all_cqs
);
4338 phwi_ctrlr
= phba
->phwi_ctrlr
;
4339 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
4340 if (blk_iopoll_enabled
) {
4341 for (i
= 0; i
< phba
->num_cpus
; i
++) {
4342 pbe_eq
= &phwi_context
->be_eq
[i
];
4343 blk_iopoll_init(&pbe_eq
->iopoll
, be_iopoll_budget
,
4345 blk_iopoll_enable(&pbe_eq
->iopoll
);
4348 ret
= beiscsi_init_irqs(phba
);
4350 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
4351 "Failed to beiscsi_init_irqs\n");
4354 hwi_enable_intr(phba
);
4356 if (beiscsi_setup_boot_info(phba
))
4358 * log error but continue, because we may not be using
4361 shost_printk(KERN_ERR
, phba
->shost
, "Could not set up "
4362 "iSCSI boot info.");
4364 SE_DEBUG(DBG_LVL_8
, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n");
4368 destroy_workqueue(phba
->wq
);
4369 if (blk_iopoll_enabled
)
4370 for (i
= 0; i
< phba
->num_cpus
; i
++) {
4371 pbe_eq
= &phwi_context
->be_eq
[i
];
4372 blk_iopoll_disable(&pbe_eq
->iopoll
);
4375 beiscsi_clean_port(phba
);
4376 beiscsi_free_mem(phba
);
4378 real_offset
= (u8
*)phba
->csr_va
+ MPU_EP_SEMAPHORE
;
4380 value
= readl((void *)real_offset
);
4382 if (value
& 0x00010000) {
4383 value
&= 0xfffeffff;
4384 writel(value
, (void *)real_offset
);
4387 pci_free_consistent(phba
->pcidev
,
4388 phba
->ctrl
.mbox_mem_alloced
.size
,
4389 phba
->ctrl
.mbox_mem_alloced
.va
,
4390 phba
->ctrl
.mbox_mem_alloced
.dma
);
4391 beiscsi_unmap_pci_function(phba
);
4393 if (phba
->msix_enabled
)
4394 pci_disable_msix(phba
->pcidev
);
4395 iscsi_host_remove(phba
->shost
);
4396 pci_dev_put(phba
->pcidev
);
4397 iscsi_host_free(phba
->shost
);
4399 pci_disable_device(pcidev
);
4403 struct iscsi_transport beiscsi_iscsi_transport
= {
4404 .owner
= THIS_MODULE
,
4406 .caps
= CAP_RECOVERY_L0
| CAP_HDRDGST
| CAP_TEXT_NEGO
|
4407 CAP_MULTI_R2T
| CAP_DATADGST
| CAP_DATA_PATH_OFFLOAD
,
4408 .create_session
= beiscsi_session_create
,
4409 .destroy_session
= beiscsi_session_destroy
,
4410 .create_conn
= beiscsi_conn_create
,
4411 .bind_conn
= beiscsi_conn_bind
,
4412 .destroy_conn
= iscsi_conn_teardown
,
4413 .attr_is_visible
= be2iscsi_attr_is_visible
,
4414 .set_param
= beiscsi_set_param
,
4415 .get_conn_param
= iscsi_conn_get_param
,
4416 .get_session_param
= iscsi_session_get_param
,
4417 .get_host_param
= beiscsi_get_host_param
,
4418 .start_conn
= beiscsi_conn_start
,
4419 .stop_conn
= iscsi_conn_stop
,
4420 .send_pdu
= iscsi_conn_send_pdu
,
4421 .xmit_task
= beiscsi_task_xmit
,
4422 .cleanup_task
= beiscsi_cleanup_task
,
4423 .alloc_pdu
= beiscsi_alloc_pdu
,
4424 .parse_pdu_itt
= beiscsi_parse_pdu
,
4425 .get_stats
= beiscsi_conn_get_stats
,
4426 .get_ep_param
= beiscsi_ep_get_param
,
4427 .ep_connect
= beiscsi_ep_connect
,
4428 .ep_poll
= beiscsi_ep_poll
,
4429 .ep_disconnect
= beiscsi_ep_disconnect
,
4430 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
4433 static struct pci_driver beiscsi_pci_driver
= {
4435 .probe
= beiscsi_dev_probe
,
4436 .remove
= beiscsi_remove
,
4437 .shutdown
= beiscsi_shutdown
,
4438 .id_table
= beiscsi_pci_id_table
4442 static int __init
beiscsi_module_init(void)
4446 beiscsi_scsi_transport
=
4447 iscsi_register_transport(&beiscsi_iscsi_transport
);
4448 if (!beiscsi_scsi_transport
) {
4450 "beiscsi_module_init - Unable to register beiscsi"
4454 SE_DEBUG(DBG_LVL_8
, "In beiscsi_module_init, tt=%p\n",
4455 &beiscsi_iscsi_transport
);
4457 ret
= pci_register_driver(&beiscsi_pci_driver
);
4460 "beiscsi_module_init - Unable to register"
4461 "beiscsi pci driver.\n");
4462 goto unregister_iscsi_transport
;
4466 unregister_iscsi_transport
:
4467 iscsi_unregister_transport(&beiscsi_iscsi_transport
);
4471 static void __exit
beiscsi_module_exit(void)
4473 pci_unregister_driver(&beiscsi_pci_driver
);
4474 iscsi_unregister_transport(&beiscsi_iscsi_transport
);
4477 module_init(beiscsi_module_init
);
4478 module_exit(beiscsi_module_exit
);