2 * Copyright (C) 2005 - 2010 ServerEngines
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version 2
7 * as published by the Free Software Foundation. The full GNU General
8 * Public License is included in this distribution in the file called COPYING.
10 * Written by: Jayamohan Kallickal (jayamohank@serverengines.com)
12 * Contact Information:
13 * linux-drivers@serverengines.com
16 * 209 N. Fair Oaks Ave
20 #include <linux/reboot.h>
21 #include <linux/delay.h>
22 #include <linux/slab.h>
23 #include <linux/interrupt.h>
24 #include <linux/blkdev.h>
25 #include <linux/pci.h>
26 #include <linux/string.h>
27 #include <linux/kernel.h>
28 #include <linux/semaphore.h>
29 #include <linux/iscsi_boot_sysfs.h>
31 #include <scsi/libiscsi.h>
32 #include <scsi/scsi_transport_iscsi.h>
33 #include <scsi/scsi_transport.h>
34 #include <scsi/scsi_cmnd.h>
35 #include <scsi/scsi_device.h>
36 #include <scsi/scsi_host.h>
37 #include <scsi/scsi.h>
42 static unsigned int be_iopoll_budget
= 10;
43 static unsigned int be_max_phys_size
= 64;
44 static unsigned int enable_msix
= 1;
45 static unsigned int gcrashmode
= 0;
46 static unsigned int num_hba
= 0;
48 MODULE_DEVICE_TABLE(pci
, beiscsi_pci_id_table
);
49 MODULE_DESCRIPTION(DRV_DESC
" " BUILD_STR
);
50 MODULE_AUTHOR("ServerEngines Corporation");
51 MODULE_LICENSE("GPL");
52 module_param(be_iopoll_budget
, int, 0);
53 module_param(enable_msix
, int, 0);
54 module_param(be_max_phys_size
, uint
, S_IRUGO
);
55 MODULE_PARM_DESC(be_max_phys_size
, "Maximum Size (In Kilobytes) of physically"
56 "contiguous memory that can be allocated."
59 static int beiscsi_slave_configure(struct scsi_device
*sdev
)
61 blk_queue_max_segment_size(sdev
->request_queue
, 65536);
65 static int beiscsi_eh_abort(struct scsi_cmnd
*sc
)
67 struct iscsi_cls_session
*cls_session
;
68 struct iscsi_task
*aborted_task
= (struct iscsi_task
*)sc
->SCp
.ptr
;
69 struct beiscsi_io_task
*aborted_io_task
;
70 struct iscsi_conn
*conn
;
71 struct beiscsi_conn
*beiscsi_conn
;
72 struct beiscsi_hba
*phba
;
73 struct iscsi_session
*session
;
74 struct invalidate_command_table
*inv_tbl
;
75 struct be_dma_mem nonemb_cmd
;
76 unsigned int cid
, tag
, num_invalidate
;
78 cls_session
= starget_to_session(scsi_target(sc
->device
));
79 session
= cls_session
->dd_data
;
81 spin_lock_bh(&session
->lock
);
82 if (!aborted_task
|| !aborted_task
->sc
) {
84 spin_unlock_bh(&session
->lock
);
88 aborted_io_task
= aborted_task
->dd_data
;
89 if (!aborted_io_task
->scsi_cmnd
) {
90 /* raced or invalid command */
91 spin_unlock_bh(&session
->lock
);
94 spin_unlock_bh(&session
->lock
);
95 conn
= aborted_task
->conn
;
96 beiscsi_conn
= conn
->dd_data
;
97 phba
= beiscsi_conn
->phba
;
100 cid
= beiscsi_conn
->beiscsi_conn_cid
;
101 inv_tbl
= phba
->inv_tbl
;
102 memset(inv_tbl
, 0x0, sizeof(*inv_tbl
));
104 inv_tbl
->icd
= aborted_io_task
->psgl_handle
->sgl_index
;
106 nonemb_cmd
.va
= pci_alloc_consistent(phba
->ctrl
.pdev
,
107 sizeof(struct invalidate_commands_params_in
),
109 if (nonemb_cmd
.va
== NULL
) {
111 "Failed to allocate memory for"
112 "mgmt_invalidate_icds\n");
115 nonemb_cmd
.size
= sizeof(struct invalidate_commands_params_in
);
117 tag
= mgmt_invalidate_icds(phba
, inv_tbl
, num_invalidate
,
120 shost_printk(KERN_WARNING
, phba
->shost
,
121 "mgmt_invalidate_icds could not be"
123 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
124 nonemb_cmd
.va
, nonemb_cmd
.dma
);
128 wait_event_interruptible(phba
->ctrl
.mcc_wait
[tag
],
129 phba
->ctrl
.mcc_numtag
[tag
]);
130 free_mcc_tag(&phba
->ctrl
, tag
);
132 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
133 nonemb_cmd
.va
, nonemb_cmd
.dma
);
134 return iscsi_eh_abort(sc
);
137 static int beiscsi_eh_device_reset(struct scsi_cmnd
*sc
)
139 struct iscsi_task
*abrt_task
;
140 struct beiscsi_io_task
*abrt_io_task
;
141 struct iscsi_conn
*conn
;
142 struct beiscsi_conn
*beiscsi_conn
;
143 struct beiscsi_hba
*phba
;
144 struct iscsi_session
*session
;
145 struct iscsi_cls_session
*cls_session
;
146 struct invalidate_command_table
*inv_tbl
;
147 struct be_dma_mem nonemb_cmd
;
148 unsigned int cid
, tag
, i
, num_invalidate
;
151 /* invalidate iocbs */
152 cls_session
= starget_to_session(scsi_target(sc
->device
));
153 session
= cls_session
->dd_data
;
154 spin_lock_bh(&session
->lock
);
155 if (!session
->leadconn
|| session
->state
!= ISCSI_STATE_LOGGED_IN
)
158 conn
= session
->leadconn
;
159 beiscsi_conn
= conn
->dd_data
;
160 phba
= beiscsi_conn
->phba
;
161 cid
= beiscsi_conn
->beiscsi_conn_cid
;
162 inv_tbl
= phba
->inv_tbl
;
163 memset(inv_tbl
, 0x0, sizeof(*inv_tbl
) * BE2_CMDS_PER_CXN
);
165 for (i
= 0; i
< conn
->session
->cmds_max
; i
++) {
166 abrt_task
= conn
->session
->cmds
[i
];
167 abrt_io_task
= abrt_task
->dd_data
;
168 if (!abrt_task
->sc
|| abrt_task
->state
== ISCSI_TASK_FREE
)
171 if (abrt_task
->sc
->device
->lun
!= abrt_task
->sc
->device
->lun
)
175 inv_tbl
->icd
= abrt_io_task
->psgl_handle
->sgl_index
;
179 spin_unlock_bh(&session
->lock
);
180 inv_tbl
= phba
->inv_tbl
;
182 nonemb_cmd
.va
= pci_alloc_consistent(phba
->ctrl
.pdev
,
183 sizeof(struct invalidate_commands_params_in
),
185 if (nonemb_cmd
.va
== NULL
) {
187 "Failed to allocate memory for"
188 "mgmt_invalidate_icds\n");
191 nonemb_cmd
.size
= sizeof(struct invalidate_commands_params_in
);
192 memset(nonemb_cmd
.va
, 0, nonemb_cmd
.size
);
193 tag
= mgmt_invalidate_icds(phba
, inv_tbl
, num_invalidate
,
196 shost_printk(KERN_WARNING
, phba
->shost
,
197 "mgmt_invalidate_icds could not be"
199 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
200 nonemb_cmd
.va
, nonemb_cmd
.dma
);
203 wait_event_interruptible(phba
->ctrl
.mcc_wait
[tag
],
204 phba
->ctrl
.mcc_numtag
[tag
]);
205 free_mcc_tag(&phba
->ctrl
, tag
);
207 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
208 nonemb_cmd
.va
, nonemb_cmd
.dma
);
209 return iscsi_eh_device_reset(sc
);
211 spin_unlock_bh(&session
->lock
);
215 static ssize_t
beiscsi_show_boot_tgt_info(void *data
, int type
, char *buf
)
217 struct beiscsi_hba
*phba
= data
;
222 case ISCSI_BOOT_TGT_NAME
:
223 rc
= sprintf(buf
, "%.*s\n",
224 (int)strlen(phba
->boot_sess
.target_name
),
225 (char *)&phba
->boot_sess
.target_name
);
227 case ISCSI_BOOT_TGT_IP_ADDR
:
228 if (phba
->boot_sess
.conn_list
[0].dest_ipaddr
.ip_type
== 0x1)
229 rc
= sprintf(buf
, "%pI4\n",
230 (char *)&phba
->boot_sess
.conn_list
[0].
231 dest_ipaddr
.ip_address
);
233 rc
= sprintf(str
, "%pI6\n",
234 (char *)&phba
->boot_sess
.conn_list
[0].
235 dest_ipaddr
.ip_address
);
237 case ISCSI_BOOT_TGT_PORT
:
238 rc
= sprintf(str
, "%d\n", phba
->boot_sess
.conn_list
[0].
242 case ISCSI_BOOT_TGT_CHAP_NAME
:
243 rc
= sprintf(str
, "%.*s\n",
244 phba
->boot_sess
.conn_list
[0].
245 negotiated_login_options
.auth_data
.chap
.
246 target_chap_name_length
,
247 (char *)&phba
->boot_sess
.conn_list
[0].
248 negotiated_login_options
.auth_data
.chap
.
251 case ISCSI_BOOT_TGT_CHAP_SECRET
:
252 rc
= sprintf(str
, "%.*s\n",
253 phba
->boot_sess
.conn_list
[0].
254 negotiated_login_options
.auth_data
.chap
.
255 target_secret_length
,
256 (char *)&phba
->boot_sess
.conn_list
[0].
257 negotiated_login_options
.auth_data
.chap
.
261 case ISCSI_BOOT_TGT_REV_CHAP_NAME
:
262 rc
= sprintf(str
, "%.*s\n",
263 phba
->boot_sess
.conn_list
[0].
264 negotiated_login_options
.auth_data
.chap
.
265 intr_chap_name_length
,
266 (char *)&phba
->boot_sess
.conn_list
[0].
267 negotiated_login_options
.auth_data
.chap
.
271 case ISCSI_BOOT_TGT_REV_CHAP_SECRET
:
272 rc
= sprintf(str
, "%.*s\n",
273 phba
->boot_sess
.conn_list
[0].
274 negotiated_login_options
.auth_data
.chap
.
276 (char *)&phba
->boot_sess
.conn_list
[0].
277 negotiated_login_options
.auth_data
.chap
.
280 case ISCSI_BOOT_TGT_FLAGS
:
281 rc
= sprintf(str
, "2\n");
283 case ISCSI_BOOT_TGT_NIC_ASSOC
:
284 rc
= sprintf(str
, "0\n");
293 static ssize_t
beiscsi_show_boot_ini_info(void *data
, int type
, char *buf
)
295 struct beiscsi_hba
*phba
= data
;
300 case ISCSI_BOOT_INI_INITIATOR_NAME
:
301 rc
= sprintf(str
, "%s\n", phba
->boot_sess
.initiator_iscsiname
);
310 static ssize_t
beiscsi_show_boot_eth_info(void *data
, int type
, char *buf
)
312 struct beiscsi_hba
*phba
= data
;
317 case ISCSI_BOOT_ETH_FLAGS
:
318 rc
= sprintf(str
, "2\n");
320 case ISCSI_BOOT_ETH_INDEX
:
321 rc
= sprintf(str
, "0\n");
323 case ISCSI_BOOT_ETH_MAC
:
324 rc
= beiscsi_get_macaddr(buf
, phba
);
326 SE_DEBUG(DBG_LVL_1
, "beiscsi_get_macaddr Failed\n");
338 static mode_t
beiscsi_tgt_get_attr_visibility(void *data
, int type
)
343 case ISCSI_BOOT_TGT_NAME
:
344 case ISCSI_BOOT_TGT_IP_ADDR
:
345 case ISCSI_BOOT_TGT_PORT
:
346 case ISCSI_BOOT_TGT_CHAP_NAME
:
347 case ISCSI_BOOT_TGT_CHAP_SECRET
:
348 case ISCSI_BOOT_TGT_REV_CHAP_NAME
:
349 case ISCSI_BOOT_TGT_REV_CHAP_SECRET
:
350 case ISCSI_BOOT_TGT_NIC_ASSOC
:
351 case ISCSI_BOOT_TGT_FLAGS
:
361 static mode_t
beiscsi_ini_get_attr_visibility(void *data
, int type
)
366 case ISCSI_BOOT_INI_INITIATOR_NAME
:
377 static mode_t
beiscsi_eth_get_attr_visibility(void *data
, int type
)
382 case ISCSI_BOOT_ETH_FLAGS
:
383 case ISCSI_BOOT_ETH_MAC
:
384 case ISCSI_BOOT_ETH_INDEX
:
394 static int beiscsi_setup_boot_info(struct beiscsi_hba
*phba
)
396 struct iscsi_boot_kobj
*boot_kobj
;
398 phba
->boot_kset
= iscsi_boot_create_host_kset(phba
->shost
->host_no
);
399 if (!phba
->boot_kset
)
402 /* get boot info using mgmt cmd */
403 boot_kobj
= iscsi_boot_create_target(phba
->boot_kset
, 0, phba
,
404 beiscsi_show_boot_tgt_info
,
405 beiscsi_tgt_get_attr_visibility
);
409 boot_kobj
= iscsi_boot_create_initiator(phba
->boot_kset
, 0, phba
,
410 beiscsi_show_boot_ini_info
,
411 beiscsi_ini_get_attr_visibility
);
415 boot_kobj
= iscsi_boot_create_ethernet(phba
->boot_kset
, 0, phba
,
416 beiscsi_show_boot_eth_info
,
417 beiscsi_eth_get_attr_visibility
);
423 iscsi_boot_destroy_kset(phba
->boot_kset
);
427 /*------------------- PCI Driver operations and data ----------------- */
428 static DEFINE_PCI_DEVICE_TABLE(beiscsi_pci_id_table
) = {
429 { PCI_DEVICE(BE_VENDOR_ID
, BE_DEVICE_ID1
) },
430 { PCI_DEVICE(BE_VENDOR_ID
, BE_DEVICE_ID2
) },
431 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID1
) },
432 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID2
) },
433 { PCI_DEVICE(BE_VENDOR_ID
, OC_DEVICE_ID3
) },
436 MODULE_DEVICE_TABLE(pci
, beiscsi_pci_id_table
);
438 static struct scsi_host_template beiscsi_sht
= {
439 .module
= THIS_MODULE
,
440 .name
= "ServerEngines 10Gbe open-iscsi Initiator Driver",
441 .proc_name
= DRV_NAME
,
442 .queuecommand
= iscsi_queuecommand
,
443 .change_queue_depth
= iscsi_change_queue_depth
,
444 .slave_configure
= beiscsi_slave_configure
,
445 .target_alloc
= iscsi_target_alloc
,
446 .eh_abort_handler
= beiscsi_eh_abort
,
447 .eh_device_reset_handler
= beiscsi_eh_device_reset
,
448 .eh_target_reset_handler
= iscsi_eh_session_reset
,
449 .sg_tablesize
= BEISCSI_SGLIST_ELEMENTS
,
450 .can_queue
= BE2_IO_DEPTH
,
452 .max_sectors
= BEISCSI_MAX_SECTORS
,
453 .cmd_per_lun
= BEISCSI_CMD_PER_LUN
,
454 .use_clustering
= ENABLE_CLUSTERING
,
457 static struct scsi_transport_template
*beiscsi_scsi_transport
;
459 static struct beiscsi_hba
*beiscsi_hba_alloc(struct pci_dev
*pcidev
)
461 struct beiscsi_hba
*phba
;
462 struct Scsi_Host
*shost
;
464 shost
= iscsi_host_alloc(&beiscsi_sht
, sizeof(*phba
), 0);
466 dev_err(&pcidev
->dev
, "beiscsi_hba_alloc -"
467 "iscsi_host_alloc failed\n");
470 shost
->dma_boundary
= pcidev
->dma_mask
;
471 shost
->max_id
= BE2_MAX_SESSIONS
;
472 shost
->max_channel
= 0;
473 shost
->max_cmd_len
= BEISCSI_MAX_CMD_LEN
;
474 shost
->max_lun
= BEISCSI_NUM_MAX_LUN
;
475 shost
->transportt
= beiscsi_scsi_transport
;
476 phba
= iscsi_host_priv(shost
);
477 memset(phba
, 0, sizeof(*phba
));
479 phba
->pcidev
= pci_dev_get(pcidev
);
480 pci_set_drvdata(pcidev
, phba
);
482 if (iscsi_host_add(shost
, &phba
->pcidev
->dev
))
485 if (beiscsi_setup_boot_info(phba
))
487 * log error but continue, because we may not be using
490 shost_printk(KERN_ERR
, phba
->shost
, "Could not set up "
496 pci_dev_put(phba
->pcidev
);
497 iscsi_host_free(phba
->shost
);
501 static void beiscsi_unmap_pci_function(struct beiscsi_hba
*phba
)
504 iounmap(phba
->csr_va
);
508 iounmap(phba
->db_va
);
512 iounmap(phba
->pci_va
);
517 static int beiscsi_map_pci_bars(struct beiscsi_hba
*phba
,
518 struct pci_dev
*pcidev
)
523 addr
= ioremap_nocache(pci_resource_start(pcidev
, 2),
524 pci_resource_len(pcidev
, 2));
527 phba
->ctrl
.csr
= addr
;
529 phba
->csr_pa
.u
.a64
.address
= pci_resource_start(pcidev
, 2);
531 addr
= ioremap_nocache(pci_resource_start(pcidev
, 4), 128 * 1024);
534 phba
->ctrl
.db
= addr
;
536 phba
->db_pa
.u
.a64
.address
= pci_resource_start(pcidev
, 4);
538 if (phba
->generation
== BE_GEN2
)
543 addr
= ioremap_nocache(pci_resource_start(pcidev
, pcicfg_reg
),
544 pci_resource_len(pcidev
, pcicfg_reg
));
548 phba
->ctrl
.pcicfg
= addr
;
550 phba
->pci_pa
.u
.a64
.address
= pci_resource_start(pcidev
, pcicfg_reg
);
554 beiscsi_unmap_pci_function(phba
);
558 static int beiscsi_enable_pci(struct pci_dev
*pcidev
)
562 ret
= pci_enable_device(pcidev
);
564 dev_err(&pcidev
->dev
, "beiscsi_enable_pci - enable device "
565 "failed. Returning -ENODEV\n");
569 pci_set_master(pcidev
);
570 if (pci_set_consistent_dma_mask(pcidev
, DMA_BIT_MASK(64))) {
571 ret
= pci_set_consistent_dma_mask(pcidev
, DMA_BIT_MASK(32));
573 dev_err(&pcidev
->dev
, "Could not set PCI DMA Mask\n");
574 pci_disable_device(pcidev
);
581 static int be_ctrl_init(struct beiscsi_hba
*phba
, struct pci_dev
*pdev
)
583 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
584 struct be_dma_mem
*mbox_mem_alloc
= &ctrl
->mbox_mem_alloced
;
585 struct be_dma_mem
*mbox_mem_align
= &ctrl
->mbox_mem
;
589 status
= beiscsi_map_pci_bars(phba
, pdev
);
592 mbox_mem_alloc
->size
= sizeof(struct be_mcc_mailbox
) + 16;
593 mbox_mem_alloc
->va
= pci_alloc_consistent(pdev
,
594 mbox_mem_alloc
->size
,
595 &mbox_mem_alloc
->dma
);
596 if (!mbox_mem_alloc
->va
) {
597 beiscsi_unmap_pci_function(phba
);
602 mbox_mem_align
->size
= sizeof(struct be_mcc_mailbox
);
603 mbox_mem_align
->va
= PTR_ALIGN(mbox_mem_alloc
->va
, 16);
604 mbox_mem_align
->dma
= PTR_ALIGN(mbox_mem_alloc
->dma
, 16);
605 memset(mbox_mem_align
->va
, 0, sizeof(struct be_mcc_mailbox
));
606 spin_lock_init(&ctrl
->mbox_lock
);
607 spin_lock_init(&phba
->ctrl
.mcc_lock
);
608 spin_lock_init(&phba
->ctrl
.mcc_cq_lock
);
613 static void beiscsi_get_params(struct beiscsi_hba
*phba
)
615 phba
->params
.ios_per_ctrl
= (phba
->fw_config
.iscsi_icd_count
616 - (phba
->fw_config
.iscsi_cid_count
619 phba
->params
.cxns_per_ctrl
= phba
->fw_config
.iscsi_cid_count
;
620 phba
->params
.asyncpdus_per_ctrl
= phba
->fw_config
.iscsi_cid_count
* 2;
621 phba
->params
.icds_per_ctrl
= phba
->fw_config
.iscsi_icd_count
;;
622 phba
->params
.num_sge_per_io
= BE2_SGE
;
623 phba
->params
.defpdu_hdr_sz
= BE2_DEFPDU_HDR_SZ
;
624 phba
->params
.defpdu_data_sz
= BE2_DEFPDU_DATA_SZ
;
625 phba
->params
.eq_timer
= 64;
626 phba
->params
.num_eq_entries
=
627 (((BE2_CMDS_PER_CXN
* 2 + phba
->fw_config
.iscsi_cid_count
* 2
628 + BE2_TMFS
) / 512) + 1) * 512;
629 phba
->params
.num_eq_entries
= (phba
->params
.num_eq_entries
< 1024)
630 ? 1024 : phba
->params
.num_eq_entries
;
631 SE_DEBUG(DBG_LVL_8
, "phba->params.num_eq_entries=%d\n",
632 phba
->params
.num_eq_entries
);
633 phba
->params
.num_cq_entries
=
634 (((BE2_CMDS_PER_CXN
* 2 + phba
->fw_config
.iscsi_cid_count
* 2
635 + BE2_TMFS
) / 512) + 1) * 512;
636 phba
->params
.wrbs_per_cxn
= 256;
639 static void hwi_ring_eq_db(struct beiscsi_hba
*phba
,
640 unsigned int id
, unsigned int clr_interrupt
,
641 unsigned int num_processed
,
642 unsigned char rearm
, unsigned char event
)
645 val
|= id
& DB_EQ_RING_ID_MASK
;
647 val
|= 1 << DB_EQ_REARM_SHIFT
;
649 val
|= 1 << DB_EQ_CLR_SHIFT
;
651 val
|= 1 << DB_EQ_EVNT_SHIFT
;
652 val
|= num_processed
<< DB_EQ_NUM_POPPED_SHIFT
;
653 iowrite32(val
, phba
->db_va
+ DB_EQ_OFFSET
);
657 * be_isr_mcc - The isr routine of the driver.
659 * @dev_id: Pointer to host adapter structure
661 static irqreturn_t
be_isr_mcc(int irq
, void *dev_id
)
663 struct beiscsi_hba
*phba
;
664 struct be_eq_entry
*eqe
= NULL
;
665 struct be_queue_info
*eq
;
666 struct be_queue_info
*mcc
;
667 unsigned int num_eq_processed
;
668 struct be_eq_obj
*pbe_eq
;
674 mcc
= &phba
->ctrl
.mcc_obj
.cq
;
675 eqe
= queue_tail_node(eq
);
677 SE_DEBUG(DBG_LVL_1
, "eqe is NULL\n");
679 num_eq_processed
= 0;
681 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
683 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
685 EQE_RESID_MASK
) >> 16) == mcc
->id
) {
686 spin_lock_irqsave(&phba
->isr_lock
, flags
);
687 phba
->todo_mcc_cq
= 1;
688 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
690 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
692 eqe
= queue_tail_node(eq
);
695 if (phba
->todo_mcc_cq
)
696 queue_work(phba
->wq
, &phba
->work_cqs
);
697 if (num_eq_processed
)
698 hwi_ring_eq_db(phba
, eq
->id
, 1, num_eq_processed
, 1, 1);
704 * be_isr_msix - The isr routine of the driver.
706 * @dev_id: Pointer to host adapter structure
708 static irqreturn_t
be_isr_msix(int irq
, void *dev_id
)
710 struct beiscsi_hba
*phba
;
711 struct be_eq_entry
*eqe
= NULL
;
712 struct be_queue_info
*eq
;
713 struct be_queue_info
*cq
;
714 unsigned int num_eq_processed
;
715 struct be_eq_obj
*pbe_eq
;
721 eqe
= queue_tail_node(eq
);
723 SE_DEBUG(DBG_LVL_1
, "eqe is NULL\n");
726 num_eq_processed
= 0;
727 if (blk_iopoll_enabled
) {
728 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
730 if (!blk_iopoll_sched_prep(&pbe_eq
->iopoll
))
731 blk_iopoll_sched(&pbe_eq
->iopoll
);
733 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
735 eqe
= queue_tail_node(eq
);
738 if (num_eq_processed
)
739 hwi_ring_eq_db(phba
, eq
->id
, 1, num_eq_processed
, 0, 1);
743 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
745 spin_lock_irqsave(&phba
->isr_lock
, flags
);
747 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
748 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
750 eqe
= queue_tail_node(eq
);
754 queue_work(phba
->wq
, &phba
->work_cqs
);
756 if (num_eq_processed
)
757 hwi_ring_eq_db(phba
, eq
->id
, 1, num_eq_processed
, 1, 1);
764 * be_isr - The isr routine of the driver.
766 * @dev_id: Pointer to host adapter structure
768 static irqreturn_t
be_isr(int irq
, void *dev_id
)
770 struct beiscsi_hba
*phba
;
771 struct hwi_controller
*phwi_ctrlr
;
772 struct hwi_context_memory
*phwi_context
;
773 struct be_eq_entry
*eqe
= NULL
;
774 struct be_queue_info
*eq
;
775 struct be_queue_info
*cq
;
776 struct be_queue_info
*mcc
;
777 unsigned long flags
, index
;
778 unsigned int num_mcceq_processed
, num_ioeq_processed
;
779 struct be_ctrl_info
*ctrl
;
780 struct be_eq_obj
*pbe_eq
;
785 isr
= ioread32(ctrl
->csr
+ CEV_ISR0_OFFSET
+
786 (PCI_FUNC(ctrl
->pdev
->devfn
) * CEV_ISR_SIZE
));
790 phwi_ctrlr
= phba
->phwi_ctrlr
;
791 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
792 pbe_eq
= &phwi_context
->be_eq
[0];
794 eq
= &phwi_context
->be_eq
[0].q
;
795 mcc
= &phba
->ctrl
.mcc_obj
.cq
;
797 eqe
= queue_tail_node(eq
);
799 SE_DEBUG(DBG_LVL_1
, "eqe is NULL\n");
801 num_ioeq_processed
= 0;
802 num_mcceq_processed
= 0;
803 if (blk_iopoll_enabled
) {
804 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
806 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
808 EQE_RESID_MASK
) >> 16) == mcc
->id
) {
809 spin_lock_irqsave(&phba
->isr_lock
, flags
);
810 phba
->todo_mcc_cq
= 1;
811 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
812 num_mcceq_processed
++;
814 if (!blk_iopoll_sched_prep(&pbe_eq
->iopoll
))
815 blk_iopoll_sched(&pbe_eq
->iopoll
);
816 num_ioeq_processed
++;
818 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
820 eqe
= queue_tail_node(eq
);
822 if (num_ioeq_processed
|| num_mcceq_processed
) {
823 if (phba
->todo_mcc_cq
)
824 queue_work(phba
->wq
, &phba
->work_cqs
);
826 if ((num_mcceq_processed
) && (!num_ioeq_processed
))
827 hwi_ring_eq_db(phba
, eq
->id
, 0,
828 (num_ioeq_processed
+
829 num_mcceq_processed
) , 1, 1);
831 hwi_ring_eq_db(phba
, eq
->id
, 0,
832 (num_ioeq_processed
+
833 num_mcceq_processed
), 0, 1);
839 cq
= &phwi_context
->be_cq
[0];
840 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
843 if (((eqe
->dw
[offsetof(struct amap_eq_entry
,
845 EQE_RESID_MASK
) >> 16) != cq
->id
) {
846 spin_lock_irqsave(&phba
->isr_lock
, flags
);
847 phba
->todo_mcc_cq
= 1;
848 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
850 spin_lock_irqsave(&phba
->isr_lock
, flags
);
852 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
854 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
856 eqe
= queue_tail_node(eq
);
857 num_ioeq_processed
++;
859 if (phba
->todo_cq
|| phba
->todo_mcc_cq
)
860 queue_work(phba
->wq
, &phba
->work_cqs
);
862 if (num_ioeq_processed
) {
863 hwi_ring_eq_db(phba
, eq
->id
, 0,
864 num_ioeq_processed
, 1, 1);
871 static int beiscsi_init_irqs(struct beiscsi_hba
*phba
)
873 struct pci_dev
*pcidev
= phba
->pcidev
;
874 struct hwi_controller
*phwi_ctrlr
;
875 struct hwi_context_memory
*phwi_context
;
876 int ret
, msix_vec
, i
, j
;
879 phwi_ctrlr
= phba
->phwi_ctrlr
;
880 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
882 if (phba
->msix_enabled
) {
883 for (i
= 0; i
< phba
->num_cpus
; i
++) {
884 sprintf(desc
, "beiscsi_msix_%04x", i
);
885 msix_vec
= phba
->msix_entries
[i
].vector
;
886 ret
= request_irq(msix_vec
, be_isr_msix
, 0, desc
,
887 &phwi_context
->be_eq
[i
]);
889 shost_printk(KERN_ERR
, phba
->shost
,
890 "beiscsi_init_irqs-Failed to"
891 "register msix for i = %d\n", i
);
897 msix_vec
= phba
->msix_entries
[i
].vector
;
898 ret
= request_irq(msix_vec
, be_isr_mcc
, 0, "beiscsi_msix_mcc",
899 &phwi_context
->be_eq
[i
]);
901 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_init_irqs-"
902 "Failed to register beiscsi_msix_mcc\n");
908 ret
= request_irq(pcidev
->irq
, be_isr
, IRQF_SHARED
,
911 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_init_irqs-"
912 "Failed to register irq\\n");
918 for (j
= i
- 1; j
== 0; j
++)
919 free_irq(msix_vec
, &phwi_context
->be_eq
[j
]);
923 static void hwi_ring_cq_db(struct beiscsi_hba
*phba
,
924 unsigned int id
, unsigned int num_processed
,
925 unsigned char rearm
, unsigned char event
)
928 val
|= id
& DB_CQ_RING_ID_MASK
;
930 val
|= 1 << DB_CQ_REARM_SHIFT
;
931 val
|= num_processed
<< DB_CQ_NUM_POPPED_SHIFT
;
932 iowrite32(val
, phba
->db_va
+ DB_CQ_OFFSET
);
936 beiscsi_process_async_pdu(struct beiscsi_conn
*beiscsi_conn
,
937 struct beiscsi_hba
*phba
,
939 struct pdu_base
*ppdu
,
940 unsigned long pdu_len
,
941 void *pbuffer
, unsigned long buf_len
)
943 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
944 struct iscsi_session
*session
= conn
->session
;
945 struct iscsi_task
*task
;
946 struct beiscsi_io_task
*io_task
;
947 struct iscsi_hdr
*login_hdr
;
949 switch (ppdu
->dw
[offsetof(struct amap_pdu_base
, opcode
) / 32] &
950 PDUBASE_OPCODE_MASK
) {
951 case ISCSI_OP_NOOP_IN
:
955 case ISCSI_OP_ASYNC_EVENT
:
957 case ISCSI_OP_REJECT
:
959 WARN_ON(!(buf_len
== 48));
960 SE_DEBUG(DBG_LVL_1
, "In ISCSI_OP_REJECT\n");
962 case ISCSI_OP_LOGIN_RSP
:
963 case ISCSI_OP_TEXT_RSP
:
964 task
= conn
->login_task
;
965 io_task
= task
->dd_data
;
966 login_hdr
= (struct iscsi_hdr
*)ppdu
;
967 login_hdr
->itt
= io_task
->libiscsi_itt
;
970 shost_printk(KERN_WARNING
, phba
->shost
,
971 "Unrecognized opcode 0x%x in async msg\n",
973 dw
[offsetof(struct amap_pdu_base
, opcode
) / 32]
974 & PDUBASE_OPCODE_MASK
));
978 spin_lock_bh(&session
->lock
);
979 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)ppdu
, pbuffer
, buf_len
);
980 spin_unlock_bh(&session
->lock
);
984 static struct sgl_handle
*alloc_io_sgl_handle(struct beiscsi_hba
*phba
)
986 struct sgl_handle
*psgl_handle
;
988 if (phba
->io_sgl_hndl_avbl
) {
990 "In alloc_io_sgl_handle,io_sgl_alloc_index=%d\n",
991 phba
->io_sgl_alloc_index
);
992 psgl_handle
= phba
->io_sgl_hndl_base
[phba
->
994 phba
->io_sgl_hndl_base
[phba
->io_sgl_alloc_index
] = NULL
;
995 phba
->io_sgl_hndl_avbl
--;
996 if (phba
->io_sgl_alloc_index
== (phba
->params
.
998 phba
->io_sgl_alloc_index
= 0;
1000 phba
->io_sgl_alloc_index
++;
1007 free_io_sgl_handle(struct beiscsi_hba
*phba
, struct sgl_handle
*psgl_handle
)
1009 SE_DEBUG(DBG_LVL_8
, "In free_,io_sgl_free_index=%d\n",
1010 phba
->io_sgl_free_index
);
1011 if (phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
]) {
1013 * this can happen if clean_task is called on a task that
1014 * failed in xmit_task or alloc_pdu.
1017 "Double Free in IO SGL io_sgl_free_index=%d,"
1018 "value there=%p\n", phba
->io_sgl_free_index
,
1019 phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
]);
1022 phba
->io_sgl_hndl_base
[phba
->io_sgl_free_index
] = psgl_handle
;
1023 phba
->io_sgl_hndl_avbl
++;
1024 if (phba
->io_sgl_free_index
== (phba
->params
.ios_per_ctrl
- 1))
1025 phba
->io_sgl_free_index
= 0;
1027 phba
->io_sgl_free_index
++;
1031 * alloc_wrb_handle - To allocate a wrb handle
1032 * @phba: The hba pointer
1033 * @cid: The cid to use for allocation
1035 * This happens under session_lock until submission to chip
1037 struct wrb_handle
*alloc_wrb_handle(struct beiscsi_hba
*phba
, unsigned int cid
)
1039 struct hwi_wrb_context
*pwrb_context
;
1040 struct hwi_controller
*phwi_ctrlr
;
1041 struct wrb_handle
*pwrb_handle
, *pwrb_handle_tmp
;
1043 phwi_ctrlr
= phba
->phwi_ctrlr
;
1044 pwrb_context
= &phwi_ctrlr
->wrb_context
[cid
];
1045 if (pwrb_context
->wrb_handles_available
>= 2) {
1046 pwrb_handle
= pwrb_context
->pwrb_handle_base
[
1047 pwrb_context
->alloc_index
];
1048 pwrb_context
->wrb_handles_available
--;
1049 if (pwrb_context
->alloc_index
==
1050 (phba
->params
.wrbs_per_cxn
- 1))
1051 pwrb_context
->alloc_index
= 0;
1053 pwrb_context
->alloc_index
++;
1054 pwrb_handle_tmp
= pwrb_context
->pwrb_handle_base
[
1055 pwrb_context
->alloc_index
];
1056 pwrb_handle
->nxt_wrb_index
= pwrb_handle_tmp
->wrb_index
;
1063 * free_wrb_handle - To free the wrb handle back to pool
1064 * @phba: The hba pointer
1065 * @pwrb_context: The context to free from
1066 * @pwrb_handle: The wrb_handle to free
1068 * This happens under session_lock until submission to chip
1071 free_wrb_handle(struct beiscsi_hba
*phba
, struct hwi_wrb_context
*pwrb_context
,
1072 struct wrb_handle
*pwrb_handle
)
1074 pwrb_context
->pwrb_handle_base
[pwrb_context
->free_index
] = pwrb_handle
;
1075 pwrb_context
->wrb_handles_available
++;
1076 if (pwrb_context
->free_index
== (phba
->params
.wrbs_per_cxn
- 1))
1077 pwrb_context
->free_index
= 0;
1079 pwrb_context
->free_index
++;
1082 "FREE WRB: pwrb_handle=%p free_index=0x%x"
1083 "wrb_handles_available=%d\n",
1084 pwrb_handle
, pwrb_context
->free_index
,
1085 pwrb_context
->wrb_handles_available
);
1088 static struct sgl_handle
*alloc_mgmt_sgl_handle(struct beiscsi_hba
*phba
)
1090 struct sgl_handle
*psgl_handle
;
1092 if (phba
->eh_sgl_hndl_avbl
) {
1093 psgl_handle
= phba
->eh_sgl_hndl_base
[phba
->eh_sgl_alloc_index
];
1094 phba
->eh_sgl_hndl_base
[phba
->eh_sgl_alloc_index
] = NULL
;
1095 SE_DEBUG(DBG_LVL_8
, "mgmt_sgl_alloc_index=%d=0x%x\n",
1096 phba
->eh_sgl_alloc_index
, phba
->eh_sgl_alloc_index
);
1097 phba
->eh_sgl_hndl_avbl
--;
1098 if (phba
->eh_sgl_alloc_index
==
1099 (phba
->params
.icds_per_ctrl
- phba
->params
.ios_per_ctrl
-
1101 phba
->eh_sgl_alloc_index
= 0;
1103 phba
->eh_sgl_alloc_index
++;
1110 free_mgmt_sgl_handle(struct beiscsi_hba
*phba
, struct sgl_handle
*psgl_handle
)
1113 SE_DEBUG(DBG_LVL_8
, "In free_mgmt_sgl_handle,eh_sgl_free_index=%d\n",
1114 phba
->eh_sgl_free_index
);
1115 if (phba
->eh_sgl_hndl_base
[phba
->eh_sgl_free_index
]) {
1117 * this can happen if clean_task is called on a task that
1118 * failed in xmit_task or alloc_pdu.
1121 "Double Free in eh SGL ,eh_sgl_free_index=%d\n",
1122 phba
->eh_sgl_free_index
);
1125 phba
->eh_sgl_hndl_base
[phba
->eh_sgl_free_index
] = psgl_handle
;
1126 phba
->eh_sgl_hndl_avbl
++;
1127 if (phba
->eh_sgl_free_index
==
1128 (phba
->params
.icds_per_ctrl
- phba
->params
.ios_per_ctrl
- 1))
1129 phba
->eh_sgl_free_index
= 0;
1131 phba
->eh_sgl_free_index
++;
1135 be_complete_io(struct beiscsi_conn
*beiscsi_conn
,
1136 struct iscsi_task
*task
, struct sol_cqe
*psol
)
1138 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1139 struct be_status_bhs
*sts_bhs
=
1140 (struct be_status_bhs
*)io_task
->cmd_bhs
;
1141 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1142 unsigned int sense_len
;
1143 unsigned char *sense
;
1144 u32 resid
= 0, exp_cmdsn
, max_cmdsn
;
1145 u8 rsp
, status
, flags
;
1148 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
1149 & SOL_EXP_CMD_SN_MASK
);
1150 max_cmdsn
= ((psol
->
1151 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
1152 & SOL_EXP_CMD_SN_MASK
) +
1153 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
1154 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
1155 rsp
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_resp
) / 32]
1156 & SOL_RESP_MASK
) >> 16);
1157 status
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_sts
) / 32]
1158 & SOL_STS_MASK
) >> 8);
1159 flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
1160 & SOL_FLAGS_MASK
) >> 24) | 0x80;
1162 task
->sc
->result
= (DID_OK
<< 16) | status
;
1163 if (rsp
!= ISCSI_STATUS_CMD_COMPLETED
) {
1164 task
->sc
->result
= DID_ERROR
<< 16;
1168 /* bidi not initially supported */
1169 if (flags
& (ISCSI_FLAG_CMD_UNDERFLOW
| ISCSI_FLAG_CMD_OVERFLOW
)) {
1170 resid
= (psol
->dw
[offsetof(struct amap_sol_cqe
, i_res_cnt
) /
1171 32] & SOL_RES_CNT_MASK
);
1173 if (!status
&& (flags
& ISCSI_FLAG_CMD_OVERFLOW
))
1174 task
->sc
->result
= DID_ERROR
<< 16;
1176 if (flags
& ISCSI_FLAG_CMD_UNDERFLOW
) {
1177 scsi_set_resid(task
->sc
, resid
);
1178 if (!status
&& (scsi_bufflen(task
->sc
) - resid
<
1179 task
->sc
->underflow
))
1180 task
->sc
->result
= DID_ERROR
<< 16;
1184 if (status
== SAM_STAT_CHECK_CONDITION
) {
1185 unsigned short *slen
= (unsigned short *)sts_bhs
->sense_info
;
1186 sense
= sts_bhs
->sense_info
+ sizeof(unsigned short);
1187 sense_len
= cpu_to_be16(*slen
);
1188 memcpy(task
->sc
->sense_buffer
, sense
,
1189 min_t(u16
, sense_len
, SCSI_SENSE_BUFFERSIZE
));
1192 if (io_task
->cmd_bhs
->iscsi_hdr
.flags
& ISCSI_FLAG_CMD_READ
) {
1193 if (psol
->dw
[offsetof(struct amap_sol_cqe
, i_res_cnt
) / 32]
1195 conn
->rxdata_octets
+= (psol
->
1196 dw
[offsetof(struct amap_sol_cqe
, i_res_cnt
) / 32]
1197 & SOL_RES_CNT_MASK
);
1200 scsi_dma_unmap(io_task
->scsi_cmnd
);
1201 iscsi_complete_scsi_task(task
, exp_cmdsn
, max_cmdsn
);
1205 be_complete_logout(struct beiscsi_conn
*beiscsi_conn
,
1206 struct iscsi_task
*task
, struct sol_cqe
*psol
)
1208 struct iscsi_logout_rsp
*hdr
;
1209 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1210 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1212 hdr
= (struct iscsi_logout_rsp
*)task
->hdr
;
1213 hdr
->opcode
= ISCSI_OP_LOGOUT_RSP
;
1216 hdr
->flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
1217 & SOL_FLAGS_MASK
) >> 24) | 0x80;
1218 hdr
->response
= (psol
->dw
[offsetof(struct amap_sol_cqe
, i_resp
) /
1219 32] & SOL_RESP_MASK
);
1220 hdr
->exp_cmdsn
= cpu_to_be32(psol
->
1221 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
1222 & SOL_EXP_CMD_SN_MASK
);
1223 hdr
->max_cmdsn
= be32_to_cpu((psol
->
1224 dw
[offsetof(struct amap_sol_cqe
, i_exp_cmd_sn
) / 32]
1225 & SOL_EXP_CMD_SN_MASK
) +
1226 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
1227 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
1228 hdr
->dlength
[0] = 0;
1229 hdr
->dlength
[1] = 0;
1230 hdr
->dlength
[2] = 0;
1232 hdr
->itt
= io_task
->libiscsi_itt
;
1233 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
1237 be_complete_tmf(struct beiscsi_conn
*beiscsi_conn
,
1238 struct iscsi_task
*task
, struct sol_cqe
*psol
)
1240 struct iscsi_tm_rsp
*hdr
;
1241 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1242 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1244 hdr
= (struct iscsi_tm_rsp
*)task
->hdr
;
1245 hdr
->opcode
= ISCSI_OP_SCSI_TMFUNC_RSP
;
1246 hdr
->flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
1247 & SOL_FLAGS_MASK
) >> 24) | 0x80;
1248 hdr
->response
= (psol
->dw
[offsetof(struct amap_sol_cqe
, i_resp
) /
1249 32] & SOL_RESP_MASK
);
1250 hdr
->exp_cmdsn
= cpu_to_be32(psol
->dw
[offsetof(struct amap_sol_cqe
,
1251 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
);
1252 hdr
->max_cmdsn
= be32_to_cpu((psol
->dw
[offsetof(struct amap_sol_cqe
,
1253 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
) +
1254 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
1255 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
1256 hdr
->itt
= io_task
->libiscsi_itt
;
1257 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
1261 hwi_complete_drvr_msgs(struct beiscsi_conn
*beiscsi_conn
,
1262 struct beiscsi_hba
*phba
, struct sol_cqe
*psol
)
1264 struct hwi_wrb_context
*pwrb_context
;
1265 struct wrb_handle
*pwrb_handle
= NULL
;
1266 struct hwi_controller
*phwi_ctrlr
;
1267 struct iscsi_task
*task
;
1268 struct beiscsi_io_task
*io_task
;
1269 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1270 struct iscsi_session
*session
= conn
->session
;
1272 phwi_ctrlr
= phba
->phwi_ctrlr
;
1273 pwrb_context
= &phwi_ctrlr
->wrb_context
[((psol
->
1274 dw
[offsetof(struct amap_sol_cqe
, cid
) / 32] &
1275 SOL_CID_MASK
) >> 6) -
1276 phba
->fw_config
.iscsi_cid_start
];
1277 pwrb_handle
= pwrb_context
->pwrb_handle_basestd
[((psol
->
1278 dw
[offsetof(struct amap_sol_cqe
, wrb_index
) /
1279 32] & SOL_WRB_INDEX_MASK
) >> 16)];
1280 task
= pwrb_handle
->pio_handle
;
1282 io_task
= task
->dd_data
;
1283 spin_lock(&phba
->mgmt_sgl_lock
);
1284 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
1285 spin_unlock(&phba
->mgmt_sgl_lock
);
1286 spin_lock_bh(&session
->lock
);
1287 free_wrb_handle(phba
, pwrb_context
, pwrb_handle
);
1288 spin_unlock_bh(&session
->lock
);
1292 be_complete_nopin_resp(struct beiscsi_conn
*beiscsi_conn
,
1293 struct iscsi_task
*task
, struct sol_cqe
*psol
)
1295 struct iscsi_nopin
*hdr
;
1296 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1297 struct beiscsi_io_task
*io_task
= task
->dd_data
;
1299 hdr
= (struct iscsi_nopin
*)task
->hdr
;
1300 hdr
->flags
= ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_flags
) / 32]
1301 & SOL_FLAGS_MASK
) >> 24) | 0x80;
1302 hdr
->exp_cmdsn
= cpu_to_be32(psol
->dw
[offsetof(struct amap_sol_cqe
,
1303 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
);
1304 hdr
->max_cmdsn
= be32_to_cpu((psol
->dw
[offsetof(struct amap_sol_cqe
,
1305 i_exp_cmd_sn
) / 32] & SOL_EXP_CMD_SN_MASK
) +
1306 ((psol
->dw
[offsetof(struct amap_sol_cqe
, i_cmd_wnd
)
1307 / 32] & SOL_CMD_WND_MASK
) >> 24) - 1);
1308 hdr
->opcode
= ISCSI_OP_NOOP_IN
;
1309 hdr
->itt
= io_task
->libiscsi_itt
;
1310 __iscsi_complete_pdu(conn
, (struct iscsi_hdr
*)hdr
, NULL
, 0);
1313 static void hwi_complete_cmd(struct beiscsi_conn
*beiscsi_conn
,
1314 struct beiscsi_hba
*phba
, struct sol_cqe
*psol
)
1316 struct hwi_wrb_context
*pwrb_context
;
1317 struct wrb_handle
*pwrb_handle
;
1318 struct iscsi_wrb
*pwrb
= NULL
;
1319 struct hwi_controller
*phwi_ctrlr
;
1320 struct iscsi_task
*task
;
1322 struct iscsi_conn
*conn
= beiscsi_conn
->conn
;
1323 struct iscsi_session
*session
= conn
->session
;
1325 phwi_ctrlr
= phba
->phwi_ctrlr
;
1326 pwrb_context
= &phwi_ctrlr
->wrb_context
[((psol
->dw
[offsetof
1327 (struct amap_sol_cqe
, cid
) / 32]
1328 & SOL_CID_MASK
) >> 6) -
1329 phba
->fw_config
.iscsi_cid_start
];
1330 pwrb_handle
= pwrb_context
->pwrb_handle_basestd
[((psol
->
1331 dw
[offsetof(struct amap_sol_cqe
, wrb_index
) /
1332 32] & SOL_WRB_INDEX_MASK
) >> 16)];
1333 task
= pwrb_handle
->pio_handle
;
1334 pwrb
= pwrb_handle
->pwrb
;
1335 type
= (pwrb
->dw
[offsetof(struct amap_iscsi_wrb
, type
) / 32] &
1336 WRB_TYPE_MASK
) >> 28;
1338 spin_lock_bh(&session
->lock
);
1341 case HWH_TYPE_IO_RD
:
1342 if ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) ==
1344 be_complete_nopin_resp(beiscsi_conn
, task
, psol
);
1346 be_complete_io(beiscsi_conn
, task
, psol
);
1349 case HWH_TYPE_LOGOUT
:
1350 if ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGOUT
)
1351 be_complete_logout(beiscsi_conn
, task
, psol
);
1353 be_complete_tmf(beiscsi_conn
, task
, psol
);
1357 case HWH_TYPE_LOGIN
:
1359 "\t\t No HWH_TYPE_LOGIN Expected in hwi_complete_cmd"
1360 "- Solicited path\n");
1364 be_complete_nopin_resp(beiscsi_conn
, task
, psol
);
1368 shost_printk(KERN_WARNING
, phba
->shost
,
1369 "In hwi_complete_cmd, unknown type = %d"
1370 "wrb_index 0x%x CID 0x%x\n", type
,
1371 ((psol
->dw
[offsetof(struct amap_iscsi_wrb
,
1372 type
) / 32] & SOL_WRB_INDEX_MASK
) >> 16),
1373 ((psol
->dw
[offsetof(struct amap_sol_cqe
,
1374 cid
) / 32] & SOL_CID_MASK
) >> 6));
1378 spin_unlock_bh(&session
->lock
);
1381 static struct list_head
*hwi_get_async_busy_list(struct hwi_async_pdu_context
1382 *pasync_ctx
, unsigned int is_header
,
1383 unsigned int host_write_ptr
)
1386 return &pasync_ctx
->async_entry
[host_write_ptr
].
1389 return &pasync_ctx
->async_entry
[host_write_ptr
].data_busy_list
;
1392 static struct async_pdu_handle
*
1393 hwi_get_async_handle(struct beiscsi_hba
*phba
,
1394 struct beiscsi_conn
*beiscsi_conn
,
1395 struct hwi_async_pdu_context
*pasync_ctx
,
1396 struct i_t_dpdu_cqe
*pdpdu_cqe
, unsigned int *pcq_index
)
1398 struct be_bus_address phys_addr
;
1399 struct list_head
*pbusy_list
;
1400 struct async_pdu_handle
*pasync_handle
= NULL
;
1402 unsigned char buffer_index
= -1;
1403 unsigned char is_header
= 0;
1405 phys_addr
.u
.a32
.address_lo
=
1406 pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, db_addr_lo
) / 32] -
1407 ((pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, dpl
) / 32]
1408 & PDUCQE_DPL_MASK
) >> 16);
1409 phys_addr
.u
.a32
.address_hi
=
1410 pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, db_addr_hi
) / 32];
1412 phys_addr
.u
.a64
.address
=
1413 *((unsigned long long *)(&phys_addr
.u
.a64
.address
));
1415 switch (pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
, code
) / 32]
1416 & PDUCQE_CODE_MASK
) {
1417 case UNSOL_HDR_NOTIFY
:
1420 pbusy_list
= hwi_get_async_busy_list(pasync_ctx
, 1,
1421 (pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1422 index
) / 32] & PDUCQE_INDEX_MASK
));
1424 buffer_len
= (unsigned int)(phys_addr
.u
.a64
.address
-
1425 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
);
1427 buffer_index
= buffer_len
/
1428 pasync_ctx
->async_header
.buffer_size
;
1431 case UNSOL_DATA_NOTIFY
:
1432 pbusy_list
= hwi_get_async_busy_list(pasync_ctx
, 0, (pdpdu_cqe
->
1433 dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1434 index
) / 32] & PDUCQE_INDEX_MASK
));
1435 buffer_len
= (unsigned long)(phys_addr
.u
.a64
.address
-
1436 pasync_ctx
->async_data
.pa_base
.u
.
1438 buffer_index
= buffer_len
/ pasync_ctx
->async_data
.buffer_size
;
1442 shost_printk(KERN_WARNING
, phba
->shost
,
1443 "Unexpected code=%d\n",
1444 pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1445 code
) / 32] & PDUCQE_CODE_MASK
);
1449 WARN_ON(!(buffer_index
<= pasync_ctx
->async_data
.num_entries
));
1450 WARN_ON(list_empty(pbusy_list
));
1451 list_for_each_entry(pasync_handle
, pbusy_list
, link
) {
1452 WARN_ON(pasync_handle
->consumed
);
1453 if (pasync_handle
->index
== buffer_index
)
1457 WARN_ON(!pasync_handle
);
1459 pasync_handle
->cri
= (unsigned short)beiscsi_conn
->beiscsi_conn_cid
-
1460 phba
->fw_config
.iscsi_cid_start
;
1461 pasync_handle
->is_header
= is_header
;
1462 pasync_handle
->buffer_len
= ((pdpdu_cqe
->
1463 dw
[offsetof(struct amap_i_t_dpdu_cqe
, dpl
) / 32]
1464 & PDUCQE_DPL_MASK
) >> 16);
1466 *pcq_index
= (pdpdu_cqe
->dw
[offsetof(struct amap_i_t_dpdu_cqe
,
1467 index
) / 32] & PDUCQE_INDEX_MASK
);
1468 return pasync_handle
;
1472 hwi_update_async_writables(struct hwi_async_pdu_context
*pasync_ctx
,
1473 unsigned int is_header
, unsigned int cq_index
)
1475 struct list_head
*pbusy_list
;
1476 struct async_pdu_handle
*pasync_handle
;
1477 unsigned int num_entries
, writables
= 0;
1478 unsigned int *pep_read_ptr
, *pwritables
;
1482 pep_read_ptr
= &pasync_ctx
->async_header
.ep_read_ptr
;
1483 pwritables
= &pasync_ctx
->async_header
.writables
;
1484 num_entries
= pasync_ctx
->async_header
.num_entries
;
1486 pep_read_ptr
= &pasync_ctx
->async_data
.ep_read_ptr
;
1487 pwritables
= &pasync_ctx
->async_data
.writables
;
1488 num_entries
= pasync_ctx
->async_data
.num_entries
;
1491 while ((*pep_read_ptr
) != cq_index
) {
1493 *pep_read_ptr
= (*pep_read_ptr
) % num_entries
;
1495 pbusy_list
= hwi_get_async_busy_list(pasync_ctx
, is_header
,
1498 WARN_ON(list_empty(pbusy_list
));
1500 if (!list_empty(pbusy_list
)) {
1501 pasync_handle
= list_entry(pbusy_list
->next
,
1502 struct async_pdu_handle
,
1504 WARN_ON(!pasync_handle
);
1505 pasync_handle
->consumed
= 1;
1513 "Duplicate notification received - index 0x%x!!\n",
1518 *pwritables
= *pwritables
+ writables
;
1522 static unsigned int hwi_free_async_msg(struct beiscsi_hba
*phba
,
1525 struct hwi_controller
*phwi_ctrlr
;
1526 struct hwi_async_pdu_context
*pasync_ctx
;
1527 struct async_pdu_handle
*pasync_handle
, *tmp_handle
;
1528 struct list_head
*plist
;
1531 phwi_ctrlr
= phba
->phwi_ctrlr
;
1532 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1534 plist
= &pasync_ctx
->async_entry
[cri
].wait_queue
.list
;
1536 list_for_each_entry_safe(pasync_handle
, tmp_handle
, plist
, link
) {
1537 list_del(&pasync_handle
->link
);
1540 list_add_tail(&pasync_handle
->link
,
1541 &pasync_ctx
->async_header
.free_list
);
1542 pasync_ctx
->async_header
.free_entries
++;
1545 list_add_tail(&pasync_handle
->link
,
1546 &pasync_ctx
->async_data
.free_list
);
1547 pasync_ctx
->async_data
.free_entries
++;
1552 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[cri
].wait_queue
.list
);
1553 pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
= 0;
1554 pasync_ctx
->async_entry
[cri
].wait_queue
.bytes_received
= 0;
1558 static struct phys_addr
*
1559 hwi_get_ring_address(struct hwi_async_pdu_context
*pasync_ctx
,
1560 unsigned int is_header
, unsigned int host_write_ptr
)
1562 struct phys_addr
*pasync_sge
= NULL
;
1565 pasync_sge
= pasync_ctx
->async_header
.ring_base
;
1567 pasync_sge
= pasync_ctx
->async_data
.ring_base
;
1569 return pasync_sge
+ host_write_ptr
;
1572 static void hwi_post_async_buffers(struct beiscsi_hba
*phba
,
1573 unsigned int is_header
)
1575 struct hwi_controller
*phwi_ctrlr
;
1576 struct hwi_async_pdu_context
*pasync_ctx
;
1577 struct async_pdu_handle
*pasync_handle
;
1578 struct list_head
*pfree_link
, *pbusy_list
;
1579 struct phys_addr
*pasync_sge
;
1580 unsigned int ring_id
, num_entries
;
1581 unsigned int host_write_num
;
1582 unsigned int writables
;
1586 phwi_ctrlr
= phba
->phwi_ctrlr
;
1587 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1590 num_entries
= pasync_ctx
->async_header
.num_entries
;
1591 writables
= min(pasync_ctx
->async_header
.writables
,
1592 pasync_ctx
->async_header
.free_entries
);
1593 pfree_link
= pasync_ctx
->async_header
.free_list
.next
;
1594 host_write_num
= pasync_ctx
->async_header
.host_write_ptr
;
1595 ring_id
= phwi_ctrlr
->default_pdu_hdr
.id
;
1597 num_entries
= pasync_ctx
->async_data
.num_entries
;
1598 writables
= min(pasync_ctx
->async_data
.writables
,
1599 pasync_ctx
->async_data
.free_entries
);
1600 pfree_link
= pasync_ctx
->async_data
.free_list
.next
;
1601 host_write_num
= pasync_ctx
->async_data
.host_write_ptr
;
1602 ring_id
= phwi_ctrlr
->default_pdu_data
.id
;
1605 writables
= (writables
/ 8) * 8;
1607 for (i
= 0; i
< writables
; i
++) {
1609 hwi_get_async_busy_list(pasync_ctx
, is_header
,
1612 list_entry(pfree_link
, struct async_pdu_handle
,
1614 WARN_ON(!pasync_handle
);
1615 pasync_handle
->consumed
= 0;
1617 pfree_link
= pfree_link
->next
;
1619 pasync_sge
= hwi_get_ring_address(pasync_ctx
,
1620 is_header
, host_write_num
);
1622 pasync_sge
->hi
= pasync_handle
->pa
.u
.a32
.address_lo
;
1623 pasync_sge
->lo
= pasync_handle
->pa
.u
.a32
.address_hi
;
1625 list_move(&pasync_handle
->link
, pbusy_list
);
1628 host_write_num
= host_write_num
% num_entries
;
1632 pasync_ctx
->async_header
.host_write_ptr
=
1634 pasync_ctx
->async_header
.free_entries
-= writables
;
1635 pasync_ctx
->async_header
.writables
-= writables
;
1636 pasync_ctx
->async_header
.busy_entries
+= writables
;
1638 pasync_ctx
->async_data
.host_write_ptr
= host_write_num
;
1639 pasync_ctx
->async_data
.free_entries
-= writables
;
1640 pasync_ctx
->async_data
.writables
-= writables
;
1641 pasync_ctx
->async_data
.busy_entries
+= writables
;
1644 doorbell
|= ring_id
& DB_DEF_PDU_RING_ID_MASK
;
1645 doorbell
|= 1 << DB_DEF_PDU_REARM_SHIFT
;
1646 doorbell
|= 0 << DB_DEF_PDU_EVENT_SHIFT
;
1647 doorbell
|= (writables
& DB_DEF_PDU_CQPROC_MASK
)
1648 << DB_DEF_PDU_CQPROC_SHIFT
;
1650 iowrite32(doorbell
, phba
->db_va
+ DB_RXULP0_OFFSET
);
1654 static void hwi_flush_default_pdu_buffer(struct beiscsi_hba
*phba
,
1655 struct beiscsi_conn
*beiscsi_conn
,
1656 struct i_t_dpdu_cqe
*pdpdu_cqe
)
1658 struct hwi_controller
*phwi_ctrlr
;
1659 struct hwi_async_pdu_context
*pasync_ctx
;
1660 struct async_pdu_handle
*pasync_handle
= NULL
;
1661 unsigned int cq_index
= -1;
1663 phwi_ctrlr
= phba
->phwi_ctrlr
;
1664 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1666 pasync_handle
= hwi_get_async_handle(phba
, beiscsi_conn
, pasync_ctx
,
1667 pdpdu_cqe
, &cq_index
);
1668 BUG_ON(pasync_handle
->is_header
!= 0);
1669 if (pasync_handle
->consumed
== 0)
1670 hwi_update_async_writables(pasync_ctx
, pasync_handle
->is_header
,
1673 hwi_free_async_msg(phba
, pasync_handle
->cri
);
1674 hwi_post_async_buffers(phba
, pasync_handle
->is_header
);
1678 hwi_fwd_async_msg(struct beiscsi_conn
*beiscsi_conn
,
1679 struct beiscsi_hba
*phba
,
1680 struct hwi_async_pdu_context
*pasync_ctx
, unsigned short cri
)
1682 struct list_head
*plist
;
1683 struct async_pdu_handle
*pasync_handle
;
1685 unsigned int hdr_len
= 0, buf_len
= 0;
1686 unsigned int status
, index
= 0, offset
= 0;
1687 void *pfirst_buffer
= NULL
;
1688 unsigned int num_buf
= 0;
1690 plist
= &pasync_ctx
->async_entry
[cri
].wait_queue
.list
;
1692 list_for_each_entry(pasync_handle
, plist
, link
) {
1694 phdr
= pasync_handle
->pbuffer
;
1695 hdr_len
= pasync_handle
->buffer_len
;
1697 buf_len
= pasync_handle
->buffer_len
;
1699 pfirst_buffer
= pasync_handle
->pbuffer
;
1702 memcpy(pfirst_buffer
+ offset
,
1703 pasync_handle
->pbuffer
, buf_len
);
1709 status
= beiscsi_process_async_pdu(beiscsi_conn
, phba
,
1710 (beiscsi_conn
->beiscsi_conn_cid
-
1711 phba
->fw_config
.iscsi_cid_start
),
1712 phdr
, hdr_len
, pfirst_buffer
,
1716 hwi_free_async_msg(phba
, cri
);
1721 hwi_gather_async_pdu(struct beiscsi_conn
*beiscsi_conn
,
1722 struct beiscsi_hba
*phba
,
1723 struct async_pdu_handle
*pasync_handle
)
1725 struct hwi_async_pdu_context
*pasync_ctx
;
1726 struct hwi_controller
*phwi_ctrlr
;
1727 unsigned int bytes_needed
= 0, status
= 0;
1728 unsigned short cri
= pasync_handle
->cri
;
1729 struct pdu_base
*ppdu
;
1731 phwi_ctrlr
= phba
->phwi_ctrlr
;
1732 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1734 list_del(&pasync_handle
->link
);
1735 if (pasync_handle
->is_header
) {
1736 pasync_ctx
->async_header
.busy_entries
--;
1737 if (pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
) {
1738 hwi_free_async_msg(phba
, cri
);
1742 pasync_ctx
->async_entry
[cri
].wait_queue
.bytes_received
= 0;
1743 pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
= 1;
1744 pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_len
=
1745 (unsigned short)pasync_handle
->buffer_len
;
1746 list_add_tail(&pasync_handle
->link
,
1747 &pasync_ctx
->async_entry
[cri
].wait_queue
.list
);
1749 ppdu
= pasync_handle
->pbuffer
;
1750 bytes_needed
= ((((ppdu
->dw
[offsetof(struct amap_pdu_base
,
1751 data_len_hi
) / 32] & PDUBASE_DATALENHI_MASK
) << 8) &
1752 0xFFFF0000) | ((be16_to_cpu((ppdu
->
1753 dw
[offsetof(struct amap_pdu_base
, data_len_lo
) / 32]
1754 & PDUBASE_DATALENLO_MASK
) >> 16)) & 0x0000FFFF));
1757 pasync_ctx
->async_entry
[cri
].wait_queue
.bytes_needed
=
1760 if (bytes_needed
== 0)
1761 status
= hwi_fwd_async_msg(beiscsi_conn
, phba
,
1765 pasync_ctx
->async_data
.busy_entries
--;
1766 if (pasync_ctx
->async_entry
[cri
].wait_queue
.hdr_received
) {
1767 list_add_tail(&pasync_handle
->link
,
1768 &pasync_ctx
->async_entry
[cri
].wait_queue
.
1770 pasync_ctx
->async_entry
[cri
].wait_queue
.
1772 (unsigned short)pasync_handle
->buffer_len
;
1774 if (pasync_ctx
->async_entry
[cri
].wait_queue
.
1776 pasync_ctx
->async_entry
[cri
].wait_queue
.
1778 status
= hwi_fwd_async_msg(beiscsi_conn
, phba
,
1785 static void hwi_process_default_pdu_ring(struct beiscsi_conn
*beiscsi_conn
,
1786 struct beiscsi_hba
*phba
,
1787 struct i_t_dpdu_cqe
*pdpdu_cqe
)
1789 struct hwi_controller
*phwi_ctrlr
;
1790 struct hwi_async_pdu_context
*pasync_ctx
;
1791 struct async_pdu_handle
*pasync_handle
= NULL
;
1792 unsigned int cq_index
= -1;
1794 phwi_ctrlr
= phba
->phwi_ctrlr
;
1795 pasync_ctx
= HWI_GET_ASYNC_PDU_CTX(phwi_ctrlr
);
1796 pasync_handle
= hwi_get_async_handle(phba
, beiscsi_conn
, pasync_ctx
,
1797 pdpdu_cqe
, &cq_index
);
1799 if (pasync_handle
->consumed
== 0)
1800 hwi_update_async_writables(pasync_ctx
, pasync_handle
->is_header
,
1802 hwi_gather_async_pdu(beiscsi_conn
, phba
, pasync_handle
);
1803 hwi_post_async_buffers(phba
, pasync_handle
->is_header
);
1806 static void beiscsi_process_mcc_isr(struct beiscsi_hba
*phba
)
1808 struct be_queue_info
*mcc_cq
;
1809 struct be_mcc_compl
*mcc_compl
;
1810 unsigned int num_processed
= 0;
1812 mcc_cq
= &phba
->ctrl
.mcc_obj
.cq
;
1813 mcc_compl
= queue_tail_node(mcc_cq
);
1814 mcc_compl
->flags
= le32_to_cpu(mcc_compl
->flags
);
1815 while (mcc_compl
->flags
& CQE_FLAGS_VALID_MASK
) {
1817 if (num_processed
>= 32) {
1818 hwi_ring_cq_db(phba
, mcc_cq
->id
,
1819 num_processed
, 0, 0);
1822 if (mcc_compl
->flags
& CQE_FLAGS_ASYNC_MASK
) {
1823 /* Interpret flags as an async trailer */
1824 if (is_link_state_evt(mcc_compl
->flags
))
1825 /* Interpret compl as a async link evt */
1826 beiscsi_async_link_state_process(phba
,
1827 (struct be_async_event_link_state
*) mcc_compl
);
1830 " Unsupported Async Event, flags"
1831 " = 0x%08x\n", mcc_compl
->flags
);
1832 } else if (mcc_compl
->flags
& CQE_FLAGS_COMPLETED_MASK
) {
1833 be_mcc_compl_process_isr(&phba
->ctrl
, mcc_compl
);
1834 atomic_dec(&phba
->ctrl
.mcc_obj
.q
.used
);
1837 mcc_compl
->flags
= 0;
1838 queue_tail_inc(mcc_cq
);
1839 mcc_compl
= queue_tail_node(mcc_cq
);
1840 mcc_compl
->flags
= le32_to_cpu(mcc_compl
->flags
);
1844 if (num_processed
> 0)
1845 hwi_ring_cq_db(phba
, mcc_cq
->id
, num_processed
, 1, 0);
1849 static unsigned int beiscsi_process_cq(struct be_eq_obj
*pbe_eq
)
1851 struct be_queue_info
*cq
;
1852 struct sol_cqe
*sol
;
1853 struct dmsg_cqe
*dmsg
;
1854 unsigned int num_processed
= 0;
1855 unsigned int tot_nump
= 0;
1856 struct beiscsi_conn
*beiscsi_conn
;
1857 struct beiscsi_endpoint
*beiscsi_ep
;
1858 struct iscsi_endpoint
*ep
;
1859 struct beiscsi_hba
*phba
;
1862 sol
= queue_tail_node(cq
);
1863 phba
= pbe_eq
->phba
;
1865 while (sol
->dw
[offsetof(struct amap_sol_cqe
, valid
) / 32] &
1867 be_dws_le_to_cpu(sol
, sizeof(struct sol_cqe
));
1869 ep
= phba
->ep_array
[(u32
) ((sol
->
1870 dw
[offsetof(struct amap_sol_cqe
, cid
) / 32] &
1871 SOL_CID_MASK
) >> 6) -
1872 phba
->fw_config
.iscsi_cid_start
];
1874 beiscsi_ep
= ep
->dd_data
;
1875 beiscsi_conn
= beiscsi_ep
->conn
;
1877 if (num_processed
>= 32) {
1878 hwi_ring_cq_db(phba
, cq
->id
,
1879 num_processed
, 0, 0);
1880 tot_nump
+= num_processed
;
1884 switch ((u32
) sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1885 32] & CQE_CODE_MASK
) {
1886 case SOL_CMD_COMPLETE
:
1887 hwi_complete_cmd(beiscsi_conn
, phba
, sol
);
1889 case DRIVERMSG_NOTIFY
:
1890 SE_DEBUG(DBG_LVL_8
, "Received DRIVERMSG_NOTIFY\n");
1891 dmsg
= (struct dmsg_cqe
*)sol
;
1892 hwi_complete_drvr_msgs(beiscsi_conn
, phba
, sol
);
1894 case UNSOL_HDR_NOTIFY
:
1895 SE_DEBUG(DBG_LVL_8
, "Received UNSOL_HDR_ NOTIFY\n");
1896 hwi_process_default_pdu_ring(beiscsi_conn
, phba
,
1897 (struct i_t_dpdu_cqe
*)sol
);
1899 case UNSOL_DATA_NOTIFY
:
1900 SE_DEBUG(DBG_LVL_8
, "Received UNSOL_DATA_NOTIFY\n");
1901 hwi_process_default_pdu_ring(beiscsi_conn
, phba
,
1902 (struct i_t_dpdu_cqe
*)sol
);
1904 case CXN_INVALIDATE_INDEX_NOTIFY
:
1905 case CMD_INVALIDATED_NOTIFY
:
1906 case CXN_INVALIDATE_NOTIFY
:
1908 "Ignoring CQ Error notification for cmd/cxn"
1911 case SOL_CMD_KILLED_DATA_DIGEST_ERR
:
1912 case CMD_KILLED_INVALID_STATSN_RCVD
:
1913 case CMD_KILLED_INVALID_R2T_RCVD
:
1914 case CMD_CXN_KILLED_LUN_INVALID
:
1915 case CMD_CXN_KILLED_ICD_INVALID
:
1916 case CMD_CXN_KILLED_ITT_INVALID
:
1917 case CMD_CXN_KILLED_SEQ_OUTOFORDER
:
1918 case CMD_CXN_KILLED_INVALID_DATASN_RCVD
:
1920 "CQ Error notification for cmd.. "
1921 "code %d cid 0x%x\n",
1922 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1923 32] & CQE_CODE_MASK
,
1924 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1925 32] & SOL_CID_MASK
));
1927 case UNSOL_DATA_DIGEST_ERROR_NOTIFY
:
1929 "Digest error on def pdu ring, dropping..\n");
1930 hwi_flush_default_pdu_buffer(phba
, beiscsi_conn
,
1931 (struct i_t_dpdu_cqe
*) sol
);
1933 case CXN_KILLED_PDU_SIZE_EXCEEDS_DSL
:
1934 case CXN_KILLED_BURST_LEN_MISMATCH
:
1935 case CXN_KILLED_AHS_RCVD
:
1936 case CXN_KILLED_HDR_DIGEST_ERR
:
1937 case CXN_KILLED_UNKNOWN_HDR
:
1938 case CXN_KILLED_STALE_ITT_TTT_RCVD
:
1939 case CXN_KILLED_INVALID_ITT_TTT_RCVD
:
1940 case CXN_KILLED_TIMED_OUT
:
1941 case CXN_KILLED_FIN_RCVD
:
1942 case CXN_KILLED_BAD_UNSOL_PDU_RCVD
:
1943 case CXN_KILLED_BAD_WRB_INDEX_ERROR
:
1944 case CXN_KILLED_OVER_RUN_RESIDUAL
:
1945 case CXN_KILLED_UNDER_RUN_RESIDUAL
:
1946 case CXN_KILLED_CMND_DATA_NOT_ON_SAME_CONN
:
1947 SE_DEBUG(DBG_LVL_1
, "CQ Error %d, reset CID "
1949 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1950 32] & CQE_CODE_MASK
,
1951 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1952 32] & CQE_CID_MASK
));
1953 iscsi_conn_failure(beiscsi_conn
->conn
,
1954 ISCSI_ERR_CONN_FAILED
);
1956 case CXN_KILLED_RST_SENT
:
1957 case CXN_KILLED_RST_RCVD
:
1958 SE_DEBUG(DBG_LVL_1
, "CQ Error %d, reset"
1959 "received/sent on CID 0x%x...\n",
1960 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1961 32] & CQE_CODE_MASK
,
1962 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1963 32] & CQE_CID_MASK
));
1964 iscsi_conn_failure(beiscsi_conn
->conn
,
1965 ISCSI_ERR_CONN_FAILED
);
1968 SE_DEBUG(DBG_LVL_1
, "CQ Error Invalid code= %d "
1969 "received on CID 0x%x...\n",
1970 sol
->dw
[offsetof(struct amap_sol_cqe
, code
) /
1971 32] & CQE_CODE_MASK
,
1972 (sol
->dw
[offsetof(struct amap_sol_cqe
, cid
) /
1973 32] & CQE_CID_MASK
));
1977 AMAP_SET_BITS(struct amap_sol_cqe
, valid
, sol
, 0);
1979 sol
= queue_tail_node(cq
);
1983 if (num_processed
> 0) {
1984 tot_nump
+= num_processed
;
1985 hwi_ring_cq_db(phba
, cq
->id
, num_processed
, 1, 0);
1990 void beiscsi_process_all_cqs(struct work_struct
*work
)
1992 unsigned long flags
;
1993 struct hwi_controller
*phwi_ctrlr
;
1994 struct hwi_context_memory
*phwi_context
;
1995 struct be_eq_obj
*pbe_eq
;
1996 struct beiscsi_hba
*phba
=
1997 container_of(work
, struct beiscsi_hba
, work_cqs
);
1999 phwi_ctrlr
= phba
->phwi_ctrlr
;
2000 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
2001 if (phba
->msix_enabled
)
2002 pbe_eq
= &phwi_context
->be_eq
[phba
->num_cpus
];
2004 pbe_eq
= &phwi_context
->be_eq
[0];
2006 if (phba
->todo_mcc_cq
) {
2007 spin_lock_irqsave(&phba
->isr_lock
, flags
);
2008 phba
->todo_mcc_cq
= 0;
2009 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
2010 beiscsi_process_mcc_isr(phba
);
2013 if (phba
->todo_cq
) {
2014 spin_lock_irqsave(&phba
->isr_lock
, flags
);
2016 spin_unlock_irqrestore(&phba
->isr_lock
, flags
);
2017 beiscsi_process_cq(pbe_eq
);
2021 static int be_iopoll(struct blk_iopoll
*iop
, int budget
)
2023 static unsigned int ret
;
2024 struct beiscsi_hba
*phba
;
2025 struct be_eq_obj
*pbe_eq
;
2027 pbe_eq
= container_of(iop
, struct be_eq_obj
, iopoll
);
2028 ret
= beiscsi_process_cq(pbe_eq
);
2030 phba
= pbe_eq
->phba
;
2031 blk_iopoll_complete(iop
);
2032 SE_DEBUG(DBG_LVL_8
, "rearm pbe_eq->q.id =%d\n", pbe_eq
->q
.id
);
2033 hwi_ring_eq_db(phba
, pbe_eq
->q
.id
, 0, 0, 1, 1);
2039 hwi_write_sgl(struct iscsi_wrb
*pwrb
, struct scatterlist
*sg
,
2040 unsigned int num_sg
, struct beiscsi_io_task
*io_task
)
2042 struct iscsi_sge
*psgl
;
2043 unsigned int sg_len
, index
;
2044 unsigned int sge_len
= 0;
2045 unsigned long long addr
;
2046 struct scatterlist
*l_sg
;
2047 unsigned int offset
;
2049 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_lo
, pwrb
,
2050 io_task
->bhs_pa
.u
.a32
.address_lo
);
2051 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_hi
, pwrb
,
2052 io_task
->bhs_pa
.u
.a32
.address_hi
);
2055 for (index
= 0; (index
< num_sg
) && (index
< 2); index
++,
2058 sg_len
= sg_dma_len(sg
);
2059 addr
= (u64
) sg_dma_address(sg
);
2060 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_lo
, pwrb
,
2061 ((u32
)(addr
& 0xFFFFFFFF)));
2062 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_hi
, pwrb
,
2063 ((u32
)(addr
>> 32)));
2064 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_len
, pwrb
,
2068 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_r2t_offset
,
2070 sg_len
= sg_dma_len(sg
);
2071 addr
= (u64
) sg_dma_address(sg
);
2072 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_addr_lo
, pwrb
,
2073 ((u32
)(addr
& 0xFFFFFFFF)));
2074 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_addr_hi
, pwrb
,
2075 ((u32
)(addr
>> 32)));
2076 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_len
, pwrb
,
2080 psgl
= (struct iscsi_sge
*)io_task
->psgl_handle
->pfrag
;
2081 memset(psgl
, 0, sizeof(*psgl
) * BE2_SGE
);
2083 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, io_task
->bhs_len
- 2);
2085 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2086 io_task
->bhs_pa
.u
.a32
.address_hi
);
2087 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2088 io_task
->bhs_pa
.u
.a32
.address_lo
);
2091 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
2093 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
,
2095 } else if (num_sg
== 2) {
2096 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
2098 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
,
2101 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
,
2103 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge1_last
, pwrb
,
2110 for (index
= 0; index
< num_sg
; index
++, sg
= sg_next(sg
), psgl
++) {
2111 sg_len
= sg_dma_len(sg
);
2112 addr
= (u64
) sg_dma_address(sg
);
2113 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2114 (addr
& 0xFFFFFFFF));
2115 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2117 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, sg_len
);
2118 AMAP_SET_BITS(struct amap_iscsi_sge
, sge_offset
, psgl
, offset
);
2119 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 0);
2123 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 1);
2126 static void hwi_write_buffer(struct iscsi_wrb
*pwrb
, struct iscsi_task
*task
)
2128 struct iscsi_sge
*psgl
;
2129 unsigned long long addr
;
2130 struct beiscsi_io_task
*io_task
= task
->dd_data
;
2131 struct beiscsi_conn
*beiscsi_conn
= io_task
->conn
;
2132 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
2134 io_task
->bhs_len
= sizeof(struct be_nonio_bhs
) - 2;
2135 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_lo
, pwrb
,
2136 io_task
->bhs_pa
.u
.a32
.address_lo
);
2137 AMAP_SET_BITS(struct amap_iscsi_wrb
, iscsi_bhs_addr_hi
, pwrb
,
2138 io_task
->bhs_pa
.u
.a32
.address_hi
);
2141 if (task
->data_count
) {
2142 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 1);
2143 addr
= (u64
) pci_map_single(phba
->pcidev
,
2145 task
->data_count
, 1);
2147 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
2150 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_lo
, pwrb
,
2151 ((u32
)(addr
& 0xFFFFFFFF)));
2152 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_addr_hi
, pwrb
,
2153 ((u32
)(addr
>> 32)));
2154 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_len
, pwrb
,
2157 AMAP_SET_BITS(struct amap_iscsi_wrb
, sge0_last
, pwrb
, 1);
2159 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
2163 psgl
= (struct iscsi_sge
*)io_task
->psgl_handle
->pfrag
;
2165 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, io_task
->bhs_len
);
2167 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2168 io_task
->bhs_pa
.u
.a32
.address_hi
);
2169 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2170 io_task
->bhs_pa
.u
.a32
.address_lo
);
2173 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
, 0);
2174 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
, 0);
2175 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, 0);
2176 AMAP_SET_BITS(struct amap_iscsi_sge
, sge_offset
, psgl
, 0);
2177 AMAP_SET_BITS(struct amap_iscsi_sge
, rsvd0
, psgl
, 0);
2178 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 0);
2182 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, psgl
,
2183 ((u32
)(addr
& 0xFFFFFFFF)));
2184 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, psgl
,
2185 ((u32
)(addr
>> 32)));
2187 AMAP_SET_BITS(struct amap_iscsi_sge
, len
, psgl
, 0x106);
2189 AMAP_SET_BITS(struct amap_iscsi_sge
, last_sge
, psgl
, 1);
2192 static void beiscsi_find_mem_req(struct beiscsi_hba
*phba
)
2194 unsigned int num_cq_pages
, num_async_pdu_buf_pages
;
2195 unsigned int num_async_pdu_data_pages
, wrb_sz_per_cxn
;
2196 unsigned int num_async_pdu_buf_sgl_pages
, num_async_pdu_data_sgl_pages
;
2198 num_cq_pages
= PAGES_REQUIRED(phba
->params
.num_cq_entries
* \
2199 sizeof(struct sol_cqe
));
2200 num_async_pdu_buf_pages
=
2201 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
2202 phba
->params
.defpdu_hdr_sz
);
2203 num_async_pdu_buf_sgl_pages
=
2204 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
2205 sizeof(struct phys_addr
));
2206 num_async_pdu_data_pages
=
2207 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
2208 phba
->params
.defpdu_data_sz
);
2209 num_async_pdu_data_sgl_pages
=
2210 PAGES_REQUIRED(phba
->params
.asyncpdus_per_ctrl
* \
2211 sizeof(struct phys_addr
));
2213 phba
->params
.hwi_ws_sz
= sizeof(struct hwi_controller
);
2215 phba
->mem_req
[ISCSI_MEM_GLOBAL_HEADER
] = 2 *
2216 BE_ISCSI_PDU_HEADER_SIZE
;
2217 phba
->mem_req
[HWI_MEM_ADDN_CONTEXT
] =
2218 sizeof(struct hwi_context_memory
);
2221 phba
->mem_req
[HWI_MEM_WRB
] = sizeof(struct iscsi_wrb
)
2222 * (phba
->params
.wrbs_per_cxn
)
2223 * phba
->params
.cxns_per_ctrl
;
2224 wrb_sz_per_cxn
= sizeof(struct wrb_handle
) *
2225 (phba
->params
.wrbs_per_cxn
);
2226 phba
->mem_req
[HWI_MEM_WRBH
] = roundup_pow_of_two((wrb_sz_per_cxn
) *
2227 phba
->params
.cxns_per_ctrl
);
2229 phba
->mem_req
[HWI_MEM_SGLH
] = sizeof(struct sgl_handle
) *
2230 phba
->params
.icds_per_ctrl
;
2231 phba
->mem_req
[HWI_MEM_SGE
] = sizeof(struct iscsi_sge
) *
2232 phba
->params
.num_sge_per_io
* phba
->params
.icds_per_ctrl
;
2234 phba
->mem_req
[HWI_MEM_ASYNC_HEADER_BUF
] =
2235 num_async_pdu_buf_pages
* PAGE_SIZE
;
2236 phba
->mem_req
[HWI_MEM_ASYNC_DATA_BUF
] =
2237 num_async_pdu_data_pages
* PAGE_SIZE
;
2238 phba
->mem_req
[HWI_MEM_ASYNC_HEADER_RING
] =
2239 num_async_pdu_buf_sgl_pages
* PAGE_SIZE
;
2240 phba
->mem_req
[HWI_MEM_ASYNC_DATA_RING
] =
2241 num_async_pdu_data_sgl_pages
* PAGE_SIZE
;
2242 phba
->mem_req
[HWI_MEM_ASYNC_HEADER_HANDLE
] =
2243 phba
->params
.asyncpdus_per_ctrl
*
2244 sizeof(struct async_pdu_handle
);
2245 phba
->mem_req
[HWI_MEM_ASYNC_DATA_HANDLE
] =
2246 phba
->params
.asyncpdus_per_ctrl
*
2247 sizeof(struct async_pdu_handle
);
2248 phba
->mem_req
[HWI_MEM_ASYNC_PDU_CONTEXT
] =
2249 sizeof(struct hwi_async_pdu_context
) +
2250 (phba
->params
.cxns_per_ctrl
* sizeof(struct hwi_async_entry
));
2253 static int beiscsi_alloc_mem(struct beiscsi_hba
*phba
)
2255 struct be_mem_descriptor
*mem_descr
;
2257 struct mem_array
*mem_arr
, *mem_arr_orig
;
2258 unsigned int i
, j
, alloc_size
, curr_alloc_size
;
2260 phba
->phwi_ctrlr
= kmalloc(phba
->params
.hwi_ws_sz
, GFP_KERNEL
);
2261 if (!phba
->phwi_ctrlr
)
2264 phba
->init_mem
= kcalloc(SE_MEM_MAX
, sizeof(*mem_descr
),
2266 if (!phba
->init_mem
) {
2267 kfree(phba
->phwi_ctrlr
);
2271 mem_arr_orig
= kmalloc(sizeof(*mem_arr_orig
) * BEISCSI_MAX_FRAGS_INIT
,
2273 if (!mem_arr_orig
) {
2274 kfree(phba
->init_mem
);
2275 kfree(phba
->phwi_ctrlr
);
2279 mem_descr
= phba
->init_mem
;
2280 for (i
= 0; i
< SE_MEM_MAX
; i
++) {
2282 mem_arr
= mem_arr_orig
;
2283 alloc_size
= phba
->mem_req
[i
];
2284 memset(mem_arr
, 0, sizeof(struct mem_array
) *
2285 BEISCSI_MAX_FRAGS_INIT
);
2286 curr_alloc_size
= min(be_max_phys_size
* 1024, alloc_size
);
2288 mem_arr
->virtual_address
= pci_alloc_consistent(
2292 if (!mem_arr
->virtual_address
) {
2293 if (curr_alloc_size
<= BE_MIN_MEM_SIZE
)
2295 if (curr_alloc_size
-
2296 rounddown_pow_of_two(curr_alloc_size
))
2297 curr_alloc_size
= rounddown_pow_of_two
2300 curr_alloc_size
= curr_alloc_size
/ 2;
2302 mem_arr
->bus_address
.u
.
2303 a64
.address
= (__u64
) bus_add
;
2304 mem_arr
->size
= curr_alloc_size
;
2305 alloc_size
-= curr_alloc_size
;
2306 curr_alloc_size
= min(be_max_phys_size
*
2311 } while (alloc_size
);
2312 mem_descr
->num_elements
= j
;
2313 mem_descr
->size_in_bytes
= phba
->mem_req
[i
];
2314 mem_descr
->mem_array
= kmalloc(sizeof(*mem_arr
) * j
,
2316 if (!mem_descr
->mem_array
)
2319 memcpy(mem_descr
->mem_array
, mem_arr_orig
,
2320 sizeof(struct mem_array
) * j
);
2323 kfree(mem_arr_orig
);
2326 mem_descr
->num_elements
= j
;
2327 while ((i
) || (j
)) {
2328 for (j
= mem_descr
->num_elements
; j
> 0; j
--) {
2329 pci_free_consistent(phba
->pcidev
,
2330 mem_descr
->mem_array
[j
- 1].size
,
2331 mem_descr
->mem_array
[j
- 1].
2333 (unsigned long)mem_descr
->
2335 bus_address
.u
.a64
.address
);
2339 kfree(mem_descr
->mem_array
);
2343 kfree(mem_arr_orig
);
2344 kfree(phba
->init_mem
);
2345 kfree(phba
->phwi_ctrlr
);
2349 static int beiscsi_get_memory(struct beiscsi_hba
*phba
)
2351 beiscsi_find_mem_req(phba
);
2352 return beiscsi_alloc_mem(phba
);
2355 static void iscsi_init_global_templates(struct beiscsi_hba
*phba
)
2357 struct pdu_data_out
*pdata_out
;
2358 struct pdu_nop_out
*pnop_out
;
2359 struct be_mem_descriptor
*mem_descr
;
2361 mem_descr
= phba
->init_mem
;
2362 mem_descr
+= ISCSI_MEM_GLOBAL_HEADER
;
2364 (struct pdu_data_out
*)mem_descr
->mem_array
[0].virtual_address
;
2365 memset(pdata_out
, 0, BE_ISCSI_PDU_HEADER_SIZE
);
2367 AMAP_SET_BITS(struct amap_pdu_data_out
, opcode
, pdata_out
,
2371 (struct pdu_nop_out
*)((unsigned char *)mem_descr
->mem_array
[0].
2372 virtual_address
+ BE_ISCSI_PDU_HEADER_SIZE
);
2374 memset(pnop_out
, 0, BE_ISCSI_PDU_HEADER_SIZE
);
2375 AMAP_SET_BITS(struct amap_pdu_nop_out
, ttt
, pnop_out
, 0xFFFFFFFF);
2376 AMAP_SET_BITS(struct amap_pdu_nop_out
, f_bit
, pnop_out
, 1);
2377 AMAP_SET_BITS(struct amap_pdu_nop_out
, i_bit
, pnop_out
, 0);
2380 static void beiscsi_init_wrb_handle(struct beiscsi_hba
*phba
)
2382 struct be_mem_descriptor
*mem_descr_wrbh
, *mem_descr_wrb
;
2383 struct wrb_handle
*pwrb_handle
;
2384 struct hwi_controller
*phwi_ctrlr
;
2385 struct hwi_wrb_context
*pwrb_context
;
2386 struct iscsi_wrb
*pwrb
;
2387 unsigned int num_cxn_wrbh
;
2388 unsigned int num_cxn_wrb
, j
, idx
, index
;
2390 mem_descr_wrbh
= phba
->init_mem
;
2391 mem_descr_wrbh
+= HWI_MEM_WRBH
;
2393 mem_descr_wrb
= phba
->init_mem
;
2394 mem_descr_wrb
+= HWI_MEM_WRB
;
2397 pwrb_handle
= mem_descr_wrbh
->mem_array
[idx
].virtual_address
;
2398 num_cxn_wrbh
= ((mem_descr_wrbh
->mem_array
[idx
].size
) /
2399 ((sizeof(struct wrb_handle
)) *
2400 phba
->params
.wrbs_per_cxn
));
2401 phwi_ctrlr
= phba
->phwi_ctrlr
;
2403 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
* 2; index
+= 2) {
2404 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
2405 pwrb_context
->pwrb_handle_base
=
2406 kzalloc(sizeof(struct wrb_handle
*) *
2407 phba
->params
.wrbs_per_cxn
, GFP_KERNEL
);
2408 pwrb_context
->pwrb_handle_basestd
=
2409 kzalloc(sizeof(struct wrb_handle
*) *
2410 phba
->params
.wrbs_per_cxn
, GFP_KERNEL
);
2412 pwrb_context
->alloc_index
= 0;
2413 pwrb_context
->wrb_handles_available
= 0;
2414 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2415 pwrb_context
->pwrb_handle_base
[j
] = pwrb_handle
;
2416 pwrb_context
->pwrb_handle_basestd
[j
] =
2418 pwrb_context
->wrb_handles_available
++;
2419 pwrb_handle
->wrb_index
= j
;
2422 pwrb_context
->free_index
= 0;
2427 mem_descr_wrbh
->mem_array
[idx
].virtual_address
;
2429 ((mem_descr_wrbh
->mem_array
[idx
].size
) /
2430 ((sizeof(struct wrb_handle
)) *
2431 phba
->params
.wrbs_per_cxn
));
2432 pwrb_context
->alloc_index
= 0;
2433 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2434 pwrb_context
->pwrb_handle_base
[j
] = pwrb_handle
;
2435 pwrb_context
->pwrb_handle_basestd
[j
] =
2437 pwrb_context
->wrb_handles_available
++;
2438 pwrb_handle
->wrb_index
= j
;
2441 pwrb_context
->free_index
= 0;
2446 pwrb
= mem_descr_wrb
->mem_array
[idx
].virtual_address
;
2447 num_cxn_wrb
= (mem_descr_wrb
->mem_array
[idx
].size
) /
2448 ((sizeof(struct iscsi_wrb
) *
2449 phba
->params
.wrbs_per_cxn
));
2450 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
* 2; index
+= 2) {
2451 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
2453 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2454 pwrb_handle
= pwrb_context
->pwrb_handle_base
[j
];
2455 pwrb_handle
->pwrb
= pwrb
;
2461 pwrb
= mem_descr_wrb
->mem_array
[idx
].virtual_address
;
2462 num_cxn_wrb
= (mem_descr_wrb
->mem_array
[idx
].size
) /
2463 ((sizeof(struct iscsi_wrb
) *
2464 phba
->params
.wrbs_per_cxn
));
2465 for (j
= 0; j
< phba
->params
.wrbs_per_cxn
; j
++) {
2466 pwrb_handle
= pwrb_context
->pwrb_handle_base
[j
];
2467 pwrb_handle
->pwrb
= pwrb
;
2475 static void hwi_init_async_pdu_ctx(struct beiscsi_hba
*phba
)
2477 struct hwi_controller
*phwi_ctrlr
;
2478 struct hba_parameters
*p
= &phba
->params
;
2479 struct hwi_async_pdu_context
*pasync_ctx
;
2480 struct async_pdu_handle
*pasync_header_h
, *pasync_data_h
;
2482 struct be_mem_descriptor
*mem_descr
;
2484 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2485 mem_descr
+= HWI_MEM_ASYNC_PDU_CONTEXT
;
2487 phwi_ctrlr
= phba
->phwi_ctrlr
;
2488 phwi_ctrlr
->phwi_ctxt
->pasync_ctx
= (struct hwi_async_pdu_context
*)
2489 mem_descr
->mem_array
[0].virtual_address
;
2490 pasync_ctx
= phwi_ctrlr
->phwi_ctxt
->pasync_ctx
;
2491 memset(pasync_ctx
, 0, sizeof(*pasync_ctx
));
2493 pasync_ctx
->async_header
.num_entries
= p
->asyncpdus_per_ctrl
;
2494 pasync_ctx
->async_header
.buffer_size
= p
->defpdu_hdr_sz
;
2495 pasync_ctx
->async_data
.buffer_size
= p
->defpdu_data_sz
;
2496 pasync_ctx
->async_data
.num_entries
= p
->asyncpdus_per_ctrl
;
2498 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2499 mem_descr
+= HWI_MEM_ASYNC_HEADER_BUF
;
2500 if (mem_descr
->mem_array
[0].virtual_address
) {
2502 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_BUF"
2503 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2505 shost_printk(KERN_WARNING
, phba
->shost
,
2506 "No Virtual address\n");
2508 pasync_ctx
->async_header
.va_base
=
2509 mem_descr
->mem_array
[0].virtual_address
;
2511 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
=
2512 mem_descr
->mem_array
[0].bus_address
.u
.a64
.address
;
2514 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2515 mem_descr
+= HWI_MEM_ASYNC_HEADER_RING
;
2516 if (mem_descr
->mem_array
[0].virtual_address
) {
2518 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_RING"
2519 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2521 shost_printk(KERN_WARNING
, phba
->shost
,
2522 "No Virtual address\n");
2523 pasync_ctx
->async_header
.ring_base
=
2524 mem_descr
->mem_array
[0].virtual_address
;
2526 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2527 mem_descr
+= HWI_MEM_ASYNC_HEADER_HANDLE
;
2528 if (mem_descr
->mem_array
[0].virtual_address
) {
2530 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_HEADER_HANDLE"
2531 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2533 shost_printk(KERN_WARNING
, phba
->shost
,
2534 "No Virtual address\n");
2536 pasync_ctx
->async_header
.handle_base
=
2537 mem_descr
->mem_array
[0].virtual_address
;
2538 pasync_ctx
->async_header
.writables
= 0;
2539 INIT_LIST_HEAD(&pasync_ctx
->async_header
.free_list
);
2541 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2542 mem_descr
+= HWI_MEM_ASYNC_DATA_BUF
;
2543 if (mem_descr
->mem_array
[0].virtual_address
) {
2545 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_BUF"
2546 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2548 shost_printk(KERN_WARNING
, phba
->shost
,
2549 "No Virtual address\n");
2550 pasync_ctx
->async_data
.va_base
=
2551 mem_descr
->mem_array
[0].virtual_address
;
2552 pasync_ctx
->async_data
.pa_base
.u
.a64
.address
=
2553 mem_descr
->mem_array
[0].bus_address
.u
.a64
.address
;
2555 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2556 mem_descr
+= HWI_MEM_ASYNC_DATA_RING
;
2557 if (mem_descr
->mem_array
[0].virtual_address
) {
2559 "hwi_init_async_pdu_ctx HWI_MEM_ASYNC_DATA_RING"
2560 "va=%p\n", mem_descr
->mem_array
[0].virtual_address
);
2562 shost_printk(KERN_WARNING
, phba
->shost
,
2563 "No Virtual address\n");
2565 pasync_ctx
->async_data
.ring_base
=
2566 mem_descr
->mem_array
[0].virtual_address
;
2568 mem_descr
= (struct be_mem_descriptor
*)phba
->init_mem
;
2569 mem_descr
+= HWI_MEM_ASYNC_DATA_HANDLE
;
2570 if (!mem_descr
->mem_array
[0].virtual_address
)
2571 shost_printk(KERN_WARNING
, phba
->shost
,
2572 "No Virtual address\n");
2574 pasync_ctx
->async_data
.handle_base
=
2575 mem_descr
->mem_array
[0].virtual_address
;
2576 pasync_ctx
->async_data
.writables
= 0;
2577 INIT_LIST_HEAD(&pasync_ctx
->async_data
.free_list
);
2580 (struct async_pdu_handle
*)pasync_ctx
->async_header
.handle_base
;
2582 (struct async_pdu_handle
*)pasync_ctx
->async_data
.handle_base
;
2584 for (index
= 0; index
< p
->asyncpdus_per_ctrl
; index
++) {
2585 pasync_header_h
->cri
= -1;
2586 pasync_header_h
->index
= (char)index
;
2587 INIT_LIST_HEAD(&pasync_header_h
->link
);
2588 pasync_header_h
->pbuffer
=
2589 (void *)((unsigned long)
2590 (pasync_ctx
->async_header
.va_base
) +
2591 (p
->defpdu_hdr_sz
* index
));
2593 pasync_header_h
->pa
.u
.a64
.address
=
2594 pasync_ctx
->async_header
.pa_base
.u
.a64
.address
+
2595 (p
->defpdu_hdr_sz
* index
);
2597 list_add_tail(&pasync_header_h
->link
,
2598 &pasync_ctx
->async_header
.free_list
);
2600 pasync_ctx
->async_header
.free_entries
++;
2601 pasync_ctx
->async_header
.writables
++;
2603 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].wait_queue
.list
);
2604 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].
2606 pasync_data_h
->cri
= -1;
2607 pasync_data_h
->index
= (char)index
;
2608 INIT_LIST_HEAD(&pasync_data_h
->link
);
2609 pasync_data_h
->pbuffer
=
2610 (void *)((unsigned long)
2611 (pasync_ctx
->async_data
.va_base
) +
2612 (p
->defpdu_data_sz
* index
));
2614 pasync_data_h
->pa
.u
.a64
.address
=
2615 pasync_ctx
->async_data
.pa_base
.u
.a64
.address
+
2616 (p
->defpdu_data_sz
* index
);
2618 list_add_tail(&pasync_data_h
->link
,
2619 &pasync_ctx
->async_data
.free_list
);
2621 pasync_ctx
->async_data
.free_entries
++;
2622 pasync_ctx
->async_data
.writables
++;
2624 INIT_LIST_HEAD(&pasync_ctx
->async_entry
[index
].data_busy_list
);
2627 pasync_ctx
->async_header
.host_write_ptr
= 0;
2628 pasync_ctx
->async_header
.ep_read_ptr
= -1;
2629 pasync_ctx
->async_data
.host_write_ptr
= 0;
2630 pasync_ctx
->async_data
.ep_read_ptr
= -1;
2634 be_sgl_create_contiguous(void *virtual_address
,
2635 u64 physical_address
, u32 length
,
2636 struct be_dma_mem
*sgl
)
2638 WARN_ON(!virtual_address
);
2639 WARN_ON(!physical_address
);
2640 WARN_ON(!length
> 0);
2643 sgl
->va
= virtual_address
;
2644 sgl
->dma
= (unsigned long)physical_address
;
2650 static void be_sgl_destroy_contiguous(struct be_dma_mem
*sgl
)
2652 memset(sgl
, 0, sizeof(*sgl
));
2656 hwi_build_be_sgl_arr(struct beiscsi_hba
*phba
,
2657 struct mem_array
*pmem
, struct be_dma_mem
*sgl
)
2660 be_sgl_destroy_contiguous(sgl
);
2662 be_sgl_create_contiguous(pmem
->virtual_address
,
2663 pmem
->bus_address
.u
.a64
.address
,
2668 hwi_build_be_sgl_by_offset(struct beiscsi_hba
*phba
,
2669 struct mem_array
*pmem
, struct be_dma_mem
*sgl
)
2672 be_sgl_destroy_contiguous(sgl
);
2674 be_sgl_create_contiguous((unsigned char *)pmem
->virtual_address
,
2675 pmem
->bus_address
.u
.a64
.address
,
2679 static int be_fill_queue(struct be_queue_info
*q
,
2680 u16 len
, u16 entry_size
, void *vaddress
)
2682 struct be_dma_mem
*mem
= &q
->dma_mem
;
2684 memset(q
, 0, sizeof(*q
));
2686 q
->entry_size
= entry_size
;
2687 mem
->size
= len
* entry_size
;
2691 memset(mem
->va
, 0, mem
->size
);
2695 static int beiscsi_create_eqs(struct beiscsi_hba
*phba
,
2696 struct hwi_context_memory
*phwi_context
)
2698 unsigned int i
, num_eq_pages
;
2699 int ret
, eq_for_mcc
;
2700 struct be_queue_info
*eq
;
2701 struct be_dma_mem
*mem
;
2705 num_eq_pages
= PAGES_REQUIRED(phba
->params
.num_eq_entries
* \
2706 sizeof(struct be_eq_entry
));
2708 if (phba
->msix_enabled
)
2712 for (i
= 0; i
< (phba
->num_cpus
+ eq_for_mcc
); i
++) {
2713 eq
= &phwi_context
->be_eq
[i
].q
;
2715 phwi_context
->be_eq
[i
].phba
= phba
;
2716 eq_vaddress
= pci_alloc_consistent(phba
->pcidev
,
2717 num_eq_pages
* PAGE_SIZE
,
2720 goto create_eq_error
;
2722 mem
->va
= eq_vaddress
;
2723 ret
= be_fill_queue(eq
, phba
->params
.num_eq_entries
,
2724 sizeof(struct be_eq_entry
), eq_vaddress
);
2726 shost_printk(KERN_ERR
, phba
->shost
,
2727 "be_fill_queue Failed for EQ\n");
2728 goto create_eq_error
;
2732 ret
= beiscsi_cmd_eq_create(&phba
->ctrl
, eq
,
2733 phwi_context
->cur_eqd
);
2735 shost_printk(KERN_ERR
, phba
->shost
,
2736 "beiscsi_cmd_eq_create"
2738 goto create_eq_error
;
2740 SE_DEBUG(DBG_LVL_8
, "eqid = %d\n", phwi_context
->be_eq
[i
].q
.id
);
2744 for (i
= 0; i
< (phba
->num_cpus
+ 1); i
++) {
2745 eq
= &phwi_context
->be_eq
[i
].q
;
2748 pci_free_consistent(phba
->pcidev
, num_eq_pages
2755 static int beiscsi_create_cqs(struct beiscsi_hba
*phba
,
2756 struct hwi_context_memory
*phwi_context
)
2758 unsigned int i
, num_cq_pages
;
2760 struct be_queue_info
*cq
, *eq
;
2761 struct be_dma_mem
*mem
;
2762 struct be_eq_obj
*pbe_eq
;
2766 num_cq_pages
= PAGES_REQUIRED(phba
->params
.num_cq_entries
* \
2767 sizeof(struct sol_cqe
));
2769 for (i
= 0; i
< phba
->num_cpus
; i
++) {
2770 cq
= &phwi_context
->be_cq
[i
];
2771 eq
= &phwi_context
->be_eq
[i
].q
;
2772 pbe_eq
= &phwi_context
->be_eq
[i
];
2774 pbe_eq
->phba
= phba
;
2776 cq_vaddress
= pci_alloc_consistent(phba
->pcidev
,
2777 num_cq_pages
* PAGE_SIZE
,
2780 goto create_cq_error
;
2781 ret
= be_fill_queue(cq
, phba
->params
.num_cq_entries
,
2782 sizeof(struct sol_cqe
), cq_vaddress
);
2784 shost_printk(KERN_ERR
, phba
->shost
,
2785 "be_fill_queue Failed for ISCSI CQ\n");
2786 goto create_cq_error
;
2790 ret
= beiscsi_cmd_cq_create(&phba
->ctrl
, cq
, eq
, false,
2793 shost_printk(KERN_ERR
, phba
->shost
,
2794 "beiscsi_cmd_eq_create"
2795 "Failed for ISCSI CQ\n");
2796 goto create_cq_error
;
2798 SE_DEBUG(DBG_LVL_8
, "iscsi cq_id is %d for eq_id %d\n",
2800 SE_DEBUG(DBG_LVL_8
, "ISCSI CQ CREATED\n");
2805 for (i
= 0; i
< phba
->num_cpus
; i
++) {
2806 cq
= &phwi_context
->be_cq
[i
];
2809 pci_free_consistent(phba
->pcidev
, num_cq_pages
2818 beiscsi_create_def_hdr(struct beiscsi_hba
*phba
,
2819 struct hwi_context_memory
*phwi_context
,
2820 struct hwi_controller
*phwi_ctrlr
,
2821 unsigned int def_pdu_ring_sz
)
2825 struct be_queue_info
*dq
, *cq
;
2826 struct be_dma_mem
*mem
;
2827 struct be_mem_descriptor
*mem_descr
;
2831 dq
= &phwi_context
->be_def_hdrq
;
2832 cq
= &phwi_context
->be_cq
[0];
2834 mem_descr
= phba
->init_mem
;
2835 mem_descr
+= HWI_MEM_ASYNC_HEADER_RING
;
2836 dq_vaddress
= mem_descr
->mem_array
[idx
].virtual_address
;
2837 ret
= be_fill_queue(dq
, mem_descr
->mem_array
[0].size
/
2838 sizeof(struct phys_addr
),
2839 sizeof(struct phys_addr
), dq_vaddress
);
2841 shost_printk(KERN_ERR
, phba
->shost
,
2842 "be_fill_queue Failed for DEF PDU HDR\n");
2845 mem
->dma
= (unsigned long)mem_descr
->mem_array
[idx
].
2846 bus_address
.u
.a64
.address
;
2847 ret
= be_cmd_create_default_pdu_queue(&phba
->ctrl
, cq
, dq
,
2849 phba
->params
.defpdu_hdr_sz
);
2851 shost_printk(KERN_ERR
, phba
->shost
,
2852 "be_cmd_create_default_pdu_queue Failed DEFHDR\n");
2855 phwi_ctrlr
->default_pdu_hdr
.id
= phwi_context
->be_def_hdrq
.id
;
2856 SE_DEBUG(DBG_LVL_8
, "iscsi def pdu id is %d\n",
2857 phwi_context
->be_def_hdrq
.id
);
2858 hwi_post_async_buffers(phba
, 1);
2863 beiscsi_create_def_data(struct beiscsi_hba
*phba
,
2864 struct hwi_context_memory
*phwi_context
,
2865 struct hwi_controller
*phwi_ctrlr
,
2866 unsigned int def_pdu_ring_sz
)
2870 struct be_queue_info
*dataq
, *cq
;
2871 struct be_dma_mem
*mem
;
2872 struct be_mem_descriptor
*mem_descr
;
2876 dataq
= &phwi_context
->be_def_dataq
;
2877 cq
= &phwi_context
->be_cq
[0];
2878 mem
= &dataq
->dma_mem
;
2879 mem_descr
= phba
->init_mem
;
2880 mem_descr
+= HWI_MEM_ASYNC_DATA_RING
;
2881 dq_vaddress
= mem_descr
->mem_array
[idx
].virtual_address
;
2882 ret
= be_fill_queue(dataq
, mem_descr
->mem_array
[0].size
/
2883 sizeof(struct phys_addr
),
2884 sizeof(struct phys_addr
), dq_vaddress
);
2886 shost_printk(KERN_ERR
, phba
->shost
,
2887 "be_fill_queue Failed for DEF PDU DATA\n");
2890 mem
->dma
= (unsigned long)mem_descr
->mem_array
[idx
].
2891 bus_address
.u
.a64
.address
;
2892 ret
= be_cmd_create_default_pdu_queue(&phba
->ctrl
, cq
, dataq
,
2894 phba
->params
.defpdu_data_sz
);
2896 shost_printk(KERN_ERR
, phba
->shost
,
2897 "be_cmd_create_default_pdu_queue Failed"
2898 " for DEF PDU DATA\n");
2901 phwi_ctrlr
->default_pdu_data
.id
= phwi_context
->be_def_dataq
.id
;
2902 SE_DEBUG(DBG_LVL_8
, "iscsi def data id is %d\n",
2903 phwi_context
->be_def_dataq
.id
);
2904 hwi_post_async_buffers(phba
, 0);
2905 SE_DEBUG(DBG_LVL_8
, "DEFAULT PDU DATA RING CREATED\n");
2910 beiscsi_post_pages(struct beiscsi_hba
*phba
)
2912 struct be_mem_descriptor
*mem_descr
;
2913 struct mem_array
*pm_arr
;
2914 unsigned int page_offset
, i
;
2915 struct be_dma_mem sgl
;
2918 mem_descr
= phba
->init_mem
;
2919 mem_descr
+= HWI_MEM_SGE
;
2920 pm_arr
= mem_descr
->mem_array
;
2922 page_offset
= (sizeof(struct iscsi_sge
) * phba
->params
.num_sge_per_io
*
2923 phba
->fw_config
.iscsi_icd_start
) / PAGE_SIZE
;
2924 for (i
= 0; i
< mem_descr
->num_elements
; i
++) {
2925 hwi_build_be_sgl_arr(phba
, pm_arr
, &sgl
);
2926 status
= be_cmd_iscsi_post_sgl_pages(&phba
->ctrl
, &sgl
,
2928 (pm_arr
->size
/ PAGE_SIZE
));
2929 page_offset
+= pm_arr
->size
/ PAGE_SIZE
;
2931 shost_printk(KERN_ERR
, phba
->shost
,
2932 "post sgl failed.\n");
2937 SE_DEBUG(DBG_LVL_8
, "POSTED PAGES\n");
2941 static void be_queue_free(struct beiscsi_hba
*phba
, struct be_queue_info
*q
)
2943 struct be_dma_mem
*mem
= &q
->dma_mem
;
2945 pci_free_consistent(phba
->pcidev
, mem
->size
,
2949 static int be_queue_alloc(struct beiscsi_hba
*phba
, struct be_queue_info
*q
,
2950 u16 len
, u16 entry_size
)
2952 struct be_dma_mem
*mem
= &q
->dma_mem
;
2954 memset(q
, 0, sizeof(*q
));
2956 q
->entry_size
= entry_size
;
2957 mem
->size
= len
* entry_size
;
2958 mem
->va
= pci_alloc_consistent(phba
->pcidev
, mem
->size
, &mem
->dma
);
2961 memset(mem
->va
, 0, mem
->size
);
2966 beiscsi_create_wrb_rings(struct beiscsi_hba
*phba
,
2967 struct hwi_context_memory
*phwi_context
,
2968 struct hwi_controller
*phwi_ctrlr
)
2970 unsigned int wrb_mem_index
, offset
, size
, num_wrb_rings
;
2972 unsigned int idx
, num
, i
;
2973 struct mem_array
*pwrb_arr
;
2975 struct be_dma_mem sgl
;
2976 struct be_mem_descriptor
*mem_descr
;
2980 mem_descr
= phba
->init_mem
;
2981 mem_descr
+= HWI_MEM_WRB
;
2982 pwrb_arr
= kmalloc(sizeof(*pwrb_arr
) * phba
->params
.cxns_per_ctrl
,
2985 shost_printk(KERN_ERR
, phba
->shost
,
2986 "Memory alloc failed in create wrb ring.\n");
2989 wrb_vaddr
= mem_descr
->mem_array
[idx
].virtual_address
;
2990 pa_addr_lo
= mem_descr
->mem_array
[idx
].bus_address
.u
.a64
.address
;
2991 num_wrb_rings
= mem_descr
->mem_array
[idx
].size
/
2992 (phba
->params
.wrbs_per_cxn
* sizeof(struct iscsi_wrb
));
2994 for (num
= 0; num
< phba
->params
.cxns_per_ctrl
; num
++) {
2995 if (num_wrb_rings
) {
2996 pwrb_arr
[num
].virtual_address
= wrb_vaddr
;
2997 pwrb_arr
[num
].bus_address
.u
.a64
.address
= pa_addr_lo
;
2998 pwrb_arr
[num
].size
= phba
->params
.wrbs_per_cxn
*
2999 sizeof(struct iscsi_wrb
);
3000 wrb_vaddr
+= pwrb_arr
[num
].size
;
3001 pa_addr_lo
+= pwrb_arr
[num
].size
;
3005 wrb_vaddr
= mem_descr
->mem_array
[idx
].virtual_address
;
3006 pa_addr_lo
= mem_descr
->mem_array
[idx
].\
3007 bus_address
.u
.a64
.address
;
3008 num_wrb_rings
= mem_descr
->mem_array
[idx
].size
/
3009 (phba
->params
.wrbs_per_cxn
*
3010 sizeof(struct iscsi_wrb
));
3011 pwrb_arr
[num
].virtual_address
= wrb_vaddr
;
3012 pwrb_arr
[num
].bus_address
.u
.a64
.address\
3014 pwrb_arr
[num
].size
= phba
->params
.wrbs_per_cxn
*
3015 sizeof(struct iscsi_wrb
);
3016 wrb_vaddr
+= pwrb_arr
[num
].size
;
3017 pa_addr_lo
+= pwrb_arr
[num
].size
;
3021 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
3026 hwi_build_be_sgl_by_offset(phba
, &pwrb_arr
[i
], &sgl
);
3027 status
= be_cmd_wrbq_create(&phba
->ctrl
, &sgl
,
3028 &phwi_context
->be_wrbq
[i
]);
3030 shost_printk(KERN_ERR
, phba
->shost
,
3031 "wrbq create failed.");
3035 phwi_ctrlr
->wrb_context
[i
* 2].cid
= phwi_context
->be_wrbq
[i
].
3042 static void free_wrb_handles(struct beiscsi_hba
*phba
)
3045 struct hwi_controller
*phwi_ctrlr
;
3046 struct hwi_wrb_context
*pwrb_context
;
3048 phwi_ctrlr
= phba
->phwi_ctrlr
;
3049 for (index
= 0; index
< phba
->params
.cxns_per_ctrl
* 2; index
+= 2) {
3050 pwrb_context
= &phwi_ctrlr
->wrb_context
[index
];
3051 kfree(pwrb_context
->pwrb_handle_base
);
3052 kfree(pwrb_context
->pwrb_handle_basestd
);
3056 static void be_mcc_queues_destroy(struct beiscsi_hba
*phba
)
3058 struct be_queue_info
*q
;
3059 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3061 q
= &phba
->ctrl
.mcc_obj
.q
;
3063 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_MCCQ
);
3064 be_queue_free(phba
, q
);
3066 q
= &phba
->ctrl
.mcc_obj
.cq
;
3068 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_CQ
);
3069 be_queue_free(phba
, q
);
3072 static void hwi_cleanup(struct beiscsi_hba
*phba
)
3074 struct be_queue_info
*q
;
3075 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3076 struct hwi_controller
*phwi_ctrlr
;
3077 struct hwi_context_memory
*phwi_context
;
3080 phwi_ctrlr
= phba
->phwi_ctrlr
;
3081 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3082 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
3083 q
= &phwi_context
->be_wrbq
[i
];
3085 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_WRBQ
);
3087 free_wrb_handles(phba
);
3089 q
= &phwi_context
->be_def_hdrq
;
3091 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_DPDUQ
);
3093 q
= &phwi_context
->be_def_dataq
;
3095 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_DPDUQ
);
3097 beiscsi_cmd_q_destroy(ctrl
, NULL
, QTYPE_SGL
);
3099 for (i
= 0; i
< (phba
->num_cpus
); i
++) {
3100 q
= &phwi_context
->be_cq
[i
];
3102 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_CQ
);
3104 if (phba
->msix_enabled
)
3108 for (i
= 0; i
< (phba
->num_cpus
+ eq_num
); i
++) {
3109 q
= &phwi_context
->be_eq
[i
].q
;
3111 beiscsi_cmd_q_destroy(ctrl
, q
, QTYPE_EQ
);
3113 be_mcc_queues_destroy(phba
);
3116 static int be_mcc_queues_create(struct beiscsi_hba
*phba
,
3117 struct hwi_context_memory
*phwi_context
)
3119 struct be_queue_info
*q
, *cq
;
3120 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3122 /* Alloc MCC compl queue */
3123 cq
= &phba
->ctrl
.mcc_obj
.cq
;
3124 if (be_queue_alloc(phba
, cq
, MCC_CQ_LEN
,
3125 sizeof(struct be_mcc_compl
)))
3127 /* Ask BE to create MCC compl queue; */
3128 if (phba
->msix_enabled
) {
3129 if (beiscsi_cmd_cq_create(ctrl
, cq
, &phwi_context
->be_eq
3130 [phba
->num_cpus
].q
, false, true, 0))
3133 if (beiscsi_cmd_cq_create(ctrl
, cq
, &phwi_context
->be_eq
[0].q
,
3138 /* Alloc MCC queue */
3139 q
= &phba
->ctrl
.mcc_obj
.q
;
3140 if (be_queue_alloc(phba
, q
, MCC_Q_LEN
, sizeof(struct be_mcc_wrb
)))
3141 goto mcc_cq_destroy
;
3143 /* Ask BE to create MCC queue */
3144 if (beiscsi_cmd_mccq_create(phba
, q
, cq
))
3150 be_queue_free(phba
, q
);
3152 beiscsi_cmd_q_destroy(ctrl
, cq
, QTYPE_CQ
);
3154 be_queue_free(phba
, cq
);
3159 static int find_num_cpus(void)
3163 num_cpus
= num_online_cpus();
3164 if (num_cpus
>= MAX_CPUS
)
3165 num_cpus
= MAX_CPUS
- 1;
3167 SE_DEBUG(DBG_LVL_8
, "num_cpus = %d\n", num_cpus
);
3171 static int hwi_init_port(struct beiscsi_hba
*phba
)
3173 struct hwi_controller
*phwi_ctrlr
;
3174 struct hwi_context_memory
*phwi_context
;
3175 unsigned int def_pdu_ring_sz
;
3176 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3180 phba
->params
.asyncpdus_per_ctrl
* sizeof(struct phys_addr
);
3181 phwi_ctrlr
= phba
->phwi_ctrlr
;
3182 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3183 phwi_context
->max_eqd
= 0;
3184 phwi_context
->min_eqd
= 0;
3185 phwi_context
->cur_eqd
= 64;
3186 be_cmd_fw_initialize(&phba
->ctrl
);
3188 status
= beiscsi_create_eqs(phba
, phwi_context
);
3190 shost_printk(KERN_ERR
, phba
->shost
, "EQ not created\n");
3194 status
= be_mcc_queues_create(phba
, phwi_context
);
3198 status
= mgmt_check_supported_fw(ctrl
, phba
);
3200 shost_printk(KERN_ERR
, phba
->shost
,
3201 "Unsupported fw version\n");
3205 status
= beiscsi_create_cqs(phba
, phwi_context
);
3207 shost_printk(KERN_ERR
, phba
->shost
, "CQ not created\n");
3211 status
= beiscsi_create_def_hdr(phba
, phwi_context
, phwi_ctrlr
,
3214 shost_printk(KERN_ERR
, phba
->shost
,
3215 "Default Header not created\n");
3219 status
= beiscsi_create_def_data(phba
, phwi_context
,
3220 phwi_ctrlr
, def_pdu_ring_sz
);
3222 shost_printk(KERN_ERR
, phba
->shost
,
3223 "Default Data not created\n");
3227 status
= beiscsi_post_pages(phba
);
3229 shost_printk(KERN_ERR
, phba
->shost
, "Post SGL Pages Failed\n");
3233 status
= beiscsi_create_wrb_rings(phba
, phwi_context
, phwi_ctrlr
);
3235 shost_printk(KERN_ERR
, phba
->shost
,
3236 "WRB Rings not created\n");
3240 SE_DEBUG(DBG_LVL_8
, "hwi_init_port success\n");
3244 shost_printk(KERN_ERR
, phba
->shost
, "hwi_init_port failed");
3249 static int hwi_init_controller(struct beiscsi_hba
*phba
)
3251 struct hwi_controller
*phwi_ctrlr
;
3253 phwi_ctrlr
= phba
->phwi_ctrlr
;
3254 if (1 == phba
->init_mem
[HWI_MEM_ADDN_CONTEXT
].num_elements
) {
3255 phwi_ctrlr
->phwi_ctxt
= (struct hwi_context_memory
*)phba
->
3256 init_mem
[HWI_MEM_ADDN_CONTEXT
].mem_array
[0].virtual_address
;
3257 SE_DEBUG(DBG_LVL_8
, " phwi_ctrlr->phwi_ctxt=%p\n",
3258 phwi_ctrlr
->phwi_ctxt
);
3260 shost_printk(KERN_ERR
, phba
->shost
,
3261 "HWI_MEM_ADDN_CONTEXT is more than one element."
3262 "Failing to load\n");
3266 iscsi_init_global_templates(phba
);
3267 beiscsi_init_wrb_handle(phba
);
3268 hwi_init_async_pdu_ctx(phba
);
3269 if (hwi_init_port(phba
) != 0) {
3270 shost_printk(KERN_ERR
, phba
->shost
,
3271 "hwi_init_controller failed\n");
3277 static void beiscsi_free_mem(struct beiscsi_hba
*phba
)
3279 struct be_mem_descriptor
*mem_descr
;
3282 mem_descr
= phba
->init_mem
;
3285 for (i
= 0; i
< SE_MEM_MAX
; i
++) {
3286 for (j
= mem_descr
->num_elements
; j
> 0; j
--) {
3287 pci_free_consistent(phba
->pcidev
,
3288 mem_descr
->mem_array
[j
- 1].size
,
3289 mem_descr
->mem_array
[j
- 1].virtual_address
,
3290 (unsigned long)mem_descr
->mem_array
[j
- 1].
3291 bus_address
.u
.a64
.address
);
3293 kfree(mem_descr
->mem_array
);
3296 kfree(phba
->init_mem
);
3297 kfree(phba
->phwi_ctrlr
);
3300 static int beiscsi_init_controller(struct beiscsi_hba
*phba
)
3304 ret
= beiscsi_get_memory(phba
);
3306 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe -"
3307 "Failed in beiscsi_alloc_memory\n");
3311 ret
= hwi_init_controller(phba
);
3314 SE_DEBUG(DBG_LVL_8
, "Return success from beiscsi_init_controller");
3318 beiscsi_free_mem(phba
);
3322 static int beiscsi_init_sgl_handle(struct beiscsi_hba
*phba
)
3324 struct be_mem_descriptor
*mem_descr_sglh
, *mem_descr_sg
;
3325 struct sgl_handle
*psgl_handle
;
3326 struct iscsi_sge
*pfrag
;
3327 unsigned int arr_index
, i
, idx
;
3329 phba
->io_sgl_hndl_avbl
= 0;
3330 phba
->eh_sgl_hndl_avbl
= 0;
3332 mem_descr_sglh
= phba
->init_mem
;
3333 mem_descr_sglh
+= HWI_MEM_SGLH
;
3334 if (1 == mem_descr_sglh
->num_elements
) {
3335 phba
->io_sgl_hndl_base
= kzalloc(sizeof(struct sgl_handle
*) *
3336 phba
->params
.ios_per_ctrl
,
3338 if (!phba
->io_sgl_hndl_base
) {
3339 shost_printk(KERN_ERR
, phba
->shost
,
3340 "Mem Alloc Failed. Failing to load\n");
3343 phba
->eh_sgl_hndl_base
= kzalloc(sizeof(struct sgl_handle
*) *
3344 (phba
->params
.icds_per_ctrl
-
3345 phba
->params
.ios_per_ctrl
),
3347 if (!phba
->eh_sgl_hndl_base
) {
3348 kfree(phba
->io_sgl_hndl_base
);
3349 shost_printk(KERN_ERR
, phba
->shost
,
3350 "Mem Alloc Failed. Failing to load\n");
3354 shost_printk(KERN_ERR
, phba
->shost
,
3355 "HWI_MEM_SGLH is more than one element."
3356 "Failing to load\n");
3362 while (idx
< mem_descr_sglh
->num_elements
) {
3363 psgl_handle
= mem_descr_sglh
->mem_array
[idx
].virtual_address
;
3365 for (i
= 0; i
< (mem_descr_sglh
->mem_array
[idx
].size
/
3366 sizeof(struct sgl_handle
)); i
++) {
3367 if (arr_index
< phba
->params
.ios_per_ctrl
) {
3368 phba
->io_sgl_hndl_base
[arr_index
] = psgl_handle
;
3369 phba
->io_sgl_hndl_avbl
++;
3372 phba
->eh_sgl_hndl_base
[arr_index
-
3373 phba
->params
.ios_per_ctrl
] =
3376 phba
->eh_sgl_hndl_avbl
++;
3383 "phba->io_sgl_hndl_avbl=%d"
3384 "phba->eh_sgl_hndl_avbl=%d\n",
3385 phba
->io_sgl_hndl_avbl
,
3386 phba
->eh_sgl_hndl_avbl
);
3387 mem_descr_sg
= phba
->init_mem
;
3388 mem_descr_sg
+= HWI_MEM_SGE
;
3389 SE_DEBUG(DBG_LVL_8
, "\n mem_descr_sg->num_elements=%d\n",
3390 mem_descr_sg
->num_elements
);
3393 while (idx
< mem_descr_sg
->num_elements
) {
3394 pfrag
= mem_descr_sg
->mem_array
[idx
].virtual_address
;
3397 i
< (mem_descr_sg
->mem_array
[idx
].size
) /
3398 (sizeof(struct iscsi_sge
) * phba
->params
.num_sge_per_io
);
3400 if (arr_index
< phba
->params
.ios_per_ctrl
)
3401 psgl_handle
= phba
->io_sgl_hndl_base
[arr_index
];
3403 psgl_handle
= phba
->eh_sgl_hndl_base
[arr_index
-
3404 phba
->params
.ios_per_ctrl
];
3405 psgl_handle
->pfrag
= pfrag
;
3406 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_hi
, pfrag
, 0);
3407 AMAP_SET_BITS(struct amap_iscsi_sge
, addr_lo
, pfrag
, 0);
3408 pfrag
+= phba
->params
.num_sge_per_io
;
3409 psgl_handle
->sgl_index
=
3410 phba
->fw_config
.iscsi_icd_start
+ arr_index
++;
3414 phba
->io_sgl_free_index
= 0;
3415 phba
->io_sgl_alloc_index
= 0;
3416 phba
->eh_sgl_free_index
= 0;
3417 phba
->eh_sgl_alloc_index
= 0;
3421 static int hba_setup_cid_tbls(struct beiscsi_hba
*phba
)
3425 phba
->cid_array
= kzalloc(sizeof(void *) * phba
->params
.cxns_per_ctrl
,
3427 if (!phba
->cid_array
) {
3428 shost_printk(KERN_ERR
, phba
->shost
,
3429 "Failed to allocate memory in "
3430 "hba_setup_cid_tbls\n");
3433 phba
->ep_array
= kzalloc(sizeof(struct iscsi_endpoint
*) *
3434 phba
->params
.cxns_per_ctrl
* 2, GFP_KERNEL
);
3435 if (!phba
->ep_array
) {
3436 shost_printk(KERN_ERR
, phba
->shost
,
3437 "Failed to allocate memory in "
3438 "hba_setup_cid_tbls\n");
3439 kfree(phba
->cid_array
);
3442 new_cid
= phba
->fw_config
.iscsi_cid_start
;
3443 for (i
= 0; i
< phba
->params
.cxns_per_ctrl
; i
++) {
3444 phba
->cid_array
[i
] = new_cid
;
3447 phba
->avlbl_cids
= phba
->params
.cxns_per_ctrl
;
3451 static void hwi_enable_intr(struct beiscsi_hba
*phba
)
3453 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3454 struct hwi_controller
*phwi_ctrlr
;
3455 struct hwi_context_memory
*phwi_context
;
3456 struct be_queue_info
*eq
;
3461 phwi_ctrlr
= phba
->phwi_ctrlr
;
3462 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3464 addr
= (u8 __iomem
*) ((u8 __iomem
*) ctrl
->pcicfg
+
3465 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET
);
3466 reg
= ioread32(addr
);
3467 SE_DEBUG(DBG_LVL_8
, "reg =x%08x\n", reg
);
3469 enabled
= reg
& MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3471 reg
|= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3472 SE_DEBUG(DBG_LVL_8
, "reg =x%08x addr=%p\n", reg
, addr
);
3473 iowrite32(reg
, addr
);
3474 if (!phba
->msix_enabled
) {
3475 eq
= &phwi_context
->be_eq
[0].q
;
3476 SE_DEBUG(DBG_LVL_8
, "eq->id=%d\n", eq
->id
);
3477 hwi_ring_eq_db(phba
, eq
->id
, 0, 0, 1, 1);
3479 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
3480 eq
= &phwi_context
->be_eq
[i
].q
;
3481 SE_DEBUG(DBG_LVL_8
, "eq->id=%d\n", eq
->id
);
3482 hwi_ring_eq_db(phba
, eq
->id
, 0, 0, 1, 1);
3488 static void hwi_disable_intr(struct beiscsi_hba
*phba
)
3490 struct be_ctrl_info
*ctrl
= &phba
->ctrl
;
3492 u8 __iomem
*addr
= ctrl
->pcicfg
+ PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET
;
3493 u32 reg
= ioread32(addr
);
3495 u32 enabled
= reg
& MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3497 reg
&= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK
;
3498 iowrite32(reg
, addr
);
3500 shost_printk(KERN_WARNING
, phba
->shost
,
3501 "In hwi_disable_intr, Already Disabled\n");
3504 static int beiscsi_get_boot_info(struct beiscsi_hba
*phba
)
3506 struct be_cmd_resp_get_boot_target
*boot_resp
;
3507 struct be_cmd_resp_get_session
*session_resp
;
3508 struct be_mcc_wrb
*wrb
;
3509 struct be_dma_mem nonemb_cmd
;
3510 unsigned int tag
, wrb_num
;
3511 unsigned short status
, extd_status
;
3512 struct be_queue_info
*mccq
= &phba
->ctrl
.mcc_obj
.q
;
3514 tag
= beiscsi_get_boot_target(phba
);
3516 SE_DEBUG(DBG_LVL_1
, "be_cmd_get_mac_addr Failed\n");
3519 wait_event_interruptible(phba
->ctrl
.mcc_wait
[tag
],
3520 phba
->ctrl
.mcc_numtag
[tag
]);
3522 wrb_num
= (phba
->ctrl
.mcc_numtag
[tag
] & 0x00FF0000) >> 16;
3523 extd_status
= (phba
->ctrl
.mcc_numtag
[tag
] & 0x0000FF00) >> 8;
3524 status
= phba
->ctrl
.mcc_numtag
[tag
] & 0x000000FF;
3525 if (status
|| extd_status
) {
3526 SE_DEBUG(DBG_LVL_1
, "be_cmd_get_mac_addr Failed"
3527 " status = %d extd_status = %d\n",
3528 status
, extd_status
);
3529 free_mcc_tag(&phba
->ctrl
, tag
);
3532 wrb
= queue_get_wrb(mccq
, wrb_num
);
3533 free_mcc_tag(&phba
->ctrl
, tag
);
3534 boot_resp
= embedded_payload(wrb
);
3536 if (boot_resp
->boot_session_handle
< 0) {
3537 printk(KERN_ERR
"No Boot Session for this pci_func,"
3538 "session Hndl = %d\n", boot_resp
->boot_session_handle
);
3542 nonemb_cmd
.va
= pci_alloc_consistent(phba
->ctrl
.pdev
,
3543 sizeof(*session_resp
),
3545 if (nonemb_cmd
.va
== NULL
) {
3547 "Failed to allocate memory for"
3548 "beiscsi_get_session_info\n");
3552 memset(nonemb_cmd
.va
, 0, sizeof(*session_resp
));
3553 tag
= beiscsi_get_session_info(phba
,
3554 boot_resp
->boot_session_handle
, &nonemb_cmd
);
3556 SE_DEBUG(DBG_LVL_1
, "beiscsi_get_session_info"
3560 wait_event_interruptible(phba
->ctrl
.mcc_wait
[tag
],
3561 phba
->ctrl
.mcc_numtag
[tag
]);
3563 wrb_num
= (phba
->ctrl
.mcc_numtag
[tag
] & 0x00FF0000) >> 16;
3564 extd_status
= (phba
->ctrl
.mcc_numtag
[tag
] & 0x0000FF00) >> 8;
3565 status
= phba
->ctrl
.mcc_numtag
[tag
] & 0x000000FF;
3566 if (status
|| extd_status
) {
3567 SE_DEBUG(DBG_LVL_1
, "beiscsi_get_session_info Failed"
3568 " status = %d extd_status = %d\n",
3569 status
, extd_status
);
3570 free_mcc_tag(&phba
->ctrl
, tag
);
3573 wrb
= queue_get_wrb(mccq
, wrb_num
);
3574 free_mcc_tag(&phba
->ctrl
, tag
);
3575 session_resp
= nonemb_cmd
.va
;
3576 memcpy(&phba
->boot_sess
, &session_resp
->session_info
,
3577 sizeof(struct mgmt_session_info
));
3578 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
3579 nonemb_cmd
.va
, nonemb_cmd
.dma
);
3582 pci_free_consistent(phba
->ctrl
.pdev
, nonemb_cmd
.size
,
3583 nonemb_cmd
.va
, nonemb_cmd
.dma
);
3587 static int beiscsi_init_port(struct beiscsi_hba
*phba
)
3591 ret
= beiscsi_init_controller(phba
);
3593 shost_printk(KERN_ERR
, phba
->shost
,
3594 "beiscsi_dev_probe - Failed in"
3595 "beiscsi_init_controller\n");
3598 ret
= beiscsi_init_sgl_handle(phba
);
3600 shost_printk(KERN_ERR
, phba
->shost
,
3601 "beiscsi_dev_probe - Failed in"
3602 "beiscsi_init_sgl_handle\n");
3603 goto do_cleanup_ctrlr
;
3606 if (hba_setup_cid_tbls(phba
)) {
3607 shost_printk(KERN_ERR
, phba
->shost
,
3608 "Failed in hba_setup_cid_tbls\n");
3609 kfree(phba
->io_sgl_hndl_base
);
3610 kfree(phba
->eh_sgl_hndl_base
);
3611 goto do_cleanup_ctrlr
;
3621 static void hwi_purge_eq(struct beiscsi_hba
*phba
)
3623 struct hwi_controller
*phwi_ctrlr
;
3624 struct hwi_context_memory
*phwi_context
;
3625 struct be_queue_info
*eq
;
3626 struct be_eq_entry
*eqe
= NULL
;
3628 unsigned int num_processed
;
3630 phwi_ctrlr
= phba
->phwi_ctrlr
;
3631 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
3632 if (phba
->msix_enabled
)
3637 for (i
= 0; i
< (phba
->num_cpus
+ eq_msix
); i
++) {
3638 eq
= &phwi_context
->be_eq
[i
].q
;
3639 eqe
= queue_tail_node(eq
);
3641 while (eqe
->dw
[offsetof(struct amap_eq_entry
, valid
) / 32]
3643 AMAP_SET_BITS(struct amap_eq_entry
, valid
, eqe
, 0);
3645 eqe
= queue_tail_node(eq
);
3650 hwi_ring_eq_db(phba
, eq
->id
, 1, num_processed
, 1, 1);
3654 static void beiscsi_clean_port(struct beiscsi_hba
*phba
)
3658 mgmt_status
= mgmt_epfw_cleanup(phba
, CMD_CONNECTION_CHUTE_0
);
3660 shost_printk(KERN_WARNING
, phba
->shost
,
3661 "mgmt_epfw_cleanup FAILED\n");
3665 kfree(phba
->io_sgl_hndl_base
);
3666 kfree(phba
->eh_sgl_hndl_base
);
3667 kfree(phba
->cid_array
);
3668 kfree(phba
->ep_array
);
3672 beiscsi_offload_connection(struct beiscsi_conn
*beiscsi_conn
,
3673 struct beiscsi_offload_params
*params
)
3675 struct wrb_handle
*pwrb_handle
;
3676 struct iscsi_target_context_update_wrb
*pwrb
= NULL
;
3677 struct be_mem_descriptor
*mem_descr
;
3678 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3682 * We can always use 0 here because it is reserved by libiscsi for
3683 * login/startup related tasks.
3685 pwrb_handle
= alloc_wrb_handle(phba
, (beiscsi_conn
->beiscsi_conn_cid
-
3686 phba
->fw_config
.iscsi_cid_start
));
3687 pwrb
= (struct iscsi_target_context_update_wrb
*)pwrb_handle
->pwrb
;
3688 memset(pwrb
, 0, sizeof(*pwrb
));
3689 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3690 max_burst_length
, pwrb
, params
->dw
[offsetof
3691 (struct amap_beiscsi_offload_params
,
3692 max_burst_length
) / 32]);
3693 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3694 max_send_data_segment_length
, pwrb
,
3695 params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3696 max_send_data_segment_length
) / 32]);
3697 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3700 params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3701 first_burst_length
) / 32]);
3703 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, erl
, pwrb
,
3704 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3705 erl
) / 32] & OFFLD_PARAMS_ERL
));
3706 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, dde
, pwrb
,
3707 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3708 dde
) / 32] & OFFLD_PARAMS_DDE
) >> 2);
3709 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, hde
, pwrb
,
3710 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3711 hde
) / 32] & OFFLD_PARAMS_HDE
) >> 3);
3712 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, ir2t
, pwrb
,
3713 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3714 ir2t
) / 32] & OFFLD_PARAMS_IR2T
) >> 4);
3715 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, imd
, pwrb
,
3716 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3717 imd
) / 32] & OFFLD_PARAMS_IMD
) >> 5);
3718 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, stat_sn
,
3720 (params
->dw
[offsetof(struct amap_beiscsi_offload_params
,
3721 exp_statsn
) / 32] + 1));
3722 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, type
, pwrb
,
3724 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, wrb_idx
,
3725 pwrb
, pwrb_handle
->wrb_index
);
3726 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, ptr2nextwrb
,
3727 pwrb
, pwrb_handle
->nxt_wrb_index
);
3728 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3729 session_state
, pwrb
, 0);
3730 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, compltonack
,
3732 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, notpredblq
,
3734 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
, mode
, pwrb
,
3737 mem_descr
= phba
->init_mem
;
3738 mem_descr
+= ISCSI_MEM_GLOBAL_HEADER
;
3740 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3741 pad_buffer_addr_hi
, pwrb
,
3742 mem_descr
->mem_array
[0].bus_address
.u
.a32
.address_hi
);
3743 AMAP_SET_BITS(struct amap_iscsi_target_context_update_wrb
,
3744 pad_buffer_addr_lo
, pwrb
,
3745 mem_descr
->mem_array
[0].bus_address
.u
.a32
.address_lo
);
3747 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_target_context_update_wrb
));
3749 doorbell
|= beiscsi_conn
->beiscsi_conn_cid
& DB_WRB_POST_CID_MASK
;
3750 doorbell
|= (pwrb_handle
->wrb_index
& DB_DEF_PDU_WRB_INDEX_MASK
)
3751 << DB_DEF_PDU_WRB_INDEX_SHIFT
;
3752 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
3754 iowrite32(doorbell
, phba
->db_va
+ DB_TXULP0_OFFSET
);
3757 static void beiscsi_parse_pdu(struct iscsi_conn
*conn
, itt_t itt
,
3758 int *index
, int *age
)
3762 *age
= conn
->session
->age
;
3766 * beiscsi_alloc_pdu - allocates pdu and related resources
3767 * @task: libiscsi task
3768 * @opcode: opcode of pdu for task
3770 * This is called with the session lock held. It will allocate
3771 * the wrb and sgl if needed for the command. And it will prep
3772 * the pdu's itt. beiscsi_parse_pdu will later translate
3773 * the pdu itt to the libiscsi task itt.
3775 static int beiscsi_alloc_pdu(struct iscsi_task
*task
, uint8_t opcode
)
3777 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3778 struct iscsi_conn
*conn
= task
->conn
;
3779 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3780 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3781 struct hwi_wrb_context
*pwrb_context
;
3782 struct hwi_controller
*phwi_ctrlr
;
3784 struct beiscsi_session
*beiscsi_sess
= beiscsi_conn
->beiscsi_sess
;
3787 io_task
->cmd_bhs
= pci_pool_alloc(beiscsi_sess
->bhs_pool
,
3788 GFP_ATOMIC
, &paddr
);
3789 if (!io_task
->cmd_bhs
)
3791 io_task
->bhs_pa
.u
.a64
.address
= paddr
;
3792 io_task
->libiscsi_itt
= (itt_t
)task
->itt
;
3793 io_task
->conn
= beiscsi_conn
;
3795 task
->hdr
= (struct iscsi_hdr
*)&io_task
->cmd_bhs
->iscsi_hdr
;
3796 task
->hdr_max
= sizeof(struct be_cmd_bhs
);
3797 io_task
->psgl_handle
= NULL
;
3798 io_task
->psgl_handle
= NULL
;
3801 spin_lock(&phba
->io_sgl_lock
);
3802 io_task
->psgl_handle
= alloc_io_sgl_handle(phba
);
3803 spin_unlock(&phba
->io_sgl_lock
);
3804 if (!io_task
->psgl_handle
)
3806 io_task
->pwrb_handle
= alloc_wrb_handle(phba
,
3807 beiscsi_conn
->beiscsi_conn_cid
-
3808 phba
->fw_config
.iscsi_cid_start
);
3809 if (!io_task
->pwrb_handle
)
3812 io_task
->scsi_cmnd
= NULL
;
3813 if ((opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGIN
) {
3814 if (!beiscsi_conn
->login_in_progress
) {
3815 spin_lock(&phba
->mgmt_sgl_lock
);
3816 io_task
->psgl_handle
= (struct sgl_handle
*)
3817 alloc_mgmt_sgl_handle(phba
);
3818 spin_unlock(&phba
->mgmt_sgl_lock
);
3819 if (!io_task
->psgl_handle
)
3822 beiscsi_conn
->login_in_progress
= 1;
3823 beiscsi_conn
->plogin_sgl_handle
=
3824 io_task
->psgl_handle
;
3825 io_task
->pwrb_handle
=
3826 alloc_wrb_handle(phba
,
3827 beiscsi_conn
->beiscsi_conn_cid
-
3828 phba
->fw_config
.iscsi_cid_start
);
3829 if (!io_task
->pwrb_handle
)
3831 beiscsi_conn
->plogin_wrb_handle
=
3832 io_task
->pwrb_handle
;
3835 io_task
->psgl_handle
=
3836 beiscsi_conn
->plogin_sgl_handle
;
3837 io_task
->pwrb_handle
=
3838 beiscsi_conn
->plogin_wrb_handle
;
3841 spin_lock(&phba
->mgmt_sgl_lock
);
3842 io_task
->psgl_handle
= alloc_mgmt_sgl_handle(phba
);
3843 spin_unlock(&phba
->mgmt_sgl_lock
);
3844 if (!io_task
->psgl_handle
)
3846 io_task
->pwrb_handle
=
3847 alloc_wrb_handle(phba
,
3848 beiscsi_conn
->beiscsi_conn_cid
-
3849 phba
->fw_config
.iscsi_cid_start
);
3850 if (!io_task
->pwrb_handle
)
3851 goto free_mgmt_hndls
;
3855 itt
= (itt_t
) cpu_to_be32(((unsigned int)io_task
->pwrb_handle
->
3856 wrb_index
<< 16) | (unsigned int)
3857 (io_task
->psgl_handle
->sgl_index
));
3858 io_task
->pwrb_handle
->pio_handle
= task
;
3860 io_task
->cmd_bhs
->iscsi_hdr
.itt
= itt
;
3864 spin_lock(&phba
->io_sgl_lock
);
3865 free_io_sgl_handle(phba
, io_task
->psgl_handle
);
3866 spin_unlock(&phba
->io_sgl_lock
);
3869 spin_lock(&phba
->mgmt_sgl_lock
);
3870 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
3871 spin_unlock(&phba
->mgmt_sgl_lock
);
3873 phwi_ctrlr
= phba
->phwi_ctrlr
;
3874 pwrb_context
= &phwi_ctrlr
->wrb_context
[
3875 beiscsi_conn
->beiscsi_conn_cid
-
3876 phba
->fw_config
.iscsi_cid_start
];
3877 if (io_task
->pwrb_handle
)
3878 free_wrb_handle(phba
, pwrb_context
, io_task
->pwrb_handle
);
3879 io_task
->pwrb_handle
= NULL
;
3880 pci_pool_free(beiscsi_sess
->bhs_pool
, io_task
->cmd_bhs
,
3881 io_task
->bhs_pa
.u
.a64
.address
);
3882 SE_DEBUG(DBG_LVL_1
, "Alloc of SGL_ICD Failed\n");
3886 static void beiscsi_cleanup_task(struct iscsi_task
*task
)
3888 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3889 struct iscsi_conn
*conn
= task
->conn
;
3890 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3891 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3892 struct beiscsi_session
*beiscsi_sess
= beiscsi_conn
->beiscsi_sess
;
3893 struct hwi_wrb_context
*pwrb_context
;
3894 struct hwi_controller
*phwi_ctrlr
;
3896 phwi_ctrlr
= phba
->phwi_ctrlr
;
3897 pwrb_context
= &phwi_ctrlr
->wrb_context
[beiscsi_conn
->beiscsi_conn_cid
3898 - phba
->fw_config
.iscsi_cid_start
];
3899 if (io_task
->pwrb_handle
) {
3900 free_wrb_handle(phba
, pwrb_context
, io_task
->pwrb_handle
);
3901 io_task
->pwrb_handle
= NULL
;
3904 if (io_task
->cmd_bhs
) {
3905 pci_pool_free(beiscsi_sess
->bhs_pool
, io_task
->cmd_bhs
,
3906 io_task
->bhs_pa
.u
.a64
.address
);
3910 if (io_task
->psgl_handle
) {
3911 spin_lock(&phba
->io_sgl_lock
);
3912 free_io_sgl_handle(phba
, io_task
->psgl_handle
);
3913 spin_unlock(&phba
->io_sgl_lock
);
3914 io_task
->psgl_handle
= NULL
;
3918 ((task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) == ISCSI_OP_LOGIN
))
3920 if (io_task
->psgl_handle
) {
3921 spin_lock(&phba
->mgmt_sgl_lock
);
3922 free_mgmt_sgl_handle(phba
, io_task
->psgl_handle
);
3923 spin_unlock(&phba
->mgmt_sgl_lock
);
3924 io_task
->psgl_handle
= NULL
;
3929 static int beiscsi_iotask(struct iscsi_task
*task
, struct scatterlist
*sg
,
3930 unsigned int num_sg
, unsigned int xferlen
,
3931 unsigned int writedir
)
3934 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3935 struct iscsi_conn
*conn
= task
->conn
;
3936 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3937 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3938 struct iscsi_wrb
*pwrb
= NULL
;
3939 unsigned int doorbell
= 0;
3941 pwrb
= io_task
->pwrb_handle
->pwrb
;
3942 io_task
->cmd_bhs
->iscsi_hdr
.exp_statsn
= 0;
3943 io_task
->bhs_len
= sizeof(struct be_cmd_bhs
);
3946 memset(&io_task
->cmd_bhs
->iscsi_data_pdu
, 0, 48);
3947 AMAP_SET_BITS(struct amap_pdu_data_out
, itt
,
3948 &io_task
->cmd_bhs
->iscsi_data_pdu
,
3949 (unsigned int)io_task
->cmd_bhs
->iscsi_hdr
.itt
);
3950 AMAP_SET_BITS(struct amap_pdu_data_out
, opcode
,
3951 &io_task
->cmd_bhs
->iscsi_data_pdu
,
3952 ISCSI_OPCODE_SCSI_DATA_OUT
);
3953 AMAP_SET_BITS(struct amap_pdu_data_out
, final_bit
,
3954 &io_task
->cmd_bhs
->iscsi_data_pdu
, 1);
3955 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3957 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 1);
3959 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
3961 AMAP_SET_BITS(struct amap_iscsi_wrb
, dsp
, pwrb
, 0);
3963 memcpy(&io_task
->cmd_bhs
->iscsi_data_pdu
.
3964 dw
[offsetof(struct amap_pdu_data_out
, lun
) / 32],
3965 io_task
->cmd_bhs
->iscsi_hdr
.lun
, sizeof(struct scsi_lun
));
3967 AMAP_SET_BITS(struct amap_iscsi_wrb
, lun
, pwrb
,
3968 cpu_to_be16((unsigned short)io_task
->cmd_bhs
->iscsi_hdr
.
3970 AMAP_SET_BITS(struct amap_iscsi_wrb
, r2t_exp_dtl
, pwrb
, xferlen
);
3971 AMAP_SET_BITS(struct amap_iscsi_wrb
, wrb_idx
, pwrb
,
3972 io_task
->pwrb_handle
->wrb_index
);
3973 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
,
3974 be32_to_cpu(task
->cmdsn
));
3975 AMAP_SET_BITS(struct amap_iscsi_wrb
, sgl_icd_idx
, pwrb
,
3976 io_task
->psgl_handle
->sgl_index
);
3978 hwi_write_sgl(pwrb
, sg
, num_sg
, io_task
);
3980 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
, pwrb
,
3981 io_task
->pwrb_handle
->nxt_wrb_index
);
3982 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_wrb
));
3984 doorbell
|= beiscsi_conn
->beiscsi_conn_cid
& DB_WRB_POST_CID_MASK
;
3985 doorbell
|= (io_task
->pwrb_handle
->wrb_index
&
3986 DB_DEF_PDU_WRB_INDEX_MASK
) << DB_DEF_PDU_WRB_INDEX_SHIFT
;
3987 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
3989 iowrite32(doorbell
, phba
->db_va
+ DB_TXULP0_OFFSET
);
3993 static int beiscsi_mtask(struct iscsi_task
*task
)
3995 struct beiscsi_io_task
*io_task
= task
->dd_data
;
3996 struct iscsi_conn
*conn
= task
->conn
;
3997 struct beiscsi_conn
*beiscsi_conn
= conn
->dd_data
;
3998 struct beiscsi_hba
*phba
= beiscsi_conn
->phba
;
3999 struct iscsi_wrb
*pwrb
= NULL
;
4000 unsigned int doorbell
= 0;
4003 cid
= beiscsi_conn
->beiscsi_conn_cid
;
4004 pwrb
= io_task
->pwrb_handle
->pwrb
;
4005 memset(pwrb
, 0, sizeof(*pwrb
));
4006 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
,
4007 be32_to_cpu(task
->cmdsn
));
4008 AMAP_SET_BITS(struct amap_iscsi_wrb
, wrb_idx
, pwrb
,
4009 io_task
->pwrb_handle
->wrb_index
);
4010 AMAP_SET_BITS(struct amap_iscsi_wrb
, sgl_icd_idx
, pwrb
,
4011 io_task
->psgl_handle
->sgl_index
);
4013 switch (task
->hdr
->opcode
& ISCSI_OPCODE_MASK
) {
4014 case ISCSI_OP_LOGIN
:
4015 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4017 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4018 AMAP_SET_BITS(struct amap_iscsi_wrb
, cmdsn_itt
, pwrb
, 1);
4019 hwi_write_buffer(pwrb
, task
);
4021 case ISCSI_OP_NOOP_OUT
:
4022 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4024 if (task
->hdr
->ttt
== ISCSI_RESERVED_TAG
)
4025 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4027 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 1);
4028 hwi_write_buffer(pwrb
, task
);
4031 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4033 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4034 hwi_write_buffer(pwrb
, task
);
4036 case ISCSI_OP_SCSI_TMFUNC
:
4037 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4039 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4040 hwi_write_buffer(pwrb
, task
);
4042 case ISCSI_OP_LOGOUT
:
4043 AMAP_SET_BITS(struct amap_iscsi_wrb
, dmsg
, pwrb
, 0);
4044 AMAP_SET_BITS(struct amap_iscsi_wrb
, type
, pwrb
,
4046 hwi_write_buffer(pwrb
, task
);
4050 SE_DEBUG(DBG_LVL_1
, "opcode =%d Not supported\n",
4051 task
->hdr
->opcode
& ISCSI_OPCODE_MASK
);
4055 AMAP_SET_BITS(struct amap_iscsi_wrb
, r2t_exp_dtl
, pwrb
,
4057 AMAP_SET_BITS(struct amap_iscsi_wrb
, ptr2nextwrb
, pwrb
,
4058 io_task
->pwrb_handle
->nxt_wrb_index
);
4059 be_dws_le_to_cpu(pwrb
, sizeof(struct iscsi_wrb
));
4061 doorbell
|= cid
& DB_WRB_POST_CID_MASK
;
4062 doorbell
|= (io_task
->pwrb_handle
->wrb_index
&
4063 DB_DEF_PDU_WRB_INDEX_MASK
) << DB_DEF_PDU_WRB_INDEX_SHIFT
;
4064 doorbell
|= 1 << DB_DEF_PDU_NUM_POSTED_SHIFT
;
4065 iowrite32(doorbell
, phba
->db_va
+ DB_TXULP0_OFFSET
);
4069 static int beiscsi_task_xmit(struct iscsi_task
*task
)
4071 struct beiscsi_io_task
*io_task
= task
->dd_data
;
4072 struct scsi_cmnd
*sc
= task
->sc
;
4073 struct scatterlist
*sg
;
4075 unsigned int writedir
= 0, xferlen
= 0;
4078 return beiscsi_mtask(task
);
4080 io_task
->scsi_cmnd
= sc
;
4081 num_sg
= scsi_dma_map(sc
);
4083 SE_DEBUG(DBG_LVL_1
, " scsi_dma_map Failed\n")
4086 xferlen
= scsi_bufflen(sc
);
4087 sg
= scsi_sglist(sc
);
4088 if (sc
->sc_data_direction
== DMA_TO_DEVICE
) {
4090 SE_DEBUG(DBG_LVL_4
, "task->imm_count=0x%08x\n",
4094 return beiscsi_iotask(task
, sg
, num_sg
, xferlen
, writedir
);
4097 static void beiscsi_remove(struct pci_dev
*pcidev
)
4099 struct beiscsi_hba
*phba
= NULL
;
4100 struct hwi_controller
*phwi_ctrlr
;
4101 struct hwi_context_memory
*phwi_context
;
4102 struct be_eq_obj
*pbe_eq
;
4103 unsigned int i
, msix_vec
;
4104 u8
*real_offset
= 0;
4107 phba
= (struct beiscsi_hba
*)pci_get_drvdata(pcidev
);
4109 dev_err(&pcidev
->dev
, "beiscsi_remove called with no phba\n");
4113 phwi_ctrlr
= phba
->phwi_ctrlr
;
4114 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
4115 hwi_disable_intr(phba
);
4116 if (phba
->msix_enabled
) {
4117 for (i
= 0; i
<= phba
->num_cpus
; i
++) {
4118 msix_vec
= phba
->msix_entries
[i
].vector
;
4119 free_irq(msix_vec
, &phwi_context
->be_eq
[i
]);
4122 if (phba
->pcidev
->irq
)
4123 free_irq(phba
->pcidev
->irq
, phba
);
4124 pci_disable_msix(phba
->pcidev
);
4125 destroy_workqueue(phba
->wq
);
4126 if (blk_iopoll_enabled
)
4127 for (i
= 0; i
< phba
->num_cpus
; i
++) {
4128 pbe_eq
= &phwi_context
->be_eq
[i
];
4129 blk_iopoll_disable(&pbe_eq
->iopoll
);
4132 beiscsi_clean_port(phba
);
4133 beiscsi_free_mem(phba
);
4134 real_offset
= (u8
*)phba
->csr_va
+ MPU_EP_SEMAPHORE
;
4136 value
= readl((void *)real_offset
);
4138 if (value
& 0x00010000) {
4139 value
&= 0xfffeffff;
4140 writel(value
, (void *)real_offset
);
4142 beiscsi_unmap_pci_function(phba
);
4143 pci_free_consistent(phba
->pcidev
,
4144 phba
->ctrl
.mbox_mem_alloced
.size
,
4145 phba
->ctrl
.mbox_mem_alloced
.va
,
4146 phba
->ctrl
.mbox_mem_alloced
.dma
);
4147 iscsi_host_remove(phba
->shost
);
4148 pci_dev_put(phba
->pcidev
);
4149 iscsi_host_free(phba
->shost
);
4150 iscsi_boot_destroy_kset(phba
->boot_kset
);
4153 static void beiscsi_msix_enable(struct beiscsi_hba
*phba
)
4157 for (i
= 0; i
<= phba
->num_cpus
; i
++)
4158 phba
->msix_entries
[i
].entry
= i
;
4160 status
= pci_enable_msix(phba
->pcidev
, phba
->msix_entries
,
4161 (phba
->num_cpus
+ 1));
4163 phba
->msix_enabled
= true;
4168 static int __devinit
beiscsi_dev_probe(struct pci_dev
*pcidev
,
4169 const struct pci_device_id
*id
)
4171 struct beiscsi_hba
*phba
= NULL
;
4172 struct hwi_controller
*phwi_ctrlr
;
4173 struct hwi_context_memory
*phwi_context
;
4174 struct be_eq_obj
*pbe_eq
;
4175 int ret
, num_cpus
, i
;
4176 u8
*real_offset
= 0;
4179 ret
= beiscsi_enable_pci(pcidev
);
4181 dev_err(&pcidev
->dev
, "beiscsi_dev_probe-"
4182 " Failed to enable pci device\n");
4186 phba
= beiscsi_hba_alloc(pcidev
);
4188 dev_err(&pcidev
->dev
, "beiscsi_dev_probe-"
4189 " Failed in beiscsi_hba_alloc\n");
4193 switch (pcidev
->device
) {
4197 phba
->generation
= BE_GEN2
;
4201 phba
->generation
= BE_GEN3
;
4204 phba
->generation
= 0;
4208 num_cpus
= find_num_cpus();
4211 phba
->num_cpus
= num_cpus
;
4212 SE_DEBUG(DBG_LVL_8
, "num_cpus = %d\n", phba
->num_cpus
);
4215 beiscsi_msix_enable(phba
);
4216 ret
= be_ctrl_init(phba
, pcidev
);
4218 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
4219 "Failed in be_ctrl_init\n");
4224 real_offset
= (u8
*)phba
->csr_va
+ MPU_EP_SEMAPHORE
;
4225 value
= readl((void *)real_offset
);
4226 if (value
& 0x00010000) {
4228 shost_printk(KERN_ERR
, phba
->shost
,
4229 "Loading Driver in crashdump mode\n");
4230 ret
= beiscsi_pci_soft_reset(phba
);
4232 shost_printk(KERN_ERR
, phba
->shost
,
4233 "Reset Failed. Aborting Crashdump\n");
4236 ret
= be_chk_reset_complete(phba
);
4238 shost_printk(KERN_ERR
, phba
->shost
,
4239 "Failed to get out of reset."
4240 "Aborting Crashdump\n");
4244 value
|= 0x00010000;
4245 writel(value
, (void *)real_offset
);
4250 spin_lock_init(&phba
->io_sgl_lock
);
4251 spin_lock_init(&phba
->mgmt_sgl_lock
);
4252 spin_lock_init(&phba
->isr_lock
);
4253 ret
= mgmt_get_fw_config(&phba
->ctrl
, phba
);
4255 shost_printk(KERN_ERR
, phba
->shost
,
4256 "Error getting fw config\n");
4259 phba
->shost
->max_id
= phba
->fw_config
.iscsi_cid_count
;
4260 beiscsi_get_params(phba
);
4261 phba
->shost
->can_queue
= phba
->params
.ios_per_ctrl
;
4262 ret
= beiscsi_init_port(phba
);
4264 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
4265 "Failed in beiscsi_init_port\n");
4269 for (i
= 0; i
< MAX_MCC_CMD
; i
++) {
4270 init_waitqueue_head(&phba
->ctrl
.mcc_wait
[i
+ 1]);
4271 phba
->ctrl
.mcc_tag
[i
] = i
+ 1;
4272 phba
->ctrl
.mcc_numtag
[i
+ 1] = 0;
4273 phba
->ctrl
.mcc_tag_available
++;
4276 phba
->ctrl
.mcc_alloc_index
= phba
->ctrl
.mcc_free_index
= 0;
4278 snprintf(phba
->wq_name
, sizeof(phba
->wq_name
), "beiscsi_q_irq%u",
4279 phba
->shost
->host_no
);
4280 phba
->wq
= alloc_workqueue(phba
->wq_name
, WQ_MEM_RECLAIM
, 1);
4282 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
4283 "Failed to allocate work queue\n");
4287 INIT_WORK(&phba
->work_cqs
, beiscsi_process_all_cqs
);
4289 phwi_ctrlr
= phba
->phwi_ctrlr
;
4290 phwi_context
= phwi_ctrlr
->phwi_ctxt
;
4291 if (blk_iopoll_enabled
) {
4292 for (i
= 0; i
< phba
->num_cpus
; i
++) {
4293 pbe_eq
= &phwi_context
->be_eq
[i
];
4294 blk_iopoll_init(&pbe_eq
->iopoll
, be_iopoll_budget
,
4296 blk_iopoll_enable(&pbe_eq
->iopoll
);
4299 ret
= beiscsi_init_irqs(phba
);
4301 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
4302 "Failed to beiscsi_init_irqs\n");
4305 hwi_enable_intr(phba
);
4306 ret
= beiscsi_get_boot_info(phba
);
4308 shost_printk(KERN_ERR
, phba
->shost
, "beiscsi_dev_probe-"
4309 "No Boot Devices !!!!!\n");
4311 SE_DEBUG(DBG_LVL_8
, "\n\n\n SUCCESS - DRIVER LOADED\n\n\n");
4315 destroy_workqueue(phba
->wq
);
4316 if (blk_iopoll_enabled
)
4317 for (i
= 0; i
< phba
->num_cpus
; i
++) {
4318 pbe_eq
= &phwi_context
->be_eq
[i
];
4319 blk_iopoll_disable(&pbe_eq
->iopoll
);
4322 beiscsi_clean_port(phba
);
4323 beiscsi_free_mem(phba
);
4325 real_offset
= (u8
*)phba
->csr_va
+ MPU_EP_SEMAPHORE
;
4327 value
= readl((void *)real_offset
);
4329 if (value
& 0x00010000) {
4330 value
&= 0xfffeffff;
4331 writel(value
, (void *)real_offset
);
4334 pci_free_consistent(phba
->pcidev
,
4335 phba
->ctrl
.mbox_mem_alloced
.size
,
4336 phba
->ctrl
.mbox_mem_alloced
.va
,
4337 phba
->ctrl
.mbox_mem_alloced
.dma
);
4338 beiscsi_unmap_pci_function(phba
);
4340 if (phba
->msix_enabled
)
4341 pci_disable_msix(phba
->pcidev
);
4342 iscsi_host_remove(phba
->shost
);
4343 pci_dev_put(phba
->pcidev
);
4344 iscsi_host_free(phba
->shost
);
4346 pci_disable_device(pcidev
);
4350 struct iscsi_transport beiscsi_iscsi_transport
= {
4351 .owner
= THIS_MODULE
,
4353 .caps
= CAP_RECOVERY_L0
| CAP_HDRDGST
| CAP_TEXT_NEGO
|
4354 CAP_MULTI_R2T
| CAP_DATADGST
| CAP_DATA_PATH_OFFLOAD
,
4355 .param_mask
= ISCSI_MAX_RECV_DLENGTH
|
4356 ISCSI_MAX_XMIT_DLENGTH
|
4359 ISCSI_INITIAL_R2T_EN
|
4364 ISCSI_PDU_INORDER_EN
|
4365 ISCSI_DATASEQ_INORDER_EN
|
4368 ISCSI_CONN_ADDRESS
|
4370 ISCSI_PERSISTENT_PORT
|
4371 ISCSI_PERSISTENT_ADDRESS
|
4372 ISCSI_TARGET_NAME
| ISCSI_TPGT
|
4373 ISCSI_USERNAME
| ISCSI_PASSWORD
|
4374 ISCSI_USERNAME_IN
| ISCSI_PASSWORD_IN
|
4375 ISCSI_FAST_ABORT
| ISCSI_ABORT_TMO
|
4376 ISCSI_LU_RESET_TMO
|
4377 ISCSI_PING_TMO
| ISCSI_RECV_TMO
|
4378 ISCSI_IFACE_NAME
| ISCSI_INITIATOR_NAME
,
4379 .host_param_mask
= ISCSI_HOST_HWADDRESS
| ISCSI_HOST_IPADDRESS
|
4380 ISCSI_HOST_INITIATOR_NAME
,
4381 .create_session
= beiscsi_session_create
,
4382 .destroy_session
= beiscsi_session_destroy
,
4383 .create_conn
= beiscsi_conn_create
,
4384 .bind_conn
= beiscsi_conn_bind
,
4385 .destroy_conn
= iscsi_conn_teardown
,
4386 .set_param
= beiscsi_set_param
,
4387 .get_conn_param
= iscsi_conn_get_param
,
4388 .get_session_param
= iscsi_session_get_param
,
4389 .get_host_param
= beiscsi_get_host_param
,
4390 .start_conn
= beiscsi_conn_start
,
4391 .stop_conn
= iscsi_conn_stop
,
4392 .send_pdu
= iscsi_conn_send_pdu
,
4393 .xmit_task
= beiscsi_task_xmit
,
4394 .cleanup_task
= beiscsi_cleanup_task
,
4395 .alloc_pdu
= beiscsi_alloc_pdu
,
4396 .parse_pdu_itt
= beiscsi_parse_pdu
,
4397 .get_stats
= beiscsi_conn_get_stats
,
4398 .get_ep_param
= beiscsi_ep_get_param
,
4399 .ep_connect
= beiscsi_ep_connect
,
4400 .ep_poll
= beiscsi_ep_poll
,
4401 .ep_disconnect
= beiscsi_ep_disconnect
,
4402 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
4405 static struct pci_driver beiscsi_pci_driver
= {
4407 .probe
= beiscsi_dev_probe
,
4408 .remove
= beiscsi_remove
,
4409 .id_table
= beiscsi_pci_id_table
4413 static int __init
beiscsi_module_init(void)
4417 beiscsi_scsi_transport
=
4418 iscsi_register_transport(&beiscsi_iscsi_transport
);
4419 if (!beiscsi_scsi_transport
) {
4421 "beiscsi_module_init - Unable to register beiscsi"
4425 SE_DEBUG(DBG_LVL_8
, "In beiscsi_module_init, tt=%p\n",
4426 &beiscsi_iscsi_transport
);
4428 ret
= pci_register_driver(&beiscsi_pci_driver
);
4431 "beiscsi_module_init - Unable to register"
4432 "beiscsi pci driver.\n");
4433 goto unregister_iscsi_transport
;
4437 unregister_iscsi_transport
:
4438 iscsi_unregister_transport(&beiscsi_iscsi_transport
);
4442 static void __exit
beiscsi_module_exit(void)
4444 pci_unregister_driver(&beiscsi_pci_driver
);
4445 iscsi_unregister_transport(&beiscsi_iscsi_transport
);
4448 module_init(beiscsi_module_init
);
4449 module_exit(beiscsi_module_exit
);