2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
4 * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
6 * This program is distributed in the hope that it will be useful, but WITHOUT
7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
9 * release for licensing terms and conditions.
11 * Written by: Dimitris Michailidis (dm@chelsio.com)
12 * Karen Xie (kxie@chelsio.com)
15 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
17 #include <linux/version.h>
18 #include <linux/module.h>
19 #include <linux/moduleparam.h>
20 #include <scsi/scsi_host.h>
25 #include "cxgb3_defs.h"
26 #include "cxgb3_ctl_defs.h"
27 #include "cxgb3_offload.h"
28 #include "firmware_exports.h"
31 static unsigned int dbg_level
;
32 #include "../libcxgbi.h"
34 #define DRV_MODULE_NAME "cxgb3i"
35 #define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver"
36 #define DRV_MODULE_VERSION "2.0.0"
37 #define DRV_MODULE_RELDATE "Jun. 2010"
39 static char version
[] =
40 DRV_MODULE_DESC
" " DRV_MODULE_NAME
41 " v" DRV_MODULE_VERSION
" (" DRV_MODULE_RELDATE
")\n";
43 MODULE_AUTHOR("Chelsio Communications, Inc.");
44 MODULE_DESCRIPTION(DRV_MODULE_DESC
);
45 MODULE_VERSION(DRV_MODULE_VERSION
);
46 MODULE_LICENSE("GPL");
48 module_param(dbg_level
, uint
, 0644);
49 MODULE_PARM_DESC(dbg_level
, "debug flag (default=0)");
51 static int cxgb3i_rcv_win
= 256 * 1024;
52 module_param(cxgb3i_rcv_win
, int, 0644);
53 MODULE_PARM_DESC(cxgb3i_rcv_win
, "TCP receive window in bytes (default=256KB)");
55 static int cxgb3i_snd_win
= 128 * 1024;
56 module_param(cxgb3i_snd_win
, int, 0644);
57 MODULE_PARM_DESC(cxgb3i_snd_win
, "TCP send window in bytes (default=128KB)");
59 static int cxgb3i_rx_credit_thres
= 10 * 1024;
60 module_param(cxgb3i_rx_credit_thres
, int, 0644);
61 MODULE_PARM_DESC(rx_credit_thres
,
62 "RX credits return threshold in bytes (default=10KB)");
64 static unsigned int cxgb3i_max_connect
= 8 * 1024;
65 module_param(cxgb3i_max_connect
, uint
, 0644);
66 MODULE_PARM_DESC(cxgb3i_max_connect
, "Max. # of connections (default=8092)");
68 static unsigned int cxgb3i_sport_base
= 20000;
69 module_param(cxgb3i_sport_base
, uint
, 0644);
70 MODULE_PARM_DESC(cxgb3i_sport_base
, "starting port number (default=20000)");
72 static void cxgb3i_dev_open(struct t3cdev
*);
73 static void cxgb3i_dev_close(struct t3cdev
*);
74 static void cxgb3i_dev_event_handler(struct t3cdev
*, u32
, u32
);
76 static struct cxgb3_client t3_client
= {
77 .name
= DRV_MODULE_NAME
,
78 .handlers
= cxgb3i_cpl_handlers
,
79 .add
= cxgb3i_dev_open
,
80 .remove
= cxgb3i_dev_close
,
81 .event_handler
= cxgb3i_dev_event_handler
,
84 static struct scsi_host_template cxgb3i_host_template
= {
85 .module
= THIS_MODULE
,
86 .name
= DRV_MODULE_NAME
,
87 .proc_name
= DRV_MODULE_NAME
,
88 .can_queue
= CXGB3I_SCSI_HOST_QDEPTH
,
89 .queuecommand
= iscsi_queuecommand
,
90 .change_queue_depth
= iscsi_change_queue_depth
,
91 .sg_tablesize
= SG_ALL
,
92 .max_sectors
= 0xFFFF,
93 .cmd_per_lun
= ISCSI_DEF_CMD_PER_LUN
,
94 .eh_abort_handler
= iscsi_eh_abort
,
95 .eh_device_reset_handler
= iscsi_eh_device_reset
,
96 .eh_target_reset_handler
= iscsi_eh_recover_target
,
97 .target_alloc
= iscsi_target_alloc
,
98 .use_clustering
= DISABLE_CLUSTERING
,
102 static struct iscsi_transport cxgb3i_iscsi_transport
= {
103 .owner
= THIS_MODULE
,
104 .name
= DRV_MODULE_NAME
,
105 /* owner and name should be set already */
106 .caps
= CAP_RECOVERY_L0
| CAP_MULTI_R2T
| CAP_HDRDGST
107 | CAP_DATADGST
| CAP_DIGEST_OFFLOAD
|
108 CAP_PADDING_OFFLOAD
| CAP_TEXT_NEGO
,
109 .param_mask
= ISCSI_MAX_RECV_DLENGTH
| ISCSI_MAX_XMIT_DLENGTH
|
110 ISCSI_HDRDGST_EN
| ISCSI_DATADGST_EN
|
111 ISCSI_INITIAL_R2T_EN
| ISCSI_MAX_R2T
|
112 ISCSI_IMM_DATA_EN
| ISCSI_FIRST_BURST
|
113 ISCSI_MAX_BURST
| ISCSI_PDU_INORDER_EN
|
114 ISCSI_DATASEQ_INORDER_EN
| ISCSI_ERL
|
115 ISCSI_CONN_PORT
| ISCSI_CONN_ADDRESS
|
116 ISCSI_EXP_STATSN
| ISCSI_PERSISTENT_PORT
|
117 ISCSI_PERSISTENT_ADDRESS
|
118 ISCSI_TARGET_NAME
| ISCSI_TPGT
|
119 ISCSI_USERNAME
| ISCSI_PASSWORD
|
120 ISCSI_USERNAME_IN
| ISCSI_PASSWORD_IN
|
121 ISCSI_FAST_ABORT
| ISCSI_ABORT_TMO
|
122 ISCSI_LU_RESET_TMO
| ISCSI_TGT_RESET_TMO
|
123 ISCSI_PING_TMO
| ISCSI_RECV_TMO
|
124 ISCSI_IFACE_NAME
| ISCSI_INITIATOR_NAME
,
125 .host_param_mask
= ISCSI_HOST_HWADDRESS
| ISCSI_HOST_IPADDRESS
|
126 ISCSI_HOST_INITIATOR_NAME
|
127 ISCSI_HOST_NETDEV_NAME
,
128 .get_host_param
= cxgbi_get_host_param
,
129 .set_host_param
= cxgbi_set_host_param
,
130 /* session management */
131 .create_session
= cxgbi_create_session
,
132 .destroy_session
= cxgbi_destroy_session
,
133 .get_session_param
= iscsi_session_get_param
,
134 /* connection management */
135 .create_conn
= cxgbi_create_conn
,
136 .bind_conn
= cxgbi_bind_conn
,
137 .destroy_conn
= iscsi_tcp_conn_teardown
,
138 .start_conn
= iscsi_conn_start
,
139 .stop_conn
= iscsi_conn_stop
,
140 .get_conn_param
= iscsi_conn_get_param
,
141 .set_param
= cxgbi_set_conn_param
,
142 .get_stats
= cxgbi_get_conn_stats
,
143 /* pdu xmit req from user space */
144 .send_pdu
= iscsi_conn_send_pdu
,
146 .init_task
= iscsi_tcp_task_init
,
147 .xmit_task
= iscsi_tcp_task_xmit
,
148 .cleanup_task
= cxgbi_cleanup_task
,
150 .alloc_pdu
= cxgbi_conn_alloc_pdu
,
151 .init_pdu
= cxgbi_conn_init_pdu
,
152 .xmit_pdu
= cxgbi_conn_xmit_pdu
,
153 .parse_pdu_itt
= cxgbi_parse_pdu_itt
,
154 /* TCP connect/disconnect */
155 .get_ep_param
= cxgbi_get_ep_param
,
156 .ep_connect
= cxgbi_ep_connect
,
157 .ep_poll
= cxgbi_ep_poll
,
158 .ep_disconnect
= cxgbi_ep_disconnect
,
159 /* Error recovery timeout call */
160 .session_recovery_timedout
= iscsi_session_recovery_timedout
,
163 static struct scsi_transport_template
*cxgb3i_stt
;
166 * CPL (Chelsio Protocol Language) defines a message passing interface between
167 * the host driver and Chelsio asic.
168 * The section below implments CPLs that related to iscsi tcp connection
169 * open/close/abort and data send/receive.
172 static int push_tx_frames(struct cxgbi_sock
*csk
, int req_completion
);
174 static void send_act_open_req(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
175 const struct l2t_entry
*e
)
177 unsigned int wscale
= cxgbi_sock_compute_wscale(cxgb3i_rcv_win
);
178 struct cpl_act_open_req
*req
= (struct cpl_act_open_req
*)skb
->head
;
180 skb
->priority
= CPL_PRIORITY_SETUP
;
182 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
183 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ
, csk
->atid
));
184 req
->local_port
= csk
->saddr
.sin_port
;
185 req
->peer_port
= csk
->daddr
.sin_port
;
186 req
->local_ip
= csk
->saddr
.sin_addr
.s_addr
;
187 req
->peer_ip
= csk
->daddr
.sin_addr
.s_addr
;
189 req
->opt0h
= htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS
|
190 V_WND_SCALE(wscale
) | V_MSS_IDX(csk
->mss_idx
) |
191 V_L2T_IDX(e
->idx
) | V_TX_CHANNEL(e
->smt_idx
));
192 req
->opt0l
= htonl(V_ULP_MODE(ULP2_MODE_ISCSI
) |
193 V_RCV_BUFSIZ(cxgb3i_rcv_win
>>10));
195 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
196 "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n",
197 csk
, csk
->state
, csk
->flags
, csk
->atid
,
198 &req
->local_ip
, ntohs(req
->local_port
),
199 &req
->peer_ip
, ntohs(req
->peer_port
),
200 csk
->mss_idx
, e
->idx
, e
->smt_idx
);
202 l2t_send(csk
->cdev
->lldev
, skb
, csk
->l2t
);
205 static inline void act_open_arp_failure(struct t3cdev
*dev
, struct sk_buff
*skb
)
207 cxgbi_sock_act_open_req_arp_failure(NULL
, skb
);
211 * CPL connection close request: host ->
213 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
214 * the write queue (i.e., after any unsent txt data).
216 static void send_close_req(struct cxgbi_sock
*csk
)
218 struct sk_buff
*skb
= csk
->cpl_close
;
219 struct cpl_close_con_req
*req
= (struct cpl_close_con_req
*)skb
->head
;
220 unsigned int tid
= csk
->tid
;
222 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
223 "csk 0x%p,%u,0x%lx,%u.\n",
224 csk
, csk
->state
, csk
->flags
, csk
->tid
);
226 csk
->cpl_close
= NULL
;
227 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON
));
228 req
->wr
.wr_lo
= htonl(V_WR_TID(tid
));
229 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ
, tid
));
230 req
->rsvd
= htonl(csk
->write_seq
);
232 cxgbi_sock_skb_entail(csk
, skb
);
233 if (csk
->state
>= CTP_ESTABLISHED
)
234 push_tx_frames(csk
, 1);
238 * CPL connection abort request: host ->
240 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
241 * for the same connection and also that we do not try to send a message
242 * after the connection has closed.
244 static void abort_arp_failure(struct t3cdev
*tdev
, struct sk_buff
*skb
)
246 struct cpl_abort_req
*req
= cplhdr(skb
);
248 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
249 "t3dev 0x%p, tid %u, skb 0x%p.\n",
250 tdev
, GET_TID(req
), skb
);
251 req
->cmd
= CPL_ABORT_NO_RST
;
252 cxgb3_ofld_send(tdev
, skb
);
255 static void send_abort_req(struct cxgbi_sock
*csk
)
257 struct sk_buff
*skb
= csk
->cpl_abort_req
;
258 struct cpl_abort_req
*req
;
260 if (unlikely(csk
->state
== CTP_ABORTING
|| !skb
))
262 cxgbi_sock_set_state(csk
, CTP_ABORTING
);
263 cxgbi_sock_set_flag(csk
, CTPF_ABORT_RPL_PENDING
);
264 /* Purge the send queue so we don't send anything after an abort. */
265 cxgbi_sock_purge_write_queue(csk
);
267 csk
->cpl_abort_req
= NULL
;
268 req
= (struct cpl_abort_req
*)skb
->head
;
269 skb
->priority
= CPL_PRIORITY_DATA
;
270 set_arp_failure_handler(skb
, abort_arp_failure
);
271 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ
));
272 req
->wr
.wr_lo
= htonl(V_WR_TID(csk
->tid
));
273 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ
, csk
->tid
));
274 req
->rsvd0
= htonl(csk
->snd_nxt
);
275 req
->rsvd1
= !cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
);
276 req
->cmd
= CPL_ABORT_SEND_RST
;
278 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
279 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
280 csk
, csk
->state
, csk
->flags
, csk
->tid
, csk
->snd_nxt
,
283 l2t_send(csk
->cdev
->lldev
, skb
, csk
->l2t
);
287 * CPL connection abort reply: host ->
289 * Send an ABORT_RPL message in response of the ABORT_REQ received.
291 static void send_abort_rpl(struct cxgbi_sock
*csk
, int rst_status
)
293 struct sk_buff
*skb
= csk
->cpl_abort_rpl
;
294 struct cpl_abort_rpl
*rpl
= (struct cpl_abort_rpl
*)skb
->head
;
296 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
297 "csk 0x%p,%u,0x%lx,%u, status %d.\n",
298 csk
, csk
->state
, csk
->flags
, csk
->tid
, rst_status
);
300 csk
->cpl_abort_rpl
= NULL
;
301 skb
->priority
= CPL_PRIORITY_DATA
;
302 rpl
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL
));
303 rpl
->wr
.wr_lo
= htonl(V_WR_TID(csk
->tid
));
304 OPCODE_TID(rpl
) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL
, csk
->tid
));
305 rpl
->cmd
= rst_status
;
306 cxgb3_ofld_send(csk
->cdev
->lldev
, skb
);
310 * CPL connection rx data ack: host ->
311 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
314 static u32
send_rx_credits(struct cxgbi_sock
*csk
, u32 credits
)
317 struct cpl_rx_data_ack
*req
;
318 u32 dack
= F_RX_DACK_CHANGE
| V_RX_DACK_MODE(1);
320 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
321 "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n",
322 csk
, csk
->state
, csk
->flags
, csk
->tid
, credits
, dack
);
324 skb
= alloc_wr(sizeof(*req
), 0, GFP_ATOMIC
);
326 pr_info("csk 0x%p, credit %u, OOM.\n", csk
, credits
);
329 req
= (struct cpl_rx_data_ack
*)skb
->head
;
330 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
331 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK
, csk
->tid
));
332 req
->credit_dack
= htonl(F_RX_DACK_CHANGE
| V_RX_DACK_MODE(1) |
333 V_RX_CREDITS(credits
));
334 skb
->priority
= CPL_PRIORITY_ACK
;
335 cxgb3_ofld_send(csk
->cdev
->lldev
, skb
);
340 * CPL connection tx data: host ->
342 * Send iscsi PDU via TX_DATA CPL message. Returns the number of
344 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
345 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
348 static unsigned int wrlen __read_mostly
;
349 static unsigned int skb_wrs
[SKB_WR_LIST_SIZE
] __read_mostly
;
351 static void init_wr_tab(unsigned int wr_len
)
355 if (skb_wrs
[1]) /* already initialized */
357 for (i
= 1; i
< SKB_WR_LIST_SIZE
; i
++) {
358 int sgl_len
= (3 * i
) / 2 + (i
& 1);
361 skb_wrs
[i
] = (sgl_len
<= wr_len
362 ? 1 : 1 + (sgl_len
- 2) / (wr_len
- 1));
367 static inline void make_tx_data_wr(struct cxgbi_sock
*csk
, struct sk_buff
*skb
,
368 int len
, int req_completion
)
370 struct tx_data_wr
*req
;
371 struct l2t_entry
*l2t
= csk
->l2t
;
373 skb_reset_transport_header(skb
);
374 req
= (struct tx_data_wr
*)__skb_push(skb
, sizeof(*req
));
375 req
->wr_hi
= htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA
) |
376 (req_completion
? F_WR_COMPL
: 0));
377 req
->wr_lo
= htonl(V_WR_TID(csk
->tid
));
378 /* len includes the length of any HW ULP additions */
379 req
->len
= htonl(len
);
380 /* V_TX_ULP_SUBMODE sets both the mode and submode */
381 req
->flags
= htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb
)) |
382 V_TX_SHOVE((skb_peek(&csk
->write_queue
) ? 0 : 1)));
383 req
->sndseq
= htonl(csk
->snd_nxt
);
384 req
->param
= htonl(V_TX_PORT(l2t
->smt_idx
));
386 if (!cxgbi_sock_flag(csk
, CTPF_TX_DATA_SENT
)) {
387 req
->flags
|= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT
|
388 V_TX_CPU_IDX(csk
->rss_qid
));
389 /* sendbuffer is in units of 32KB. */
390 req
->param
|= htonl(V_TX_SNDBUF(cxgb3i_snd_win
>> 15));
391 cxgbi_sock_set_flag(csk
, CTPF_TX_DATA_SENT
);
396 * push_tx_frames -- start transmit
397 * @c3cn: the offloaded connection
398 * @req_completion: request wr_ack or not
400 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
401 * connection's send queue and sends them on to T3. Must be called with the
402 * connection's lock held. Returns the amount of send buffer space that was
403 * freed as a result of sending queued data to T3.
406 static void arp_failure_skb_discard(struct t3cdev
*dev
, struct sk_buff
*skb
)
411 static int push_tx_frames(struct cxgbi_sock
*csk
, int req_completion
)
416 if (unlikely(csk
->state
< CTP_ESTABLISHED
||
417 csk
->state
== CTP_CLOSE_WAIT_1
|| csk
->state
>= CTP_ABORTING
)) {
418 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_TX
,
419 "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
420 csk
, csk
->state
, csk
->flags
, csk
->tid
);
424 while (csk
->wr_cred
&& (skb
= skb_peek(&csk
->write_queue
)) != NULL
) {
425 int len
= skb
->len
; /* length before skb_push */
426 int frags
= skb_shinfo(skb
)->nr_frags
+ (len
!= skb
->data_len
);
427 int wrs_needed
= skb_wrs
[frags
];
429 if (wrs_needed
> 1 && len
+ sizeof(struct tx_data_wr
) <= wrlen
)
432 WARN_ON(frags
>= SKB_WR_LIST_SIZE
|| wrs_needed
< 1);
434 if (csk
->wr_cred
< wrs_needed
) {
435 log_debug(1 << CXGBI_DBG_PDU_TX
,
436 "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n",
437 csk
, skb
->len
, skb
->data_len
, frags
,
438 wrs_needed
, csk
->wr_cred
);
442 __skb_unlink(skb
, &csk
->write_queue
);
443 skb
->priority
= CPL_PRIORITY_DATA
;
444 skb
->csum
= wrs_needed
; /* remember this until the WR_ACK */
445 csk
->wr_cred
-= wrs_needed
;
446 csk
->wr_una_cred
+= wrs_needed
;
447 cxgbi_sock_enqueue_wr(csk
, skb
);
449 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_TX
,
450 "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, "
451 "left %u, unack %u.\n",
452 csk
, skb
->len
, skb
->data_len
, frags
, skb
->csum
,
453 csk
->wr_cred
, csk
->wr_una_cred
);
455 if (likely(cxgbi_skcb_test_flag(skb
, SKCBF_TX_NEED_HDR
))) {
456 if ((req_completion
&&
457 csk
->wr_una_cred
== wrs_needed
) ||
458 csk
->wr_una_cred
>= csk
->wr_max_cred
/ 2) {
460 csk
->wr_una_cred
= 0;
462 len
+= cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb
));
463 make_tx_data_wr(csk
, skb
, len
, req_completion
);
465 cxgbi_skcb_clear_flag(skb
, SKCBF_TX_NEED_HDR
);
467 total_size
+= skb
->truesize
;
468 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_TX
,
469 "csk 0x%p, tid 0x%x, send skb 0x%p.\n",
471 set_arp_failure_handler(skb
, arp_failure_skb_discard
);
472 l2t_send(csk
->cdev
->lldev
, skb
, csk
->l2t
);
478 * Process a CPL_ACT_ESTABLISH message: -> host
479 * Updates connection state from an active establish CPL message. Runs with
480 * the connection lock held.
483 static inline void free_atid(struct cxgbi_sock
*csk
)
485 if (cxgbi_sock_flag(csk
, CTPF_HAS_ATID
)) {
486 cxgb3_free_atid(csk
->cdev
->lldev
, csk
->atid
);
487 cxgbi_sock_clear_flag(csk
, CTPF_HAS_ATID
);
492 static int do_act_establish(struct t3cdev
*tdev
, struct sk_buff
*skb
, void *ctx
)
494 struct cxgbi_sock
*csk
= ctx
;
495 struct cpl_act_establish
*req
= cplhdr(skb
);
496 unsigned int tid
= GET_TID(req
);
497 unsigned int atid
= G_PASS_OPEN_TID(ntohl(req
->tos_tid
));
498 u32 rcv_isn
= ntohl(req
->rcv_isn
); /* real RCV_ISN + 1 */
500 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
501 "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n",
502 atid
, atid
, csk
, csk
->state
, csk
->flags
, rcv_isn
);
505 cxgbi_sock_set_flag(csk
, CTPF_HAS_TID
);
507 cxgb3_insert_tid(csk
->cdev
->lldev
, &t3_client
, csk
, tid
);
511 csk
->rss_qid
= G_QNUM(ntohs(skb
->csum
));
513 spin_lock_bh(&csk
->lock
);
514 if (csk
->retry_timer
.function
) {
515 del_timer(&csk
->retry_timer
);
516 csk
->retry_timer
.function
= NULL
;
519 if (unlikely(csk
->state
!= CTP_ACTIVE_OPEN
))
520 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
521 csk
, csk
->state
, csk
->flags
, csk
->tid
);
523 csk
->copied_seq
= csk
->rcv_wup
= csk
->rcv_nxt
= rcv_isn
;
524 if (cxgb3i_rcv_win
> (M_RCV_BUFSIZ
<< 10))
525 csk
->rcv_wup
-= cxgb3i_rcv_win
- (M_RCV_BUFSIZ
<< 10);
527 cxgbi_sock_established(csk
, ntohl(req
->snd_isn
), ntohs(req
->tcp_opt
));
529 if (unlikely(cxgbi_sock_flag(csk
, CTPF_ACTIVE_CLOSE_NEEDED
)))
530 /* upper layer has requested closing */
533 if (skb_queue_len(&csk
->write_queue
))
534 push_tx_frames(csk
, 1);
535 cxgbi_conn_tx_open(csk
);
538 spin_unlock_bh(&csk
->lock
);
544 * Process a CPL_ACT_OPEN_RPL message: -> host
545 * Handle active open failures.
547 static int act_open_rpl_status_to_errno(int status
)
550 case CPL_ERR_CONN_RESET
:
551 return -ECONNREFUSED
;
552 case CPL_ERR_ARP_MISS
:
553 return -EHOSTUNREACH
;
554 case CPL_ERR_CONN_TIMEDOUT
:
556 case CPL_ERR_TCAM_FULL
:
558 case CPL_ERR_CONN_EXIST
:
565 static void act_open_retry_timer(unsigned long data
)
568 struct cxgbi_sock
*csk
= (struct cxgbi_sock
*)data
;
570 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
571 "csk 0x%p,%u,0x%lx,%u.\n",
572 csk
, csk
->state
, csk
->flags
, csk
->tid
);
575 spin_lock_bh(&csk
->lock
);
576 skb
= alloc_wr(sizeof(struct cpl_act_open_req
), 0, GFP_ATOMIC
);
578 cxgbi_sock_fail_act_open(csk
, -ENOMEM
);
580 skb
->sk
= (struct sock
*)csk
;
581 set_arp_failure_handler(skb
, act_open_arp_failure
);
582 send_act_open_req(csk
, skb
, csk
->l2t
);
584 spin_unlock_bh(&csk
->lock
);
588 static int do_act_open_rpl(struct t3cdev
*tdev
, struct sk_buff
*skb
, void *ctx
)
590 struct cxgbi_sock
*csk
= ctx
;
591 struct cpl_act_open_rpl
*rpl
= cplhdr(skb
);
593 pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n",
594 csk
, csk
->state
, csk
->flags
, csk
->atid
, rpl
->status
,
595 &csk
->saddr
.sin_addr
.s_addr
, ntohs(csk
->saddr
.sin_port
),
596 &csk
->daddr
.sin_addr
.s_addr
, ntohs(csk
->daddr
.sin_port
));
598 if (rpl
->status
!= CPL_ERR_TCAM_FULL
&&
599 rpl
->status
!= CPL_ERR_CONN_EXIST
&&
600 rpl
->status
!= CPL_ERR_ARP_MISS
)
601 cxgb3_queue_tid_release(tdev
, GET_TID(rpl
));
604 spin_lock_bh(&csk
->lock
);
605 if (rpl
->status
== CPL_ERR_CONN_EXIST
&&
606 csk
->retry_timer
.function
!= act_open_retry_timer
) {
607 csk
->retry_timer
.function
= act_open_retry_timer
;
608 mod_timer(&csk
->retry_timer
, jiffies
+ HZ
/ 2);
610 cxgbi_sock_fail_act_open(csk
,
611 act_open_rpl_status_to_errno(rpl
->status
));
613 spin_unlock_bh(&csk
->lock
);
620 * Process PEER_CLOSE CPL messages: -> host
623 static int do_peer_close(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
625 struct cxgbi_sock
*csk
= ctx
;
627 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
628 "csk 0x%p,%u,0x%lx,%u.\n",
629 csk
, csk
->state
, csk
->flags
, csk
->tid
);
631 cxgbi_sock_rcv_peer_close(csk
);
637 * Process CLOSE_CONN_RPL CPL message: -> host
638 * Process a peer ACK to our FIN.
640 static int do_close_con_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
,
643 struct cxgbi_sock
*csk
= ctx
;
644 struct cpl_close_con_rpl
*rpl
= cplhdr(skb
);
646 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
647 "csk 0x%p,%u,0x%lx,%u, snxt %u.\n",
648 csk
, csk
->state
, csk
->flags
, csk
->tid
, ntohl(rpl
->snd_nxt
));
650 cxgbi_sock_rcv_close_conn_rpl(csk
, ntohl(rpl
->snd_nxt
));
656 * Process ABORT_REQ_RSS CPL message: -> host
657 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
658 * request except that we need to reply to it.
661 static int abort_status_to_errno(struct cxgbi_sock
*csk
, int abort_reason
,
664 switch (abort_reason
) {
665 case CPL_ERR_BAD_SYN
: /* fall through */
666 case CPL_ERR_CONN_RESET
:
667 return csk
->state
> CTP_ESTABLISHED
? -EPIPE
: -ECONNRESET
;
668 case CPL_ERR_XMIT_TIMEDOUT
:
669 case CPL_ERR_PERSIST_TIMEDOUT
:
670 case CPL_ERR_FINWAIT2_TIMEDOUT
:
671 case CPL_ERR_KEEPALIVE_TIMEDOUT
:
678 static int do_abort_req(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
680 const struct cpl_abort_req_rss
*req
= cplhdr(skb
);
681 struct cxgbi_sock
*csk
= ctx
;
682 int rst_status
= CPL_ABORT_NO_RST
;
684 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
685 "csk 0x%p,%u,0x%lx,%u.\n",
686 csk
, csk
->state
, csk
->flags
, csk
->tid
);
688 if (req
->status
== CPL_ERR_RTX_NEG_ADVICE
||
689 req
->status
== CPL_ERR_PERSIST_NEG_ADVICE
) {
694 spin_lock_bh(&csk
->lock
);
696 if (!cxgbi_sock_flag(csk
, CTPF_ABORT_REQ_RCVD
)) {
697 cxgbi_sock_set_flag(csk
, CTPF_ABORT_REQ_RCVD
);
698 cxgbi_sock_set_state(csk
, CTP_ABORTING
);
702 cxgbi_sock_clear_flag(csk
, CTPF_ABORT_REQ_RCVD
);
703 send_abort_rpl(csk
, rst_status
);
705 if (!cxgbi_sock_flag(csk
, CTPF_ABORT_RPL_PENDING
)) {
706 csk
->err
= abort_status_to_errno(csk
, req
->status
, &rst_status
);
707 cxgbi_sock_closed(csk
);
711 spin_unlock_bh(&csk
->lock
);
719 * Process ABORT_RPL_RSS CPL message: -> host
720 * Process abort replies. We only process these messages if we anticipate
721 * them as the coordination between SW and HW in this area is somewhat lacking
722 * and sometimes we get ABORT_RPLs after we are done with the connection that
723 * originated the ABORT_REQ.
725 static int do_abort_rpl(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
727 struct cpl_abort_rpl_rss
*rpl
= cplhdr(skb
);
728 struct cxgbi_sock
*csk
= ctx
;
730 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
731 "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
732 rpl
->status
, csk
, csk
? csk
->state
: 0,
733 csk
? csk
->flags
: 0UL);
735 * Ignore replies to post-close aborts indicating that the abort was
736 * requested too late. These connections are terminated when we get
737 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
738 * arrives the TID is either no longer used or it has been recycled.
740 if (rpl
->status
== CPL_ERR_ABORT_FAILED
)
743 * Sometimes we've already closed the connection, e.g., a post-close
744 * abort races with ABORT_REQ_RSS, the latter frees the connection
745 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
746 * but FW turns the ABORT_REQ into a regular one and so we get
747 * ABORT_RPL_RSS with status 0 and no connection.
750 cxgbi_sock_rcv_abort_rpl(csk
);
757 * Process RX_ISCSI_HDR CPL message: -> host
758 * Handle received PDUs, the payload could be DDP'ed. If not, the payload
759 * follow after the bhs.
761 static int do_iscsi_hdr(struct t3cdev
*t3dev
, struct sk_buff
*skb
, void *ctx
)
763 struct cxgbi_sock
*csk
= ctx
;
764 struct cpl_iscsi_hdr
*hdr_cpl
= cplhdr(skb
);
765 struct cpl_iscsi_hdr_norss data_cpl
;
766 struct cpl_rx_data_ddp_norss ddp_cpl
;
767 unsigned int hdr_len
, data_len
, status
;
771 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
772 "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n",
773 csk
, csk
->state
, csk
->flags
, csk
->tid
, skb
, skb
->len
);
775 spin_lock_bh(&csk
->lock
);
777 if (unlikely(csk
->state
>= CTP_PASSIVE_CLOSE
)) {
778 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
779 "csk 0x%p,%u,0x%lx,%u, bad state.\n",
780 csk
, csk
->state
, csk
->flags
, csk
->tid
);
781 if (csk
->state
!= CTP_ABORTING
)
787 cxgbi_skcb_tcp_seq(skb
) = ntohl(hdr_cpl
->seq
);
788 cxgbi_skcb_flags(skb
) = 0;
790 skb_reset_transport_header(skb
);
791 __skb_pull(skb
, sizeof(struct cpl_iscsi_hdr
));
793 len
= hdr_len
= ntohs(hdr_cpl
->len
);
794 /* msg coalesce is off or not enough data received */
795 if (skb
->len
<= hdr_len
) {
796 pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n",
797 csk
->cdev
->ports
[csk
->port_id
]->name
, csk
->tid
,
801 cxgbi_skcb_set_flag(skb
, SKCBF_RX_COALESCED
);
803 err
= skb_copy_bits(skb
, skb
->len
- sizeof(ddp_cpl
), &ddp_cpl
,
806 pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n",
807 csk
->cdev
->ports
[csk
->port_id
]->name
, csk
->tid
,
808 skb
->len
, sizeof(ddp_cpl
), err
);
812 cxgbi_skcb_set_flag(skb
, SKCBF_RX_STATUS
);
813 cxgbi_skcb_rx_pdulen(skb
) = ntohs(ddp_cpl
.len
);
814 cxgbi_skcb_rx_ddigest(skb
) = ntohl(ddp_cpl
.ulp_crc
);
815 status
= ntohl(ddp_cpl
.ddp_status
);
817 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
818 "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n",
819 csk
, skb
, skb
->len
, cxgbi_skcb_rx_pdulen(skb
), status
);
821 if (status
& (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT
))
822 cxgbi_skcb_set_flag(skb
, SKCBF_RX_HCRC_ERR
);
823 if (status
& (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT
))
824 cxgbi_skcb_set_flag(skb
, SKCBF_RX_DCRC_ERR
);
825 if (status
& (1 << CPL_RX_DDP_STATUS_PAD_SHIFT
))
826 cxgbi_skcb_set_flag(skb
, SKCBF_RX_PAD_ERR
);
828 if (skb
->len
> (hdr_len
+ sizeof(ddp_cpl
))) {
829 err
= skb_copy_bits(skb
, hdr_len
, &data_cpl
, sizeof(data_cpl
));
831 pr_err("%s: tid %u, cp %zu/%u failed %d.\n",
832 csk
->cdev
->ports
[csk
->port_id
]->name
,
833 csk
->tid
, sizeof(data_cpl
), skb
->len
, err
);
836 data_len
= ntohs(data_cpl
.len
);
837 log_debug(1 << CXGBI_DBG_DDP
| 1 << CXGBI_DBG_PDU_RX
,
838 "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n",
839 skb
, data_len
, cxgbi_skcb_rx_pdulen(skb
), status
);
840 len
+= sizeof(data_cpl
) + data_len
;
841 } else if (status
& (1 << CPL_RX_DDP_STATUS_DDP_SHIFT
))
842 cxgbi_skcb_set_flag(skb
, SKCBF_RX_DATA_DDPD
);
844 csk
->rcv_nxt
= ntohl(ddp_cpl
.seq
) + cxgbi_skcb_rx_pdulen(skb
);
845 __pskb_trim(skb
, len
);
846 __skb_queue_tail(&csk
->receive_queue
, skb
);
847 cxgbi_conn_pdu_ready(csk
);
849 spin_unlock_bh(&csk
->lock
);
855 spin_unlock_bh(&csk
->lock
);
861 * Process TX_DATA_ACK CPL messages: -> host
862 * Process an acknowledgment of WR completion. Advance snd_una and send the
863 * next batch of work requests from the write queue.
865 static int do_wr_ack(struct t3cdev
*cdev
, struct sk_buff
*skb
, void *ctx
)
867 struct cxgbi_sock
*csk
= ctx
;
868 struct cpl_wr_ack
*hdr
= cplhdr(skb
);
870 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_PDU_RX
,
871 "csk 0x%p,%u,0x%lx,%u, cr %u.\n",
872 csk
, csk
->state
, csk
->flags
, csk
->tid
, ntohs(hdr
->credits
));
874 cxgbi_sock_rcv_wr_ack(csk
, ntohs(hdr
->credits
), ntohl(hdr
->snd_una
), 1);
880 * for each connection, pre-allocate skbs needed for close/abort requests. So
881 * that we can service the request right away.
883 static int alloc_cpls(struct cxgbi_sock
*csk
)
885 csk
->cpl_close
= alloc_wr(sizeof(struct cpl_close_con_req
), 0,
889 csk
->cpl_abort_req
= alloc_wr(sizeof(struct cpl_abort_req
), 0,
891 if (!csk
->cpl_abort_req
)
894 csk
->cpl_abort_rpl
= alloc_wr(sizeof(struct cpl_abort_rpl
), 0,
896 if (!csk
->cpl_abort_rpl
)
902 cxgbi_sock_free_cpl_skbs(csk
);
907 * release_offload_resources - release offload resource
908 * @c3cn: the offloaded iscsi tcp connection.
909 * Release resources held by an offload connection (TID, L2T entry, etc.)
911 static void l2t_put(struct cxgbi_sock
*csk
)
913 struct t3cdev
*t3dev
= (struct t3cdev
*)csk
->cdev
->lldev
;
916 l2t_release(L2DATA(t3dev
), csk
->l2t
);
922 static void release_offload_resources(struct cxgbi_sock
*csk
)
924 struct t3cdev
*t3dev
= (struct t3cdev
*)csk
->cdev
->lldev
;
926 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
927 "csk 0x%p,%u,0x%lx,%u.\n",
928 csk
, csk
->state
, csk
->flags
, csk
->tid
);
931 cxgbi_sock_free_cpl_skbs(csk
);
933 if (csk
->wr_cred
!= csk
->wr_max_cred
) {
934 cxgbi_sock_purge_wr_queue(csk
);
935 cxgbi_sock_reset_wr_list(csk
);
938 if (cxgbi_sock_flag(csk
, CTPF_HAS_ATID
))
940 else if (cxgbi_sock_flag(csk
, CTPF_HAS_TID
)) {
941 cxgb3_remove_tid(t3dev
, (void *)csk
, csk
->tid
);
942 cxgbi_sock_clear_flag(csk
, CTPF_HAS_TID
);
949 static void update_address(struct cxgbi_hba
*chba
)
951 if (chba
->ipv4addr
) {
953 chba
->ipv4addr
!= cxgb3i_get_private_ipv4addr(chba
->vdev
)) {
954 cxgb3i_set_private_ipv4addr(chba
->vdev
, chba
->ipv4addr
);
955 cxgb3i_set_private_ipv4addr(chba
->ndev
, 0);
956 pr_info("%s set %pI4.\n",
957 chba
->vdev
->name
, &chba
->ipv4addr
);
958 } else if (chba
->ipv4addr
!=
959 cxgb3i_get_private_ipv4addr(chba
->ndev
)) {
960 cxgb3i_set_private_ipv4addr(chba
->ndev
, chba
->ipv4addr
);
961 pr_info("%s set %pI4.\n",
962 chba
->ndev
->name
, &chba
->ipv4addr
);
964 } else if (cxgb3i_get_private_ipv4addr(chba
->ndev
)) {
966 cxgb3i_set_private_ipv4addr(chba
->vdev
, 0);
967 cxgb3i_set_private_ipv4addr(chba
->ndev
, 0);
971 static int init_act_open(struct cxgbi_sock
*csk
)
973 struct dst_entry
*dst
= csk
->dst
;
974 struct cxgbi_device
*cdev
= csk
->cdev
;
975 struct t3cdev
*t3dev
= (struct t3cdev
*)cdev
->lldev
;
976 struct net_device
*ndev
= cdev
->ports
[csk
->port_id
];
977 struct cxgbi_hba
*chba
= cdev
->hbas
[csk
->port_id
];
978 struct sk_buff
*skb
= NULL
;
980 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
981 "csk 0x%p,%u,0x%lx.\n", csk
, csk
->state
, csk
->flags
);
983 update_address(chba
);
985 csk
->saddr
.sin_addr
.s_addr
= chba
->ipv4addr
;
988 csk
->l2t
= t3_l2t_get(t3dev
, dst_get_neighbour(dst
), ndev
);
990 pr_err("NO l2t available.\n");
995 csk
->atid
= cxgb3_alloc_atid(t3dev
, &t3_client
, csk
);
997 pr_err("NO atid available.\n");
1000 cxgbi_sock_set_flag(csk
, CTPF_HAS_ATID
);
1001 cxgbi_sock_get(csk
);
1003 skb
= alloc_wr(sizeof(struct cpl_act_open_req
), 0, GFP_KERNEL
);
1006 skb
->sk
= (struct sock
*)csk
;
1007 set_arp_failure_handler(skb
, act_open_arp_failure
);
1009 csk
->wr_max_cred
= csk
->wr_cred
= T3C_DATA(t3dev
)->max_wrs
- 1;
1010 csk
->wr_una_cred
= 0;
1011 csk
->mss_idx
= cxgbi_sock_select_mss(csk
, dst_mtu(dst
));
1012 cxgbi_sock_reset_wr_list(csk
);
1015 log_debug(1 << CXGBI_DBG_TOE
| 1 << CXGBI_DBG_SOCK
,
1016 "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n",
1017 csk
, csk
->state
, csk
->flags
,
1018 &csk
->saddr
.sin_addr
.s_addr
, ntohs(csk
->saddr
.sin_port
),
1019 &csk
->daddr
.sin_addr
.s_addr
, ntohs(csk
->daddr
.sin_port
));
1021 cxgbi_sock_set_state(csk
, CTP_ACTIVE_OPEN
);
1022 send_act_open_req(csk
, skb
, csk
->l2t
);
1031 cxgb3_cpl_handler_func cxgb3i_cpl_handlers
[NUM_CPL_CMDS
] = {
1032 [CPL_ACT_ESTABLISH
] = do_act_establish
,
1033 [CPL_ACT_OPEN_RPL
] = do_act_open_rpl
,
1034 [CPL_PEER_CLOSE
] = do_peer_close
,
1035 [CPL_ABORT_REQ_RSS
] = do_abort_req
,
1036 [CPL_ABORT_RPL_RSS
] = do_abort_rpl
,
1037 [CPL_CLOSE_CON_RPL
] = do_close_con_rpl
,
1038 [CPL_TX_DMA_ACK
] = do_wr_ack
,
1039 [CPL_ISCSI_HDR
] = do_iscsi_hdr
,
1043 * cxgb3i_ofld_init - allocate and initialize resources for each adapter found
1044 * @cdev: cxgbi adapter
1046 int cxgb3i_ofld_init(struct cxgbi_device
*cdev
)
1048 struct t3cdev
*t3dev
= (struct t3cdev
*)cdev
->lldev
;
1049 struct adap_ports port
;
1050 struct ofld_page_info rx_page_info
;
1051 unsigned int wr_len
;
1054 if (t3dev
->ctl(t3dev
, GET_WR_LEN
, &wr_len
) < 0 ||
1055 t3dev
->ctl(t3dev
, GET_PORTS
, &port
) < 0 ||
1056 t3dev
->ctl(t3dev
, GET_RX_PAGE_INFO
, &rx_page_info
) < 0) {
1057 pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev
);
1061 if (cxgb3i_max_connect
> CXGBI_MAX_CONN
)
1062 cxgb3i_max_connect
= CXGBI_MAX_CONN
;
1064 rc
= cxgbi_device_portmap_create(cdev
, cxgb3i_sport_base
,
1065 cxgb3i_max_connect
);
1069 init_wr_tab(wr_len
);
1070 cdev
->csk_release_offload_resources
= release_offload_resources
;
1071 cdev
->csk_push_tx_frames
= push_tx_frames
;
1072 cdev
->csk_send_abort_req
= send_abort_req
;
1073 cdev
->csk_send_close_req
= send_close_req
;
1074 cdev
->csk_send_rx_credits
= send_rx_credits
;
1075 cdev
->csk_alloc_cpls
= alloc_cpls
;
1076 cdev
->csk_init_act_open
= init_act_open
;
1078 pr_info("cdev 0x%p, offload up, added.\n", cdev
);
1083 * functions to program the pagepod in h/w
1085 static inline void ulp_mem_io_set_hdr(struct sk_buff
*skb
, unsigned int addr
)
1087 struct ulp_mem_io
*req
= (struct ulp_mem_io
*)skb
->head
;
1089 memset(req
, 0, sizeof(*req
));
1091 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_BYPASS
));
1092 req
->cmd_lock_addr
= htonl(V_ULP_MEMIO_ADDR(addr
>> 5) |
1093 V_ULPTX_CMD(ULP_MEM_WRITE
));
1094 req
->len
= htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE
>> 5) |
1095 V_ULPTX_NFLITS((PPOD_SIZE
>> 3) + 1));
1098 static int ddp_set_map(struct cxgbi_sock
*csk
, struct cxgbi_pagepod_hdr
*hdr
,
1099 unsigned int idx
, unsigned int npods
,
1100 struct cxgbi_gather_list
*gl
)
1102 struct cxgbi_device
*cdev
= csk
->cdev
;
1103 struct cxgbi_ddp_info
*ddp
= cdev
->ddp
;
1104 unsigned int pm_addr
= (idx
<< PPOD_SIZE_SHIFT
) + ddp
->llimit
;
1107 log_debug(1 << CXGBI_DBG_DDP
,
1108 "csk 0x%p, idx %u, npods %u, gl 0x%p.\n",
1109 csk
, idx
, npods
, gl
);
1111 for (i
= 0; i
< npods
; i
++, idx
++, pm_addr
+= PPOD_SIZE
) {
1112 struct sk_buff
*skb
= alloc_wr(sizeof(struct ulp_mem_io
) +
1113 PPOD_SIZE
, 0, GFP_ATOMIC
);
1118 ulp_mem_io_set_hdr(skb
, pm_addr
);
1119 cxgbi_ddp_ppod_set((struct cxgbi_pagepod
*)(skb
->head
+
1120 sizeof(struct ulp_mem_io
)),
1121 hdr
, gl
, i
* PPOD_PAGES_MAX
);
1122 skb
->priority
= CPL_PRIORITY_CONTROL
;
1123 cxgb3_ofld_send(cdev
->lldev
, skb
);
1128 static void ddp_clear_map(struct cxgbi_hba
*chba
, unsigned int tag
,
1129 unsigned int idx
, unsigned int npods
)
1131 struct cxgbi_device
*cdev
= chba
->cdev
;
1132 struct cxgbi_ddp_info
*ddp
= cdev
->ddp
;
1133 unsigned int pm_addr
= (idx
<< PPOD_SIZE_SHIFT
) + ddp
->llimit
;
1136 log_debug(1 << CXGBI_DBG_DDP
,
1137 "cdev 0x%p, idx %u, npods %u, tag 0x%x.\n",
1138 cdev
, idx
, npods
, tag
);
1140 for (i
= 0; i
< npods
; i
++, idx
++, pm_addr
+= PPOD_SIZE
) {
1141 struct sk_buff
*skb
= alloc_wr(sizeof(struct ulp_mem_io
) +
1142 PPOD_SIZE
, 0, GFP_ATOMIC
);
1145 pr_err("tag 0x%x, 0x%x, %d/%u, skb OOM.\n",
1146 tag
, idx
, i
, npods
);
1149 ulp_mem_io_set_hdr(skb
, pm_addr
);
1150 skb
->priority
= CPL_PRIORITY_CONTROL
;
1151 cxgb3_ofld_send(cdev
->lldev
, skb
);
1155 static int ddp_setup_conn_pgidx(struct cxgbi_sock
*csk
,
1156 unsigned int tid
, int pg_idx
, bool reply
)
1158 struct sk_buff
*skb
= alloc_wr(sizeof(struct cpl_set_tcb_field
), 0,
1160 struct cpl_set_tcb_field
*req
;
1161 u64 val
= pg_idx
< DDP_PGIDX_MAX
? pg_idx
: 0;
1163 log_debug(1 << CXGBI_DBG_DDP
,
1164 "csk 0x%p, tid %u, pg_idx %d.\n", csk
, tid
, pg_idx
);
1168 /* set up ulp submode and page size */
1169 req
= (struct cpl_set_tcb_field
*)skb
->head
;
1170 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
1171 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, tid
));
1172 req
->reply
= V_NO_REPLY(reply
? 0 : 1);
1174 req
->word
= htons(31);
1175 req
->mask
= cpu_to_be64(0xF0000000);
1176 req
->val
= cpu_to_be64(val
<< 28);
1177 skb
->priority
= CPL_PRIORITY_CONTROL
;
1179 cxgb3_ofld_send(csk
->cdev
->lldev
, skb
);
1184 * cxgb3i_setup_conn_digest - setup conn. digest setting
1185 * @csk: cxgb tcp socket
1186 * @tid: connection id
1187 * @hcrc: header digest enabled
1188 * @dcrc: data digest enabled
1189 * @reply: request reply from h/w
1190 * set up the iscsi digest settings for a connection identified by tid
1192 static int ddp_setup_conn_digest(struct cxgbi_sock
*csk
, unsigned int tid
,
1193 int hcrc
, int dcrc
, int reply
)
1195 struct sk_buff
*skb
= alloc_wr(sizeof(struct cpl_set_tcb_field
), 0,
1197 struct cpl_set_tcb_field
*req
;
1198 u64 val
= (hcrc
? 1 : 0) | (dcrc
? 2 : 0);
1200 log_debug(1 << CXGBI_DBG_DDP
,
1201 "csk 0x%p, tid %u, crc %d,%d.\n", csk
, tid
, hcrc
, dcrc
);
1205 /* set up ulp submode and page size */
1206 req
= (struct cpl_set_tcb_field
*)skb
->head
;
1207 req
->wr
.wr_hi
= htonl(V_WR_OP(FW_WROPCODE_FORWARD
));
1208 OPCODE_TID(req
) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD
, tid
));
1209 req
->reply
= V_NO_REPLY(reply
? 0 : 1);
1211 req
->word
= htons(31);
1212 req
->mask
= cpu_to_be64(0x0F000000);
1213 req
->val
= cpu_to_be64(val
<< 24);
1214 skb
->priority
= CPL_PRIORITY_CONTROL
;
1216 cxgb3_ofld_send(csk
->cdev
->lldev
, skb
);
1221 * t3_ddp_cleanup - release the cxgb3 adapter's ddp resource
1222 * @cdev: cxgb3i adapter
1223 * release all the resource held by the ddp pagepod manager for a given
1227 static void t3_ddp_cleanup(struct cxgbi_device
*cdev
)
1229 struct t3cdev
*tdev
= (struct t3cdev
*)cdev
->lldev
;
1231 if (cxgbi_ddp_cleanup(cdev
)) {
1232 pr_info("t3dev 0x%p, ulp_iscsi no more user.\n", tdev
);
1233 tdev
->ulp_iscsi
= NULL
;
1238 * ddp_init - initialize the cxgb3 adapter's ddp resource
1239 * @cdev: cxgb3i adapter
1240 * initialize the ddp pagepod manager for a given adapter
1242 static int cxgb3i_ddp_init(struct cxgbi_device
*cdev
)
1244 struct t3cdev
*tdev
= (struct t3cdev
*)cdev
->lldev
;
1245 struct cxgbi_ddp_info
*ddp
= tdev
->ulp_iscsi
;
1246 struct ulp_iscsi_info uinfo
;
1247 unsigned int pgsz_factor
[4];
1251 kref_get(&ddp
->refcnt
);
1252 pr_warn("t3dev 0x%p, ddp 0x%p already set up.\n",
1253 tdev
, tdev
->ulp_iscsi
);
1258 err
= tdev
->ctl(tdev
, ULP_ISCSI_GET_PARAMS
, &uinfo
);
1260 pr_err("%s, failed to get iscsi param err=%d.\n",
1265 err
= cxgbi_ddp_init(cdev
, uinfo
.llimit
, uinfo
.ulimit
,
1266 uinfo
.max_txsz
, uinfo
.max_rxsz
);
1272 uinfo
.tagmask
= ddp
->idx_mask
<< PPOD_IDX_SHIFT
;
1273 cxgbi_ddp_page_size_factor(pgsz_factor
);
1274 for (i
= 0; i
< 4; i
++)
1275 uinfo
.pgsz_factor
[i
] = pgsz_factor
[i
];
1276 uinfo
.ulimit
= uinfo
.llimit
+ (ddp
->nppods
<< PPOD_SIZE_SHIFT
);
1278 err
= tdev
->ctl(tdev
, ULP_ISCSI_SET_PARAMS
, &uinfo
);
1280 pr_warn("%s unable to set iscsi param err=%d, ddp disabled.\n",
1282 cxgbi_ddp_cleanup(cdev
);
1285 tdev
->ulp_iscsi
= ddp
;
1287 cdev
->csk_ddp_setup_digest
= ddp_setup_conn_digest
;
1288 cdev
->csk_ddp_setup_pgidx
= ddp_setup_conn_pgidx
;
1289 cdev
->csk_ddp_set
= ddp_set_map
;
1290 cdev
->csk_ddp_clear
= ddp_clear_map
;
1292 pr_info("tdev 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, "
1294 tdev
, ddp
->nppods
, ddp
->idx_bits
, ddp
->idx_mask
,
1295 ddp
->rsvd_tag_mask
, ddp
->max_txsz
, uinfo
.max_txsz
,
1296 ddp
->max_rxsz
, uinfo
.max_rxsz
);
1300 static void cxgb3i_dev_close(struct t3cdev
*t3dev
)
1302 struct cxgbi_device
*cdev
= cxgbi_device_find_by_lldev(t3dev
);
1304 if (!cdev
|| cdev
->flags
& CXGBI_FLAG_ADAPTER_RESET
) {
1305 pr_info("0x%p close, f 0x%x.\n", cdev
, cdev
? cdev
->flags
: 0);
1309 cxgbi_device_unregister(cdev
);
1313 * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings
1314 * @t3dev: t3cdev adapter
1316 static void cxgb3i_dev_open(struct t3cdev
*t3dev
)
1318 struct cxgbi_device
*cdev
= cxgbi_device_find_by_lldev(t3dev
);
1319 struct adapter
*adapter
= tdev2adap(t3dev
);
1323 pr_info("0x%p, updating.\n", cdev
);
1327 cdev
= cxgbi_device_register(0, adapter
->params
.nports
);
1329 pr_warn("device 0x%p register failed.\n", t3dev
);
1333 cdev
->flags
= CXGBI_FLAG_DEV_T3
| CXGBI_FLAG_IPV4_SET
;
1334 cdev
->lldev
= t3dev
;
1335 cdev
->pdev
= adapter
->pdev
;
1336 cdev
->ports
= adapter
->port
;
1337 cdev
->nports
= adapter
->params
.nports
;
1338 cdev
->mtus
= adapter
->params
.mtus
;
1339 cdev
->nmtus
= NMTUS
;
1340 cdev
->snd_win
= cxgb3i_snd_win
;
1341 cdev
->rcv_win
= cxgb3i_rcv_win
;
1342 cdev
->rx_credit_thres
= cxgb3i_rx_credit_thres
;
1343 cdev
->skb_tx_rsvd
= CXGB3I_TX_HEADER_LEN
;
1344 cdev
->skb_rx_extra
= sizeof(struct cpl_iscsi_hdr_norss
);
1345 cdev
->dev_ddp_cleanup
= t3_ddp_cleanup
;
1346 cdev
->itp
= &cxgb3i_iscsi_transport
;
1348 err
= cxgb3i_ddp_init(cdev
);
1350 pr_info("0x%p ddp init failed\n", cdev
);
1354 err
= cxgb3i_ofld_init(cdev
);
1356 pr_info("0x%p offload init failed\n", cdev
);
1360 err
= cxgbi_hbas_add(cdev
, CXGB3I_MAX_LUN
, CXGBI_MAX_CONN
,
1361 &cxgb3i_host_template
, cxgb3i_stt
);
1365 for (i
= 0; i
< cdev
->nports
; i
++)
1366 cdev
->hbas
[i
]->ipv4addr
=
1367 cxgb3i_get_private_ipv4addr(cdev
->ports
[i
]);
1369 pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n",
1370 cdev
, cdev
? cdev
->flags
: 0, t3dev
, err
);
1374 cxgbi_device_unregister(cdev
);
1377 static void cxgb3i_dev_event_handler(struct t3cdev
*t3dev
, u32 event
, u32 port
)
1379 struct cxgbi_device
*cdev
= cxgbi_device_find_by_lldev(t3dev
);
1381 log_debug(1 << CXGBI_DBG_TOE
,
1382 "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n",
1383 t3dev
, cdev
, event
, port
);
1388 case OFFLOAD_STATUS_DOWN
:
1389 cdev
->flags
|= CXGBI_FLAG_ADAPTER_RESET
;
1391 case OFFLOAD_STATUS_UP
:
1392 cdev
->flags
&= ~CXGBI_FLAG_ADAPTER_RESET
;
1398 * cxgb3i_init_module - module init entry point
1400 * initialize any driver wide global data structures and register itself
1401 * with the cxgb3 module
1403 static int __init
cxgb3i_init_module(void)
1407 printk(KERN_INFO
"%s", version
);
1409 rc
= cxgbi_iscsi_init(&cxgb3i_iscsi_transport
, &cxgb3i_stt
);
1413 cxgb3_register_client(&t3_client
);
1418 * cxgb3i_exit_module - module cleanup/exit entry point
1420 * go through the driver hba list and for each hba, release any resource held.
1421 * and unregisters iscsi transport and the cxgb3 module
1423 static void __exit
cxgb3i_exit_module(void)
1425 cxgb3_unregister_client(&t3_client
);
1426 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3
);
1427 cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport
, &cxgb3i_stt
);
1430 module_init(cxgb3i_init_module
);
1431 module_exit(cxgb3i_exit_module
);