?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add linux-next specific files for 20110831
[linux-2.6/next.git] / drivers / scsi / cxgbi / cxgb4i / cxgb4i.c
blob628a6983a20b1a0ab8be9aca5ebff6777a1fbaca
1 /*
2 * cxgb4i.c: Chelsio T4 iSCSI driver.
3 *
4 * Copyright (c) 2010 Chelsio Communications, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation.
9 *
10 * Written by: Karen Xie (kxie@chelsio.com)
11 * Rakesh Ranjan (rranjan@chelsio.com)
12 */
13
14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
15
16 #include <linux/version.h>
17 #include <linux/module.h>
18 #include <linux/moduleparam.h>
19 #include <scsi/scsi_host.h>
20 #include <net/tcp.h>
21 #include <net/dst.h>
22 #include <linux/netdevice.h>
23
24 #include "t4_msg.h"
25 #include "cxgb4.h"
26 #include "cxgb4_uld.h"
27 #include "t4fw_api.h"
28 #include "l2t.h"
29 #include "cxgb4i.h"
30
31 static unsigned int dbg_level;
32
33 #include "../libcxgbi.h"
34
35 #define DRV_MODULE_NAME "cxgb4i"
36 #define DRV_MODULE_DESC "Chelsio T4 iSCSI Driver"
37 #define DRV_MODULE_VERSION "0.9.1"
38 #define DRV_MODULE_RELDATE "Aug. 2010"
39
40 static char version[] =
41 DRV_MODULE_DESC " " DRV_MODULE_NAME
42 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
43
44 MODULE_AUTHOR("Chelsio Communications, Inc.");
45 MODULE_DESCRIPTION(DRV_MODULE_DESC);
46 MODULE_VERSION(DRV_MODULE_VERSION);
47 MODULE_LICENSE("GPL");
48
49 module_param(dbg_level, uint, 0644);
50 MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)");
51
52 static int cxgb4i_rcv_win = 256 * 1024;
53 module_param(cxgb4i_rcv_win, int, 0644);
54 MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP reveive window in bytes");
55
56 static int cxgb4i_snd_win = 128 * 1024;
57 module_param(cxgb4i_snd_win, int, 0644);
58 MODULE_PARM_DESC(cxgb4i_snd_win, "TCP send window in bytes");
59
60 static int cxgb4i_rx_credit_thres = 10 * 1024;
61 module_param(cxgb4i_rx_credit_thres, int, 0644);
62 MODULE_PARM_DESC(cxgb4i_rx_credit_thres,
63 "RX credits return threshold in bytes (default=10KB)");
64
65 static unsigned int cxgb4i_max_connect = (8 * 1024);
66 module_param(cxgb4i_max_connect, uint, 0644);
67 MODULE_PARM_DESC(cxgb4i_max_connect, "Maximum number of connections");
68
69 static unsigned short cxgb4i_sport_base = 20000;
70 module_param(cxgb4i_sport_base, ushort, 0644);
71 MODULE_PARM_DESC(cxgb4i_sport_base, "Starting port number (default 20000)");
72
73 typedef void (*cxgb4i_cplhandler_func)(struct cxgbi_device *, struct sk_buff *);
74
75 static void *t4_uld_add(const struct cxgb4_lld_info *);
76 static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *);
77 static int t4_uld_state_change(void *, enum cxgb4_state state);
78
79 static const struct cxgb4_uld_info cxgb4i_uld_info = {
80 .name = DRV_MODULE_NAME,
81 .add = t4_uld_add,
82 .rx_handler = t4_uld_rx_handler,
83 .state_change = t4_uld_state_change,
84 };
85
86 static struct scsi_host_template cxgb4i_host_template = {
87 .module = THIS_MODULE,
88 .name = DRV_MODULE_NAME,
89 .proc_name = DRV_MODULE_NAME,
90 .can_queue = CXGB4I_SCSI_HOST_QDEPTH,
91 .queuecommand = iscsi_queuecommand,
92 .change_queue_depth = iscsi_change_queue_depth,
93 .sg_tablesize = SG_ALL,
94 .max_sectors = 0xFFFF,
95 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN,
96 .eh_abort_handler = iscsi_eh_abort,
97 .eh_device_reset_handler = iscsi_eh_device_reset,
98 .eh_target_reset_handler = iscsi_eh_recover_target,
99 .target_alloc = iscsi_target_alloc,
100 .use_clustering = DISABLE_CLUSTERING,
101 .this_id = -1,
102 };
103
104 static struct iscsi_transport cxgb4i_iscsi_transport = {
105 .owner = THIS_MODULE,
106 .name = DRV_MODULE_NAME,
107 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST |
108 CAP_DATADGST | CAP_DIGEST_OFFLOAD |
109 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO,
110 .attr_is_visible = cxgbi_attr_is_visible,
111 .get_host_param = cxgbi_get_host_param,
112 .set_host_param = cxgbi_set_host_param,
113 /* session management */
114 .create_session = cxgbi_create_session,
115 .destroy_session = cxgbi_destroy_session,
116 .get_session_param = iscsi_session_get_param,
117 /* connection management */
118 .create_conn = cxgbi_create_conn,
119 .bind_conn = cxgbi_bind_conn,
120 .destroy_conn = iscsi_tcp_conn_teardown,
121 .start_conn = iscsi_conn_start,
122 .stop_conn = iscsi_conn_stop,
123 .get_conn_param = iscsi_conn_get_param,
124 .set_param = cxgbi_set_conn_param,
125 .get_stats = cxgbi_get_conn_stats,
126 /* pdu xmit req from user space */
127 .send_pdu = iscsi_conn_send_pdu,
128 /* task */
129 .init_task = iscsi_tcp_task_init,
130 .xmit_task = iscsi_tcp_task_xmit,
131 .cleanup_task = cxgbi_cleanup_task,
132 /* pdu */
133 .alloc_pdu = cxgbi_conn_alloc_pdu,
134 .init_pdu = cxgbi_conn_init_pdu,
135 .xmit_pdu = cxgbi_conn_xmit_pdu,
136 .parse_pdu_itt = cxgbi_parse_pdu_itt,
137 /* TCP connect/disconnect */
138 .get_ep_param = cxgbi_get_ep_param,
139 .ep_connect = cxgbi_ep_connect,
140 .ep_poll = cxgbi_ep_poll,
141 .ep_disconnect = cxgbi_ep_disconnect,
142 /* Error recovery timeout call */
143 .session_recovery_timedout = iscsi_session_recovery_timedout,
144 };
145
146 static struct scsi_transport_template *cxgb4i_stt;
147
148 /*
149 * CPL (Chelsio Protocol Language) defines a message passing interface between
150 * the host driver and Chelsio asic.
151 * The section below implments CPLs that related to iscsi tcp connection
152 * open/close/abort and data send/receive.
153 */
154 #define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
155 #define RCV_BUFSIZ_MASK 0x3FFU
156 #define MAX_IMM_TX_PKT_LEN 128
157
158 static inline void set_queue(struct sk_buff *skb, unsigned int queue,
159 const struct cxgbi_sock *csk)
160 {
161 skb->queue_mapping = queue;
162 }
163
164 static int push_tx_frames(struct cxgbi_sock *, int);
165
166 /*
167 * is_ofld_imm - check whether a packet can be sent as immediate data
168 * @skb: the packet
169 *
170 * Returns true if a packet can be sent as an offload WR with immediate
171 * data. We currently use the same limit as for Ethernet packets.
172 */
173 static inline int is_ofld_imm(const struct sk_buff *skb)
174 {
175 return skb->len <= (MAX_IMM_TX_PKT_LEN -
176 sizeof(struct fw_ofld_tx_data_wr));
177 }
178
179 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb,
180 struct l2t_entry *e)
181 {
182 struct cpl_act_open_req *req;
183 int wscale = cxgbi_sock_compute_wscale(csk->mss_idx);
184 unsigned long long opt0;
185 unsigned int opt2;
186 unsigned int qid_atid = ((unsigned int)csk->atid) |
187 (((unsigned int)csk->rss_qid) << 14);
188
189 opt0 = KEEP_ALIVE(1) |
190 WND_SCALE(wscale) |
191 MSS_IDX(csk->mss_idx) |
192 L2T_IDX(((struct l2t_entry *)csk->l2t)->idx) |
193 TX_CHAN(csk->tx_chan) |
194 SMAC_SEL(csk->smac_idx) |
195 ULP_MODE(ULP_MODE_ISCSI) |
196 RCV_BUFSIZ(cxgb4i_rcv_win >> 10);
197 opt2 = RX_CHANNEL(0) |
198 RSS_QUEUE_VALID |
199 (1 << 20) | (1 << 22) |
200 RSS_QUEUE(csk->rss_qid);
201
202 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
203 req = (struct cpl_act_open_req *)skb->head;
204
205 INIT_TP_WR(req, 0);
206 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ,
207 qid_atid));
208 req->local_port = csk->saddr.sin_port;
209 req->peer_port = csk->daddr.sin_port;
210 req->local_ip = csk->saddr.sin_addr.s_addr;
211 req->peer_ip = csk->daddr.sin_addr.s_addr;
212 req->opt0 = cpu_to_be64(opt0);
213 req->params = 0;
214 req->opt2 = cpu_to_be32(opt2);
215
216 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
217 "csk 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n",
218 csk, &req->local_ip, ntohs(req->local_port),
219 &req->peer_ip, ntohs(req->peer_port),
220 csk->atid, csk->rss_qid);
221
222 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
223 }
224
225 static void send_close_req(struct cxgbi_sock *csk)
226 {
227 struct sk_buff *skb = csk->cpl_close;
228 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
229 unsigned int tid = csk->tid;
230
231 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
232 "csk 0x%p,%u,0x%lx, tid %u.\n",
233 csk, csk->state, csk->flags, csk->tid);
234 csk->cpl_close = NULL;
235 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id);
236 INIT_TP_WR(req, tid);
237 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
238 req->rsvd = 0;
239
240 cxgbi_sock_skb_entail(csk, skb);
241 if (csk->state >= CTP_ESTABLISHED)
242 push_tx_frames(csk, 1);
243 }
244
245 static void abort_arp_failure(void *handle, struct sk_buff *skb)
246 {
247 struct cxgbi_sock *csk = (struct cxgbi_sock *)handle;
248 struct cpl_abort_req *req;
249
250 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
251 "csk 0x%p,%u,0x%lx, tid %u, abort.\n",
252 csk, csk->state, csk->flags, csk->tid);
253 req = (struct cpl_abort_req *)skb->data;
254 req->cmd = CPL_ABORT_NO_RST;
255 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
256 }
257
258 static void send_abort_req(struct cxgbi_sock *csk)
259 {
260 struct cpl_abort_req *req;
261 struct sk_buff *skb = csk->cpl_abort_req;
262
263 if (unlikely(csk->state == CTP_ABORTING) || !skb || !csk->cdev)
264 return;
265 cxgbi_sock_set_state(csk, CTP_ABORTING);
266 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING);
267 cxgbi_sock_purge_write_queue(csk);
268
269 csk->cpl_abort_req = NULL;
270 req = (struct cpl_abort_req *)skb->head;
271 set_queue(skb, CPL_PRIORITY_DATA, csk);
272 req->cmd = CPL_ABORT_SEND_RST;
273 t4_set_arp_err_handler(skb, csk, abort_arp_failure);
274 INIT_TP_WR(req, csk->tid);
275 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid));
276 req->rsvd0 = htonl(csk->snd_nxt);
277 req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT);
278
279 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
280 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n",
281 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt,
282 req->rsvd1);
283
284 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
285 }
286
287 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status)
288 {
289 struct sk_buff *skb = csk->cpl_abort_rpl;
290 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
291
292 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
293 "csk 0x%p,%u,0x%lx,%u, status %d.\n",
294 csk, csk->state, csk->flags, csk->tid, rst_status);
295
296 csk->cpl_abort_rpl = NULL;
297 set_queue(skb, CPL_PRIORITY_DATA, csk);
298 INIT_TP_WR(rpl, csk->tid);
299 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid));
300 rpl->cmd = rst_status;
301 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
302 }
303
304 /*
305 * CPL connection rx data ack: host ->
306 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
307 * credits sent.
308 */
309 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits)
310 {
311 struct sk_buff *skb;
312 struct cpl_rx_data_ack *req;
313
314 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
315 "csk 0x%p,%u,0x%lx,%u, credit %u.\n",
316 csk, csk->state, csk->flags, csk->tid, credits);
317
318 skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC);
319 if (!skb) {
320 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits);
321 return 0;
322 }
323 req = (struct cpl_rx_data_ack *)skb->head;
324
325 set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id);
326 INIT_TP_WR(req, csk->tid);
327 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK,
328 csk->tid));
329 req->credit_dack = cpu_to_be32(RX_CREDITS(credits) | RX_FORCE_ACK(1));
330 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
331 return credits;
332 }
333
334 /*
335 * sgl_len - calculates the size of an SGL of the given capacity
336 * @n: the number of SGL entries
337 * Calculates the number of flits needed for a scatter/gather list that
338 * can hold the given number of entries.
339 */
340 static inline unsigned int sgl_len(unsigned int n)
341 {
342 n--;
343 return (3 * n) / 2 + (n & 1) + 2;
344 }
345
346 /*
347 * calc_tx_flits_ofld - calculate # of flits for an offload packet
348 * @skb: the packet
349 *
350 * Returns the number of flits needed for the given offload packet.
351 * These packets are already fully constructed and no additional headers
352 * will be added.
353 */
354 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
355 {
356 unsigned int flits, cnt;
357
358 if (is_ofld_imm(skb))
359 return DIV_ROUND_UP(skb->len, 8);
360 flits = skb_transport_offset(skb) / 8;
361 cnt = skb_shinfo(skb)->nr_frags;
362 if (skb->tail != skb->transport_header)
363 cnt++;
364 return flits + sgl_len(cnt);
365 }
366
367 static inline void send_tx_flowc_wr(struct cxgbi_sock *csk)
368 {
369 struct sk_buff *skb;
370 struct fw_flowc_wr *flowc;
371 int flowclen, i;
372
373 flowclen = 80;
374 skb = alloc_wr(flowclen, 0, GFP_ATOMIC);
375 flowc = (struct fw_flowc_wr *)skb->head;
376 flowc->op_to_nparams =
377 htonl(FW_WR_OP(FW_FLOWC_WR) | FW_FLOWC_WR_NPARAMS(8));
378 flowc->flowid_len16 =
379 htonl(FW_WR_LEN16(DIV_ROUND_UP(72, 16)) |
380 FW_WR_FLOWID(csk->tid));
381 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
382 flowc->mnemval[0].val = htonl(csk->cdev->pfvf);
383 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
384 flowc->mnemval[1].val = htonl(csk->tx_chan);
385 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
386 flowc->mnemval[2].val = htonl(csk->tx_chan);
387 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
388 flowc->mnemval[3].val = htonl(csk->rss_qid);
389 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
390 flowc->mnemval[4].val = htonl(csk->snd_nxt);
391 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
392 flowc->mnemval[5].val = htonl(csk->rcv_nxt);
393 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
394 flowc->mnemval[6].val = htonl(cxgb4i_snd_win);
395 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
396 flowc->mnemval[7].val = htonl(csk->advmss);
397 flowc->mnemval[8].mnemonic = 0;
398 flowc->mnemval[8].val = 0;
399 for (i = 0; i < 9; i++) {
400 flowc->mnemval[i].r4[0] = 0;
401 flowc->mnemval[i].r4[1] = 0;
402 flowc->mnemval[i].r4[2] = 0;
403 }
404 set_queue(skb, CPL_PRIORITY_DATA, csk);
405
406 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
407 "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n",
408 csk, csk->tid, 0, csk->tx_chan, csk->rss_qid,
409 csk->snd_nxt, csk->rcv_nxt, cxgb4i_snd_win,
410 csk->advmss);
411
412 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
413 }
414
415 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb,
416 int dlen, int len, u32 credits, int compl)
417 {
418 struct fw_ofld_tx_data_wr *req;
419 unsigned int submode = cxgbi_skcb_ulp_mode(skb) & 3;
420 unsigned int wr_ulp_mode = 0;
421
422 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req));
423
424 if (is_ofld_imm(skb)) {
425 req->op_to_immdlen = htonl(FW_WR_OP(FW_OFLD_TX_DATA_WR) |
426 FW_WR_COMPL(1) |
427 FW_WR_IMMDLEN(dlen));
428 req->flowid_len16 = htonl(FW_WR_FLOWID(csk->tid) |
429 FW_WR_LEN16(credits));
430 } else {
431 req->op_to_immdlen =
432 cpu_to_be32(FW_WR_OP(FW_OFLD_TX_DATA_WR) |
433 FW_WR_COMPL(1) |
434 FW_WR_IMMDLEN(0));
435 req->flowid_len16 =
436 cpu_to_be32(FW_WR_FLOWID(csk->tid) |
437 FW_WR_LEN16(credits));
438 }
439 if (submode)
440 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE(ULP2_MODE_ISCSI) |
441 FW_OFLD_TX_DATA_WR_ULPSUBMODE(submode);
442 req->tunnel_to_proxy = htonl(wr_ulp_mode) |
443 FW_OFLD_TX_DATA_WR_SHOVE(skb_peek(&csk->write_queue) ? 0 : 1);
444 req->plen = htonl(len);
445 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT))
446 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT);
447 }
448
449 static void arp_failure_skb_discard(void *handle, struct sk_buff *skb)
450 {
451 kfree_skb(skb);
452 }
453
454 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion)
455 {
456 int total_size = 0;
457 struct sk_buff *skb;
458
459 if (unlikely(csk->state < CTP_ESTABLISHED ||
460 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) {
461 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK |
462 1 << CXGBI_DBG_PDU_TX,
463 "csk 0x%p,%u,0x%lx,%u, in closing state.\n",
464 csk, csk->state, csk->flags, csk->tid);
465 return 0;
466 }
467
468 while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) {
469 int dlen = skb->len;
470 int len = skb->len;
471 unsigned int credits_needed;
472
473 skb_reset_transport_header(skb);
474 if (is_ofld_imm(skb))
475 credits_needed = DIV_ROUND_UP(dlen +
476 sizeof(struct fw_ofld_tx_data_wr), 16);
477 else
478 credits_needed = DIV_ROUND_UP(8*calc_tx_flits_ofld(skb)
479 + sizeof(struct fw_ofld_tx_data_wr),
480 16);
481
482 if (csk->wr_cred < credits_needed) {
483 log_debug(1 << CXGBI_DBG_PDU_TX,
484 "csk 0x%p, skb %u/%u, wr %d < %u.\n",
485 csk, skb->len, skb->data_len,
486 credits_needed, csk->wr_cred);
487 break;
488 }
489 __skb_unlink(skb, &csk->write_queue);
490 set_queue(skb, CPL_PRIORITY_DATA, csk);
491 skb->csum = credits_needed;
492 csk->wr_cred -= credits_needed;
493 csk->wr_una_cred += credits_needed;
494 cxgbi_sock_enqueue_wr(csk, skb);
495
496 log_debug(1 << CXGBI_DBG_PDU_TX,
497 "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
498 csk, skb->len, skb->data_len, credits_needed,
499 csk->wr_cred, csk->wr_una_cred);
500
501 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) {
502 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) {
503 send_tx_flowc_wr(csk);
504 skb->csum += 5;
505 csk->wr_cred -= 5;
506 csk->wr_una_cred += 5;
507 }
508 len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb));
509 make_tx_data_wr(csk, skb, dlen, len, credits_needed,
510 req_completion);
511 csk->snd_nxt += len;
512 cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR);
513 }
514 total_size += skb->truesize;
515 t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard);
516
517 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX,
518 "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n",
519 csk, csk->state, csk->flags, csk->tid, skb, len);
520
521 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t);
522 }
523 return total_size;
524 }
525
526 static inline void free_atid(struct cxgbi_sock *csk)
527 {
528 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev);
529
530 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) {
531 cxgb4_free_atid(lldi->tids, csk->atid);
532 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID);
533 cxgbi_sock_put(csk);
534 }
535 }
536
537 static void do_act_establish(struct cxgbi_device *cdev, struct sk_buff *skb)
538 {
539 struct cxgbi_sock *csk;
540 struct cpl_act_establish *req = (struct cpl_act_establish *)skb->data;
541 unsigned short tcp_opt = ntohs(req->tcp_opt);
542 unsigned int tid = GET_TID(req);
543 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid));
544 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
545 struct tid_info *t = lldi->tids;
546 u32 rcv_isn = be32_to_cpu(req->rcv_isn);
547
548 csk = lookup_atid(t, atid);
549 if (unlikely(!csk)) {
550 pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid, cdev);
551 goto rel_skb;
552 }
553
554 if (csk->atid != atid) {
555 pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n",
556 atid, csk, csk->state, csk->flags, csk->tid, csk->atid);
557 goto rel_skb;
558 }
559
560 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
561 "csk 0x%p,%u,0x%lx, tid %u, atid %u, rseq %u.\n",
562 csk, csk->state, csk->flags, tid, atid, rcv_isn);
563
564 cxgbi_sock_get(csk);
565 csk->tid = tid;
566 cxgb4_insert_tid(lldi->tids, csk, tid);
567 cxgbi_sock_set_flag(csk, CTPF_HAS_TID);
568
569 free_atid(csk);
570
571 spin_lock_bh(&csk->lock);
572 if (unlikely(csk->state != CTP_ACTIVE_OPEN))
573 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n",
574 csk, csk->state, csk->flags, csk->tid);
575
576 if (csk->retry_timer.function) {
577 del_timer(&csk->retry_timer);
578 csk->retry_timer.function = NULL;
579 }
580
581 csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn;
582 /*
583 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
584 * pass through opt0.
585 */
586 if (cxgb4i_rcv_win > (RCV_BUFSIZ_MASK << 10))
587 csk->rcv_wup -= cxgb4i_rcv_win - (RCV_BUFSIZ_MASK << 10);
588
589 csk->advmss = lldi->mtus[GET_TCPOPT_MSS(tcp_opt)] - 40;
590 if (GET_TCPOPT_TSTAMP(tcp_opt))
591 csk->advmss -= 12;
592 if (csk->advmss < 128)
593 csk->advmss = 128;
594
595 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
596 "csk 0x%p, mss_idx %u, advmss %u.\n",
597 csk, GET_TCPOPT_MSS(tcp_opt), csk->advmss);
598
599 cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
600
601 if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED)))
602 send_abort_req(csk);
603 else {
604 if (skb_queue_len(&csk->write_queue))
605 push_tx_frames(csk, 0);
606 cxgbi_conn_tx_open(csk);
607 }
608 spin_unlock_bh(&csk->lock);
609
610 rel_skb:
611 __kfree_skb(skb);
612 }
613
614 static int act_open_rpl_status_to_errno(int status)
615 {
616 switch (status) {
617 case CPL_ERR_CONN_RESET:
618 return -ECONNREFUSED;
619 case CPL_ERR_ARP_MISS:
620 return -EHOSTUNREACH;
621 case CPL_ERR_CONN_TIMEDOUT:
622 return -ETIMEDOUT;
623 case CPL_ERR_TCAM_FULL:
624 return -ENOMEM;
625 case CPL_ERR_CONN_EXIST:
626 return -EADDRINUSE;
627 default:
628 return -EIO;
629 }
630 }
631
632 static void csk_act_open_retry_timer(unsigned long data)
633 {
634 struct sk_buff *skb;
635 struct cxgbi_sock *csk = (struct cxgbi_sock *)data;
636
637 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
638 "csk 0x%p,%u,0x%lx,%u.\n",
639 csk, csk->state, csk->flags, csk->tid);
640
641 cxgbi_sock_get(csk);
642 spin_lock_bh(&csk->lock);
643 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC);
644 if (!skb)
645 cxgbi_sock_fail_act_open(csk, -ENOMEM);
646 else {
647 skb->sk = (struct sock *)csk;
648 t4_set_arp_err_handler(skb, csk,
649 cxgbi_sock_act_open_req_arp_failure);
650 send_act_open_req(csk, skb, csk->l2t);
651 }
652 spin_unlock_bh(&csk->lock);
653 cxgbi_sock_put(csk);
654 }
655
656 static void do_act_open_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
657 {
658 struct cxgbi_sock *csk;
659 struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)skb->data;
660 unsigned int tid = GET_TID(rpl);
661 unsigned int atid =
662 GET_TID_TID(GET_AOPEN_ATID(be32_to_cpu(rpl->atid_status)));
663 unsigned int status = GET_AOPEN_STATUS(be32_to_cpu(rpl->atid_status));
664 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
665 struct tid_info *t = lldi->tids;
666
667 csk = lookup_atid(t, atid);
668 if (unlikely(!csk)) {
669 pr_err("NO matching conn. atid %u, tid %u.\n", atid, tid);
670 goto rel_skb;
671 }
672
673 pr_info("%pI4:%u-%pI4:%u, atid %u,%u, status %u, csk 0x%p,%u,0x%lx.\n",
674 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port),
675 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port),
676 atid, tid, status, csk, csk->state, csk->flags);
677
678 if (status == CPL_ERR_RTX_NEG_ADVICE)
679 goto rel_skb;
680
681 if (status && status != CPL_ERR_TCAM_FULL &&
682 status != CPL_ERR_CONN_EXIST &&
683 status != CPL_ERR_ARP_MISS)
684 cxgb4_remove_tid(lldi->tids, csk->port_id, GET_TID(rpl));
685
686 cxgbi_sock_get(csk);
687 spin_lock_bh(&csk->lock);
688
689 if (status == CPL_ERR_CONN_EXIST &&
690 csk->retry_timer.function != csk_act_open_retry_timer) {
691 csk->retry_timer.function = csk_act_open_retry_timer;
692 mod_timer(&csk->retry_timer, jiffies + HZ / 2);
693 } else
694 cxgbi_sock_fail_act_open(csk,
695 act_open_rpl_status_to_errno(status));
696
697 spin_unlock_bh(&csk->lock);
698 cxgbi_sock_put(csk);
699 rel_skb:
700 __kfree_skb(skb);
701 }
702
703 static void do_peer_close(struct cxgbi_device *cdev, struct sk_buff *skb)
704 {
705 struct cxgbi_sock *csk;
706 struct cpl_peer_close *req = (struct cpl_peer_close *)skb->data;
707 unsigned int tid = GET_TID(req);
708 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
709 struct tid_info *t = lldi->tids;
710
711 csk = lookup_tid(t, tid);
712 if (unlikely(!csk)) {
713 pr_err("can't find connection for tid %u.\n", tid);
714 goto rel_skb;
715 }
716 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
717 "csk 0x%p,%u,0x%lx,%u.\n",
718 csk, csk->state, csk->flags, csk->tid);
719 cxgbi_sock_rcv_peer_close(csk);
720 rel_skb:
721 __kfree_skb(skb);
722 }
723
724 static void do_close_con_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
725 {
726 struct cxgbi_sock *csk;
727 struct cpl_close_con_rpl *rpl = (struct cpl_close_con_rpl *)skb->data;
728 unsigned int tid = GET_TID(rpl);
729 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
730 struct tid_info *t = lldi->tids;
731
732 csk = lookup_tid(t, tid);
733 if (unlikely(!csk)) {
734 pr_err("can't find connection for tid %u.\n", tid);
735 goto rel_skb;
736 }
737 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
738 "csk 0x%p,%u,0x%lx,%u.\n",
739 csk, csk->state, csk->flags, csk->tid);
740 cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt));
741 rel_skb:
742 __kfree_skb(skb);
743 }
744
745 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason,
746 int *need_rst)
747 {
748 switch (abort_reason) {
749 case CPL_ERR_BAD_SYN: /* fall through */
750 case CPL_ERR_CONN_RESET:
751 return csk->state > CTP_ESTABLISHED ?
752 -EPIPE : -ECONNRESET;
753 case CPL_ERR_XMIT_TIMEDOUT:
754 case CPL_ERR_PERSIST_TIMEDOUT:
755 case CPL_ERR_FINWAIT2_TIMEDOUT:
756 case CPL_ERR_KEEPALIVE_TIMEDOUT:
757 return -ETIMEDOUT;
758 default:
759 return -EIO;
760 }
761 }
762
763 static void do_abort_req_rss(struct cxgbi_device *cdev, struct sk_buff *skb)
764 {
765 struct cxgbi_sock *csk;
766 struct cpl_abort_req_rss *req = (struct cpl_abort_req_rss *)skb->data;
767 unsigned int tid = GET_TID(req);
768 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
769 struct tid_info *t = lldi->tids;
770 int rst_status = CPL_ABORT_NO_RST;
771
772 csk = lookup_tid(t, tid);
773 if (unlikely(!csk)) {
774 pr_err("can't find connection for tid %u.\n", tid);
775 goto rel_skb;
776 }
777
778 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
779 "csk 0x%p,%u,0x%lx, tid %u, status 0x%x.\n",
780 csk, csk->state, csk->flags, csk->tid, req->status);
781
782 if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
783 req->status == CPL_ERR_PERSIST_NEG_ADVICE)
784 goto rel_skb;
785
786 cxgbi_sock_get(csk);
787 spin_lock_bh(&csk->lock);
788
789 if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) {
790 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD);
791 cxgbi_sock_set_state(csk, CTP_ABORTING);
792 goto done;
793 }
794
795 cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD);
796 send_abort_rpl(csk, rst_status);
797
798 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) {
799 csk->err = abort_status_to_errno(csk, req->status, &rst_status);
800 cxgbi_sock_closed(csk);
801 }
802 done:
803 spin_unlock_bh(&csk->lock);
804 cxgbi_sock_put(csk);
805 rel_skb:
806 __kfree_skb(skb);
807 }
808
809 static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb)
810 {
811 struct cxgbi_sock *csk;
812 struct cpl_abort_rpl_rss *rpl = (struct cpl_abort_rpl_rss *)skb->data;
813 unsigned int tid = GET_TID(rpl);
814 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
815 struct tid_info *t = lldi->tids;
816
817 csk = lookup_tid(t, tid);
818 if (!csk)
819 goto rel_skb;
820
821 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
822 "status 0x%x, csk 0x%p, s %u, 0x%lx.\n",
823 rpl->status, csk, csk ? csk->state : 0,
824 csk ? csk->flags : 0UL);
825
826 if (rpl->status == CPL_ERR_ABORT_FAILED)
827 goto rel_skb;
828
829 cxgbi_sock_rcv_abort_rpl(csk);
830 rel_skb:
831 __kfree_skb(skb);
832 }
833
834 static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb)
835 {
836 struct cxgbi_sock *csk;
837 struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data;
838 unsigned short pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp);
839 unsigned int tid = GET_TID(cpl);
840 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
841 struct tid_info *t = lldi->tids;
842
843 csk = lookup_tid(t, tid);
844 if (unlikely(!csk)) {
845 pr_err("can't find conn. for tid %u.\n", tid);
846 goto rel_skb;
847 }
848
849 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
850 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n",
851 csk, csk->state, csk->flags, csk->tid, skb, skb->len,
852 pdu_len_ddp);
853
854 spin_lock_bh(&csk->lock);
855
856 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
857 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
858 "csk 0x%p,%u,0x%lx,%u, bad state.\n",
859 csk, csk->state, csk->flags, csk->tid);
860 if (csk->state != CTP_ABORTING)
861 goto abort_conn;
862 else
863 goto discard;
864 }
865
866 cxgbi_skcb_tcp_seq(skb) = ntohl(cpl->seq);
867 cxgbi_skcb_flags(skb) = 0;
868
869 skb_reset_transport_header(skb);
870 __skb_pull(skb, sizeof(*cpl));
871 __pskb_trim(skb, ntohs(cpl->len));
872
873 if (!csk->skb_ulp_lhdr) {
874 unsigned char *bhs;
875 unsigned int hlen, dlen;
876
877 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
878 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n",
879 csk, csk->state, csk->flags, csk->tid, skb);
880 csk->skb_ulp_lhdr = skb;
881 cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR);
882
883 if (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt) {
884 pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n",
885 csk->tid, cxgbi_skcb_tcp_seq(skb),
886 csk->rcv_nxt);
887 goto abort_conn;
888 }
889
890 bhs = skb->data;
891 hlen = ntohs(cpl->len);
892 dlen = ntohl(*(unsigned int *)(bhs + 4)) & 0xFFFFFF;
893
894 if ((hlen + dlen) != ISCSI_PDU_LEN(pdu_len_ddp) - 40) {
895 pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len "
896 "mismatch %u != %u + %u, seq 0x%x.\n",
897 csk->tid, ISCSI_PDU_LEN(pdu_len_ddp) - 40,
898 hlen, dlen, cxgbi_skcb_tcp_seq(skb));
899 goto abort_conn;
900 }
901
902 cxgbi_skcb_rx_pdulen(skb) = (hlen + dlen + 3) & (~0x3);
903 if (dlen)
904 cxgbi_skcb_rx_pdulen(skb) += csk->dcrc_len;
905 csk->rcv_nxt += cxgbi_skcb_rx_pdulen(skb);
906
907 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
908 "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n",
909 csk, skb, *bhs, hlen, dlen,
910 ntohl(*((unsigned int *)(bhs + 16))),
911 ntohl(*((unsigned int *)(bhs + 24))));
912
913 } else {
914 struct sk_buff *lskb = csk->skb_ulp_lhdr;
915
916 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA);
917 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
918 "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n",
919 csk, csk->state, csk->flags, skb, lskb);
920 }
921
922 __skb_queue_tail(&csk->receive_queue, skb);
923 spin_unlock_bh(&csk->lock);
924 return;
925
926 abort_conn:
927 send_abort_req(csk);
928 discard:
929 spin_unlock_bh(&csk->lock);
930 rel_skb:
931 __kfree_skb(skb);
932 }
933
934 static void do_rx_data_ddp(struct cxgbi_device *cdev,
935 struct sk_buff *skb)
936 {
937 struct cxgbi_sock *csk;
938 struct sk_buff *lskb;
939 struct cpl_rx_data_ddp *rpl = (struct cpl_rx_data_ddp *)skb->data;
940 unsigned int tid = GET_TID(rpl);
941 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
942 struct tid_info *t = lldi->tids;
943 unsigned int status = ntohl(rpl->ddpvld);
944
945 csk = lookup_tid(t, tid);
946 if (unlikely(!csk)) {
947 pr_err("can't find connection for tid %u.\n", tid);
948 goto rel_skb;
949 }
950
951 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX,
952 "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n",
953 csk, csk->state, csk->flags, skb, status, csk->skb_ulp_lhdr);
954
955 spin_lock_bh(&csk->lock);
956
957 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) {
958 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
959 "csk 0x%p,%u,0x%lx,%u, bad state.\n",
960 csk, csk->state, csk->flags, csk->tid);
961 if (csk->state != CTP_ABORTING)
962 goto abort_conn;
963 else
964 goto discard;
965 }
966
967 if (!csk->skb_ulp_lhdr) {
968 pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk->tid);
969 goto abort_conn;
970 }
971
972 lskb = csk->skb_ulp_lhdr;
973 csk->skb_ulp_lhdr = NULL;
974
975 cxgbi_skcb_rx_ddigest(lskb) = ntohl(rpl->ulp_crc);
976
977 if (ntohs(rpl->len) != cxgbi_skcb_rx_pdulen(lskb))
978 pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n",
979 csk->tid, ntohs(rpl->len), cxgbi_skcb_rx_pdulen(lskb));
980
981 if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) {
982 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n",
983 csk, lskb, status, cxgbi_skcb_flags(lskb));
984 cxgbi_skcb_set_flag(lskb, SKCBF_RX_HCRC_ERR);
985 }
986 if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) {
987 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n",
988 csk, lskb, status, cxgbi_skcb_flags(lskb));
989 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DCRC_ERR);
990 }
991 if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) {
992 log_debug(1 << CXGBI_DBG_PDU_RX,
993 "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n",
994 csk, lskb, status);
995 cxgbi_skcb_set_flag(lskb, SKCBF_RX_PAD_ERR);
996 }
997 if ((status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) &&
998 !cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA)) {
999 log_debug(1 << CXGBI_DBG_PDU_RX,
1000 "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n",
1001 csk, lskb, status);
1002 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA_DDPD);
1003 }
1004 log_debug(1 << CXGBI_DBG_PDU_RX,
1005 "csk 0x%p, lskb 0x%p, f 0x%lx.\n",
1006 csk, lskb, cxgbi_skcb_flags(lskb));
1007
1008 cxgbi_skcb_set_flag(lskb, SKCBF_RX_STATUS);
1009 cxgbi_conn_pdu_ready(csk);
1010 spin_unlock_bh(&csk->lock);
1011 goto rel_skb;
1012
1013 abort_conn:
1014 send_abort_req(csk);
1015 discard:
1016 spin_unlock_bh(&csk->lock);
1017 rel_skb:
1018 __kfree_skb(skb);
1019 }
1020
1021 static void do_fw4_ack(struct cxgbi_device *cdev, struct sk_buff *skb)
1022 {
1023 struct cxgbi_sock *csk;
1024 struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)skb->data;
1025 unsigned int tid = GET_TID(rpl);
1026 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1027 struct tid_info *t = lldi->tids;
1028
1029 csk = lookup_tid(t, tid);
1030 if (unlikely(!csk))
1031 pr_err("can't find connection for tid %u.\n", tid);
1032 else {
1033 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1034 "csk 0x%p,%u,0x%lx,%u.\n",
1035 csk, csk->state, csk->flags, csk->tid);
1036 cxgbi_sock_rcv_wr_ack(csk, rpl->credits, ntohl(rpl->snd_una),
1037 rpl->seq_vld);
1038 }
1039 __kfree_skb(skb);
1040 }
1041
1042 static void do_set_tcb_rpl(struct cxgbi_device *cdev, struct sk_buff *skb)
1043 {
1044 struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data;
1045 unsigned int tid = GET_TID(rpl);
1046 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1047 struct tid_info *t = lldi->tids;
1048 struct cxgbi_sock *csk;
1049
1050 csk = lookup_tid(t, tid);
1051 if (!csk)
1052 pr_err("can't find conn. for tid %u.\n", tid);
1053
1054 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1055 "csk 0x%p,%u,%lx,%u, status 0x%x.\n",
1056 csk, csk->state, csk->flags, csk->tid, rpl->status);
1057
1058 if (rpl->status != CPL_ERR_NONE)
1059 pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n",
1060 csk, tid, rpl->status);
1061
1062 __kfree_skb(skb);
1063 }
1064
1065 static int alloc_cpls(struct cxgbi_sock *csk)
1066 {
1067 csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req),
1068 0, GFP_KERNEL);
1069 if (!csk->cpl_close)
1070 return -ENOMEM;
1071
1072 csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req),
1073 0, GFP_KERNEL);
1074 if (!csk->cpl_abort_req)
1075 goto free_cpls;
1076
1077 csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl),
1078 0, GFP_KERNEL);
1079 if (!csk->cpl_abort_rpl)
1080 goto free_cpls;
1081 return 0;
1082
1083 free_cpls:
1084 cxgbi_sock_free_cpl_skbs(csk);
1085 return -ENOMEM;
1086 }
1087
1088 static inline void l2t_put(struct cxgbi_sock *csk)
1089 {
1090 if (csk->l2t) {
1091 cxgb4_l2t_release(csk->l2t);
1092 csk->l2t = NULL;
1093 cxgbi_sock_put(csk);
1094 }
1095 }
1096
1097 static void release_offload_resources(struct cxgbi_sock *csk)
1098 {
1099 struct cxgb4_lld_info *lldi;
1100
1101 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1102 "csk 0x%p,%u,0x%lx,%u.\n",
1103 csk, csk->state, csk->flags, csk->tid);
1104
1105 cxgbi_sock_free_cpl_skbs(csk);
1106 if (csk->wr_cred != csk->wr_max_cred) {
1107 cxgbi_sock_purge_wr_queue(csk);
1108 cxgbi_sock_reset_wr_list(csk);
1109 }
1110
1111 l2t_put(csk);
1112 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID))
1113 free_atid(csk);
1114 else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) {
1115 lldi = cxgbi_cdev_priv(csk->cdev);
1116 cxgb4_remove_tid(lldi->tids, 0, csk->tid);
1117 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID);
1118 cxgbi_sock_put(csk);
1119 }
1120 csk->dst = NULL;
1121 csk->cdev = NULL;
1122 }
1123
1124 static int init_act_open(struct cxgbi_sock *csk)
1125 {
1126 struct cxgbi_device *cdev = csk->cdev;
1127 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1128 struct net_device *ndev = cdev->ports[csk->port_id];
1129 struct port_info *pi = netdev_priv(ndev);
1130 struct sk_buff *skb = NULL;
1131 unsigned int step;
1132
1133 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1134 "csk 0x%p,%u,0x%lx,%u.\n",
1135 csk, csk->state, csk->flags, csk->tid);
1136
1137 csk->atid = cxgb4_alloc_atid(lldi->tids, csk);
1138 if (csk->atid < 0) {
1139 pr_err("%s, NO atid available.\n", ndev->name);
1140 return -EINVAL;
1141 }
1142 cxgbi_sock_set_flag(csk, CTPF_HAS_ATID);
1143 cxgbi_sock_get(csk);
1144
1145 csk->l2t = cxgb4_l2t_get(lldi->l2t, dst_get_neighbour(csk->dst), ndev, 0);
1146 if (!csk->l2t) {
1147 pr_err("%s, cannot alloc l2t.\n", ndev->name);
1148 goto rel_resource;
1149 }
1150 cxgbi_sock_get(csk);
1151
1152 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL);
1153 if (!skb)
1154 goto rel_resource;
1155 skb->sk = (struct sock *)csk;
1156 t4_set_arp_err_handler(skb, csk, cxgbi_sock_act_open_req_arp_failure);
1157
1158 if (!csk->mtu)
1159 csk->mtu = dst_mtu(csk->dst);
1160 cxgb4_best_mtu(lldi->mtus, csk->mtu, &csk->mss_idx);
1161 csk->tx_chan = cxgb4_port_chan(ndev);
1162 /* SMT two entries per row */
1163 csk->smac_idx = ((cxgb4_port_viid(ndev) & 0x7F)) << 1;
1164 step = lldi->ntxq / lldi->nchan;
1165 csk->txq_idx = cxgb4_port_idx(ndev) * step;
1166 step = lldi->nrxq / lldi->nchan;
1167 csk->rss_qid = lldi->rxq_ids[cxgb4_port_idx(ndev) * step];
1168 csk->wr_max_cred = csk->wr_cred = lldi->wr_cred;
1169 csk->wr_una_cred = 0;
1170 cxgbi_sock_reset_wr_list(csk);
1171 csk->err = 0;
1172 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1173 "csk 0x%p,p%d,%s, %u,%u,%u, mss %u,%u, smac %u.\n",
1174 csk, pi->port_id, ndev->name, csk->tx_chan,
1175 csk->txq_idx, csk->rss_qid, csk->mtu, csk->mss_idx,
1176 csk->smac_idx);
1177
1178 cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN);
1179 send_act_open_req(csk, skb, csk->l2t);
1180 return 0;
1181
1182 rel_resource:
1183 if (skb)
1184 __kfree_skb(skb);
1185 return -EINVAL;
1186 }
1187
1188 cxgb4i_cplhandler_func cxgb4i_cplhandlers[NUM_CPL_CMDS] = {
1189 [CPL_ACT_ESTABLISH] = do_act_establish,
1190 [CPL_ACT_OPEN_RPL] = do_act_open_rpl,
1191 [CPL_PEER_CLOSE] = do_peer_close,
1192 [CPL_ABORT_REQ_RSS] = do_abort_req_rss,
1193 [CPL_ABORT_RPL_RSS] = do_abort_rpl_rss,
1194 [CPL_CLOSE_CON_RPL] = do_close_con_rpl,
1195 [CPL_FW4_ACK] = do_fw4_ack,
1196 [CPL_ISCSI_HDR] = do_rx_iscsi_hdr,
1197 [CPL_SET_TCB_RPL] = do_set_tcb_rpl,
1198 [CPL_RX_DATA_DDP] = do_rx_data_ddp,
1199 };
1200
1201 int cxgb4i_ofld_init(struct cxgbi_device *cdev)
1202 {
1203 int rc;
1204
1205 if (cxgb4i_max_connect > CXGB4I_MAX_CONN)
1206 cxgb4i_max_connect = CXGB4I_MAX_CONN;
1207
1208 rc = cxgbi_device_portmap_create(cdev, cxgb4i_sport_base,
1209 cxgb4i_max_connect);
1210 if (rc < 0)
1211 return rc;
1212
1213 cdev->csk_release_offload_resources = release_offload_resources;
1214 cdev->csk_push_tx_frames = push_tx_frames;
1215 cdev->csk_send_abort_req = send_abort_req;
1216 cdev->csk_send_close_req = send_close_req;
1217 cdev->csk_send_rx_credits = send_rx_credits;
1218 cdev->csk_alloc_cpls = alloc_cpls;
1219 cdev->csk_init_act_open = init_act_open;
1220
1221 pr_info("cdev 0x%p, offload up, added.\n", cdev);
1222 return 0;
1223 }
1224
1225 /*
1226 * functions to program the pagepod in h/w
1227 */
1228 #define ULPMEM_IDATA_MAX_NPPODS 4 /* 256/PPOD_SIZE */
1229 static inline void ulp_mem_io_set_hdr(struct ulp_mem_io *req,
1230 unsigned int wr_len, unsigned int dlen,
1231 unsigned int pm_addr)
1232 {
1233 struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1);
1234
1235 INIT_ULPTX_WR(req, wr_len, 0, 0);
1236 req->cmd = htonl(ULPTX_CMD(ULP_TX_MEM_WRITE) | (1 << 23));
1237 req->dlen = htonl(ULP_MEMIO_DATA_LEN(dlen >> 5));
1238 req->lock_addr = htonl(ULP_MEMIO_ADDR(pm_addr >> 5));
1239 req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16));
1240
1241 idata->cmd_more = htonl(ULPTX_CMD(ULP_TX_SC_IMM));
1242 idata->len = htonl(dlen);
1243 }
1244
1245 static int ddp_ppod_write_idata(struct cxgbi_device *cdev, unsigned int port_id,
1246 struct cxgbi_pagepod_hdr *hdr, unsigned int idx,
1247 unsigned int npods,
1248 struct cxgbi_gather_list *gl,
1249 unsigned int gl_pidx)
1250 {
1251 struct cxgbi_ddp_info *ddp = cdev->ddp;
1252 struct sk_buff *skb;
1253 struct ulp_mem_io *req;
1254 struct ulptx_idata *idata;
1255 struct cxgbi_pagepod *ppod;
1256 unsigned int pm_addr = idx * PPOD_SIZE + ddp->llimit;
1257 unsigned int dlen = PPOD_SIZE * npods;
1258 unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) +
1259 sizeof(struct ulptx_idata) + dlen, 16);
1260 unsigned int i;
1261
1262 skb = alloc_wr(wr_len, 0, GFP_ATOMIC);
1263 if (!skb) {
1264 pr_err("cdev 0x%p, idx %u, npods %u, OOM.\n",
1265 cdev, idx, npods);
1266 return -ENOMEM;
1267 }
1268 req = (struct ulp_mem_io *)skb->head;
1269 set_queue(skb, CPL_PRIORITY_CONTROL, NULL);
1270
1271 ulp_mem_io_set_hdr(req, wr_len, dlen, pm_addr);
1272 idata = (struct ulptx_idata *)(req + 1);
1273 ppod = (struct cxgbi_pagepod *)(idata + 1);
1274
1275 for (i = 0; i < npods; i++, ppod++, gl_pidx += PPOD_PAGES_MAX) {
1276 if (!hdr && !gl)
1277 cxgbi_ddp_ppod_clear(ppod);
1278 else
1279 cxgbi_ddp_ppod_set(ppod, hdr, gl, gl_pidx);
1280 }
1281
1282 cxgb4_ofld_send(cdev->ports[port_id], skb);
1283 return 0;
1284 }
1285
1286 static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr,
1287 unsigned int idx, unsigned int npods,
1288 struct cxgbi_gather_list *gl)
1289 {
1290 unsigned int i, cnt;
1291 int err = 0;
1292
1293 for (i = 0; i < npods; i += cnt, idx += cnt) {
1294 cnt = npods - i;
1295 if (cnt > ULPMEM_IDATA_MAX_NPPODS)
1296 cnt = ULPMEM_IDATA_MAX_NPPODS;
1297 err = ddp_ppod_write_idata(csk->cdev, csk->port_id, hdr,
1298 idx, cnt, gl, 4 * i);
1299 if (err < 0)
1300 break;
1301 }
1302 return err;
1303 }
1304
1305 static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag,
1306 unsigned int idx, unsigned int npods)
1307 {
1308 unsigned int i, cnt;
1309 int err;
1310
1311 for (i = 0; i < npods; i += cnt, idx += cnt) {
1312 cnt = npods - i;
1313 if (cnt > ULPMEM_IDATA_MAX_NPPODS)
1314 cnt = ULPMEM_IDATA_MAX_NPPODS;
1315 err = ddp_ppod_write_idata(chba->cdev, chba->port_id, NULL,
1316 idx, cnt, NULL, 0);
1317 if (err < 0)
1318 break;
1319 }
1320 }
1321
1322 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, unsigned int tid,
1323 int pg_idx, bool reply)
1324 {
1325 struct sk_buff *skb;
1326 struct cpl_set_tcb_field *req;
1327
1328 if (!pg_idx || pg_idx >= DDP_PGIDX_MAX)
1329 return 0;
1330
1331 skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL);
1332 if (!skb)
1333 return -ENOMEM;
1334
1335 /* set up ulp page size */
1336 req = (struct cpl_set_tcb_field *)skb->head;
1337 INIT_TP_WR(req, csk->tid);
1338 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid));
1339 req->reply_ctrl = htons(NO_REPLY(reply) | QUEUENO(csk->rss_qid));
1340 req->word_cookie = htons(0);
1341 req->mask = cpu_to_be64(0x3 << 8);
1342 req->val = cpu_to_be64(pg_idx << 8);
1343 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
1344
1345 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1346 "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk, csk->tid, pg_idx);
1347
1348 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
1349 return 0;
1350 }
1351
1352 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid,
1353 int hcrc, int dcrc, int reply)
1354 {
1355 struct sk_buff *skb;
1356 struct cpl_set_tcb_field *req;
1357
1358 if (!hcrc && !dcrc)
1359 return 0;
1360
1361 skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL);
1362 if (!skb)
1363 return -ENOMEM;
1364
1365 csk->hcrc_len = (hcrc ? 4 : 0);
1366 csk->dcrc_len = (dcrc ? 4 : 0);
1367 /* set up ulp submode */
1368 req = (struct cpl_set_tcb_field *)skb->head;
1369 INIT_TP_WR(req, tid);
1370 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
1371 req->reply_ctrl = htons(NO_REPLY(reply) | QUEUENO(csk->rss_qid));
1372 req->word_cookie = htons(0);
1373 req->mask = cpu_to_be64(0x3 << 4);
1374 req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) |
1375 (dcrc ? ULP_CRC_DATA : 0)) << 4);
1376 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id);
1377
1378 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK,
1379 "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk, csk->tid, hcrc, dcrc);
1380
1381 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb);
1382 return 0;
1383 }
1384
1385 static int cxgb4i_ddp_init(struct cxgbi_device *cdev)
1386 {
1387 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev);
1388 struct cxgbi_ddp_info *ddp = cdev->ddp;
1389 unsigned int tagmask, pgsz_factor[4];
1390 int err;
1391
1392 if (ddp) {
1393 kref_get(&ddp->refcnt);
1394 pr_warn("cdev 0x%p, ddp 0x%p already set up.\n",
1395 cdev, cdev->ddp);
1396 return -EALREADY;
1397 }
1398
1399 err = cxgbi_ddp_init(cdev, lldi->vr->iscsi.start,
1400 lldi->vr->iscsi.start + lldi->vr->iscsi.size - 1,
1401 lldi->iscsi_iolen, lldi->iscsi_iolen);
1402 if (err < 0)
1403 return err;
1404
1405 ddp = cdev->ddp;
1406
1407 tagmask = ddp->idx_mask << PPOD_IDX_SHIFT;
1408 cxgbi_ddp_page_size_factor(pgsz_factor);
1409 cxgb4_iscsi_init(lldi->ports[0], tagmask, pgsz_factor);
1410
1411 cdev->csk_ddp_setup_digest = ddp_setup_conn_digest;
1412 cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx;
1413 cdev->csk_ddp_set = ddp_set_map;
1414 cdev->csk_ddp_clear = ddp_clear_map;
1415
1416 pr_info("cxgb4i 0x%p tag: sw %u, rsvd %u,%u, mask 0x%x.\n",
1417 cdev, cdev->tag_format.sw_bits, cdev->tag_format.rsvd_bits,
1418 cdev->tag_format.rsvd_shift, cdev->tag_format.rsvd_mask);
1419 pr_info("cxgb4i 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, "
1420 " %u/%u.\n",
1421 cdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask,
1422 ddp->rsvd_tag_mask, ddp->max_txsz, lldi->iscsi_iolen,
1423 ddp->max_rxsz, lldi->iscsi_iolen);
1424 pr_info("cxgb4i 0x%p max payload size: %u/%u, %u/%u.\n",
1425 cdev, cdev->tx_max_size, ddp->max_txsz, cdev->rx_max_size,
1426 ddp->max_rxsz);
1427 return 0;
1428 }
1429
1430 static void *t4_uld_add(const struct cxgb4_lld_info *lldi)
1431 {
1432 struct cxgbi_device *cdev;
1433 struct port_info *pi;
1434 int i, rc;
1435
1436 cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports);
1437 if (!cdev) {
1438 pr_info("t4 device 0x%p, register failed.\n", lldi);
1439 return NULL;
1440 }
1441 pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n",
1442 cdev, lldi->adapter_type, lldi->nports,
1443 lldi->ports[0]->name, lldi->nchan, lldi->ntxq,
1444 lldi->nrxq, lldi->wr_cred);
1445 for (i = 0; i < lldi->nrxq; i++)
1446 log_debug(1 << CXGBI_DBG_DEV,
1447 "t4 0x%p, rxq id #%d: %u.\n",
1448 cdev, i, lldi->rxq_ids[i]);
1449
1450 memcpy(cxgbi_cdev_priv(cdev), lldi, sizeof(*lldi));
1451 cdev->flags = CXGBI_FLAG_DEV_T4;
1452 cdev->pdev = lldi->pdev;
1453 cdev->ports = lldi->ports;
1454 cdev->nports = lldi->nports;
1455 cdev->mtus = lldi->mtus;
1456 cdev->nmtus = NMTUS;
1457 cdev->snd_win = cxgb4i_snd_win;
1458 cdev->rcv_win = cxgb4i_rcv_win;
1459 cdev->rx_credit_thres = cxgb4i_rx_credit_thres;
1460 cdev->skb_tx_rsvd = CXGB4I_TX_HEADER_LEN;
1461 cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr);
1462 cdev->itp = &cxgb4i_iscsi_transport;
1463
1464 cdev->pfvf = FW_VIID_PFN_GET(cxgb4_port_viid(lldi->ports[0])) << 8;
1465 pr_info("cdev 0x%p,%s, pfvf %u.\n",
1466 cdev, lldi->ports[0]->name, cdev->pfvf);
1467
1468 rc = cxgb4i_ddp_init(cdev);
1469 if (rc) {
1470 pr_info("t4 0x%p ddp init failed.\n", cdev);
1471 goto err_out;
1472 }
1473 rc = cxgb4i_ofld_init(cdev);
1474 if (rc) {
1475 pr_info("t4 0x%p ofld init failed.\n", cdev);
1476 goto err_out;
1477 }
1478
1479 rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, CXGBI_MAX_CONN,
1480 &cxgb4i_host_template, cxgb4i_stt);
1481 if (rc)
1482 goto err_out;
1483
1484 for (i = 0; i < cdev->nports; i++) {
1485 pi = netdev_priv(lldi->ports[i]);
1486 cdev->hbas[i]->port_id = pi->port_id;
1487 }
1488 return cdev;
1489
1490 err_out:
1491 cxgbi_device_unregister(cdev);
1492 return ERR_PTR(-ENOMEM);
1493 }
1494
1495 #define RX_PULL_LEN 128
1496 static int t4_uld_rx_handler(void *handle, const __be64 *rsp,
1497 const struct pkt_gl *pgl)
1498 {
1499 const struct cpl_act_establish *rpl;
1500 struct sk_buff *skb;
1501 unsigned int opc;
1502 struct cxgbi_device *cdev = handle;
1503
1504 if (pgl == NULL) {
1505 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8;
1506
1507 skb = alloc_wr(len, 0, GFP_ATOMIC);
1508 if (!skb)
1509 goto nomem;
1510 skb_copy_to_linear_data(skb, &rsp[1], len);
1511 } else {
1512 if (unlikely(*(u8 *)rsp != *(u8 *)pgl->va)) {
1513 pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n",
1514 pgl->va, be64_to_cpu(*rsp),
1515 be64_to_cpu(*(u64 *)pgl->va),
1516 pgl->tot_len);
1517 return 0;
1518 }
1519 skb = cxgb4_pktgl_to_skb(pgl, RX_PULL_LEN, RX_PULL_LEN);
1520 if (unlikely(!skb))
1521 goto nomem;
1522 }
1523
1524 rpl = (struct cpl_act_establish *)skb->data;
1525 opc = rpl->ot.opcode;
1526 log_debug(1 << CXGBI_DBG_TOE,
1527 "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n",
1528 cdev, opc, rpl->ot.opcode_tid, ntohl(rpl->ot.opcode_tid), skb);
1529 if (cxgb4i_cplhandlers[opc])
1530 cxgb4i_cplhandlers[opc](cdev, skb);
1531 else {
1532 pr_err("No handler for opcode 0x%x.\n", opc);
1533 __kfree_skb(skb);
1534 }
1535 return 0;
1536 nomem:
1537 log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out.\n");
1538 return 1;
1539 }
1540
1541 static int t4_uld_state_change(void *handle, enum cxgb4_state state)
1542 {
1543 struct cxgbi_device *cdev = handle;
1544
1545 switch (state) {
1546 case CXGB4_STATE_UP:
1547 pr_info("cdev 0x%p, UP.\n", cdev);
1548 /* re-initialize */
1549 break;
1550 case CXGB4_STATE_START_RECOVERY:
1551 pr_info("cdev 0x%p, RECOVERY.\n", cdev);
1552 /* close all connections */
1553 break;
1554 case CXGB4_STATE_DOWN:
1555 pr_info("cdev 0x%p, DOWN.\n", cdev);
1556 break;
1557 case CXGB4_STATE_DETACH:
1558 pr_info("cdev 0x%p, DETACH.\n", cdev);
1559 break;
1560 default:
1561 pr_info("cdev 0x%p, unknown state %d.\n", cdev, state);
1562 break;
1563 }
1564 return 0;
1565 }
1566
1567 static int __init cxgb4i_init_module(void)
1568 {
1569 int rc;
1570
1571 printk(KERN_INFO "%s", version);
1572
1573 rc = cxgbi_iscsi_init(&cxgb4i_iscsi_transport, &cxgb4i_stt);
1574 if (rc < 0)
1575 return rc;
1576 cxgb4_register_uld(CXGB4_ULD_ISCSI, &cxgb4i_uld_info);
1577 return 0;
1578 }
1579
1580 static void __exit cxgb4i_exit_module(void)
1581 {
1582 cxgb4_unregister_uld(CXGB4_ULD_ISCSI);
1583 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4);
1584 cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport, &cxgb4i_stt);
1585 }
1586
1587 module_init(cxgb4i_init_module);
1588 module_exit(cxgb4i_exit_module);
linux-next