?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 3.11-rc3
[cris-mirror.git] / drivers / infiniband / hw / cxgb3 / iwch_cm.c
blob3e094cd6a0e345e1e239c96348e162cc53184fa7
1 /*
2 * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #include <linux/module.h>
33 #include <linux/list.h>
34 #include <linux/slab.h>
35 #include <linux/workqueue.h>
36 #include <linux/skbuff.h>
37 #include <linux/timer.h>
38 #include <linux/notifier.h>
39 #include <linux/inetdevice.h>
40
41 #include <net/neighbour.h>
42 #include <net/netevent.h>
43 #include <net/route.h>
44
45 #include "tcb.h"
46 #include "cxgb3_offload.h"
47 #include "iwch.h"
48 #include "iwch_provider.h"
49 #include "iwch_cm.h"
50
51 static char *states[] = {
52 "idle",
53 "listen",
54 "connecting",
55 "mpa_wait_req",
56 "mpa_req_sent",
57 "mpa_req_rcvd",
58 "mpa_rep_sent",
59 "fpdu_mode",
60 "aborting",
61 "closing",
62 "moribund",
63 "dead",
64 NULL,
65 };
66
67 int peer2peer = 0;
68 module_param(peer2peer, int, 0644);
69 MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)");
70
71 static int ep_timeout_secs = 60;
72 module_param(ep_timeout_secs, int, 0644);
73 MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout "
74 "in seconds (default=60)");
75
76 static int mpa_rev = 1;
77 module_param(mpa_rev, int, 0644);
78 MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, "
79 "1 is spec compliant. (default=1)");
80
81 static int markers_enabled = 0;
82 module_param(markers_enabled, int, 0644);
83 MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)");
84
85 static int crc_enabled = 1;
86 module_param(crc_enabled, int, 0644);
87 MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)");
88
89 static int rcv_win = 256 * 1024;
90 module_param(rcv_win, int, 0644);
91 MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256)");
92
93 static int snd_win = 32 * 1024;
94 module_param(snd_win, int, 0644);
95 MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=32KB)");
96
97 static unsigned int nocong = 0;
98 module_param(nocong, uint, 0644);
99 MODULE_PARM_DESC(nocong, "Turn off congestion control (default=0)");
100
101 static unsigned int cong_flavor = 1;
102 module_param(cong_flavor, uint, 0644);
103 MODULE_PARM_DESC(cong_flavor, "TCP Congestion control flavor (default=1)");
104
105 static struct workqueue_struct *workq;
106
107 static struct sk_buff_head rxq;
108
109 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp);
110 static void ep_timeout(unsigned long arg);
111 static void connect_reply_upcall(struct iwch_ep *ep, int status);
112
113 static void start_ep_timer(struct iwch_ep *ep)
114 {
115 PDBG("%s ep %p\n", __func__, ep);
116 if (timer_pending(&ep->timer)) {
117 PDBG("%s stopped / restarted timer ep %p\n", __func__, ep);
118 del_timer_sync(&ep->timer);
119 } else
120 get_ep(&ep->com);
121 ep->timer.expires = jiffies + ep_timeout_secs * HZ;
122 ep->timer.data = (unsigned long)ep;
123 ep->timer.function = ep_timeout;
124 add_timer(&ep->timer);
125 }
126
127 static void stop_ep_timer(struct iwch_ep *ep)
128 {
129 PDBG("%s ep %p\n", __func__, ep);
130 if (!timer_pending(&ep->timer)) {
131 WARN(1, "%s timer stopped when its not running! ep %p state %u\n",
132 __func__, ep, ep->com.state);
133 return;
134 }
135 del_timer_sync(&ep->timer);
136 put_ep(&ep->com);
137 }
138
139 static int iwch_l2t_send(struct t3cdev *tdev, struct sk_buff *skb, struct l2t_entry *l2e)
140 {
141 int error = 0;
142 struct cxio_rdev *rdev;
143
144 rdev = (struct cxio_rdev *)tdev->ulp;
145 if (cxio_fatal_error(rdev)) {
146 kfree_skb(skb);
147 return -EIO;
148 }
149 error = l2t_send(tdev, skb, l2e);
150 if (error < 0)
151 kfree_skb(skb);
152 return error;
153 }
154
155 int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb)
156 {
157 int error = 0;
158 struct cxio_rdev *rdev;
159
160 rdev = (struct cxio_rdev *)tdev->ulp;
161 if (cxio_fatal_error(rdev)) {
162 kfree_skb(skb);
163 return -EIO;
164 }
165 error = cxgb3_ofld_send(tdev, skb);
166 if (error < 0)
167 kfree_skb(skb);
168 return error;
169 }
170
171 static void release_tid(struct t3cdev *tdev, u32 hwtid, struct sk_buff *skb)
172 {
173 struct cpl_tid_release *req;
174
175 skb = get_skb(skb, sizeof *req, GFP_KERNEL);
176 if (!skb)
177 return;
178 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req));
179 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
180 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid));
181 skb->priority = CPL_PRIORITY_SETUP;
182 iwch_cxgb3_ofld_send(tdev, skb);
183 return;
184 }
185
186 int iwch_quiesce_tid(struct iwch_ep *ep)
187 {
188 struct cpl_set_tcb_field *req;
189 struct sk_buff *skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
190
191 if (!skb)
192 return -ENOMEM;
193 req = (struct cpl_set_tcb_field *) skb_put(skb, sizeof(*req));
194 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
195 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
196 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep->hwtid));
197 req->reply = 0;
198 req->cpu_idx = 0;
199 req->word = htons(W_TCB_RX_QUIESCE);
200 req->mask = cpu_to_be64(1ULL << S_TCB_RX_QUIESCE);
201 req->val = cpu_to_be64(1 << S_TCB_RX_QUIESCE);
202
203 skb->priority = CPL_PRIORITY_DATA;
204 return iwch_cxgb3_ofld_send(ep->com.tdev, skb);
205 }
206
207 int iwch_resume_tid(struct iwch_ep *ep)
208 {
209 struct cpl_set_tcb_field *req;
210 struct sk_buff *skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
211
212 if (!skb)
213 return -ENOMEM;
214 req = (struct cpl_set_tcb_field *) skb_put(skb, sizeof(*req));
215 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
216 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
217 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, ep->hwtid));
218 req->reply = 0;
219 req->cpu_idx = 0;
220 req->word = htons(W_TCB_RX_QUIESCE);
221 req->mask = cpu_to_be64(1ULL << S_TCB_RX_QUIESCE);
222 req->val = 0;
223
224 skb->priority = CPL_PRIORITY_DATA;
225 return iwch_cxgb3_ofld_send(ep->com.tdev, skb);
226 }
227
228 static void set_emss(struct iwch_ep *ep, u16 opt)
229 {
230 PDBG("%s ep %p opt %u\n", __func__, ep, opt);
231 ep->emss = T3C_DATA(ep->com.tdev)->mtus[G_TCPOPT_MSS(opt)] - 40;
232 if (G_TCPOPT_TSTAMP(opt))
233 ep->emss -= 12;
234 if (ep->emss < 128)
235 ep->emss = 128;
236 PDBG("emss=%d\n", ep->emss);
237 }
238
239 static enum iwch_ep_state state_read(struct iwch_ep_common *epc)
240 {
241 unsigned long flags;
242 enum iwch_ep_state state;
243
244 spin_lock_irqsave(&epc->lock, flags);
245 state = epc->state;
246 spin_unlock_irqrestore(&epc->lock, flags);
247 return state;
248 }
249
250 static void __state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
251 {
252 epc->state = new;
253 }
254
255 static void state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
256 {
257 unsigned long flags;
258
259 spin_lock_irqsave(&epc->lock, flags);
260 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]);
261 __state_set(epc, new);
262 spin_unlock_irqrestore(&epc->lock, flags);
263 return;
264 }
265
266 static void *alloc_ep(int size, gfp_t gfp)
267 {
268 struct iwch_ep_common *epc;
269
270 epc = kzalloc(size, gfp);
271 if (epc) {
272 kref_init(&epc->kref);
273 spin_lock_init(&epc->lock);
274 init_waitqueue_head(&epc->waitq);
275 }
276 PDBG("%s alloc ep %p\n", __func__, epc);
277 return epc;
278 }
279
280 void __free_ep(struct kref *kref)
281 {
282 struct iwch_ep *ep;
283 ep = container_of(container_of(kref, struct iwch_ep_common, kref),
284 struct iwch_ep, com);
285 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]);
286 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) {
287 cxgb3_remove_tid(ep->com.tdev, (void *)ep, ep->hwtid);
288 dst_release(ep->dst);
289 l2t_release(ep->com.tdev, ep->l2t);
290 }
291 kfree(ep);
292 }
293
294 static void release_ep_resources(struct iwch_ep *ep)
295 {
296 PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid);
297 set_bit(RELEASE_RESOURCES, &ep->com.flags);
298 put_ep(&ep->com);
299 }
300
301 static int status2errno(int status)
302 {
303 switch (status) {
304 case CPL_ERR_NONE:
305 return 0;
306 case CPL_ERR_CONN_RESET:
307 return -ECONNRESET;
308 case CPL_ERR_ARP_MISS:
309 return -EHOSTUNREACH;
310 case CPL_ERR_CONN_TIMEDOUT:
311 return -ETIMEDOUT;
312 case CPL_ERR_TCAM_FULL:
313 return -ENOMEM;
314 case CPL_ERR_CONN_EXIST:
315 return -EADDRINUSE;
316 default:
317 return -EIO;
318 }
319 }
320
321 /*
322 * Try and reuse skbs already allocated...
323 */
324 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp)
325 {
326 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) {
327 skb_trim(skb, 0);
328 skb_get(skb);
329 } else {
330 skb = alloc_skb(len, gfp);
331 }
332 return skb;
333 }
334
335 static struct rtable *find_route(struct t3cdev *dev, __be32 local_ip,
336 __be32 peer_ip, __be16 local_port,
337 __be16 peer_port, u8 tos)
338 {
339 struct rtable *rt;
340 struct flowi4 fl4;
341
342 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip,
343 peer_port, local_port, IPPROTO_TCP,
344 tos, 0);
345 if (IS_ERR(rt))
346 return NULL;
347 return rt;
348 }
349
350 static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
351 {
352 int i = 0;
353
354 while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
355 ++i;
356 return i;
357 }
358
359 static void arp_failure_discard(struct t3cdev *dev, struct sk_buff *skb)
360 {
361 PDBG("%s t3cdev %p\n", __func__, dev);
362 kfree_skb(skb);
363 }
364
365 /*
366 * Handle an ARP failure for an active open.
367 */
368 static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
369 {
370 printk(KERN_ERR MOD "ARP failure duing connect\n");
371 kfree_skb(skb);
372 }
373
374 /*
375 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant
376 * and send it along.
377 */
378 static void abort_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
379 {
380 struct cpl_abort_req *req = cplhdr(skb);
381
382 PDBG("%s t3cdev %p\n", __func__, dev);
383 req->cmd = CPL_ABORT_NO_RST;
384 iwch_cxgb3_ofld_send(dev, skb);
385 }
386
387 static int send_halfclose(struct iwch_ep *ep, gfp_t gfp)
388 {
389 struct cpl_close_con_req *req;
390 struct sk_buff *skb;
391
392 PDBG("%s ep %p\n", __func__, ep);
393 skb = get_skb(NULL, sizeof(*req), gfp);
394 if (!skb) {
395 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
396 return -ENOMEM;
397 }
398 skb->priority = CPL_PRIORITY_DATA;
399 set_arp_failure_handler(skb, arp_failure_discard);
400 req = (struct cpl_close_con_req *) skb_put(skb, sizeof(*req));
401 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
402 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
403 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, ep->hwtid));
404 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
405 }
406
407 static int send_abort(struct iwch_ep *ep, struct sk_buff *skb, gfp_t gfp)
408 {
409 struct cpl_abort_req *req;
410
411 PDBG("%s ep %p\n", __func__, ep);
412 skb = get_skb(skb, sizeof(*req), gfp);
413 if (!skb) {
414 printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
415 __func__);
416 return -ENOMEM;
417 }
418 skb->priority = CPL_PRIORITY_DATA;
419 set_arp_failure_handler(skb, abort_arp_failure);
420 req = (struct cpl_abort_req *) skb_put(skb, sizeof(*req));
421 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
422 req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
423 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid));
424 req->cmd = CPL_ABORT_SEND_RST;
425 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
426 }
427
428 static int send_connect(struct iwch_ep *ep)
429 {
430 struct cpl_act_open_req *req;
431 struct sk_buff *skb;
432 u32 opt0h, opt0l, opt2;
433 unsigned int mtu_idx;
434 int wscale;
435
436 PDBG("%s ep %p\n", __func__, ep);
437
438 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
439 if (!skb) {
440 printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
441 __func__);
442 return -ENOMEM;
443 }
444 mtu_idx = find_best_mtu(T3C_DATA(ep->com.tdev), dst_mtu(ep->dst));
445 wscale = compute_wscale(rcv_win);
446 opt0h = V_NAGLE(0) |
447 V_NO_CONG(nocong) |
448 V_KEEP_ALIVE(1) |
449 F_TCAM_BYPASS |
450 V_WND_SCALE(wscale) |
451 V_MSS_IDX(mtu_idx) |
452 V_L2T_IDX(ep->l2t->idx) | V_TX_CHANNEL(ep->l2t->smt_idx);
453 opt0l = V_TOS((ep->tos >> 2) & M_TOS) | V_RCV_BUFSIZ(rcv_win>>10);
454 opt2 = F_RX_COALESCE_VALID | V_RX_COALESCE(0) | V_FLAVORS_VALID(1) |
455 V_CONG_CONTROL_FLAVOR(cong_flavor);
456 skb->priority = CPL_PRIORITY_SETUP;
457 set_arp_failure_handler(skb, act_open_req_arp_failure);
458
459 req = (struct cpl_act_open_req *) skb_put(skb, sizeof(*req));
460 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
461 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, ep->atid));
462 req->local_port = ep->com.local_addr.sin_port;
463 req->peer_port = ep->com.remote_addr.sin_port;
464 req->local_ip = ep->com.local_addr.sin_addr.s_addr;
465 req->peer_ip = ep->com.remote_addr.sin_addr.s_addr;
466 req->opt0h = htonl(opt0h);
467 req->opt0l = htonl(opt0l);
468 req->params = 0;
469 req->opt2 = htonl(opt2);
470 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
471 }
472
473 static void send_mpa_req(struct iwch_ep *ep, struct sk_buff *skb)
474 {
475 int mpalen;
476 struct tx_data_wr *req;
477 struct mpa_message *mpa;
478 int len;
479
480 PDBG("%s ep %p pd_len %d\n", __func__, ep, ep->plen);
481
482 BUG_ON(skb_cloned(skb));
483
484 mpalen = sizeof(*mpa) + ep->plen;
485 if (skb->data + mpalen + sizeof(*req) > skb_end_pointer(skb)) {
486 kfree_skb(skb);
487 skb=alloc_skb(mpalen + sizeof(*req), GFP_KERNEL);
488 if (!skb) {
489 connect_reply_upcall(ep, -ENOMEM);
490 return;
491 }
492 }
493 skb_trim(skb, 0);
494 skb_reserve(skb, sizeof(*req));
495 skb_put(skb, mpalen);
496 skb->priority = CPL_PRIORITY_DATA;
497 mpa = (struct mpa_message *) skb->data;
498 memset(mpa, 0, sizeof(*mpa));
499 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key));
500 mpa->flags = (crc_enabled ? MPA_CRC : 0) |
501 (markers_enabled ? MPA_MARKERS : 0);
502 mpa->private_data_size = htons(ep->plen);
503 mpa->revision = mpa_rev;
504
505 if (ep->plen)
506 memcpy(mpa->private_data, ep->mpa_pkt + sizeof(*mpa), ep->plen);
507
508 /*
509 * Reference the mpa skb. This ensures the data area
510 * will remain in memory until the hw acks the tx.
511 * Function tx_ack() will deref it.
512 */
513 skb_get(skb);
514 set_arp_failure_handler(skb, arp_failure_discard);
515 skb_reset_transport_header(skb);
516 len = skb->len;
517 req = (struct tx_data_wr *) skb_push(skb, sizeof(*req));
518 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);
519 req->wr_lo = htonl(V_WR_TID(ep->hwtid));
520 req->len = htonl(len);
521 req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) |
522 V_TX_SNDBUF(snd_win>>15));
523 req->flags = htonl(F_TX_INIT);
524 req->sndseq = htonl(ep->snd_seq);
525 BUG_ON(ep->mpa_skb);
526 ep->mpa_skb = skb;
527 iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
528 start_ep_timer(ep);
529 state_set(&ep->com, MPA_REQ_SENT);
530 return;
531 }
532
533 static int send_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen)
534 {
535 int mpalen;
536 struct tx_data_wr *req;
537 struct mpa_message *mpa;
538 struct sk_buff *skb;
539
540 PDBG("%s ep %p plen %d\n", __func__, ep, plen);
541
542 mpalen = sizeof(*mpa) + plen;
543
544 skb = get_skb(NULL, mpalen + sizeof(*req), GFP_KERNEL);
545 if (!skb) {
546 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
547 return -ENOMEM;
548 }
549 skb_reserve(skb, sizeof(*req));
550 mpa = (struct mpa_message *) skb_put(skb, mpalen);
551 memset(mpa, 0, sizeof(*mpa));
552 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
553 mpa->flags = MPA_REJECT;
554 mpa->revision = mpa_rev;
555 mpa->private_data_size = htons(plen);
556 if (plen)
557 memcpy(mpa->private_data, pdata, plen);
558
559 /*
560 * Reference the mpa skb again. This ensures the data area
561 * will remain in memory until the hw acks the tx.
562 * Function tx_ack() will deref it.
563 */
564 skb_get(skb);
565 skb->priority = CPL_PRIORITY_DATA;
566 set_arp_failure_handler(skb, arp_failure_discard);
567 skb_reset_transport_header(skb);
568 req = (struct tx_data_wr *) skb_push(skb, sizeof(*req));
569 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);
570 req->wr_lo = htonl(V_WR_TID(ep->hwtid));
571 req->len = htonl(mpalen);
572 req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) |
573 V_TX_SNDBUF(snd_win>>15));
574 req->flags = htonl(F_TX_INIT);
575 req->sndseq = htonl(ep->snd_seq);
576 BUG_ON(ep->mpa_skb);
577 ep->mpa_skb = skb;
578 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
579 }
580
581 static int send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen)
582 {
583 int mpalen;
584 struct tx_data_wr *req;
585 struct mpa_message *mpa;
586 int len;
587 struct sk_buff *skb;
588
589 PDBG("%s ep %p plen %d\n", __func__, ep, plen);
590
591 mpalen = sizeof(*mpa) + plen;
592
593 skb = get_skb(NULL, mpalen + sizeof(*req), GFP_KERNEL);
594 if (!skb) {
595 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
596 return -ENOMEM;
597 }
598 skb->priority = CPL_PRIORITY_DATA;
599 skb_reserve(skb, sizeof(*req));
600 mpa = (struct mpa_message *) skb_put(skb, mpalen);
601 memset(mpa, 0, sizeof(*mpa));
602 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key));
603 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) |
604 (markers_enabled ? MPA_MARKERS : 0);
605 mpa->revision = mpa_rev;
606 mpa->private_data_size = htons(plen);
607 if (plen)
608 memcpy(mpa->private_data, pdata, plen);
609
610 /*
611 * Reference the mpa skb. This ensures the data area
612 * will remain in memory until the hw acks the tx.
613 * Function tx_ack() will deref it.
614 */
615 skb_get(skb);
616 set_arp_failure_handler(skb, arp_failure_discard);
617 skb_reset_transport_header(skb);
618 len = skb->len;
619 req = (struct tx_data_wr *) skb_push(skb, sizeof(*req));
620 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);
621 req->wr_lo = htonl(V_WR_TID(ep->hwtid));
622 req->len = htonl(len);
623 req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) |
624 V_TX_SNDBUF(snd_win>>15));
625 req->flags = htonl(F_TX_INIT);
626 req->sndseq = htonl(ep->snd_seq);
627 ep->mpa_skb = skb;
628 state_set(&ep->com, MPA_REP_SENT);
629 return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
630 }
631
632 static int act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
633 {
634 struct iwch_ep *ep = ctx;
635 struct cpl_act_establish *req = cplhdr(skb);
636 unsigned int tid = GET_TID(req);
637
638 PDBG("%s ep %p tid %d\n", __func__, ep, tid);
639
640 dst_confirm(ep->dst);
641
642 /* setup the hwtid for this connection */
643 ep->hwtid = tid;
644 cxgb3_insert_tid(ep->com.tdev, &t3c_client, ep, tid);
645
646 ep->snd_seq = ntohl(req->snd_isn);
647 ep->rcv_seq = ntohl(req->rcv_isn);
648
649 set_emss(ep, ntohs(req->tcp_opt));
650
651 /* dealloc the atid */
652 cxgb3_free_atid(ep->com.tdev, ep->atid);
653
654 /* start MPA negotiation */
655 send_mpa_req(ep, skb);
656
657 return 0;
658 }
659
660 static void abort_connection(struct iwch_ep *ep, struct sk_buff *skb, gfp_t gfp)
661 {
662 PDBG("%s ep %p\n", __FILE__, ep);
663 state_set(&ep->com, ABORTING);
664 send_abort(ep, skb, gfp);
665 }
666
667 static void close_complete_upcall(struct iwch_ep *ep)
668 {
669 struct iw_cm_event event;
670
671 PDBG("%s ep %p\n", __func__, ep);
672 memset(&event, 0, sizeof(event));
673 event.event = IW_CM_EVENT_CLOSE;
674 if (ep->com.cm_id) {
675 PDBG("close complete delivered ep %p cm_id %p tid %d\n",
676 ep, ep->com.cm_id, ep->hwtid);
677 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
678 ep->com.cm_id->rem_ref(ep->com.cm_id);
679 ep->com.cm_id = NULL;
680 ep->com.qp = NULL;
681 }
682 }
683
684 static void peer_close_upcall(struct iwch_ep *ep)
685 {
686 struct iw_cm_event event;
687
688 PDBG("%s ep %p\n", __func__, ep);
689 memset(&event, 0, sizeof(event));
690 event.event = IW_CM_EVENT_DISCONNECT;
691 if (ep->com.cm_id) {
692 PDBG("peer close delivered ep %p cm_id %p tid %d\n",
693 ep, ep->com.cm_id, ep->hwtid);
694 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
695 }
696 }
697
698 static void peer_abort_upcall(struct iwch_ep *ep)
699 {
700 struct iw_cm_event event;
701
702 PDBG("%s ep %p\n", __func__, ep);
703 memset(&event, 0, sizeof(event));
704 event.event = IW_CM_EVENT_CLOSE;
705 event.status = -ECONNRESET;
706 if (ep->com.cm_id) {
707 PDBG("abort delivered ep %p cm_id %p tid %d\n", ep,
708 ep->com.cm_id, ep->hwtid);
709 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
710 ep->com.cm_id->rem_ref(ep->com.cm_id);
711 ep->com.cm_id = NULL;
712 ep->com.qp = NULL;
713 }
714 }
715
716 static void connect_reply_upcall(struct iwch_ep *ep, int status)
717 {
718 struct iw_cm_event event;
719
720 PDBG("%s ep %p status %d\n", __func__, ep, status);
721 memset(&event, 0, sizeof(event));
722 event.event = IW_CM_EVENT_CONNECT_REPLY;
723 event.status = status;
724 event.local_addr = ep->com.local_addr;
725 event.remote_addr = ep->com.remote_addr;
726
727 if ((status == 0) || (status == -ECONNREFUSED)) {
728 event.private_data_len = ep->plen;
729 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
730 }
731 if (ep->com.cm_id) {
732 PDBG("%s ep %p tid %d status %d\n", __func__, ep,
733 ep->hwtid, status);
734 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
735 }
736 if (status < 0) {
737 ep->com.cm_id->rem_ref(ep->com.cm_id);
738 ep->com.cm_id = NULL;
739 ep->com.qp = NULL;
740 }
741 }
742
743 static void connect_request_upcall(struct iwch_ep *ep)
744 {
745 struct iw_cm_event event;
746
747 PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid);
748 memset(&event, 0, sizeof(event));
749 event.event = IW_CM_EVENT_CONNECT_REQUEST;
750 event.local_addr = ep->com.local_addr;
751 event.remote_addr = ep->com.remote_addr;
752 event.private_data_len = ep->plen;
753 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
754 event.provider_data = ep;
755 /*
756 * Until ird/ord negotiation via MPAv2 support is added, send max
757 * supported values
758 */
759 event.ird = event.ord = 8;
760 if (state_read(&ep->parent_ep->com) != DEAD) {
761 get_ep(&ep->com);
762 ep->parent_ep->com.cm_id->event_handler(
763 ep->parent_ep->com.cm_id,
764 &event);
765 }
766 put_ep(&ep->parent_ep->com);
767 ep->parent_ep = NULL;
768 }
769
770 static void established_upcall(struct iwch_ep *ep)
771 {
772 struct iw_cm_event event;
773
774 PDBG("%s ep %p\n", __func__, ep);
775 memset(&event, 0, sizeof(event));
776 event.event = IW_CM_EVENT_ESTABLISHED;
777 /*
778 * Until ird/ord negotiation via MPAv2 support is added, send max
779 * supported values
780 */
781 event.ird = event.ord = 8;
782 if (ep->com.cm_id) {
783 PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid);
784 ep->com.cm_id->event_handler(ep->com.cm_id, &event);
785 }
786 }
787
788 static int update_rx_credits(struct iwch_ep *ep, u32 credits)
789 {
790 struct cpl_rx_data_ack *req;
791 struct sk_buff *skb;
792
793 PDBG("%s ep %p credits %u\n", __func__, ep, credits);
794 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
795 if (!skb) {
796 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n");
797 return 0;
798 }
799
800 req = (struct cpl_rx_data_ack *) skb_put(skb, sizeof(*req));
801 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
802 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, ep->hwtid));
803 req->credit_dack = htonl(V_RX_CREDITS(credits) | V_RX_FORCE_ACK(1));
804 skb->priority = CPL_PRIORITY_ACK;
805 iwch_cxgb3_ofld_send(ep->com.tdev, skb);
806 return credits;
807 }
808
809 static void process_mpa_reply(struct iwch_ep *ep, struct sk_buff *skb)
810 {
811 struct mpa_message *mpa;
812 u16 plen;
813 struct iwch_qp_attributes attrs;
814 enum iwch_qp_attr_mask mask;
815 int err;
816
817 PDBG("%s ep %p\n", __func__, ep);
818
819 /*
820 * Stop mpa timer. If it expired, then the state has
821 * changed and we bail since ep_timeout already aborted
822 * the connection.
823 */
824 stop_ep_timer(ep);
825 if (state_read(&ep->com) != MPA_REQ_SENT)
826 return;
827
828 /*
829 * If we get more than the supported amount of private data
830 * then we must fail this connection.
831 */
832 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) {
833 err = -EINVAL;
834 goto err;
835 }
836
837 /*
838 * copy the new data into our accumulation buffer.
839 */
840 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]),
841 skb->len);
842 ep->mpa_pkt_len += skb->len;
843
844 /*
845 * if we don't even have the mpa message, then bail.
846 */
847 if (ep->mpa_pkt_len < sizeof(*mpa))
848 return;
849 mpa = (struct mpa_message *) ep->mpa_pkt;
850
851 /* Validate MPA header. */
852 if (mpa->revision != mpa_rev) {
853 err = -EPROTO;
854 goto err;
855 }
856 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) {
857 err = -EPROTO;
858 goto err;
859 }
860
861 plen = ntohs(mpa->private_data_size);
862
863 /*
864 * Fail if there's too much private data.
865 */
866 if (plen > MPA_MAX_PRIVATE_DATA) {
867 err = -EPROTO;
868 goto err;
869 }
870
871 /*
872 * If plen does not account for pkt size
873 */
874 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
875 err = -EPROTO;
876 goto err;
877 }
878
879 ep->plen = (u8) plen;
880
881 /*
882 * If we don't have all the pdata yet, then bail.
883 * We'll continue process when more data arrives.
884 */
885 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
886 return;
887
888 if (mpa->flags & MPA_REJECT) {
889 err = -ECONNREFUSED;
890 goto err;
891 }
892
893 /*
894 * If we get here we have accumulated the entire mpa
895 * start reply message including private data. And
896 * the MPA header is valid.
897 */
898 state_set(&ep->com, FPDU_MODE);
899 ep->mpa_attr.initiator = 1;
900 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
901 ep->mpa_attr.recv_marker_enabled = markers_enabled;
902 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
903 ep->mpa_attr.version = mpa_rev;
904 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
905 "xmit_marker_enabled=%d, version=%d\n", __func__,
906 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
907 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
908
909 attrs.mpa_attr = ep->mpa_attr;
910 attrs.max_ird = ep->ird;
911 attrs.max_ord = ep->ord;
912 attrs.llp_stream_handle = ep;
913 attrs.next_state = IWCH_QP_STATE_RTS;
914
915 mask = IWCH_QP_ATTR_NEXT_STATE |
916 IWCH_QP_ATTR_LLP_STREAM_HANDLE | IWCH_QP_ATTR_MPA_ATTR |
917 IWCH_QP_ATTR_MAX_IRD | IWCH_QP_ATTR_MAX_ORD;
918
919 /* bind QP and TID with INIT_WR */
920 err = iwch_modify_qp(ep->com.qp->rhp,
921 ep->com.qp, mask, &attrs, 1);
922 if (err)
923 goto err;
924
925 if (peer2peer && iwch_rqes_posted(ep->com.qp) == 0) {
926 iwch_post_zb_read(ep);
927 }
928
929 goto out;
930 err:
931 abort_connection(ep, skb, GFP_KERNEL);
932 out:
933 connect_reply_upcall(ep, err);
934 return;
935 }
936
937 static void process_mpa_request(struct iwch_ep *ep, struct sk_buff *skb)
938 {
939 struct mpa_message *mpa;
940 u16 plen;
941
942 PDBG("%s ep %p\n", __func__, ep);
943
944 /*
945 * Stop mpa timer. If it expired, then the state has
946 * changed and we bail since ep_timeout already aborted
947 * the connection.
948 */
949 stop_ep_timer(ep);
950 if (state_read(&ep->com) != MPA_REQ_WAIT)
951 return;
952
953 /*
954 * If we get more than the supported amount of private data
955 * then we must fail this connection.
956 */
957 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) {
958 abort_connection(ep, skb, GFP_KERNEL);
959 return;
960 }
961
962 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);
963
964 /*
965 * Copy the new data into our accumulation buffer.
966 */
967 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]),
968 skb->len);
969 ep->mpa_pkt_len += skb->len;
970
971 /*
972 * If we don't even have the mpa message, then bail.
973 * We'll continue process when more data arrives.
974 */
975 if (ep->mpa_pkt_len < sizeof(*mpa))
976 return;
977 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);
978 mpa = (struct mpa_message *) ep->mpa_pkt;
979
980 /*
981 * Validate MPA Header.
982 */
983 if (mpa->revision != mpa_rev) {
984 abort_connection(ep, skb, GFP_KERNEL);
985 return;
986 }
987
988 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) {
989 abort_connection(ep, skb, GFP_KERNEL);
990 return;
991 }
992
993 plen = ntohs(mpa->private_data_size);
994
995 /*
996 * Fail if there's too much private data.
997 */
998 if (plen > MPA_MAX_PRIVATE_DATA) {
999 abort_connection(ep, skb, GFP_KERNEL);
1000 return;
1001 }
1002
1003 /*
1004 * If plen does not account for pkt size
1005 */
1006 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) {
1007 abort_connection(ep, skb, GFP_KERNEL);
1008 return;
1009 }
1010 ep->plen = (u8) plen;
1011
1012 /*
1013 * If we don't have all the pdata yet, then bail.
1014 */
1015 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen))
1016 return;
1017
1018 /*
1019 * If we get here we have accumulated the entire mpa
1020 * start reply message including private data.
1021 */
1022 ep->mpa_attr.initiator = 0;
1023 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
1024 ep->mpa_attr.recv_marker_enabled = markers_enabled;
1025 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
1026 ep->mpa_attr.version = mpa_rev;
1027 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
1028 "xmit_marker_enabled=%d, version=%d\n", __func__,
1029 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
1030 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
1031
1032 state_set(&ep->com, MPA_REQ_RCVD);
1033
1034 /* drive upcall */
1035 connect_request_upcall(ep);
1036 return;
1037 }
1038
1039 static int rx_data(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1040 {
1041 struct iwch_ep *ep = ctx;
1042 struct cpl_rx_data *hdr = cplhdr(skb);
1043 unsigned int dlen = ntohs(hdr->len);
1044
1045 PDBG("%s ep %p dlen %u\n", __func__, ep, dlen);
1046
1047 skb_pull(skb, sizeof(*hdr));
1048 skb_trim(skb, dlen);
1049
1050 ep->rcv_seq += dlen;
1051 BUG_ON(ep->rcv_seq != (ntohl(hdr->seq) + dlen));
1052
1053 switch (state_read(&ep->com)) {
1054 case MPA_REQ_SENT:
1055 process_mpa_reply(ep, skb);
1056 break;
1057 case MPA_REQ_WAIT:
1058 process_mpa_request(ep, skb);
1059 break;
1060 case MPA_REP_SENT:
1061 break;
1062 default:
1063 printk(KERN_ERR MOD "%s Unexpected streaming data."
1064 " ep %p state %d tid %d\n",
1065 __func__, ep, state_read(&ep->com), ep->hwtid);
1066
1067 /*
1068 * The ep will timeout and inform the ULP of the failure.
1069 * See ep_timeout().
1070 */
1071 break;
1072 }
1073
1074 /* update RX credits */
1075 update_rx_credits(ep, dlen);
1076
1077 return CPL_RET_BUF_DONE;
1078 }
1079
1080 /*
1081 * Upcall from the adapter indicating data has been transmitted.
1082 * For us its just the single MPA request or reply. We can now free
1083 * the skb holding the mpa message.
1084 */
1085 static int tx_ack(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1086 {
1087 struct iwch_ep *ep = ctx;
1088 struct cpl_wr_ack *hdr = cplhdr(skb);
1089 unsigned int credits = ntohs(hdr->credits);
1090 unsigned long flags;
1091 int post_zb = 0;
1092
1093 PDBG("%s ep %p credits %u\n", __func__, ep, credits);
1094
1095 if (credits == 0) {
1096 PDBG("%s 0 credit ack ep %p state %u\n",
1097 __func__, ep, state_read(&ep->com));
1098 return CPL_RET_BUF_DONE;
1099 }
1100
1101 spin_lock_irqsave(&ep->com.lock, flags);
1102 BUG_ON(credits != 1);
1103 dst_confirm(ep->dst);
1104 if (!ep->mpa_skb) {
1105 PDBG("%s rdma_init wr_ack ep %p state %u\n",
1106 __func__, ep, ep->com.state);
1107 if (ep->mpa_attr.initiator) {
1108 PDBG("%s initiator ep %p state %u\n",
1109 __func__, ep, ep->com.state);
1110 if (peer2peer && ep->com.state == FPDU_MODE)
1111 post_zb = 1;
1112 } else {
1113 PDBG("%s responder ep %p state %u\n",
1114 __func__, ep, ep->com.state);
1115 if (ep->com.state == MPA_REQ_RCVD) {
1116 ep->com.rpl_done = 1;
1117 wake_up(&ep->com.waitq);
1118 }
1119 }
1120 } else {
1121 PDBG("%s lsm ack ep %p state %u freeing skb\n",
1122 __func__, ep, ep->com.state);
1123 kfree_skb(ep->mpa_skb);
1124 ep->mpa_skb = NULL;
1125 }
1126 spin_unlock_irqrestore(&ep->com.lock, flags);
1127 if (post_zb)
1128 iwch_post_zb_read(ep);
1129 return CPL_RET_BUF_DONE;
1130 }
1131
1132 static int abort_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1133 {
1134 struct iwch_ep *ep = ctx;
1135 unsigned long flags;
1136 int release = 0;
1137
1138 PDBG("%s ep %p\n", __func__, ep);
1139 BUG_ON(!ep);
1140
1141 /*
1142 * We get 2 abort replies from the HW. The first one must
1143 * be ignored except for scribbling that we need one more.
1144 */
1145 if (!test_and_set_bit(ABORT_REQ_IN_PROGRESS, &ep->com.flags)) {
1146 return CPL_RET_BUF_DONE;
1147 }
1148
1149 spin_lock_irqsave(&ep->com.lock, flags);
1150 switch (ep->com.state) {
1151 case ABORTING:
1152 close_complete_upcall(ep);
1153 __state_set(&ep->com, DEAD);
1154 release = 1;
1155 break;
1156 default:
1157 printk(KERN_ERR "%s ep %p state %d\n",
1158 __func__, ep, ep->com.state);
1159 break;
1160 }
1161 spin_unlock_irqrestore(&ep->com.lock, flags);
1162
1163 if (release)
1164 release_ep_resources(ep);
1165 return CPL_RET_BUF_DONE;
1166 }
1167
1168 /*
1169 * Return whether a failed active open has allocated a TID
1170 */
1171 static inline int act_open_has_tid(int status)
1172 {
1173 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST &&
1174 status != CPL_ERR_ARP_MISS;
1175 }
1176
1177 static int act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1178 {
1179 struct iwch_ep *ep = ctx;
1180 struct cpl_act_open_rpl *rpl = cplhdr(skb);
1181
1182 PDBG("%s ep %p status %u errno %d\n", __func__, ep, rpl->status,
1183 status2errno(rpl->status));
1184 connect_reply_upcall(ep, status2errno(rpl->status));
1185 state_set(&ep->com, DEAD);
1186 if (ep->com.tdev->type != T3A && act_open_has_tid(rpl->status))
1187 release_tid(ep->com.tdev, GET_TID(rpl), NULL);
1188 cxgb3_free_atid(ep->com.tdev, ep->atid);
1189 dst_release(ep->dst);
1190 l2t_release(ep->com.tdev, ep->l2t);
1191 put_ep(&ep->com);
1192 return CPL_RET_BUF_DONE;
1193 }
1194
1195 static int listen_start(struct iwch_listen_ep *ep)
1196 {
1197 struct sk_buff *skb;
1198 struct cpl_pass_open_req *req;
1199
1200 PDBG("%s ep %p\n", __func__, ep);
1201 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
1202 if (!skb) {
1203 printk(KERN_ERR MOD "t3c_listen_start failed to alloc skb!\n");
1204 return -ENOMEM;
1205 }
1206
1207 req = (struct cpl_pass_open_req *) skb_put(skb, sizeof(*req));
1208 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1209 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, ep->stid));
1210 req->local_port = ep->com.local_addr.sin_port;
1211 req->local_ip = ep->com.local_addr.sin_addr.s_addr;
1212 req->peer_port = 0;
1213 req->peer_ip = 0;
1214 req->peer_netmask = 0;
1215 req->opt0h = htonl(F_DELACK | F_TCAM_BYPASS);
1216 req->opt0l = htonl(V_RCV_BUFSIZ(rcv_win>>10));
1217 req->opt1 = htonl(V_CONN_POLICY(CPL_CONN_POLICY_ASK));
1218
1219 skb->priority = 1;
1220 return iwch_cxgb3_ofld_send(ep->com.tdev, skb);
1221 }
1222
1223 static int pass_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1224 {
1225 struct iwch_listen_ep *ep = ctx;
1226 struct cpl_pass_open_rpl *rpl = cplhdr(skb);
1227
1228 PDBG("%s ep %p status %d error %d\n", __func__, ep,
1229 rpl->status, status2errno(rpl->status));
1230 ep->com.rpl_err = status2errno(rpl->status);
1231 ep->com.rpl_done = 1;
1232 wake_up(&ep->com.waitq);
1233
1234 return CPL_RET_BUF_DONE;
1235 }
1236
1237 static int listen_stop(struct iwch_listen_ep *ep)
1238 {
1239 struct sk_buff *skb;
1240 struct cpl_close_listserv_req *req;
1241
1242 PDBG("%s ep %p\n", __func__, ep);
1243 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
1244 if (!skb) {
1245 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
1246 return -ENOMEM;
1247 }
1248 req = (struct cpl_close_listserv_req *) skb_put(skb, sizeof(*req));
1249 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1250 req->cpu_idx = 0;
1251 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, ep->stid));
1252 skb->priority = 1;
1253 return iwch_cxgb3_ofld_send(ep->com.tdev, skb);
1254 }
1255
1256 static int close_listsrv_rpl(struct t3cdev *tdev, struct sk_buff *skb,
1257 void *ctx)
1258 {
1259 struct iwch_listen_ep *ep = ctx;
1260 struct cpl_close_listserv_rpl *rpl = cplhdr(skb);
1261
1262 PDBG("%s ep %p\n", __func__, ep);
1263 ep->com.rpl_err = status2errno(rpl->status);
1264 ep->com.rpl_done = 1;
1265 wake_up(&ep->com.waitq);
1266 return CPL_RET_BUF_DONE;
1267 }
1268
1269 static void accept_cr(struct iwch_ep *ep, __be32 peer_ip, struct sk_buff *skb)
1270 {
1271 struct cpl_pass_accept_rpl *rpl;
1272 unsigned int mtu_idx;
1273 u32 opt0h, opt0l, opt2;
1274 int wscale;
1275
1276 PDBG("%s ep %p\n", __func__, ep);
1277 BUG_ON(skb_cloned(skb));
1278 skb_trim(skb, sizeof(*rpl));
1279 skb_get(skb);
1280 mtu_idx = find_best_mtu(T3C_DATA(ep->com.tdev), dst_mtu(ep->dst));
1281 wscale = compute_wscale(rcv_win);
1282 opt0h = V_NAGLE(0) |
1283 V_NO_CONG(nocong) |
1284 V_KEEP_ALIVE(1) |
1285 F_TCAM_BYPASS |
1286 V_WND_SCALE(wscale) |
1287 V_MSS_IDX(mtu_idx) |
1288 V_L2T_IDX(ep->l2t->idx) | V_TX_CHANNEL(ep->l2t->smt_idx);
1289 opt0l = V_TOS((ep->tos >> 2) & M_TOS) | V_RCV_BUFSIZ(rcv_win>>10);
1290 opt2 = F_RX_COALESCE_VALID | V_RX_COALESCE(0) | V_FLAVORS_VALID(1) |
1291 V_CONG_CONTROL_FLAVOR(cong_flavor);
1292
1293 rpl = cplhdr(skb);
1294 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1295 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, ep->hwtid));
1296 rpl->peer_ip = peer_ip;
1297 rpl->opt0h = htonl(opt0h);
1298 rpl->opt0l_status = htonl(opt0l | CPL_PASS_OPEN_ACCEPT);
1299 rpl->opt2 = htonl(opt2);
1300 rpl->rsvd = rpl->opt2; /* workaround for HW bug */
1301 skb->priority = CPL_PRIORITY_SETUP;
1302 iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
1303
1304 return;
1305 }
1306
1307 static void reject_cr(struct t3cdev *tdev, u32 hwtid, __be32 peer_ip,
1308 struct sk_buff *skb)
1309 {
1310 PDBG("%s t3cdev %p tid %u peer_ip %x\n", __func__, tdev, hwtid,
1311 peer_ip);
1312 BUG_ON(skb_cloned(skb));
1313 skb_trim(skb, sizeof(struct cpl_tid_release));
1314 skb_get(skb);
1315
1316 if (tdev->type != T3A)
1317 release_tid(tdev, hwtid, skb);
1318 else {
1319 struct cpl_pass_accept_rpl *rpl;
1320
1321 rpl = cplhdr(skb);
1322 skb->priority = CPL_PRIORITY_SETUP;
1323 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1324 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
1325 hwtid));
1326 rpl->peer_ip = peer_ip;
1327 rpl->opt0h = htonl(F_TCAM_BYPASS);
1328 rpl->opt0l_status = htonl(CPL_PASS_OPEN_REJECT);
1329 rpl->opt2 = 0;
1330 rpl->rsvd = rpl->opt2;
1331 iwch_cxgb3_ofld_send(tdev, skb);
1332 }
1333 }
1334
1335 static int pass_accept_req(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1336 {
1337 struct iwch_ep *child_ep, *parent_ep = ctx;
1338 struct cpl_pass_accept_req *req = cplhdr(skb);
1339 unsigned int hwtid = GET_TID(req);
1340 struct dst_entry *dst;
1341 struct l2t_entry *l2t;
1342 struct rtable *rt;
1343 struct iff_mac tim;
1344
1345 PDBG("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid);
1346
1347 if (state_read(&parent_ep->com) != LISTEN) {
1348 printk(KERN_ERR "%s - listening ep not in LISTEN\n",
1349 __func__);
1350 goto reject;
1351 }
1352
1353 /*
1354 * Find the netdev for this connection request.
1355 */
1356 tim.mac_addr = req->dst_mac;
1357 tim.vlan_tag = ntohs(req->vlan_tag);
1358 if (tdev->ctl(tdev, GET_IFF_FROM_MAC, &tim) < 0 || !tim.dev) {
1359 printk(KERN_ERR "%s bad dst mac %pM\n",
1360 __func__, req->dst_mac);
1361 goto reject;
1362 }
1363
1364 /* Find output route */
1365 rt = find_route(tdev,
1366 req->local_ip,
1367 req->peer_ip,
1368 req->local_port,
1369 req->peer_port, G_PASS_OPEN_TOS(ntohl(req->tos_tid)));
1370 if (!rt) {
1371 printk(KERN_ERR MOD "%s - failed to find dst entry!\n",
1372 __func__);
1373 goto reject;
1374 }
1375 dst = &rt->dst;
1376 l2t = t3_l2t_get(tdev, dst, NULL, &req->peer_ip);
1377 if (!l2t) {
1378 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n",
1379 __func__);
1380 dst_release(dst);
1381 goto reject;
1382 }
1383 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL);
1384 if (!child_ep) {
1385 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n",
1386 __func__);
1387 l2t_release(tdev, l2t);
1388 dst_release(dst);
1389 goto reject;
1390 }
1391 state_set(&child_ep->com, CONNECTING);
1392 child_ep->com.tdev = tdev;
1393 child_ep->com.cm_id = NULL;
1394 child_ep->com.local_addr.sin_family = PF_INET;
1395 child_ep->com.local_addr.sin_port = req->local_port;
1396 child_ep->com.local_addr.sin_addr.s_addr = req->local_ip;
1397 child_ep->com.remote_addr.sin_family = PF_INET;
1398 child_ep->com.remote_addr.sin_port = req->peer_port;
1399 child_ep->com.remote_addr.sin_addr.s_addr = req->peer_ip;
1400 get_ep(&parent_ep->com);
1401 child_ep->parent_ep = parent_ep;
1402 child_ep->tos = G_PASS_OPEN_TOS(ntohl(req->tos_tid));
1403 child_ep->l2t = l2t;
1404 child_ep->dst = dst;
1405 child_ep->hwtid = hwtid;
1406 init_timer(&child_ep->timer);
1407 cxgb3_insert_tid(tdev, &t3c_client, child_ep, hwtid);
1408 accept_cr(child_ep, req->peer_ip, skb);
1409 goto out;
1410 reject:
1411 reject_cr(tdev, hwtid, req->peer_ip, skb);
1412 out:
1413 return CPL_RET_BUF_DONE;
1414 }
1415
1416 static int pass_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1417 {
1418 struct iwch_ep *ep = ctx;
1419 struct cpl_pass_establish *req = cplhdr(skb);
1420
1421 PDBG("%s ep %p\n", __func__, ep);
1422 ep->snd_seq = ntohl(req->snd_isn);
1423 ep->rcv_seq = ntohl(req->rcv_isn);
1424
1425 set_emss(ep, ntohs(req->tcp_opt));
1426
1427 dst_confirm(ep->dst);
1428 state_set(&ep->com, MPA_REQ_WAIT);
1429 start_ep_timer(ep);
1430
1431 return CPL_RET_BUF_DONE;
1432 }
1433
1434 static int peer_close(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1435 {
1436 struct iwch_ep *ep = ctx;
1437 struct iwch_qp_attributes attrs;
1438 unsigned long flags;
1439 int disconnect = 1;
1440 int release = 0;
1441
1442 PDBG("%s ep %p\n", __func__, ep);
1443 dst_confirm(ep->dst);
1444
1445 spin_lock_irqsave(&ep->com.lock, flags);
1446 switch (ep->com.state) {
1447 case MPA_REQ_WAIT:
1448 __state_set(&ep->com, CLOSING);
1449 break;
1450 case MPA_REQ_SENT:
1451 __state_set(&ep->com, CLOSING);
1452 connect_reply_upcall(ep, -ECONNRESET);
1453 break;
1454 case MPA_REQ_RCVD:
1455
1456 /*
1457 * We're gonna mark this puppy DEAD, but keep
1458 * the reference on it until the ULP accepts or
1459 * rejects the CR. Also wake up anyone waiting
1460 * in rdma connection migration (see iwch_accept_cr()).
1461 */
1462 __state_set(&ep->com, CLOSING);
1463 ep->com.rpl_done = 1;
1464 ep->com.rpl_err = -ECONNRESET;
1465 PDBG("waking up ep %p\n", ep);
1466 wake_up(&ep->com.waitq);
1467 break;
1468 case MPA_REP_SENT:
1469 __state_set(&ep->com, CLOSING);
1470 ep->com.rpl_done = 1;
1471 ep->com.rpl_err = -ECONNRESET;
1472 PDBG("waking up ep %p\n", ep);
1473 wake_up(&ep->com.waitq);
1474 break;
1475 case FPDU_MODE:
1476 start_ep_timer(ep);
1477 __state_set(&ep->com, CLOSING);
1478 attrs.next_state = IWCH_QP_STATE_CLOSING;
1479 iwch_modify_qp(ep->com.qp->rhp, ep->com.qp,
1480 IWCH_QP_ATTR_NEXT_STATE, &attrs, 1);
1481 peer_close_upcall(ep);
1482 break;
1483 case ABORTING:
1484 disconnect = 0;
1485 break;
1486 case CLOSING:
1487 __state_set(&ep->com, MORIBUND);
1488 disconnect = 0;
1489 break;
1490 case MORIBUND:
1491 stop_ep_timer(ep);
1492 if (ep->com.cm_id && ep->com.qp) {
1493 attrs.next_state = IWCH_QP_STATE_IDLE;
1494 iwch_modify_qp(ep->com.qp->rhp, ep->com.qp,
1495 IWCH_QP_ATTR_NEXT_STATE, &attrs, 1);
1496 }
1497 close_complete_upcall(ep);
1498 __state_set(&ep->com, DEAD);
1499 release = 1;
1500 disconnect = 0;
1501 break;
1502 case DEAD:
1503 disconnect = 0;
1504 break;
1505 default:
1506 BUG_ON(1);
1507 }
1508 spin_unlock_irqrestore(&ep->com.lock, flags);
1509 if (disconnect)
1510 iwch_ep_disconnect(ep, 0, GFP_KERNEL);
1511 if (release)
1512 release_ep_resources(ep);
1513 return CPL_RET_BUF_DONE;
1514 }
1515
1516 /*
1517 * Returns whether an ABORT_REQ_RSS message is a negative advice.
1518 */
1519 static int is_neg_adv_abort(unsigned int status)
1520 {
1521 return status == CPL_ERR_RTX_NEG_ADVICE ||
1522 status == CPL_ERR_PERSIST_NEG_ADVICE;
1523 }
1524
1525 static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1526 {
1527 struct cpl_abort_req_rss *req = cplhdr(skb);
1528 struct iwch_ep *ep = ctx;
1529 struct cpl_abort_rpl *rpl;
1530 struct sk_buff *rpl_skb;
1531 struct iwch_qp_attributes attrs;
1532 int ret;
1533 int release = 0;
1534 unsigned long flags;
1535
1536 if (is_neg_adv_abort(req->status)) {
1537 PDBG("%s neg_adv_abort ep %p tid %d\n", __func__, ep,
1538 ep->hwtid);
1539 t3_l2t_send_event(ep->com.tdev, ep->l2t);
1540 return CPL_RET_BUF_DONE;
1541 }
1542
1543 /*
1544 * We get 2 peer aborts from the HW. The first one must
1545 * be ignored except for scribbling that we need one more.
1546 */
1547 if (!test_and_set_bit(PEER_ABORT_IN_PROGRESS, &ep->com.flags)) {
1548 return CPL_RET_BUF_DONE;
1549 }
1550
1551 spin_lock_irqsave(&ep->com.lock, flags);
1552 PDBG("%s ep %p state %u\n", __func__, ep, ep->com.state);
1553 switch (ep->com.state) {
1554 case CONNECTING:
1555 break;
1556 case MPA_REQ_WAIT:
1557 stop_ep_timer(ep);
1558 break;
1559 case MPA_REQ_SENT:
1560 stop_ep_timer(ep);
1561 connect_reply_upcall(ep, -ECONNRESET);
1562 break;
1563 case MPA_REP_SENT:
1564 ep->com.rpl_done = 1;
1565 ep->com.rpl_err = -ECONNRESET;
1566 PDBG("waking up ep %p\n", ep);
1567 wake_up(&ep->com.waitq);
1568 break;
1569 case MPA_REQ_RCVD:
1570
1571 /*
1572 * We're gonna mark this puppy DEAD, but keep
1573 * the reference on it until the ULP accepts or
1574 * rejects the CR. Also wake up anyone waiting
1575 * in rdma connection migration (see iwch_accept_cr()).
1576 */
1577 ep->com.rpl_done = 1;
1578 ep->com.rpl_err = -ECONNRESET;
1579 PDBG("waking up ep %p\n", ep);
1580 wake_up(&ep->com.waitq);
1581 break;
1582 case MORIBUND:
1583 case CLOSING:
1584 stop_ep_timer(ep);
1585 /*FALLTHROUGH*/
1586 case FPDU_MODE:
1587 if (ep->com.cm_id && ep->com.qp) {
1588 attrs.next_state = IWCH_QP_STATE_ERROR;
1589 ret = iwch_modify_qp(ep->com.qp->rhp,
1590 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
1591 &attrs, 1);
1592 if (ret)
1593 printk(KERN_ERR MOD
1594 "%s - qp <- error failed!\n",
1595 __func__);
1596 }
1597 peer_abort_upcall(ep);
1598 break;
1599 case ABORTING:
1600 break;
1601 case DEAD:
1602 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__);
1603 spin_unlock_irqrestore(&ep->com.lock, flags);
1604 return CPL_RET_BUF_DONE;
1605 default:
1606 BUG_ON(1);
1607 break;
1608 }
1609 dst_confirm(ep->dst);
1610 if (ep->com.state != ABORTING) {
1611 __state_set(&ep->com, DEAD);
1612 release = 1;
1613 }
1614 spin_unlock_irqrestore(&ep->com.lock, flags);
1615
1616 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL);
1617 if (!rpl_skb) {
1618 printk(KERN_ERR MOD "%s - cannot allocate skb!\n",
1619 __func__);
1620 release = 1;
1621 goto out;
1622 }
1623 rpl_skb->priority = CPL_PRIORITY_DATA;
1624 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl));
1625 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
1626 rpl->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
1627 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid));
1628 rpl->cmd = CPL_ABORT_NO_RST;
1629 iwch_cxgb3_ofld_send(ep->com.tdev, rpl_skb);
1630 out:
1631 if (release)
1632 release_ep_resources(ep);
1633 return CPL_RET_BUF_DONE;
1634 }
1635
1636 static int close_con_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1637 {
1638 struct iwch_ep *ep = ctx;
1639 struct iwch_qp_attributes attrs;
1640 unsigned long flags;
1641 int release = 0;
1642
1643 PDBG("%s ep %p\n", __func__, ep);
1644 BUG_ON(!ep);
1645
1646 /* The cm_id may be null if we failed to connect */
1647 spin_lock_irqsave(&ep->com.lock, flags);
1648 switch (ep->com.state) {
1649 case CLOSING:
1650 __state_set(&ep->com, MORIBUND);
1651 break;
1652 case MORIBUND:
1653 stop_ep_timer(ep);
1654 if ((ep->com.cm_id) && (ep->com.qp)) {
1655 attrs.next_state = IWCH_QP_STATE_IDLE;
1656 iwch_modify_qp(ep->com.qp->rhp,
1657 ep->com.qp,
1658 IWCH_QP_ATTR_NEXT_STATE,
1659 &attrs, 1);
1660 }
1661 close_complete_upcall(ep);
1662 __state_set(&ep->com, DEAD);
1663 release = 1;
1664 break;
1665 case ABORTING:
1666 case DEAD:
1667 break;
1668 default:
1669 BUG_ON(1);
1670 break;
1671 }
1672 spin_unlock_irqrestore(&ep->com.lock, flags);
1673 if (release)
1674 release_ep_resources(ep);
1675 return CPL_RET_BUF_DONE;
1676 }
1677
1678 /*
1679 * T3A does 3 things when a TERM is received:
1680 * 1) send up a CPL_RDMA_TERMINATE message with the TERM packet
1681 * 2) generate an async event on the QP with the TERMINATE opcode
1682 * 3) post a TERMINATE opcode cqe into the associated CQ.
1683 *
1684 * For (1), we save the message in the qp for later consumer consumption.
1685 * For (2), we move the QP into TERMINATE, post a QP event and disconnect.
1686 * For (3), we toss the CQE in cxio_poll_cq().
1687 *
1688 * terminate() handles case (1)...
1689 */
1690 static int terminate(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1691 {
1692 struct iwch_ep *ep = ctx;
1693
1694 if (state_read(&ep->com) != FPDU_MODE)
1695 return CPL_RET_BUF_DONE;
1696
1697 PDBG("%s ep %p\n", __func__, ep);
1698 skb_pull(skb, sizeof(struct cpl_rdma_terminate));
1699 PDBG("%s saving %d bytes of term msg\n", __func__, skb->len);
1700 skb_copy_from_linear_data(skb, ep->com.qp->attr.terminate_buffer,
1701 skb->len);
1702 ep->com.qp->attr.terminate_msg_len = skb->len;
1703 ep->com.qp->attr.is_terminate_local = 0;
1704 return CPL_RET_BUF_DONE;
1705 }
1706
1707 static int ec_status(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
1708 {
1709 struct cpl_rdma_ec_status *rep = cplhdr(skb);
1710 struct iwch_ep *ep = ctx;
1711
1712 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid,
1713 rep->status);
1714 if (rep->status) {
1715 struct iwch_qp_attributes attrs;
1716
1717 printk(KERN_ERR MOD "%s BAD CLOSE - Aborting tid %u\n",
1718 __func__, ep->hwtid);
1719 stop_ep_timer(ep);
1720 attrs.next_state = IWCH_QP_STATE_ERROR;
1721 iwch_modify_qp(ep->com.qp->rhp,
1722 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
1723 &attrs, 1);
1724 abort_connection(ep, NULL, GFP_KERNEL);
1725 }
1726 return CPL_RET_BUF_DONE;
1727 }
1728
1729 static void ep_timeout(unsigned long arg)
1730 {
1731 struct iwch_ep *ep = (struct iwch_ep *)arg;
1732 struct iwch_qp_attributes attrs;
1733 unsigned long flags;
1734 int abort = 1;
1735
1736 spin_lock_irqsave(&ep->com.lock, flags);
1737 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid,
1738 ep->com.state);
1739 switch (ep->com.state) {
1740 case MPA_REQ_SENT:
1741 __state_set(&ep->com, ABORTING);
1742 connect_reply_upcall(ep, -ETIMEDOUT);
1743 break;
1744 case MPA_REQ_WAIT:
1745 __state_set(&ep->com, ABORTING);
1746 break;
1747 case CLOSING:
1748 case MORIBUND:
1749 if (ep->com.cm_id && ep->com.qp) {
1750 attrs.next_state = IWCH_QP_STATE_ERROR;
1751 iwch_modify_qp(ep->com.qp->rhp,
1752 ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
1753 &attrs, 1);
1754 }
1755 __state_set(&ep->com, ABORTING);
1756 break;
1757 default:
1758 WARN(1, "%s unexpected state ep %p state %u\n",
1759 __func__, ep, ep->com.state);
1760 abort = 0;
1761 }
1762 spin_unlock_irqrestore(&ep->com.lock, flags);
1763 if (abort)
1764 abort_connection(ep, NULL, GFP_ATOMIC);
1765 put_ep(&ep->com);
1766 }
1767
1768 int iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
1769 {
1770 int err;
1771 struct iwch_ep *ep = to_ep(cm_id);
1772 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1773
1774 if (state_read(&ep->com) == DEAD) {
1775 put_ep(&ep->com);
1776 return -ECONNRESET;
1777 }
1778 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD);
1779 if (mpa_rev == 0)
1780 abort_connection(ep, NULL, GFP_KERNEL);
1781 else {
1782 err = send_mpa_reject(ep, pdata, pdata_len);
1783 err = iwch_ep_disconnect(ep, 0, GFP_KERNEL);
1784 }
1785 put_ep(&ep->com);
1786 return 0;
1787 }
1788
1789 int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1790 {
1791 int err;
1792 struct iwch_qp_attributes attrs;
1793 enum iwch_qp_attr_mask mask;
1794 struct iwch_ep *ep = to_ep(cm_id);
1795 struct iwch_dev *h = to_iwch_dev(cm_id->device);
1796 struct iwch_qp *qp = get_qhp(h, conn_param->qpn);
1797
1798 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
1799 if (state_read(&ep->com) == DEAD) {
1800 err = -ECONNRESET;
1801 goto err;
1802 }
1803
1804 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD);
1805 BUG_ON(!qp);
1806
1807 if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) ||
1808 (conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) {
1809 abort_connection(ep, NULL, GFP_KERNEL);
1810 err = -EINVAL;
1811 goto err;
1812 }
1813
1814 cm_id->add_ref(cm_id);
1815 ep->com.cm_id = cm_id;
1816 ep->com.qp = qp;
1817
1818 ep->ird = conn_param->ird;
1819 ep->ord = conn_param->ord;
1820
1821 if (peer2peer && ep->ird == 0)
1822 ep->ird = 1;
1823
1824 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord);
1825
1826 /* bind QP to EP and move to RTS */
1827 attrs.mpa_attr = ep->mpa_attr;
1828 attrs.max_ird = ep->ird;
1829 attrs.max_ord = ep->ord;
1830 attrs.llp_stream_handle = ep;
1831 attrs.next_state = IWCH_QP_STATE_RTS;
1832
1833 /* bind QP and TID with INIT_WR */
1834 mask = IWCH_QP_ATTR_NEXT_STATE |
1835 IWCH_QP_ATTR_LLP_STREAM_HANDLE |
1836 IWCH_QP_ATTR_MPA_ATTR |
1837 IWCH_QP_ATTR_MAX_IRD |
1838 IWCH_QP_ATTR_MAX_ORD;
1839
1840 err = iwch_modify_qp(ep->com.qp->rhp,
1841 ep->com.qp, mask, &attrs, 1);
1842 if (err)
1843 goto err1;
1844
1845 /* if needed, wait for wr_ack */
1846 if (iwch_rqes_posted(qp)) {
1847 wait_event(ep->com.waitq, ep->com.rpl_done);
1848 err = ep->com.rpl_err;
1849 if (err)
1850 goto err1;
1851 }
1852
1853 err = send_mpa_reply(ep, conn_param->private_data,
1854 conn_param->private_data_len);
1855 if (err)
1856 goto err1;
1857
1858
1859 state_set(&ep->com, FPDU_MODE);
1860 established_upcall(ep);
1861 put_ep(&ep->com);
1862 return 0;
1863 err1:
1864 ep->com.cm_id = NULL;
1865 ep->com.qp = NULL;
1866 cm_id->rem_ref(cm_id);
1867 err:
1868 put_ep(&ep->com);
1869 return err;
1870 }
1871
1872 static int is_loopback_dst(struct iw_cm_id *cm_id)
1873 {
1874 struct net_device *dev;
1875
1876 dev = ip_dev_find(&init_net, cm_id->remote_addr.sin_addr.s_addr);
1877 if (!dev)
1878 return 0;
1879 dev_put(dev);
1880 return 1;
1881 }
1882
1883 int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
1884 {
1885 struct iwch_dev *h = to_iwch_dev(cm_id->device);
1886 struct iwch_ep *ep;
1887 struct rtable *rt;
1888 int err = 0;
1889
1890 if (is_loopback_dst(cm_id)) {
1891 err = -ENOSYS;
1892 goto out;
1893 }
1894
1895 ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
1896 if (!ep) {
1897 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__);
1898 err = -ENOMEM;
1899 goto out;
1900 }
1901 init_timer(&ep->timer);
1902 ep->plen = conn_param->private_data_len;
1903 if (ep->plen)
1904 memcpy(ep->mpa_pkt + sizeof(struct mpa_message),
1905 conn_param->private_data, ep->plen);
1906 ep->ird = conn_param->ird;
1907 ep->ord = conn_param->ord;
1908
1909 if (peer2peer && ep->ord == 0)
1910 ep->ord = 1;
1911
1912 ep->com.tdev = h->rdev.t3cdev_p;
1913
1914 cm_id->add_ref(cm_id);
1915 ep->com.cm_id = cm_id;
1916 ep->com.qp = get_qhp(h, conn_param->qpn);
1917 BUG_ON(!ep->com.qp);
1918 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn,
1919 ep->com.qp, cm_id);
1920
1921 /*
1922 * Allocate an active TID to initiate a TCP connection.
1923 */
1924 ep->atid = cxgb3_alloc_atid(h->rdev.t3cdev_p, &t3c_client, ep);
1925 if (ep->atid == -1) {
1926 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__);
1927 err = -ENOMEM;
1928 goto fail2;
1929 }
1930
1931 /* find a route */
1932 rt = find_route(h->rdev.t3cdev_p,
1933 cm_id->local_addr.sin_addr.s_addr,
1934 cm_id->remote_addr.sin_addr.s_addr,
1935 cm_id->local_addr.sin_port,
1936 cm_id->remote_addr.sin_port, IPTOS_LOWDELAY);
1937 if (!rt) {
1938 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__);
1939 err = -EHOSTUNREACH;
1940 goto fail3;
1941 }
1942 ep->dst = &rt->dst;
1943 ep->l2t = t3_l2t_get(ep->com.tdev, ep->dst, NULL,
1944 &cm_id->remote_addr.sin_addr.s_addr);
1945 if (!ep->l2t) {
1946 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__);
1947 err = -ENOMEM;
1948 goto fail4;
1949 }
1950
1951 state_set(&ep->com, CONNECTING);
1952 ep->tos = IPTOS_LOWDELAY;
1953 ep->com.local_addr = cm_id->local_addr;
1954 ep->com.remote_addr = cm_id->remote_addr;
1955
1956 /* send connect request to rnic */
1957 err = send_connect(ep);
1958 if (!err)
1959 goto out;
1960
1961 l2t_release(h->rdev.t3cdev_p, ep->l2t);
1962 fail4:
1963 dst_release(ep->dst);
1964 fail3:
1965 cxgb3_free_atid(ep->com.tdev, ep->atid);
1966 fail2:
1967 cm_id->rem_ref(cm_id);
1968 put_ep(&ep->com);
1969 out:
1970 return err;
1971 }
1972
1973 int iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
1974 {
1975 int err = 0;
1976 struct iwch_dev *h = to_iwch_dev(cm_id->device);
1977 struct iwch_listen_ep *ep;
1978
1979
1980 might_sleep();
1981
1982 ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
1983 if (!ep) {
1984 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__);
1985 err = -ENOMEM;
1986 goto fail1;
1987 }
1988 PDBG("%s ep %p\n", __func__, ep);
1989 ep->com.tdev = h->rdev.t3cdev_p;
1990 cm_id->add_ref(cm_id);
1991 ep->com.cm_id = cm_id;
1992 ep->backlog = backlog;
1993 ep->com.local_addr = cm_id->local_addr;
1994
1995 /*
1996 * Allocate a server TID.
1997 */
1998 ep->stid = cxgb3_alloc_stid(h->rdev.t3cdev_p, &t3c_client, ep);
1999 if (ep->stid == -1) {
2000 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__);
2001 err = -ENOMEM;
2002 goto fail2;
2003 }
2004
2005 state_set(&ep->com, LISTEN);
2006 err = listen_start(ep);
2007 if (err)
2008 goto fail3;
2009
2010 /* wait for pass_open_rpl */
2011 wait_event(ep->com.waitq, ep->com.rpl_done);
2012 err = ep->com.rpl_err;
2013 if (!err) {
2014 cm_id->provider_data = ep;
2015 goto out;
2016 }
2017 fail3:
2018 cxgb3_free_stid(ep->com.tdev, ep->stid);
2019 fail2:
2020 cm_id->rem_ref(cm_id);
2021 put_ep(&ep->com);
2022 fail1:
2023 out:
2024 return err;
2025 }
2026
2027 int iwch_destroy_listen(struct iw_cm_id *cm_id)
2028 {
2029 int err;
2030 struct iwch_listen_ep *ep = to_listen_ep(cm_id);
2031
2032 PDBG("%s ep %p\n", __func__, ep);
2033
2034 might_sleep();
2035 state_set(&ep->com, DEAD);
2036 ep->com.rpl_done = 0;
2037 ep->com.rpl_err = 0;
2038 err = listen_stop(ep);
2039 if (err)
2040 goto done;
2041 wait_event(ep->com.waitq, ep->com.rpl_done);
2042 cxgb3_free_stid(ep->com.tdev, ep->stid);
2043 done:
2044 err = ep->com.rpl_err;
2045 cm_id->rem_ref(cm_id);
2046 put_ep(&ep->com);
2047 return err;
2048 }
2049
2050 int iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, gfp_t gfp)
2051 {
2052 int ret=0;
2053 unsigned long flags;
2054 int close = 0;
2055 int fatal = 0;
2056 struct t3cdev *tdev;
2057 struct cxio_rdev *rdev;
2058
2059 spin_lock_irqsave(&ep->com.lock, flags);
2060
2061 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep,
2062 states[ep->com.state], abrupt);
2063
2064 tdev = (struct t3cdev *)ep->com.tdev;
2065 rdev = (struct cxio_rdev *)tdev->ulp;
2066 if (cxio_fatal_error(rdev)) {
2067 fatal = 1;
2068 close_complete_upcall(ep);
2069 ep->com.state = DEAD;
2070 }
2071 switch (ep->com.state) {
2072 case MPA_REQ_WAIT:
2073 case MPA_REQ_SENT:
2074 case MPA_REQ_RCVD:
2075 case MPA_REP_SENT:
2076 case FPDU_MODE:
2077 close = 1;
2078 if (abrupt)
2079 ep->com.state = ABORTING;
2080 else {
2081 ep->com.state = CLOSING;
2082 start_ep_timer(ep);
2083 }
2084 set_bit(CLOSE_SENT, &ep->com.flags);
2085 break;
2086 case CLOSING:
2087 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) {
2088 close = 1;
2089 if (abrupt) {
2090 stop_ep_timer(ep);
2091 ep->com.state = ABORTING;
2092 } else
2093 ep->com.state = MORIBUND;
2094 }
2095 break;
2096 case MORIBUND:
2097 case ABORTING:
2098 case DEAD:
2099 PDBG("%s ignoring disconnect ep %p state %u\n",
2100 __func__, ep, ep->com.state);
2101 break;
2102 default:
2103 BUG();
2104 break;
2105 }
2106
2107 spin_unlock_irqrestore(&ep->com.lock, flags);
2108 if (close) {
2109 if (abrupt)
2110 ret = send_abort(ep, NULL, gfp);
2111 else
2112 ret = send_halfclose(ep, gfp);
2113 if (ret)
2114 fatal = 1;
2115 }
2116 if (fatal)
2117 release_ep_resources(ep);
2118 return ret;
2119 }
2120
2121 int iwch_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
2122 struct l2t_entry *l2t)
2123 {
2124 struct iwch_ep *ep = ctx;
2125
2126 if (ep->dst != old)
2127 return 0;
2128
2129 PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new,
2130 l2t);
2131 dst_hold(new);
2132 l2t_release(ep->com.tdev, ep->l2t);
2133 ep->l2t = l2t;
2134 dst_release(old);
2135 ep->dst = new;
2136 return 1;
2137 }
2138
2139 /*
2140 * All the CM events are handled on a work queue to have a safe context.
2141 * These are the real handlers that are called from the work queue.
2142 */
2143 static const cxgb3_cpl_handler_func work_handlers[NUM_CPL_CMDS] = {
2144 [CPL_ACT_ESTABLISH] = act_establish,
2145 [CPL_ACT_OPEN_RPL] = act_open_rpl,
2146 [CPL_RX_DATA] = rx_data,
2147 [CPL_TX_DMA_ACK] = tx_ack,
2148 [CPL_ABORT_RPL_RSS] = abort_rpl,
2149 [CPL_ABORT_RPL] = abort_rpl,
2150 [CPL_PASS_OPEN_RPL] = pass_open_rpl,
2151 [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl,
2152 [CPL_PASS_ACCEPT_REQ] = pass_accept_req,
2153 [CPL_PASS_ESTABLISH] = pass_establish,
2154 [CPL_PEER_CLOSE] = peer_close,
2155 [CPL_ABORT_REQ_RSS] = peer_abort,
2156 [CPL_CLOSE_CON_RPL] = close_con_rpl,
2157 [CPL_RDMA_TERMINATE] = terminate,
2158 [CPL_RDMA_EC_STATUS] = ec_status,
2159 };
2160
2161 static void process_work(struct work_struct *work)
2162 {
2163 struct sk_buff *skb = NULL;
2164 void *ep;
2165 struct t3cdev *tdev;
2166 int ret;
2167
2168 while ((skb = skb_dequeue(&rxq))) {
2169 ep = *((void **) (skb->cb));
2170 tdev = *((struct t3cdev **) (skb->cb + sizeof(void *)));
2171 ret = work_handlers[G_OPCODE(ntohl((__force __be32)skb->csum))](tdev, skb, ep);
2172 if (ret & CPL_RET_BUF_DONE)
2173 kfree_skb(skb);
2174
2175 /*
2176 * ep was referenced in sched(), and is freed here.
2177 */
2178 put_ep((struct iwch_ep_common *)ep);
2179 }
2180 }
2181
2182 static DECLARE_WORK(skb_work, process_work);
2183
2184 static int sched(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
2185 {
2186 struct iwch_ep_common *epc = ctx;
2187
2188 get_ep(epc);
2189
2190 /*
2191 * Save ctx and tdev in the skb->cb area.
2192 */
2193 *((void **) skb->cb) = ctx;
2194 *((struct t3cdev **) (skb->cb + sizeof(void *))) = tdev;
2195
2196 /*
2197 * Queue the skb and schedule the worker thread.
2198 */
2199 skb_queue_tail(&rxq, skb);
2200 queue_work(workq, &skb_work);
2201 return 0;
2202 }
2203
2204 static int set_tcb_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
2205 {
2206 struct cpl_set_tcb_rpl *rpl = cplhdr(skb);
2207
2208 if (rpl->status != CPL_ERR_NONE) {
2209 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u "
2210 "for tid %u\n", rpl->status, GET_TID(rpl));
2211 }
2212 return CPL_RET_BUF_DONE;
2213 }
2214
2215 /*
2216 * All upcalls from the T3 Core go to sched() to schedule the
2217 * processing on a work queue.
2218 */
2219 cxgb3_cpl_handler_func t3c_handlers[NUM_CPL_CMDS] = {
2220 [CPL_ACT_ESTABLISH] = sched,
2221 [CPL_ACT_OPEN_RPL] = sched,
2222 [CPL_RX_DATA] = sched,
2223 [CPL_TX_DMA_ACK] = sched,
2224 [CPL_ABORT_RPL_RSS] = sched,
2225 [CPL_ABORT_RPL] = sched,
2226 [CPL_PASS_OPEN_RPL] = sched,
2227 [CPL_CLOSE_LISTSRV_RPL] = sched,
2228 [CPL_PASS_ACCEPT_REQ] = sched,
2229 [CPL_PASS_ESTABLISH] = sched,
2230 [CPL_PEER_CLOSE] = sched,
2231 [CPL_CLOSE_CON_RPL] = sched,
2232 [CPL_ABORT_REQ_RSS] = sched,
2233 [CPL_RDMA_TERMINATE] = sched,
2234 [CPL_RDMA_EC_STATUS] = sched,
2235 [CPL_SET_TCB_RPL] = set_tcb_rpl,
2236 };
2237
2238 int __init iwch_cm_init(void)
2239 {
2240 skb_queue_head_init(&rxq);
2241
2242 workq = create_singlethread_workqueue("iw_cxgb3");
2243 if (!workq)
2244 return -ENOMEM;
2245
2246 return 0;
2247 }
2248
2249 void __exit iwch_cm_term(void)
2250 {
2251 flush_workqueue(workq);
2252 destroy_workqueue(workq);
2253 }
CRIS linux kernel tree