?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
spi-topcliff-pch: add recovery processing in case wait-event timeout
[zen-stable.git] / drivers / infiniband / hw / nes / nes_hw.c
blobd42c9f435b1b7a62ad55a331c1f6bc76c84fdbab
1 /*
2 * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <linux/module.h>
35 #include <linux/moduleparam.h>
36 #include <linux/netdevice.h>
37 #include <linux/etherdevice.h>
38 #include <linux/ip.h>
39 #include <linux/tcp.h>
40 #include <linux/if_vlan.h>
41 #include <linux/inet_lro.h>
42 #include <linux/slab.h>
43
44 #include "nes.h"
45
46 static unsigned int nes_lro_max_aggr = NES_LRO_MAX_AGGR;
47 module_param(nes_lro_max_aggr, uint, 0444);
48 MODULE_PARM_DESC(nes_lro_max_aggr, "NIC LRO max packet aggregation");
49
50 static int wide_ppm_offset;
51 module_param(wide_ppm_offset, int, 0644);
52 MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm");
53
54 static u32 crit_err_count;
55 u32 int_mod_timer_init;
56 u32 int_mod_cq_depth_256;
57 u32 int_mod_cq_depth_128;
58 u32 int_mod_cq_depth_32;
59 u32 int_mod_cq_depth_24;
60 u32 int_mod_cq_depth_16;
61 u32 int_mod_cq_depth_4;
62 u32 int_mod_cq_depth_1;
63 static const u8 nes_max_critical_error_count = 100;
64 #include "nes_cm.h"
65
66 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq);
67 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count);
68 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
69 struct nes_adapter *nesadapter, u8 OneG_Mode);
70 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq);
71 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq);
72 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq);
73 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
74 struct nes_hw_aeqe *aeqe);
75 static void process_critical_error(struct nes_device *nesdev);
76 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number);
77 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode);
78 static void nes_terminate_timeout(unsigned long context);
79 static void nes_terminate_start_timer(struct nes_qp *nesqp);
80
81 #ifdef CONFIG_INFINIBAND_NES_DEBUG
82 static unsigned char *nes_iwarp_state_str[] = {
83 "Non-Existent",
84 "Idle",
85 "RTS",
86 "Closing",
87 "RSVD1",
88 "Terminate",
89 "Error",
90 "RSVD2",
91 };
92
93 static unsigned char *nes_tcp_state_str[] = {
94 "Non-Existent",
95 "Closed",
96 "Listen",
97 "SYN Sent",
98 "SYN Rcvd",
99 "Established",
100 "Close Wait",
101 "FIN Wait 1",
102 "Closing",
103 "Last Ack",
104 "FIN Wait 2",
105 "Time Wait",
106 "RSVD1",
107 "RSVD2",
108 "RSVD3",
109 "RSVD4",
110 };
111 #endif
112
113 static inline void print_ip(struct nes_cm_node *cm_node)
114 {
115 unsigned char *rem_addr;
116 if (cm_node) {
117 rem_addr = (unsigned char *)&cm_node->rem_addr;
118 printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr);
119 }
120 }
121
122 /**
123 * nes_nic_init_timer_defaults
124 */
125 void nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode)
126 {
127 unsigned long flags;
128 struct nes_adapter *nesadapter = nesdev->nesadapter;
129 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
130
131 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
132
133 shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW;
134 shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH;
135 if (jumbomode) {
136 shared_timer->threshold_low = DEFAULT_JUMBO_NES_QL_LOW;
137 shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET;
138 shared_timer->threshold_high = DEFAULT_JUMBO_NES_QL_HIGH;
139 } else {
140 shared_timer->threshold_low = DEFAULT_NES_QL_LOW;
141 shared_timer->threshold_target = DEFAULT_NES_QL_TARGET;
142 shared_timer->threshold_high = DEFAULT_NES_QL_HIGH;
143 }
144
145 /* todo use netdev->mtu to set thresholds */
146 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
147 }
148
149
150 /**
151 * nes_nic_init_timer
152 */
153 static void nes_nic_init_timer(struct nes_device *nesdev)
154 {
155 unsigned long flags;
156 struct nes_adapter *nesadapter = nesdev->nesadapter;
157 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
158
159 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
160
161 if (shared_timer->timer_in_use_old == 0) {
162 nesdev->deepcq_count = 0;
163 shared_timer->timer_direction_upward = 0;
164 shared_timer->timer_direction_downward = 0;
165 shared_timer->timer_in_use = NES_NIC_FAST_TIMER;
166 shared_timer->timer_in_use_old = 0;
167
168 }
169 if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) {
170 shared_timer->timer_in_use_old = shared_timer->timer_in_use;
171 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL,
172 0x80000000 | ((u32)(shared_timer->timer_in_use*8)));
173 }
174 /* todo use netdev->mtu to set thresholds */
175 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
176 }
177
178
179 /**
180 * nes_nic_tune_timer
181 */
182 static void nes_nic_tune_timer(struct nes_device *nesdev)
183 {
184 unsigned long flags;
185 struct nes_adapter *nesadapter = nesdev->nesadapter;
186 struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer;
187 u16 cq_count = nesdev->currcq_count;
188
189 spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags);
190
191 if (shared_timer->cq_count_old <= cq_count)
192 shared_timer->cq_direction_downward = 0;
193 else
194 shared_timer->cq_direction_downward++;
195 shared_timer->cq_count_old = cq_count;
196 if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) {
197 if (cq_count <= shared_timer->threshold_low &&
198 shared_timer->threshold_low > 4) {
199 shared_timer->threshold_low = shared_timer->threshold_low/2;
200 shared_timer->cq_direction_downward=0;
201 nesdev->currcq_count = 0;
202 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
203 return;
204 }
205 }
206
207 if (cq_count > 1) {
208 nesdev->deepcq_count += cq_count;
209 if (cq_count <= shared_timer->threshold_low) { /* increase timer gently */
210 shared_timer->timer_direction_upward++;
211 shared_timer->timer_direction_downward = 0;
212 } else if (cq_count <= shared_timer->threshold_target) { /* balanced */
213 shared_timer->timer_direction_upward = 0;
214 shared_timer->timer_direction_downward = 0;
215 } else if (cq_count <= shared_timer->threshold_high) { /* decrease timer gently */
216 shared_timer->timer_direction_downward++;
217 shared_timer->timer_direction_upward = 0;
218 } else if (cq_count <= (shared_timer->threshold_high) * 2) {
219 shared_timer->timer_in_use -= 2;
220 shared_timer->timer_direction_upward = 0;
221 shared_timer->timer_direction_downward++;
222 } else {
223 shared_timer->timer_in_use -= 4;
224 shared_timer->timer_direction_upward = 0;
225 shared_timer->timer_direction_downward++;
226 }
227
228 if (shared_timer->timer_direction_upward > 3 ) { /* using history */
229 shared_timer->timer_in_use += 3;
230 shared_timer->timer_direction_upward = 0;
231 shared_timer->timer_direction_downward = 0;
232 }
233 if (shared_timer->timer_direction_downward > 5) { /* using history */
234 shared_timer->timer_in_use -= 4 ;
235 shared_timer->timer_direction_downward = 0;
236 shared_timer->timer_direction_upward = 0;
237 }
238 }
239
240 /* boundary checking */
241 if (shared_timer->timer_in_use > shared_timer->threshold_high)
242 shared_timer->timer_in_use = shared_timer->threshold_high;
243 else if (shared_timer->timer_in_use < shared_timer->threshold_low)
244 shared_timer->timer_in_use = shared_timer->threshold_low;
245
246 nesdev->currcq_count = 0;
247
248 spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags);
249 }
250
251
252 /**
253 * nes_init_adapter - initialize adapter
254 */
255 struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) {
256 struct nes_adapter *nesadapter = NULL;
257 unsigned long num_pds;
258 u32 u32temp;
259 u32 port_count;
260 u16 max_rq_wrs;
261 u16 max_sq_wrs;
262 u32 max_mr;
263 u32 max_256pbl;
264 u32 max_4kpbl;
265 u32 max_qp;
266 u32 max_irrq;
267 u32 max_cq;
268 u32 hte_index_mask;
269 u32 adapter_size;
270 u32 arp_table_size;
271 u16 vendor_id;
272 u16 device_id;
273 u8 OneG_Mode;
274 u8 func_index;
275
276 /* search the list of existing adapters */
277 list_for_each_entry(nesadapter, &nes_adapter_list, list) {
278 nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X,"
279 " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n",
280 nesdev->pcidev->devfn,
281 PCI_SLOT(nesadapter->devfn),
282 nesadapter->bus_number,
283 PCI_SLOT(nesdev->pcidev->devfn),
284 nesdev->pcidev->bus->number );
285 if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) &&
286 (nesadapter->bus_number == nesdev->pcidev->bus->number)) {
287 nesadapter->ref_count++;
288 return nesadapter;
289 }
290 }
291
292 /* no adapter found */
293 num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT;
294 if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) {
295 nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n",
296 hw_rev);
297 return NULL;
298 }
299
300 nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n",
301 nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8),
302 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS),
303 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4),
304 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8));
305
306 nes_debug(NES_DBG_INIT, "Reset and init NE020\n");
307
308
309 if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0)
310 return NULL;
311
312 max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE);
313 nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp);
314
315 u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE);
316 if (max_qp > ((u32)1 << (u32temp & 0x001f))) {
317 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n",
318 max_qp, u32temp);
319 max_qp = (u32)1 << (u32temp & 0x001f);
320 }
321
322 hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1;
323 nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n",
324 max_qp, hte_index_mask);
325
326 u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT);
327
328 max_irrq = 1 << (u32temp & 0x001f);
329
330 if (max_qp > max_irrq) {
331 max_qp = max_irrq;
332 nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n",
333 max_qp);
334 }
335
336 /* there should be no reason to allocate more pds than qps */
337 if (num_pds > max_qp)
338 num_pds = max_qp;
339
340 u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE);
341 max_mr = (u32)8192 << (u32temp & 0x7);
342
343 u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE);
344 max_256pbl = (u32)1 << (u32temp & 0x0000001f);
345 max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f);
346 max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE);
347
348 u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE);
349 arp_table_size = 1 << u32temp;
350
351 adapter_size = (sizeof(struct nes_adapter) +
352 (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1));
353 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp);
354 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr);
355 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq);
356 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds);
357 adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size);
358 adapter_size += sizeof(struct nes_qp **) * max_qp;
359
360 /* allocate a new adapter struct */
361 nesadapter = kzalloc(adapter_size, GFP_KERNEL);
362 if (nesadapter == NULL) {
363 return NULL;
364 }
365
366 nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n",
367 nesadapter, (u32)sizeof(struct nes_adapter), adapter_size);
368
369 if (nes_read_eeprom_values(nesdev, nesadapter)) {
370 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
371 kfree(nesadapter);
372 return NULL;
373 }
374
375 nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) |
376 (nesadapter->mac_addr_low >> 24);
377
378 pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn,
379 PCI_DEVICE_ID, &device_id);
380 nesadapter->vendor_part_id = device_id;
381
382 if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter,
383 OneG_Mode)) {
384 kfree(nesadapter);
385 return NULL;
386 }
387 nes_init_csr_ne020(nesdev, hw_rev, port_count);
388
389 memset(nesadapter->pft_mcast_map, 255,
390 sizeof nesadapter->pft_mcast_map);
391
392 /* populate the new nesadapter */
393 nesadapter->devfn = nesdev->pcidev->devfn;
394 nesadapter->bus_number = nesdev->pcidev->bus->number;
395 nesadapter->ref_count = 1;
396 nesadapter->timer_int_req = 0xffff0000;
397 nesadapter->OneG_Mode = OneG_Mode;
398 nesadapter->doorbell_start = nesdev->doorbell_region;
399
400 /* nesadapter->tick_delta = clk_divisor; */
401 nesadapter->hw_rev = hw_rev;
402 nesadapter->port_count = port_count;
403
404 nesadapter->max_qp = max_qp;
405 nesadapter->hte_index_mask = hte_index_mask;
406 nesadapter->max_irrq = max_irrq;
407 nesadapter->max_mr = max_mr;
408 nesadapter->max_256pbl = max_256pbl - 1;
409 nesadapter->max_4kpbl = max_4kpbl - 1;
410 nesadapter->max_cq = max_cq;
411 nesadapter->free_256pbl = max_256pbl - 1;
412 nesadapter->free_4kpbl = max_4kpbl - 1;
413 nesadapter->max_pd = num_pds;
414 nesadapter->arp_table_size = arp_table_size;
415
416 nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT;
417 if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) {
418 nesadapter->et_use_adaptive_rx_coalesce = 0;
419 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT;
420 nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval;
421 } else {
422 nesadapter->et_use_adaptive_rx_coalesce = 1;
423 nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC;
424 nesadapter->et_rx_coalesce_usecs_irq = 0;
425 printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__);
426 }
427 /* Setup and enable the periodic timer */
428 if (nesadapter->et_rx_coalesce_usecs_irq)
429 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 |
430 ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8)));
431 else
432 nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000);
433
434 nesadapter->base_pd = 1;
435
436 nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
437 IB_DEVICE_MEM_WINDOW |
438 IB_DEVICE_MEM_MGT_EXTENSIONS;
439
440 nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter)
441 [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]);
442 nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)];
443 nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)];
444 nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)];
445 nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)];
446 nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]);
447
448
449 /* mark the usual suspect QPs, MR and CQs as in use */
450 for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) {
451 set_bit(u32temp, nesadapter->allocated_qps);
452 set_bit(u32temp, nesadapter->allocated_cqs);
453 }
454 set_bit(0, nesadapter->allocated_mrs);
455
456 for (u32temp = 0; u32temp < 20; u32temp++)
457 set_bit(u32temp, nesadapter->allocated_pds);
458 u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES);
459
460 max_rq_wrs = ((u32temp >> 8) & 3);
461 switch (max_rq_wrs) {
462 case 0:
463 max_rq_wrs = 4;
464 break;
465 case 1:
466 max_rq_wrs = 16;
467 break;
468 case 2:
469 max_rq_wrs = 32;
470 break;
471 case 3:
472 max_rq_wrs = 512;
473 break;
474 }
475
476 max_sq_wrs = (u32temp & 3);
477 switch (max_sq_wrs) {
478 case 0:
479 max_sq_wrs = 4;
480 break;
481 case 1:
482 max_sq_wrs = 16;
483 break;
484 case 2:
485 max_sq_wrs = 32;
486 break;
487 case 3:
488 max_sq_wrs = 512;
489 break;
490 }
491 nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs);
492 nesadapter->max_irrq_wr = (u32temp >> 16) & 3;
493
494 nesadapter->max_sge = 4;
495 nesadapter->max_cqe = 32766;
496
497 if (nes_read_eeprom_values(nesdev, nesadapter)) {
498 printk(KERN_ERR PFX "Unable to read EEPROM data.\n");
499 kfree(nesadapter);
500 return NULL;
501 }
502
503 u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG);
504 nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG,
505 (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff));
506
507 /* setup port configuration */
508 if (nesadapter->port_count == 1) {
509 nesadapter->log_port = 0x00000000;
510 if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT)
511 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002);
512 else
513 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
514 } else {
515 if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) {
516 nesadapter->log_port = 0x000000D8;
517 } else {
518 if (nesadapter->port_count == 2)
519 nesadapter->log_port = 0x00000044;
520 else
521 nesadapter->log_port = 0x000000e4;
522 }
523 nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003);
524 }
525
526 nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT,
527 nesadapter->log_port);
528 nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n",
529 nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT));
530
531 spin_lock_init(&nesadapter->resource_lock);
532 spin_lock_init(&nesadapter->phy_lock);
533 spin_lock_init(&nesadapter->pbl_lock);
534 spin_lock_init(&nesadapter->periodic_timer_lock);
535
536 INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]);
537 INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]);
538 INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]);
539 INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]);
540
541 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
542 u32 pcs_control_status0, pcs_control_status1;
543 u32 reset_value;
544 u32 i = 0;
545 u32 int_cnt = 0;
546 u32 ext_cnt = 0;
547 unsigned long flags;
548 u32 j = 0;
549
550 pcs_control_status0 = nes_read_indexed(nesdev,
551 NES_IDX_PHY_PCS_CONTROL_STATUS0);
552 pcs_control_status1 = nes_read_indexed(nesdev,
553 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
554
555 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
556 pcs_control_status0 = nes_read_indexed(nesdev,
557 NES_IDX_PHY_PCS_CONTROL_STATUS0);
558 pcs_control_status1 = nes_read_indexed(nesdev,
559 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
560 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
561 || (0x0F000100 == (pcs_control_status1 & 0x0F000100)))
562 int_cnt++;
563 msleep(1);
564 }
565 if (int_cnt > 1) {
566 spin_lock_irqsave(&nesadapter->phy_lock, flags);
567 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
568 mh_detected++;
569 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
570 reset_value |= 0x0000003d;
571 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
572
573 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
574 & 0x00000040) != 0x00000040) && (j++ < 5000));
575 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
576
577 pcs_control_status0 = nes_read_indexed(nesdev,
578 NES_IDX_PHY_PCS_CONTROL_STATUS0);
579 pcs_control_status1 = nes_read_indexed(nesdev,
580 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
581
582 for (i = 0; i < NES_MAX_LINK_CHECK; i++) {
583 pcs_control_status0 = nes_read_indexed(nesdev,
584 NES_IDX_PHY_PCS_CONTROL_STATUS0);
585 pcs_control_status1 = nes_read_indexed(nesdev,
586 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
587 if ((0x0F000100 == (pcs_control_status0 & 0x0F000100))
588 || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) {
589 if (++ext_cnt > int_cnt) {
590 spin_lock_irqsave(&nesadapter->phy_lock, flags);
591 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1,
592 0x0000F088);
593 mh_detected++;
594 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
595 reset_value |= 0x0000003d;
596 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
597
598 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
599 & 0x00000040) != 0x00000040) && (j++ < 5000));
600 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
601 break;
602 }
603 }
604 msleep(1);
605 }
606 }
607 }
608
609 if (nesadapter->hw_rev == NE020_REV) {
610 init_timer(&nesadapter->mh_timer);
611 nesadapter->mh_timer.function = nes_mh_fix;
612 nesadapter->mh_timer.expires = jiffies + (HZ/5); /* 1 second */
613 nesadapter->mh_timer.data = (unsigned long)nesdev;
614 add_timer(&nesadapter->mh_timer);
615 } else {
616 nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000);
617 }
618
619 init_timer(&nesadapter->lc_timer);
620 nesadapter->lc_timer.function = nes_clc;
621 nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */
622 nesadapter->lc_timer.data = (unsigned long)nesdev;
623 add_timer(&nesadapter->lc_timer);
624
625 list_add_tail(&nesadapter->list, &nes_adapter_list);
626
627 for (func_index = 0; func_index < 8; func_index++) {
628 pci_bus_read_config_word(nesdev->pcidev->bus,
629 PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn),
630 func_index), 0, &vendor_id);
631 if (vendor_id == 0xffff)
632 break;
633 }
634 nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__,
635 func_index, pci_name(nesdev->pcidev));
636 nesadapter->adapter_fcn_count = func_index;
637
638 return nesadapter;
639 }
640
641
642 /**
643 * nes_reset_adapter_ne020
644 */
645 static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode)
646 {
647 u32 port_count;
648 u32 u32temp;
649 u32 i;
650
651 u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
652 port_count = ((u32temp & 0x00000300) >> 8) + 1;
653 /* TODO: assuming that both SERDES are set the same for now */
654 *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1;
655 nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n",
656 u32temp, port_count);
657 if (*OneG_Mode)
658 nes_debug(NES_DBG_INIT, "Running in 1G mode.\n");
659 u32temp &= 0xff00ffc0;
660 switch (port_count) {
661 case 1:
662 u32temp |= 0x00ee0000;
663 break;
664 case 2:
665 u32temp |= 0x00cc0000;
666 break;
667 case 4:
668 u32temp |= 0x00000000;
669 break;
670 default:
671 return 0;
672 break;
673 }
674
675 /* check and do full reset if needed */
676 if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) {
677 nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd);
678 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
679
680 i = 0;
681 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
682 mdelay(1);
683 if (i > 10000) {
684 nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n");
685 return 0;
686 }
687
688 i = 0;
689 while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000)
690 mdelay(1);
691 if (i > 10000) {
692 printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n",
693 nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS));
694 return 0;
695 }
696 }
697
698 /* port reset */
699 switch (port_count) {
700 case 1:
701 u32temp |= 0x00ee0010;
702 break;
703 case 2:
704 u32temp |= 0x00cc0030;
705 break;
706 case 4:
707 u32temp |= 0x00000030;
708 break;
709 }
710
711 nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd);
712 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd);
713
714 i = 0;
715 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000)
716 mdelay(1);
717 if (i > 10000) {
718 nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n");
719 return 0;
720 }
721
722 /* serdes 0 */
723 i = 0;
724 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
725 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
726 mdelay(1);
727 if (i > 5000) {
728 nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp);
729 return 0;
730 }
731
732 /* serdes 1 */
733 if (port_count > 1) {
734 i = 0;
735 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
736 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
737 mdelay(1);
738 if (i > 5000) {
739 nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp);
740 return 0;
741 }
742 }
743
744 return port_count;
745 }
746
747
748 /**
749 * nes_init_serdes
750 */
751 static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count,
752 struct nes_adapter *nesadapter, u8 OneG_Mode)
753 {
754 int i;
755 u32 u32temp;
756 u32 sds;
757
758 if (hw_rev != NE020_REV) {
759 /* init serdes 0 */
760 switch (nesadapter->phy_type[0]) {
761 case NES_PHY_TYPE_CX4:
762 if (wide_ppm_offset)
763 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA);
764 else
765 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
766 break;
767 case NES_PHY_TYPE_KR:
768 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
769 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
770 break;
771 case NES_PHY_TYPE_PUMA_1G:
772 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
773 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0);
774 sds |= 0x00000100;
775 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds);
776 break;
777 default:
778 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF);
779 break;
780 }
781
782 if (!OneG_Mode)
783 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000);
784
785 if (port_count < 2)
786 return 0;
787
788 /* init serdes 1 */
789 if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G)))
790 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF);
791
792 switch (nesadapter->phy_type[1]) {
793 case NES_PHY_TYPE_ARGUS:
794 case NES_PHY_TYPE_SFP_D:
795 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000);
796 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
797 break;
798 case NES_PHY_TYPE_CX4:
799 if (wide_ppm_offset)
800 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA);
801 break;
802 case NES_PHY_TYPE_KR:
803 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000);
804 break;
805 case NES_PHY_TYPE_PUMA_1G:
806 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
807 sds |= 0x000000100;
808 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
809 }
810 if (!OneG_Mode) {
811 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000);
812 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
813 sds &= 0xFFFFFFBF;
814 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds);
815 }
816 } else {
817 /* init serdes 0 */
818 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008);
819 i = 0;
820 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0)
821 & 0x0000000f)) != 0x0000000f) && i++ < 5000)
822 mdelay(1);
823 if (i > 5000) {
824 nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp);
825 return 1;
826 }
827 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7);
828 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000);
829 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000);
830 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000);
831 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000);
832 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000);
833 if (OneG_Mode)
834 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222);
835 else
836 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222);
837
838 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff);
839 if (port_count > 1) {
840 /* init serdes 1 */
841 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048);
842 i = 0;
843 while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1)
844 & 0x0000000f)) != 0x0000000f) && (i++ < 5000))
845 mdelay(1);
846 if (i > 5000) {
847 printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp);
848 /* return 1; */
849 }
850 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7);
851 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000);
852 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000);
853 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000);
854 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000);
855 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000);
856 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222);
857 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff);
858 }
859 }
860 return 0;
861 }
862
863
864 /**
865 * nes_init_csr_ne020
866 * Initialize registers for ne020 hardware
867 */
868 static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count)
869 {
870 u32 u32temp;
871
872 nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count);
873
874 nes_write_indexed(nesdev, 0x000001E4, 0x00000007);
875 /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */
876 nes_write_indexed(nesdev, 0x000001E8, 0x00020874);
877 nes_write_indexed(nesdev, 0x000001D8, 0x00048002);
878 /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */
879 nes_write_indexed(nesdev, 0x000001FC, 0x00050005);
880 nes_write_indexed(nesdev, 0x00000600, 0x55555555);
881 nes_write_indexed(nesdev, 0x00000604, 0x55555555);
882
883 /* TODO: move these MAC register settings to NIC bringup */
884 nes_write_indexed(nesdev, 0x00002000, 0x00000001);
885 nes_write_indexed(nesdev, 0x00002004, 0x00000001);
886 nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF);
887 nes_write_indexed(nesdev, 0x0000200C, 0x00000001);
888 nes_write_indexed(nesdev, 0x00002010, 0x000003c1);
889 nes_write_indexed(nesdev, 0x0000201C, 0x75345678);
890 if (port_count > 1) {
891 nes_write_indexed(nesdev, 0x00002200, 0x00000001);
892 nes_write_indexed(nesdev, 0x00002204, 0x00000001);
893 nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF);
894 nes_write_indexed(nesdev, 0x0000220C, 0x00000001);
895 nes_write_indexed(nesdev, 0x00002210, 0x000003c1);
896 nes_write_indexed(nesdev, 0x0000221C, 0x75345678);
897 nes_write_indexed(nesdev, 0x00000908, 0x20000001);
898 }
899 if (port_count > 2) {
900 nes_write_indexed(nesdev, 0x00002400, 0x00000001);
901 nes_write_indexed(nesdev, 0x00002404, 0x00000001);
902 nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF);
903 nes_write_indexed(nesdev, 0x0000240C, 0x00000001);
904 nes_write_indexed(nesdev, 0x00002410, 0x000003c1);
905 nes_write_indexed(nesdev, 0x0000241C, 0x75345678);
906 nes_write_indexed(nesdev, 0x00000910, 0x20000001);
907
908 nes_write_indexed(nesdev, 0x00002600, 0x00000001);
909 nes_write_indexed(nesdev, 0x00002604, 0x00000001);
910 nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF);
911 nes_write_indexed(nesdev, 0x0000260C, 0x00000001);
912 nes_write_indexed(nesdev, 0x00002610, 0x000003c1);
913 nes_write_indexed(nesdev, 0x0000261C, 0x75345678);
914 nes_write_indexed(nesdev, 0x00000918, 0x20000001);
915 }
916
917 nes_write_indexed(nesdev, 0x00005000, 0x00018000);
918 /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */
919 nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) |
920 0x00000001);
921 nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F);
922 nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F);
923 nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F);
924 nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F);
925 nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF);
926
927 /* TODO: move this to code, get from EEPROM */
928 nes_write_indexed(nesdev, 0x00000900, 0x20000001);
929 nes_write_indexed(nesdev, 0x000060C0, 0x0000028e);
930 nes_write_indexed(nesdev, 0x000060C8, 0x00000020);
931
932 nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0);
933 /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */
934
935 if (hw_rev != NE020_REV) {
936 u32temp = nes_read_indexed(nesdev, 0x000008e8);
937 u32temp |= 0x80000000;
938 nes_write_indexed(nesdev, 0x000008e8, u32temp);
939 u32temp = nes_read_indexed(nesdev, 0x000021f8);
940 u32temp &= 0x7fffffff;
941 u32temp |= 0x7fff0010;
942 nes_write_indexed(nesdev, 0x000021f8, u32temp);
943 if (port_count > 1) {
944 u32temp = nes_read_indexed(nesdev, 0x000023f8);
945 u32temp &= 0x7fffffff;
946 u32temp |= 0x7fff0010;
947 nes_write_indexed(nesdev, 0x000023f8, u32temp);
948 }
949 }
950 }
951
952
953 /**
954 * nes_destroy_adapter - destroy the adapter structure
955 */
956 void nes_destroy_adapter(struct nes_adapter *nesadapter)
957 {
958 struct nes_adapter *tmp_adapter;
959
960 list_for_each_entry(tmp_adapter, &nes_adapter_list, list) {
961 nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n",
962 tmp_adapter);
963 }
964
965 nesadapter->ref_count--;
966 if (!nesadapter->ref_count) {
967 if (nesadapter->hw_rev == NE020_REV) {
968 del_timer(&nesadapter->mh_timer);
969 }
970 del_timer(&nesadapter->lc_timer);
971
972 list_del(&nesadapter->list);
973 kfree(nesadapter);
974 }
975 }
976
977
978 /**
979 * nes_init_cqp
980 */
981 int nes_init_cqp(struct nes_device *nesdev)
982 {
983 struct nes_adapter *nesadapter = nesdev->nesadapter;
984 struct nes_hw_cqp_qp_context *cqp_qp_context;
985 struct nes_hw_cqp_wqe *cqp_wqe;
986 struct nes_hw_ceq *ceq;
987 struct nes_hw_ceq *nic_ceq;
988 struct nes_hw_aeq *aeq;
989 void *vmem;
990 dma_addr_t pmem;
991 u32 count=0;
992 u32 cqp_head;
993 u64 u64temp;
994 u32 u32temp;
995
996 /* allocate CQP memory */
997 /* Need to add max_cq to the aeq size once cq overflow checking is added back */
998 /* SQ is 512 byte aligned, others are 256 byte aligned */
999 nesdev->cqp_mem_size = 512 +
1000 (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) +
1001 (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) +
1002 max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) +
1003 max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) +
1004 (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) +
1005 sizeof(struct nes_hw_cqp_qp_context);
1006
1007 nesdev->cqp_vbase = pci_alloc_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1008 &nesdev->cqp_pbase);
1009 if (!nesdev->cqp_vbase) {
1010 nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n");
1011 return -ENOMEM;
1012 }
1013 memset(nesdev->cqp_vbase, 0, nesdev->cqp_mem_size);
1014
1015 /* Allocate a twice the number of CQP requests as the SQ size */
1016 nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) *
1017 2 * NES_CQP_SQ_SIZE, GFP_KERNEL);
1018 if (nesdev->nes_cqp_requests == NULL) {
1019 nes_debug(NES_DBG_INIT, "Unable to allocate memory CQP request entries.\n");
1020 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1021 nesdev->cqp.sq_pbase);
1022 return -ENOMEM;
1023 }
1024
1025 nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n",
1026 nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size);
1027
1028 spin_lock_init(&nesdev->cqp.lock);
1029 init_waitqueue_head(&nesdev->cqp.waitq);
1030
1031 /* Setup Various Structures */
1032 vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) &
1033 ~(unsigned long)(512 - 1));
1034 pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) &
1035 ~(unsigned long long)(512 - 1));
1036
1037 nesdev->cqp.sq_vbase = vmem;
1038 nesdev->cqp.sq_pbase = pmem;
1039 nesdev->cqp.sq_size = NES_CQP_SQ_SIZE;
1040 nesdev->cqp.sq_head = 0;
1041 nesdev->cqp.sq_tail = 0;
1042 nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn);
1043
1044 vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1045 pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size);
1046
1047 nesdev->ccq.cq_vbase = vmem;
1048 nesdev->ccq.cq_pbase = pmem;
1049 nesdev->ccq.cq_size = NES_CCQ_SIZE;
1050 nesdev->ccq.cq_head = 0;
1051 nesdev->ccq.ce_handler = nes_cqp_ce_handler;
1052 nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn);
1053
1054 vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1055 pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size);
1056
1057 nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn);
1058 ceq = &nesadapter->ceq[nesdev->ceq_index];
1059 ceq->ceq_vbase = vmem;
1060 ceq->ceq_pbase = pmem;
1061 ceq->ceq_size = NES_CCEQ_SIZE;
1062 ceq->ceq_head = 0;
1063
1064 vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1065 pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256);
1066
1067 nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8;
1068 nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index];
1069 nic_ceq->ceq_vbase = vmem;
1070 nic_ceq->ceq_pbase = pmem;
1071 nic_ceq->ceq_size = NES_NIC_CEQ_SIZE;
1072 nic_ceq->ceq_head = 0;
1073
1074 vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1075 pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256);
1076
1077 aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)];
1078 aeq->aeq_vbase = vmem;
1079 aeq->aeq_pbase = pmem;
1080 aeq->aeq_size = nesadapter->max_qp;
1081 aeq->aeq_head = 0;
1082
1083 /* Setup QP Context */
1084 vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1085 pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size);
1086
1087 cqp_qp_context = vmem;
1088 cqp_qp_context->context_words[0] =
1089 cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10));
1090 cqp_qp_context->context_words[1] = 0;
1091 cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase);
1092 cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32);
1093
1094
1095 /* Write the address to Create CQP */
1096 if ((sizeof(dma_addr_t) > 4)) {
1097 nes_write_indexed(nesdev,
1098 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1099 ((u64)pmem) >> 32);
1100 } else {
1101 nes_write_indexed(nesdev,
1102 NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0);
1103 }
1104 nes_write_indexed(nesdev,
1105 NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8),
1106 (u32)pmem);
1107
1108 INIT_LIST_HEAD(&nesdev->cqp_avail_reqs);
1109 INIT_LIST_HEAD(&nesdev->cqp_pending_reqs);
1110
1111 for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) {
1112 init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq);
1113 list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs);
1114 }
1115
1116 /* Write Create CCQ WQE */
1117 cqp_head = nesdev->cqp.sq_head++;
1118 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1119 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1120 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1121 (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1122 NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16)));
1123 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
1124 (nesdev->ccq.cq_number |
1125 ((u32)nesdev->ceq_index << 16)));
1126 u64temp = (u64)nesdev->ccq.cq_pbase;
1127 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1128 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1129 u64temp = (unsigned long)&nesdev->ccq;
1130 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] =
1131 cpu_to_le32((u32)(u64temp >> 1));
1132 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1133 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1134 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1135
1136 /* Write Create CEQ WQE */
1137 cqp_head = nesdev->cqp.sq_head++;
1138 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1139 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1140 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1141 (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8)));
1142 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size);
1143 u64temp = (u64)ceq->ceq_pbase;
1144 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1145
1146 /* Write Create AEQ WQE */
1147 cqp_head = nesdev->cqp.sq_head++;
1148 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1149 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1150 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1151 (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)));
1152 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size);
1153 u64temp = (u64)aeq->aeq_pbase;
1154 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1155
1156 /* Write Create NIC CEQ WQE */
1157 cqp_head = nesdev->cqp.sq_head++;
1158 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1159 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1160 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
1161 (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8)));
1162 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size);
1163 u64temp = (u64)nic_ceq->ceq_pbase;
1164 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1165
1166 /* Poll until CCQP done */
1167 count = 0;
1168 do {
1169 if (count++ > 1000) {
1170 printk(KERN_ERR PFX "Error creating CQP\n");
1171 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1172 nesdev->cqp_vbase, nesdev->cqp_pbase);
1173 return -1;
1174 }
1175 udelay(10);
1176 } while (!(nes_read_indexed(nesdev,
1177 NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8)));
1178
1179 nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev,
1180 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1181
1182 u32temp = 0x04800000;
1183 nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id);
1184
1185 /* wait for the CCQ, CEQ, and AEQ to get created */
1186 count = 0;
1187 do {
1188 if (count++ > 1000) {
1189 printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n");
1190 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size,
1191 nesdev->cqp_vbase, nesdev->cqp_pbase);
1192 return -1;
1193 }
1194 udelay(10);
1195 } while (((nes_read_indexed(nesdev,
1196 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8)));
1197
1198 /* dump the QP status value */
1199 nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev,
1200 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1201
1202 nesdev->cqp.sq_tail++;
1203
1204 return 0;
1205 }
1206
1207
1208 /**
1209 * nes_destroy_cqp
1210 */
1211 int nes_destroy_cqp(struct nes_device *nesdev)
1212 {
1213 struct nes_hw_cqp_wqe *cqp_wqe;
1214 u32 count = 0;
1215 u32 cqp_head;
1216 unsigned long flags;
1217
1218 do {
1219 if (count++ > 1000)
1220 break;
1221 udelay(10);
1222 } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail));
1223
1224 /* Reset CCQ */
1225 nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET |
1226 nesdev->ccq.cq_number);
1227
1228 /* Disable device interrupts */
1229 nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff);
1230
1231 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1232
1233 /* Destroy the AEQ */
1234 cqp_head = nesdev->cqp.sq_head++;
1235 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1236 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1237 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ |
1238 ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8));
1239 cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0;
1240
1241 /* Destroy the NIC CEQ */
1242 cqp_head = nesdev->cqp.sq_head++;
1243 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1244 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1245 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1246 ((u32)nesdev->nic_ceq_index << 8));
1247
1248 /* Destroy the CEQ */
1249 cqp_head = nesdev->cqp.sq_head++;
1250 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1251 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1252 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ |
1253 (nesdev->ceq_index << 8));
1254
1255 /* Destroy the CCQ */
1256 cqp_head = nesdev->cqp.sq_head++;
1257 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1258 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1259 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ);
1260 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number |
1261 ((u32)nesdev->ceq_index << 16));
1262
1263 /* Destroy CQP */
1264 cqp_head = nesdev->cqp.sq_head++;
1265 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
1266 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1267 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP |
1268 NES_CQP_QP_TYPE_CQP);
1269 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id);
1270
1271 barrier();
1272 /* Ring doorbell (5 WQEs) */
1273 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id);
1274
1275 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1276
1277 /* wait for the CCQ, CEQ, and AEQ to get destroyed */
1278 count = 0;
1279 do {
1280 if (count++ > 1000) {
1281 printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n",
1282 PCI_FUNC(nesdev->pcidev->devfn));
1283 break;
1284 }
1285 udelay(10);
1286 } while (((nes_read_indexed(nesdev,
1287 NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0));
1288
1289 /* dump the QP status value */
1290 nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n",
1291 PCI_FUNC(nesdev->pcidev->devfn),
1292 nes_read_indexed(nesdev,
1293 NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)));
1294
1295 kfree(nesdev->nes_cqp_requests);
1296
1297 /* Free the control structures */
1298 pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase,
1299 nesdev->cqp.sq_pbase);
1300
1301 return 0;
1302 }
1303
1304
1305 /**
1306 * nes_init_1g_phy
1307 */
1308 static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1309 {
1310 u32 counter = 0;
1311 u16 phy_data;
1312 int ret = 0;
1313
1314 nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data);
1315 nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000);
1316
1317 /* Reset the PHY */
1318 nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000);
1319 udelay(100);
1320 counter = 0;
1321 do {
1322 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1323 if (counter++ > 100) {
1324 ret = -1;
1325 break;
1326 }
1327 } while (phy_data & 0x8000);
1328
1329 /* Setting no phy loopback */
1330 phy_data &= 0xbfff;
1331 phy_data |= 0x1140;
1332 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data);
1333 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1334 nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data);
1335 nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data);
1336
1337 /* Setting the interrupt mask */
1338 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1339 nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee);
1340 nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data);
1341
1342 /* turning on flow control */
1343 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1344 nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00);
1345 nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data);
1346
1347 /* Clear Half duplex */
1348 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1349 nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100));
1350 nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data);
1351
1352 nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data);
1353 nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300);
1354
1355 return ret;
1356 }
1357
1358
1359 /**
1360 * nes_init_2025_phy
1361 */
1362 static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index)
1363 {
1364 u32 temp_phy_data = 0;
1365 u32 temp_phy_data2 = 0;
1366 u32 counter = 0;
1367 u32 sds;
1368 u32 mac_index = nesdev->mac_index;
1369 int ret = 0;
1370 unsigned int first_attempt = 1;
1371
1372 /* Check firmware heartbeat */
1373 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1374 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1375 udelay(1500);
1376 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1377 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1378
1379 if (temp_phy_data != temp_phy_data2) {
1380 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1381 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1382 if ((temp_phy_data & 0xff) > 0x20)
1383 return 0;
1384 printk(PFX "Reinitialize external PHY\n");
1385 }
1386
1387 /* no heartbeat, configure the PHY */
1388 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000);
1389 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000);
1390 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1391 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1392
1393 switch (phy_type) {
1394 case NES_PHY_TYPE_ARGUS:
1395 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1396 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1397 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1398 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008);
1399 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001);
1400 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1401 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1402
1403 /* setup LEDs */
1404 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1405 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1406 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1407 break;
1408
1409 case NES_PHY_TYPE_SFP_D:
1410 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1411 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1412 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004);
1413 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038);
1414 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1415 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098);
1416 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1417
1418 /* setup LEDs */
1419 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007);
1420 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A);
1421 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009);
1422 break;
1423
1424 case NES_PHY_TYPE_KR:
1425 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A);
1426 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052);
1427 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C);
1428 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010);
1429 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013);
1430 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080);
1431 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00);
1432
1433 /* setup LEDs */
1434 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B);
1435 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003);
1436 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004);
1437
1438 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D);
1439 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020);
1440 break;
1441 }
1442
1443 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528);
1444
1445 /* Bring PHY out of reset */
1446 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002);
1447
1448 /* Check for heartbeat */
1449 counter = 0;
1450 mdelay(690);
1451 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1452 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1453 do {
1454 if (counter++ > 150) {
1455 printk(PFX "No PHY heartbeat\n");
1456 break;
1457 }
1458 mdelay(1);
1459 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee);
1460 temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1461 } while ((temp_phy_data2 == temp_phy_data));
1462
1463 /* wait for tracking */
1464 counter = 0;
1465 do {
1466 nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd);
1467 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
1468 if (counter++ > 300) {
1469 if (((temp_phy_data & 0xff) == 0x0) && first_attempt) {
1470 first_attempt = 0;
1471 counter = 0;
1472 /* reset AMCC PHY and try again */
1473 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0);
1474 nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040);
1475 continue;
1476 } else {
1477 ret = 1;
1478 break;
1479 }
1480 }
1481 mdelay(10);
1482 } while ((temp_phy_data & 0xff) < 0x30);
1483
1484 /* setup signal integrity */
1485 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000);
1486 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE);
1487 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032);
1488 if (phy_type == NES_PHY_TYPE_KR) {
1489 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C);
1490 } else {
1491 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002);
1492 nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063);
1493 }
1494
1495 /* reset serdes */
1496 sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200);
1497 sds |= 0x1;
1498 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1499 sds &= 0xfffffffe;
1500 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds);
1501
1502 counter = 0;
1503 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040)
1504 && (counter++ < 5000))
1505 ;
1506
1507 return ret;
1508 }
1509
1510
1511 /**
1512 * nes_init_phy
1513 */
1514 int nes_init_phy(struct nes_device *nesdev)
1515 {
1516 struct nes_adapter *nesadapter = nesdev->nesadapter;
1517 u32 mac_index = nesdev->mac_index;
1518 u32 tx_config = 0;
1519 unsigned long flags;
1520 u8 phy_type = nesadapter->phy_type[mac_index];
1521 u8 phy_index = nesadapter->phy_index[mac_index];
1522 int ret = 0;
1523
1524 tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG);
1525 if (phy_type == NES_PHY_TYPE_1G) {
1526 /* setup 1G MDIO operation */
1527 tx_config &= 0xFFFFFFE3;
1528 tx_config |= 0x04;
1529 } else {
1530 /* setup 10G MDIO operation */
1531 tx_config &= 0xFFFFFFE3;
1532 tx_config |= 0x1D;
1533 }
1534 nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config);
1535
1536 spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags);
1537
1538 switch (phy_type) {
1539 case NES_PHY_TYPE_1G:
1540 ret = nes_init_1g_phy(nesdev, phy_type, phy_index);
1541 break;
1542 case NES_PHY_TYPE_ARGUS:
1543 case NES_PHY_TYPE_SFP_D:
1544 case NES_PHY_TYPE_KR:
1545 ret = nes_init_2025_phy(nesdev, phy_type, phy_index);
1546 break;
1547 }
1548
1549 spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags);
1550
1551 return ret;
1552 }
1553
1554
1555 /**
1556 * nes_replenish_nic_rq
1557 */
1558 static void nes_replenish_nic_rq(struct nes_vnic *nesvnic)
1559 {
1560 unsigned long flags;
1561 dma_addr_t bus_address;
1562 struct sk_buff *skb;
1563 struct nes_hw_nic_rq_wqe *nic_rqe;
1564 struct nes_hw_nic *nesnic;
1565 struct nes_device *nesdev;
1566 struct nes_rskb_cb *cb;
1567 u32 rx_wqes_posted = 0;
1568
1569 nesnic = &nesvnic->nic;
1570 nesdev = nesvnic->nesdev;
1571 spin_lock_irqsave(&nesnic->rq_lock, flags);
1572 if (nesnic->replenishing_rq !=0) {
1573 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1574 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1575 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1576 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1577 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
1578 add_timer(&nesvnic->rq_wqes_timer);
1579 } else
1580 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1581 return;
1582 }
1583 nesnic->replenishing_rq = 1;
1584 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1585 do {
1586 skb = dev_alloc_skb(nesvnic->max_frame_size);
1587 if (skb) {
1588 skb->dev = nesvnic->netdev;
1589
1590 bus_address = pci_map_single(nesdev->pcidev,
1591 skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1592 cb = (struct nes_rskb_cb *)&skb->cb[0];
1593 cb->busaddr = bus_address;
1594 cb->maplen = nesvnic->max_frame_size;
1595
1596 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head];
1597 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] =
1598 cpu_to_le32(nesvnic->max_frame_size);
1599 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1600 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] =
1601 cpu_to_le32((u32)bus_address);
1602 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] =
1603 cpu_to_le32((u32)((u64)bus_address >> 32));
1604 nesnic->rx_skb[nesnic->rq_head] = skb;
1605 nesnic->rq_head++;
1606 nesnic->rq_head &= nesnic->rq_size - 1;
1607 atomic_dec(&nesvnic->rx_skbs_needed);
1608 barrier();
1609 if (++rx_wqes_posted == 255) {
1610 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1611 rx_wqes_posted = 0;
1612 }
1613 } else {
1614 spin_lock_irqsave(&nesnic->rq_lock, flags);
1615 if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) &&
1616 (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) {
1617 atomic_set(&nesvnic->rx_skb_timer_running, 1);
1618 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1619 nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */
1620 add_timer(&nesvnic->rq_wqes_timer);
1621 } else
1622 spin_unlock_irqrestore(&nesnic->rq_lock, flags);
1623 break;
1624 }
1625 } while (atomic_read(&nesvnic->rx_skbs_needed));
1626 barrier();
1627 if (rx_wqes_posted)
1628 nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id);
1629 nesnic->replenishing_rq = 0;
1630 }
1631
1632
1633 /**
1634 * nes_rq_wqes_timeout
1635 */
1636 static void nes_rq_wqes_timeout(unsigned long parm)
1637 {
1638 struct nes_vnic *nesvnic = (struct nes_vnic *)parm;
1639 printk("%s: Timer fired.\n", __func__);
1640 atomic_set(&nesvnic->rx_skb_timer_running, 0);
1641 if (atomic_read(&nesvnic->rx_skbs_needed))
1642 nes_replenish_nic_rq(nesvnic);
1643 }
1644
1645
1646 static int nes_lro_get_skb_hdr(struct sk_buff *skb, void **iphdr,
1647 void **tcph, u64 *hdr_flags, void *priv)
1648 {
1649 unsigned int ip_len;
1650 struct iphdr *iph;
1651 skb_reset_network_header(skb);
1652 iph = ip_hdr(skb);
1653 if (iph->protocol != IPPROTO_TCP)
1654 return -1;
1655 ip_len = ip_hdrlen(skb);
1656 skb_set_transport_header(skb, ip_len);
1657 *tcph = tcp_hdr(skb);
1658
1659 *hdr_flags = LRO_IPV4 | LRO_TCP;
1660 *iphdr = iph;
1661 return 0;
1662 }
1663
1664
1665 /**
1666 * nes_init_nic_qp
1667 */
1668 int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev)
1669 {
1670 struct nes_hw_cqp_wqe *cqp_wqe;
1671 struct nes_hw_nic_sq_wqe *nic_sqe;
1672 struct nes_hw_nic_qp_context *nic_context;
1673 struct sk_buff *skb;
1674 struct nes_hw_nic_rq_wqe *nic_rqe;
1675 struct nes_vnic *nesvnic = netdev_priv(netdev);
1676 unsigned long flags;
1677 void *vmem;
1678 dma_addr_t pmem;
1679 u64 u64temp;
1680 int ret;
1681 u32 cqp_head;
1682 u32 counter;
1683 u32 wqe_count;
1684 struct nes_rskb_cb *cb;
1685 u8 jumbomode=0;
1686
1687 /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */
1688 nesvnic->nic_mem_size = 256 +
1689 (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) +
1690 (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) +
1691 (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) +
1692 (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) +
1693 sizeof(struct nes_hw_nic_qp_context);
1694
1695 nesvnic->nic_vbase = pci_alloc_consistent(nesdev->pcidev, nesvnic->nic_mem_size,
1696 &nesvnic->nic_pbase);
1697 if (!nesvnic->nic_vbase) {
1698 nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n");
1699 return -ENOMEM;
1700 }
1701 memset(nesvnic->nic_vbase, 0, nesvnic->nic_mem_size);
1702 nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n",
1703 nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size);
1704
1705 vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) &
1706 ~(unsigned long)(256 - 1));
1707 pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) &
1708 ~(unsigned long long)(256 - 1));
1709
1710 /* Setup the first Fragment buffers */
1711 nesvnic->nic.first_frag_vbase = vmem;
1712
1713 for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1714 nesvnic->nic.frag_paddr[counter] = pmem;
1715 pmem += sizeof(struct nes_first_frag);
1716 }
1717
1718 /* setup the SQ */
1719 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag));
1720
1721 nesvnic->nic.sq_vbase = (void *)vmem;
1722 nesvnic->nic.sq_pbase = pmem;
1723 nesvnic->nic.sq_head = 0;
1724 nesvnic->nic.sq_tail = 0;
1725 nesvnic->nic.sq_size = NES_NIC_WQ_SIZE;
1726 for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) {
1727 nic_sqe = &nesvnic->nic.sq_vbase[counter];
1728 nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] =
1729 cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM |
1730 NES_NIC_SQ_WQE_COMPLETION);
1731 nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] =
1732 cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16);
1733 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] =
1734 cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]);
1735 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] =
1736 cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32));
1737 }
1738
1739 nesvnic->get_cqp_request = nes_get_cqp_request;
1740 nesvnic->post_cqp_request = nes_post_cqp_request;
1741 nesvnic->mcrq_mcast_filter = NULL;
1742
1743 spin_lock_init(&nesvnic->nic.rq_lock);
1744
1745 /* setup the RQ */
1746 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1747 pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe));
1748
1749
1750 nesvnic->nic.rq_vbase = vmem;
1751 nesvnic->nic.rq_pbase = pmem;
1752 nesvnic->nic.rq_head = 0;
1753 nesvnic->nic.rq_tail = 0;
1754 nesvnic->nic.rq_size = NES_NIC_WQ_SIZE;
1755
1756 /* setup the CQ */
1757 vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1758 pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe));
1759
1760 if (nesdev->nesadapter->netdev_count > 2)
1761 nesvnic->mcrq_qp_id = nesvnic->nic_index + 32;
1762 else
1763 nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4;
1764
1765 nesvnic->nic_cq.cq_vbase = vmem;
1766 nesvnic->nic_cq.cq_pbase = pmem;
1767 nesvnic->nic_cq.cq_head = 0;
1768 nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2;
1769
1770 nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler;
1771
1772 /* Send CreateCQ request to CQP */
1773 spin_lock_irqsave(&nesdev->cqp.lock, flags);
1774 cqp_head = nesdev->cqp.sq_head;
1775
1776 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1777 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1778
1779 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
1780 NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID |
1781 ((u32)nesvnic->nic_cq.cq_size << 16));
1782 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(
1783 nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16));
1784 u64temp = (u64)nesvnic->nic_cq.cq_pbase;
1785 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp);
1786 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0;
1787 u64temp = (unsigned long)&nesvnic->nic_cq;
1788 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1));
1789 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] =
1790 cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF);
1791 cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0;
1792 if (++cqp_head >= nesdev->cqp.sq_size)
1793 cqp_head = 0;
1794 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
1795 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
1796
1797 /* Send CreateQP request to CQP */
1798 nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]);
1799 nic_context->context_words[NES_NIC_CTX_MISC_IDX] =
1800 cpu_to_le32((u32)NES_NIC_CTX_SIZE |
1801 ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12));
1802 nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n",
1803 nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE),
1804 nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE));
1805 if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) {
1806 nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE);
1807 }
1808
1809 u64temp = (u64)nesvnic->nic.sq_pbase;
1810 nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1811 nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1812 u64temp = (u64)nesvnic->nic.rq_pbase;
1813 nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp);
1814 nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32));
1815
1816 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP |
1817 NES_CQP_QP_TYPE_NIC);
1818 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id);
1819 u64temp = (u64)nesvnic->nic_cq.cq_pbase +
1820 (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe));
1821 set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp);
1822
1823 if (++cqp_head >= nesdev->cqp.sq_size)
1824 cqp_head = 0;
1825 nesdev->cqp.sq_head = cqp_head;
1826
1827 barrier();
1828
1829 /* Ring doorbell (2 WQEs) */
1830 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
1831
1832 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
1833 nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n",
1834 nesvnic->nic.qp_id);
1835
1836 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
1837 NES_EVENT_TIMEOUT);
1838 nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n",
1839 nesvnic->nic.qp_id, ret);
1840 if (!ret) {
1841 nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id);
1842 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
1843 nesvnic->nic_pbase);
1844 return -EIO;
1845 }
1846
1847 /* Populate the RQ */
1848 for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) {
1849 skb = dev_alloc_skb(nesvnic->max_frame_size);
1850 if (!skb) {
1851 nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name);
1852
1853 nes_destroy_nic_qp(nesvnic);
1854 return -ENOMEM;
1855 }
1856
1857 skb->dev = netdev;
1858
1859 pmem = pci_map_single(nesdev->pcidev, skb->data,
1860 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
1861 cb = (struct nes_rskb_cb *)&skb->cb[0];
1862 cb->busaddr = pmem;
1863 cb->maplen = nesvnic->max_frame_size;
1864
1865 nic_rqe = &nesvnic->nic.rq_vbase[counter];
1866 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size);
1867 nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0;
1868 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem);
1869 nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32));
1870 nesvnic->nic.rx_skb[counter] = skb;
1871 }
1872
1873 wqe_count = NES_NIC_WQ_SIZE - 1;
1874 nesvnic->nic.rq_head = wqe_count;
1875 barrier();
1876 do {
1877 counter = min(wqe_count, ((u32)255));
1878 wqe_count -= counter;
1879 nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id);
1880 } while (wqe_count);
1881 init_timer(&nesvnic->rq_wqes_timer);
1882 nesvnic->rq_wqes_timer.function = nes_rq_wqes_timeout;
1883 nesvnic->rq_wqes_timer.data = (unsigned long)nesvnic;
1884 nes_debug(NES_DBG_INIT, "NAPI support Enabled\n");
1885 if (nesdev->nesadapter->et_use_adaptive_rx_coalesce)
1886 {
1887 nes_nic_init_timer(nesdev);
1888 if (netdev->mtu > 1500)
1889 jumbomode = 1;
1890 nes_nic_init_timer_defaults(nesdev, jumbomode);
1891 }
1892 if ((nesdev->nesadapter->allow_unaligned_fpdus) &&
1893 (nes_init_mgt_qp(nesdev, netdev, nesvnic))) {
1894 nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", netdev->name);
1895 nes_destroy_nic_qp(nesvnic);
1896 return -ENOMEM;
1897 }
1898
1899 nesvnic->lro_mgr.max_aggr = nes_lro_max_aggr;
1900 nesvnic->lro_mgr.max_desc = NES_MAX_LRO_DESCRIPTORS;
1901 nesvnic->lro_mgr.lro_arr = nesvnic->lro_desc;
1902 nesvnic->lro_mgr.get_skb_header = nes_lro_get_skb_hdr;
1903 nesvnic->lro_mgr.features = LRO_F_NAPI | LRO_F_EXTRACT_VLAN_ID;
1904 nesvnic->lro_mgr.dev = netdev;
1905 nesvnic->lro_mgr.ip_summed = CHECKSUM_UNNECESSARY;
1906 nesvnic->lro_mgr.ip_summed_aggr = CHECKSUM_UNNECESSARY;
1907 return 0;
1908 }
1909
1910
1911 /**
1912 * nes_destroy_nic_qp
1913 */
1914 void nes_destroy_nic_qp(struct nes_vnic *nesvnic)
1915 {
1916 u64 u64temp;
1917 dma_addr_t bus_address;
1918 struct nes_device *nesdev = nesvnic->nesdev;
1919 struct nes_hw_cqp_wqe *cqp_wqe;
1920 struct nes_hw_nic_sq_wqe *nic_sqe;
1921 __le16 *wqe_fragment_length;
1922 u16 wqe_fragment_index;
1923 u32 cqp_head;
1924 u32 wqm_cfg0;
1925 unsigned long flags;
1926 struct sk_buff *rx_skb;
1927 struct nes_rskb_cb *cb;
1928 int ret;
1929
1930 if (nesdev->nesadapter->allow_unaligned_fpdus)
1931 nes_destroy_mgt(nesvnic);
1932
1933 /* clear wqe stall before destroying NIC QP */
1934 wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0);
1935 nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF);
1936
1937 /* Free remaining NIC receive buffers */
1938 while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) {
1939 rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail];
1940 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
1941 pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen,
1942 PCI_DMA_FROMDEVICE);
1943
1944 dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]);
1945 nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1);
1946 }
1947
1948 /* Free remaining NIC transmit buffers */
1949 while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) {
1950 nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail];
1951 wqe_fragment_index = 1;
1952 wqe_fragment_length = (__le16 *)
1953 &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
1954 /* bump past the vlan tag */
1955 wqe_fragment_length++;
1956 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
1957 u64temp = (u64)le32_to_cpu(
1958 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1959 wqe_fragment_index*2]);
1960 u64temp += ((u64)le32_to_cpu(
1961 nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
1962 + wqe_fragment_index*2]))<<32;
1963 bus_address = (dma_addr_t)u64temp;
1964 if (test_and_clear_bit(nesvnic->nic.sq_tail,
1965 nesvnic->nic.first_frag_overflow)) {
1966 pci_unmap_single(nesdev->pcidev,
1967 bus_address,
1968 le16_to_cpu(wqe_fragment_length[
1969 wqe_fragment_index++]),
1970 PCI_DMA_TODEVICE);
1971 }
1972 for (; wqe_fragment_index < 5; wqe_fragment_index++) {
1973 if (wqe_fragment_length[wqe_fragment_index]) {
1974 u64temp = le32_to_cpu(
1975 nic_sqe->wqe_words[
1976 NES_NIC_SQ_WQE_FRAG0_LOW_IDX+
1977 wqe_fragment_index*2]);
1978 u64temp += ((u64)le32_to_cpu(
1979 nic_sqe->wqe_words[
1980 NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+
1981 wqe_fragment_index*2]))<<32;
1982 bus_address = (dma_addr_t)u64temp;
1983 pci_unmap_page(nesdev->pcidev,
1984 bus_address,
1985 le16_to_cpu(
1986 wqe_fragment_length[
1987 wqe_fragment_index]),
1988 PCI_DMA_TODEVICE);
1989 } else
1990 break;
1991 }
1992 }
1993 if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail])
1994 dev_kfree_skb(
1995 nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]);
1996
1997 nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1)
1998 & (nesvnic->nic.sq_size - 1);
1999 }
2000
2001 spin_lock_irqsave(&nesdev->cqp.lock, flags);
2002
2003 /* Destroy NIC QP */
2004 cqp_head = nesdev->cqp.sq_head;
2005 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2006 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2007
2008 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2009 (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC));
2010 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2011 nesvnic->nic.qp_id);
2012
2013 if (++cqp_head >= nesdev->cqp.sq_size)
2014 cqp_head = 0;
2015
2016 cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head];
2017
2018 /* Destroy NIC CQ */
2019 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
2020 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX,
2021 (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16)));
2022 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
2023 (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)));
2024
2025 if (++cqp_head >= nesdev->cqp.sq_size)
2026 cqp_head = 0;
2027
2028 nesdev->cqp.sq_head = cqp_head;
2029 barrier();
2030
2031 /* Ring doorbell (2 WQEs) */
2032 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id);
2033
2034 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
2035 nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u,"
2036 " cqp.sq_tail=%u, cqp.sq_size=%u\n",
2037 cqp_head, nesdev->cqp.sq_head,
2038 nesdev->cqp.sq_tail, nesdev->cqp.sq_size);
2039
2040 ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head),
2041 NES_EVENT_TIMEOUT);
2042
2043 nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u,"
2044 " cqp.sq_head=%u, cqp.sq_tail=%u\n",
2045 ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
2046 if (!ret) {
2047 nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n",
2048 nesvnic->nic.qp_id);
2049 }
2050
2051 pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase,
2052 nesvnic->nic_pbase);
2053
2054 /* restore old wqm_cfg0 value */
2055 nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0);
2056 }
2057
2058 /**
2059 * nes_napi_isr
2060 */
2061 int nes_napi_isr(struct nes_device *nesdev)
2062 {
2063 struct nes_adapter *nesadapter = nesdev->nesadapter;
2064 u32 int_stat;
2065
2066 if (nesdev->napi_isr_ran) {
2067 /* interrupt status has already been read in ISR */
2068 int_stat = nesdev->int_stat;
2069 } else {
2070 int_stat = nes_read32(nesdev->regs + NES_INT_STAT);
2071 nesdev->int_stat = int_stat;
2072 nesdev->napi_isr_ran = 1;
2073 }
2074
2075 int_stat &= nesdev->int_req;
2076 /* iff NIC, process here, else wait for DPC */
2077 if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) {
2078 nesdev->napi_isr_ran = 0;
2079 nes_write32(nesdev->regs + NES_INT_STAT,
2080 (int_stat &
2081 ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2082
2083 /* Process the CEQs */
2084 nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]);
2085
2086 if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) &&
2087 (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2088 ((nesadapter->et_use_adaptive_rx_coalesce) &&
2089 (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) {
2090 if ((nesdev->int_req & NES_INT_TIMER) == 0) {
2091 /* Enable Periodic timer interrupts */
2092 nesdev->int_req |= NES_INT_TIMER;
2093 /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */
2094 /* TODO: need to also ack other unused periodic timer values, get from nesadapter */
2095 nes_write32(nesdev->regs+NES_TIMER_STAT,
2096 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2097 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2098 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2099 }
2100
2101 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2102 {
2103 nes_nic_init_timer(nesdev);
2104 }
2105 /* Enable interrupts, except CEQs */
2106 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2107 } else {
2108 /* Enable interrupts, make sure timer is off */
2109 nesdev->int_req &= ~NES_INT_TIMER;
2110 nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2111 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2112 }
2113 nesdev->deepcq_count = 0;
2114 return 1;
2115 } else {
2116 return 0;
2117 }
2118 }
2119
2120 static void process_critical_error(struct nes_device *nesdev)
2121 {
2122 u32 debug_error;
2123 u32 nes_idx_debug_error_masks0 = 0;
2124 u16 error_module = 0;
2125
2126 debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS);
2127 printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n",
2128 (u16)debug_error);
2129 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS,
2130 0x01010000 | (debug_error & 0x0000ffff));
2131 if (crit_err_count++ > 10)
2132 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17);
2133 error_module = (u16) (debug_error & 0x1F00) >> 8;
2134 if (++nesdev->nesadapter->crit_error_count[error_module-1] >=
2135 nes_max_critical_error_count) {
2136 printk(KERN_ERR PFX "Masking off critical error for module "
2137 "0x%02X\n", (u16)error_module);
2138 nes_idx_debug_error_masks0 = nes_read_indexed(nesdev,
2139 NES_IDX_DEBUG_ERROR_MASKS0);
2140 nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0,
2141 nes_idx_debug_error_masks0 | (1 << error_module));
2142 }
2143 }
2144 /**
2145 * nes_dpc
2146 */
2147 void nes_dpc(unsigned long param)
2148 {
2149 struct nes_device *nesdev = (struct nes_device *)param;
2150 struct nes_adapter *nesadapter = nesdev->nesadapter;
2151 u32 counter;
2152 u32 loop_counter = 0;
2153 u32 int_status_bit;
2154 u32 int_stat;
2155 u32 timer_stat;
2156 u32 temp_int_stat;
2157 u32 intf_int_stat;
2158 u32 processed_intf_int = 0;
2159 u16 processed_timer_int = 0;
2160 u16 completion_ints = 0;
2161 u16 timer_ints = 0;
2162
2163 /* nes_debug(NES_DBG_ISR, "\n"); */
2164
2165 do {
2166 timer_stat = 0;
2167 if (nesdev->napi_isr_ran) {
2168 nesdev->napi_isr_ran = 0;
2169 int_stat = nesdev->int_stat;
2170 } else
2171 int_stat = nes_read32(nesdev->regs+NES_INT_STAT);
2172 if (processed_intf_int != 0)
2173 int_stat &= nesdev->int_req & ~NES_INT_INTF;
2174 else
2175 int_stat &= nesdev->int_req;
2176 if (processed_timer_int == 0) {
2177 processed_timer_int = 1;
2178 if (int_stat & NES_INT_TIMER) {
2179 timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT);
2180 if ((timer_stat & nesdev->timer_int_req) == 0) {
2181 int_stat &= ~NES_INT_TIMER;
2182 }
2183 }
2184 } else {
2185 int_stat &= ~NES_INT_TIMER;
2186 }
2187
2188 if (int_stat) {
2189 if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2190 NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) {
2191 /* Ack the interrupts */
2192 nes_write32(nesdev->regs+NES_INT_STAT,
2193 (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0|
2194 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3)));
2195 }
2196
2197 temp_int_stat = int_stat;
2198 for (counter = 0, int_status_bit = 1; counter < 16; counter++) {
2199 if (int_stat & int_status_bit) {
2200 nes_process_ceq(nesdev, &nesadapter->ceq[counter]);
2201 temp_int_stat &= ~int_status_bit;
2202 completion_ints = 1;
2203 }
2204 if (!(temp_int_stat & 0x0000ffff))
2205 break;
2206 int_status_bit <<= 1;
2207 }
2208
2209 /* Process the AEQ for this pci function */
2210 int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn));
2211 if (int_stat & int_status_bit) {
2212 nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]);
2213 }
2214
2215 /* Process the MAC interrupt for this pci function */
2216 int_status_bit = 1 << (24 + nesdev->mac_index);
2217 if (int_stat & int_status_bit) {
2218 nes_process_mac_intr(nesdev, nesdev->mac_index);
2219 }
2220
2221 if (int_stat & NES_INT_TIMER) {
2222 if (timer_stat & nesdev->timer_int_req) {
2223 nes_write32(nesdev->regs + NES_TIMER_STAT,
2224 (timer_stat & nesdev->timer_int_req) |
2225 ~(nesdev->nesadapter->timer_int_req));
2226 timer_ints = 1;
2227 }
2228 }
2229
2230 if (int_stat & NES_INT_INTF) {
2231 processed_intf_int = 1;
2232 intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT);
2233 intf_int_stat &= nesdev->intf_int_req;
2234 if (NES_INTF_INT_CRITERR & intf_int_stat) {
2235 process_critical_error(nesdev);
2236 }
2237 if (NES_INTF_INT_PCIERR & intf_int_stat) {
2238 printk(KERN_ERR PFX "PCI Error reported by device!!!\n");
2239 BUG();
2240 }
2241 if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) {
2242 printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n");
2243 BUG();
2244 }
2245 nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat);
2246 }
2247
2248 if (int_stat & NES_INT_TSW) {
2249 }
2250 }
2251 /* Don't use the interface interrupt bit stay in loop */
2252 int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 |
2253 NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3;
2254 } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS));
2255
2256 if (timer_ints == 1) {
2257 if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) {
2258 if (completion_ints == 0) {
2259 nesdev->timer_only_int_count++;
2260 if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) {
2261 nesdev->timer_only_int_count = 0;
2262 nesdev->int_req &= ~NES_INT_TIMER;
2263 nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2264 nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req);
2265 } else {
2266 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2267 }
2268 } else {
2269 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
2270 {
2271 nes_nic_init_timer(nesdev);
2272 }
2273 nesdev->timer_only_int_count = 0;
2274 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2275 }
2276 } else {
2277 nesdev->timer_only_int_count = 0;
2278 nesdev->int_req &= ~NES_INT_TIMER;
2279 nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req));
2280 nes_write32(nesdev->regs+NES_TIMER_STAT,
2281 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2282 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2283 }
2284 } else {
2285 if ( (completion_ints == 1) &&
2286 (((nesadapter->et_rx_coalesce_usecs_irq) &&
2287 (!nesadapter->et_use_adaptive_rx_coalesce)) ||
2288 ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) &&
2289 (nesadapter->et_use_adaptive_rx_coalesce) )) ) {
2290 /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */
2291 nesdev->timer_only_int_count = 0;
2292 nesdev->int_req |= NES_INT_TIMER;
2293 nes_write32(nesdev->regs+NES_TIMER_STAT,
2294 nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req));
2295 nes_write32(nesdev->regs+NES_INTF_INT_MASK,
2296 ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER));
2297 nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req));
2298 } else {
2299 nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req);
2300 }
2301 }
2302 nesdev->deepcq_count = 0;
2303 }
2304
2305
2306 /**
2307 * nes_process_ceq
2308 */
2309 static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq)
2310 {
2311 u64 u64temp;
2312 struct nes_hw_cq *cq;
2313 u32 head;
2314 u32 ceq_size;
2315
2316 /* nes_debug(NES_DBG_CQ, "\n"); */
2317 head = ceq->ceq_head;
2318 ceq_size = ceq->ceq_size;
2319
2320 do {
2321 if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) &
2322 NES_CEQE_VALID) {
2323 u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) |
2324 ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX])));
2325 u64temp <<= 1;
2326 cq = *((struct nes_hw_cq **)&u64temp);
2327 /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */
2328 barrier();
2329 ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0;
2330
2331 /* call the event handler */
2332 cq->ce_handler(nesdev, cq);
2333
2334 if (++head >= ceq_size)
2335 head = 0;
2336 } else {
2337 break;
2338 }
2339
2340 } while (1);
2341
2342 ceq->ceq_head = head;
2343 }
2344
2345
2346 /**
2347 * nes_process_aeq
2348 */
2349 static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq)
2350 {
2351 /* u64 u64temp; */
2352 u32 head;
2353 u32 aeq_size;
2354 u32 aeqe_misc;
2355 u32 aeqe_cq_id;
2356 struct nes_hw_aeqe volatile *aeqe;
2357
2358 head = aeq->aeq_head;
2359 aeq_size = aeq->aeq_size;
2360
2361 do {
2362 aeqe = &aeq->aeq_vbase[head];
2363 if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0)
2364 break;
2365 aeqe_misc = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
2366 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
2367 if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) {
2368 if (aeqe_cq_id >= NES_FIRST_QPN) {
2369 /* dealing with an accelerated QP related AE */
2370 /*
2371 * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) |
2372 * ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX])));
2373 */
2374 nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe);
2375 } else {
2376 /* TODO: dealing with a CQP related AE */
2377 nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n",
2378 (u16)(aeqe_misc >> 16));
2379 }
2380 }
2381
2382 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0;
2383
2384 if (++head >= aeq_size)
2385 head = 0;
2386
2387 nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16);
2388 }
2389 while (1);
2390 aeq->aeq_head = head;
2391 }
2392
2393 static void nes_reset_link(struct nes_device *nesdev, u32 mac_index)
2394 {
2395 struct nes_adapter *nesadapter = nesdev->nesadapter;
2396 u32 reset_value;
2397 u32 i=0;
2398 u32 u32temp;
2399
2400 if (nesadapter->hw_rev == NE020_REV) {
2401 return;
2402 }
2403 mh_detected++;
2404
2405 reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET);
2406
2407 if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode)))
2408 reset_value |= 0x0000001d;
2409 else
2410 reset_value |= 0x0000002d;
2411
2412 if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) {
2413 if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) {
2414 nesadapter->link_interrupt_count[0] = 0;
2415 nesadapter->link_interrupt_count[1] = 0;
2416 u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2417 if (0x00000040 & u32temp)
2418 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2419 else
2420 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2421
2422 reset_value |= 0x0000003d;
2423 }
2424 nesadapter->link_interrupt_count[mac_index] = 0;
2425 }
2426
2427 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2428
2429 while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET)
2430 & 0x00000040) != 0x00000040) && (i++ < 5000));
2431
2432 if (0x0000003d == (reset_value & 0x0000003d)) {
2433 u32 pcs_control_status0, pcs_control_status1;
2434
2435 for (i = 0; i < 10; i++) {
2436 pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0);
2437 pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2438 if (((0x0F000000 == (pcs_control_status0 & 0x0F000000))
2439 && (pcs_control_status0 & 0x00100000))
2440 || ((0x0F000000 == (pcs_control_status1 & 0x0F000000))
2441 && (pcs_control_status1 & 0x00100000)))
2442 continue;
2443 else
2444 break;
2445 }
2446 if (10 == i) {
2447 u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1);
2448 if (0x00000040 & u32temp)
2449 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088);
2450 else
2451 nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8);
2452
2453 nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value);
2454
2455 while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET)
2456 & 0x00000040) != 0x00000040) && (i++ < 5000));
2457 }
2458 }
2459 }
2460
2461 /**
2462 * nes_process_mac_intr
2463 */
2464 static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number)
2465 {
2466 unsigned long flags;
2467 u32 pcs_control_status;
2468 struct nes_adapter *nesadapter = nesdev->nesadapter;
2469 struct nes_vnic *nesvnic;
2470 u32 mac_status;
2471 u32 mac_index = nesdev->mac_index;
2472 u32 u32temp;
2473 u16 phy_data;
2474 u16 temp_phy_data;
2475 u32 pcs_val = 0x0f0f0000;
2476 u32 pcs_mask = 0x0f1f0000;
2477 u32 cdr_ctrl;
2478
2479 spin_lock_irqsave(&nesadapter->phy_lock, flags);
2480 if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) {
2481 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2482 return;
2483 }
2484 nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT;
2485
2486 /* ack the MAC interrupt */
2487 mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200));
2488 /* Clear the interrupt */
2489 nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status);
2490
2491 nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status);
2492
2493 if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) {
2494 nesdev->link_status_interrupts++;
2495 if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS)))
2496 nes_reset_link(nesdev, mac_index);
2497
2498 /* read the PHY interrupt status register */
2499 if ((nesadapter->OneG_Mode) &&
2500 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2501 do {
2502 nes_read_1G_phy_reg(nesdev, 0x1a,
2503 nesadapter->phy_index[mac_index], &phy_data);
2504 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n",
2505 nesadapter->phy_index[mac_index], phy_data);
2506 } while (phy_data&0x8000);
2507
2508 temp_phy_data = 0;
2509 do {
2510 nes_read_1G_phy_reg(nesdev, 0x11,
2511 nesadapter->phy_index[mac_index], &phy_data);
2512 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n",
2513 nesadapter->phy_index[mac_index], phy_data);
2514 if (temp_phy_data == phy_data)
2515 break;
2516 temp_phy_data = phy_data;
2517 } while (1);
2518
2519 nes_read_1G_phy_reg(nesdev, 0x1e,
2520 nesadapter->phy_index[mac_index], &phy_data);
2521 nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n",
2522 nesadapter->phy_index[mac_index], phy_data);
2523
2524 nes_read_1G_phy_reg(nesdev, 1,
2525 nesadapter->phy_index[mac_index], &phy_data);
2526 nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n",
2527 nesadapter->phy_index[mac_index], phy_data);
2528
2529 if (temp_phy_data & 0x1000) {
2530 nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n");
2531 phy_data = 4;
2532 } else {
2533 nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n");
2534 }
2535 }
2536 nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n",
2537 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0),
2538 nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200));
2539
2540 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) {
2541 switch (mac_index) {
2542 case 1:
2543 case 3:
2544 pcs_control_status = nes_read_indexed(nesdev,
2545 NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200);
2546 break;
2547 default:
2548 pcs_control_status = nes_read_indexed(nesdev,
2549 NES_IDX_PHY_PCS_CONTROL_STATUS0);
2550 break;
2551 }
2552 } else {
2553 pcs_control_status = nes_read_indexed(nesdev,
2554 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2555 pcs_control_status = nes_read_indexed(nesdev,
2556 NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200));
2557 }
2558
2559 nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n",
2560 mac_index, pcs_control_status);
2561 if ((nesadapter->OneG_Mode) &&
2562 (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) {
2563 u32temp = 0x01010000;
2564 if (nesadapter->port_count > 2) {
2565 u32temp |= 0x02020000;
2566 }
2567 if ((pcs_control_status & u32temp)!= u32temp) {
2568 phy_data = 0;
2569 nes_debug(NES_DBG_PHY, "PCS says the link is down\n");
2570 }
2571 } else {
2572 switch (nesadapter->phy_type[mac_index]) {
2573 case NES_PHY_TYPE_ARGUS:
2574 case NES_PHY_TYPE_SFP_D:
2575 case NES_PHY_TYPE_KR:
2576 /* clear the alarms */
2577 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008);
2578 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001);
2579 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002);
2580 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005);
2581 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006);
2582 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2583 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004);
2584 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005);
2585 /* check link status */
2586 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2587 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2588
2589 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2590 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2591 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2592 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2593
2594 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2595
2596 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2597 __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2598 break;
2599
2600 case NES_PHY_TYPE_PUMA_1G:
2601 if (mac_index < 2)
2602 pcs_val = pcs_mask = 0x01010000;
2603 else
2604 pcs_val = pcs_mask = 0x02020000;
2605 /* fall through */
2606 default:
2607 phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0;
2608 break;
2609 }
2610 }
2611
2612 if (phy_data & 0x0004) {
2613 if (wide_ppm_offset &&
2614 (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2615 (nesadapter->hw_rev != NE020_REV)) {
2616 cdr_ctrl = nes_read_indexed(nesdev,
2617 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2618 mac_index * 0x200);
2619 nes_write_indexed(nesdev,
2620 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2621 mac_index * 0x200,
2622 cdr_ctrl | 0x000F0000);
2623 }
2624 nesadapter->mac_link_down[mac_index] = 0;
2625 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2626 nes_debug(NES_DBG_PHY, "The Link is UP!!. linkup was %d\n",
2627 nesvnic->linkup);
2628 if (nesvnic->linkup == 0) {
2629 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2630 nesvnic->netdev->name, nesvnic->netdev);
2631 if (netif_queue_stopped(nesvnic->netdev))
2632 netif_start_queue(nesvnic->netdev);
2633 nesvnic->linkup = 1;
2634 netif_carrier_on(nesvnic->netdev);
2635
2636 spin_lock(&nesvnic->port_ibevent_lock);
2637 if (nesvnic->of_device_registered) {
2638 if (nesdev->iw_status == 0) {
2639 nesdev->iw_status = 1;
2640 nes_port_ibevent(nesvnic);
2641 }
2642 }
2643 spin_unlock(&nesvnic->port_ibevent_lock);
2644 }
2645 }
2646 } else {
2647 if (wide_ppm_offset &&
2648 (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) &&
2649 (nesadapter->hw_rev != NE020_REV)) {
2650 cdr_ctrl = nes_read_indexed(nesdev,
2651 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2652 mac_index * 0x200);
2653 nes_write_indexed(nesdev,
2654 NES_IDX_ETH_SERDES_CDR_CONTROL0 +
2655 mac_index * 0x200,
2656 cdr_ctrl & 0xFFF0FFFF);
2657 }
2658 nesadapter->mac_link_down[mac_index] = 1;
2659 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2660 nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n",
2661 nesvnic->linkup);
2662 if (nesvnic->linkup == 1) {
2663 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2664 nesvnic->netdev->name, nesvnic->netdev);
2665 if (!(netif_queue_stopped(nesvnic->netdev)))
2666 netif_stop_queue(nesvnic->netdev);
2667 nesvnic->linkup = 0;
2668 netif_carrier_off(nesvnic->netdev);
2669
2670 spin_lock(&nesvnic->port_ibevent_lock);
2671 if (nesvnic->of_device_registered) {
2672 if (nesdev->iw_status == 1) {
2673 nesdev->iw_status = 0;
2674 nes_port_ibevent(nesvnic);
2675 }
2676 }
2677 spin_unlock(&nesvnic->port_ibevent_lock);
2678 }
2679 }
2680 }
2681 if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) {
2682 if (nesdev->link_recheck)
2683 cancel_delayed_work(&nesdev->work);
2684 nesdev->link_recheck = 1;
2685 schedule_delayed_work(&nesdev->work,
2686 NES_LINK_RECHECK_DELAY);
2687 }
2688 }
2689
2690 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2691
2692 nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE;
2693 }
2694
2695 void nes_recheck_link_status(struct work_struct *work)
2696 {
2697 unsigned long flags;
2698 struct nes_device *nesdev = container_of(work, struct nes_device, work.work);
2699 struct nes_adapter *nesadapter = nesdev->nesadapter;
2700 struct nes_vnic *nesvnic;
2701 u32 mac_index = nesdev->mac_index;
2702 u16 phy_data;
2703 u16 temp_phy_data;
2704
2705 spin_lock_irqsave(&nesadapter->phy_lock, flags);
2706
2707 /* check link status */
2708 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003);
2709 temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2710
2711 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2712 nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2713 nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021);
2714 phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL);
2715
2716 phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0;
2717
2718 nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n",
2719 __func__, phy_data,
2720 nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP");
2721
2722 if (phy_data & 0x0004) {
2723 nesadapter->mac_link_down[mac_index] = 0;
2724 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2725 if (nesvnic->linkup == 0) {
2726 printk(PFX "The Link is now up for port %s, netdev %p.\n",
2727 nesvnic->netdev->name, nesvnic->netdev);
2728 if (netif_queue_stopped(nesvnic->netdev))
2729 netif_start_queue(nesvnic->netdev);
2730 nesvnic->linkup = 1;
2731 netif_carrier_on(nesvnic->netdev);
2732
2733 spin_lock(&nesvnic->port_ibevent_lock);
2734 if (nesvnic->of_device_registered) {
2735 if (nesdev->iw_status == 0) {
2736 nesdev->iw_status = 1;
2737 nes_port_ibevent(nesvnic);
2738 }
2739 }
2740 spin_unlock(&nesvnic->port_ibevent_lock);
2741 }
2742 }
2743
2744 } else {
2745 nesadapter->mac_link_down[mac_index] = 1;
2746 list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) {
2747 if (nesvnic->linkup == 1) {
2748 printk(PFX "The Link is now down for port %s, netdev %p.\n",
2749 nesvnic->netdev->name, nesvnic->netdev);
2750 if (!(netif_queue_stopped(nesvnic->netdev)))
2751 netif_stop_queue(nesvnic->netdev);
2752 nesvnic->linkup = 0;
2753 netif_carrier_off(nesvnic->netdev);
2754
2755 spin_lock(&nesvnic->port_ibevent_lock);
2756 if (nesvnic->of_device_registered) {
2757 if (nesdev->iw_status == 1) {
2758 nesdev->iw_status = 0;
2759 nes_port_ibevent(nesvnic);
2760 }
2761 }
2762 spin_unlock(&nesvnic->port_ibevent_lock);
2763 }
2764 }
2765 }
2766 if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX)
2767 schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY);
2768 else
2769 nesdev->link_recheck = 0;
2770
2771 spin_unlock_irqrestore(&nesadapter->phy_lock, flags);
2772 }
2773
2774
2775 static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2776 {
2777 struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2778
2779 napi_schedule(&nesvnic->napi);
2780 }
2781
2782
2783 /* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before
2784 * getting out of nic_ce_handler
2785 */
2786 #define MAX_RQES_TO_PROCESS 384
2787
2788 /**
2789 * nes_nic_ce_handler
2790 */
2791 void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq)
2792 {
2793 u64 u64temp;
2794 dma_addr_t bus_address;
2795 struct nes_hw_nic *nesnic;
2796 struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq);
2797 struct nes_adapter *nesadapter = nesdev->nesadapter;
2798 struct nes_hw_nic_rq_wqe *nic_rqe;
2799 struct nes_hw_nic_sq_wqe *nic_sqe;
2800 struct sk_buff *skb;
2801 struct sk_buff *rx_skb;
2802 struct nes_rskb_cb *cb;
2803 __le16 *wqe_fragment_length;
2804 u32 head;
2805 u32 cq_size;
2806 u32 rx_pkt_size;
2807 u32 cqe_count=0;
2808 u32 cqe_errv;
2809 u32 cqe_misc;
2810 u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */
2811 u16 vlan_tag;
2812 u16 pkt_type;
2813 u16 rqes_processed = 0;
2814 u8 sq_cqes = 0;
2815 u8 nes_use_lro = 0;
2816
2817 head = cq->cq_head;
2818 cq_size = cq->cq_size;
2819 cq->cqes_pending = 1;
2820 if (nesvnic->netdev->features & NETIF_F_LRO)
2821 nes_use_lro = 1;
2822 do {
2823 if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) &
2824 NES_NIC_CQE_VALID) {
2825 nesnic = &nesvnic->nic;
2826 cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]);
2827 if (cqe_misc & NES_NIC_CQE_SQ) {
2828 sq_cqes++;
2829 wqe_fragment_index = 1;
2830 nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail];
2831 skb = nesnic->tx_skb[nesnic->sq_tail];
2832 wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX];
2833 /* bump past the vlan tag */
2834 wqe_fragment_length++;
2835 if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) {
2836 u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2837 wqe_fragment_index * 2]);
2838 u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX +
2839 wqe_fragment_index * 2])) << 32;
2840 bus_address = (dma_addr_t)u64temp;
2841 if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) {
2842 pci_unmap_single(nesdev->pcidev,
2843 bus_address,
2844 le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]),
2845 PCI_DMA_TODEVICE);
2846 }
2847 for (; wqe_fragment_index < 5; wqe_fragment_index++) {
2848 if (wqe_fragment_length[wqe_fragment_index]) {
2849 u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX +
2850 wqe_fragment_index * 2]);
2851 u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX
2852 + wqe_fragment_index * 2])) <<32;
2853 bus_address = (dma_addr_t)u64temp;
2854 pci_unmap_page(nesdev->pcidev,
2855 bus_address,
2856 le16_to_cpu(wqe_fragment_length[wqe_fragment_index]),
2857 PCI_DMA_TODEVICE);
2858 } else
2859 break;
2860 }
2861 }
2862 if (skb)
2863 dev_kfree_skb_any(skb);
2864 nesnic->sq_tail++;
2865 nesnic->sq_tail &= nesnic->sq_size-1;
2866 if (sq_cqes > 128) {
2867 barrier();
2868 /* restart the queue if it had been stopped */
2869 if (netif_queue_stopped(nesvnic->netdev))
2870 netif_wake_queue(nesvnic->netdev);
2871 sq_cqes = 0;
2872 }
2873 } else {
2874 rqes_processed ++;
2875
2876 cq->rx_cqes_completed++;
2877 cq->rx_pkts_indicated++;
2878 rx_pkt_size = cqe_misc & 0x0000ffff;
2879 nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail];
2880 /* Get the skb */
2881 rx_skb = nesnic->rx_skb[nesnic->rq_tail];
2882 nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail];
2883 bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]);
2884 bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32;
2885 pci_unmap_single(nesdev->pcidev, bus_address,
2886 nesvnic->max_frame_size, PCI_DMA_FROMDEVICE);
2887 cb = (struct nes_rskb_cb *)&rx_skb->cb[0];
2888 cb->busaddr = 0;
2889 /* rx_skb->tail = rx_skb->data + rx_pkt_size; */
2890 /* rx_skb->len = rx_pkt_size; */
2891 rx_skb->len = 0; /* TODO: see if this is necessary */
2892 skb_put(rx_skb, rx_pkt_size);
2893 rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev);
2894 nesnic->rq_tail++;
2895 nesnic->rq_tail &= nesnic->rq_size - 1;
2896
2897 atomic_inc(&nesvnic->rx_skbs_needed);
2898 if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) {
2899 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2900 cq->cq_number | (cqe_count << 16));
2901 /* nesadapter->tune_timer.cq_count += cqe_count; */
2902 nesdev->currcq_count += cqe_count;
2903 cqe_count = 0;
2904 nes_replenish_nic_rq(nesvnic);
2905 }
2906 pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]));
2907 cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT;
2908 rx_skb->ip_summed = CHECKSUM_NONE;
2909
2910 if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) ||
2911 (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) {
2912 if ((cqe_errv &
2913 (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR |
2914 NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2915 if (nesvnic->netdev->features & NETIF_F_RXCSUM)
2916 rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2917 } else
2918 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2919 " errv = 0x%X, pkt_type = 0x%X.\n",
2920 nesvnic->netdev->name, cqe_errv, pkt_type);
2921
2922 } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) {
2923 if ((cqe_errv &
2924 (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR |
2925 NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) {
2926 if (nesvnic->netdev->features & NETIF_F_RXCSUM) {
2927 rx_skb->ip_summed = CHECKSUM_UNNECESSARY;
2928 /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n",
2929 nesvnic->netdev->name); */
2930 }
2931 } else
2932 nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet."
2933 " errv = 0x%X, pkt_type = 0x%X.\n",
2934 nesvnic->netdev->name, cqe_errv, pkt_type);
2935 }
2936 /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n",
2937 pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */
2938
2939 if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) {
2940 if (nes_cm_recv(rx_skb, nesvnic->netdev))
2941 rx_skb = NULL;
2942 }
2943 if (rx_skb == NULL)
2944 goto skip_rx_indicate0;
2945
2946
2947 if (cqe_misc & NES_NIC_CQE_TAG_VALID) {
2948 vlan_tag = (u16)(le32_to_cpu(
2949 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])
2950 >> 16);
2951 nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n",
2952 nesvnic->netdev->name, vlan_tag);
2953
2954 __vlan_hwaccel_put_tag(rx_skb, vlan_tag);
2955 }
2956 if (nes_use_lro)
2957 lro_receive_skb(&nesvnic->lro_mgr, rx_skb, NULL);
2958 else
2959 netif_receive_skb(rx_skb);
2960
2961 skip_rx_indicate0:
2962 ;
2963 /* nesvnic->netstats.rx_packets++; */
2964 /* nesvnic->netstats.rx_bytes += rx_pkt_size; */
2965 }
2966
2967 cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0;
2968 /* Accounting... */
2969 cqe_count++;
2970 if (++head >= cq_size)
2971 head = 0;
2972 if (cqe_count == 255) {
2973 /* Replenish Nic CQ */
2974 nes_write32(nesdev->regs+NES_CQE_ALLOC,
2975 cq->cq_number | (cqe_count << 16));
2976 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
2977 nesdev->currcq_count += cqe_count;
2978 cqe_count = 0;
2979 }
2980
2981 if (cq->rx_cqes_completed >= nesvnic->budget)
2982 break;
2983 } else {
2984 cq->cqes_pending = 0;
2985 break;
2986 }
2987
2988 } while (1);
2989
2990 if (nes_use_lro)
2991 lro_flush_all(&nesvnic->lro_mgr);
2992 if (sq_cqes) {
2993 barrier();
2994 /* restart the queue if it had been stopped */
2995 if (netif_queue_stopped(nesvnic->netdev))
2996 netif_wake_queue(nesvnic->netdev);
2997 }
2998 cq->cq_head = head;
2999 /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n",
3000 cq->cq_number, cqe_count, cq->cq_head); */
3001 cq->cqe_allocs_pending = cqe_count;
3002 if (unlikely(nesadapter->et_use_adaptive_rx_coalesce))
3003 {
3004 /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */
3005 nesdev->currcq_count += cqe_count;
3006 nes_nic_tune_timer(nesdev);
3007 }
3008 if (atomic_read(&nesvnic->rx_skbs_needed))
3009 nes_replenish_nic_rq(nesvnic);
3010 }
3011
3012
3013
3014 /**
3015 * nes_cqp_ce_handler
3016 */
3017 static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq)
3018 {
3019 u64 u64temp;
3020 unsigned long flags;
3021 struct nes_hw_cqp *cqp = NULL;
3022 struct nes_cqp_request *cqp_request;
3023 struct nes_hw_cqp_wqe *cqp_wqe;
3024 u32 head;
3025 u32 cq_size;
3026 u32 cqe_count=0;
3027 u32 error_code;
3028 u32 opcode;
3029 u32 ctx_index;
3030 /* u32 counter; */
3031
3032 head = cq->cq_head;
3033 cq_size = cq->cq_size;
3034
3035 do {
3036 /* process the CQE */
3037 /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head,
3038 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */
3039
3040 opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]);
3041 if (opcode & NES_CQE_VALID) {
3042 cqp = &nesdev->cqp;
3043
3044 error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]);
3045 if (error_code) {
3046 nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP,"
3047 " Major/Minor codes = 0x%04X:%04X.\n",
3048 le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f,
3049 (u16)(error_code >> 16),
3050 (u16)error_code);
3051 }
3052
3053 u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head].
3054 cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) |
3055 ((u64)(le32_to_cpu(cq->cq_vbase[head].
3056 cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX])));
3057
3058 cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp;
3059 if (cqp_request) {
3060 if (cqp_request->waiting) {
3061 /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */
3062 cqp_request->major_code = (u16)(error_code >> 16);
3063 cqp_request->minor_code = (u16)error_code;
3064 barrier();
3065 cqp_request->request_done = 1;
3066 wake_up(&cqp_request->waitq);
3067 nes_put_cqp_request(nesdev, cqp_request);
3068 } else {
3069 if (cqp_request->callback)
3070 cqp_request->cqp_callback(nesdev, cqp_request);
3071 nes_free_cqp_request(nesdev, cqp_request);
3072 }
3073 } else {
3074 wake_up(&nesdev->cqp.waitq);
3075 }
3076
3077 cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0;
3078 nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16));
3079 if (++cqp->sq_tail >= cqp->sq_size)
3080 cqp->sq_tail = 0;
3081
3082 /* Accounting... */
3083 cqe_count++;
3084 if (++head >= cq_size)
3085 head = 0;
3086 } else {
3087 break;
3088 }
3089 } while (1);
3090 cq->cq_head = head;
3091
3092 spin_lock_irqsave(&nesdev->cqp.lock, flags);
3093 while ((!list_empty(&nesdev->cqp_pending_reqs)) &&
3094 ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) &
3095 (nesdev->cqp.sq_size - 1)) != 1)) {
3096 cqp_request = list_entry(nesdev->cqp_pending_reqs.next,
3097 struct nes_cqp_request, list);
3098 list_del_init(&cqp_request->list);
3099 head = nesdev->cqp.sq_head++;
3100 nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1;
3101 cqp_wqe = &nesdev->cqp.sq_vbase[head];
3102 memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe));
3103 barrier();
3104
3105 opcode = cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX];
3106 if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT)
3107 ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX;
3108 else
3109 ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX;
3110 cqp_wqe->wqe_words[ctx_index] =
3111 cpu_to_le32((u32)((unsigned long)cqp_request));
3112 cqp_wqe->wqe_words[ctx_index + 1] =
3113 cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request)));
3114 nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n",
3115 cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head);
3116 /* Ring doorbell (1 WQEs) */
3117 barrier();
3118 nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id);
3119 }
3120 spin_unlock_irqrestore(&nesdev->cqp.lock, flags);
3121
3122 /* Arm the CCQ */
3123 nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT |
3124 cq->cq_number);
3125 nes_read32(nesdev->regs+NES_CQE_ALLOC);
3126 }
3127
3128 static u8 *locate_mpa(u8 *pkt, u32 aeq_info)
3129 {
3130 if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) {
3131 /* skip over ethernet header */
3132 pkt += ETH_HLEN;
3133
3134 /* Skip over IP and TCP headers */
3135 pkt += 4 * (pkt[0] & 0x0f);
3136 pkt += 4 * ((pkt[12] >> 4) & 0x0f);
3137 }
3138 return pkt;
3139 }
3140
3141 /* Determine if incoming error pkt is rdma layer */
3142 static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info)
3143 {
3144 u8 *pkt;
3145 u16 *mpa;
3146 u32 opcode = 0xffffffff;
3147
3148 if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3149 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3150 mpa = (u16 *)locate_mpa(pkt, aeq_info);
3151 opcode = be16_to_cpu(mpa[1]) & 0xf;
3152 }
3153
3154 return opcode;
3155 }
3156
3157 /* Build iWARP terminate header */
3158 static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info)
3159 {
3160 u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3161 u16 ddp_seg_len;
3162 int copy_len = 0;
3163 u8 is_tagged = 0;
3164 u8 flush_code = 0;
3165 struct nes_terminate_hdr *termhdr;
3166
3167 termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase;
3168 memset(termhdr, 0, 64);
3169
3170 if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3171
3172 /* Use data from offending packet to fill in ddp & rdma hdrs */
3173 pkt = locate_mpa(pkt, aeq_info);
3174 ddp_seg_len = be16_to_cpu(*(u16 *)pkt);
3175 if (ddp_seg_len) {
3176 copy_len = 2;
3177 termhdr->hdrct = DDP_LEN_FLAG;
3178 if (pkt[2] & 0x80) {
3179 is_tagged = 1;
3180 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
3181 copy_len += TERM_DDP_LEN_TAGGED;
3182 termhdr->hdrct |= DDP_HDR_FLAG;
3183 }
3184 } else {
3185 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
3186 copy_len += TERM_DDP_LEN_UNTAGGED;
3187 termhdr->hdrct |= DDP_HDR_FLAG;
3188 }
3189
3190 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
3191 if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
3192 copy_len += TERM_RDMA_LEN;
3193 termhdr->hdrct |= RDMA_HDR_FLAG;
3194 }
3195 }
3196 }
3197 }
3198 }
3199
3200 switch (async_event_id) {
3201 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3202 switch (iwarp_opcode(nesqp, aeq_info)) {
3203 case IWARP_OPCODE_WRITE:
3204 flush_code = IB_WC_LOC_PROT_ERR;
3205 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3206 termhdr->error_code = DDP_TAGGED_INV_STAG;
3207 break;
3208 default:
3209 flush_code = IB_WC_REM_ACCESS_ERR;
3210 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3211 termhdr->error_code = RDMAP_INV_STAG;
3212 }
3213 break;
3214 case NES_AEQE_AEID_AMP_INVALID_STAG:
3215 flush_code = IB_WC_REM_ACCESS_ERR;
3216 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3217 termhdr->error_code = RDMAP_INV_STAG;
3218 break;
3219 case NES_AEQE_AEID_AMP_BAD_QP:
3220 flush_code = IB_WC_LOC_QP_OP_ERR;
3221 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3222 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3223 break;
3224 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3225 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3226 switch (iwarp_opcode(nesqp, aeq_info)) {
3227 case IWARP_OPCODE_SEND_INV:
3228 case IWARP_OPCODE_SEND_SE_INV:
3229 flush_code = IB_WC_REM_OP_ERR;
3230 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3231 termhdr->error_code = RDMAP_CANT_INV_STAG;
3232 break;
3233 default:
3234 flush_code = IB_WC_REM_ACCESS_ERR;
3235 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3236 termhdr->error_code = RDMAP_INV_STAG;
3237 }
3238 break;
3239 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3240 if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) {
3241 flush_code = IB_WC_LOC_PROT_ERR;
3242 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3243 termhdr->error_code = DDP_TAGGED_BOUNDS;
3244 } else {
3245 flush_code = IB_WC_REM_ACCESS_ERR;
3246 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3247 termhdr->error_code = RDMAP_INV_BOUNDS;
3248 }
3249 break;
3250 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3251 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3252 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3253 flush_code = IB_WC_REM_ACCESS_ERR;
3254 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3255 termhdr->error_code = RDMAP_ACCESS;
3256 break;
3257 case NES_AEQE_AEID_AMP_TO_WRAP:
3258 flush_code = IB_WC_REM_ACCESS_ERR;
3259 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3260 termhdr->error_code = RDMAP_TO_WRAP;
3261 break;
3262 case NES_AEQE_AEID_AMP_BAD_PD:
3263 switch (iwarp_opcode(nesqp, aeq_info)) {
3264 case IWARP_OPCODE_WRITE:
3265 flush_code = IB_WC_LOC_PROT_ERR;
3266 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3267 termhdr->error_code = DDP_TAGGED_UNASSOC_STAG;
3268 break;
3269 case IWARP_OPCODE_SEND_INV:
3270 case IWARP_OPCODE_SEND_SE_INV:
3271 flush_code = IB_WC_REM_ACCESS_ERR;
3272 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3273 termhdr->error_code = RDMAP_CANT_INV_STAG;
3274 break;
3275 default:
3276 flush_code = IB_WC_REM_ACCESS_ERR;
3277 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT;
3278 termhdr->error_code = RDMAP_UNASSOC_STAG;
3279 }
3280 break;
3281 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3282 flush_code = IB_WC_LOC_LEN_ERR;
3283 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3284 termhdr->error_code = MPA_MARKER;
3285 break;
3286 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3287 flush_code = IB_WC_GENERAL_ERR;
3288 termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP;
3289 termhdr->error_code = MPA_CRC;
3290 break;
3291 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3292 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3293 flush_code = IB_WC_LOC_LEN_ERR;
3294 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3295 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3296 break;
3297 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3298 case NES_AEQE_AEID_DDP_NO_L_BIT:
3299 flush_code = IB_WC_FATAL_ERR;
3300 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC;
3301 termhdr->error_code = DDP_CATASTROPHIC_LOCAL;
3302 break;
3303 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3304 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3305 flush_code = IB_WC_GENERAL_ERR;
3306 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3307 termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE;
3308 break;
3309 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3310 flush_code = IB_WC_LOC_LEN_ERR;
3311 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3312 termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG;
3313 break;
3314 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3315 flush_code = IB_WC_GENERAL_ERR;
3316 if (is_tagged) {
3317 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER;
3318 termhdr->error_code = DDP_TAGGED_INV_DDP_VER;
3319 } else {
3320 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3321 termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER;
3322 }
3323 break;
3324 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3325 flush_code = IB_WC_GENERAL_ERR;
3326 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3327 termhdr->error_code = DDP_UNTAGGED_INV_MO;
3328 break;
3329 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3330 flush_code = IB_WC_REM_OP_ERR;
3331 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3332 termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF;
3333 break;
3334 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3335 flush_code = IB_WC_GENERAL_ERR;
3336 termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER;
3337 termhdr->error_code = DDP_UNTAGGED_INV_QN;
3338 break;
3339 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3340 flush_code = IB_WC_GENERAL_ERR;
3341 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3342 termhdr->error_code = RDMAP_INV_RDMAP_VER;
3343 break;
3344 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3345 flush_code = IB_WC_LOC_QP_OP_ERR;
3346 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3347 termhdr->error_code = RDMAP_UNEXPECTED_OP;
3348 break;
3349 default:
3350 flush_code = IB_WC_FATAL_ERR;
3351 termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP;
3352 termhdr->error_code = RDMAP_UNSPECIFIED;
3353 break;
3354 }
3355
3356 if (copy_len)
3357 memcpy(termhdr + 1, pkt, copy_len);
3358
3359 if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) {
3360 if (aeq_info & NES_AEQE_SQ)
3361 nesqp->term_sq_flush_code = flush_code;
3362 else
3363 nesqp->term_rq_flush_code = flush_code;
3364 }
3365
3366 return sizeof(struct nes_terminate_hdr) + copy_len;
3367 }
3368
3369 static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp,
3370 struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype)
3371 {
3372 u64 context;
3373 unsigned long flags;
3374 u32 aeq_info;
3375 u16 async_event_id;
3376 u8 tcp_state;
3377 u8 iwarp_state;
3378 u32 termlen = 0;
3379 u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE |
3380 NES_CQP_QP_TERM_DONT_SEND_FIN;
3381 struct nes_adapter *nesadapter = nesdev->nesadapter;
3382
3383 if (nesqp->term_flags & NES_TERM_SENT)
3384 return; /* Sanity check */
3385
3386 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3387 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3388 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3389 async_event_id = (u16)aeq_info;
3390
3391 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3392 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3393 if (!context) {
3394 WARN_ON(!context);
3395 return;
3396 }
3397
3398 nesqp = (struct nes_qp *)(unsigned long)context;
3399 spin_lock_irqsave(&nesqp->lock, flags);
3400 nesqp->hw_iwarp_state = iwarp_state;
3401 nesqp->hw_tcp_state = tcp_state;
3402 nesqp->last_aeq = async_event_id;
3403 nesqp->terminate_eventtype = eventtype;
3404 spin_unlock_irqrestore(&nesqp->lock, flags);
3405
3406 if (nesadapter->send_term_ok)
3407 termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info);
3408 else
3409 mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG;
3410
3411 if (!nesdev->iw_status) {
3412 nesqp->term_flags = NES_TERM_DONE;
3413 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0);
3414 nes_cm_disconn(nesqp);
3415 } else {
3416 nes_terminate_start_timer(nesqp);
3417 nesqp->term_flags |= NES_TERM_SENT;
3418 nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0);
3419 }
3420 }
3421
3422 static void nes_terminate_send_fin(struct nes_device *nesdev,
3423 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3424 {
3425 u32 aeq_info;
3426 u16 async_event_id;
3427 u8 tcp_state;
3428 u8 iwarp_state;
3429 unsigned long flags;
3430
3431 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3432 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3433 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3434 async_event_id = (u16)aeq_info;
3435
3436 spin_lock_irqsave(&nesqp->lock, flags);
3437 nesqp->hw_iwarp_state = iwarp_state;
3438 nesqp->hw_tcp_state = tcp_state;
3439 nesqp->last_aeq = async_event_id;
3440 spin_unlock_irqrestore(&nesqp->lock, flags);
3441
3442 /* Send the fin only */
3443 nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE |
3444 NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0);
3445 }
3446
3447 /* Cleanup after a terminate sent or received */
3448 static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred)
3449 {
3450 u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR;
3451 unsigned long flags;
3452 struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device);
3453 struct nes_device *nesdev = nesvnic->nesdev;
3454 u8 first_time = 0;
3455
3456 spin_lock_irqsave(&nesqp->lock, flags);
3457 if (nesqp->hte_added) {
3458 nesqp->hte_added = 0;
3459 next_iwarp_state |= NES_CQP_QP_DEL_HTE;
3460 }
3461
3462 first_time = (nesqp->term_flags & NES_TERM_DONE) == 0;
3463 nesqp->term_flags |= NES_TERM_DONE;
3464 spin_unlock_irqrestore(&nesqp->lock, flags);
3465
3466 /* Make sure we go through this only once */
3467 if (first_time) {
3468 if (timeout_occurred == 0)
3469 del_timer(&nesqp->terminate_timer);
3470 else
3471 next_iwarp_state |= NES_CQP_QP_RESET;
3472
3473 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3474 nes_cm_disconn(nesqp);
3475 }
3476 }
3477
3478 static void nes_terminate_received(struct nes_device *nesdev,
3479 struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe)
3480 {
3481 u32 aeq_info;
3482 u8 *pkt;
3483 u32 *mpa;
3484 u8 ddp_ctl;
3485 u8 rdma_ctl;
3486 u16 aeq_id = 0;
3487
3488 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3489 if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) {
3490 /* Terminate is not a performance path so the silicon */
3491 /* did not validate the frame - do it now */
3492 pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET;
3493 mpa = (u32 *)locate_mpa(pkt, aeq_info);
3494 ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff;
3495 rdma_ctl = be32_to_cpu(mpa[0]) & 0xff;
3496 if ((ddp_ctl & 0xc0) != 0x40)
3497 aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC;
3498 else if ((ddp_ctl & 0x03) != 1)
3499 aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION;
3500 else if (be32_to_cpu(mpa[2]) != 2)
3501 aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN;
3502 else if (be32_to_cpu(mpa[3]) != 1)
3503 aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN;
3504 else if (be32_to_cpu(mpa[4]) != 0)
3505 aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO;
3506 else if ((rdma_ctl & 0xc0) != 0x40)
3507 aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION;
3508
3509 if (aeq_id) {
3510 /* Bad terminate recvd - send back a terminate */
3511 aeq_info = (aeq_info & 0xffff0000) | aeq_id;
3512 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3513 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3514 return;
3515 }
3516 }
3517
3518 nesqp->term_flags |= NES_TERM_RCVD;
3519 nesqp->terminate_eventtype = IB_EVENT_QP_FATAL;
3520 nes_terminate_start_timer(nesqp);
3521 nes_terminate_send_fin(nesdev, nesqp, aeqe);
3522 }
3523
3524 /* Timeout routine in case terminate fails to complete */
3525 static void nes_terminate_timeout(unsigned long context)
3526 {
3527 struct nes_qp *nesqp = (struct nes_qp *)(unsigned long)context;
3528
3529 nes_terminate_done(nesqp, 1);
3530 }
3531
3532 /* Set a timer in case hw cannot complete the terminate sequence */
3533 static void nes_terminate_start_timer(struct nes_qp *nesqp)
3534 {
3535 init_timer(&nesqp->terminate_timer);
3536 nesqp->terminate_timer.function = nes_terminate_timeout;
3537 nesqp->terminate_timer.expires = jiffies + HZ;
3538 nesqp->terminate_timer.data = (unsigned long)nesqp;
3539 add_timer(&nesqp->terminate_timer);
3540 }
3541
3542 /**
3543 * nes_process_iwarp_aeqe
3544 */
3545 static void nes_process_iwarp_aeqe(struct nes_device *nesdev,
3546 struct nes_hw_aeqe *aeqe)
3547 {
3548 u64 context;
3549 unsigned long flags;
3550 struct nes_qp *nesqp;
3551 struct nes_hw_cq *hw_cq;
3552 struct nes_cq *nescq;
3553 int resource_allocated;
3554 struct nes_adapter *nesadapter = nesdev->nesadapter;
3555 u32 aeq_info;
3556 u32 next_iwarp_state = 0;
3557 u32 aeqe_cq_id;
3558 u16 async_event_id;
3559 u8 tcp_state;
3560 u8 iwarp_state;
3561 struct ib_event ibevent;
3562
3563 nes_debug(NES_DBG_AEQ, "\n");
3564 aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]);
3565 if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) {
3566 context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]);
3567 context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32;
3568 } else {
3569 context = (unsigned long)nesadapter->qp_table[le32_to_cpu(
3570 aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN];
3571 BUG_ON(!context);
3572 }
3573
3574 /* context is nesqp unless async_event_id == CQ ERROR */
3575 nesqp = (struct nes_qp *)(unsigned long)context;
3576 async_event_id = (u16)aeq_info;
3577 tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT;
3578 iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT;
3579 nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p,"
3580 " Tcp state = %s, iWARP state = %s\n",
3581 async_event_id,
3582 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe,
3583 nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]);
3584
3585 aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]);
3586 if (aeq_info & NES_AEQE_QP) {
3587 if (!nes_is_resource_allocated(nesadapter,
3588 nesadapter->allocated_qps,
3589 aeqe_cq_id))
3590 return;
3591 }
3592
3593 switch (async_event_id) {
3594 case NES_AEQE_AEID_LLP_FIN_RECEIVED:
3595 if (nesqp->term_flags)
3596 return; /* Ignore it, wait for close complete */
3597
3598 if (atomic_inc_return(&nesqp->close_timer_started) == 1) {
3599 if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) &&
3600 (nesqp->ibqp_state == IB_QPS_RTS)) {
3601 spin_lock_irqsave(&nesqp->lock, flags);
3602 nesqp->hw_iwarp_state = iwarp_state;
3603 nesqp->hw_tcp_state = tcp_state;
3604 nesqp->last_aeq = async_event_id;
3605 next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING;
3606 nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;
3607 spin_unlock_irqrestore(&nesqp->lock, flags);
3608 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3609 nes_cm_disconn(nesqp);
3610 }
3611 nesqp->cm_id->add_ref(nesqp->cm_id);
3612 schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp,
3613 NES_TIMER_TYPE_CLOSE, 1, 0);
3614 nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d),"
3615 " need ae to finish up, original_last_aeq = 0x%04X."
3616 " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n",
3617 nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),
3618 async_event_id, nesqp->last_aeq, tcp_state);
3619 }
3620 break;
3621 case NES_AEQE_AEID_LLP_CLOSE_COMPLETE:
3622 spin_lock_irqsave(&nesqp->lock, flags);
3623 nesqp->hw_iwarp_state = iwarp_state;
3624 nesqp->hw_tcp_state = tcp_state;
3625 nesqp->last_aeq = async_event_id;
3626 spin_unlock_irqrestore(&nesqp->lock, flags);
3627 nes_cm_disconn(nesqp);
3628 break;
3629
3630 case NES_AEQE_AEID_RESET_SENT:
3631 tcp_state = NES_AEQE_TCP_STATE_CLOSED;
3632 spin_lock_irqsave(&nesqp->lock, flags);
3633 nesqp->hw_iwarp_state = iwarp_state;
3634 nesqp->hw_tcp_state = tcp_state;
3635 nesqp->last_aeq = async_event_id;
3636 nesqp->hte_added = 0;
3637 spin_unlock_irqrestore(&nesqp->lock, flags);
3638 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE;
3639 nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0);
3640 nes_cm_disconn(nesqp);
3641 break;
3642
3643 case NES_AEQE_AEID_LLP_CONNECTION_RESET:
3644 if (atomic_read(&nesqp->close_timer_started))
3645 return;
3646 spin_lock_irqsave(&nesqp->lock, flags);
3647 nesqp->hw_iwarp_state = iwarp_state;
3648 nesqp->hw_tcp_state = tcp_state;
3649 nesqp->last_aeq = async_event_id;
3650 spin_unlock_irqrestore(&nesqp->lock, flags);
3651 nes_cm_disconn(nesqp);
3652 break;
3653
3654 case NES_AEQE_AEID_TERMINATE_SENT:
3655 nes_terminate_send_fin(nesdev, nesqp, aeqe);
3656 break;
3657
3658 case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED:
3659 nes_terminate_received(nesdev, nesqp, aeqe);
3660 break;
3661
3662 case NES_AEQE_AEID_AMP_BAD_STAG_KEY:
3663 case NES_AEQE_AEID_AMP_BAD_STAG_INDEX:
3664 case NES_AEQE_AEID_AMP_UNALLOCATED_STAG:
3665 case NES_AEQE_AEID_AMP_INVALID_STAG:
3666 case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION:
3667 case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
3668 case NES_AEQE_AEID_PRIV_OPERATION_DENIED:
3669 case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
3670 case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION:
3671 case NES_AEQE_AEID_AMP_TO_WRAP:
3672 printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n",
3673 nesqp->hwqp.qp_id, async_event_id);
3674 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR);
3675 break;
3676
3677 case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE:
3678 case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL:
3679 case NES_AEQE_AEID_DDP_UBE_INVALID_MO:
3680 case NES_AEQE_AEID_DDP_UBE_INVALID_QN:
3681 if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) {
3682 aeq_info &= 0xffff0000;
3683 aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE;
3684 aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info);
3685 }
3686
3687 case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE:
3688 case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES:
3689 case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
3690 case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR:
3691 case NES_AEQE_AEID_AMP_BAD_QP:
3692 case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH:
3693 case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC:
3694 case NES_AEQE_AEID_DDP_NO_L_BIT:
3695 case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN:
3696 case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID:
3697 case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION:
3698 case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION:
3699 case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE:
3700 case NES_AEQE_AEID_AMP_BAD_PD:
3701 case NES_AEQE_AEID_AMP_FASTREG_SHARED:
3702 case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG:
3703 case NES_AEQE_AEID_AMP_FASTREG_MW_STAG:
3704 case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS:
3705 case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW:
3706 case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH:
3707 case NES_AEQE_AEID_AMP_INVALIDATE_SHARED:
3708 case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS:
3709 case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG:
3710 case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG:
3711 case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG:
3712 case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG:
3713 case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS:
3714 case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS:
3715 case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT:
3716 case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED:
3717 case NES_AEQE_AEID_BAD_CLOSE:
3718 case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO:
3719 case NES_AEQE_AEID_STAG_ZERO_INVALID:
3720 case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST:
3721 case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
3722 printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n",
3723 nesqp->hwqp.qp_id, async_event_id);
3724 print_ip(nesqp->cm_node);
3725 if (!atomic_read(&nesqp->close_timer_started))
3726 nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL);
3727 break;
3728
3729 case NES_AEQE_AEID_CQ_OPERATION_ERROR:
3730 context <<= 1;
3731 nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n",
3732 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context);
3733 resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs,
3734 le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3735 if (resource_allocated) {
3736 printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n",
3737 __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]));
3738 hw_cq = (struct nes_hw_cq *)(unsigned long)context;
3739 if (hw_cq) {
3740 nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3741 if (nescq->ibcq.event_handler) {
3742 ibevent.device = nescq->ibcq.device;
3743 ibevent.event = IB_EVENT_CQ_ERR;
3744 ibevent.element.cq = &nescq->ibcq;
3745 nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context);
3746 }
3747 }
3748 }
3749 break;
3750
3751 default:
3752 nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n",
3753 async_event_id);
3754 break;
3755 }
3756
3757 }
3758
3759 /**
3760 * nes_iwarp_ce_handler
3761 */
3762 void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq)
3763 {
3764 struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq);
3765
3766 /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n",
3767 nescq->hw_cq.cq_number); */
3768 nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number);
3769
3770 if (nescq->ibcq.comp_handler)
3771 nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context);
3772
3773 return;
3774 }
3775
3776
3777 /**
3778 * nes_manage_apbvt()
3779 */
3780 int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port,
3781 u32 nic_index, u32 add_port)
3782 {
3783 struct nes_device *nesdev = nesvnic->nesdev;
3784 struct nes_hw_cqp_wqe *cqp_wqe;
3785 struct nes_cqp_request *cqp_request;
3786 int ret = 0;
3787 u16 major_code;
3788
3789 /* Send manage APBVT request to CQP */
3790 cqp_request = nes_get_cqp_request(nesdev);
3791 if (cqp_request == NULL) {
3792 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3793 return -ENOMEM;
3794 }
3795 cqp_request->waiting = 1;
3796 cqp_wqe = &cqp_request->cqp_wqe;
3797
3798 nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n",
3799 (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL",
3800 accel_local_port, accel_local_port, nic_index);
3801
3802 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3803 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT |
3804 ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0)));
3805 set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX,
3806 ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port));
3807
3808 nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n");
3809
3810 atomic_set(&cqp_request->refcount, 2);
3811 nes_post_cqp_request(nesdev, cqp_request);
3812
3813 if (add_port == NES_MANAGE_APBVT_ADD)
3814 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3815 NES_EVENT_TIMEOUT);
3816 nes_debug(NES_DBG_QP, "Completed, ret=%u, CQP Major:Minor codes = 0x%04X:0x%04X\n",
3817 ret, cqp_request->major_code, cqp_request->minor_code);
3818 major_code = cqp_request->major_code;
3819
3820 nes_put_cqp_request(nesdev, cqp_request);
3821
3822 if (!ret)
3823 return -ETIME;
3824 else if (major_code)
3825 return -EIO;
3826 else
3827 return 0;
3828 }
3829
3830
3831 /**
3832 * nes_manage_arp_cache
3833 */
3834 void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr,
3835 u32 ip_addr, u32 action)
3836 {
3837 struct nes_hw_cqp_wqe *cqp_wqe;
3838 struct nes_vnic *nesvnic = netdev_priv(netdev);
3839 struct nes_device *nesdev;
3840 struct nes_cqp_request *cqp_request;
3841 int arp_index;
3842
3843 nesdev = nesvnic->nesdev;
3844 arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action);
3845 if (arp_index == -1) {
3846 return;
3847 }
3848
3849 /* update the ARP entry */
3850 cqp_request = nes_get_cqp_request(nesdev);
3851 if (cqp_request == NULL) {
3852 nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n");
3853 return;
3854 }
3855 cqp_request->waiting = 0;
3856 cqp_wqe = &cqp_request->cqp_wqe;
3857 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3858
3859 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(
3860 NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM);
3861 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(
3862 (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT);
3863 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index);
3864
3865 if (action == NES_ARP_ADD) {
3866 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID);
3867 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32(
3868 (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) |
3869 (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]);
3870 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32(
3871 (((u32)mac_addr[0]) << 16) | (u32)mac_addr[1]);
3872 } else {
3873 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0;
3874 cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0;
3875 }
3876
3877 nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n",
3878 nesdev->cqp.sq_head, nesdev->cqp.sq_tail);
3879
3880 atomic_set(&cqp_request->refcount, 1);
3881 nes_post_cqp_request(nesdev, cqp_request);
3882 }
3883
3884
3885 /**
3886 * flush_wqes
3887 */
3888 void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp,
3889 u32 which_wq, u32 wait_completion)
3890 {
3891 struct nes_cqp_request *cqp_request;
3892 struct nes_hw_cqp_wqe *cqp_wqe;
3893 u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3894 u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH;
3895 int ret;
3896
3897 cqp_request = nes_get_cqp_request(nesdev);
3898 if (cqp_request == NULL) {
3899 nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n");
3900 return;
3901 }
3902 if (wait_completion) {
3903 cqp_request->waiting = 1;
3904 atomic_set(&cqp_request->refcount, 2);
3905 } else {
3906 cqp_request->waiting = 0;
3907 }
3908 cqp_wqe = &cqp_request->cqp_wqe;
3909 nes_fill_init_cqp_wqe(cqp_wqe, nesdev);
3910
3911 /* If wqe in error was identified, set code to be put into cqe */
3912 if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) {
3913 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3914 sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code;
3915 nesqp->term_sq_flush_code = 0;
3916 }
3917
3918 if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) {
3919 which_wq |= NES_CQP_FLUSH_MAJ_MIN;
3920 rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code;
3921 nesqp->term_rq_flush_code = 0;
3922 }
3923
3924 if (which_wq & NES_CQP_FLUSH_MAJ_MIN) {
3925 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code);
3926 cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code);
3927 }
3928
3929 cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] =
3930 cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq);
3931 cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id);
3932
3933 nes_post_cqp_request(nesdev, cqp_request);
3934
3935 if (wait_completion) {
3936 /* Wait for CQP */
3937 ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0),
3938 NES_EVENT_TIMEOUT);
3939 nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u,"
3940 " CQP Major:Minor codes = 0x%04X:0x%04X\n",
3941 ret, cqp_request->major_code, cqp_request->minor_code);
3942 nes_put_cqp_request(nesdev, cqp_request);
3943 }
3944 }