?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 2.6.35-rc1
[orion.git] / drivers / net / mlx4 / en_rx.c
blob8e2fcb7103c3a12a3072549881a7529d3062bc85
1 /*
2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 *
32 */
33
34 #include <linux/mlx4/cq.h>
35 #include <linux/slab.h>
36 #include <linux/mlx4/qp.h>
37 #include <linux/skbuff.h>
38 #include <linux/if_ether.h>
39 #include <linux/if_vlan.h>
40 #include <linux/vmalloc.h>
41
42 #include "mlx4_en.h"
43
44
45 static int mlx4_en_get_frag_header(struct skb_frag_struct *frags, void **mac_hdr,
46 void **ip_hdr, void **tcpudp_hdr,
47 u64 *hdr_flags, void *priv)
48 {
49 *mac_hdr = page_address(frags->page) + frags->page_offset;
50 *ip_hdr = *mac_hdr + ETH_HLEN;
51 *tcpudp_hdr = (struct tcphdr *)(*ip_hdr + sizeof(struct iphdr));
52 *hdr_flags = LRO_IPV4 | LRO_TCP;
53
54 return 0;
55 }
56
57 static int mlx4_en_alloc_frag(struct mlx4_en_priv *priv,
58 struct mlx4_en_rx_desc *rx_desc,
59 struct skb_frag_struct *skb_frags,
60 struct mlx4_en_rx_alloc *ring_alloc,
61 int i)
62 {
63 struct mlx4_en_dev *mdev = priv->mdev;
64 struct mlx4_en_frag_info *frag_info = &priv->frag_info[i];
65 struct mlx4_en_rx_alloc *page_alloc = &ring_alloc[i];
66 struct page *page;
67 dma_addr_t dma;
68
69 if (page_alloc->offset == frag_info->last_offset) {
70 /* Allocate new page */
71 page = alloc_pages(GFP_ATOMIC | __GFP_COMP, MLX4_EN_ALLOC_ORDER);
72 if (!page)
73 return -ENOMEM;
74
75 skb_frags[i].page = page_alloc->page;
76 skb_frags[i].page_offset = page_alloc->offset;
77 page_alloc->page = page;
78 page_alloc->offset = frag_info->frag_align;
79 } else {
80 page = page_alloc->page;
81 get_page(page);
82
83 skb_frags[i].page = page;
84 skb_frags[i].page_offset = page_alloc->offset;
85 page_alloc->offset += frag_info->frag_stride;
86 }
87 dma = pci_map_single(mdev->pdev, page_address(skb_frags[i].page) +
88 skb_frags[i].page_offset, frag_info->frag_size,
89 PCI_DMA_FROMDEVICE);
90 rx_desc->data[i].addr = cpu_to_be64(dma);
91 return 0;
92 }
93
94 static int mlx4_en_init_allocator(struct mlx4_en_priv *priv,
95 struct mlx4_en_rx_ring *ring)
96 {
97 struct mlx4_en_rx_alloc *page_alloc;
98 int i;
99
100 for (i = 0; i < priv->num_frags; i++) {
101 page_alloc = &ring->page_alloc[i];
102 page_alloc->page = alloc_pages(GFP_ATOMIC | __GFP_COMP,
103 MLX4_EN_ALLOC_ORDER);
104 if (!page_alloc->page)
105 goto out;
106
107 page_alloc->offset = priv->frag_info[i].frag_align;
108 en_dbg(DRV, priv, "Initialized allocator:%d with page:%p\n",
109 i, page_alloc->page);
110 }
111 return 0;
112
113 out:
114 while (i--) {
115 page_alloc = &ring->page_alloc[i];
116 put_page(page_alloc->page);
117 page_alloc->page = NULL;
118 }
119 return -ENOMEM;
120 }
121
122 static void mlx4_en_destroy_allocator(struct mlx4_en_priv *priv,
123 struct mlx4_en_rx_ring *ring)
124 {
125 struct mlx4_en_rx_alloc *page_alloc;
126 int i;
127
128 for (i = 0; i < priv->num_frags; i++) {
129 page_alloc = &ring->page_alloc[i];
130 en_dbg(DRV, priv, "Freeing allocator:%d count:%d\n",
131 i, page_count(page_alloc->page));
132
133 put_page(page_alloc->page);
134 page_alloc->page = NULL;
135 }
136 }
137
138
139 static void mlx4_en_init_rx_desc(struct mlx4_en_priv *priv,
140 struct mlx4_en_rx_ring *ring, int index)
141 {
142 struct mlx4_en_rx_desc *rx_desc = ring->buf + ring->stride * index;
143 struct skb_frag_struct *skb_frags = ring->rx_info +
144 (index << priv->log_rx_info);
145 int possible_frags;
146 int i;
147
148 /* Set size and memtype fields */
149 for (i = 0; i < priv->num_frags; i++) {
150 skb_frags[i].size = priv->frag_info[i].frag_size;
151 rx_desc->data[i].byte_count =
152 cpu_to_be32(priv->frag_info[i].frag_size);
153 rx_desc->data[i].lkey = cpu_to_be32(priv->mdev->mr.key);
154 }
155
156 /* If the number of used fragments does not fill up the ring stride,
157 * remaining (unused) fragments must be padded with null address/size
158 * and a special memory key */
159 possible_frags = (ring->stride - sizeof(struct mlx4_en_rx_desc)) / DS_SIZE;
160 for (i = priv->num_frags; i < possible_frags; i++) {
161 rx_desc->data[i].byte_count = 0;
162 rx_desc->data[i].lkey = cpu_to_be32(MLX4_EN_MEMTYPE_PAD);
163 rx_desc->data[i].addr = 0;
164 }
165 }
166
167
168 static int mlx4_en_prepare_rx_desc(struct mlx4_en_priv *priv,
169 struct mlx4_en_rx_ring *ring, int index)
170 {
171 struct mlx4_en_rx_desc *rx_desc = ring->buf + (index * ring->stride);
172 struct skb_frag_struct *skb_frags = ring->rx_info +
173 (index << priv->log_rx_info);
174 int i;
175
176 for (i = 0; i < priv->num_frags; i++)
177 if (mlx4_en_alloc_frag(priv, rx_desc, skb_frags, ring->page_alloc, i))
178 goto err;
179
180 return 0;
181
182 err:
183 while (i--)
184 put_page(skb_frags[i].page);
185 return -ENOMEM;
186 }
187
188 static inline void mlx4_en_update_rx_prod_db(struct mlx4_en_rx_ring *ring)
189 {
190 *ring->wqres.db.db = cpu_to_be32(ring->prod & 0xffff);
191 }
192
193 static void mlx4_en_free_rx_desc(struct mlx4_en_priv *priv,
194 struct mlx4_en_rx_ring *ring,
195 int index)
196 {
197 struct mlx4_en_dev *mdev = priv->mdev;
198 struct skb_frag_struct *skb_frags;
199 struct mlx4_en_rx_desc *rx_desc = ring->buf + (index << ring->log_stride);
200 dma_addr_t dma;
201 int nr;
202
203 skb_frags = ring->rx_info + (index << priv->log_rx_info);
204 for (nr = 0; nr < priv->num_frags; nr++) {
205 en_dbg(DRV, priv, "Freeing fragment:%d\n", nr);
206 dma = be64_to_cpu(rx_desc->data[nr].addr);
207
208 en_dbg(DRV, priv, "Unmapping buffer at dma:0x%llx\n", (u64) dma);
209 pci_unmap_single(mdev->pdev, dma, skb_frags[nr].size,
210 PCI_DMA_FROMDEVICE);
211 put_page(skb_frags[nr].page);
212 }
213 }
214
215 static int mlx4_en_fill_rx_buffers(struct mlx4_en_priv *priv)
216 {
217 struct mlx4_en_rx_ring *ring;
218 int ring_ind;
219 int buf_ind;
220 int new_size;
221
222 for (buf_ind = 0; buf_ind < priv->prof->rx_ring_size; buf_ind++) {
223 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
224 ring = &priv->rx_ring[ring_ind];
225
226 if (mlx4_en_prepare_rx_desc(priv, ring,
227 ring->actual_size)) {
228 if (ring->actual_size < MLX4_EN_MIN_RX_SIZE) {
229 en_err(priv, "Failed to allocate "
230 "enough rx buffers\n");
231 return -ENOMEM;
232 } else {
233 new_size = rounddown_pow_of_two(ring->actual_size);
234 en_warn(priv, "Only %d buffers allocated "
235 "reducing ring size to %d",
236 ring->actual_size, new_size);
237 goto reduce_rings;
238 }
239 }
240 ring->actual_size++;
241 ring->prod++;
242 }
243 }
244 return 0;
245
246 reduce_rings:
247 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
248 ring = &priv->rx_ring[ring_ind];
249 while (ring->actual_size > new_size) {
250 ring->actual_size--;
251 ring->prod--;
252 mlx4_en_free_rx_desc(priv, ring, ring->actual_size);
253 }
254 ring->size_mask = ring->actual_size - 1;
255 }
256
257 return 0;
258 }
259
260 static void mlx4_en_free_rx_buf(struct mlx4_en_priv *priv,
261 struct mlx4_en_rx_ring *ring)
262 {
263 int index;
264
265 en_dbg(DRV, priv, "Freeing Rx buf - cons:%d prod:%d\n",
266 ring->cons, ring->prod);
267
268 /* Unmap and free Rx buffers */
269 BUG_ON((u32) (ring->prod - ring->cons) > ring->actual_size);
270 while (ring->cons != ring->prod) {
271 index = ring->cons & ring->size_mask;
272 en_dbg(DRV, priv, "Processing descriptor:%d\n", index);
273 mlx4_en_free_rx_desc(priv, ring, index);
274 ++ring->cons;
275 }
276 }
277
278 int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv,
279 struct mlx4_en_rx_ring *ring, u32 size, u16 stride)
280 {
281 struct mlx4_en_dev *mdev = priv->mdev;
282 int err;
283 int tmp;
284
285
286 ring->prod = 0;
287 ring->cons = 0;
288 ring->size = size;
289 ring->size_mask = size - 1;
290 ring->stride = stride;
291 ring->log_stride = ffs(ring->stride) - 1;
292 ring->buf_size = ring->size * ring->stride + TXBB_SIZE;
293
294 tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS *
295 sizeof(struct skb_frag_struct));
296 ring->rx_info = vmalloc(tmp);
297 if (!ring->rx_info) {
298 en_err(priv, "Failed allocating rx_info ring\n");
299 return -ENOMEM;
300 }
301 en_dbg(DRV, priv, "Allocated rx_info ring at addr:%p size:%d\n",
302 ring->rx_info, tmp);
303
304 err = mlx4_alloc_hwq_res(mdev->dev, &ring->wqres,
305 ring->buf_size, 2 * PAGE_SIZE);
306 if (err)
307 goto err_ring;
308
309 err = mlx4_en_map_buffer(&ring->wqres.buf);
310 if (err) {
311 en_err(priv, "Failed to map RX buffer\n");
312 goto err_hwq;
313 }
314 ring->buf = ring->wqres.buf.direct.buf;
315
316 /* Configure lro mngr */
317 memset(&ring->lro, 0, sizeof(struct net_lro_mgr));
318 ring->lro.dev = priv->dev;
319 ring->lro.features = LRO_F_NAPI;
320 ring->lro.frag_align_pad = NET_IP_ALIGN;
321 ring->lro.ip_summed = CHECKSUM_UNNECESSARY;
322 ring->lro.ip_summed_aggr = CHECKSUM_UNNECESSARY;
323 ring->lro.max_desc = mdev->profile.num_lro;
324 ring->lro.max_aggr = MAX_SKB_FRAGS;
325 ring->lro.lro_arr = kzalloc(mdev->profile.num_lro *
326 sizeof(struct net_lro_desc),
327 GFP_KERNEL);
328 if (!ring->lro.lro_arr) {
329 en_err(priv, "Failed to allocate lro array\n");
330 goto err_map;
331 }
332 ring->lro.get_frag_header = mlx4_en_get_frag_header;
333
334 return 0;
335
336 err_map:
337 mlx4_en_unmap_buffer(&ring->wqres.buf);
338 err_hwq:
339 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size);
340 err_ring:
341 vfree(ring->rx_info);
342 ring->rx_info = NULL;
343 return err;
344 }
345
346 int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv)
347 {
348 struct mlx4_en_rx_ring *ring;
349 int i;
350 int ring_ind;
351 int err;
352 int stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) +
353 DS_SIZE * priv->num_frags);
354
355 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
356 ring = &priv->rx_ring[ring_ind];
357
358 ring->prod = 0;
359 ring->cons = 0;
360 ring->actual_size = 0;
361 ring->cqn = priv->rx_cq[ring_ind].mcq.cqn;
362
363 ring->stride = stride;
364 if (ring->stride <= TXBB_SIZE)
365 ring->buf += TXBB_SIZE;
366
367 ring->log_stride = ffs(ring->stride) - 1;
368 ring->buf_size = ring->size * ring->stride;
369
370 memset(ring->buf, 0, ring->buf_size);
371 mlx4_en_update_rx_prod_db(ring);
372
373 /* Initailize all descriptors */
374 for (i = 0; i < ring->size; i++)
375 mlx4_en_init_rx_desc(priv, ring, i);
376
377 /* Initialize page allocators */
378 err = mlx4_en_init_allocator(priv, ring);
379 if (err) {
380 en_err(priv, "Failed initializing ring allocator\n");
381 ring_ind--;
382 goto err_allocator;
383 }
384 }
385 err = mlx4_en_fill_rx_buffers(priv);
386 if (err)
387 goto err_buffers;
388
389 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++) {
390 ring = &priv->rx_ring[ring_ind];
391
392 mlx4_en_update_rx_prod_db(ring);
393 }
394
395 return 0;
396
397 err_buffers:
398 for (ring_ind = 0; ring_ind < priv->rx_ring_num; ring_ind++)
399 mlx4_en_free_rx_buf(priv, &priv->rx_ring[ring_ind]);
400
401 ring_ind = priv->rx_ring_num - 1;
402 err_allocator:
403 while (ring_ind >= 0) {
404 mlx4_en_destroy_allocator(priv, &priv->rx_ring[ring_ind]);
405 ring_ind--;
406 }
407 return err;
408 }
409
410 void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
411 struct mlx4_en_rx_ring *ring)
412 {
413 struct mlx4_en_dev *mdev = priv->mdev;
414
415 kfree(ring->lro.lro_arr);
416 mlx4_en_unmap_buffer(&ring->wqres.buf);
417 mlx4_free_hwq_res(mdev->dev, &ring->wqres, ring->buf_size + TXBB_SIZE);
418 vfree(ring->rx_info);
419 ring->rx_info = NULL;
420 }
421
422 void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv,
423 struct mlx4_en_rx_ring *ring)
424 {
425 mlx4_en_free_rx_buf(priv, ring);
426 if (ring->stride <= TXBB_SIZE)
427 ring->buf -= TXBB_SIZE;
428 mlx4_en_destroy_allocator(priv, ring);
429 }
430
431
432 /* Unmap a completed descriptor and free unused pages */
433 static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
434 struct mlx4_en_rx_desc *rx_desc,
435 struct skb_frag_struct *skb_frags,
436 struct skb_frag_struct *skb_frags_rx,
437 struct mlx4_en_rx_alloc *page_alloc,
438 int length)
439 {
440 struct mlx4_en_dev *mdev = priv->mdev;
441 struct mlx4_en_frag_info *frag_info;
442 int nr;
443 dma_addr_t dma;
444
445 /* Collect used fragments while replacing them in the HW descirptors */
446 for (nr = 0; nr < priv->num_frags; nr++) {
447 frag_info = &priv->frag_info[nr];
448 if (length <= frag_info->frag_prefix_size)
449 break;
450
451 /* Save page reference in skb */
452 skb_frags_rx[nr].page = skb_frags[nr].page;
453 skb_frags_rx[nr].size = skb_frags[nr].size;
454 skb_frags_rx[nr].page_offset = skb_frags[nr].page_offset;
455 dma = be64_to_cpu(rx_desc->data[nr].addr);
456
457 /* Allocate a replacement page */
458 if (mlx4_en_alloc_frag(priv, rx_desc, skb_frags, page_alloc, nr))
459 goto fail;
460
461 /* Unmap buffer */
462 pci_unmap_single(mdev->pdev, dma, skb_frags[nr].size,
463 PCI_DMA_FROMDEVICE);
464 }
465 /* Adjust size of last fragment to match actual length */
466 if (nr > 0)
467 skb_frags_rx[nr - 1].size = length -
468 priv->frag_info[nr - 1].frag_prefix_size;
469 return nr;
470
471 fail:
472 /* Drop all accumulated fragments (which have already been replaced in
473 * the descriptor) of this packet; remaining fragments are reused... */
474 while (nr > 0) {
475 nr--;
476 put_page(skb_frags_rx[nr].page);
477 }
478 return 0;
479 }
480
481
482 static struct sk_buff *mlx4_en_rx_skb(struct mlx4_en_priv *priv,
483 struct mlx4_en_rx_desc *rx_desc,
484 struct skb_frag_struct *skb_frags,
485 struct mlx4_en_rx_alloc *page_alloc,
486 unsigned int length)
487 {
488 struct mlx4_en_dev *mdev = priv->mdev;
489 struct sk_buff *skb;
490 void *va;
491 int used_frags;
492 dma_addr_t dma;
493
494 skb = dev_alloc_skb(SMALL_PACKET_SIZE + NET_IP_ALIGN);
495 if (!skb) {
496 en_dbg(RX_ERR, priv, "Failed allocating skb\n");
497 return NULL;
498 }
499 skb->dev = priv->dev;
500 skb_reserve(skb, NET_IP_ALIGN);
501 skb->len = length;
502 skb->truesize = length + sizeof(struct sk_buff);
503
504 /* Get pointer to first fragment so we could copy the headers into the
505 * (linear part of the) skb */
506 va = page_address(skb_frags[0].page) + skb_frags[0].page_offset;
507
508 if (length <= SMALL_PACKET_SIZE) {
509 /* We are copying all relevant data to the skb - temporarily
510 * synch buffers for the copy */
511 dma = be64_to_cpu(rx_desc->data[0].addr);
512 dma_sync_single_for_cpu(&mdev->pdev->dev, dma, length,
513 DMA_FROM_DEVICE);
514 skb_copy_to_linear_data(skb, va, length);
515 dma_sync_single_for_device(&mdev->pdev->dev, dma, length,
516 DMA_FROM_DEVICE);
517 skb->tail += length;
518 } else {
519
520 /* Move relevant fragments to skb */
521 used_frags = mlx4_en_complete_rx_desc(priv, rx_desc, skb_frags,
522 skb_shinfo(skb)->frags,
523 page_alloc, length);
524 if (unlikely(!used_frags)) {
525 kfree_skb(skb);
526 return NULL;
527 }
528 skb_shinfo(skb)->nr_frags = used_frags;
529
530 /* Copy headers into the skb linear buffer */
531 memcpy(skb->data, va, HEADER_COPY_SIZE);
532 skb->tail += HEADER_COPY_SIZE;
533
534 /* Skip headers in first fragment */
535 skb_shinfo(skb)->frags[0].page_offset += HEADER_COPY_SIZE;
536
537 /* Adjust size of first fragment */
538 skb_shinfo(skb)->frags[0].size -= HEADER_COPY_SIZE;
539 skb->data_len = length - HEADER_COPY_SIZE;
540 }
541 return skb;
542 }
543
544
545 int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget)
546 {
547 struct mlx4_en_priv *priv = netdev_priv(dev);
548 struct mlx4_cqe *cqe;
549 struct mlx4_en_rx_ring *ring = &priv->rx_ring[cq->ring];
550 struct skb_frag_struct *skb_frags;
551 struct skb_frag_struct lro_frags[MLX4_EN_MAX_RX_FRAGS];
552 struct mlx4_en_rx_desc *rx_desc;
553 struct sk_buff *skb;
554 int index;
555 int nr;
556 unsigned int length;
557 int polled = 0;
558 int ip_summed;
559
560 if (!priv->port_up)
561 return 0;
562
563 /* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
564 * descriptor offset can be deduced from the CQE index instead of
565 * reading 'cqe->index' */
566 index = cq->mcq.cons_index & ring->size_mask;
567 cqe = &cq->buf[index];
568
569 /* Process all completed CQEs */
570 while (XNOR(cqe->owner_sr_opcode & MLX4_CQE_OWNER_MASK,
571 cq->mcq.cons_index & cq->size)) {
572
573 skb_frags = ring->rx_info + (index << priv->log_rx_info);
574 rx_desc = ring->buf + (index << ring->log_stride);
575
576 /*
577 * make sure we read the CQE after we read the ownership bit
578 */
579 rmb();
580
581 /* Drop packet on bad receive or bad checksum */
582 if (unlikely((cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
583 MLX4_CQE_OPCODE_ERROR)) {
584 en_err(priv, "CQE completed in error - vendor "
585 "syndrom:%d syndrom:%d\n",
586 ((struct mlx4_err_cqe *) cqe)->vendor_err_syndrome,
587 ((struct mlx4_err_cqe *) cqe)->syndrome);
588 goto next;
589 }
590 if (unlikely(cqe->badfcs_enc & MLX4_CQE_BAD_FCS)) {
591 en_dbg(RX_ERR, priv, "Accepted frame with bad FCS\n");
592 goto next;
593 }
594
595 /*
596 * Packet is OK - process it.
597 */
598 length = be32_to_cpu(cqe->byte_cnt);
599 ring->bytes += length;
600 ring->packets++;
601
602 if (likely(priv->rx_csum)) {
603 if ((cqe->status & cpu_to_be16(MLX4_CQE_STATUS_IPOK)) &&
604 (cqe->checksum == cpu_to_be16(0xffff))) {
605 priv->port_stats.rx_chksum_good++;
606 /* This packet is eligible for LRO if it is:
607 * - DIX Ethernet (type interpretation)
608 * - TCP/IP (v4)
609 * - without IP options
610 * - not an IP fragment */
611 if (mlx4_en_can_lro(cqe->status) &&
612 dev->features & NETIF_F_LRO) {
613
614 nr = mlx4_en_complete_rx_desc(
615 priv, rx_desc,
616 skb_frags, lro_frags,
617 ring->page_alloc, length);
618 if (!nr)
619 goto next;
620
621 if (priv->vlgrp && (cqe->vlan_my_qpn &
622 cpu_to_be32(MLX4_CQE_VLAN_PRESENT_MASK))) {
623 lro_vlan_hwaccel_receive_frags(
624 &ring->lro, lro_frags,
625 length, length,
626 priv->vlgrp,
627 be16_to_cpu(cqe->sl_vid),
628 NULL, 0);
629 } else
630 lro_receive_frags(&ring->lro,
631 lro_frags,
632 length,
633 length,
634 NULL, 0);
635
636 goto next;
637 }
638
639 /* LRO not possible, complete processing here */
640 ip_summed = CHECKSUM_UNNECESSARY;
641 INC_PERF_COUNTER(priv->pstats.lro_misses);
642 } else {
643 ip_summed = CHECKSUM_NONE;
644 priv->port_stats.rx_chksum_none++;
645 }
646 } else {
647 ip_summed = CHECKSUM_NONE;
648 priv->port_stats.rx_chksum_none++;
649 }
650
651 skb = mlx4_en_rx_skb(priv, rx_desc, skb_frags,
652 ring->page_alloc, length);
653 if (!skb) {
654 priv->stats.rx_dropped++;
655 goto next;
656 }
657
658 skb->ip_summed = ip_summed;
659 skb->protocol = eth_type_trans(skb, dev);
660 skb_record_rx_queue(skb, cq->ring);
661
662 /* Push it up the stack */
663 if (priv->vlgrp && (be32_to_cpu(cqe->vlan_my_qpn) &
664 MLX4_CQE_VLAN_PRESENT_MASK)) {
665 vlan_hwaccel_receive_skb(skb, priv->vlgrp,
666 be16_to_cpu(cqe->sl_vid));
667 } else
668 netif_receive_skb(skb);
669
670 next:
671 ++cq->mcq.cons_index;
672 index = (cq->mcq.cons_index) & ring->size_mask;
673 cqe = &cq->buf[index];
674 if (++polled == budget) {
675 /* We are here because we reached the NAPI budget -
676 * flush only pending LRO sessions */
677 lro_flush_all(&ring->lro);
678 goto out;
679 }
680 }
681
682 /* If CQ is empty flush all LRO sessions unconditionally */
683 lro_flush_all(&ring->lro);
684
685 out:
686 AVG_PERF_COUNTER(priv->pstats.rx_coal_avg, polled);
687 mlx4_cq_set_ci(&cq->mcq);
688 wmb(); /* ensure HW sees CQ consumer before we post new buffers */
689 ring->cons = cq->mcq.cons_index;
690 ring->prod += polled; /* Polled descriptors were realocated in place */
691 mlx4_en_update_rx_prod_db(ring);
692 return polled;
693 }
694
695
696 void mlx4_en_rx_irq(struct mlx4_cq *mcq)
697 {
698 struct mlx4_en_cq *cq = container_of(mcq, struct mlx4_en_cq, mcq);
699 struct mlx4_en_priv *priv = netdev_priv(cq->dev);
700
701 if (priv->port_up)
702 napi_schedule(&cq->napi);
703 else
704 mlx4_en_arm_cq(priv, cq);
705 }
706
707 /* Rx CQ polling - called by NAPI */
708 int mlx4_en_poll_rx_cq(struct napi_struct *napi, int budget)
709 {
710 struct mlx4_en_cq *cq = container_of(napi, struct mlx4_en_cq, napi);
711 struct net_device *dev = cq->dev;
712 struct mlx4_en_priv *priv = netdev_priv(dev);
713 int done;
714
715 done = mlx4_en_process_rx_cq(dev, cq, budget);
716
717 /* If we used up all the quota - we're probably not done yet... */
718 if (done == budget)
719 INC_PERF_COUNTER(priv->pstats.napi_quota);
720 else {
721 /* Done for now */
722 napi_complete(napi);
723 mlx4_en_arm_cq(priv, cq);
724 }
725 return done;
726 }
727
728
729 /* Calculate the last offset position that accomodates a full fragment
730 * (assuming fagment size = stride-align) */
731 static int mlx4_en_last_alloc_offset(struct mlx4_en_priv *priv, u16 stride, u16 align)
732 {
733 u16 res = MLX4_EN_ALLOC_SIZE % stride;
734 u16 offset = MLX4_EN_ALLOC_SIZE - stride - res + align;
735
736 en_dbg(DRV, priv, "Calculated last offset for stride:%d align:%d "
737 "res:%d offset:%d\n", stride, align, res, offset);
738 return offset;
739 }
740
741
742 static int frag_sizes[] = {
743 FRAG_SZ0,
744 FRAG_SZ1,
745 FRAG_SZ2,
746 FRAG_SZ3
747 };
748
749 void mlx4_en_calc_rx_buf(struct net_device *dev)
750 {
751 struct mlx4_en_priv *priv = netdev_priv(dev);
752 int eff_mtu = dev->mtu + ETH_HLEN + VLAN_HLEN + ETH_LLC_SNAP_SIZE;
753 int buf_size = 0;
754 int i = 0;
755
756 while (buf_size < eff_mtu) {
757 priv->frag_info[i].frag_size =
758 (eff_mtu > buf_size + frag_sizes[i]) ?
759 frag_sizes[i] : eff_mtu - buf_size;
760 priv->frag_info[i].frag_prefix_size = buf_size;
761 if (!i) {
762 priv->frag_info[i].frag_align = NET_IP_ALIGN;
763 priv->frag_info[i].frag_stride =
764 ALIGN(frag_sizes[i] + NET_IP_ALIGN, SMP_CACHE_BYTES);
765 } else {
766 priv->frag_info[i].frag_align = 0;
767 priv->frag_info[i].frag_stride =
768 ALIGN(frag_sizes[i], SMP_CACHE_BYTES);
769 }
770 priv->frag_info[i].last_offset = mlx4_en_last_alloc_offset(
771 priv, priv->frag_info[i].frag_stride,
772 priv->frag_info[i].frag_align);
773 buf_size += priv->frag_info[i].frag_size;
774 i++;
775 }
776
777 priv->num_frags = i;
778 priv->rx_skb_size = eff_mtu;
779 priv->log_rx_info = ROUNDUP_LOG2(i * sizeof(struct skb_frag_struct));
780
781 en_dbg(DRV, priv, "Rx buffer scatter-list (effective-mtu:%d "
782 "num_frags:%d):\n", eff_mtu, priv->num_frags);
783 for (i = 0; i < priv->num_frags; i++) {
784 en_dbg(DRV, priv, " frag:%d - size:%d prefix:%d align:%d "
785 "stride:%d last_offset:%d\n", i,
786 priv->frag_info[i].frag_size,
787 priv->frag_info[i].frag_prefix_size,
788 priv->frag_info[i].frag_align,
789 priv->frag_info[i].frag_stride,
790 priv->frag_info[i].last_offset);
791 }
792 }
793
794 /* RSS related functions */
795
796 static int mlx4_en_config_rss_qp(struct mlx4_en_priv *priv, int qpn,
797 struct mlx4_en_rx_ring *ring,
798 enum mlx4_qp_state *state,
799 struct mlx4_qp *qp)
800 {
801 struct mlx4_en_dev *mdev = priv->mdev;
802 struct mlx4_qp_context *context;
803 int err = 0;
804
805 context = kmalloc(sizeof *context , GFP_KERNEL);
806 if (!context) {
807 en_err(priv, "Failed to allocate qp context\n");
808 return -ENOMEM;
809 }
810
811 err = mlx4_qp_alloc(mdev->dev, qpn, qp);
812 if (err) {
813 en_err(priv, "Failed to allocate qp #%x\n", qpn);
814 goto out;
815 }
816 qp->event = mlx4_en_sqp_event;
817
818 memset(context, 0, sizeof *context);
819 mlx4_en_fill_qp_context(priv, ring->size, ring->stride, 0, 0,
820 qpn, ring->cqn, context);
821 context->db_rec_addr = cpu_to_be64(ring->wqres.db.dma);
822
823 err = mlx4_qp_to_ready(mdev->dev, &ring->wqres.mtt, context, qp, state);
824 if (err) {
825 mlx4_qp_remove(mdev->dev, qp);
826 mlx4_qp_free(mdev->dev, qp);
827 }
828 mlx4_en_update_rx_prod_db(ring);
829 out:
830 kfree(context);
831 return err;
832 }
833
834 /* Allocate rx qp's and configure them according to rss map */
835 int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv)
836 {
837 struct mlx4_en_dev *mdev = priv->mdev;
838 struct mlx4_en_rss_map *rss_map = &priv->rss_map;
839 struct mlx4_qp_context context;
840 struct mlx4_en_rss_context *rss_context;
841 void *ptr;
842 int rss_xor = mdev->profile.rss_xor;
843 u8 rss_mask = mdev->profile.rss_mask;
844 int i, qpn;
845 int err = 0;
846 int good_qps = 0;
847
848 en_dbg(DRV, priv, "Configuring rss steering\n");
849 err = mlx4_qp_reserve_range(mdev->dev, priv->rx_ring_num,
850 priv->rx_ring_num,
851 &rss_map->base_qpn);
852 if (err) {
853 en_err(priv, "Failed reserving %d qps\n", priv->rx_ring_num);
854 return err;
855 }
856
857 for (i = 0; i < priv->rx_ring_num; i++) {
858 qpn = rss_map->base_qpn + i;
859 err = mlx4_en_config_rss_qp(priv, qpn, &priv->rx_ring[i],
860 &rss_map->state[i],
861 &rss_map->qps[i]);
862 if (err)
863 goto rss_err;
864
865 ++good_qps;
866 }
867
868 /* Configure RSS indirection qp */
869 err = mlx4_qp_reserve_range(mdev->dev, 1, 1, &priv->base_qpn);
870 if (err) {
871 en_err(priv, "Failed to reserve range for RSS "
872 "indirection qp\n");
873 goto rss_err;
874 }
875 err = mlx4_qp_alloc(mdev->dev, priv->base_qpn, &rss_map->indir_qp);
876 if (err) {
877 en_err(priv, "Failed to allocate RSS indirection QP\n");
878 goto reserve_err;
879 }
880 rss_map->indir_qp.event = mlx4_en_sqp_event;
881 mlx4_en_fill_qp_context(priv, 0, 0, 0, 1, priv->base_qpn,
882 priv->rx_ring[0].cqn, &context);
883
884 ptr = ((void *) &context) + 0x3c;
885 rss_context = (struct mlx4_en_rss_context *) ptr;
886 rss_context->base_qpn = cpu_to_be32(ilog2(priv->rx_ring_num) << 24 |
887 (rss_map->base_qpn));
888 rss_context->default_qpn = cpu_to_be32(rss_map->base_qpn);
889 rss_context->hash_fn = rss_xor & 0x3;
890 rss_context->flags = rss_mask << 2;
891
892 err = mlx4_qp_to_ready(mdev->dev, &priv->res.mtt, &context,
893 &rss_map->indir_qp, &rss_map->indir_state);
894 if (err)
895 goto indir_err;
896
897 return 0;
898
899 indir_err:
900 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
901 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
902 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
903 mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
904 reserve_err:
905 mlx4_qp_release_range(mdev->dev, priv->base_qpn, 1);
906 rss_err:
907 for (i = 0; i < good_qps; i++) {
908 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
909 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
910 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
911 mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
912 }
913 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
914 return err;
915 }
916
917 void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv)
918 {
919 struct mlx4_en_dev *mdev = priv->mdev;
920 struct mlx4_en_rss_map *rss_map = &priv->rss_map;
921 int i;
922
923 mlx4_qp_modify(mdev->dev, NULL, rss_map->indir_state,
924 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->indir_qp);
925 mlx4_qp_remove(mdev->dev, &rss_map->indir_qp);
926 mlx4_qp_free(mdev->dev, &rss_map->indir_qp);
927 mlx4_qp_release_range(mdev->dev, priv->base_qpn, 1);
928
929 for (i = 0; i < priv->rx_ring_num; i++) {
930 mlx4_qp_modify(mdev->dev, NULL, rss_map->state[i],
931 MLX4_QP_STATE_RST, NULL, 0, 0, &rss_map->qps[i]);
932 mlx4_qp_remove(mdev->dev, &rss_map->qps[i]);
933 mlx4_qp_free(mdev->dev, &rss_map->qps[i]);
934 }
935 mlx4_qp_release_range(mdev->dev, rss_map->base_qpn, priv->rx_ring_num);
936 }
937
938
939
940
941
Mirror of Nicolas Pitre's orion git repository