cmd64x: don't clear the other channel's interrupt
[linux-2.6/linux-mips/linux-dm7025.git] / drivers / net / cxgb3 / sge.c
blobc15e43a8543b983f82b0517010979d5eb782bbf8
1 /*
2 * Copyright (c) 2005-2007 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
32 #include <linux/skbuff.h>
33 #include <linux/netdevice.h>
34 #include <linux/etherdevice.h>
35 #include <linux/if_vlan.h>
36 #include <linux/ip.h>
37 #include <linux/tcp.h>
38 #include <linux/dma-mapping.h>
39 #include "common.h"
40 #include "regs.h"
41 #include "sge_defs.h"
42 #include "t3_cpl.h"
43 #include "firmware_exports.h"
45 #define USE_GTS 0
47 #define SGE_RX_SM_BUF_SIZE 1536
49 #define SGE_RX_COPY_THRES 256
50 #define SGE_RX_PULL_LEN 128
53 * Page chunk size for FL0 buffers if FL0 is to be populated with page chunks.
54 * It must be a divisor of PAGE_SIZE. If set to 0 FL0 will use sk_buffs
55 * directly.
57 #define FL0_PG_CHUNK_SIZE 2048
59 #define SGE_RX_DROP_THRES 16
62 * Period of the Tx buffer reclaim timer. This timer does not need to run
63 * frequently as Tx buffers are usually reclaimed by new Tx packets.
65 #define TX_RECLAIM_PERIOD (HZ / 4)
67 /* WR size in bytes */
68 #define WR_LEN (WR_FLITS * 8)
71 * Types of Tx queues in each queue set. Order here matters, do not change.
73 enum { TXQ_ETH, TXQ_OFLD, TXQ_CTRL };
75 /* Values for sge_txq.flags */
76 enum {
77 TXQ_RUNNING = 1 << 0, /* fetch engine is running */
78 TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */
81 struct tx_desc {
82 __be64 flit[TX_DESC_FLITS];
85 struct rx_desc {
86 __be32 addr_lo;
87 __be32 len_gen;
88 __be32 gen2;
89 __be32 addr_hi;
92 struct tx_sw_desc { /* SW state per Tx descriptor */
93 struct sk_buff *skb;
96 struct rx_sw_desc { /* SW state per Rx descriptor */
97 union {
98 struct sk_buff *skb;
99 struct fl_pg_chunk pg_chunk;
101 DECLARE_PCI_UNMAP_ADDR(dma_addr);
104 struct rsp_desc { /* response queue descriptor */
105 struct rss_header rss_hdr;
106 __be32 flags;
107 __be32 len_cq;
108 u8 imm_data[47];
109 u8 intr_gen;
112 struct unmap_info { /* packet unmapping info, overlays skb->cb */
113 int sflit; /* start flit of first SGL entry in Tx descriptor */
114 u16 fragidx; /* first page fragment in current Tx descriptor */
115 u16 addr_idx; /* buffer index of first SGL entry in descriptor */
116 u32 len; /* mapped length of skb main body */
120 * Holds unmapping information for Tx packets that need deferred unmapping.
121 * This structure lives at skb->head and must be allocated by callers.
123 struct deferred_unmap_info {
124 struct pci_dev *pdev;
125 dma_addr_t addr[MAX_SKB_FRAGS + 1];
129 * Maps a number of flits to the number of Tx descriptors that can hold them.
130 * The formula is
132 * desc = 1 + (flits - 2) / (WR_FLITS - 1).
134 * HW allows up to 4 descriptors to be combined into a WR.
136 static u8 flit_desc_map[] = {
138 #if SGE_NUM_GENBITS == 1
139 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
140 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
141 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
142 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
143 #elif SGE_NUM_GENBITS == 2
144 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
145 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
146 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
147 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
148 #else
149 # error "SGE_NUM_GENBITS must be 1 or 2"
150 #endif
153 static inline struct sge_qset *fl_to_qset(const struct sge_fl *q, int qidx)
155 return container_of(q, struct sge_qset, fl[qidx]);
158 static inline struct sge_qset *rspq_to_qset(const struct sge_rspq *q)
160 return container_of(q, struct sge_qset, rspq);
163 static inline struct sge_qset *txq_to_qset(const struct sge_txq *q, int qidx)
165 return container_of(q, struct sge_qset, txq[qidx]);
169 * refill_rspq - replenish an SGE response queue
170 * @adapter: the adapter
171 * @q: the response queue to replenish
172 * @credits: how many new responses to make available
174 * Replenishes a response queue by making the supplied number of responses
175 * available to HW.
177 static inline void refill_rspq(struct adapter *adapter,
178 const struct sge_rspq *q, unsigned int credits)
180 t3_write_reg(adapter, A_SG_RSPQ_CREDIT_RETURN,
181 V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
185 * need_skb_unmap - does the platform need unmapping of sk_buffs?
187 * Returns true if the platfrom needs sk_buff unmapping. The compiler
188 * optimizes away unecessary code if this returns true.
190 static inline int need_skb_unmap(void)
193 * This structure is used to tell if the platfrom needs buffer
194 * unmapping by checking if DECLARE_PCI_UNMAP_ADDR defines anything.
196 struct dummy {
197 DECLARE_PCI_UNMAP_ADDR(addr);
200 return sizeof(struct dummy) != 0;
204 * unmap_skb - unmap a packet main body and its page fragments
205 * @skb: the packet
206 * @q: the Tx queue containing Tx descriptors for the packet
207 * @cidx: index of Tx descriptor
208 * @pdev: the PCI device
210 * Unmap the main body of an sk_buff and its page fragments, if any.
211 * Because of the fairly complicated structure of our SGLs and the desire
212 * to conserve space for metadata, we keep the information necessary to
213 * unmap an sk_buff partly in the sk_buff itself (in its cb), and partly
214 * in the Tx descriptors (the physical addresses of the various data
215 * buffers). The send functions initialize the state in skb->cb so we
216 * can unmap the buffers held in the first Tx descriptor here, and we
217 * have enough information at this point to update the state for the next
218 * Tx descriptor.
220 static inline void unmap_skb(struct sk_buff *skb, struct sge_txq *q,
221 unsigned int cidx, struct pci_dev *pdev)
223 const struct sg_ent *sgp;
224 struct unmap_info *ui = (struct unmap_info *)skb->cb;
225 int nfrags, frag_idx, curflit, j = ui->addr_idx;
227 sgp = (struct sg_ent *)&q->desc[cidx].flit[ui->sflit];
229 if (ui->len) {
230 pci_unmap_single(pdev, be64_to_cpu(sgp->addr[0]), ui->len,
231 PCI_DMA_TODEVICE);
232 ui->len = 0; /* so we know for next descriptor for this skb */
233 j = 1;
236 frag_idx = ui->fragidx;
237 curflit = ui->sflit + 1 + j;
238 nfrags = skb_shinfo(skb)->nr_frags;
240 while (frag_idx < nfrags && curflit < WR_FLITS) {
241 pci_unmap_page(pdev, be64_to_cpu(sgp->addr[j]),
242 skb_shinfo(skb)->frags[frag_idx].size,
243 PCI_DMA_TODEVICE);
244 j ^= 1;
245 if (j == 0) {
246 sgp++;
247 curflit++;
249 curflit++;
250 frag_idx++;
253 if (frag_idx < nfrags) { /* SGL continues into next Tx descriptor */
254 ui->fragidx = frag_idx;
255 ui->addr_idx = j;
256 ui->sflit = curflit - WR_FLITS - j; /* sflit can be -1 */
261 * free_tx_desc - reclaims Tx descriptors and their buffers
262 * @adapter: the adapter
263 * @q: the Tx queue to reclaim descriptors from
264 * @n: the number of descriptors to reclaim
266 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
267 * Tx buffers. Called with the Tx queue lock held.
269 static void free_tx_desc(struct adapter *adapter, struct sge_txq *q,
270 unsigned int n)
272 struct tx_sw_desc *d;
273 struct pci_dev *pdev = adapter->pdev;
274 unsigned int cidx = q->cidx;
276 const int need_unmap = need_skb_unmap() &&
277 q->cntxt_id >= FW_TUNNEL_SGEEC_START;
279 d = &q->sdesc[cidx];
280 while (n--) {
281 if (d->skb) { /* an SGL is present */
282 if (need_unmap)
283 unmap_skb(d->skb, q, cidx, pdev);
284 if (d->skb->priority == cidx)
285 kfree_skb(d->skb);
287 ++d;
288 if (++cidx == q->size) {
289 cidx = 0;
290 d = q->sdesc;
293 q->cidx = cidx;
297 * reclaim_completed_tx - reclaims completed Tx descriptors
298 * @adapter: the adapter
299 * @q: the Tx queue to reclaim completed descriptors from
301 * Reclaims Tx descriptors that the SGE has indicated it has processed,
302 * and frees the associated buffers if possible. Called with the Tx
303 * queue's lock held.
305 static inline void reclaim_completed_tx(struct adapter *adapter,
306 struct sge_txq *q)
308 unsigned int reclaim = q->processed - q->cleaned;
310 if (reclaim) {
311 free_tx_desc(adapter, q, reclaim);
312 q->cleaned += reclaim;
313 q->in_use -= reclaim;
318 * should_restart_tx - are there enough resources to restart a Tx queue?
319 * @q: the Tx queue
321 * Checks if there are enough descriptors to restart a suspended Tx queue.
323 static inline int should_restart_tx(const struct sge_txq *q)
325 unsigned int r = q->processed - q->cleaned;
327 return q->in_use - r < (q->size >> 1);
331 * free_rx_bufs - free the Rx buffers on an SGE free list
332 * @pdev: the PCI device associated with the adapter
333 * @rxq: the SGE free list to clean up
335 * Release the buffers on an SGE free-buffer Rx queue. HW fetching from
336 * this queue should be stopped before calling this function.
338 static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q)
340 unsigned int cidx = q->cidx;
342 while (q->credits--) {
343 struct rx_sw_desc *d = &q->sdesc[cidx];
345 pci_unmap_single(pdev, pci_unmap_addr(d, dma_addr),
346 q->buf_size, PCI_DMA_FROMDEVICE);
347 if (q->use_pages) {
348 put_page(d->pg_chunk.page);
349 d->pg_chunk.page = NULL;
350 } else {
351 kfree_skb(d->skb);
352 d->skb = NULL;
354 if (++cidx == q->size)
355 cidx = 0;
358 if (q->pg_chunk.page) {
359 __free_page(q->pg_chunk.page);
360 q->pg_chunk.page = NULL;
365 * add_one_rx_buf - add a packet buffer to a free-buffer list
366 * @va: buffer start VA
367 * @len: the buffer length
368 * @d: the HW Rx descriptor to write
369 * @sd: the SW Rx descriptor to write
370 * @gen: the generation bit value
371 * @pdev: the PCI device associated with the adapter
373 * Add a buffer of the given length to the supplied HW and SW Rx
374 * descriptors.
376 static inline void add_one_rx_buf(void *va, unsigned int len,
377 struct rx_desc *d, struct rx_sw_desc *sd,
378 unsigned int gen, struct pci_dev *pdev)
380 dma_addr_t mapping;
382 mapping = pci_map_single(pdev, va, len, PCI_DMA_FROMDEVICE);
383 pci_unmap_addr_set(sd, dma_addr, mapping);
385 d->addr_lo = cpu_to_be32(mapping);
386 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
387 wmb();
388 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
389 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
392 static int alloc_pg_chunk(struct sge_fl *q, struct rx_sw_desc *sd, gfp_t gfp)
394 if (!q->pg_chunk.page) {
395 q->pg_chunk.page = alloc_page(gfp);
396 if (unlikely(!q->pg_chunk.page))
397 return -ENOMEM;
398 q->pg_chunk.va = page_address(q->pg_chunk.page);
399 q->pg_chunk.offset = 0;
401 sd->pg_chunk = q->pg_chunk;
403 q->pg_chunk.offset += q->buf_size;
404 if (q->pg_chunk.offset == PAGE_SIZE)
405 q->pg_chunk.page = NULL;
406 else {
407 q->pg_chunk.va += q->buf_size;
408 get_page(q->pg_chunk.page);
410 return 0;
414 * refill_fl - refill an SGE free-buffer list
415 * @adapter: the adapter
416 * @q: the free-list to refill
417 * @n: the number of new buffers to allocate
418 * @gfp: the gfp flags for allocating new buffers
420 * (Re)populate an SGE free-buffer list with up to @n new packet buffers,
421 * allocated with the supplied gfp flags. The caller must assure that
422 * @n does not exceed the queue's capacity.
424 static void refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp)
426 void *buf_start;
427 struct rx_sw_desc *sd = &q->sdesc[q->pidx];
428 struct rx_desc *d = &q->desc[q->pidx];
430 while (n--) {
431 if (q->use_pages) {
432 if (unlikely(alloc_pg_chunk(q, sd, gfp))) {
433 nomem: q->alloc_failed++;
434 break;
436 buf_start = sd->pg_chunk.va;
437 } else {
438 struct sk_buff *skb = alloc_skb(q->buf_size, gfp);
440 if (!skb)
441 goto nomem;
443 sd->skb = skb;
444 buf_start = skb->data;
447 add_one_rx_buf(buf_start, q->buf_size, d, sd, q->gen,
448 adap->pdev);
449 d++;
450 sd++;
451 if (++q->pidx == q->size) {
452 q->pidx = 0;
453 q->gen ^= 1;
454 sd = q->sdesc;
455 d = q->desc;
457 q->credits++;
460 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
463 static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
465 refill_fl(adap, fl, min(16U, fl->size - fl->credits), GFP_ATOMIC);
469 * recycle_rx_buf - recycle a receive buffer
470 * @adapter: the adapter
471 * @q: the SGE free list
472 * @idx: index of buffer to recycle
474 * Recycles the specified buffer on the given free list by adding it at
475 * the next available slot on the list.
477 static void recycle_rx_buf(struct adapter *adap, struct sge_fl *q,
478 unsigned int idx)
480 struct rx_desc *from = &q->desc[idx];
481 struct rx_desc *to = &q->desc[q->pidx];
483 q->sdesc[q->pidx] = q->sdesc[idx];
484 to->addr_lo = from->addr_lo; /* already big endian */
485 to->addr_hi = from->addr_hi; /* likewise */
486 wmb();
487 to->len_gen = cpu_to_be32(V_FLD_GEN1(q->gen));
488 to->gen2 = cpu_to_be32(V_FLD_GEN2(q->gen));
489 q->credits++;
491 if (++q->pidx == q->size) {
492 q->pidx = 0;
493 q->gen ^= 1;
495 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
499 * alloc_ring - allocate resources for an SGE descriptor ring
500 * @pdev: the PCI device
501 * @nelem: the number of descriptors
502 * @elem_size: the size of each descriptor
503 * @sw_size: the size of the SW state associated with each ring element
504 * @phys: the physical address of the allocated ring
505 * @metadata: address of the array holding the SW state for the ring
507 * Allocates resources for an SGE descriptor ring, such as Tx queues,
508 * free buffer lists, or response queues. Each SGE ring requires
509 * space for its HW descriptors plus, optionally, space for the SW state
510 * associated with each HW entry (the metadata). The function returns
511 * three values: the virtual address for the HW ring (the return value
512 * of the function), the physical address of the HW ring, and the address
513 * of the SW ring.
515 static void *alloc_ring(struct pci_dev *pdev, size_t nelem, size_t elem_size,
516 size_t sw_size, dma_addr_t * phys, void *metadata)
518 size_t len = nelem * elem_size;
519 void *s = NULL;
520 void *p = dma_alloc_coherent(&pdev->dev, len, phys, GFP_KERNEL);
522 if (!p)
523 return NULL;
524 if (sw_size) {
525 s = kcalloc(nelem, sw_size, GFP_KERNEL);
527 if (!s) {
528 dma_free_coherent(&pdev->dev, len, p, *phys);
529 return NULL;
532 if (metadata)
533 *(void **)metadata = s;
534 memset(p, 0, len);
535 return p;
539 * free_qset - free the resources of an SGE queue set
540 * @adapter: the adapter owning the queue set
541 * @q: the queue set
543 * Release the HW and SW resources associated with an SGE queue set, such
544 * as HW contexts, packet buffers, and descriptor rings. Traffic to the
545 * queue set must be quiesced prior to calling this.
547 static void t3_free_qset(struct adapter *adapter, struct sge_qset *q)
549 int i;
550 struct pci_dev *pdev = adapter->pdev;
552 if (q->tx_reclaim_timer.function)
553 del_timer_sync(&q->tx_reclaim_timer);
555 for (i = 0; i < SGE_RXQ_PER_SET; ++i)
556 if (q->fl[i].desc) {
557 spin_lock(&adapter->sge.reg_lock);
558 t3_sge_disable_fl(adapter, q->fl[i].cntxt_id);
559 spin_unlock(&adapter->sge.reg_lock);
560 free_rx_bufs(pdev, &q->fl[i]);
561 kfree(q->fl[i].sdesc);
562 dma_free_coherent(&pdev->dev,
563 q->fl[i].size *
564 sizeof(struct rx_desc), q->fl[i].desc,
565 q->fl[i].phys_addr);
568 for (i = 0; i < SGE_TXQ_PER_SET; ++i)
569 if (q->txq[i].desc) {
570 spin_lock(&adapter->sge.reg_lock);
571 t3_sge_enable_ecntxt(adapter, q->txq[i].cntxt_id, 0);
572 spin_unlock(&adapter->sge.reg_lock);
573 if (q->txq[i].sdesc) {
574 free_tx_desc(adapter, &q->txq[i],
575 q->txq[i].in_use);
576 kfree(q->txq[i].sdesc);
578 dma_free_coherent(&pdev->dev,
579 q->txq[i].size *
580 sizeof(struct tx_desc),
581 q->txq[i].desc, q->txq[i].phys_addr);
582 __skb_queue_purge(&q->txq[i].sendq);
585 if (q->rspq.desc) {
586 spin_lock(&adapter->sge.reg_lock);
587 t3_sge_disable_rspcntxt(adapter, q->rspq.cntxt_id);
588 spin_unlock(&adapter->sge.reg_lock);
589 dma_free_coherent(&pdev->dev,
590 q->rspq.size * sizeof(struct rsp_desc),
591 q->rspq.desc, q->rspq.phys_addr);
594 memset(q, 0, sizeof(*q));
598 * init_qset_cntxt - initialize an SGE queue set context info
599 * @qs: the queue set
600 * @id: the queue set id
602 * Initializes the TIDs and context ids for the queues of a queue set.
604 static void init_qset_cntxt(struct sge_qset *qs, unsigned int id)
606 qs->rspq.cntxt_id = id;
607 qs->fl[0].cntxt_id = 2 * id;
608 qs->fl[1].cntxt_id = 2 * id + 1;
609 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
610 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
611 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
612 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
613 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
617 * sgl_len - calculates the size of an SGL of the given capacity
618 * @n: the number of SGL entries
620 * Calculates the number of flits needed for a scatter/gather list that
621 * can hold the given number of entries.
623 static inline unsigned int sgl_len(unsigned int n)
625 /* alternatively: 3 * (n / 2) + 2 * (n & 1) */
626 return (3 * n) / 2 + (n & 1);
630 * flits_to_desc - returns the num of Tx descriptors for the given flits
631 * @n: the number of flits
633 * Calculates the number of Tx descriptors needed for the supplied number
634 * of flits.
636 static inline unsigned int flits_to_desc(unsigned int n)
638 BUG_ON(n >= ARRAY_SIZE(flit_desc_map));
639 return flit_desc_map[n];
643 * get_packet - return the next ingress packet buffer from a free list
644 * @adap: the adapter that received the packet
645 * @fl: the SGE free list holding the packet
646 * @len: the packet length including any SGE padding
647 * @drop_thres: # of remaining buffers before we start dropping packets
649 * Get the next packet from a free list and complete setup of the
650 * sk_buff. If the packet is small we make a copy and recycle the
651 * original buffer, otherwise we use the original buffer itself. If a
652 * positive drop threshold is supplied packets are dropped and their
653 * buffers recycled if (a) the number of remaining buffers is under the
654 * threshold and the packet is too big to copy, or (b) the packet should
655 * be copied but there is no memory for the copy.
657 static struct sk_buff *get_packet(struct adapter *adap, struct sge_fl *fl,
658 unsigned int len, unsigned int drop_thres)
660 struct sk_buff *skb = NULL;
661 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
663 prefetch(sd->skb->data);
664 fl->credits--;
666 if (len <= SGE_RX_COPY_THRES) {
667 skb = alloc_skb(len, GFP_ATOMIC);
668 if (likely(skb != NULL)) {
669 __skb_put(skb, len);
670 pci_dma_sync_single_for_cpu(adap->pdev,
671 pci_unmap_addr(sd, dma_addr), len,
672 PCI_DMA_FROMDEVICE);
673 memcpy(skb->data, sd->skb->data, len);
674 pci_dma_sync_single_for_device(adap->pdev,
675 pci_unmap_addr(sd, dma_addr), len,
676 PCI_DMA_FROMDEVICE);
677 } else if (!drop_thres)
678 goto use_orig_buf;
679 recycle:
680 recycle_rx_buf(adap, fl, fl->cidx);
681 return skb;
684 if (unlikely(fl->credits < drop_thres))
685 goto recycle;
687 use_orig_buf:
688 pci_unmap_single(adap->pdev, pci_unmap_addr(sd, dma_addr),
689 fl->buf_size, PCI_DMA_FROMDEVICE);
690 skb = sd->skb;
691 skb_put(skb, len);
692 __refill_fl(adap, fl);
693 return skb;
697 * get_packet_pg - return the next ingress packet buffer from a free list
698 * @adap: the adapter that received the packet
699 * @fl: the SGE free list holding the packet
700 * @len: the packet length including any SGE padding
701 * @drop_thres: # of remaining buffers before we start dropping packets
703 * Get the next packet from a free list populated with page chunks.
704 * If the packet is small we make a copy and recycle the original buffer,
705 * otherwise we attach the original buffer as a page fragment to a fresh
706 * sk_buff. If a positive drop threshold is supplied packets are dropped
707 * and their buffers recycled if (a) the number of remaining buffers is
708 * under the threshold and the packet is too big to copy, or (b) there's
709 * no system memory.
711 * Note: this function is similar to @get_packet but deals with Rx buffers
712 * that are page chunks rather than sk_buffs.
714 static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl,
715 unsigned int len, unsigned int drop_thres)
717 struct sk_buff *skb = NULL;
718 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
720 if (len <= SGE_RX_COPY_THRES) {
721 skb = alloc_skb(len, GFP_ATOMIC);
722 if (likely(skb != NULL)) {
723 __skb_put(skb, len);
724 pci_dma_sync_single_for_cpu(adap->pdev,
725 pci_unmap_addr(sd, dma_addr), len,
726 PCI_DMA_FROMDEVICE);
727 memcpy(skb->data, sd->pg_chunk.va, len);
728 pci_dma_sync_single_for_device(adap->pdev,
729 pci_unmap_addr(sd, dma_addr), len,
730 PCI_DMA_FROMDEVICE);
731 } else if (!drop_thres)
732 return NULL;
733 recycle:
734 fl->credits--;
735 recycle_rx_buf(adap, fl, fl->cidx);
736 return skb;
739 if (unlikely(fl->credits <= drop_thres))
740 goto recycle;
742 skb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC);
743 if (unlikely(!skb)) {
744 if (!drop_thres)
745 return NULL;
746 goto recycle;
749 pci_unmap_single(adap->pdev, pci_unmap_addr(sd, dma_addr),
750 fl->buf_size, PCI_DMA_FROMDEVICE);
751 __skb_put(skb, SGE_RX_PULL_LEN);
752 memcpy(skb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN);
753 skb_fill_page_desc(skb, 0, sd->pg_chunk.page,
754 sd->pg_chunk.offset + SGE_RX_PULL_LEN,
755 len - SGE_RX_PULL_LEN);
756 skb->len = len;
757 skb->data_len = len - SGE_RX_PULL_LEN;
758 skb->truesize += skb->data_len;
760 fl->credits--;
762 * We do not refill FLs here, we let the caller do it to overlap a
763 * prefetch.
765 return skb;
769 * get_imm_packet - return the next ingress packet buffer from a response
770 * @resp: the response descriptor containing the packet data
772 * Return a packet containing the immediate data of the given response.
774 static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp)
776 struct sk_buff *skb = alloc_skb(IMMED_PKT_SIZE, GFP_ATOMIC);
778 if (skb) {
779 __skb_put(skb, IMMED_PKT_SIZE);
780 skb_copy_to_linear_data(skb, resp->imm_data, IMMED_PKT_SIZE);
782 return skb;
786 * calc_tx_descs - calculate the number of Tx descriptors for a packet
787 * @skb: the packet
789 * Returns the number of Tx descriptors needed for the given Ethernet
790 * packet. Ethernet packets require addition of WR and CPL headers.
792 static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
794 unsigned int flits;
796 if (skb->len <= WR_LEN - sizeof(struct cpl_tx_pkt))
797 return 1;
799 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 2;
800 if (skb_shinfo(skb)->gso_size)
801 flits++;
802 return flits_to_desc(flits);
806 * make_sgl - populate a scatter/gather list for a packet
807 * @skb: the packet
808 * @sgp: the SGL to populate
809 * @start: start address of skb main body data to include in the SGL
810 * @len: length of skb main body data to include in the SGL
811 * @pdev: the PCI device
813 * Generates a scatter/gather list for the buffers that make up a packet
814 * and returns the SGL size in 8-byte words. The caller must size the SGL
815 * appropriately.
817 static inline unsigned int make_sgl(const struct sk_buff *skb,
818 struct sg_ent *sgp, unsigned char *start,
819 unsigned int len, struct pci_dev *pdev)
821 dma_addr_t mapping;
822 unsigned int i, j = 0, nfrags;
824 if (len) {
825 mapping = pci_map_single(pdev, start, len, PCI_DMA_TODEVICE);
826 sgp->len[0] = cpu_to_be32(len);
827 sgp->addr[0] = cpu_to_be64(mapping);
828 j = 1;
831 nfrags = skb_shinfo(skb)->nr_frags;
832 for (i = 0; i < nfrags; i++) {
833 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
835 mapping = pci_map_page(pdev, frag->page, frag->page_offset,
836 frag->size, PCI_DMA_TODEVICE);
837 sgp->len[j] = cpu_to_be32(frag->size);
838 sgp->addr[j] = cpu_to_be64(mapping);
839 j ^= 1;
840 if (j == 0)
841 ++sgp;
843 if (j)
844 sgp->len[j] = 0;
845 return ((nfrags + (len != 0)) * 3) / 2 + j;
849 * check_ring_tx_db - check and potentially ring a Tx queue's doorbell
850 * @adap: the adapter
851 * @q: the Tx queue
853 * Ring the doorbel if a Tx queue is asleep. There is a natural race,
854 * where the HW is going to sleep just after we checked, however,
855 * then the interrupt handler will detect the outstanding TX packet
856 * and ring the doorbell for us.
858 * When GTS is disabled we unconditionally ring the doorbell.
860 static inline void check_ring_tx_db(struct adapter *adap, struct sge_txq *q)
862 #if USE_GTS
863 clear_bit(TXQ_LAST_PKT_DB, &q->flags);
864 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
865 set_bit(TXQ_LAST_PKT_DB, &q->flags);
866 t3_write_reg(adap, A_SG_KDOORBELL,
867 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
869 #else
870 wmb(); /* write descriptors before telling HW */
871 t3_write_reg(adap, A_SG_KDOORBELL,
872 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
873 #endif
876 static inline void wr_gen2(struct tx_desc *d, unsigned int gen)
878 #if SGE_NUM_GENBITS == 2
879 d->flit[TX_DESC_FLITS - 1] = cpu_to_be64(gen);
880 #endif
884 * write_wr_hdr_sgl - write a WR header and, optionally, SGL
885 * @ndesc: number of Tx descriptors spanned by the SGL
886 * @skb: the packet corresponding to the WR
887 * @d: first Tx descriptor to be written
888 * @pidx: index of above descriptors
889 * @q: the SGE Tx queue
890 * @sgl: the SGL
891 * @flits: number of flits to the start of the SGL in the first descriptor
892 * @sgl_flits: the SGL size in flits
893 * @gen: the Tx descriptor generation
894 * @wr_hi: top 32 bits of WR header based on WR type (big endian)
895 * @wr_lo: low 32 bits of WR header based on WR type (big endian)
897 * Write a work request header and an associated SGL. If the SGL is
898 * small enough to fit into one Tx descriptor it has already been written
899 * and we just need to write the WR header. Otherwise we distribute the
900 * SGL across the number of descriptors it spans.
902 static void write_wr_hdr_sgl(unsigned int ndesc, struct sk_buff *skb,
903 struct tx_desc *d, unsigned int pidx,
904 const struct sge_txq *q,
905 const struct sg_ent *sgl,
906 unsigned int flits, unsigned int sgl_flits,
907 unsigned int gen, __be32 wr_hi,
908 __be32 wr_lo)
910 struct work_request_hdr *wrp = (struct work_request_hdr *)d;
911 struct tx_sw_desc *sd = &q->sdesc[pidx];
913 sd->skb = skb;
914 if (need_skb_unmap()) {
915 struct unmap_info *ui = (struct unmap_info *)skb->cb;
917 ui->fragidx = 0;
918 ui->addr_idx = 0;
919 ui->sflit = flits;
922 if (likely(ndesc == 1)) {
923 skb->priority = pidx;
924 wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
925 V_WR_SGLSFLT(flits)) | wr_hi;
926 wmb();
927 wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) |
928 V_WR_GEN(gen)) | wr_lo;
929 wr_gen2(d, gen);
930 } else {
931 unsigned int ogen = gen;
932 const u64 *fp = (const u64 *)sgl;
933 struct work_request_hdr *wp = wrp;
935 wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
936 V_WR_SGLSFLT(flits)) | wr_hi;
938 while (sgl_flits) {
939 unsigned int avail = WR_FLITS - flits;
941 if (avail > sgl_flits)
942 avail = sgl_flits;
943 memcpy(&d->flit[flits], fp, avail * sizeof(*fp));
944 sgl_flits -= avail;
945 ndesc--;
946 if (!sgl_flits)
947 break;
949 fp += avail;
950 d++;
951 sd++;
952 if (++pidx == q->size) {
953 pidx = 0;
954 gen ^= 1;
955 d = q->desc;
956 sd = q->sdesc;
959 sd->skb = skb;
960 wrp = (struct work_request_hdr *)d;
961 wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
962 V_WR_SGLSFLT(1)) | wr_hi;
963 wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS,
964 sgl_flits + 1)) |
965 V_WR_GEN(gen)) | wr_lo;
966 wr_gen2(d, gen);
967 flits = 1;
969 skb->priority = pidx;
970 wrp->wr_hi |= htonl(F_WR_EOP);
971 wmb();
972 wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
973 wr_gen2((struct tx_desc *)wp, ogen);
974 WARN_ON(ndesc != 0);
979 * write_tx_pkt_wr - write a TX_PKT work request
980 * @adap: the adapter
981 * @skb: the packet to send
982 * @pi: the egress interface
983 * @pidx: index of the first Tx descriptor to write
984 * @gen: the generation value to use
985 * @q: the Tx queue
986 * @ndesc: number of descriptors the packet will occupy
987 * @compl: the value of the COMPL bit to use
989 * Generate a TX_PKT work request to send the supplied packet.
991 static void write_tx_pkt_wr(struct adapter *adap, struct sk_buff *skb,
992 const struct port_info *pi,
993 unsigned int pidx, unsigned int gen,
994 struct sge_txq *q, unsigned int ndesc,
995 unsigned int compl)
997 unsigned int flits, sgl_flits, cntrl, tso_info;
998 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
999 struct tx_desc *d = &q->desc[pidx];
1000 struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)d;
1002 cpl->len = htonl(skb->len | 0x80000000);
1003 cntrl = V_TXPKT_INTF(pi->port_id);
1005 if (vlan_tx_tag_present(skb) && pi->vlan_grp)
1006 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(vlan_tx_tag_get(skb));
1008 tso_info = V_LSO_MSS(skb_shinfo(skb)->gso_size);
1009 if (tso_info) {
1010 int eth_type;
1011 struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)cpl;
1013 d->flit[2] = 0;
1014 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
1015 hdr->cntrl = htonl(cntrl);
1016 eth_type = skb_network_offset(skb) == ETH_HLEN ?
1017 CPL_ETH_II : CPL_ETH_II_VLAN;
1018 tso_info |= V_LSO_ETH_TYPE(eth_type) |
1019 V_LSO_IPHDR_WORDS(ip_hdr(skb)->ihl) |
1020 V_LSO_TCPHDR_WORDS(tcp_hdr(skb)->doff);
1021 hdr->lso_info = htonl(tso_info);
1022 flits = 3;
1023 } else {
1024 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
1025 cntrl |= F_TXPKT_IPCSUM_DIS; /* SW calculates IP csum */
1026 cntrl |= V_TXPKT_L4CSUM_DIS(skb->ip_summed != CHECKSUM_PARTIAL);
1027 cpl->cntrl = htonl(cntrl);
1029 if (skb->len <= WR_LEN - sizeof(*cpl)) {
1030 q->sdesc[pidx].skb = NULL;
1031 if (!skb->data_len)
1032 skb_copy_from_linear_data(skb, &d->flit[2],
1033 skb->len);
1034 else
1035 skb_copy_bits(skb, 0, &d->flit[2], skb->len);
1037 flits = (skb->len + 7) / 8 + 2;
1038 cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(skb->len & 7) |
1039 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT)
1040 | F_WR_SOP | F_WR_EOP | compl);
1041 wmb();
1042 cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(gen) |
1043 V_WR_TID(q->token));
1044 wr_gen2(d, gen);
1045 kfree_skb(skb);
1046 return;
1049 flits = 2;
1052 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1053 sgl_flits = make_sgl(skb, sgp, skb->data, skb_headlen(skb), adap->pdev);
1054 if (need_skb_unmap())
1055 ((struct unmap_info *)skb->cb)->len = skb_headlen(skb);
1057 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits, gen,
1058 htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | compl),
1059 htonl(V_WR_TID(q->token)));
1063 * eth_xmit - add a packet to the Ethernet Tx queue
1064 * @skb: the packet
1065 * @dev: the egress net device
1067 * Add a packet to an SGE Tx queue. Runs with softirqs disabled.
1069 int t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1071 unsigned int ndesc, pidx, credits, gen, compl;
1072 const struct port_info *pi = netdev_priv(dev);
1073 struct adapter *adap = pi->adapter;
1074 struct sge_qset *qs = pi->qs;
1075 struct sge_txq *q = &qs->txq[TXQ_ETH];
1078 * The chip min packet length is 9 octets but play safe and reject
1079 * anything shorter than an Ethernet header.
1081 if (unlikely(skb->len < ETH_HLEN)) {
1082 dev_kfree_skb(skb);
1083 return NETDEV_TX_OK;
1086 spin_lock(&q->lock);
1087 reclaim_completed_tx(adap, q);
1089 credits = q->size - q->in_use;
1090 ndesc = calc_tx_descs(skb);
1092 if (unlikely(credits < ndesc)) {
1093 if (!netif_queue_stopped(dev)) {
1094 netif_stop_queue(dev);
1095 set_bit(TXQ_ETH, &qs->txq_stopped);
1096 q->stops++;
1097 dev_err(&adap->pdev->dev,
1098 "%s: Tx ring %u full while queue awake!\n",
1099 dev->name, q->cntxt_id & 7);
1101 spin_unlock(&q->lock);
1102 return NETDEV_TX_BUSY;
1105 q->in_use += ndesc;
1106 if (unlikely(credits - ndesc < q->stop_thres)) {
1107 q->stops++;
1108 netif_stop_queue(dev);
1109 set_bit(TXQ_ETH, &qs->txq_stopped);
1110 #if !USE_GTS
1111 if (should_restart_tx(q) &&
1112 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1113 q->restarts++;
1114 netif_wake_queue(dev);
1116 #endif
1119 gen = q->gen;
1120 q->unacked += ndesc;
1121 compl = (q->unacked & 8) << (S_WR_COMPL - 3);
1122 q->unacked &= 7;
1123 pidx = q->pidx;
1124 q->pidx += ndesc;
1125 if (q->pidx >= q->size) {
1126 q->pidx -= q->size;
1127 q->gen ^= 1;
1130 /* update port statistics */
1131 if (skb->ip_summed == CHECKSUM_COMPLETE)
1132 qs->port_stats[SGE_PSTAT_TX_CSUM]++;
1133 if (skb_shinfo(skb)->gso_size)
1134 qs->port_stats[SGE_PSTAT_TSO]++;
1135 if (vlan_tx_tag_present(skb) && pi->vlan_grp)
1136 qs->port_stats[SGE_PSTAT_VLANINS]++;
1138 dev->trans_start = jiffies;
1139 spin_unlock(&q->lock);
1142 * We do not use Tx completion interrupts to free DMAd Tx packets.
1143 * This is good for performamce but means that we rely on new Tx
1144 * packets arriving to run the destructors of completed packets,
1145 * which open up space in their sockets' send queues. Sometimes
1146 * we do not get such new packets causing Tx to stall. A single
1147 * UDP transmitter is a good example of this situation. We have
1148 * a clean up timer that periodically reclaims completed packets
1149 * but it doesn't run often enough (nor do we want it to) to prevent
1150 * lengthy stalls. A solution to this problem is to run the
1151 * destructor early, after the packet is queued but before it's DMAd.
1152 * A cons is that we lie to socket memory accounting, but the amount
1153 * of extra memory is reasonable (limited by the number of Tx
1154 * descriptors), the packets do actually get freed quickly by new
1155 * packets almost always, and for protocols like TCP that wait for
1156 * acks to really free up the data the extra memory is even less.
1157 * On the positive side we run the destructors on the sending CPU
1158 * rather than on a potentially different completing CPU, usually a
1159 * good thing. We also run them without holding our Tx queue lock,
1160 * unlike what reclaim_completed_tx() would otherwise do.
1162 * Run the destructor before telling the DMA engine about the packet
1163 * to make sure it doesn't complete and get freed prematurely.
1165 if (likely(!skb_shared(skb)))
1166 skb_orphan(skb);
1168 write_tx_pkt_wr(adap, skb, pi, pidx, gen, q, ndesc, compl);
1169 check_ring_tx_db(adap, q);
1170 return NETDEV_TX_OK;
1174 * write_imm - write a packet into a Tx descriptor as immediate data
1175 * @d: the Tx descriptor to write
1176 * @skb: the packet
1177 * @len: the length of packet data to write as immediate data
1178 * @gen: the generation bit value to write
1180 * Writes a packet as immediate data into a Tx descriptor. The packet
1181 * contains a work request at its beginning. We must write the packet
1182 * carefully so the SGE doesn't read it accidentally before it's written
1183 * in its entirety.
1185 static inline void write_imm(struct tx_desc *d, struct sk_buff *skb,
1186 unsigned int len, unsigned int gen)
1188 struct work_request_hdr *from = (struct work_request_hdr *)skb->data;
1189 struct work_request_hdr *to = (struct work_request_hdr *)d;
1191 if (likely(!skb->data_len))
1192 memcpy(&to[1], &from[1], len - sizeof(*from));
1193 else
1194 skb_copy_bits(skb, sizeof(*from), &to[1], len - sizeof(*from));
1196 to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP |
1197 V_WR_BCNTLFLT(len & 7));
1198 wmb();
1199 to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) |
1200 V_WR_LEN((len + 7) / 8));
1201 wr_gen2(d, gen);
1202 kfree_skb(skb);
1206 * check_desc_avail - check descriptor availability on a send queue
1207 * @adap: the adapter
1208 * @q: the send queue
1209 * @skb: the packet needing the descriptors
1210 * @ndesc: the number of Tx descriptors needed
1211 * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
1213 * Checks if the requested number of Tx descriptors is available on an
1214 * SGE send queue. If the queue is already suspended or not enough
1215 * descriptors are available the packet is queued for later transmission.
1216 * Must be called with the Tx queue locked.
1218 * Returns 0 if enough descriptors are available, 1 if there aren't
1219 * enough descriptors and the packet has been queued, and 2 if the caller
1220 * needs to retry because there weren't enough descriptors at the
1221 * beginning of the call but some freed up in the mean time.
1223 static inline int check_desc_avail(struct adapter *adap, struct sge_txq *q,
1224 struct sk_buff *skb, unsigned int ndesc,
1225 unsigned int qid)
1227 if (unlikely(!skb_queue_empty(&q->sendq))) {
1228 addq_exit:__skb_queue_tail(&q->sendq, skb);
1229 return 1;
1231 if (unlikely(q->size - q->in_use < ndesc)) {
1232 struct sge_qset *qs = txq_to_qset(q, qid);
1234 set_bit(qid, &qs->txq_stopped);
1235 smp_mb__after_clear_bit();
1237 if (should_restart_tx(q) &&
1238 test_and_clear_bit(qid, &qs->txq_stopped))
1239 return 2;
1241 q->stops++;
1242 goto addq_exit;
1244 return 0;
1248 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1249 * @q: the SGE control Tx queue
1251 * This is a variant of reclaim_completed_tx() that is used for Tx queues
1252 * that send only immediate data (presently just the control queues) and
1253 * thus do not have any sk_buffs to release.
1255 static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1257 unsigned int reclaim = q->processed - q->cleaned;
1259 q->in_use -= reclaim;
1260 q->cleaned += reclaim;
1263 static inline int immediate(const struct sk_buff *skb)
1265 return skb->len <= WR_LEN;
1269 * ctrl_xmit - send a packet through an SGE control Tx queue
1270 * @adap: the adapter
1271 * @q: the control queue
1272 * @skb: the packet
1274 * Send a packet through an SGE control Tx queue. Packets sent through
1275 * a control queue must fit entirely as immediate data in a single Tx
1276 * descriptor and have no page fragments.
1278 static int ctrl_xmit(struct adapter *adap, struct sge_txq *q,
1279 struct sk_buff *skb)
1281 int ret;
1282 struct work_request_hdr *wrp = (struct work_request_hdr *)skb->data;
1284 if (unlikely(!immediate(skb))) {
1285 WARN_ON(1);
1286 dev_kfree_skb(skb);
1287 return NET_XMIT_SUCCESS;
1290 wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP);
1291 wrp->wr_lo = htonl(V_WR_TID(q->token));
1293 spin_lock(&q->lock);
1294 again:reclaim_completed_tx_imm(q);
1296 ret = check_desc_avail(adap, q, skb, 1, TXQ_CTRL);
1297 if (unlikely(ret)) {
1298 if (ret == 1) {
1299 spin_unlock(&q->lock);
1300 return NET_XMIT_CN;
1302 goto again;
1305 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1307 q->in_use++;
1308 if (++q->pidx >= q->size) {
1309 q->pidx = 0;
1310 q->gen ^= 1;
1312 spin_unlock(&q->lock);
1313 wmb();
1314 t3_write_reg(adap, A_SG_KDOORBELL,
1315 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1316 return NET_XMIT_SUCCESS;
1320 * restart_ctrlq - restart a suspended control queue
1321 * @qs: the queue set cotaining the control queue
1323 * Resumes transmission on a suspended Tx control queue.
1325 static void restart_ctrlq(unsigned long data)
1327 struct sk_buff *skb;
1328 struct sge_qset *qs = (struct sge_qset *)data;
1329 struct sge_txq *q = &qs->txq[TXQ_CTRL];
1331 spin_lock(&q->lock);
1332 again:reclaim_completed_tx_imm(q);
1334 while (q->in_use < q->size &&
1335 (skb = __skb_dequeue(&q->sendq)) != NULL) {
1337 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1339 if (++q->pidx >= q->size) {
1340 q->pidx = 0;
1341 q->gen ^= 1;
1343 q->in_use++;
1346 if (!skb_queue_empty(&q->sendq)) {
1347 set_bit(TXQ_CTRL, &qs->txq_stopped);
1348 smp_mb__after_clear_bit();
1350 if (should_restart_tx(q) &&
1351 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
1352 goto again;
1353 q->stops++;
1356 spin_unlock(&q->lock);
1357 t3_write_reg(qs->adap, A_SG_KDOORBELL,
1358 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1362 * Send a management message through control queue 0
1364 int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
1366 return ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], skb);
1370 * deferred_unmap_destructor - unmap a packet when it is freed
1371 * @skb: the packet
1373 * This is the packet destructor used for Tx packets that need to remain
1374 * mapped until they are freed rather than until their Tx descriptors are
1375 * freed.
1377 static void deferred_unmap_destructor(struct sk_buff *skb)
1379 int i;
1380 const dma_addr_t *p;
1381 const struct skb_shared_info *si;
1382 const struct deferred_unmap_info *dui;
1383 const struct unmap_info *ui = (struct unmap_info *)skb->cb;
1385 dui = (struct deferred_unmap_info *)skb->head;
1386 p = dui->addr;
1388 if (ui->len)
1389 pci_unmap_single(dui->pdev, *p++, ui->len, PCI_DMA_TODEVICE);
1391 si = skb_shinfo(skb);
1392 for (i = 0; i < si->nr_frags; i++)
1393 pci_unmap_page(dui->pdev, *p++, si->frags[i].size,
1394 PCI_DMA_TODEVICE);
1397 static void setup_deferred_unmapping(struct sk_buff *skb, struct pci_dev *pdev,
1398 const struct sg_ent *sgl, int sgl_flits)
1400 dma_addr_t *p;
1401 struct deferred_unmap_info *dui;
1403 dui = (struct deferred_unmap_info *)skb->head;
1404 dui->pdev = pdev;
1405 for (p = dui->addr; sgl_flits >= 3; sgl++, sgl_flits -= 3) {
1406 *p++ = be64_to_cpu(sgl->addr[0]);
1407 *p++ = be64_to_cpu(sgl->addr[1]);
1409 if (sgl_flits)
1410 *p = be64_to_cpu(sgl->addr[0]);
1414 * write_ofld_wr - write an offload work request
1415 * @adap: the adapter
1416 * @skb: the packet to send
1417 * @q: the Tx queue
1418 * @pidx: index of the first Tx descriptor to write
1419 * @gen: the generation value to use
1420 * @ndesc: number of descriptors the packet will occupy
1422 * Write an offload work request to send the supplied packet. The packet
1423 * data already carry the work request with most fields populated.
1425 static void write_ofld_wr(struct adapter *adap, struct sk_buff *skb,
1426 struct sge_txq *q, unsigned int pidx,
1427 unsigned int gen, unsigned int ndesc)
1429 unsigned int sgl_flits, flits;
1430 struct work_request_hdr *from;
1431 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1432 struct tx_desc *d = &q->desc[pidx];
1434 if (immediate(skb)) {
1435 q->sdesc[pidx].skb = NULL;
1436 write_imm(d, skb, skb->len, gen);
1437 return;
1440 /* Only TX_DATA builds SGLs */
1442 from = (struct work_request_hdr *)skb->data;
1443 memcpy(&d->flit[1], &from[1],
1444 skb_transport_offset(skb) - sizeof(*from));
1446 flits = skb_transport_offset(skb) / 8;
1447 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1448 sgl_flits = make_sgl(skb, sgp, skb_transport_header(skb),
1449 skb->tail - skb->transport_header,
1450 adap->pdev);
1451 if (need_skb_unmap()) {
1452 setup_deferred_unmapping(skb, adap->pdev, sgp, sgl_flits);
1453 skb->destructor = deferred_unmap_destructor;
1454 ((struct unmap_info *)skb->cb)->len = (skb->tail -
1455 skb->transport_header);
1458 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits,
1459 gen, from->wr_hi, from->wr_lo);
1463 * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet
1464 * @skb: the packet
1466 * Returns the number of Tx descriptors needed for the given offload
1467 * packet. These packets are already fully constructed.
1469 static inline unsigned int calc_tx_descs_ofld(const struct sk_buff *skb)
1471 unsigned int flits, cnt;
1473 if (skb->len <= WR_LEN)
1474 return 1; /* packet fits as immediate data */
1476 flits = skb_transport_offset(skb) / 8; /* headers */
1477 cnt = skb_shinfo(skb)->nr_frags;
1478 if (skb->tail != skb->transport_header)
1479 cnt++;
1480 return flits_to_desc(flits + sgl_len(cnt));
1484 * ofld_xmit - send a packet through an offload queue
1485 * @adap: the adapter
1486 * @q: the Tx offload queue
1487 * @skb: the packet
1489 * Send an offload packet through an SGE offload queue.
1491 static int ofld_xmit(struct adapter *adap, struct sge_txq *q,
1492 struct sk_buff *skb)
1494 int ret;
1495 unsigned int ndesc = calc_tx_descs_ofld(skb), pidx, gen;
1497 spin_lock(&q->lock);
1498 again:reclaim_completed_tx(adap, q);
1500 ret = check_desc_avail(adap, q, skb, ndesc, TXQ_OFLD);
1501 if (unlikely(ret)) {
1502 if (ret == 1) {
1503 skb->priority = ndesc; /* save for restart */
1504 spin_unlock(&q->lock);
1505 return NET_XMIT_CN;
1507 goto again;
1510 gen = q->gen;
1511 q->in_use += ndesc;
1512 pidx = q->pidx;
1513 q->pidx += ndesc;
1514 if (q->pidx >= q->size) {
1515 q->pidx -= q->size;
1516 q->gen ^= 1;
1518 spin_unlock(&q->lock);
1520 write_ofld_wr(adap, skb, q, pidx, gen, ndesc);
1521 check_ring_tx_db(adap, q);
1522 return NET_XMIT_SUCCESS;
1526 * restart_offloadq - restart a suspended offload queue
1527 * @qs: the queue set cotaining the offload queue
1529 * Resumes transmission on a suspended Tx offload queue.
1531 static void restart_offloadq(unsigned long data)
1533 struct sk_buff *skb;
1534 struct sge_qset *qs = (struct sge_qset *)data;
1535 struct sge_txq *q = &qs->txq[TXQ_OFLD];
1536 const struct port_info *pi = netdev_priv(qs->netdev);
1537 struct adapter *adap = pi->adapter;
1539 spin_lock(&q->lock);
1540 again:reclaim_completed_tx(adap, q);
1542 while ((skb = skb_peek(&q->sendq)) != NULL) {
1543 unsigned int gen, pidx;
1544 unsigned int ndesc = skb->priority;
1546 if (unlikely(q->size - q->in_use < ndesc)) {
1547 set_bit(TXQ_OFLD, &qs->txq_stopped);
1548 smp_mb__after_clear_bit();
1550 if (should_restart_tx(q) &&
1551 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
1552 goto again;
1553 q->stops++;
1554 break;
1557 gen = q->gen;
1558 q->in_use += ndesc;
1559 pidx = q->pidx;
1560 q->pidx += ndesc;
1561 if (q->pidx >= q->size) {
1562 q->pidx -= q->size;
1563 q->gen ^= 1;
1565 __skb_unlink(skb, &q->sendq);
1566 spin_unlock(&q->lock);
1568 write_ofld_wr(adap, skb, q, pidx, gen, ndesc);
1569 spin_lock(&q->lock);
1571 spin_unlock(&q->lock);
1573 #if USE_GTS
1574 set_bit(TXQ_RUNNING, &q->flags);
1575 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1576 #endif
1577 t3_write_reg(adap, A_SG_KDOORBELL,
1578 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1582 * queue_set - return the queue set a packet should use
1583 * @skb: the packet
1585 * Maps a packet to the SGE queue set it should use. The desired queue
1586 * set is carried in bits 1-3 in the packet's priority.
1588 static inline int queue_set(const struct sk_buff *skb)
1590 return skb->priority >> 1;
1594 * is_ctrl_pkt - return whether an offload packet is a control packet
1595 * @skb: the packet
1597 * Determines whether an offload packet should use an OFLD or a CTRL
1598 * Tx queue. This is indicated by bit 0 in the packet's priority.
1600 static inline int is_ctrl_pkt(const struct sk_buff *skb)
1602 return skb->priority & 1;
1606 * t3_offload_tx - send an offload packet
1607 * @tdev: the offload device to send to
1608 * @skb: the packet
1610 * Sends an offload packet. We use the packet priority to select the
1611 * appropriate Tx queue as follows: bit 0 indicates whether the packet
1612 * should be sent as regular or control, bits 1-3 select the queue set.
1614 int t3_offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
1616 struct adapter *adap = tdev2adap(tdev);
1617 struct sge_qset *qs = &adap->sge.qs[queue_set(skb)];
1619 if (unlikely(is_ctrl_pkt(skb)))
1620 return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], skb);
1622 return ofld_xmit(adap, &qs->txq[TXQ_OFLD], skb);
1626 * offload_enqueue - add an offload packet to an SGE offload receive queue
1627 * @q: the SGE response queue
1628 * @skb: the packet
1630 * Add a new offload packet to an SGE response queue's offload packet
1631 * queue. If the packet is the first on the queue it schedules the RX
1632 * softirq to process the queue.
1634 static inline void offload_enqueue(struct sge_rspq *q, struct sk_buff *skb)
1636 skb->next = skb->prev = NULL;
1637 if (q->rx_tail)
1638 q->rx_tail->next = skb;
1639 else {
1640 struct sge_qset *qs = rspq_to_qset(q);
1642 napi_schedule(&qs->napi);
1643 q->rx_head = skb;
1645 q->rx_tail = skb;
1649 * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts
1650 * @tdev: the offload device that will be receiving the packets
1651 * @q: the SGE response queue that assembled the bundle
1652 * @skbs: the partial bundle
1653 * @n: the number of packets in the bundle
1655 * Delivers a (partial) bundle of Rx offload packets to an offload device.
1657 static inline void deliver_partial_bundle(struct t3cdev *tdev,
1658 struct sge_rspq *q,
1659 struct sk_buff *skbs[], int n)
1661 if (n) {
1662 q->offload_bundles++;
1663 tdev->recv(tdev, skbs, n);
1668 * ofld_poll - NAPI handler for offload packets in interrupt mode
1669 * @dev: the network device doing the polling
1670 * @budget: polling budget
1672 * The NAPI handler for offload packets when a response queue is serviced
1673 * by the hard interrupt handler, i.e., when it's operating in non-polling
1674 * mode. Creates small packet batches and sends them through the offload
1675 * receive handler. Batches need to be of modest size as we do prefetches
1676 * on the packets in each.
1678 static int ofld_poll(struct napi_struct *napi, int budget)
1680 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
1681 struct sge_rspq *q = &qs->rspq;
1682 struct adapter *adapter = qs->adap;
1683 int work_done = 0;
1685 while (work_done < budget) {
1686 struct sk_buff *head, *tail, *skbs[RX_BUNDLE_SIZE];
1687 int ngathered;
1689 spin_lock_irq(&q->lock);
1690 head = q->rx_head;
1691 if (!head) {
1692 napi_complete(napi);
1693 spin_unlock_irq(&q->lock);
1694 return work_done;
1697 tail = q->rx_tail;
1698 q->rx_head = q->rx_tail = NULL;
1699 spin_unlock_irq(&q->lock);
1701 for (ngathered = 0; work_done < budget && head; work_done++) {
1702 prefetch(head->data);
1703 skbs[ngathered] = head;
1704 head = head->next;
1705 skbs[ngathered]->next = NULL;
1706 if (++ngathered == RX_BUNDLE_SIZE) {
1707 q->offload_bundles++;
1708 adapter->tdev.recv(&adapter->tdev, skbs,
1709 ngathered);
1710 ngathered = 0;
1713 if (head) { /* splice remaining packets back onto Rx queue */
1714 spin_lock_irq(&q->lock);
1715 tail->next = q->rx_head;
1716 if (!q->rx_head)
1717 q->rx_tail = tail;
1718 q->rx_head = head;
1719 spin_unlock_irq(&q->lock);
1721 deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered);
1724 return work_done;
1728 * rx_offload - process a received offload packet
1729 * @tdev: the offload device receiving the packet
1730 * @rq: the response queue that received the packet
1731 * @skb: the packet
1732 * @rx_gather: a gather list of packets if we are building a bundle
1733 * @gather_idx: index of the next available slot in the bundle
1735 * Process an ingress offload pakcet and add it to the offload ingress
1736 * queue. Returns the index of the next available slot in the bundle.
1738 static inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq,
1739 struct sk_buff *skb, struct sk_buff *rx_gather[],
1740 unsigned int gather_idx)
1742 rq->offload_pkts++;
1743 skb_reset_mac_header(skb);
1744 skb_reset_network_header(skb);
1745 skb_reset_transport_header(skb);
1747 if (rq->polling) {
1748 rx_gather[gather_idx++] = skb;
1749 if (gather_idx == RX_BUNDLE_SIZE) {
1750 tdev->recv(tdev, rx_gather, RX_BUNDLE_SIZE);
1751 gather_idx = 0;
1752 rq->offload_bundles++;
1754 } else
1755 offload_enqueue(rq, skb);
1757 return gather_idx;
1761 * restart_tx - check whether to restart suspended Tx queues
1762 * @qs: the queue set to resume
1764 * Restarts suspended Tx queues of an SGE queue set if they have enough
1765 * free resources to resume operation.
1767 static void restart_tx(struct sge_qset *qs)
1769 if (test_bit(TXQ_ETH, &qs->txq_stopped) &&
1770 should_restart_tx(&qs->txq[TXQ_ETH]) &&
1771 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1772 qs->txq[TXQ_ETH].restarts++;
1773 if (netif_running(qs->netdev))
1774 netif_wake_queue(qs->netdev);
1777 if (test_bit(TXQ_OFLD, &qs->txq_stopped) &&
1778 should_restart_tx(&qs->txq[TXQ_OFLD]) &&
1779 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
1780 qs->txq[TXQ_OFLD].restarts++;
1781 tasklet_schedule(&qs->txq[TXQ_OFLD].qresume_tsk);
1783 if (test_bit(TXQ_CTRL, &qs->txq_stopped) &&
1784 should_restart_tx(&qs->txq[TXQ_CTRL]) &&
1785 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
1786 qs->txq[TXQ_CTRL].restarts++;
1787 tasklet_schedule(&qs->txq[TXQ_CTRL].qresume_tsk);
1792 * rx_eth - process an ingress ethernet packet
1793 * @adap: the adapter
1794 * @rq: the response queue that received the packet
1795 * @skb: the packet
1796 * @pad: amount of padding at the start of the buffer
1798 * Process an ingress ethernet pakcet and deliver it to the stack.
1799 * The padding is 2 if the packet was delivered in an Rx buffer and 0
1800 * if it was immediate data in a response.
1802 static void rx_eth(struct adapter *adap, struct sge_rspq *rq,
1803 struct sk_buff *skb, int pad)
1805 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)(skb->data + pad);
1806 struct port_info *pi;
1808 skb_pull(skb, sizeof(*p) + pad);
1809 skb->protocol = eth_type_trans(skb, adap->port[p->iff]);
1810 skb->dev->last_rx = jiffies;
1811 pi = netdev_priv(skb->dev);
1812 if (pi->rx_csum_offload && p->csum_valid && p->csum == 0xffff &&
1813 !p->fragment) {
1814 rspq_to_qset(rq)->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
1815 skb->ip_summed = CHECKSUM_UNNECESSARY;
1816 } else
1817 skb->ip_summed = CHECKSUM_NONE;
1819 if (unlikely(p->vlan_valid)) {
1820 struct vlan_group *grp = pi->vlan_grp;
1822 rspq_to_qset(rq)->port_stats[SGE_PSTAT_VLANEX]++;
1823 if (likely(grp))
1824 __vlan_hwaccel_rx(skb, grp, ntohs(p->vlan),
1825 rq->polling);
1826 else
1827 dev_kfree_skb_any(skb);
1828 } else if (rq->polling)
1829 netif_receive_skb(skb);
1830 else
1831 netif_rx(skb);
1835 * handle_rsp_cntrl_info - handles control information in a response
1836 * @qs: the queue set corresponding to the response
1837 * @flags: the response control flags
1839 * Handles the control information of an SGE response, such as GTS
1840 * indications and completion credits for the queue set's Tx queues.
1841 * HW coalesces credits, we don't do any extra SW coalescing.
1843 static inline void handle_rsp_cntrl_info(struct sge_qset *qs, u32 flags)
1845 unsigned int credits;
1847 #if USE_GTS
1848 if (flags & F_RSPD_TXQ0_GTS)
1849 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
1850 #endif
1852 credits = G_RSPD_TXQ0_CR(flags);
1853 if (credits)
1854 qs->txq[TXQ_ETH].processed += credits;
1856 credits = G_RSPD_TXQ2_CR(flags);
1857 if (credits)
1858 qs->txq[TXQ_CTRL].processed += credits;
1860 # if USE_GTS
1861 if (flags & F_RSPD_TXQ1_GTS)
1862 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);
1863 # endif
1864 credits = G_RSPD_TXQ1_CR(flags);
1865 if (credits)
1866 qs->txq[TXQ_OFLD].processed += credits;
1870 * check_ring_db - check if we need to ring any doorbells
1871 * @adapter: the adapter
1872 * @qs: the queue set whose Tx queues are to be examined
1873 * @sleeping: indicates which Tx queue sent GTS
1875 * Checks if some of a queue set's Tx queues need to ring their doorbells
1876 * to resume transmission after idling while they still have unprocessed
1877 * descriptors.
1879 static void check_ring_db(struct adapter *adap, struct sge_qset *qs,
1880 unsigned int sleeping)
1882 if (sleeping & F_RSPD_TXQ0_GTS) {
1883 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1885 if (txq->cleaned + txq->in_use != txq->processed &&
1886 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
1887 set_bit(TXQ_RUNNING, &txq->flags);
1888 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
1889 V_EGRCNTX(txq->cntxt_id));
1893 if (sleeping & F_RSPD_TXQ1_GTS) {
1894 struct sge_txq *txq = &qs->txq[TXQ_OFLD];
1896 if (txq->cleaned + txq->in_use != txq->processed &&
1897 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
1898 set_bit(TXQ_RUNNING, &txq->flags);
1899 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
1900 V_EGRCNTX(txq->cntxt_id));
1906 * is_new_response - check if a response is newly written
1907 * @r: the response descriptor
1908 * @q: the response queue
1910 * Returns true if a response descriptor contains a yet unprocessed
1911 * response.
1913 static inline int is_new_response(const struct rsp_desc *r,
1914 const struct sge_rspq *q)
1916 return (r->intr_gen & F_RSPD_GEN2) == q->gen;
1919 #define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
1920 #define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
1921 V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
1922 V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
1923 V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
1925 /* How long to delay the next interrupt in case of memory shortage, in 0.1us. */
1926 #define NOMEM_INTR_DELAY 2500
1929 * process_responses - process responses from an SGE response queue
1930 * @adap: the adapter
1931 * @qs: the queue set to which the response queue belongs
1932 * @budget: how many responses can be processed in this round
1934 * Process responses from an SGE response queue up to the supplied budget.
1935 * Responses include received packets as well as credits and other events
1936 * for the queues that belong to the response queue's queue set.
1937 * A negative budget is effectively unlimited.
1939 * Additionally choose the interrupt holdoff time for the next interrupt
1940 * on this queue. If the system is under memory shortage use a fairly
1941 * long delay to help recovery.
1943 static int process_responses(struct adapter *adap, struct sge_qset *qs,
1944 int budget)
1946 struct sge_rspq *q = &qs->rspq;
1947 struct rsp_desc *r = &q->desc[q->cidx];
1948 int budget_left = budget;
1949 unsigned int sleeping = 0;
1950 struct sk_buff *offload_skbs[RX_BUNDLE_SIZE];
1951 int ngathered = 0;
1953 q->next_holdoff = q->holdoff_tmr;
1955 while (likely(budget_left && is_new_response(r, q))) {
1956 int eth, ethpad = 2;
1957 struct sk_buff *skb = NULL;
1958 u32 len, flags = ntohl(r->flags);
1959 u32 rss_hi = *(const u32 *)r, rss_lo = r->rss_hdr.rss_hash_val;
1961 eth = r->rss_hdr.opcode == CPL_RX_PKT;
1963 if (unlikely(flags & F_RSPD_ASYNC_NOTIF)) {
1964 skb = alloc_skb(AN_PKT_SIZE, GFP_ATOMIC);
1965 if (!skb)
1966 goto no_mem;
1968 memcpy(__skb_put(skb, AN_PKT_SIZE), r, AN_PKT_SIZE);
1969 skb->data[0] = CPL_ASYNC_NOTIF;
1970 rss_hi = htonl(CPL_ASYNC_NOTIF << 24);
1971 q->async_notif++;
1972 } else if (flags & F_RSPD_IMM_DATA_VALID) {
1973 skb = get_imm_packet(r);
1974 if (unlikely(!skb)) {
1975 no_mem:
1976 q->next_holdoff = NOMEM_INTR_DELAY;
1977 q->nomem++;
1978 /* consume one credit since we tried */
1979 budget_left--;
1980 break;
1982 q->imm_data++;
1983 ethpad = 0;
1984 } else if ((len = ntohl(r->len_cq)) != 0) {
1985 struct sge_fl *fl;
1987 fl = (len & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];
1988 if (fl->use_pages) {
1989 void *addr = fl->sdesc[fl->cidx].pg_chunk.va;
1991 prefetch(addr);
1992 #if L1_CACHE_BYTES < 128
1993 prefetch(addr + L1_CACHE_BYTES);
1994 #endif
1995 __refill_fl(adap, fl);
1997 skb = get_packet_pg(adap, fl, G_RSPD_LEN(len),
1998 eth ? SGE_RX_DROP_THRES : 0);
1999 } else
2000 skb = get_packet(adap, fl, G_RSPD_LEN(len),
2001 eth ? SGE_RX_DROP_THRES : 0);
2002 if (unlikely(!skb)) {
2003 if (!eth)
2004 goto no_mem;
2005 q->rx_drops++;
2006 } else if (unlikely(r->rss_hdr.opcode == CPL_TRACE_PKT))
2007 __skb_pull(skb, 2);
2009 if (++fl->cidx == fl->size)
2010 fl->cidx = 0;
2011 } else
2012 q->pure_rsps++;
2014 if (flags & RSPD_CTRL_MASK) {
2015 sleeping |= flags & RSPD_GTS_MASK;
2016 handle_rsp_cntrl_info(qs, flags);
2019 r++;
2020 if (unlikely(++q->cidx == q->size)) {
2021 q->cidx = 0;
2022 q->gen ^= 1;
2023 r = q->desc;
2025 prefetch(r);
2027 if (++q->credits >= (q->size / 4)) {
2028 refill_rspq(adap, q, q->credits);
2029 q->credits = 0;
2032 if (likely(skb != NULL)) {
2033 if (eth)
2034 rx_eth(adap, q, skb, ethpad);
2035 else {
2036 /* Preserve the RSS info in csum & priority */
2037 skb->csum = rss_hi;
2038 skb->priority = rss_lo;
2039 ngathered = rx_offload(&adap->tdev, q, skb,
2040 offload_skbs,
2041 ngathered);
2044 --budget_left;
2047 deliver_partial_bundle(&adap->tdev, q, offload_skbs, ngathered);
2048 if (sleeping)
2049 check_ring_db(adap, qs, sleeping);
2051 smp_mb(); /* commit Tx queue .processed updates */
2052 if (unlikely(qs->txq_stopped != 0))
2053 restart_tx(qs);
2055 budget -= budget_left;
2056 return budget;
2059 static inline int is_pure_response(const struct rsp_desc *r)
2061 u32 n = ntohl(r->flags) & (F_RSPD_ASYNC_NOTIF | F_RSPD_IMM_DATA_VALID);
2063 return (n | r->len_cq) == 0;
2067 * napi_rx_handler - the NAPI handler for Rx processing
2068 * @napi: the napi instance
2069 * @budget: how many packets we can process in this round
2071 * Handler for new data events when using NAPI.
2073 static int napi_rx_handler(struct napi_struct *napi, int budget)
2075 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
2076 struct adapter *adap = qs->adap;
2077 int work_done = process_responses(adap, qs, budget);
2079 if (likely(work_done < budget)) {
2080 napi_complete(napi);
2083 * Because we don't atomically flush the following
2084 * write it is possible that in very rare cases it can
2085 * reach the device in a way that races with a new
2086 * response being written plus an error interrupt
2087 * causing the NAPI interrupt handler below to return
2088 * unhandled status to the OS. To protect against
2089 * this would require flushing the write and doing
2090 * both the write and the flush with interrupts off.
2091 * Way too expensive and unjustifiable given the
2092 * rarity of the race.
2094 * The race cannot happen at all with MSI-X.
2096 t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |
2097 V_NEWTIMER(qs->rspq.next_holdoff) |
2098 V_NEWINDEX(qs->rspq.cidx));
2100 return work_done;
2104 * Returns true if the device is already scheduled for polling.
2106 static inline int napi_is_scheduled(struct napi_struct *napi)
2108 return test_bit(NAPI_STATE_SCHED, &napi->state);
2112 * process_pure_responses - process pure responses from a response queue
2113 * @adap: the adapter
2114 * @qs: the queue set owning the response queue
2115 * @r: the first pure response to process
2117 * A simpler version of process_responses() that handles only pure (i.e.,
2118 * non data-carrying) responses. Such respones are too light-weight to
2119 * justify calling a softirq under NAPI, so we handle them specially in
2120 * the interrupt handler. The function is called with a pointer to a
2121 * response, which the caller must ensure is a valid pure response.
2123 * Returns 1 if it encounters a valid data-carrying response, 0 otherwise.
2125 static int process_pure_responses(struct adapter *adap, struct sge_qset *qs,
2126 struct rsp_desc *r)
2128 struct sge_rspq *q = &qs->rspq;
2129 unsigned int sleeping = 0;
2131 do {
2132 u32 flags = ntohl(r->flags);
2134 r++;
2135 if (unlikely(++q->cidx == q->size)) {
2136 q->cidx = 0;
2137 q->gen ^= 1;
2138 r = q->desc;
2140 prefetch(r);
2142 if (flags & RSPD_CTRL_MASK) {
2143 sleeping |= flags & RSPD_GTS_MASK;
2144 handle_rsp_cntrl_info(qs, flags);
2147 q->pure_rsps++;
2148 if (++q->credits >= (q->size / 4)) {
2149 refill_rspq(adap, q, q->credits);
2150 q->credits = 0;
2152 } while (is_new_response(r, q) && is_pure_response(r));
2154 if (sleeping)
2155 check_ring_db(adap, qs, sleeping);
2157 smp_mb(); /* commit Tx queue .processed updates */
2158 if (unlikely(qs->txq_stopped != 0))
2159 restart_tx(qs);
2161 return is_new_response(r, q);
2165 * handle_responses - decide what to do with new responses in NAPI mode
2166 * @adap: the adapter
2167 * @q: the response queue
2169 * This is used by the NAPI interrupt handlers to decide what to do with
2170 * new SGE responses. If there are no new responses it returns -1. If
2171 * there are new responses and they are pure (i.e., non-data carrying)
2172 * it handles them straight in hard interrupt context as they are very
2173 * cheap and don't deliver any packets. Finally, if there are any data
2174 * signaling responses it schedules the NAPI handler. Returns 1 if it
2175 * schedules NAPI, 0 if all new responses were pure.
2177 * The caller must ascertain NAPI is not already running.
2179 static inline int handle_responses(struct adapter *adap, struct sge_rspq *q)
2181 struct sge_qset *qs = rspq_to_qset(q);
2182 struct rsp_desc *r = &q->desc[q->cidx];
2184 if (!is_new_response(r, q))
2185 return -1;
2186 if (is_pure_response(r) && process_pure_responses(adap, qs, r) == 0) {
2187 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2188 V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx));
2189 return 0;
2191 napi_schedule(&qs->napi);
2192 return 1;
2196 * The MSI-X interrupt handler for an SGE response queue for the non-NAPI case
2197 * (i.e., response queue serviced in hard interrupt).
2199 irqreturn_t t3_sge_intr_msix(int irq, void *cookie)
2201 struct sge_qset *qs = cookie;
2202 struct adapter *adap = qs->adap;
2203 struct sge_rspq *q = &qs->rspq;
2205 spin_lock(&q->lock);
2206 if (process_responses(adap, qs, -1) == 0)
2207 q->unhandled_irqs++;
2208 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2209 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2210 spin_unlock(&q->lock);
2211 return IRQ_HANDLED;
2215 * The MSI-X interrupt handler for an SGE response queue for the NAPI case
2216 * (i.e., response queue serviced by NAPI polling).
2218 static irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie)
2220 struct sge_qset *qs = cookie;
2221 struct sge_rspq *q = &qs->rspq;
2223 spin_lock(&q->lock);
2225 if (handle_responses(qs->adap, q) < 0)
2226 q->unhandled_irqs++;
2227 spin_unlock(&q->lock);
2228 return IRQ_HANDLED;
2232 * The non-NAPI MSI interrupt handler. This needs to handle data events from
2233 * SGE response queues as well as error and other async events as they all use
2234 * the same MSI vector. We use one SGE response queue per port in this mode
2235 * and protect all response queues with queue 0's lock.
2237 static irqreturn_t t3_intr_msi(int irq, void *cookie)
2239 int new_packets = 0;
2240 struct adapter *adap = cookie;
2241 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2243 spin_lock(&q->lock);
2245 if (process_responses(adap, &adap->sge.qs[0], -1)) {
2246 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2247 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2248 new_packets = 1;
2251 if (adap->params.nports == 2 &&
2252 process_responses(adap, &adap->sge.qs[1], -1)) {
2253 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2255 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q1->cntxt_id) |
2256 V_NEWTIMER(q1->next_holdoff) |
2257 V_NEWINDEX(q1->cidx));
2258 new_packets = 1;
2261 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2262 q->unhandled_irqs++;
2264 spin_unlock(&q->lock);
2265 return IRQ_HANDLED;
2268 static int rspq_check_napi(struct sge_qset *qs)
2270 struct sge_rspq *q = &qs->rspq;
2272 if (!napi_is_scheduled(&qs->napi) &&
2273 is_new_response(&q->desc[q->cidx], q)) {
2274 napi_schedule(&qs->napi);
2275 return 1;
2277 return 0;
2281 * The MSI interrupt handler for the NAPI case (i.e., response queues serviced
2282 * by NAPI polling). Handles data events from SGE response queues as well as
2283 * error and other async events as they all use the same MSI vector. We use
2284 * one SGE response queue per port in this mode and protect all response
2285 * queues with queue 0's lock.
2287 static irqreturn_t t3_intr_msi_napi(int irq, void *cookie)
2289 int new_packets;
2290 struct adapter *adap = cookie;
2291 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2293 spin_lock(&q->lock);
2295 new_packets = rspq_check_napi(&adap->sge.qs[0]);
2296 if (adap->params.nports == 2)
2297 new_packets += rspq_check_napi(&adap->sge.qs[1]);
2298 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2299 q->unhandled_irqs++;
2301 spin_unlock(&q->lock);
2302 return IRQ_HANDLED;
2306 * A helper function that processes responses and issues GTS.
2308 static inline int process_responses_gts(struct adapter *adap,
2309 struct sge_rspq *rq)
2311 int work;
2313 work = process_responses(adap, rspq_to_qset(rq), -1);
2314 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) |
2315 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx));
2316 return work;
2320 * The legacy INTx interrupt handler. This needs to handle data events from
2321 * SGE response queues as well as error and other async events as they all use
2322 * the same interrupt pin. We use one SGE response queue per port in this mode
2323 * and protect all response queues with queue 0's lock.
2325 static irqreturn_t t3_intr(int irq, void *cookie)
2327 int work_done, w0, w1;
2328 struct adapter *adap = cookie;
2329 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2330 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2332 spin_lock(&q0->lock);
2334 w0 = is_new_response(&q0->desc[q0->cidx], q0);
2335 w1 = adap->params.nports == 2 &&
2336 is_new_response(&q1->desc[q1->cidx], q1);
2338 if (likely(w0 | w1)) {
2339 t3_write_reg(adap, A_PL_CLI, 0);
2340 t3_read_reg(adap, A_PL_CLI); /* flush */
2342 if (likely(w0))
2343 process_responses_gts(adap, q0);
2345 if (w1)
2346 process_responses_gts(adap, q1);
2348 work_done = w0 | w1;
2349 } else
2350 work_done = t3_slow_intr_handler(adap);
2352 spin_unlock(&q0->lock);
2353 return IRQ_RETVAL(work_done != 0);
2357 * Interrupt handler for legacy INTx interrupts for T3B-based cards.
2358 * Handles data events from SGE response queues as well as error and other
2359 * async events as they all use the same interrupt pin. We use one SGE
2360 * response queue per port in this mode and protect all response queues with
2361 * queue 0's lock.
2363 static irqreturn_t t3b_intr(int irq, void *cookie)
2365 u32 map;
2366 struct adapter *adap = cookie;
2367 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2369 t3_write_reg(adap, A_PL_CLI, 0);
2370 map = t3_read_reg(adap, A_SG_DATA_INTR);
2372 if (unlikely(!map)) /* shared interrupt, most likely */
2373 return IRQ_NONE;
2375 spin_lock(&q0->lock);
2377 if (unlikely(map & F_ERRINTR))
2378 t3_slow_intr_handler(adap);
2380 if (likely(map & 1))
2381 process_responses_gts(adap, q0);
2383 if (map & 2)
2384 process_responses_gts(adap, &adap->sge.qs[1].rspq);
2386 spin_unlock(&q0->lock);
2387 return IRQ_HANDLED;
2391 * NAPI interrupt handler for legacy INTx interrupts for T3B-based cards.
2392 * Handles data events from SGE response queues as well as error and other
2393 * async events as they all use the same interrupt pin. We use one SGE
2394 * response queue per port in this mode and protect all response queues with
2395 * queue 0's lock.
2397 static irqreturn_t t3b_intr_napi(int irq, void *cookie)
2399 u32 map;
2400 struct adapter *adap = cookie;
2401 struct sge_qset *qs0 = &adap->sge.qs[0];
2402 struct sge_rspq *q0 = &qs0->rspq;
2404 t3_write_reg(adap, A_PL_CLI, 0);
2405 map = t3_read_reg(adap, A_SG_DATA_INTR);
2407 if (unlikely(!map)) /* shared interrupt, most likely */
2408 return IRQ_NONE;
2410 spin_lock(&q0->lock);
2412 if (unlikely(map & F_ERRINTR))
2413 t3_slow_intr_handler(adap);
2415 if (likely(map & 1))
2416 napi_schedule(&qs0->napi);
2418 if (map & 2)
2419 napi_schedule(&adap->sge.qs[1].napi);
2421 spin_unlock(&q0->lock);
2422 return IRQ_HANDLED;
2426 * t3_intr_handler - select the top-level interrupt handler
2427 * @adap: the adapter
2428 * @polling: whether using NAPI to service response queues
2430 * Selects the top-level interrupt handler based on the type of interrupts
2431 * (MSI-X, MSI, or legacy) and whether NAPI will be used to service the
2432 * response queues.
2434 irq_handler_t t3_intr_handler(struct adapter *adap, int polling)
2436 if (adap->flags & USING_MSIX)
2437 return polling ? t3_sge_intr_msix_napi : t3_sge_intr_msix;
2438 if (adap->flags & USING_MSI)
2439 return polling ? t3_intr_msi_napi : t3_intr_msi;
2440 if (adap->params.rev > 0)
2441 return polling ? t3b_intr_napi : t3b_intr;
2442 return t3_intr;
2446 * t3_sge_err_intr_handler - SGE async event interrupt handler
2447 * @adapter: the adapter
2449 * Interrupt handler for SGE asynchronous (non-data) events.
2451 void t3_sge_err_intr_handler(struct adapter *adapter)
2453 unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2455 if (status & F_RSPQCREDITOVERFOW)
2456 CH_ALERT(adapter, "SGE response queue credit overflow\n");
2458 if (status & F_RSPQDISABLED) {
2459 v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS);
2461 CH_ALERT(adapter,
2462 "packet delivered to disabled response queue "
2463 "(0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff);
2466 if (status & (F_HIPIODRBDROPERR | F_LOPIODRBDROPERR))
2467 CH_ALERT(adapter, "SGE dropped %s priority doorbell\n",
2468 status & F_HIPIODRBDROPERR ? "high" : "lo");
2470 t3_write_reg(adapter, A_SG_INT_CAUSE, status);
2471 if (status & (F_RSPQCREDITOVERFOW | F_RSPQDISABLED))
2472 t3_fatal_err(adapter);
2476 * sge_timer_cb - perform periodic maintenance of an SGE qset
2477 * @data: the SGE queue set to maintain
2479 * Runs periodically from a timer to perform maintenance of an SGE queue
2480 * set. It performs two tasks:
2482 * a) Cleans up any completed Tx descriptors that may still be pending.
2483 * Normal descriptor cleanup happens when new packets are added to a Tx
2484 * queue so this timer is relatively infrequent and does any cleanup only
2485 * if the Tx queue has not seen any new packets in a while. We make a
2486 * best effort attempt to reclaim descriptors, in that we don't wait
2487 * around if we cannot get a queue's lock (which most likely is because
2488 * someone else is queueing new packets and so will also handle the clean
2489 * up). Since control queues use immediate data exclusively we don't
2490 * bother cleaning them up here.
2492 * b) Replenishes Rx queues that have run out due to memory shortage.
2493 * Normally new Rx buffers are added when existing ones are consumed but
2494 * when out of memory a queue can become empty. We try to add only a few
2495 * buffers here, the queue will be replenished fully as these new buffers
2496 * are used up if memory shortage has subsided.
2498 static void sge_timer_cb(unsigned long data)
2500 spinlock_t *lock;
2501 struct sge_qset *qs = (struct sge_qset *)data;
2502 struct adapter *adap = qs->adap;
2504 if (spin_trylock(&qs->txq[TXQ_ETH].lock)) {
2505 reclaim_completed_tx(adap, &qs->txq[TXQ_ETH]);
2506 spin_unlock(&qs->txq[TXQ_ETH].lock);
2508 if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {
2509 reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD]);
2510 spin_unlock(&qs->txq[TXQ_OFLD].lock);
2512 lock = (adap->flags & USING_MSIX) ? &qs->rspq.lock :
2513 &adap->sge.qs[0].rspq.lock;
2514 if (spin_trylock_irq(lock)) {
2515 if (!napi_is_scheduled(&qs->napi)) {
2516 u32 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS);
2518 if (qs->fl[0].credits < qs->fl[0].size)
2519 __refill_fl(adap, &qs->fl[0]);
2520 if (qs->fl[1].credits < qs->fl[1].size)
2521 __refill_fl(adap, &qs->fl[1]);
2523 if (status & (1 << qs->rspq.cntxt_id)) {
2524 qs->rspq.starved++;
2525 if (qs->rspq.credits) {
2526 refill_rspq(adap, &qs->rspq, 1);
2527 qs->rspq.credits--;
2528 qs->rspq.restarted++;
2529 t3_write_reg(adap, A_SG_RSPQ_FL_STATUS,
2530 1 << qs->rspq.cntxt_id);
2534 spin_unlock_irq(lock);
2536 mod_timer(&qs->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
2540 * t3_update_qset_coalesce - update coalescing settings for a queue set
2541 * @qs: the SGE queue set
2542 * @p: new queue set parameters
2544 * Update the coalescing settings for an SGE queue set. Nothing is done
2545 * if the queue set is not initialized yet.
2547 void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)
2549 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);/* can't be 0 */
2550 qs->rspq.polling = p->polling;
2551 qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll;
2555 * t3_sge_alloc_qset - initialize an SGE queue set
2556 * @adapter: the adapter
2557 * @id: the queue set id
2558 * @nports: how many Ethernet ports will be using this queue set
2559 * @irq_vec_idx: the IRQ vector index for response queue interrupts
2560 * @p: configuration parameters for this queue set
2561 * @ntxq: number of Tx queues for the queue set
2562 * @netdev: net device associated with this queue set
2564 * Allocate resources and initialize an SGE queue set. A queue set
2565 * comprises a response queue, two Rx free-buffer queues, and up to 3
2566 * Tx queues. The Tx queues are assigned roles in the order Ethernet
2567 * queue, offload queue, and control queue.
2569 int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
2570 int irq_vec_idx, const struct qset_params *p,
2571 int ntxq, struct net_device *dev)
2573 int i, ret = -ENOMEM;
2574 struct sge_qset *q = &adapter->sge.qs[id];
2576 init_qset_cntxt(q, id);
2577 init_timer(&q->tx_reclaim_timer);
2578 q->tx_reclaim_timer.data = (unsigned long)q;
2579 q->tx_reclaim_timer.function = sge_timer_cb;
2581 q->fl[0].desc = alloc_ring(adapter->pdev, p->fl_size,
2582 sizeof(struct rx_desc),
2583 sizeof(struct rx_sw_desc),
2584 &q->fl[0].phys_addr, &q->fl[0].sdesc);
2585 if (!q->fl[0].desc)
2586 goto err;
2588 q->fl[1].desc = alloc_ring(adapter->pdev, p->jumbo_size,
2589 sizeof(struct rx_desc),
2590 sizeof(struct rx_sw_desc),
2591 &q->fl[1].phys_addr, &q->fl[1].sdesc);
2592 if (!q->fl[1].desc)
2593 goto err;
2595 q->rspq.desc = alloc_ring(adapter->pdev, p->rspq_size,
2596 sizeof(struct rsp_desc), 0,
2597 &q->rspq.phys_addr, NULL);
2598 if (!q->rspq.desc)
2599 goto err;
2601 for (i = 0; i < ntxq; ++i) {
2603 * The control queue always uses immediate data so does not
2604 * need to keep track of any sk_buffs.
2606 size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
2608 q->txq[i].desc = alloc_ring(adapter->pdev, p->txq_size[i],
2609 sizeof(struct tx_desc), sz,
2610 &q->txq[i].phys_addr,
2611 &q->txq[i].sdesc);
2612 if (!q->txq[i].desc)
2613 goto err;
2615 q->txq[i].gen = 1;
2616 q->txq[i].size = p->txq_size[i];
2617 spin_lock_init(&q->txq[i].lock);
2618 skb_queue_head_init(&q->txq[i].sendq);
2621 tasklet_init(&q->txq[TXQ_OFLD].qresume_tsk, restart_offloadq,
2622 (unsigned long)q);
2623 tasklet_init(&q->txq[TXQ_CTRL].qresume_tsk, restart_ctrlq,
2624 (unsigned long)q);
2626 q->fl[0].gen = q->fl[1].gen = 1;
2627 q->fl[0].size = p->fl_size;
2628 q->fl[1].size = p->jumbo_size;
2630 q->rspq.gen = 1;
2631 q->rspq.size = p->rspq_size;
2632 spin_lock_init(&q->rspq.lock);
2634 q->txq[TXQ_ETH].stop_thres = nports *
2635 flits_to_desc(sgl_len(MAX_SKB_FRAGS + 1) + 3);
2637 #if FL0_PG_CHUNK_SIZE > 0
2638 q->fl[0].buf_size = FL0_PG_CHUNK_SIZE;
2639 #else
2640 q->fl[0].buf_size = SGE_RX_SM_BUF_SIZE + sizeof(struct cpl_rx_data);
2641 #endif
2642 q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0;
2643 q->fl[1].buf_size = is_offload(adapter) ?
2644 (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
2645 MAX_FRAME_SIZE + 2 + sizeof(struct cpl_rx_pkt);
2647 spin_lock(&adapter->sge.reg_lock);
2649 /* FL threshold comparison uses < */
2650 ret = t3_sge_init_rspcntxt(adapter, q->rspq.cntxt_id, irq_vec_idx,
2651 q->rspq.phys_addr, q->rspq.size,
2652 q->fl[0].buf_size, 1, 0);
2653 if (ret)
2654 goto err_unlock;
2656 for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
2657 ret = t3_sge_init_flcntxt(adapter, q->fl[i].cntxt_id, 0,
2658 q->fl[i].phys_addr, q->fl[i].size,
2659 q->fl[i].buf_size, p->cong_thres, 1,
2661 if (ret)
2662 goto err_unlock;
2665 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
2666 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
2667 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
2668 1, 0);
2669 if (ret)
2670 goto err_unlock;
2672 if (ntxq > 1) {
2673 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_OFLD].cntxt_id,
2674 USE_GTS, SGE_CNTXT_OFLD, id,
2675 q->txq[TXQ_OFLD].phys_addr,
2676 q->txq[TXQ_OFLD].size, 0, 1, 0);
2677 if (ret)
2678 goto err_unlock;
2681 if (ntxq > 2) {
2682 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_CTRL].cntxt_id, 0,
2683 SGE_CNTXT_CTRL, id,
2684 q->txq[TXQ_CTRL].phys_addr,
2685 q->txq[TXQ_CTRL].size,
2686 q->txq[TXQ_CTRL].token, 1, 0);
2687 if (ret)
2688 goto err_unlock;
2691 spin_unlock(&adapter->sge.reg_lock);
2693 q->adap = adapter;
2694 q->netdev = dev;
2695 t3_update_qset_coalesce(q, p);
2697 refill_fl(adapter, &q->fl[0], q->fl[0].size, GFP_KERNEL);
2698 refill_fl(adapter, &q->fl[1], q->fl[1].size, GFP_KERNEL);
2699 refill_rspq(adapter, &q->rspq, q->rspq.size - 1);
2701 t3_write_reg(adapter, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
2702 V_NEWTIMER(q->rspq.holdoff_tmr));
2704 mod_timer(&q->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
2705 return 0;
2707 err_unlock:
2708 spin_unlock(&adapter->sge.reg_lock);
2709 err:
2710 t3_free_qset(adapter, q);
2711 return ret;
2715 * t3_free_sge_resources - free SGE resources
2716 * @adap: the adapter
2718 * Frees resources used by the SGE queue sets.
2720 void t3_free_sge_resources(struct adapter *adap)
2722 int i;
2724 for (i = 0; i < SGE_QSETS; ++i)
2725 t3_free_qset(adap, &adap->sge.qs[i]);
2729 * t3_sge_start - enable SGE
2730 * @adap: the adapter
2732 * Enables the SGE for DMAs. This is the last step in starting packet
2733 * transfers.
2735 void t3_sge_start(struct adapter *adap)
2737 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
2741 * t3_sge_stop - disable SGE operation
2742 * @adap: the adapter
2744 * Disables the DMA engine. This can be called in emeregencies (e.g.,
2745 * from error interrupts) or from normal process context. In the latter
2746 * case it also disables any pending queue restart tasklets. Note that
2747 * if it is called in interrupt context it cannot disable the restart
2748 * tasklets as it cannot wait, however the tasklets will have no effect
2749 * since the doorbells are disabled and the driver will call this again
2750 * later from process context, at which time the tasklets will be stopped
2751 * if they are still running.
2753 void t3_sge_stop(struct adapter *adap)
2755 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, 0);
2756 if (!in_interrupt()) {
2757 int i;
2759 for (i = 0; i < SGE_QSETS; ++i) {
2760 struct sge_qset *qs = &adap->sge.qs[i];
2762 tasklet_kill(&qs->txq[TXQ_OFLD].qresume_tsk);
2763 tasklet_kill(&qs->txq[TXQ_CTRL].qresume_tsk);
2769 * t3_sge_init - initialize SGE
2770 * @adap: the adapter
2771 * @p: the SGE parameters
2773 * Performs SGE initialization needed every time after a chip reset.
2774 * We do not initialize any of the queue sets here, instead the driver
2775 * top-level must request those individually. We also do not enable DMA
2776 * here, that should be done after the queues have been set up.
2778 void t3_sge_init(struct adapter *adap, struct sge_params *p)
2780 unsigned int ctrl, ups = ffs(pci_resource_len(adap->pdev, 2) >> 12);
2782 ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
2783 F_CQCRDTCTRL |
2784 V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
2785 V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
2786 #if SGE_NUM_GENBITS == 1
2787 ctrl |= F_EGRGENCTRL;
2788 #endif
2789 if (adap->params.rev > 0) {
2790 if (!(adap->flags & (USING_MSIX | USING_MSI)))
2791 ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
2792 ctrl |= F_CQCRDTCTRL | F_AVOIDCQOVFL;
2794 t3_write_reg(adap, A_SG_CONTROL, ctrl);
2795 t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
2796 V_LORCQDRBTHRSH(512));
2797 t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
2798 t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
2799 V_TIMEOUT(200 * core_ticks_per_usec(adap)));
2800 t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH, 1000);
2801 t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
2802 t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
2803 t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
2804 t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff));
2805 t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024);
2809 * t3_sge_prep - one-time SGE initialization
2810 * @adap: the associated adapter
2811 * @p: SGE parameters
2813 * Performs one-time initialization of SGE SW state. Includes determining
2814 * defaults for the assorted SGE parameters, which admins can change until
2815 * they are used to initialize the SGE.
2817 void __devinit t3_sge_prep(struct adapter *adap, struct sge_params *p)
2819 int i;
2821 p->max_pkt_size = (16 * 1024) - sizeof(struct cpl_rx_data) -
2822 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
2824 for (i = 0; i < SGE_QSETS; ++i) {
2825 struct qset_params *q = p->qset + i;
2827 q->polling = adap->params.rev > 0;
2828 q->coalesce_usecs = 5;
2829 q->rspq_size = 1024;
2830 q->fl_size = 1024;
2831 q->jumbo_size = 512;
2832 q->txq_size[TXQ_ETH] = 1024;
2833 q->txq_size[TXQ_OFLD] = 1024;
2834 q->txq_size[TXQ_CTRL] = 256;
2835 q->cong_thres = 0;
2838 spin_lock_init(&adap->sge.reg_lock);
2842 * t3_get_desc - dump an SGE descriptor for debugging purposes
2843 * @qs: the queue set
2844 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
2845 * @idx: the descriptor index in the queue
2846 * @data: where to dump the descriptor contents
2848 * Dumps the contents of a HW descriptor of an SGE queue. Returns the
2849 * size of the descriptor.
2851 int t3_get_desc(const struct sge_qset *qs, unsigned int qnum, unsigned int idx,
2852 unsigned char *data)
2854 if (qnum >= 6)
2855 return -EINVAL;
2857 if (qnum < 3) {
2858 if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size)
2859 return -EINVAL;
2860 memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc));
2861 return sizeof(struct tx_desc);
2864 if (qnum == 3) {
2865 if (!qs->rspq.desc || idx >= qs->rspq.size)
2866 return -EINVAL;
2867 memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc));
2868 return sizeof(struct rsp_desc);
2871 qnum -= 4;
2872 if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size)
2873 return -EINVAL;
2874 memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc));
2875 return sizeof(struct rx_desc);