?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'origin/master'
[unleashed/lotheac.git] / usr / src / uts / common / io / cxgbe / t4nex / t4_sge.c
blob22971740c9fd69f49c185214fa7717a487eb423b
1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 #include <sys/ddi.h>
24 #include <sys/sunddi.h>
25 #include <sys/sunndi.h>
26 #include <sys/atomic.h>
27 #include <sys/dlpi.h>
28 #include <sys/pattr.h>
29 #include <sys/strsubr.h>
30 #include <sys/stream.h>
31 #include <sys/strsun.h>
32 #include <inet/ip.h>
33 #include <inet/tcp.h>
34
35 #include "version.h"
36 #include "common/common.h"
37 #include "common/t4_msg.h"
38 #include "common/t4_regs.h"
39 #include "common/t4_regs_values.h"
40
41 /* TODO: Tune. */
42 int rx_buf_size = 8192;
43 int tx_copy_threshold = 256;
44 uint16_t rx_copy_threshold = 256;
45
46 /* Used to track coalesced tx work request */
47 struct txpkts {
48 mblk_t *tail; /* head is in the software descriptor */
49 uint64_t *flitp; /* ptr to flit where next pkt should start */
50 uint8_t npkt; /* # of packets in this work request */
51 uint8_t nflits; /* # of flits used by this work request */
52 uint16_t plen; /* total payload (sum of all packets) */
53 };
54
55 /* All information needed to tx a frame */
56 struct txinfo {
57 uint32_t len; /* Total length of frame */
58 uint32_t flags; /* Checksum and LSO flags */
59 uint32_t mss; /* MSS for LSO */
60 uint8_t nsegs; /* # of segments in the SGL, 0 means imm. tx */
61 uint8_t nflits; /* # of flits needed for the SGL */
62 uint8_t hdls_used; /* # of DMA handles used */
63 uint32_t txb_used; /* txb_space used */
64 struct ulptx_sgl sgl __attribute__((aligned(8)));
65 struct ulptx_sge_pair reserved[TX_SGL_SEGS / 2];
66 };
67
68 static int service_iq(struct sge_iq *iq, int budget);
69 static inline void init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx,
70 int8_t pktc_idx, int qsize, uint8_t esize);
71 static inline void init_fl(struct sge_fl *fl, uint16_t qsize);
72 static inline void init_eq(struct adapter *sc, struct sge_eq *eq,
73 uint16_t eqtype, uint16_t qsize,uint8_t tx_chan, uint16_t iqid);
74 static int alloc_iq_fl(struct port_info *pi, struct sge_iq *iq,
75 struct sge_fl *fl, int intr_idx, int cong);
76 static int free_iq_fl(struct port_info *pi, struct sge_iq *iq,
77 struct sge_fl *fl);
78 static int alloc_fwq(struct adapter *sc);
79 static int free_fwq(struct adapter *sc);
80 #ifdef TCP_OFFLOAD_ENABLE
81 static int alloc_mgmtq(struct adapter *sc);
82 #endif
83 static int alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx,
84 int i);
85 static int free_rxq(struct port_info *pi, struct sge_rxq *rxq);
86 #ifdef TCP_OFFLOAD_ENABLE
87 static int alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
88 int intr_idx);
89 static int free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq);
90 #endif
91 static int ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq);
92 static int eth_eq_alloc(struct adapter *sc, struct port_info *pi,
93 struct sge_eq *eq);
94 #ifdef TCP_OFFLOAD_ENABLE
95 static int ofld_eq_alloc(struct adapter *sc, struct port_info *pi,
96 struct sge_eq *eq);
97 #endif
98 static int alloc_eq(struct adapter *sc, struct port_info *pi,
99 struct sge_eq *eq);
100 static int free_eq(struct adapter *sc, struct sge_eq *eq);
101 #ifdef TCP_OFFLOAD_ENABLE
102 static int alloc_wrq(struct adapter *sc, struct port_info *pi,
103 struct sge_wrq *wrq, int idx);
104 static int free_wrq(struct adapter *sc, struct sge_wrq *wrq);
105 #endif
106 static int alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx);
107 static int free_txq(struct port_info *pi, struct sge_txq *txq);
108 static int alloc_dma_memory(struct adapter *sc, size_t len, int flags,
109 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
110 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
111 caddr_t *pva);
112 static int free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
113 static int alloc_desc_ring(struct adapter *sc, size_t len, int rw,
114 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
115 caddr_t *pva);
116 static int free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl);
117 static int alloc_tx_copybuffer(struct adapter *sc, size_t len,
118 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl, uint64_t *pba,
119 caddr_t *pva);
120 static inline bool is_new_response(const struct sge_iq *iq,
121 struct rsp_ctrl **ctrl);
122 static inline void iq_next(struct sge_iq *iq);
123 static int refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs);
124 static void refill_sfl(void *arg);
125 static void add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl);
126 static void free_fl_bufs(struct sge_fl *fl);
127 static mblk_t *get_fl_payload(struct adapter *sc, struct sge_fl *fl,
128 uint32_t len_newbuf, int *fl_bufs_used);
129 static int get_frame_txinfo(struct sge_txq *txq, mblk_t **fp,
130 struct txinfo *txinfo, int sgl_only);
131 static inline int fits_in_txb(struct sge_txq *txq, int len, int *waste);
132 static inline int copy_into_txb(struct sge_txq *txq, mblk_t *m, int len,
133 struct txinfo *txinfo);
134 static inline void add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len);
135 static inline int add_mblk(struct sge_txq *txq, struct txinfo *txinfo,
136 mblk_t *m, int len);
137 static void free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo);
138 static int add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
139 struct txinfo *txinfo);
140 static void write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts);
141 static int write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
142 struct txinfo *txinfo);
143 static inline void write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
144 struct txpkts *txpkts, struct txinfo *txinfo);
145 static inline void copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to,
146 int len);
147 static inline void ring_tx_db(struct adapter *sc, struct sge_eq *eq);
148 static int reclaim_tx_descs(struct sge_txq *txq, int howmany);
149 static void write_txqflush_wr(struct sge_txq *txq);
150 static int t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss,
151 mblk_t *m);
152 static inline void ring_fl_db(struct adapter *sc, struct sge_fl *fl);
153 static kstat_t *setup_port_config_kstats(struct port_info *pi);
154 static kstat_t *setup_port_info_kstats(struct port_info *pi);
155 static kstat_t *setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq,
156 int idx);
157 static int update_rxq_kstats(kstat_t *ksp, int rw);
158 static int update_port_info_kstats(kstat_t *ksp, int rw);
159 static kstat_t *setup_txq_kstats(struct port_info *pi, struct sge_txq *txq,
160 int idx);
161 static int update_txq_kstats(kstat_t *ksp, int rw);
162 static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
163 mblk_t *);
164 static int handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss,
165 mblk_t *m);
166
167 static inline int
168 reclaimable(struct sge_eq *eq)
169 {
170 unsigned int cidx;
171
172 cidx = eq->spg->cidx; /* stable snapshot */
173 cidx = be16_to_cpu(cidx);
174
175 if (cidx >= eq->cidx)
176 return (cidx - eq->cidx);
177 else
178 return (cidx + eq->cap - eq->cidx);
179 }
180
181 void
182 t4_sge_init(struct adapter *sc)
183 {
184 struct driver_properties *p = &sc->props;
185 ddi_dma_attr_t *dma_attr;
186 ddi_device_acc_attr_t *acc_attr;
187 uint32_t sge_control, sge_conm_ctrl;
188 int egress_threshold;
189
190 /*
191 * Device access and DMA attributes for descriptor rings
192 */
193 acc_attr = &sc->sge.acc_attr_desc;
194 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
195 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
196 acc_attr->devacc_attr_dataorder = DDI_STRICTORDER_ACC;
197
198 dma_attr = &sc->sge.dma_attr_desc;
199 dma_attr->dma_attr_version = DMA_ATTR_V0;
200 dma_attr->dma_attr_addr_lo = 0;
201 dma_attr->dma_attr_addr_hi = UINT64_MAX;
202 dma_attr->dma_attr_count_max = UINT64_MAX;
203 dma_attr->dma_attr_align = 512;
204 dma_attr->dma_attr_burstsizes = 0xfff;
205 dma_attr->dma_attr_minxfer = 1;
206 dma_attr->dma_attr_maxxfer = UINT64_MAX;
207 dma_attr->dma_attr_seg = UINT64_MAX;
208 dma_attr->dma_attr_sgllen = 1;
209 dma_attr->dma_attr_granular = 1;
210 dma_attr->dma_attr_flags = 0;
211
212 /*
213 * Device access and DMA attributes for tx buffers
214 */
215 acc_attr = &sc->sge.acc_attr_tx;
216 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
217 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
218
219 dma_attr = &sc->sge.dma_attr_tx;
220 dma_attr->dma_attr_version = DMA_ATTR_V0;
221 dma_attr->dma_attr_addr_lo = 0;
222 dma_attr->dma_attr_addr_hi = UINT64_MAX;
223 dma_attr->dma_attr_count_max = UINT64_MAX;
224 dma_attr->dma_attr_align = 1;
225 dma_attr->dma_attr_burstsizes = 0xfff;
226 dma_attr->dma_attr_minxfer = 1;
227 dma_attr->dma_attr_maxxfer = UINT64_MAX;
228 dma_attr->dma_attr_seg = UINT64_MAX;
229 dma_attr->dma_attr_sgllen = TX_SGL_SEGS;
230 dma_attr->dma_attr_granular = 1;
231 dma_attr->dma_attr_flags = 0;
232
233 /*
234 * Ingress Padding Boundary and Egress Status Page Size are set up by
235 * t4_fixup_host_params().
236 */
237 sge_control = t4_read_reg(sc, A_SGE_CONTROL);
238 sc->sge.pktshift = G_PKTSHIFT(sge_control);
239 sc->sge.stat_len = (sge_control & F_EGRSTATUSPAGESIZE) ? 128 : 64;
240
241 /* t4_nex uses FLM packed mode */
242 sc->sge.fl_align = t4_fl_pkt_align(sc, true);
243
244 /*
245 * Device access and DMA attributes for rx buffers
246 */
247 sc->sge.rxb_params.dip = sc->dip;
248 sc->sge.rxb_params.buf_size = rx_buf_size;
249
250 acc_attr = &sc->sge.rxb_params.acc_attr_rx;
251 acc_attr->devacc_attr_version = DDI_DEVICE_ATTR_V0;
252 acc_attr->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
253
254 dma_attr = &sc->sge.rxb_params.dma_attr_rx;
255 dma_attr->dma_attr_version = DMA_ATTR_V0;
256 dma_attr->dma_attr_addr_lo = 0;
257 dma_attr->dma_attr_addr_hi = UINT64_MAX;
258 dma_attr->dma_attr_count_max = UINT64_MAX;
259 /*
260 * Low 4 bits of an rx buffer address have a special meaning to the SGE
261 * and an rx buf cannot have an address with any of these bits set.
262 * FL_ALIGN is >= 32 so we're sure things are ok.
263 */
264 dma_attr->dma_attr_align = sc->sge.fl_align;
265 dma_attr->dma_attr_burstsizes = 0xfff;
266 dma_attr->dma_attr_minxfer = 1;
267 dma_attr->dma_attr_maxxfer = UINT64_MAX;
268 dma_attr->dma_attr_seg = UINT64_MAX;
269 dma_attr->dma_attr_sgllen = 1;
270 dma_attr->dma_attr_granular = 1;
271 dma_attr->dma_attr_flags = 0;
272
273 sc->sge.rxbuf_cache = rxbuf_cache_create(&sc->sge.rxb_params);
274
275 /*
276 * A FL with <= fl_starve_thres buffers is starving and a periodic
277 * timer will attempt to refill it. This needs to be larger than the
278 * SGE's Egress Congestion Threshold. If it isn't, then we can get
279 * stuck waiting for new packets while the SGE is waiting for us to
280 * give it more Free List entries. (Note that the SGE's Egress
281 * Congestion Threshold is in units of 2 Free List pointers.) For T4,
282 * there was only a single field to control this. For T5 there's the
283 * original field which now only applies to Unpacked Mode Free List
284 * buffers and a new field which only applies to Packed Mode Free List
285 * buffers.
286 */
287
288 sge_conm_ctrl = t4_read_reg(sc, A_SGE_CONM_CTRL);
289 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) {
290 case CHELSIO_T4:
291 egress_threshold = G_EGRTHRESHOLD(sge_conm_ctrl);
292 break;
293 case CHELSIO_T5:
294 egress_threshold = G_EGRTHRESHOLDPACKING(sge_conm_ctrl);
295 break;
296 case CHELSIO_T6:
297 default:
298 egress_threshold = G_T6_EGRTHRESHOLDPACKING(sge_conm_ctrl);
299 }
300 sc->sge.fl_starve_threshold = 2*egress_threshold + 1;
301
302 t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0, rx_buf_size);
303
304 t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
305 V_THRESHOLD_0(p->counter_val[0]) |
306 V_THRESHOLD_1(p->counter_val[1]) |
307 V_THRESHOLD_2(p->counter_val[2]) |
308 V_THRESHOLD_3(p->counter_val[3]));
309
310 t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
311 V_TIMERVALUE0(us_to_core_ticks(sc, p->timer_val[0])) |
312 V_TIMERVALUE1(us_to_core_ticks(sc, p->timer_val[1])));
313 t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
314 V_TIMERVALUE2(us_to_core_ticks(sc, p->timer_val[2])) |
315 V_TIMERVALUE3(us_to_core_ticks(sc, p->timer_val[3])));
316 t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
317 V_TIMERVALUE4(us_to_core_ticks(sc, p->timer_val[4])) |
318 V_TIMERVALUE5(us_to_core_ticks(sc, p->timer_val[5])));
319
320 (void) t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl);
321 (void) t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl);
322 (void) t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
323 (void) t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
324 (void) t4_register_fw_msg_handler(sc, FW6_TYPE_CMD_RPL,
325 t4_handle_fw_rpl);
326 }
327
328 /*
329 * Allocate and initialize the firmware event queue and the forwarded interrupt
330 * queues, if any. The adapter owns all these queues as they are not associated
331 * with any particular port.
332 *
333 * Returns errno on failure. Resources allocated up to that point may still be
334 * allocated. Caller is responsible for cleanup in case this function fails.
335 */
336 int
337 t4_setup_adapter_queues(struct adapter *sc)
338 {
339 int rc;
340
341 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
342
343 /*
344 * Firmware event queue
345 */
346 rc = alloc_fwq(sc);
347 if (rc != 0)
348 return (rc);
349
350 #ifdef TCP_OFFLOAD_ENABLE
351 /*
352 * Management queue. This is just a control queue that uses the fwq as
353 * its associated iq.
354 */
355 rc = alloc_mgmtq(sc);
356 #endif
357
358 return (rc);
359 }
360
361 /*
362 * Idempotent
363 */
364 int
365 t4_teardown_adapter_queues(struct adapter *sc)
366 {
367
368 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
369
370 (void) free_fwq(sc);
371
372 return (0);
373 }
374
375 static inline int
376 first_vector(struct port_info *pi)
377 {
378 struct adapter *sc = pi->adapter;
379 int rc = T4_EXTRA_INTR, i;
380
381 if (sc->intr_count == 1)
382 return (0);
383
384 for_each_port(sc, i) {
385 struct port_info *p = sc->port[i];
386
387 if (i == pi->port_id)
388 break;
389
390 #ifdef TCP_OFFLOAD_ENABLE
391 if (!(sc->flags & INTR_FWD))
392 rc += p->nrxq + p->nofldrxq;
393 else
394 rc += max(p->nrxq, p->nofldrxq);
395 #else
396 /*
397 * Not compiled with offload support and intr_count > 1. Only
398 * NIC queues exist and they'd better be taking direct
399 * interrupts.
400 */
401 ASSERT(!(sc->flags & INTR_FWD));
402 rc += p->nrxq;
403 #endif
404 }
405 return (rc);
406 }
407
408 /*
409 * Given an arbitrary "index," come up with an iq that can be used by other
410 * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
411 * The iq returned is guaranteed to be something that takes direct interrupts.
412 */
413 static struct sge_iq *
414 port_intr_iq(struct port_info *pi, int idx)
415 {
416 struct adapter *sc = pi->adapter;
417 struct sge *s = &sc->sge;
418 struct sge_iq *iq = NULL;
419
420 if (sc->intr_count == 1)
421 return (&sc->sge.fwq);
422
423 #ifdef TCP_OFFLOAD_ENABLE
424 if (!(sc->flags & INTR_FWD)) {
425 idx %= pi->nrxq + pi->nofldrxq;
426
427 if (idx >= pi->nrxq) {
428 idx -= pi->nrxq;
429 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
430 } else
431 iq = &s->rxq[pi->first_rxq + idx].iq;
432
433 } else {
434 idx %= max(pi->nrxq, pi->nofldrxq);
435
436 if (pi->nrxq >= pi->nofldrxq)
437 iq = &s->rxq[pi->first_rxq + idx].iq;
438 else
439 iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
440 }
441 #else
442 /*
443 * Not compiled with offload support and intr_count > 1. Only NIC
444 * queues exist and they'd better be taking direct interrupts.
445 */
446 ASSERT(!(sc->flags & INTR_FWD));
447
448 idx %= pi->nrxq;
449 iq = &s->rxq[pi->first_rxq + idx].iq;
450 #endif
451
452 return (iq);
453 }
454
455 int
456 t4_setup_port_queues(struct port_info *pi)
457 {
458 int rc = 0, i, intr_idx, j;
459 struct sge_rxq *rxq;
460 struct sge_txq *txq;
461 #ifdef TCP_OFFLOAD_ENABLE
462 int iqid;
463 struct sge_wrq *ctrlq;
464 struct sge_ofld_rxq *ofld_rxq;
465 struct sge_wrq *ofld_txq;
466 #endif
467 struct adapter *sc = pi->adapter;
468 struct driver_properties *p = &sc->props;
469
470 pi->ksp_config = setup_port_config_kstats(pi);
471 pi->ksp_info = setup_port_info_kstats(pi);
472
473 /* Interrupt vector to start from (when using multiple vectors) */
474 intr_idx = first_vector(pi);
475
476 /*
477 * First pass over all rx queues (NIC and TOE):
478 * a) initialize iq and fl
479 * b) allocate queue iff it will take direct interrupts.
480 */
481
482 for_each_rxq(pi, i, rxq) {
483
484 init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, p->qsize_rxq,
485 RX_IQ_ESIZE);
486
487 init_fl(&rxq->fl, p->qsize_rxq / 8); /* 8 bufs in each entry */
488
489 if ((!(sc->flags & INTR_FWD))
490 #ifdef TCP_OFFLOAD_ENABLE
491 || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
492 #else
493 || (sc->intr_count > 1 && pi->nrxq)
494 #endif
495 ) {
496 rxq->iq.flags |= IQ_INTR;
497 rc = alloc_rxq(pi, rxq, intr_idx, i);
498 if (rc != 0)
499 goto done;
500 intr_idx++;
501 }
502
503 }
504
505 #ifdef TCP_OFFLOAD_ENABLE
506 for_each_ofld_rxq(pi, i, ofld_rxq) {
507
508 init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
509 p->qsize_rxq, RX_IQ_ESIZE);
510
511 init_fl(&ofld_rxq->fl, p->qsize_rxq / 8);
512
513 if (!(sc->flags & INTR_FWD) ||
514 (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
515 ofld_rxq->iq.flags = IQ_INTR;
516 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
517 if (rc != 0)
518 goto done;
519
520 intr_idx++;
521 }
522 }
523 #endif
524
525 /*
526 * Second pass over all rx queues (NIC and TOE). The queues forwarding
527 * their interrupts are allocated now.
528 */
529 j = 0;
530 for_each_rxq(pi, i, rxq) {
531 if (rxq->iq.flags & IQ_INTR)
532 continue;
533
534 intr_idx = port_intr_iq(pi, j)->abs_id;
535
536 rc = alloc_rxq(pi, rxq, intr_idx, i);
537 if (rc != 0)
538 goto done;
539 j++;
540 }
541
542 #ifdef TCP_OFFLOAD_ENABLE
543 for_each_ofld_rxq(pi, i, ofld_rxq) {
544 if (ofld_rxq->iq.flags & IQ_INTR)
545 continue;
546
547 intr_idx = port_intr_iq(pi, j)->abs_id;
548 rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx);
549 if (rc != 0)
550 goto done;
551 j++;
552 }
553 #endif
554 /*
555 * Now the tx queues. Only one pass needed.
556 */
557 j = 0;
558 for_each_txq(pi, i, txq) {
559 uint16_t iqid;
560
561 iqid = port_intr_iq(pi, j)->cntxt_id;
562 init_eq(sc, &txq->eq, EQ_ETH, p->qsize_txq, pi->tx_chan, iqid);
563 rc = alloc_txq(pi, txq, i);
564 if (rc != 0)
565 goto done;
566 }
567
568 #ifdef TCP_OFFLOAD_ENABLE
569 for_each_ofld_txq(pi, i, ofld_txq) {
570 uint16_t iqid;
571
572 iqid = port_intr_iq(pi, j)->cntxt_id;
573 init_eq(sc, &ofld_txq->eq, EQ_OFLD, p->qsize_txq, pi->tx_chan,
574 iqid);
575 rc = alloc_wrq(sc, pi, ofld_txq, i);
576 if (rc != 0)
577 goto done;
578 }
579
580 /*
581 * Finally, the control queue.
582 */
583 ctrlq = &sc->sge.ctrlq[pi->port_id];
584 iqid = port_intr_iq(pi, 0)->cntxt_id;
585 init_eq(sc, &ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid);
586 rc = alloc_wrq(sc, pi, ctrlq, 0);
587 #endif
588
589 done:
590 if (rc != 0)
591 (void) t4_teardown_port_queues(pi);
592
593 return (rc);
594 }
595
596 /*
597 * Idempotent
598 */
599 int
600 t4_teardown_port_queues(struct port_info *pi)
601 {
602 int i;
603 struct sge_rxq *rxq;
604 struct sge_txq *txq;
605 #ifdef TCP_OFFLOAD_ENABLE
606 struct adapter *sc = pi->adapter;
607 struct sge_ofld_rxq *ofld_rxq;
608 struct sge_wrq *ofld_txq;
609 #endif
610
611 if (pi->ksp_config != NULL) {
612 kstat_delete(pi->ksp_config);
613 pi->ksp_config = NULL;
614 }
615 if (pi->ksp_info != NULL) {
616 kstat_delete(pi->ksp_info);
617 pi->ksp_info = NULL;
618 }
619
620 #ifdef TCP_OFFLOAD_ENABLE
621 (void) free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
622 #endif
623
624 for_each_txq(pi, i, txq) {
625 (void) free_txq(pi, txq);
626 }
627
628 #ifdef TCP_OFFLOAD_ENABLE
629 for_each_ofld_txq(pi, i, ofld_txq) {
630 (void) free_wrq(sc, ofld_txq);
631 }
632
633 for_each_ofld_rxq(pi, i, ofld_rxq) {
634 if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
635 (void) free_ofld_rxq(pi, ofld_rxq);
636 }
637 #endif
638
639 for_each_rxq(pi, i, rxq) {
640 if ((rxq->iq.flags & IQ_INTR) == 0)
641 (void) free_rxq(pi, rxq);
642 }
643
644 /*
645 * Then take down the rx queues that take direct interrupts.
646 */
647
648 for_each_rxq(pi, i, rxq) {
649 if (rxq->iq.flags & IQ_INTR)
650 (void) free_rxq(pi, rxq);
651 }
652
653 #ifdef TCP_OFFLOAD_ENABLE
654 for_each_ofld_rxq(pi, i, ofld_rxq) {
655 if (ofld_rxq->iq.flags & IQ_INTR)
656 (void) free_ofld_rxq(pi, ofld_rxq);
657 }
658 #endif
659
660 return (0);
661 }
662
663 /* Deals with errors and forwarded interrupts */
664 uint_t
665 t4_intr_all(caddr_t arg1, caddr_t arg2)
666 {
667
668 (void) t4_intr_err(arg1, arg2);
669 (void) t4_intr(arg1, arg2);
670
671 return (DDI_INTR_CLAIMED);
672 }
673
674 static void
675 t4_intr_rx_work(struct sge_iq *iq)
676 {
677 mblk_t *mp = NULL;
678 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
679 RXQ_LOCK(rxq);
680 if (!iq->polling) {
681 mp = t4_ring_rx(rxq, iq->qsize/8);
682 t4_write_reg(iq->adapter, MYPF_REG(A_SGE_PF_GTS),
683 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
684 }
685 RXQ_UNLOCK(rxq);
686 if (mp != NULL)
687 mac_rx_ring(rxq->port->mh, rxq->ring_handle, mp,
688 rxq->ring_gen_num);
689 }
690
691 /* Deals with interrupts on the given ingress queue */
692 /* ARGSUSED */
693 uint_t
694 t4_intr(caddr_t arg1, caddr_t arg2)
695 {
696 struct sge_iq *iq = (struct sge_iq *)arg2;
697 int state;
698
699 /* Right now receive polling is only enabled for MSI-X and
700 * when we have enough msi-x vectors i.e no interrupt forwarding.
701 */
702 if (iq->adapter->props.multi_rings) {
703 t4_intr_rx_work(iq);
704 } else {
705 state = atomic_cas_uint(&iq->state, IQS_IDLE, IQS_BUSY);
706 if (state == IQS_IDLE) {
707 (void) service_iq(iq, 0);
708 (void) atomic_cas_uint(&iq->state, IQS_BUSY, IQS_IDLE);
709 }
710 }
711 return (DDI_INTR_CLAIMED);
712 }
713
714 /* Deals with error interrupts */
715 /* ARGSUSED */
716 uint_t
717 t4_intr_err(caddr_t arg1, caddr_t arg2)
718 {
719 /* LINTED: E_BAD_PTR_CAST_ALIGN */
720 struct adapter *sc = (struct adapter *)arg1;
721
722 t4_write_reg(sc, MYPF_REG(A_PCIE_PF_CLI), 0);
723 (void) t4_slow_intr_handler(sc);
724
725 return (DDI_INTR_CLAIMED);
726 }
727
728 /*
729 * t4_ring_rx - Process responses from an SGE response queue.
730 *
731 * This function processes responses from an SGE response queue up to the supplied budget.
732 * Responses include received packets as well as control messages from FW
733 * or HW.
734 * It returns a chain of mblks containing the received data, to be
735 * passed up to mac_ring_rx().
736 */
737 mblk_t *
738 t4_ring_rx(struct sge_rxq *rxq, int budget)
739 {
740 struct sge_iq *iq = &rxq->iq;
741 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
742 struct adapter *sc = iq->adapter;
743 struct rsp_ctrl *ctrl;
744 const struct rss_header *rss;
745 int ndescs = 0, fl_bufs_used = 0;
746 int rsp_type;
747 uint32_t lq;
748 mblk_t *mblk_head = NULL, **mblk_tail, *m;
749 struct cpl_rx_pkt *cpl;
750 uint32_t received_bytes = 0, pkt_len = 0;
751 bool csum_ok;
752 uint16_t err_vec;
753
754 mblk_tail = &mblk_head;
755
756 while (is_new_response(iq, &ctrl)) {
757
758 membar_consumer();
759
760 m = NULL;
761 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
762 lq = be32_to_cpu(ctrl->pldbuflen_qid);
763 rss = (const void *)iq->cdesc;
764
765 switch (rsp_type) {
766 case X_RSPD_TYPE_FLBUF:
767
768 ASSERT(iq->flags & IQ_HAS_FL);
769
770 if (CPL_RX_PKT == rss->opcode) {
771 cpl = (void *)(rss + 1);
772 pkt_len = be16_to_cpu(cpl->len);
773
774 if (iq->polling && ((received_bytes + pkt_len) > budget))
775 goto done;
776
777 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
778 if (m == NULL)
779 goto done;
780
781 iq->intr_next = iq->intr_params;
782 m->b_rptr += sc->sge.pktshift;
783 if (sc->params.tp.rx_pkt_encap)
784 /* It is enabled only in T6 config file */
785 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
786 else
787 err_vec = ntohs(cpl->err_vec);
788
789 csum_ok = cpl->csum_calc && !err_vec;
790
791 /* TODO: what about cpl->ip_frag? */
792 if (csum_ok && !cpl->ip_frag) {
793 mac_hcksum_set(m, 0, 0, 0, 0xffff,
794 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
795 HCK_IPV4_HDRCKSUM_OK);
796 rxq->rxcsum++;
797 }
798 rxq->rxpkts++;
799 rxq->rxbytes += pkt_len;
800 received_bytes += pkt_len;
801
802 *mblk_tail = m;
803 mblk_tail = &m->b_next;
804
805 break;
806 }
807
808 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
809 if (m == NULL)
810 goto done;
811 /* FALLTHROUGH */
812
813 case X_RSPD_TYPE_CPL:
814 ASSERT(rss->opcode < NUM_CPL_CMDS);
815 sc->cpl_handler[rss->opcode](iq, rss, m);
816 break;
817
818 default:
819 break;
820 }
821 iq_next(iq);
822 ++ndescs;
823 if (!iq->polling && (ndescs == budget))
824 break;
825 }
826
827 done:
828
829 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
830 V_CIDXINC(ndescs) | V_INGRESSQID(iq->cntxt_id) |
831 V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
832
833 if ((fl_bufs_used > 0) || (iq->flags & IQ_HAS_FL)) {
834 int starved;
835 FL_LOCK(fl);
836 fl->needed += fl_bufs_used;
837 starved = refill_fl(sc, fl, fl->cap / 8);
838 FL_UNLOCK(fl);
839 if (starved)
840 add_fl_to_sfl(sc, fl);
841 }
842 return (mblk_head);
843 }
844
845 /*
846 * Deals with anything and everything on the given ingress queue.
847 */
848 static int
849 service_iq(struct sge_iq *iq, int budget)
850 {
851 struct sge_iq *q;
852 struct sge_rxq *rxq = iq_to_rxq(iq); /* Use iff iq is part of rxq */
853 struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
854 struct adapter *sc = iq->adapter;
855 struct rsp_ctrl *ctrl;
856 const struct rss_header *rss;
857 int ndescs = 0, limit, fl_bufs_used = 0;
858 int rsp_type;
859 uint32_t lq;
860 int starved;
861 mblk_t *m;
862 STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
863
864 limit = budget ? budget : iq->qsize / 8;
865
866 /*
867 * We always come back and check the descriptor ring for new indirect
868 * interrupts and other responses after running a single handler.
869 */
870 for (;;) {
871 while (is_new_response(iq, &ctrl)) {
872
873 membar_consumer();
874
875 m = NULL;
876 rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
877 lq = be32_to_cpu(ctrl->pldbuflen_qid);
878 rss = (const void *)iq->cdesc;
879
880 switch (rsp_type) {
881 case X_RSPD_TYPE_FLBUF:
882
883 ASSERT(iq->flags & IQ_HAS_FL);
884
885 m = get_fl_payload(sc, fl, lq, &fl_bufs_used);
886 if (m == NULL) {
887 /*
888 * Rearm the iq with a
889 * longer-than-default timer
890 */
891 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
892 V_INGRESSQID((u32)iq->cntxt_id) |
893 V_SEINTARM(V_QINTR_TIMER_IDX(SGE_NTIMERS-1)));
894 if (fl_bufs_used > 0) {
895 ASSERT(iq->flags & IQ_HAS_FL);
896 FL_LOCK(fl);
897 fl->needed += fl_bufs_used;
898 starved = refill_fl(sc, fl, fl->cap / 8);
899 FL_UNLOCK(fl);
900 if (starved)
901 add_fl_to_sfl(sc, fl);
902 }
903 return (0);
904 }
905
906 /* FALLTHRU */
907 case X_RSPD_TYPE_CPL:
908
909 ASSERT(rss->opcode < NUM_CPL_CMDS);
910 sc->cpl_handler[rss->opcode](iq, rss, m);
911 break;
912
913 case X_RSPD_TYPE_INTR:
914
915 /*
916 * Interrupts should be forwarded only to queues
917 * that are not forwarding their interrupts.
918 * This means service_iq can recurse but only 1
919 * level deep.
920 */
921 ASSERT(budget == 0);
922
923 q = sc->sge.iqmap[lq - sc->sge.iq_start];
924 if (atomic_cas_uint(&q->state, IQS_IDLE,
925 IQS_BUSY) == IQS_IDLE) {
926 if (service_iq(q, q->qsize / 8) == 0) {
927 (void) atomic_cas_uint(
928 &q->state, IQS_BUSY,
929 IQS_IDLE);
930 } else {
931 STAILQ_INSERT_TAIL(&iql, q,
932 link);
933 }
934 }
935 break;
936
937 default:
938 break;
939 }
940
941 iq_next(iq);
942 if (++ndescs == limit) {
943 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
944 V_CIDXINC(ndescs) |
945 V_INGRESSQID(iq->cntxt_id) |
946 V_SEINTARM(V_QINTR_TIMER_IDX(
947 X_TIMERREG_UPDATE_CIDX)));
948 ndescs = 0;
949
950 if (fl_bufs_used > 0) {
951 ASSERT(iq->flags & IQ_HAS_FL);
952 FL_LOCK(fl);
953 fl->needed += fl_bufs_used;
954 (void) refill_fl(sc, fl, fl->cap / 8);
955 FL_UNLOCK(fl);
956 fl_bufs_used = 0;
957 }
958
959 if (budget != 0)
960 return (EINPROGRESS);
961 }
962 }
963
964 if (STAILQ_EMPTY(&iql) != 0)
965 break;
966
967 /*
968 * Process the head only, and send it to the back of the list if
969 * it's still not done.
970 */
971 q = STAILQ_FIRST(&iql);
972 STAILQ_REMOVE_HEAD(&iql, link);
973 if (service_iq(q, q->qsize / 8) == 0)
974 (void) atomic_cas_uint(&q->state, IQS_BUSY, IQS_IDLE);
975 else
976 STAILQ_INSERT_TAIL(&iql, q, link);
977 }
978
979 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
980 V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
981
982 if (iq->flags & IQ_HAS_FL) {
983
984 FL_LOCK(fl);
985 fl->needed += fl_bufs_used;
986 starved = refill_fl(sc, fl, fl->cap / 4);
987 FL_UNLOCK(fl);
988 if (starved != 0)
989 add_fl_to_sfl(sc, fl);
990 }
991
992 return (0);
993 }
994
995 #ifdef TCP_OFFLOAD_ENABLE
996 int
997 t4_mgmt_tx(struct adapter *sc, mblk_t *m)
998 {
999 return (t4_wrq_tx(sc, &sc->sge.mgmtq, m));
1000 }
1001
1002 /*
1003 * Doesn't fail. Holds on to work requests it can't send right away.
1004 */
1005 int
1006 t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0)
1007 {
1008 struct sge_eq *eq = &wrq->eq;
1009 struct mblk_pair *wr_list = &wrq->wr_list;
1010 int can_reclaim;
1011 caddr_t dst;
1012 mblk_t *wr, *next;
1013
1014 TXQ_LOCK_ASSERT_OWNED(wrq);
1015 #ifdef TCP_OFFLOAD_ENABLE
1016 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
1017 (eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1018 #else
1019 ASSERT((eq->flags & EQ_TYPEMASK) == EQ_CTRL);
1020 #endif
1021
1022 if (m0 != NULL) {
1023 if (wr_list->head != NULL)
1024 wr_list->tail->b_next = m0;
1025 else
1026 wr_list->head = m0;
1027 while (m0->b_next)
1028 m0 = m0->b_next;
1029 wr_list->tail = m0;
1030 }
1031
1032 can_reclaim = reclaimable(eq);
1033 eq->cidx += can_reclaim;
1034 eq->avail += can_reclaim;
1035 if (eq->cidx >= eq->cap)
1036 eq->cidx -= eq->cap;
1037
1038 for (wr = wr_list->head; wr; wr = next) {
1039 int ndesc, len = 0;
1040 mblk_t *m;
1041
1042 next = wr->b_next;
1043 wr->b_next = NULL;
1044
1045 for (m = wr; m; m = m->b_cont)
1046 len += MBLKL(m);
1047
1048 ASSERT(len > 0 && (len & 0x7) == 0);
1049 ASSERT(len <= SGE_MAX_WR_LEN);
1050
1051 ndesc = howmany(len, EQ_ESIZE);
1052 if (eq->avail < ndesc) {
1053 wr->b_next = next;
1054 wrq->no_desc++;
1055 break;
1056 }
1057
1058 dst = (void *)&eq->desc[eq->pidx];
1059 for (m = wr; m; m = m->b_cont)
1060 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
1061
1062 eq->pidx += ndesc;
1063 eq->avail -= ndesc;
1064 if (eq->pidx >= eq->cap)
1065 eq->pidx -= eq->cap;
1066
1067 eq->pending += ndesc;
1068 if (eq->pending > 16)
1069 ring_tx_db(sc, eq);
1070
1071 wrq->tx_wrs++;
1072 freemsg(wr);
1073
1074 if (eq->avail < 8) {
1075 can_reclaim = reclaimable(eq);
1076 eq->cidx += can_reclaim;
1077 eq->avail += can_reclaim;
1078 if (eq->cidx >= eq->cap)
1079 eq->cidx -= eq->cap;
1080 }
1081 }
1082
1083 if (eq->pending != 0)
1084 ring_tx_db(sc, eq);
1085
1086 if (wr == NULL)
1087 wr_list->head = wr_list->tail = NULL;
1088 else {
1089 wr_list->head = wr;
1090
1091 ASSERT(wr_list->tail->b_next == NULL);
1092 }
1093
1094 return (0);
1095 }
1096 #endif
1097
1098 /* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
1099 #define TXPKTS_PKT_HDR ((\
1100 sizeof (struct ulp_txpkt) + \
1101 sizeof (struct ulptx_idata) + \
1102 sizeof (struct cpl_tx_pkt_core)) / 8)
1103
1104 /* Header of a coalesced tx WR, before SGL of first packet (in flits) */
1105 #define TXPKTS_WR_HDR (\
1106 sizeof (struct fw_eth_tx_pkts_wr) / 8 + \
1107 TXPKTS_PKT_HDR)
1108
1109 /* Header of a tx WR, before SGL of first packet (in flits) */
1110 #define TXPKT_WR_HDR ((\
1111 sizeof (struct fw_eth_tx_pkt_wr) + \
1112 sizeof (struct cpl_tx_pkt_core)) / 8)
1113
1114 /* Header of a tx LSO WR, before SGL of first packet (in flits) */
1115 #define TXPKT_LSO_WR_HDR ((\
1116 sizeof (struct fw_eth_tx_pkt_wr) + \
1117 sizeof(struct cpl_tx_pkt_lso_core) + \
1118 sizeof (struct cpl_tx_pkt_core)) / 8)
1119
1120 mblk_t *
1121 t4_eth_tx(void *arg, mblk_t *frame)
1122 {
1123 struct sge_txq *txq = (struct sge_txq *) arg;
1124 struct port_info *pi = txq->port;
1125 struct adapter *sc = pi->adapter;
1126 struct sge_eq *eq = &txq->eq;
1127 mblk_t *next_frame;
1128 int rc, coalescing;
1129 struct txpkts txpkts;
1130 struct txinfo txinfo;
1131
1132 txpkts.npkt = 0; /* indicates there's nothing in txpkts */
1133 coalescing = 0;
1134
1135 TXQ_LOCK(txq);
1136 if (eq->avail < 8)
1137 (void) reclaim_tx_descs(txq, 8);
1138 for (; frame; frame = next_frame) {
1139
1140 if (eq->avail < 8)
1141 break;
1142
1143 next_frame = frame->b_next;
1144 frame->b_next = NULL;
1145
1146 if (next_frame != NULL)
1147 coalescing = 1;
1148
1149 rc = get_frame_txinfo(txq, &frame, &txinfo, coalescing);
1150 if (rc != 0) {
1151 if (rc == ENOMEM) {
1152
1153 /* Short of resources, suspend tx */
1154
1155 frame->b_next = next_frame;
1156 break;
1157 }
1158
1159 /*
1160 * Unrecoverable error for this frame, throw it
1161 * away and move on to the next.
1162 */
1163
1164 freemsg(frame);
1165 continue;
1166 }
1167
1168 if (coalescing != 0 &&
1169 add_to_txpkts(txq, &txpkts, frame, &txinfo) == 0) {
1170
1171 /* Successfully absorbed into txpkts */
1172
1173 write_ulp_cpl_sgl(pi, txq, &txpkts, &txinfo);
1174 goto doorbell;
1175 }
1176
1177 /*
1178 * We weren't coalescing to begin with, or current frame could
1179 * not be coalesced (add_to_txpkts flushes txpkts if a frame
1180 * given to it can't be coalesced). Either way there should be
1181 * nothing in txpkts.
1182 */
1183 ASSERT(txpkts.npkt == 0);
1184
1185 /* We're sending out individual frames now */
1186 coalescing = 0;
1187
1188 if (eq->avail < 8)
1189 (void) reclaim_tx_descs(txq, 8);
1190 rc = write_txpkt_wr(pi, txq, frame, &txinfo);
1191 if (rc != 0) {
1192
1193 /* Short of hardware descriptors, suspend tx */
1194
1195 /*
1196 * This is an unlikely but expensive failure. We've
1197 * done all the hard work (DMA bindings etc.) and now we
1198 * can't send out the frame. What's worse, we have to
1199 * spend even more time freeing up everything in txinfo.
1200 */
1201 txq->qfull++;
1202 free_txinfo_resources(txq, &txinfo);
1203
1204 frame->b_next = next_frame;
1205 break;
1206 }
1207
1208 doorbell:
1209 /* Fewer and fewer doorbells as the queue fills up */
1210 if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2))) {
1211 txq->txbytes += txinfo.len;
1212 txq->txpkts++;
1213 ring_tx_db(sc, eq);
1214 }
1215 (void) reclaim_tx_descs(txq, 32);
1216 }
1217
1218 if (txpkts.npkt > 0)
1219 write_txpkts_wr(txq, &txpkts);
1220
1221 /*
1222 * frame not NULL means there was an error but we haven't thrown it
1223 * away. This can happen when we're short of tx descriptors (qfull) or
1224 * maybe even DMA handles (dma_hdl_failed). Either way, a credit flush
1225 * and reclaim will get things going again.
1226 *
1227 * If eq->avail is already 0 we know a credit flush was requested in the
1228 * WR that reduced it to 0 so we don't need another flush (we don't have
1229 * any descriptor for a flush WR anyway, duh).
1230 */
1231 if (frame && eq->avail > 0)
1232 write_txqflush_wr(txq);
1233
1234 if (eq->pending != 0)
1235 ring_tx_db(sc, eq);
1236
1237 (void) reclaim_tx_descs(txq, eq->qsize);
1238 TXQ_UNLOCK(txq);
1239
1240 return (frame);
1241 }
1242
1243 static inline void
1244 init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int8_t pktc_idx,
1245 int qsize, uint8_t esize)
1246 {
1247 ASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS);
1248 ASSERT(pktc_idx < SGE_NCOUNTERS); /* -ve is ok, means don't use */
1249
1250 iq->flags = 0;
1251 iq->adapter = sc;
1252 iq->intr_params = V_QINTR_TIMER_IDX(tmr_idx);
1253 iq->intr_pktc_idx = SGE_NCOUNTERS - 1;
1254 if (pktc_idx >= 0) {
1255 iq->intr_params |= F_QINTR_CNT_EN;
1256 iq->intr_pktc_idx = pktc_idx;
1257 }
1258 iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
1259 iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
1260 }
1261
1262 static inline void
1263 init_fl(struct sge_fl *fl, uint16_t qsize)
1264 {
1265
1266 fl->qsize = qsize;
1267 fl->allocb_fail = 0;
1268 }
1269
1270 static inline void
1271 init_eq(struct adapter *sc, struct sge_eq *eq, uint16_t eqtype, uint16_t qsize,
1272 uint8_t tx_chan, uint16_t iqid)
1273 {
1274 struct sge *s = &sc->sge;
1275 uint32_t r;
1276
1277 ASSERT(tx_chan < NCHAN);
1278 ASSERT(eqtype <= EQ_TYPEMASK);
1279
1280 if (is_t5(sc->params.chip)) {
1281 r = t4_read_reg(sc, A_SGE_EGRESS_QUEUES_PER_PAGE_PF);
1282 r >>= S_QUEUESPERPAGEPF0 +
1283 (S_QUEUESPERPAGEPF1 - S_QUEUESPERPAGEPF0) * sc->pf;
1284 s->s_qpp = r & M_QUEUESPERPAGEPF0;
1285 }
1286
1287 eq->flags = eqtype & EQ_TYPEMASK;
1288 eq->tx_chan = tx_chan;
1289 eq->iqid = iqid;
1290 eq->qsize = qsize;
1291 }
1292
1293 /*
1294 * Allocates the ring for an ingress queue and an optional freelist. If the
1295 * freelist is specified it will be allocated and then associated with the
1296 * ingress queue.
1297 *
1298 * Returns errno on failure. Resources allocated up to that point may still be
1299 * allocated. Caller is responsible for cleanup in case this function fails.
1300 *
1301 * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
1302 * the intr_idx specifies the vector, starting from 0. Otherwise it specifies
1303 * the index of the queue to which its interrupts will be forwarded.
1304 */
1305 static int
1306 alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
1307 int intr_idx, int cong)
1308 {
1309 int rc, i, cntxt_id;
1310 size_t len;
1311 struct fw_iq_cmd c;
1312 struct adapter *sc = iq->adapter;
1313 uint32_t v = 0;
1314
1315 len = iq->qsize * iq->esize;
1316 rc = alloc_desc_ring(sc, len, DDI_DMA_READ, &iq->dhdl, &iq->ahdl,
1317 &iq->ba, (caddr_t *)&iq->desc);
1318 if (rc != 0)
1319 return (rc);
1320
1321 bzero(&c, sizeof (c));
1322 c.op_to_vfn = cpu_to_be32(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST |
1323 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(sc->pf) |
1324 V_FW_IQ_CMD_VFN(0));
1325
1326 c.alloc_to_len16 = cpu_to_be32(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART |
1327 FW_LEN16(c));
1328
1329 /* Special handling for firmware event queue */
1330 if (iq == &sc->sge.fwq)
1331 v |= F_FW_IQ_CMD_IQASYNCH;
1332
1333 if (iq->flags & IQ_INTR)
1334 ASSERT(intr_idx < sc->intr_count);
1335 else
1336 v |= F_FW_IQ_CMD_IQANDST;
1337 v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
1338
1339 c.type_to_iqandstindex = cpu_to_be32(v |
1340 V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
1341 V_FW_IQ_CMD_VIID(pi->viid) |
1342 V_FW_IQ_CMD_IQANUD(X_UPDATEDELIVERY_INTERRUPT));
1343 c.iqdroprss_to_iqesize = cpu_to_be16(V_FW_IQ_CMD_IQPCIECH(pi->tx_chan) |
1344 F_FW_IQ_CMD_IQGTSMODE |
1345 V_FW_IQ_CMD_IQINTCNTTHRESH(iq->intr_pktc_idx) |
1346 V_FW_IQ_CMD_IQESIZE(ilog2(iq->esize) - 4));
1347 c.iqsize = cpu_to_be16(iq->qsize);
1348 c.iqaddr = cpu_to_be64(iq->ba);
1349 if (cong >= 0)
1350 c.iqns_to_fl0congen = BE_32(F_FW_IQ_CMD_IQFLINTCONGEN);
1351
1352 if (fl != NULL) {
1353 unsigned int chip_ver = CHELSIO_CHIP_VERSION(sc->params.chip);
1354
1355 mutex_init(&fl->lock, NULL, MUTEX_DRIVER,
1356 DDI_INTR_PRI(sc->intr_pri));
1357 fl->flags |= FL_MTX;
1358
1359 len = fl->qsize * RX_FL_ESIZE;
1360 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &fl->dhdl,
1361 &fl->ahdl, &fl->ba, (caddr_t *)&fl->desc);
1362 if (rc != 0)
1363 return (rc);
1364
1365 /* Allocate space for one software descriptor per buffer. */
1366 fl->cap = (fl->qsize - sc->sge.stat_len / RX_FL_ESIZE) * 8;
1367 fl->sdesc = kmem_zalloc(sizeof (struct fl_sdesc) * fl->cap,
1368 KM_SLEEP);
1369 fl->needed = fl->cap;
1370 fl->lowat = roundup(sc->sge.fl_starve_threshold, 8);
1371
1372 c.iqns_to_fl0congen |=
1373 cpu_to_be32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
1374 F_FW_IQ_CMD_FL0PACKEN | F_FW_IQ_CMD_FL0PADEN);
1375 if (cong >= 0) {
1376 c.iqns_to_fl0congen |=
1377 BE_32(V_FW_IQ_CMD_FL0CNGCHMAP(cong) |
1378 F_FW_IQ_CMD_FL0CONGCIF |
1379 F_FW_IQ_CMD_FL0CONGEN);
1380 }
1381
1382 /* In T6, for egress queue type FL there is internal overhead
1383 * of 16B for header going into FLM module. Hence the maximum
1384 * allowed burst size is 448 bytes. For T4/T5, the hardware
1385 * doesn't coalesce fetch requests if more than 64 bytes of
1386 * Free List pointers are provided, so we use a 128-byte Fetch
1387 * Burst Minimum there (T6 implements coalescing so we can use
1388 * the smaller 64-byte value there).
1389 */
1390
1391 c.fl0dcaen_to_fl0cidxfthresh =
1392 cpu_to_be16(V_FW_IQ_CMD_FL0FBMIN(chip_ver <= CHELSIO_T5
1393 ? X_FETCHBURSTMIN_128B
1394 : X_FETCHBURSTMIN_64B) |
1395 V_FW_IQ_CMD_FL0FBMAX(chip_ver <= CHELSIO_T5
1396 ? X_FETCHBURSTMAX_512B
1397 : X_FETCHBURSTMAX_256B));
1398 c.fl0size = cpu_to_be16(fl->qsize);
1399 c.fl0addr = cpu_to_be64(fl->ba);
1400 }
1401
1402 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1403 if (rc != 0) {
1404 cxgb_printf(sc->dip, CE_WARN,
1405 "failed to create ingress queue: %d", rc);
1406 return (rc);
1407 }
1408
1409 iq->cdesc = iq->desc;
1410 iq->cidx = 0;
1411 iq->gen = 1;
1412 iq->intr_next = iq->intr_params;
1413 iq->adapter = sc;
1414 iq->cntxt_id = be16_to_cpu(c.iqid);
1415 iq->abs_id = be16_to_cpu(c.physiqid);
1416 iq->flags |= IQ_ALLOCATED;
1417 mutex_init(&iq->lock, NULL,
1418 MUTEX_DRIVER, DDI_INTR_PRI(DDI_INTR_PRI(sc->intr_pri)));
1419 iq->polling = 0;
1420
1421 cntxt_id = iq->cntxt_id - sc->sge.iq_start;
1422 if (cntxt_id >= sc->sge.niq) {
1423 panic("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
1424 cntxt_id, sc->sge.niq - 1);
1425 }
1426 sc->sge.iqmap[cntxt_id] = iq;
1427
1428 if (fl != NULL) {
1429 fl->cntxt_id = be16_to_cpu(c.fl0id);
1430 fl->pidx = fl->cidx = 0;
1431 fl->copy_threshold = rx_copy_threshold;
1432
1433 cntxt_id = fl->cntxt_id - sc->sge.eq_start;
1434 if (cntxt_id >= sc->sge.neq) {
1435 panic("%s: fl->cntxt_id (%d) more than the max (%d)",
1436 __func__, cntxt_id, sc->sge.neq - 1);
1437 }
1438 sc->sge.eqmap[cntxt_id] = (void *)fl;
1439
1440 FL_LOCK(fl);
1441 (void) refill_fl(sc, fl, fl->lowat);
1442 FL_UNLOCK(fl);
1443
1444 iq->flags |= IQ_HAS_FL;
1445 }
1446
1447 if (is_t5(sc->params.chip) && cong >= 0) {
1448 uint32_t param, val;
1449
1450 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1451 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_CONM_CTXT) |
1452 V_FW_PARAMS_PARAM_YZ(iq->cntxt_id);
1453 if (cong == 0)
1454 val = 1 << 19;
1455 else {
1456 val = 2 << 19;
1457 for (i = 0; i < 4; i++) {
1458 if (cong & (1 << i))
1459 val |= 1 << (i << 2);
1460 }
1461 }
1462
1463 rc = -t4_set_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
1464 if (rc != 0) {
1465 /* report error but carry on */
1466 cxgb_printf(sc->dip, CE_WARN,
1467 "failed to set congestion manager context for "
1468 "ingress queue %d: %d", iq->cntxt_id, rc);
1469 }
1470 }
1471
1472 /* Enable IQ interrupts */
1473 iq->state = IQS_IDLE;
1474 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
1475 V_INGRESSQID(iq->cntxt_id));
1476
1477 return (0);
1478 }
1479
1480 static int
1481 free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
1482 {
1483 int rc;
1484 struct adapter *sc = iq->adapter;
1485 dev_info_t *dip;
1486
1487 dip = pi ? pi->dip : sc->dip;
1488
1489 if (iq != NULL) {
1490 if (iq->flags & IQ_ALLOCATED) {
1491 rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
1492 FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
1493 fl ? fl->cntxt_id : 0xffff, 0xffff);
1494 if (rc != 0) {
1495 cxgb_printf(dip, CE_WARN,
1496 "failed to free queue %p: %d", iq, rc);
1497 return (rc);
1498 }
1499 mutex_destroy(&iq->lock);
1500 iq->flags &= ~IQ_ALLOCATED;
1501 }
1502
1503 if (iq->desc != NULL) {
1504 (void) free_desc_ring(&iq->dhdl, &iq->ahdl);
1505 iq->desc = NULL;
1506 }
1507
1508 bzero(iq, sizeof (*iq));
1509 }
1510
1511 if (fl != NULL) {
1512 if (fl->sdesc != NULL) {
1513 FL_LOCK(fl);
1514 free_fl_bufs(fl);
1515 FL_UNLOCK(fl);
1516
1517 kmem_free(fl->sdesc, sizeof (struct fl_sdesc) *
1518 fl->cap);
1519 fl->sdesc = NULL;
1520 }
1521
1522 if (fl->desc != NULL) {
1523 (void) free_desc_ring(&fl->dhdl, &fl->ahdl);
1524 fl->desc = NULL;
1525 }
1526
1527 if (fl->flags & FL_MTX) {
1528 mutex_destroy(&fl->lock);
1529 fl->flags &= ~FL_MTX;
1530 }
1531
1532 bzero(fl, sizeof (struct sge_fl));
1533 }
1534
1535 return (0);
1536 }
1537
1538 static int
1539 alloc_fwq(struct adapter *sc)
1540 {
1541 int rc, intr_idx;
1542 struct sge_iq *fwq = &sc->sge.fwq;
1543
1544 init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE);
1545 fwq->flags |= IQ_INTR; /* always */
1546 intr_idx = sc->intr_count > 1 ? 1 : 0;
1547 rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
1548 if (rc != 0) {
1549 cxgb_printf(sc->dip, CE_WARN,
1550 "failed to create firmware event queue: %d.", rc);
1551 return (rc);
1552 }
1553
1554 return (0);
1555 }
1556
1557 static int
1558 free_fwq(struct adapter *sc)
1559 {
1560
1561 return (free_iq_fl(NULL, &sc->sge.fwq, NULL));
1562 }
1563
1564 #ifdef TCP_OFFLOAD_ENABLE
1565 static int
1566 alloc_mgmtq(struct adapter *sc)
1567 {
1568 int rc;
1569 struct sge_wrq *mgmtq = &sc->sge.mgmtq;
1570
1571 init_eq(sc, &mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
1572 sc->sge.fwq.cntxt_id);
1573 rc = alloc_wrq(sc, NULL, mgmtq, 0);
1574 if (rc != 0) {
1575 cxgb_printf(sc->dip, CE_WARN,
1576 "failed to create management queue: %d\n", rc);
1577 return (rc);
1578 }
1579
1580 return (0);
1581 }
1582 #endif
1583
1584 static int
1585 alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int i)
1586 {
1587 int rc;
1588
1589 rxq->port = pi;
1590 rc = alloc_iq_fl(pi, &rxq->iq, &rxq->fl, intr_idx, 1 << pi->tx_chan);
1591 if (rc != 0)
1592 return (rc);
1593
1594 rxq->ksp = setup_rxq_kstats(pi, rxq, i);
1595
1596 return (rc);
1597 }
1598
1599 static int
1600 free_rxq(struct port_info *pi, struct sge_rxq *rxq)
1601 {
1602 int rc;
1603
1604 if (rxq->ksp != NULL) {
1605 kstat_delete(rxq->ksp);
1606 rxq->ksp = NULL;
1607 }
1608
1609 rc = free_iq_fl(pi, &rxq->iq, &rxq->fl);
1610 if (rc == 0)
1611 bzero(&rxq->fl, sizeof (*rxq) - offsetof(struct sge_rxq, fl));
1612
1613 return (rc);
1614 }
1615
1616 #ifdef TCP_OFFLOAD_ENABLE
1617 static int
1618 alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
1619 int intr_idx)
1620 {
1621 int rc;
1622
1623 rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
1624 1 << pi->tx_chan);
1625 if (rc != 0)
1626 return (rc);
1627
1628 return (rc);
1629 }
1630
1631 static int
1632 free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
1633 {
1634 int rc;
1635
1636 rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
1637 if (rc == 0)
1638 bzero(&ofld_rxq->fl, sizeof (*ofld_rxq) -
1639 offsetof(struct sge_ofld_rxq, fl));
1640
1641 return (rc);
1642 }
1643 #endif
1644
1645 static int
1646 ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
1647 {
1648 int rc, cntxt_id;
1649 struct fw_eq_ctrl_cmd c;
1650
1651 bzero(&c, sizeof (c));
1652
1653 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_CTRL_CMD) | F_FW_CMD_REQUEST |
1654 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_CTRL_CMD_PFN(sc->pf) |
1655 V_FW_EQ_CTRL_CMD_VFN(0));
1656 c.alloc_to_len16 = BE_32(F_FW_EQ_CTRL_CMD_ALLOC |
1657 F_FW_EQ_CTRL_CMD_EQSTART | FW_LEN16(c));
1658 c.cmpliqid_eqid = htonl(V_FW_EQ_CTRL_CMD_CMPLIQID(eq->iqid)); /* TODO */
1659 c.physeqid_pkd = BE_32(0);
1660 c.fetchszm_to_iqid =
1661 BE_32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1662 V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
1663 F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
1664 c.dcaen_to_eqsize =
1665 BE_32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1666 V_FW_EQ_CTRL_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1667 V_FW_EQ_CTRL_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1668 V_FW_EQ_CTRL_CMD_EQSIZE(eq->qsize));
1669 c.eqaddr = BE_64(eq->ba);
1670
1671 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1672 if (rc != 0) {
1673 cxgb_printf(sc->dip, CE_WARN,
1674 "failed to create control queue %d: %d", eq->tx_chan, rc);
1675 return (rc);
1676 }
1677 eq->flags |= EQ_ALLOCATED;
1678
1679 eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(BE_32(c.cmpliqid_eqid));
1680 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1681 if (cntxt_id >= sc->sge.neq)
1682 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1683 cntxt_id, sc->sge.neq - 1);
1684 sc->sge.eqmap[cntxt_id] = eq;
1685
1686 return (rc);
1687 }
1688
1689 static int
1690 eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1691 {
1692 int rc, cntxt_id;
1693 struct fw_eq_eth_cmd c;
1694
1695 bzero(&c, sizeof (c));
1696
1697 c.op_to_vfn = BE_32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
1698 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
1699 V_FW_EQ_ETH_CMD_VFN(0));
1700 c.alloc_to_len16 = BE_32(F_FW_EQ_ETH_CMD_ALLOC |
1701 F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
1702 c.autoequiqe_to_viid = BE_32(F_FW_EQ_ETH_CMD_AUTOEQUIQE |
1703 F_FW_EQ_ETH_CMD_AUTOEQUEQE | V_FW_EQ_ETH_CMD_VIID(pi->viid));
1704 c.fetchszm_to_iqid =
1705 BE_32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1706 V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
1707 V_FW_EQ_ETH_CMD_IQID(eq->iqid));
1708 c.dcaen_to_eqsize = BE_32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1709 V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1710 V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1711 V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
1712 c.eqaddr = BE_64(eq->ba);
1713
1714 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1715 if (rc != 0) {
1716 cxgb_printf(pi->dip, CE_WARN,
1717 "failed to create Ethernet egress queue: %d", rc);
1718 return (rc);
1719 }
1720 eq->flags |= EQ_ALLOCATED;
1721
1722 eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(BE_32(c.eqid_pkd));
1723 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1724 if (cntxt_id >= sc->sge.neq)
1725 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1726 cntxt_id, sc->sge.neq - 1);
1727 sc->sge.eqmap[cntxt_id] = eq;
1728
1729 return (rc);
1730 }
1731
1732 #ifdef TCP_OFFLOAD_ENABLE
1733 static int
1734 ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1735 {
1736 int rc, cntxt_id;
1737 struct fw_eq_ofld_cmd c;
1738
1739 bzero(&c, sizeof (c));
1740
1741 c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
1742 F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
1743 V_FW_EQ_OFLD_CMD_VFN(0));
1744 c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
1745 F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
1746 c.fetchszm_to_iqid =
1747 htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
1748 V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
1749 F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
1750 c.dcaen_to_eqsize =
1751 BE_32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
1752 V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
1753 V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
1754 V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
1755 c.eqaddr = BE_64(eq->ba);
1756
1757 rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof (c), &c);
1758 if (rc != 0) {
1759 cxgb_printf(pi->dip, CE_WARN,
1760 "failed to create egress queue for TCP offload: %d", rc);
1761 return (rc);
1762 }
1763 eq->flags |= EQ_ALLOCATED;
1764
1765 eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(BE_32(c.eqid_pkd));
1766 cntxt_id = eq->cntxt_id - sc->sge.eq_start;
1767 if (cntxt_id >= sc->sge.neq)
1768 panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
1769 cntxt_id, sc->sge.neq - 1);
1770 sc->sge.eqmap[cntxt_id] = eq;
1771
1772 return (rc);
1773 }
1774 #endif
1775
1776 static int
1777 alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
1778 {
1779 int rc;
1780 size_t len;
1781
1782 mutex_init(&eq->lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(sc->intr_pri));
1783 eq->flags |= EQ_MTX;
1784
1785 len = eq->qsize * EQ_ESIZE;
1786 rc = alloc_desc_ring(sc, len, DDI_DMA_WRITE, &eq->desc_dhdl,
1787 &eq->desc_ahdl, &eq->ba, (caddr_t *)&eq->desc);
1788 if (rc != 0)
1789 return (rc);
1790
1791 eq->cap = eq->qsize - sc->sge.stat_len / EQ_ESIZE;
1792 eq->spg = (void *)&eq->desc[eq->cap];
1793 eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
1794 eq->pidx = eq->cidx = 0;
1795 eq->doorbells = sc->doorbells;
1796
1797 switch (eq->flags & EQ_TYPEMASK) {
1798 case EQ_CTRL:
1799 rc = ctrl_eq_alloc(sc, eq);
1800 break;
1801
1802 case EQ_ETH:
1803 rc = eth_eq_alloc(sc, pi, eq);
1804 break;
1805
1806 #ifdef TCP_OFFLOAD_ENABLE
1807 case EQ_OFLD:
1808 rc = ofld_eq_alloc(sc, pi, eq);
1809 break;
1810 #endif
1811
1812 default:
1813 panic("%s: invalid eq type %d.", __func__,
1814 eq->flags & EQ_TYPEMASK);
1815 }
1816
1817 if (eq->doorbells &
1818 (DOORBELL_UDB | DOORBELL_UDBWC | DOORBELL_WCWR)) {
1819 uint32_t s_qpp = sc->sge.s_qpp;
1820 uint32_t mask = (1 << s_qpp) - 1;
1821 volatile uint8_t *udb;
1822
1823 udb = (volatile uint8_t *)sc->reg1p + UDBS_DB_OFFSET;
1824 udb += (eq->cntxt_id >> s_qpp) << PAGE_SHIFT; /* pg offset */
1825 eq->udb_qid = eq->cntxt_id & mask; /* id in page */
1826 if (eq->udb_qid > PAGE_SIZE / UDBS_SEG_SIZE)
1827 eq->doorbells &= ~DOORBELL_WCWR;
1828 else {
1829 udb += eq->udb_qid << UDBS_SEG_SHIFT; /* seg offset */
1830 eq->udb_qid = 0;
1831 }
1832 eq->udb = (volatile void *)udb;
1833 }
1834
1835 if (rc != 0) {
1836 cxgb_printf(sc->dip, CE_WARN,
1837 "failed to allocate egress queue(%d): %d",
1838 eq->flags & EQ_TYPEMASK, rc);
1839 }
1840
1841 return (rc);
1842 }
1843
1844 static int
1845 free_eq(struct adapter *sc, struct sge_eq *eq)
1846 {
1847 int rc;
1848
1849 if (eq->flags & EQ_ALLOCATED) {
1850 switch (eq->flags & EQ_TYPEMASK) {
1851 case EQ_CTRL:
1852 rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
1853 eq->cntxt_id);
1854 break;
1855
1856 case EQ_ETH:
1857 rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
1858 eq->cntxt_id);
1859 break;
1860
1861 #ifdef TCP_OFFLOAD_ENABLE
1862 case EQ_OFLD:
1863 rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
1864 eq->cntxt_id);
1865 break;
1866 #endif
1867
1868 default:
1869 panic("%s: invalid eq type %d.", __func__,
1870 eq->flags & EQ_TYPEMASK);
1871 }
1872 if (rc != 0) {
1873 cxgb_printf(sc->dip, CE_WARN,
1874 "failed to free egress queue (%d): %d",
1875 eq->flags & EQ_TYPEMASK, rc);
1876 return (rc);
1877 }
1878 eq->flags &= ~EQ_ALLOCATED;
1879 }
1880
1881 if (eq->desc != NULL) {
1882 (void) free_desc_ring(&eq->desc_dhdl, &eq->desc_ahdl);
1883 eq->desc = NULL;
1884 }
1885
1886 if (eq->flags & EQ_MTX)
1887 mutex_destroy(&eq->lock);
1888
1889 bzero(eq, sizeof (*eq));
1890 return (0);
1891 }
1892
1893 #ifdef TCP_OFFLOAD_ENABLE
1894 /* ARGSUSED */
1895 static int
1896 alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq,
1897 int idx)
1898 {
1899 int rc;
1900
1901 rc = alloc_eq(sc, pi, &wrq->eq);
1902 if (rc != 0)
1903 return (rc);
1904
1905 wrq->adapter = sc;
1906 wrq->wr_list.head = NULL;
1907 wrq->wr_list.tail = NULL;
1908
1909 /*
1910 * TODO: use idx to figure out what kind of wrq this is and install
1911 * useful kstats for it.
1912 */
1913
1914 return (rc);
1915 }
1916
1917 static int
1918 free_wrq(struct adapter *sc, struct sge_wrq *wrq)
1919 {
1920 int rc;
1921
1922 rc = free_eq(sc, &wrq->eq);
1923 if (rc != 0)
1924 return (rc);
1925
1926 bzero(wrq, sizeof (*wrq));
1927 return (0);
1928 }
1929 #endif
1930
1931 static int
1932 alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
1933 {
1934 int rc, i;
1935 struct adapter *sc = pi->adapter;
1936 struct sge_eq *eq = &txq->eq;
1937
1938 rc = alloc_eq(sc, pi, eq);
1939 if (rc != 0)
1940 return (rc);
1941
1942 txq->port = pi;
1943 txq->sdesc = kmem_zalloc(sizeof (struct tx_sdesc) * eq->cap, KM_SLEEP);
1944 txq->txb_size = eq->qsize * tx_copy_threshold;
1945 rc = alloc_tx_copybuffer(sc, txq->txb_size, &txq->txb_dhdl,
1946 &txq->txb_ahdl, &txq->txb_ba, &txq->txb_va);
1947 if (rc == 0)
1948 txq->txb_avail = txq->txb_size;
1949 else
1950 txq->txb_avail = txq->txb_size = 0;
1951
1952 /*
1953 * TODO: is this too low? Worst case would need around 4 times qsize
1954 * (all tx descriptors filled to the brim with SGLs, with each entry in
1955 * the SGL coming from a distinct DMA handle). Increase tx_dhdl_total
1956 * if you see too many dma_hdl_failed.
1957 */
1958 txq->tx_dhdl_total = eq->qsize * 2;
1959 txq->tx_dhdl = kmem_zalloc(sizeof (ddi_dma_handle_t) *
1960 txq->tx_dhdl_total, KM_SLEEP);
1961 for (i = 0; i < txq->tx_dhdl_total; i++) {
1962 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
1963 DDI_DMA_SLEEP, 0, &txq->tx_dhdl[i]);
1964 if (rc != DDI_SUCCESS) {
1965 cxgb_printf(sc->dip, CE_WARN,
1966 "%s: failed to allocate DMA handle (%d)",
1967 __func__, rc);
1968 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
1969 }
1970 txq->tx_dhdl_avail++;
1971 }
1972
1973 txq->ksp = setup_txq_kstats(pi, txq, idx);
1974
1975 return (rc);
1976 }
1977
1978 static int
1979 free_txq(struct port_info *pi, struct sge_txq *txq)
1980 {
1981 int i;
1982 struct adapter *sc = pi->adapter;
1983 struct sge_eq *eq = &txq->eq;
1984
1985 if (txq->ksp != NULL) {
1986 kstat_delete(txq->ksp);
1987 txq->ksp = NULL;
1988 }
1989
1990 if (txq->txb_va != NULL) {
1991 (void) free_desc_ring(&txq->txb_dhdl, &txq->txb_ahdl);
1992 txq->txb_va = NULL;
1993 }
1994
1995 if (txq->sdesc != NULL) {
1996 struct tx_sdesc *sd;
1997 ddi_dma_handle_t hdl;
1998
1999 TXQ_LOCK(txq);
2000 while (eq->cidx != eq->pidx) {
2001 sd = &txq->sdesc[eq->cidx];
2002
2003 for (i = sd->hdls_used; i; i--) {
2004 hdl = txq->tx_dhdl[txq->tx_dhdl_cidx];
2005 (void) ddi_dma_unbind_handle(hdl);
2006 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
2007 txq->tx_dhdl_cidx = 0;
2008 }
2009
2010 ASSERT(sd->m);
2011 freemsgchain(sd->m);
2012
2013 eq->cidx += sd->desc_used;
2014 if (eq->cidx >= eq->cap)
2015 eq->cidx -= eq->cap;
2016
2017 txq->txb_avail += txq->txb_used;
2018 }
2019 ASSERT(txq->tx_dhdl_cidx == txq->tx_dhdl_pidx);
2020 ASSERT(txq->txb_avail == txq->txb_size);
2021 TXQ_UNLOCK(txq);
2022
2023 kmem_free(txq->sdesc, sizeof (struct tx_sdesc) * eq->cap);
2024 txq->sdesc = NULL;
2025 }
2026
2027 if (txq->tx_dhdl != NULL) {
2028 for (i = 0; i < txq->tx_dhdl_total; i++) {
2029 if (txq->tx_dhdl[i] != NULL)
2030 ddi_dma_free_handle(&txq->tx_dhdl[i]);
2031 }
2032 }
2033
2034 (void) free_eq(sc, &txq->eq);
2035
2036 bzero(txq, sizeof (*txq));
2037 return (0);
2038 }
2039
2040 /*
2041 * Allocates a block of contiguous memory for DMA. Can be used to allocate
2042 * memory for descriptor rings or for tx/rx copy buffers.
2043 *
2044 * Caller does not have to clean up anything if this function fails, it cleans
2045 * up after itself.
2046 *
2047 * Caller provides the following:
2048 * len length of the block of memory to allocate.
2049 * flags DDI_DMA_* flags to use (CONSISTENT/STREAMING, READ/WRITE/RDWR)
2050 * acc_attr device access attributes for the allocation.
2051 * dma_attr DMA attributes for the allocation
2052 *
2053 * If the function is successful it fills up this information:
2054 * dma_hdl DMA handle for the allocated memory
2055 * acc_hdl access handle for the allocated memory
2056 * ba bus address of the allocated memory
2057 * va KVA of the allocated memory.
2058 */
2059 static int
2060 alloc_dma_memory(struct adapter *sc, size_t len, int flags,
2061 ddi_device_acc_attr_t *acc_attr, ddi_dma_attr_t *dma_attr,
2062 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2063 uint64_t *pba, caddr_t *pva)
2064 {
2065 int rc;
2066 ddi_dma_handle_t dhdl;
2067 ddi_acc_handle_t ahdl;
2068 ddi_dma_cookie_t cookie;
2069 uint_t ccount;
2070 caddr_t va;
2071 size_t real_len;
2072
2073 *pva = NULL;
2074
2075 /*
2076 * DMA handle.
2077 */
2078 rc = ddi_dma_alloc_handle(sc->dip, dma_attr, DDI_DMA_SLEEP, 0, &dhdl);
2079 if (rc != DDI_SUCCESS) {
2080 cxgb_printf(sc->dip, CE_WARN,
2081 "failed to allocate DMA handle: %d", rc);
2082
2083 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
2084 }
2085
2086 /*
2087 * Memory suitable for DMA.
2088 */
2089 rc = ddi_dma_mem_alloc(dhdl, len, acc_attr,
2090 flags & DDI_DMA_CONSISTENT ? DDI_DMA_CONSISTENT : DDI_DMA_STREAMING,
2091 DDI_DMA_SLEEP, 0, &va, &real_len, &ahdl);
2092 if (rc != DDI_SUCCESS) {
2093 cxgb_printf(sc->dip, CE_WARN,
2094 "failed to allocate DMA memory: %d", rc);
2095
2096 ddi_dma_free_handle(&dhdl);
2097 return (ENOMEM);
2098 }
2099
2100 if (len != real_len) {
2101 cxgb_printf(sc->dip, CE_WARN,
2102 "%s: len (%u) != real_len (%u)\n", len, real_len);
2103 }
2104
2105 /*
2106 * DMA bindings.
2107 */
2108 rc = ddi_dma_addr_bind_handle(dhdl, NULL, va, real_len, flags, NULL,
2109 NULL, &cookie, &ccount);
2110 if (rc != DDI_DMA_MAPPED) {
2111 cxgb_printf(sc->dip, CE_WARN,
2112 "failed to map DMA memory: %d", rc);
2113
2114 ddi_dma_mem_free(&ahdl);
2115 ddi_dma_free_handle(&dhdl);
2116 return (ENOMEM);
2117 }
2118 if (ccount != 1) {
2119 cxgb_printf(sc->dip, CE_WARN,
2120 "unusable DMA mapping (%d segments)", ccount);
2121 (void) free_desc_ring(&dhdl, &ahdl);
2122 }
2123
2124 bzero(va, real_len);
2125 *dma_hdl = dhdl;
2126 *acc_hdl = ahdl;
2127 *pba = cookie.dmac_laddress;
2128 *pva = va;
2129
2130 return (0);
2131 }
2132
2133 static int
2134 free_dma_memory(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2135 {
2136 (void) ddi_dma_unbind_handle(*dhdl);
2137 ddi_dma_mem_free(ahdl);
2138 ddi_dma_free_handle(dhdl);
2139
2140 return (0);
2141 }
2142
2143 static int
2144 alloc_desc_ring(struct adapter *sc, size_t len, int rw,
2145 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2146 uint64_t *pba, caddr_t *pva)
2147 {
2148 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_desc;
2149 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc;
2150
2151 return (alloc_dma_memory(sc, len, DDI_DMA_CONSISTENT | rw, acc_attr,
2152 dma_attr, dma_hdl, acc_hdl, pba, pva));
2153 }
2154
2155 static int
2156 free_desc_ring(ddi_dma_handle_t *dhdl, ddi_acc_handle_t *ahdl)
2157 {
2158 return (free_dma_memory(dhdl, ahdl));
2159 }
2160
2161 static int
2162 alloc_tx_copybuffer(struct adapter *sc, size_t len,
2163 ddi_dma_handle_t *dma_hdl, ddi_acc_handle_t *acc_hdl,
2164 uint64_t *pba, caddr_t *pva)
2165 {
2166 ddi_device_acc_attr_t *acc_attr = &sc->sge.acc_attr_tx;
2167 ddi_dma_attr_t *dma_attr = &sc->sge.dma_attr_desc; /* NOT dma_attr_tx */
2168
2169 return (alloc_dma_memory(sc, len, DDI_DMA_STREAMING | DDI_DMA_WRITE,
2170 acc_attr, dma_attr, dma_hdl, acc_hdl, pba, pva));
2171 }
2172
2173 static inline bool
2174 is_new_response(const struct sge_iq *iq, struct rsp_ctrl **ctrl)
2175 {
2176 (void) ddi_dma_sync(iq->dhdl, (uintptr_t)iq->cdesc -
2177 (uintptr_t)iq->desc, iq->esize, DDI_DMA_SYNC_FORKERNEL);
2178
2179 *ctrl = (void *)((uintptr_t)iq->cdesc +
2180 (iq->esize - sizeof (struct rsp_ctrl)));
2181
2182 return ((((*ctrl)->u.type_gen >> S_RSPD_GEN) == iq->gen));
2183 }
2184
2185 static inline void
2186 iq_next(struct sge_iq *iq)
2187 {
2188 iq->cdesc = (void *) ((uintptr_t)iq->cdesc + iq->esize);
2189 if (++iq->cidx == iq->qsize - 1) {
2190 iq->cidx = 0;
2191 iq->gen ^= 1;
2192 iq->cdesc = iq->desc;
2193 }
2194 }
2195
2196 /*
2197 * Fill up the freelist by upto nbufs and maybe ring its doorbell.
2198 *
2199 * Returns non-zero to indicate that it should be added to the list of starving
2200 * freelists.
2201 */
2202 static int
2203 refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
2204 {
2205 uint64_t *d = &fl->desc[fl->pidx];
2206 struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
2207
2208 FL_LOCK_ASSERT_OWNED(fl);
2209 ASSERT(nbufs >= 0);
2210
2211 if (nbufs > fl->needed)
2212 nbufs = fl->needed;
2213
2214 while (nbufs--) {
2215 if (sd->rxb != NULL) {
2216 if (sd->rxb->ref_cnt == 1) {
2217 /*
2218 * Buffer is available for recycling. Two ways
2219 * this can happen:
2220 *
2221 * a) All the packets DMA'd into it last time
2222 * around were within the rx_copy_threshold
2223 * and no part of the buffer was ever passed
2224 * up (ref_cnt never went over 1).
2225 *
2226 * b) Packets DMA'd into the buffer were passed
2227 * up but have all been freed by the upper
2228 * layers by now (ref_cnt went over 1 but is
2229 * now back to 1).
2230 *
2231 * Either way the bus address in the descriptor
2232 * ring is already valid.
2233 */
2234 ASSERT(*d == cpu_to_be64(sd->rxb->ba));
2235 d++;
2236 goto recycled;
2237 } else {
2238 /*
2239 * Buffer still in use and we need a
2240 * replacement. But first release our reference
2241 * on the existing buffer.
2242 */
2243 rxbuf_free(sd->rxb);
2244 }
2245 }
2246
2247 sd->rxb = rxbuf_alloc(sc->sge.rxbuf_cache, KM_NOSLEEP, 1);
2248 if (sd->rxb == NULL)
2249 break;
2250 *d++ = cpu_to_be64(sd->rxb->ba);
2251
2252 recycled: fl->pending++;
2253 sd++;
2254 fl->needed--;
2255 if (++fl->pidx == fl->cap) {
2256 fl->pidx = 0;
2257 sd = fl->sdesc;
2258 d = fl->desc;
2259 }
2260 }
2261
2262 if (fl->pending >= 8)
2263 ring_fl_db(sc, fl);
2264
2265 return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
2266 }
2267
2268 #ifndef TAILQ_FOREACH_SAFE
2269 #define TAILQ_FOREACH_SAFE(var, head, field, tvar) \
2270 for ((var) = TAILQ_FIRST((head)); \
2271 (var) && ((tvar) = TAILQ_NEXT((var), field), 1); \
2272 (var) = (tvar))
2273 #endif
2274
2275 /*
2276 * Attempt to refill all starving freelists.
2277 */
2278 static void
2279 refill_sfl(void *arg)
2280 {
2281 struct adapter *sc = arg;
2282 struct sge_fl *fl, *fl_temp;
2283
2284 mutex_enter(&sc->sfl_lock);
2285 TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
2286 FL_LOCK(fl);
2287 (void) refill_fl(sc, fl, 64);
2288 if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
2289 TAILQ_REMOVE(&sc->sfl, fl, link);
2290 fl->flags &= ~FL_STARVING;
2291 }
2292 FL_UNLOCK(fl);
2293 }
2294
2295 if (!TAILQ_EMPTY(&sc->sfl) != 0)
2296 sc->sfl_timer = timeout(refill_sfl, sc, drv_usectohz(100000));
2297 mutex_exit(&sc->sfl_lock);
2298 }
2299
2300 static void
2301 add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
2302 {
2303 mutex_enter(&sc->sfl_lock);
2304 FL_LOCK(fl);
2305 if ((fl->flags & FL_DOOMED) == 0) {
2306 if (TAILQ_EMPTY(&sc->sfl) != 0) {
2307 sc->sfl_timer = timeout(refill_sfl, sc,
2308 drv_usectohz(100000));
2309 }
2310 fl->flags |= FL_STARVING;
2311 TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
2312 }
2313 FL_UNLOCK(fl);
2314 mutex_exit(&sc->sfl_lock);
2315 }
2316
2317 static void
2318 free_fl_bufs(struct sge_fl *fl)
2319 {
2320 struct fl_sdesc *sd;
2321 unsigned int i;
2322
2323 FL_LOCK_ASSERT_OWNED(fl);
2324
2325 for (i = 0; i < fl->cap; i++) {
2326 sd = &fl->sdesc[i];
2327
2328 if (sd->rxb != NULL) {
2329 rxbuf_free(sd->rxb);
2330 sd->rxb = NULL;
2331 }
2332 }
2333 }
2334
2335 /*
2336 * Note that fl->cidx and fl->offset are left unchanged in case of failure.
2337 */
2338 static mblk_t *
2339 get_fl_payload(struct adapter *sc, struct sge_fl *fl,
2340 uint32_t len_newbuf, int *fl_bufs_used)
2341 {
2342 struct mblk_pair frame = {0};
2343 struct rxbuf *rxb;
2344 mblk_t *m = NULL;
2345 uint_t nbuf = 0, len, copy, n;
2346 uint32_t cidx, offset, rcidx, roffset;
2347
2348 /*
2349 * The SGE won't pack a new frame into the current buffer if the entire
2350 * payload doesn't fit in the remaining space. Move on to the next buf
2351 * in that case.
2352 */
2353 rcidx = fl->cidx;
2354 roffset = fl->offset;
2355 if (fl->offset > 0 && len_newbuf & F_RSPD_NEWBUF) {
2356 fl->offset = 0;
2357 if (++fl->cidx == fl->cap)
2358 fl->cidx = 0;
2359 nbuf++;
2360 }
2361 cidx = fl->cidx;
2362 offset = fl->offset;
2363
2364 len = G_RSPD_LEN(len_newbuf); /* pktshift + payload length */
2365 copy = (len <= fl->copy_threshold);
2366 if (copy != 0) {
2367 frame.head = m = allocb(len, BPRI_HI);
2368 if (m == NULL) {
2369 fl->allocb_fail++;
2370 cmn_err(CE_WARN,"%s: mbuf allocation failure "
2371 "count = %llu", __func__,
2372 (unsigned long long)fl->allocb_fail);
2373 fl->cidx = rcidx;
2374 fl->offset = roffset;
2375 return (NULL);
2376 }
2377 }
2378
2379 while (len) {
2380 rxb = fl->sdesc[cidx].rxb;
2381 n = min(len, rxb->buf_size - offset);
2382
2383 (void) ddi_dma_sync(rxb->dhdl, offset, n,
2384 DDI_DMA_SYNC_FORKERNEL);
2385
2386 if (copy != 0)
2387 bcopy(rxb->va + offset, m->b_wptr, n);
2388 else {
2389 m = desballoc((unsigned char *)rxb->va + offset, n,
2390 BPRI_HI, &rxb->freefunc);
2391 if (m == NULL) {
2392 fl->allocb_fail++;
2393 cmn_err(CE_WARN,
2394 "%s: mbuf allocation failure "
2395 "count = %llu", __func__,
2396 (unsigned long long)fl->allocb_fail);
2397 if (frame.head)
2398 freemsgchain(frame.head);
2399 fl->cidx = rcidx;
2400 fl->offset = roffset;
2401 return (NULL);
2402 }
2403 atomic_inc_uint(&rxb->ref_cnt);
2404 if (frame.head != NULL)
2405 frame.tail->b_cont = m;
2406 else
2407 frame.head = m;
2408 frame.tail = m;
2409 }
2410 m->b_wptr += n;
2411 len -= n;
2412 offset += roundup(n, sc->sge.fl_align);
2413 ASSERT(offset <= rxb->buf_size);
2414 if (offset == rxb->buf_size) {
2415 offset = 0;
2416 if (++cidx == fl->cap)
2417 cidx = 0;
2418 nbuf++;
2419 }
2420 }
2421
2422 fl->cidx = cidx;
2423 fl->offset = offset;
2424 (*fl_bufs_used) += nbuf;
2425
2426 ASSERT(frame.head != NULL);
2427 return (frame.head);
2428 }
2429
2430 /*
2431 * We'll do immediate data tx for non-LSO, but only when not coalescing. We're
2432 * willing to use upto 2 hardware descriptors which means a maximum of 96 bytes
2433 * of immediate data.
2434 */
2435 #define IMM_LEN ( \
2436 2 * EQ_ESIZE \
2437 - sizeof (struct fw_eth_tx_pkt_wr) \
2438 - sizeof (struct cpl_tx_pkt_core))
2439
2440 /*
2441 * Returns non-zero on failure, no need to cleanup anything in that case.
2442 *
2443 * Note 1: We always try to pull up the mblk if required and return E2BIG only
2444 * if this fails.
2445 *
2446 * Note 2: We'll also pullup incoming mblk if HW_LSO is set and the first mblk
2447 * does not have the TCP header in it.
2448 */
2449 static int
2450 get_frame_txinfo(struct sge_txq *txq, mblk_t **fp, struct txinfo *txinfo,
2451 int sgl_only)
2452 {
2453 uint32_t flags = 0, len, n;
2454 mblk_t *m = *fp;
2455 int rc;
2456
2457 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb and dma_hdls */
2458
2459 mac_hcksum_get(m, NULL, NULL, NULL, NULL, &flags);
2460 txinfo->flags = flags;
2461
2462 mac_lso_get(m, &txinfo->mss, &flags);
2463 txinfo->flags |= flags;
2464
2465 if (flags & HW_LSO)
2466 sgl_only = 1; /* Do not allow immediate data with LSO */
2467
2468 start: txinfo->nsegs = 0;
2469 txinfo->hdls_used = 0;
2470 txinfo->txb_used = 0;
2471 txinfo->len = 0;
2472
2473 /* total length and a rough estimate of # of segments */
2474 n = 0;
2475 for (; m; m = m->b_cont) {
2476 len = MBLKL(m);
2477 n += (len / PAGE_SIZE) + 1;
2478 txinfo->len += len;
2479 }
2480 m = *fp;
2481
2482 if (n >= TX_SGL_SEGS || (flags & HW_LSO && MBLKL(m) < 50)) {
2483 txq->pullup_early++;
2484 m = msgpullup(*fp, -1);
2485 if (m == NULL) {
2486 txq->pullup_failed++;
2487 return (E2BIG); /* (*fp) left as it was */
2488 }
2489 freemsg(*fp);
2490 *fp = m;
2491 mac_hcksum_set(m, 0, 0, 0, 0, txinfo->flags);
2492 }
2493
2494 if (txinfo->len <= IMM_LEN && !sgl_only)
2495 return (0); /* nsegs = 0 tells caller to use imm. tx */
2496
2497 if (txinfo->len <= txq->copy_threshold &&
2498 copy_into_txb(txq, m, txinfo->len, txinfo) == 0)
2499 goto done;
2500
2501 for (; m; m = m->b_cont) {
2502
2503 len = MBLKL(m);
2504
2505 /* Use tx copy buffer if this mblk is small enough */
2506 if (len <= txq->copy_threshold &&
2507 copy_into_txb(txq, m, len, txinfo) == 0)
2508 continue;
2509
2510 /* Add DMA bindings for this mblk to the SGL */
2511 rc = add_mblk(txq, txinfo, m, len);
2512
2513 if (rc == E2BIG ||
2514 (txinfo->nsegs == TX_SGL_SEGS && m->b_cont)) {
2515
2516 txq->pullup_late++;
2517 m = msgpullup(*fp, -1);
2518 if (m != NULL) {
2519 free_txinfo_resources(txq, txinfo);
2520 freemsg(*fp);
2521 *fp = m;
2522 mac_hcksum_set(m, 0, 0, 0, 0,
2523 txinfo->flags);
2524 goto start;
2525 }
2526
2527 txq->pullup_failed++;
2528 rc = E2BIG;
2529 }
2530
2531 if (rc != 0) {
2532 free_txinfo_resources(txq, txinfo);
2533 return (rc);
2534 }
2535 }
2536
2537 ASSERT(txinfo->nsegs > 0 && txinfo->nsegs <= TX_SGL_SEGS);
2538
2539 done:
2540
2541 /*
2542 * Store the # of flits required to hold this frame's SGL in nflits. An
2543 * SGL has a (ULPTX header + len0, addr0) tuple optionally followed by
2544 * multiple (len0 + len1, addr0, addr1) tuples. If addr1 is not used
2545 * then len1 must be set to 0.
2546 */
2547 n = txinfo->nsegs - 1;
2548 txinfo->nflits = (3 * n) / 2 + (n & 1) + 2;
2549 if (n & 1)
2550 txinfo->sgl.sge[n / 2].len[1] = cpu_to_be32(0);
2551
2552 txinfo->sgl.cmd_nsge = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_DSGL) |
2553 V_ULPTX_NSGE(txinfo->nsegs));
2554
2555 return (0);
2556 }
2557
2558 static inline int
2559 fits_in_txb(struct sge_txq *txq, int len, int *waste)
2560 {
2561 if (txq->txb_avail < len)
2562 return (0);
2563
2564 if (txq->txb_next + len <= txq->txb_size) {
2565 *waste = 0;
2566 return (1);
2567 }
2568
2569 *waste = txq->txb_size - txq->txb_next;
2570
2571 return (txq->txb_avail - *waste < len ? 0 : 1);
2572 }
2573
2574 #define TXB_CHUNK 64
2575
2576 /*
2577 * Copies the specified # of bytes into txq's tx copy buffer and updates txinfo
2578 * and txq to indicate resources used. Caller has to make sure that those many
2579 * bytes are available in the mblk chain (b_cont linked).
2580 */
2581 static inline int
2582 copy_into_txb(struct sge_txq *txq, mblk_t *m, int len, struct txinfo *txinfo)
2583 {
2584 int waste, n;
2585
2586 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate txb */
2587
2588 if (!fits_in_txb(txq, len, &waste)) {
2589 txq->txb_full++;
2590 return (ENOMEM);
2591 }
2592
2593 if (waste != 0) {
2594 ASSERT((waste & (TXB_CHUNK - 1)) == 0);
2595 txinfo->txb_used += waste;
2596 txq->txb_avail -= waste;
2597 txq->txb_next = 0;
2598 }
2599
2600 for (n = 0; n < len; m = m->b_cont) {
2601 bcopy(m->b_rptr, txq->txb_va + txq->txb_next + n, MBLKL(m));
2602 n += MBLKL(m);
2603 }
2604
2605 add_seg(txinfo, txq->txb_ba + txq->txb_next, len);
2606
2607 n = roundup(len, TXB_CHUNK);
2608 txinfo->txb_used += n;
2609 txq->txb_avail -= n;
2610 txq->txb_next += n;
2611 ASSERT(txq->txb_next <= txq->txb_size);
2612 if (txq->txb_next == txq->txb_size)
2613 txq->txb_next = 0;
2614
2615 return (0);
2616 }
2617
2618 static inline void
2619 add_seg(struct txinfo *txinfo, uint64_t ba, uint32_t len)
2620 {
2621 ASSERT(txinfo->nsegs < TX_SGL_SEGS); /* must have room */
2622
2623 if (txinfo->nsegs != 0) {
2624 int idx = txinfo->nsegs - 1;
2625 txinfo->sgl.sge[idx / 2].len[idx & 1] = cpu_to_be32(len);
2626 txinfo->sgl.sge[idx / 2].addr[idx & 1] = cpu_to_be64(ba);
2627 } else {
2628 txinfo->sgl.len0 = cpu_to_be32(len);
2629 txinfo->sgl.addr0 = cpu_to_be64(ba);
2630 }
2631 txinfo->nsegs++;
2632 }
2633
2634 /*
2635 * This function cleans up any partially allocated resources when it fails so
2636 * there's nothing for the caller to clean up in that case.
2637 *
2638 * EIO indicates permanent failure. Caller should drop the frame containing
2639 * this mblk and continue.
2640 *
2641 * E2BIG indicates that the SGL length for this mblk exceeds the hardware
2642 * limit. Caller should pull up the frame before trying to send it out.
2643 * (This error means our pullup_early heuristic did not work for this frame)
2644 *
2645 * ENOMEM indicates a temporary shortage of resources (DMA handles, other DMA
2646 * resources, etc.). Caller should suspend the tx queue and wait for reclaim to
2647 * free up resources.
2648 */
2649 static inline int
2650 add_mblk(struct sge_txq *txq, struct txinfo *txinfo, mblk_t *m, int len)
2651 {
2652 ddi_dma_handle_t dhdl;
2653 ddi_dma_cookie_t cookie;
2654 uint_t ccount = 0;
2655 int rc;
2656
2657 TXQ_LOCK_ASSERT_OWNED(txq); /* will manipulate dhdls */
2658
2659 if (txq->tx_dhdl_avail == 0) {
2660 txq->dma_hdl_failed++;
2661 return (ENOMEM);
2662 }
2663
2664 dhdl = txq->tx_dhdl[txq->tx_dhdl_pidx];
2665 rc = ddi_dma_addr_bind_handle(dhdl, NULL, (caddr_t)m->b_rptr, len,
2666 DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, NULL, &cookie,
2667 &ccount);
2668 if (rc != DDI_DMA_MAPPED) {
2669 txq->dma_map_failed++;
2670
2671 ASSERT(rc != DDI_DMA_INUSE && rc != DDI_DMA_PARTIAL_MAP);
2672
2673 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EIO);
2674 }
2675
2676 if (ccount + txinfo->nsegs > TX_SGL_SEGS) {
2677 (void) ddi_dma_unbind_handle(dhdl);
2678 return (E2BIG);
2679 }
2680
2681 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2682 while (--ccount) {
2683 ddi_dma_nextcookie(dhdl, &cookie);
2684 add_seg(txinfo, cookie.dmac_laddress, cookie.dmac_size);
2685 }
2686
2687 if (++txq->tx_dhdl_pidx == txq->tx_dhdl_total)
2688 txq->tx_dhdl_pidx = 0;
2689 txq->tx_dhdl_avail--;
2690 txinfo->hdls_used++;
2691
2692 return (0);
2693 }
2694
2695 /*
2696 * Releases all the txq resources used up in the specified txinfo.
2697 */
2698 static void
2699 free_txinfo_resources(struct sge_txq *txq, struct txinfo *txinfo)
2700 {
2701 int n;
2702
2703 TXQ_LOCK_ASSERT_OWNED(txq); /* dhdls, txb */
2704
2705 n = txinfo->txb_used;
2706 if (n > 0) {
2707 txq->txb_avail += n;
2708 if (n <= txq->txb_next)
2709 txq->txb_next -= n;
2710 else {
2711 n -= txq->txb_next;
2712 txq->txb_next = txq->txb_size - n;
2713 }
2714 }
2715
2716 for (n = txinfo->hdls_used; n > 0; n--) {
2717 if (txq->tx_dhdl_pidx > 0)
2718 txq->tx_dhdl_pidx--;
2719 else
2720 txq->tx_dhdl_pidx = txq->tx_dhdl_total - 1;
2721 txq->tx_dhdl_avail++;
2722 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_pidx]);
2723 }
2724 }
2725
2726 /*
2727 * Returns 0 to indicate that m has been accepted into a coalesced tx work
2728 * request. It has either been folded into txpkts or txpkts was flushed and m
2729 * has started a new coalesced work request (as the first frame in a fresh
2730 * txpkts).
2731 *
2732 * Returns non-zero to indicate a failure - caller is responsible for
2733 * transmitting m, if there was anything in txpkts it has been flushed.
2734 */
2735 static int
2736 add_to_txpkts(struct sge_txq *txq, struct txpkts *txpkts, mblk_t *m,
2737 struct txinfo *txinfo)
2738 {
2739 struct sge_eq *eq = &txq->eq;
2740 int can_coalesce;
2741 struct tx_sdesc *txsd;
2742 uint8_t flits;
2743
2744 TXQ_LOCK_ASSERT_OWNED(txq);
2745
2746 if (txpkts->npkt > 0) {
2747 flits = TXPKTS_PKT_HDR + txinfo->nflits;
2748 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2749 txpkts->nflits + flits <= TX_WR_FLITS &&
2750 txpkts->nflits + flits <= eq->avail * 8 &&
2751 txpkts->plen + txinfo->len < 65536;
2752
2753 if (can_coalesce != 0) {
2754 txpkts->tail->b_next = m;
2755 txpkts->tail = m;
2756 txpkts->npkt++;
2757 txpkts->nflits += flits;
2758 txpkts->plen += txinfo->len;
2759
2760 txsd = &txq->sdesc[eq->pidx];
2761 txsd->txb_used += txinfo->txb_used;
2762 txsd->hdls_used += txinfo->hdls_used;
2763
2764 return (0);
2765 }
2766
2767 /*
2768 * Couldn't coalesce m into txpkts. The first order of business
2769 * is to send txpkts on its way. Then we'll revisit m.
2770 */
2771 write_txpkts_wr(txq, txpkts);
2772 }
2773
2774 /*
2775 * Check if we can start a new coalesced tx work request with m as
2776 * the first packet in it.
2777 */
2778
2779 ASSERT(txpkts->npkt == 0);
2780 ASSERT(txinfo->len < 65536);
2781
2782 flits = TXPKTS_WR_HDR + txinfo->nflits;
2783 can_coalesce = (txinfo->flags & HW_LSO) == 0 &&
2784 flits <= eq->avail * 8 && flits <= TX_WR_FLITS;
2785
2786 if (can_coalesce == 0)
2787 return (EINVAL);
2788
2789 /*
2790 * Start a fresh coalesced tx WR with m as the first frame in it.
2791 */
2792 txpkts->tail = m;
2793 txpkts->npkt = 1;
2794 txpkts->nflits = flits;
2795 txpkts->flitp = &eq->desc[eq->pidx].flit[2];
2796 txpkts->plen = txinfo->len;
2797
2798 txsd = &txq->sdesc[eq->pidx];
2799 txsd->m = m;
2800 txsd->txb_used = txinfo->txb_used;
2801 txsd->hdls_used = txinfo->hdls_used;
2802
2803 return (0);
2804 }
2805
2806 /*
2807 * Note that write_txpkts_wr can never run out of hardware descriptors (but
2808 * write_txpkt_wr can). add_to_txpkts ensures that a frame is accepted for
2809 * coalescing only if sufficient hardware descriptors are available.
2810 */
2811 static void
2812 write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
2813 {
2814 struct sge_eq *eq = &txq->eq;
2815 struct fw_eth_tx_pkts_wr *wr;
2816 struct tx_sdesc *txsd;
2817 uint32_t ctrl;
2818 uint16_t ndesc;
2819
2820 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2821
2822 ndesc = howmany(txpkts->nflits, 8);
2823
2824 wr = (void *)&eq->desc[eq->pidx];
2825 wr->op_pkd = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) |
2826 V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */
2827 ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
2828 if (eq->avail == ndesc)
2829 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2830 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2831 wr->plen = cpu_to_be16(txpkts->plen);
2832 wr->npkt = txpkts->npkt;
2833 wr->r3 = wr->type = 0;
2834
2835 /* Everything else already written */
2836
2837 txsd = &txq->sdesc[eq->pidx];
2838 txsd->desc_used = ndesc;
2839
2840 txq->txb_used += txsd->txb_used / TXB_CHUNK;
2841 txq->hdl_used += txsd->hdls_used;
2842
2843 ASSERT(eq->avail >= ndesc);
2844
2845 eq->pending += ndesc;
2846 eq->avail -= ndesc;
2847 eq->pidx += ndesc;
2848 if (eq->pidx >= eq->cap)
2849 eq->pidx -= eq->cap;
2850
2851 txq->txpkts_pkts += txpkts->npkt;
2852 txq->txpkts_wrs++;
2853 txpkts->npkt = 0; /* emptied */
2854 }
2855
2856 static int
2857 write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, mblk_t *m,
2858 struct txinfo *txinfo)
2859 {
2860 struct sge_eq *eq = &txq->eq;
2861 struct fw_eth_tx_pkt_wr *wr;
2862 struct cpl_tx_pkt_core *cpl;
2863 uint32_t ctrl; /* used in many unrelated places */
2864 uint64_t ctrl1;
2865 int nflits, ndesc;
2866 struct tx_sdesc *txsd;
2867 caddr_t dst;
2868
2869 TXQ_LOCK_ASSERT_OWNED(txq); /* pidx, avail */
2870
2871 /*
2872 * Do we have enough flits to send this frame out?
2873 */
2874 ctrl = sizeof (struct cpl_tx_pkt_core);
2875 if (txinfo->flags & HW_LSO) {
2876 nflits = TXPKT_LSO_WR_HDR;
2877 ctrl += sizeof(struct cpl_tx_pkt_lso_core);
2878 } else
2879 nflits = TXPKT_WR_HDR;
2880 if (txinfo->nsegs > 0)
2881 nflits += txinfo->nflits;
2882 else {
2883 nflits += howmany(txinfo->len, 8);
2884 ctrl += txinfo->len;
2885 }
2886 ndesc = howmany(nflits, 8);
2887 if (ndesc > eq->avail)
2888 return (ENOMEM);
2889
2890 /* Firmware work request header */
2891 wr = (void *)&eq->desc[eq->pidx];
2892 wr->op_immdlen = cpu_to_be32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
2893 V_FW_WR_IMMDLEN(ctrl));
2894 ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
2895 if (eq->avail == ndesc)
2896 ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
2897 wr->equiq_to_len16 = cpu_to_be32(ctrl);
2898 wr->r3 = 0;
2899
2900 if (txinfo->flags & HW_LSO) {
2901 struct cpl_tx_pkt_lso_core *lso = (void *)(wr + 1);
2902 char *p = (void *)m->b_rptr;
2903 ctrl = V_LSO_OPCODE((u32)CPL_TX_PKT_LSO) | F_LSO_FIRST_SLICE |
2904 F_LSO_LAST_SLICE;
2905
2906 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2907 if (((struct ether_header *)p)->ether_type ==
2908 htons(ETHERTYPE_VLAN)) {
2909 ctrl |= V_LSO_ETHHDR_LEN(1);
2910 p += sizeof (struct ether_vlan_header);
2911 } else
2912 p += sizeof (struct ether_header);
2913
2914 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2915 ctrl |= V_LSO_IPHDR_LEN(IPH_HDR_LENGTH(p) / 4);
2916 /* LINTED: E_BAD_PTR_CAST_ALIGN for IPH_HDR_LENGTH() */
2917 p += IPH_HDR_LENGTH(p);
2918 ctrl |= V_LSO_TCPHDR_LEN(TCP_HDR_LENGTH((tcph_t *)p) / 4);
2919
2920 lso->lso_ctrl = cpu_to_be32(ctrl);
2921 lso->ipid_ofst = cpu_to_be16(0);
2922 lso->mss = cpu_to_be16(txinfo->mss);
2923 lso->seqno_offset = cpu_to_be32(0);
2924 if (is_t4(pi->adapter->params.chip))
2925 lso->len = cpu_to_be32(txinfo->len);
2926 else
2927 lso->len = cpu_to_be32(V_LSO_T5_XFER_SIZE(txinfo->len));
2928
2929 cpl = (void *)(lso + 1);
2930
2931 txq->tso_wrs++;
2932 } else
2933 cpl = (void *)(wr + 1);
2934
2935 /* Checksum offload */
2936 ctrl1 = 0;
2937 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
2938 ctrl1 |= F_TXPKT_IPCSUM_DIS;
2939 if (!(txinfo->flags & HCK_FULLCKSUM))
2940 ctrl1 |= F_TXPKT_L4CSUM_DIS;
2941 if (ctrl1 == 0)
2942 txq->txcsum++; /* some hardware assistance provided */
2943
2944 /* CPL header */
2945 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
2946 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
2947 cpl->pack = 0;
2948 cpl->len = cpu_to_be16(txinfo->len);
2949 cpl->ctrl1 = cpu_to_be64(ctrl1);
2950
2951 /* Software descriptor */
2952 txsd = &txq->sdesc[eq->pidx];
2953 txsd->m = m;
2954 txsd->txb_used = txinfo->txb_used;
2955 txsd->hdls_used = txinfo->hdls_used;
2956 /* LINTED: E_ASSIGN_NARROW_CONV */
2957 txsd->desc_used = ndesc;
2958
2959 txq->txb_used += txinfo->txb_used / TXB_CHUNK;
2960 txq->hdl_used += txinfo->hdls_used;
2961
2962 eq->pending += ndesc;
2963 eq->avail -= ndesc;
2964 eq->pidx += ndesc;
2965 if (eq->pidx >= eq->cap)
2966 eq->pidx -= eq->cap;
2967
2968 /* SGL */
2969 dst = (void *)(cpl + 1);
2970 if (txinfo->nsegs > 0) {
2971 txq->sgl_wrs++;
2972 copy_to_txd(eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
2973
2974 /* Need to zero-pad to a 16 byte boundary if not on one */
2975 if ((uintptr_t)dst & 0xf)
2976 /* LINTED: E_BAD_PTR_CAST_ALIGN */
2977 *(uint64_t *)dst = 0;
2978
2979 } else {
2980 txq->imm_wrs++;
2981 #ifdef DEBUG
2982 ctrl = txinfo->len;
2983 #endif
2984 for (; m; m = m->b_cont) {
2985 copy_to_txd(eq, (void *)m->b_rptr, &dst, MBLKL(m));
2986 #ifdef DEBUG
2987 ctrl -= MBLKL(m);
2988 #endif
2989 }
2990 ASSERT(ctrl == 0);
2991 }
2992
2993 txq->txpkt_wrs++;
2994 return (0);
2995 }
2996
2997 static inline void
2998 write_ulp_cpl_sgl(struct port_info *pi, struct sge_txq *txq,
2999 struct txpkts *txpkts, struct txinfo *txinfo)
3000 {
3001 struct ulp_txpkt *ulpmc;
3002 struct ulptx_idata *ulpsc;
3003 struct cpl_tx_pkt_core *cpl;
3004 uintptr_t flitp, start, end;
3005 uint64_t ctrl;
3006 caddr_t dst;
3007
3008 ASSERT(txpkts->npkt > 0);
3009
3010 start = (uintptr_t)txq->eq.desc;
3011 end = (uintptr_t)txq->eq.spg;
3012
3013 /* Checksum offload */
3014 ctrl = 0;
3015 if (!(txinfo->flags & HCK_IPV4_HDRCKSUM))
3016 ctrl |= F_TXPKT_IPCSUM_DIS;
3017 if (!(txinfo->flags & HCK_FULLCKSUM))
3018 ctrl |= F_TXPKT_L4CSUM_DIS;
3019 if (ctrl == 0)
3020 txq->txcsum++; /* some hardware assistance provided */
3021
3022 /*
3023 * The previous packet's SGL must have ended at a 16 byte boundary (this
3024 * is required by the firmware/hardware). It follows that flitp cannot
3025 * wrap around between the ULPTX master command and ULPTX subcommand (8
3026 * bytes each), and that it can not wrap around in the middle of the
3027 * cpl_tx_pkt_core either.
3028 */
3029 flitp = (uintptr_t)txpkts->flitp;
3030 ASSERT((flitp & 0xf) == 0);
3031
3032 /* ULP master command */
3033 ulpmc = (void *)flitp;
3034 ulpmc->cmd_dest = htonl(V_ULPTX_CMD(ULP_TX_PKT) | V_ULP_TXPKT_DEST(0));
3035 ulpmc->len = htonl(howmany(sizeof (*ulpmc) + sizeof (*ulpsc) +
3036 sizeof (*cpl) + 8 * txinfo->nflits, 16));
3037
3038 /* ULP subcommand */
3039 ulpsc = (void *)(ulpmc + 1);
3040 ulpsc->cmd_more = cpu_to_be32(V_ULPTX_CMD((u32)ULP_TX_SC_IMM) |
3041 F_ULP_TX_SC_MORE);
3042 ulpsc->len = cpu_to_be32(sizeof (struct cpl_tx_pkt_core));
3043
3044 flitp += sizeof (*ulpmc) + sizeof (*ulpsc);
3045 if (flitp == end)
3046 flitp = start;
3047
3048 /* CPL_TX_PKT */
3049 cpl = (void *)flitp;
3050 cpl->ctrl0 = cpu_to_be32(V_TXPKT_OPCODE(CPL_TX_PKT) |
3051 V_TXPKT_INTF(pi->tx_chan) | V_TXPKT_PF(pi->adapter->pf));
3052 cpl->pack = 0;
3053 cpl->len = cpu_to_be16(txinfo->len);
3054 cpl->ctrl1 = cpu_to_be64(ctrl);
3055
3056 flitp += sizeof (*cpl);
3057 if (flitp == end)
3058 flitp = start;
3059
3060 /* SGL for this frame */
3061 dst = (caddr_t)flitp;
3062 copy_to_txd(&txq->eq, (void *)&txinfo->sgl, &dst, txinfo->nflits * 8);
3063 flitp = (uintptr_t)dst;
3064
3065 /* Zero pad and advance to a 16 byte boundary if not already at one. */
3066 if (flitp & 0xf) {
3067
3068 /* no matter what, flitp should be on an 8 byte boundary */
3069 ASSERT((flitp & 0x7) == 0);
3070
3071 *(uint64_t *)flitp = 0;
3072 flitp += sizeof (uint64_t);
3073 txpkts->nflits++;
3074 }
3075
3076 if (flitp == end)
3077 flitp = start;
3078
3079 txpkts->flitp = (void *)flitp;
3080 }
3081
3082 static inline void
3083 copy_to_txd(struct sge_eq *eq, caddr_t from, caddr_t *to, int len)
3084 {
3085 if ((uintptr_t)(*to) + len <= (uintptr_t)eq->spg) {
3086 bcopy(from, *to, len);
3087 (*to) += len;
3088 } else {
3089 int portion = (uintptr_t)eq->spg - (uintptr_t)(*to);
3090
3091 bcopy(from, *to, portion);
3092 from += portion;
3093 portion = len - portion; /* remaining */
3094 bcopy(from, (void *)eq->desc, portion);
3095 (*to) = (caddr_t)eq->desc + portion;
3096 }
3097 }
3098
3099 static inline void
3100 ring_tx_db(struct adapter *sc, struct sge_eq *eq)
3101 {
3102 int val, db_mode;
3103 u_int db = eq->doorbells;
3104
3105 if (eq->pending > 1)
3106 db &= ~DOORBELL_WCWR;
3107
3108 if (eq->pending > eq->pidx) {
3109 int offset = eq->cap - (eq->pending - eq->pidx);
3110
3111 /* pidx has wrapped around since last doorbell */
3112
3113 (void) ddi_dma_sync(eq->desc_dhdl,
3114 offset * sizeof (struct tx_desc), 0,
3115 DDI_DMA_SYNC_FORDEV);
3116 (void) ddi_dma_sync(eq->desc_dhdl,
3117 0, eq->pidx * sizeof (struct tx_desc),
3118 DDI_DMA_SYNC_FORDEV);
3119 } else if (eq->pending > 0) {
3120 (void) ddi_dma_sync(eq->desc_dhdl,
3121 (eq->pidx - eq->pending) * sizeof (struct tx_desc),
3122 eq->pending * sizeof (struct tx_desc),
3123 DDI_DMA_SYNC_FORDEV);
3124 }
3125
3126 membar_producer();
3127
3128 if (is_t4(sc->params.chip))
3129 val = V_PIDX(eq->pending);
3130 else
3131 val = V_PIDX_T5(eq->pending);
3132
3133 db_mode = (1 << (ffs(db) - 1));
3134 switch (db_mode) {
3135 case DOORBELL_UDB:
3136 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3137 break;
3138
3139 case DOORBELL_WCWR:
3140 {
3141 volatile uint64_t *dst, *src;
3142 int i;
3143 /*
3144 * Queues whose 128B doorbell segment fits in
3145 * the page do not use relative qid
3146 * (udb_qid is always 0). Only queues with
3147 * doorbell segments can do WCWR.
3148 */
3149 ASSERT(eq->udb_qid == 0 && eq->pending == 1);
3150
3151 dst = (volatile void *)((uintptr_t)eq->udb +
3152 UDBS_WR_OFFSET - UDBS_DB_OFFSET);
3153 i = eq->pidx ? eq->pidx - 1 : eq->cap - 1;
3154 src = (void *)&eq->desc[i];
3155 while (src != (void *)&eq->desc[i + 1])
3156 *dst++ = *src++;
3157 membar_producer();
3158 break;
3159 }
3160
3161 case DOORBELL_UDBWC:
3162 *eq->udb = LE_32(V_QID(eq->udb_qid) | val);
3163 membar_producer();
3164 break;
3165
3166 case DOORBELL_KDB:
3167 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL),
3168 V_QID(eq->cntxt_id) | val);
3169 break;
3170 }
3171
3172 eq->pending = 0;
3173 }
3174
3175 static int
3176 reclaim_tx_descs(struct sge_txq *txq, int howmany)
3177 {
3178 struct tx_sdesc *txsd;
3179 uint_t cidx, can_reclaim, reclaimed, txb_freed, hdls_freed;
3180 struct sge_eq *eq = &txq->eq;
3181
3182 EQ_LOCK_ASSERT_OWNED(eq);
3183
3184 cidx = eq->spg->cidx; /* stable snapshot */
3185 cidx = be16_to_cpu(cidx);
3186
3187 if (cidx >= eq->cidx)
3188 can_reclaim = cidx - eq->cidx;
3189 else
3190 can_reclaim = cidx + eq->cap - eq->cidx;
3191
3192 if (can_reclaim == 0)
3193 return (0);
3194
3195 txb_freed = hdls_freed = reclaimed = 0;
3196 do {
3197 int ndesc;
3198
3199 txsd = &txq->sdesc[eq->cidx];
3200 ndesc = txsd->desc_used;
3201
3202 /* Firmware doesn't return "partial" credits. */
3203 ASSERT(can_reclaim >= ndesc);
3204
3205 /*
3206 * We always keep mblk around, even for immediate data. If mblk
3207 * is NULL, this has to be the software descriptor for a credit
3208 * flush work request.
3209 */
3210 if (txsd->m != NULL)
3211 freemsgchain(txsd->m);
3212 #ifdef DEBUG
3213 else {
3214 ASSERT(txsd->txb_used == 0);
3215 ASSERT(txsd->hdls_used == 0);
3216 ASSERT(ndesc == 1);
3217 }
3218 #endif
3219
3220 txb_freed += txsd->txb_used;
3221 hdls_freed += txsd->hdls_used;
3222 reclaimed += ndesc;
3223
3224 eq->cidx += ndesc;
3225 if (eq->cidx >= eq->cap)
3226 eq->cidx -= eq->cap;
3227
3228 can_reclaim -= ndesc;
3229
3230 } while (can_reclaim && reclaimed < howmany);
3231
3232 eq->avail += reclaimed;
3233 ASSERT(eq->avail < eq->cap); /* avail tops out at (cap - 1) */
3234
3235 txq->txb_avail += txb_freed;
3236
3237 txq->tx_dhdl_avail += hdls_freed;
3238 ASSERT(txq->tx_dhdl_avail <= txq->tx_dhdl_total);
3239 for (; hdls_freed; hdls_freed--) {
3240 (void) ddi_dma_unbind_handle(txq->tx_dhdl[txq->tx_dhdl_cidx]);
3241 if (++txq->tx_dhdl_cidx == txq->tx_dhdl_total)
3242 txq->tx_dhdl_cidx = 0;
3243 }
3244
3245 return (reclaimed);
3246 }
3247
3248 static void
3249 write_txqflush_wr(struct sge_txq *txq)
3250 {
3251 struct sge_eq *eq = &txq->eq;
3252 struct fw_eq_flush_wr *wr;
3253 struct tx_sdesc *txsd;
3254
3255 EQ_LOCK_ASSERT_OWNED(eq);
3256 ASSERT(eq->avail > 0);
3257
3258 wr = (void *)&eq->desc[eq->pidx];
3259 bzero(wr, sizeof (*wr));
3260 wr->opcode = FW_EQ_FLUSH_WR;
3261 wr->equiq_to_len16 = cpu_to_be32(V_FW_WR_LEN16(sizeof (*wr) / 16) |
3262 F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
3263
3264 txsd = &txq->sdesc[eq->pidx];
3265 txsd->m = NULL;
3266 txsd->txb_used = 0;
3267 txsd->hdls_used = 0;
3268 txsd->desc_used = 1;
3269
3270 eq->pending++;
3271 eq->avail--;
3272 if (++eq->pidx == eq->cap)
3273 eq->pidx = 0;
3274 }
3275
3276 static int
3277 t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3278 {
3279 bool csum_ok;
3280 uint16_t err_vec;
3281 struct sge_rxq *rxq = (void *)iq;
3282 struct mblk_pair chain = {0};
3283 struct adapter *sc = iq->adapter;
3284 const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
3285
3286 iq->intr_next = iq->intr_params;
3287
3288 m->b_rptr += sc->sge.pktshift;
3289
3290 /* Compressed error vector is enabled for T6 only */
3291 if (sc->params.tp.rx_pkt_encap)
3292 /* It is enabled only in T6 config file */
3293 err_vec = G_T6_COMPR_RXERR_VEC(ntohs(cpl->err_vec));
3294 else
3295 err_vec = ntohs(cpl->err_vec);
3296
3297 csum_ok = cpl->csum_calc && !err_vec;
3298 /* TODO: what about cpl->ip_frag? */
3299 if (csum_ok && !cpl->ip_frag) {
3300 mac_hcksum_set(m, 0, 0, 0, 0xffff,
3301 HCK_FULLCKSUM_OK | HCK_FULLCKSUM |
3302 HCK_IPV4_HDRCKSUM_OK);
3303 rxq->rxcsum++;
3304 }
3305
3306 /* Add to the chain that we'll send up */
3307 if (chain.head != NULL)
3308 chain.tail->b_next = m;
3309 else
3310 chain.head = m;
3311 chain.tail = m;
3312
3313 t4_mac_rx(rxq->port, rxq, chain.head);
3314
3315 rxq->rxpkts++;
3316 rxq->rxbytes += be16_to_cpu(cpl->len);
3317 return (0);
3318 }
3319
3320 #define FL_HW_IDX(idx) ((idx) >> 3)
3321
3322 static inline void
3323 ring_fl_db(struct adapter *sc, struct sge_fl *fl)
3324 {
3325 int desc_start, desc_last, ndesc;
3326 uint32_t v = sc->params.arch.sge_fl_db ;
3327
3328 ndesc = FL_HW_IDX(fl->pending);
3329
3330 /* Hold back one credit if pidx = cidx */
3331 if (FL_HW_IDX(fl->pidx) == FL_HW_IDX(fl->cidx))
3332 ndesc--;
3333
3334 /*
3335 * There are chances of ndesc modified above (to avoid pidx = cidx).
3336 * If there is nothing to post, return.
3337 */
3338 if (ndesc <= 0)
3339 return;
3340
3341 desc_last = FL_HW_IDX(fl->pidx);
3342
3343 if (fl->pidx < fl->pending) {
3344 /* There was a wrap */
3345 desc_start = FL_HW_IDX(fl->pidx + fl->cap - fl->pending);
3346
3347 /* From desc_start to the end of list */
3348 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE, 0,
3349 DDI_DMA_SYNC_FORDEV);
3350
3351 /* From start of list to the desc_last */
3352 if (desc_last != 0)
3353 (void) ddi_dma_sync(fl->dhdl, 0, desc_last *
3354 RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3355 } else {
3356 /* There was no wrap, sync from start_desc to last_desc */
3357 desc_start = FL_HW_IDX(fl->pidx - fl->pending);
3358 (void) ddi_dma_sync(fl->dhdl, desc_start * RX_FL_ESIZE,
3359 ndesc * RX_FL_ESIZE, DDI_DMA_SYNC_FORDEV);
3360 }
3361
3362 if (is_t4(sc->params.chip))
3363 v |= V_PIDX(ndesc);
3364 else
3365 v |= V_PIDX_T5(ndesc);
3366 v |= V_QID(fl->cntxt_id) | V_PIDX(ndesc);
3367
3368 membar_producer();
3369
3370 t4_write_reg(sc, MYPF_REG(A_SGE_PF_KDOORBELL), v);
3371
3372 /*
3373 * Update pending count:
3374 * Deduct the number of descriptors posted
3375 */
3376 fl->pending -= ndesc * 8;
3377 }
3378
3379 static void
3380 tx_reclaim_task(void *arg)
3381 {
3382 struct sge_txq *txq = arg;
3383
3384 TXQ_LOCK(txq);
3385 reclaim_tx_descs(txq, txq->eq.qsize);
3386 TXQ_UNLOCK(txq);
3387 }
3388
3389 /* ARGSUSED */
3390 static int
3391 handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
3392 mblk_t *m)
3393 {
3394 const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
3395 unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
3396 struct adapter *sc = iq->adapter;
3397 struct sge *s = &sc->sge;
3398 struct sge_eq *eq;
3399 struct sge_txq *txq;
3400
3401 txq = (void *)s->eqmap[qid - s->eq_start];
3402 eq = &txq->eq;
3403 txq->qflush++;
3404 t4_mac_tx_update(txq->port, txq);
3405
3406 ddi_taskq_dispatch(sc->tq[eq->tx_chan], tx_reclaim_task,
3407 (void *)txq, DDI_NOSLEEP);
3408
3409 return (0);
3410 }
3411
3412 static int
3413 handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m)
3414 {
3415 struct adapter *sc = iq->adapter;
3416 const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
3417
3418 ASSERT(m == NULL);
3419
3420 if (cpl->type == FW_TYPE_RSSCPL || cpl->type == FW6_TYPE_RSSCPL) {
3421 const struct rss_header *rss2;
3422
3423 rss2 = (const struct rss_header *)&cpl->data[0];
3424 return (sc->cpl_handler[rss2->opcode](iq, rss2, m));
3425 }
3426 return (sc->fw_msg_handler[cpl->type](sc, &cpl->data[0]));
3427 }
3428
3429 int
3430 t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
3431 int flags)
3432 {
3433 int i, rc;
3434
3435 txmaps->map_total = count;
3436 txmaps->map_avail = txmaps->map_cidx = txmaps->map_pidx = 0;
3437
3438 txmaps->map = kmem_zalloc(sizeof (ddi_dma_handle_t) *
3439 txmaps->map_total, flags);
3440
3441 for (i = 0; i < count; i++) {
3442 rc = ddi_dma_alloc_handle(sc->dip, &sc->sge.dma_attr_tx,
3443 DDI_DMA_SLEEP, 0, &txmaps->map[i]);
3444 if (rc != DDI_SUCCESS) {
3445 cxgb_printf(sc->dip, CE_WARN,
3446 "%s: failed to allocate DMA handle (%d)",
3447 __func__, rc);
3448 return (rc == DDI_DMA_NORESOURCES ? ENOMEM : EINVAL);
3449 }
3450 txmaps->map_avail++;
3451 }
3452
3453 return (0);
3454 }
3455
3456 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG)
3457 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR)
3458 #define KS_U_SET(x, y) kstatp->x.value.ul = (y)
3459 #define KS_U_FROM(x, y) kstatp->x.value.ul = (y)->x
3460 #define KS_C_SET(x, ...) \
3461 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__)
3462
3463 /*
3464 * cxgbe:X:config
3465 */
3466 struct cxgbe_port_config_kstats {
3467 kstat_named_t idx;
3468 kstat_named_t nrxq;
3469 kstat_named_t ntxq;
3470 kstat_named_t first_rxq;
3471 kstat_named_t first_txq;
3472 kstat_named_t controller;
3473 kstat_named_t factory_mac_address;
3474 };
3475
3476 /*
3477 * cxgbe:X:info
3478 */
3479 struct cxgbe_port_info_kstats {
3480 kstat_named_t transceiver;
3481 kstat_named_t rx_ovflow0;
3482 kstat_named_t rx_ovflow1;
3483 kstat_named_t rx_ovflow2;
3484 kstat_named_t rx_ovflow3;
3485 kstat_named_t rx_trunc0;
3486 kstat_named_t rx_trunc1;
3487 kstat_named_t rx_trunc2;
3488 kstat_named_t rx_trunc3;
3489 kstat_named_t tx_pause;
3490 kstat_named_t rx_pause;
3491 };
3492
3493 static kstat_t *
3494 setup_port_config_kstats(struct port_info *pi)
3495 {
3496 kstat_t *ksp;
3497 struct cxgbe_port_config_kstats *kstatp;
3498 int ndata;
3499 dev_info_t *pdip = ddi_get_parent(pi->dip);
3500 uint8_t *ma = &pi->hw_addr[0];
3501
3502 ndata = sizeof (struct cxgbe_port_config_kstats) /
3503 sizeof (kstat_named_t);
3504
3505 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "config",
3506 "net", KSTAT_TYPE_NAMED, ndata, 0);
3507 if (ksp == NULL) {
3508 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3509 return (NULL);
3510 }
3511
3512 kstatp = (struct cxgbe_port_config_kstats *)ksp->ks_data;
3513
3514 KS_UINIT(idx);
3515 KS_UINIT(nrxq);
3516 KS_UINIT(ntxq);
3517 KS_UINIT(first_rxq);
3518 KS_UINIT(first_txq);
3519 KS_CINIT(controller);
3520 KS_CINIT(factory_mac_address);
3521
3522 KS_U_SET(idx, pi->port_id);
3523 KS_U_SET(nrxq, pi->nrxq);
3524 KS_U_SET(ntxq, pi->ntxq);
3525 KS_U_SET(first_rxq, pi->first_rxq);
3526 KS_U_SET(first_txq, pi->first_txq);
3527 KS_C_SET(controller, "%s%d", ddi_driver_name(pdip),
3528 ddi_get_instance(pdip));
3529 KS_C_SET(factory_mac_address, "%02X%02X%02X%02X%02X%02X",
3530 ma[0], ma[1], ma[2], ma[3], ma[4], ma[5]);
3531
3532 /* Do NOT set ksp->ks_update. These kstats do not change. */
3533
3534 /* Install the kstat */
3535 ksp->ks_private = (void *)pi;
3536 kstat_install(ksp);
3537
3538 return (ksp);
3539 }
3540
3541 static kstat_t *
3542 setup_port_info_kstats(struct port_info *pi)
3543 {
3544 kstat_t *ksp;
3545 struct cxgbe_port_info_kstats *kstatp;
3546 int ndata;
3547
3548 ndata = sizeof (struct cxgbe_port_info_kstats) / sizeof (kstat_named_t);
3549
3550 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), "info",
3551 "net", KSTAT_TYPE_NAMED, ndata, 0);
3552 if (ksp == NULL) {
3553 cxgb_printf(pi->dip, CE_WARN, "failed to initialize kstats.");
3554 return (NULL);
3555 }
3556
3557 kstatp = (struct cxgbe_port_info_kstats *)ksp->ks_data;
3558
3559 KS_CINIT(transceiver);
3560 KS_UINIT(rx_ovflow0);
3561 KS_UINIT(rx_ovflow1);
3562 KS_UINIT(rx_ovflow2);
3563 KS_UINIT(rx_ovflow3);
3564 KS_UINIT(rx_trunc0);
3565 KS_UINIT(rx_trunc1);
3566 KS_UINIT(rx_trunc2);
3567 KS_UINIT(rx_trunc3);
3568 KS_UINIT(tx_pause);
3569 KS_UINIT(rx_pause);
3570
3571 /* Install the kstat */
3572 ksp->ks_update = update_port_info_kstats;
3573 ksp->ks_private = (void *)pi;
3574 kstat_install(ksp);
3575
3576 return (ksp);
3577 }
3578
3579 static int
3580 update_port_info_kstats(kstat_t *ksp, int rw)
3581 {
3582 struct cxgbe_port_info_kstats *kstatp =
3583 (struct cxgbe_port_info_kstats *)ksp->ks_data;
3584 struct port_info *pi = ksp->ks_private;
3585 static const char *mod_str[] = { NULL, "LR", "SR", "ER", "TWINAX",
3586 "active TWINAX", "LRM" };
3587 uint32_t bgmap;
3588
3589 if (rw == KSTAT_WRITE)
3590 return (0);
3591
3592 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
3593 KS_C_SET(transceiver, "unplugged");
3594 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
3595 KS_C_SET(transceiver, "unknown");
3596 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
3597 KS_C_SET(transceiver, "unsupported");
3598 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str))
3599 KS_C_SET(transceiver, "%s", mod_str[pi->mod_type]);
3600 else
3601 KS_C_SET(transceiver, "type %d", pi->mod_type);
3602
3603 #define GET_STAT(name) t4_read_reg64(pi->adapter, \
3604 PORT_REG(pi->port_id, A_MPS_PORT_STAT_##name##_L))
3605 #define GET_STAT_COM(name) t4_read_reg64(pi->adapter, \
3606 A_MPS_STAT_##name##_L)
3607
3608 bgmap = G_NUMPORTS(t4_read_reg(pi->adapter, A_MPS_CMN_CTL));
3609 if (bgmap == 0)
3610 bgmap = (pi->port_id == 0) ? 0xf : 0;
3611 else if (bgmap == 1)
3612 bgmap = (pi->port_id < 2) ? (3 << (2 * pi->port_id)) : 0;
3613 else
3614 bgmap = 1;
3615
3616 KS_U_SET(rx_ovflow0, (bgmap & 1) ?
3617 GET_STAT_COM(RX_BG_0_MAC_DROP_FRAME) : 0);
3618 KS_U_SET(rx_ovflow1, (bgmap & 2) ?
3619 GET_STAT_COM(RX_BG_1_MAC_DROP_FRAME) : 0);
3620 KS_U_SET(rx_ovflow2, (bgmap & 4) ?
3621 GET_STAT_COM(RX_BG_2_MAC_DROP_FRAME) : 0);
3622 KS_U_SET(rx_ovflow3, (bgmap & 8) ?
3623 GET_STAT_COM(RX_BG_3_MAC_DROP_FRAME) : 0);
3624 KS_U_SET(rx_trunc0, (bgmap & 1) ?
3625 GET_STAT_COM(RX_BG_0_MAC_TRUNC_FRAME) : 0);
3626 KS_U_SET(rx_trunc1, (bgmap & 2) ?
3627 GET_STAT_COM(RX_BG_1_MAC_TRUNC_FRAME) : 0);
3628 KS_U_SET(rx_trunc2, (bgmap & 4) ?
3629 GET_STAT_COM(RX_BG_2_MAC_TRUNC_FRAME) : 0);
3630 KS_U_SET(rx_trunc3, (bgmap & 8) ?
3631 GET_STAT_COM(RX_BG_3_MAC_TRUNC_FRAME) : 0);
3632
3633 KS_U_SET(tx_pause, GET_STAT(TX_PORT_PAUSE));
3634 KS_U_SET(rx_pause, GET_STAT(RX_PORT_PAUSE));
3635
3636 return (0);
3637
3638 }
3639
3640 /*
3641 * cxgbe:X:rxqY
3642 */
3643 struct rxq_kstats {
3644 kstat_named_t rxcsum;
3645 kstat_named_t rxpkts;
3646 kstat_named_t rxbytes;
3647 kstat_named_t nomem;
3648 };
3649
3650 static kstat_t *
3651 setup_rxq_kstats(struct port_info *pi, struct sge_rxq *rxq, int idx)
3652 {
3653 struct kstat *ksp;
3654 struct rxq_kstats *kstatp;
3655 int ndata;
3656 char str[16];
3657
3658 ndata = sizeof (struct rxq_kstats) / sizeof (kstat_named_t);
3659 (void) snprintf(str, sizeof (str), "rxq%u", idx);
3660
3661 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "rxq",
3662 KSTAT_TYPE_NAMED, ndata, 0);
3663 if (ksp == NULL) {
3664 cxgb_printf(pi->dip, CE_WARN,
3665 "%s: failed to initialize rxq kstats for queue %d.",
3666 __func__, idx);
3667 return (NULL);
3668 }
3669
3670 kstatp = (struct rxq_kstats *)ksp->ks_data;
3671
3672 KS_UINIT(rxcsum);
3673 KS_UINIT(rxpkts);
3674 KS_UINIT(rxbytes);
3675 KS_UINIT(nomem);
3676
3677 ksp->ks_update = update_rxq_kstats;
3678 ksp->ks_private = (void *)rxq;
3679 kstat_install(ksp);
3680
3681 return (ksp);
3682 }
3683
3684 static int
3685 update_rxq_kstats(kstat_t *ksp, int rw)
3686 {
3687 struct rxq_kstats *kstatp = (struct rxq_kstats *)ksp->ks_data;
3688 struct sge_rxq *rxq = ksp->ks_private;
3689
3690 if (rw == KSTAT_WRITE)
3691 return (0);
3692
3693 KS_U_FROM(rxcsum, rxq);
3694 KS_U_FROM(rxpkts, rxq);
3695 KS_U_FROM(rxbytes, rxq);
3696 KS_U_FROM(nomem, rxq);
3697
3698 return (0);
3699 }
3700
3701 /*
3702 * cxgbe:X:txqY
3703 */
3704 struct txq_kstats {
3705 kstat_named_t txcsum;
3706 kstat_named_t tso_wrs;
3707 kstat_named_t imm_wrs;
3708 kstat_named_t sgl_wrs;
3709 kstat_named_t txpkt_wrs;
3710 kstat_named_t txpkts_wrs;
3711 kstat_named_t txpkts_pkts;
3712 kstat_named_t txb_used;
3713 kstat_named_t hdl_used;
3714 kstat_named_t txb_full;
3715 kstat_named_t dma_hdl_failed;
3716 kstat_named_t dma_map_failed;
3717 kstat_named_t qfull;
3718 kstat_named_t qflush;
3719 kstat_named_t pullup_early;
3720 kstat_named_t pullup_late;
3721 kstat_named_t pullup_failed;
3722 };
3723
3724 static kstat_t *
3725 setup_txq_kstats(struct port_info *pi, struct sge_txq *txq, int idx)
3726 {
3727 struct kstat *ksp;
3728 struct txq_kstats *kstatp;
3729 int ndata;
3730 char str[16];
3731
3732 ndata = sizeof (struct txq_kstats) / sizeof (kstat_named_t);
3733 (void) snprintf(str, sizeof (str), "txq%u", idx);
3734
3735 ksp = kstat_create(T4_PORT_NAME, ddi_get_instance(pi->dip), str, "txq",
3736 KSTAT_TYPE_NAMED, ndata, 0);
3737 if (ksp == NULL) {
3738 cxgb_printf(pi->dip, CE_WARN,
3739 "%s: failed to initialize txq kstats for queue %d.",
3740 __func__, idx);
3741 return (NULL);
3742 }
3743
3744 kstatp = (struct txq_kstats *)ksp->ks_data;
3745
3746 KS_UINIT(txcsum);
3747 KS_UINIT(tso_wrs);
3748 KS_UINIT(imm_wrs);
3749 KS_UINIT(sgl_wrs);
3750 KS_UINIT(txpkt_wrs);
3751 KS_UINIT(txpkts_wrs);
3752 KS_UINIT(txpkts_pkts);
3753 KS_UINIT(txb_used);
3754 KS_UINIT(hdl_used);
3755 KS_UINIT(txb_full);
3756 KS_UINIT(dma_hdl_failed);
3757 KS_UINIT(dma_map_failed);
3758 KS_UINIT(qfull);
3759 KS_UINIT(qflush);
3760 KS_UINIT(pullup_early);
3761 KS_UINIT(pullup_late);
3762 KS_UINIT(pullup_failed);
3763
3764 ksp->ks_update = update_txq_kstats;
3765 ksp->ks_private = (void *)txq;
3766 kstat_install(ksp);
3767
3768 return (ksp);
3769 }
3770
3771 static int
3772 update_txq_kstats(kstat_t *ksp, int rw)
3773 {
3774 struct txq_kstats *kstatp = (struct txq_kstats *)ksp->ks_data;
3775 struct sge_txq *txq = ksp->ks_private;
3776
3777 if (rw == KSTAT_WRITE)
3778 return (0);
3779
3780 KS_U_FROM(txcsum, txq);
3781 KS_U_FROM(tso_wrs, txq);
3782 KS_U_FROM(imm_wrs, txq);
3783 KS_U_FROM(sgl_wrs, txq);
3784 KS_U_FROM(txpkt_wrs, txq);
3785 KS_U_FROM(txpkts_wrs, txq);
3786 KS_U_FROM(txpkts_pkts, txq);
3787 KS_U_FROM(txb_used, txq);
3788 KS_U_FROM(hdl_used, txq);
3789 KS_U_FROM(txb_full, txq);
3790 KS_U_FROM(dma_hdl_failed, txq);
3791 KS_U_FROM(dma_map_failed, txq);
3792 KS_U_FROM(qfull, txq);
3793 KS_U_FROM(qflush, txq);
3794 KS_U_FROM(pullup_early, txq);
3795 KS_U_FROM(pullup_late, txq);
3796 KS_U_FROM(pullup_failed, txq);
3797
3798 return (0);
3799 }