sys/dev/pci/cxgb_lro.c

   1 /**************************************************************************
   2
   3 Copyright (c) 2007, Chelsio Inc.
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions are met:
   8
   9  1. Redistributions of source code must retain the above copyright notice,
  10     this list of conditions and the following disclaimer.
  11
  12 2. Neither the name of the Chelsio Corporation nor the names of its
  13     contributors may be used to endorse or promote products derived from
  14     this software without specific prior written permission.
  15
  16 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  19 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  20 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  21 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  22 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  23 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  24 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  25 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  26 POSSIBILITY OF SUCH DAMAGE.
  27
  28 ***************************************************************************/
  29
  30 #include <sys/cdefs.h>
  31 #ifdef __NetBSD__
  32 __KERNEL_RCSID(0, "$NetBSD: cxgb_lro.c,v 1.5 2007/12/15 00:39:29 perry Exp $");
  33 #endif
  34 #ifdef __FreeBSD__
  35 __FBSDID("$FreeBSD: src/sys/dev/cxgb/cxgb_lro.c,v 1.8 2007/08/25 21:07:36 kmacy Exp $");
  36 #endif
  37
  38
  39 #include <sys/param.h>
  40 #include <sys/systm.h>
  41 #include <sys/kernel.h>
  42 #ifdef __FreeBSD__
  43 #include <sys/module.h>
  44 #include <sys/bus.h>
  45 #endif
  46 #include <sys/conf.h>
  47 #include <machine/bus.h>
  48 #ifdef __FreeBSD__
  49 #include <machine/resource.h>
  50 #include <sys/bus_dma.h>
  51 #include <sys/rman.h>
  52 #endif
  53 #include <sys/queue.h>
  54 #ifdef __FreeBSD__
  55 #include <sys/taskqueue.h>
  56 #endif
  57
  58 #include <netinet/in_systm.h>
  59 #include <netinet/in.h>
  60 #include <netinet/ip.h>
  61 #include <netinet/tcp.h>
  62
  63
  64 #ifdef CONFIG_DEFINED
  65 #ifdef __FreeBSD__
  66 #include <cxgb_include.h>
  67 #else
  68 #include <dev/cxgb/cxgb_include.h>
  69 #endif
  70
  71 #include <machine/in_cksum.h>
  72 #endif
  73
  74 #ifdef __NetBSD__
  75 #include "cxgb_include.h"
  76 #endif
  77
  78 #ifndef M_LRO
  79 #define M_LRO    0x0200
  80 #endif
  81
  82 #ifdef DEBUG
  83 #define MBUF_HEADER_CHECK(m) do { \
  84     if ((m->m_len == 0) || (m->m_pkthdr.len == 0)   \
  85         || ((m->m_flags & M_PKTHDR) == 0))              \
  86         panic("lro_flush_session - mbuf len=%d pktlen=%d flags=0x%x\n", \
  87             m->m_len, m->m_pkthdr.len, m->m_flags); \
  88     if ((m->m_flags & M_PKTHDR) == 0)               \
  89         panic("first mbuf is not packet header - flags=0x%x\n", \
  90             m->m_flags);  \
  91     if ((m->m_len < ETHER_HDR_LEN) || (m->m_pkthdr.len < ETHER_HDR_LEN)) \
  92         panic("packet too small len=%d pktlen=%d\n", \
  93             m->m_len, m->m_pkthdr.len);\
  94 } while (0)
  95 #else
  96 #define MBUF_HEADER_CHECK(m)
  97 #endif
  98
  99 #define IPH_OFFSET (2 + sizeof (struct cpl_rx_pkt) + ETHER_HDR_LEN)
 100 #define LRO_SESSION_IDX_HINT_HASH(hash) (hash & (MAX_LRO_SES - 1))
 101 #define LRO_IDX_INC(idx) idx = (idx + 1) & (MAX_LRO_SES - 1)
 102
 103 static __inline int
 104 lro_match(struct mbuf *m, struct ip *ih, struct tcphdr *th)
 105 {
 106     struct ip *sih = (struct ip *)(mtod(m, uint8_t *) + IPH_OFFSET);
 107     struct tcphdr *sth = (struct tcphdr *) (sih + 1);
 108
 109     return (th->th_sport == sth->th_sport &&
 110         th->th_dport == sth->th_dport &&
 111         ih->ip_src.s_addr == sih->ip_src.s_addr &&
 112         ih->ip_dst.s_addr == sih->ip_dst.s_addr);
 113 }
 114
 115 static __inline struct t3_lro_session *
 116 lro_lookup(struct lro_state *l, int idx, struct ip *ih, struct tcphdr *th)
 117 {
 118     struct t3_lro_session *s = NULL;
 119     int active = l->nactive;
 120
 121     while (active) {
 122         s = &l->sess[idx];
 123         if (s->head) {
 124             if (lro_match(s->head, ih, th))
 125                 break;
 126             active--;
 127         }
 128         LRO_IDX_INC(idx);
 129     }
 130
 131     return (s);
 132 }
 133
 134 static __inline int
 135 can_lro_packet(struct cpl_rx_pkt *cpl, unsigned int rss_hi)
 136 {
 137     struct ether_header *eh = (struct ether_header *)(cpl + 1);
 138     struct ip *ih = (struct ip *)(eh + 1);
 139
 140     /*
 141      * XXX VLAN support?
 142      */
 143     if (__predict_false(G_HASHTYPE(ntohl(rss_hi)) != RSS_HASH_4_TUPLE ||
 144              (*((uint8_t *)cpl + 1) & 0x90) != 0x10 ||
 145              cpl->csum != 0xffff || eh->ether_type != ntohs(ETHERTYPE_IP) ||
 146              ih->ip_hl != (sizeof (*ih) >> 2))) {
 147         return 0;
 148     }
 149
 150     return 1;
 151 }
 152
 153 static int
 154 can_lro_tcpsegment(struct tcphdr *th)
 155 {
 156     int olen = (th->th_off << 2) - sizeof (*th);
 157     u8 control_bits = *((u8 *)th + 13);
 158
 159     if (__predict_false((control_bits & 0xB7) != 0x10))
 160         goto no_lro;
 161
 162     if (olen) {
 163         uint32_t *ptr = (u32 *)(th + 1);
 164         if (__predict_false(olen != TCPOLEN_TSTAMP_APPA ||
 165                  *ptr != ntohl((TCPOPT_NOP << 24) |
 166                        (TCPOPT_NOP << 16) |
 167                        (TCPOPT_TIMESTAMP << 8) |
 168                         TCPOLEN_TIMESTAMP)))
 169             goto no_lro;
 170     }
 171
 172     return 1;
 173
 174  no_lro:
 175     return 0;
 176 }
 177
 178 static __inline void
 179 lro_new_session_init(struct t3_lro_session *s, struct mbuf *m)
 180 {
 181     struct ip *ih = (struct ip *)(mtod(m, uint8_t *) + IPH_OFFSET);
 182     struct tcphdr *th = (struct tcphdr *) (ih + 1);
 183     int ip_len = ntohs(ih->ip_len);
 184
 185     DPRINTF("%s(s=%p, m=%p)\n", __func__, s, m);
 186
 187     s->head = m;
 188
 189     MBUF_HEADER_CHECK(m);
 190     s->ip_len = ip_len;
 191     s->seq = ntohl(th->th_seq) + ip_len - sizeof(*ih) - (th->th_off << 2);
 192
 193 }
 194
 195 static void
 196 lro_flush_session(struct sge_qset *qs, struct t3_lro_session *s, struct mbuf *m)
 197 {
 198     struct lro_state *l = &qs->lro;
 199     struct mbuf *sm = s->head;
 200     struct ip *ih = (struct ip *)(mtod(sm, uint8_t *) + IPH_OFFSET);
 201
 202
 203     DPRINTF("%s(qs=%p, s=%p, ", __func__,
 204         qs, s);
 205
 206     if (m)
 207         DPRINTF("m=%p)\n", m);
 208     else
 209         DPRINTF("m=NULL)\n");
 210
 211     ih->ip_len = htons(s->ip_len);
 212     ih->ip_sum = 0;
 213     ih->ip_sum = in_cksum_hdr(ih);
 214
 215     MBUF_HEADER_CHECK(sm);
 216
 217     sm->m_flags |= M_LRO;
 218     t3_rx_eth(qs->port->adapter, &qs->rspq, sm, 2);
 219
 220     if (m) {
 221         s->head = m;
 222         lro_new_session_init(s, m);
 223     } else {
 224         s->head = NULL;
 225         l->nactive--;
 226     }
 227
 228     qs->port_stats[SGE_PSTATS_LRO_FLUSHED]++;
 229 }
 230
 231 static __inline struct t3_lro_session *
 232 lro_new_session(struct sge_qset *qs, struct mbuf *m, uint32_t rss_hash)
 233 {
 234     struct lro_state *l = &qs->lro;
 235     int idx = LRO_SESSION_IDX_HINT_HASH(rss_hash);
 236     struct t3_lro_session *s = &l->sess[idx];
 237
 238     DPRINTF("%s(qs=%p,  m=%p, rss_hash=0x%x)\n", __func__,
 239         qs, m, rss_hash);
 240
 241     if (__predict_true(!s->head))
 242         goto done;
 243
 244     if (l->nactive > MAX_LRO_SES)
 245         panic("MAX_LRO_PER_QSET exceeded");
 246
 247     if (l->nactive == MAX_LRO_SES) {
 248         lro_flush_session(qs, s, m);
 249         qs->port_stats[SGE_PSTATS_LRO_X_STREAMS]++;
 250         return s;
 251     }
 252
 253     while (1) {
 254         LRO_IDX_INC(idx);
 255         s = &l->sess[idx];
 256         if (!s->head)
 257             break;
 258     }
 259 done:
 260     lro_new_session_init(s, m);
 261     l->nactive++;
 262
 263     return s;
 264 }
 265
 266 static __inline int
 267 lro_update_session(struct t3_lro_session *s, struct mbuf *m)
 268 {
 269     struct mbuf *sm = s->head;
 270     struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(sm, uint8_t *) + 2);
 271     struct cpl_rx_pkt *ncpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + 2);
 272     struct ip *nih = (struct ip *)(mtod(m, uint8_t *) + IPH_OFFSET);
 273     struct tcphdr *th, *nth = (struct tcphdr *)(nih + 1);
 274     uint32_t seq = ntohl(nth->th_seq);
 275     int plen, tcpiphlen, olen = (nth->th_off << 2) - sizeof (*nth);
 276
 277
 278     DPRINTF("%s(s=%p,  m=%p)\n", __func__, s, m);
 279     if (cpl->vlan_valid && cpl->vlan != ncpl->vlan) {
 280         return -1;
 281     }
 282     if (__predict_false(seq != s->seq)) {
 283         DPRINTF("sequence mismatch\n");
 284         return -1;
 285     }
 286
 287     MBUF_HEADER_CHECK(sm);
 288     th = (struct tcphdr *)(mtod(sm, uint8_t *) + IPH_OFFSET + sizeof (struct ip));
 289
 290     if (olen) {
 291         uint32_t *ptr = (uint32_t *)(th + 1);
 292         uint32_t *nptr = (uint32_t *)(nth + 1);
 293
 294         if (__predict_false(ntohl(*(ptr + 1)) > ntohl(*(nptr + 1)) ||
 295                  !*(nptr + 2))) {
 296             return -1;
 297         }
 298         *(ptr + 1) = *(nptr + 1);
 299         *(ptr + 2) = *(nptr + 2);
 300     }
 301     th->th_ack = nth->th_ack;
 302     th->th_win = nth->th_win;
 303
 304     tcpiphlen = (nth->th_off << 2) + sizeof (*nih);
 305     plen = ntohs(nih->ip_len) - tcpiphlen;
 306     s->seq += plen;
 307     s->ip_len += plen;
 308     sm->m_pkthdr.len += plen;
 309
 310     /*
 311      * XXX FIX ME
 312      *
 313      *
 314      */
 315
 316 #if 0
 317     /* XXX this I *do not* understand */
 318     if (plen > skb_shinfo(s->skb)->gso_size)
 319         skb_shinfo(s->skb)->gso_size = plen;
 320 #endif
 321 #if __FreeBSD_version > 700000
 322     if (plen > sm->m_pkthdr.tso_segsz)
 323         sm->m_pkthdr.tso_segsz = plen;
 324 #endif
 325     DPRINTF("m_adj(%d)\n", (int)(IPH_OFFSET + tcpiphlen));
 326     m_adj(m, IPH_OFFSET + tcpiphlen);
 327 #if 0
 328     if (__predict_false(!skb_shinfo(s->skb)->frag_list))
 329         skb_shinfo(s->skb)->frag_list = skb;
 330
 331 #endif
 332
 333 #if 0
 334
 335     /*
 336      * XXX we really need to be able to
 337      * support vectors of buffers in FreeBSD
 338      */
 339     int nr = skb_shinfo(s->skb)->nr_frags;
 340     skb_shinfo(s->skb)->frags[nr].page = frag->page;
 341     skb_shinfo(s->skb)->frags[nr].page_offset =
 342         frag->page_offset + IPH_OFFSET + tcpiphlen;
 343     skb_shinfo(s->skb)->frags[nr].size = plen;
 344     skb_shinfo(s->skb)->nr_frags = ++nr;
 345
 346 #endif
 347     return (0);
 348 }
 349
 350 void
 351 t3_rx_eth_lro(adapter_t *adap, struct sge_rspq *rq, struct mbuf *m,
 352     int ethpad, uint32_t rss_hash, uint32_t rss_csum, int lro)
 353 {
 354     struct sge_qset *qs = rspq_to_qset(rq);
 355     struct cpl_rx_pkt *cpl = (struct cpl_rx_pkt *)(mtod(m, uint8_t *) + ethpad);
 356     struct ether_header *eh = (struct ether_header *)(cpl + 1);
 357     struct ip *ih;
 358     struct tcphdr *th;
 359     struct t3_lro_session *s = NULL;
 360
 361     if (lro == 0)
 362         goto no_lro;
 363
 364     if (!can_lro_packet(cpl, rss_csum))
 365         goto no_lro;
 366
 367     ih = (struct ip *)(eh + 1);
 368     th = (struct tcphdr *)(ih + 1);
 369
 370     s = lro_lookup(&qs->lro,
 371         LRO_SESSION_IDX_HINT_HASH(rss_hash), ih, th);
 372
 373     if (__predict_false(!can_lro_tcpsegment(th))) {
 374         goto no_lro;
 375     } else if (__predict_false(!s)) {
 376         s = lro_new_session(qs, m, rss_hash);
 377     } else {
 378         if (lro_update_session(s, m)) {
 379             lro_flush_session(qs, s, m);
 380         }
 381 #ifdef notyet
 382         if (__predict_false(s->head->m_pkthdr.len + pi->ifp->if_mtu > 65535)) {
 383             lro_flush_session(qs, s, NULL);
 384         }
 385 #endif
 386     }
 387
 388     qs->port_stats[SGE_PSTATS_LRO_QUEUED]++;
 389     return;
 390 no_lro:
 391     if (s)
 392         lro_flush_session(qs, s, NULL);
 393
 394     if (m->m_len == 0 || m->m_pkthdr.len == 0 || (m->m_flags & M_PKTHDR) == 0)
 395         DPRINTF("rx_eth_lro mbuf len=%d pktlen=%d flags=0x%x\n",
 396             m->m_len, m->m_pkthdr.len, m->m_flags);
 397
 398     t3_rx_eth(adap, rq, m, ethpad);
 399 }
 400
 401 void
 402 t3_lro_flush(adapter_t *adap, struct sge_qset *qs, struct lro_state *state)
 403 {
 404     unsigned int idx = state->active_idx;
 405
 406     while (state->nactive) {
 407         struct t3_lro_session *s = &state->sess[idx];
 408
 409         if (s->head)
 410             lro_flush_session(qs, s, NULL);
 411         LRO_IDX_INC(idx);
 412     }
 413 }