sys/netinet/tcp_lro.c

   1 /******************************************************************************
   2
   3 Copyright (c) 2007, Myricom Inc.
   4 Copyright (c) 2008, Intel Corporation.
   5 All rights reserved.
   6
   7 Redistribution and use in source and binary forms, with or without
   8 modification, are permitted provided that the following conditions are met:
   9
  10  1. Redistributions of source code must retain the above copyright notice,
  11     this list of conditions and the following disclaimer.
  12
  13  2. Neither the name of the Myricom Inc, nor the names of its
  14     contributors may be used to endorse or promote products derived from
  15     this software without specific prior written permission.
  16
  17  3. Neither the name of the Intel Corporation, nor the names of its
  18     contributors may be used to endorse or promote products derived from
  19     this software without specific prior written permission.
  20
  21 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  22 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  23 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  24 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  25 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  26 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  27 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  28 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  29 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  30 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  31 POSSIBILITY OF SUCH DAMAGE.
  32
  33 $FreeBSD$
  34 ***************************************************************************/
  35
  36 #include <sys/param.h>
  37 #include <sys/systm.h>
  38 #include <sys/endian.h>
  39 #include <sys/mbuf.h>
  40 #include <sys/kernel.h>
  41 #include <sys/socket.h>
  42
  43 #include <net/if.h>
  44 #include <net/ethernet.h>
  45 #include <net/if_media.h>
  46
  47 #include <netinet/in_systm.h>
  48 #include <netinet/in.h>
  49 #include <netinet/ip.h>
  50 #include <netinet/tcp.h>
  51 #include <netinet/tcp_lro.h>
  52
  53 #include <machine/bus.h>
  54 #include <machine/in_cksum.h>
  55
  56
  57 static uint16_t do_csum_data(uint16_t *raw, int len)
  58 {
  59         uint32_t csum;
  60         csum = 0;
  61         while (len > 0) {
  62                 csum += *raw;
  63                 raw++;
  64                 csum += *raw;
  65                 raw++;
  66                 len -= 4;
  67         }
  68         csum = (csum >> 16) + (csum & 0xffff);
  69         csum = (csum >> 16) + (csum & 0xffff);
  70         return (uint16_t)csum;
  71 }
  72
  73 /*
  74  * Allocate and init the LRO data structures
  75  */
  76 int
  77 tcp_lro_init(struct lro_ctrl *cntl)
  78 {
  79         struct lro_entry *lro;
  80         int i, error = 0;
  81
  82         SLIST_INIT(&cntl->lro_free);
  83         SLIST_INIT(&cntl->lro_active);
  84
  85         cntl->lro_bad_csum = 0;
  86         cntl->lro_queued = 0;
  87         cntl->lro_flushed = 0;
  88
  89         for (i = 0; i < LRO_ENTRIES; i++) {
  90                 lro = (struct lro_entry *) malloc(sizeof (struct lro_entry),
  91                     M_DEVBUF, M_NOWAIT | M_ZERO);
  92                 if (lro == NULL) {
  93                         if (i == 0)
  94                                 error = ENOMEM;
  95                         break;
  96                 }
  97                 cntl->lro_cnt = i;
  98                 SLIST_INSERT_HEAD(&cntl->lro_free, lro, next);
  99         }
 100
 101         return (error);
 102 }
 103
 104 void
 105 tcp_lro_free(struct lro_ctrl *cntl)
 106 {
 107         struct lro_entry *entry;
 108
 109         while (!SLIST_EMPTY(&cntl->lro_free)) {
 110                 entry = SLIST_FIRST(&cntl->lro_free);
 111                 SLIST_REMOVE_HEAD(&cntl->lro_free, next);
 112                 free(entry, M_DEVBUF);
 113         }
 114 }
 115
 116 void
 117 tcp_lro_flush(struct lro_ctrl *cntl, struct lro_entry *lro)
 118 {
 119         struct ifnet *ifp;
 120         struct ip *ip;
 121         struct tcphdr *tcp;
 122         uint32_t *ts_ptr;
 123         uint32_t tcplen, tcp_csum;
 124
 125
 126         if (lro->append_cnt) {
 127                 /* incorporate the new len into the ip header and
 128                  * re-calculate the checksum */
 129                 ip = lro->ip;
 130                 ip->ip_len = htons(lro->len - ETHER_HDR_LEN);
 131                 ip->ip_sum = 0;
 132                 ip->ip_sum = 0xffff ^
 133                         do_csum_data((uint16_t*)ip,
 134                                               sizeof (*ip));
 135
 136                 lro->m_head->m_pkthdr.csum_flags = CSUM_IP_CHECKED |
 137                         CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
 138                 lro->m_head->m_pkthdr.csum_data = 0xffff;
 139                 lro->m_head->m_pkthdr.len = lro->len;
 140
 141                 /* incorporate the latest ack into the tcp header */
 142                 tcp = (struct tcphdr *) (ip + 1);
 143                 tcp->th_ack = lro->ack_seq;
 144                 tcp->th_win = lro->window;
 145                 /* incorporate latest timestamp into the tcp header */
 146                 if (lro->timestamp) {
 147                         ts_ptr = (uint32_t *)(tcp + 1);
 148                         ts_ptr[1] = htonl(lro->tsval);
 149                         ts_ptr[2] = lro->tsecr;
 150                 }
 151                 /*
 152                  * update checksum in tcp header by re-calculating the
 153                  * tcp pseudoheader checksum, and adding it to the checksum
 154                  * of the tcp payload data
 155                  */
 156                 tcp->th_sum = 0;
 157                 tcplen = lro->len - sizeof(*ip) - ETHER_HDR_LEN;
 158                 tcp_csum = lro->data_csum;
 159                 tcp_csum += in_pseudo(ip->ip_src.s_addr, ip->ip_dst.s_addr,
 160                                       htons(tcplen + IPPROTO_TCP));
 161                 tcp_csum += do_csum_data((uint16_t*)tcp,
 162                                                   tcp->th_off << 2);
 163                 tcp_csum = (tcp_csum & 0xffff) + (tcp_csum >> 16);
 164                 tcp_csum = (tcp_csum & 0xffff) + (tcp_csum >> 16);
 165                 tcp->th_sum = 0xffff ^ tcp_csum;
 166         }
 167         ifp = cntl->ifp;
 168         (*ifp->if_input)(cntl->ifp, lro->m_head);
 169         cntl->lro_queued += lro->append_cnt + 1;
 170         cntl->lro_flushed++;
 171         lro->m_head = NULL;
 172         lro->timestamp = 0;
 173         lro->append_cnt = 0;
 174         SLIST_INSERT_HEAD(&cntl->lro_free, lro, next);
 175 }
 176
 177 int
 178 tcp_lro_rx(struct lro_ctrl *cntl, struct mbuf *m_head, uint32_t csum)
 179 {
 180         struct ether_header *eh;
 181         struct ip *ip;
 182         struct tcphdr *tcp;
 183         uint32_t *ts_ptr;
 184         struct mbuf *m_nxt, *m_tail;
 185         struct lro_entry *lro;
 186         int hlen, ip_len, tcp_hdr_len, tcp_data_len, tot_len;
 187         int opt_bytes, trim, csum_flags;
 188         uint32_t seq, tmp_csum, device_mtu;
 189
 190
 191         eh = mtod(m_head, struct ether_header *);
 192         if (eh->ether_type != htons(ETHERTYPE_IP))
 193                 return 1;
 194         ip = (struct ip *) (eh + 1);
 195         if (ip->ip_p != IPPROTO_TCP)
 196                 return 1;
 197
 198         /* ensure there are no options */
 199         if ((ip->ip_hl << 2) != sizeof (*ip))
 200                 return -1;
 201
 202         /* .. and the packet is not fragmented */
 203         if (ip->ip_off & htons(IP_MF|IP_OFFMASK))
 204                 return -1;
 205
 206         /* verify that the IP header checksum is correct */
 207         csum_flags = m_head->m_pkthdr.csum_flags;
 208         if (csum_flags & CSUM_IP_CHECKED) {
 209                 if (__predict_false((csum_flags & CSUM_IP_VALID) == 0)) {
 210                         cntl->lro_bad_csum++;
 211                         return -1;
 212                 }
 213         } else {
 214                 tmp_csum = do_csum_data((uint16_t *)ip, sizeof (*ip));
 215                 if (__predict_false((tmp_csum ^ 0xffff) != 0)) {
 216                         cntl->lro_bad_csum++;
 217                         return -1;
 218                 }
 219         }
 220
 221         /* find the TCP header */
 222         tcp = (struct tcphdr *) (ip + 1);
 223
 224         /* Get the TCP checksum if we dont have it */
 225         if (!csum)
 226                 csum = tcp->th_sum;
 227
 228         /* ensure no bits set besides ack or psh */
 229         if ((tcp->th_flags & ~(TH_ACK | TH_PUSH)) != 0)
 230                 return -1;
 231
 232         /* check for timestamps. Since the only option we handle are
 233            timestamps, we only have to handle the simple case of
 234            aligned timestamps */
 235
 236         opt_bytes = (tcp->th_off << 2) - sizeof (*tcp);
 237         tcp_hdr_len =  sizeof (*tcp) + opt_bytes;
 238         ts_ptr = (uint32_t *)(tcp + 1);
 239         if (opt_bytes != 0) {
 240                 if (__predict_false(opt_bytes != TCPOLEN_TSTAMP_APPA) ||
 241                     (*ts_ptr !=  ntohl(TCPOPT_NOP<<24|TCPOPT_NOP<<16|
 242                     TCPOPT_TIMESTAMP<<8|TCPOLEN_TIMESTAMP)))
 243                         return -1;
 244         }
 245
 246         ip_len = ntohs(ip->ip_len);
 247         tcp_data_len = ip_len - (tcp->th_off << 2) - sizeof (*ip);
 248
 249
 250         /*
 251          * If frame is padded beyond the end of the IP packet,
 252          * then we must trim the extra bytes off the end.
 253          */
 254         tot_len = m_head->m_pkthdr.len;
 255         trim = tot_len - (ip_len + ETHER_HDR_LEN);
 256         if (trim != 0) {
 257                 if (trim < 0) {
 258                         /* truncated packet */
 259                         return -1;
 260                 }
 261                 m_adj(m_head, -trim);
 262                 tot_len = m_head->m_pkthdr.len;
 263         }
 264
 265         m_nxt = m_head;
 266         m_tail = NULL; /* -Wuninitialized */
 267         while (m_nxt != NULL) {
 268                 m_tail = m_nxt;
 269                 m_nxt = m_tail->m_next;
 270         }
 271
 272         hlen = ip_len + ETHER_HDR_LEN - tcp_data_len;
 273         seq = ntohl(tcp->th_seq);
 274
 275         SLIST_FOREACH(lro, &cntl->lro_active, next) {
 276                 if (lro->source_port == tcp->th_sport &&
 277                     lro->dest_port == tcp->th_dport &&
 278                     lro->source_ip == ip->ip_src.s_addr &&
 279                     lro->dest_ip == ip->ip_dst.s_addr) {
 280                         /* Try to append it */
 281
 282                         if (__predict_false(seq != lro->next_seq)) {
 283                                 /* out of order packet */
 284                                 SLIST_REMOVE(&cntl->lro_active, lro,
 285                                              lro_entry, next);
 286                                 tcp_lro_flush(cntl, lro);
 287                                 return -1;
 288                         }
 289
 290                         if (opt_bytes) {
 291                                 uint32_t tsval = ntohl(*(ts_ptr + 1));
 292                                 /* make sure timestamp values are increasing */
 293                                 if (__predict_false(lro->tsval > tsval ||
 294                                              *(ts_ptr + 2) == 0)) {
 295                                         return -1;
 296                                 }
 297                                 lro->tsval = tsval;
 298                                 lro->tsecr = *(ts_ptr + 2);
 299                         }
 300
 301                         lro->next_seq += tcp_data_len;
 302                         lro->ack_seq = tcp->th_ack;
 303                         lro->window = tcp->th_win;
 304                         lro->append_cnt++;
 305                         if (tcp_data_len == 0) {
 306                                 m_freem(m_head);
 307                                 return 0;
 308                         }
 309                         /* subtract off the checksum of the tcp header
 310                          * from the hardware checksum, and add it to the
 311                          * stored tcp data checksum.  Byteswap the checksum
 312                          * if the total length so far is odd
 313                          */
 314                         tmp_csum = do_csum_data((uint16_t*)tcp,
 315                                                          tcp_hdr_len);
 316                         csum = csum + (tmp_csum ^ 0xffff);
 317                         csum = (csum & 0xffff) + (csum >> 16);
 318                         csum = (csum & 0xffff) + (csum >> 16);
 319                         if (lro->len & 0x1) {
 320                                 /* Odd number of bytes so far, flip bytes */
 321                                 csum = ((csum << 8) | (csum >> 8)) & 0xffff;
 322                         }
 323                         csum = csum + lro->data_csum;
 324                         csum = (csum & 0xffff) + (csum >> 16);
 325                         csum = (csum & 0xffff) + (csum >> 16);
 326                         lro->data_csum = csum;
 327
 328                         lro->len += tcp_data_len;
 329
 330                         /* adjust mbuf so that m->m_data points to
 331                            the first byte of the payload */
 332                         m_adj(m_head, hlen);
 333                         /* append mbuf chain */
 334                         lro->m_tail->m_next = m_head;
 335                         /* advance the last pointer */
 336                         lro->m_tail = m_tail;
 337                         /* flush packet if required */
 338                         device_mtu = cntl->ifp->if_mtu;
 339                         if (lro->len > (65535 - device_mtu)) {
 340                                 SLIST_REMOVE(&cntl->lro_active, lro,
 341                                              lro_entry, next);
 342                                 tcp_lro_flush(cntl, lro);
 343                         }
 344                         return 0;
 345                 }
 346         }
 347
 348         if (SLIST_EMPTY(&cntl->lro_free))
 349             return -1;
 350
 351         /* start a new chain */
 352         lro = SLIST_FIRST(&cntl->lro_free);
 353         SLIST_REMOVE_HEAD(&cntl->lro_free, next);
 354         SLIST_INSERT_HEAD(&cntl->lro_active, lro, next);
 355         lro->source_port = tcp->th_sport;
 356         lro->dest_port = tcp->th_dport;
 357         lro->source_ip = ip->ip_src.s_addr;
 358         lro->dest_ip = ip->ip_dst.s_addr;
 359         lro->next_seq = seq + tcp_data_len;
 360         lro->mss = tcp_data_len;
 361         lro->ack_seq = tcp->th_ack;
 362         lro->window = tcp->th_win;
 363
 364         /* save the checksum of just the TCP payload by
 365          * subtracting off the checksum of the TCP header from
 366          * the entire hardware checksum
 367          * Since IP header checksum is correct, checksum over
 368          * the IP header is -0.  Substracting -0 is unnecessary.
 369          */
 370         tmp_csum = do_csum_data((uint16_t*)tcp, tcp_hdr_len);
 371         csum = csum + (tmp_csum ^ 0xffff);
 372         csum = (csum & 0xffff) + (csum >> 16);
 373         csum = (csum & 0xffff) + (csum >> 16);
 374         lro->data_csum = csum;
 375
 376         lro->ip = ip;
 377         /* record timestamp if it is present */
 378         if (opt_bytes) {
 379                 lro->timestamp = 1;
 380                 lro->tsval = ntohl(*(ts_ptr + 1));
 381                 lro->tsecr = *(ts_ptr + 2);
 382         }
 383         lro->len = tot_len;
 384         lro->m_head = m_head;
 385         lro->m_tail = m_tail;
 386         return 0;
 387 }