?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Expand PMF_FN_* macros.
[netbsd-mini2440.git] / sys / netinet / tcp_output.c
blob2e980b4d117f5541376c9d93231a9ff535977365
1 /* $NetBSD: tcp_output.c,v 1.167 2008/04/28 20:24:09 martin Exp $ */
2
3 /*
4 * Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. Neither the name of the project nor the names of its contributors
16 * may be used to endorse or promote products derived from this software
17 * without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 /*
33 * @(#)COPYRIGHT 1.1 (NRL) 17 January 1995
34 *
35 * NRL grants permission for redistribution and use in source and binary
36 * forms, with or without modification, of the software and documentation
37 * created at NRL provided that the following conditions are met:
38 *
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 * 3. All advertising materials mentioning features or use of this software
45 * must display the following acknowledgements:
46 * This product includes software developed by the University of
47 * California, Berkeley and its contributors.
48 * This product includes software developed at the Information
49 * Technology Division, US Naval Research Laboratory.
50 * 4. Neither the name of the NRL nor the names of its contributors
51 * may be used to endorse or promote products derived from this software
52 * without specific prior written permission.
53 *
54 * THE SOFTWARE PROVIDED BY NRL IS PROVIDED BY NRL AND CONTRIBUTORS ``AS
55 * IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
57 * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NRL OR
58 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
59 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
60 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
61 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
62 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
63 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
64 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
65 *
66 * The views and conclusions contained in the software and documentation
67 * are those of the authors and should not be interpreted as representing
68 * official policies, either expressed or implied, of the US Naval
69 * Research Laboratory (NRL).
70 */
71
72 /*-
73 * Copyright (c) 1997, 1998, 2001, 2005, 2006 The NetBSD Foundation, Inc.
74 * All rights reserved.
75 *
76 * This code is derived from software contributed to The NetBSD Foundation
77 * by Jason R. Thorpe and Kevin M. Lahey of the Numerical Aerospace Simulation
78 * Facility, NASA Ames Research Center.
79 * This code is derived from software contributed to The NetBSD Foundation
80 * by Charles M. Hannum.
81 * This code is derived from software contributed to The NetBSD Foundation
82 * by Rui Paulo.
83 *
84 * Redistribution and use in source and binary forms, with or without
85 * modification, are permitted provided that the following conditions
86 * are met:
87 * 1. Redistributions of source code must retain the above copyright
88 * notice, this list of conditions and the following disclaimer.
89 * 2. Redistributions in binary form must reproduce the above copyright
90 * notice, this list of conditions and the following disclaimer in the
91 * documentation and/or other materials provided with the distribution.
92 *
93 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
94 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
95 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
96 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
97 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
98 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
99 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
100 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
101 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
102 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
103 * POSSIBILITY OF SUCH DAMAGE.
104 */
105
106 /*
107 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
108 * The Regents of the University of California. All rights reserved.
109 *
110 * Redistribution and use in source and binary forms, with or without
111 * modification, are permitted provided that the following conditions
112 * are met:
113 * 1. Redistributions of source code must retain the above copyright
114 * notice, this list of conditions and the following disclaimer.
115 * 2. Redistributions in binary form must reproduce the above copyright
116 * notice, this list of conditions and the following disclaimer in the
117 * documentation and/or other materials provided with the distribution.
118 * 3. Neither the name of the University nor the names of its contributors
119 * may be used to endorse or promote products derived from this software
120 * without specific prior written permission.
121 *
122 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
123 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
124 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
125 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
126 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
127 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
128 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
129 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
130 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
131 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
132 * SUCH DAMAGE.
133 *
134 * @(#)tcp_output.c 8.4 (Berkeley) 5/24/95
135 */
136
137 #include <sys/cdefs.h>
138 __KERNEL_RCSID(0, "$NetBSD: tcp_output.c,v 1.167 2008/04/28 20:24:09 martin Exp $");
139
140 #include "opt_inet.h"
141 #include "opt_ipsec.h"
142 #include "opt_tcp_debug.h"
143
144 #include <sys/param.h>
145 #include <sys/systm.h>
146 #include <sys/malloc.h>
147 #include <sys/mbuf.h>
148 #include <sys/protosw.h>
149 #include <sys/socket.h>
150 #include <sys/socketvar.h>
151 #include <sys/errno.h>
152 #include <sys/domain.h>
153 #include <sys/kernel.h>
154 #ifdef TCP_SIGNATURE
155 #include <sys/md5.h>
156 #endif
157
158 #include <net/if.h>
159 #include <net/route.h>
160
161 #include <netinet/in.h>
162 #include <netinet/in_systm.h>
163 #include <netinet/ip.h>
164 #include <netinet/in_pcb.h>
165 #include <netinet/ip_var.h>
166
167 #ifdef INET6
168 #ifndef INET
169 #include <netinet/in.h>
170 #endif
171 #include <netinet/ip6.h>
172 #include <netinet6/in6_var.h>
173 #include <netinet6/ip6_var.h>
174 #include <netinet6/in6_pcb.h>
175 #include <netinet6/nd6.h>
176 #endif
177
178 #ifdef FAST_IPSEC
179 #include <netipsec/ipsec.h>
180 #include <netipsec/key.h>
181 #ifdef INET6
182 #include <netipsec/ipsec6.h>
183 #endif
184 #endif /* FAST_IPSEC*/
185 #ifdef IPSEC
186 #include <netinet6/ipsec.h>
187 #endif
188
189 #include <netinet/tcp.h>
190 #define TCPOUTFLAGS
191 #include <netinet/tcp_fsm.h>
192 #include <netinet/tcp_seq.h>
193 #include <netinet/tcp_timer.h>
194 #include <netinet/tcp_var.h>
195 #include <netinet/tcp_private.h>
196 #include <netinet/tcp_congctl.h>
197 #include <netinet/tcpip.h>
198 #include <netinet/tcp_debug.h>
199 #include <netinet/in_offload.h>
200 #include <netinet6/in6_offload.h>
201
202 #ifdef IPSEC
203 #include <netkey/key.h>
204 #endif
205
206 #ifdef notyet
207 extern struct mbuf *m_copypack();
208 #endif
209
210 /*
211 * Knob to enable Congestion Window Monitoring, and control
212 * the burst size it allows. Default burst is 4 packets, per
213 * the Internet draft.
214 */
215 int tcp_cwm = 0;
216 int tcp_cwm_burstsize = 4;
217
218 int tcp_do_autosndbuf = 0;
219 int tcp_autosndbuf_inc = 8 * 1024;
220 int tcp_autosndbuf_max = 256 * 1024;
221
222 #ifdef TCP_OUTPUT_COUNTERS
223 #include <sys/device.h>
224
225 extern struct evcnt tcp_output_bigheader;
226 extern struct evcnt tcp_output_predict_hit;
227 extern struct evcnt tcp_output_predict_miss;
228 extern struct evcnt tcp_output_copysmall;
229 extern struct evcnt tcp_output_copybig;
230 extern struct evcnt tcp_output_refbig;
231
232 #define TCP_OUTPUT_COUNTER_INCR(ev) (ev)->ev_count++
233 #else
234
235 #define TCP_OUTPUT_COUNTER_INCR(ev) /* nothing */
236
237 #endif /* TCP_OUTPUT_COUNTERS */
238
239 static
240 #ifndef GPROF
241 inline
242 #endif
243 int
244 tcp_segsize(struct tcpcb *tp, int *txsegsizep, int *rxsegsizep,
245 bool *alwaysfragp)
246 {
247 #ifdef INET
248 struct inpcb *inp = tp->t_inpcb;
249 #endif
250 #ifdef INET6
251 struct in6pcb *in6p = tp->t_in6pcb;
252 #endif
253 struct socket *so = NULL;
254 struct rtentry *rt;
255 struct ifnet *ifp;
256 int size;
257 int hdrlen;
258 int optlen;
259
260 *alwaysfragp = false;
261
262 #ifdef DIAGNOSTIC
263 if (tp->t_inpcb && tp->t_in6pcb)
264 panic("tcp_segsize: both t_inpcb and t_in6pcb are set");
265 #endif
266 switch (tp->t_family) {
267 #ifdef INET
268 case AF_INET:
269 hdrlen = sizeof(struct ip) + sizeof(struct tcphdr);
270 break;
271 #endif
272 #ifdef INET6
273 case AF_INET6:
274 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
275 break;
276 #endif
277 default:
278 size = tcp_mssdflt;
279 goto out;
280 }
281
282 rt = NULL;
283 #ifdef INET
284 if (inp) {
285 rt = in_pcbrtentry(inp);
286 so = inp->inp_socket;
287 }
288 #endif
289 #ifdef INET6
290 if (in6p) {
291 rt = in6_pcbrtentry(in6p);
292 so = in6p->in6p_socket;
293 }
294 #endif
295 if (rt == NULL) {
296 size = tcp_mssdflt;
297 goto out;
298 }
299
300 ifp = rt->rt_ifp;
301
302 size = tcp_mssdflt;
303 if (tp->t_mtudisc && rt->rt_rmx.rmx_mtu != 0) {
304 #ifdef INET6
305 if (in6p && rt->rt_rmx.rmx_mtu < IPV6_MMTU) {
306 /*
307 * RFC2460 section 5, last paragraph: if path MTU is
308 * smaller than 1280, use 1280 as packet size and
309 * attach fragment header.
310 */
311 size = IPV6_MMTU - hdrlen - sizeof(struct ip6_frag);
312 *alwaysfragp = true;
313 } else
314 size = rt->rt_rmx.rmx_mtu - hdrlen;
315 #else
316 size = rt->rt_rmx.rmx_mtu - hdrlen;
317 #endif
318 } else if (ifp->if_flags & IFF_LOOPBACK)
319 size = ifp->if_mtu - hdrlen;
320 #ifdef INET
321 else if (inp && tp->t_mtudisc)
322 size = ifp->if_mtu - hdrlen;
323 else if (inp && in_localaddr(inp->inp_faddr))
324 size = ifp->if_mtu - hdrlen;
325 #endif
326 #ifdef INET6
327 else if (in6p) {
328 #ifdef INET
329 if (IN6_IS_ADDR_V4MAPPED(&in6p->in6p_faddr)) {
330 /* mapped addr case */
331 struct in_addr d;
332 bcopy(&in6p->in6p_faddr.s6_addr32[3], &d, sizeof(d));
333 if (tp->t_mtudisc || in_localaddr(d))
334 size = ifp->if_mtu - hdrlen;
335 } else
336 #endif
337 {
338 /*
339 * for IPv6, path MTU discovery is always turned on,
340 * or the node must use packet size <= 1280.
341 */
342 size = tp->t_mtudisc ? IN6_LINKMTU(ifp) : IPV6_MMTU;
343 size -= hdrlen;
344 }
345 }
346 #endif
347 out:
348 /*
349 * Now we must make room for whatever extra TCP/IP options are in
350 * the packet.
351 */
352 optlen = tcp_optlen(tp);
353
354 /*
355 * XXX tp->t_ourmss should have the right size, but without this code
356 * fragmentation will occur... need more investigation
357 */
358 #ifdef INET
359 if (inp) {
360 #if defined(IPSEC) || defined(FAST_IPSEC)
361 if (! IPSEC_PCB_SKIP_IPSEC(inp->inp_sp, IPSEC_DIR_OUTBOUND))
362 optlen += ipsec4_hdrsiz_tcp(tp);
363 #endif
364 optlen += ip_optlen(inp);
365 }
366 #endif
367 #ifdef INET6
368 #ifdef INET
369 if (in6p && tp->t_family == AF_INET) {
370 #if defined(IPSEC) || defined(FAST_IPSEC)
371 if (! IPSEC_PCB_SKIP_IPSEC(in6p->in6p_sp, IPSEC_DIR_OUTBOUND))
372 optlen += ipsec4_hdrsiz_tcp(tp);
373 #endif
374 /* XXX size -= ip_optlen(in6p); */
375 } else
376 #endif
377 if (in6p && tp->t_family == AF_INET6) {
378 #if defined(IPSEC) || defined(FAST_IPSEC)
379 if (! IPSEC_PCB_SKIP_IPSEC(in6p->in6p_sp, IPSEC_DIR_OUTBOUND))
380 optlen += ipsec6_hdrsiz_tcp(tp);
381 #endif
382 optlen += ip6_optlen(in6p);
383 }
384 #endif
385 size -= optlen;
386
387 /* there may not be any room for data if mtu is too small */
388 if (size < 0)
389 return (EMSGSIZE);
390
391 /*
392 * *rxsegsizep holds *estimated* inbound segment size (estimation
393 * assumes that path MTU is the same for both ways). this is only
394 * for silly window avoidance, do not use the value for other purposes.
395 *
396 * ipseclen is subtracted from both sides, this may not be right.
397 * I'm not quite sure about this (could someone comment).
398 */
399 *txsegsizep = min(tp->t_peermss - optlen, size);
400 /*
401 * Never send more than half a buffer full. This insures that we can
402 * always keep 2 packets on the wire, no matter what SO_SNDBUF is, and
403 * therefore acks will never be delayed unless we run out of data to
404 * transmit.
405 */
406 if (so)
407 *txsegsizep = min(so->so_snd.sb_hiwat >> 1, *txsegsizep);
408 *rxsegsizep = min(tp->t_ourmss - optlen, size);
409
410 if (*txsegsizep != tp->t_segsz) {
411 /*
412 * If the new segment size is larger, we don't want to
413 * mess up the congestion window, but if it is smaller
414 * we'll have to reduce the congestion window to ensure
415 * that we don't get into trouble with initial windows
416 * and the rest. In any case, if the segment size
417 * has changed, chances are the path has, too, and
418 * our congestion window will be different.
419 */
420 if (*txsegsizep < tp->t_segsz) {
421 tp->snd_cwnd = max((tp->snd_cwnd / tp->t_segsz)
422 * *txsegsizep, *txsegsizep);
423 tp->snd_ssthresh = max((tp->snd_ssthresh / tp->t_segsz)
424 * *txsegsizep, *txsegsizep);
425 }
426 tp->t_segsz = *txsegsizep;
427 }
428
429 return (0);
430 }
431
432 static
433 #ifndef GPROF
434 inline
435 #endif
436 int
437 tcp_build_datapkt(struct tcpcb *tp, struct socket *so, int off,
438 long len, int hdrlen, struct mbuf **mp)
439 {
440 struct mbuf *m, *m0;
441 uint64_t *tcps;
442
443 tcps = TCP_STAT_GETREF();
444 if (tp->t_force && len == 1)
445 tcps[TCP_STAT_SNDPROBE]++;
446 else if (SEQ_LT(tp->snd_nxt, tp->snd_max)) {
447 tcps[TCP_STAT_SNDREXMITPACK]++;
448 tcps[TCP_STAT_SNDREXMITBYTE] += len;
449 } else {
450 tcps[TCP_STAT_SNDPACK]++;
451 tcps[TCP_STAT_SNDBYTE] += len;
452 }
453 TCP_STAT_PUTREF();
454 #ifdef notyet
455 if ((m = m_copypack(so->so_snd.sb_mb, off,
456 (int)len, max_linkhdr + hdrlen)) == 0)
457 return (ENOBUFS);
458 /*
459 * m_copypack left space for our hdr; use it.
460 */
461 m->m_len += hdrlen;
462 m->m_data -= hdrlen;
463 #else
464 MGETHDR(m, M_DONTWAIT, MT_HEADER);
465 if (__predict_false(m == NULL))
466 return (ENOBUFS);
467 MCLAIM(m, &tcp_tx_mowner);
468
469 /*
470 * XXX Because other code assumes headers will fit in
471 * XXX one header mbuf.
472 *
473 * (This code should almost *never* be run.)
474 */
475 if (__predict_false((max_linkhdr + hdrlen) > MHLEN)) {
476 TCP_OUTPUT_COUNTER_INCR(&tcp_output_bigheader);
477 MCLGET(m, M_DONTWAIT);
478 if ((m->m_flags & M_EXT) == 0) {
479 m_freem(m);
480 return (ENOBUFS);
481 }
482 }
483
484 m->m_data += max_linkhdr;
485 m->m_len = hdrlen;
486
487 /*
488 * To avoid traversing the whole sb_mb chain for correct
489 * data to send, remember last sent mbuf, its offset and
490 * the sent size. When called the next time, see if the
491 * data to send is directly following the previous transfer.
492 * This is important for large TCP windows.
493 */
494 if (off == 0 || tp->t_lastm == NULL ||
495 (tp->t_lastoff + tp->t_lastlen) != off) {
496 TCP_OUTPUT_COUNTER_INCR(&tcp_output_predict_miss);
497 /*
498 * Either a new packet or a retransmit.
499 * Start from the beginning.
500 */
501 tp->t_lastm = so->so_snd.sb_mb;
502 tp->t_inoff = off;
503 } else {
504 TCP_OUTPUT_COUNTER_INCR(&tcp_output_predict_hit);
505 tp->t_inoff += tp->t_lastlen;
506 }
507
508 /* Traverse forward to next packet */
509 while (tp->t_inoff > 0) {
510 if (tp->t_lastm == NULL)
511 panic("tp->t_lastm == NULL");
512 if (tp->t_inoff < tp->t_lastm->m_len)
513 break;
514 tp->t_inoff -= tp->t_lastm->m_len;
515 tp->t_lastm = tp->t_lastm->m_next;
516 }
517
518 tp->t_lastoff = off;
519 tp->t_lastlen = len;
520 m0 = tp->t_lastm;
521 off = tp->t_inoff;
522
523 if (len <= M_TRAILINGSPACE(m)) {
524 m_copydata(m0, off, (int) len, mtod(m, char *) + hdrlen);
525 m->m_len += len;
526 TCP_OUTPUT_COUNTER_INCR(&tcp_output_copysmall);
527 } else {
528 m->m_next = m_copym(m0, off, (int) len, M_DONTWAIT);
529 if (m->m_next == NULL) {
530 m_freem(m);
531 return (ENOBUFS);
532 }
533 #ifdef TCP_OUTPUT_COUNTERS
534 if (m->m_next->m_flags & M_EXT)
535 TCP_OUTPUT_COUNTER_INCR(&tcp_output_refbig);
536 else
537 TCP_OUTPUT_COUNTER_INCR(&tcp_output_copybig);
538 #endif /* TCP_OUTPUT_COUNTERS */
539 }
540 #endif
541
542 *mp = m;
543 return (0);
544 }
545
546 /*
547 * Tcp output routine: figure out what should be sent and send it.
548 */
549 int
550 tcp_output(struct tcpcb *tp)
551 {
552 struct rtentry *rt;
553 struct socket *so;
554 struct route *ro;
555 long len, win;
556 int off, flags, error;
557 struct mbuf *m;
558 struct ip *ip;
559 #ifdef INET6
560 struct ip6_hdr *ip6;
561 #endif
562 struct tcphdr *th;
563 u_char opt[MAX_TCPOPTLEN];
564 unsigned optlen, hdrlen, packetlen;
565 unsigned int sack_numblks;
566 int idle, sendalot, txsegsize, rxsegsize;
567 int txsegsize_nosack;
568 int maxburst = TCP_MAXBURST;
569 int af; /* address family on the wire */
570 int iphdrlen;
571 int has_tso4, has_tso6;
572 int has_tso, use_tso;
573 bool alwaysfrag;
574 int sack_rxmit;
575 int sack_bytes_rxmt;
576 struct sackhole *p;
577 #ifdef TCP_SIGNATURE
578 int sigoff = 0;
579 #endif
580 uint64_t *tcps;
581
582 #ifdef DIAGNOSTIC
583 if (tp->t_inpcb && tp->t_in6pcb)
584 panic("tcp_output: both t_inpcb and t_in6pcb are set");
585 #endif
586 so = NULL;
587 ro = NULL;
588 if (tp->t_inpcb) {
589 so = tp->t_inpcb->inp_socket;
590 ro = &tp->t_inpcb->inp_route;
591 }
592 #ifdef INET6
593 else if (tp->t_in6pcb) {
594 so = tp->t_in6pcb->in6p_socket;
595 ro = &tp->t_in6pcb->in6p_route;
596 }
597 #endif
598
599 switch (af = tp->t_family) {
600 #ifdef INET
601 case AF_INET:
602 if (tp->t_inpcb)
603 break;
604 #ifdef INET6
605 /* mapped addr case */
606 if (tp->t_in6pcb)
607 break;
608 #endif
609 return (EINVAL);
610 #endif
611 #ifdef INET6
612 case AF_INET6:
613 if (tp->t_in6pcb)
614 break;
615 return (EINVAL);
616 #endif
617 default:
618 return (EAFNOSUPPORT);
619 }
620
621 if (tcp_segsize(tp, &txsegsize, &rxsegsize, &alwaysfrag))
622 return (EMSGSIZE);
623
624 idle = (tp->snd_max == tp->snd_una);
625
626 /*
627 * Determine if we can use TCP segmentation offload:
628 * - If we're using IPv4
629 * - If there is not an IPsec policy that prevents it
630 * - If the interface can do it
631 */
632 has_tso4 = has_tso6 = false;
633 #if defined(INET)
634 has_tso4 = tp->t_inpcb != NULL &&
635 #if defined(IPSEC) || defined(FAST_IPSEC)
636 IPSEC_PCB_SKIP_IPSEC(tp->t_inpcb->inp_sp,
637 IPSEC_DIR_OUTBOUND) &&
638 #endif
639 (rt = rtcache_validate(&tp->t_inpcb->inp_route)) != NULL &&
640 (rt->rt_ifp->if_capenable & IFCAP_TSOv4) != 0;
641 #endif /* defined(INET) */
642 #if defined(INET6)
643 has_tso6 = tp->t_in6pcb != NULL &&
644 #if defined(IPSEC) || defined(FAST_IPSEC)
645 IPSEC_PCB_SKIP_IPSEC(tp->t_in6pcb->in6p_sp,
646 IPSEC_DIR_OUTBOUND) &&
647 #endif
648 (rt = rtcache_validate(&tp->t_in6pcb->in6p_route)) != NULL &&
649 (rt->rt_ifp->if_capenable & IFCAP_TSOv6) != 0;
650 #endif /* defined(INET6) */
651 has_tso = (has_tso4 || has_tso6) && !alwaysfrag;
652
653 /*
654 * Restart Window computation. From draft-floyd-incr-init-win-03:
655 *
656 * Optionally, a TCP MAY set the restart window to the
657 * minimum of the value used for the initial window and
658 * the current value of cwnd (in other words, using a
659 * larger value for the restart window should never increase
660 * the size of cwnd).
661 */
662 if (tcp_cwm) {
663 /*
664 * Hughes/Touch/Heidemann Congestion Window Monitoring.
665 * Count the number of packets currently pending
666 * acknowledgement, and limit our congestion window
667 * to a pre-determined allowed burst size plus that count.
668 * This prevents bursting once all pending packets have
669 * been acknowledged (i.e. transmission is idle).
670 *
671 * XXX Link this to Initial Window?
672 */
673 tp->snd_cwnd = min(tp->snd_cwnd,
674 (tcp_cwm_burstsize * txsegsize) +
675 (tp->snd_nxt - tp->snd_una));
676 } else {
677 if (idle && (tcp_now - tp->t_rcvtime) >= tp->t_rxtcur) {
678 /*
679 * We have been idle for "a while" and no acks are
680 * expected to clock out any data we send --
681 * slow start to get ack "clock" running again.
682 */
683 int ss = tcp_init_win;
684 #ifdef INET
685 if (tp->t_inpcb &&
686 in_localaddr(tp->t_inpcb->inp_faddr))
687 ss = tcp_init_win_local;
688 #endif
689 #ifdef INET6
690 if (tp->t_in6pcb &&
691 in6_localaddr(&tp->t_in6pcb->in6p_faddr))
692 ss = tcp_init_win_local;
693 #endif
694 tp->snd_cwnd = min(tp->snd_cwnd,
695 TCP_INITIAL_WINDOW(ss, txsegsize));
696 }
697 }
698
699 txsegsize_nosack = txsegsize;
700 again:
701 use_tso = has_tso;
702 if ((tp->t_flags & (TF_ECN_SND_CWR|TF_ECN_SND_ECE)) != 0) {
703 /* don't duplicate CWR/ECE. */
704 use_tso = 0;
705 }
706 TCP_REASS_LOCK(tp);
707 sack_numblks = tcp_sack_numblks(tp);
708 if (sack_numblks) {
709 int sackoptlen;
710
711 sackoptlen = TCP_SACK_OPTLEN(sack_numblks);
712 if (sackoptlen > txsegsize_nosack) {
713 sack_numblks = 0; /* give up SACK */
714 txsegsize = txsegsize_nosack;
715 } else {
716 if ((tp->rcv_sack_flags & TCPSACK_HAVED) != 0) {
717 /* don't duplicate D-SACK. */
718 use_tso = 0;
719 }
720 txsegsize = txsegsize_nosack - sackoptlen;
721 }
722 } else {
723 txsegsize = txsegsize_nosack;
724 }
725
726 /*
727 * Determine length of data that should be transmitted, and
728 * flags that should be used. If there is some data or critical
729 * controls (SYN, RST) to send, then transmit; otherwise,
730 * investigate further.
731 *
732 * Readjust SACK information to avoid resending duplicate data.
733 */
734 if (TCP_SACK_ENABLED(tp) && SEQ_LT(tp->snd_nxt, tp->snd_max))
735 tcp_sack_adjust(tp);
736 sendalot = 0;
737 off = tp->snd_nxt - tp->snd_una;
738 win = min(tp->snd_wnd, tp->snd_cwnd);
739
740 flags = tcp_outflags[tp->t_state];
741
742 /*
743 * Send any SACK-generated retransmissions. If we're explicitly trying
744 * to send out new data (when sendalot is 1), bypass this function.
745 * If we retransmit in fast recovery mode, decrement snd_cwnd, since
746 * we're replacing a (future) new transmission with a retransmission
747 * now, and we previously incremented snd_cwnd in tcp_input().
748 */
749 /*
750 * Still in sack recovery , reset rxmit flag to zero.
751 */
752 sack_rxmit = 0;
753 sack_bytes_rxmt = 0;
754 len = 0;
755 p = NULL;
756 do {
757 long cwin;
758 if (!TCP_SACK_ENABLED(tp))
759 break;
760 if (tp->t_partialacks < 0)
761 break;
762 p = tcp_sack_output(tp, &sack_bytes_rxmt);
763 if (p == NULL)
764 break;
765
766 cwin = min(tp->snd_wnd, tp->snd_cwnd) - sack_bytes_rxmt;
767 if (cwin < 0)
768 cwin = 0;
769 /* Do not retransmit SACK segments beyond snd_recover */
770 if (SEQ_GT(p->end, tp->snd_recover)) {
771 /*
772 * (At least) part of sack hole extends beyond
773 * snd_recover. Check to see if we can rexmit data
774 * for this hole.
775 */
776 if (SEQ_GEQ(p->rxmit, tp->snd_recover)) {
777 /*
778 * Can't rexmit any more data for this hole.
779 * That data will be rexmitted in the next
780 * sack recovery episode, when snd_recover
781 * moves past p->rxmit.
782 */
783 p = NULL;
784 break;
785 }
786 /* Can rexmit part of the current hole */
787 len = ((long)ulmin(cwin, tp->snd_recover - p->rxmit));
788 } else
789 len = ((long)ulmin(cwin, p->end - p->rxmit));
790 off = p->rxmit - tp->snd_una;
791 if (off + len > so->so_snd.sb_cc) {
792 /* 1 for TH_FIN */
793 KASSERT(off + len == so->so_snd.sb_cc + 1);
794 KASSERT(p->rxmit + len == tp->snd_max);
795 len = so->so_snd.sb_cc - off;
796 }
797 if (len > 0) {
798 sack_rxmit = 1;
799 sendalot = 1;
800 }
801 } while (/*CONSTCOND*/0);
802
803 /*
804 * If in persist timeout with window of 0, send 1 byte.
805 * Otherwise, if window is small but nonzero
806 * and timer expired, we will send what we can
807 * and go to transmit state.
808 */
809 if (tp->t_force) {
810 if (win == 0) {
811 /*
812 * If we still have some data to send, then
813 * clear the FIN bit. Usually this would
814 * happen below when it realizes that we
815 * aren't sending all the data. However,
816 * if we have exactly 1 byte of unset data,
817 * then it won't clear the FIN bit below,
818 * and if we are in persist state, we wind
819 * up sending the packet without recording
820 * that we sent the FIN bit.
821 *
822 * We can't just blindly clear the FIN bit,
823 * because if we don't have any more data
824 * to send then the probe will be the FIN
825 * itself.
826 */
827 if (off < so->so_snd.sb_cc)
828 flags &= ~TH_FIN;
829 win = 1;
830 } else {
831 TCP_TIMER_DISARM(tp, TCPT_PERSIST);
832 tp->t_rxtshift = 0;
833 }
834 }
835
836 if (sack_rxmit == 0) {
837 if (TCP_SACK_ENABLED(tp) && tp->t_partialacks >= 0) {
838 long cwin;
839
840 /*
841 * We are inside of a SACK recovery episode and are
842 * sending new data, having retransmitted all the
843 * data possible in the scoreboard.
844 */
845 if (tp->snd_wnd < so->so_snd.sb_cc) {
846 len = tp->snd_wnd - off;
847 flags &= ~TH_FIN;
848 } else {
849 len = so->so_snd.sb_cc - off;
850 }
851
852 /*
853 * From FreeBSD:
854 * Don't remove this (len > 0) check !
855 * We explicitly check for len > 0 here (although it
856 * isn't really necessary), to work around a gcc
857 * optimization issue - to force gcc to compute
858 * len above. Without this check, the computation
859 * of len is bungled by the optimizer.
860 */
861 if (len > 0) {
862 cwin = tp->snd_cwnd -
863 (tp->snd_nxt - tp->sack_newdata) -
864 sack_bytes_rxmt;
865 if (cwin < 0)
866 cwin = 0;
867 if (cwin < len) {
868 len = cwin;
869 flags &= ~TH_FIN;
870 }
871 }
872 } else if (win < so->so_snd.sb_cc) {
873 len = win - off;
874 flags &= ~TH_FIN;
875 } else {
876 len = so->so_snd.sb_cc - off;
877 }
878 }
879
880 if (len < 0) {
881 /*
882 * If FIN has been sent but not acked,
883 * but we haven't been called to retransmit,
884 * len will be -1. Otherwise, window shrank
885 * after we sent into it. If window shrank to 0,
886 * cancel pending retransmit, pull snd_nxt back
887 * to (closed) window, and set the persist timer
888 * if it isn't already going. If the window didn't
889 * close completely, just wait for an ACK.
890 *
891 * If we have a pending FIN, either it has already been
892 * transmitted or it is outside the window, so drop it.
893 * If the FIN has been transmitted, but this is not a
894 * retransmission, then len must be -1. Therefore we also
895 * prevent here the sending of `gratuitous FINs'. This
896 * eliminates the need to check for that case below (e.g.
897 * to back up snd_nxt before the FIN so that the sequence
898 * number is correct).
899 */
900 len = 0;
901 flags &= ~TH_FIN;
902 if (win == 0) {
903 TCP_TIMER_DISARM(tp, TCPT_REXMT);
904 tp->t_rxtshift = 0;
905 tp->snd_nxt = tp->snd_una;
906 if (TCP_TIMER_ISARMED(tp, TCPT_PERSIST) == 0)
907 tcp_setpersist(tp);
908 }
909 }
910
911 /*
912 * Automatic sizing enables the performance of large buffers
913 * and most of the efficiency of small ones by only allocating
914 * space when it is needed.
915 *
916 * The criteria to step up the send buffer one notch are:
917 * 1. receive window of remote host is larger than send buffer
918 * (with a fudge factor of 5/4th);
919 * 2. send buffer is filled to 7/8th with data (so we actually
920 * have data to make use of it);
921 * 3. send buffer fill has not hit maximal automatic size;
922 * 4. our send window (slow start and cogestion controlled) is
923 * larger than sent but unacknowledged data in send buffer.
924 *
925 * The remote host receive window scaling factor may limit the
926 * growing of the send buffer before it reaches its allowed
927 * maximum.
928 *
929 * It scales directly with slow start or congestion window
930 * and does at most one step per received ACK. This fast
931 * scaling has the drawback of growing the send buffer beyond
932 * what is strictly necessary to make full use of a given
933 * delay*bandwith product. However testing has shown this not
934 * to be much of an problem. At worst we are trading wasting
935 * of available bandwith (the non-use of it) for wasting some
936 * socket buffer memory.
937 *
938 * TODO: Shrink send buffer during idle periods together
939 * with congestion window. Requires another timer.
940 */
941 if (tcp_do_autosndbuf && so->so_snd.sb_flags & SB_AUTOSIZE) {
942 if ((tp->snd_wnd / 4 * 5) >= so->so_snd.sb_hiwat &&
943 so->so_snd.sb_cc >= (so->so_snd.sb_hiwat / 8 * 7) &&
944 so->so_snd.sb_cc < tcp_autosndbuf_max &&
945 win >= (so->so_snd.sb_cc - (tp->snd_nxt - tp->snd_una))) {
946 if (!sbreserve(&so->so_snd,
947 min(so->so_snd.sb_hiwat + tcp_autosndbuf_inc,
948 tcp_autosndbuf_max), so))
949 so->so_snd.sb_flags &= ~SB_AUTOSIZE;
950 }
951 }
952
953 if (len > txsegsize) {
954 if (use_tso) {
955 /*
956 * Truncate TSO transfers to IP_MAXPACKET, and make
957 * sure that we send equal size transfers down the
958 * stack (rather than big-small-big-small-...).
959 */
960 #ifdef INET6
961 #if IPV6_MAXPACKET != IP_MAXPACKET
962 #error IPV6_MAXPACKET != IP_MAXPACKET
963 #endif
964 #endif
965 len = (min(len, IP_MAXPACKET) / txsegsize) * txsegsize;
966 if (len <= txsegsize) {
967 use_tso = 0;
968 }
969 } else
970 len = txsegsize;
971 flags &= ~TH_FIN;
972 sendalot = 1;
973 } else
974 use_tso = 0;
975 if (sack_rxmit) {
976 if (SEQ_LT(p->rxmit + len, tp->snd_una + so->so_snd.sb_cc))
977 flags &= ~TH_FIN;
978 }
979
980 win = sbspace(&so->so_rcv);
981
982 /*
983 * Sender silly window avoidance. If connection is idle
984 * and can send all data, a maximum segment,
985 * at least a maximum default-size segment do it,
986 * or are forced, do it; otherwise don't bother.
987 * If peer's buffer is tiny, then send
988 * when window is at least half open.
989 * If retransmitting (possibly after persist timer forced us
990 * to send into a small window), then must resend.
991 */
992 if (len) {
993 if (len >= txsegsize)
994 goto send;
995 if ((so->so_state & SS_MORETOCOME) == 0 &&
996 ((idle || tp->t_flags & TF_NODELAY) &&
997 len + off >= so->so_snd.sb_cc))
998 goto send;
999 if (tp->t_force)
1000 goto send;
1001 if (len >= tp->max_sndwnd / 2)
1002 goto send;
1003 if (SEQ_LT(tp->snd_nxt, tp->snd_max))
1004 goto send;
1005 if (sack_rxmit)
1006 goto send;
1007 }
1008
1009 /*
1010 * Compare available window to amount of window known to peer
1011 * (as advertised window less next expected input). If the
1012 * difference is at least twice the size of the largest segment
1013 * we expect to receive (i.e. two segments) or at least 50% of
1014 * the maximum possible window, then want to send a window update
1015 * to peer.
1016 */
1017 if (win > 0) {
1018 /*
1019 * "adv" is the amount we can increase the window,
1020 * taking into account that we are limited by
1021 * TCP_MAXWIN << tp->rcv_scale.
1022 */
1023 long adv = min(win, (long)TCP_MAXWIN << tp->rcv_scale) -
1024 (tp->rcv_adv - tp->rcv_nxt);
1025
1026 if (adv >= (long) (2 * rxsegsize))
1027 goto send;
1028 if (2 * adv >= (long) so->so_rcv.sb_hiwat)
1029 goto send;
1030 }
1031
1032 /*
1033 * Send if we owe peer an ACK.
1034 */
1035 if (tp->t_flags & TF_ACKNOW)
1036 goto send;
1037 if (flags & (TH_SYN|TH_FIN|TH_RST))
1038 goto send;
1039 if (SEQ_GT(tp->snd_up, tp->snd_una))
1040 goto send;
1041 /*
1042 * In SACK, it is possible for tcp_output to fail to send a segment
1043 * after the retransmission timer has been turned off. Make sure
1044 * that the retransmission timer is set.
1045 */
1046 if (TCP_SACK_ENABLED(tp) && SEQ_GT(tp->snd_max, tp->snd_una) &&
1047 !TCP_TIMER_ISARMED(tp, TCPT_REXMT) &&
1048 !TCP_TIMER_ISARMED(tp, TCPT_PERSIST)) {
1049 TCP_TIMER_ARM(tp, TCPT_REXMT, tp->t_rxtcur);
1050 goto just_return;
1051 }
1052
1053 /*
1054 * TCP window updates are not reliable, rather a polling protocol
1055 * using ``persist'' packets is used to insure receipt of window
1056 * updates. The three ``states'' for the output side are:
1057 * idle not doing retransmits or persists
1058 * persisting to move a small or zero window
1059 * (re)transmitting and thereby not persisting
1060 *
1061 * tp->t_timer[TCPT_PERSIST]
1062 * is set when we are in persist state.
1063 * tp->t_force
1064 * is set when we are called to send a persist packet.
1065 * tp->t_timer[TCPT_REXMT]
1066 * is set when we are retransmitting
1067 * The output side is idle when both timers are zero.
1068 *
1069 * If send window is too small, there is data to transmit, and no
1070 * retransmit or persist is pending, then go to persist state.
1071 * If nothing happens soon, send when timer expires:
1072 * if window is nonzero, transmit what we can,
1073 * otherwise force out a byte.
1074 */
1075 if (so->so_snd.sb_cc && TCP_TIMER_ISARMED(tp, TCPT_REXMT) == 0 &&
1076 TCP_TIMER_ISARMED(tp, TCPT_PERSIST) == 0) {
1077 tp->t_rxtshift = 0;
1078 tcp_setpersist(tp);
1079 }
1080
1081 /*
1082 * No reason to send a segment, just return.
1083 */
1084 just_return:
1085 TCP_REASS_UNLOCK(tp);
1086 return (0);
1087
1088 send:
1089 /*
1090 * Before ESTABLISHED, force sending of initial options
1091 * unless TCP set not to do any options.
1092 * NOTE: we assume that the IP/TCP header plus TCP options
1093 * always fit in a single mbuf, leaving room for a maximum
1094 * link header, i.e.
1095 * max_linkhdr + sizeof (struct tcpiphdr) + optlen <= MCLBYTES
1096 */
1097 optlen = 0;
1098 switch (af) {
1099 #ifdef INET
1100 case AF_INET:
1101 iphdrlen = sizeof(struct ip) + sizeof(struct tcphdr);
1102 break;
1103 #endif
1104 #ifdef INET6
1105 case AF_INET6:
1106 iphdrlen = sizeof(struct ip6_hdr) + sizeof(struct tcphdr);
1107 break;
1108 #endif
1109 default: /*pacify gcc*/
1110 iphdrlen = 0;
1111 break;
1112 }
1113 hdrlen = iphdrlen;
1114 if (flags & TH_SYN) {
1115 struct rtentry *synrt;
1116
1117 synrt = NULL;
1118 #ifdef INET
1119 if (tp->t_inpcb)
1120 synrt = in_pcbrtentry(tp->t_inpcb);
1121 #endif
1122 #ifdef INET6
1123 if (tp->t_in6pcb)
1124 synrt = in6_pcbrtentry(tp->t_in6pcb);
1125 #endif
1126
1127 tp->snd_nxt = tp->iss;
1128 tp->t_ourmss = tcp_mss_to_advertise(synrt != NULL ?
1129 synrt->rt_ifp : NULL, af);
1130 if ((tp->t_flags & TF_NOOPT) == 0) {
1131 opt[0] = TCPOPT_MAXSEG;
1132 opt[1] = 4;
1133 opt[2] = (tp->t_ourmss >> 8) & 0xff;
1134 opt[3] = tp->t_ourmss & 0xff;
1135 optlen = 4;
1136
1137 if ((tp->t_flags & TF_REQ_SCALE) &&
1138 ((flags & TH_ACK) == 0 ||
1139 (tp->t_flags & TF_RCVD_SCALE))) {
1140 *((u_int32_t *) (opt + optlen)) = htonl(
1141 TCPOPT_NOP << 24 |
1142 TCPOPT_WINDOW << 16 |
1143 TCPOLEN_WINDOW << 8 |
1144 tp->request_r_scale);
1145 optlen += 4;
1146 }
1147 if (tcp_do_sack) {
1148 u_int8_t *cp = (u_int8_t *)(opt + optlen);
1149
1150 cp[0] = TCPOPT_SACK_PERMITTED;
1151 cp[1] = 2;
1152 cp[2] = TCPOPT_NOP;
1153 cp[3] = TCPOPT_NOP;
1154 optlen += 4;
1155 }
1156 }
1157 }
1158
1159 /*
1160 * Send a timestamp and echo-reply if this is a SYN and our side
1161 * wants to use timestamps (TF_REQ_TSTMP is set) or both our side
1162 * and our peer have sent timestamps in our SYN's.
1163 */
1164 if ((tp->t_flags & (TF_REQ_TSTMP|TF_NOOPT)) == TF_REQ_TSTMP &&
1165 (flags & TH_RST) == 0 &&
1166 ((flags & (TH_SYN|TH_ACK)) == TH_SYN ||
1167 (tp->t_flags & TF_RCVD_TSTMP))) {
1168 u_int32_t *lp = (u_int32_t *)(opt + optlen);
1169
1170 /* Form timestamp option as shown in appendix A of RFC 1323. */
1171 *lp++ = htonl(TCPOPT_TSTAMP_HDR);
1172 *lp++ = htonl(TCP_TIMESTAMP(tp));
1173 *lp = htonl(tp->ts_recent);
1174 optlen += TCPOLEN_TSTAMP_APPA;
1175
1176 /* Set receive buffer autosizing timestamp. */
1177 if (tp->rfbuf_ts == 0 && (so->so_rcv.sb_flags & SB_AUTOSIZE))
1178 tp->rfbuf_ts = TCP_TIMESTAMP(tp);
1179 }
1180
1181 /*
1182 * Tack on the SACK block if it is necessary.
1183 */
1184 if (sack_numblks) {
1185 int sack_len;
1186 u_char *bp = (u_char *)(opt + optlen);
1187 u_int32_t *lp = (u_int32_t *)(bp + 4);
1188 struct ipqent *tiqe;
1189
1190 sack_len = sack_numblks * 8 + 2;
1191 bp[0] = TCPOPT_NOP;
1192 bp[1] = TCPOPT_NOP;
1193 bp[2] = TCPOPT_SACK;
1194 bp[3] = sack_len;
1195 if ((tp->rcv_sack_flags & TCPSACK_HAVED) != 0) {
1196 sack_numblks--;
1197 *lp++ = htonl(tp->rcv_dsack_block.left);
1198 *lp++ = htonl(tp->rcv_dsack_block.right);
1199 tp->rcv_sack_flags &= ~TCPSACK_HAVED;
1200 }
1201 for (tiqe = TAILQ_FIRST(&tp->timeq);
1202 sack_numblks > 0; tiqe = TAILQ_NEXT(tiqe, ipqe_timeq)) {
1203 KASSERT(tiqe != NULL);
1204 sack_numblks--;
1205 *lp++ = htonl(tiqe->ipqe_seq);
1206 *lp++ = htonl(tiqe->ipqe_seq + tiqe->ipqe_len +
1207 ((tiqe->ipqe_flags & TH_FIN) != 0 ? 1 : 0));
1208 }
1209 optlen += sack_len + 2;
1210 }
1211 TCP_REASS_UNLOCK(tp);
1212
1213 #ifdef TCP_SIGNATURE
1214 if (tp->t_flags & TF_SIGNATURE) {
1215 u_char *bp;
1216 /*
1217 * Initialize TCP-MD5 option (RFC2385)
1218 */
1219 bp = (u_char *)opt + optlen;
1220 *bp++ = TCPOPT_SIGNATURE;
1221 *bp++ = TCPOLEN_SIGNATURE;
1222 sigoff = optlen + 2;
1223 memset(bp, 0, TCP_SIGLEN);
1224 bp += TCP_SIGLEN;
1225 optlen += TCPOLEN_SIGNATURE;
1226 /*
1227 * Terminate options list and maintain 32-bit alignment.
1228 */
1229 *bp++ = TCPOPT_NOP;
1230 *bp++ = TCPOPT_EOL;
1231 optlen += 2;
1232 }
1233 #endif /* TCP_SIGNATURE */
1234
1235 hdrlen += optlen;
1236
1237 #ifdef DIAGNOSTIC
1238 if (!use_tso && len > txsegsize)
1239 panic("tcp data to be sent is larger than segment");
1240 else if (use_tso && len > IP_MAXPACKET)
1241 panic("tcp data to be sent is larger than max TSO size");
1242 if (max_linkhdr + hdrlen > MCLBYTES)
1243 panic("tcphdr too big");
1244 #endif
1245
1246 /*
1247 * Grab a header mbuf, attaching a copy of data to
1248 * be transmitted, and initialize the header from
1249 * the template for sends on this connection.
1250 */
1251 if (len) {
1252 error = tcp_build_datapkt(tp, so, off, len, hdrlen, &m);
1253 if (error)
1254 goto out;
1255 /*
1256 * If we're sending everything we've got, set PUSH.
1257 * (This will keep happy those implementations which only
1258 * give data to the user when a buffer fills or
1259 * a PUSH comes in.)
1260 */
1261 if (off + len == so->so_snd.sb_cc)
1262 flags |= TH_PUSH;
1263 } else {
1264 tcps = TCP_STAT_GETREF();
1265 if (tp->t_flags & TF_ACKNOW)
1266 tcps[TCP_STAT_SNDACKS]++;
1267 else if (flags & (TH_SYN|TH_FIN|TH_RST))
1268 tcps[TCP_STAT_SNDCTRL]++;
1269 else if (SEQ_GT(tp->snd_up, tp->snd_una))
1270 tcps[TCP_STAT_SNDURG]++;
1271 else
1272 tcps[TCP_STAT_SNDWINUP]++;
1273 TCP_STAT_PUTREF();
1274
1275 MGETHDR(m, M_DONTWAIT, MT_HEADER);
1276 if (m != NULL && max_linkhdr + hdrlen > MHLEN) {
1277 MCLGET(m, M_DONTWAIT);
1278 if ((m->m_flags & M_EXT) == 0) {
1279 m_freem(m);
1280 m = NULL;
1281 }
1282 }
1283 if (m == NULL) {
1284 error = ENOBUFS;
1285 goto out;
1286 }
1287 MCLAIM(m, &tcp_tx_mowner);
1288 m->m_data += max_linkhdr;
1289 m->m_len = hdrlen;
1290 }
1291 m->m_pkthdr.rcvif = (struct ifnet *)0;
1292 switch (af) {
1293 #ifdef INET
1294 case AF_INET:
1295 ip = mtod(m, struct ip *);
1296 #ifdef INET6
1297 ip6 = NULL;
1298 #endif
1299 th = (struct tcphdr *)(ip + 1);
1300 break;
1301 #endif
1302 #ifdef INET6
1303 case AF_INET6:
1304 ip = NULL;
1305 ip6 = mtod(m, struct ip6_hdr *);
1306 th = (struct tcphdr *)(ip6 + 1);
1307 break;
1308 #endif
1309 default: /*pacify gcc*/
1310 ip = NULL;
1311 #ifdef INET6
1312 ip6 = NULL;
1313 #endif
1314 th = NULL;
1315 break;
1316 }
1317 if (tp->t_template == 0)
1318 panic("tcp_output");
1319 if (tp->t_template->m_len < iphdrlen)
1320 panic("tcp_output");
1321 bcopy(mtod(tp->t_template, void *), mtod(m, void *), iphdrlen);
1322
1323 /*
1324 * If we are starting a connection, send ECN setup
1325 * SYN packet. If we are on a retransmit, we may
1326 * resend those bits a number of times as per
1327 * RFC 3168.
1328 */
1329 if (tp->t_state == TCPS_SYN_SENT && tcp_do_ecn) {
1330 if (tp->t_flags & TF_SYN_REXMT) {
1331 if (tp->t_ecn_retries--)
1332 flags |= TH_ECE|TH_CWR;
1333 } else {
1334 flags |= TH_ECE|TH_CWR;
1335 tp->t_ecn_retries = tcp_ecn_maxretries;
1336 }
1337 }
1338
1339 if (TCP_ECN_ALLOWED(tp)) {
1340 /*
1341 * If the peer has ECN, mark data packets
1342 * ECN capable. Ignore pure ack packets, retransmissions
1343 * and window probes.
1344 */
1345 if (len > 0 && SEQ_GEQ(tp->snd_nxt, tp->snd_max) &&
1346 !(tp->t_force && len == 1)) {
1347 switch (af) {
1348 #ifdef INET
1349 case AF_INET:
1350 tp->t_inpcb->inp_ip.ip_tos |= IPTOS_ECN_ECT0;
1351 break;
1352 #endif
1353 #ifdef INET6
1354 case AF_INET6:
1355 ip6->ip6_flow |= htonl(IPTOS_ECN_ECT0 << 20);
1356 break;
1357 #endif
1358 }
1359 TCP_STATINC(TCP_STAT_ECN_ECT);
1360 }
1361
1362 /*
1363 * Reply with proper ECN notifications.
1364 */
1365 if (tp->t_flags & TF_ECN_SND_CWR) {
1366 flags |= TH_CWR;
1367 tp->t_flags &= ~TF_ECN_SND_CWR;
1368 }
1369 if (tp->t_flags & TF_ECN_SND_ECE) {
1370 flags |= TH_ECE;
1371 }
1372 }
1373
1374
1375 /*
1376 * If we are doing retransmissions, then snd_nxt will
1377 * not reflect the first unsent octet. For ACK only
1378 * packets, we do not want the sequence number of the
1379 * retransmitted packet, we want the sequence number
1380 * of the next unsent octet. So, if there is no data
1381 * (and no SYN or FIN), use snd_max instead of snd_nxt
1382 * when filling in ti_seq. But if we are in persist
1383 * state, snd_max might reflect one byte beyond the
1384 * right edge of the window, so use snd_nxt in that
1385 * case, since we know we aren't doing a retransmission.
1386 * (retransmit and persist are mutually exclusive...)
1387 */
1388 if (TCP_SACK_ENABLED(tp) && sack_rxmit) {
1389 th->th_seq = htonl(p->rxmit);
1390 p->rxmit += len;
1391 } else {
1392 if (len || (flags & (TH_SYN|TH_FIN)) ||
1393 TCP_TIMER_ISARMED(tp, TCPT_PERSIST))
1394 th->th_seq = htonl(tp->snd_nxt);
1395 else
1396 th->th_seq = htonl(tp->snd_max);
1397 }
1398 th->th_ack = htonl(tp->rcv_nxt);
1399 if (optlen) {
1400 bcopy((void *)opt, (void *)(th + 1), optlen);
1401 th->th_off = (sizeof (struct tcphdr) + optlen) >> 2;
1402 }
1403 th->th_flags = flags;
1404 /*
1405 * Calculate receive window. Don't shrink window,
1406 * but avoid silly window syndrome.
1407 */
1408 if (win < (long)(so->so_rcv.sb_hiwat / 4) && win < (long)rxsegsize)
1409 win = 0;
1410 if (win > (long)TCP_MAXWIN << tp->rcv_scale)
1411 win = (long)TCP_MAXWIN << tp->rcv_scale;
1412 if (win < (long)(int32_t)(tp->rcv_adv - tp->rcv_nxt))
1413 win = (long)(int32_t)(tp->rcv_adv - tp->rcv_nxt);
1414 th->th_win = htons((u_int16_t) (win>>tp->rcv_scale));
1415 if (SEQ_GT(tp->snd_up, tp->snd_nxt)) {
1416 u_int32_t urp = tp->snd_up - tp->snd_nxt;
1417 if (urp > IP_MAXPACKET)
1418 urp = IP_MAXPACKET;
1419 th->th_urp = htons((u_int16_t)urp);
1420 th->th_flags |= TH_URG;
1421 } else
1422 /*
1423 * If no urgent pointer to send, then we pull
1424 * the urgent pointer to the left edge of the send window
1425 * so that it doesn't drift into the send window on sequence
1426 * number wraparound.
1427 */
1428 tp->snd_up = tp->snd_una; /* drag it along */
1429
1430 #ifdef TCP_SIGNATURE
1431 if (sigoff && (tp->t_flags & TF_SIGNATURE)) {
1432 struct secasvar *sav;
1433 u_int8_t *sigp;
1434
1435 sav = tcp_signature_getsav(m, th);
1436
1437 if (sav == NULL) {
1438 if (m)
1439 m_freem(m);
1440 return (EPERM);
1441 }
1442
1443 m->m_pkthdr.len = hdrlen + len;
1444 sigp = (char *)th + sizeof(*th) + sigoff;
1445 tcp_signature(m, th, (char *)th - mtod(m, char *), sav, sigp);
1446
1447 key_sa_recordxfer(sav, m);
1448 #ifdef FAST_IPSEC
1449 KEY_FREESAV(&sav);
1450 #else
1451 key_freesav(sav);
1452 #endif
1453 }
1454 #endif
1455
1456 /*
1457 * Set ourselves up to be checksummed just before the packet
1458 * hits the wire.
1459 */
1460 switch (af) {
1461 #ifdef INET
1462 case AF_INET:
1463 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
1464 if (use_tso) {
1465 m->m_pkthdr.segsz = txsegsize;
1466 m->m_pkthdr.csum_flags = M_CSUM_TSOv4;
1467 } else {
1468 m->m_pkthdr.csum_flags = M_CSUM_TCPv4;
1469 if (len + optlen) {
1470 /* Fixup the pseudo-header checksum. */
1471 /* XXXJRT Not IP Jumbogram safe. */
1472 th->th_sum = in_cksum_addword(th->th_sum,
1473 htons((u_int16_t) (len + optlen)));
1474 }
1475 }
1476 break;
1477 #endif
1478 #ifdef INET6
1479 case AF_INET6:
1480 m->m_pkthdr.csum_data = offsetof(struct tcphdr, th_sum);
1481 if (use_tso) {
1482 m->m_pkthdr.segsz = txsegsize;
1483 m->m_pkthdr.csum_flags = M_CSUM_TSOv6;
1484 } else {
1485 m->m_pkthdr.csum_flags = M_CSUM_TCPv6;
1486 if (len + optlen) {
1487 /* Fixup the pseudo-header checksum. */
1488 /* XXXJRT: Not IPv6 Jumbogram safe. */
1489 th->th_sum = in_cksum_addword(th->th_sum,
1490 htons((u_int16_t) (len + optlen)));
1491 }
1492 }
1493 break;
1494 #endif
1495 }
1496
1497 /*
1498 * In transmit state, time the transmission and arrange for
1499 * the retransmit. In persist state, just set snd_max.
1500 */
1501 if (tp->t_force == 0 || TCP_TIMER_ISARMED(tp, TCPT_PERSIST) == 0) {
1502 tcp_seq startseq = tp->snd_nxt;
1503
1504 /*
1505 * Advance snd_nxt over sequence space of this segment.
1506 * There are no states in which we send both a SYN and a FIN,
1507 * so we collapse the tests for these flags.
1508 */
1509 if (flags & (TH_SYN|TH_FIN))
1510 tp->snd_nxt++;
1511 if (sack_rxmit)
1512 goto timer;
1513 tp->snd_nxt += len;
1514 if (SEQ_GT(tp->snd_nxt, tp->snd_max)) {
1515 tp->snd_max = tp->snd_nxt;
1516 /*
1517 * Time this transmission if not a retransmission and
1518 * not currently timing anything.
1519 */
1520 if (tp->t_rtttime == 0) {
1521 tp->t_rtttime = tcp_now;
1522 tp->t_rtseq = startseq;
1523 TCP_STATINC(TCP_STAT_SEGSTIMED);
1524 }
1525 }
1526
1527 /*
1528 * Set retransmit timer if not currently set,
1529 * and not doing an ack or a keep-alive probe.
1530 * Initial value for retransmit timer is smoothed
1531 * round-trip time + 2 * round-trip time variance.
1532 * Initialize shift counter which is used for backoff
1533 * of retransmit time.
1534 */
1535 timer:
1536 if (TCP_TIMER_ISARMED(tp, TCPT_REXMT) == 0 &&
1537 ((sack_rxmit && tp->snd_nxt != tp->snd_max) ||
1538 tp->snd_nxt != tp->snd_una)) {
1539 if (TCP_TIMER_ISARMED(tp, TCPT_PERSIST)) {
1540 TCP_TIMER_DISARM(tp, TCPT_PERSIST);
1541 tp->t_rxtshift = 0;
1542 }
1543 TCP_TIMER_ARM(tp, TCPT_REXMT, tp->t_rxtcur);
1544 }
1545 } else
1546 if (SEQ_GT(tp->snd_nxt + len, tp->snd_max))
1547 tp->snd_max = tp->snd_nxt + len;
1548
1549 #ifdef TCP_DEBUG
1550 /*
1551 * Trace.
1552 */
1553 if (so->so_options & SO_DEBUG)
1554 tcp_trace(TA_OUTPUT, tp->t_state, tp, m, 0);
1555 #endif
1556
1557 /*
1558 * Fill in IP length and desired time to live and
1559 * send to IP level. There should be a better way
1560 * to handle ttl and tos; we could keep them in
1561 * the template, but need a way to checksum without them.
1562 */
1563 m->m_pkthdr.len = hdrlen + len;
1564
1565 switch (af) {
1566 #ifdef INET
1567 case AF_INET:
1568 ip->ip_len = htons(m->m_pkthdr.len);
1569 packetlen = m->m_pkthdr.len;
1570 if (tp->t_inpcb) {
1571 ip->ip_ttl = tp->t_inpcb->inp_ip.ip_ttl;
1572 ip->ip_tos = tp->t_inpcb->inp_ip.ip_tos;
1573 }
1574 #ifdef INET6
1575 else if (tp->t_in6pcb) {
1576 ip->ip_ttl = in6_selecthlim(tp->t_in6pcb, NULL); /*XXX*/
1577 ip->ip_tos = 0; /*XXX*/
1578 }
1579 #endif
1580 break;
1581 #endif
1582 #ifdef INET6
1583 case AF_INET6:
1584 packetlen = m->m_pkthdr.len;
1585 ip6->ip6_nxt = IPPROTO_TCP;
1586 if (tp->t_in6pcb) {
1587 /*
1588 * we separately set hoplimit for every segment, since
1589 * the user might want to change the value via
1590 * setsockopt. Also, desired default hop limit might
1591 * be changed via Neighbor Discovery.
1592 */
1593 ip6->ip6_hlim = in6_selecthlim(tp->t_in6pcb,
1594 (rt = rtcache_validate(ro)) != NULL ? rt->rt_ifp
1595 : NULL);
1596 }
1597 /* ip6->ip6_flow = ??? */
1598 /* ip6_plen will be filled in ip6_output(). */
1599 break;
1600 #endif
1601 default: /*pacify gcc*/
1602 packetlen = 0;
1603 break;
1604 }
1605
1606 switch (af) {
1607 #ifdef INET
1608 case AF_INET:
1609 {
1610 struct mbuf *opts;
1611
1612 if (tp->t_inpcb)
1613 opts = tp->t_inpcb->inp_options;
1614 else
1615 opts = NULL;
1616 error = ip_output(m, opts, ro,
1617 (tp->t_mtudisc ? IP_MTUDISC : 0) |
1618 (so->so_options & SO_DONTROUTE),
1619 (struct ip_moptions *)0, so);
1620 break;
1621 }
1622 #endif
1623 #ifdef INET6
1624 case AF_INET6:
1625 {
1626 struct ip6_pktopts *opts;
1627
1628 if (tp->t_in6pcb)
1629 opts = tp->t_in6pcb->in6p_outputopts;
1630 else
1631 opts = NULL;
1632 error = ip6_output(m, opts, ro, so->so_options & SO_DONTROUTE,
1633 NULL, so, NULL);
1634 break;
1635 }
1636 #endif
1637 default:
1638 error = EAFNOSUPPORT;
1639 break;
1640 }
1641 if (error) {
1642 out:
1643 if (error == ENOBUFS) {
1644 TCP_STATINC(TCP_STAT_SELFQUENCH);
1645 #ifdef INET
1646 if (tp->t_inpcb)
1647 tcp_quench(tp->t_inpcb, 0);
1648 #endif
1649 #ifdef INET6
1650 if (tp->t_in6pcb)
1651 tcp6_quench(tp->t_in6pcb, 0);
1652 #endif
1653 error = 0;
1654 } else if ((error == EHOSTUNREACH || error == ENETDOWN) &&
1655 TCPS_HAVERCVDSYN(tp->t_state)) {
1656 tp->t_softerror = error;
1657 error = 0;
1658 }
1659
1660 /* Back out the seqence number advance. */
1661 if (sack_rxmit)
1662 p->rxmit -= len;
1663
1664 /* Restart the delayed ACK timer, if necessary. */
1665 if (tp->t_flags & TF_DELACK)
1666 TCP_RESTART_DELACK(tp);
1667
1668 return (error);
1669 }
1670
1671 if (packetlen > tp->t_pmtud_mtu_sent)
1672 tp->t_pmtud_mtu_sent = packetlen;
1673
1674 tcps = TCP_STAT_GETREF();
1675 tcps[TCP_STAT_SNDTOTAL]++;
1676 if (tp->t_flags & TF_DELACK)
1677 tcps[TCP_STAT_DELACK]++;
1678 TCP_STAT_PUTREF();
1679
1680 /*
1681 * Data sent (as far as we can tell).
1682 * If this advertises a larger window than any other segment,
1683 * then remember the size of the advertised window.
1684 * Any pending ACK has now been sent.
1685 */
1686 if (win > 0 && SEQ_GT(tp->rcv_nxt+win, tp->rcv_adv))
1687 tp->rcv_adv = tp->rcv_nxt + win;
1688 tp->last_ack_sent = tp->rcv_nxt;
1689 tp->t_flags &= ~TF_ACKNOW;
1690 TCP_CLEAR_DELACK(tp);
1691 #ifdef DIAGNOSTIC
1692 if (maxburst < 0)
1693 printf("tcp_output: maxburst exceeded by %d\n", -maxburst);
1694 #endif
1695 if (sendalot && (tp->t_congctl == &tcp_reno_ctl || --maxburst))
1696 goto again;
1697 return (0);
1698 }
1699
1700 void
1701 tcp_setpersist(struct tcpcb *tp)
1702 {
1703 int t = ((tp->t_srtt >> 2) + tp->t_rttvar) >> (1 + 2);
1704 int nticks;
1705
1706 if (TCP_TIMER_ISARMED(tp, TCPT_REXMT))
1707 panic("tcp_output REXMT");
1708 /*
1709 * Start/restart persistance timer.
1710 */
1711 if (t < tp->t_rttmin)
1712 t = tp->t_rttmin;
1713 TCPT_RANGESET(nticks, t * tcp_backoff[tp->t_rxtshift],
1714 TCPTV_PERSMIN, TCPTV_PERSMAX);
1715 TCP_TIMER_ARM(tp, TCPT_PERSIST, nticks);
1716 if (tp->t_rxtshift < TCP_MAXRXTSHIFT)
1717 tp->t_rxtshift++;
1718 }
NetBSD on the FriendlyARM MINI2440