| blob | 902845ad34f6d1912eeaecc0219f18fcbf995331 |
1 /*-
2 * Copyright (c) 2016-2020 Netflix, Inc.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 */
27 #include <sys/cdefs.h>
33 #if defined(INET) || defined(INET6)
34 #include <sys/param.h>
35 #include <sys/arb.h>
36 #include <sys/module.h>
37 #include <sys/kernel.h>
38 #ifdef TCP_HHOOK
39 #include <sys/hhook.h>
40 #endif
41 #include <sys/lock.h>
42 #include <sys/malloc.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/mbuf.h>
47 #include <sys/socket.h>
48 #include <sys/socketvar.h>
49 #include <sys/sysctl.h>
50 #include <sys/systm.h>
51 #ifdef STATS
52 #include <sys/qmath.h>
53 #include <sys/tree.h>
55 #else
56 #include <sys/tree.h>
57 #endif
58 #include <sys/refcount.h>
59 #include <sys/queue.h>
60 #include <sys/tim_filter.h>
61 #include <sys/smp.h>
62 #include <sys/kthread.h>
63 #include <sys/kern_prefetch.h>
64 #include <sys/protosw.h>
65 #ifdef TCP_ACCOUNTING
66 #include <sys/sched.h>
67 #include <machine/cpu.h>
68 #endif
69 #include <vm/uma.h>
71 #include <net/route.h>
72 #include <net/route/nhop.h>
73 #include <net/vnet.h>
77 #include <netinet/in.h>
78 #include <netinet/in_kdtrace.h>
79 #include <netinet/in_pcb.h>
80 #include <netinet/ip.h>
83 #include <netinet/ip_var.h>
84 #include <netinet/ip6.h>
85 #include <netinet6/in6_pcb.h>
86 #include <netinet6/ip6_var.h>
87 #include <netinet/tcp.h>
88 #define TCPOUTFLAGS
89 #include <netinet/tcp_fsm.h>
90 #include <netinet/tcp_seq.h>
91 #include <netinet/tcp_timer.h>
92 #include <netinet/tcp_var.h>
93 #include <netinet/tcp_log_buf.h>
94 #include <netinet/tcp_syncache.h>
95 #include <netinet/tcp_hpts.h>
96 #include <netinet/tcp_ratelimit.h>
97 #include <netinet/tcp_accounting.h>
98 #include <netinet/tcpip.h>
99 #include <netinet/cc/cc.h>
100 #include <netinet/cc/cc_newreno.h>
101 #include <netinet/tcp_fastopen.h>
102 #include <netinet/tcp_lro.h>
103 #ifdef NETFLIX_SHARED_CWND
104 #include <netinet/tcp_shared_cwnd.h>
105 #endif
106 #ifdef TCP_OFFLOAD
107 #include <netinet/tcp_offload.h>
108 #endif
109 #ifdef INET6
110 #include <netinet6/tcp6_var.h>
111 #endif
112 #include <netinet/tcp_ecn.h>
114 #include <netipsec/ipsec_support.h>
116 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
117 #include <netipsec/ipsec.h>
118 #include <netipsec/ipsec6.h>
121 #include <netinet/udp.h>
122 #include <netinet/udp_var.h>
123 #include <machine/in_cksum.h>
125 #ifdef MAC
126 #include <security/mac/mac_framework.h>
127 #endif
133 uma_zone_t rack_zone;
134 uma_zone_t rack_pcb_zone;
136 #ifndef TICKS2SBT
137 #define TICKS2SBT(__t) (tick_sbt * ((sbintime_t)(__t)))
138 #endif
142 #define V_newreno_beta VNET(newreno_beta)
143 #define V_newreno_beta_ecn VNET(newreno_beta_ecn)
145 #define M_TCPFSB __CONCAT(M_TCPFSB, STACKNAME)
146 #define M_TCPDO __CONCAT(M_TCPDO, STACKNAME)
155 #define CUM_ACKED 1
156 #define SACKED 2
158 /*
159 * The RACK module incorporates a number of
160 * TCP ideas that have been put out into the IETF
161 * over the last few years:
162 * - Matt Mathis's Rate Halving which slowly drops
163 * the congestion window so that the ack clock can
164 * be maintained during a recovery.
165 * - Yuchung Cheng's RACK TCP (for which its named) that
166 * will stop us using the number of dup acks and instead
167 * use time as the gage of when we retransmit.
168 * - Reorder Detection of RFC4737 and the Tail-Loss probe draft
169 * of Dukkipati et.al.
170 * RACK depends on SACK, so if an endpoint arrives that
171 * cannot do SACK the state machine below will shuttle the
172 * connection back to using the "default" TCP stack that is
173 * in FreeBSD.
174 *
175 * To implement RACK the original TCP stack was first decomposed
176 * into a functional state machine with individual states
177 * for each of the possible TCP connection states. The do_segment
178 * functions role in life is to mandate the connection supports SACK
179 * initially and then assure that the RACK state matches the conenction
180 * state before calling the states do_segment function. Each
181 * state is simplified due to the fact that the original do_segment
182 * has been decomposed and we *know* what state we are in (no
183 * switches on the state) and all tests for SACK are gone. This
184 * greatly simplifies what each state does.
185 *
186 * TCP output is also over-written with a new version since it
187 * must maintain the new rack scoreboard.
188 *
189 */
195 * - 60 seconds */
199 static uint8_t rack_ssthresh_rest_rto_rec = 0; /* Do we restore ssthresh when we have rec -> rto -> rec */
201 static uint32_t rack_gp_gain_req = 1200; /* Amount percent wise required to gain to record a round has "gaining" */
202 static uint32_t rack_rnd_cnt_req = 0x10005; /* Default number of rounds if we are below rack_gp_gain_req where we exit ss */
210 /* Attack threshold detections */
243 static int32_t rack_dsack_std_based = 0x3; /* bit field bit 1 sets rc_rack_tmr_std_based and bit 2 sets rc_rack_use_dsack */
245 #ifdef TCP_ACCOUNTING
247 #endif
253 static int32_t rack_non_rxt_use_cr = 0; /* does a non-rxt in recovery use the configured rate (ss/ca)? */
256 static int32_t rack_honors_hpts_min_to = 1; /* Do we honor the hpts minimum time out for pacing timers */
261 static int32_t rack_enobuf_hw_boost_mult = 0; /* How many times the hw rate we boost slot using time_between */
264 static int32_t rack_hw_rwnd_factor = 2; /* How many max_segs the rwnd must be before we hold off sending */
267 /*
268 * Currently regular tcp has a rto_min of 30ms
269 * the backoff goes 12 times so that ends up
270 * being a total of 122.850 seconds before a
271 * connection is killed.
272 */
299 /* Probertt */
303 static uint16_t rack_atexit_prtt_hbp = 130; /* Clamp to 130% on exit prtt if highly buffered path */
304 static uint16_t rack_atexit_prtt = 130; /* Clamp to 100% on exit prtt if non highly buffered path */
308 static uint32_t rack_probertt_use_min_rtt_entry = 1; /* Use the min to calculate the goal else gp_srtt */
313 static uint32_t rack_probertt_gpsrtt_cnt_mul = 0; /* How many srtt periods does probe-rtt last top fraction */
314 static uint32_t rack_probertt_gpsrtt_cnt_div = 0; /* How many srtt periods does probe-rtt last bottom fraction */
318 static uint32_t rack_min_rtt_movement = 250000; /* Must move at least 250ms (in microseconds) to count as a lowering */
324 /* Part of pacing */
327 /* Timely information:
328 *
329 * Here we have various control parameters on how
330 * timely may change the multiplier. rack_gain_p5_ub
331 * is associated with timely but not directly influencing
332 * the rate decision like the other variables. It controls
333 * the way fill-cw interacts with timely and caps how much
334 * timely can boost the fill-cw b/w.
335 *
336 * The other values are various boost/shrink numbers as well
337 * as potential caps when adjustments are made to the timely
338 * gain (returned by rack_get_output_gain(). Remember too that
339 * the gain returned can be overriden by other factors such as
340 * probeRTT as well as fixed-rate-pacing.
341 */
360 static int32_t rack_timely_int_timely_only = 0; /* do interim timely's only use the timely algo (no b/w changes)? */
368 /* Rack specific counters */
369 counter_u64_t rack_saw_enobuf;
370 counter_u64_t rack_saw_enobuf_hw;
371 counter_u64_t rack_saw_enetunreach;
372 counter_u64_t rack_persists_sends;
373 counter_u64_t rack_persists_acks;
374 counter_u64_t rack_persists_loss;
375 counter_u64_t rack_persists_lost_ends;
376 counter_u64_t rack_total_bytes;
377 #ifdef INVARIANTS
378 counter_u64_t rack_adjust_map_bw;
379 #endif
380 /* Tail loss probe counters */
381 counter_u64_t rack_tlp_tot;
382 counter_u64_t rack_tlp_newdata;
383 counter_u64_t rack_tlp_retran;
384 counter_u64_t rack_tlp_retran_bytes;
385 counter_u64_t rack_to_tot;
386 counter_u64_t rack_hot_alloc;
387 counter_u64_t rack_to_alloc;
388 counter_u64_t rack_to_alloc_hard;
389 counter_u64_t rack_to_alloc_emerg;
390 counter_u64_t rack_to_alloc_limited;
391 counter_u64_t rack_alloc_limited_conns;
392 counter_u64_t rack_split_limited;
393 counter_u64_t rack_rxt_clamps_cwnd;
394 counter_u64_t rack_rxt_clamps_cwnd_uniq;
396 counter_u64_t rack_multi_single_eq;
397 counter_u64_t rack_proc_non_comp_ack;
399 counter_u64_t rack_fto_send;
400 counter_u64_t rack_fto_rsm_send;
401 counter_u64_t rack_nfto_resend;
402 counter_u64_t rack_non_fto_send;
403 counter_u64_t rack_extended_rfo;
405 counter_u64_t rack_sack_proc_all;
406 counter_u64_t rack_sack_proc_short;
407 counter_u64_t rack_sack_proc_restart;
408 counter_u64_t rack_sack_attacks_detected;
409 counter_u64_t rack_sack_attacks_reversed;
410 counter_u64_t rack_sack_attacks_suspect;
411 counter_u64_t rack_sack_used_next_merge;
412 counter_u64_t rack_sack_splits;
413 counter_u64_t rack_sack_used_prev_merge;
414 counter_u64_t rack_sack_skipped_acked;
415 counter_u64_t rack_ack_total;
416 counter_u64_t rack_express_sack;
417 counter_u64_t rack_sack_total;
418 counter_u64_t rack_move_none;
419 counter_u64_t rack_move_some;
421 counter_u64_t rack_input_idle_reduces;
422 counter_u64_t rack_collapsed_win;
423 counter_u64_t rack_collapsed_win_seen;
424 counter_u64_t rack_collapsed_win_rxt;
425 counter_u64_t rack_collapsed_win_rxt_bytes;
426 counter_u64_t rack_try_scwnd;
427 counter_u64_t rack_hw_pace_init_fail;
428 counter_u64_t rack_hw_pace_lost;
434 #define RACK_REXMTVAL(tp) max(rack_rto_min, ((tp)->t_srtt + ((tp)->t_rttvar << 2)))
436 #define RACK_TCPT_RANGESET(tv, value, tvmin, tvmax, slop) do { \
437 (tv) = (value) + slop; \
438 if ((u_long)(tv) < (u_long)(tvmin)) \
439 (tv) = (tvmin); \
440 if ((u_long)(tv) > (u_long)(tvmax)) \
441 (tv) = (tvmax); \
442 } while (0)
444 static void
445 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, int event, int line);
447 static int
450 uint32_t tiwin, int32_t tlen, int32_t * ofia, int32_t thflags, int32_t * ret_val, int32_t orig_tlen);
451 static int
455 static void
464 static uint32_t
466 static void
470 static int
473 static void
474 rack_set_pace_segments(struct tcpcb *tp, struct tcp_rack *rack, uint32_t line, uint64_t *fill_override);
475 static void
479 static void
486 static void
497 static void
500 static void
501 rack_log_type_pacing_sizes(struct tcpcb *tp, struct tcp_rack *rack, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint8_t frm);
503 static uint32_t
509 static void
513 static void
516 struct rack_sendmap *hintrsm, uint32_t add_flags, struct mbuf *s_mb, uint32_t s_moff, int hw_tls, int segsiz);
521 static void
527 static uint32_t
537 static uint32_t
540 static void
543 static int
547 static int
552 static int
556 static int
560 static int
564 static int
568 static int
572 static int
576 static int
580 static int
584 static void rack_chk_req_and_hybrid_on_out(struct tcp_rack *rack, tcp_seq seq, uint32_t len, uint64_t cts);
590 static void
592 static int
594 static void
599 static uint64_t
601 {
608 /* Include all the current bytes too */
612 }
615 else
617 }
619 static void
621 {
630 /* Tcb is leaving */
632 }
635 /* Not new-reno we can't play games with beta! */
639 }
641 /* Huh, not using new-reno so no swaps.? */
644 }
646 /* Get the current values out */
654 }
661 }
664 /* Now lets set in the values we have stored */
672 }
679 }
680 /* Save off the values for restoral */
682 out:
706 }
707 }
709 static void
711 {
714 /*
715 * Use the swap utility placing in 3 for flex8 to id a
716 * set of a new set of values.
717 */
720 }
722 static void
724 {
727 /*
728 * Use the swap utility placing in 4 for flex8 to id a
729 * restoral of the old values.
730 */
733 }
735 static void
737 {
749 }
751 static void
754 {
779 }
786 }
787 }
789 static int
791 {
803 #ifdef INVARIANTS
805 #endif
819 #ifdef INVARIANTS
821 #endif
860 #ifdef INVARIANTS
862 #endif
867 #ifdef INVARIANTS
869 #endif
871 }
874 }
876 static void
878 {
893 OID_AUTO,
899 OID_AUTO,
908 /* Probe rtt related controls */
911 OID_AUTO,
1020 /* Pacing related sysctls */
1023 OID_AUTO,
1081 "What is the limit multiplier of the current gp_est that fillcw can increase the b/w too, 200 == 200% (0 = off)?");
1142 /* Hardware pacing */
1145 OID_AUTO,
1153 "How many times does snd_wnd need to be bigger than pace_max_seg so we will hold off and get more acks?");
1208 "If the rates between software and hardware match precisely how many extra time_betweens do we get?");
1211 OID_AUTO,
1215 /* Timely based GP dynmics */
1266 "Rack timely highest percentage we allow GP multiplier in SS to raise to (0 is no upperbound)");
1328 /* TLP and Rack related parameters */
1331 OID_AUTO,
1401 /* Timer related controls */
1404 OID_AUTO,
1458 /* Measure controls */
1461 OID_AUTO,
1500 /* Features */
1503 OID_AUTO,
1542 /* Misc rack controls */
1545 OID_AUTO,
1549 #ifdef TCP_ACCOUNTING
1555 #endif
1580 "When a persist or keep-alive probe is not answered do we calculate rtt on subsequent answers?");
1590 "What is the maximum number of MSS we allow to be added back if prr can't send all its data?");
1595 "What do we feed the stats framework (1 = ms_rtt, 0 = us_rtt, 2 = ms_rtt from hdwr, > 2 usec rtt from hdwr)?");
1600 "Client low buffer level (below this we are more aggressive in DGP exiting recovery (0 = off)?");
1648 /* Sack Attacker detection stuff */
1726 /* Counters */
1896 #ifdef INVARIANTS
1903 #endif
1997 }
1999 static uint32_t
2001 {
2004 }
2006 static uint64_t
2008 {
2013 else
2015 }
2017 static void
2021 {
2022 #ifdef TCP_REQUEST_TRK
2025 /*
2026 * The rate cap one is noisy and only should come out when normal BB logging
2027 * is enabled, the other logs (not RATE_CAP and NOT CAP_CALC) only come out
2028 * once per chunk and make up the BBpoint that can be turned on by the client.
2029 */
2031 /*
2032 * The very noisy two need to only come out when
2033 * we have verbose logging on.
2034 */
2037 else
2040 /*
2041 * All other less noisy logs here except the measure which
2042 * also needs to come out on the point and the log.
2043 */
2047 }
2054 /* Convert our ms to a microsecond */
2065 /* lt_bw = < flex3 | flex2 > */
2068 /* Record the last obtained us rtt in inflight */
2070 /* Make sure we are looking at the right log if an overide comes in */
2072 }
2076 /* Use the last known rtt i.e. the rack-rtt */
2078 }
2084 /* start = < lost | pkt_epoch > */
2090 /* start = < lost | pkt_epoch > */
2093 /* end = < pkts_out | flex6 > */
2096 }
2097 /* first_send = <lt_epoch | epoch> */
2100 /* localtime = <delivered | applimited>*/
2103 #ifdef TCP_REQUEST_TRK
2106 #endif
2114 }
2115 /*
2116 * Compose bbr_state to be a bit wise 0000ADHF
2117 * where A is the always_pace flag
2118 * where D is the dgp_on flag
2119 * where H is the hybrid_mode on flag
2120 * where F is the use_fixed_rate flag.
2121 */
2136 }
2137 #endif
2138 }
2140 #ifdef TCP_REQUEST_TRK
2141 static void
2143 {
2149 /* Convert our ms to a microsecond */
2156 /*
2157 * We did not get a new Rules Applied to set so
2158 * no overlapping send occured, this means the
2159 * current byte counts are correct.
2160 */
2164 /*
2165 * Overlapping send case, we switched to a new
2166 * send and did a rules applied.
2167 */
2170 }
2175 /* start = < flex1 | flex2 > */
2178 /* end = < flex3 | flex4 > */
2182 /* localtime = <delivered | applimited>*/
2185 /* client timestamp = <lt_epoch | epoch>*/
2188 /* now set all the flags in */
2192 /*
2193 * Last send time = <flex5 | pkt_epoch> note we do not distinguish cases
2194 * where a false retransmit occurred so first_send <-> lastsend may
2195 * include longer time then it actually took if we have a false rxt.
2196 */
2199 /*
2200 * Compose bbr_state to be a bit wise 0000ADHF
2201 * where A is the always_pace flag
2202 * where D is the dgp_on flag
2203 * where H is the hybrid_mode on flag
2204 * where F is the use_fixed_rate flag.
2205 */
2220 }
2221 }
2222 #endif
2226 {
2232 #ifdef INET6
2234 #endif
2237 #ifdef INET
2239 #endif
2241 }
2247 }
2249 static void
2251 {
2252 #ifdef TCP_REQUEST_TRK
2255 #endif
2259 #ifdef TCP_REQUEST_TRK
2262 /*
2263 * We have a dynamic cap. The original target
2264 * is in bw_rate_cap, but we need to look at
2265 * how long it is until we hit the deadline.
2266 */
2273 /* No time left we do DGP only */
2278 }
2279 /* We have the time */
2282 /* If there is less than a ms left just use DGPs rate */
2287 }
2288 /*
2289 * Now lets find the amount of data left to send.
2290 *
2291 * Now ideally we want to use the end_seq to figure out how much more
2292 * but it might not be possible (only if we have the TRACK_FG_COMP on the entry..
2293 */
2298 /* TSNH, we should catch it at the send */
2303 }
2305 /*
2306 * The hard way, figure out how much is gone and then
2307 * take that away from the total the client asked for
2308 * (thats off by tls overhead if this is tls).
2309 */
2312 else
2317 /* TSNH, we should catch it at the send */
2322 }
2323 }
2325 /* We have it all sent */
2330 else
2332 }
2335 /* Now we must compensate for IP/TCP overhead */
2337 /* Update the bit rate cap */
2342 /* Lets set in a smaller mss possibly here to match our rate-cap */
2347 rack->r_ctl.rc_pace_max_segs = rack_get_pacing_len(rack, calcbw, ctf_fixed_maxseg(rack->rc_tp));
2348 rack_log_type_pacing_sizes(rack->rc_tp, rack, rack->r_ctl.client_suggested_maxseg, orig_max, __LINE__, 5);
2349 }
2357 }
2359 }
2360 normal_ratecap:
2361 #endif
2363 #ifdef TCP_REQUEST_TRK
2370 /* Lets set in a smaller mss possibly here to match our rate-cap */
2375 rack->r_ctl.rc_pace_max_segs = rack_get_pacing_len(rack, rack->r_ctl.bw_rate_cap, ctf_fixed_maxseg(rack->rc_tp));
2376 rack_log_type_pacing_sizes(rack->rc_tp, rack, rack->r_ctl.client_suggested_maxseg, orig_max, __LINE__, 5);
2377 }
2378 #endif
2384 }
2385 }
2387 static uint64_t
2389 {
2393 /*
2394 * We have yet no b/w measurement,
2395 * if we have a user set initial bw
2396 * return it. If we don't have that and
2397 * we have an srtt, use the tcp IW (10) to
2398 * calculate a fictional b/w over the SRTT
2399 * which is more or less a guess. Note
2400 * we don't use our IW from rack on purpose
2401 * so if we have like IW=30, we are not
2402 * calculating a "huge" b/w.
2403 */
2408 else
2411 /*
2412 * No goodput bw but a long-term b/w does exist
2413 * lets use that.
2414 */
2417 }
2421 /* Ok lets come up with the IW guess, if we have a srtt */
2423 /*
2424 * Go with old pacing method
2425 * i.e. burst mitigation only.
2426 */
2428 }
2429 /* Ok lets get the initial TCP win (not racks) */
2437 }
2439 /* Averaging is done, we can return the value */
2442 /* Still doing initial average must calculate */
2444 }
2446 /* We are not using lt-bw */
2449 }
2452 /* If we don't have one then equate it to the gp_bw */
2454 }
2458 else
2463 else
2465 }
2466 /*
2467 * Now lets compensate based on the TCP/IP overhead. Our
2468 * Goodput estimate does not include this so we must pace out
2469 * a bit faster since our pacing calculations do. The pacing
2470 * calculations use the base ETHERNET_SEGMENT_SIZE and the segsiz
2471 * we are using to do this, so we do that here in the opposite
2472 * direction as well. This means that if we are tunneled and the
2473 * segsiz is say 1200 bytes we will get quite a boost, but its
2474 * compensated for in the pacing time the opposite way.
2475 */
2476 compensate:
2479 }
2482 static uint64_t
2484 {
2488 /* Return the fixed pacing rate */
2490 }
2493 }
2495 static uint16_t
2497 {
2505 /* a retransmission always use the recovery rate */
2508 /* Directed to use the configured rate */
2512 /* No PRR, lets just use the b/w estimate only */
2515 /*
2516 * Here we may have a non-retransmit but we
2517 * have no overrides, so just use the recovery
2518 * rate (prr is in effect).
2519 */
2521 }
2522 }
2523 configured_rate:
2524 /* For the configured rate we look at our cwnd vs the ssthresh */
2527 else
2529 }
2531 static void
2532 rack_log_dsack_event(struct tcp_rack *rack, uint8_t mod, uint32_t flex4, uint32_t flex5, uint32_t flex6)
2533 {
2534 /*
2535 * Types of logs (mod value)
2536 * 1 = dsack_persists reduced by 1 via T-O or fast recovery exit.
2537 * 2 = a dsack round begins, persist is reset to 16.
2538 * 3 = a dsack round ends
2539 * 4 = Dsack option increases rack rtt flex5 is the srtt input, flex6 is thresh
2540 * 5 = Socket option set changing the control flags rc_rack_tmr_std_based, rc_rack_use_dsack
2541 * 6 = Final rack rtt, flex4 is srtt and flex6 is final limited thresh.
2542 */
2568 }
2569 }
2571 static void
2575 {
2596 }
2610 else
2620 }
2621 }
2623 static uint64_t
2625 {
2626 /*
2627 * We allow rack_per_of_gp_xx to dictate our bw rate we want.
2628 */
2635 /* Never fall below the minimum (def 64kbps) */
2639 /* Rate caps are in place */
2641 /* We have a hdwr rate already */
2644 /* We are capping bw at the highest rate table entry */
2649 }
2656 }
2662 /*
2663 * Special case, we have not yet attempted hardware
2664 * pacing, and yet we may, when we do, find out if we are
2665 * above the highest rate. We need to know the maxbw for the interface
2666 * in question (if it supports ratelimiting). We get back
2667 * a 0, if the interface is not found in the RL lists.
2668 */
2671 /* Yep, we have a rate is it above this rate? */
2676 }
2677 }
2678 }
2679 }
2680 done:
2682 }
2684 static void
2685 rack_log_retran_reason(struct tcp_rack *rack, struct rack_sendmap *rsm, uint32_t tsused, uint32_t thresh, int mod)
2686 {
2692 /*
2693 * We get 3 values currently for mod
2694 * 1 - We are retransmitting and this tells the reason.
2695 * 2 - We are clearing a dup-ack count.
2696 * 3 - We are incrementing a dup-ack count.
2697 *
2698 * The clear/increment are only logged
2699 * if you have BBverbose on.
2700 */
2702 }
2724 }
2725 }
2727 static void
2728 rack_log_to_start(struct tcp_rack *rack, uint32_t cts, uint32_t to, int32_t slot, uint8_t which)
2729 {
2745 else
2767 }
2768 }
2770 static void
2772 {
2784 else
2788 else
2803 }
2804 }
2806 static void
2812 {
2828 }
2833 }
2838 }
2845 else
2855 }
2856 }
2858 static void
2861 {
2884 /* We loose any upper of the 24 bits */
2887 /* Its a SYN */
2892 }
2893 /* Write out general bits of interest rrs here */
2925 }
2926 }
2928 static void
2930 {
2931 /*
2932 * Log the rtt sample we are
2933 * applying to the srtt algorithm in
2934 * useconds.
2935 */
2940 /* Convert our ms to a microsecond */
2950 /*
2951 * We capture in delRate the upper 32 bits as
2952 * the confidence level we had declared, and the
2953 * lower 32 bits as the actual RTT using the arrival
2954 * timestamp.
2955 */
2959 /* Lets capture all the things that make up t_rtxcur */
2974 }
2975 }
2977 static void
2978 rack_log_rtt_sample_calc(struct tcp_rack *rack, uint32_t rtt, uint32_t send_time, uint32_t ack_time, int where)
2979 {
2984 /* Convert our ms to a microsecond */
3000 }
3001 }
3004 static void
3006 {
3011 /* Convert our ms to a microsecond */
3027 }
3028 }
3032 rack_log_progress_event(struct tcp_rack *rack, struct tcpcb *tp, uint32_t tick, int event, int line)
3033 {
3058 }
3059 }
3061 static void
3062 rack_log_type_bbrsnd(struct tcp_rack *rack, uint32_t len, uint32_t slot, uint32_t cts, struct timeval *tv, int line)
3063 {
3072 else
3088 }
3089 }
3091 static void
3092 rack_log_doseg_done(struct tcp_rack *rack, uint32_t cts, int32_t nxt_pkt, int32_t did_out, int way_out, int nsegs)
3093 {
3105 else
3136 }
3137 }
3139 static void
3140 rack_log_type_pacing_sizes(struct tcpcb *tp, struct tcp_rack *rack, uint32_t arg1, uint32_t arg2, uint32_t arg3, uint8_t frm)
3141 {
3163 }
3164 }
3166 static void
3169 {
3181 else
3200 }
3201 }
3203 static void
3206 {
3218 else
3237 }
3238 }
3240 static void
3246 {
3252 /* No you can't use 1, its for the real to cancel */
3254 }
3270 }
3271 }
3273 static void
3275 {
3288 else
3300 }
3301 }
3303 static void
3305 {
3315 else
3333 }
3334 }
3336 static void
3338 {
3391 #ifdef INVARIANTS
3393 #endif
3396 }
3400 {
3403 /*
3404 * First get the top of the list it in
3405 * theory is the "hottest" rsm we have,
3406 * possibly just freed by ack processing.
3407 */
3414 }
3415 /*
3416 * Once we get under our free cache we probably
3417 * no longer have a "hot" one available. Lets
3418 * get one from UMA.
3419 */
3425 }
3426 /*
3427 * Dig in to our aux rsm's (the last two) since
3428 * UMA failed to get us one.
3429 */
3436 }
3438 }
3442 {
3449 }
3451 }
3453 }
3455 /* wrapper to allocate a sendmap entry, subject to a specific limit */
3458 {
3462 /* currently there is only one limit type */
3469 }
3471 }
3472 }
3474 /* allocate and mark in the limit type, if set */
3479 }
3481 }
3483 static void
3485 {
3488 /*
3489 * Free up all the tail entries until
3490 * we get our list down to the limit.
3491 */
3498 }
3499 }
3501 static void
3503 {
3507 }
3508 }
3510 /* currently there is only one limit type */
3512 }
3518 else
3520 }
3530 /* Make sure we are not going to overrun our count limit of 0xff */
3533 }
3536 }
3538 static uint32_t
3540 {
3547 /*
3548 * We have no measurement (IW is in flight?) so
3549 * we can only guess using our data_window sysctl
3550 * value (usually 20MSS).
3551 */
3553 }
3554 /*
3555 * Now we have a number of factors to consider.
3556 *
3557 * 1) We have a desired BDP which is usually
3558 * at least 2.
3559 * 2) We have a minimum number of rtt's usually 1 SRTT
3560 * but we allow it too to be more.
3561 * 3) We want to make sure a measurement last N useconds (if
3562 * we have set rack_min_measure_usec.
3563 *
3564 * We handle the first concern here by trying to create a data
3565 * window of max(rack_def_data_window, DesiredBDP). The
3566 * second concern we handle in not letting the measurement
3567 * window end normally until at least the required SRTT's
3568 * have gone by which is done further below in
3569 * rack_enough_for_measurement(). Finally the third concern
3570 * we also handle here by calculating how long that time
3571 * would take at the current BW and then return the
3572 * max of our first calculation and that length. Note
3573 * that if rack_min_measure_usec is 0, we don't deal
3574 * with concern 3. Also for both Concern 1 and 3 an
3575 * application limited period could end the measurement
3576 * earlier.
3577 *
3578 * So lets calculate the BDP with the "known" b/w using
3579 * the SRTT has our rtt and then multiply it by the
3580 * goal.
3581 */
3587 /* Now we need to round up to the nearest MSS */
3590 /* Now calculate our min length for this b/w */
3598 }
3599 /*
3600 * Now if we have a very small window we want
3601 * to attempt to get the window that is
3602 * as small as possible. This happens on
3603 * low b/w connections and we don't want to
3604 * span huge numbers of rtt's between measurements.
3605 *
3606 * We basically include 2 over our "MIN window" so
3607 * that the measurement can be shortened (possibly) by
3608 * an ack'ed packet.
3609 */
3612 else
3615 }
3617 static int
3618 rack_enough_for_measurement(struct tcpcb *tp, struct tcp_rack *rack, tcp_seq th_ack, uint8_t *quality)
3619 {
3622 /*
3623 * Has enough time passed for the GP measurement to be valid?
3624 */
3626 /* Not enough bytes yet */
3628 }
3631 /*
3632 * All is acked quality of all acked is
3633 * usually low or medium, but we in theory could split
3634 * all acked into two cases, where you got
3635 * a signifigant amount of your window and
3636 * where you did not. For now we leave it
3637 * but it is something to contemplate in the
3638 * future. The danger here is that delayed ack
3639 * is effecting the last byte (which is a 50:50 chance).
3640 */
3643 }
3645 /*
3646 * We obtained our entire window of data we wanted
3647 * no matter if we are in recovery or not then
3648 * its ok since expanding the window does not
3649 * make things fuzzy (or at least not as much).
3650 */
3653 }
3657 /* Not enough bytes yet */
3659 }
3662 /*
3663 * We are up to the app limited send point
3664 * we have to measure irrespective of the time..
3665 */
3668 }
3669 /* Now what about time? */
3673 /*
3674 * We do not allow a measurement if we are in recovery
3675 * that would shrink the goodput window we wanted.
3676 * This is to prevent cloudyness of when the last send
3677 * was actually made.
3678 */
3681 }
3682 /* Nope not even a full SRTT has passed */
3684 }
3686 static void
3690 {
3733 }
3734 }
3736 static int
3737 rack_bw_can_be_raised(struct tcp_rack *rack, uint64_t cur_bw, uint64_t last_bw_est, uint16_t mult)
3738 {
3739 /*
3740 * Before we increase we need to know if
3741 * the estimate just made was less than
3742 * our pacing goal (i.e. (cur_bw * mult) > last_bw_est)
3743 *
3744 * If we already are pacing at a fast enough
3745 * rate to push us faster there is no sense of
3746 * increasing.
3747 *
3748 * We first caculate our actual pacing rate (ss or ca multiplier
3749 * times our cur_bw).
3750 *
3751 * Then we take the last measured rate and multipy by our
3752 * maximum pacing overage to give us a max allowable rate.
3753 *
3754 * If our act_rate is smaller than our max_allowable rate
3755 * then we should increase. Else we should hold steady.
3756 *
3757 */
3764 /*
3765 * Initial startup case or
3766 * everything is acked case.
3767 */
3771 }
3773 /*
3774 * We can always pace at or slightly above our rate.
3775 */
3779 }
3785 /*
3786 * Here the rate we are actually pacing at
3787 * is smaller than 10% above our last measurement.
3788 * This means we are pacing below what we would
3789 * like to try to achieve (plus some wiggle room).
3790 */
3795 /*
3796 * Here we are already pacing at least rack_max_per_above(10%)
3797 * what we are getting back. This indicates most likely
3798 * that we are being limited (cwnd/rwnd/app) and can't
3799 * get any more b/w. There is no sense of trying to
3800 * raise up the pacing rate its not speeding us up
3801 * and we already are pacing faster than we are getting.
3802 */
3806 }
3807 }
3809 static void
3811 {
3812 /*
3813 * When we drag bottom, we want to assure
3814 * that no multiplier is below 1.0, if so
3815 * we want to restore it to at least that.
3816 */
3818 /* This is unlikely we usually do not touch recovery */
3820 }
3823 }
3826 }
3827 }
3829 static void
3831 {
3834 }
3837 }
3838 }
3840 static void
3841 rack_increase_bw_mul(struct tcp_rack *rack, int timely_says, uint64_t cur_bw, uint64_t last_bw_est, int override)
3842 {
3850 /*
3851 * override is passed when we are
3852 * loosing b/w and making one last
3853 * gasp at trying to not loose out
3854 * to a new-reno flow.
3855 */
3857 }
3858 /* In classic timely we boost by 5x if we have 5 increases in a row, lets not */
3861 /*
3862 * Reset and get 5 strokes more before the boost. Note
3863 * that the count is 0 based so we have to add one.
3864 */
3865 extra_boost:
3870 /* Must be at least 1% increase for true timely increases */
3878 /* We have been in recovery ding it too */
3888 }
3893 /* In CA */
3903 }
3907 /* In SS */
3917 }
3920 /* Go into increment mode */
3923 }
3925 logged &&
3928 }
3931 }
3933 static uint32_t
3935 {
3936 /*-
3937 * norm_grad = rtt_diff / minrtt;
3938 * new_per = curper * (1 - B * norm_grad)
3939 *
3940 * B = rack_gp_decrease_per (default 80%)
3941 * rtt_dif = input var current rtt-diff
3942 * curper = input var current percentage
3943 * minrtt = from rack filter
3944 *
3945 * In order to do the floating point calculations above we
3946 * do an integer conversion. The code looks confusing so let me
3947 * translate it into something that use more variables and
3948 * is clearer for us humans :)
3949 *
3950 * uint64_t norm_grad, inverse, reduce_by, final_result;
3951 * uint32_t perf;
3952 *
3953 * norm_grad = (((uint64_t)rtt_diff * 1000000) /
3954 * (uint64_t)get_filter_small(&rack->r_ctl.rc_gp_min_rtt));
3955 * inverse = ((uint64_t)rack_gp_decrease * (uint64_t)1000000) * norm_grad;
3956 * inverse /= 1000000;
3957 * reduce_by = (1000000 - inverse);
3958 * final_result = (cur_per * reduce_by) / 1000000;
3959 * perf = (uint32_t)final_result;
3960 */
3970 /* TSNH */
3972 }
3974 }
3976 static uint32_t
3978 {
3979 /*
3980 * highrttthresh
3981 * result = curper * (1 - (B * ( 1 - ------ ))
3982 * gp_srtt
3983 *
3984 * B = rack_gp_decrease_per (default .8 i.e. 80)
3985 * highrttthresh = filter_min * rack_gp_rtt_maxmul
3986 */
4003 rack_gp_decrease_per,
4005 log1,
4006 perf,
4007 __LINE__,
4009 }
4011 }
4013 static void
4015 {
4022 /* Turn off increment counting */
4025 }
4028 /* Calculate the reduction value */
4031 }
4032 /* Must be at least 1% reduction */
4034 /* We have been in recovery ding it too */
4040 else
4050 }
4054 }
4056 /* Sent in SS */
4062 else
4084 }
4089 /* Sent in CA */
4095 else
4117 }
4121 }
4127 }
4133 }
4135 static void
4138 {
4183 }
4184 }
4186 static void
4188 {
4196 /*
4197 * A window protocol must be able to have 4 packets
4198 * outstanding as the floor in order to function
4199 * (especially considering delayed ack :D).
4200 */
4202 }
4203 }
4205 static void
4207 {
4208 /**
4209 * ProbeRTT is a bit different in rack_pacing than in
4210 * BBR. It is like BBR in that it uses the lowering of
4211 * the RTT as a signal that we saw something new and
4212 * counts from there for how long between. But it is
4213 * different in that its quite simple. It does not
4214 * play with the cwnd and wait until we get down
4215 * to N segments outstanding and hold that for
4216 * 200ms. Instead it just sets the pacing reduction
4217 * rate to a set percentage (70 by default) and hold
4218 * that for a number of recent GP Srtt's.
4219 */
4227 /* We are idle */
4229 }
4232 /*
4233 * Stop the goodput now, the idea here is
4234 * that future measurements with in_probe_rtt
4235 * won't register if they are not greater so
4236 * we want to get what info (if any) is available
4237 * now.
4238 */
4241 RACK_QUALITY_PROBERTT);
4242 }
4253 else
4257 }
4259 static void
4261 {
4270 /*
4271 * Stop the goodput now, the idea here is
4272 * that future measurements with in_probe_rtt
4273 * won't register if they are not greater so
4274 * we want to get what info (if any) is available
4275 * now.
4276 */
4279 RACK_QUALITY_PROBERTT);
4281 /*
4282 * We don't have enough data to make a measurement.
4283 * So lets just stop and start here after exiting
4284 * probe-rtt. We probably are not interested in
4285 * the results anyway.
4286 */
4288 }
4289 /*
4290 * Measurements through the current snd_max are going
4291 * to be limited by the slower pacing rate.
4292 *
4293 * We need to mark these as app-limited so we
4294 * don't collapse the b/w.
4295 */
4301 /*
4302 * Go out to the end app limited and mark
4303 * this new one as next and move the end_appl up
4304 * to this guy.
4305 */
4309 }
4312 }
4313 /*
4314 * Now, we need to examine our pacing rate multipliers.
4315 * If its under 100%, we need to kick it back up to
4316 * 100%. We also don't let it be over our "max" above
4317 * the actual rate i.e. 100% + rack_clamp_atexit_prtt.
4318 * Note setting clamp_atexit_prtt to 0 has the effect
4319 * of setting CA/SS to 100% always at exit (which is
4320 * the default behavior).
4321 */
4327 }
4328 /* Do we do any clamping at exit? */
4332 }
4336 }
4337 /*
4338 * Lets set rtt_diff to 0, so that we will get a "boost"
4339 * after exiting.
4340 */
4343 /* Clear all flags so we start fresh */
4346 /*
4347 * If configured to, set the cwnd and ssthresh to
4348 * our targets.
4349 */
4354 /* Set ssthresh so we get into CA once we hit our target */
4356 /* Set to min rtt */
4360 /* Set to current gp rtt */
4364 /* Set to entry gp rtt */
4375 /*
4376 * A highly buffered path needs
4377 * cwnd space for timely to work.
4378 * Lets set things up as if
4379 * we are heading back here again.
4380 */
4383 /*
4384 * Lets take the smaller of the
4385 * two since we are just somewhat
4386 * buffered.
4387 */
4392 /*
4393 * Here we are not highly buffered
4394 * and should pick the min we can to
4395 * keep from causing loss.
4396 */
4398 }
4400 setval);
4401 }
4403 /* There is a percentage here to boost */
4412 /* Enforce a min */
4414 }
4415 /* If we set in the cwnd also set the ssthresh point so we are in CA */
4417 }
4421 /* Clear times last so log has all the info */
4426 }
4428 static void
4430 {
4431 /* Check in on probe-rtt */
4434 /* We do not do p-rtt unless we have gp measurements */
4436 }
4443 /*
4444 * We went idle during prtt, just exit now.
4445 */
4450 /*
4451 * Probe RTT safety value triggered!
4452 */
4457 }
4458 /* Calculate the max we will wait */
4459 endtime = rack->r_ctl.rc_time_probertt_entered + (rack->r_ctl.rc_gp_srtt * rack_max_drain_wait);
4462 /* Calculate the min we must wait */
4464 if ((ctf_flight_size(rack->rc_tp, rack->r_ctl.rc_sacked) > rack->r_ctl.rc_target_probertt_flight) &&
4467 /* Do we lower more? */
4468 no_exit:
4471 else
4475 /* Maybe */
4479 else
4481 /* Limit it too */
4484 }
4485 /* We must reach target or the time set */
4487 }
4493 /* We are not past the must_stay time */
4495 }
4502 /* Restore back to our rate we want to pace at in prtt */
4504 }
4505 /*
4506 * Setup our end time, some number of gp_srtts plus 200ms.
4507 */
4512 else
4517 /* yes, exit probertt */
4519 }
4524 /* Go into probertt, its been too long since we went lower */
4526 }
4527 }
4529 static void
4532 {
4540 /* No dynamic GP multiplier in play */
4542 }
4545 /* Calculate our up and down range */
4554 /*
4555 * This is the case where our RTT is above
4556 * the max target and we have been configured
4557 * to just do timely no bonus up stuff in that case.
4558 *
4559 * There are two configurations, set to 1, and we
4560 * just do timely if we are over our max. If its
4561 * set above 1 then we slam the multipliers down
4562 * to 100 and then decrement per timely.
4563 */
4570 /*
4571 * We are decreasing this is a bit complicated this
4572 * means we are loosing ground. This could be
4573 * because another flow entered and we are competing
4574 * for b/w with it. This will push the RTT up which
4575 * makes timely unusable unless we want to get shoved
4576 * into a corner and just be backed off (the age
4577 * old problem with delay based CC).
4578 *
4579 * On the other hand if it was a route change we
4580 * would like to stay somewhat contained and not
4581 * blow out the buffers.
4582 */
4587 /* Go into reduction counting */
4590 }
4592 /*
4593 * Push another time with a faster pacing
4594 * to try to gain back (we include override to
4595 * get a full raise factor).
4596 */
4601 /*
4602 * Do an override up in b/w if we were
4603 * below the threshold or if the threshold
4604 * is zero we always do the raise.
4605 */
4608 /* Log it stays the same */
4611 }
4613 /* We are not incrementing really no-count */
4617 /*
4618 * Lets just use the RTT
4619 * information and give up
4620 * pushing.
4621 */
4623 }
4627 /*
4628 * We are increasing b/w lets keep going, updating
4629 * our b/w and ignoring any timely input, unless
4630 * of course we are at our max raise (if there is one).
4631 */
4639 /*
4640 * In cases where we can't go higher
4641 * we should just use timely.
4642 */
4644 }
4648 /*
4649 * In cases where we can't go higher
4650 * we should just use timely.
4651 */
4653 }
4656 /* You get a set number of pushes if timely is trying to reduce */
4660 /* Log it stays the same */
4663 }
4666 /*
4667 * We are staying between the lower and upper range bounds
4668 * so use timely to decide.
4669 */
4672 use_timely:
4679 /* We are loosing ground */
4682 /* We are not incrementing really no-count */
4691 }
4692 }
4693 }
4695 static int32_t
4696 rack_make_timely_judgement(struct tcp_rack *rack, uint32_t rtt, int32_t rtt_diff, uint32_t prev_rtt)
4697 {
4705 rack_gp_rtt_maxmul)) {
4706 /* Reduce the b/w multiplier */
4717 /* Increase the b/w multiplier */
4728 /*
4729 * Use a gradient to find it the timely gradient
4730 * is:
4731 * grad = rc_rtt_diff / min_rtt;
4732 *
4733 * anything below or equal to 0 will be
4734 * a increase indication. Anything above
4735 * zero is a decrease. Note we take care
4736 * of the actual gradient calculation
4737 * in the reduction (its not needed for
4738 * increase).
4739 */
4742 /*
4743 * Rttdiff is less than zero, increase the
4744 * b/w multiplier (its 0 or negative)
4745 */
4750 /* Reduce the b/w multiplier */
4754 }
4755 }
4757 }
4761 {
4764 /**
4765 * This covers the case that the
4766 * resent is completely inside
4767 * the gp range or up to it.
4768 * |----------------|
4769 * |-----| <or>
4770 * |----|
4771 * <or> |---|
4772 */
4776 /**
4777 * This covers the case of
4778 * |--------------|
4779 * |-------->|
4780 */
4786 /**
4787 * This covers the case of
4788 * |--------------|
4789 * |-------->|
4790 */
4792 }
4794 }
4798 {
4802 /*
4803 * We have a Goodput measurement in progress. Mark
4804 * the send if its within the window. If its not
4805 * in the window make sure it does not have the mark.
4806 */
4809 else
4811 }
4815 {
4816 /* A GP measurement is ending, clear all marks on the send map*/
4822 }
4823 /* Nothing left? */
4827 }
4828 }
4833 {
4837 /* Nothing outstanding yet, nothing to do here */
4839 }
4841 /*
4842 * We are measuring ahead of some outstanding
4843 * data. We need to walk through up until we get
4844 * to gp_seq marking so that no rsm is set incorrectly
4845 * with RACK_IN_GP_WIN.
4846 */
4853 }
4854 }
4856 /*
4857 * Need to find the GP seq, if rsm is
4858 * set we stopped as we hit it.
4859 */
4864 }
4865 /*
4866 * Now we may need to mark already sent rsm, ahead of
4867 * gput_seq in the window since they may have been sent
4868 * *before* we started our measurment. The rsm, if non-null
4869 * has been marked (note if rsm would have been NULL we would have
4870 * returned in the previous block). So we go to the next, and continue
4871 * until we run out of entries or we exceed the gp_ack value.
4872 */
4879 }
4880 }
4882 static void
4883 rack_log_gp_calc(struct tcp_rack *rack, uint32_t add_part, uint32_t sub_part, uint32_t srtt, uint64_t meas_bw, uint64_t utim, uint8_t meth, uint32_t line)
4884 {
4905 }
4906 }
4908 static void
4911 {
4922 else
4926 else
4928 /*
4929 * Use the larger of the send time or ack time. This prevents us
4930 * from being influenced by ack artifacts to come up with too
4931 * high of measurement. Note that since we are spanning over many more
4932 * bytes in most of our measurements hopefully that is less likely to
4933 * occur.
4934 */
4937 else
4942 /*
4943 * Invalid measurement time, maybe
4944 * all on one ack/one send?
4945 */
4951 }
4953 /* We never made a us_rtt measurement? */
4959 }
4960 /*
4961 * Calculate the maximum possible b/w this connection
4962 * could have. We base our calculation on the lowest
4963 * rtt we have seen during the measurement and the
4964 * largest rwnd the client has given us in that time. This
4965 * forms a BDP that is the maximum that we could ever
4966 * get to the client. Anything larger is not valid.
4967 *
4968 * I originally had code here that rejected measurements
4969 * where the time was less than 1/2 the latest us_rtt.
4970 * But after thinking on that I realized its wrong since
4971 * say you had a 150Mbps or even 1Gbps link, and you
4972 * were a long way away.. example I am in Europe (100ms rtt)
4973 * talking to my 1Gbps link in S.C. Now measuring say 150,000
4974 * bytes my time would be 1.2ms, and yet my rtt would say
4975 * the measurement was invalid the time was < 50ms. The
4976 * same thing is true for 150Mb (8ms of time).
4977 *
4978 * A better way I realized is to look at what the maximum
4979 * the connection could possibly do. This is gated on
4980 * the lowest RTT we have seen and the highest rwnd.
4981 * We should in theory never exceed that, if we are
4982 * then something on the path is storing up packets
4983 * and then feeding them all at once to our endpoint
4984 * messing up our measurement.
4985 */
4990 /* No measurement can be made */
4999 /*
5000 * Don't measure a b/w for pacing unless we have gotten at least
5001 * an initial windows worth of data in this measurement interval.
5002 *
5003 * Small numbers of bytes get badly influenced by delayed ack and
5004 * other artifacts. Note we take the initial window or our
5005 * defined minimum GP (defaulting to 10 which hopefully is the
5006 * IW).
5007 */
5009 /*
5010 * The initial estimate is special. We
5011 * have blasted out an IW worth of packets
5012 * without a real valid ack ts results. We
5013 * then setup the app_limited_needs_set flag,
5014 * this should get the first ack in (probably 2
5015 * MSS worth) to be recorded as the timestamp.
5016 * We thus allow a smaller number of bytes i.e.
5017 * IW - 2MSS.
5018 */
5020 /* Also lets fill previous for our first measurement to be neutral */
5022 }
5028 }
5029 /*
5030 * We now need to calculate the Timely like status so
5031 * we can update (possibly) the b/w multipliers.
5032 */
5035 /* No previous reading */
5039 /*
5040 * We don't want a probertt to be counted
5041 * since it will be negative incorrectly. We
5042 * expect to be reducing the RTT when we
5043 * pace at a slower rate.
5044 */
5047 }
5048 }
5053 );
5057 /*
5058 * Something is on path playing
5059 * since this b/w is not possible based
5060 * on our BDP (highest rwnd and lowest rtt
5061 * we saw in the measurement window).
5062 *
5063 * Another option here would be to
5064 * instead skip the measurement.
5065 */
5070 }
5071 /* We store gp for b/w in bytes per second */
5073 /* Initial measurement */
5083 }
5086 /*
5087 * Ok we can't trust the pacer in this case
5088 * where we transition from un-paced to paced.
5089 * Or for that matter when the burst mitigation
5090 * was making a wild guess and got it wrong.
5091 * Stop the pacer and clear up all the aggregate
5092 * delays etc.
5093 */
5097 }
5100 /* Still a small number run an average */
5105 /* We have collected enough to move forward */
5107 }
5111 /*
5112 * We want to take 1/wma of the goodput and add in to 7/8th
5113 * of the old value weighted by the srtt. So if your measurement
5114 * period is say 2 SRTT's long you would get 1/4 as the
5115 * value, if it was like 1/2 SRTT then you would get 1/16th.
5116 *
5117 * But we must be careful not to take too much i.e. if the
5118 * srtt is say 20ms and the measurement is taken over
5119 * 400ms our weight would be 400/20 i.e. 20. On the
5120 * other hand if we get a measurement over 1ms with a
5121 * 10ms rtt we only want to take a much smaller portion.
5122 */
5127 }
5130 /*
5131 * Strange why did t_srtt go back to zero?
5132 */
5135 else
5137 }
5138 /*
5139 * XXXrrs: Note for reviewers, in playing with
5140 * dynamic pacing I discovered this GP calculation
5141 * as done originally leads to some undesired results.
5142 * Basically you can get longer measurements contributing
5143 * too much to the WMA. Thus I changed it if you are doing
5144 * dynamic adjustments to only do the aportioned adjustment
5145 * if we have a very small (time wise) measurement. Longer
5146 * measurements just get there weight (defaulting to 1/8)
5147 * add to the WMA. We may want to think about changing
5148 * this to always do that for both sides i.e. dynamic
5149 * and non-dynamic... but considering lots of folks
5150 * were playing with this I did not want to change the
5151 * calculation per.se. without your thoughts.. Lawerence?
5152 * Peter??
5153 */
5158 /*
5159 * The b/w update takes no more
5160 * away then 1/2 our running total
5161 * so factor it in.
5162 */
5167 /*
5168 * Don't allow a single measurement
5169 * to account for more than 1/2 of the
5170 * WMA. This could happen on a retransmission
5171 * where utim becomes huge compared to
5172 * srtt (multiple retransmissions when using
5173 * the sending rate which factors in all the
5174 * transmissions from the first one).
5175 */
5179 }
5186 /*
5187 * The b/w update was over a small period
5188 * of time. The idea here is to prevent a small
5189 * measurement time period from counting
5190 * too much. So we scale it based on the
5191 * time so it attributes less than 1/rack_wma_divisor
5192 * of its measurement.
5193 */
5200 /*
5201 * The scaled measurement was long
5202 * enough so lets just add in the
5203 * portion of the measurement i.e. 1/rack_wma_divisor
5204 */
5208 }
5211 /*
5212 * For probe-rtt we only add it in
5213 * if its larger, all others we just
5214 * add in.
5215 */
5220 }
5221 }
5223 }
5224 /*
5225 * We only watch the growth of the GP during the initial startup
5226 * or first-slowstart that ensues. If we ever needed to watch
5227 * growth of gp outside of that period all we need to do is
5228 * remove the first clause of this if (rc_initial_ss_comp).
5229 */
5248 }
5251 /*
5252 * The round we get our measurement averaging going
5253 * is the base round so it always is the source point
5254 * for when we had our first increment. From there on
5255 * we only record the round that had a rise.
5256 */
5260 /*
5261 * Test to see if its gone up enough
5262 * to set the round count up to now. Note
5263 * that on the seeding of the 4th measurement we
5264 */
5268 /*
5269 * We went up enough to record the round.
5270 */
5285 }
5289 else
5291 }
5292 }
5293 }
5296 /* We have enough measurements now */
5303 }
5306 /* We do not update any multipliers if we are in or have seen a probe-rtt */
5314 }
5315 }
5325 /* reset the gp srtt and setup the new prev */
5327 /* Record the lost count for the next measurement */
5329 skip_measurement:
5330 /*
5331 * We restart our diffs based on the gpsrtt in the
5332 * measurement window.
5333 */
5340 /*
5341 * Gput in the stats world is in kbps where bytes_ps is
5342 * bytes per second so we do ((x * 8)/ 1000).
5343 */
5345 #ifdef STATS
5347 gput);
5348 /*
5349 * XXXLAS: This is a temporary hack, and should be
5350 * chained off VOI_TCP_GPUT when stats(9) grows an
5351 * API to deal with chained VOIs.
5352 */
5355 VOI_TCP_GPUT_ND,
5358 #endif
5360 }
5362 /*
5363 * Now are we app limited now and there is space from where we
5364 * were to where we want to go?
5365 *
5366 * We don't do the other case i.e. non-applimited here since
5367 * the next send will trigger us picking up the missing data.
5368 */
5375 /*
5376 * Yep there is enough outstanding to make a measurement here.
5377 */
5391 /* There is a full window to gain info from */
5394 /* We can only measure up to the applimited point */
5397 /*
5398 * We don't have enough to make a measurement.
5399 */
5404 }
5405 }
5407 /*
5408 * We will get no more data into the SB
5409 * this means we need to have the data available
5410 * before we start a measurement.
5411 */
5413 /* Nope not enough data. */
5415 }
5416 }
5418 /*
5419 * Now we need to find the timestamp of the send at tp->gput_seq
5420 * for the send based measurement.
5421 */
5425 /* Ok send-based limit is set */
5427 /*
5428 * Move back to include the earlier part
5429 * so our ack time lines up right (this may
5430 * make an overlapping measurement but thats
5431 * ok).
5432 */
5434 }
5445 }
5448 /* We always go from the first send */
5451 /*
5452 * If we don't find the rsm due to some
5453 * send-limit set the current time, which
5454 * basically disables the send-limit.
5455 */
5460 }
5472 /*
5473 * To make sure proper timestamp merging occurs, we need to clear
5474 * all GP marks if we don't start a measurement.
5475 */
5477 }
5478 }
5480 /*
5481 * CC wrapper hook functions
5482 */
5483 static void
5486 {
5500 }
5501 }
5502 #ifdef STATS
5505 #endif
5507 /*
5508 * We will ack all the data, time to end any
5509 * lt_bw_up we have running until something
5510 * new is sent. Note we need to use the actual
5511 * ack_rcv_time which with pacing may be different.
5512 */
5520 }
5523 }
5527 /* Measure the Goodput */
5529 }
5530 /* Which way our we limited, if not cwnd limited no advance in CA */
5533 else
5538 /* For the setting of a window past use the actual scwnd we are using */
5542 }
5546 }
5552 else
5570 }
5572 /* XXXLAS: Find a way to live without this */
5577 }
5580 }
5583 /*
5584 * We now are beyond the rxt point so lets disable
5585 * the flag.
5586 */
5590 /*
5591 * Only decrement the rc_out_at_rto if the cwnd advances
5592 * at least a whole segment. Otherwise next time the peer
5593 * acks, we won't be able to send this generaly happens
5594 * when we are in Congestion Avoidance.
5595 */
5600 }
5601 }
5602 }
5603 #ifdef STATS
5605 #endif
5608 }
5611 /*
5612 * The cwnd has grown beyond ssthresh we have
5613 * entered ca and completed our first Slowstart.
5614 */
5616 }
5617 }
5619 static void
5621 {
5626 /*
5627 * If we are doing PRR and have enough
5628 * room to send <or> we are pacing and prr
5629 * is disabled we will want to see if we
5630 * can send data (by setting r_wanted_output to
5631 * true).
5632 */
5636 }
5638 static void
5640 {
5641 /*
5642 * Now exit recovery.
5643 */
5645 }
5647 static void
5649 {
5656 /* only alert CC if we alerted when we entered */
5661 /*
5662 * Rack has burst control and pacing
5663 * so lets not set this any lower than
5664 * snd_ssthresh per RFC-6582 (option 2).
5665 */
5667 }
5668 }
5686 }
5690 /*
5691 * Suck the next prr cnt back into cwnd, but
5692 * only do that if we are not application limited.
5693 */
5695 /*
5696 * We are allowed to add back to the cwnd the amount we did
5697 * not get out if:
5698 * a) no_prr_addback is off.
5699 * b) we are not app limited
5700 * c) we are doing prr
5701 * <and>
5702 * d) it is bounded by rack_prr_addbackmax (if addback is 0, then none).
5703 */
5706 }
5709 }
5716 }
5718 }
5723 }
5725 }
5727 static void
5729 {
5734 #ifdef STATS
5736 #endif
5749 /* Check if this is the end of the initial Start-up i.e. initial slow-start */
5751 /* Yep it is the end of the initial slowstart */
5753 }
5760 }
5765 }
5769 /*
5770 * Allow ECN reaction on ACK to CWR, if
5771 * that data segment was also CE marked.
5772 */
5780 }
5791 /* TSNH */
5796 }
5802 /* RTO was unnecessary, so reset everything. */
5809 }
5813 }
5817 }
5822 }
5830 }
5831 }
5835 {
5845 else
5848 /*
5849 * Being idle is no different than the initial window. If the cc
5850 * clamps it down below the initial window raise it to the initial
5851 * window.
5852 */
5855 }
5856 }
5858 /*
5859 * Indicate whether this ack should be delayed. We can delay the ack if
5860 * following conditions are met:
5861 * - There is no delayed ack timer in progress.
5862 * - Our last ack wasn't a 0-sized window. We never want to delay
5863 * the ack that opens up a 0-sized window.
5864 * - LRO wasn't used for this segment. We make sure by checking that the
5865 * segment size is not larger than the MSS.
5866 * - Delayed acks are enabled or this is a half-synchronized T/TCP
5867 * connection.
5868 */
5869 #define DELAY_ACK(tp, tlen) \
5870 (((tp->t_flags & TF_RXWIN0SENT) == 0) && \
5871 ((tp->t_flags & TF_DELACK) == 0) && \
5872 (tlen <= tp->t_maxseg) && \
5873 (tp->t_delayed_ack || (tp->t_flags & TF_NEEDSYN)))
5877 {
5880 /*
5881 * Walk the time-order transmitted list looking for an rsm that is
5882 * not acked. This will be the one that was sent the longest time
5883 * ago that is still outstanding.
5884 */
5888 }
5890 }
5891 finish:
5893 }
5897 {
5900 /*
5901 * Walk the sequence order list backward until we hit and arrive at
5902 * the highest seq not acked. In theory when this is called it
5903 * should be the last segment (which it was not).
5904 */
5910 }
5912 }
5914 }
5916 static uint32_t
5917 rack_calc_thresh_rack(struct tcp_rack *rack, uint32_t srtt, uint32_t cts, int line, int log_allowed)
5918 {
5922 /*
5923 * lro is the flag we use to determine if we have seen reordering.
5924 * If it gets set we have seen reordering. The reorder logic either
5925 * works in one of two ways:
5926 *
5927 * If reorder-fade is configured, then we track the last time we saw
5928 * re-ordering occur. If we reach the point where enough time as
5929 * passed we no longer consider reordering has occuring.
5930 *
5931 * Or if reorder-face is 0, then once we see reordering we consider
5932 * the connection to alway be subject to reordering and just set lro
5933 * to 1.
5934 *
5935 * In the end if lro is non-zero we add the extra time for
5936 * reordering in.
5937 */
5945 /*
5946 * No time as passed since the last
5947 * reorder, mark it as reordering.
5948 */
5950 }
5952 /* Negative time? */
5954 }
5956 /* Turn off reordering seen too */
5959 }
5961 /* Reodering does not fade */
5963 }
5966 }
5970 /* Standards based pkt-delay is 1/4 srtt */
5972 }
5974 /* It must be set, if not you get 1/4 rtt */
5977 else
5979 }
5981 lro &&
5983 /*
5984 * We only increase the reordering window if we
5985 * have seen reordering <and> we have a DSACK count.
5986 */
5990 }
5991 /* SRTT * 2 is the ceiling */
5994 }
5995 /* And we don't want it above the RTO max either */
5998 }
6002 }
6004 static uint32_t
6007 {
6016 else
6019 /* Get the previous sent packet, if any */
6023 /* Exactly like the ID */
6024 if (((tp->snd_max - tp->snd_una) - rack->r_ctl.rc_sacked + rack->r_ctl.rc_holes_rxt) <= segsiz) {
6026 /*
6027 * Compensate for delayed-ack with the d-ack time.
6028 */
6032 }
6034 /* 2.1 behavior */
6037 /*
6038 * Two packets outstanding, thresh should be (2*srtt) +
6039 * possible inter-packet delay (if any).
6040 */
6047 /* Yes it was sent later (or at the same time) */
6049 }
6052 /*
6053 * Possibly compensate for delayed-ack.
6054 */
6060 }
6062 /* 2.2 behavior */
6065 /*
6066 * Compensate for delayed-ack with the d-ack time.
6067 */
6071 }
6072 }
6073 /* Not above an RTO */
6076 }
6077 /* Not above a RTO max */
6080 }
6081 /* Apply user supplied min TLP */
6084 }
6086 }
6088 static uint32_t
6090 {
6091 /*
6092 * We want the rack_rtt which is the
6093 * last rtt we measured. However if that
6094 * does not exist we fallback to the srtt (which
6095 * we probably will never do) and then as a last
6096 * resort we use RACK_INITIAL_RTO if no srtt is
6097 * yet set.
6098 */
6104 }
6108 {
6109 /*
6110 * Check to see that we don't need to fall into recovery. We will
6111 * need to do so if our oldest transmit is past the time we should
6112 * have had an ack.
6113 */
6122 }
6132 }
6138 }
6141 }
6142 /* Ok if we reach here we are over-due and this guy can be sent */
6145 }
6147 static uint32_t
6149 {
6160 }
6162 static uint32_t
6164 {
6165 /*
6166 * Start the FR timer, we do this based on getting the first one in
6167 * the rc_tmap. Note that if its NULL we must stop the timer. in all
6168 * events we need to stop the running timer (if its running) before
6169 * starting the new one.
6170 */
6178 /* All timers have been stopped none are to run */
6180 }
6182 /* We can't start any timer in persists */
6184 }
6189 }
6192 /* Nothing on the send map or no rack */
6193 activate_rxt:
6197 /*
6198 * Should we discount the RTX timer any?
6199 *
6200 * We want to discount it the smallest amount.
6201 * If a timer (Rack/TLP or RXT) has gone off more
6202 * recently thats the discount we want to use (now - timer time).
6203 * If the retransmit of the oldest packet was more recent then
6204 * we want to use that (now - oldest-packet-last_transmit_time).
6205 *
6206 */
6210 else
6214 }
6221 else
6225 /* Special case for KEEPINIT */
6228 rsm) {
6229 /*
6230 * We have to put a ceiling on the rxt timer
6231 * of the keep-init timeout.
6232 */
6240 else
6242 }
6243 /* Reduce timeout to the keep value if needed */
6246 }
6248 }
6250 }
6254 /* No lowest? */
6256 }
6257 }
6258 /* Convert from ms to usecs */
6265 /*
6266 * We don't start a rack timer if all we have is a
6267 * FIN outstanding.
6268 */
6270 }
6275 /*
6276 * We are not cheating, in recovery and
6277 * not enough ack's to yet get our next
6278 * retransmission out.
6279 *
6280 * Note that classified attackers do not
6281 * get to use the rack-cheat.
6282 */
6284 }
6295 }
6300 }
6302 /* Ok we need to do a TLP not RACK */
6303 activate_tlp:
6306 /*
6307 * The previous send was a TLP and we have sent
6308 * N TLP's without sending new data.
6309 */
6311 }
6314 /* We found no rsm to TLP with. */
6316 }
6318 /* If its a FIN we dont do TLP */
6321 }
6326 else
6334 /*
6335 * If another stack as run and set srtt to 1,
6336 * then the srtt was 0, so lets use the initial.
6337 */
6342 }
6345 /*
6346 * If the SRTT is not keeping up and the
6347 * rack RTT has spiked we want to use
6348 * the last RTT not the smoothed one.
6349 */
6354 }
6366 srtt,
6368 }
6371 }
6373 /*
6374 * If the TLP time works out to larger than the max
6375 * RTO lets not do TLP.. just RTO.
6376 */
6378 }
6379 }
6384 }
6388 }
6390 static void
6392 {
6395 /*
6396 * Stop the goodput now, the calling of the
6397 * measurement function clears the flag.
6398 */
6400 RACK_QUALITY_PERSIST);
6401 }
6402 #ifdef NETFLIX_SHARED_CWND
6406 }
6407 #endif
6412 /* Suspend our LT BW measurement */
6420 }
6424 }
6433 }
6434 }
6436 static void
6438 {
6442 }
6443 #ifdef NETFLIX_SHARED_CWND
6447 }
6448 #endif
6452 /*
6453 * Do we count this as if a probe-rtt just
6454 * finished?
6455 */
6466 }
6468 /* Yes, we count it as a probe-rtt. */
6479 }
6480 }
6481 }
6483 /* Continue where we left off */
6487 }
6497 }
6499 static void
6502 {
6515 /* Hijack other fields as needed */
6534 }
6536 }
6538 static void
6539 rack_log_wakeup(struct tcpcb *tp, struct tcp_rack *rack, struct sockbuf *sb, uint32_t len, int type)
6540 {
6556 }
6557 }
6559 static void
6562 {
6576 }
6578 /* Already on the pacer */
6580 }
6584 }
6588 /* Now early/late accounting */
6591 /*
6592 * We have a early carry over set,
6593 * we can always add more time so we
6594 * can always make this compensation.
6595 *
6596 * Note if ack's are allowed to wake us do not
6597 * penalize the next timer for being awoke
6598 * by an ack aka the rc_agg_early (non-paced mode).
6599 */
6603 }
6606 /*
6607 * This is harder, we can
6608 * compensate some but it
6609 * really depends on what
6610 * the current pacing time is.
6611 */
6613 /*
6614 * We can't compensate for it all.
6615 * And we have to have some time
6616 * on the clock. We always have a min
6617 * 10 slots (10 x 10 i.e. 100 usecs).
6618 */
6620 /* We gain delay */
6624 /* We take off some */
6627 }
6631 /* Make sure we have 100 useconds at minimum */
6635 }
6638 }
6640 /* r_use_hpts_min is on and so is DGP */
6650 }
6651 }
6655 /*
6656 * We are enforcing a min pacing timer
6657 * based on our hpts min timeout.
6658 */
6664 }
6665 }
6670 }
6674 else
6676 /*
6677 * If no timers are going to run and we will fall off the hptsi
6678 * wheel, we resort to a keep-alive timer if its configured.
6679 */
6684 /*
6685 * Ok we have no timer (persists, rack, tlp, rxt or
6686 * del-ack), we don't have segments being paced. So
6687 * all that is left is the keepalive timer.
6688 */
6690 /* Get the established keep-alive time */
6693 /*
6694 * Get the initial setup keep-alive time,
6695 * note that this is probably not going to
6696 * happen, since rack will be running a rxt timer
6697 * if a SYN of some sort is outstanding. It is
6698 * actually handled in rack_timeout_rxt().
6699 */
6701 }
6704 /*
6705 * We want to instead not wake up a long time from
6706 * now but to wake up about the time we would
6707 * exit probe-rtt and initiate a keep-alive ack.
6708 * This will get us out of probe-rtt and update
6709 * our min-rtt.
6710 */
6712 }
6713 }
6714 }
6717 /*
6718 * RACK, TLP, persists and RXT timers all are restartable
6719 * based on actions input .. i.e we received a packet (ack
6720 * or sack) and that changes things (rw, or snd_una etc).
6721 * Thus we can restart them with a new value. For
6722 * keep-alive, delayed_ack we keep track of what was left
6723 * and restart the timer with a smaller value.
6724 */
6727 }
6729 /*
6730 * Hack alert for now we can't time-out over 2,147,483
6731 * seconds (a bit more than 596 hours), which is probably ok
6732 * :).
6733 */
6737 }
6743 /*
6744 * We have no good estimate yet for the
6745 * old clunky burst mitigation or the
6746 * real pacing. And the tlp or rxt is smaller
6747 * than the pacing calculation. Lets not
6748 * pace that long since we know the calculation
6749 * so far is not accurate.
6750 */
6752 }
6753 /**
6754 * Turn off all the flags for queuing by default. The
6755 * flags have important meanings to what happens when
6756 * LRO interacts with the transport. Most likely (by default now)
6757 * mbuf_queueing and ack compression are on. So the transport
6758 * has a couple of flags that control what happens (if those
6759 * are not on then these flags won't have any effect since it
6760 * won't go through the queuing LRO path).
6761 *
6762 * TF2_MBUF_QUEUE_READY - This flags says that I am busy
6763 * pacing output, so don't disturb. But
6764 * it also means LRO can wake me if there
6765 * is a SACK arrival.
6766 *
6767 * TF2_DONT_SACK_QUEUE - This flag is used in conjunction
6768 * with the above flag (QUEUE_READY) and
6769 * when present it says don't even wake me
6770 * if a SACK arrives.
6771 *
6772 * The idea behind these flags is that if we are pacing we
6773 * set the MBUF_QUEUE_READY and only get woken up if
6774 * a SACK arrives (which could change things) or if
6775 * our pacing timer expires. If, however, we have a rack
6776 * timer running, then we don't even want a sack to wake
6777 * us since the rack timer has to expire before we can send.
6778 *
6779 * Other cases should usually have none of the flags set
6780 * so LRO can call into us.
6781 */
6786 /*
6787 * A pacing timer (slot) is being set, in
6788 * such a case we cannot send (we are blocked by
6789 * the timer). So lets tell LRO that it should not
6790 * wake us unless there is a SACK. Note this only
6791 * will be effective if mbuf queueing is on or
6792 * compressed acks are being processed.
6793 */
6795 /*
6796 * But wait if we have a Rack timer running
6797 * even a SACK should not disturb us (with
6798 * the exception of r_rr_config 3).
6799 */
6805 /*
6806 * When DND is on, we only let a sack
6807 * interrupt us if we are not in recovery.
6808 *
6809 * If DND is off, then we never hit here
6810 * and let all sacks wake us up.
6811 *
6812 */
6814 }
6815 }
6817 /*
6818 * Ahh but wait, this is that special case
6819 * where the pacing timer can be disturbed
6820 * backout the changes (used for non-paced
6821 * burst limiting).
6822 */
6824 TF2_MBUF_QUEUE_READY);
6825 }
6829 /*
6830 * Arrange for the hpts to kick back in after the
6831 * t-o if the t-o does not cause a send.
6832 */
6842 }
6844 /*
6845 * With respect to t_flags2(?) here, lets let any new acks wake
6846 * us up here. Since we are not pacing (no pacing timer), output
6847 * can happen so we should let it. If its a Rack timer, then any inbound
6848 * packet probably won't change the sending (we will be blocked)
6849 * but it may change the prr stats so letting it in (the set defaults
6850 * at the start of this block) are good enough.
6851 */
6858 /* No timer starting */
6859 #ifdef INVARIANTS
6863 }
6864 #endif
6865 }
6869 }
6871 static void
6874 {
6882 /* Got up to all that were marked sack-passed */
6884 }
6888 /* We now consider it lost */
6892 /* Past here it won't be lost so stop */
6894 }
6895 }
6896 }
6897 }
6899 /*
6900 * RACK Timer, here we simply do logging and house keeping.
6901 * the normal rack_output() function will call the
6902 * appropriate thing to check if we need to do a RACK retransmit.
6903 * We return 1, saying don't proceed with rack_output only
6904 * when all timers have been stopped (destroyed PCB?).
6905 */
6906 static int
6908 {
6909 /*
6910 * This timer simply provides an internal trigger to send out data.
6911 * The check_recovery_mode call will see if there are needed
6912 * retransmissions, if so we will enter fast-recovery. The output
6913 * call may or may not do the same thing depending on sysctl
6914 * settings.
6915 */
6925 /* We need to stroke any lost that are now declared as lost */
6930 /*
6931 * Don't accumulate extra pacing delay
6932 * we are allowing the rack timer to
6933 * over-ride pacing i.e. rrr takes precedence
6934 * if the pacing interval is longer than the rrr
6935 * time (in other words we get the min pacing
6936 * time versus rrr pacing time).
6937 */
6939 }
6940 }
6943 /* restart a timer and return 1 */
6947 }
6949 }
6953 static void
6955 {
6958 /*
6959 * The trailing space changed, mbufs can grow
6960 * at the tail but they can't shrink from
6961 * it, KASSERT that. Adjust the orig_m_len to
6962 * compensate for this change.
6963 */
6967 rsm,
6974 }
6976 /*
6977 * Mbuf shrank, trimmed off the top by an ack, our
6978 * offset changes.
6979 */
6987 else
6990 #ifdef INVARIANTS
6994 #endif
6995 }
6996 }
6998 static void
6999 rack_setup_offset_for_rsm(struct tcp_rack *rack, struct rack_sendmap *src_rsm, struct rack_sendmap *rsm)
7000 {
7007 /* Fix up the orig_m_len and possibly the mbuf offset */
7009 }
7013 /* Move out past this mbuf */
7020 /* This should *not* happen which is why there is a kassert */
7032 }
7033 }
7038 }
7043 {
7060 }
7061 /* Now if we have SYN flag we keep it on the left edge */
7064 /* Now if we have a FIN flag we keep it on the right edge */
7067 /* Push bit must go to the right edge as well */
7070 /* Clone over the state of the hw_tls flag */
7072 /*
7073 * Now we need to find nrsm's new location in the mbuf chain
7074 * we basically calculate a new offset, which is soff +
7075 * how much is left in original rsm. Then we walk out the mbuf
7076 * chain to find the righ position, it may be the same mbuf
7077 * or maybe not.
7078 */
7084 }
7090 {
7091 /*
7092 * We are merging two ack'd RSM's,
7093 * the l_rsm is on the left (lower seq
7094 * values) and the r_rsm is on the right
7095 * (higher seq value). The simplest way
7096 * to merge these is to move the right
7097 * one into the left. I don't think there
7098 * is any reason we need to try to find
7099 * the oldest (or last oldest retransmitted).
7100 */
7109 /* This really should not happen */
7112 }
7114 /* Now the flags */
7123 /*
7124 * If both are app-limited then let the
7125 * free lower the count. If right is app
7126 * limited and left is not, transfer.
7127 */
7132 }
7134 /*
7135 * We keep the largest value, which is the newest
7136 * send. We do this in case a segment that is
7137 * joined together and not part of a GP estimate
7138 * later gets expanded into the GP estimate.
7139 *
7140 * We prohibit the merging of unlike kinds i.e.
7141 * all pieces that are in the GP estimate can be
7142 * merged and all pieces that are not in a GP estimate
7143 * can be merged, but not disimilar pieces. Combine
7144 * this with taking the highest here and we should
7145 * be ok unless of course the client reneges. Then
7146 * all bets are off.
7147 */
7151 }
7152 /*
7153 * When merging two RSM's we also need to consider the ack time and keep
7154 * newest. If the ack gets merged into a measurement then that is the
7155 * one we will want to be using.
7156 */
7161 /* Transfer the split limit to the map we free */
7164 }
7168 }
7170 /*
7171 * TLP Timer, here we simply setup what segment we want to
7172 * have the TLP expire on, the normal rack_output() will then
7173 * send it out.
7174 *
7175 * We return 1, saying don't proceed with rack_output only
7176 * when all timers have been stopped (destroyed PCB?).
7177 */
7178 static int
7180 {
7181 /*
7182 * Tail Loss Probe.
7183 */
7192 /* Its not time yet */
7194 }
7198 }
7199 /*
7200 * A TLP timer has expired. We have been idle for 2 rtts. So we now
7201 * need to figure out how to force a full MSS segment out.
7202 */
7213 /* special case, we need a retransmission */
7216 }
7221 }
7223 }
7226 /*
7227 * If this is the second in a row
7228 * TLP and we are doing a measurement
7229 * its time to abandon the measurement.
7230 * Something is likely broken on
7231 * the clients network and measuring a
7232 * broken network does us no good.
7233 */
7239 }
7240 /*
7241 * Check our send oldest always settings, and if
7242 * there is an oldest to send jump to the need_retran.
7243 */
7248 /* New data is available */
7253 /* We are rwnd limited */
7255 }
7257 /* not enough to fill a MTU */
7259 }
7261 /* Unlikely */
7267 }
7271 /* Set the send-new override */
7274 else
7276 }
7280 }
7281 need_retran:
7282 /*
7283 * Ok we need to arrange the last un-acked segment to be re-sent, or
7284 * optionally the first un-acked segment.
7285 */
7293 }
7294 }
7296 #ifdef TCP_BLACKBOX
7298 #endif
7300 }
7302 /*
7303 * We had a collapsed window, lets find
7304 * the point before the collapse.
7305 */
7310 }
7312 /* Huh */
7314 }
7315 }
7317 /*
7318 * We need to split this the last segment in two.
7319 */
7324 /*
7325 * No memory to split, we will just exit and punt
7326 * off to the RXT timer.
7327 */
7329 }
7333 #ifndef INVARIANTS
7335 #else
7339 }
7340 #endif
7344 }
7346 }
7348 send:
7349 /* Make sure output path knows we are doing a TLP */
7354 out:
7357 }
7359 /*
7360 * Delayed ack Timer, here we simply need to setup the
7361 * ACK_NOW flag and remove the DELACK flag. From there
7362 * the output routine will send the ack out.
7363 *
7364 * We only return 1, saying don't proceed, if all timers
7365 * are stopped (destroyed PCB?).
7366 */
7367 static int
7369 {
7377 }
7381 {
7391 }
7396 /* This does send an ack so kill any D-ack timer */
7403 }
7405 /*
7406 * Persists timer, here we simply send the
7407 * same thing as a keepalive will.
7408 * the one byte send.
7409 *
7410 * We only return 1, saying don't proceed, if all timers
7411 * are stopped (destroyed PCB?).
7412 */
7413 static int
7415 {
7425 }
7426 /*
7427 * Persistence timer into zero window. Force a byte to be output, if
7428 * possible.
7429 */
7431 /*
7432 * Hack: if the peer is dead/unreachable, we do not time out if the
7433 * window is closed. After a full backoff, drop the connection if
7434 * the idle time (no responses to probes) reaches the maximum
7435 * backoff that we would use if retransmitting.
7436 */
7445 }
7450 /*
7451 * If the user has closed the socket then drop a persisting
7452 * connection after a much reduced timeout.
7453 */
7461 }
7463 /* only set it if we were answered */
7467 }
7470 }
7473 out:
7478 }
7480 /*
7481 * If a keepalive goes off, we had no other timers
7482 * happening. We always return 1 here since this
7483 * routine either drops the connection or sends
7484 * out a segment with respond.
7485 */
7486 static int
7488 {
7493 /*
7494 * Keep-alive timer went off; send something or drop connection if
7495 * idle for too long.
7496 */
7504 /*
7505 * Send a packet designed to force a response if the peer is
7506 * up and reachable: either an ACK if the connection is
7507 * still alive, or an RST if the peer has closed the
7508 * connection due to timeout or reboot. Using sequence
7509 * number tp->snd_una-1 causes the transmitted zero-length
7510 * segment to lie outside the receive window; by the
7511 * protocol spec, this requires the correspondent TCP to
7512 * respond.
7513 */
7516 }
7519 dropit:
7523 }
7525 /*
7526 * Retransmit helper function, clear up all the ack
7527 * flags and take care of important book keeping.
7528 */
7529 static void
7531 {
7532 /*
7533 * The retransmit timer went off, all sack'd blocks must be
7534 * un-acked.
7535 */
7552 /*
7553 * We do not clear the scoreboard until we have had
7554 * more than rack_rxt_scoreboard_clear_thresh time-outs.
7555 */
7561 }
7562 /*
7563 * Ideally we would like to be able to
7564 * mark SACK-PASS on anything not acked here.
7565 *
7566 * However, if we do that we would burst out
7567 * all that data 1ms apart. This would be unwise,
7568 * so for now we will just let the normal rxt timer
7569 * and tlp timer take care of it.
7570 *
7571 * Also we really need to stick them back in sequence
7572 * order. This way we send in the proper order and any
7573 * sacks that come floating in will "re-ack" the data.
7574 * To do this we zap the tmap with an INIT and then
7575 * walk through and place every rsm in the tail queue
7576 * hash table back in its seq ordered place.
7577 */
7584 /* We must re-add it back to the tlist */
7589 }
7594 rsm->r_flags &= ~(RACK_ACKED | RACK_SACK_PASSED | RACK_WAS_SACKPASS | RACK_RWND_COLLAPSED | RACK_WAS_LOST);
7596 }
7597 /* zero the lost since it's all gone */
7599 /* Clear the count (we just un-acked them) */
7602 /* Clear the tlp rtx mark */
7613 /*
7614 * For non-sack customers new data
7615 * needs to go out as retransmits until
7616 * we retransmit up to snd_max.
7617 */
7621 }
7622 }
7624 static void
7626 {
7631 }
7634 }
7635 }
7637 static void
7639 {
7646 /*
7647 * Now convert to rack's internal format,
7648 * if required.
7649 */
7652 /*
7653 * We want a chance to stay in slowstart as
7654 * we create a connection. TCP spec says that
7655 * initially ssthresh is infinite. For our
7656 * purposes that is the snd_wnd.
7657 */
7660 }
7661 /*
7662 * We also want to assure a IW worth of
7663 * data can get inflight.
7664 */
7667 }
7669 /*
7670 * Re-transmit timeout! If we drop the PCB we will return 1, otherwise
7671 * we will setup to retransmit the lowest seq number outstanding.
7672 */
7673 static int
7675 {
7683 /*
7684 * We have had a second timeout
7685 * measurements on successive rxt's are not profitable.
7686 * It is unlikely to be of any use (the network is
7687 * broken or the client went away).
7688 */
7694 }
7699 }
7706 /*
7707 * Mark that we had a rto while in recovery
7708 * and save the ssthresh so if we go back
7709 * into recovery we will have a chance
7710 * to slowstart back to the level.
7711 */
7714 }
7717 else
7721 else
7725 /* Nothing outstanding .. nothing to do */
7727 }
7732 }
7734 }
7735 /*
7736 * Rack can only run one timer at a time, so we cannot
7737 * run a KEEPINIT (gating SYN sending) and a retransmit
7738 * timer for the SYN. So if we are in a front state and
7739 * have a KEEPINIT timer we need to check the first transmit
7740 * against now to see if we have exceeded the KEEPINIT time
7741 * (if one is set).
7742 */
7749 /* Ok we have something outstanding to test keepinit with */
7752 /* We have exceeded the KEEPINIT time */
7755 }
7756 }
7757 }
7758 /*
7759 * Retransmission timer went off. Message has not been acked within
7760 * retransmit interval. Back off to a longer retransmit interval
7761 * and retransmit one segment.
7762 */
7765 /*
7766 * If the rwnd collapsed on
7767 * the one we are retransmitting
7768 * it does not count against the
7769 * rxt count.
7770 */
7772 }
7776 drop_it:
7779 /* XXXGL: previously t_softerror was casted to uint16_t */
7783 }
7785 /*
7786 * If the SYN was retransmitted, indicate CWND to be limited
7787 * to 1 segment in cc_conn_init().
7788 */
7791 /*
7792 * first retransmit; record ssthresh and cwnd so they can be
7793 * recovered if this turns out to be a "bad" retransmit. A
7794 * retransmit is considered "bad" if an ACK for this segment
7795 * is received within RTT/2 interval; the assumption here is
7796 * that the ACK was already in flight. See "On Estimating
7797 * End-to-End Network Path Properties" by Allman and Paxson
7798 * for more details.
7799 */
7811 else
7816 /*
7817 * We enter the path for PLMTUD if connection is established or, if
7818 * connection is FIN_WAIT_1 status, reason for the last is that if
7819 * amount of data we send is very small, we could send it in couple
7820 * of packets and process straight to FIN. In that case we won't
7821 * catch ESTABLISHED state.
7822 */
7823 #ifdef INET6
7825 #else
7827 #endif
7833 /*
7834 * Idea here is that at each stage of mtu probe (usually,
7835 * 1448 -> 1188 -> 524) should be given 2 chances to recover
7836 * before further clamping down. 'tp->t_rxtshift % 2 == 0'
7837 * should take care of that.
7838 */
7843 /*
7844 * Enter Path MTU Black-hole Detection mechanism: -
7845 * Disable Path MTU Discovery (IP "DF" bit). -
7846 * Reduce MTU to lower value than what we negotiated
7847 * with peer.
7848 */
7850 /* Record that we may have found a black hole. */
7852 /* Keep track of previous MSS. */
7854 }
7856 /*
7857 * Reduce the MSS to blackhole value or to the
7858 * default in an attempt to retransmit.
7859 */
7860 #ifdef INET6
7863 /* Use the sysctl tuneable blackhole MSS. */
7867 /* Use the default MSS. */
7869 /*
7870 * Disable Path MTU Discovery when we switch
7871 * to minmss.
7872 */
7875 }
7876 #endif
7877 #if defined(INET6) && defined(INET)
7878 else
7879 #endif
7880 #ifdef INET
7882 /* Use the sysctl tuneable blackhole MSS. */
7886 /* Use the default MSS. */
7888 /*
7889 * Disable Path MTU Discovery when we switch
7890 * to minmss.
7891 */
7894 }
7895 #endif
7897 /*
7898 * If further retransmissions are still unsuccessful
7899 * with a lowered MTU, maybe this isn't a blackhole
7900 * and we restore the previous MSS and blackhole
7901 * detection flags. The limit '6' is determined by
7902 * giving each probe stage (1448, 1188, 524) 2
7903 * chances to recover.
7904 */
7911 /*
7912 * The MSS is so small we should not
7913 * process incoming SACK's since we are
7914 * subject to attack in such a case.
7915 */
7919 }
7921 }
7922 }
7923 }
7924 /*
7925 * Disable RFC1323 and SACK if we haven't got any response to
7926 * our third SYN to work-around some broken terminal servers
7927 * (most of which have hopefully been retired) that have bad VJ
7928 * header compression code which trashes TCP segments containing
7929 * unknown-to-them TCP options.
7930 */
7934 /*
7935 * If we backed off this far, our srtt estimate is probably bogus.
7936 * Clobber it so we'll take the next rtt measurement as our srtt;
7937 * move the current srtt into rttvar to keep the current retransmit
7938 * times until then.
7939 */
7941 #ifdef INET6
7944 else
7945 #endif
7949 }
7955 out:
7957 }
7959 static int
7960 rack_process_timers(struct tcpcb *tp, struct tcp_rack *rack, uint32_t cts, uint8_t hpts_calling, uint8_t *doing_tlp)
7961 {
7967 /*
7968 * We have a goodput in progress
7969 * and we have entered a late state.
7970 * Do we have enough data in the sb
7971 * to handle the GPUT request?
7972 */
7979 /*
7980 * There are not enough bytes in the socket
7981 * buffer that have been sent to cover this
7982 * measurement. Cancel it.
7983 */
7989 }
7990 }
7993 }
7995 /* no timers on listen sockets */
7999 }
8002 /*
8003 * For the rack timer when we
8004 * are on a min-timeout (which means rrr_conf = 3)
8005 * we don't want to check the timer. It may
8006 * be going off for a pace and thats ok we
8007 * want to send the retransmit (if its ready).
8008 *
8009 * If its on a normal rack timer (non-min) then
8010 * we will check if its expired.
8011 */
8013 }
8021 }
8023 /*
8024 * A user send or queued mbuf (sack) has called us? We
8025 * return 0 and let the pacing guards
8026 * deal with it if they should or
8027 * should not cause a send.
8028 */
8032 }
8033 /*
8034 * Ok our timer went off early and we are not paced false
8035 * alarm, go back to sleep. We make sure we don't have
8036 * no-sack wakeup on since we no longer have a PKT_OUTPUT
8037 * flag in place.
8038 */
8045 }
8046 skip_time_check:
8066 }
8069 }
8071 static void
8073 {
8085 /* If we were not delayed cancel out the flag. */
8089 }
8094 /*
8095 * Canceling timer's when we have no output being
8096 * paced. We also must remove ourselves from the
8097 * hpts.
8098 */
8101 }
8103 }
8106 }
8108 static int
8110 {
8119 }
8121 static void
8123 {
8124 /*
8125 * Assure no timers are running.
8126 */
8128 /* We enter in persists, set the flag appropriately */
8130 }
8133 }
8134 }
8136 static void
8139 {
8146 }
8148 /* Peg the count/index */
8154 }
8156 /*
8157 * We retransmitted it putting it back in flight
8158 * remove the lost desgination and reduce the
8159 * bytes considered lost.
8160 */
8166 else
8168 }
8171 /*
8172 * Here we don't add in the len of send, since its already
8173 * in snduna <->snd_max.
8174 */
8178 /* Problably MTU discovery messing with us */
8181 }
8185 }
8186 /* Lets make sure it really is in or not the GP window */
8191 /* Take off the must retransmit flag, if its on */
8196 /*
8197 * We have retransmitted all we need. Clear
8198 * any must retransmit flags.
8199 */
8202 }
8204 }
8205 /* Remove any collapsed flag */
8208 /* We have retransmitted due to the SACK pass */
8211 }
8212 }
8214 static uint32_t
8217 {
8218 /*
8219 * We (re-)transmitted starting at rsm->r_start for some length
8220 * (possibly less than r_end.
8221 */
8230 /*
8231 * We retransmitted the whole piece or more than the whole
8232 * slopping into the next rsm.
8233 */
8241 /* Hangs over the end return whats left */
8245 }
8246 /* We don't get out of this block. */
8247 }
8248 /*
8249 * Here we retransmitted less than the whole thing which means we
8250 * have to split this into what was transmitted and what was not.
8251 */
8254 /*
8255 * We can't get memory, so lets not proceed.
8256 */
8259 }
8260 /*
8261 * So here we are going to take the original rsm and make it what we
8262 * retransmitted. nrsm will be the tail portion we did not
8263 * retransmit. For example say the chunk was 1, 11 (10 bytes). And
8264 * we retransmitted 5 bytes i.e. 1, 5. The original piece shrinks to
8265 * 1, 6 and the new piece will be 6, 11.
8266 */
8270 #ifndef INVARIANTS
8272 #else
8276 }
8277 #endif
8281 }
8284 /* Log a split of rsm into rsm and nrsm */
8288 }
8290 static void
8295 {
8302 /*
8303 * Add to the RACK log of packets in flight or retransmitted. If
8304 * there is a TS option we will use the TS echoed, if not we will
8305 * grab a TS.
8306 *
8307 * Retransmissions will increment the count and move the ts to its
8308 * proper place. Note that if options do not include TS's then we
8309 * won't be able to effectively use the ACK for an RTT on a retran.
8310 *
8311 * Notes about r_start and r_end. Lets consider a send starting at
8312 * sequence 1 for 10 bytes. In such an example the r_start would be
8313 * 1 (starting sequence) but the r_end would be r_start+len i.e. 11.
8314 * This means that r_end is actually the first sequence for the next
8315 * slot (11).
8316 *
8317 */
8318 /*
8319 * If err is set what do we do XXXrrs? should we not add the thing?
8320 * -- i.e. return if err != 0 or should we pretend we sent it? --
8321 * i.e. proceed with add ** do this for now.
8322 */
8325 /*
8326 * We don't log errors -- we could but snd_max does not
8327 * advance in this case either.
8328 */
8332 /*
8333 * We don't log resets and we return immediately from
8334 * sending
8335 */
8337 }
8342 /*
8343 * The call to rack_log_output is made before bumping
8344 * snd_max. This means we can record one extra byte on a SYN
8345 * or FIN if seq_out is adding more on and a FIN is present
8346 * (and we are not resending).
8347 */
8349 len++;
8351 len++;
8352 }
8354 /* Are sending an old segment to induce an ack (keep-alive)? */
8356 }
8358 /* huh? should we panic? */
8365 else
8367 }
8369 /* We don't log zero window probes */
8371 }
8374 }
8375 /* First question is it a retransmission or new? */
8377 /* Its new */
8379 again:
8382 /*
8383 * Hmm out of memory and the tcb got destroyed while
8384 * we tried to wait.
8385 */
8387 }
8392 }
8400 /* The data space is one beyond snd_una */
8402 }
8407 /*
8408 * save off the mbuf location that
8409 * sndmbuf_noadv returned (which is
8410 * where we started copying from)..
8411 */
8414 /*
8415 * Here we do add in the len of send, since its not yet
8416 * reflected in in snduna <->snd_max
8417 */
8424 }
8425 /* rsm->m will be NULL if RACK_HAS_SYN or RACK_HAS_FIN is set */
8428 /*
8429 * XXXrrs Question, will this happen?
8430 *
8431 * If sbsndptr is set at the correct place
8432 * then s_moff should always be somewhere
8433 * within rsm->m. But if the sbsndptr was
8434 * off then that won't be true. If it occurs
8435 * we need to walkout to the correct location.
8436 */
8445 }
8447 }
8453 }
8456 /* Log a new rsm */
8458 #ifndef INVARIANTS
8460 #else
8464 }
8465 #endif
8471 /* First time through we set the start too */
8474 }
8475 /*
8476 * Special case detection, is there just a single
8477 * packet outstanding when we are not in recovery?
8478 *
8479 * If this is true mark it so.
8480 */
8488 }
8490 }
8491 /*
8492 * If we reach here its a retransmission and we need to find it.
8493 */
8494 more:
8499 /* No hints sorry */
8501 }
8508 }
8509 }
8510 /* Ok it was not the last pointer go through it the hard way. */
8511 refind:
8520 }
8521 }
8523 /* Transmitted within this piece */
8524 /*
8525 * Ok we must split off the front and then let the
8526 * update do the rest
8527 */
8532 }
8533 /*
8534 * copy rsm to nrsm and then trim the front of rsm
8535 * to not include this part.
8536 */
8539 #ifndef INVARIANTS
8541 #else
8545 }
8546 #endif
8550 }
8557 }
8558 }
8559 /*
8560 * Hmm not found in map did they retransmit both old and on into the
8561 * new?
8562 */
8566 #ifdef INVARIANTS
8573 }
8577 #endif
8579 #ifdef INVARIANTS
8580 /*
8581 * Hmm beyond sndmax? (only if we are using the new rtt-pack
8582 * flag)
8583 */
8586 #endif
8587 }
8588 }
8590 /*
8591 * Record one of the RTT updates from an ack into
8592 * our sample structure.
8593 */
8595 static void
8598 {
8602 }
8606 }
8612 }
8618 /*
8619 * If the rsm had a just return
8620 * hit it then we can't trust the
8621 * rtt measurement for buffer deterimination
8622 * Note that a confidence of 2, indicates
8623 * SACK'd which overrides the r_just_ret or
8624 * the r_one_out_nr. If it was a CUM-ACK and
8625 * we had only two outstanding, but get an
8626 * ack for only 1. Then that also lowers our
8627 * confidence.
8628 */
8630 }
8634 /*
8635 * We take anything with no current confidence
8636 * saved.
8637 */
8642 /*
8643 * Once we have a confident number,
8644 * we can update it with a smaller
8645 * value since this confident number
8646 * may include the DSACK time until
8647 * the next segment (the second one) arrived.
8648 */
8652 }
8653 }
8658 }
8660 /*
8661 * Collect new round-trip time estimate
8662 * and update averages and current timeout.
8663 */
8664 static void
8666 {
8671 /* No valid sample */
8674 /* We are to use the lowest RTT seen in a single ack */
8677 /* We are to use the highest RTT seen in a single ack */
8680 /* We are to use the average RTT seen in a single ack */
8684 #ifdef INVARIANTS
8686 #endif
8688 }
8692 /*
8693 * With no RTT we have to accept
8694 * even one we are not confident of.
8695 */
8699 /* update the running gp srtt */
8702 }
8704 /*
8705 * record the low and high for highly buffered path computation,
8706 * we only do this if we are confident (not a retransmission).
8707 */
8710 }
8712 /*
8713 * Currently once we declare a path has
8714 * highly buffered there is no going
8715 * back, which may be a problem...
8716 */
8721 RACK_RTTS_SEEHBP);
8723 }
8724 }
8725 }
8728 /*
8729 * If we are highly confident of it <or> it was
8730 * never retransmitted we accept it as the last us_rtt.
8731 */
8733 /* The lowest rtt can be set if its was not retransmited */
8738 }
8739 }
8742 /*
8743 * We keep a simple srtt in microseconds, like our rtt
8744 * measurement. We don't need to do any tricks with shifting
8745 * etc. Instead we just add in 1/8th of the new measurement
8746 * and subtract out 1/8 of the old srtt. We do the same with
8747 * the variance after finding the absolute value of the
8748 * difference between this sample and the current srtt.
8749 */
8751 /* Take off 1/8th of the current sRTT */
8753 /* Add in 1/8th of the new RTT just measured */
8757 /* Now lets make the absolute value of the variance */
8760 /* Subtract out 1/8th */
8762 /* Add in 1/8th of the new variance we just saw */
8767 /*
8768 * No rtt measurement yet - use the unsmoothed rtt. Set the
8769 * variance to half the rtt (so our first retransmit happens
8770 * at 3*rtt).
8771 */
8774 }
8779 #ifdef STATS
8781 /* Send in the microsecond rtt used for rxt timeout purposes */
8784 /* Send in the millisecond rtt used for rxt timeout purposes */
8787 /* Round up */
8791 /* Send in the millisecond rtt has close to the path RTT as we can get */
8794 /* Round up */
8798 /* Send in the microsecond rtt has close to the path RTT as we can get */
8800 }
8801 stats_voi_update_abs_u32(tp->t_stats, VOI_TCP_PATHRTT, imax(0, rack->r_ctl.rack_rs.rs_us_rtt));
8802 #endif
8804 /*
8805 * the retransmit should happen at rtt + 4 * rttvar. Because of the
8806 * way we do the smoothing, srtt and rttvar will each average +1/2
8807 * tick of bias. When we compute the retransmit timer, we want 1/2
8808 * tick of rounding and 1 extra tick because of +-1/2 tick
8809 * uncertainty in the firing of the timer. The bias will give us
8810 * exactly the 1.5 tick we need. But, because the bias is
8811 * statistical, we have to test that we don't drop below the minimum
8812 * feasible timer (which is 2 ticks).
8813 */
8819 }
8822 static void
8824 {
8825 /*
8826 * Apply to filter the inbound us-rtt at us_cts.
8827 */
8834 /* We just hit a new lower rtt time */
8837 /*
8838 * Only count it if its lower than what we saw within our
8839 * calculated range.
8840 */
8846 /*
8847 * We are seeing a new lower rtt very close
8848 * to the time that we would have entered probe-rtt.
8849 * This is probably due to the fact that a peer flow
8850 * has entered probe-rtt. Lets go in now too.
8851 */
8860 }
8861 }
8863 }
8864 }
8865 }
8867 static int
8870 {
8877 /* Already done */
8880 /* Not allowed */
8882 }
8890 }
8894 }
8907 }
8908 }
8909 if (TSTMP_GT(tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time), rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)]))
8910 us_rtt = tcp_tv_to_usectick(&rack->r_ctl.act_rcv_time) - (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt-1)];
8911 else
8916 /* Kick the RTT to the CC */
8918 }
8921 rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)], cts, 1);
8924 /*
8925 * We need to setup what our confidence
8926 * is in this ack.
8927 *
8928 * If the rsm was app limited and it is
8929 * less than a mss in length (the end
8930 * of the send) then we have a gap. If we
8931 * were app limited but say we were sending
8932 * multiple MSS's then we are more confident
8933 * int it.
8934 *
8935 * When we are not app-limited then we see if
8936 * the rsm is being included in the current
8937 * measurement, we tell this by the app_limited_needs_set
8938 * flag.
8939 *
8940 * Note that being cwnd blocked is not applimited
8941 * as well as the pacing delay between packets which
8942 * are sending only 1 or 2 MSS's also will show up
8943 * in the RTT. We probably need to examine this algorithm
8944 * a bit more and enhance it to account for the delay
8945 * between rsm's. We could do that by saving off the
8946 * pacing delay of each rsm (in an rsm) and then
8947 * factoring that in somehow though for now I am
8948 * not sure how :)
8949 */
8955 else
8961 }
8962 rack_log_rtt_sample_calc(rack, t, (uint32_t)rsm->r_tim_lastsent[(rsm->r_rtr_cnt - 1)], cts, 2);
8965 }
8968 /* Segment was a TLP and our retrans matched */
8971 }
8972 }
8976 /* New more recent rack_tmit_time */
8981 }
8983 }
8984 /*
8985 * We clear the soft/rxtshift since we got an ack.
8986 * There is no assurance we will call the commit() function
8987 * so we need to clear these to avoid incorrect handling.
8988 */
8997 /*
8998 * Now which timestamp does it match? In this block the ACK
8999 * must be coming from a previous transmission.
9000 */
9007 /*
9008 * Kick the RTT to the CC, here
9009 * we lie a bit in that we know the
9010 * retransmission is correct even though
9011 * we retransmitted. This is because
9012 * we match the timestamps.
9013 */
9016 else
9019 }
9021 /*
9022 * The peer ack'd from our previous
9023 * transmission. We have a spurious
9024 * retransmission and thus we dont
9025 * want to update our rack_rtt.
9026 *
9027 * Hmm should there be a CC revert here?
9028 *
9029 */
9031 }
9038 }
9039 }
9043 /* New more recent rack_tmit_time */
9048 }
9053 }
9054 }
9055 /* If we are logging log out the sendmap */
9059 }
9060 }
9063 /*
9064 * Ok its a SACK block that we retransmitted. or a windows
9065 * machine without timestamps. We can tell nothing from the
9066 * time-stamp since its not there or the time the peer last
9067 * received a segment that moved forward its cum-ack point.
9068 */
9069 ts_not_found:
9075 /*
9076 * We retransmitted and the ack came back in less
9077 * than the smallest rtt we have observed. We most
9078 * likely did an improper retransmit as outlined in
9079 * 6.2 Step 2 point 2 in the rack-draft so we
9080 * don't want to update our rack_rtt. We in
9081 * theory (in future) might want to think about reverting our
9082 * cwnd state but we won't for now.
9083 */
9086 /*
9087 * We retransmitted it and the retransmit did the
9088 * job.
9089 */
9095 }
9096 }
9100 /* New more recent rack_tmit_time */
9105 }
9107 }
9108 }
9110 }
9112 /*
9113 * Mark the SACK_PASSED flag on all entries prior to rsm send wise.
9114 */
9115 static void
9118 {
9122 /* Get our rxt threshold for lost consideration */
9124 /* Now start looking at rsm's */
9129 /* Skip original segment he is acked */
9131 }
9133 /*
9134 * Skip ack'd segments, though we
9135 * should not see these, since tmap
9136 * should not have ack'd segments.
9137 */
9139 }
9141 /*
9142 * If the peer dropped the rwnd on
9143 * these then we don't worry about them.
9144 */
9146 }
9147 /* Check lost state */
9153 /* We consider it lost */
9156 }
9157 }
9159 /*
9160 * We found one that is already marked
9161 * passed, we have been here before and
9162 * so all others below this are marked.
9163 */
9165 }
9168 }
9169 }
9171 static void
9175 tcp_seq th_ack,
9178 {
9183 /*
9184 * We were app limited, and this ack
9185 * butts up or goes beyond the point where we want
9186 * to start our next measurement. We need
9187 * to record the new gput_ts as here and
9188 * possibly update the start sequence.
9189 */
9193 /*
9194 * This is a retransmit, can we
9195 * really make any assessment at this
9196 * point? We are not really sure of
9197 * the timestamp, is it this or the
9198 * previous transmission?
9199 *
9200 * Lets wait for something better that
9201 * is not retransmitted.
9202 */
9204 }
9209 /* Do we start at a new end? */
9212 /*
9213 * When we get an ACK that just eats
9214 * up some of the rsm, we set RACK_USE_BEG
9215 * since whats at r_start (i.e. th_ack)
9216 * is left unacked and thats where the
9217 * measurement now starts.
9218 */
9220 }
9223 /*
9224 * We use the end when the cumack
9225 * is moving forward and completely
9226 * deleting the rsm passed so basically
9227 * r_end holds th_ack.
9228 *
9229 * For SACK's we also want to use the end
9230 * since this piece just got sacked and
9231 * we want to target anything after that
9232 * in our measurement.
9233 */
9235 }
9237 /*
9238 * special case for ack moving forward,
9239 * not a sack, we need to move all the
9240 * way up to where this ack cum-ack moves
9241 * to.
9242 */
9245 else
9247 }
9250 else
9252 /*
9253 * Pick up the correct send time if we can the rsm passed in
9254 * may be equal to s_rsm if the RACK_USE_BEG was set. For the other
9255 * two cases (RACK_USE_THACK or RACK_USE_END) most likely we will
9256 * find a different seq i.e. the next send up.
9257 *
9258 * If that has not been sent, s_rsm will be NULL and we must
9259 * arrange it so this function will get called again by setting
9260 * app_limited_needs_set.
9261 */
9265 /* If we hit here we have to have *not* sent tp->gput_seq */
9267 /* Set it up so we will go through here again */
9269 }
9271 /*
9272 * We moved beyond this guy's range, re-calculate
9273 * the new end point.
9274 */
9279 }
9280 }
9281 /*
9282 * We are moving the goal post, we may be able to clear the
9283 * measure_saw_probe_rtt flag.
9284 */
9302 /*
9303 * There is no sense of continuing this measurement
9304 * because its too small to gain us anything we
9305 * trust. Skip it and that way we can start a new
9306 * measurement quicker.
9307 */
9315 /*
9316 * Reset the window further out.
9317 */
9319 }
9320 }
9323 }
9324 }
9328 {
9330 /* Behind our TLP definition or right at */
9332 }
9334 /* The start is beyond or right at our end of TLP definition */
9336 }
9337 /* It has to be a sub-part of the original TLP recorded */
9339 }
9341 static uint32_t
9345 {
9357 do_rest_ofb:
9362 /*
9363 * We are not in the right spot,
9364 * find the correct spot in the tree.
9365 */
9368 }
9370 /* TSNH */
9372 }
9373 /* Ok we have an ACK for some piece of this rsm */
9376 /*
9377 * Before any splitting or hookery is
9378 * done is it a TLP of interest i.e. rxt?
9379 */
9382 /*
9383 * We are splitting a rxt TLP, check
9384 * if we need to save off the start/end
9385 */
9388 /*
9389 * We already turned this on since we are inside
9390 * the previous one was a partially sack now we
9391 * are getting another one (maybe all of it).
9392 *
9393 */
9395 /*
9396 * Lets make sure we have all of it though.
9397 */
9402 }
9407 }
9414 }
9415 }
9416 /**
9417 * Need to split this in two pieces the before and after,
9418 * the before remains in the map, the after must be
9419 * added. In other words we have:
9420 * rsm |--------------|
9421 * sackblk |------->
9422 * rsm will become
9423 * rsm |---|
9424 * and nrsm will be the sacked piece
9425 * nrsm |----------|
9426 *
9427 * But before we start down that path lets
9428 * see if the sack spans over on top of
9429 * the next guy and it is already sacked.
9430 *
9431 */
9432 /*
9433 * Hookery can only be used if the two entries
9434 * are in the same bucket and neither one of
9435 * them staddle the bucket line.
9436 */
9447 else
9452 /**
9453 * So the next one is already acked, and
9454 * we can thus by hookery use our stack_map
9455 * to reflect the piece being sacked and
9456 * then adjust the two tree entries moving
9457 * the start and ends around. So we start like:
9458 * rsm |------------| (not-acked)
9459 * next |-----------| (acked)
9460 * sackblk |-------->
9461 * We want to end like so:
9462 * rsm |------| (not-acked)
9463 * next |-----------------| (acked)
9464 * nrsm |-----|
9465 * Where nrsm is a temporary stack piece we
9466 * use to update all the gizmos.
9467 */
9468 /* Copy up our fudge block */
9471 /* Now adjust our tree blocks */
9476 /* Now we must adjust back where next->m is */
9478 /*
9479 * Which timestamp do we keep? It is rather
9480 * important in GP measurements to have the
9481 * accurate end of the send window.
9482 *
9483 * We keep the largest value, which is the newest
9484 * send. We do this in case a segment that is
9485 * joined together and not part of a GP estimate
9486 * later gets expanded into the GP estimate.
9487 *
9488 * We prohibit the merging of unlike kinds i.e.
9489 * all pieces that are in the GP estimate can be
9490 * merged and all pieces that are not in a GP estimate
9491 * can be merged, but not disimilar pieces. Combine
9492 * this with taking the highest here and we should
9493 * be ok unless of course the client reneges. Then
9494 * all bets are off.
9495 */
9499 /*
9500 * And we must keep the newest ack arrival time.
9501 */
9507 /* We don't need to adjust rsm, it did not change */
9508 /* Clear out the dup ack count of the remainder */
9512 /* Now lets make sure our fudge block is right */
9514 /* Now lets update all the stats and such */
9528 else
9530 }
9535 }
9536 /*
9537 * Now we want to go up from rsm (the
9538 * one left un-acked) to the next one
9539 * in the tmap. We do this so when
9540 * we walk backwards we include marking
9541 * sack-passed on rsm (The one passed in
9542 * is skipped since it is generally called
9543 * on something sacked before removing it
9544 * from the tmap).
9545 */
9548 /*
9549 * Now that we have the next
9550 * one walk backwards from there.
9551 */
9554 }
9555 /* Now are we done? */
9558 /* Done with block */
9560 }
9563 /* Postion for the next block */
9569 /**
9570 * We can't use any hookery here, so we
9571 * need to split the map. We enter like
9572 * so:
9573 * rsm |--------|
9574 * sackblk |----->
9575 * We will add the new block nrsm and
9576 * that will be the new portion, and then
9577 * fall through after reseting rsm. So we
9578 * split and look like this:
9579 * rsm |----|
9580 * sackblk |----->
9581 * nrsm |---|
9582 * We then fall through reseting
9583 * rsm to nrsm, so the next block
9584 * picks it up.
9585 */
9588 /*
9589 * failed XXXrrs what can we do but loose the sack
9590 * info?
9591 */
9593 }
9597 #ifndef INVARIANTS
9599 #else
9603 }
9604 #endif
9608 }
9611 /* Position us to point to the new nrsm that starts the sack blk */
9613 }
9615 /* Already sacked this piece */
9618 /* Done with block */
9622 /* A partial sack to a already sacked block */
9626 /*
9627 * The end goes beyond this guy
9628 * reposition the start to the
9629 * next block.
9630 */
9635 }
9636 }
9637 }
9639 /**
9640 * The end of this block is either beyond this guy or right
9641 * at this guy. I.e.:
9642 * rsm --- |-----|
9643 * end |-----|
9644 * <or>
9645 * end |---------|
9646 */
9648 /*
9649 * Is it a TLP of interest?
9650 */
9653 /*
9654 * We are splitting a rxt TLP, check
9655 * if we need to save off the start/end
9656 */
9659 /*
9660 * We already turned this on since we are inside
9661 * the previous one was a partially sack now we
9662 * are getting another one (maybe all of it).
9663 */
9665 /*
9666 * Lets make sure we have all of it though.
9667 */
9672 }
9677 }
9684 }
9685 }
9688 /* You get a count for acking a whole segment or more */
9698 else
9700 }
9704 /* Is Reordering occuring? */
9710 }
9719 }
9723 }
9725 /* This block only - done, setup for next */
9727 }
9728 /*
9729 * There is more not coverend by this rsm move on
9730 * to the next block in the tail queue hash table.
9731 */
9738 }
9739 /**
9740 * The end of this sack block is smaller than
9741 * our rsm i.e.:
9742 * rsm --- |-----|
9743 * end |--|
9744 */
9746 /*
9747 * Is it a TLP of interest?
9748 */
9751 /*
9752 * We are splitting a rxt TLP, check
9753 * if we need to save off the start/end
9754 */
9757 /*
9758 * We already turned this on since we are inside
9759 * the previous one was a partially sack now we
9760 * are getting another one (maybe all of it).
9761 */
9763 /*
9764 * Lets make sure we have all of it though.
9765 */
9770 }
9775 }
9782 }
9783 }
9784 /*
9785 * Hookery can only be used if the two entries
9786 * are in the same bucket and neither one of
9787 * them staddle the bucket line.
9788 */
9799 else
9803 /**
9804 * Goal, we want the right remainder of rsm to shrink
9805 * in place and span from (rsm->r_start = end) to rsm->r_end.
9806 * We want to expand prev to go all the way
9807 * to prev->r_end <- end.
9808 * so in the tree we have before:
9809 * prev |--------| (acked)
9810 * rsm |-------| (non-acked)
9811 * sackblk |-|
9812 * We churn it so we end up with
9813 * prev |----------| (acked)
9814 * rsm |-----| (non-acked)
9815 * nrsm |-| (temporary)
9816 *
9817 * Note if either prev/rsm is a TLP we don't
9818 * do this.
9819 */
9826 /* Now adjust nrsm (stack copy) to be
9827 * the one that is the small
9828 * piece that was "sacked".
9829 */
9832 /*
9833 * Which timestamp do we keep? It is rather
9834 * important in GP measurements to have the
9835 * accurate end of the send window.
9836 *
9837 * We keep the largest value, which is the newest
9838 * send. We do this in case a segment that is
9839 * joined together and not part of a GP estimate
9840 * later gets expanded into the GP estimate.
9841 *
9842 * We prohibit the merging of unlike kinds i.e.
9843 * all pieces that are in the GP estimate can be
9844 * merged and all pieces that are not in a GP estimate
9845 * can be merged, but not disimilar pieces. Combine
9846 * this with taking the highest here and we should
9847 * be ok unless of course the client reneges. Then
9848 * all bets are off.
9849 */
9853 }
9854 /*
9855 * And we must keep the newest ack arrival time.
9856 */
9863 /*
9864 * Now that the rsm has had its start moved forward
9865 * lets go ahead and get its new place in the world.
9866 */
9868 /*
9869 * Now nrsm is our new little piece
9870 * that is acked (which was merged
9871 * to prev). Update the rtt and changed
9872 * based on that. Also check for reordering.
9873 */
9887 else
9889 }
9894 }
9899 /**
9900 * This is the case where our previous
9901 * block is not acked either, so we must
9902 * split the block in two.
9903 */
9906 /* failed rrs what can we do but loose the sack info? */
9908 }
9911 /*
9912 * We are splitting a rxt TLP, check
9913 * if we need to save off the start/end
9914 */
9917 /*
9918 * We already turned this on since this block is inside
9919 * the previous one was a partially sack now we
9920 * are getting another one (maybe all of it).
9921 */
9923 /*
9924 * Lets make sure we have all of it though.
9925 */
9930 }
9935 }
9942 }
9943 }
9944 /**
9945 * In this case nrsm becomes
9946 * nrsm->r_start = end;
9947 * nrsm->r_end = rsm->r_end;
9948 * which is un-acked.
9949 * <and>
9950 * rsm->r_end = nrsm->r_start;
9951 * i.e. the remaining un-acked
9952 * piece is left on the left
9953 * hand side.
9954 *
9955 * So we start like this
9956 * rsm |----------| (not acked)
9957 * sackblk |---|
9958 * build it so we have
9959 * rsm |---| (acked)
9960 * nrsm |------| (not acked)
9961 */
9966 #ifndef INVARIANTS
9968 #else
9972 }
9973 #endif
9977 }
9991 else
9993 }
9998 /* Is Reordering occuring? */
10004 }
10014 }
10015 }
10017 /*
10018 * The block was already acked.
10019 */
10021 }
10022 out:
10026 /*
10027 * Now can we merge where we worked
10028 * with either the previous or
10029 * next block?
10030 */
10035 /* Only allow merges between ones in or out of GP window */
10039 }
10043 }
10055 /* yep this and next can be merged */
10060 }
10061 /* Now what about the previous? */
10066 /* Only allow merges between ones in or out of GP window */
10070 }
10074 }
10086 /* yep the previous and this can be merged */
10091 }
10092 }
10097 }
10098 /* Save off the next one for quick reference. */
10102 }
10106 {
10111 /* Its no longer sacked, mark it so */
10113 #ifdef INVARIANTS
10117 }
10118 #endif
10120 /* Rebuild it into our tmap */
10127 }
10130 }
10131 /*
10132 * Now lets possibly clear the sack filter so we start
10133 * recognizing sacks that cover this area.
10134 */
10137 }
10141 rack_rsm_sender_update(struct tcp_rack *rack, struct tcpcb *tp, struct rack_sendmap *rsm, uint8_t from)
10142 {
10143 /*
10144 * We look at advancing the end send time for our GP
10145 * measurement tracking only as the cumulative acknowledgment
10146 * moves forward. You might wonder about this, why not
10147 * at every transmission or retransmission within the
10148 * GP window update the rc_gp_cumack_ts? Well its rather
10149 * nuanced but basically the GP window *may* expand (as
10150 * it does below) or worse and harder to track it may shrink.
10151 *
10152 * This last makes it impossible to track at the time of
10153 * the send, since you may set forward your rc_gp_cumack_ts
10154 * when you send, because that send *is* in your currently
10155 * "guessed" window, but then it shrinks. Now which was
10156 * the send time of the last bytes in the window, by the
10157 * time you ask that question that part of the sendmap
10158 * is freed. So you don't know and you will have too
10159 * long of send window. Instead by updating the time
10160 * marker only when the cumack advances this assures us
10161 * that we will have only the sends in the window of our
10162 * GP measurement.
10163 *
10164 * Another complication from this is the
10165 * merging of sendmap entries. During SACK processing this
10166 * can happen to conserve the sendmap size. That breaks
10167 * everything down in tracking the send window of the GP
10168 * estimate. So to prevent that and keep it working with
10169 * a tiny bit more limited merging, we only allow like
10170 * types to be merged. I.e. if two sends are in the GP window
10171 * then its ok to merge them together. If two sends are not
10172 * in the GP window its ok to merge them together too. Though
10173 * one send in and one send out cannot be merged. We combine
10174 * this with never allowing the shrinking of the GP window when
10175 * we are in recovery so that we can properly calculate the
10176 * sending times.
10177 *
10178 * This all of course seems complicated, because it is.. :)
10179 *
10180 * The cum-ack is being advanced upon the sendmap.
10181 * If we are not doing a GP estimate don't
10182 * proceed.
10183 */
10188 /*
10189 * If this sendmap entry is going
10190 * beyond the measurement window we had picked,
10191 * expand the measurement window by that much.
10192 */
10195 }
10196 /*
10197 * If we have not setup a ack, then we
10198 * have no idea if the newly acked pieces
10199 * will be "in our seq measurement range". If
10200 * it is when we clear the app_limited_needs_set
10201 * flag the timestamp will be updated.
10202 */
10205 /*
10206 * Finally, we grab out the latest timestamp
10207 * that this packet was sent and then see
10208 * if:
10209 * a) The packet touches are newly defined GP range.
10210 * b) The time is greater than (newer) than the
10211 * one we currently have. If so we update
10212 * our sending end time window.
10213 *
10214 * Note we *do not* do this at send time. The reason
10215 * is that if you do you *may* pick up a newer timestamp
10216 * for a range you are not going to measure. We project
10217 * out how far and then sometimes modify that to be
10218 * smaller. If that occurs then you will have a send
10219 * that does not belong to the range included.
10220 */
10228 }
10229 }
10231 static void
10232 rack_process_to_cumack(struct tcpcb *tp, struct tcp_rack *rack, register uint32_t th_ack, uint32_t cts, struct tcpopt *to, uint64_t acktime)
10233 {
10235 /*
10236 * The ACK point is advancing to th_ack, we must drop off
10237 * the packets in the rack log and calculate any eligble
10238 * RTT's.
10239 */
10242 /*
10243 * If we have some sack blocks in the filter
10244 * lets prune them out by calling sfb with no blocks.
10245 */
10247 }
10249 /* Clear any app ack remembered settings */
10251 }
10256 /* Tend any TLP that has been marked for 1/2 the seq space (its old) */
10260 /*
10261 * We have reached the point where our last rack
10262 * tlp retransmit sequence is ahead of the cum-ack.
10263 * This can only happen when the cum-ack moves all
10264 * the way around (its been a full 2^^31+1 bytes
10265 * or more since we sent a retransmitted TLP). Lets
10266 * turn off the valid flag since its not really valid.
10267 *
10268 * Note since sack's also turn on this event we have
10269 * a complication, we have to wait to age it out until
10270 * the cum-ack is by the TLP before checking which is
10271 * what the next else clause does.
10272 */
10281 /*
10282 * It is safe to start aging TLP's out.
10283 */
10285 }
10286 /* We do the same for the tlp send seq as well */
10299 /*
10300 * It is safe to start aging TLP's send.
10301 */
10303 }
10304 more:
10308 /*
10309 * For the SYN incoming case we will not
10310 * have called tcp_output for the sending of
10311 * the SYN, so there will be no map. All
10312 * other cases should probably be a panic.
10313 */
10315 }
10317 /* if we sent a FIN we often will not have map */
10319 }
10320 #ifdef INVARIANTS
10322 tp,
10325 #endif
10327 }
10329 /* Huh map is missing this */
10330 #ifdef INVARIANTS
10334 #endif
10336 }
10339 /* Now was it a retransmitted TLP? */
10342 /*
10343 * Yes, this rsm was a TLP and retransmitted, remember that
10344 * since if a DSACK comes back on this we don't want
10345 * to think of it as a reordered segment. This may
10346 * get updated again with possibly even other TLPs
10347 * in flight, but thats ok. Only when we don't send
10348 * a retransmitted TLP for 1/2 the sequences space
10349 * will it get turned off (above).
10350 */
10353 /*
10354 * We already turned this on since the end matches,
10355 * the previous one was a partially ack now we
10356 * are getting another one (maybe all of it).
10357 */
10359 /*
10360 * Lets make sure we have all of it though.
10361 */
10366 }
10371 }
10378 }
10379 }
10380 /* Now do we consume the whole thing? */
10383 /* Its all consumed. */
10388 /*
10389 * This can happen when we marked it as lost
10390 * and yet before retransmitting we get an ack
10391 * which can happen due to reordering.
10392 */
10398 else
10400 }
10404 /*
10405 * Record the time of highest cumack sent if its in our measurement
10406 * window and possibly bump out the end.
10407 */
10413 }
10416 /*
10417 * It was acked on the scoreboard -- remove
10418 * it from total
10419 */
10423 /*
10424 * There are segments ACKED on the
10425 * scoreboard further up. We are seeing
10426 * reordering.
10427 */
10435 /*
10436 * We have sent no more, and we saw an sack
10437 * then ack arrive.
10438 */
10440 }
10444 }
10450 /*
10451 * We can use the timestamp to see
10452 * if this retransmission was from the
10453 * first transmit. If so we made a mistake.
10454 */
10457 /* The first transmit is what this ack is for */
10459 }
10460 }
10464 /* Free back to zone */
10468 }
10469 /* Check for reneging */
10472 /*
10473 * The peer has moved snd_una up to
10474 * the edge of this send, i.e. one
10475 * that it had previously acked. The only
10476 * way that can be true if the peer threw
10477 * away data (space issues) that it had
10478 * previously sacked (else it would have
10479 * given us snd_una up to (rsm->r_end).
10480 * We need to undo the acked markings here.
10481 *
10482 * Note we have to look to make sure th_ack is
10483 * our rsm->r_start in case we get an old ack
10484 * where th_ack is behind snd_una.
10485 */
10487 }
10489 }
10491 /*
10492 * It was acked on the scoreboard -- remove it from
10493 * total for the part being cum-acked.
10494 */
10498 }
10499 /* And what about the lost flag? */
10501 /*
10502 * This can happen when we marked it as lost
10503 * and yet before retransmitting we get an ack
10504 * which can happen due to reordering. In this
10505 * case its only a partial ack of the send.
10506 */
10511 else
10513 }
10514 /*
10515 * Clear the dup ack count for
10516 * the piece that remains.
10517 */
10521 /*
10522 * It was retransmitted adjust the
10523 * sack holes for what was acked.
10524 */
10531 }
10532 }
10533 /*
10534 * Update where the piece starts and record
10535 * the time of send of highest cumack sent if
10536 * its in our GP range.
10537 */
10539 /* Now we need to move our offset forward too */
10543 /* Fix up the orig_m_len and possibly the mbuf offset */
10545 }
10548 /* The trim will move th_ack into r_start for us */
10550 /* Now do we need to move the mbuf fwd too? */
10551 {
10565 /*
10566 * This is a fall-back that prevents a panic. In reality
10567 * we should be able to walk the mbuf's and find our place.
10568 * At this point snd_una has not been updated with the sbcut() yet
10569 * but tqhash_trim did update rsm->r_start so the offset calcuation
10570 * should work fine. This is undesirable since we will take cache
10571 * hits to access the socket buffer. And even more puzzling is that
10572 * it happens occasionally. It should not :(
10573 */
10578 }
10579 }
10580 /*
10581 * Now save in our updated values.
10582 */
10587 }
10588 }
10592 }
10594 static void
10596 {
10601 /*
10602 * Ok we have reordering, have not sent anything, we
10603 * might want to revert the congestion state if nothing
10604 * further has SACK_PASSED on it. Lets check.
10605 *
10606 * We also get here when we have DSACKs come in for
10607 * all the data that we FR'd. Note that a rxt or tlp
10608 * timer clears this from happening.
10609 */
10615 }
10616 }
10618 /*
10619 * We went into recovery
10620 * incorrectly due to reordering!
10621 */
10632 /*
10633 * We were in recovery, had an RTO
10634 * and then re-entered recovery (more sack's arrived)
10635 * and we have properly recorded the old ssthresh from
10636 * the first recovery. We want to be able to slow-start
10637 * back to this level. The ssthresh from the timeout
10638 * and then back into recovery will end up most likely
10639 * to be min(cwnd=1mss, 2mss). Which makes it basically
10640 * so we get no slow-start after our RTO.
10641 */
10645 }
10646 }
10647 }
10649 }
10650 }
10653 static int
10655 {
10662 else
10667 /*
10668 * The DSACK is because of a TLP which we don't
10669 * do anything with the reordering window over since
10670 * it was not reordering that caused the DSACK but
10671 * our previous retransmit TLP.
10672 */
10676 }
10681 /*
10682 * This dsack is from the last sent TLP, ignore it
10683 * for reordering purposes.
10684 */
10688 }
10689 }
10696 }
10697 skip_dsack_round:
10698 /*
10699 * We keep track of how many DSACK blocks we get
10700 * after a recovery incident.
10701 */
10706 /*
10707 * False recovery most likely culprit is reordering. If
10708 * nothing else is missing we need to revert.
10709 */
10715 }
10717 }
10719 static uint32_t
10721 {
10724 }
10726 static int32_t
10728 {
10732 }
10734 static void
10736 {
10737 /* Deal with changed and PRR here (in recovery only) */
10743 /*
10744 * It is all outstanding, we are application limited
10745 * and thus we don't need more room to send anything.
10746 * Note we use tp->snd_una here and not th_ack because
10747 * the data as yet not been cut from the sb.
10748 */
10751 }
10752 /* Compute prr_sndcnt */
10757 }
10769 }
10770 sndcnt++;
10773 else
10782 else
10793 }
10794 }
10795 }
10797 static void
10798 rack_log_ack(struct tcpcb *tp, struct tcpopt *to, struct tcphdr *th, int entered_recovery, int dup_ack_struck,
10800 {
10815 /* We don't log resets */
10817 }
10825 /*
10826 * You only get credit for
10827 * MSS and greater (and you get extra
10828 * credit for larger cum-ack moves).
10829 */
10834 }
10838 }
10844 }
10846 /* We are done nothing left and no sack. */
10848 /*
10849 * For cases where we struck a dup-ack
10850 * with no SACK, add to the changes so
10851 * PRR will work right.
10852 */
10855 }
10857 }
10858 /* Sack block processing */
10861 else
10874 num_sack_blks++;
10882 /*
10883 * Its a D-SACK block.
10884 */
10886 }
10887 }
10889 /* Is the dsack roound over? */
10891 /* Yes it is */
10894 }
10895 }
10896 /*
10897 * Sort the SACK blocks so we can update the rack scoreboard with
10898 * just one pass.
10899 */
10906 /* Nothing to sack, but we need to update counts */
10908 }
10909 /* Its a sack of some sort */
10911 /* Only one, we don't need to sort */
10913 }
10914 /* Sort the sacks */
10921 }
10922 }
10923 }
10924 /*
10925 * Now are any of the sack block ends the same (yes some
10926 * implementations send these)?
10927 */
10928 again:
10935 /*
10936 * Ok these two have the same end we
10937 * want the smallest end and then
10938 * throw away the larger and start
10939 * again.
10940 */
10942 /*
10943 * The second block covers
10944 * more area use that
10945 */
10947 }
10948 /*
10949 * Now collapse out the dup-sack and
10950 * lower the count
10951 */
10955 j++;
10956 }
10957 num_sack_blks--;
10959 }
10960 }
10961 }
10962 }
10963 do_sack_work:
10964 /*
10965 * First lets look to see if
10966 * we have retransmitted and
10967 * can use the transmit next?
10968 */
10973 /*
10974 * We probably did the FR and the next
10975 * SACK in continues as we would expect.
10976 */
10981 }
10983 /*
10984 * This is what we would expect from
10985 * a normal implementation to happen
10986 * after we have retransmitted the FR,
10987 * i.e the sack-filter pushes down
10988 * to 1 block and the next to be retransmitted
10989 * is the sequence in the sack block (has more
10990 * are acked). Count this as ACK'd data to boost
10991 * up the chances of recovering any false positives.
10992 */
10997 /*
10998 * Start the loop through the
10999 * rest of blocks, past the first block.
11000 */
11002 }
11003 }
11011 }
11012 }
11013 out_with_totals:
11015 /*
11016 * You get an extra stroke if
11017 * you have more than one sack-blk, this
11018 * could be where we are skipping forward
11019 * and the sack-filter is still working, or
11020 * it could be an attacker constantly
11021 * moving us.
11022 */
11024 }
11025 out:
11027 /* Something changed cancel the rack timer */
11029 }
11033 rsm &&
11035 /* Enter recovery */
11038 /*
11039 * When we enter recovery we need to assure we send
11040 * one packet.
11041 */
11045 }
11050 rsm &&
11052 /*
11053 * Assure we can output and we get no
11054 * remembered pace time except the retransmit.
11055 */
11059 }
11067 /*
11068 * If you are pacing output you don't want
11069 * to override.
11070 */
11074 }
11075 }
11076 }
11078 static void
11080 {
11085 /*
11086 * We need to skip anything already set
11087 * to be retransmitted.
11088 */
11093 }
11095 }
11101 /*
11102 * Here we see if we need to retransmit. For
11103 * a SACK type connection if enough time has passed
11104 * we will get a return of the rsm. For a non-sack
11105 * connection we will get the rsm returned if the
11106 * dupack value is 3 or more.
11107 */
11114 }
11118 }
11121 }
11122 }
11123 }
11125 static void
11129 {
11130 /*
11131 * So what is dragging bottom?
11132 *
11133 * Dragging bottom means you were under pacing and had a
11134 * delay in processing inbound acks waiting on our pacing
11135 * timer to expire. While you were waiting all of the acknowledgments
11136 * for the packets you sent have arrived. This means we are pacing
11137 * way underneath the bottleneck to the point where our Goodput
11138 * measurements stop working, since they require more than one
11139 * ack (usually at least 8 packets worth with multiple acks so we can
11140 * gauge the inter-ack times). If that occurs we have a real problem
11141 * since we are stuck in a hole that we can't get out of without
11142 * something speeding us up.
11143 *
11144 * We also check to see if we are widdling down to just one segment
11145 * outstanding. If this occurs and we have room to send in our cwnd/rwnd
11146 * then we are adding the delayed ack interval into our measurments and
11147 * we need to speed up slightly.
11148 */
11154 /*
11155 * We are doing dynamic pacing and we are way
11156 * under. Basically everything got acked while
11157 * we were still waiting on the pacer to expire.
11158 *
11159 * This means we need to boost the b/w in
11160 * addition to any earlier boosting of
11161 * the multiplier.
11162 */
11173 /*
11174 * Lets use the long-term b/w we have
11175 * been getting as a base.
11176 */
11179 /*
11180 * If we have no measurement
11181 * don't let us set in more than
11182 * 1.2Mbps. If we are still too
11183 * low after pacing with this we
11184 * will hopefully have a max b/w
11185 * available to sanity check things.
11186 */
11188 }
11205 /* We have enough measurements now */
11212 }
11214 /*
11215 * zero rtt possibly?, settle for just an old increase.
11216 */
11218 }
11221 minseg)) &&
11226 /*
11227 * We are doing dynamic GP pacing and
11228 * we have everything except 1MSS or less
11229 * bytes left out. We are still pacing away.
11230 * And there is data that could be sent, This
11231 * means we are inserting delayed ack time in
11232 * our measurements because we are pacing too slow.
11233 */
11237 }
11238 }
11240 #ifdef TCP_REQUEST_TRK
11241 static void
11244 {
11251 /* We only allow the three below with point logging on */
11257 }
11262 /* Convert our ms to a microsecond */
11284 #ifdef TCP_REQUEST_TRK
11287 #endif
11290 }
11291 /*
11292 * Fill in flex7 to be CHD (catchup|hybrid|DGP)
11293 */
11299 /*
11300 * Compose bbr_state to be a bit wise 0000ADHF
11301 * where A is the always_pace flag
11302 * where D is the dgp_on flag
11303 * where H is the hybrid_mode on flag
11304 * where F is the use_fixed_rate flag.
11305 */
11325 }
11326 }
11327 #endif
11329 #ifdef TCP_REQUEST_TRK
11330 static void
11332 {
11340 /* If not in the beginning what about the end piece */
11346 }
11347 /* If we find no parameters we are in straight DGP mode */
11349 /* None found for this seq, just DGP for now */
11355 else
11357 }
11360 }
11363 }
11365 }
11367 /* This entry was never setup for hybrid pacing on/off etc */
11372 }
11375 }
11381 }
11383 }
11384 /*
11385 * Ok if we have a new entry *or* have never
11386 * set up an entry we need to proceed. If
11387 * we have already set it up this entry we
11388 * just continue along with what we already
11389 * setup.
11390 */
11394 /* Its already in place */
11398 }
11405 }
11408 }
11410 /* Compensate for all the header overhead's */
11413 else
11419 else
11421 }
11422 }
11425 else
11428 /*
11429 * It is the same timestamp as the previous one
11430 * add the hybrid flag that will indicate we use
11431 * sendtime not arrival time for catch-up mode.
11432 */
11434 }
11440 /*
11441 * Calculate the deadline time, first set the
11442 * time to when the request arrived.
11443 */
11445 /*
11446 * For cases where its a duplicate tm (we received more
11447 * than one request for a tm) we want to use now, the point
11448 * where we are just sending the first bit of the request.
11449 */
11452 /*
11453 * Here we have a different tm from the last request
11454 * so we want to use arrival time as our base.
11455 */
11457 }
11458 /*
11459 * Next calculate the length and compensate for
11460 * TLS if need be.
11461 */
11464 /*
11465 * This session is doing TLS. Take a swag guess
11466 * at the overhead.
11467 */
11469 }
11470 /*
11471 * Now considering the size, and the cspr, what is the time that
11472 * would be required at the cspr rate. Here we use the raw
11473 * cspr value since the client only looks at the raw data. We
11474 * do use len which includes TLS overhead, but not the TCP/IP etc.
11475 * That will get made up for in the CU pacing rate set.
11476 */
11483 }
11485 /*
11486 * We need to reset the max pace segs if we have a
11487 * client_suggested_maxseg.
11488 */
11490 }
11495 }
11497 /* Remember it for next time and for CU mode */
11500 }
11501 #endif
11503 static void
11504 rack_chk_req_and_hybrid_on_out(struct tcp_rack *rack, tcp_seq seq, uint32_t len, uint64_t cts)
11505 {
11506 #ifdef TCP_REQUEST_TRK
11513 /* Time to update the track. */
11516 }
11517 /* Out of all */
11520 }
11522 /*
11523 * This is the case where our end_seq guess
11524 * was wrong. This is usually due to TLS having
11525 * more bytes then our guess. It could also be the
11526 * case that the client sent in two requests closely
11527 * and the SB is full of both so we are sending part
11528 * of each (end|beg). In such a case lets move this
11529 * guys end to match the end of this send. That
11530 * way it will complete when all of it is acked.
11531 */
11535 }
11536 /* Now validate we have set the send time of this one */
11542 }
11543 #endif
11544 }
11546 static void
11547 rack_gain_for_fastoutput(struct tcp_rack *rack, struct tcpcb *tp, struct socket *so, uint32_t acked_amount)
11548 {
11549 /*
11550 * The fast output path is enabled and we
11551 * have moved the cumack forward. Lets see if
11552 * we can expand forward the fast path length by
11553 * that amount. What we would ideally like to
11554 * do is increase the number of bytes in the
11555 * fast path block (left_to_send) by the
11556 * acked amount. However we have to gate that
11557 * by two factors:
11558 * 1) The amount outstanding and the rwnd of the peer
11559 * (i.e. we don't want to exceed the rwnd of the peer).
11560 * <and>
11561 * 2) The amount of data left in the socket buffer (i.e.
11562 * we can't send beyond what is in the buffer).
11563 *
11564 * Note that this does not take into account any increase
11565 * in the cwnd. We will only extend the fast path by
11566 * what was acked.
11567 */
11574 /* We can increase left_to_send by the acked amount */
11577 KASSERT((rack->r_ctl.fsb.left_to_send <= (sbavail(&rack->rc_inp->inp_socket->so_snd) - (tp->snd_max - tp->snd_una))),
11583 }
11584 }
11586 static void
11588 {
11589 /*
11590 * Here any sendmap entry that points to the
11591 * beginning mbuf must be adjusted to the correct
11592 * offset. This must be called with:
11593 * 1) The socket buffer locked
11594 * 2) snd_una adjusted to its new position.
11595 *
11596 * Note that (2) implies rack_ack_received has also
11597 * been called and all the sbcut's have been done.
11598 *
11599 * We grab the first mbuf in the socket buffer and
11600 * then go through the front of the sendmap, recalculating
11601 * the stored offset for any sendmap entry that has
11602 * that mbuf. We must use the sb functions to do this
11603 * since its possible an add was done has well as
11604 * the subtraction we may have just completed. This should
11605 * not be a penalty though, since we just referenced the sb
11606 * to go in and trim off the mbufs that we freed (of course
11607 * there will be a penalty for the sendmap references though).
11608 *
11609 * Note also with INVARIANT on, we validate with a KASSERT
11610 * that the first sendmap entry has a soff of 0.
11611 *
11612 */
11615 tcp_seq snd_una;
11616 #ifdef INVARIANTS
11618 #endif
11625 /* Nothing outstanding */
11627 }
11628 /* The very first RSM's mbuf must point to the head mbuf in the sb */
11633 /* one to adjust */
11634 #ifdef INVARIANTS
11642 }
11648 }
11650 /*
11651 * This is not a fatal error, we anticipate it
11652 * might happen (the else code), so we count it here
11653 * so that under invariant we can see that it really
11654 * does happen.
11655 */
11657 }
11666 }
11667 #else
11675 }
11676 #endif
11680 }
11681 }
11683 #ifdef TCP_REQUEST_TRK
11686 {
11692 /*
11693 * Just do normal completions hybrid pacing is not on
11694 * and CLDL is off as well.
11695 */
11698 }
11699 /*
11700 * Originally I was just going to find the th_ack associated
11701 * with an entry. But then I realized a large strech ack could
11702 * in theory ack two or more requests at once. So instead we
11703 * need to find all entries that are completed by th_ack not
11704 * just a single entry and do our logging.
11705 */
11708 /*
11709 * We may be doing hybrid pacing or CLDL and need more details possibly
11710 * so we do it manually instead of calling
11711 * tcp_req_check_for_comp()
11712 */
11715 /* Ok this ack frees it */
11719 /* calculate the time based on the ack arrival */
11725 else
11728 /* TSNH */
11730 }
11733 else
11738 else
11741 /*
11742 * Check to see if we are freeing what we are pointing to send wise
11743 * if so be sure to NULL the pointer so we know we are no longer
11744 * set to anything.
11745 */
11752 else
11755 }
11756 }
11757 /* Generate the log that the tcp_netflix call would have */
11760 /* Free it and see if there is another one */
11763 }
11764 }
11765 #endif
11768 /*
11769 * Return value of 1, we do not need to call rack_process_data().
11770 * return value of 0, rack_process_data can be called.
11771 * For ret_val if its 0 the TCP is locked, if its non-zero
11772 * its unlocked and probably unsafe to touch the TCB.
11773 */
11774 static int
11779 {
11793 /* Checking SEG.ACK against ISS is definitely redundant. */
11795 }
11797 tcp_seq seq_min;
11801 /* Check for too old ACKs (RFC 5961, Section 5.2). */
11806 /* Checking for ghost ACKs is stricter. */
11810 /*
11811 * Checking for too old ACKs (RFC 5961,
11812 * Section 5.2) is stricter.
11813 */
11816 }
11817 }
11821 else
11823 /* Send challenge ACK. */
11827 }
11828 }
11833 }
11837 }
11847 }
11855 }
11859 }
11861 /*
11862 * Old ack, behind (or duplicate to) the last one rcv'd
11863 * Note: We mark reordering is occuring if its
11864 * less than and we have not closed our window.
11865 */
11870 }
11872 }
11873 /*
11874 * If we reach this point, ACK is not a duplicate, i.e., it ACKs
11875 * something we sent.
11876 */
11878 /*
11879 * T/TCP: Connection was half-synchronized, and our SYN has
11880 * been ACK'd (so connection is now fully synchronized). Go
11881 * to non-starred state, increment snd_una for ACK of SYN,
11882 * and check if we can do window scaling.
11883 */
11886 /* Do window scaling? */
11890 /* Send window already scaled. */
11891 }
11892 }
11897 /*
11898 * Any time we move the cum-ack forward clear
11899 * keep-alive tied probe-not-answered. The
11900 * persists clears its own on entry.
11901 */
11903 }
11906 /*
11907 * If we just performed our first retransmit, and the ACK arrives
11908 * within our recovery window, then it was a mistake to do the
11909 * retransmit in the first place. Recover our original cwnd and
11910 * ssthresh, and proceed to transmit where we left off.
11911 */
11918 }
11920 /* assure we are not backed off */
11926 /*
11927 * If it is the RXT timer we want to
11928 * stop it, so we can restart a TLP.
11929 */
11932 #ifdef TCP_REQUEST_TRK
11934 #endif
11935 }
11936 /*
11937 * If we have a timestamp reply, update smoothed round trip time. If
11938 * no timestamp is present but transmit timer is running and timed
11939 * sequence number was acked, update smoothed round trip time. Since
11940 * we now have an rtt measurement, cancel the timer backoff (cf.,
11941 * Phil Karn's retransmit alg.). Recompute the initial retransmit
11942 * timer.
11943 *
11944 * Some boxes send broken timestamp replies during the SYN+ACK
11945 * phase, ignore timestamps of 0 or we could calculate a huge RTT
11946 * and blow up the retransmit timer.
11947 */
11948 /*
11949 * If all outstanding data is acked, stop retransmit timer and
11950 * remember to restart (more output or persist). If there is more
11951 * data to be acked, restart retransmit timer, using current
11952 * (possibly backed-off) value.
11953 */
11958 }
11966 /*
11967 * Grab the segsiz, multiply by 2 and add the snd_cwnd
11968 * that is the max the CC should add if we are exiting
11969 * recovery and doing a late add.
11970 */
11974 }
11977 /*
11978 * We were in recovery, hit a rxt timeout
11979 * and never re-entered recovery. The timeout(s)
11980 * made up all the lost data. In such a case
11981 * we need to clear the rto_from_rec flag.
11982 */
11984 }
11985 /*
11986 * Let the congestion control algorithm update congestion control
11987 * related information. This typically means increasing the
11988 * congestion window.
11989 */
11993 /* Must be non-newreno (cubic) getting too ahead of itself */
11995 }
12004 /*
12005 * We must be sure our fin
12006 * was sent and acked (we can be
12007 * in FIN_WAIT_1 without having
12008 * sent the fin).
12009 */
12011 }
12013 /* wakeups? */
12017 /* NB: sowwakeup_locked() does an implicit unlock. */
12025 }
12031 /* Check if we are dragging bottom */
12033 }
12035 /* Nothing left outstanding */
12046 /* Set need output so persist might get set */
12052 /*
12053 * The socket was gone and the
12054 * peer sent data (now or in the past), time to
12055 * reset him.
12056 */
12058 /* tcp_close will kill the inp pre-log the Reset */
12063 }
12064 }
12068 }
12071 static void
12072 rack_log_collapse(struct tcp_rack *rack, uint32_t cnt, uint32_t split, uint32_t out, int line,
12074 {
12088 * 1 is collapsed, 0 is uncollapsed,
12089 * 2 is log of a rsm being marked, 3 is a split.
12090 */
12093 else
12102 }
12103 }
12105 static void
12107 {
12108 /*
12109 * Here all we do is mark the collapsed point and set the flag.
12110 * This may happen again and again, but there is no
12111 * sense splitting our map until we know where the
12112 * peer finally lands in the collapse.
12113 */
12123 }
12125 static void
12127 {
12137 /* Nothing to do maybe the peer ack'ed it all */
12140 }
12141 /* Now do we need to split this one? */
12147 /* We can't get a rsm, mark all? */
12150 }
12151 /* Clone it */
12154 #ifndef INVARIANTS
12156 #else
12160 }
12161 #endif
12167 }
12168 /*
12169 * Set in the new RSM as the
12170 * collapsed starting point
12171 */
12173 }
12175 no_split:
12177 cnt++;
12180 cnt++;
12181 }
12184 }
12186 }
12188 static void
12191 {
12199 }
12200 }
12202 static void
12204 {
12205 /*
12206 * If fast output is in progress, lets validate that
12207 * the new window did not shrink on us and make it
12208 * so fast output should end.
12209 */
12213 /*
12214 * Calculate what we will send if left as is
12215 * and compare that to our send window.
12216 */
12219 /* ok we have an issue */
12221 /* Turn off fast output the window is met or collapsed */
12224 /* we have some room left */
12227 /* If not at least 1 full segment never mind */
12229 }
12230 }
12231 }
12232 }
12233 }
12235 /*
12236 * Return value of 1, the TCB is unlocked and most
12237 * likely gone, return value of 0, the TCP is still
12238 * locked.
12239 */
12240 static int
12244 {
12245 /*
12246 * Update window information. Don't look at window if no ACK: TAC's
12247 * send garbage on first SYN.
12248 */
12261 /* keep track of pure window updates */
12278 }
12279 }
12281 /* The peer collapsed the window */
12288 /* Was persist timer active and now we have window space? */
12294 /* Make sure we output to start the timer */
12296 }
12297 /* Do we enter persists? */
12304 /*
12305 * Here the rwnd is less than
12306 * the pacing size, we are established,
12307 * nothing is outstanding, and there is
12308 * data to send. Enter persists.
12309 */
12311 }
12315 }
12316 /*
12317 * don't process the URG bit, ignore them drag
12318 * along the up.
12319 */
12322 /*
12323 * Process the segment text, merging it into the TCP sequencing
12324 * queue, and arranging for acknowledgment of receipt if necessary.
12325 * This process logically involves adjusting tp->rcv_wnd as data is
12326 * presented to the user (this happens in tcp_usrreq.c, case
12327 * PRU_RCVD). If a FIN has already been received on this connection
12328 * then we just ignore the text.
12329 */
12339 /*
12340 * Insert segment which includes th into TCP reassembly
12341 * queue with control block tp. Set thflags to whether
12342 * reassembly now includes a segment with FIN. This handles
12343 * the common case inline (segment is the next to be
12344 * received on an established connection, and the queue is
12345 * empty), avoiding linkage into and removal from the queue
12346 * and repetition of various conversions. Set DELACK for
12347 * segments received in order, but ack immediately when
12348 * segments are out of order (so fast retransmit can work).
12349 */
12353 tfo_syn)) {
12354 #ifdef NETFLIX_SB_LIMITS
12365 }
12366 }
12367 #endif
12378 }
12393 }
12394 #ifdef NETFLIX_SB_LIMITS
12395 appended =
12396 #endif
12398 }
12400 /* NB: sorwakeup_locked() does an implicit unlock. */
12402 #ifdef NETFLIX_SB_LIMITS
12406 #endif
12408 /*
12409 * XXX: Due to the header drop above "th" is
12410 * theoretically invalid by now. Fortunately
12411 * m_adj() doesn't actually frees any mbufs when
12412 * trimming from the head.
12413 */
12420 /* NB: sorwakeup_locked() does an implicit unlock. */
12422 }
12423 }
12428 /*
12429 * DSACK actually handled in the fastpath
12430 * above.
12431 */
12437 /*
12438 * Partial overlap, recorded at todrop
12439 * above.
12440 */
12447 }
12449 /* Update of sackblks. */
12455 }
12456 }
12460 }
12462 /*
12463 * If FIN is received ACK the FIN and let the user know that the
12464 * connection is closing.
12465 */
12468 /* The socket upcall is handled by socantrcvmore. */
12470 /*
12471 * If connection is half-synchronized (ie NEEDSYN
12472 * flag on) then delay ACK, so it may be piggybacked
12473 * when SYN is sent. Otherwise, since we received a
12474 * FIN then no more input can be expected, send ACK
12475 * now.
12476 */
12483 }
12485 }
12487 /*
12488 * In SYN_RECEIVED and ESTABLISHED STATES enter the
12489 * CLOSE_WAIT state.
12490 */
12493 /* FALLTHROUGH */
12500 /*
12501 * If still in FIN_WAIT_1 STATE FIN has not been
12502 * acked so enter the CLOSING state.
12503 */
12510 /*
12511 * In FIN_WAIT_2 state enter the TIME_WAIT state,
12512 * starting the time-wait timer, turning off the
12513 * other standard timers.
12514 */
12520 }
12521 }
12522 /*
12523 * Return any desired output.
12524 */
12528 }
12530 }
12532 /*
12533 * Here nothing is really faster, its just that we
12534 * have broken out the fast-data path also just like
12535 * the fast-ack.
12536 */
12537 static int
12541 {
12545 #ifdef NETFLIX_SB_LIMITS
12547 #endif
12549 /*
12550 * If last ACK falls within this segment's sequence numbers, record
12551 * the timestamp. NOTE that the test is modified according to the
12552 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
12553 */
12556 }
12559 }
12562 }
12566 }
12569 }
12572 }
12577 }
12579 /*
12580 * This is a pure, in-sequence data packet with nothing on the
12581 * reassembly queue and we have enough buffer space to take it.
12582 */
12585 #ifdef NETFLIX_SB_LIMITS
12594 }
12595 }
12596 #endif
12597 /* Clean receiver SACK report if present */
12610 }
12611 /*
12612 * Pull snd_wl1 up to prevent seq wrap relative to th_seq.
12613 */
12615 /*
12616 * Pull rcv_up up to prevent seq wrap relative to rcv_nxt.
12617 */
12623 /* Add data to socket buffer. */
12628 /*
12629 * Set new socket buffer size. Give up when limit is
12630 * reached.
12631 */
12636 #ifdef NETFLIX_SB_LIMITS
12637 appended =
12638 #endif
12641 }
12643 /* NB: sorwakeup_locked() does an implicit unlock. */
12645 #ifdef NETFLIX_SB_LIMITS
12648 #endif
12653 }
12655 /*
12656 * This subfunction is used to try to highly optimize the
12657 * fast path. We again allow window updates that are
12658 * in sequence to remain in the fast-path. We also add
12659 * in the __predict's to attempt to help the compiler.
12660 * Note that if we return a 0, then we can *not* process
12661 * it and the caller should push the packet into the
12662 * slow-path.
12663 */
12664 static int
12668 {
12675 /* Old ack, behind (or duplicate to) the last one rcv'd */
12677 }
12679 /* Above what we have sent? */
12681 }
12683 /* zero window */
12685 }
12687 /* We need a SYN or a FIN, unlikely.. */
12689 }
12691 /* Timestamp is behind .. old ack with seq wrap? */
12693 }
12695 /* Still recovering */
12697 }
12700 /* We have sack holes on our scoreboard */
12702 }
12703 /* Ok if we reach here, we can process a fast-ack */
12707 }
12710 /* Did the window get updated? */
12717 }
12718 /* Do we exit persists? */
12723 }
12724 /* Do we enter persists? */
12731 /*
12732 * Here the rwnd is less than
12733 * the pacing size, we are established,
12734 * nothing is outstanding, and there is
12735 * data to send. Enter persists.
12736 */
12738 }
12739 /*
12740 * If last ACK falls within this segment's sequence numbers, record
12741 * the timestamp. NOTE that the test is modified according to the
12742 * latest proposal of the tcplw@cray.com list (Braden 1993/04/26).
12743 */
12748 }
12749 /*
12750 * This is a pure ack for outstanding data.
12751 */
12754 /*
12755 * "bad retransmit" recovery.
12756 */
12763 }
12764 /*
12765 * Recalculate the transmit timer / rtt.
12766 *
12767 * Some boxes send broken timestamp replies during the SYN+ACK
12768 * phase, ignore timestamps of 0 or we could calculate a huge RTT
12769 * and blow up the retransmit timer.
12770 */
12773 #ifdef TCP_HHOOK
12774 /* Run HHOOK_TCP_ESTABLISHED_IN helper hooks. */
12776 #endif
12786 /* Note we want to hold the sb lock through the sendmap adjust */
12788 /* Wake up the socket if we have room to write more */
12797 /*
12798 * If it is the RXT timer we want to
12799 * stop it, so we can restart a TLP.
12800 */
12804 #ifdef TCP_REQUEST_TRK
12806 #endif
12807 }
12808 /*
12809 * Let the congestion control algorithm update congestion control
12810 * related information. This typically means increasing the
12811 * congestion window.
12812 */
12814 /* The peer collapsed the window */
12821 /*
12822 * Pull snd_wl2 up to prevent seq wrap relative to th_ack.
12823 */
12829 /*
12830 * If all outstanding data are acked, stop retransmit timer,
12831 * otherwise restart timer using current (possibly backed-off)
12832 * value. If process is waiting for space, wakeup/selwakeup/signal.
12833 * If data are ready to send, let tcp_output decide between more
12834 * output or persist.
12835 */
12841 /* Check if we are dragging bottom */
12843 }
12856 }
12861 }
12863 }
12865 /*
12866 * Return value of 1, the TCB is unlocked and most
12867 * likely gone, return value of 0, the TCP is still
12868 * locked.
12869 */
12870 static int
12874 {
12884 /*
12885 * If the state is SYN_SENT: if seg contains an ACK, but not for our
12886 * SYN, drop the input. if seg contains a RST, then drop the
12887 * connection. if seg does not contain SYN, then drop it. Otherwise
12888 * this is an acceptable SYN segment initialize tp->rcv_nxt and
12889 * tp->irs if seg contains ack then advance tp->snd_una if seg
12890 * contains an ECE and ECN support is enabled, the stream is ECN
12891 * capable. if SYN has been acked change to ESTABLISHED else
12892 * SYN_RCVD state arrange for segment to be acked (eventually)
12893 * continue processing rest of data/controls.
12894 */
12901 }
12908 }
12912 }
12916 }
12925 #ifdef MAC
12927 #endif
12928 /* Do window scaling on this connection? */
12932 }
12935 /*
12936 * If not all the data that was sent in the TFO SYN
12937 * has been acked, resend the remainder right away.
12938 */
12941 /* Was it a partial ack? */
12944 }
12945 /*
12946 * If there's data, delay ACK; if there's also a FIN ACKNOW
12947 * will be turned on later.
12948 */
12956 }
12961 /*
12962 * We advance snd_una for the
12963 * fast open case. If th_ack is
12964 * acknowledging data beyond
12965 * snd_una we can't just call
12966 * ack-processing since the
12967 * data stream in our send-map
12968 * will start at snd_una + 1 (one
12969 * beyond the SYN). If its just
12970 * equal we don't need to do that
12971 * and there is no send_map.
12972 */
12975 /*
12976 * We sent a SYN with data, and thus have a
12977 * sendmap entry with a SYN set. Lets find it
12978 * and take off the send bit and the byte and
12979 * set it up to be what we send (send it next).
12980 */
12988 }
12990 }
12991 }
12992 }
12993 /*
12994 * Received <SYN,ACK> in SYN_SENT[*] state. Transitions:
12995 * SYN_SENT --> ESTABLISHED SYN_SENT* --> FIN_WAIT_1
12996 */
13007 }
13009 /*
13010 * Received initial SYN in SYN-SENT[*] state => simultaneous
13011 * open. If segment contains CC option and there is a
13012 * cached CC, apply TAO test. If it succeeds, connection is *
13013 * half-synchronized. Otherwise, do 3-way handshake:
13014 * SYN-SENT -> SYN-RECEIVED SYN-SENT* -> SYN-RECEIVED* If
13015 * there was no CC option, clear cached CC value.
13016 */
13019 }
13020 /*
13021 * Advance th->th_seq to correspond to first data byte. If data,
13022 * trim to stay within window, dropping FIN if necessary.
13023 */
13032 }
13035 /*
13036 * Client side of transaction: already sent SYN and data. If the
13037 * remote host used T/TCP to validate the SYN, our data will be
13038 * ACK'd; if so, enter normal data segment processing in the middle
13039 * of step 5, ack processing. Otherwise, goto step 6.
13040 */
13042 /* For syn-sent we need to possibly update the rtt */
13053 }
13054 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen))
13056 /* We may have changed to FIN_WAIT_1 above */
13058 /*
13059 * In FIN_WAIT_1 STATE in addition to the processing
13060 * for the ESTABLISHED state if our FIN is now
13061 * acknowledged then enter FIN_WAIT_2.
13062 */
13064 /*
13065 * If we can't receive any more data, then
13066 * closing user can proceed. Starting the
13067 * timer is contrary to the specification,
13068 * but if we don't get a FIN we'll hang
13069 * forever.
13070 *
13071 * XXXjl: we should release the tp also, and
13072 * use a compressed state.
13073 */
13078 tcp_finwait2_timeout :
13080 }
13082 }
13083 }
13084 }
13087 }
13089 /*
13090 * Return value of 1, the TCB is unlocked and most
13091 * likely gone, return value of 0, the TCP is still
13092 * locked.
13093 */
13094 static int
13098 {
13115 }
13117 /*
13118 * When a TFO connection is in SYN_RECEIVED, the
13119 * only valid packets are the initial SYN, a
13120 * retransmit/copy of the initial SYN (possibly with
13121 * a subset of the original data), a valid ACK, a
13122 * FIN, or a RST.
13123 */
13129 /* non-initial SYN is ignored */
13135 }
13139 }
13140 }
13142 /*
13143 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13144 * it's less than ts_recent, drop it.
13145 */
13150 }
13151 /*
13152 * In the SYN-RECEIVED state, validate that the packet belongs to
13153 * this connection before trimming the data to fit the receive
13154 * window. Check the sequence number versus IRS since we know the
13155 * sequence numbers haven't wrapped. This is a partial fix for the
13156 * "LAND" DoS attack.
13157 */
13162 }
13165 }
13166 /*
13167 * If last ACK falls within this segment's sequence numbers, record
13168 * its timestamp. NOTE: 1) That the test incorporates suggestions
13169 * from the latest proposal of the tcplw@cray.com list (Braden
13170 * 1993/04/26). 2) That updating only on newer timestamps interferes
13171 * with our earlier PAWS tests, so this check should be solely
13172 * predicated on the sequence space of this segment. 3) That we
13173 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13174 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13175 * SEG.Len, This modified check allows us to overcome RFC1323's
13176 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13177 * p.869. In such cases, we can still calculate the RTT correctly
13178 * when RCV.NXT == Last.ACK.Sent.
13179 */
13186 }
13189 /*
13190 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13191 * is on (half-synchronized state), then queue data for later
13192 * processing; else drop segment and return.
13193 */
13197 }
13200 }
13205 }
13206 /* Do window scaling? */
13210 }
13211 /*
13212 * Make transitions: SYN-RECEIVED -> ESTABLISHED SYN-RECEIVED* ->
13213 * FIN-WAIT-1
13214 */
13219 }
13227 /*
13228 * TFO connections call cc_conn_init() during SYN
13229 * processing. Calling it again here for such connections
13230 * is not harmless as it would undo the snd_cwnd reduction
13231 * that occurs when a TFO SYN|ACK is retransmitted.
13232 */
13235 }
13236 /*
13237 * Account for the ACK of our SYN prior to
13238 * regular ACK processing below, except for
13239 * simultaneous SYN, which is handled later.
13240 */
13243 /*
13244 * If segment contains data or ACK, will call tcp_reass() later; if
13245 * not, do so now to pass queued data to user.
13246 */
13252 /* NB: sorwakeup_locked() does an implicit unlock. */
13254 }
13255 }
13257 /* For syn-recv we need to possibly update the rtt */
13268 }
13269 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13271 }
13273 /* We could have went to FIN_WAIT_1 (or EST) above */
13274 /*
13275 * In FIN_WAIT_1 STATE in addition to the processing for the
13276 * ESTABLISHED state if our FIN is now acknowledged then
13277 * enter FIN_WAIT_2.
13278 */
13280 /*
13281 * If we can't receive any more data, then closing
13282 * user can proceed. Starting the timer is contrary
13283 * to the specification, but if we don't get a FIN
13284 * we'll hang forever.
13285 *
13286 * XXXjl: we should release the tp also, and use a
13287 * compressed state.
13288 */
13293 tcp_finwait2_timeout :
13295 }
13297 }
13298 }
13301 }
13303 /*
13304 * Return value of 1, the TCB is unlocked and most
13305 * likely gone, return value of 0, the TCP is still
13306 * locked.
13307 */
13308 static int
13312 {
13317 /*
13318 * Header prediction: check for the two common cases of a
13319 * uni-directional data xfer. If the packet has no control flags,
13320 * is in-sequence, the window didn't change and we're not
13321 * retransmitting, it's a candidate. If the length is zero and the
13322 * ack moved forward, we're the sender side of the xfer. Just free
13323 * the data acked & wake any higher level process that was blocked
13324 * waiting for space. If the length is non-zero and the ack didn't
13325 * move, we're the receiver side. If we're getting packets in-order
13326 * (the reassembly queue is empty), add the data toc The socket
13327 * buffer and note that we need a delayed ack. Make sure that the
13328 * hidden state-flags are also off. Since we check for
13329 * TCPS_ESTABLISHED first, it can only be TH_NEEDSYN.
13330 */
13340 }
13345 }
13346 }
13347 }
13354 /*
13355 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13356 * synchronized state.
13357 */
13361 }
13362 /*
13363 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13364 * it's less than ts_recent, drop it.
13365 */
13370 }
13373 }
13374 /*
13375 * If last ACK falls within this segment's sequence numbers, record
13376 * its timestamp. NOTE: 1) That the test incorporates suggestions
13377 * from the latest proposal of the tcplw@cray.com list (Braden
13378 * 1993/04/26). 2) That updating only on newer timestamps interferes
13379 * with our earlier PAWS tests, so this check should be solely
13380 * predicated on the sequence space of this segment. 3) That we
13381 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13382 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13383 * SEG.Len, This modified check allows us to overcome RFC1323's
13384 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13385 * p.869. In such cases, we can still calculate the RTT correctly
13386 * when RCV.NXT == Last.ACK.Sent.
13387 */
13394 }
13395 /*
13396 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13397 * is on (half-synchronized state), then queue data for later
13398 * processing; else drop segment and return.
13399 */
13412 }
13413 }
13414 /*
13415 * Ack processing.
13416 */
13419 }
13425 }
13426 }
13427 /* State changes only happen in rack_process_data() */
13430 }
13432 /*
13433 * Return value of 1, the TCB is unlocked and most
13434 * likely gone, return value of 0, the TCP is still
13435 * locked.
13436 */
13437 static int
13441 {
13449 /*
13450 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13451 * synchronized state.
13452 */
13456 }
13457 /*
13458 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13459 * it's less than ts_recent, drop it.
13460 */
13465 }
13468 }
13469 /*
13470 * If last ACK falls within this segment's sequence numbers, record
13471 * its timestamp. NOTE: 1) That the test incorporates suggestions
13472 * from the latest proposal of the tcplw@cray.com list (Braden
13473 * 1993/04/26). 2) That updating only on newer timestamps interferes
13474 * with our earlier PAWS tests, so this check should be solely
13475 * predicated on the sequence space of this segment. 3) That we
13476 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13477 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13478 * SEG.Len, This modified check allows us to overcome RFC1323's
13479 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13480 * p.869. In such cases, we can still calculate the RTT correctly
13481 * when RCV.NXT == Last.ACK.Sent.
13482 */
13489 }
13490 /*
13491 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13492 * is on (half-synchronized state), then queue data for later
13493 * processing; else drop segment and return.
13494 */
13507 }
13508 }
13509 /*
13510 * Ack processing.
13511 */
13514 }
13521 }
13522 }
13525 }
13527 static int
13530 {
13535 close_now:
13537 /* tcp_close will kill the inp pre-log the Reset */
13543 }
13546 /* Ok we allow data that is ignored and a followup reset */
13553 }
13555 /*
13556 * Return value of 1, the TCB is unlocked and most
13557 * likely gone, return value of 0, the TCP is still
13558 * locked.
13559 */
13560 static int
13564 {
13574 /*
13575 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13576 * synchronized state.
13577 */
13581 }
13582 /*
13583 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13584 * it's less than ts_recent, drop it.
13585 */
13590 }
13593 }
13594 /*
13595 * If new data are received on a connection after the user processes
13596 * are gone, then RST the other end.
13597 */
13601 /*
13602 * If last ACK falls within this segment's sequence numbers, record
13603 * its timestamp. NOTE: 1) That the test incorporates suggestions
13604 * from the latest proposal of the tcplw@cray.com list (Braden
13605 * 1993/04/26). 2) That updating only on newer timestamps interferes
13606 * with our earlier PAWS tests, so this check should be solely
13607 * predicated on the sequence space of this segment. 3) That we
13608 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13609 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13610 * SEG.Len, This modified check allows us to overcome RFC1323's
13611 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13612 * p.869. In such cases, we can still calculate the RTT correctly
13613 * when RCV.NXT == Last.ACK.Sent.
13614 */
13621 }
13622 /*
13623 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13624 * is on (half-synchronized state), then queue data for later
13625 * processing; else drop segment and return.
13626 */
13638 }
13639 }
13640 /*
13641 * Ack processing.
13642 */
13643 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13645 }
13647 /*
13648 * If we can't receive any more data, then closing user can
13649 * proceed. Starting the timer is contrary to the
13650 * specification, but if we don't get a FIN we'll hang
13651 * forever.
13652 *
13653 * XXXjl: we should release the tp also, and use a
13654 * compressed state.
13655 */
13660 tcp_finwait2_timeout :
13662 }
13664 }
13671 }
13672 }
13675 }
13677 /*
13678 * Return value of 1, the TCB is unlocked and most
13679 * likely gone, return value of 0, the TCP is still
13680 * locked.
13681 */
13682 static int
13686 {
13696 /*
13697 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13698 * synchronized state.
13699 */
13703 }
13704 /*
13705 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13706 * it's less than ts_recent, drop it.
13707 */
13712 }
13715 }
13716 /*
13717 * If last ACK falls within this segment's sequence numbers, record
13718 * its timestamp. NOTE: 1) That the test incorporates suggestions
13719 * from the latest proposal of the tcplw@cray.com list (Braden
13720 * 1993/04/26). 2) That updating only on newer timestamps interferes
13721 * with our earlier PAWS tests, so this check should be solely
13722 * predicated on the sequence space of this segment. 3) That we
13723 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13724 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13725 * SEG.Len, This modified check allows us to overcome RFC1323's
13726 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13727 * p.869. In such cases, we can still calculate the RTT correctly
13728 * when RCV.NXT == Last.ACK.Sent.
13729 */
13736 }
13737 /*
13738 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13739 * is on (half-synchronized state), then queue data for later
13740 * processing; else drop segment and return.
13741 */
13753 }
13754 }
13755 /*
13756 * Ack processing.
13757 */
13758 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13760 }
13765 }
13772 }
13773 }
13776 }
13778 /*
13779 * Return value of 1, the TCB is unlocked and most
13780 * likely gone, return value of 0, the TCP is still
13781 * locked.
13782 */
13783 static int
13787 {
13797 /*
13798 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13799 * synchronized state.
13800 */
13804 }
13805 /*
13806 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13807 * it's less than ts_recent, drop it.
13808 */
13813 }
13817 }
13818 /*
13819 * If last ACK falls within this segment's sequence numbers, record
13820 * its timestamp. NOTE: 1) That the test incorporates suggestions
13821 * from the latest proposal of the tcplw@cray.com list (Braden
13822 * 1993/04/26). 2) That updating only on newer timestamps interferes
13823 * with our earlier PAWS tests, so this check should be solely
13824 * predicated on the sequence space of this segment. 3) That we
13825 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13826 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13827 * SEG.Len, This modified check allows us to overcome RFC1323's
13828 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13829 * p.869. In such cases, we can still calculate the RTT correctly
13830 * when RCV.NXT == Last.ACK.Sent.
13831 */
13838 }
13839 /*
13840 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13841 * is on (half-synchronized state), then queue data for later
13842 * processing; else drop segment and return.
13843 */
13855 }
13856 }
13857 /*
13858 * case TCPS_LAST_ACK: Ack processing.
13859 */
13860 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13862 }
13867 }
13874 }
13875 }
13878 }
13880 /*
13881 * Return value of 1, the TCB is unlocked and most
13882 * likely gone, return value of 0, the TCP is still
13883 * locked.
13884 */
13885 static int
13889 {
13896 /* Reset receive buffer auto scaling when not in bulk receive mode. */
13900 /*
13901 * RFC5961 Section 4.2 Send challenge ACK for any SYN in
13902 * synchronized state.
13903 */
13907 }
13908 /*
13909 * RFC 1323 PAWS: If we have a timestamp reply on this segment and
13910 * it's less than ts_recent, drop it.
13911 */
13916 }
13919 }
13920 /*
13921 * If new data are received on a connection after the user processes
13922 * are gone, then RST the other end.
13923 */
13927 /*
13928 * If last ACK falls within this segment's sequence numbers, record
13929 * its timestamp. NOTE: 1) That the test incorporates suggestions
13930 * from the latest proposal of the tcplw@cray.com list (Braden
13931 * 1993/04/26). 2) That updating only on newer timestamps interferes
13932 * with our earlier PAWS tests, so this check should be solely
13933 * predicated on the sequence space of this segment. 3) That we
13934 * modify the segment boundary check to be Last.ACK.Sent <= SEG.SEQ
13935 * + SEG.Len instead of RFC1323's Last.ACK.Sent < SEG.SEQ +
13936 * SEG.Len, This modified check allows us to overcome RFC1323's
13937 * limitations as described in Stevens TCP/IP Illustrated Vol. 2
13938 * p.869. In such cases, we can still calculate the RTT correctly
13939 * when RCV.NXT == Last.ACK.Sent.
13940 */
13947 }
13948 /*
13949 * If the ACK bit is off: if in SYN-RECEIVED state or SENDSYN flag
13950 * is on (half-synchronized state), then queue data for later
13951 * processing; else drop segment and return.
13952 */
13964 }
13965 }
13966 /*
13967 * Ack processing.
13968 */
13969 if (rack_process_ack(m, th, so, tp, to, tiwin, tlen, &ourfinisacked, thflags, &ret_val, orig_tlen)) {
13971 }
13978 }
13979 }
13982 }
13986 {
13990 }
13992 static void
13993 rack_set_pace_segments(struct tcpcb *tp, struct tcp_rack *rack, uint32_t line, uint64_t *fill_override)
13994 {
13999 #ifdef TCP_REQUEST_TRK
14006 }
14007 #endif
14017 }
14026 /* We are pacing right at the hardware rate */
14032 else
14039 }
14043 /* We have a rate of some sort set */
14050 else
14053 /* We have something */
14055 rate_wanted,
14063 /*
14064 * If we have nothing limit us to bursting
14065 * out IW sized pieces.
14066 */
14069 }
14070 }
14074 }
14077 }
14080 static void
14082 {
14083 #ifdef INET6
14085 #endif
14086 #ifdef INET
14088 #endif
14091 /* Ok lets fill in the fast block, it can only be used with no IP options! */
14092 #ifdef INET6
14104 {
14107 }
14114 #ifdef INET
14115 {
14126 {
14129 }
14134 }
14135 #endif
14139 }
14141 static int
14143 {
14144 /*
14145 * Allocate the larger of spaces V6 if available else just
14146 * V4 and include udphdr (overbook)
14147 */
14148 #ifdef INET6
14149 rack->r_ctl.fsb.tcp_ip_hdr_len = sizeof(struct ip6_hdr) + sizeof(struct tcphdr) + sizeof(struct udphdr);
14150 #else
14152 #endif
14157 }
14160 }
14162 static void
14164 {
14165 /*
14166 * Types of logs (mod value)
14167 * 20 - Initial round setup
14168 * 21 - Rack declares a new round.
14169 */
14191 }
14192 }
14194 static void
14196 {
14201 }
14203 static void
14205 {
14206 /* Retransmit bit controls.
14207 *
14208 * The setting of these values control one of
14209 * three settings you can have and dictate
14210 * how rack does retransmissions. Note this
14211 * is in *any* mode i.e. pacing on or off DGP
14212 * fixed rate pacing, or just bursting rack.
14213 *
14214 * 1 - Use full sized retransmits i.e. limit
14215 * the size to whatever the pace_max_segments
14216 * size is.
14217 *
14218 * 2 - Use pacer min granularity as a guide to
14219 * the size combined with the current calculated
14220 * goodput b/w measurement. So for example if
14221 * the goodput is measured at 20Mbps we would
14222 * calculate 8125 (pacer minimum 250usec in
14223 * that b/w) and then round it up to the next
14224 * MSS i.e. for 1448 mss 6 MSS or 8688 bytes.
14225 *
14226 * 0 - The rack default 1 MSS (anything not 0/1/2
14227 * fall here too if we are setting via rack_init()).
14228 *
14229 */
14239 }
14240 }
14242 static void
14247 {
14260 }
14261 }
14263 static int
14265 {
14276 /* Unlikely */
14278 }
14281 /* Can't find that seq -- unlikely */
14283 }
14304 /* no timers up */
14306 }
14310 }
14313 }
14321 /* Reordering items */
14324 /* Timerstamps and timers */
14330 /* PRR data */
14337 /* TLP and persists info */
14346 }
14349 else
14355 }
14356 }
14358 static void
14360 {
14361 /*
14362 * This method gets called if a stack switch was
14363 * attempted and it failed. We are left
14364 * but our hpts timers were stopped and we
14365 * need to validate time units and t_flags2.
14366 */
14377 else
14382 /* Strange */
14384 }
14390 /* one slot please */
14392 }
14397 /* one slot please */
14399 }
14405 }
14407 static int
14409 {
14412 /*
14413 * When initing outstanding, we must be quite careful
14414 * to not refer to tp->t_fb_ptr. This has the old rack
14415 * pointer in it, not the "new" one (when we are doing
14416 * a stack switch).
14417 */
14421 /* Create a send map for the current outstanding data */
14427 }
14436 /* The data space is one beyond snd_una */
14451 }
14453 /*
14454 * This can happen if we have a stand-alone FIN or
14455 * SYN.
14456 */
14461 }
14462 #ifdef INVARIANTS
14466 }
14467 #else
14469 #endif
14473 /* We have a query mechanism, lets use it */
14476 tcp_seq at;
14485 /* Move forward */
14487 /* Now lets build the entry for this one */
14492 }
14494 /* Now configure the rsm and insert it */
14500 else
14502 /*
14503 * We have carefully aligned the bits
14504 * so that all we have to do is copy over
14505 * the bits with the mask.
14506 */
14521 }
14522 #ifdef INVARIANTS
14526 }
14527 #else
14529 #endif
14534 /*
14535 * If the existing ersm was sent at
14536 * a later time than the new one, then
14537 * the new one should appear ahead of this
14538 * ersm.
14539 */
14543 }
14544 }
14546 /*
14547 * Not found so shove it on the tail.
14548 */
14551 }
14556 }
14557 }
14562 }
14563 }
14565 }
14568 static int32_t
14570 {
14579 /*
14580 * First are we the initial or are we a switched stack?
14581 * If we are initing via tcp_newtcppcb the ptr passed
14582 * will be tp->t_fb_ptr. If its a stack switch that
14583 * has a previous stack we can query it will be a local
14584 * var that will in the end be set into t_fb_ptr.
14585 */
14588 else
14592 /*
14593 * We need to allocate memory but cant. The INP and INP_INFO
14594 * locks and they are recursive (happens during setup. So a
14595 * scheme to drop the locks fails :(
14596 *
14597 */
14599 }
14606 }
14612 /* Set the flag */
14614 /* Probably not needed but lets be sure */
14616 /*
14617 * Save off the default values, socket options will poke
14618 * at these if pacing is not on or we have not yet
14619 * reached where pacing is on (gp_ready/fixed enabled).
14620 * When they get set into the CC module (when gp_ready
14621 * is enabled or we enable fixed) then we will set these
14622 * values into the CC and place in here the old values
14623 * so we have a restoral. Then we will set the flag
14624 * rc_pacing_cc_set. That way whenever we turn off pacing
14625 * or switch off this stack, we will know to go restore
14626 * the saved values.
14627 *
14628 * We specifically put into the beta the ecn value for pacing.
14629 */
14632 /* We want abe like behavior as well */
14646 }
14649 else
14651 #ifdef TCP_ACCOUNTING
14654 }
14655 #endif
14661 }
14662 #ifdef NETFLIX_STATS
14664 #endif
14682 }
14685 }
14686 /*
14687 * We initialize to all ones so we never match 0
14688 * just in case the client sends in 0, it hopefully
14689 * will never have all 1's in ms :-)
14690 */
14717 else
14722 else
14730 /*
14731 * Make sure any TCP timers are not running.
14732 */
14735 /*
14736 * Server side, we are called from the
14737 * syn-cache. This means none of the
14738 * snd_una/max are set yet so we have
14739 * to defer this until the first send.
14740 */
14742 }
14754 /* When dynamic adjustment is on CA needs to start at 100% */
14763 }
14768 }
14774 rack_probertt_filter_life);
14787 }
14789 /* Basically this means all rack timers are at least (srtt + 1/4 srtt) */
14791 }
14793 /* Basically this means rack timers are extended based on dsack by up to (2 * srtt) */
14795 }
14796 /* We require at least one measurement, even if the sysctl is 0 */
14799 else
14807 /* Lets setup the fsb block */
14813 }
14820 }
14821 /* Log what we will do with queries */
14826 /* Cancel the GP measurement in progress */
14830 /*
14831 * We are already open, we may
14832 * need to adjust a few things.
14833 */
14836 else
14841 /* We are not past the initial window
14842 * on the first init (i.e. a stack switch
14843 * has not yet occured) so we need to make
14844 * sure cwnd and ssthresh is correct.
14845 */
14848 /*
14849 * If we are within the initial window
14850 * we want ssthresh to be unlimited. Setting
14851 * it to the rwnd (which the default stack does
14852 * and older racks) is not really a good idea
14853 * since we want to be in SS and grow both the
14854 * cwnd and the rwnd (via dynamic rwnd growth). If
14855 * we set it to the rwnd then as the peer grows its
14856 * rwnd we will be stuck in CA and never hit SS.
14857 *
14858 * Its far better to raise it up high (this takes the
14859 * risk that there as been a loss already, probably
14860 * we should have an indicator in all stacks of loss
14861 * but we don't), but considering the normal use this
14862 * is a risk worth taking. The consequences of not
14863 * hitting SS are far worse than going one more time
14864 * into it early on (before we have sent even a IW).
14865 * It is highly unlikely that we will have had a loss
14866 * before getting the IW out.
14867 */
14869 }
14870 /*
14871 * Any init based on sequence numbers
14872 * should be done in the deferred init path
14873 * since we can be CLOSED and not have them
14874 * inited when rack_init() is called. We
14875 * are not closed so lets call it.
14876 */
14878 }
14887 }
14888 }
14890 /* Setup all the t_flags2 */
14893 else
14897 /*
14898 * Timers in Rack are kept in microseconds so lets
14899 * convert any initial incoming variables
14900 * from ticks into usecs. Note that we
14901 * also change the values of t_srtt and t_rttvar, if
14902 * they are non-zero. They are kept with a 5
14903 * bit decimal so we have to carefully convert
14904 * these to get the full precision.
14905 */
14909 /* We do not start any timers on DROPPED connections */
14918 /* Get the misc time stamps and such for rack */
14940 }
14945 #ifdef NETFLIX_SHARED_CWND
14949 }
14950 #endif
14958 }
14965 }
14966 /* Get the old stack timers */
14971 /*
14972 * non-zero return means we have a timer('s)
14973 * to start. Zero means no timer (no keepalive
14974 * I suppose).
14975 */
14983 else
14985 }
14991 else
14993 }
14994 }
15005 }
15006 }
15007 }
15010 }
15012 }
15014 static int
15016 {
15019 /* Sure no problem though it may not stick */
15021 }
15024 /*
15025 * We really don't know if you support sack,
15026 * you have to get to ESTAB or beyond to tell.
15027 */
15029 }
15031 /*
15032 * Rack will only send a FIN after all data is acknowledged.
15033 * So in this case we have more data outstanding. We can't
15034 * switch stacks until either all data and only the FIN
15035 * is left (in which case rack_init() now knows how
15036 * to deal with that) <or> all is acknowledged and we
15037 * are only left with incoming data, though why you
15038 * would want to switch to rack after all data is acknowledged
15039 * I have no idea (rrs)!
15040 */
15042 }
15045 }
15046 /*
15047 * If we reach here we don't do SACK on this connection so we can
15048 * never do rack.
15049 */
15051 }
15053 static void
15055 {
15072 #ifdef NETFLIX_SHARED_CWND
15078 else
15082 limit);
15084 }
15085 #endif
15090 }
15093 }
15094 /* Clean up any options if they were not applied */
15101 }
15102 /* rack does not use force data but other stacks may clear it */
15107 }
15108 #ifdef TCP_BLACKBOX
15110 #endif
15111 /*
15112 * Lets take a different approach to purging just
15113 * get each one and free it like a cum-ack would and
15114 * not use a foreach loop.
15115 */
15122 }
15128 cnt_free++;
15131 }
15137 }
15157 }
15160 rack,
15167 }
15168 /* Make sure snd_nxt is correctly set */
15170 }
15172 static void
15174 {
15177 }
15220 };
15224 }
15226 static void
15228 {
15229 /*
15230 * We received an ack, and then did not
15231 * call send or were bounced out due to the
15232 * hpts was running. Now a timer is up as well, is
15233 * it the right timer?
15234 */
15240 /*
15241 * Ok we probably need some timer up, but no
15242 * matter what the mask we are not in hpts. We
15243 * may have received an old ack and thus did nothing.
15244 */
15248 }
15254 /* Should be an RXT */
15256 }
15258 /* Nothing outstanding? */
15261 /* We are supposed to have delayed ack up and we do */
15268 /* We should have keep alive up and we do */
15270 }
15271 }
15276 /*
15277 * Either a Rack, TLP or RXT is fine if we
15278 * have outstanding data.
15279 */
15282 /*
15283 * If the delayed ack was going to go off
15284 * before the rtx/tlp/rack timer were going to
15285 * expire, then that would be the timer in control.
15286 * Note we don't check the time here trusting the
15287 * code is correct.
15288 */
15290 }
15291 /*
15292 * Ok the timer originally started is not what we want now.
15293 * We will force the hpts to be stopped if any, and restart
15294 * with the slot set to what was in the saved slot.
15295 */
15304 }
15306 }
15308 }
15311 }
15314 static void
15315 rack_do_win_updates(struct tcpcb *tp, struct tcp_rack *rack, uint32_t tiwin, uint32_t seq, uint32_t ack, uint32_t cts)
15316 {
15320 /* keep track of pure window updates */
15336 /* Not a valid win update */
15338 }
15341 /* Do we exit persists? */
15346 }
15347 /* Do we enter persists? */
15354 /*
15355 * Here the rwnd is less than
15356 * the pacing size, we are established,
15357 * nothing is outstanding, and there is
15358 * data to send. Enter persists.
15359 */
15361 }
15362 }
15364 static void
15365 rack_log_input_packet(struct tcpcb *tp, struct tcp_rack *rack, struct tcp_ackent *ae, int ackval, uint32_t high_seq)
15366 {
15378 #ifdef TCP_REQUEST_TRK
15385 }
15386 #endif
15391 else
15406 /* Record the hardware timestamp if present */
15414 /* Record the LRO the arrival timestamp */
15421 }
15423 /* Log the rcv time */
15425 #ifdef TCP_REQUEST_TRK
15432 /* Copy out any client req info */
15433 /* seconds */
15435 /* useconds */
15444 }
15449 }
15450 }
15451 #endif
15457 /* Now fill in the ports */
15461 /* Now do we have a timestamp option? */
15469 cp++;
15471 cp++;
15473 cp++;
15475 cp++;
15485 /*
15486 * For sane logging we need to play a little trick.
15487 * If the ack were fully processed we would have moved
15488 * snd_una to high_seq, but since compressed acks are
15489 * processed in two phases, at this point (logging) snd_una
15490 * won't be advanced. So we would see multiple acks showing
15491 * the advancement. We can prevent that by "pretending" that
15492 * snd_una was advanced and then un-advancing it so that the
15493 * logging code has the right value for tlb_snd_una.
15494 */
15507 }
15508 }
15510 }
15512 static void
15514 {
15516 /*
15517 * A persist or keep-alive was forced out, update our
15518 * min rtt time. Note now worry about lost responses.
15519 * When a subsequent keep-alive or persist times out
15520 * and forced_ack is still on, then the last probe
15521 * was not responded to. In such cases we have a
15522 * sysctl that controls the behavior. Either we apply
15523 * the rtt but with reduced confidence (0). Or we just
15524 * plain don't apply the rtt estimate. Having data flow
15525 * will clear the probe_not_answered flag i.e. cum-ack
15526 * move forward <or> exiting and reentering persists.
15527 */
15534 /*
15535 * In persists only apply the RTT update if this is
15536 * a response to our window probe. And that
15537 * means the rwnd sent must match the current
15538 * snd_wnd. If it does not, then we got a
15539 * window update ack instead. For keepalive
15540 * we allow the answer no matter what the window.
15541 *
15542 * Note that if the probe_not_answered is set then
15543 * the forced_ack_ts is the oldest one i.e. the first
15544 * probe sent that might have been lost. This assures
15545 * us that if we do calculate an RTT it is longer not
15546 * some short thing.
15547 */
15557 /* We have a retransmitted probe here too */
15561 }
15562 }
15563 }
15564 }
15566 static void
15568 {
15569 /*
15570 * The next send has occurred mark the end of the round
15571 * as when that data gets acknowledged. We can
15572 * also do common things we might need to do when
15573 * a round begins.
15574 */
15578 }
15581 static void
15584 {
15603 log.u_bbr.bbr_substate |= rack->pcm_enabled; /* bits are NIE for Needed, Inprogress, Enabled */
15606 }
15607 }
15609 static void
15611 {
15612 /*
15613 * The round (current_round) has ended. We now
15614 * setup for the next round by incrementing the
15615 * round numnber and doing any round specific
15616 * things.
15617 */
15620 /* New round (current_round) begins at next send */
15625 /*
15626 * If we have enabled PCM, then we need to
15627 * check if the round has adanced to the state
15628 * where one is required.
15629 */
15635 rack_log_pcm(rack, 3, rack->r_ctl.last_pcm_round, rack_pcm_every_n_rounds, rack->r_ctl.current_round );
15637 rack_log_pcm(rack, 3, rack->r_ctl.last_pcm_round, rack_pcm_every_n_rounds, rack->r_ctl.current_round );
15638 }
15639 }
15641 /* We have hystart enabled send the round info in */
15644 }
15645 }
15646 /*
15647 * For DGP an initial startup check. We want to validate
15648 * that we are not just pushing on slow-start and just
15649 * not gaining.. i.e. filling buffers without getting any
15650 * boost in b/w during the inital slow-start.
15651 */
15657 ((rack->r_ctl.current_round - rack->r_ctl.last_rnd_of_gp_rise) >= rack->r_ctl.gp_rnd_thresh)) {
15659 /*
15660 * We are in the initial SS and we have hd rack_rnd_cnt_req rounds(def:5) where
15661 * we have not gained the required amount in the gp_est (120.0% aka 1200). Lets
15662 * exit SS.
15663 *
15664 * Pick up the flight size now as we enter slowstart (not the
15665 * cwnd which may be inflated).
15666 */
15683 }
15687 }
15689 /* Turn off any fast output running */
15691 }
15692 }
15694 static int
15695 rack_do_compressed_ack_processing(struct tcpcb *tp, struct socket *so, struct mbuf *m, int nxt_pkt, struct timeval *tv)
15696 {
15697 /*
15698 * Handle a "special" compressed ack mbuf. Each incoming
15699 * ack has only four possible dispositions:
15700 *
15701 * A) It moves the cum-ack forward
15702 * B) It is behind the cum-ack.
15703 * C) It is a window-update ack.
15704 * D) It is a dup-ack.
15705 *
15706 * Note that we can have between 1 -> TCP_COMP_ACK_ENTRIES
15707 * in the incoming mbuf. We also need to still pay attention
15708 * to nxt_pkt since there may be another packet after this
15709 * one.
15710 */
15711 #ifdef TCP_ACCOUNTING
15714 #endif
15722 #ifdef TCP_ACCOUNTING
15724 #endif
15728 #ifdef TCP_ACCOUNTING
15730 #endif
15740 /*
15741 * We have a goodput in progress
15742 * and we have entered a late state.
15743 * Do we have enough data in the sb
15744 * to handle the GPUT request?
15745 */
15752 /*
15753 * There are not enough bytes in the socket
15754 * buffer that have been sent to cover this
15755 * measurement. Cancel it.
15756 */
15762 }
15763 }
15781 /* Check in on probertt */
15783 }
15785 #ifdef TCP_ACCOUNTING
15787 #endif
15798 /* Setup the window */
15802 /* figure out the type of ack */
15804 /* Case B*/
15807 /* Case A */
15810 /* Case D */
15813 /* Case C */
15815 }
15818 /* Validate timestamp */
15820 /* Setup for a timestamp */
15825 /*
15826 * If echoed timestamp is later than the current time, fall back to
15827 * non RFC1323 RTT calculation. Normalize timestamp if syncookies
15828 * were used when this connection was established.
15829 */
15835 #ifdef TCP_ACCOUNTING
15840 }
15841 }
15842 #endif
15844 }
15845 }
15850 }
15852 /* Setup for a no options */
15854 }
15855 /* Update the rcv time and perform idle reduction possibly */
15861 }
15863 /* Now what about ECN of a chain of pure ACKs? */
15868 #ifdef TCP_ACCOUNTING
15869 /* Count for the specific type of ack in */
15872 }
15873 #endif
15874 /*
15875 * Note how we could move up these in the determination
15876 * above, but we don't so that way the timestamp checks (and ECN)
15877 * is done first before we do any processing on the ACK.
15878 * The non-compressed path through the code has this
15879 * weakness (noted by @jtl) that it actually does some
15880 * processing before verifying the timestamp information.
15881 * We don't take that path here which is why we set
15882 * the ack_val_set first, do the timestamp and ecn
15883 * processing, and then look at what we have setup.
15884 */
15886 /*
15887 * Case B flag reordering, if window is not closed
15888 * or it could be a keep-alive or persists
15889 */
15894 }
15896 /* Case D */
15899 /* Case C */
15907 }
15911 }
15912 #ifdef TCP_ACCOUNTING
15914 #endif
15920 /* Case A */
15922 /*
15923 * We just send an ack since the incoming
15924 * ack is beyond the largest seq we sent.
15925 */
15930 }
15932 nsegs++;
15933 /* If the window changed setup to update */
15939 }
15940 #ifdef TCP_ACCOUNTING
15941 /* Account for the acks */
15944 }
15945 #endif
15947 /* Setup our act_rcv_time */
15955 }
15958 #ifdef TCP_REQUEST_TRK
15960 #endif
15962 /* Is the dsack round over? */
15964 /* Yes it is */
15967 }
15968 }
15969 }
15970 }
15971 /* And lets be sure to commit the rtt measurements for this ack */
15973 #ifdef TCP_ACCOUNTING
15980 }
15981 }
15982 #endif
15983 }
15984 #ifdef TCP_ACCOUNTING
15986 #endif
15987 /* Tend to any collapsed window */
15989 /* The peer collapsed the window */
15998 /*
15999 * The draft (v3) calls for us to use SEQ_GEQ, but that
16000 * causes issues when we are just going app limited. Lets
16001 * instead use SEQ_GT <or> where its equal but more data
16002 * is outstanding.
16003 *
16004 * Also make sure we are on the last ack of a series. We
16005 * have to have all the ack's processed in queue to know
16006 * if there is something left outstanding.
16007 *
16008 */
16012 /*
16013 * We have crossed into a new round with
16014 * this th_ack value.
16015 */
16017 }
16018 /*
16019 * Clear the probe not answered flag
16020 * since cum-ack moved forward.
16021 */
16024 /*
16025 * T/TCP: Connection was half-synchronized, and our SYN has
16026 * been ACK'd (so connection is now fully synchronized). Go
16027 * to non-starred state, increment snd_una for ACK of SYN,
16028 * and check if we can do window scaling.
16029 */
16033 }
16046 }
16049 /*
16050 * We were in recovery, hit a rxt timeout
16051 * and never re-entered recovery. The timeout(s)
16052 * made up all the lost data. In such a case
16053 * we need to clear the rto_from_rec flag.
16054 */
16056 }
16057 /* Handle the rack-log-ack part (sendmap) */
16062 /*
16063 * We must be sure our fin
16064 * was sent and acked (we can be
16065 * in FIN_WAIT_1 without having
16066 * sent the fin).
16067 */
16069 /*
16070 * Lets make sure snd_una is updated
16071 * since most likely acked_amount = 0 (it
16072 * should be).
16073 */
16075 }
16076 /* Did we make a RTO error? */
16083 }
16084 /* Handle the data in the socket buffer */
16092 /*
16093 * Grab the segsiz, multiply by 2 and add the snd_cwnd
16094 * that is the max the CC should add if we are exiting
16095 * recovery and doing a late add.
16096 */
16100 }
16103 /* Must be non-newreno (cubic) getting too ahead of itself */
16105 }
16109 /* Note we want to hold the sb lock through the sendmap adjust */
16111 /* Wake up the socket if we have room to write more */
16115 }
16116 /* update progress */
16120 /* Clear out shifts and such */
16126 /* Send recover and snd_nxt must be dragged along */
16131 /*
16132 * If the RXT timer is running we want to
16133 * stop it, so we can restart a TLP (or new RXT).
16134 */
16144 /* Check if we are dragging bottom */
16146 }
16158 /* Set so we might enter persists... */
16165 /*
16166 * The socket was gone and the
16167 * peer sent data (not now in the past), time to
16168 * reset him.
16169 */
16171 /* tcp_close will kill the inp pre-log the Reset */
16173 #ifdef TCP_ACCOUNTING
16179 }
16180 }
16181 #endif
16185 #ifdef TCP_ACCOUNTING
16187 #endif
16189 }
16190 /*
16191 * We would normally do drop-with-reset which would
16192 * send back a reset. We can't since we don't have
16193 * all the needed bits. Instead lets arrange for
16194 * a call to tcp_output(). That way since we
16195 * are in the closed state we will generate a reset.
16196 *
16197 * Note if tcp_accounting is on we don't unpin since
16198 * we do that after the goto label.
16199 */
16201 }
16205 /*
16206 * If we can't receive any more data, then closing user can
16207 * proceed. Starting the timer is contrary to the
16208 * specification, but if we don't get a FIN we'll hang
16209 * forever.
16210 *
16211 */
16216 tcp_finwait2_timeout :
16218 }
16220 /*
16221 * We don't change to fin-wait-2 if we have our fin acked
16222 * which means we are probably in TCPS_CLOSING.
16223 */
16225 }
16226 }
16227 }
16228 /* Wake up the socket if we have room to write more */
16234 /*
16235 * We cheat here and don't send a RST, we should send one
16236 * when the pacer drops the connection.
16237 */
16238 #ifdef TCP_ACCOUNTING
16244 }
16245 }
16247 #endif
16251 }
16252 }
16256 #ifdef TCP_ACCOUNTING
16262 }
16263 }
16265 #endif
16271 #ifdef TCP_ACCOUNTING
16277 }
16278 }
16280 #endif
16286 #ifdef TCP_ACCOUNTING
16292 }
16293 }
16294 #endif
16299 tcp_finwait2_timeout :
16301 }
16306 }
16307 }
16309 /*
16310 * We re doing fast output.. can we expand that?
16311 */
16313 }
16314 #ifdef TCP_ACCOUNTING
16320 }
16321 }
16328 }
16329 }
16330 #endif
16331 }
16332 /* Now is there a next packet, if so we are done */
16336 #ifdef TCP_ACCOUNTING
16338 #endif
16341 }
16347 send_out_a_rst:
16349 #ifdef TCP_ACCOUNTING
16351 #endif
16353 }
16355 }
16359 #ifdef TCP_ACCOUNTING
16361 #endif
16365 }
16367 #define TCP_LRO_TS_OPTION \
16368 ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | \
16369 (TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)
16371 static int
16375 {
16378 #ifdef TCP_ACCOUNTING
16380 #endif
16383 /*
16384 * cts - is the current time from tv (caller gets ts) in microseconds.
16385 * ms_cts - is the current time from tv in milliseconds.
16386 * us_cts - is the time that LRO or hardware actually got the packet in microseconds.
16387 */
16397 #ifdef TCP_ACCOUNTING
16399 #endif
16405 /*
16406 * tv passed from common code is from either M_TSTMP_LRO or
16407 * tcp_get_usecs() if no LRO m_pkthdr timestamp is present.
16408 */
16411 /*
16412 * If we are the connecting socket we will
16413 * hit rack_init() when no sequence numbers
16414 * are setup. This makes it so we must defer
16415 * some initialization. Call that now.
16416 */
16418 }
16419 /*
16420 * Check to see if we need to skip any output plans. This
16421 * can happen in the non-LRO path where we are pacing and
16422 * must process the ack coming in but need to defer sending
16423 * anything becase a pacing timer is running.
16424 */
16427 /*
16428 * All compressed ack's are ack's by definition so
16429 * remove any ack required flag and then do the processing.
16430 */
16433 }
16439 /*
16440 * Ok conditions are right for queuing the packets
16441 * but we do have to check the flags in the inp, it
16442 * could be, if a sack is present, we want to be awoken and
16443 * so should process the packets.
16444 */
16449 /*
16450 * If there is no options, or just a
16451 * timestamp option, we will want to queue
16452 * the packets. This is the same that LRO does
16453 * and will need to change with accurate ECN.
16454 */
16464 }
16466 /*
16467 * It is unrealistic to think we can pace in less than
16468 * the minimum granularity of the pacer (def:250usec). So
16469 * if we have less than that time remaining we should go
16470 * ahead and allow output to be "early". We will attempt to
16471 * make up for it in any pacing time we try to apply on
16472 * the outbound packet.
16473 */
16475 }
16476 }
16477 /*
16478 * If there is a RST or FIN lets dump out the bw
16479 * with a FIN the connection may go on but we
16480 * may not.
16481 */
16492 }
16497 #ifdef TCP_ACCOUNTING
16501 #endif
16511 /*
16512 * Unscale the window into a 32-bit value. For the SYN_SENT state
16513 * the scale is zero.
16514 */
16516 #ifdef TCP_ACCOUNTING
16518 /*
16519 * We have a tradeoff here. We can either do what we are
16520 * doing i.e. pinning to this CPU and then doing the accounting
16521 * <or> we could do a critical enter, setup the rdtsc and cpu
16522 * as in below, and then validate we are on the same CPU on
16523 * exit. I have choosen to not do the critical enter since
16524 * that often will gain you a context switch, and instead lock
16525 * us (line above this if) to the same CPU with sched_pin(). This
16526 * means we may be context switched out for a higher priority
16527 * interupt but we won't be moved to another CPU.
16528 *
16529 * If this occurs (which it won't very often since we most likely
16530 * are running this code in interupt context and only a higher
16531 * priority will bump us ... clock?) we will falsely add in
16532 * to the time the interupt processing time plus the ack processing
16533 * time. This is ok since its a rare event.
16534 */
16537 }
16538 #endif
16539 /*
16540 * Parse options on any incoming segment.
16541 */
16547 __func__));
16549 __func__));
16551 /*
16552 * We don't look at sack's from the
16553 * peer because the MSS is too small which
16554 * can subject us to an attack.
16555 */
16557 }
16560 /*
16561 * We have a goodput in progress
16562 * and we have entered a late state.
16563 * Do we have enough data in the sb
16564 * to handle the GPUT request?
16565 */
16572 /*
16573 * There are not enough bytes in the socket
16574 * buffer that have been sent to cover this
16575 * measurement. Cancel it.
16576 */
16582 }
16583 }
16587 #ifdef TCP_REQUEST_TRK
16594 }
16595 #endif
16600 else
16613 #ifdef TCP_ACCOUNTING
16615 #endif
16618 /* Record the hardware timestamp if present */
16624 /* Record the LRO the arrival timestamp */
16629 }
16631 /* Log the rcv time */
16633 #ifdef TCP_REQUEST_TRK
16640 /* Copy out any client req info */
16641 /* seconds */
16643 /* useconds */
16652 }
16657 }
16658 }
16659 #endif
16662 }
16663 /* Remove ack required flag if set, we have one */
16672 }
16673 /*
16674 * If a segment with the ACK-bit set arrives in the SYN-SENT state
16675 * check SEQ.ACK first as described on page 66 of RFC 793, section 3.9.
16676 */
16681 #ifdef TCP_ACCOUNTING
16683 #endif
16685 }
16686 /*
16687 * If timestamps were negotiated during SYN/ACK and a
16688 * segment without a timestamp is received, silently drop
16689 * the segment, unless it is a RST segment or missing timestamps are
16690 * tolerated.
16691 * See section 3.2 of RFC 7323.
16692 */
16699 }
16700 /*
16701 * Segment received on connection. Reset idle time and keep-alive
16702 * timer. XXX: This should be done after segment validation to
16703 * ignore broken/spoofed segs.
16704 */
16710 }
16712 #ifdef STATS
16714 #endif
16717 /*
16718 * TCP ECN processing. XXXJTL: If we ever use ECN, we need to move
16719 * this to occur after we've validated the segment.
16720 */
16723 iptos))
16726 /*
16727 * If echoed timestamp is later than the current time, fall back to
16728 * non RFC1323 RTT calculation. Normalize timestamp if syncookies
16729 * were used when this connection was established.
16730 */
16735 }
16741 /*
16742 * We are receiving only and thus not sending
16743 * data to do an RTT. We set a flag when we first
16744 * sent this TS to the peer. We now have it back
16745 * and have an RTT to share. We log it as a conf
16746 * 4, we are not so sure about it.. since we
16747 * may have lost an ack.
16748 */
16752 /* Submit and commit the timer */
16756 }
16757 }
16758 /*
16759 * If its the first time in we need to take care of options and
16760 * verify we can do SACK for rack!
16761 */
16763 /* Should be init'd by rack_init() */
16768 }
16770 /*
16771 * Process options only when we get SYN/ACK back. The SYN
16772 * case for incoming connections is handled in tcp_syncache.
16773 * According to RFC1323 the window field in a SYN (i.e., a
16774 * <SYN> or <SYN,ACK>) segment itself is never scaled. XXX
16775 * this is traditional behavior, may need to be cleaned up.
16776 */
16778 /* Handle parallel SYN for ECN */
16786 /*
16787 * Initial send window. It will be updated with the
16788 * next incoming segment to the scaled value.
16789 */
16801 }
16811 else
16814 else
16820 }
16821 }
16822 /*
16823 * At this point we are at the initial call. Here we decide
16824 * if we are doing RACK or not. We do this by seeing if
16825 * TF_SACK_PERMIT is set and the sack-not-required is clear.
16826 * The code now does do dup-ack counting so if you don't
16827 * switch back you won't get rack & TLP, but you will still
16828 * get this stack.
16829 */
16836 #ifdef TCP_ACCOUNTING
16838 #endif
16840 }
16843 }
16850 /* Check in on probertt */
16852 }
16857 }
16858 /*
16859 * This is the one exception case where we set the rack state
16860 * always. All other times (timers etc) we must have a rack-state
16861 * set (so we assure we have done the checks above for SACK).
16862 */
16874 /* The connection will be killed by a reset check the tracepoint */
16876 }
16881 /*
16882 * If retval is 1 the tcb is unlocked and most likely the tp
16883 * is gone.
16884 */
16891 /*
16892 * If we are going for target, lets recheck before
16893 * we output.
16894 */
16896 }
16898 /* How much has been acked? */
16900 /* We have enough to set in the pacing segment size */
16903 }
16904 }
16906 #ifdef TCP_ACCOUNTING
16907 /*
16908 * If we set the ack_val_se to what ack processing we are doing
16909 * we also want to track how many cycles we burned. Note
16910 * the bits after tcp_output we let be "free". This is because
16911 * we are also tracking the tcp_output times as well. Note the
16912 * use of 0xf here since we only have 11 counter (0 - 0xa) and
16913 * 0xf cannot be returned and is what we initialize it too to
16914 * indicate we are not doing the tabulations.
16915 */
16922 }
16923 }
16924 #endif
16930 do_output_now:
16932 #ifdef TCP_ACCOUNTING
16934 #endif
16936 }
16938 }
16946 /*
16947 * We are not in hpts and we had a pacing timer up. Use
16948 * the remaining time (slot_remaining) to restart the timer.
16949 */
16953 }
16954 /* Clear the flag, it may have been cleared by output but we may not have */
16957 /*
16958 * The draft (v3) calls for us to use SEQ_GEQ, but that
16959 * causes issues when we are just going app limited. Lets
16960 * instead use SEQ_GT <or> where its equal but more data
16961 * is outstanding.
16962 *
16963 * Also make sure we are on the last ack of a series. We
16964 * have to have all the ack's processed in queue to know
16965 * if there is something left outstanding.
16966 */
16970 /*
16971 * We have crossed into a new round with
16972 * the new snd_unae.
16973 */
16975 }
16982 /* We could not send (probably in the hpts but stopped the timer earlier)? */
16987 /* keep alive not needed if we are hptsi output yet */
16988 ;
17000 }
17002 }
17004 /*
17005 * We are late in the sending
17006 * and we did not call the output
17007 * (this probably should not happen).
17008 */
17010 }
17012 }
17015 /* Do we have the correct timer running? */
17018 }
17019 done_with_input:
17023 }
17025 #ifdef TCP_ACCOUNTING
17027 #endif
17029 }
17031 static void
17034 {
17037 /* First lets see if we have old packets */
17042 }
17043 }
17047 /* Should not be should we kassert instead? */
17049 }
17053 }
17054 }
17058 {
17063 /* Return the next guy to be re-transmitted */
17066 }
17068 /* retran the end FIN? */
17070 }
17071 /* ok lets look at this one */
17075 }
17078 }
17082 }
17083 check_it:
17086 /*
17087 * No sack so we automatically do the 3 strikes and
17088 * retransmit (no rack timer would be started).
17089 */
17091 }
17094 }
17097 /* Its not yet ready */
17099 }
17106 /* No time since sending */
17108 }
17110 /* It has not been long enough yet */
17112 }
17115 /*
17116 * We have passed the dup-ack threshold <or>
17117 * a SACK has indicated this is missing.
17118 * Note that if you are a declared attacker
17119 * it is only the dup-ack threshold that
17120 * will cause retransmits.
17121 */
17122 /* log retransmit reason */
17126 }
17128 }
17130 static void
17134 {
17140 /*
17141 * We are not verbose screen out all but
17142 * ones we always want.
17143 */
17151 }
17152 }
17183 else
17190 /* We are in slow start */
17193 /* we are on congestion avoidance */
17195 }
17214 }
17215 }
17217 static uint32_t
17219 {
17225 }
17229 /* Use the user mss since we are not exactly matched */
17231 }
17235 else
17245 }
17250 }
17252 static uint64_t
17253 rack_arrive_at_discounted_rate(struct tcp_rack *rack, uint64_t window_input, uint32_t *rate_set, uint32_t *gain_b)
17254 {
17259 /*
17260 * The cwnd is collapsed to
17261 * nearly zero, maybe because of a time-out?
17262 * Lets drop back to the lt-bw.
17263 */
17265 /* Set the flag so the caller knows its a rate and not a reduced window */
17269 /*
17270 * If we are in recover our cwnd needs to be less for
17271 * our pacing consideration.
17272 */
17280 }
17282 /*
17283 * Apply Timely factor to increase/decrease the
17284 * amount we are pacing at.
17285 */
17289 }
17292 }
17295 /*
17296 * What is being returned here is a trimmed down
17297 * window values in all cases where rate_set is left
17298 * at 0. In one case we actually return the rate (lt_bw).
17299 * the "reduced_win" is returned as a slimmed down cwnd that
17300 * is then calculated by the caller into a rate when rate_set
17301 * is 0.
17302 */
17304 }
17306 static int32_t
17307 pace_to_fill_cwnd(struct tcp_rack *rack, int32_t slot, uint32_t len, uint32_t segsiz, int *capped, uint64_t *rate_wanted, uint8_t non_paced)
17308 {
17321 /* The rtt is huge, N * smallest, lets not fill */
17323 }
17326 /*
17327 * first lets calculate the b/w based on the last us-rtt
17328 * and the the smallest send window.
17329 */
17337 }
17338 }
17339 /* Take the rwnd if its smaller */
17342 /* Now lets make it into a b/w */
17345 /* Adjust to any cap */
17349 at_lt_bw:
17351 /*
17352 * We want to limit fill-cw to the some multiplier
17353 * of the max(lt_bw, gp_est). The normal default
17354 * is 0 for off, so a sysctl has enabled it.
17355 */
17362 else
17383 }
17386 }
17387 /* We are below the min b/w */
17399 /* We are capping bw at the highest rate table entry */
17401 /* The original rate was also capped */
17403 }
17410 }
17418 /*
17419 * Ok we may have a first attempt that is greater than our top rate
17420 * lets check.
17421 */
17430 }
17431 }
17432 }
17437 }
17438 /*
17439 * Ok fill_bw holds our mythical b/w to fill the cwnd
17440 * in an rtt (unless it was capped), what does that
17441 * time wise equate too?
17442 */
17452 }
17454 static int32_t
17455 rack_get_pacing_delay(struct tcp_rack *rack, struct tcpcb *tp, uint32_t len, struct rack_sendmap *rsm, uint32_t segsiz, int line)
17456 {
17466 else
17469 /*
17470 * We use the most optimistic possible cwnd/srtt for
17471 * sending calculations. This will make our
17472 * calculation anticipate getting more through
17473 * quicker then possible. But thats ok we don't want
17474 * the peer to have a gap in data sending.
17475 */
17479 old_method:
17480 /*
17481 * We keep no precise pacing with the old method
17482 * instead we use the pacer to mitigate bursts.
17483 */
17486 else
17490 else
17492 /* Inflate cwnd by 1000 so srtt of usecs is in ms */
17496 }
17497 /*
17498 * Calculate how long this will take to drain, if
17499 * the calculation comes out to zero, thats ok we
17500 * will use send_a_lot to possibly spin around for
17501 * more increasing tot_len_this_send to the point
17502 * that its going to require a pace, or we hit the
17503 * cwnd. Which in that case we are just waiting for
17504 * a ACK.
17505 */
17507 /* Now do we reduce the time so we don't run dry? */
17525 /*******************************************************/
17526 /* RRS: We insert non-paced call to stats here for len */
17527 /*******************************************************/
17536 }
17541 /* no way to yet do an estimate */
17554 /* Now lets make it into a b/w */
17557 }
17564 }
17567 /*
17568 * No way yet to make a b/w estimate or
17569 * our raise is set incorrectly.
17570 */
17572 }
17574 /* We need to account for all the overheads */
17576 /*
17577 * We need the diff between 1514 bytes (e-mtu with e-hdr)
17578 * and how much data we put in each packet. Yes this
17579 * means we may be off if we are larger than 1500 bytes
17580 * or smaller. But this just makes us more conservative.
17581 */
17585 #ifdef INET6
17587 #endif
17589 #ifdef INET
17591 #endif
17592 }
17593 /* We add a fixed 14 for the ethernet header */
17603 /*
17604 * Ok we need to release it, we
17605 * have fallen too low.
17606 */
17611 }
17612 }
17615 /*
17616 * We want more than the hardware can give us,
17617 * don't start any hw pacing.
17618 */
17621 /*
17622 * Ok we need to release it, we
17623 * want more than the card can give us and
17624 * no rate cap is in place. Set it up so
17625 * when we want less we can retry.
17626 */
17631 }
17632 }
17634 /*
17635 * We lost our rate somehow, this can happen
17636 * if the interface changed underneath us.
17637 */
17640 /* Lets re-allow attempting to setup pacing */
17646 }
17654 /*
17655 * We want to pace at our rate *or* faster to
17656 * fill the cwnd to the max if its not full.
17657 */
17659 /* Re-check to make sure we are not exceeding our max b/w */
17662 /*
17663 * We want more than the hardware can give us,
17664 * don't start any hw pacing.
17665 */
17668 /*
17669 * Ok we need to release it, we
17670 * want more than the card can give us and
17671 * no rate cap is in place. Set it up so
17672 * when we want less we can retry.
17673 */
17679 }
17680 }
17681 }
17688 /*
17689 * Lets attempt to turn on hardware pacing
17690 * if we can.
17691 */
17695 rate_wanted,
17696 RS_PACING_GEQ,
17709 }
17712 /* Do we need to adjust our rate? */
17717 /**
17718 * We have four possible states here
17719 * having to do with the previous time
17720 * and this time.
17721 * previous | this-time
17722 * A) 0 | 0 -- fill_cw not in the picture
17723 * B) 1 | 0 -- we were doing a fill-cw but now are not
17724 * C) 1 | 1 -- all rates from fill_cw
17725 * D) 0 | 1 -- we were doing non-fill and now we are filling
17726 *
17727 * For case A, C and D we don't allow a drop. But for
17728 * case B where we now our on our steady rate we do
17729 * allow a drop.
17730 *
17731 */
17734 }
17739 /*
17740 * The pacing rate is too low for hardware, but
17741 * do allow hardware pacing to be restarted.
17742 */
17752 }
17756 rate_wanted,
17757 RS_PACING_GEQ,
17760 /*
17761 * Lost the rate, lets drop hardware pacing
17762 * period.
17763 */
17780 }
17782 /* We just need to adjust the segment size */
17788 }
17789 }
17790 }
17791 done_w_hdwr:
17795 /*
17796 * Sanity check, we do not allow the pacing delay
17797 * to be longer than the SRTT of the path. If it is
17798 * a slow path, then adding a packet should increase
17799 * the RTT and compensate for this i.e. the srtt will
17800 * be greater so the allowed pacing time will be greater.
17801 *
17802 * Note this restriction is not for where a peak rate
17803 * is set, we are doing fixed pacing or hardware pacing.
17804 */
17807 else
17810 rack_log_pacing_delay_calc(rack, srtt, slot, rate_wanted, bw_est, lentim, 99, __LINE__, NULL, 0);
17812 }
17813 }
17814 /*******************************************************************/
17815 /* RRS: We insert paced call to stats here for len and rate_wanted */
17816 /*******************************************************************/
17817 rack_log_pacing_delay_calc(rack, len, slot, rate_wanted, bw_est, lentim, 2, __LINE__, rsm, 0);
17818 }
17820 /*
17821 * If this rate is seeing enobufs when it
17822 * goes to send then either the nic is out
17823 * of gas or we are mis-estimating the time
17824 * somehow and not letting the queue empty
17825 * completely. Lets add to the pacing time.
17826 */
17835 }
17837 }
17839 static void
17842 {
17846 /*
17847 * We don't start any measurements if we are
17848 * not at least established.
17849 */
17851 }
17853 /*
17854 * We will get no more data into the SB
17855 * this means we need to have the data available
17856 * before we start a measurement.
17857 */
17862 /* Nope not enough data */
17864 }
17865 }
17880 /* Special case initial measurement */
17885 }
17886 /*
17887 * We take a guess out into the future,
17888 * if we have no measurement and no
17889 * initial rate, we measure the first
17890 * initial-windows worth of data to
17891 * speed up getting some GP measurement and
17892 * thus start pacing.
17893 */
17909 }
17911 /*
17912 * We are out somewhere in the sb
17913 * can we use the already outstanding data?
17914 */
17917 /*
17918 * Yes first one is good and in this case
17919 * the tp->gput_ts is correctly set based on
17920 * the last ack that arrived (no need to
17921 * set things up when an ack comes in).
17922 */
17926 /* retransmission? */
17928 }
17931 /*
17932 * If rc_first_appl is NULL
17933 * then the cnt should be 0.
17934 * This is probably an error, maybe
17935 * a KASSERT would be approprate.
17936 */
17938 }
17939 /*
17940 * If we have a marker pointer to the last one that is
17941 * app limited we can use that, but we need to set
17942 * things up so that when it gets ack'ed we record
17943 * the ack time (if its not already acked).
17944 */
17946 /*
17947 * We want to get to the rsm that is either
17948 * next with space i.e. over 1 MSS or the one
17949 * after that (after the app-limited).
17950 */
17954 /* Have to use the next one */
17957 /* Use after the first MSS of it is acked */
17960 }
17961 }
17964 /*
17965 * Either its a retransmit or
17966 * the last is the app-limited one.
17967 */
17969 }
17970 }
17972 start_set:
17974 /*
17975 * This one has been acked use the arrival ack time
17976 */
17981 /*
17982 * Ok in this path we need to use the r_end now
17983 * since this guy is the starting ack.
17984 */
17986 /*
17987 * We also need to adjust up the sendtime
17988 * to the send of the next data after my_rsm.
17989 */
17994 /*
17995 * The next as not been sent, thats the
17996 * case for using the latest.
17997 */
17999 }
18000 }
18006 /*
18007 * We just cleared an application limited period
18008 * so the next seq out needs to skip the first
18009 * ack.
18010 */
18013 }
18022 /* Now lets make sure all are marked as they should be */
18026 }
18028 use_latest:
18029 /*
18030 * We don't know how long we may have been
18031 * idle or if this is the first-send. Lets
18032 * setup the flag so we will trim off
18033 * the first ack'd data so we get a true
18034 * measurement.
18035 */
18039 /* Find this guy so we can pull the send time */
18044 /*
18045 * Unlikely since its probably what was
18046 * just transmitted (but I am paranoid).
18047 */
18050 }
18052 /* This also is unlikely */
18054 }
18056 /*
18057 * TSNH unless we have some send-map limit,
18058 * and even at that it should not be hitting
18059 * that limit (we should have stopped sending).
18060 */
18065 }
18075 }
18080 {
18086 else
18089 /* We never want to go over our peers rcv-window */
18096 /*
18097 * We have in flight what we are allowed by cwnd (if
18098 * it was rwnd blocking it would have hit above out
18099 * >= tp->snd_wnd).
18100 */
18102 }
18105 /* We would send too much (beyond the rwnd) */
18107 }
18109 /*
18110 * We don't have that much in the SB, how much is
18111 * there?
18112 */
18114 }
18115 }
18117 }
18119 static void
18123 {
18149 }
18150 }
18157 {
18158 #ifdef KERN_TLS
18160 #ifdef INVARIANTS
18162 #endif
18163 #endif
18176 #ifdef KERN_TLS
18179 else
18181 #ifdef INVARIANTS
18183 #endif
18184 #endif
18189 }
18190 #ifdef KERN_TLS
18194 else
18197 /*
18198 * Avoid mixing TLS records with handshake
18199 * data or TLS records from different
18200 * sessions.
18201 */
18206 }
18207 }
18208 #endif
18211 /*
18212 * For M_EXTPG mbufs, add 3 segments
18213 * + 1 in case we are crossing page boundaries
18214 * + 2 in case the TLS hdr/trailer are used
18215 * It is cheaper to just add the segments
18216 * than it is to take the cache miss to look
18217 * at the mbuf ext_pgs state in detail.
18218 */
18225 }
18227 /* Break if we really can't fit anymore. */
18231 }
18233 /*
18234 * Reduce size if you can't copy the whole
18235 * mbuf. If we can't copy the whole mbuf, also
18236 * adjust len so the loop will end after this
18237 * mbuf.
18238 */
18243 }
18246 frags++;
18250 }
18264 }
18270 /*
18271 * We have more so we move forward or
18272 * we have consumed the entire mbuf and
18273 * len has fell to 0.
18274 */
18277 }
18279 }
18284 /*
18285 * Save off the size of the mbuf. We do
18286 * this so that we can recognize when it
18287 * has been trimmed by sbcut() as acks
18288 * come in.
18289 */
18293 /*
18294 * This is the case where the next mbuf went to NULL. This
18295 * means with this copy we have sent everything in the sb.
18296 * In theory we could clear the fast_output flag, but lets
18297 * not since its possible that we could get more added
18298 * and acks that call the extend function which would let
18299 * us send more.
18300 */
18303 }
18304 }
18306 nospace:
18311 }
18313 /*
18314 * This is a copy of m_copym(), taking the TSO segment size/limit
18315 * constraints into account, and advancing the sndptr as it goes.
18316 */
18320 {
18326 /*
18327 * The trailing space changed, mbufs can grow
18328 * at the tail but they can't shrink from
18329 * it, KASSERT that. Adjust the orig_m_len to
18330 * compensate for this change.
18331 */
18334 m,
18335 rack,
18342 }
18344 /*
18345 * Mbuf shrank, trimmed off the top by an ack, our
18346 * offset changes.
18347 */
18356 else
18359 #ifdef INVARIANTS
18363 #endif
18364 }
18369 __FUNCTION__,
18371 /* Save off the right location before we copy and advance */
18378 }
18380 /* Log the buffer level */
18381 static void
18385 {
18389 #ifdef RATELIMIT
18392 #endif
18409 }
18411 static uint32_t
18414 {
18416 #ifdef RATELIMIT
18422 /* Failed or queue is zero */
18426 }
18431 }
18432 /*
18433 * If we reach here we have some bytes in
18434 * the queue. The number returned is a value
18435 * between 0 and 0xffff where ffff is full
18436 * and 0 is empty. So how best to make this into
18437 * something usable?
18438 *
18439 * The "safer" way is lets take the b/w gotten
18440 * from the query (which should be our b/w rate)
18441 * and pretend that a full send (our rc_pace_max_segs)
18442 * is outstanding. We factor it so its as if a full
18443 * number of our MSS segment is terms of full
18444 * ethernet segments are outstanding.
18445 */
18453 /* TSNH -- KASSERT? */
18455 }
18456 out:
18474 }
18475 #endif
18477 }
18479 static int
18481 uint64_t ts_val, uint32_t cts, uint32_t ms_cts, struct timeval *tv, int len, uint8_t doing_tlp)
18482 {
18483 /*
18484 * Enter the fast retransmit path. We are given that a sched_pin is
18485 * in place (if accounting is compliled in) and the cycle count taken
18486 * at the entry is in the ts_val. The concept her is that the rsm
18487 * now holds the mbuf offsets and such so we can directly transmit
18488 * without a lot of overhead, the len field is already set for
18489 * us to prohibit us from sending too much (usually its 1MSS).
18490 */
18498 #ifdef TCP_ACCOUNTING
18501 #endif
18511 #ifdef INET6
18519 {
18522 }
18525 }
18527 /* Its a TLP add the flag, it may already be there but be sure */
18530 /* If it was a TLP it is not not on this retransmit */
18532 }
18540 }
18542 /* We can't send a FIN here */
18544 }
18546 /* We never send a FIN */
18548 }
18553 }
18554 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
18555 /* TCP-MD5 (RFC2385). */
18558 #endif
18568 else
18571 V_tcp_do_tso &&
18575 #ifdef INET6
18578 else
18579 #endif
18586 /* Establish the len to send */
18593 /* extract TSO information */
18597 /*
18598 * Check if we should limit by maximum payload
18599 * length:
18600 */
18602 /* compute maximum TSO length */
18604 max_linkhdr);
18609 }
18610 }
18612 /*
18613 * In case there are too many small fragments don't
18614 * use TSO:
18615 */
18617 }
18620 }
18627 }
18630 /*
18631 * The PUSH bit should only be applied
18632 * if the full retransmission is made. If
18633 * we are sending less than this is the
18634 * left hand edge and should not have
18635 * the PUSH bit.
18636 */
18647 /*
18648 * TLP should not count in retran count, but
18649 * in its own bin
18650 */
18657 }
18658 #ifdef STATS
18660 len);
18661 #endif
18667 /* Fix up the orig_m_len and possibly the mbuf offset */
18669 }
18670 m->m_next = rack_fo_base_copym(rsm->m, rsm->soff, &len, NULL, if_hw_tsomaxsegcount, if_hw_tsomaxsegsize, rsm->r_hw_tls);
18672 /*
18673 * Must have ran out of mbufs for the copy
18674 * shorten it to no longer need tso. Lets
18675 * not put on sendalot since we are low on
18676 * mbufs.
18677 */
18679 }
18682 }
18686 else
18689 }
18697 #ifdef INET6
18701 }
18702 else
18703 #endif
18704 {
18707 }
18708 }
18710 /* See if we can send via the hw queue */
18712 /* If there is nothing in queue (no pacing time) we can send via the hw queue */
18715 }
18718 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
18720 /*
18721 * Calculate MD5 signature and put it into the place
18722 * determined before.
18723 * NOTE: since TCP options buffer doesn't point into
18724 * mbuf's data, calculate offset and use it.
18725 */
18728 /*
18729 * Do not send segment if the calculation of MD5
18730 * digest has failed.
18731 */
18733 }
18734 }
18735 #endif
18736 #ifdef INET6
18750 }
18751 }
18752 #endif
18753 #if defined(INET6) && defined(INET)
18754 else
18755 #endif
18756 #ifdef INET
18757 {
18771 }
18772 /* IP version must be set here for ipv4/ipv6 checking later */
18775 }
18776 #endif
18778 /*
18779 * Here we use segsiz since we have no added options besides
18780 * any standard timestamp options (no DSACKs or SACKS are sent
18781 * via either fast-path).
18782 */
18787 }
18788 #ifdef INET6
18794 else
18796 }
18797 #endif
18798 #if defined(INET) && defined(INET6)
18799 else
18800 #endif
18801 #ifdef INET
18802 {
18809 }
18812 }
18813 }
18814 #endif
18816 /* Set we retransmitted */
18819 /* Its a TLP set ca or ss */
18821 /* Set we sent in CA */
18824 /* Set we sent in SS */
18826 }
18827 }
18828 /* Time to copy in our header */
18837 }
18845 }
18850 else
18855 /* Save off the early/late values */
18862 else
18870 /*
18871 * When we have a retransmit we want to log the
18872 * burst at send and flight at send from before.
18873 */
18877 /*
18878 * This is currently unlikely until we do the
18879 * packet pair probes but I will add it for completeness.
18880 */
18883 }
18900 }
18901 #ifdef INET6
18906 }
18907 else
18908 #endif
18909 #ifdef INET
18910 {
18914 }
18915 #endif
18920 }
18921 /* Move snd_nxt to snd_max so we don't have false retransmissions */
18931 }
18937 }
18946 }
18949 else
18951 }
18956 {
18962 else
18964 }
18969 }
18986 }
18990 }
18992 #ifdef TCP_ACCOUNTING
18996 }
18999 }
19002 }
19004 #endif
19006 failed:
19010 }
19012 static void
19014 {
19015 /*
19016 * Automatic sizing of send socket buffer. Often the send buffer
19017 * size is not optimally adjusted to the actual network conditions
19018 * at hand (delay bandwidth product). Setting the buffer size too
19019 * small limits throughput on links with high bandwidth and high
19020 * delay (eg. trans-continental/oceanic links). Setting the
19021 * buffer size too big consumes too much real kernel memory,
19022 * especially with many connections on busy servers.
19023 *
19024 * The criteria to step up the send buffer one notch are:
19025 * 1. receive window of remote host is larger than send buffer
19026 * (with a fudge factor of 5/4th);
19027 * 2. send buffer is filled to 7/8th with data (so we actually
19028 * have data to make use of it);
19029 * 3. send buffer fill has not hit maximal automatic size;
19030 * 4. our send window (slow start and cogestion controlled) is
19031 * larger than sent but unacknowledged data in send buffer.
19032 *
19033 * Note that the rack version moves things much faster since
19034 * we want to avoid hitting cache lines in the rack_fast_output()
19035 * path so this is called much less often and thus moves
19036 * the SB forward by a percentage.
19037 */
19054 else
19063 }
19064 }
19065 }
19067 static int
19070 {
19071 /*
19072 * Enter to do fast output. We are given that the sched_pin is
19073 * in place (if accounting is compiled in) and the cycle count taken
19074 * at entry is in place in ts_val. The idea here is that
19075 * we know how many more bytes needs to be sent (presumably either
19076 * during pacing or to fill the cwnd and that was greater than
19077 * the max-burst). We have how much to send and all the info we
19078 * need to just send.
19079 */
19080 #ifdef INET
19082 #endif
19089 #ifdef TCP_ACCOUNTING
19091 #endif
19095 #ifdef TCP_ACCOUNTING
19097 #endif
19104 #ifdef INET6
19112 {
19113 #ifdef INET
19116 #endif
19117 }
19121 }
19133 }
19134 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
19135 /* TCP-MD5 (RFC2385). */
19138 #endif
19148 else
19151 V_tcp_do_tso &&
19155 again:
19156 #ifdef INET6
19159 else
19160 #endif
19167 /* Establish the len to send */
19174 /* extract TSO information */
19178 /*
19179 * Check if we should limit by maximum payload
19180 * length:
19181 */
19183 /* compute maximum TSO length */
19185 max_linkhdr);
19190 }
19191 }
19193 /*
19194 * In case there are too many small fragments don't
19195 * use TSO:
19196 */
19198 }
19201 }
19208 }
19221 #ifdef STATS
19223 len);
19224 #endif
19228 /* s_mb and s_soff are saved for rack_log_output */
19232 /*
19233 * Must have ran out of mbufs for the copy
19234 * shorten it to no longer need tso. Lets
19235 * not put on sendalot since we are low on
19236 * mbufs.
19237 */
19239 }
19244 }
19247 }
19251 else
19254 }
19262 #ifdef INET6
19266 }
19267 else
19268 #endif
19269 {
19270 #ifdef INET
19273 #endif
19274 }
19275 }
19278 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
19280 /*
19281 * Calculate MD5 signature and put it into the place
19282 * determined before.
19283 * NOTE: since TCP options buffer doesn't point into
19284 * mbuf's data, calculate offset and use it.
19285 */
19288 /*
19289 * Do not send segment if the calculation of MD5
19290 * digest has failed.
19291 */
19293 }
19294 }
19295 #endif
19296 #ifdef INET6
19310 }
19311 }
19312 #endif
19313 #if defined(INET6) && defined(INET)
19314 else
19315 #endif
19316 #ifdef INET
19317 {
19331 }
19332 /* IP version must be set here for ipv4/ipv6 checking later */
19335 }
19336 #endif
19338 /*
19339 * Here we use segsiz since we have no added options besides
19340 * any standard timestamp options (no DSACKs or SACKS are sent
19341 * via either fast-path).
19342 */
19347 }
19348 #ifdef INET6
19354 else
19356 }
19357 #endif
19358 #if defined(INET) && defined(INET6)
19359 else
19360 #endif
19361 #ifdef INET
19362 {
19369 }
19372 }
19373 }
19374 #endif
19376 /* Set we sent in CA */
19379 /* Set we sent in SS */
19381 }
19382 /* Time to copy in our header */
19391 }
19395 }
19403 else
19408 /* Save off the early/late values */
19426 /* For fast output no retrans so just inflight and how many mss we send */
19433 #ifdef INET6
19438 }
19439 #endif
19440 #if defined(INET) && defined(INET6)
19441 else
19442 #endif
19443 #ifdef INET
19444 {
19448 }
19449 #endif
19453 }
19464 }
19466 /* Unlikely */
19472 /*
19473 * Need to record what we have since we are
19474 * approaching seq wrap.
19475 */
19485 }
19486 }
19493 }
19505 }
19506 {
19512 else
19514 }
19517 else
19522 /* At the end of fast_output scale up the sb */
19526 }
19531 }
19538 #ifdef TCP_ACCOUNTING
19539 cnt_thru++;
19540 #endif
19542 }
19547 #ifdef TCP_ACCOUNTING
19551 }
19554 }
19557 }
19559 #endif
19561 failed:
19566 }
19574 {
19582 /* Less than a full sized pace, lets not */
19586 /* Round down to the nearest pace_max_seg */
19588 }
19591 else
19602 else
19604 }
19605 }
19607 static uint32_t
19609 {
19617 }
19621 {
19625 restart:
19628 /* Nothing, strange turn off validity */
19631 }
19632 /* Can we send it yet? */
19634 /*
19635 * Receiver window has not grown enough for
19636 * the segment to be put on the wire.
19637 */
19639 }
19641 /*
19642 * It has been sacked, lets move to the
19643 * next one if possible.
19644 */
19646 /* Are we done? */
19651 }
19653 }
19654 /* Now has it been long enough ? */
19661 }
19662 /* Not enough time */
19667 }
19670 rack_validate_sizes(struct tcp_rack *rack, int32_t *len, int32_t segsiz, uint32_t pace_max_seg)
19671 {
19678 /* We use pacing min as shaping len req */
19685 /*
19686 * The else is full_size_rxt is on so send it all
19687 * note we do need to check this for exceeding
19688 * our max segment size due to the fact that
19689 * we do sometimes merge chunks together i.e.
19690 * we cannot just assume that we will never have
19691 * a chunk greater than pace_max_seg
19692 */
19695 }
19696 }
19698 static int
19700 {
19712 #ifdef INET
19714 #endif
19726 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
19729 #endif
19741 /* The doing_tlp flag will be set by the actual rack_timeout_tlp() */
19753 #ifdef TCP_ACCOUNTING
19755 #endif
19756 #ifdef INET6
19759 #endif
19765 /* setup and take the cache hits here */
19767 #ifdef TCP_ACCOUNTING
19770 #endif
19773 #ifdef TCP_OFFLOAD
19775 #ifdef TCP_ACCOUNTING
19777 #endif
19779 }
19780 #endif
19782 /*
19783 * If we are the connecting socket we will
19784 * hit rack_init() when no sequence numbers
19785 * are setup. This makes it so we must defer
19786 * some initialization. Call that now.
19787 */
19789 }
19790 /*
19791 * For TFO connections in SYN_RECEIVED, only allow the initial
19792 * SYN|ACK and those sent by the retransmit timer.
19793 */
19798 #ifdef TCP_ACCOUNTING
19800 #endif
19802 }
19803 #ifdef INET6
19805 /* Use the cache line loaded if possible */
19809 }
19810 #endif
19816 /*
19817 * We are on the hpts for some timer but not hptsi output.
19818 * Remove from the hpts unconditionally.
19819 */
19821 }
19822 /* Are we pacing and late? */
19825 /* We are delayed */
19829 }
19830 /* Do the timers, which may override the pacer */
19838 #ifdef TCP_ACCOUNTING
19840 #endif
19841 /*
19842 * If timers want tcp_drop(), then pass error out,
19843 * otherwise suppress it.
19844 */
19846 }
19847 }
19850 /* Timer is not running */
19852 }
19853 #ifdef TCP_ACCOUNTING
19855 #endif
19857 }
19860 /* A timeout occurred and no ack has arrived */
19862 /* Timer is not running */
19864 }
19865 #ifdef TCP_ACCOUNTING
19867 #endif
19869 }
19878 /*
19879 * On the hpts you can't pass even if ACKNOW is on, we will
19880 * when the hpts fires.
19881 */
19882 #ifdef TCP_ACCOUNTING
19886 }
19889 }
19891 #endif
19894 }
19895 /* Finish out both pacing early and late accounting */
19908 /* Non-paced we are not late */
19911 }
19912 /* Now that early/late accounting is done turn off the flag */
19919 }
19934 }
19935 }
19937 /*
19938 * For TFO connections in SYN_SENT or SYN_RECEIVED,
19939 * only allow the initial SYN or SYN|ACK and those sent
19940 * by the retransmit timer.
19941 */
19948 #ifdef TCP_ACCOUNTING
19950 #endif
19952 }
19953 /*
19954 * Determine length of data that should be transmitted, and flags
19955 * that will be used. If there is some data or critical controls
19956 * (SYN, RST) to send, then transmit; otherwise, investigate
19957 * further.
19958 */
19963 }
19969 }
19970 }
19976 /* Count as a probe rtt */
19984 }
19985 }
19995 /*
19996 * Make sure any TCP timers (keep-alive) is not running.
19997 */
19999 }
20002 /*
20003 * Sanity check before sending, if we have
20004 * no-pacing enabled and prr is turned off that
20005 * is a logistics error. Correct this by turnning
20006 * prr back on. A user *must* set some form of
20007 * pacing in order to turn PRR off. We do this
20008 * in the output path so that we can avoid socket
20009 * option ordering issues that would occur if we
20010 * tried to do it while setting rack_no_prr on.
20011 */
20013 }
20017 /*
20018 * We have been idle some micro seconds. We need
20019 * to factor this in to see if a PCM is needed.
20020 */
20025 else
20032 }
20033 }
20034 again:
20044 else
20048 /*
20049 * We set in our first send so we know that the ctf_fixed_maxseg
20050 * has been fully set. If we do it in rack_init() we most likely
20051 * see 512 bytes so we end up at 5120, not desirable.
20052 */
20055 /*
20056 * Assure our initial PCM probe is at least 10 MSS.
20057 */
20059 }
20060 }
20066 else
20070 else
20074 /* Raise up the max seg for this trip through */
20076 /* Disable any fast output */
20078 }
20082 }
20083 }
20091 /* Retry in a ms */
20096 }
20100 }
20109 }
20111 /* Retransmit timer */
20125 /*
20126 * If an RSM is returned then enough time has passed
20127 * for us to retransmit it. Move up the collapse point,
20128 * since this rsm has its chance to retransmit now.
20129 */
20132 /* Are we done? */
20137 /* We are not doing a TLP */
20144 /* We have a retransmit that takes precedence */
20148 /* Enter recovery if not induced by a time-out */
20150 }
20151 #ifdef INVARIANTS
20155 }
20156 #endif
20170 }
20172 /* Tail loss probe */
20176 /*
20177 * Check if we can do a TLP with a RACK'd packet
20178 * this can happen if we are not doing the rack
20179 * cheat and we skipped to a TLP and it
20180 * went off.
20181 */
20183 /* We are doing a TLP make sure the flag is preent */
20197 }
20202 /*
20203 * There are two different ways that we
20204 * can get into this block:
20205 * a) This is a non-sack connection, we had a time-out
20206 * and thus r_must_retran was set and everything
20207 * left outstanding as been marked for retransmit.
20208 * b) The MTU of the path shrank, so that everything
20209 * was marked to be retransmitted with the smaller
20210 * mtu and r_must_retran was set.
20211 *
20212 * This means that we expect the sendmap (outstanding)
20213 * to all be marked must. We can use the tmap to
20214 * look at them.
20215 *
20216 */
20222 /*
20223 * We can't send yet.
20224 */
20228 }
20229 /*
20230 * This is the case a/b mentioned above. All
20231 * outstanding/not-acked should be marked.
20232 * We can use the tmap to find them.
20233 */
20236 /* TSNH */
20242 }
20244 /*
20245 * The first one does not have the flag, did we collapse
20246 * further up in our list?
20247 */
20263 /* We use pacing min as shaping len req */
20269 }
20270 /*
20271 * Delay removing the flag RACK_MUST_RXT so
20272 * that the fastpath for retransmit will
20273 * work with this rsm.
20274 */
20275 }
20276 }
20277 /*
20278 * Enforce a connection sendmap count limit if set
20279 * as long as we are not retransmiting.
20280 */
20288 }
20292 }
20294 /* we are retransmitting the fin */
20295 len--;
20297 /*
20298 * When retransmitting data do *not* include the
20299 * FIN. This could happen from a TLP probe.
20300 */
20302 }
20303 }
20305 rack_use_rsm_rfo &&
20312 }
20318 }
20319 #ifdef NETFLIX_SHARED_CWND
20322 /* We are doing cwnd sharing */
20327 /* The pcbid is in, lets make an attempt */
20332 segsiz);
20333 }
20337 /* we are no longer out of data */
20340 }
20342 /* First lets update and get the cwnd */
20346 }
20347 }
20348 #endif
20349 /*
20350 * Get standard flags, and add SYN or FIN if requested by 'hidden'
20351 * state flags.
20352 */
20363 }
20368 /*
20369 * We are not retransmitting (sack_rxmit is 0) so we
20370 * are sending new data. This is always based on snd_max.
20371 * Now in theory snd_max may be equal to snd_una, if so
20372 * then nothing is outstanding and the offset would be 0.
20373 */
20379 else
20383 /* TLP is forcing out new data */
20386 }
20390 else
20394 }
20398 }
20404 /*
20405 * For prr=off, we need to send only 1 MSS
20406 * at a time. We do this because another sack could
20407 * be arriving that causes us to send retransmits and
20408 * we don't want to be on a long pace due to a larger send
20409 * that keeps us from sending out the retransmit.
20410 */
20414 /* We use pacing min as shaping len req */
20423 /*
20424 * We are inside of a Fast recovery episode, this
20425 * is caused by a SACK or 3 dup acks. At this point
20426 * we have sent all the retransmissions and we rely
20427 * on PRR to dictate what we will send in the form of
20428 * new data.
20429 */
20435 /* Check to see if we have the data */
20437 /* It does not all fit */
20440 else
20442 }
20445 }
20450 }
20454 }
20457 }
20458 }
20460 /*
20461 * We should never send more than a MSS when
20462 * retransmitting or sending new data in prr
20463 * mode unless the override flag is on. Most
20464 * likely the PRR algorithm is not going to
20465 * let us send a lot as well :-)
20466 */
20469 }
20471 /*
20472 * Do we send any? The idea here is if the
20473 * send empty's the socket buffer we want to
20474 * do it. However if not then lets just wait
20475 * for our prr_sndcnt to get bigger.
20476 */
20481 /* This send does not empty the sb */
20483 }
20484 }
20485 }
20487 /*
20488 * If you have not established
20489 * and are not doing FAST OPEN
20490 * no data please.
20491 */
20496 }
20497 }
20501 }
20503 /*
20504 * Lop off SYN bit if it has already been sent. However, if this is
20505 * SYN-SENT state and if segment contains data and if we don't know
20506 * that foreign host supports TAO, suppress sending segment.
20507 */
20512 /*
20513 * When sending additional segments following a TFO SYN|ACK,
20514 * do not include the SYN bit.
20515 */
20519 }
20520 /*
20521 * Be careful not to send data and/or FIN on SYN segments. This
20522 * measure is needed to prevent interoperability problems with not
20523 * fully conformant TCP implementations.
20524 */
20528 }
20529 /*
20530 * On TFO sockets, ensure no data is sent in the following cases:
20531 *
20532 * - When retransmitting SYN|ACK on a passively-created socket
20533 *
20534 * - When retransmitting SYN on an actively created socket
20535 *
20536 * - When sending a zero-length cookie (cookie request) on an
20537 * actively created socket
20538 *
20539 * - When the socket is in the CLOSED state (RST is being sent)
20540 */
20548 }
20549 /* Without fast-open there should never be data sent on a SYN */
20552 }
20554 /* We only send 1 MSS if we have a DSACK block */
20557 }
20559 /*
20560 * We have nothing to send, or the window shrank, or
20561 * is closed, do we need to go into persists?
20562 */
20569 }
20573 /*
20574 * We are not sending a maximum sized segment for
20575 * some reason. Should we not send anything (think
20576 * sws or persists)?
20577 */
20582 /*
20583 * Here the rwnd is less than
20584 * the minimum pacing size, this is not a retransmit,
20585 * we are established and
20586 * the send is not the last in the socket buffer
20587 * we send nothing, and we may enter persists
20588 * if nothing is outstanding.
20589 */
20592 /*
20593 * Nothing out we can
20594 * go into persists.
20595 */
20597 }
20602 /*
20603 * Here we are not retransmitting, and
20604 * the cwnd is not so small that we could
20605 * not send at least a min size (rxt timer
20606 * not having gone off), We have 2 segments or
20607 * more already in flight, its not the tail end
20608 * of the socket buffer and the cwnd is blocking
20609 * us from sending out a minimum pacing segment size.
20610 * Lets not send anything.
20611 */
20618 /*
20619 * Here we have a send window but we have
20620 * filled it up and we can't send another pacing segment.
20621 * We also have in flight more than 2 segments
20622 * and we are not completing the sb i.e. we allow
20623 * the last bytes of the sb to go out even if
20624 * its not a full pacing segment.
20625 */
20632 /*
20633 * Here we are doing hardware pacing, this is not a TLP,
20634 * we are not sending a pace max segment size, there is rwnd
20635 * room to send at least N pace_max_seg, the cwnd is greater
20636 * than or equal to a full pacing segments plus 4 mss and we have 2 or
20637 * more segments in flight and its not the tail of the socket buffer.
20638 *
20639 * We don't want to send instead we need to get more ack's in to
20640 * allow us to send a full pacing segment. Normally, if we are pacing
20641 * about the right speed, we should have finished our pacing
20642 * send as most of the acks have come back if we are at the
20643 * right rate. This is a bit fuzzy since return path delay
20644 * can delay the acks, which is why we want to make sure we
20645 * have cwnd space to have a bit more than a max pace segments in flight.
20646 *
20647 * If we have not gotten our acks back we are pacing at too high a
20648 * rate delaying will not hurt and will bring our GP estimate down by
20649 * injecting the delay. If we don't do this we will send
20650 * 2 MSS out in response to the acks being clocked in which
20651 * defeats the point of hw-pacing (i.e. to help us get
20652 * larger TSO's out).
20653 */
20655 }
20657 }
20658 /* len will be >= 0 after this point. */
20661 /*
20662 * Decide if we can use TCP Segmentation Offloading (if supported by
20663 * hardware).
20664 *
20665 * TSO may only be used if we are in a pure bulk sending state. The
20666 * presence of TCP-MD5, SACK retransmits, SACK advertizements and IP
20667 * options prevent using TSO. With TSO the TCP header is the same
20668 * (except for the sequence number) for all generated packets. This
20669 * makes it impossible to transmit any options which vary per
20670 * generated segment or packet.
20671 *
20672 * IPv4 handling has a clear separation of ip options and ip header
20673 * flags while IPv6 combines both in in6p_outputopts. ip6_optlen() does
20674 * the right thing below to provide length of just ip options and thus
20675 * checking for ipoptlen is enough to decide if ip options are present.
20676 */
20678 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
20679 /*
20680 * Pre-calculate here as we save another lookup into the darknesses
20681 * of IPsec that way and can actually decide if TSO is ok.
20682 */
20683 #ifdef INET6
20686 #ifdef INET
20687 else
20688 #endif
20690 #ifdef INET
20694 #endif
20696 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
20698 #endif
20705 {
20710 /*
20711 * If we sent a fin, snd_max is 1 higher than
20712 * snd_una
20713 */
20714 outstanding--;
20715 }
20719 }
20720 }
20724 /*
20725 * Sender silly window avoidance. We transmit under the following
20726 * conditions when len is non-zero:
20727 *
20728 * - We have a full segment (or more with TSO) - This is the last
20729 * buffer in a write()/send() and we are either idle or running
20730 * NODELAY - we've timed out (e.g. persist timer) - we have more
20731 * then 1/2 the maximum send window's worth of data (receiver may be
20732 * limited the window size) - we need to retransmit
20733 */
20737 }
20738 /*
20739 * NOTE! on localhost connections an 'ack' from the remote
20740 * end may occur synchronously with the output and cause us
20741 * to flush a buffer queued with moretocome. XXX
20742 *
20743 */
20750 }
20754 }
20758 }
20762 }
20765 /*
20766 * We have less than two MSS outstanding (delayed ack)
20767 * and our rwnd will not let us send a full sized
20768 * MSS. Lets go ahead and let this small segment
20769 * out because we want to try to have at least two
20770 * packets inflight to not be caught by delayed ack.
20771 */
20774 }
20775 }
20776 /*
20777 * Sending of standalone window updates.
20778 *
20779 * Window updates are important when we close our window due to a
20780 * full socket buffer and are opening it again after the application
20781 * reads data from it. Once the window has opened again and the
20782 * remote end starts to send again the ACK clock takes over and
20783 * provides the most current window information.
20784 *
20785 * We must avoid the silly window syndrome whereas every read from
20786 * the receive buffer, no matter how small, causes a window update
20787 * to be sent. We also should avoid sending a flurry of window
20788 * updates when the socket buffer had queued a lot of data and the
20789 * application is doing small reads.
20790 *
20791 * Prevent a flurry of pointless window updates by only sending an
20792 * update when we can increase the advertized window by more than
20793 * 1/4th of the socket buffer capacity. When the buffer is getting
20794 * full or is very small be more aggressive and send an update
20795 * whenever we can increase by two mss sized segments. In all other
20796 * situations the ACK's to new incoming data will carry further
20797 * window increases.
20798 *
20799 * Don't send an independent window update if a delayed ACK is
20800 * pending (it will get piggy-backed on it) or the remote side
20801 * already has done a half-close and won't send more data. Skip
20802 * this if the connection is in T/TCP half-open state.
20803 */
20807 /*
20808 * "adv" is the amount we could increase the window, taking
20809 * into account that we are limited by TCP_MAXWIN <<
20810 * tp->rcv_scale.
20811 */
20821 /* We can't increase the window */
20823 }
20827 /*
20828 * If the new window size ends up being the same as or less
20829 * than the old size when it is scaled, then don't force
20830 * a window update.
20831 */
20841 }
20845 }
20846 }
20847 dontupdate:
20849 /*
20850 * Send if we owe the peer an ACK, RST, SYN, or urgent data. ACKNOW
20851 * is also a catch-all for the retransmit timer timeout case.
20852 */
20856 }
20860 }
20861 /*
20862 * If our state indicates that FIN should be sent and we have not
20863 * yet done so, then we need to send.
20864 */
20869 }
20870 /*
20871 * No reason to send a segment, just return.
20872 */
20873 just_return:
20875 just_return_nolock:
20876 {
20884 /*
20885 * Ok less than or right at a MSS is
20886 * outstanding. The original FreeBSD stack would
20887 * have sent a FIN, which can speed things up for
20888 * a transactional application doing a MSG_WAITALL.
20889 * To speed things up since we do *not* send a FIN
20890 * if data is outstanding, we send a "challenge ack".
20891 * The idea behind that is instead of having to have
20892 * the peer wait for the delayed-ack timer to run off
20893 * we send an ack that makes the peer send us an ack.
20894 */
20896 }
20901 rack_use_rfo &&
20914 /* We can send at least one more MSS using our fsb */
20922 /* Assure when we leave that snd_nxt will point to top */
20931 /*
20932 * Mark the last sent that we just-returned (hinting
20933 * that delayed ack may play a role in any rtt measurement).
20934 */
20936 }
20945 /* We are limited by the rwnd */
20950 /* We are limited by whats available -- app limited */
20958 /*
20959 * No delay is not on and the
20960 * user is sending less than 1MSS. This
20961 * brings out SWS avoidance so we
20962 * don't send. Another app-limited case.
20963 */
20966 /*
20967 * The user has requested no push of
20968 * the last segment and we are
20969 * at the last segment. Another app
20970 * limited case.
20971 */
20974 /* Its the cwnd */
20981 /* Now why here are we not sending? */
20982 #ifdef NOW
20983 #ifdef INVARIANTS
20985 #endif
20986 #endif
20988 }
20989 /*
20990 * App limited in some fashion, for our pacing GP
20991 * measurements we don't want any gap (even cwnd).
20992 * Close down the measurement window.
20993 */
20997 /*
20998 * The reason we are not sending is
20999 * the cwnd (or prr). We have been configured
21000 * to end the measurement window in
21001 * this case.
21002 */
21006 /*
21007 * We are rwnd limited and have been
21008 * configured to end the measurement
21009 * window in this case.
21010 */
21013 /*
21014 * A true application limited period, we have
21015 * ran out of data.
21016 */
21019 /*
21020 * In the assessing case we hit the end of
21021 * the if/else and had no known reason
21022 * This will panic us under invariants..
21023 *
21024 * If we get this out in logs we need to
21025 * investagate which reason we missed.
21026 */
21028 }
21032 /* Adjust the Gput measurement */
21037 /*
21038 * There is not enough to measure.
21039 */
21047 }
21048 /* Mark the last packet has app limited */
21054 /*
21055 * Go out to the end app limited and mark
21056 * this new one as next and move the end_appl up
21057 * to this guy.
21058 */
21062 }
21065 }
21070 }
21071 }
21072 /* Check if we need to go into persists or not */
21078 /* Yes lets make sure to move to persist before timer-start */
21080 }
21082 rack_log_type_just_return(rack, cts, tot_len_this_send, slot, hpts_calling, app_limited, cwnd_to_use);
21083 }
21084 #ifdef NETFLIX_SHARED_CWND
21089 }
21090 #endif
21091 #ifdef TCP_ACCOUNTING
21096 }
21099 }
21102 }
21107 }
21110 }
21111 }
21113 #endif
21116 send:
21121 /*
21122 * We only want to do this once with the hw_check_queue,
21123 * for the enobuf case we would only do it once if
21124 * we come around to again, the flag will be clear.
21125 */
21135 }
21136 }
21139 else
21143 /*
21144 * We do not transmit a FIN
21145 * with data outstanding. We
21146 * need to make it so all data
21147 * is acked first.
21148 */
21153 /*
21154 * Ok less than or right at a MSS is
21155 * outstanding. The original FreeBSD stack would
21156 * have sent a FIN, which can speed things up for
21157 * a transactional application doing a MSG_WAITALL.
21158 * To speed things up since we do *not* send a FIN
21159 * if data is outstanding, we send a "challenge ack".
21160 * The idea behind that is instead of having to have
21161 * the peer wait for the delayed-ack timer to run off
21162 * we send an ack that makes the peer send us an ack.
21163 */
21165 }
21166 }
21167 /* Enforce stack imposed max seg size if we have one */
21172 }
21177 /* It is large enough for a measurement */
21182 }
21188 else
21190 }
21191 /*
21192 * Before ESTABLISHED, force sending of initial options unless TCP
21193 * set not to do any options. NOTE: we assume that the IP/TCP header
21194 * plus TCP options always fit in a single mbuf, leaving room for a
21195 * maximum link header, i.e. max_linkhdr + sizeof (struct tcpiphdr)
21196 * + optlen <= MCLBYTES
21197 */
21199 #ifdef INET6
21202 else
21203 #endif
21206 /*
21207 * Ok what seq are we sending from. If we have
21208 * no rsm to use, then we look at various bits,
21209 * if we are putting out a SYN it will be ISS.
21210 * If we are retransmitting a FIN it will
21211 * be snd_max-1 else its snd_max.
21212 */
21219 else
21223 }
21224 /*
21225 * Compute options for segment. We only have to care about SYN and
21226 * established connection segments. Options for SYN-ACK segments
21227 * are handled in TCP syncache.
21228 */
21231 /* Maximum segment size. */
21238 /*
21239 * On SYN or SYN|ACK transmits on TFO connections,
21240 * only include the TFO option if it is not a
21241 * retransmit, as the presence of the TFO option may
21242 * have caused the original SYN or SYN|ACK to have
21243 * been dropped by a middlebox.
21244 */
21260 /*
21261 * If we wind up having more data to
21262 * send with the SYN than can fit in
21263 * one segment, don't send any more
21264 * until the SYN|ACK comes back from
21265 * the other end.
21266 */
21268 }
21269 }
21270 }
21271 /* Window scaling. */
21275 }
21276 /* Timestamps. */
21283 /*
21284 * When we are doing a rcv_rtt probe all
21285 * other timestamps use the next msec. This
21286 * is safe since our previous ack is in the
21287 * air and we will just have a few more
21288 * on the next ms. This assures that only
21289 * the one ack has the ms_cts that was on
21290 * our ack-probe.
21291 */
21295 }
21311 /* Subtract 1 from seq to force a response */
21312 rack_seq--;
21313 }
21314 }
21315 /* Set receive buffer autosizing timestamp. */
21319 }
21320 /* Selective ACK's. */
21329 }
21330 }
21331 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
21332 /* TCP-MD5 (RFC2385). */
21335 #endif
21337 /* Processing the options. */
21339 /*
21340 * If we wanted a TFO option to be added, but it was unable
21341 * to fit, ensure no data is sent.
21342 */
21346 }
21349 /* The port was removed?? */
21351 #ifdef TCP_ACCOUNTING
21355 }
21358 }
21360 #endif
21362 }
21364 }
21365 #ifdef INET6
21368 else
21369 #endif
21373 else
21375 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
21377 #endif
21379 /*
21380 * Adjust data length if insertion of options will bump the packet
21381 * length beyond the t_maxseg length. Clear the FIN bit because we
21382 * cut off the tail of the segment.
21383 */
21390 /* extract TSO information */
21397 /*
21398 * Check if we should limit by maximum payload
21399 * length:
21400 */
21402 /* compute maximum TSO length */
21404 max_linkhdr);
21411 }
21412 }
21413 /*
21414 * Prevent the last segment from being fractional
21415 * unless the send sockbuf can be emptied:
21416 */
21423 }
21424 }
21425 /*
21426 * In case there are too many small fragments don't
21427 * use TSO:
21428 */
21432 }
21433 /*
21434 * Send the FIN in a separate segment after the bulk
21435 * sending is done. We don't trust the TSO
21436 * implementations to clear the FIN flag on all but
21437 * the last segment.
21438 */
21441 }
21445 /*
21446 * Since we don't have enough space to put
21447 * the IP header chain and the TCP header in
21448 * one packet as required by RFC 7112, don't
21449 * send it. Also ensure that at least one
21450 * byte of the payload can be put into the
21451 * TCP segment.
21452 */
21457 }
21460 }
21464 }
21467 #ifdef DIAGNOSTIC
21468 #ifdef INET6
21470 #else
21472 #endif
21474 #endif
21476 /*
21477 * This KASSERT is here to catch edge cases at a well defined place.
21478 * Before, those had triggered (random) panic conditions further
21479 * down.
21480 */
21485 /*
21486 * We have outstanding data, don't send a fin by itself!.
21487 *
21488 * Check to see if we need to send a challenge ack.
21489 */
21492 /*
21493 * Ok less than or right at a MSS is
21494 * outstanding. The original FreeBSD stack would
21495 * have sent a FIN, which can speed things up for
21496 * a transactional application doing a MSG_WAITALL.
21497 * To speed things up since we do *not* send a FIN
21498 * if data is outstanding, we send a "challenge ack".
21499 * The idea behind that is instead of having to have
21500 * the peer wait for the delayed-ack timer to run off
21501 * we send an ack that makes the peer send us an ack.
21502 */
21504 }
21506 }
21507 /*
21508 * Grab a header mbuf, attaching a copy of data to be transmitted,
21509 * and initialize the header from the template for sends on this
21510 * connection.
21511 */
21519 else
21521 /*
21522 * We allow a limit on sending with hptsi.
21523 */
21527 }
21528 #ifdef INET6
21531 else
21532 #endif
21540 }
21544 /*
21545 * Start the m_copy functions from the closest mbuf to the
21546 * sb_offset in the socket buffer chain.
21547 */
21554 /*
21555 * If we are not retransmitting advance the
21556 * sndptr to help remember the next place in
21557 * the sb.
21558 */
21565 /*
21566 * If we are not retransmitting pass in msb so
21567 * the socket buffer can be advanced. Otherwise
21568 * set it to NULL if its a retransmission since
21569 * we don't want to change the sb remembered
21570 * location.
21571 */
21574 else
21580 #ifdef NETFLIX_COPY_ARGS
21582 #endif
21583 );
21585 /*
21586 * Must have ran out of mbufs for the copy
21587 * shorten it to no longer need tso. Lets
21588 * not put on sendalot since we are low on
21589 * mbufs.
21590 */
21592 }
21599 }
21600 }
21603 /*
21604 * TLP should not count in retran count, but
21605 * in its own bin
21606 */
21613 }
21614 #ifdef STATS
21616 len);
21617 #endif
21621 #ifdef STATS
21623 len);
21624 #endif
21625 }
21626 /*
21627 * If we're sending everything we've got, set PUSH. (This
21628 * will keep happy those implementations which only give
21629 * data to the user when a buffer fills or a PUSH comes in.)
21630 */
21636 }
21645 else
21653 }
21654 #ifdef INET6
21659 #endif
21662 }
21665 #ifdef MAC
21667 #endif
21669 #ifdef INET6
21672 else
21674 #ifdef INET
21676 #endif
21680 #ifdef INET6
21683 else
21687 }
21689 #ifdef INET6
21704 {
21705 #ifdef INET
21717 #endif
21718 }
21719 }
21720 /*
21721 * If we are starting a connection, send ECN setup SYN packet. If we
21722 * are on a retransmit, we may resend those bits a number of times
21723 * as per RFC 3168.
21724 */
21727 }
21728 /* Also handle parallel SYN for ECN */
21735 #ifdef INET6
21739 }
21740 else
21741 #endif
21742 {
21743 #ifdef INET
21746 #endif
21747 }
21748 }
21752 /*
21753 * Calculate receive window. Don't shrink window, but avoid silly
21754 * window syndrome.
21755 * If a RST segment is sent, advertise a window of zero.
21756 */
21763 }
21767 }
21769 /*
21770 * According to RFC1323 the window field in a SYN (i.e., a <SYN> or
21771 * <SYN,ACK>) segment itself is never scaled. The <SYN,ACK> case is
21772 * handled in syncache.
21773 */
21778 /* Avoid shrinking window with window scaling. */
21781 }
21782 /*
21783 * Adjust the RXWIN0SENT flag - indicate that we have advertised a 0
21784 * window. This may cause the remote transmitter to stall. This
21785 * flag tells soreceive() to disable delayed acknowledgements when
21786 * draining the buffer. This can occur if the receiver is
21787 * attempting to read more data than can be buffered prior to
21788 * transmitting on the connection.
21789 */
21796 /* Now are we using fsb?, if so copy the template data to the mbuf */
21802 /*
21803 * We have just copied in:
21804 * IP/IP6
21805 * <optional udphdr>
21806 * tcphdr (no options)
21807 *
21808 * We need to grab the correct pointers into the mbuf
21809 * for both the tcp header, and possibly the udp header (if tunneling).
21810 * We do this by using the offset in the copy buffer and adding it
21811 * to the mbuf base pointer (cpto).
21812 */
21813 #ifdef INET6
21816 else
21818 #ifdef INET
21820 #endif
21822 /* If we have a udp header lets set it into the mbuf as well */
21824 udp = (struct udphdr *)(cpto + ((uint8_t *)rack->r_ctl.fsb.udp - rack->r_ctl.fsb.tcp_ip_hdr));
21825 }
21829 }
21830 /*
21831 * Put TCP length in extended header, and then checksum extended
21832 * header and data.
21833 */
21835 #if defined(IPSEC_SUPPORT) || defined(TCP_SIGNATURE)
21837 /*
21838 * Calculate MD5 signature and put it into the place
21839 * determined before.
21840 * NOTE: since TCP options buffer doesn't point into
21841 * mbuf's data, calculate offset and use it.
21842 */
21845 /*
21846 * Do not send segment if the calculation of MD5
21847 * digest has failed.
21848 */
21850 }
21851 }
21852 #endif
21853 #ifdef INET6
21855 /*
21856 * ip6_plen is not need to be filled now, and will be filled
21857 * in ip6_output.
21858 */
21871 }
21872 }
21873 #endif
21874 #if defined(INET6) && defined(INET)
21875 else
21876 #endif
21877 #ifdef INET
21878 {
21892 }
21893 /* IP version must be set here for ipv4/ipv6 checking later */
21896 }
21897 #endif
21898 /*
21899 * Enable TSO and specify the size of the segments. The TCP pseudo
21900 * header checksum is always provided. XXX: Fixme: This is currently
21901 * not the case for IPv6.
21902 */
21904 /*
21905 * Here we must use t_maxseg and the optlen since
21906 * the optlen may include SACK's (or DSACK).
21907 */
21912 }
21917 #ifdef TCP_HHOOK
21918 /* Run HHOOK_TCP_ESTABLISHED_OUT helper hooks. */
21920 #endif
21925 }
21926 /* We're getting ready to send; log now. */
21934 else
21939 /* Save off the early/late values */
21950 }
21953 else
21958 }
21967 /*
21968 * When we have a retransmit we want to log the
21969 * burst at send and flight at send from before.
21970 */
21974 /*
21975 * New transmits we log in flex5 the inflight again as
21976 * well as the number of segments in our send in the
21977 * substate field.
21978 */
21981 }
21996 }
22002 /*
22003 * Fill in IP length and desired time to live and send to IP level.
22004 * There should be a better way to handle ttl and tos; we could keep
22005 * them in the template, but need a way to checksum without them.
22006 */
22007 /*
22008 * m->m_pkthdr.len should have been set before cksum calcuration,
22009 * because in6_cksum() need it.
22010 */
22011 #ifdef INET6
22013 /*
22014 * we separately set hoplimit for every segment, since the
22015 * user might want to change the value via setsockopt. Also,
22016 * desired default hop limit might be changed via Neighbor
22017 * Discovery.
22018 */
22021 /*
22022 * Set the packet size here for the benefit of DTrace
22023 * probes. ip6_output() will set it properly; it's supposed
22024 * to include the option header lengths as well.
22025 */
22030 else
22037 /* TODO: IPv6 IP6TOS_ECT bit on */
22046 }
22048 #if defined(INET) && defined(INET6)
22049 else
22050 #endif
22051 #ifdef INET
22052 {
22054 #ifdef INET6
22059 /*
22060 * If we do path MTU discovery, then we set DF on every
22061 * packet. This might not be the best thing to do according
22062 * to RFC3390 Section 2. However the tcp hostcache migitates
22063 * the problem so it affects only the first tcp connection
22064 * with a host.
22065 *
22066 * NB: Don't set DF on small MTU/MSS to have a safe
22067 * fallback.
22068 */
22073 }
22076 }
22084 #if defined(IPSEC) || defined(IPSEC_SUPPORT)
22086 #else
22087 NULL,
22088 #endif
22091 inp);
22094 }
22099 }
22101 out:
22102 /*
22103 * In transmit state, time the transmission and arrange for the
22104 * retransmit. In persist state, just set snd_max.
22105 */
22111 /* We just launched a PCM */
22112 /* rrs here log */
22116 }
22123 /*
22124 * Need to record what we have since we are
22125 * approaching seq wrap.
22126 */
22135 }
22136 }
22137 }
22146 }
22153 }
22155 /* Set we retransmitted */
22159 /* Set we sent in CA */
22162 /* Set we sent in SS */
22164 }
22165 }
22179 else
22181 }
22182 }
22184 sub_from_prr &&
22188 else
22190 }
22193 /* Make sure the TLP is added */
22196 /* If its a resend without TLP then it must not have the flag */
22198 }
22206 {
22207 /*
22208 * This block is not associated with the above error == 0 test.
22209 * It is used to advance snd_max if we have a new transmit.
22210 */
22217 /* We don't log or do anything with errors */
22221 /*
22222 * Not a retransmission of some
22223 * sort, new data is going out so
22224 * clear our TLP count and flag.
22225 */
22228 }
22230 /*
22231 * We have just sent a TLP, mark that it is true
22232 * and make sure our in progress is set so we
22233 * continue to check the count.
22234 */
22237 }
22238 /*
22239 * If we are retransmitting we are done, snd_max
22240 * does not get updated.
22241 */
22245 /*
22246 * Update the time we just added data since
22247 * nothing was outstanding.
22248 */
22251 }
22252 /*
22253 * Now for special SYN/FIN handling.
22254 */
22260 }
22265 }
22266 }
22270 }
22271 /*
22272 * Time this transmission if not a retransmission and
22273 * not currently timing anything.
22274 * This is only relevant in case of switching back to
22275 * the base stack.
22276 */
22281 }
22285 /*
22286 * If we are doing FO we need to update the mbuf position and subtract
22287 * this happens when the peer sends us duplicate information and
22288 * we thus want to send a DSACK.
22289 *
22290 * XXXRRS: This brings to mind a ?, when we send a DSACK block is TSO
22291 * turned off? If not then we are going to echo multiple DSACK blocks
22292 * out (with the TSO), which we should not be doing.
22293 */
22297 else
22305 }
22306 }
22312 /*
22313 * We are doing a PCM measurement and we did
22314 * not get enough data in the TSO to meet the
22315 * burst requirement.
22316 */
22324 /* Re-lock for the next spin */
22327 }
22332 /*
22333 * We are doing a PCM measurement and we did
22334 * not get enough data in the TSO to meet the
22335 * burst requirement.
22336 */
22343 /* Re-lock for the next spin */
22346 }
22347 }
22348 }
22349 nomore:
22356 /*
22357 * Failures do not advance the seq counter above. For the
22358 * case of ENOBUFS we will fall out and retry in 1ms with
22359 * the hpts. Everything else will just have to retransmit
22360 * with the timer.
22361 *
22362 * In any case, we do not want to loop around for another
22363 * send without a good reason.
22364 */
22370 #ifdef TCP_ACCOUNTING
22374 }
22377 }
22379 #endif
22382 /*
22383 * Pace us right away to retry in a some
22384 * time
22385 */
22400 }
22404 /*
22405 * For some reason the interface we used initially
22406 * to send segments changed to another or lowered
22407 * its MTU. If TSO was active we either got an
22408 * interface without TSO capabilits or TSO was
22409 * turned off. If we obtained mtu from ip_output()
22410 * then update it and try again.
22411 */
22421 }
22422 }
22425 #ifdef TCP_ACCOUNTING
22429 }
22432 }
22434 #endif
22438 /* FALLTHROUGH */
22445 }
22446 /* FALLTHROUGH */
22450 #ifdef TCP_ACCOUNTING
22454 }
22457 }
22459 #endif
22461 }
22466 }
22469 /*
22470 * Data sent (as far as we can tell). If this advertises a larger
22471 * window than any other segment, then remember the size of the
22472 * advertised window. Any pending ACK has now been sent.
22473 */
22479 enobufs:
22481 /* Do we need to turn off sendalot? */
22484 /* We hit our max. */
22486 }
22487 }
22491 /*
22492 * We don't send again after sending a RST.
22493 */
22499 /*
22500 * Get our pacing rate, if an error
22501 * occurred in sending (ENOBUF) we would
22502 * hit the else if with slot preset. Other
22503 * errors return.
22504 */
22506 }
22507 /* We have sent clear the flag */
22513 /*
22514 * We have retransmitted all.
22515 */
22518 }
22520 /*
22521 * Sending new data will also kill
22522 * the loop.
22523 */
22526 }
22527 }
22531 /*
22532 * We hit an RTO and now have past snd_max at the RTO
22533 * clear all the WAS flags.
22534 */
22536 }
22538 /* set the rack tcb into the slot N */
22540 rack_use_rfo &&
22554 /* We can send at least one more MSS using our fsb */
22567 rack_use_rfo &&
22581 /* we can use fast_output for more */
22592 }
22593 }
22595 }
22596 skip_all_send:
22597 /* Assure when we leave that snd_nxt will point to top */
22601 #ifdef TCP_ACCOUNTING
22606 }
22609 }
22612 }
22616 }
22619 }
22620 }
22622 #endif
22626 }
22628 static void
22630 {
22637 }
22639 static void
22641 {
22642 /*
22643 * The MSS may have changed
22644 */
22650 /*
22651 * The MTU has changed we need to resend everything
22652 * since all we have sent is lost. We first fix
22653 * up the mtu though.
22654 */
22656 /* We treat this like a full retransmit timeout without the cwnd adjustment */
22663 /* Mark all inflight to needing to be rxt'd */
22666 }
22667 }
22669 /* We don't use snd_nxt to retransmit */
22671 }
22673 static int
22675 {
22680 /*
22681 * We are already pacing another
22682 * way.
22683 */
22685 }
22688 }
22701 /* rxt settings */
22704 /* cmpack=1 */
22709 /* scwnd=1 */
22711 /* dynamic=100 */
22713 /* gp_inc_ca */
22715 /* rrr_conf=3 */
22717 /* npush=2 */
22719 /* fillcw=1 */
22723 /* noprr=1 */
22725 /* lscwnd=1 */
22727 /* gp_inc_rec */
22730 }
22732 static int
22734 {
22737 /*
22738 * Profile 1 is "standard" DGP. It ignores
22739 * client buffer level.
22740 */
22748 /*
22749 * Profile 6 tweaks DGP so that it will apply to
22750 * fill-cw the same settings that profile5 does
22751 * to replace DGP. It gets then the max(dgp-rate, fillcw(discounted).
22752 */
22755 /* This changes things back to the default settings */
22759 /* Make sure any stray flags are off */
22763 }
22767 else
22778 /* Basically this means all rack timers are at least (srtt + 1/4 srtt) */
22780 }
22782 /* Basically this means rack timers are extended based on dsack by up to (2 * srtt) */
22784 }
22787 else
22791 else
22795 else
22805 }
22808 else
22811 /* When dynamic adjustment is on CA needs to start at 100% */
22818 }
22826 else
22830 else
22834 else
22838 }
22840 }
22842 static int
22844 {
22850 /*
22851 * No space yikes -- fail out..
22852 */
22854 }
22859 }
22861 static int
22863 {
22864 #ifdef TCP_REQUEST_TRK
22867 tcp_seq seq;
22872 /* Make sure no fixed rate is on */
22877 /* Now allocate or find our entry that will have these settings */
22881 /* no space, where would it have gone? */
22885 }
22886 /* mask our internal flags */
22888 /* The seq will be snd_una + everything in the buffer */
22891 /* Disabling hybrid pacing */
22895 }
22898 }
22900 /*
22901 * If we have not yet turned DGP on, do so
22902 * now setting pure DGP mode, no buffer level
22903 * response.
22904 */
22906 /* Failed to turn pacing on */
22910 }
22911 }
22912 /*
22913 * Now we must switch to hybrid mode as well which also
22914 * means moving to regular pacing.
22915 */
22917 /* First time */
22923 }
22925 /*
22926 * This should be true.
22927 */
22930 }
22931 }
22932 /* Now set in our flags */
22936 else
22940 else
22945 #else
22947 #endif
22948 }
22950 static int
22952 {
22953 /* We pulled a SSI info log out what was there */
22957 }
22959 static int
22963 {
22976 /*
22977 * You must send in 0, 1 or 2 all else is
22978 * invalid.
22979 */
22981 }
22989 }
22994 }
23004 else
23006 }
23014 /*
23015 * User wants to set a custom beta.
23016 */
23021 }
23028 }
23037 /*
23038 * If we have an SRTT lets update t_rxtcur
23039 * to have the new slop.
23040 */
23044 }
23049 /* This only works for newreno. */
23052 }
23054 /*
23055 * Set them into the real CC module
23056 * whats in the rack pcb is the old values
23057 * to be used on restoral/
23058 */
23064 else
23067 /*
23068 * Not pacing yet so set it into our local
23069 * rack pcb storage.
23070 */
23073 }
23079 /* Too late */
23082 }
23098 else
23105 else
23112 else
23123 /*
23124 * If we are doing DGP we need to switch
23125 * to using the pacing limit.
23126 */
23130 }
23131 /*
23132 * Now change up the flags and counts to be correct.
23133 */
23137 }
23144 }
23148 }
23154 else
23164 /* You can't turn it off once its on! */
23170 }
23178 else
23190 }
23192 rack_gp_rtt_maxmul &&
23199 }
23207 else
23214 else
23218 /* Now do we use the LRO mbuf-queue feature */
23222 else
23226 else
23233 else
23244 else
23250 else
23260 /*
23261 * If the user sets something 100 or more
23262 * its the gp_ca value.
23263 */
23265 }
23266 }
23275 }
23280 /* RACK TLP cwnd reduction (bool) */
23284 /* Pacing related ones */
23286 /*
23287 * zero is old rack method, 1 is new
23288 * method using a pacing rate.
23289 */
23294 }
23304 }
23308 }
23312 }
23313 }
23316 else
23318 /* A rate may be set irate or other, if so set seg size */
23324 /* Change from kbits per second to bytes per second */
23338 else
23347 /* Max segments size in a pace in bytes */
23351 /*
23352 * If we set a max-seg and are doing DGP then
23353 * we now fall under the pacing limits not the
23354 * DGP ones.
23355 */
23359 }
23360 /*
23361 * Now change up the flags and counts to be correct.
23362 */
23366 }
23369 else
23374 /* Set the fixed pacing rate in Bytes per second ca */
23379 }
23381 /*
23382 * We are already pacing another
23383 * way.
23384 */
23387 }
23404 /* Set the fixed pacing rate in Bytes per second ca */
23409 }
23411 /*
23412 * We are already pacing another
23413 * way.
23414 */
23417 }
23434 /* Set the fixed pacing rate in Bytes per second ca */
23439 }
23441 /*
23442 * We are already pacing another
23443 * way.
23444 */
23447 }
23475 /*
23476 * We don't allow any reduction
23477 * over the GP b/w.
23478 */
23481 }
23493 /*
23494 * We don't allow any reduction
23495 * over the GP b/w.
23496 */
23499 }
23511 else
23517 else
23525 else
23529 }
23532 {
23539 }
23547 }
23552 }
23559 }
23561 /* We do not do ss early exit at all */
23564 }
23580 #ifdef RATELIMIT
23586 }
23587 #endif
23588 }
23590 /* End Pacing related ones */
23592 /* Allow PRR to send more than one seg */
23597 /* Minimum time between rack t-o's in ms */
23602 /* If early recovery max segments */
23607 {
23616 }
23617 }
23620 /* RACK reorder threshold (shift amount) */
23624 else
23628 /* Does reordering fade after ms time */
23633 /* RACK TLP theshold i.e. srtt+(srtt/N) */
23637 else
23644 else
23648 /* RACK added ms i.e. rack-rtt + reord + N */
23656 else
23664 }
23673 else
23680 /*
23681 * Must be between 2 - 80% to be a reduction else
23682 * we keep the default (10%).
23683 */
23686 }
23694 else
23706 }
23718 }
23724 else
23729 }
23732 }
23734 static void
23736 {
23737 /*
23738 * A new connection has been created (tp) and
23739 * the parent is the inpcb given. We want to
23740 * apply a read-lock to the parent (we are already
23741 * holding a write lock on the tp) and copy anything
23742 * out of the rack specific data as long as its tfb is
23743 * the same as ours i.e. we are the same stack. Otherwise
23744 * we just return.
23745 */
23752 /* Not the same stack */
23755 }
23756 /* Ok if we reach here lets setup the two rack pointers */
23760 /* Huh? */
23763 }
23764 /* Now copy out anything we wish to inherit i.e. things in socket-options */
23765 /* TCP_RACK_PROFILE we can't know but we can set DGP if its on */
23767 /* Profile 1 had to be set via sock opt */
23769 cnt++;
23770 }
23771 /* TCP_RACK_SET_RXT_OPTIONS */
23774 cnt++;
23775 }
23778 cnt++;
23779 }
23780 /* TCP_RACK_DSACK_OPT */
23783 cnt++;
23784 }
23787 cnt++;
23788 }
23789 /* TCP_RACK_PACING_DIVISOR */
23792 cnt++;
23793 }
23794 /* TCP_RACK_HI_BETA */
23796 cnt++;
23802 }
23803 }
23804 /* TCP_RACK_TIMER_SLOP */
23807 cnt++;
23808 }
23809 /* TCP_RACK_PACING_BETA_ECN */
23812 cnt++;
23813 }
23816 cnt++;
23817 }
23818 /* We do not do TCP_DEFER_OPTIONS */
23819 /* TCP_RACK_MEASURE_CNT */
23822 cnt++;
23823 }
23824 /* TCP_HDWR_UP_ONLY */
23827 cnt++;
23828 }
23829 /* TCP_FILLCW_RATE_CAP */
23832 cnt++;
23833 }
23834 /* TCP_PACING_RATE_CAP */
23837 cnt++;
23838 }
23839 /* A listener can't set TCP_HYBRID_PACING */
23840 /* TCP_SIDECHAN_DIS */
23843 cnt++;
23844 }
23845 /* TCP_SHARED_CWND_TIME_LIMIT */
23848 cnt++;
23849 }
23850 /* TCP_RACK_PACE_TO_FILL */
23853 cnt++;
23854 }
23857 cnt++;
23858 }
23861 cnt++;
23862 }
23863 /* TCP_RACK_NO_PUSH_AT_MAX */
23866 cnt++;
23867 }
23868 /* TCP_SHARED_CWND_ENABLE */
23871 cnt++;
23872 }
23873 /* TCP_USE_CMP_ACKS */
23876 cnt++;
23877 }
23881 cnt++;
23882 }
23883 /* TCP_RACK_MBUF_QUEUE */
23886 cnt++;
23887 }
23892 }
23895 }
23896 /* TCP_RACK_NONRXT_CFG_RATE */
23899 cnt++;
23900 }
23901 /* TCP_NO_PRR */
23904 cnt++;
23905 }
23908 cnt++;
23909 }
23910 /* RACK_CSPR_IS_FCC */
23913 cnt++;
23914 }
23915 /* TCP_TIMELY_DYN_ADJ */
23918 cnt++;
23919 }
23922 cnt++;
23923 }
23924 /* TCP_RACK_TLP_USE */
23927 cnt++;
23928 }
23929 /* we don't allow inheritence of TCP_RACK_PACE_ALWAYS */
23930 /* TCP_BBR_RACK_INIT_RATE */
23933 cnt++;
23934 }
23935 /* TCP_RACK_FORCE_MSEG */
23938 cnt++;
23939 }
23940 /* TCP_RACK_PACE_MIN_SEG */
23943 cnt++;
23944 }
23945 /* we don't allow TCP_RACK_PACE_MAX_SEG */
23946 /* TCP_RACK_PACE_RATE_REC, TCP_RACK_PACE_RATE_SS, TCP_RACK_PACE_RATE_CA */
23949 cnt++;
23950 }
23953 cnt++;
23954 }
23957 cnt++;
23958 }
23959 /* TCP_RACK_GP_INCREASE_REC, TCP_RACK_GP_INCREASE_CA, TCP_RACK_GP_INCREASE_SS */
23962 cnt++;
23963 }
23966 cnt++;
23967 }
23971 cnt++;
23972 }
23973 /* TCP_RACK_RR_CONF */
23976 cnt++;
23977 }
23978 /* TCP_PACING_DND */
23981 cnt++;
23982 }
23983 /* TCP_HDWR_RATE_CAP */
23986 cnt++;
23987 }
23988 /* TCP_DGP_UPPER_BOUNDS */
23991 cnt++;
23992 }
23995 cnt++;
23996 }
23997 /* TCP_SS_EEXIT */
24000 cnt++;
24001 }
24004 cnt++;
24005 }
24008 cnt++;
24009 }
24012 cnt++;
24013 }
24014 /* TCP_BBR_HDWR_PACE */
24017 cnt++;
24018 }
24021 cnt++;
24022 }
24023 /* TCP_RACK_PRR_SENDALOT */
24026 cnt++;
24027 }
24028 /* TCP_RACK_MIN_TO */
24031 cnt++;
24032 }
24033 /* TCP_RACK_EARLY_SEG */
24036 cnt++;
24037 }
24038 /* TCP_RACK_ENABLE_HYSTART */
24040 cnt++;
24049 }
24050 }
24051 /* TCP_RACK_REORD_THRESH */
24054 cnt++;
24055 }
24056 /* TCP_RACK_REORD_FADE */
24059 cnt++;
24060 }
24061 /* TCP_RACK_TLP_THRESH */
24064 cnt++;
24065 }
24066 /* TCP_BBR_USE_RACK_RR */
24069 cnt++;
24070 }
24071 /* TCP_RACK_PKT_DELAY */
24074 cnt++;
24075 }
24076 /* TCP_DELACK will get copied via the main code if applicable */
24077 /* TCP_BBR_RACK_RTT_USE */
24080 cnt++;
24081 }
24082 /* TCP_HONOR_HPTS_MIN */
24085 cnt++;
24086 }
24089 cnt++;
24090 }
24091 /* TCP_REC_IS_DYN */
24094 cnt++;
24095 }
24098 cnt++;
24099 }
24100 /* TCP_DATA_AFTER_CLOSE */
24103 cnt++;
24104 }
24105 /* TCP_GP_USE_LTBW */
24108 cnt++;
24109 }
24112 cnt++;
24113 }
24115 }
24118 static void
24120 {
24126 /* Disadvantage of deferal is you loose the error return */
24130 }
24131 }
24133 static void
24135 {
24136 /* Update HW tls state */
24142 else
24144 }
24146 static int
24148 {
24152 }
24154 static bool
24156 {
24165 /*
24166 * Pacing timer is up, check if we are ready.
24167 */
24171 /*
24172 * A timer is up, check if we are ready.
24173 */
24176 }
24177 }
24179 }
24204 };
24206 /*
24207 * rack_ctloutput() must drop the inpcb lock before performing copyin on
24208 * socket option arguments. When it re-acquires the lock after the copy, it
24209 * has to revalidate that the connection is still valid for the socket
24210 * option.
24211 */
24212 static int
24214 {
24216 #ifdef INET
24218 #endif
24228 }
24229 #ifdef INET
24231 #endif
24234 #ifdef INET6
24241 }
24244 #endif
24245 #ifdef INET
24249 /*
24250 * The DSCP codepoint has changed, update the fsb.
24251 */
24255 /*
24256 * The TTL has changed, update the fsb.
24257 */
24260 }
24263 #endif
24264 #ifdef SO_PEERPRIO
24268 /* Already read in and sanity checked in sosetopt(). */
24271 }
24273 }
24276 #endif
24280 /* Pacing related ones */
24300 /* End pacing related */
24346 /* Filter off all unknown options to the base stack */
24349 }
24353 }
24354 process_opt:
24359 /*
24360 * We truncate it down to 32 bits for the socket-option trace this
24361 * means rates > 34Gbps won't show right, but thats probably ok.
24362 */
24368 /* Save it in 64 bit form too */
24370 }
24377 }
24384 /* Options are being deferred */
24389 /* No memory to defer, fail */
24392 }
24393 }
24397 }
24399 static void
24401 {
24415 }
24420 /* still kept in ticks is t_rcvtime */
24422 /* Since we hold everything in precise useconds this is easy */
24428 /*
24429 * FreeBSD-specific extension fields for tcp_info.
24430 */
24444 #ifdef NETFLIX_STATS
24446 #endif
24447 #ifdef TCP_OFFLOAD
24451 }
24452 #endif
24453 }
24455 static int
24457 {
24463 /*
24464 * Because all our options are either boolean or an int, we can just
24465 * pull everything into optval and then unlock and copy. If we ever
24466 * add a option that is not a int, then this will have quite an
24467 * impact to this routine.
24468 */
24474 }
24477 /* First get the info filled */
24479 /* Fix up the rtt related fields if needed */
24483 /*
24484 * Beta is the congestion control value for NewReno that influences how
24485 * much of a backoff happens when loss is detected. It is normally set
24486 * to 50 for 50% i.e. the cwnd is reduced to 50% of its previous value
24487 * when you exit recovery.
24488 */
24495 /*
24496 * Reach out into the CC data and report back what
24497 * I have previously set. Yeah it looks hackish but
24498 * we don't want to report the saved values.
24499 */
24502 else
24504 }
24506 /*
24507 * Beta_ecn is the congestion control value for NewReno that influences how
24508 * much of a backoff happens when a ECN mark is detected. It is normally set
24509 * to 80 for 80% i.e. the cwnd is reduced by 20% of its previous value when
24510 * you exit recovery. Note that classic ECN has a beta of 50, it is only
24511 * ABE Ecn that uses this "less" value, but we do too with pacing :)
24512 */
24519 /*
24520 * Reach out into the CC data and report back what
24521 * I have previously set. Yeah it looks hackish but
24522 * we don't want to report the saved values.
24523 */
24526 else
24528 }
24534 }
24537 }
24540 {
24549 }
24550 }
24580 /* You cannot retrieve a profile, its write only */
24587 /* You cannot retrieve hybrid pacing information, its write only */
24610 else
24615 /* It is not used */
24618 /* we use min() */
24621 /* we use max() */
24623 }
24629 /* Now do we use the LRO mbuf-queue feature */
24642 /* RACK TLP cwnd reduction (bool) */
24647 /* convert to kbits per sec */
24659 /* Max segments in a pace */
24663 /* Use the always pace method */
24667 /* Allow PRR to send more than one seg */
24671 /* Minimum time between rack t-o's in ms */
24678 /* If early recovery max segments */
24682 /* RACK reorder threshold (shift amount) */
24698 /* Does reordering fade after ms time */
24702 /* Do we use the rack cheat for rxt */
24715 /* RACK TLP theshold i.e. srtt+(srtt/N) */
24719 /* RACK added ms i.e. rack-rtt + reord + N */
24766 else
24781 }
24787 else
24789 }
24791 }
24793 static int
24795 {
24802 }
24803 }
24807 #ifdef STACKALIAS
24809 #endif
24810 };
24812 static int
24814 {
24817 }
24819 static void
24821 {
24823 }
24827 static int
24829 {
24846 OID_AUTO,
24847 #ifdef STACKALIAS
24849 #else
24851 #endif
24858 }
24868 free_uma:
24874 }
24891 }
24897 }
24899 }
24905 };