From 488a847e6e0785ac583e0f76a8dd84c5ef248629 Mon Sep 17 00:00:00 2001 From: CN_SZTL Date: Wed, 26 Feb 2020 00:09:53 +0800 Subject: [PATCH] Revert "BBR Fix (Testing (#37)" This reverts commit 60552cddea81e56b46a47049e637884ab80d545d. --- ...-based-on-ack-aggregation-estimation.patch | 504 +++++++----------- 1 file changed, 182 insertions(+), 322 deletions(-) diff --git a/target/linux/generic/pending-4.14/607-tcp_bbr-adapt-cwnd-based-on-ack-aggregation-estimation.patch b/target/linux/generic/pending-4.14/607-tcp_bbr-adapt-cwnd-based-on-ack-aggregation-estimation.patch index 18b1b8b16b..bd0e470161 100644 --- a/target/linux/generic/pending-4.14/607-tcp_bbr-adapt-cwnd-based-on-ack-aggregation-estimation.patch +++ b/target/linux/generic/pending-4.14/607-tcp_bbr-adapt-cwnd-based-on-ack-aggregation-estimation.patch @@ -15,38 +15,23 @@ Signed-off-by: Neal Cardwell Signed-off-by: Yuchung Cheng Signed-off-by: David S. Miller --- -diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h ---- a/include/net/inet_connection_sock.h -+++ b/include/net/inet_connection_sock.h -@@ -136,8 +136,8 @@ - } icsk_mtup; - u32 icsk_user_timeout; - -- u64 icsk_ca_priv[88 / sizeof(u64)]; --#define ICSK_CA_PRIV_SIZE (11 * sizeof(u64)) -+ u64 icsk_ca_priv[104 / sizeof(u64)]; -+#define ICSK_CA_PRIV_SIZE (13 * sizeof(u64)) - }; - - #define ICSK_TIME_RETRANS 1 /* Retransmit timer */ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c --- a/net/ipv4/tcp_bbr.c +++ b/net/ipv4/tcp_bbr.c -@@ -95,12 +95,10 @@ - u32 mode:3, /* current bbr_mode in state machine */ - prev_ca_state:3, /* CA state on previous ACK */ +@@ -97,10 +97,11 @@ packet_conservation:1, /* use packet conservation? */ -- restore_cwnd:1, /* decided to revert cwnd to old value */ + restore_cwnd:1, /* decided to revert cwnd to old value */ round_start:1, /* start of packet-timed tx->ack round? */ -- tso_segs_goal:7, /* segments we want in each skb we send */ ++ cycle_len:4, /* phases in this PROBE_BW gain cycle */ + tso_segs_goal:7, /* segments we want in each skb we send */ idle_restart:1, /* restarting after idle? */ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ - unused:5, -+ unused:13, ++ unused:8, lt_is_sampling:1, /* taking long-term ("LT") samples now? */ lt_rtt_cnt:7, /* round trips in long-term interval */ lt_use_bw:1; /* use lt_bw as our bw estimate? */ -@@ -117,6 +115,14 @@ +@@ -117,6 +118,14 @@ unused_b:5; u32 prior_cwnd; /* prior cwnd upon entering loss recovery */ u32 full_bw; /* recent bw, to estimate if pipe is full */ @@ -57,29 +42,49 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c + u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */ + extra_acked_win_rtts:5, /* age of extra_acked, in round trips */ + extra_acked_win_idx:1, /* current index in extra_acked array */ -+ unused_c:6; ++ unused1:6; }; #define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */ -@@ -176,6 +182,17 @@ +@@ -143,6 +152,11 @@ + /* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs: */ + static const int bbr_cwnd_gain = BBR_UNIT * 2; + /* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw: */ ++enum bbr_pacing_gain_phase { ++ BBR_BW_PROBE_UP = 0, ++ BBR_BW_PROBE_DOWN = 1, ++ BBR_BW_PROBE_CRUISE = 2, ++}; + static const int bbr_pacing_gain[] = { + BBR_UNIT * 5 / 4, /* probe for more available bw */ + BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */ +@@ -176,6 +190,25 @@ /* If we estimate we're policed, use lt_bw for this many round trips: */ static const u32 bbr_lt_bw_max_rtts = 48; +/* Gain factor for adding extra_acked to target cwnd: */ -+static const int bbr_extra_acked_gain = BBR_UNIT; -+/* Window length of extra_acked window. */ -+static const u32 bbr_extra_acked_win_rtts = 5; ++static int bbr_extra_acked_gain = BBR_UNIT; ++/* Window length of extra_acked window. Max allowed val is 31. */ ++static const u32 bbr_extra_acked_win_rtts = 10; +/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */ +static const u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20; +/* Time period for clamping cwnd increment due to ack aggregation */ +static const u32 bbr_extra_acked_max_us = 100 * 1000; + -+static void bbr_check_probe_rtt_done(struct sock *sk); ++/* Each cycle, try to hold sub-unity gain until inflight <= BDP. */ ++static bool bbr_drain_to_target = true; /* default: enabled */ ++ ++extern bool tcp_snd_wnd_test(const struct tcp_sock *tp, ++ const struct sk_buff *skb, ++ unsigned int cur_mss); ++ ++module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664); ++module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664); + /* Do we estimate that STARTUP filled the pipe? */ static bool bbr_full_bw_reached(const struct sock *sk) { -@@ -200,13 +217,31 @@ +@@ -200,6 +233,16 @@ return bbr->lt_use_bw ? bbr->lt_bw : bbr_max_bw(sk); } @@ -96,60 +101,7 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c /* Return rate in bytes per second, optionally with a gain. * The order here is chosen carefully to avoid overflow of u64. This should * work for input rates of up to 2.9Tbit/sec and gain of 2.89x. - */ -+static bool tcp_needs_internal_pacing(const struct sock *sk) -+{ -+ return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; -+} - static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain) - { -- rate *= tcp_mss_to_mtu(sk, tcp_sk(sk)->mss_cache); -+ unsigned int mss = tcp_sk(sk)->mss_cache; -+ -+ if (!tcp_needs_internal_pacing(sk)) -+ mss = tcp_mss_to_mtu(sk, mss); -+ rate *= mss; - rate *= gain; - rate >>= BBR_SCALE; - rate *= USEC_PER_SEC; -@@ -261,23 +296,25 @@ - sk->sk_pacing_rate = rate; - } - --/* Return count of segments we want in the skbs we send, or 0 for default. */ --static u32 bbr_tso_segs_goal(struct sock *sk) -+/* override sysctl_tcp_min_tso_segs */ -+static u32 bbr_min_tso_segs(struct sock *sk) - { -- struct bbr *bbr = inet_csk_ca(sk); -- -- return bbr->tso_segs_goal; -+ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; - } - --static void bbr_set_tso_segs_goal(struct sock *sk) -+static u32 bbr_tso_segs_goal(struct sock *sk) - { - struct tcp_sock *tp = tcp_sk(sk); -- struct bbr *bbr = inet_csk_ca(sk); -- u32 min_segs; -+ u32 segs, bytes; - -- min_segs = sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; -- bbr->tso_segs_goal = min(tcp_tso_autosize(sk, tp->mss_cache, min_segs), -- 0x7FU); -+ /* Sort of tcp_tso_autosize() but ignoring -+ * driver provided sk_gso_max_size. -+ */ -+ bytes = min_t(u32, sk->sk_pacing_rate >> sk->sk_pacing_shift, -+ GSO_MAX_SIZE - 1 - MAX_TCP_HEADER); -+ segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); -+ -+ return min(segs, 0x7FU); - } - - /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ -@@ -299,38 +336,31 @@ +@@ -299,6 +342,8 @@ if (event == CA_EVENT_TX_START && tp->app_limited) { bbr->idle_restart = 1; @@ -158,10 +110,7 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c /* Avoid pointless buffer overflows: pace at est. bw if we don't * need more speed (we're restarting from idle and app-limited). */ - if (bbr->mode == BBR_PROBE_BW) - bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); -+ else if (bbr->mode == BBR_PROBE_RTT) -+ bbr_check_probe_rtt_done(sk); +@@ -307,30 +352,19 @@ } } @@ -196,13 +145,13 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c u64 w; /* If we've never had a valid RTT sample, cap cwnd at the initial -@@ -345,21 +375,65 @@ +@@ -345,8 +379,24 @@ w = (u64)bw * bbr->min_rtt_us; /* Apply a gain to the given value, then remove the BW_SCALE shift. */ - cwnd = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; + bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; -+ + + return bdp; +} + @@ -216,22 +165,13 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c + * which allows 2 outstanding 2-packet sequences, to try to keep pipe + * full even with ACK-every-other-packet delayed ACKs. + */ -+static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) ++static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd, int gain) +{ + struct bbr *bbr = inet_csk_ca(sk); - /* Allow enough full-sized skbs in flight to utilize end systems. */ -- cwnd += 3 * bbr->tso_segs_goal; -+ cwnd += 3 * bbr_tso_segs_goal(sk); - - /* Reduce delayed ACKs by rounding up cwnd to the next even number. */ - cwnd = (cwnd + 1) & ~1U; - - /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ -- if (bbr->mode == BBR_PROBE_BW && gain > BBR_UNIT) -+ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == 0) - cwnd += 2; + cwnd += 3 * bbr->tso_segs_goal; +@@ -360,6 +410,33 @@ return cwnd; } @@ -241,7 +181,7 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c + u32 inflight; + + inflight = bbr_bdp(sk, bw, gain); -+ inflight = bbr_quantization_budget(sk, inflight); ++ inflight = bbr_quantization_budget(sk, inflight, gain); + + return inflight; +} @@ -265,53 +205,23 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c /* An optimization in BBR to reduce losses: On the first round of recovery, we * follow the packet conservation principle: send P packets per P packets acked. * After that, we slow-start and send at most 2*P packets per P packets acked. -@@ -391,17 +465,11 @@ - cwnd = tcp_packets_in_flight(tp) + acked; - } else if (prev_state >= TCP_CA_Recovery && state < TCP_CA_Recovery) { - /* Exiting loss recovery; restore cwnd saved before recovery. */ -- bbr->restore_cwnd = 1; -+ cwnd = max(cwnd, bbr->prior_cwnd); - bbr->packet_conservation = 0; - } - bbr->prev_ca_state = state; - -- if (bbr->restore_cwnd) { -- /* Restore cwnd after exiting loss recovery or PROBE_RTT. */ -- cwnd = max(cwnd, bbr->prior_cwnd); -- bbr->restore_cwnd = 0; -- } -- - if (bbr->packet_conservation) { - *new_cwnd = max(cwnd, tcp_packets_in_flight(tp) + acked); - return true; /* yes, using packet conservation */ -@@ -418,16 +486,23 @@ - { - struct tcp_sock *tp = tcp_sk(sk); - struct bbr *bbr = inet_csk_ca(sk); -- u32 cwnd = 0, target_cwnd = 0; -+ u32 cwnd = tp->snd_cwnd, target_cwnd = 0; - - if (!acked) -- return; -+ goto done; /* no packet fully ACKed; just apply caps */ - - if (bbr_set_cwnd_to_recover_or_restore(sk, rs, acked, &cwnd)) +@@ -427,7 +504,14 @@ goto done; + /* If we're below target cwnd, slow start cwnd toward target cwnd. */ +- target_cwnd = bbr_target_cwnd(sk, bw, gain); + target_cwnd = bbr_bdp(sk, bw, gain); + + /* Increment the cwnd to account for excess ACKed data that seems + * due to aggregation (of data and/or ACKs) visible in the ACK stream. + */ + target_cwnd += bbr_ack_aggregation_cwnd(sk); -+ target_cwnd = bbr_quantization_budget(sk, target_cwnd); ++ target_cwnd = bbr_quantization_budget(sk, target_cwnd, gain); + - /* If we're below target cwnd, slow start cwnd toward target cwnd. */ -- target_cwnd = bbr_target_cwnd(sk, bw, gain); if (bbr_full_bw_reached(sk)) /* only cut cwnd if we filled the pipe */ cwnd = min(cwnd + acked, target_cwnd); else if (cwnd < target_cwnd || tp->delivered < TCP_INIT_CWND) -@@ -468,14 +543,14 @@ +@@ -468,14 +552,80 @@ if (bbr->pacing_gain > BBR_UNIT) return is_full_length && (rs->losses || /* perhaps pacing_gain*BDP won't fit */ @@ -325,47 +235,88 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c return is_full_length || - inflight <= bbr_target_cwnd(sk, bw, BBR_UNIT); + inflight <= bbr_inflight(sk, bw, BBR_UNIT); ++} ++ ++static void bbr_set_cycle_idx(struct sock *sk, int cycle_idx) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ bbr->cycle_idx = cycle_idx; ++ bbr->pacing_gain = bbr->lt_use_bw ? ++ BBR_UNIT : bbr_pacing_gain[bbr->cycle_idx]; ++} ++ ++static void bbr_drain_to_target_cycling(struct sock *sk, ++ const struct rate_sample *rs) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ u32 elapsed_us = ++ tcp_stamp_us_delta(tp->delivered_mstamp, bbr->cycle_mstamp); ++ u32 inflight, bw; ++ ++ if (bbr->mode != BBR_PROBE_BW) ++ return; ++ ++ /* Always need to probe for bw before we forget good bw estimate. */ ++ if (elapsed_us > bbr->cycle_len * bbr->min_rtt_us) { ++ /* Start a new PROBE_BW probing cycle of [2 to 8] x min_rtt. */ ++ bbr->cycle_mstamp = tp->delivered_mstamp; ++ bbr->cycle_len = CYCLE_LEN - prandom_u32_max(bbr_cycle_rand); ++ bbr_set_cycle_idx(sk, BBR_BW_PROBE_UP); /* probe bandwidth */ ++ return; ++ } ++ ++ /* The pacing_gain of 1.0 paces at the estimated bw to try to fully ++ * use the pipe without increasing the queue. ++ */ ++ if (bbr->pacing_gain == BBR_UNIT) ++ return; ++ ++ inflight = rs->prior_in_flight; /* what was in-flight before ACK? */ ++ bw = bbr_max_bw(sk); ++ ++ /* A pacing_gain < 1.0 tries to drain extra queue we added if bw ++ * probing didn't find more bw. If inflight falls to match BDP then we ++ * estimate queue is drained; persisting would underutilize the pipe. ++ */ ++ if (bbr->pacing_gain < BBR_UNIT) { ++ if (inflight <= bbr_inflight(sk, bw, BBR_UNIT)) ++ bbr_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE); /* cruise */ ++ return; ++ } ++ ++ /* A pacing_gain > 1.0 probes for bw by trying to raise inflight to at ++ * least pacing_gain*BDP; this may take more than min_rtt if min_rtt is ++ * small (e.g. on a LAN). We do not persist if packets are lost, since ++ * a path with small buffers may not hold that much. Similarly we exit ++ * if we were prevented by app/recv-win from reaching the target. ++ */ ++ if (elapsed_us > bbr->min_rtt_us && ++ (inflight >= bbr_inflight(sk, bw, bbr->pacing_gain) || ++ rs->losses || /* perhaps pacing_gain*BDP won't fit */ ++ rs->is_app_limited || /* previously app-limited */ ++ !tcp_send_head(sk) || /* currently app/rwin-limited */ ++ !tcp_snd_wnd_test(tp, tcp_send_head(sk), tp->mss_cache))) { ++ bbr_set_cycle_idx(sk, BBR_BW_PROBE_DOWN); /* drain queue */ ++ return; ++ } } static void bbr_advance_cycle_phase(struct sock *sk) -@@ -485,8 +560,6 @@ - - bbr->cycle_idx = (bbr->cycle_idx + 1) & (CYCLE_LEN - 1); - bbr->cycle_mstamp = tp->delivered_mstamp; -- bbr->pacing_gain = bbr->lt_use_bw ? BBR_UNIT : -- bbr_pacing_gain[bbr->cycle_idx]; - } - - /* Gain cycling: cycle pacing gain to converge to fair share of available bw. */ -@@ -504,8 +577,6 @@ +@@ -495,6 +645,11 @@ + { struct bbr *bbr = inet_csk_ca(sk); - bbr->mode = BBR_STARTUP; -- bbr->pacing_gain = bbr_high_gain; -- bbr->cwnd_gain = bbr_high_gain; ++ if (bbr_drain_to_target) { ++ bbr_drain_to_target_cycling(sk, rs); ++ return; ++ } ++ + if (bbr->mode == BBR_PROBE_BW && bbr_is_next_cycle_phase(sk, rs)) + bbr_advance_cycle_phase(sk); } - - static void bbr_reset_probe_bw_mode(struct sock *sk) -@@ -513,8 +584,6 @@ - struct bbr *bbr = inet_csk_ca(sk); - - bbr->mode = BBR_PROBE_BW; -- bbr->pacing_gain = BBR_UNIT; -- bbr->cwnd_gain = bbr_cwnd_gain; - bbr->cycle_idx = CYCLE_LEN - 1 - prandom_u32_max(bbr_cycle_rand); - bbr_advance_cycle_phase(sk); /* flip to next phase of gain cycle */ - } -@@ -678,8 +747,7 @@ - * bandwidth sample. Delivered is in packets and interval_us in uS and - * ratio will be <<1 for most connections. So delivered is first scaled. - */ -- bw = (u64)rs->delivered * BW_UNIT; -- do_div(bw, rs->interval_us); -+ bw = div64_long((u64)rs->delivered * BW_UNIT, rs->interval_us); - - /* If this sample is application-limited, it is likely to have a very - * low delivered count that represents application behavior rather than -@@ -698,6 +766,67 @@ +@@ -698,6 +853,67 @@ } } @@ -379,7 +330,7 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c + * cwnd += max_extra_acked + * + * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms). -+ * Max filter is an approximate sliding window of 5-10 (packet timed) round ++ * Max filter is an approximate sliding window of 10-20 (packet timed) round + * trips. + */ +static void bbr_update_ack_aggregation(struct sock *sk, @@ -422,8 +373,8 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c + } + + /* Compute excess data delivered, beyond what was expected. */ -+ bbr->ack_epoch_acked = min_t(u32, 0xFFFFF, -+ bbr->ack_epoch_acked + rs->acked_sacked); ++ bbr->ack_epoch_acked = min(0xFFFFFU, ++ bbr->ack_epoch_acked + rs->acked_sacked); + extra_acked = bbr->ack_epoch_acked - expected_acked; + extra_acked = min(extra_acked, tp->snd_cwnd); + if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx]) @@ -433,11 +384,10 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c /* Estimate when the pipe is full, using the change in delivery rate: BBR * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited -@@ -732,15 +861,29 @@ - +@@ -733,11 +949,12 @@ if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { bbr->mode = BBR_DRAIN; /* drain queue we created */ -- bbr->pacing_gain = bbr_drain_gain; /* pace slow to drain */ + bbr->pacing_gain = bbr_drain_gain; /* pace slow to drain */ - bbr->cwnd_gain = bbr_high_gain; /* maintain cwnd */ + tcp_sk(sk)->snd_ssthresh = + bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); @@ -449,90 +399,7 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c bbr_reset_probe_bw_mode(sk); /* we estimate queue is drained */ } -+static void bbr_check_probe_rtt_done(struct sock *sk) -+{ -+ struct tcp_sock *tp = tcp_sk(sk); -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ if (!(bbr->probe_rtt_done_stamp && -+ after(tcp_jiffies32, bbr->probe_rtt_done_stamp))) -+ return; -+ -+ bbr->min_rtt_stamp = tcp_jiffies32; /* wait a while until PROBE_RTT */ -+ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); -+ bbr_reset_mode(sk); -+} -+ - /* The goal of PROBE_RTT mode is to have BBR flows cooperatively and - * periodically drain the bottleneck queue, to converge to measure the true - * min_rtt (unloaded propagation delay). This allows the flows to keep queues -@@ -770,7 +913,8 @@ - filter_expired = after(tcp_jiffies32, - bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ); - if (rs->rtt_us >= 0 && -- (rs->rtt_us <= bbr->min_rtt_us || filter_expired)) { -+ (rs->rtt_us <= bbr->min_rtt_us || -+ (filter_expired && !rs->is_ack_delayed))) { - bbr->min_rtt_us = rs->rtt_us; - bbr->min_rtt_stamp = tcp_jiffies32; - } -@@ -778,8 +922,6 @@ - if (bbr_probe_rtt_mode_ms > 0 && filter_expired && - !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { - bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ -- bbr->pacing_gain = BBR_UNIT; -- bbr->cwnd_gain = BBR_UNIT; - bbr_save_cwnd(sk); /* note cwnd so we can restore it */ - bbr->probe_rtt_done_stamp = 0; - } -@@ -798,12 +940,8 @@ - } else if (bbr->probe_rtt_done_stamp) { - if (bbr->round_start) - bbr->probe_rtt_round_done = 1; -- if (bbr->probe_rtt_round_done && -- after(tcp_jiffies32, bbr->probe_rtt_done_stamp)) { -- bbr->min_rtt_stamp = tcp_jiffies32; -- bbr->restore_cwnd = 1; /* snap to prior_cwnd */ -- bbr_reset_mode(sk); -- } -+ if (bbr->probe_rtt_round_done) -+ bbr_check_probe_rtt_done(sk); - } - } - /* Restart after idle ends only once we process a new S/ACK for data */ -@@ -811,13 +949,44 @@ - bbr->idle_restart = 0; - } - -+static void bbr_update_gains(struct sock *sk) -+{ -+ struct bbr *bbr = inet_csk_ca(sk); -+ -+ switch (bbr->mode) { -+ case BBR_STARTUP: -+ bbr->pacing_gain = bbr_high_gain; -+ bbr->cwnd_gain = bbr_high_gain; -+ break; -+ case BBR_DRAIN: -+ bbr->pacing_gain = bbr_drain_gain; /* slow, to drain */ -+ bbr->cwnd_gain = bbr_high_gain; /* keep cwnd */ -+ break; -+ case BBR_PROBE_BW: -+ bbr->pacing_gain = (bbr->lt_use_bw ? -+ BBR_UNIT : -+ bbr_pacing_gain[bbr->cycle_idx]); -+ bbr->cwnd_gain = bbr_cwnd_gain; -+ break; -+ case BBR_PROBE_RTT: -+ bbr->pacing_gain = BBR_UNIT; -+ bbr->cwnd_gain = BBR_UNIT; -+ break; -+ default: -+ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); -+ break; -+ } -+} -+ +@@ -814,6 +1031,7 @@ static void bbr_update_model(struct sock *sk, const struct rate_sample *rs) { bbr_update_bw(sk, rs); @@ -540,37 +407,11 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c bbr_update_cycle_phase(sk, rs); bbr_check_full_bw_reached(sk, rs); bbr_check_drain(sk, rs); - bbr_update_min_rtt(sk, rs); -+ bbr_update_gains(sk); - } - - static void bbr_main(struct sock *sk, const struct rate_sample *rs) -@@ -829,7 +998,6 @@ - - bw = bbr_bw(sk); - bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); -- bbr_set_tso_segs_goal(sk); - bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain); - } - -@@ -839,7 +1007,7 @@ - struct bbr *bbr = inet_csk_ca(sk); - - bbr->prior_cwnd = 0; -- bbr->tso_segs_goal = 0; /* default segs per skb until first ACK */ -+ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; - bbr->rtt_cnt = 0; - bbr->next_rtt_delivered = 0; - bbr->prev_ca_state = TCP_CA_Open; -@@ -855,7 +1023,6 @@ - bbr->has_seen_rtt = 0; - bbr_init_pacing_rate_from_rtt(sk); - -- bbr->restore_cwnd = 0; - bbr->round_start = 0; - bbr->idle_restart = 0; - bbr->full_bw_reached = 0; -@@ -866,6 +1033,13 @@ +@@ -863,9 +1081,17 @@ + bbr->full_bw_cnt = 0; + bbr->cycle_mstamp = 0; + bbr->cycle_idx = 0; ++ bbr->cycle_len = 0; bbr_reset_lt_bw_sampling(sk); bbr_reset_startup_mode(sk); @@ -584,42 +425,61 @@ diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); } -@@ -892,7 +1066,7 @@ - static u32 bbr_ssthresh(struct sock *sk) +@@ -948,7 +1174,7 @@ + + static int __init bbr_register(void) { - bbr_save_cwnd(sk); -- return TCP_INFINITE_SSTHRESH; /* BBR does not use ssthresh */ -+ return tcp_sk(sk)->snd_ssthresh; +- BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE); ++/* BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE);*/ + return tcp_register_congestion_control(&tcp_bbr_cong_ops); + } + +diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c +--- a/net/ipv4/tcp_output.c ++++ b/net/ipv4/tcp_output.c +@@ -1824,8 +1824,7 @@ } - static size_t bbr_get_info(struct sock *sk, u32 ext, int *attr, -@@ -941,7 +1115,7 @@ - .undo_cwnd = bbr_undo_cwnd, - .cwnd_event = bbr_cwnd_event, - .ssthresh = bbr_ssthresh, -- .tso_segs_goal = bbr_tso_segs_goal, -+ .min_tso_segs = bbr_min_tso_segs, - .get_info = bbr_get_info, - .set_state = bbr_set_state, - }; + /* Does at least the first segment of SKB fit into the send window? */ +-static bool tcp_snd_wnd_test(const struct tcp_sock *tp, +- const struct sk_buff *skb, ++bool tcp_snd_wnd_test(const struct tcp_sock *tp, const struct sk_buff *skb, + unsigned int cur_mss) + { + u32 end_seq = TCP_SKB_CB(skb)->end_seq; +@@ -1835,6 +1834,7 @@ + + return !after(end_seq, tcp_wnd_end(tp)); + } ++EXPORT_SYMBOL(tcp_snd_wnd_test); + + /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet + * which is put after SKB on the list. It is very much like diff --git a/include/net/tcp.h b/include/net/tcp.h --- a/include/net/tcp.h +++ b/include/net/tcp.h -@@ -998,6 +998,7 @@ - u32 prior_in_flight; /* in flight before this ACK */ - bool is_app_limited; /* is sample from packet with bubble in pipe? */ - bool is_retrans; /* is sample from retransmission? */ -+ bool is_ack_delayed; /* is this (likely) a delayed ACK? */ +@@ -553,6 +553,8 @@ + + u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, + int min_tso_segs); ++bool tcp_snd_wnd_test(const struct tcp_sock *tp, const struct sk_buff *skb, ++ unsigned int cur_mss); + void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, + int nonagle); + int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs); + +diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h +--- a/include/net/inet_connection_sock.h ++++ b/include/net/inet_connection_sock.h +@@ -136,8 +136,8 @@ + } icsk_mtup; + u32 icsk_user_timeout; + +- u64 icsk_ca_priv[88 / sizeof(u64)]; +-#define ICSK_CA_PRIV_SIZE (11 * sizeof(u64)) ++ u64 icsk_ca_priv[104 / sizeof(u64)]; ++#define ICSK_CA_PRIV_SIZE (13 * sizeof(u64)) }; - struct tcp_congestion_ops { -@@ -1024,6 +1025,8 @@ - u32 (*undo_cwnd)(struct sock *sk); - /* hook for packet ack accounting (optional) */ - void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); -+ /* override sysctl_tcp_min_tso_segs */ -+ u32 (*min_tso_segs)(struct sock *sk); - /* suggest number of segments for each skb to transmit (optional) */ - u32 (*tso_segs_goal)(struct sock *sk); - /* returns the multiplier used in tcp_sndbuf_expand (optional) */ \ No newline at end of file + #define ICSK_TIME_RETRANS 1 /* Retransmit timer */