diff --git a/target/linux/generic/backport-4.9/605-0001-tcp-internal-implementation-for-pacing.patch b/target/linux/generic/backport-4.9/605-0001-tcp-internal-implementation-for-pacing.patch new file mode 100644 index 0000000000..1632631561 --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0001-tcp-internal-implementation-for-pacing.patch @@ -0,0 +1,340 @@ +From 4f7737d75080d6d28b8a90c7499d85b6d4b4dd49 Mon Sep 17 00:00:00 2001 +From: Eric Dumazet +Date: Tue, 16 May 2017 04:24:36 -0700 +Subject: [PATCH] BACKPORT: tcp: internal implementation for pacing + +BBR congestion control depends on pacing, and pacing is +currently handled by sch_fq packet scheduler for performance reasons, +and also because implemening pacing with FQ was convenient to truly +avoid bursts. + +However there are many cases where this packet scheduler constraint +is not practical. +- Many linux hosts are not focusing on handling thousands of TCP + flows in the most efficient way. +- Some routers use fq_codel or other AQM, but still would like + to use BBR for the few TCP flows they initiate/terminate. + +This patch implements an automatic fallback to internal pacing. + +Pacing is requested either by BBR or use of SO_MAX_PACING_RATE option. + +If sch_fq happens to be in the egress path, pacing is delegated to +the qdisc, otherwise pacing is done by TCP itself. + +One advantage of pacing from TCP stack is to get more precise rtt +estimations, and less work done from TX completion, since TCP Small +queue limits are not generally hit. Setups with single TX queue but +many cpus might even benefit from this. + +Note that unlike sch_fq, we do not take into account header sizes. +Taking care of these headers would add additional complexity for +no practical differences in behavior. + +Some performance numbers using 800 TCP_STREAM flows rate limited to +~48 Mbit per second on 40Gbit NIC. + +If MQ+pfifo_fast is used on the NIC : + +$ sar -n DEV 1 5 | grep eth +14:48:44 eth0 725743.00 2932134.00 46776.76 4335184.68 0.00 0.00 1.00 +14:48:45 eth0 725349.00 2932112.00 46751.86 4335158.90 0.00 0.00 0.00 +14:48:46 eth0 725101.00 2931153.00 46735.07 4333748.63 0.00 0.00 0.00 +14:48:47 eth0 725099.00 2931161.00 46735.11 4333760.44 0.00 0.00 1.00 +14:48:48 eth0 725160.00 2931731.00 46738.88 4334606.07 0.00 0.00 0.00 +Average: eth0 725290.40 2931658.20 46747.54 4334491.74 0.00 0.00 0.40 +$ vmstat 1 5 +procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu----- + r b swpd free buff cache si so bi bo in cs us sy id wa st + 4 0 0 259825920 45644 2708324 0 0 21 2 247 98 0 0 100 0 0 + 4 0 0 259823744 45644 2708356 0 0 0 0 2400825 159843 0 19 81 0 0 + 0 0 0 259824208 45644 2708072 0 0 0 0 2407351 159929 0 19 81 0 0 + 1 0 0 259824592 45644 2708128 0 0 0 0 2405183 160386 0 19 80 0 0 + 1 0 0 259824272 45644 2707868 0 0 0 32 2396361 158037 0 19 81 0 0 + +Now use MQ+FQ : + +lpaa23:~# echo fq >/proc/sys/net/core/default_qdisc +lpaa23:~# tc qdisc replace dev eth0 root mq + +$ sar -n DEV 1 5 | grep eth +14:49:57 eth0 678614.00 2727930.00 43739.13 4033279.14 0.00 0.00 0.00 +14:49:58 eth0 677620.00 2723971.00 43674.69 4027429.62 0.00 0.00 1.00 +14:49:59 eth0 676396.00 2719050.00 43596.83 4020125.02 0.00 0.00 0.00 +14:50:00 eth0 675197.00 2714173.00 43518.62 4012938.90 0.00 0.00 1.00 +14:50:01 eth0 676388.00 2719063.00 43595.47 4020171.64 0.00 0.00 0.00 +Average: eth0 676843.00 2720837.40 43624.95 4022788.86 0.00 0.00 0.40 +$ vmstat 1 5 +procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu----- + r b swpd free buff cache si so bi bo in cs us sy id wa st + 2 0 0 259832240 46008 2710912 0 0 21 2 223 192 0 1 99 0 0 + 1 0 0 259832896 46008 2710744 0 0 0 0 1702206 198078 0 17 82 0 0 + 0 0 0 259830272 46008 2710596 0 0 0 0 1696340 197756 1 17 83 0 0 + 4 0 0 259829168 46024 2710584 0 0 16 0 1688472 197158 1 17 82 0 0 + 3 0 0 259830224 46024 2710408 0 0 0 0 1692450 197212 0 18 82 0 0 + +As expected, number of interrupts per second is very different. + +Signed-off-by: Eric Dumazet +Acked-by: Soheil Hassas Yeganeh +Cc: Neal Cardwell +Cc: Yuchung Cheng +Cc: Van Jacobson +Cc: Jerry Chu +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + include/linux/tcp.h | 2 ++ + include/net/sock.h | 8 +++++ + include/net/tcp.h | 3 ++ + net/core/sock.c | 4 +++ + net/ipv4/tcp_bbr.c | 9 ++--- + net/ipv4/tcp_output.c | 80 +++++++++++++++++++++++++++++++++++++++++++ + net/ipv4/tcp_timer.c | 3 ++ + net/sched/sch_fq.c | 8 +++++ + 8 files changed, 113 insertions(+), 4 deletions(-) + +--- a/include/linux/tcp.h ++++ b/include/linux/tcp.h +@@ -290,6 +290,8 @@ struct tcp_sock { + u32 sacked_out; /* SACK'd packets */ + u32 fackets_out; /* FACK'd packets */ + ++ struct hrtimer pacing_timer; ++ + /* from STCP, retrans queue hinting */ + struct sk_buff* lost_skb_hint; + struct sk_buff *retransmit_skb_hint; +--- a/include/net/sock.h ++++ b/include/net/sock.h +@@ -250,6 +250,7 @@ struct sock_common { + * @sk_ll_usec: usecs to busypoll when there is no data + * @sk_allocation: allocation mode + * @sk_pacing_rate: Pacing rate (if supported by transport/packet scheduler) ++ * @sk_pacing_status: Pacing status (requested, handled by sch_fq) + * @sk_max_pacing_rate: Maximum pacing rate (%SO_MAX_PACING_RATE) + * @sk_sndbuf: size of send buffer in bytes + * @sk_padding: unused element for alignment +@@ -394,6 +395,7 @@ struct sock { + struct sk_buff_head sk_write_queue; + __s32 sk_peek_off; + int sk_write_pending; ++ u32 sk_pacing_status; /* see enum sk_pacing */ + long sk_sndtimeo; + struct timer_list sk_timer; + __u32 sk_priority; +@@ -460,6 +462,12 @@ struct sock { + struct rcu_head sk_rcu; + }; + ++enum sk_pacing { ++ SK_PACING_NONE = 0, ++ SK_PACING_NEEDED = 1, ++ SK_PACING_FQ = 2, ++}; ++ + #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data))) + + #define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk))) +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -584,6 +584,7 @@ void tcp_fin(struct sock *sk); + void tcp_init_xmit_timers(struct sock *); + static inline void tcp_clear_xmit_timers(struct sock *sk) + { ++ hrtimer_cancel(&tcp_sk(sk)->pacing_timer); + inet_csk_clear_xmit_timers(sk); + } + +@@ -1970,4 +1971,6 @@ static inline void tcp_listendrop(const + __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); + } + ++enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); ++ + #endif /* _TCP_H */ +--- a/net/core/sock.c ++++ b/net/core/sock.c +@@ -989,6 +989,10 @@ set_rcvbuf: + #endif + + case SO_MAX_PACING_RATE: ++ if (val != ~0U) ++ cmpxchg(&sk->sk_pacing_status, ++ SK_PACING_NONE, ++ SK_PACING_NEEDED); + sk->sk_max_pacing_rate = val; + sk->sk_pacing_rate = min(sk->sk_pacing_rate, + sk->sk_max_pacing_rate); +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -22,10 +22,9 @@ + * There is a public e-mail list for discussing BBR development and testing: + * https://groups.google.com/forum/#!forum/bbr-dev + * +- * NOTE: BBR *must* be used with the fq qdisc ("man tc-fq") with pacing enabled, +- * since pacing is integral to the BBR design and implementation. +- * BBR without pacing would not function properly, and may incur unnecessary +- * high packet loss rates. ++ * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, ++ * otherwise TCP stack falls back to an internal pacing using one high ++ * resolution timer per TCP socket and may use more resources. + */ + #include + #include +@@ -835,6 +834,8 @@ static void bbr_init(struct sock *sk) + bbr->cycle_idx = 0; + bbr_reset_lt_bw_sampling(sk); + bbr_reset_startup_mode(sk); ++ ++ cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); + } + + static u32 bbr_sndbuf_expand(struct sock *sk) +--- a/net/ipv4/tcp_output.c ++++ b/net/ipv4/tcp_output.c +@@ -913,6 +913,72 @@ out: + sk_free(sk); + } + ++/* Note: Called under hard irq. ++ * We can not call TCP stack right away. ++ */ ++enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer) ++{ ++ struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer); ++ struct sock *sk = (struct sock *)tp; ++ unsigned long nval, oval; ++ ++ for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) { ++ struct tsq_tasklet *tsq; ++ bool empty; ++ ++ if (oval & TSQF_QUEUED) ++ break; ++ ++ nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED; ++ nval = cmpxchg(&sk->sk_tsq_flags, oval, nval); ++ if (nval != oval) ++ continue; ++ ++ if (!atomic_inc_not_zero(&sk->sk_wmem_alloc)) ++ break; ++ /* queue this socket to tasklet queue */ ++ tsq = this_cpu_ptr(&tsq_tasklet); ++ empty = list_empty(&tsq->head); ++ list_add(&tp->tsq_node, &tsq->head); ++ if (empty) ++ tasklet_schedule(&tsq->tasklet); ++ break; ++ } ++ return HRTIMER_NORESTART; ++} ++ ++/* BBR congestion control needs pacing. ++ * Same remark for SO_MAX_PACING_RATE. ++ * sch_fq packet scheduler is efficiently handling pacing, ++ * but is not always installed/used. ++ * Return true if TCP stack should pace packets itself. ++ */ ++static bool tcp_needs_internal_pacing(const struct sock *sk) ++{ ++ return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; ++} ++ ++static void tcp_internal_pacing(struct sock *sk, const struct sk_buff *skb) ++{ ++ u64 len_ns; ++ u32 rate; ++ ++ if (!tcp_needs_internal_pacing(sk)) ++ return; ++ rate = sk->sk_pacing_rate; ++ if (!rate || rate == ~0U) ++ return; ++ ++ /* Should account for header sizes as sch_fq does, ++ * but lets make things simple. ++ */ ++ len_ns = (u64)skb->len * NSEC_PER_SEC; ++ do_div(len_ns, rate); ++ hrtimer_start(&tcp_sk(sk)->pacing_timer, ++ ktime_add_ns(ktime_get(), len_ns), ++ HRTIMER_MODE_ABS_PINNED); ++} ++ + /* This routine actually transmits TCP packets queued in by + * tcp_do_sendmsg(). This is used by both the initial + * transmission and possible later retransmissions. +@@ -1034,6 +1100,7 @@ static int __tcp_transmit_skb(struct soc + if (skb->len != tcp_header_size) { + tcp_event_data_sent(tp, sk); + tp->data_segs_out += tcp_skb_pcount(skb); ++ tcp_internal_pacing(sk, skb); + } + + if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq) +@@ -2125,6 +2192,12 @@ static int tcp_mtu_probe(struct sock *sk + return -1; + } + ++static bool tcp_pacing_check(const struct sock *sk) ++{ ++ return tcp_needs_internal_pacing(sk) && ++ hrtimer_active(&tcp_sk(sk)->pacing_timer); ++} ++ + /* TCP Small Queues : + * Control number of packets in qdisc/devices to two packets / or ~1 ms. + * (These limits are doubled for retransmits) +@@ -2208,6 +2281,9 @@ static bool tcp_write_xmit(struct sock * + while ((skb = tcp_send_head(sk))) { + unsigned int limit; + ++ if (tcp_pacing_check(sk)) ++ break; ++ + tso_segs = tcp_init_tso_segs(skb, mss_now); + BUG_ON(!tso_segs); + +@@ -2932,6 +3008,10 @@ void tcp_xmit_retransmit_queue(struct so + + if (skb == tcp_send_head(sk)) + break; ++ ++ if (tcp_pacing_check(sk)) ++ break; ++ + /* we could do better than to assign each time */ + if (!hole) + tp->retransmit_skb_hint = skb; +--- a/net/ipv4/tcp_timer.c ++++ b/net/ipv4/tcp_timer.c +@@ -727,4 +727,7 @@ void tcp_init_xmit_timers(struct sock *s + { + inet_csk_init_xmit_timers(sk, &tcp_write_timer, &tcp_delack_timer, + &tcp_keepalive_timer); ++ hrtimer_init(&tcp_sk(sk)->pacing_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_ABS_PINNED); ++ tcp_sk(sk)->pacing_timer.function = tcp_pace_kick; + } +--- a/net/sched/sch_fq.c ++++ b/net/sched/sch_fq.c +@@ -405,9 +405,17 @@ static int fq_enqueue(struct sk_buff *sk + q->stat_tcp_retrans++; + qdisc_qstats_backlog_inc(sch, skb); + if (fq_flow_is_detached(f)) { ++ struct sock *sk = skb->sk; ++ + fq_flow_add_tail(&q->new_flows, f); + if (time_after(jiffies, f->age + q->flow_refill_delay)) + f->credit = max_t(u32, f->credit, q->quantum); ++ if (sk && q->rate_enable) { ++ if (unlikely(smp_load_acquire(&sk->sk_pacing_status) != ++ SK_PACING_FQ)) ++ smp_store_release(&sk->sk_pacing_status, ++ SK_PACING_FQ); ++ } + q->inactive_flows--; + } + diff --git a/target/linux/generic/backport-4.9/605-0002-tcp-avoid-min-RTT-overestimation-from-delayed-ACK.patch b/target/linux/generic/backport-4.9/605-0002-tcp-avoid-min-RTT-overestimation-from-delayed-ACK.patch new file mode 100644 index 0000000000..63a4eb0b6c --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0002-tcp-avoid-min-RTT-overestimation-from-delayed-ACK.patch @@ -0,0 +1,80 @@ +From 549a5e6d301748438255a9c014df6079e5a7dd28 Mon Sep 17 00:00:00 2001 +From: Yuchung Cheng +Date: Wed, 17 Jan 2018 12:11:00 -0800 +Subject: [PATCH] BACKPORT: tcp: avoid min-RTT overestimation from delayed ACKs + +This patch avoids having TCP sender or congestion control +overestimate the min RTT by orders of magnitude. This happens when +all the samples in the windowed filter are one-packet transfer +like small request and health-check like chit-chat, which is farily +common for applications using persistent connections. This patch +tries to conservatively labels and skip RTT samples obtained from +this type of workload. + +Signed-off-by: Yuchung Cheng +Signed-off-by: Soheil Hassas Yeganeh +Acked-by: Neal Cardwell +Acked-by: Eric Dumazet +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_input.c | 23 +++++++++++++++++++++-- + 1 file changed, 21 insertions(+), 2 deletions(-) + +--- a/net/ipv4/tcp_input.c ++++ b/net/ipv4/tcp_input.c +@@ -116,6 +116,7 @@ int sysctl_tcp_invalid_ratelimit __read_ + #define FLAG_SACK_RENEGING 0x2000 /* snd_una advanced to a sacked seq */ + #define FLAG_UPDATE_TS_RECENT 0x4000 /* tcp_replace_ts_recent() */ + #define FLAG_NO_CHALLENGE_ACK 0x8000 /* do not call tcp_send_challenge_ack() */ ++#define FLAG_ACK_MAYBE_DELAYED 0x10000 /* Likely a delayed ACK */ + + #define FLAG_ACKED (FLAG_DATA_ACKED|FLAG_SYN_ACKED) + #define FLAG_NOT_DUP (FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED) +@@ -2972,11 +2973,18 @@ static void tcp_fastretrans_alert(struct + *rexmit = REXMIT_LOST; + } + +-static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us) ++static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us, const int flag) + { + struct tcp_sock *tp = tcp_sk(sk); + u32 wlen = sysctl_tcp_min_rtt_wlen * HZ; + ++ if ((flag & FLAG_ACK_MAYBE_DELAYED) && rtt_us > tcp_min_rtt(tp)) { ++ /* If the remote keeps returning delayed ACKs, eventually ++ * the min filter would pick it up and overestimate the ++ * prop. delay when it expires. Skip suspected delayed ACKs. ++ */ ++ return; ++ } + minmax_running_min(&tp->rtt_min, wlen, tcp_time_stamp, + rtt_us ? : jiffies_to_usecs(1)); + } +@@ -3012,7 +3020,7 @@ static inline bool tcp_ack_update_rtt(st + * always taken together with ACK, SACK, or TS-opts. Any negative + * values will be skipped with the seq_rtt_us < 0 check above. + */ +- tcp_update_rtt_min(sk, ca_rtt_us); ++ tcp_update_rtt_min(sk, ca_rtt_us, flag); + tcp_rtt_estimator(sk, seq_rtt_us); + tcp_set_rto(sk); + +@@ -3247,6 +3255,17 @@ static int tcp_clean_rtx_queue(struct so + if (likely(first_ackt.v64) && !(flag & FLAG_RETRANS_DATA_ACKED)) { + seq_rtt_us = skb_mstamp_us_delta(now, &first_ackt); + ca_rtt_us = skb_mstamp_us_delta(now, &last_ackt); ++ ++ if (pkts_acked == 1 && last_in_flight < tp->mss_cache && ++ last_in_flight && !prior_sacked && fully_acked && ++ sack->rate->prior_delivered + 1 == tp->delivered && ++ !(flag & (FLAG_CA_ALERT | FLAG_SYN_ACKED))) { ++ /* Conservatively mark a delayed ACK. It's typically ++ * from a lone runt packet over the round trip to ++ * a receiver w/o out-of-order or CE events. ++ */ ++ flag |= FLAG_ACK_MAYBE_DELAYED; ++ } + } + if (sack->first_sackt.v64) { + sack_rtt_us = skb_mstamp_us_delta(now, &sack->first_sackt); diff --git a/target/linux/generic/backport-4.9/605-0003-tcp-avoid-min-RTT-bloat-by-skipping-RTT-from-delayed-ACK-.patch b/target/linux/generic/backport-4.9/605-0003-tcp-avoid-min-RTT-bloat-by-skipping-RTT-from-delayed-ACK-.patch new file mode 100644 index 0000000000..7ccb01bead --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0003-tcp-avoid-min-RTT-bloat-by-skipping-RTT-from-delayed-ACK-.patch @@ -0,0 +1,60 @@ +From b2f3b3121c19c43877654079753325c16fe2b811 Mon Sep 17 00:00:00 2001 +From: Yuchung Cheng +Date: Wed, 17 Jan 2018 12:11:01 -0800 +Subject: [PATCH] BACKPORT: tcp: avoid min RTT bloat by skipping RTT from + delayed-ACK in BBR + +A persistent connection may send tiny amount of data (e.g. health-check) +for a long period of time. BBR's windowed min RTT filter may only see +RTT samples from delayed ACKs causing BBR to grossly over-estimate +the path delay depending how much the ACK was delayed at the receiver. + +This patch skips RTT samples that are likely coming from delayed ACKs. Note +that it is possible the sender never obtains a valid measure to set the +min RTT. In this case BBR will continue to set cwnd to initial window +which seems fine because the connection is thin stream. + +Signed-off-by: Yuchung Cheng +Acked-by: Neal Cardwell +Acked-by: Soheil Hassas Yeganeh +Acked-by: Priyaranjan Jha +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + include/net/tcp.h | 1 + + net/ipv4/tcp_bbr.c | 3 ++- + net/ipv4/tcp_input.c | 1 + + 3 files changed, 4 insertions(+), 1 deletion(-) + +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -911,6 +911,7 @@ struct rate_sample { + u32 prior_in_flight; /* in flight before this ACK */ + bool is_app_limited; /* is sample from packet with bubble in pipe? */ + bool is_retrans; /* is sample from retransmission? */ ++ bool is_ack_delayed; /* is this (likely) a delayed ACK? */ + }; + + struct tcp_congestion_ops { +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -739,7 +739,8 @@ static void bbr_update_min_rtt(struct so + filter_expired = after(tcp_time_stamp, + bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ); + if (rs->rtt_us >= 0 && +- (rs->rtt_us < bbr->min_rtt_us || filter_expired)) { ++ (rs->rtt_us < bbr->min_rtt_us || ++ (filter_expired && !rs->is_ack_delayed))) { + bbr->min_rtt_us = rs->rtt_us; + bbr->min_rtt_stamp = tcp_time_stamp; + } +--- a/net/ipv4/tcp_input.c ++++ b/net/ipv4/tcp_input.c +@@ -3779,6 +3779,7 @@ static int tcp_ack(struct sock *sk, cons + tcp_schedule_loss_probe(sk); + delivered = tp->delivered - delivered; /* freshly ACKed or SACKed */ + lost = tp->lost - lost; /* freshly marked lost */ ++ rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); + tcp_rate_gen(sk, delivered, lost, is_sack_reneg, &now, &rs); + tcp_cong_control(sk, ack, delivered, flag, &rs); + tcp_xmit_recovery(sk, rexmit); diff --git a/target/linux/generic/backport-4.9/605-0004-tcp_bbr-better-deal-with-suboptimal-GSO-II.patch b/target/linux/generic/backport-4.9/605-0004-tcp_bbr-better-deal-with-suboptimal-GSO-II.patch new file mode 100644 index 0000000000..62ed2d7184 --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0004-tcp_bbr-better-deal-with-suboptimal-GSO-II.patch @@ -0,0 +1,140 @@ +From a03defe0b7f9d9378599c986aaf1418ee6cec231 Mon Sep 17 00:00:00 2001 +From: Eric Dumazet +Date: Wed, 28 Feb 2018 14:40:46 -0800 +Subject: [PATCH] BACKPORT: tcp_bbr: better deal with suboptimal GSO (II) + +This is second part of dealing with suboptimal device gso parameters. +In first patch (45fa6615258e "tcp_bbr: better deal with suboptimal GSO") +we dealt with devices having low gso_max_segs + +Some devices lower gso_max_size from 64KB to 16 KB (r8152 is an example) + +In order to probe an optimal cwnd, we want BBR being not sensitive +to whatever GSO constraint a device can have. + +This patch removes tso_segs_goal() CC callback in favor of +min_tso_segs() for CC wanting to override sysctl_tcp_min_tso_segs + +Next patch will remove bbr->tso_segs_goal since it does not have +to be persistent. + +Signed-off-by: Eric Dumazet +Acked-by: Neal Cardwell +Signed-off-by: David S. Miller +[kras: Replace sk_pacing_shift with 10 (which is default value) to + avoid backporting mainline commit 3a9b76fd0db9] +Signed-off-by: Albert I +--- + include/net/tcp.h | 6 ++---- + net/ipv4/tcp_bbr.c | 23 +++++++++++++---------- + net/ipv4/tcp_output.c | 15 ++++++++------- + 3 files changed, 23 insertions(+), 21 deletions(-) + +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -544,8 +544,6 @@ __u32 cookie_v6_init_sequence(const stru + #endif + /* tcp_output.c */ + +-u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, +- int min_tso_segs); + void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, + int nonagle); + bool tcp_may_send_now(struct sock *sk); +@@ -938,8 +936,8 @@ struct tcp_congestion_ops { + u32 (*undo_cwnd)(struct sock *sk); + /* hook for packet ack accounting (optional) */ + void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); +- /* suggest number of segments for each skb to transmit (optional) */ +- u32 (*tso_segs_goal)(struct sock *sk); ++ /* override sysctl_tcp_min_tso_segs */ ++ u32 (*min_tso_segs)(struct sock *sk); + /* returns the multiplier used in tcp_sndbuf_expand (optional) */ + u32 (*sndbuf_expand)(struct sock *sk); + /* call when packets are delivered to update cwnd and pacing rate, +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -231,23 +231,26 @@ static void bbr_set_pacing_rate(struct s + sk->sk_pacing_rate = rate; + } + +-/* Return count of segments we want in the skbs we send, or 0 for default. */ +-static u32 bbr_tso_segs_goal(struct sock *sk) ++/* override sysctl_tcp_min_tso_segs */ ++static u32 bbr_min_tso_segs(struct sock *sk) + { +- struct bbr *bbr = inet_csk_ca(sk); +- +- return bbr->tso_segs_goal; ++ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; + } + + static void bbr_set_tso_segs_goal(struct sock *sk) + { + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); +- u32 min_segs; ++ u32 segs, bytes; ++ ++ /* Sort of tcp_tso_autosize() but ignoring ++ * driver provided sk_gso_max_size. ++ */ ++ bytes = min_t(u32, sk->sk_pacing_rate >> 10, ++ GSO_MAX_SIZE - 1 - MAX_TCP_HEADER); ++ segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); + +- min_segs = sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; +- bbr->tso_segs_goal = min(tcp_tso_autosize(sk, tp->mss_cache, min_segs), +- 0x7FU); ++ bbr->tso_segs_goal = min(segs, 0x7FU); + } + + /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ +@@ -911,7 +914,7 @@ static struct tcp_congestion_ops tcp_bbr + .undo_cwnd = bbr_undo_cwnd, + .cwnd_event = bbr_cwnd_event, + .ssthresh = bbr_ssthresh, +- .tso_segs_goal = bbr_tso_segs_goal, ++ .min_tso_segs = bbr_min_tso_segs, + .get_info = bbr_get_info, + .set_state = bbr_set_state, + }; +--- a/net/ipv4/tcp_output.c ++++ b/net/ipv4/tcp_output.c +@@ -1672,8 +1672,8 @@ static bool tcp_nagle_check(bool partial + /* Return how many segs we'd like on a TSO packet, + * to send one TSO packet per ms + */ +-u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, +- int min_tso_segs) ++static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, ++ int min_tso_segs) + { + u32 bytes, segs; + +@@ -1689,7 +1689,6 @@ u32 tcp_tso_autosize(const struct sock * + + return segs; + } +-EXPORT_SYMBOL(tcp_tso_autosize); + + /* Return the number of segments we want in the skb we are transmitting. + * See if congestion control module wants to decide; otherwise, autosize. +@@ -1697,11 +1696,13 @@ EXPORT_SYMBOL(tcp_tso_autosize); + static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) + { + const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; +- u32 tso_segs = ca_ops->tso_segs_goal ? ca_ops->tso_segs_goal(sk) : 0; ++ u32 min_tso, tso_segs; + +- if (!tso_segs) +- tso_segs = tcp_tso_autosize(sk, mss_now, +- sysctl_tcp_min_tso_segs); ++ min_tso = ca_ops->min_tso_segs ? ++ ca_ops->min_tso_segs(sk) : ++ sysctl_tcp_min_tso_segs; ++ ++ tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); + return min_t(u32, tso_segs, sk->sk_gso_max_segs); + } + diff --git a/target/linux/generic/backport-4.9/605-0005-tcp_bbr-remove-bbr-tso_segs_goal.patch b/target/linux/generic/backport-4.9/605-0005-tcp_bbr-remove-bbr-tso_segs_goal.patch new file mode 100644 index 0000000000..f0a36056bb --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0005-tcp_bbr-remove-bbr-tso_segs_goal.patch @@ -0,0 +1,76 @@ +From 27ab772b56968575bc3217bbd2b208ef9dbfff57 Mon Sep 17 00:00:00 2001 +From: Eric Dumazet +Date: Wed, 28 Feb 2018 14:40:47 -0800 +Subject: [PATCH] BACKPORT: tcp_bbr: remove bbr->tso_segs_goal + +Its value is computed then immediately used, +there is no need to store it. + +Signed-off-by: Eric Dumazet +Acked-by: Neal Cardwell +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 12 ++++-------- + 1 file changed, 4 insertions(+), 8 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -67,10 +67,9 @@ struct bbr { + packet_conservation:1, /* use packet conservation? */ + restore_cwnd:1, /* decided to revert cwnd to old value */ + round_start:1, /* start of packet-timed tx->ack round? */ +- tso_segs_goal:7, /* segments we want in each skb we send */ + idle_restart:1, /* restarting after idle? */ + probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ +- unused:5, ++ unused:12, + lt_is_sampling:1, /* taking long-term ("LT") samples now? */ + lt_rtt_cnt:7, /* round trips in long-term interval */ + lt_use_bw:1; /* use lt_bw as our bw estimate? */ +@@ -237,10 +236,9 @@ static u32 bbr_min_tso_segs(struct sock + return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; + } + +-static void bbr_set_tso_segs_goal(struct sock *sk) ++static u32 bbr_tso_segs_goal(struct sock *sk) + { + struct tcp_sock *tp = tcp_sk(sk); +- struct bbr *bbr = inet_csk_ca(sk); + u32 segs, bytes; + + /* Sort of tcp_tso_autosize() but ignoring +@@ -250,7 +248,7 @@ static void bbr_set_tso_segs_goal(struct + GSO_MAX_SIZE - 1 - MAX_TCP_HEADER); + segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); + +- bbr->tso_segs_goal = min(segs, 0x7FU); ++ return min(segs, 0x7FU); + } + + /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ +@@ -321,7 +319,7 @@ static u32 bbr_target_cwnd(struct sock * + cwnd = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; + + /* Allow enough full-sized skbs in flight to utilize end systems. */ +- cwnd += 3 * bbr->tso_segs_goal; ++ cwnd += 3 * bbr_tso_segs_goal(sk); + + /* Reduce delayed ACKs by rounding up cwnd to the next even number. */ + cwnd = (cwnd + 1) & ~1U; +@@ -802,7 +800,6 @@ static void bbr_main(struct sock *sk, co + + bw = bbr_bw(sk); + bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); +- bbr_set_tso_segs_goal(sk); + bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain); + } + +@@ -812,7 +809,6 @@ static void bbr_init(struct sock *sk) + struct bbr *bbr = inet_csk_ca(sk); + + bbr->prior_cwnd = 0; +- bbr->tso_segs_goal = 0; /* default segs per skb until first ACK */ + bbr->rtt_cnt = 0; + bbr->next_rtt_delivered = 0; + bbr->prev_ca_state = TCP_CA_Open; diff --git a/target/linux/generic/backport-4.9/605-0006-net-tcp_bbr-set-tp-snd_ssthresh-to-BDP-upon-STARTUP-exit.patch b/target/linux/generic/backport-4.9/605-0006-net-tcp_bbr-set-tp-snd_ssthresh-to-BDP-upon-STARTUP-exit.patch new file mode 100644 index 0000000000..c01e721b6c --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0006-net-tcp_bbr-set-tp-snd_ssthresh-to-BDP-upon-STARTUP-exit.patch @@ -0,0 +1,50 @@ +From b1f4eb4e4f9626fccb396f7998841506117a5916 Mon Sep 17 00:00:00 2001 +From: Yousuk Seung +Date: Fri, 16 Mar 2018 10:51:49 -0700 +Subject: [PATCH] BACKPORT: net-tcp_bbr: set tp->snd_ssthresh to BDP upon + STARTUP exit + +Set tp->snd_ssthresh to BDP upon STARTUP exit. This allows us +to check if a BBR flow exited STARTUP and the BDP at the +time of STARTUP exit with SCM_TIMESTAMPING_OPT_STATS. Since BBR does not +use snd_ssthresh this fix has no impact on BBR's behavior. + +Signed-off-by: Yousuk Seung +Signed-off-by: Neal Cardwell +Signed-off-by: Priyaranjan Jha +Signed-off-by: Soheil Hassas Yeganeh +Signed-off-by: Yuchung Cheng +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -704,6 +704,8 @@ static void bbr_check_drain(struct sock + bbr->mode = BBR_DRAIN; /* drain queue we created */ + bbr->pacing_gain = bbr_drain_gain; /* pace slow to drain */ + bbr->cwnd_gain = bbr_high_gain; /* maintain cwnd */ ++ tcp_sk(sk)->snd_ssthresh = ++ bbr_target_cwnd(sk, bbr_max_bw(sk), BBR_UNIT); + } /* fall through to check if in-flight is already small: */ + if (bbr->mode == BBR_DRAIN && + tcp_packets_in_flight(tcp_sk(sk)) <= +@@ -809,6 +811,7 @@ static void bbr_init(struct sock *sk) + struct bbr *bbr = inet_csk_ca(sk); + + bbr->prior_cwnd = 0; ++ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; + bbr->rtt_cnt = 0; + bbr->next_rtt_delivered = 0; + bbr->prev_ca_state = TCP_CA_Open; +@@ -861,7 +864,7 @@ static u32 bbr_undo_cwnd(struct sock *sk + static u32 bbr_ssthresh(struct sock *sk) + { + bbr_save_cwnd(sk); +- return TCP_INFINITE_SSTHRESH; /* BBR does not use ssthresh */ ++ return tcp_sk(sk)->snd_ssthresh; + } + + static size_t bbr_get_info(struct sock *sk, u32 ext, int *attr, diff --git a/target/linux/generic/backport-4.9/605-0007-tcp_bbr-fix-bbr-pacing-rate-for-internal-pacing.patch b/target/linux/generic/backport-4.9/605-0007-tcp_bbr-fix-bbr-pacing-rate-for-internal-pacing.patch new file mode 100644 index 0000000000..202ed2d9aa --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0007-tcp_bbr-fix-bbr-pacing-rate-for-internal-pacing.patch @@ -0,0 +1,88 @@ +From dcbf673adcbf5275e98d31c2833b2a5311a7eb1e Mon Sep 17 00:00:00 2001 +From: Eric Dumazet +Date: Wed, 20 Jun 2018 16:07:35 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: fix bbr pacing rate for internal pacing + +This commit makes BBR use only the MSS (without any headers) to +calculate pacing rates when internal TCP-layer pacing is used. + +This is necessary to achieve the correct pacing behavior in this case, +since tcp_internal_pacing() uses only the payload length to calculate +pacing delays. + +Signed-off-by: Kevin Yang +Signed-off-by: Eric Dumazet +Reviewed-by: Neal Cardwell +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + include/net/tcp.h | 11 +++++++++++ + net/ipv4/tcp_bbr.c | 6 +++++- + net/ipv4/tcp_output.c | 14 -------------- + 3 files changed, 16 insertions(+), 15 deletions(-) + +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -1161,6 +1161,17 @@ static inline bool tcp_is_cwnd_limited(c + return tp->is_cwnd_limited; + } + ++/* BBR congestion control needs pacing. ++ * Same remark for SO_MAX_PACING_RATE. ++ * sch_fq packet scheduler is efficiently handling pacing, ++ * but is not always installed/used. ++ * Return true if TCP stack should pace packets itself. ++ */ ++static inline bool tcp_needs_internal_pacing(const struct sock *sk) ++{ ++ return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; ++} ++ + /* Something is really bad, we could not queue an additional packet, + * because qdisc is full or receiver sent a 0 window. + * We do not want to add fuel to the fire, or abort too early, +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -175,7 +175,11 @@ static u32 bbr_bw(const struct sock *sk) + */ + static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain) + { +- rate *= tcp_mss_to_mtu(sk, tcp_sk(sk)->mss_cache); ++ unsigned int mss = tcp_sk(sk)->mss_cache; ++ ++ if (!tcp_needs_internal_pacing(sk)) ++ mss = tcp_mss_to_mtu(sk, mss); ++ rate *= mss; + rate *= gain; + rate >>= BBR_SCALE; + rate *= USEC_PER_SEC; +--- a/net/ipv4/tcp_output.c ++++ b/net/ipv4/tcp_output.c +@@ -947,17 +947,6 @@ enum hrtimer_restart tcp_pace_kick(struc + return HRTIMER_NORESTART; + } + +-/* BBR congestion control needs pacing. +- * Same remark for SO_MAX_PACING_RATE. +- * sch_fq packet scheduler is efficiently handling pacing, +- * but is not always installed/used. +- * Return true if TCP stack should pace packets itself. +- */ +-static bool tcp_needs_internal_pacing(const struct sock *sk) +-{ +- return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; +-} +- + static void tcp_internal_pacing(struct sock *sk, const struct sk_buff *skb) + { + u64 len_ns; +@@ -969,9 +958,6 @@ static void tcp_internal_pacing(struct s + if (!rate || rate == ~0U) + return; + +- /* Should account for header sizes as sch_fq does, +- * but lets make things simple. +- */ + len_ns = (u64)skb->len * NSEC_PER_SEC; + do_div(len_ns, rate); + hrtimer_start(&tcp_sk(sk)->pacing_timer, diff --git a/target/linux/generic/backport-4.9/605-0008-tcp_bbr-add-bbr_check_probe_rtt_done-helper.patch b/target/linux/generic/backport-4.9/605-0008-tcp_bbr-add-bbr_check_probe_rtt_done-helper.patch new file mode 100644 index 0000000000..05728e827f --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0008-tcp_bbr-add-bbr_check_probe_rtt_done-helper.patch @@ -0,0 +1,99 @@ +From 94d2a6a1e040fa0707ea2835d425356b88a673e9 Mon Sep 17 00:00:00 2001 +From: Kevin Yang +Date: Wed, 22 Aug 2018 17:43:14 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: add bbr_check_probe_rtt_done() helper + +This patch add a helper function bbr_check_probe_rtt_done() to + 1. check the condition to see if bbr should exit probe_rtt mode; + 2. process the logic of exiting probe_rtt mode. + +Fixes: 0f8782ea1497 ("tcp_bbr: add BBR congestion control") +Signed-off-by: Kevin Yang +Signed-off-by: Neal Cardwell +Signed-off-by: Yuchung Cheng +Reviewed-by: Soheil Hassas Yeganeh +Signed-off-by: David S. Miller +[kras: Replace tcp_jiffies32 with tcp_time_stamp for 4.9] +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 34 ++++++++++++++++++---------------- + 1 file changed, 18 insertions(+), 16 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -65,11 +65,10 @@ struct bbr { + u32 mode:3, /* current bbr_mode in state machine */ + prev_ca_state:3, /* CA state on previous ACK */ + packet_conservation:1, /* use packet conservation? */ +- restore_cwnd:1, /* decided to revert cwnd to old value */ + round_start:1, /* start of packet-timed tx->ack round? */ + idle_restart:1, /* restarting after idle? */ + probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ +- unused:12, ++ unused:13, + lt_is_sampling:1, /* taking long-term ("LT") samples now? */ + lt_rtt_cnt:7, /* round trips in long-term interval */ + lt_use_bw:1; /* use lt_bw as our bw estimate? */ +@@ -366,17 +365,11 @@ static bool bbr_set_cwnd_to_recover_or_r + cwnd = tcp_packets_in_flight(tp) + acked; + } else if (prev_state >= TCP_CA_Recovery && state < TCP_CA_Recovery) { + /* Exiting loss recovery; restore cwnd saved before recovery. */ +- bbr->restore_cwnd = 1; ++ cwnd = max(cwnd, bbr->prior_cwnd); + bbr->packet_conservation = 0; + } + bbr->prev_ca_state = state; + +- if (bbr->restore_cwnd) { +- /* Restore cwnd after exiting loss recovery or PROBE_RTT. */ +- cwnd = max(cwnd, bbr->prior_cwnd); +- bbr->restore_cwnd = 0; +- } +- + if (bbr->packet_conservation) { + *new_cwnd = max(cwnd, tcp_packets_in_flight(tp) + acked); + return true; /* yes, using packet conservation */ +@@ -717,6 +710,20 @@ static void bbr_check_drain(struct sock + bbr_reset_probe_bw_mode(sk); /* we estimate queue is drained */ + } + ++static void bbr_check_probe_rtt_done(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (!(bbr->probe_rtt_done_stamp && ++ after(tcp_time_stamp, bbr->probe_rtt_done_stamp))) ++ return; ++ ++ bbr->min_rtt_stamp = tcp_time_stamp; /* wait a while until PROBE_RTT */ ++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); ++ bbr_reset_mode(sk); ++} ++ + /* The goal of PROBE_RTT mode is to have BBR flows cooperatively and + * periodically drain the bottleneck queue, to converge to measure the true + * min_rtt (unloaded propagation delay). This allows the flows to keep queues +@@ -775,12 +782,8 @@ static void bbr_update_min_rtt(struct so + } else if (bbr->probe_rtt_done_stamp) { + if (bbr->round_start) + bbr->probe_rtt_round_done = 1; +- if (bbr->probe_rtt_round_done && +- after(tcp_time_stamp, bbr->probe_rtt_done_stamp)) { +- bbr->min_rtt_stamp = tcp_time_stamp; +- bbr->restore_cwnd = 1; /* snap to prior_cwnd */ +- bbr_reset_mode(sk); +- } ++ if (bbr->probe_rtt_round_done) ++ bbr_check_probe_rtt_done(sk); + } + } + /* Restart after idle ends only once we process a new S/ACK for data */ +@@ -831,7 +834,6 @@ static void bbr_init(struct sock *sk) + bbr->has_seen_rtt = 0; + bbr_init_pacing_rate_from_rtt(sk); + +- bbr->restore_cwnd = 0; + bbr->round_start = 0; + bbr->idle_restart = 0; + bbr->full_bw_reached = 0; diff --git a/target/linux/generic/backport-4.9/605-0009-tcp_bbr-in-restart-from-idle-see-if-we-should-exit.patch b/target/linux/generic/backport-4.9/605-0009-tcp_bbr-in-restart-from-idle-see-if-we-should-exit.patch new file mode 100644 index 0000000000..500f2eee7e --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0009-tcp_bbr-in-restart-from-idle-see-if-we-should-exit.patch @@ -0,0 +1,43 @@ +From 43ac328b26f344c6954a4cf857e9a5a382ec92e6 Mon Sep 17 00:00:00 2001 +From: Kevin Yang +Date: Wed, 22 Aug 2018 17:43:15 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: in restart from idle, see if we should + exit PROBE_RTT + +This patch fix the case where BBR does not exit PROBE_RTT mode when +it restarts from idle. When BBR restarts from idle and if BBR is in +PROBE_RTT mode, BBR should check if it's time to exit PROBE_RTT. If +yes, then BBR should exit PROBE_RTT mode and restore the cwnd to its +full value. + +Fixes: 0f8782ea1497 ("tcp_bbr: add BBR congestion control") +Signed-off-by: Kevin Yang +Signed-off-by: Neal Cardwell +Reviewed-by: Yuchung Cheng +Reviewed-by: Soheil Hassas Yeganeh +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 4 ++++ + 1 file changed, 4 insertions(+) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -144,6 +144,8 @@ static const u32 bbr_lt_bw_diff = 4000 / + /* If we estimate we're policed, use lt_bw for this many round trips: */ + static const u32 bbr_lt_bw_max_rtts = 48; + ++static void bbr_check_probe_rtt_done(struct sock *sk); ++ + /* Do we estimate that STARTUP filled the pipe? */ + static bool bbr_full_bw_reached(const struct sock *sk) + { +@@ -278,6 +280,8 @@ static void bbr_cwnd_event(struct sock * + */ + if (bbr->mode == BBR_PROBE_BW) + bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); ++ else if (bbr->mode == BBR_PROBE_RTT) ++ bbr_check_probe_rtt_done(sk); + } + } + diff --git a/target/linux/generic/backport-4.9/605-0010-tcp_bbr-apply-PROBE_RTT-cwnd-cap-even-if-acked-0.patch b/target/linux/generic/backport-4.9/605-0010-tcp_bbr-apply-PROBE_RTT-cwnd-cap-even-if-acked-0.patch new file mode 100644 index 0000000000..b3a0a0e7a5 --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0010-tcp_bbr-apply-PROBE_RTT-cwnd-cap-even-if-acked-0.patch @@ -0,0 +1,41 @@ +From 3638356eff0cedaae1ad779c0920662ab068e312 Mon Sep 17 00:00:00 2001 +From: Kevin Yang +Date: Wed, 22 Aug 2018 17:43:16 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: apply PROBE_RTT cwnd cap even if acked==0 + +This commit fixes a corner case where TCP BBR would enter PROBE_RTT +mode but not reduce its cwnd. If a TCP receiver ACKed less than one +full segment, the number of delivered/acked packets was 0, so that +bbr_set_cwnd() would short-circuit and exit early, without cutting +cwnd to the value we want for PROBE_RTT. + +The fix is to instead make sure that even when 0 full packets are +ACKed, we do apply all the appropriate caps, including the cap that +applies in PROBE_RTT mode. + +Fixes: 0f8782ea1497 ("tcp_bbr: add BBR congestion control") +Signed-off-by: Kevin Yang +Signed-off-by: Neal Cardwell +Reviewed-by: Yuchung Cheng +Reviewed-by: Soheil Hassas Yeganeh +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -390,10 +390,10 @@ static void bbr_set_cwnd(struct sock *sk + { + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); +- u32 cwnd = 0, target_cwnd = 0; ++ u32 cwnd = tp->snd_cwnd, target_cwnd = 0; + + if (!acked) +- return; ++ goto done; /* no packet fully ACKed; just apply caps */ + + if (bbr_set_cwnd_to_recover_or_restore(sk, rs, acked, &cwnd)) + goto done; diff --git a/target/linux/generic/backport-4.9/605-0011-tcp_bbr-centralize-code-to-set-gains.patch b/target/linux/generic/backport-4.9/605-0011-tcp_bbr-centralize-code-to-set-gains.patch new file mode 100644 index 0000000000..833c3d7c32 --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0011-tcp_bbr-centralize-code-to-set-gains.patch @@ -0,0 +1,112 @@ +From 460cd4eb19924d12123c5002c203a87c886c5726 Mon Sep 17 00:00:00 2001 +From: Neal Cardwell +Date: Tue, 16 Oct 2018 20:16:45 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: centralize code to set gains + +Centralize the code that sets gains used for computing cwnd and pacing +rate. This simplifies the code and makes it easier to change the state +machine or (in the future) dynamically change the gain values and +ensure that the correct gain values are always used. + +Signed-off-by: Neal Cardwell +Signed-off-by: Yuchung Cheng +Signed-off-by: Soheil Hassas Yeganeh +Signed-off-by: Priyaranjan Jha +Signed-off-by: Eric Dumazet +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 40 ++++++++++++++++++++++++++++++---------- + 1 file changed, 30 insertions(+), 10 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -457,8 +457,6 @@ static void bbr_advance_cycle_phase(stru + + bbr->cycle_idx = (bbr->cycle_idx + 1) & (CYCLE_LEN - 1); + bbr->cycle_mstamp = tp->delivered_mstamp; +- bbr->pacing_gain = bbr->lt_use_bw ? BBR_UNIT : +- bbr_pacing_gain[bbr->cycle_idx]; + } + + /* Gain cycling: cycle pacing gain to converge to fair share of available bw. */ +@@ -476,8 +474,6 @@ static void bbr_reset_startup_mode(struc + struct bbr *bbr = inet_csk_ca(sk); + + bbr->mode = BBR_STARTUP; +- bbr->pacing_gain = bbr_high_gain; +- bbr->cwnd_gain = bbr_high_gain; + } + + static void bbr_reset_probe_bw_mode(struct sock *sk) +@@ -485,8 +481,6 @@ static void bbr_reset_probe_bw_mode(stru + struct bbr *bbr = inet_csk_ca(sk); + + bbr->mode = BBR_PROBE_BW; +- bbr->pacing_gain = BBR_UNIT; +- bbr->cwnd_gain = bbr_cwnd_gain; + bbr->cycle_idx = CYCLE_LEN - 1 - prandom_u32_max(bbr_cycle_rand); + bbr_advance_cycle_phase(sk); /* flip to next phase of gain cycle */ + } +@@ -703,8 +697,6 @@ static void bbr_check_drain(struct sock + + if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { + bbr->mode = BBR_DRAIN; /* drain queue we created */ +- bbr->pacing_gain = bbr_drain_gain; /* pace slow to drain */ +- bbr->cwnd_gain = bbr_high_gain; /* maintain cwnd */ + tcp_sk(sk)->snd_ssthresh = + bbr_target_cwnd(sk, bbr_max_bw(sk), BBR_UNIT); + } /* fall through to check if in-flight is already small: */ +@@ -766,8 +758,6 @@ static void bbr_update_min_rtt(struct so + if (bbr_probe_rtt_mode_ms > 0 && filter_expired && + !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { + bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ +- bbr->pacing_gain = BBR_UNIT; +- bbr->cwnd_gain = BBR_UNIT; + bbr_save_cwnd(sk); /* note cwnd so we can restore it */ + bbr->probe_rtt_done_stamp = 0; + } +@@ -795,6 +785,35 @@ static void bbr_update_min_rtt(struct so + bbr->idle_restart = 0; + } + ++static void bbr_update_gains(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ switch (bbr->mode) { ++ case BBR_STARTUP: ++ bbr->pacing_gain = bbr_high_gain; ++ bbr->cwnd_gain = bbr_high_gain; ++ break; ++ case BBR_DRAIN: ++ bbr->pacing_gain = bbr_drain_gain; /* slow, to drain */ ++ bbr->cwnd_gain = bbr_high_gain; /* keep cwnd */ ++ break; ++ case BBR_PROBE_BW: ++ bbr->pacing_gain = (bbr->lt_use_bw ? ++ BBR_UNIT : ++ bbr_pacing_gain[bbr->cycle_idx]); ++ bbr->cwnd_gain = bbr_cwnd_gain; ++ break; ++ case BBR_PROBE_RTT: ++ bbr->pacing_gain = BBR_UNIT; ++ bbr->cwnd_gain = BBR_UNIT; ++ break; ++ default: ++ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); ++ break; ++ } ++} ++ + static void bbr_update_model(struct sock *sk, const struct rate_sample *rs) + { + bbr_update_bw(sk, rs); +@@ -802,6 +821,7 @@ static void bbr_update_model(struct sock + bbr_check_full_bw_reached(sk, rs); + bbr_check_drain(sk, rs); + bbr_update_min_rtt(sk, rs); ++ bbr_update_gains(sk); + } + + static void bbr_main(struct sock *sk, const struct rate_sample *rs) diff --git a/target/linux/generic/backport-4.9/605-0012-tcp_bbr-refactor-bbr_target_cwnd-for-general-inflight-pro.patch b/target/linux/generic/backport-4.9/605-0012-tcp_bbr-refactor-bbr_target_cwnd-for-general-inflight-pro.patch new file mode 100644 index 0000000000..f7486a592b --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0012-tcp_bbr-refactor-bbr_target_cwnd-for-general-inflight-pro.patch @@ -0,0 +1,144 @@ +From 6a69f3fe4b25ddd777369b7c836fffadb4f387ce Mon Sep 17 00:00:00 2001 +From: Priyaranjan Jha +Date: Wed, 23 Jan 2019 12:04:53 -0800 +Subject: [PATCH] BACKPORT: tcp_bbr: refactor bbr_target_cwnd() for general + inflight provisioning + +Because bbr_target_cwnd() is really a general-purpose BBR helper for +computing some volume of inflight data as a function of the estimated +BDP, refactor it into following helper functions: +- bbr_bdp() +- bbr_quantization_budget() +- bbr_inflight() + +Signed-off-by: Priyaranjan Jha +Signed-off-by: Neal Cardwell +Signed-off-by: Yuchung Cheng +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 60 ++++++++++++++++++++++++++++++---------------- + 1 file changed, 39 insertions(+), 21 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -285,30 +285,19 @@ static void bbr_cwnd_event(struct sock * + } + } + +-/* Find target cwnd. Right-size the cwnd based on min RTT and the +- * estimated bottleneck bandwidth: ++/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: + * +- * cwnd = bw * min_rtt * gain = BDP * gain ++ * bdp = bw * min_rtt * gain + * + * The key factor, gain, controls the amount of queue. While a small gain + * builds a smaller queue, it becomes more vulnerable to noise in RTT + * measurements (e.g., delayed ACKs or other ACK compression effects). This + * noise may cause BBR to under-estimate the rate. +- * +- * To achieve full performance in high-speed paths, we budget enough cwnd to +- * fit full-sized skbs in-flight on both end hosts to fully utilize the path: +- * - one skb in sending host Qdisc, +- * - one skb in sending host TSO/GSO engine +- * - one skb being received by receiver host LRO/GRO/delayed-ACK engine +- * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because +- * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, +- * which allows 2 outstanding 2-packet sequences, to try to keep pipe +- * full even with ACK-every-other-packet delayed ACKs. + */ +-static u32 bbr_target_cwnd(struct sock *sk, u32 bw, int gain) ++static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) + { + struct bbr *bbr = inet_csk_ca(sk); +- u32 cwnd; ++ u32 bdp; + u64 w; + + /* If we've never had a valid RTT sample, cap cwnd at the initial +@@ -323,7 +312,24 @@ static u32 bbr_target_cwnd(struct sock * + w = (u64)bw * bbr->min_rtt_us; + + /* Apply a gain to the given value, then remove the BW_SCALE shift. */ +- cwnd = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; ++ bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; ++ ++ return bdp; ++} ++ ++/* To achieve full performance in high-speed paths, we budget enough cwnd to ++ * fit full-sized skbs in-flight on both end hosts to fully utilize the path: ++ * - one skb in sending host Qdisc, ++ * - one skb in sending host TSO/GSO engine ++ * - one skb being received by receiver host LRO/GRO/delayed-ACK engine ++ * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because ++ * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, ++ * which allows 2 outstanding 2-packet sequences, to try to keep pipe ++ * full even with ACK-every-other-packet delayed ACKs. ++ */ ++static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd, int gain) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); + + /* Allow enough full-sized skbs in flight to utilize end systems. */ + cwnd += 3 * bbr_tso_segs_goal(sk); +@@ -338,6 +344,17 @@ static u32 bbr_target_cwnd(struct sock * + return cwnd; + } + ++/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ ++static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) ++{ ++ u32 inflight; ++ ++ inflight = bbr_bdp(sk, bw, gain); ++ inflight = bbr_quantization_budget(sk, inflight, gain); ++ ++ return inflight; ++} ++ + /* An optimization in BBR to reduce losses: On the first round of recovery, we + * follow the packet conservation principle: send P packets per P packets acked. + * After that, we slow-start and send at most 2*P packets per P packets acked. +@@ -399,7 +416,8 @@ static void bbr_set_cwnd(struct sock *sk + goto done; + + /* If we're below target cwnd, slow start cwnd toward target cwnd. */ +- target_cwnd = bbr_target_cwnd(sk, bw, gain); ++ target_cwnd = bbr_bdp(sk, bw, gain); ++ target_cwnd = bbr_quantization_budget(sk, target_cwnd, gain); + if (bbr_full_bw_reached(sk)) /* only cut cwnd if we filled the pipe */ + cwnd = min(cwnd + acked, target_cwnd); + else if (cwnd < target_cwnd || tp->delivered < TCP_INIT_CWND) +@@ -440,14 +458,14 @@ static bool bbr_is_next_cycle_phase(stru + if (bbr->pacing_gain > BBR_UNIT) + return is_full_length && + (rs->losses || /* perhaps pacing_gain*BDP won't fit */ +- inflight >= bbr_target_cwnd(sk, bw, bbr->pacing_gain)); ++ inflight >= bbr_inflight(sk, bw, bbr->pacing_gain)); + + /* A pacing_gain < 1.0 tries to drain extra queue we added if bw + * probing didn't find more bw. If inflight falls to match BDP then we + * estimate queue is drained; persisting would underutilize the pipe. + */ + return is_full_length || +- inflight <= bbr_target_cwnd(sk, bw, BBR_UNIT); ++ inflight <= bbr_inflight(sk, bw, BBR_UNIT); + } + + static void bbr_advance_cycle_phase(struct sock *sk) +@@ -698,11 +716,11 @@ static void bbr_check_drain(struct sock + if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { + bbr->mode = BBR_DRAIN; /* drain queue we created */ + tcp_sk(sk)->snd_ssthresh = +- bbr_target_cwnd(sk, bbr_max_bw(sk), BBR_UNIT); ++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + } /* fall through to check if in-flight is already small: */ + if (bbr->mode == BBR_DRAIN && + tcp_packets_in_flight(tcp_sk(sk)) <= +- bbr_target_cwnd(sk, bbr_max_bw(sk), BBR_UNIT)) ++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) + bbr_reset_probe_bw_mode(sk); /* we estimate queue is drained */ + } + diff --git a/target/linux/generic/backport-4.9/605-0013-tcp_bbr-clarify-that-bbr_bdp-rounds-up-in-comments.patch b/target/linux/generic/backport-4.9/605-0013-tcp_bbr-clarify-that-bbr_bdp-rounds-up-in-comments.patch new file mode 100644 index 0000000000..4d9c501e9f --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0013-tcp_bbr-clarify-that-bbr_bdp-rounds-up-in-comments.patch @@ -0,0 +1,41 @@ +From 697ee116a6411b47c42d4b2facb9fbffad3f1abd Mon Sep 17 00:00:00 2001 +From: Luke Hsiao +Date: Thu, 29 Aug 2019 10:02:44 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: clarify that bbr_bdp() rounds up in + comments + +This explicitly clarifies that bbr_bdp() returns the rounded-up value of +the bandwidth-delay product and why in the comments. + +Signed-off-by: Luke Hsiao +Acked-by: Soheil Hassas Yeganeh +Acked-by: Neal Cardwell +Acked-by: Priyaranjan Jha +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -287,7 +287,7 @@ static void bbr_cwnd_event(struct sock * + + /* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: + * +- * bdp = bw * min_rtt * gain ++ * bdp = ceil(bw * min_rtt * gain) + * + * The key factor, gain, controls the amount of queue. While a small gain + * builds a smaller queue, it becomes more vulnerable to noise in RTT +@@ -311,7 +311,9 @@ static u32 bbr_bdp(struct sock *sk, u32 + + w = (u64)bw * bbr->min_rtt_us; + +- /* Apply a gain to the given value, then remove the BW_SCALE shift. */ ++ /* Apply a gain to the given value, remove the BW_SCALE shift, and ++ * round the value up to avoid a negative feedback loop. ++ */ + bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; + + return bdp; diff --git a/target/linux/generic/backport-4.9/605-0014-tcp_bbr-fix-quantization-code-to-not-raise-cwnd-if-not-pr.patch b/target/linux/generic/backport-4.9/605-0014-tcp_bbr-fix-quantization-code-to-not-raise-cwnd-if-not-pr.patch new file mode 100644 index 0000000000..49ff9193be --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0014-tcp_bbr-fix-quantization-code-to-not-raise-cwnd-if-not-pr.patch @@ -0,0 +1,66 @@ +From 6c283578cb270c599c6fc6a6e4e332edf9f32425 Mon Sep 17 00:00:00 2001 +From: "Kevin(Yudong) Yang" +Date: Thu, 26 Sep 2019 10:30:05 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: fix quantization code to not raise cwnd if + not probing bandwidth + +There was a bug in the previous logic that attempted to ensure gain cycling +gets inflight above BDP even for small BDPs. This code correctly raised and +lowered target inflight values during the gain cycle. And this code +correctly ensured that cwnd was raised when probing bandwidth. However, it +did not correspondingly ensure that cwnd was *not* raised in this way when +*not* probing for bandwidth. The result was that small-BDP flows that were +always cwnd-bound could go for many cycles with a fixed cwnd, and not probe +or yield bandwidth at all. This meant that multiple small-BDP flows could +fail to converge in their bandwidth allocations. + +Fixes: 3c346b233c68 ("tcp_bbr: fix bw probing to raise in-flight data for very small BDPs") +Signed-off-by: Kevin(Yudong) Yang +Acked-by: Neal Cardwell +Acked-by: Yuchung Cheng +Acked-by: Soheil Hassas Yeganeh +Acked-by: Priyaranjan Jha +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -329,7 +329,7 @@ static u32 bbr_bdp(struct sock *sk, u32 + * which allows 2 outstanding 2-packet sequences, to try to keep pipe + * full even with ACK-every-other-packet delayed ACKs. + */ +-static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd, int gain) ++static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) + { + struct bbr *bbr = inet_csk_ca(sk); + +@@ -340,7 +340,7 @@ static u32 bbr_quantization_budget(struc + cwnd = (cwnd + 1) & ~1U; + + /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ +- if (bbr->mode == BBR_PROBE_BW && gain > BBR_UNIT) ++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == 0) + cwnd += 2; + + return cwnd; +@@ -352,7 +352,7 @@ static u32 bbr_inflight(struct sock *sk, + u32 inflight; + + inflight = bbr_bdp(sk, bw, gain); +- inflight = bbr_quantization_budget(sk, inflight, gain); ++ inflight = bbr_quantization_budget(sk, inflight); + + return inflight; + } +@@ -419,7 +419,7 @@ static void bbr_set_cwnd(struct sock *sk + + /* If we're below target cwnd, slow start cwnd toward target cwnd. */ + target_cwnd = bbr_bdp(sk, bw, gain); +- target_cwnd = bbr_quantization_budget(sk, target_cwnd, gain); ++ target_cwnd = bbr_quantization_budget(sk, target_cwnd); + if (bbr_full_bw_reached(sk)) /* only cut cwnd if we filled the pipe */ + cwnd = min(cwnd + acked, target_cwnd); + else if (cwnd < target_cwnd || tp->delivered < TCP_INIT_CWND)