From b1e3f877a0c185d1f202b731917c635b6a2b3f4b Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sun, 7 Mar 2021 14:20:54 +0800 Subject: [PATCH 1/6] kernel: 4.9: backport BBR fixes Pulled from: https://github.com/Bikram557/android_kernel_xiaomi_santoni_msm4.9/tree/eleven Signed-off-by: Tianling Shen --- ...p-internal-implementation-for-pacing.patch | 340 ++++++++++++++++++ ...-RTT-overestimation-from-delayed-ACK.patch | 80 +++++ ...at-by-skipping-RTT-from-delayed-ACK-.patch | 60 ++++ ...r-better-deal-with-suboptimal-GSO-II.patch | 140 ++++++++ ...005-tcp_bbr-remove-bbr-tso_segs_goal.patch | 76 ++++ ...nd_ssthresh-to-BDP-upon-STARTUP-exit.patch | 50 +++ ...-bbr-pacing-rate-for-internal-pacing.patch | 88 +++++ ...-add-bbr_check_probe_rtt_done-helper.patch | 99 +++++ ...tart-from-idle-see-if-we-should-exit.patch | 43 +++ ...y-PROBE_RTT-cwnd-cap-even-if-acked-0.patch | 41 +++ ...tcp_bbr-centralize-code-to-set-gains.patch | 112 ++++++ ...target_cwnd-for-general-inflight-pro.patch | 144 ++++++++ ...y-that-bbr_bdp-rounds-up-in-comments.patch | 41 +++ ...ion-code-to-not-raise-cwnd-if-not-pr.patch | 66 ++++ 14 files changed, 1380 insertions(+) create mode 100644 target/linux/generic/backport-4.9/605-0001-tcp-internal-implementation-for-pacing.patch create mode 100644 target/linux/generic/backport-4.9/605-0002-tcp-avoid-min-RTT-overestimation-from-delayed-ACK.patch create mode 100644 target/linux/generic/backport-4.9/605-0003-tcp-avoid-min-RTT-bloat-by-skipping-RTT-from-delayed-ACK-.patch create mode 100644 target/linux/generic/backport-4.9/605-0004-tcp_bbr-better-deal-with-suboptimal-GSO-II.patch create mode 100644 target/linux/generic/backport-4.9/605-0005-tcp_bbr-remove-bbr-tso_segs_goal.patch create mode 100644 target/linux/generic/backport-4.9/605-0006-net-tcp_bbr-set-tp-snd_ssthresh-to-BDP-upon-STARTUP-exit.patch create mode 100644 target/linux/generic/backport-4.9/605-0007-tcp_bbr-fix-bbr-pacing-rate-for-internal-pacing.patch create mode 100644 target/linux/generic/backport-4.9/605-0008-tcp_bbr-add-bbr_check_probe_rtt_done-helper.patch create mode 100644 target/linux/generic/backport-4.9/605-0009-tcp_bbr-in-restart-from-idle-see-if-we-should-exit.patch create mode 100644 target/linux/generic/backport-4.9/605-0010-tcp_bbr-apply-PROBE_RTT-cwnd-cap-even-if-acked-0.patch create mode 100644 target/linux/generic/backport-4.9/605-0011-tcp_bbr-centralize-code-to-set-gains.patch create mode 100644 target/linux/generic/backport-4.9/605-0012-tcp_bbr-refactor-bbr_target_cwnd-for-general-inflight-pro.patch create mode 100644 target/linux/generic/backport-4.9/605-0013-tcp_bbr-clarify-that-bbr_bdp-rounds-up-in-comments.patch create mode 100644 target/linux/generic/backport-4.9/605-0014-tcp_bbr-fix-quantization-code-to-not-raise-cwnd-if-not-pr.patch diff --git a/target/linux/generic/backport-4.9/605-0001-tcp-internal-implementation-for-pacing.patch b/target/linux/generic/backport-4.9/605-0001-tcp-internal-implementation-for-pacing.patch new file mode 100644 index 0000000000..1632631561 --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0001-tcp-internal-implementation-for-pacing.patch @@ -0,0 +1,340 @@ +From 4f7737d75080d6d28b8a90c7499d85b6d4b4dd49 Mon Sep 17 00:00:00 2001 +From: Eric Dumazet +Date: Tue, 16 May 2017 04:24:36 -0700 +Subject: [PATCH] BACKPORT: tcp: internal implementation for pacing + +BBR congestion control depends on pacing, and pacing is +currently handled by sch_fq packet scheduler for performance reasons, +and also because implemening pacing with FQ was convenient to truly +avoid bursts. + +However there are many cases where this packet scheduler constraint +is not practical. +- Many linux hosts are not focusing on handling thousands of TCP + flows in the most efficient way. +- Some routers use fq_codel or other AQM, but still would like + to use BBR for the few TCP flows they initiate/terminate. + +This patch implements an automatic fallback to internal pacing. + +Pacing is requested either by BBR or use of SO_MAX_PACING_RATE option. + +If sch_fq happens to be in the egress path, pacing is delegated to +the qdisc, otherwise pacing is done by TCP itself. + +One advantage of pacing from TCP stack is to get more precise rtt +estimations, and less work done from TX completion, since TCP Small +queue limits are not generally hit. Setups with single TX queue but +many cpus might even benefit from this. + +Note that unlike sch_fq, we do not take into account header sizes. +Taking care of these headers would add additional complexity for +no practical differences in behavior. + +Some performance numbers using 800 TCP_STREAM flows rate limited to +~48 Mbit per second on 40Gbit NIC. + +If MQ+pfifo_fast is used on the NIC : + +$ sar -n DEV 1 5 | grep eth +14:48:44 eth0 725743.00 2932134.00 46776.76 4335184.68 0.00 0.00 1.00 +14:48:45 eth0 725349.00 2932112.00 46751.86 4335158.90 0.00 0.00 0.00 +14:48:46 eth0 725101.00 2931153.00 46735.07 4333748.63 0.00 0.00 0.00 +14:48:47 eth0 725099.00 2931161.00 46735.11 4333760.44 0.00 0.00 1.00 +14:48:48 eth0 725160.00 2931731.00 46738.88 4334606.07 0.00 0.00 0.00 +Average: eth0 725290.40 2931658.20 46747.54 4334491.74 0.00 0.00 0.40 +$ vmstat 1 5 +procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu----- + r b swpd free buff cache si so bi bo in cs us sy id wa st + 4 0 0 259825920 45644 2708324 0 0 21 2 247 98 0 0 100 0 0 + 4 0 0 259823744 45644 2708356 0 0 0 0 2400825 159843 0 19 81 0 0 + 0 0 0 259824208 45644 2708072 0 0 0 0 2407351 159929 0 19 81 0 0 + 1 0 0 259824592 45644 2708128 0 0 0 0 2405183 160386 0 19 80 0 0 + 1 0 0 259824272 45644 2707868 0 0 0 32 2396361 158037 0 19 81 0 0 + +Now use MQ+FQ : + +lpaa23:~# echo fq >/proc/sys/net/core/default_qdisc +lpaa23:~# tc qdisc replace dev eth0 root mq + +$ sar -n DEV 1 5 | grep eth +14:49:57 eth0 678614.00 2727930.00 43739.13 4033279.14 0.00 0.00 0.00 +14:49:58 eth0 677620.00 2723971.00 43674.69 4027429.62 0.00 0.00 1.00 +14:49:59 eth0 676396.00 2719050.00 43596.83 4020125.02 0.00 0.00 0.00 +14:50:00 eth0 675197.00 2714173.00 43518.62 4012938.90 0.00 0.00 1.00 +14:50:01 eth0 676388.00 2719063.00 43595.47 4020171.64 0.00 0.00 0.00 +Average: eth0 676843.00 2720837.40 43624.95 4022788.86 0.00 0.00 0.40 +$ vmstat 1 5 +procs -----------memory---------- ---swap-- -----io---- -system-- ------cpu----- + r b swpd free buff cache si so bi bo in cs us sy id wa st + 2 0 0 259832240 46008 2710912 0 0 21 2 223 192 0 1 99 0 0 + 1 0 0 259832896 46008 2710744 0 0 0 0 1702206 198078 0 17 82 0 0 + 0 0 0 259830272 46008 2710596 0 0 0 0 1696340 197756 1 17 83 0 0 + 4 0 0 259829168 46024 2710584 0 0 16 0 1688472 197158 1 17 82 0 0 + 3 0 0 259830224 46024 2710408 0 0 0 0 1692450 197212 0 18 82 0 0 + +As expected, number of interrupts per second is very different. + +Signed-off-by: Eric Dumazet +Acked-by: Soheil Hassas Yeganeh +Cc: Neal Cardwell +Cc: Yuchung Cheng +Cc: Van Jacobson +Cc: Jerry Chu +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + include/linux/tcp.h | 2 ++ + include/net/sock.h | 8 +++++ + include/net/tcp.h | 3 ++ + net/core/sock.c | 4 +++ + net/ipv4/tcp_bbr.c | 9 ++--- + net/ipv4/tcp_output.c | 80 +++++++++++++++++++++++++++++++++++++++++++ + net/ipv4/tcp_timer.c | 3 ++ + net/sched/sch_fq.c | 8 +++++ + 8 files changed, 113 insertions(+), 4 deletions(-) + +--- a/include/linux/tcp.h ++++ b/include/linux/tcp.h +@@ -290,6 +290,8 @@ struct tcp_sock { + u32 sacked_out; /* SACK'd packets */ + u32 fackets_out; /* FACK'd packets */ + ++ struct hrtimer pacing_timer; ++ + /* from STCP, retrans queue hinting */ + struct sk_buff* lost_skb_hint; + struct sk_buff *retransmit_skb_hint; +--- a/include/net/sock.h ++++ b/include/net/sock.h +@@ -250,6 +250,7 @@ struct sock_common { + * @sk_ll_usec: usecs to busypoll when there is no data + * @sk_allocation: allocation mode + * @sk_pacing_rate: Pacing rate (if supported by transport/packet scheduler) ++ * @sk_pacing_status: Pacing status (requested, handled by sch_fq) + * @sk_max_pacing_rate: Maximum pacing rate (%SO_MAX_PACING_RATE) + * @sk_sndbuf: size of send buffer in bytes + * @sk_padding: unused element for alignment +@@ -394,6 +395,7 @@ struct sock { + struct sk_buff_head sk_write_queue; + __s32 sk_peek_off; + int sk_write_pending; ++ u32 sk_pacing_status; /* see enum sk_pacing */ + long sk_sndtimeo; + struct timer_list sk_timer; + __u32 sk_priority; +@@ -460,6 +462,12 @@ struct sock { + struct rcu_head sk_rcu; + }; + ++enum sk_pacing { ++ SK_PACING_NONE = 0, ++ SK_PACING_NEEDED = 1, ++ SK_PACING_FQ = 2, ++}; ++ + #define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data))) + + #define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk))) +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -584,6 +584,7 @@ void tcp_fin(struct sock *sk); + void tcp_init_xmit_timers(struct sock *); + static inline void tcp_clear_xmit_timers(struct sock *sk) + { ++ hrtimer_cancel(&tcp_sk(sk)->pacing_timer); + inet_csk_clear_xmit_timers(sk); + } + +@@ -1970,4 +1971,6 @@ static inline void tcp_listendrop(const + __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); + } + ++enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer); ++ + #endif /* _TCP_H */ +--- a/net/core/sock.c ++++ b/net/core/sock.c +@@ -989,6 +989,10 @@ set_rcvbuf: + #endif + + case SO_MAX_PACING_RATE: ++ if (val != ~0U) ++ cmpxchg(&sk->sk_pacing_status, ++ SK_PACING_NONE, ++ SK_PACING_NEEDED); + sk->sk_max_pacing_rate = val; + sk->sk_pacing_rate = min(sk->sk_pacing_rate, + sk->sk_max_pacing_rate); +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -22,10 +22,9 @@ + * There is a public e-mail list for discussing BBR development and testing: + * https://groups.google.com/forum/#!forum/bbr-dev + * +- * NOTE: BBR *must* be used with the fq qdisc ("man tc-fq") with pacing enabled, +- * since pacing is integral to the BBR design and implementation. +- * BBR without pacing would not function properly, and may incur unnecessary +- * high packet loss rates. ++ * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled, ++ * otherwise TCP stack falls back to an internal pacing using one high ++ * resolution timer per TCP socket and may use more resources. + */ + #include + #include +@@ -835,6 +834,8 @@ static void bbr_init(struct sock *sk) + bbr->cycle_idx = 0; + bbr_reset_lt_bw_sampling(sk); + bbr_reset_startup_mode(sk); ++ ++ cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED); + } + + static u32 bbr_sndbuf_expand(struct sock *sk) +--- a/net/ipv4/tcp_output.c ++++ b/net/ipv4/tcp_output.c +@@ -913,6 +913,72 @@ out: + sk_free(sk); + } + ++/* Note: Called under hard irq. ++ * We can not call TCP stack right away. ++ */ ++enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer) ++{ ++ struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer); ++ struct sock *sk = (struct sock *)tp; ++ unsigned long nval, oval; ++ ++ for (oval = READ_ONCE(sk->sk_tsq_flags);; oval = nval) { ++ struct tsq_tasklet *tsq; ++ bool empty; ++ ++ if (oval & TSQF_QUEUED) ++ break; ++ ++ nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED; ++ nval = cmpxchg(&sk->sk_tsq_flags, oval, nval); ++ if (nval != oval) ++ continue; ++ ++ if (!atomic_inc_not_zero(&sk->sk_wmem_alloc)) ++ break; ++ /* queue this socket to tasklet queue */ ++ tsq = this_cpu_ptr(&tsq_tasklet); ++ empty = list_empty(&tsq->head); ++ list_add(&tp->tsq_node, &tsq->head); ++ if (empty) ++ tasklet_schedule(&tsq->tasklet); ++ break; ++ } ++ return HRTIMER_NORESTART; ++} ++ ++/* BBR congestion control needs pacing. ++ * Same remark for SO_MAX_PACING_RATE. ++ * sch_fq packet scheduler is efficiently handling pacing, ++ * but is not always installed/used. ++ * Return true if TCP stack should pace packets itself. ++ */ ++static bool tcp_needs_internal_pacing(const struct sock *sk) ++{ ++ return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; ++} ++ ++static void tcp_internal_pacing(struct sock *sk, const struct sk_buff *skb) ++{ ++ u64 len_ns; ++ u32 rate; ++ ++ if (!tcp_needs_internal_pacing(sk)) ++ return; ++ rate = sk->sk_pacing_rate; ++ if (!rate || rate == ~0U) ++ return; ++ ++ /* Should account for header sizes as sch_fq does, ++ * but lets make things simple. ++ */ ++ len_ns = (u64)skb->len * NSEC_PER_SEC; ++ do_div(len_ns, rate); ++ hrtimer_start(&tcp_sk(sk)->pacing_timer, ++ ktime_add_ns(ktime_get(), len_ns), ++ HRTIMER_MODE_ABS_PINNED); ++} ++ + /* This routine actually transmits TCP packets queued in by + * tcp_do_sendmsg(). This is used by both the initial + * transmission and possible later retransmissions. +@@ -1034,6 +1100,7 @@ static int __tcp_transmit_skb(struct soc + if (skb->len != tcp_header_size) { + tcp_event_data_sent(tp, sk); + tp->data_segs_out += tcp_skb_pcount(skb); ++ tcp_internal_pacing(sk, skb); + } + + if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq) +@@ -2125,6 +2192,12 @@ static int tcp_mtu_probe(struct sock *sk + return -1; + } + ++static bool tcp_pacing_check(const struct sock *sk) ++{ ++ return tcp_needs_internal_pacing(sk) && ++ hrtimer_active(&tcp_sk(sk)->pacing_timer); ++} ++ + /* TCP Small Queues : + * Control number of packets in qdisc/devices to two packets / or ~1 ms. + * (These limits are doubled for retransmits) +@@ -2208,6 +2281,9 @@ static bool tcp_write_xmit(struct sock * + while ((skb = tcp_send_head(sk))) { + unsigned int limit; + ++ if (tcp_pacing_check(sk)) ++ break; ++ + tso_segs = tcp_init_tso_segs(skb, mss_now); + BUG_ON(!tso_segs); + +@@ -2932,6 +3008,10 @@ void tcp_xmit_retransmit_queue(struct so + + if (skb == tcp_send_head(sk)) + break; ++ ++ if (tcp_pacing_check(sk)) ++ break; ++ + /* we could do better than to assign each time */ + if (!hole) + tp->retransmit_skb_hint = skb; +--- a/net/ipv4/tcp_timer.c ++++ b/net/ipv4/tcp_timer.c +@@ -727,4 +727,7 @@ void tcp_init_xmit_timers(struct sock *s + { + inet_csk_init_xmit_timers(sk, &tcp_write_timer, &tcp_delack_timer, + &tcp_keepalive_timer); ++ hrtimer_init(&tcp_sk(sk)->pacing_timer, CLOCK_MONOTONIC, ++ HRTIMER_MODE_ABS_PINNED); ++ tcp_sk(sk)->pacing_timer.function = tcp_pace_kick; + } +--- a/net/sched/sch_fq.c ++++ b/net/sched/sch_fq.c +@@ -405,9 +405,17 @@ static int fq_enqueue(struct sk_buff *sk + q->stat_tcp_retrans++; + qdisc_qstats_backlog_inc(sch, skb); + if (fq_flow_is_detached(f)) { ++ struct sock *sk = skb->sk; ++ + fq_flow_add_tail(&q->new_flows, f); + if (time_after(jiffies, f->age + q->flow_refill_delay)) + f->credit = max_t(u32, f->credit, q->quantum); ++ if (sk && q->rate_enable) { ++ if (unlikely(smp_load_acquire(&sk->sk_pacing_status) != ++ SK_PACING_FQ)) ++ smp_store_release(&sk->sk_pacing_status, ++ SK_PACING_FQ); ++ } + q->inactive_flows--; + } + diff --git a/target/linux/generic/backport-4.9/605-0002-tcp-avoid-min-RTT-overestimation-from-delayed-ACK.patch b/target/linux/generic/backport-4.9/605-0002-tcp-avoid-min-RTT-overestimation-from-delayed-ACK.patch new file mode 100644 index 0000000000..63a4eb0b6c --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0002-tcp-avoid-min-RTT-overestimation-from-delayed-ACK.patch @@ -0,0 +1,80 @@ +From 549a5e6d301748438255a9c014df6079e5a7dd28 Mon Sep 17 00:00:00 2001 +From: Yuchung Cheng +Date: Wed, 17 Jan 2018 12:11:00 -0800 +Subject: [PATCH] BACKPORT: tcp: avoid min-RTT overestimation from delayed ACKs + +This patch avoids having TCP sender or congestion control +overestimate the min RTT by orders of magnitude. This happens when +all the samples in the windowed filter are one-packet transfer +like small request and health-check like chit-chat, which is farily +common for applications using persistent connections. This patch +tries to conservatively labels and skip RTT samples obtained from +this type of workload. + +Signed-off-by: Yuchung Cheng +Signed-off-by: Soheil Hassas Yeganeh +Acked-by: Neal Cardwell +Acked-by: Eric Dumazet +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_input.c | 23 +++++++++++++++++++++-- + 1 file changed, 21 insertions(+), 2 deletions(-) + +--- a/net/ipv4/tcp_input.c ++++ b/net/ipv4/tcp_input.c +@@ -116,6 +116,7 @@ int sysctl_tcp_invalid_ratelimit __read_ + #define FLAG_SACK_RENEGING 0x2000 /* snd_una advanced to a sacked seq */ + #define FLAG_UPDATE_TS_RECENT 0x4000 /* tcp_replace_ts_recent() */ + #define FLAG_NO_CHALLENGE_ACK 0x8000 /* do not call tcp_send_challenge_ack() */ ++#define FLAG_ACK_MAYBE_DELAYED 0x10000 /* Likely a delayed ACK */ + + #define FLAG_ACKED (FLAG_DATA_ACKED|FLAG_SYN_ACKED) + #define FLAG_NOT_DUP (FLAG_DATA|FLAG_WIN_UPDATE|FLAG_ACKED) +@@ -2972,11 +2973,18 @@ static void tcp_fastretrans_alert(struct + *rexmit = REXMIT_LOST; + } + +-static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us) ++static void tcp_update_rtt_min(struct sock *sk, u32 rtt_us, const int flag) + { + struct tcp_sock *tp = tcp_sk(sk); + u32 wlen = sysctl_tcp_min_rtt_wlen * HZ; + ++ if ((flag & FLAG_ACK_MAYBE_DELAYED) && rtt_us > tcp_min_rtt(tp)) { ++ /* If the remote keeps returning delayed ACKs, eventually ++ * the min filter would pick it up and overestimate the ++ * prop. delay when it expires. Skip suspected delayed ACKs. ++ */ ++ return; ++ } + minmax_running_min(&tp->rtt_min, wlen, tcp_time_stamp, + rtt_us ? : jiffies_to_usecs(1)); + } +@@ -3012,7 +3020,7 @@ static inline bool tcp_ack_update_rtt(st + * always taken together with ACK, SACK, or TS-opts. Any negative + * values will be skipped with the seq_rtt_us < 0 check above. + */ +- tcp_update_rtt_min(sk, ca_rtt_us); ++ tcp_update_rtt_min(sk, ca_rtt_us, flag); + tcp_rtt_estimator(sk, seq_rtt_us); + tcp_set_rto(sk); + +@@ -3247,6 +3255,17 @@ static int tcp_clean_rtx_queue(struct so + if (likely(first_ackt.v64) && !(flag & FLAG_RETRANS_DATA_ACKED)) { + seq_rtt_us = skb_mstamp_us_delta(now, &first_ackt); + ca_rtt_us = skb_mstamp_us_delta(now, &last_ackt); ++ ++ if (pkts_acked == 1 && last_in_flight < tp->mss_cache && ++ last_in_flight && !prior_sacked && fully_acked && ++ sack->rate->prior_delivered + 1 == tp->delivered && ++ !(flag & (FLAG_CA_ALERT | FLAG_SYN_ACKED))) { ++ /* Conservatively mark a delayed ACK. It's typically ++ * from a lone runt packet over the round trip to ++ * a receiver w/o out-of-order or CE events. ++ */ ++ flag |= FLAG_ACK_MAYBE_DELAYED; ++ } + } + if (sack->first_sackt.v64) { + sack_rtt_us = skb_mstamp_us_delta(now, &sack->first_sackt); diff --git a/target/linux/generic/backport-4.9/605-0003-tcp-avoid-min-RTT-bloat-by-skipping-RTT-from-delayed-ACK-.patch b/target/linux/generic/backport-4.9/605-0003-tcp-avoid-min-RTT-bloat-by-skipping-RTT-from-delayed-ACK-.patch new file mode 100644 index 0000000000..7ccb01bead --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0003-tcp-avoid-min-RTT-bloat-by-skipping-RTT-from-delayed-ACK-.patch @@ -0,0 +1,60 @@ +From b2f3b3121c19c43877654079753325c16fe2b811 Mon Sep 17 00:00:00 2001 +From: Yuchung Cheng +Date: Wed, 17 Jan 2018 12:11:01 -0800 +Subject: [PATCH] BACKPORT: tcp: avoid min RTT bloat by skipping RTT from + delayed-ACK in BBR + +A persistent connection may send tiny amount of data (e.g. health-check) +for a long period of time. BBR's windowed min RTT filter may only see +RTT samples from delayed ACKs causing BBR to grossly over-estimate +the path delay depending how much the ACK was delayed at the receiver. + +This patch skips RTT samples that are likely coming from delayed ACKs. Note +that it is possible the sender never obtains a valid measure to set the +min RTT. In this case BBR will continue to set cwnd to initial window +which seems fine because the connection is thin stream. + +Signed-off-by: Yuchung Cheng +Acked-by: Neal Cardwell +Acked-by: Soheil Hassas Yeganeh +Acked-by: Priyaranjan Jha +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + include/net/tcp.h | 1 + + net/ipv4/tcp_bbr.c | 3 ++- + net/ipv4/tcp_input.c | 1 + + 3 files changed, 4 insertions(+), 1 deletion(-) + +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -911,6 +911,7 @@ struct rate_sample { + u32 prior_in_flight; /* in flight before this ACK */ + bool is_app_limited; /* is sample from packet with bubble in pipe? */ + bool is_retrans; /* is sample from retransmission? */ ++ bool is_ack_delayed; /* is this (likely) a delayed ACK? */ + }; + + struct tcp_congestion_ops { +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -739,7 +739,8 @@ static void bbr_update_min_rtt(struct so + filter_expired = after(tcp_time_stamp, + bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ); + if (rs->rtt_us >= 0 && +- (rs->rtt_us < bbr->min_rtt_us || filter_expired)) { ++ (rs->rtt_us < bbr->min_rtt_us || ++ (filter_expired && !rs->is_ack_delayed))) { + bbr->min_rtt_us = rs->rtt_us; + bbr->min_rtt_stamp = tcp_time_stamp; + } +--- a/net/ipv4/tcp_input.c ++++ b/net/ipv4/tcp_input.c +@@ -3779,6 +3779,7 @@ static int tcp_ack(struct sock *sk, cons + tcp_schedule_loss_probe(sk); + delivered = tp->delivered - delivered; /* freshly ACKed or SACKed */ + lost = tp->lost - lost; /* freshly marked lost */ ++ rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED); + tcp_rate_gen(sk, delivered, lost, is_sack_reneg, &now, &rs); + tcp_cong_control(sk, ack, delivered, flag, &rs); + tcp_xmit_recovery(sk, rexmit); diff --git a/target/linux/generic/backport-4.9/605-0004-tcp_bbr-better-deal-with-suboptimal-GSO-II.patch b/target/linux/generic/backport-4.9/605-0004-tcp_bbr-better-deal-with-suboptimal-GSO-II.patch new file mode 100644 index 0000000000..62ed2d7184 --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0004-tcp_bbr-better-deal-with-suboptimal-GSO-II.patch @@ -0,0 +1,140 @@ +From a03defe0b7f9d9378599c986aaf1418ee6cec231 Mon Sep 17 00:00:00 2001 +From: Eric Dumazet +Date: Wed, 28 Feb 2018 14:40:46 -0800 +Subject: [PATCH] BACKPORT: tcp_bbr: better deal with suboptimal GSO (II) + +This is second part of dealing with suboptimal device gso parameters. +In first patch (45fa6615258e "tcp_bbr: better deal with suboptimal GSO") +we dealt with devices having low gso_max_segs + +Some devices lower gso_max_size from 64KB to 16 KB (r8152 is an example) + +In order to probe an optimal cwnd, we want BBR being not sensitive +to whatever GSO constraint a device can have. + +This patch removes tso_segs_goal() CC callback in favor of +min_tso_segs() for CC wanting to override sysctl_tcp_min_tso_segs + +Next patch will remove bbr->tso_segs_goal since it does not have +to be persistent. + +Signed-off-by: Eric Dumazet +Acked-by: Neal Cardwell +Signed-off-by: David S. Miller +[kras: Replace sk_pacing_shift with 10 (which is default value) to + avoid backporting mainline commit 3a9b76fd0db9] +Signed-off-by: Albert I +--- + include/net/tcp.h | 6 ++---- + net/ipv4/tcp_bbr.c | 23 +++++++++++++---------- + net/ipv4/tcp_output.c | 15 ++++++++------- + 3 files changed, 23 insertions(+), 21 deletions(-) + +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -544,8 +544,6 @@ __u32 cookie_v6_init_sequence(const stru + #endif + /* tcp_output.c */ + +-u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, +- int min_tso_segs); + void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss, + int nonagle); + bool tcp_may_send_now(struct sock *sk); +@@ -938,8 +936,8 @@ struct tcp_congestion_ops { + u32 (*undo_cwnd)(struct sock *sk); + /* hook for packet ack accounting (optional) */ + void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample); +- /* suggest number of segments for each skb to transmit (optional) */ +- u32 (*tso_segs_goal)(struct sock *sk); ++ /* override sysctl_tcp_min_tso_segs */ ++ u32 (*min_tso_segs)(struct sock *sk); + /* returns the multiplier used in tcp_sndbuf_expand (optional) */ + u32 (*sndbuf_expand)(struct sock *sk); + /* call when packets are delivered to update cwnd and pacing rate, +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -231,23 +231,26 @@ static void bbr_set_pacing_rate(struct s + sk->sk_pacing_rate = rate; + } + +-/* Return count of segments we want in the skbs we send, or 0 for default. */ +-static u32 bbr_tso_segs_goal(struct sock *sk) ++/* override sysctl_tcp_min_tso_segs */ ++static u32 bbr_min_tso_segs(struct sock *sk) + { +- struct bbr *bbr = inet_csk_ca(sk); +- +- return bbr->tso_segs_goal; ++ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; + } + + static void bbr_set_tso_segs_goal(struct sock *sk) + { + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); +- u32 min_segs; ++ u32 segs, bytes; ++ ++ /* Sort of tcp_tso_autosize() but ignoring ++ * driver provided sk_gso_max_size. ++ */ ++ bytes = min_t(u32, sk->sk_pacing_rate >> 10, ++ GSO_MAX_SIZE - 1 - MAX_TCP_HEADER); ++ segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); + +- min_segs = sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; +- bbr->tso_segs_goal = min(tcp_tso_autosize(sk, tp->mss_cache, min_segs), +- 0x7FU); ++ bbr->tso_segs_goal = min(segs, 0x7FU); + } + + /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ +@@ -911,7 +914,7 @@ static struct tcp_congestion_ops tcp_bbr + .undo_cwnd = bbr_undo_cwnd, + .cwnd_event = bbr_cwnd_event, + .ssthresh = bbr_ssthresh, +- .tso_segs_goal = bbr_tso_segs_goal, ++ .min_tso_segs = bbr_min_tso_segs, + .get_info = bbr_get_info, + .set_state = bbr_set_state, + }; +--- a/net/ipv4/tcp_output.c ++++ b/net/ipv4/tcp_output.c +@@ -1672,8 +1672,8 @@ static bool tcp_nagle_check(bool partial + /* Return how many segs we'd like on a TSO packet, + * to send one TSO packet per ms + */ +-u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, +- int min_tso_segs) ++static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now, ++ int min_tso_segs) + { + u32 bytes, segs; + +@@ -1689,7 +1689,6 @@ u32 tcp_tso_autosize(const struct sock * + + return segs; + } +-EXPORT_SYMBOL(tcp_tso_autosize); + + /* Return the number of segments we want in the skb we are transmitting. + * See if congestion control module wants to decide; otherwise, autosize. +@@ -1697,11 +1696,13 @@ EXPORT_SYMBOL(tcp_tso_autosize); + static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now) + { + const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; +- u32 tso_segs = ca_ops->tso_segs_goal ? ca_ops->tso_segs_goal(sk) : 0; ++ u32 min_tso, tso_segs; + +- if (!tso_segs) +- tso_segs = tcp_tso_autosize(sk, mss_now, +- sysctl_tcp_min_tso_segs); ++ min_tso = ca_ops->min_tso_segs ? ++ ca_ops->min_tso_segs(sk) : ++ sysctl_tcp_min_tso_segs; ++ ++ tso_segs = tcp_tso_autosize(sk, mss_now, min_tso); + return min_t(u32, tso_segs, sk->sk_gso_max_segs); + } + diff --git a/target/linux/generic/backport-4.9/605-0005-tcp_bbr-remove-bbr-tso_segs_goal.patch b/target/linux/generic/backport-4.9/605-0005-tcp_bbr-remove-bbr-tso_segs_goal.patch new file mode 100644 index 0000000000..f0a36056bb --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0005-tcp_bbr-remove-bbr-tso_segs_goal.patch @@ -0,0 +1,76 @@ +From 27ab772b56968575bc3217bbd2b208ef9dbfff57 Mon Sep 17 00:00:00 2001 +From: Eric Dumazet +Date: Wed, 28 Feb 2018 14:40:47 -0800 +Subject: [PATCH] BACKPORT: tcp_bbr: remove bbr->tso_segs_goal + +Its value is computed then immediately used, +there is no need to store it. + +Signed-off-by: Eric Dumazet +Acked-by: Neal Cardwell +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 12 ++++-------- + 1 file changed, 4 insertions(+), 8 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -67,10 +67,9 @@ struct bbr { + packet_conservation:1, /* use packet conservation? */ + restore_cwnd:1, /* decided to revert cwnd to old value */ + round_start:1, /* start of packet-timed tx->ack round? */ +- tso_segs_goal:7, /* segments we want in each skb we send */ + idle_restart:1, /* restarting after idle? */ + probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ +- unused:5, ++ unused:12, + lt_is_sampling:1, /* taking long-term ("LT") samples now? */ + lt_rtt_cnt:7, /* round trips in long-term interval */ + lt_use_bw:1; /* use lt_bw as our bw estimate? */ +@@ -237,10 +236,9 @@ static u32 bbr_min_tso_segs(struct sock + return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2; + } + +-static void bbr_set_tso_segs_goal(struct sock *sk) ++static u32 bbr_tso_segs_goal(struct sock *sk) + { + struct tcp_sock *tp = tcp_sk(sk); +- struct bbr *bbr = inet_csk_ca(sk); + u32 segs, bytes; + + /* Sort of tcp_tso_autosize() but ignoring +@@ -250,7 +248,7 @@ static void bbr_set_tso_segs_goal(struct + GSO_MAX_SIZE - 1 - MAX_TCP_HEADER); + segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk)); + +- bbr->tso_segs_goal = min(segs, 0x7FU); ++ return min(segs, 0x7FU); + } + + /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */ +@@ -321,7 +319,7 @@ static u32 bbr_target_cwnd(struct sock * + cwnd = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; + + /* Allow enough full-sized skbs in flight to utilize end systems. */ +- cwnd += 3 * bbr->tso_segs_goal; ++ cwnd += 3 * bbr_tso_segs_goal(sk); + + /* Reduce delayed ACKs by rounding up cwnd to the next even number. */ + cwnd = (cwnd + 1) & ~1U; +@@ -802,7 +800,6 @@ static void bbr_main(struct sock *sk, co + + bw = bbr_bw(sk); + bbr_set_pacing_rate(sk, bw, bbr->pacing_gain); +- bbr_set_tso_segs_goal(sk); + bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain); + } + +@@ -812,7 +809,6 @@ static void bbr_init(struct sock *sk) + struct bbr *bbr = inet_csk_ca(sk); + + bbr->prior_cwnd = 0; +- bbr->tso_segs_goal = 0; /* default segs per skb until first ACK */ + bbr->rtt_cnt = 0; + bbr->next_rtt_delivered = 0; + bbr->prev_ca_state = TCP_CA_Open; diff --git a/target/linux/generic/backport-4.9/605-0006-net-tcp_bbr-set-tp-snd_ssthresh-to-BDP-upon-STARTUP-exit.patch b/target/linux/generic/backport-4.9/605-0006-net-tcp_bbr-set-tp-snd_ssthresh-to-BDP-upon-STARTUP-exit.patch new file mode 100644 index 0000000000..c01e721b6c --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0006-net-tcp_bbr-set-tp-snd_ssthresh-to-BDP-upon-STARTUP-exit.patch @@ -0,0 +1,50 @@ +From b1f4eb4e4f9626fccb396f7998841506117a5916 Mon Sep 17 00:00:00 2001 +From: Yousuk Seung +Date: Fri, 16 Mar 2018 10:51:49 -0700 +Subject: [PATCH] BACKPORT: net-tcp_bbr: set tp->snd_ssthresh to BDP upon + STARTUP exit + +Set tp->snd_ssthresh to BDP upon STARTUP exit. This allows us +to check if a BBR flow exited STARTUP and the BDP at the +time of STARTUP exit with SCM_TIMESTAMPING_OPT_STATS. Since BBR does not +use snd_ssthresh this fix has no impact on BBR's behavior. + +Signed-off-by: Yousuk Seung +Signed-off-by: Neal Cardwell +Signed-off-by: Priyaranjan Jha +Signed-off-by: Soheil Hassas Yeganeh +Signed-off-by: Yuchung Cheng +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -704,6 +704,8 @@ static void bbr_check_drain(struct sock + bbr->mode = BBR_DRAIN; /* drain queue we created */ + bbr->pacing_gain = bbr_drain_gain; /* pace slow to drain */ + bbr->cwnd_gain = bbr_high_gain; /* maintain cwnd */ ++ tcp_sk(sk)->snd_ssthresh = ++ bbr_target_cwnd(sk, bbr_max_bw(sk), BBR_UNIT); + } /* fall through to check if in-flight is already small: */ + if (bbr->mode == BBR_DRAIN && + tcp_packets_in_flight(tcp_sk(sk)) <= +@@ -809,6 +811,7 @@ static void bbr_init(struct sock *sk) + struct bbr *bbr = inet_csk_ca(sk); + + bbr->prior_cwnd = 0; ++ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH; + bbr->rtt_cnt = 0; + bbr->next_rtt_delivered = 0; + bbr->prev_ca_state = TCP_CA_Open; +@@ -861,7 +864,7 @@ static u32 bbr_undo_cwnd(struct sock *sk + static u32 bbr_ssthresh(struct sock *sk) + { + bbr_save_cwnd(sk); +- return TCP_INFINITE_SSTHRESH; /* BBR does not use ssthresh */ ++ return tcp_sk(sk)->snd_ssthresh; + } + + static size_t bbr_get_info(struct sock *sk, u32 ext, int *attr, diff --git a/target/linux/generic/backport-4.9/605-0007-tcp_bbr-fix-bbr-pacing-rate-for-internal-pacing.patch b/target/linux/generic/backport-4.9/605-0007-tcp_bbr-fix-bbr-pacing-rate-for-internal-pacing.patch new file mode 100644 index 0000000000..202ed2d9aa --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0007-tcp_bbr-fix-bbr-pacing-rate-for-internal-pacing.patch @@ -0,0 +1,88 @@ +From dcbf673adcbf5275e98d31c2833b2a5311a7eb1e Mon Sep 17 00:00:00 2001 +From: Eric Dumazet +Date: Wed, 20 Jun 2018 16:07:35 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: fix bbr pacing rate for internal pacing + +This commit makes BBR use only the MSS (without any headers) to +calculate pacing rates when internal TCP-layer pacing is used. + +This is necessary to achieve the correct pacing behavior in this case, +since tcp_internal_pacing() uses only the payload length to calculate +pacing delays. + +Signed-off-by: Kevin Yang +Signed-off-by: Eric Dumazet +Reviewed-by: Neal Cardwell +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + include/net/tcp.h | 11 +++++++++++ + net/ipv4/tcp_bbr.c | 6 +++++- + net/ipv4/tcp_output.c | 14 -------------- + 3 files changed, 16 insertions(+), 15 deletions(-) + +--- a/include/net/tcp.h ++++ b/include/net/tcp.h +@@ -1161,6 +1161,17 @@ static inline bool tcp_is_cwnd_limited(c + return tp->is_cwnd_limited; + } + ++/* BBR congestion control needs pacing. ++ * Same remark for SO_MAX_PACING_RATE. ++ * sch_fq packet scheduler is efficiently handling pacing, ++ * but is not always installed/used. ++ * Return true if TCP stack should pace packets itself. ++ */ ++static inline bool tcp_needs_internal_pacing(const struct sock *sk) ++{ ++ return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; ++} ++ + /* Something is really bad, we could not queue an additional packet, + * because qdisc is full or receiver sent a 0 window. + * We do not want to add fuel to the fire, or abort too early, +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -175,7 +175,11 @@ static u32 bbr_bw(const struct sock *sk) + */ + static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain) + { +- rate *= tcp_mss_to_mtu(sk, tcp_sk(sk)->mss_cache); ++ unsigned int mss = tcp_sk(sk)->mss_cache; ++ ++ if (!tcp_needs_internal_pacing(sk)) ++ mss = tcp_mss_to_mtu(sk, mss); ++ rate *= mss; + rate *= gain; + rate >>= BBR_SCALE; + rate *= USEC_PER_SEC; +--- a/net/ipv4/tcp_output.c ++++ b/net/ipv4/tcp_output.c +@@ -947,17 +947,6 @@ enum hrtimer_restart tcp_pace_kick(struc + return HRTIMER_NORESTART; + } + +-/* BBR congestion control needs pacing. +- * Same remark for SO_MAX_PACING_RATE. +- * sch_fq packet scheduler is efficiently handling pacing, +- * but is not always installed/used. +- * Return true if TCP stack should pace packets itself. +- */ +-static bool tcp_needs_internal_pacing(const struct sock *sk) +-{ +- return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; +-} +- + static void tcp_internal_pacing(struct sock *sk, const struct sk_buff *skb) + { + u64 len_ns; +@@ -969,9 +958,6 @@ static void tcp_internal_pacing(struct s + if (!rate || rate == ~0U) + return; + +- /* Should account for header sizes as sch_fq does, +- * but lets make things simple. +- */ + len_ns = (u64)skb->len * NSEC_PER_SEC; + do_div(len_ns, rate); + hrtimer_start(&tcp_sk(sk)->pacing_timer, diff --git a/target/linux/generic/backport-4.9/605-0008-tcp_bbr-add-bbr_check_probe_rtt_done-helper.patch b/target/linux/generic/backport-4.9/605-0008-tcp_bbr-add-bbr_check_probe_rtt_done-helper.patch new file mode 100644 index 0000000000..05728e827f --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0008-tcp_bbr-add-bbr_check_probe_rtt_done-helper.patch @@ -0,0 +1,99 @@ +From 94d2a6a1e040fa0707ea2835d425356b88a673e9 Mon Sep 17 00:00:00 2001 +From: Kevin Yang +Date: Wed, 22 Aug 2018 17:43:14 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: add bbr_check_probe_rtt_done() helper + +This patch add a helper function bbr_check_probe_rtt_done() to + 1. check the condition to see if bbr should exit probe_rtt mode; + 2. process the logic of exiting probe_rtt mode. + +Fixes: 0f8782ea1497 ("tcp_bbr: add BBR congestion control") +Signed-off-by: Kevin Yang +Signed-off-by: Neal Cardwell +Signed-off-by: Yuchung Cheng +Reviewed-by: Soheil Hassas Yeganeh +Signed-off-by: David S. Miller +[kras: Replace tcp_jiffies32 with tcp_time_stamp for 4.9] +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 34 ++++++++++++++++++---------------- + 1 file changed, 18 insertions(+), 16 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -65,11 +65,10 @@ struct bbr { + u32 mode:3, /* current bbr_mode in state machine */ + prev_ca_state:3, /* CA state on previous ACK */ + packet_conservation:1, /* use packet conservation? */ +- restore_cwnd:1, /* decided to revert cwnd to old value */ + round_start:1, /* start of packet-timed tx->ack round? */ + idle_restart:1, /* restarting after idle? */ + probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */ +- unused:12, ++ unused:13, + lt_is_sampling:1, /* taking long-term ("LT") samples now? */ + lt_rtt_cnt:7, /* round trips in long-term interval */ + lt_use_bw:1; /* use lt_bw as our bw estimate? */ +@@ -366,17 +365,11 @@ static bool bbr_set_cwnd_to_recover_or_r + cwnd = tcp_packets_in_flight(tp) + acked; + } else if (prev_state >= TCP_CA_Recovery && state < TCP_CA_Recovery) { + /* Exiting loss recovery; restore cwnd saved before recovery. */ +- bbr->restore_cwnd = 1; ++ cwnd = max(cwnd, bbr->prior_cwnd); + bbr->packet_conservation = 0; + } + bbr->prev_ca_state = state; + +- if (bbr->restore_cwnd) { +- /* Restore cwnd after exiting loss recovery or PROBE_RTT. */ +- cwnd = max(cwnd, bbr->prior_cwnd); +- bbr->restore_cwnd = 0; +- } +- + if (bbr->packet_conservation) { + *new_cwnd = max(cwnd, tcp_packets_in_flight(tp) + acked); + return true; /* yes, using packet conservation */ +@@ -717,6 +710,20 @@ static void bbr_check_drain(struct sock + bbr_reset_probe_bw_mode(sk); /* we estimate queue is drained */ + } + ++static void bbr_check_probe_rtt_done(struct sock *sk) ++{ ++ struct tcp_sock *tp = tcp_sk(sk); ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ if (!(bbr->probe_rtt_done_stamp && ++ after(tcp_time_stamp, bbr->probe_rtt_done_stamp))) ++ return; ++ ++ bbr->min_rtt_stamp = tcp_time_stamp; /* wait a while until PROBE_RTT */ ++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd); ++ bbr_reset_mode(sk); ++} ++ + /* The goal of PROBE_RTT mode is to have BBR flows cooperatively and + * periodically drain the bottleneck queue, to converge to measure the true + * min_rtt (unloaded propagation delay). This allows the flows to keep queues +@@ -775,12 +782,8 @@ static void bbr_update_min_rtt(struct so + } else if (bbr->probe_rtt_done_stamp) { + if (bbr->round_start) + bbr->probe_rtt_round_done = 1; +- if (bbr->probe_rtt_round_done && +- after(tcp_time_stamp, bbr->probe_rtt_done_stamp)) { +- bbr->min_rtt_stamp = tcp_time_stamp; +- bbr->restore_cwnd = 1; /* snap to prior_cwnd */ +- bbr_reset_mode(sk); +- } ++ if (bbr->probe_rtt_round_done) ++ bbr_check_probe_rtt_done(sk); + } + } + /* Restart after idle ends only once we process a new S/ACK for data */ +@@ -831,7 +834,6 @@ static void bbr_init(struct sock *sk) + bbr->has_seen_rtt = 0; + bbr_init_pacing_rate_from_rtt(sk); + +- bbr->restore_cwnd = 0; + bbr->round_start = 0; + bbr->idle_restart = 0; + bbr->full_bw_reached = 0; diff --git a/target/linux/generic/backport-4.9/605-0009-tcp_bbr-in-restart-from-idle-see-if-we-should-exit.patch b/target/linux/generic/backport-4.9/605-0009-tcp_bbr-in-restart-from-idle-see-if-we-should-exit.patch new file mode 100644 index 0000000000..500f2eee7e --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0009-tcp_bbr-in-restart-from-idle-see-if-we-should-exit.patch @@ -0,0 +1,43 @@ +From 43ac328b26f344c6954a4cf857e9a5a382ec92e6 Mon Sep 17 00:00:00 2001 +From: Kevin Yang +Date: Wed, 22 Aug 2018 17:43:15 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: in restart from idle, see if we should + exit PROBE_RTT + +This patch fix the case where BBR does not exit PROBE_RTT mode when +it restarts from idle. When BBR restarts from idle and if BBR is in +PROBE_RTT mode, BBR should check if it's time to exit PROBE_RTT. If +yes, then BBR should exit PROBE_RTT mode and restore the cwnd to its +full value. + +Fixes: 0f8782ea1497 ("tcp_bbr: add BBR congestion control") +Signed-off-by: Kevin Yang +Signed-off-by: Neal Cardwell +Reviewed-by: Yuchung Cheng +Reviewed-by: Soheil Hassas Yeganeh +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 4 ++++ + 1 file changed, 4 insertions(+) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -144,6 +144,8 @@ static const u32 bbr_lt_bw_diff = 4000 / + /* If we estimate we're policed, use lt_bw for this many round trips: */ + static const u32 bbr_lt_bw_max_rtts = 48; + ++static void bbr_check_probe_rtt_done(struct sock *sk); ++ + /* Do we estimate that STARTUP filled the pipe? */ + static bool bbr_full_bw_reached(const struct sock *sk) + { +@@ -278,6 +280,8 @@ static void bbr_cwnd_event(struct sock * + */ + if (bbr->mode == BBR_PROBE_BW) + bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT); ++ else if (bbr->mode == BBR_PROBE_RTT) ++ bbr_check_probe_rtt_done(sk); + } + } + diff --git a/target/linux/generic/backport-4.9/605-0010-tcp_bbr-apply-PROBE_RTT-cwnd-cap-even-if-acked-0.patch b/target/linux/generic/backport-4.9/605-0010-tcp_bbr-apply-PROBE_RTT-cwnd-cap-even-if-acked-0.patch new file mode 100644 index 0000000000..b3a0a0e7a5 --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0010-tcp_bbr-apply-PROBE_RTT-cwnd-cap-even-if-acked-0.patch @@ -0,0 +1,41 @@ +From 3638356eff0cedaae1ad779c0920662ab068e312 Mon Sep 17 00:00:00 2001 +From: Kevin Yang +Date: Wed, 22 Aug 2018 17:43:16 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: apply PROBE_RTT cwnd cap even if acked==0 + +This commit fixes a corner case where TCP BBR would enter PROBE_RTT +mode but not reduce its cwnd. If a TCP receiver ACKed less than one +full segment, the number of delivered/acked packets was 0, so that +bbr_set_cwnd() would short-circuit and exit early, without cutting +cwnd to the value we want for PROBE_RTT. + +The fix is to instead make sure that even when 0 full packets are +ACKed, we do apply all the appropriate caps, including the cap that +applies in PROBE_RTT mode. + +Fixes: 0f8782ea1497 ("tcp_bbr: add BBR congestion control") +Signed-off-by: Kevin Yang +Signed-off-by: Neal Cardwell +Reviewed-by: Yuchung Cheng +Reviewed-by: Soheil Hassas Yeganeh +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -390,10 +390,10 @@ static void bbr_set_cwnd(struct sock *sk + { + struct tcp_sock *tp = tcp_sk(sk); + struct bbr *bbr = inet_csk_ca(sk); +- u32 cwnd = 0, target_cwnd = 0; ++ u32 cwnd = tp->snd_cwnd, target_cwnd = 0; + + if (!acked) +- return; ++ goto done; /* no packet fully ACKed; just apply caps */ + + if (bbr_set_cwnd_to_recover_or_restore(sk, rs, acked, &cwnd)) + goto done; diff --git a/target/linux/generic/backport-4.9/605-0011-tcp_bbr-centralize-code-to-set-gains.patch b/target/linux/generic/backport-4.9/605-0011-tcp_bbr-centralize-code-to-set-gains.patch new file mode 100644 index 0000000000..833c3d7c32 --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0011-tcp_bbr-centralize-code-to-set-gains.patch @@ -0,0 +1,112 @@ +From 460cd4eb19924d12123c5002c203a87c886c5726 Mon Sep 17 00:00:00 2001 +From: Neal Cardwell +Date: Tue, 16 Oct 2018 20:16:45 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: centralize code to set gains + +Centralize the code that sets gains used for computing cwnd and pacing +rate. This simplifies the code and makes it easier to change the state +machine or (in the future) dynamically change the gain values and +ensure that the correct gain values are always used. + +Signed-off-by: Neal Cardwell +Signed-off-by: Yuchung Cheng +Signed-off-by: Soheil Hassas Yeganeh +Signed-off-by: Priyaranjan Jha +Signed-off-by: Eric Dumazet +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 40 ++++++++++++++++++++++++++++++---------- + 1 file changed, 30 insertions(+), 10 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -457,8 +457,6 @@ static void bbr_advance_cycle_phase(stru + + bbr->cycle_idx = (bbr->cycle_idx + 1) & (CYCLE_LEN - 1); + bbr->cycle_mstamp = tp->delivered_mstamp; +- bbr->pacing_gain = bbr->lt_use_bw ? BBR_UNIT : +- bbr_pacing_gain[bbr->cycle_idx]; + } + + /* Gain cycling: cycle pacing gain to converge to fair share of available bw. */ +@@ -476,8 +474,6 @@ static void bbr_reset_startup_mode(struc + struct bbr *bbr = inet_csk_ca(sk); + + bbr->mode = BBR_STARTUP; +- bbr->pacing_gain = bbr_high_gain; +- bbr->cwnd_gain = bbr_high_gain; + } + + static void bbr_reset_probe_bw_mode(struct sock *sk) +@@ -485,8 +481,6 @@ static void bbr_reset_probe_bw_mode(stru + struct bbr *bbr = inet_csk_ca(sk); + + bbr->mode = BBR_PROBE_BW; +- bbr->pacing_gain = BBR_UNIT; +- bbr->cwnd_gain = bbr_cwnd_gain; + bbr->cycle_idx = CYCLE_LEN - 1 - prandom_u32_max(bbr_cycle_rand); + bbr_advance_cycle_phase(sk); /* flip to next phase of gain cycle */ + } +@@ -703,8 +697,6 @@ static void bbr_check_drain(struct sock + + if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { + bbr->mode = BBR_DRAIN; /* drain queue we created */ +- bbr->pacing_gain = bbr_drain_gain; /* pace slow to drain */ +- bbr->cwnd_gain = bbr_high_gain; /* maintain cwnd */ + tcp_sk(sk)->snd_ssthresh = + bbr_target_cwnd(sk, bbr_max_bw(sk), BBR_UNIT); + } /* fall through to check if in-flight is already small: */ +@@ -766,8 +758,6 @@ static void bbr_update_min_rtt(struct so + if (bbr_probe_rtt_mode_ms > 0 && filter_expired && + !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) { + bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */ +- bbr->pacing_gain = BBR_UNIT; +- bbr->cwnd_gain = BBR_UNIT; + bbr_save_cwnd(sk); /* note cwnd so we can restore it */ + bbr->probe_rtt_done_stamp = 0; + } +@@ -795,6 +785,35 @@ static void bbr_update_min_rtt(struct so + bbr->idle_restart = 0; + } + ++static void bbr_update_gains(struct sock *sk) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); ++ ++ switch (bbr->mode) { ++ case BBR_STARTUP: ++ bbr->pacing_gain = bbr_high_gain; ++ bbr->cwnd_gain = bbr_high_gain; ++ break; ++ case BBR_DRAIN: ++ bbr->pacing_gain = bbr_drain_gain; /* slow, to drain */ ++ bbr->cwnd_gain = bbr_high_gain; /* keep cwnd */ ++ break; ++ case BBR_PROBE_BW: ++ bbr->pacing_gain = (bbr->lt_use_bw ? ++ BBR_UNIT : ++ bbr_pacing_gain[bbr->cycle_idx]); ++ bbr->cwnd_gain = bbr_cwnd_gain; ++ break; ++ case BBR_PROBE_RTT: ++ bbr->pacing_gain = BBR_UNIT; ++ bbr->cwnd_gain = BBR_UNIT; ++ break; ++ default: ++ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode); ++ break; ++ } ++} ++ + static void bbr_update_model(struct sock *sk, const struct rate_sample *rs) + { + bbr_update_bw(sk, rs); +@@ -802,6 +821,7 @@ static void bbr_update_model(struct sock + bbr_check_full_bw_reached(sk, rs); + bbr_check_drain(sk, rs); + bbr_update_min_rtt(sk, rs); ++ bbr_update_gains(sk); + } + + static void bbr_main(struct sock *sk, const struct rate_sample *rs) diff --git a/target/linux/generic/backport-4.9/605-0012-tcp_bbr-refactor-bbr_target_cwnd-for-general-inflight-pro.patch b/target/linux/generic/backport-4.9/605-0012-tcp_bbr-refactor-bbr_target_cwnd-for-general-inflight-pro.patch new file mode 100644 index 0000000000..f7486a592b --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0012-tcp_bbr-refactor-bbr_target_cwnd-for-general-inflight-pro.patch @@ -0,0 +1,144 @@ +From 6a69f3fe4b25ddd777369b7c836fffadb4f387ce Mon Sep 17 00:00:00 2001 +From: Priyaranjan Jha +Date: Wed, 23 Jan 2019 12:04:53 -0800 +Subject: [PATCH] BACKPORT: tcp_bbr: refactor bbr_target_cwnd() for general + inflight provisioning + +Because bbr_target_cwnd() is really a general-purpose BBR helper for +computing some volume of inflight data as a function of the estimated +BDP, refactor it into following helper functions: +- bbr_bdp() +- bbr_quantization_budget() +- bbr_inflight() + +Signed-off-by: Priyaranjan Jha +Signed-off-by: Neal Cardwell +Signed-off-by: Yuchung Cheng +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 60 ++++++++++++++++++++++++++++++---------------- + 1 file changed, 39 insertions(+), 21 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -285,30 +285,19 @@ static void bbr_cwnd_event(struct sock * + } + } + +-/* Find target cwnd. Right-size the cwnd based on min RTT and the +- * estimated bottleneck bandwidth: ++/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: + * +- * cwnd = bw * min_rtt * gain = BDP * gain ++ * bdp = bw * min_rtt * gain + * + * The key factor, gain, controls the amount of queue. While a small gain + * builds a smaller queue, it becomes more vulnerable to noise in RTT + * measurements (e.g., delayed ACKs or other ACK compression effects). This + * noise may cause BBR to under-estimate the rate. +- * +- * To achieve full performance in high-speed paths, we budget enough cwnd to +- * fit full-sized skbs in-flight on both end hosts to fully utilize the path: +- * - one skb in sending host Qdisc, +- * - one skb in sending host TSO/GSO engine +- * - one skb being received by receiver host LRO/GRO/delayed-ACK engine +- * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because +- * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, +- * which allows 2 outstanding 2-packet sequences, to try to keep pipe +- * full even with ACK-every-other-packet delayed ACKs. + */ +-static u32 bbr_target_cwnd(struct sock *sk, u32 bw, int gain) ++static u32 bbr_bdp(struct sock *sk, u32 bw, int gain) + { + struct bbr *bbr = inet_csk_ca(sk); +- u32 cwnd; ++ u32 bdp; + u64 w; + + /* If we've never had a valid RTT sample, cap cwnd at the initial +@@ -323,7 +312,24 @@ static u32 bbr_target_cwnd(struct sock * + w = (u64)bw * bbr->min_rtt_us; + + /* Apply a gain to the given value, then remove the BW_SCALE shift. */ +- cwnd = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; ++ bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; ++ ++ return bdp; ++} ++ ++/* To achieve full performance in high-speed paths, we budget enough cwnd to ++ * fit full-sized skbs in-flight on both end hosts to fully utilize the path: ++ * - one skb in sending host Qdisc, ++ * - one skb in sending host TSO/GSO engine ++ * - one skb being received by receiver host LRO/GRO/delayed-ACK engine ++ * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because ++ * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets, ++ * which allows 2 outstanding 2-packet sequences, to try to keep pipe ++ * full even with ACK-every-other-packet delayed ACKs. ++ */ ++static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd, int gain) ++{ ++ struct bbr *bbr = inet_csk_ca(sk); + + /* Allow enough full-sized skbs in flight to utilize end systems. */ + cwnd += 3 * bbr_tso_segs_goal(sk); +@@ -338,6 +344,17 @@ static u32 bbr_target_cwnd(struct sock * + return cwnd; + } + ++/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */ ++static u32 bbr_inflight(struct sock *sk, u32 bw, int gain) ++{ ++ u32 inflight; ++ ++ inflight = bbr_bdp(sk, bw, gain); ++ inflight = bbr_quantization_budget(sk, inflight, gain); ++ ++ return inflight; ++} ++ + /* An optimization in BBR to reduce losses: On the first round of recovery, we + * follow the packet conservation principle: send P packets per P packets acked. + * After that, we slow-start and send at most 2*P packets per P packets acked. +@@ -399,7 +416,8 @@ static void bbr_set_cwnd(struct sock *sk + goto done; + + /* If we're below target cwnd, slow start cwnd toward target cwnd. */ +- target_cwnd = bbr_target_cwnd(sk, bw, gain); ++ target_cwnd = bbr_bdp(sk, bw, gain); ++ target_cwnd = bbr_quantization_budget(sk, target_cwnd, gain); + if (bbr_full_bw_reached(sk)) /* only cut cwnd if we filled the pipe */ + cwnd = min(cwnd + acked, target_cwnd); + else if (cwnd < target_cwnd || tp->delivered < TCP_INIT_CWND) +@@ -440,14 +458,14 @@ static bool bbr_is_next_cycle_phase(stru + if (bbr->pacing_gain > BBR_UNIT) + return is_full_length && + (rs->losses || /* perhaps pacing_gain*BDP won't fit */ +- inflight >= bbr_target_cwnd(sk, bw, bbr->pacing_gain)); ++ inflight >= bbr_inflight(sk, bw, bbr->pacing_gain)); + + /* A pacing_gain < 1.0 tries to drain extra queue we added if bw + * probing didn't find more bw. If inflight falls to match BDP then we + * estimate queue is drained; persisting would underutilize the pipe. + */ + return is_full_length || +- inflight <= bbr_target_cwnd(sk, bw, BBR_UNIT); ++ inflight <= bbr_inflight(sk, bw, BBR_UNIT); + } + + static void bbr_advance_cycle_phase(struct sock *sk) +@@ -698,11 +716,11 @@ static void bbr_check_drain(struct sock + if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) { + bbr->mode = BBR_DRAIN; /* drain queue we created */ + tcp_sk(sk)->snd_ssthresh = +- bbr_target_cwnd(sk, bbr_max_bw(sk), BBR_UNIT); ++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT); + } /* fall through to check if in-flight is already small: */ + if (bbr->mode == BBR_DRAIN && + tcp_packets_in_flight(tcp_sk(sk)) <= +- bbr_target_cwnd(sk, bbr_max_bw(sk), BBR_UNIT)) ++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT)) + bbr_reset_probe_bw_mode(sk); /* we estimate queue is drained */ + } + diff --git a/target/linux/generic/backport-4.9/605-0013-tcp_bbr-clarify-that-bbr_bdp-rounds-up-in-comments.patch b/target/linux/generic/backport-4.9/605-0013-tcp_bbr-clarify-that-bbr_bdp-rounds-up-in-comments.patch new file mode 100644 index 0000000000..4d9c501e9f --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0013-tcp_bbr-clarify-that-bbr_bdp-rounds-up-in-comments.patch @@ -0,0 +1,41 @@ +From 697ee116a6411b47c42d4b2facb9fbffad3f1abd Mon Sep 17 00:00:00 2001 +From: Luke Hsiao +Date: Thu, 29 Aug 2019 10:02:44 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: clarify that bbr_bdp() rounds up in + comments + +This explicitly clarifies that bbr_bdp() returns the rounded-up value of +the bandwidth-delay product and why in the comments. + +Signed-off-by: Luke Hsiao +Acked-by: Soheil Hassas Yeganeh +Acked-by: Neal Cardwell +Acked-by: Priyaranjan Jha +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -287,7 +287,7 @@ static void bbr_cwnd_event(struct sock * + + /* Calculate bdp based on min RTT and the estimated bottleneck bandwidth: + * +- * bdp = bw * min_rtt * gain ++ * bdp = ceil(bw * min_rtt * gain) + * + * The key factor, gain, controls the amount of queue. While a small gain + * builds a smaller queue, it becomes more vulnerable to noise in RTT +@@ -311,7 +311,9 @@ static u32 bbr_bdp(struct sock *sk, u32 + + w = (u64)bw * bbr->min_rtt_us; + +- /* Apply a gain to the given value, then remove the BW_SCALE shift. */ ++ /* Apply a gain to the given value, remove the BW_SCALE shift, and ++ * round the value up to avoid a negative feedback loop. ++ */ + bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT; + + return bdp; diff --git a/target/linux/generic/backport-4.9/605-0014-tcp_bbr-fix-quantization-code-to-not-raise-cwnd-if-not-pr.patch b/target/linux/generic/backport-4.9/605-0014-tcp_bbr-fix-quantization-code-to-not-raise-cwnd-if-not-pr.patch new file mode 100644 index 0000000000..49ff9193be --- /dev/null +++ b/target/linux/generic/backport-4.9/605-0014-tcp_bbr-fix-quantization-code-to-not-raise-cwnd-if-not-pr.patch @@ -0,0 +1,66 @@ +From 6c283578cb270c599c6fc6a6e4e332edf9f32425 Mon Sep 17 00:00:00 2001 +From: "Kevin(Yudong) Yang" +Date: Thu, 26 Sep 2019 10:30:05 -0400 +Subject: [PATCH] BACKPORT: tcp_bbr: fix quantization code to not raise cwnd if + not probing bandwidth + +There was a bug in the previous logic that attempted to ensure gain cycling +gets inflight above BDP even for small BDPs. This code correctly raised and +lowered target inflight values during the gain cycle. And this code +correctly ensured that cwnd was raised when probing bandwidth. However, it +did not correspondingly ensure that cwnd was *not* raised in this way when +*not* probing for bandwidth. The result was that small-BDP flows that were +always cwnd-bound could go for many cycles with a fixed cwnd, and not probe +or yield bandwidth at all. This meant that multiple small-BDP flows could +fail to converge in their bandwidth allocations. + +Fixes: 3c346b233c68 ("tcp_bbr: fix bw probing to raise in-flight data for very small BDPs") +Signed-off-by: Kevin(Yudong) Yang +Acked-by: Neal Cardwell +Acked-by: Yuchung Cheng +Acked-by: Soheil Hassas Yeganeh +Acked-by: Priyaranjan Jha +Signed-off-by: David S. Miller +Signed-off-by: Albert I +--- + net/ipv4/tcp_bbr.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +--- a/net/ipv4/tcp_bbr.c ++++ b/net/ipv4/tcp_bbr.c +@@ -329,7 +329,7 @@ static u32 bbr_bdp(struct sock *sk, u32 + * which allows 2 outstanding 2-packet sequences, to try to keep pipe + * full even with ACK-every-other-packet delayed ACKs. + */ +-static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd, int gain) ++static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd) + { + struct bbr *bbr = inet_csk_ca(sk); + +@@ -340,7 +340,7 @@ static u32 bbr_quantization_budget(struc + cwnd = (cwnd + 1) & ~1U; + + /* Ensure gain cycling gets inflight above BDP even for small BDPs. */ +- if (bbr->mode == BBR_PROBE_BW && gain > BBR_UNIT) ++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == 0) + cwnd += 2; + + return cwnd; +@@ -352,7 +352,7 @@ static u32 bbr_inflight(struct sock *sk, + u32 inflight; + + inflight = bbr_bdp(sk, bw, gain); +- inflight = bbr_quantization_budget(sk, inflight, gain); ++ inflight = bbr_quantization_budget(sk, inflight); + + return inflight; + } +@@ -419,7 +419,7 @@ static void bbr_set_cwnd(struct sock *sk + + /* If we're below target cwnd, slow start cwnd toward target cwnd. */ + target_cwnd = bbr_bdp(sk, bw, gain); +- target_cwnd = bbr_quantization_budget(sk, target_cwnd, gain); ++ target_cwnd = bbr_quantization_budget(sk, target_cwnd); + if (bbr_full_bw_reached(sk)) /* only cut cwnd if we filled the pipe */ + cwnd = min(cwnd + acked, target_cwnd); + else if (cwnd < target_cwnd || tp->delivered < TCP_INIT_CWND) From 490fd504c63afa1c544aef2465196a549e4b51d6 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sun, 7 Mar 2021 14:38:44 +0800 Subject: [PATCH 2/6] luci-app-turboacc: enable BBR for kernel 4.9 by default Signed-off-by: Tianling Shen --- package/ctcgfw/luci-app-turboacc/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/package/ctcgfw/luci-app-turboacc/Makefile b/package/ctcgfw/luci-app-turboacc/Makefile index 788f6ec5cd..d060bdd05b 100644 --- a/package/ctcgfw/luci-app-turboacc/Makefile +++ b/package/ctcgfw/luci-app-turboacc/Makefile @@ -33,7 +33,6 @@ config PACKAGE_$(PKG_NAME)_INCLUDE_flow-offload config PACKAGE_$(PKG_NAME)_INCLUDE_bbr-cca bool "Include BBR CCA" - default n if LINUX_4_9 default y config PACKAGE_$(PKG_NAME)_INCLUDE_dnsforwarder From 691dbcb8ac5b9a1cc47b5f8e0d360d3ade029bbf Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sun, 7 Mar 2021 15:12:56 +0800 Subject: [PATCH 3/6] naiveproxy: bump to 89.0.4389.72-1 Removed upstreamed patches, refreshed all dependencies. Signed-off-by: Tianling Shen --- package/ctcgfw/naiveproxy/Makefile | 23 +++++++++---------- .../patches/010-Fix-OpenWrt-x86-builds.patch | 22 ------------------ package/ctcgfw/naiveproxy/src/init_env.sh | 6 ++--- 3 files changed, 14 insertions(+), 37 deletions(-) delete mode 100644 package/ctcgfw/naiveproxy/patches/010-Fix-OpenWrt-x86-builds.patch diff --git a/package/ctcgfw/naiveproxy/Makefile b/package/ctcgfw/naiveproxy/Makefile index 65b53c5316..1940a5abc7 100644 --- a/package/ctcgfw/naiveproxy/Makefile +++ b/package/ctcgfw/naiveproxy/Makefile @@ -8,13 +8,12 @@ include $(TOPDIR)/rules.mk PKG_NAME:=naiveproxy -PKG_VERSION:=88.0.4324.96-1 -PKG_RELEASE:=2 +PKG_VERSION:=89.0.4389.72-1 +PKG_RELEASE:=1 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://codeload.github.com/klzgrad/naiveproxy/tar.gz/v$(PKG_VERSION)? -PKG_HASH:=287c5291d0371a0d1e06f08fcb29c1d80da4a07e557fb8d2bf770b357a8eb29a -PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_NAME)-$(PKG_VERSION) +PKG_HASH:=94b1c3f1e63ecab43eb8006a38ed1a76f1e510364ba651abaef0ec575449f687 PKG_LICENSE:=BSD 3-Clause PKG_LICENSE_FILES:=LICENSE @@ -63,32 +62,32 @@ ifneq ($(CONFIG_CCACHE),) export naive_ccache_flags=cc_wrapper="ccache" endif -AFDO_VER:=88.0.4324.91_rc-r1-merged -AFDO_FILE:=chromeos-chrome-amd64-$$(AFDO_VER).afdo.bz2 +AFDO_VER:=89.0.4389.65_rc-r1-merged +AFDO_FILE:=chromeos-chrome-amd64-$(AFDO_VER).afdo.bz2 define Download/AFDO_PROF URL:=https://storage.googleapis.com.cnpmjs.org/chromeos-prebuilt/afdo-job/llvm \ https://storage.googleapis.com/chromeos-prebuilt/afdo-job/llvm URL_FILE:=$(AFDO_FILE) FILE:=$(AFDO_FILE) - HASH:=d918ae36f535ffd7a3c36c7d06a0a58a3d93f03d07a670cee0f7cb4538925b34 + HASH:=75a176d57c9b5fb12bff7e9442ddf21226df8aa25de19266f7ceb426eafd3421 endef -CLANG_VER:=llvmorg-12-init-11462-g418f18c6-1 -CLANG_FILE:=clang-$$(CLANG_VER).tgz +CLANG_VER:=llvmorg-12-init-16296-g5e476061-1 +CLANG_FILE:=clang-$(CLANG_VER).tgz define Download/CLANG URL:=https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64 URL_FILE:=$(CLANG_FILE) FILE:=$(CLANG_FILE) - HASH:=59e4682179bbda9b1cf7fcc5c6e3e24445d6e9e8411bd1c893882f63c6b65308 + HASH:=78729dfab39fb9d52ede2841e408cf505959b26a5d0b086ed7cf4f784f2b393a endef -GN_VER:=53d92014bf94c3893886470a1c7c1289f8818db0 +GN_VER:=595e3be7c8381d4eeefce62a63ec12bae9ce5140 GN_FILE:=gn-git_revision-$(GN_VER).zip define Download/GN_TOOL URL:=https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-amd64/+ URL_FILE:=git_revision:$(GN_VER) FILE:=$(GN_FILE) - HASH:=e39d3da74cb2802614c0edc4877483ff50045abedb98adb40eaadb68ab4066ff + HASH:=9d61dcab3814868786768c9dfab71e0cd850a61dbad9a049536d2f1aae784fe4 endef PGO_VER:=4324-1610733451-8576f5ff23113bddfb13d417df9ef2a9be1f740d diff --git a/package/ctcgfw/naiveproxy/patches/010-Fix-OpenWrt-x86-builds.patch b/package/ctcgfw/naiveproxy/patches/010-Fix-OpenWrt-x86-builds.patch deleted file mode 100644 index ab85c7b063..0000000000 --- a/package/ctcgfw/naiveproxy/patches/010-Fix-OpenWrt-x86-builds.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 63f89cd454bcd1f2cdd14e63de0587a2f692cd07 Mon Sep 17 00:00:00 2001 -From: klzgrad -Date: Sun, 31 Jan 2021 20:15:28 +0800 -Subject: [PATCH] Fix OpenWrt x86 builds - ---- - src/build/toolchain/linux/BUILD.gn | 6 +++--- - 1 file changed, 3 insertions(+), 3 deletions(-) - ---- a/src/build/toolchain/linux/BUILD.gn -+++ b/src/build/toolchain/linux/BUILD.gn -@@ -99,8 +99,8 @@ clang_toolchain("clang_x86") { - clang_toolchain("clang_x86_openwrt") { - # Output linker map files for binary size analysis. - enable_linker_map = true -- extra_cppflags = "--target=i386-openwrt-linux-musl -D_LIBCPP_HAS_MUSL_LIBC -D__UCLIBC__" -- extra_ldflags = "--target=i386-openwrt-linux-musl" -+ extra_cppflags = "--target=i486-openwrt-linux-musl -D_LIBCPP_HAS_MUSL_LIBC -D__UCLIBC__" -+ extra_ldflags = "--target=i486-openwrt-linux-musl" - - toolchain_args = { - current_cpu = "x86" diff --git a/package/ctcgfw/naiveproxy/src/init_env.sh b/package/ctcgfw/naiveproxy/src/init_env.sh index 1aaf1760f7..0260616f02 100755 --- a/package/ctcgfw/naiveproxy/src/init_env.sh +++ b/package/ctcgfw/naiveproxy/src/init_env.sh @@ -16,7 +16,7 @@ naive_arch="${target_arch}" [ "${target_arch}" == "i386" ] && naive_arch="x86" [ "${target_arch}" == "x86_64" ] && naive_arch="x64" [ "${target_arch}" == "aarch64" ] && naive_arch="arm64" -ldso_path="/lib/$(find "${toolchain_dir}/" | grep -Eo "ld-musl-[a-z0-9_-]+\\.so\\.1")" +# ldso_path="/lib/$(find "${toolchain_dir}/" | grep -Eo "ld-musl-[a-z0-9_-]+\\.so\\.1")" # OS detection [ "$(uname)" != "Linux" -o "$(uname -m)" != "x86_64" ] && { echo -e "Support Linux AMD64 only."; exit 1; } @@ -81,8 +81,8 @@ ozone_platform_headless=true current_os=\"linux\" current_cpu=\"${naive_arch}\" sysroot=\"${toolchain_dir}\" -custom_toolchain=\"//build/toolchain/linux:clang_${naive_arch}_openwrt\" -ldso_path=\"${ldso_path}\"" +custom_toolchain=\"//build/toolchain/linux:clang_${naive_arch}_openwrt\"" +# ldso_path=\"${ldso_path}\"" [ "${target_arch}" == "arm" ] && { naive_flags="${naive_flags} arm_version=0 arm_cpu=\"${cpu_type}\"" [ -n "${cpu_subtype}" ] && { echo "${cpu_subtype}" | grep -q "neon" && neon_flag="arm_use_neon=true" || neon_flag="arm_use_neon=false"; naive_flags="${naive_flags} arm_fpu=\"${cpu_subtype}\" arm_float_abi=\"hard\" ${neon_flag}"; } || naive_flags="${naive_flags} arm_float_abi=\"soft\" arm_use_neon=false" From a1836ccb539e3d11ef4199968ba216f3139bf75a Mon Sep 17 00:00:00 2001 From: QiuSimons <45143996+QiuSimons@users.noreply.github.com> Date: Sun, 7 Mar 2021 15:56:25 +0800 Subject: [PATCH 4/6] mbedtls: Implements AES and GCM with ARMv8 Crypto Extensions A compact patch that provides AES and GCM implementations that utilize the ARMv8 Crypto Extensions. The config flag is MBEDTLS_ARMV8CE_AES_C, which is disabled by default as we don't do runtime checking for the feature. The new implementation lives in armv8ce_aes.c. Provides similar functionality to https://github.com/ARMmbed/mbedtls/pull/432 Thanks to Barry O'Rourke and others for that contribtion. Tested on a Cortex A53 device and QEMU. On a midrange phone the real AES-GCM throughput increases about 4x, while raw AES speed is up to 10x faster. [updated Makefile to enable this function, adjusted commit message] Signed-off-by: Tianling Shen --- package/libs/mbedtls/Makefile | 28 +- ...and-GCM-with-ARMv8-Crypto-Extensions.patch | 401 ++++++++++++++++++ 2 files changed, 423 insertions(+), 6 deletions(-) create mode 100644 package/libs/mbedtls/patches/100-Implements-AES-and-GCM-with-ARMv8-Crypto-Extensions.patch diff --git a/package/libs/mbedtls/Makefile b/package/libs/mbedtls/Makefile index 4fcef3dd9c..6f51abbdc9 100644 --- a/package/libs/mbedtls/Makefile +++ b/package/libs/mbedtls/Makefile @@ -23,6 +23,7 @@ PKG_CPE_ID:=cpe:/a:arm:mbed_tls PKG_CONFIG_DEPENDS := \ CONFIG_LIBMBEDTLS_DEBUG_C \ + CONFIG_LIBMBEDTLS_HAVE_ARMV8CE_AES \ CONFIG_LIBMBEDTLS_HAVE_SSE2 \ CONFIG_LIBMBEDTLS_HKDF_C @@ -61,6 +62,20 @@ config LIBMBEDTLS_DEBUG_C Usually, you don't need this, so don't select this if you're unsure. +config LIBMBEDTLS_HAVE_ARMV8CE_AES + depends on PACKAGE_libmbedtls + bool + default y if !TARGET_bcm27xx + prompt "Enable use of the ARMv8 Crypto Extensions" + depends on aarch64 + help + Use of the ARMv8 Crypto Extensions greatly increase performance + (up to 4x faster on AES-GCM while 10x faster on raw AES). + + Related instructions should be included in all modern Aarch64 + devices, except some wastes like Broadcom. + If you don't sure, say Y here. + config LIBMBEDTLS_HAVE_SSE2 depends on PACKAGE_libmbedtls bool @@ -68,12 +83,12 @@ config LIBMBEDTLS_HAVE_SSE2 prompt "Enable use of x86 SSE2 instructions" depends on x86_64 || i386 help - Use of SSE2 instructions greatly increase performance (up to - 3x faster) with a minimum (~0.2%, or 23KB) increase in package - size, but it will bring no benefit if your hardware does not - support them, such as Geode GX and LX. In this case you may - save 23KB by saying yes here. AMD Geode NX, and Intel - Pentium 4 and above support SSE2. + Use of SSE2 instructions greatly increase performance (up to + 3x faster) with a minimum (~0.2%, or 23KB) increase in package + size, but it will bring no benefit if your hardware does not + support them, such as Geode GX and LX. In this case you may + save 23KB by saying yes here. AMD Geode NX, and Intel + Pentium 4 and above support SSE2. config LIBMBEDTLS_HKDF_C depends on PACKAGE_libmbedtls @@ -118,6 +133,7 @@ define Build/Configure awk 'BEGIN { rc = 1 } \ /#define MBEDTLS_DEBUG_C/ { $$$$0 = "$(if $(CONFIG_LIBMBEDTLS_DEBUG_C),,// )#define MBEDTLS_DEBUG_C"; rc = 0 } \ + /#define MBEDTLS_ARMV8CE_AES_C/ { $$$$0 = "$(if $(CONFIG_LIBMBEDTLS_HAVE_ARMV8CE_AES),,// )#define MBEDTLS_ARMV8CE_AES_C"; rc = 0 } \ /#define MBEDTLS_HAVE_SSE2/ { $$$$0 = "$(if $(CONFIG_LIBMBEDTLS_HAVE_SSE2),,// )#define MBEDTLS_HAVE_SSE2"; rc = 0 } \ { print } \ END { exit(rc) }' $(PKG_BUILD_DIR)/include/mbedtls/config.h \ diff --git a/package/libs/mbedtls/patches/100-Implements-AES-and-GCM-with-ARMv8-Crypto-Extensions.patch b/package/libs/mbedtls/patches/100-Implements-AES-and-GCM-with-ARMv8-Crypto-Extensions.patch new file mode 100644 index 0000000000..b6531181f0 --- /dev/null +++ b/package/libs/mbedtls/patches/100-Implements-AES-and-GCM-with-ARMv8-Crypto-Extensions.patch @@ -0,0 +1,401 @@ +From dfb6015ca79a9fee28f7fcb0af7e350a83574b83 Mon Sep 17 00:00:00 2001 +From: "Markku-Juhani O. Saarinen" +Date: Mon, 20 Nov 2017 14:58:41 +0000 +Subject: Implements AES and GCM with ARMv8 Crypto Extensions + +A compact patch that provides AES and GCM implementations that utilize the +ARMv8 Crypto Extensions. The config flag is MBEDTLS_ARMV8CE_AES_C, which +is disabled by default as we don't do runtime checking for the feature. +The new implementation lives in armv8ce_aes.c. + +Provides similar functionality to https://github.com/ARMmbed/mbedtls/pull/432 +Thanks to Barry O'Rourke and others for that contribtion. + +Tested on a Cortex A53 device and QEMU. On a midrange phone the real AES-GCM +throughput increases about 4x, while raw AES speed is up to 10x faster. + +When cross-compiling, you want to set something like: + + export CC='aarch64-linux-gnu-gcc' + export CFLAGS='-Ofast -march=armv8-a+crypto' + scripts/config.pl set MBEDTLS_ARMV8CE_AES_C + +QEMU seems to also need + + export LDFLAGS='-static' + +Then run normal make or cmake etc. +--- + +diff -ruNa --binary a/ChangeLog.d/armv8_crypto_extensions.txt b/ChangeLog.d/armv8_crypto_extensions.txt +--- a/ChangeLog.d/armv8_crypto_extensions.txt 1970-01-01 08:00:00.000000000 +0800 ++++ b/ChangeLog.d/armv8_crypto_extensions.txt 2021-03-07 15:07:17.781911791 +0800 +@@ -0,0 +1,2 @@ ++Features ++ * Support ARMv8 Cryptography Extensions for AES and GCM. +diff -ruNa --binary a/include/mbedtls/armv8ce_aes.h b/include/mbedtls/armv8ce_aes.h +--- a/include/mbedtls/armv8ce_aes.h 1970-01-01 08:00:00.000000000 +0800 ++++ b/include/mbedtls/armv8ce_aes.h 2021-03-07 15:07:17.781911791 +0800 +@@ -0,0 +1,63 @@ ++/** ++ * \file armv8ce_aes.h ++ * ++ * \brief ARMv8 Cryptography Extensions -- Optimized code for AES and GCM ++ */ ++ ++/* ++ * ++ * Copyright (C) 2006-2017, ARM Limited, All Rights Reserved ++ * SPDX-License-Identifier: Apache-2.0 ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); you may ++ * not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ * ++ * This file is part of mbed TLS (https://tls.mbed.org) ++ */ ++ ++#ifndef MBEDTLS_ARMV8CE_AES_H ++#define MBEDTLS_ARMV8CE_AES_H ++ ++#include "aes.h" ++ ++/** ++ * \brief [ARMv8 Crypto Extensions] AES-ECB block en(de)cryption ++ * ++ * \param ctx AES context ++ * \param mode MBEDTLS_AES_ENCRYPT or MBEDTLS_AES_DECRYPT ++ * \param input 16-byte input block ++ * \param output 16-byte output block ++ * ++ * \return 0 on success (cannot fail) ++ */ ++ ++int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx, ++ int mode, ++ const unsigned char input[16], ++ unsigned char output[16] ); ++ ++/** ++ * \brief [ARMv8 Crypto Extensions] Multiply in GF(2^128) for GCM ++ * ++ * \param c Result ++ * \param a First operand ++ * \param b Second operand ++ * ++ * \note Both operands and result are bit strings interpreted as ++ * elements of GF(2^128) as per the GCM spec. ++ */ ++ ++void mbedtls_armv8ce_gcm_mult( unsigned char c[16], ++ const unsigned char a[16], ++ const unsigned char b[16] ); ++ ++#endif /* MBEDTLS_ARMV8CE_AES_H */ +diff -ruNa --binary a/include/mbedtls/check_config.h b/include/mbedtls/check_config.h +--- a/include/mbedtls/check_config.h 2020-12-10 20:54:15.000000000 +0800 ++++ b/include/mbedtls/check_config.h 2021-03-07 15:06:45.625543309 +0800 +@@ -95,6 +95,10 @@ + #error "MBEDTLS_AESNI_C defined, but not all prerequisites" + #endif + ++#if defined(MBEDTLS_ARMV8CE_AES_C) && !defined(MBEDTLS_HAVE_ASM) ++#error "MBEDTLS_ARMV8CE_AES_C defined, but not all prerequisites" ++#endif ++ + #if defined(MBEDTLS_CTR_DRBG_C) && !defined(MBEDTLS_AES_C) + #error "MBEDTLS_CTR_DRBG_C defined, but not all prerequisites" + #endif +@@ -772,3 +776,4 @@ + typedef int mbedtls_iso_c_forbids_empty_translation_units; + + #endif /* MBEDTLS_CHECK_CONFIG_H */ ++ +diff -ruNa --binary a/include/mbedtls/config.h b/include/mbedtls/config.h +--- a/include/mbedtls/config.h 2020-12-10 20:54:15.000000000 +0800 ++++ b/include/mbedtls/config.h 2021-03-07 15:14:27.957855484 +0800 +@@ -73,6 +73,7 @@ + * Requires support for asm() in compiler. + * + * Used in: ++ * library/armv8ce_aes.c + * library/aria.c + * library/timing.c + * include/mbedtls/bn_mul.h +@@ -1888,6 +1889,21 @@ + #define MBEDTLS_AESNI_C + + /** ++ * \def MBEDTLS_ARMV8CE_AES_C ++ * ++ * Enable ARMv8 Crypto Extensions for AES and GCM ++ * ++ * Module: library/armv8ce_aes.c ++ * Caller: library/aes.c ++ * library/gcm.c ++ * ++ * Requires: MBEDTLS_HAVE_ASM ++ * ++ * This module adds support for Armv8 Cryptography Extensions for AES and GCM. ++ */ ++//#define MBEDTLS_ARMV8CE_AES_C ++ ++/** + * \def MBEDTLS_AES_C + * + * Enable the AES block cipher. +diff -ruNa --binary a/library/aes.c b/library/aes.c +--- a/library/aes.c 2020-12-10 20:54:15.000000000 +0800 ++++ b/library/aes.c 2021-03-07 15:06:45.625543309 +0800 +@@ -69,7 +69,9 @@ + #if defined(MBEDTLS_AESNI_C) + #include "mbedtls/aesni.h" + #endif +- ++#if defined(MBEDTLS_ARMV8CE_AES_C) ++#include "mbedtls/armv8ce_aes.h" ++#endif + #if defined(MBEDTLS_SELF_TEST) + #if defined(MBEDTLS_PLATFORM_C) + #include "mbedtls/platform.h" +@@ -1052,6 +1054,11 @@ + return( mbedtls_aesni_crypt_ecb( ctx, mode, input, output ) ); + #endif + ++#if defined(MBEDTLS_ARMV8CE_AES_C) ++ // We don't do runtime checking for ARMv8 Crypto Extensions ++ return mbedtls_armv8ce_aes_crypt_ecb( ctx, mode, input, output ); ++#endif ++ + #if defined(MBEDTLS_PADLOCK_C) && defined(MBEDTLS_HAVE_X86) + if( aes_padlock_ace ) + { +diff -ruNa --binary a/library/armv8ce_aes.c b/library/armv8ce_aes.c +--- a/library/armv8ce_aes.c 1970-01-01 08:00:00.000000000 +0800 ++++ b/library/armv8ce_aes.c 2021-03-07 15:07:17.781911791 +0800 +@@ -0,0 +1,142 @@ ++/* ++ * ARMv8 Cryptography Extensions -- Optimized code for AES and GCM ++ * ++ * Copyright (C) 2006-2017, ARM Limited, All Rights Reserved ++ * SPDX-License-Identifier: Apache-2.0 ++ * ++ * Licensed under the Apache License, Version 2.0 (the "License"); you may ++ * not use this file except in compliance with the License. ++ * You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ * ++ * This file is part of mbed TLS (https://tls.mbed.org) ++ */ ++ ++#if !defined(MBEDTLS_CONFIG_FILE) ++#include "mbedtls/config.h" ++#else ++#include MBEDTLS_CONFIG_FILE ++#endif ++ ++#if defined(MBEDTLS_ARMV8CE_AES_C) ++ ++#include ++#include "mbedtls/armv8ce_aes.h" ++ ++#ifndef asm ++#define asm __asm ++#endif ++ ++/* ++ * [Armv8 Cryptography Extensions] AES-ECB block en(de)cryption ++ */ ++ ++#if defined(MBEDTLS_AES_C) ++ ++int mbedtls_armv8ce_aes_crypt_ecb( mbedtls_aes_context *ctx, ++ int mode, ++ const unsigned char input[16], ++ unsigned char output[16] ) ++{ ++ unsigned int i; ++ const uint8_t *rk; ++ uint8x16_t x, k; ++ ++ x = vld1q_u8( input ); /* input block */ ++ rk = (const uint8_t *) ctx->rk; /* round keys */ ++ ++ if( mode == MBEDTLS_AES_ENCRYPT ) ++ { ++ for( i = ctx->nr - 1; i != 0; i-- ) /* encryption loop */ ++ { ++ k = vld1q_u8( rk ); ++ rk += 16; ++ x = vaeseq_u8( x, k ); ++ x = vaesmcq_u8( x ); ++ } ++ k = vld1q_u8( rk ); ++ rk += 16; ++ x = vaeseq_u8( x, k ); ++ } ++ else ++ { ++ for( i = ctx->nr - 1; i != 0 ; i-- ) /* decryption loop */ ++ { ++ k = vld1q_u8( rk ); ++ rk += 16; ++ x = vaesdq_u8( x, k ); ++ x = vaesimcq_u8( x ); ++ } ++ k = vld1q_u8( rk ); ++ rk += 16; ++ x = vaesdq_u8( x, k ); ++ } ++ ++ k = vld1q_u8( rk ); /* final key just XORed */ ++ x = veorq_u8( x, k ); ++ vst1q_u8( output, x ); /* write out */ ++ ++ return ( 0 ); ++} ++ ++#endif /* MBEDTLS_AES_C */ ++ ++ ++/* ++ * [Armv8 Cryptography Extensions] Multiply in GF(2^128) for GCM ++ */ ++ ++#if defined(MBEDTLS_GCM_C) ++ ++void mbedtls_armv8ce_gcm_mult( unsigned char c[16], ++ const unsigned char a[16], ++ const unsigned char b[16] ) ++{ ++ /* GCM's GF(2^128) polynomial basis is x^128 + x^7 + x^2 + x + 1 */ ++ const uint64x2_t base = { 0, 0x86 }; /* note missing LS bit */ ++ ++ register uint8x16_t vc asm( "v0" ); /* named registers */ ++ register uint8x16_t va asm( "v1" ); /* (to avoid conflict) */ ++ register uint8x16_t vb asm( "v2" ); ++ register uint64x2_t vp asm( "v3" ); ++ ++ va = vld1q_u8( a ); /* load inputs */ ++ vb = vld1q_u8( b ); ++ vp = base; ++ ++ asm ( ++ "rbit %1.16b, %1.16b \n\t" /* reverse bit order */ ++ "rbit %2.16b, %2.16b \n\t" ++ "pmull2 %0.1q, %1.2d, %2.2d \n\t" /* v0 = a.hi * b.hi */ ++ "pmull2 v4.1q, %0.2d, %3.2d \n\t" /* mul v0 by x^64, reduce */ ++ "ext %0.16b, %0.16b, %0.16b, #8 \n\t" ++ "eor %0.16b, %0.16b, v4.16b \n\t" ++ "ext v5.16b, %2.16b, %2.16b, #8 \n\t" /* (swap hi and lo in b) */ ++ "pmull v4.1q, %1.1d, v5.1d \n\t" /* v0 ^= a.lo * b.hi */ ++ "eor %0.16b, %0.16b, v4.16b \n\t" ++ "pmull2 v4.1q, %1.2d, v5.2d \n\t" /* v0 ^= a.hi * b.lo */ ++ "eor %0.16b, %0.16b, v4.16b \n\t" ++ "pmull2 v4.1q, %0.2d, %3.2d \n\t" /* mul v0 by x^64, reduce */ ++ "ext %0.16b, %0.16b, %0.16b, #8 \n\t" ++ "eor %0.16b, %0.16b, v4.16b \n\t" ++ "pmull v4.1q, %1.1d, %2.1d \n\t" /* v0 ^= a.lo * b.lo */ ++ "eor %0.16b, %0.16b, v4.16b \n\t" ++ "rbit %0.16b, %0.16b \n\t" /* reverse bits for output */ ++ : "=w" (vc) /* q0: output */ ++ : "w" (va), "w" (vb), "w" (vp) /* q1, q2: input */ ++ : "v4", "v5" /* q4, q5: clobbered */ ++ ); ++ ++ vst1q_u8( c, vc ); /* write out */ ++} ++ ++#endif /* MBEDTLS_GCM_C */ ++ ++#endif /* MBEDTLS_ARMV8CE_AES_C */ +diff -ruNa --binary a/library/CMakeLists.txt b/library/CMakeLists.txt +--- a/library/CMakeLists.txt 2020-12-10 20:54:15.000000000 +0800 ++++ b/library/CMakeLists.txt 2021-03-07 15:06:45.625543309 +0800 +@@ -7,6 +7,7 @@ + aesni.c + arc4.c + aria.c ++ armv8ce_aes.c + asn1parse.c + asn1write.c + base64.c +diff -ruNa --binary a/library/gcm.c b/library/gcm.c +--- a/library/gcm.c 2020-12-10 20:54:15.000000000 +0800 ++++ b/library/gcm.c 2021-03-07 15:06:45.625543309 +0800 +@@ -71,6 +71,10 @@ + #include "mbedtls/aesni.h" + #endif + ++#if defined(MBEDTLS_ARMV8CE_AES_C) ++#include "mbedtls/armv8ce_aes.h" ++#endif ++ + #if defined(MBEDTLS_SELF_TEST) && defined(MBEDTLS_AES_C) + #include "mbedtls/aes.h" + #include "mbedtls/platform.h" +@@ -140,6 +144,12 @@ + if( ( ret = mbedtls_cipher_update( &ctx->cipher_ctx, h, 16, h, &olen ) ) != 0 ) + return( ret ); + ++#if defined(MBEDTLS_ARMV8CE_AES_C) ++ // we don't do feature testing with ARMv8 cryptography extensions ++ memcpy( ctx ->HL, h, 16 ); // put H at the beginning of buffer ++ return( 0 ); // that's all we need ++#endif ++ + /* pack h as two 64-bits ints, big-endian */ + GET_UINT32_BE( hi, h, 0 ); + GET_UINT32_BE( lo, h, 4 ); +@@ -248,6 +258,11 @@ + unsigned char lo, hi, rem; + uint64_t zh, zl; + ++#if defined(MBEDTLS_ARMV8CE_AES_C) ++ mbedtls_armv8ce_gcm_mult( output, x, (const unsigned char *) ctx->HL ); ++ return; ++#endif ++ + #if defined(MBEDTLS_AESNI_C) && defined(MBEDTLS_HAVE_X86_64) + if( mbedtls_aesni_has_support( MBEDTLS_AESNI_CLMUL ) ) { + unsigned char h[16]; +diff -ruNa --binary a/library/Makefile b/library/Makefile +--- a/library/Makefile 2020-12-10 20:54:15.000000000 +0800 ++++ b/library/Makefile 2021-03-07 15:12:49.277078224 +0800 +@@ -65,6 +65,7 @@ + + OBJS_CRYPTO= aes.o aesni.o arc4.o \ + aria.o asn1parse.o asn1write.o \ ++ armv8ce_aes.o \ + base64.o bignum.o blowfish.o \ + camellia.o ccm.o chacha20.o \ + chachapoly.o cipher.o cipher_wrap.o \ +diff -ruNa --binary a/library/version_features.c b/library/version_features.c +--- a/library/version_features.c 2020-12-10 20:54:15.000000000 +0800 ++++ b/library/version_features.c 2021-03-07 15:06:45.625543309 +0800 +@@ -583,6 +583,9 @@ + #if defined(MBEDTLS_AESNI_C) + "MBEDTLS_AESNI_C", + #endif /* MBEDTLS_AESNI_C */ ++#if defined(MBEDTLS_ARMV8CE_AES_C) ++ "MBEDTLS_ARMV8CE_AES_C", ++#endif /* MBEDTLS_ARMV8CE_AES_C */ + #if defined(MBEDTLS_AES_C) + "MBEDTLS_AES_C", + #endif /* MBEDTLS_AES_C */ + From 755dc0ffd7b393be61a3a6bce6c8899f46f42553 Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sun, 7 Mar 2021 20:43:25 +0800 Subject: [PATCH 5/6] target/aarch64: append crypto optimization to cflags Signed-off-by: Tianling Shen --- include/target.mk | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/target.mk b/include/target.mk index de9fb5016b..a54799d877 100644 --- a/include/target.mk +++ b/include/target.mk @@ -207,6 +207,9 @@ ifeq ($(DUMP),1) endif ifeq ($(ARCH),aarch64) CPU_TYPE ?= generic + ifneq ($(filter $(FEATURES),fpu),) + CPU_CFLAGS += -march=armv8-a+crypto + endif CPU_CFLAGS_generic = -mcpu=generic CPU_CFLAGS_cortex-a53 = -mcpu=cortex-a53 endif From 4d7d6034d272a6715ba08e9042636988f84c609a Mon Sep 17 00:00:00 2001 From: Tianling Shen Date: Sun, 7 Mar 2021 20:46:08 +0800 Subject: [PATCH 6/6] mbedtls: hidden `ARMv8 Crypto Extensions` option for broadcom devices Signed-off-by: Tianling Shen --- package/libs/mbedtls/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/libs/mbedtls/Makefile b/package/libs/mbedtls/Makefile index 6f51abbdc9..69632a77d0 100644 --- a/package/libs/mbedtls/Makefile +++ b/package/libs/mbedtls/Makefile @@ -65,9 +65,9 @@ config LIBMBEDTLS_DEBUG_C config LIBMBEDTLS_HAVE_ARMV8CE_AES depends on PACKAGE_libmbedtls bool - default y if !TARGET_bcm27xx + default y prompt "Enable use of the ARMv8 Crypto Extensions" - depends on aarch64 + depends on aarch64 && !TARGET_bcm27xx help Use of the ARMv8 Crypto Extensions greatly increase performance (up to 4x faster on AES-GCM while 10x faster on raw AES).