From 95d267365e82205bada1b50fe699fb2284aa090e Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Mon, 6 Sep 2004 20:01:22 -0700 Subject: [PATCH] [TCP]: Calculate SKB tso factor more accurately. Eliminate tp->mss_tso_factor. Instead, we calculate the SKB tso factor as we walk the write queue for initial transmit or fragment SKBs. Signed-off-by: David S. Miller --- include/linux/tcp.h | 1 - include/net/tcp.h | 8 +++---- net/ipv4/tcp.c | 21 ++++++++----------- net/ipv4/tcp_input.c | 1 - net/ipv4/tcp_ipv4.c | 1 - net/ipv4/tcp_output.c | 49 ++++++++++++++++++++++++++++++------------- net/ipv6/tcp_ipv6.c | 1 - 7 files changed, 47 insertions(+), 35 deletions(-) diff --git a/include/linux/tcp.h b/include/linux/tcp.h index ebf15b6a8162f..86ca98c5ef8f0 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -254,7 +254,6 @@ struct tcp_opt { __u32 max_window; /* Maximal window ever seen from peer */ __u32 pmtu_cookie; /* Last pmtu seen by socket */ __u32 mss_cache; /* Cached effective mss, not including SACKS */ - __u32 mss_tso_factor; /* Real packets per TSO packet */ __u16 mss_cache_std; /* Like mss_cache, but without TSO */ __u16 mss_clamp; /* Maximal mss, negotiated at connection setup */ __u16 ext_header_len; /* Network protocol overhead (IP/IPv6 options) */ diff --git a/include/net/tcp.h b/include/net/tcp.h index efda37b84207e..1de15c7a560ad 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1047,7 +1047,7 @@ static inline void tcp_reset_xmit_timer(struct sock *sk, int what, unsigned long * is not a big flaw. */ -static inline unsigned int tcp_current_mss(struct sock *sk, int large, int *factor) +static inline unsigned int tcp_current_mss(struct sock *sk, int large) { struct tcp_opt *tp = tcp_sk(sk); struct dst_entry *dst = __sk_dst_get(sk); @@ -1057,8 +1057,6 @@ static inline unsigned int tcp_current_mss(struct sock *sk, int large, int *fact (sk->sk_route_caps & NETIF_F_TSO) && !tp->urg_mode); mss_now = do_large ? tp->mss_cache : tp->mss_cache_std; - if (factor) - *factor = do_large ? tp->mss_tso_factor : 1; if (dst) { u32 mtu = dst_pmtu(dst); @@ -1545,7 +1543,7 @@ static __inline__ void __tcp_push_pending_frames(struct sock *sk, static __inline__ void tcp_push_pending_frames(struct sock *sk, struct tcp_opt *tp) { - __tcp_push_pending_frames(sk, tp, tcp_current_mss(sk, 1, NULL), tp->nonagle); + __tcp_push_pending_frames(sk, tp, tcp_current_mss(sk, 1), tp->nonagle); } static __inline__ int tcp_may_send_now(struct sock *sk, struct tcp_opt *tp) @@ -1553,7 +1551,7 @@ static __inline__ int tcp_may_send_now(struct sock *sk, struct tcp_opt *tp) struct sk_buff *skb = sk->sk_send_head; return (skb && - tcp_snd_test(tp, skb, tcp_current_mss(sk, 1, NULL), + tcp_snd_test(tp, skb, tcp_current_mss(sk, 1), tcp_skb_is_last(sk, skb) ? TCP_NAGLE_PUSH : tp->nonagle)); } diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 9df826c8e22b5..36953ef7e6c2b 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -590,14 +590,13 @@ static inline int forced_push(struct tcp_opt *tp) } static inline void skb_entail(struct sock *sk, struct tcp_opt *tp, - struct sk_buff *skb, int tso_factor) + struct sk_buff *skb) { skb->csum = 0; TCP_SKB_CB(skb)->seq = tp->write_seq; TCP_SKB_CB(skb)->end_seq = tp->write_seq; TCP_SKB_CB(skb)->flags = TCPCB_FLAG_ACK; TCP_SKB_CB(skb)->sacked = 0; - TCP_SKB_CB(skb)->tso_factor = tso_factor; __skb_queue_tail(&sk->sk_write_queue, skb); sk_charge_skb(sk, skb); if (!sk->sk_send_head) @@ -633,7 +632,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse size_t psize, int flags) { struct tcp_opt *tp = tcp_sk(sk); - int mss_now, mss_factor_now; + int mss_now; int err; ssize_t copied; long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT); @@ -645,7 +644,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); - mss_now = tcp_current_mss(sk, !(flags&MSG_OOB), &mss_factor_now); + mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); copied = 0; err = -EPIPE; @@ -669,7 +668,7 @@ new_segment: if (!skb) goto wait_for_memory; - skb_entail(sk, tp, skb, mss_factor_now); + skb_entail(sk, tp, skb); copy = mss_now; } @@ -720,8 +719,7 @@ wait_for_memory: if ((err = sk_stream_wait_memory(sk, &timeo)) != 0) goto do_error; - mss_now = tcp_current_mss(sk, !(flags&MSG_OOB), - &mss_factor_now); + mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); } out: @@ -782,7 +780,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, struct tcp_opt *tp = tcp_sk(sk); struct sk_buff *skb; int iovlen, flags; - int mss_now, mss_factor_now; + int mss_now; int err, copied; long timeo; @@ -800,7 +798,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg, /* This should be in poll */ clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags); - mss_now = tcp_current_mss(sk, !(flags&MSG_OOB), &mss_factor_now); + mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); /* Ok commence sending. */ iovlen = msg->msg_iovlen; @@ -845,7 +843,7 @@ new_segment: NETIF_F_HW_CSUM)) skb->ip_summed = CHECKSUM_HW; - skb_entail(sk, tp, skb, mss_factor_now); + skb_entail(sk, tp, skb); copy = mss_now; } @@ -964,8 +962,7 @@ wait_for_memory: if ((err = sk_stream_wait_memory(sk, &timeo)) != 0) goto do_error; - mss_now = tcp_current_mss(sk, !(flags&MSG_OOB), - &mss_factor_now); + mss_now = tcp_current_mss(sk, !(flags&MSG_OOB)); } } diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index f4ec161699062..d7fb3cde4f20e 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -974,7 +974,6 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_ sk->sk_route_caps &= ~NETIF_F_TSO; sk->sk_no_largesend = 1; tp->mss_cache = tp->mss_cache_std; - tp->mss_tso_factor = 1; } if (!tcp_get_pcount(&tp->sacked_out)) diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 0fb326e84f284..73f12904c7c31 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -2076,7 +2076,6 @@ static int tcp_v4_init_sock(struct sock *sk) tp->snd_ssthresh = 0x7fffffff; /* Infinity */ tp->snd_cwnd_clamp = ~0; tp->mss_cache_std = tp->mss_cache = 536; - tp->mss_tso_factor = 1; tp->reordering = sysctl_tcp_reordering; diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 0a70d082028ce..336c7121b6b6a 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -422,6 +422,23 @@ void tcp_push_one(struct sock *sk, unsigned cur_mss) } } +static void tcp_set_skb_tso_factor(struct sk_buff *skb, unsigned int mss, + unsigned int mss_std) +{ + if (skb->len <= mss_std) { + /* Avoid the costly divide in the normal + * non-TSO case. + */ + TCP_SKB_CB(skb)->tso_factor = 1; + } else { + unsigned int factor; + + factor = skb->len + (mss_std - 1); + factor /= mss; + TCP_SKB_CB(skb)->tso_factor = factor; + } +} + /* Function to create two new TCP segments. Shrinks the given segment * to the specified size and appends a new segment with the rest of the * packet to the list. This won't be called frequently, I hope. @@ -457,7 +474,6 @@ static int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len) TCP_SKB_CB(buff)->sacked = (TCP_SKB_CB(skb)->sacked & (TCPCB_LOST | TCPCB_EVER_RETRANS | TCPCB_AT_TAIL)); - TCP_SKB_CB(buff)->tso_factor = tp->mss_tso_factor; if (TCP_SKB_CB(buff)->sacked&TCPCB_LOST) { tcp_inc_pcount(&tp->lost_out, buff); tcp_inc_pcount(&tp->left_out, buff); @@ -484,6 +500,10 @@ static int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len) */ TCP_SKB_CB(buff)->when = TCP_SKB_CB(skb)->when; + /* Fix up tso_factor for both original and new SKB. */ + tcp_set_skb_tso_factor(skb, tp->mss_cache, tp->mss_cache_std); + tcp_set_skb_tso_factor(buff, tp->mss_cache, tp->mss_cache_std); + /* Link BUFF into the send queue. */ __skb_append(skb, buff); @@ -598,7 +618,6 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu) /* And store cached results */ tp->pmtu_cookie = pmtu; tp->mss_cache = tp->mss_cache_std = mss_now; - tp->mss_tso_factor = 1; if (sk->sk_route_caps & NETIF_F_TSO) { int large_mss, factor; @@ -617,7 +636,6 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu) factor = tp->snd_cwnd; tp->mss_cache = mss_now * factor; - tp->mss_tso_factor = factor; } return mss_now; @@ -634,7 +652,7 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu) int tcp_write_xmit(struct sock *sk, int nonagle) { struct tcp_opt *tp = tcp_sk(sk); - unsigned int mss_now; + unsigned int mss_now, mss_std; /* If we are closed, the bytes will have to remain here. * In time closedown will finish, we empty the write queue and all @@ -649,7 +667,8 @@ int tcp_write_xmit(struct sock *sk, int nonagle) * We also handle things correctly when the user adds some * IP options mid-stream. Silly to do, but cover it. */ - mss_now = tcp_current_mss(sk, 1, NULL); + mss_now = tcp_current_mss(sk, 1); + mss_std = tp->mss_cache_std; while ((skb = sk->sk_send_head) && tcp_snd_test(tp, skb, mss_now, @@ -658,7 +677,8 @@ int tcp_write_xmit(struct sock *sk, int nonagle) if (skb->len > mss_now) { if (tcp_fragment(sk, skb, mss_now)) break; - } + } else + tcp_set_skb_tso_factor(skb, mss_now, mss_std); TCP_SKB_CB(skb)->when = tcp_time_stamp; if (tcp_transmit_skb(sk, skb_clone(skb, GFP_ATOMIC))) @@ -871,7 +891,7 @@ void tcp_simple_retransmit(struct sock *sk) { struct tcp_opt *tp = tcp_sk(sk); struct sk_buff *skb; - unsigned int mss = tcp_current_mss(sk, 0, NULL); + unsigned int mss = tcp_current_mss(sk, 0); int lost = 0; sk_stream_for_retrans_queue(skb, sk) { @@ -916,7 +936,7 @@ void tcp_simple_retransmit(struct sock *sk) int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb) { struct tcp_opt *tp = tcp_sk(sk); - unsigned int cur_mss = tcp_current_mss(sk, 0, NULL); + unsigned int cur_mss = tcp_current_mss(sk, 0); int err; /* Do not sent more than we queued. 1/4 is reserved for possible @@ -934,7 +954,6 @@ int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb) sk->sk_route_caps &= ~NETIF_F_TSO; sk->sk_no_largesend = 1; tp->mss_cache = tp->mss_cache_std; - tp->mss_tso_factor = 1; } if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq)) @@ -1125,7 +1144,7 @@ void tcp_send_fin(struct sock *sk) * unsent frames. But be careful about outgoing SACKS * and IP options. */ - mss_now = tcp_current_mss(sk, 1, NULL); + mss_now = tcp_current_mss(sk, 1); if (sk->sk_send_head != NULL) { TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN; @@ -1516,8 +1535,9 @@ int tcp_write_wakeup(struct sock *sk) if ((skb = sk->sk_send_head) != NULL && before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)) { int err; - int mss = tcp_current_mss(sk, 0, NULL); - int seg_size = tp->snd_una+tp->snd_wnd-TCP_SKB_CB(skb)->seq; + unsigned int mss = tcp_current_mss(sk, 0); + unsigned int mss_std = tp->mss_cache_std; + unsigned int seg_size = tp->snd_una+tp->snd_wnd-TCP_SKB_CB(skb)->seq; if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq)) tp->pushed_seq = TCP_SKB_CB(skb)->end_seq; @@ -1538,9 +1558,10 @@ int tcp_write_wakeup(struct sock *sk) sk->sk_no_largesend = 1; sk->sk_route_caps &= ~NETIF_F_TSO; tp->mss_cache = tp->mss_cache_std; - tp->mss_tso_factor = 1; } - } + } else + tcp_set_skb_tso_factor(skb, mss, mss_std); + TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH; TCP_SKB_CB(skb)->when = tcp_time_stamp; err = tcp_transmit_skb(sk, skb_clone(skb, GFP_ATOMIC)); diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 0ecd0d8dfa0f5..ebed7e197aac2 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -1930,7 +1930,6 @@ static int tcp_v6_init_sock(struct sock *sk) tp->snd_ssthresh = 0x7fffffff; tp->snd_cwnd_clamp = ~0; tp->mss_cache_std = tp->mss_cache = 536; - tp->mss_tso_factor = 1; tp->reordering = sysctl_tcp_reordering; -- 2.53.0