[TCP]: Calculate SKB tso factor more accurately.

Eliminate tp->mss_tso_factor.  Instead, we calculate the
SKB tso factor as we walk the write queue for initial
transmit or fragment SKBs.

Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/include/linux/tcp.h b/include/linux/tcp.h
index ebf15b6a8162f2cb5957c42a286d9b2bdbfa621f..86ca98c5ef8f05240b0eb52a85987709221f7c09 100644 (file)
--- a/include/linux/tcp.h
+++ b/include/linux/tcp.h
@@ -254,7 +254,6 @@ struct tcp_opt {
        __u32   max_window;     /* Maximal window ever seen from peer   */
        __u32   pmtu_cookie;    /* Last pmtu seen by socket             */
        __u32   mss_cache;      /* Cached effective mss, not including SACKS */
-       __u32   mss_tso_factor; /* Real packets per TSO packet */
        __u16   mss_cache_std;  /* Like mss_cache, but without TSO */
        __u16   mss_clamp;      /* Maximal mss, negotiated at connection setup */
        __u16   ext_header_len; /* Network protocol overhead (IP/IPv6 options) */

diff --git a/include/net/tcp.h b/include/net/tcp.h
index efda37b84207efe11496f5943fe5c14f290c0527..1de15c7a560ad4e6682873f52ba0077a09e0b04e 100644 (file)
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -1047,7 +1047,7 @@ static inline void tcp_reset_xmit_timer(struct sock *sk, int what, unsigned long
  * is not a big flaw.
  */
 
-static inline unsigned int tcp_current_mss(struct sock *sk, int large, int *factor)
+static inline unsigned int tcp_current_mss(struct sock *sk, int large)
 {
        struct tcp_opt *tp = tcp_sk(sk);
        struct dst_entry *dst = __sk_dst_get(sk);
@@ -1057,8 +1057,6 @@ static inline unsigned int tcp_current_mss(struct sock *sk, int large, int *fact
                    (sk->sk_route_caps & NETIF_F_TSO) &&
                    !tp->urg_mode);
        mss_now = do_large ? tp->mss_cache : tp->mss_cache_std;
-       if (factor)
-               *factor = do_large ? tp->mss_tso_factor : 1;
 
        if (dst) {
                u32 mtu = dst_pmtu(dst);
@@ -1545,7 +1543,7 @@ static __inline__ void __tcp_push_pending_frames(struct sock *sk,
 static __inline__ void tcp_push_pending_frames(struct sock *sk,
                                               struct tcp_opt *tp)
 {
-       __tcp_push_pending_frames(sk, tp, tcp_current_mss(sk, 1, NULL), tp->nonagle);
+       __tcp_push_pending_frames(sk, tp, tcp_current_mss(sk, 1), tp->nonagle);
 }
 
 static __inline__ int tcp_may_send_now(struct sock *sk, struct tcp_opt *tp)
@@ -1553,7 +1551,7 @@ static __inline__ int tcp_may_send_now(struct sock *sk, struct tcp_opt *tp)
        struct sk_buff *skb = sk->sk_send_head;
 
        return (skb &&
-               tcp_snd_test(tp, skb, tcp_current_mss(sk, 1, NULL),
+               tcp_snd_test(tp, skb, tcp_current_mss(sk, 1),
                             tcp_skb_is_last(sk, skb) ? TCP_NAGLE_PUSH : tp->nonagle));
 }
 

diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 9df826c8e22b5f79b600ac600140e723b17a741b..36953ef7e6c2b0043806fb71b94d7febce93d11f 100644 (file)
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -590,14 +590,13 @@ static inline int forced_push(struct tcp_opt *tp)
 }
 
 static inline void skb_entail(struct sock *sk, struct tcp_opt *tp,
-                             struct sk_buff *skb, int tso_factor)
+                             struct sk_buff *skb)
 {
        skb->csum = 0;
        TCP_SKB_CB(skb)->seq = tp->write_seq;
        TCP_SKB_CB(skb)->end_seq = tp->write_seq;
        TCP_SKB_CB(skb)->flags = TCPCB_FLAG_ACK;
        TCP_SKB_CB(skb)->sacked = 0;
-       TCP_SKB_CB(skb)->tso_factor = tso_factor;
        __skb_queue_tail(&sk->sk_write_queue, skb);
        sk_charge_skb(sk, skb);
        if (!sk->sk_send_head)
@@ -633,7 +632,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse
                         size_t psize, int flags)
 {
        struct tcp_opt *tp = tcp_sk(sk);
-       int mss_now, mss_factor_now;
+       int mss_now;
        int err;
        ssize_t copied;
        long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
@@ -645,7 +644,7 @@ static ssize_t do_tcp_sendpages(struct sock *sk, struct page **pages, int poffse
 
        clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 
-       mss_now = tcp_current_mss(sk, !(flags&MSG_OOB), &mss_factor_now);
+       mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
        copied = 0;
 
        err = -EPIPE;
@@ -669,7 +668,7 @@ new_segment:
                        if (!skb)
                                goto wait_for_memory;
 
-                       skb_entail(sk, tp, skb, mss_factor_now);
+                       skb_entail(sk, tp, skb);
                        copy = mss_now;
                }
 
@@ -720,8 +719,7 @@ wait_for_memory:
                if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
                        goto do_error;
 
-               mss_now = tcp_current_mss(sk, !(flags&MSG_OOB),
-                                         &mss_factor_now);
+               mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
        }
 
 out:
@@ -782,7 +780,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
        struct tcp_opt *tp = tcp_sk(sk);
        struct sk_buff *skb;
        int iovlen, flags;
-       int mss_now, mss_factor_now;
+       int mss_now;
        int err, copied;
        long timeo;
 
@@ -800,7 +798,7 @@ int tcp_sendmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
        /* This should be in poll */
        clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags);
 
-       mss_now = tcp_current_mss(sk, !(flags&MSG_OOB), &mss_factor_now);
+       mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
 
        /* Ok commence sending. */
        iovlen = msg->msg_iovlen;
@@ -845,7 +843,7 @@ new_segment:
                                     NETIF_F_HW_CSUM))
                                        skb->ip_summed = CHECKSUM_HW;
 
-                               skb_entail(sk, tp, skb, mss_factor_now);
+                               skb_entail(sk, tp, skb);
                                copy = mss_now;
                        }
 
@@ -964,8 +962,7 @@ wait_for_memory:
                        if ((err = sk_stream_wait_memory(sk, &timeo)) != 0)
                                goto do_error;
 
-                       mss_now = tcp_current_mss(sk, !(flags&MSG_OOB),
-                                                 &mss_factor_now);
+                       mss_now = tcp_current_mss(sk, !(flags&MSG_OOB));
                }
        }
 

diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index f4ec161699062ebc89d680fb47ec388f7af24cb6..d7fb3cde4f20e107794322320cc11c5a4064aece 100644 (file)
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -974,7 +974,6 @@ tcp_sacktag_write_queue(struct sock *sk, struct sk_buff *ack_skb, u32 prior_snd_
                sk->sk_route_caps &= ~NETIF_F_TSO;
                sk->sk_no_largesend = 1;
                tp->mss_cache = tp->mss_cache_std;
-               tp->mss_tso_factor = 1;
        }
 
        if (!tcp_get_pcount(&tp->sacked_out))

diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 0fb326e84f284fef000e4fbc26e68fbdfb3e6b2b..73f12904c7c31d18569e39157ad7601839cdfd2b 100644 (file)
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -2076,7 +2076,6 @@ static int tcp_v4_init_sock(struct sock *sk)
        tp->snd_ssthresh = 0x7fffffff;  /* Infinity */
        tp->snd_cwnd_clamp = ~0;
        tp->mss_cache_std = tp->mss_cache = 536;
-       tp->mss_tso_factor = 1;
 
        tp->reordering = sysctl_tcp_reordering;
 

diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index 0a70d082028ce3ee035136744a79fd4565929a2d..336c7121b6b6a8dd8cb0d2bd5b84604d77af7c12 100644 (file)
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -422,6 +422,23 @@ void tcp_push_one(struct sock *sk, unsigned cur_mss)
        }
 }
 
+static void tcp_set_skb_tso_factor(struct sk_buff *skb, unsigned int mss,
+                                  unsigned int mss_std)
+{
+       if (skb->len <= mss_std) {
+               /* Avoid the costly divide in the normal
+                * non-TSO case.
+                */
+               TCP_SKB_CB(skb)->tso_factor = 1;
+       } else {
+               unsigned int factor;
+
+               factor = skb->len + (mss_std - 1);
+               factor /= mss;
+               TCP_SKB_CB(skb)->tso_factor = factor;
+       }
+}
+
 /* Function to create two new TCP segments.  Shrinks the given segment
  * to the specified size and appends a new segment with the rest of the
  * packet to the list.  This won't be called frequently, I hope. 
@@ -457,7 +474,6 @@ static int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len)
        TCP_SKB_CB(buff)->sacked =
                (TCP_SKB_CB(skb)->sacked &
                 (TCPCB_LOST | TCPCB_EVER_RETRANS | TCPCB_AT_TAIL));
-       TCP_SKB_CB(buff)->tso_factor = tp->mss_tso_factor;
        if (TCP_SKB_CB(buff)->sacked&TCPCB_LOST) {
                tcp_inc_pcount(&tp->lost_out, buff);
                tcp_inc_pcount(&tp->left_out, buff);
@@ -484,6 +500,10 @@ static int tcp_fragment(struct sock *sk, struct sk_buff *skb, u32 len)
         */
        TCP_SKB_CB(buff)->when = TCP_SKB_CB(skb)->when;
 
+       /* Fix up tso_factor for both original and new SKB.  */
+       tcp_set_skb_tso_factor(skb, tp->mss_cache, tp->mss_cache_std);
+       tcp_set_skb_tso_factor(buff, tp->mss_cache, tp->mss_cache_std);
+
        /* Link BUFF into the send queue. */
        __skb_append(skb, buff);
 
@@ -598,7 +618,6 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu)
        /* And store cached results */
        tp->pmtu_cookie = pmtu;
        tp->mss_cache = tp->mss_cache_std = mss_now;
-       tp->mss_tso_factor = 1;
 
        if (sk->sk_route_caps & NETIF_F_TSO) {
                int large_mss, factor;
@@ -617,7 +636,6 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu)
                        factor = tp->snd_cwnd;
 
                tp->mss_cache = mss_now * factor;
-               tp->mss_tso_factor = factor;
        }
 
        return mss_now;
@@ -634,7 +652,7 @@ int tcp_sync_mss(struct sock *sk, u32 pmtu)
 int tcp_write_xmit(struct sock *sk, int nonagle)
 {
        struct tcp_opt *tp = tcp_sk(sk);
-       unsigned int mss_now;
+       unsigned int mss_now, mss_std;
 
        /* If we are closed, the bytes will have to remain here.
         * In time closedown will finish, we empty the write queue and all
@@ -649,7 +667,8 @@ int tcp_write_xmit(struct sock *sk, int nonagle)
                 * We also handle things correctly when the user adds some
                 * IP options mid-stream.  Silly to do, but cover it.
                 */
-               mss_now = tcp_current_mss(sk, 1, NULL);
+               mss_now = tcp_current_mss(sk, 1);
+               mss_std = tp->mss_cache_std;
 
                while ((skb = sk->sk_send_head) &&
                       tcp_snd_test(tp, skb, mss_now,
@@ -658,7 +677,8 @@ int tcp_write_xmit(struct sock *sk, int nonagle)
                        if (skb->len > mss_now) {
                                if (tcp_fragment(sk, skb, mss_now))
                                        break;
-                       }
+                       } else
+                               tcp_set_skb_tso_factor(skb, mss_now, mss_std);
 
                        TCP_SKB_CB(skb)->when = tcp_time_stamp;
                        if (tcp_transmit_skb(sk, skb_clone(skb, GFP_ATOMIC)))
@@ -871,7 +891,7 @@ void tcp_simple_retransmit(struct sock *sk)
 {
        struct tcp_opt *tp = tcp_sk(sk);
        struct sk_buff *skb;
-       unsigned int mss = tcp_current_mss(sk, 0, NULL);
+       unsigned int mss = tcp_current_mss(sk, 0);
        int lost = 0;
 
        sk_stream_for_retrans_queue(skb, sk) {
@@ -916,7 +936,7 @@ void tcp_simple_retransmit(struct sock *sk)
 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
 {
        struct tcp_opt *tp = tcp_sk(sk);
-       unsigned int cur_mss = tcp_current_mss(sk, 0, NULL);
+       unsigned int cur_mss = tcp_current_mss(sk, 0);
        int err;
 
        /* Do not sent more than we queued. 1/4 is reserved for possible
@@ -934,7 +954,6 @@ int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb)
                        sk->sk_route_caps &= ~NETIF_F_TSO;
                        sk->sk_no_largesend = 1;
                        tp->mss_cache = tp->mss_cache_std;
-                       tp->mss_tso_factor = 1;
                }
 
                if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
@@ -1125,7 +1144,7 @@ void tcp_send_fin(struct sock *sk)
         * unsent frames.  But be careful about outgoing SACKS
         * and IP options.
         */
-       mss_now = tcp_current_mss(sk, 1, NULL);
+       mss_now = tcp_current_mss(sk, 1);
 
        if (sk->sk_send_head != NULL) {
                TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_FIN;
@@ -1516,8 +1535,9 @@ int tcp_write_wakeup(struct sock *sk)
                if ((skb = sk->sk_send_head) != NULL &&
                    before(TCP_SKB_CB(skb)->seq, tp->snd_una+tp->snd_wnd)) {
                        int err;
-                       int mss = tcp_current_mss(sk, 0, NULL);
-                       int seg_size = tp->snd_una+tp->snd_wnd-TCP_SKB_CB(skb)->seq;
+                       unsigned int mss = tcp_current_mss(sk, 0);
+                       unsigned int mss_std = tp->mss_cache_std;
+                       unsigned int seg_size = tp->snd_una+tp->snd_wnd-TCP_SKB_CB(skb)->seq;
 
                        if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
                                tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
@@ -1538,9 +1558,10 @@ int tcp_write_wakeup(struct sock *sk)
                                        sk->sk_no_largesend = 1;
                                        sk->sk_route_caps &= ~NETIF_F_TSO;
                                        tp->mss_cache = tp->mss_cache_std;
-                                       tp->mss_tso_factor = 1;
                                }
-                       }
+                       } else
+                               tcp_set_skb_tso_factor(skb, mss, mss_std);
+
                        TCP_SKB_CB(skb)->flags |= TCPCB_FLAG_PSH;
                        TCP_SKB_CB(skb)->when = tcp_time_stamp;
                        err = tcp_transmit_skb(sk, skb_clone(skb, GFP_ATOMIC));

diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index 0ecd0d8dfa0f5028e66f3c3366ce831666485319..ebed7e197aac2df00c0511135be3401ccb24e6d7 100644 (file)
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -1930,7 +1930,6 @@ static int tcp_v6_init_sock(struct sock *sk)
        tp->snd_ssthresh = 0x7fffffff;
        tp->snd_cwnd_clamp = ~0;
        tp->mss_cache_std = tp->mss_cache = 536;
-       tp->mss_tso_factor = 1;
 
        tp->reordering = sysctl_tcp_reordering;