diff options
| author | Eric Dumazet <edumazet@google.com> | 2016-12-03 11:14:52 -0800 |
|---|---|---|
| committer | David S. Miller <davem@davemloft.net> | 2016-12-05 13:32:22 -0500 |
| commit | b223feb9de2a65c533ff95c08e834fa732906ea5 (patch) | |
| tree | 0327e31d3fe77301791b56fd3a4082cb3229737e /net/ipv4/tcp_output.c | |
| parent | 408f0a6c21e124cc4f6c7aa370b38aa47e55428d (diff) | |
tcp: tsq: add shortcut in tcp_tasklet_func()
Under high stress, I've seen tcp_tasklet_func() consuming
~700 usec, handling ~150 tcp sockets.
By setting TCP_TSQ_DEFERRED in tcp_wfree(), we give a chance
for other cpus/threads entering tcp_write_xmit() to grab it,
allowing tcp_tasklet_func() to skip sockets that already did
an xmit cycle.
In the future, we might give to ACK processing an increased
budget to reduce even more tcp_tasklet_func() amount of work.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/ipv4/tcp_output.c')
| -rw-r--r-- | net/ipv4/tcp_output.c | 22 |
1 files changed, 12 insertions, 10 deletions
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 4adaf8e1bb63..fa23b688a6f3 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -767,19 +767,19 @@ static void tcp_tasklet_func(unsigned long data) list_for_each_safe(q, n, &list) { tp = list_entry(q, struct tcp_sock, tsq_node); list_del(&tp->tsq_node); + clear_bit(TSQ_QUEUED, &tp->tsq_flags); sk = (struct sock *)tp; - bh_lock_sock(sk); - - if (!sock_owned_by_user(sk)) { - tcp_tsq_handler(sk); - } else { - /* defer the work to tcp_release_cb() */ - set_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags); + if (!sk->sk_lock.owned && + test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags)) { + bh_lock_sock(sk); + if (!sock_owned_by_user(sk)) { + clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags); + tcp_tsq_handler(sk); + } + bh_unlock_sock(sk); } - bh_unlock_sock(sk); - clear_bit(TSQ_QUEUED, &tp->tsq_flags); sk_free(sk); } } @@ -884,7 +884,7 @@ void tcp_wfree(struct sk_buff *skb) if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED)) goto out; - nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED; + nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED | TCPF_TSQ_DEFERRED; nval = cmpxchg(&tp->tsq_flags, oval, nval); if (nval != oval) continue; @@ -2229,6 +2229,8 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle, unlikely(tso_fragment(sk, skb, limit, mss_now, gfp))) break; + if (test_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags)) + clear_bit(TCP_TSQ_DEFERRED, &tp->tsq_flags); if (tcp_small_queue_check(sk, skb, 0)) break; |