diff options
| author | Eric Dumazet <edumazet@google.com> | 2024-04-29 13:40:25 +0000 |
|---|---|---|
| committer | Jakub Kicinski <kuba@kernel.org> | 2024-04-30 18:46:52 -0700 |
| commit | c204fef97ee62ca3310f43d12bbda4eb10266c7c (patch) | |
| tree | 6cc12ade57310fb99b3a64b72516bee33488c1b3 | |
| parent | f3d93817fba30a8d3508fa990405039c0820dca3 (diff) | |
net: move sysctl_mem_pcpu_rsv to net_hotdata
sysctl_mem_pcpu_rsv is used in TCP fast path,
move it to net_hodata for better cache locality.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Link: https://lore.kernel.org/r/20240429134025.1233626-6-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
| -rw-r--r-- | include/net/hotdata.h | 1 | ||||
| -rw-r--r-- | include/net/proto_memory.h | 6 | ||||
| -rw-r--r-- | net/core/hotdata.c | 5 | ||||
| -rw-r--r-- | net/core/sock.c | 1 | ||||
| -rw-r--r-- | net/core/sysctl_net_core.c | 2 |
5 files changed, 8 insertions, 7 deletions
diff --git a/include/net/hotdata.h b/include/net/hotdata.h index 290499f72e18..30e9570beb2a 100644 --- a/include/net/hotdata.h +++ b/include/net/hotdata.h @@ -40,6 +40,7 @@ struct net_hotdata { int dev_rx_weight; int sysctl_max_skb_frags; int sysctl_skb_defer_max; + int sysctl_mem_pcpu_rsv; }; #define inet_ehash_secret net_hotdata.tcp_protocol.secret diff --git a/include/net/proto_memory.h b/include/net/proto_memory.h index 41404d4bb6f0..a6ab2f4f5e28 100644 --- a/include/net/proto_memory.h +++ b/include/net/proto_memory.h @@ -3,10 +3,10 @@ #define _PROTO_MEMORY_H #include <net/sock.h> +#include <net/hotdata.h> /* 1 MB per cpu, in page units */ #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT)) -extern int sysctl_mem_pcpu_rsv; static inline bool sk_has_memory_pressure(const struct sock *sk) { @@ -65,7 +65,7 @@ sk_memory_allocated_add(const struct sock *sk, int val) val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val); - if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv))) + if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv))) proto_memory_pcpu_drain(proto); } @@ -76,7 +76,7 @@ sk_memory_allocated_sub(const struct sock *sk, int val) val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val); - if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv))) + if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv))) proto_memory_pcpu_drain(proto); } diff --git a/net/core/hotdata.c b/net/core/hotdata.c index a359ff160d54..d0aaaaa556f2 100644 --- a/net/core/hotdata.c +++ b/net/core/hotdata.c @@ -1,9 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later -#include <net/hotdata.h> #include <linux/cache.h> #include <linux/jiffies.h> #include <linux/list.h> - +#include <net/hotdata.h> +#include <net/proto_memory.h> struct net_hotdata net_hotdata __cacheline_aligned = { .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base), @@ -20,5 +20,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = { .dev_rx_weight = 64, .sysctl_max_skb_frags = MAX_SKB_FRAGS, .sysctl_skb_defer_max = 64, + .sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE }; EXPORT_SYMBOL(net_hotdata); diff --git a/net/core/sock.c b/net/core/sock.c index e0692b752369..8d6e638b5426 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -284,7 +284,6 @@ __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX; EXPORT_SYMBOL(sysctl_rmem_max); __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX; __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX; -int sysctl_mem_pcpu_rsv __read_mostly = SK_MEMORY_PCPU_RESERVE; int sysctl_tstamp_allow_data __read_mostly = 1; diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c index a452a330d0ed..6da5995ac86a 100644 --- a/net/core/sysctl_net_core.c +++ b/net/core/sysctl_net_core.c @@ -416,7 +416,7 @@ static struct ctl_table net_core_table[] = { }, { .procname = "mem_pcpu_rsv", - .data = &sysctl_mem_pcpu_rsv, + .data = &net_hotdata.sysctl_mem_pcpu_rsv, .maxlen = sizeof(int), .mode = 0644, .proc_handler = proc_dointvec_minmax, |