switch from sysctl to setsockopt for setting recv/send buffer sizes (s…

…olana-labs#3594)

rm default buffer size from sysctl and switch to setsockopt at runtime

ci/setup-new-buildkite-agent/setup-procfs-knobs.sh

@@ -11,9 +11,7 @@ ensure_env || exit 1
 cat > /etc/sysctl.d/20-solana-node.conf <<EOF
 
 # Solana networking requirements
-net.core.rmem_default=134217728
 net.core.rmem_max=134217728
-net.core.wmem_default=134217728
 net.core.wmem_max=134217728
 
 # Solana earlyoom setup

core/src/system_monitor_service.rs

@@ -402,15 +402,7 @@ enum InterestingLimit {
 #[cfg(target_os = "linux")]
 const INTERESTING_LIMITS: &[(&str, InterestingLimit)] = &[
     ("net.core.rmem_max", InterestingLimit::Recommend(134217728)),
-    (
-        "net.core.rmem_default",
-        InterestingLimit::Recommend(134217728),
-    ),
     ("net.core.wmem_max", InterestingLimit::Recommend(134217728)),
-    (
-        "net.core.wmem_default",
-        InterestingLimit::Recommend(134217728),
-    ),
     ("vm.max_map_count", InterestingLimit::Recommend(1000000)),
     ("net.core.optmem_max", InterestingLimit::QueryOnly),
     ("net.core.netdev_max_backlog", InterestingLimit::QueryOnly),

docs/src/operations/guides/validator-start.md

@@ -48,10 +48,8 @@ the following commands.
 
 ```bash
 sudo bash -c "cat >/etc/sysctl.d/21-agave-validator.conf <<EOF
-# Increase UDP buffer sizes
-net.core.rmem_default = 134217728
+# Increase max UDP buffer sizes
 net.core.rmem_max = 134217728
-net.core.wmem_default = 134217728
 net.core.wmem_max = 134217728
 
 # Increase memory mapped files limit

docs/src/operations/setup-a-validator.md

@@ -308,10 +308,8 @@ not start without the settings below.
 
 ```bash
 sudo bash -c "cat >/etc/sysctl.d/21-agave-validator.conf <<EOF
-# Increase UDP buffer sizes
-net.core.rmem_default = 134217728
+# Increase max UDP buffer sizes
 net.core.rmem_max = 134217728
-net.core.wmem_default = 134217728
 net.core.wmem_max = 134217728
 
 # Increase memory mapped files limit

net-utils/src/lib.rs

@@ -34,6 +34,11 @@ pub type PortRange = (u16, u16);
 pub const VALIDATOR_PORT_RANGE: PortRange = (8000, 10_000);
 pub const MINIMUM_VALIDATOR_PORT_RANGE_WIDTH: u16 = 17; // VALIDATOR_PORT_RANGE must be at least this wide
 
+#[cfg(not(any(windows, target_os = "ios")))]
+const DEFAULT_RECV_BUFFER_SIZE: usize = 64 * 1024 * 1024; // 64 MB - Doubled to 128MB by the kernel
+#[cfg(not(any(windows, target_os = "ios")))]
+const DEFAULT_SEND_BUFFER_SIZE: usize = 64 * 1024 * 1024; // 64 MB - Doubled to 128MB by the kernel
+
 pub(crate) const HEADER_LENGTH: usize = 4;
 pub(crate) const IP_ECHO_SERVER_RESPONSE_LENGTH: usize = HEADER_LENGTH + 23;
 
@@ -421,6 +426,10 @@ fn udp_socket_with_config(config: SocketConfig) -> io::Result<Socket> {
 
     let sock = Socket::new(Domain::IPV4, Type::DGRAM, None)?;
 
+    // Set recv and send buffer sizes to 128MB
+    sock.set_recv_buffer_size(DEFAULT_RECV_BUFFER_SIZE)?;
+    sock.set_send_buffer_size(DEFAULT_SEND_BUFFER_SIZE)?;
+
     if reuseport {
         setsockopt(&sock, ReusePort, &true).ok();
     }

net/scripts/network-config.sh

@@ -4,10 +4,8 @@ set -ex
 [[ $(uname) = Linux ]] || exit 1
 [[ $USER = root ]] || exit 1
 
-sudo sysctl -w net.core.rmem_default=134217728
 sudo sysctl -w net.core.rmem_max=134217728
 
-sudo sysctl -w net.core.wmem_default=134217728
 sudo sysctl -w net.core.wmem_max=134217728
 
 # Increase memory mapped files limit