utils/ulimit: ensure default/runtime is valid or log/error (#1232)

Author: hexfusion

app/process/process.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ava-labs/avalanchego/utils/dynamicip"
 	"github.com/ava-labs/avalanchego/utils/logging"
 	"github.com/ava-labs/avalanchego/utils/perms"
+	"github.com/ava-labs/avalanchego/utils/ulimit"
 	"github.com/ava-labs/avalanchego/version"
 )
 
@@ -71,6 +72,12 @@ func (p *process) Start() error {
 		return err
 	}
 
+	// update fd limit
+	fdLimit := p.config.FdLimit
+	if err := ulimit.Set(fdLimit, log); err != nil {
+		return err
+	}
+
 	// start the db manager
 	var dbManager manager.Manager
 	switch p.config.DatabaseConfig.Name {

config/config.go

@@ -43,7 +43,6 @@ import (
 	"github.com/ava-labs/avalanchego/utils/profiler"
 	"github.com/ava-labs/avalanchego/utils/storage"
 	"github.com/ava-labs/avalanchego/utils/timer"
-	"github.com/ava-labs/avalanchego/utils/ulimit"
 	"github.com/ava-labs/avalanchego/vms"
 )
 
@@ -1107,16 +1106,12 @@ func GetNodeConfig(v *viper.Viper, buildDir string) (node.Config, error) {
 	}
 
 	// File Descriptor Limit
-	fdLimit := v.GetUint64(FdLimitKey)
-	if err := ulimit.Set(fdLimit); err != nil {
-		return node.Config{}, fmt.Errorf("failed to set fd limit correctly due to: %w", err)
-	}
+	nodeConfig.FdLimit = v.GetUint64(FdLimitKey)
 
 	// Tx Fee
 	nodeConfig.TxFeeConfig = getTxFeeConfig(v, nodeConfig.NetworkID)
 
 	// Genesis Data
-
 	nodeConfig.GenesisBytes, nodeConfig.AvaxAssetID, err = getGenesisData(v, nodeConfig.NetworkID)
 	if err != nil {
 		return node.Config{}, fmt.Errorf("unable to load genesis file: %w", err)

node/config.go

@@ -169,6 +169,9 @@ type Config struct {
 	// Plugin directory
 	PluginDir string `json:"pluginDir"`
 
+	// File Descriptor Limit
+	FdLimit uint64 `json:"fdLimit"`
+
 	// Consensus configuration
 	ConsensusParams avalanche.Parameters `json:"consensusParams"`
 

utils/ulimit/ulimit.go

@@ -0,0 +1,55 @@
+// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build freebsd
+// +build freebsd
+
+package ulimit
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/ava-labs/avalanchego/utils/logging"
+)
+
+const DefaultFDLimit = 32 * 1024
+
+// Set attempts to bump the Rlimit which has a soft (Cur) and a hard (Max) value.
+// The soft limit is what is used by the kernel to report EMFILE errors. The hard
+// limit is a secondary limit which the process can be bumped to without additional
+// privileges. Bumping the Max limit further would require superuser privileges.
+// If the value is below the recommendation warn on start.
+// see: http://0pointer.net/blog/file-descriptor-limits.html
+func Set(max uint64, log logging.Logger) error {
+	// Note: BSD Rlimit is type int64
+	// ref: https://cs.opensource.google/go/x/sys/+/b874c991:unix/ztypes_freebsd_amd64.go
+	bsdMax := int64(max)
+	var rLimit syscall.Rlimit
+	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	if bsdMax > rLimit.Max {
+		return fmt.Errorf("error fd-limit: (%d) greater than max: (%d)", limit, rLimit.Max)
+	}
+
+	rLimit.Cur = bsdMax
+
+	// set new limit
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error setting fd-limit: %w", err)
+	}
+
+	// verify limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	if rLimit.Cur < DefaultFDLimit {
+		log.Warn("fd-limit: (%d) is less than recommended: (%d) and could result in reduced performance", rLimit.Cur, DefaultFDLimit)
+	}
+
+	return nil
+}

utils/ulimit/ulimit_bsd.go

@@ -0,0 +1,56 @@
+// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build darwin
+// +build darwin
+
+package ulimit
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/ava-labs/avalanchego/utils/logging"
+)
+
+const DefaultFDLimit = 10 * 1024
+
+// Set attempts to bump the Rlimit which has a soft (Cur) and a hard (Max) value.
+// The soft limit is what is used by the kernel to report EMFILE errors. The hard
+// limit is a secondary limit which the process can be bumped to without additional
+// privileges. Bumping the Max limit further would require superuser privileges.
+// If the value is below the recommendation warn on start.
+// see: http://0pointer.net/blog/file-descriptor-limits.html
+func Set(max uint64, log logging.Logger) error {
+	var rLimit syscall.Rlimit
+	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
+	if err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	// Darwin max number of FDs to allocatable for darwin systems.
+	// The max file limit is 10240, even though the max returned by
+	// Getrlimit is 1<<63-1. This is OPEN_MAX in sys/syslimits.h.
+	// See https://github.com/golang/go/issues/30401
+	if max > DefaultFDLimit {
+		return fmt.Errorf("error fd-limit: (%d) greater than max: (%d)", max, DefaultFDLimit)
+	}
+
+	rLimit.Cur = max
+
+	// set new limit
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error setting fd-limit: %w", err)
+	}
+
+	// verify limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	if rLimit.Cur < DefaultFDLimit {
+		log.Warn("fd-limit: (%d) is less than recommended: (%d) and could result in reduced performance", rLimit.Cur, DefaultFDLimit)
+	}
+
+	return nil
+}

utils/ulimit/ulimit_darwin.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+package ulimit
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/ava-labs/avalanchego/utils/logging"
+)
+
+// Test_SetDefault performs sanity checks for the os default.
+func Test_SetDefault(t *testing.T) {
+	assert := assert.New(t)
+	err := Set(DefaultFDLimit, logging.NoLog{})
+	assert.NoErrorf(err, "default fd-limit failed %v", err)
+}

utils/ulimit/ulimit_non_unix.go.go

@@ -1,51 +1,51 @@
-// This file was taken from:
-// https://github.com/OpenBazaar/openbazaar-go/blob/master/core/ulimit_non_unix.go
+// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
 
-//go:build darwin || linux || netbsd || openbsd
-// +build darwin linux netbsd openbsd
+//go:build linux || netbsd || openbsd
+// +build linux netbsd openbsd
 
 package ulimit
 
 import (
 	"fmt"
-	"runtime"
 	"syscall"
+
+	"github.com/ava-labs/avalanchego/utils/logging"
 )
 
-// Set the file descriptor limit
-func Set(limit uint64) error {
+const DefaultFDLimit = 32 * 1024
+
+// Set attempts to bump the Rlimit which has a soft (Cur) and a hard (Max) value.
+// The soft limit is what is used by the kernel to report EMFILE errors. The hard
+// limit is a secondary limit which the process can be bumped to without additional
+// privileges. Bumping the Max limit further would require superuser privileges.
+// If the current Max is below our recommendation we will warn on start.
+// see: http://0pointer.net/blog/file-descriptor-limits.html
+func Set(max uint64, log logging.Logger) error {
 	var rLimit syscall.Rlimit
 	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit)
 	if err != nil {
 		return fmt.Errorf("error getting rlimit: %w", err)
 	}
 
-	oldMax := rLimit.Max
-	if rLimit.Cur < limit {
-		if rLimit.Max < limit {
-			rLimit.Max = limit
-		}
-		rLimit.Cur = limit
+	if max > rLimit.Max {
+		return fmt.Errorf("error fd-limit: (%d) greater than max: (%d)", max, rLimit.Max)
 	}
 
-	// If we're on darwin, work around the fact that Getrlimit reports the wrong
-	// value. See https://github.com/golang/go/issues/30401
-	if runtime.GOOS == "darwin" && rLimit.Cur > 10240 {
-		// The max file limit is 10240, even though the max returned by
-		// Getrlimit is 1<<63-1. This is OPEN_MAX in sys/syslimits.h.
-		rLimit.Max = 10240
-		rLimit.Cur = 10240
-	}
+	rLimit.Cur = max
 
-	// Try updating the limit. If it fails, try using the previous maximum
-	// instead of our new maximum. Not all users have permissions to increase
-	// the maximum.
+	// set new limit
 	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
-		rLimit.Max = oldMax
-		rLimit.Cur = oldMax
-		if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
-			return fmt.Errorf("error setting ulimit: %w", err)
-		}
+		return fmt.Errorf("error setting fd-limit: %w", err)
+	}
+
+	// verify limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rLimit); err != nil {
+		return fmt.Errorf("error getting rlimit: %w", err)
+	}
+
+	if rLimit.Cur < DefaultFDLimit {
+		log.Warn("fd-limit: (%d) is less than recommended: (%d) and could result in reduced performance", rLimit.Cur, DefaultFDLimit)
 	}
 
 	return nil

utils/ulimit/ulimit_test.go

@@ -0,0 +1,19 @@
+// Copyright (C) 2019-2021, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+
+//go:build windows
+// +build windows
+
+package ulimit
+
+import "github.com/ava-labs/avalanchego/utils/logging"
+
+const DefaultFDLimit = 16384
+
+// Set is a no-op for windows and will warn if the default is not used.
+func Set(max uint64, log logging.Logger) error {
+	if max != DefaultFDLimit {
+		log.Warn("fd-limit is not supported for windows")
+	}
+	return nil
+}