add a function to automatically scale limits to system memory / FDs

go.mod

@@ -6,8 +6,10 @@ require (
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/libp2p/go-libp2p-core v0.19.0
 	github.com/multiformats/go-multiaddr v0.6.0
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/stretchr/testify v1.8.0
 	go.opencensus.io v0.23.0
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c
 )
 
 require (
@@ -37,6 +39,5 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.19.1 // indirect
 	golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf // indirect
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -122,6 +122,8 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

limit_defaults.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/libp2p/go-libp2p-core/protocol"
+
+	"github.com/pbnjay/memory"
 )
 
 type baseLimitConfig struct {
@@ -176,6 +178,13 @@ func (cfg *ScalingLimitConfig) Scale(memory int64, numFD int) LimitConfig {
 	return lc
 }
 
+func (cfg *ScalingLimitConfig) AutoScale() LimitConfig {
+	return cfg.Scale(
+		int64(memory.TotalMemory())/8,
+		getNumFDs()/2,
+	)
+}
+
 // factor is the number of MBs above the minimum (128 MB)
 func scale(base BaseLimit, inc BaseLimitIncrease, factor int, numFD int) BaseLimit {
 	l := BaseLimit{
@@ -188,7 +197,7 @@ func scale(base BaseLimit, inc BaseLimitIncrease, factor int, numFD int) BaseLim
 		Memory:          base.Memory + (inc.Memory*int64(factor))>>10,
 		FD:              base.FD,
 	}
-	if inc.FDFraction > 0 {
+	if inc.FDFraction > 0 && numFD > 0 {
 		l.FD = int(inc.FDFraction * float64(numFD))
 	}
 	return l

limit_test.go

@@ -1,11 +1,21 @@
 package rcmgr
 
 import (
+	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
+func TestFileDescriptorCounting(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("can't read file descriptors on Windows")
+	}
+	n := getNumFDs()
+	require.NotZero(t, n)
+	require.Less(t, n, int(1e6))
+}
+
 func TestScaling(t *testing.T) {
 	base := BaseLimit{
 		Streams:         100,

sys_not_unix.go

@@ -0,0 +1,11 @@
+//go:build !linux && !darwin
+
+package rcmgr
+
+import "runtime"
+
+// TODO: figure out how to get the number of file descriptors on Windows and other systems
+func getNumFDs() int {
+	log.Warnf("cannot determine number of file descriptors on %s", runtime.GOOS)
+	return 0
+}

sys_unix.go

@@ -0,0 +1,17 @@
+//go:build linux || darwin
+// +build linux darwin
+
+package rcmgr
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+func getNumFDs() int {
+	var l unix.Rlimit
+	if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &l); err != nil {
+		log.Errorw("failed to get fd limit", "error", err)
+		return 0
+	}
+	return int(l.Cur)
+}