pick defaults based on ram size

elastic · Oct 5, 2021 · b5a3e14 · b5a3e14
1 parent cb956b8
commit b5a3e14
Show file tree

Hide file tree

Showing 14 changed files with 634 additions and 25 deletions.
diff --git a/Makefile b/Makefile
@@ -58,6 +58,7 @@ generate: ## - Generate schema models
 .PHONY: check
 check: ## - Run all checks
 	@$(MAKE) generate
+	@$(MAKE) defaults
 	@$(MAKE) check-headers
 	@$(MAKE) check-go
 	@$(MAKE) notice
@@ -88,6 +89,11 @@ notice: ## - Generates the NOTICE.txt file.
 		-noticeOut NOTICE.txt \
 		-depsOut ""
 
+.PHONY: defaults
+defaults: ## -Generate defaults based on limits files.
+	@echo "Generating env_defaults.go"
+	@go run dev-tools/buildlimits/buildlimits.go --in "internal/pkg/config/defaults/*.yml" --out internal/pkg/config/env_defaults.go
+
 .PHONY: check-no-changes
 check-no-changes:
 	@git diff | cat

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -2167,6 +2167,45 @@ SOFTWARE.
 
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/pbnjay/memory
+Version: v0.0.0-20210728143218-7b4eea64cf58
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/pbnjay/memory@v0.0.0-20210728143218-7b4eea64cf58/LICENSE:
+
+BSD 3-Clause License
+
+Copyright (c) 2017, Jeremy Jay
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/pkg/errors
 Version: v0.9.1

diff --git a/dev-tools/buildlimits/buildlimits.go b/dev-tools/buildlimits/buildlimits.go
diff --git a/go.mod b/go.mod
@@ -17,6 +17,7 @@ require (
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/mailru/easyjson v0.7.7
 	github.com/miolini/datacounter v1.0.2
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/pkg/errors v0.9.1
 	github.com/rs/xid v1.2.1
 	github.com/rs/zerolog v1.19.0

diff --git a/go.sum b/go.sum
@@ -621,6 +621,8 @@ github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc=
 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso=
 github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=

diff --git a/internal/pkg/config/cache.go b/internal/pkg/config/cache.go
@@ -9,13 +9,11 @@ import (
 )
 
 const (
-	defaultCacheNumCounters = 500000           // 10x times expected count
-	defaultCacheMaxCost     = 50 * 1024 * 1024 // 50MiB cache size
-	defaultActionTTL        = time.Minute * 5
-	defaultEnrollKeyTTL     = time.Minute
-	defaultArtifactTTL      = time.Hour * 24
-	defaultApiKeyTTL        = time.Minute * 15 // ApiKey validation is a bottleneck.
-	defaultApiKeyJitter     = time.Minute * 5  // Jitter allows some randomness on ApiKeyTTL, zero to disable
+	defaultActionTTL    = time.Minute * 5
+	defaultEnrollKeyTTL = time.Minute
+	defaultArtifactTTL  = time.Hour * 24
+	defaultApiKeyTTL    = time.Minute * 15 // ApiKey validation is a bottleneck.
+	defaultApiKeyJitter = time.Minute * 5  // Jitter allows some randomness on ApiKeyTTL, zero to disable
 )
 
 type Cache struct {
@@ -29,8 +27,10 @@ type Cache struct {
 }
 
 func (c *Cache) InitDefaults() {
-	c.NumCounters = defaultCacheNumCounters
-	c.MaxCost = defaultCacheMaxCost
+	l := loadLimits().Cache
+
+	c.NumCounters = l.NumCounters
+	c.MaxCost = l.MaxCost
 	c.ActionTTL = defaultActionTTL
 	c.EnrollKeyTTL = defaultEnrollKeyTTL
 	c.ArtifactTTL = defaultArtifactTTL

diff --git a/internal/pkg/config/defaults/1024_limits.yml b/internal/pkg/config/defaults/1024_limits.yml
@@ -0,0 +1,23 @@
+max_ram: 1024
+cache_limits:
+  num_counters: 20000
+  max_cost: 20971520
+server_limits:
+  policy_throttle: 50ms
+  max_connections: 7000
+  checkin_limit:
+    interval: 5ms
+    burst: 500
+    max: 5001
+  artifact_limit: 
+    interval: 5ms
+    burst: 500
+    max: 1000
+  enroll_limit: 
+    interval: 20ms
+    burst: 50
+    max: 100
+  ack_limit:
+    interval: 4ms
+    burst: 500
+    max: 1000