Added ldclient evaluation benchmarks

[alexwforsythe]

Requirements

• I have added test coverage for new or changed functionality
• I have followed the repository's pull request submission guidelines
• I have validated my changes against all supported platform versions

Related issues

N/A

Describe the solution you've provided

I've added benchmarks to measure the throughput and memory allocations of several evaluation functions.

Describe alternatives you've considered

N/A

Additional context

Results:

Running tool: /usr/local/go/bin/go test -benchmem -run=^$ gopkg.in/launchdarkly/go-server-sdk.v4 -bench ^(BenchmarkBoolVariation|BenchmarkIntVariation|BenchmarkStringVariation|BenchmarkJSONVariation)$

goos: linux
goarch: amd64
pkg: gopkg.in/launchdarkly/go-server-sdk.v4
BenchmarkBoolVariation/{numUsers:1000_numFlags:1000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16         	 1212894	      1050 ns/op	     593 B/op	       4 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:10000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16       	 1407657	       858 ns/op	     579 B/op	       4 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:10000_numVariations:2_numTargets:10_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16      	 1367804	       895 ns/op	     581 B/op	       4 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:}-16        	  120196	      9596 ns/op	    6863 B/op	      65 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:3_prereqsWidth:0_prereqsDepth:0_operator:}-16        	  117769	      9706 ns/op	    6868 B/op	      65 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:5_numClauses:3_prereqsWidth:0_prereqsDepth:0_operator:}-16        	   26331	     44985 ns/op	   32055 B/op	     309 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:5_prereqsDepth:1_operator:}-16        	  110031	     10680 ns/op	    7705 B/op	      69 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:1_prereqsDepth:5_operator:}-16        	  108960	     10678 ns/op	    7704 B/op	      69 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:2_prereqsDepth:2_operator:}-16        	  658082	      1913 ns/op	    1405 B/op	       8 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:5_prereqsDepth:5_operator:}-16        	   76873	     15517 ns/op	   14525 B/op	      86 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:greaterThan}-16         	 1000000	      1092 ns/op	     599 B/op	       7 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:contains}-16            	 1000000	      1044 ns/op	     591 B/op	       5 allocs/op
BenchmarkBoolVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:in}-16                  	  995553	      1146 ns/op	     592 B/op	       5 allocs/op
BenchmarkIntVariation/{numUsers:1000_numFlags:1000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16                      	 1351836	       915 ns/op	     590 B/op	       5 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:10000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16                    	 1338130	       922 ns/op	     591 B/op	       5 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:10000_numVariations:2_numTargets:10_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16                   	 1349911	       934 ns/op	     590 B/op	       5 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:}-16                     	  117642	      9812 ns/op	    6871 B/op	      66 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:3_prereqsWidth:0_prereqsDepth:0_operator:}-16                     	  117109	      9951 ns/op	    6874 B/op	      66 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:5_numClauses:3_prereqsWidth:0_prereqsDepth:0_operator:}-16                     	   26042	     44585 ns/op	   32022 B/op	     310 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:5_prereqsDepth:1_operator:}-16                     	  109071	     10850 ns/op	    7712 B/op	      70 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:1_prereqsDepth:5_operator:}-16                     	  108716	     10636 ns/op	    7713 B/op	      70 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:2_prereqsDepth:2_operator:}-16                     	  683667	      1875 ns/op	    1407 B/op	       9 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:5_prereqsDepth:5_operator:}-16                     	   76902	     15611 ns/op	   14530 B/op	      87 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:greaterThan}-16          	 1000000	      1139 ns/op	     607 B/op	       8 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:contains}-16             	 1000000	      1052 ns/op	     599 B/op	       6 allocs/op
BenchmarkIntVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:in}-16                   	  980880	      1166 ns/op	     601 B/op	       6 allocs/op
BenchmarkStringVariation/{numUsers:1000_numFlags:1000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16                   	 1000000	      1001 ns/op	     607 B/op	       6 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:10000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16                 	 1277961	      1000 ns/op	     620 B/op	       6 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:10000_numVariations:2_numTargets:10_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16                	 1236304	       952 ns/op	     623 B/op	       6 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:}-16                  	  115327	     10022 ns/op	    6893 B/op	      67 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:3_prereqsWidth:0_prereqsDepth:0_operator:}-16                  	  113648	     10062 ns/op	    6893 B/op	      67 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:5_numClauses:3_prereqsWidth:0_prereqsDepth:0_operator:}-16                  	   26416	     45946 ns/op	   32024 B/op	     311 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:5_prereqsDepth:1_operator:}-16                  	  105981	     11005 ns/op	    7728 B/op	      73 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:1_prereqsDepth:5_operator:}-16                  	  107774	     10939 ns/op	    7770 B/op	      73 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:2_prereqsDepth:2_operator:}-16                  	  624622	      2037 ns/op	    1478 B/op	      12 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:5_prereqsDepth:5_operator:}-16                  	   76010	     15926 ns/op	   14594 B/op	      90 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:greaterThan}-16       	  985740	      1199 ns/op	     632 B/op	       9 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:contains}-16          	 1000000	      1159 ns/op	     623 B/op	       7 allocs/op
BenchmarkStringVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:in}-16                	  935442	      1268 ns/op	     629 B/op	       7 allocs/op
BenchmarkJSONVariation/{numUsers:1000_numFlags:1000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16                     	 1000000	      1056 ns/op	     703 B/op	       8 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:10000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16                   	 1000000	      1064 ns/op	     703 B/op	       8 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:10000_numVariations:2_numTargets:10_numRules:0_numClauses:0_prereqsWidth:0_prereqsDepth:0_operator:}-16                  	 1000000	      1094 ns/op	     703 B/op	       8 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:}-16                    	  116539	      9967 ns/op	    6980 B/op	      69 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:3_prereqsWidth:0_prereqsDepth:0_operator:}-16                    	  115340	     10069 ns/op	    6984 B/op	      69 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:5_numClauses:3_prereqsWidth:0_prereqsDepth:0_operator:}-16                    	   25959	     45521 ns/op	   32118 B/op	     313 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:5_prereqsDepth:1_operator:}-16                    	  105205	     11329 ns/op	    7918 B/op	      77 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:1_prereqsDepth:5_operator:}-16                    	  103836	     11189 ns/op	    7920 B/op	      77 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:1_numRules:0_numClauses:0_prereqsWidth:2_prereqsDepth:2_operator:}-16                    	  550192	      2267 ns/op	    1694 B/op	      16 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:5_prereqsDepth:5_operator:}-16                    	   73746	     16134 ns/op	   14800 B/op	      94 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:greaterThan}-16         	  908146	      1301 ns/op	     736 B/op	      11 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:contains}-16            	  936613	      1935 ns/op	     725 B/op	       9 allocs/op
BenchmarkJSONVariation/{numUsers:10000_numFlags:1000_numVariations:2_numTargets:0_numRules:1_numClauses:1_prereqsWidth:0_prereqsDepth:0_operator:in}-16                  	  854199	      1363 ns/op	     713 B/op	       9 allocs/op
PASS
ok  	gopkg.in/launchdarkly/go-server-sdk.v4	76.605s

[eli-darkly]

Thanks. This partly duplicates some work we've been doing to compare the performance of the current release with our upcoming 5.0 rewrite, but it looks like a more thorough approach.

I do have some questions about the intended behavior, since there aren't any comments in the part that sets up the flag configurations. You're generating a bunch of rules and clauses, but (at first glance; I haven't stepped through this in detail, it is a bit hard to follow) I'm not sure that they are actually having any effect. If I'm reading correctly, you are always evaluating with a user whose key is user-0-- and you are always adding Targets to the flag which always have user-0 as their first user key. If so, every evaluation will immediately match that first target list, so the flag rules will be ignored, the operators and clause values you're building are irrelevant, and most of the flag evaluation logic is not actually being benchmarked.

[alexwforsythe]

Hey @eli-darkly -- thanks for taking a look. I think I may have introduced a typo when I was renaming some of the values to prepare this for the PR. I've updated the benchmark cases (and results in the description) to never match, so they shouldn't short-circuit the evaluation logic anymore.

[eli-darkly]

Since your use of force-pushing makes it impossible for me to see the incremental diff, would you mind pointing out to me what your recent change was?

[alexwforsythe]

Since your use of force-pushing makes it impossible for me to see the incremental diff, would you mind pointing out to me what your recent change was?

@@ -51,10 +51,10 @@ func (env *benchmarkEnv) tearDown() {
  */

 var (
-       evalBenchmarkUser = NewUserBuilder("user-0").
-               Name("Checkit Nice").
-               Custom("stringAttr", ldvalue.String("stringAttr-2")).
-               Custom("numAttr", ldvalue.Int(100)).Build()
+       evalBenchmarkUser = NewUserBuilder("user-nomatch").
+               Name("name-nomatch").
+               Custom("stringAttr", ldvalue.String("stringAttr-nomatch")).
+               Custom("numAttr", ldvalue.Int(0)).Build()
 )

 type benchmarkCase struct {
@@ -332,29 +332,29 @@ func makeClauses(numClauses int, op Operator) []Clause {
                clause := Clause{Op: op}
                switch op {
                case OperatorGreaterThan:
-                       clause.Attribute = "numDevices"
+                       clause.Attribute = "numAttr"
                        clause.Values = []interface{}{i}
                case OperatorContains:
                        clause.Attribute = "name"
                        clause.Values = []interface{}{
-                               fmt.Sprintf("val-%d", i),
-                               fmt.Sprintf("val-%d", i+1),
-                               "Checkit Nice",
+                               fmt.Sprintf("name-%d", i),
+                               fmt.Sprintf("name-%d", i+1),
+                               fmt.Sprintf("name-%d", i+2),
                        }
                case OperatorIn:

[eli-darkly]

I'm pulling this into another branch so we can adapt and extend it before releasing it on the main branch. Thanks again.