feat: add -compare option (#10)

* feat: add -compare option

* test: fix arguments

Author: knqyf263

README.md

@@ -29,6 +29,7 @@
   - [Show only benchmarks with worse score](#show-only-benchmarks-with-worse-score)
   - [Specify a threshold](#specify-a-threshold)
   - [Specify a base commit compared with HEAD](#specify-a-base-commit-compared-with-head)
+  - [Compare only memory allocation](#compare-only-memory-allocation)
 - [Usage](#usage)
 - [Q&A](#qa)
   - [A result of benchmarks is unstable](#a-result-of-benchmarks-is-unstable)
@@ -172,7 +173,6 @@ Comparison
 +-----------------------------+---------+-------------------+
 
 2020/01/12 17:48:39 This commit makes benchmarks worse
-
 ```
 
 </details>
@@ -192,6 +192,51 @@ By default, `cob` uses `HEAD~1`. If you compare benchmarks with different commit
 $ cob --base origin/master ./...
 ```
 
+## Compare only memory allocation
+You can use `-compare` option.
+
+```
+$ cob -compare B/op
+```
+
+<details>
+<summary>Result</summary>
+
+```
+2020/01/15 14:46:31 Run Benchmark: 4363944cbed3da7a8245cbcdc8d8240b8976eb24 HEAD~1
+2020/01/15 14:46:33 Run Benchmark: 599a5523729d4d99a331b9d3f71dde9e1e6daef0 HEAD
+
+Result
+======
+
++-----------------------------+----------+---------------+-------------------+
+|            Name             |  Commit  |    NsPerOp    | AllocedBytesPerOp |
++-----------------------------+----------+---------------+-------------------+
+| BenchmarkAppend_Allocate-16 |   HEAD   |  179.00 ns/op |      121 B/op     |
++                             +----------+---------------+-------------------+
+|                             | HEAD@{1} |  104.00 ns/op |      23 B/op      |
++-----------------------------+----------+---------------+-------------------+
+|      BenchmarkCall-16       |   HEAD   |   0.50 ns/op  |       0 B/op      |
++                             +----------+---------------+                   +
+|                             | HEAD@{1} |   0.49 ns/op  |                   |
++-----------------------------+----------+---------------+-------------------+
+
+Comparison
+==========
+
++-----------------------------+---------+-------------------+
+|            Name             | NsPerOp | AllocedBytesPerOp |
++-----------------------------+---------+-------------------+
+| BenchmarkAppend_Allocate-16 |    -    |      426.09%      |
++-----------------------------+---------+-------------------+
+|      BenchmarkCall-16       |    -    |       0.00%       |
++-----------------------------+---------+-------------------+
+
+2020/01/15 14:46:35 This commit makes benchmarks worse
+```
+
+</details>
+
 # Usage
 
 ```
@@ -208,6 +253,7 @@ GLOBAL OPTIONS:
    --only-degression   Show only benchmarks with worse score (default: false)
    --threshold value   The program fails if the benchmark gets worse than the threshold (default: 0.2)
    --base value        Specify a base commit compared with HEAD (default: "HEAD~1")
+   --compare value     Which score to compare (default: "ns/op,B/op")
    --bench-cmd value   Specify a command to measure benchmarks (default: "go")
    --bench-args value  Specify arguments passed to -cmd (default: "test -run '^$' -bench . -benchmem ./...")
    --help, -h          show help (default: false)

config.go

@@ -10,6 +10,7 @@ type config struct {
 	onlyDegression bool
 	threshold      float64
 	base           string
+	compare        []string
 	benchCmd       string
 	benchArgs      []string
 }
@@ -19,6 +20,7 @@ func newConfig(c *cli.Context) config {
 		onlyDegression: c.Bool("only-degression"),
 		threshold:      c.Float64("threshold"),
 		base:           c.String("base"),
+		compare:        strings.Split(c.String("compare"), ","),
 		benchCmd:       c.String("bench-cmd"),
 		benchArgs:      strings.Fields(c.String("bench-args")),
 	}

main.go

@@ -25,6 +25,11 @@ type result struct {
 	RatioAllocedBytesPerOp float64
 }
 
+type comparedScore struct {
+	nsPerOp           bool
+	allocedBytesPerOp bool
+}
+
 func main() {
 	app := &cli.App{
 		Name:  "cob",
@@ -47,6 +52,11 @@ func main() {
 				Usage: "Specify a base commit compared with HEAD",
 				Value: "HEAD~1",
 			},
+			&cli.StringFlag{
+				Name:  "compare",
+				Usage: "Which score to compare",
+				Value: "ns/op,B/op",
+			},
 			&cli.StringFlag{
 				Name:  "bench-cmd",
 				Usage: "Specify a command to measure benchmarks",
@@ -159,7 +169,7 @@ func run(c config) error {
 		showResult(os.Stdout, rows)
 	}
 
-	degression := showRatio(os.Stdout, ratios, c.threshold, c.onlyDegression)
+	degression := showRatio(os.Stdout, ratios, c.threshold, whichScoreToCompare(c.compare), c.onlyDegression)
 	if degression {
 		return xerrors.New("This commit makes benchmarks worse")
 	}
@@ -201,7 +211,7 @@ func showResult(w io.Writer, rows [][]string) {
 	table.Render()
 }
 
-func showRatio(w io.Writer, results []result, threshold float64, onlyDegression bool) bool {
+func showRatio(w io.Writer, results []result, threshold float64, comparedScore comparedScore, onlyDegression bool) bool {
 	table := tablewriter.NewWriter(w)
 	table.SetAutoFormatHeaders(false)
 	table.SetAlignment(tablewriter.ALIGN_CENTER)
@@ -211,17 +221,25 @@ func showRatio(w io.Writer, results []result, threshold float64, onlyDegression
 
 	var degression bool
 	for _, result := range results {
-		if threshold < result.RatioNsPerOp || threshold < result.RatioAllocedBytesPerOp {
+		if comparedScore.nsPerOp && threshold < result.RatioNsPerOp {
+			degression = true
+		} else if comparedScore.allocedBytesPerOp && threshold < result.RatioAllocedBytesPerOp {
 			degression = true
 		} else {
 			if onlyDegression {
 				continue
 			}
 		}
 		row := []string{result.Name, generateRatioItem(result.RatioNsPerOp), generateRatioItem(result.RatioAllocedBytesPerOp)}
-		colors := []tablewriter.Colors{{}}
-		colors = append(colors, generateColor(result.RatioNsPerOp))
-		colors = append(colors, generateColor(result.RatioAllocedBytesPerOp))
+		colors := []tablewriter.Colors{{}, generateColor(result.RatioNsPerOp), generateColor(result.RatioAllocedBytesPerOp)}
+		if !comparedScore.nsPerOp {
+			row[1] = "-"
+			colors[1] = tablewriter.Colors{}
+		}
+		if !comparedScore.allocedBytesPerOp {
+			row[2] = "-"
+			colors[2] = tablewriter.Colors{}
+		}
 		table.Rich(row, colors)
 	}
 	if table.NumLines() > 0 {
@@ -250,3 +268,18 @@ func generateColor(ratio float64) tablewriter.Colors {
 	}
 	return tablewriter.Colors{tablewriter.Bold, tablewriter.FgBlueColor}
 }
+
+func whichScoreToCompare(c []string) comparedScore {
+	var comparedScore comparedScore
+	for _, cc := range c {
+		switch cc {
+		case "ns/op":
+			fmt.Println("cpu")
+			comparedScore.nsPerOp = true
+		case "B/op":
+			fmt.Println("memory")
+			comparedScore.allocedBytesPerOp = true
+		}
+	}
+	return comparedScore
+}

main_test.go

@@ -61,6 +61,7 @@ func Test_showRatio(t *testing.T) {
 	type args struct {
 		results        []result
 		threshold      float64
+		compare        comparedScore
 		onlyDegression bool
 	}
 	tests := []struct {
@@ -79,7 +80,11 @@ func Test_showRatio(t *testing.T) {
 						RatioAllocedBytesPerOp: 0.5,
 					},
 				},
-				threshold:      0.2,
+				threshold: 0.2,
+				compare: comparedScore{
+					nsPerOp:           true,
+					allocedBytesPerOp: true,
+				},
 				onlyDegression: false,
 			},
 			want: true,
@@ -110,7 +115,11 @@ Comparison
 						RatioAllocedBytesPerOp: 0.5,
 					},
 				},
-				threshold:      0.95,
+				threshold: 0.95,
+				compare: comparedScore{
+					nsPerOp:           true,
+					allocedBytesPerOp: true,
+				},
 				onlyDegression: false,
 			},
 			want: false,
@@ -143,7 +152,11 @@ Comparison
 						RatioAllocedBytesPerOp: 0.5,
 					},
 				},
-				threshold:      0.4,
+				threshold: 0.4,
+				compare: comparedScore{
+					nsPerOp:           true,
+					allocedBytesPerOp: true,
+				},
 				onlyDegression: true,
 			},
 			want: true,
@@ -174,7 +187,11 @@ Comparison
 						RatioAllocedBytesPerOp: 0.5,
 					},
 				},
-				threshold:      0.6,
+				threshold: 0.6,
+				compare: comparedScore{
+					nsPerOp:           true,
+					allocedBytesPerOp: true,
+				},
 				onlyDegression: true,
 			},
 			want:      false,
@@ -184,7 +201,7 @@ Comparison
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			w := &bytes.Buffer{}
-			got := showRatio(w, tt.args.results, tt.args.threshold, tt.args.onlyDegression)
+			got := showRatio(w, tt.args.results, tt.args.threshold, tt.args.compare, tt.args.onlyDegression)
 			gotTable := w.String()
 			assert.Equal(t, tt.wantTable, gotTable, tt.name)
 			assert.Equal(t, tt.want, got, tt.name)