Add metadata-retrieval kind

.gitignore

@@ -0,0 +1,9 @@
+# downloads
+binaries
+
+#ci
+.ci/
+
+# coverage
+coverage.out
+coverage.txt

.travis.yml

@@ -9,7 +9,7 @@ service:
   - docker
 
 env:
-  - GO111MODULE=on GOPROXY=https://proxy.golang.org
+  - GO111MODULE=on GOPROXY=https://proxy.golang.org REG_TOKEN=$GITHUB_TOKEN
 
 matrix:
   fast_finish: true

README.md

@@ -8,7 +8,7 @@ Usage:
 
 data-retrieval utils regression tester.
 
-This tool executes several versions of data-retrieval utils and compares query times and resource usage. There should be at least two versions specified as arguments in the following way:
+This tool executes several versions of data-retrieval utils and compares query times and resource usage. There should be at least one version specified as an argument in the following way:
 
 * v0.12.1 - release name from github (https://github.com/src-d/your-data-retrieval-util/releases). The binary will be downloaded.
 * latest - latest release from github. The binary will be downloaded.

cmd/regression-retrieval/main.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/src-d/regression-retrieval/test"
 	_ "github.com/src-d/regression-retrieval/test/gitcollector"
+	_ "github.com/src-d/regression-retrieval/test/metadata-retrieval"
 
 	"github.com/jessevdk/go-flags"
 	"github.com/src-d/regression-core"
@@ -13,7 +14,7 @@ import (
 
 var description = `data-retrieval utils regression tester.
 
-This tool executes several versions of data-retrieval utils and compares query times and resource usage. There should be at least two versions specified as arguments in the following way:
+This tool executes several versions of data-retrieval utils and compares query times and resource usage. There should be at least one version specified as an argument in the following way:
 
 * v0.12.1 - release name from github (https://github.com/src-d/your-data-retrieval-util/releases). The binary will be downloaded.
 * latest - latest release from github. The binary will be downloaded.
@@ -26,6 +27,7 @@ This tool executes several versions of data-retrieval utils and compares query t
 The repositories and downloaded/built binaries are cached by default in "repos" and "binaries" repositories from the current directory.
 `
 
+// Options CLI options
 type Options struct {
 	regression.Config
 

go.mod

@@ -8,6 +8,8 @@ require (
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/mattn/go-colorable v0.1.4 // indirect
 	github.com/mattn/go-isatty v0.0.9 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/prometheus/client_golang v1.1.0
 	github.com/src-d/regression-core v0.0.0-20191003075028-b476aeec74d4
 	github.com/stretchr/testify v1.3.0

test/gitcollector/gitcollector.go

@@ -2,7 +2,6 @@ package gitcollector
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strings"
 	"text/tabwriter"
@@ -13,11 +12,10 @@ import (
 
 	"github.com/src-d/regression-core"
 	"gopkg.in/src-d/go-log.v1"
-	"gopkg.in/yaml.v2"
 )
 
 type (
-	gitCollectorResults map[string][]*Result
+	gitCollectorResults map[string][]*test.Result
 	versionResults      map[string]gitCollectorResults
 
 	// Test holds the information about a gitcollector test
@@ -38,12 +36,6 @@ func init() {
 	test.Register(Kind, NewTest)
 }
 
-// Result is a wrapper around regression.Result that additionally contains organizations that were processed
-type Result struct {
-	*regression.Result
-	Organizations string
-}
-
 // NewTest creates a new Test struct
 func NewTest(config regression.Config) (test.Test, error) {
 	l, err := (&log.LoggerFactory{Level: log.InfoLevel}).New(log.Fields{})
@@ -105,14 +97,14 @@ func (t *Test) RunLoad() error {
 		}
 
 		rf := gitCollector.ExtraFile("regression.yml")
-		if organizations, err := loadOrganizationsYaml(rf); err != nil {
+		if organizations, err := test.LoadOrganizationsYaml(rf); err != nil {
 			return err
 		} else {
 			t.organizations = organizations
 		}
 
 		for _, orgs := range t.organizations {
-			results[version][orgs] = make([]*Result, times)
+			results[version][orgs] = make([]*test.Result, times)
 			for i := 0; i < times; i++ {
 				l.New(log.Fields{
 					"orgs": orgs,
@@ -137,7 +129,7 @@ func (t *Test) RunLoad() error {
 func (t *Test) runLoadTest(
 	gitCollector *regression.Binary,
 	orgs string,
-) (*Result, error) {
+) (*test.Result, error) {
 	t.log.Infof("Executing gitcollector test")
 
 	command := NewCommand(gitCollector.Path, orgs)
@@ -167,7 +159,7 @@ func (t *Test) runLoadTest(
 		Memory: rusage.Maxrss * 1024,
 	}
 
-	r := &Result{
+	r := &test.Result{
 		Result:        result,
 		Organizations: orgs,
 	}
@@ -273,8 +265,8 @@ func (t *Test) GetResults() bool {
 			queryA := a[orgs][0]
 			queryB := b[orgs][0]
 
-			queryA.Result = average(a[orgs])
-			queryB.Result = average(b[orgs])
+			queryA.Result = test.Average(a[orgs])
+			queryB.Result = test.Average(b[orgs])
 			c := queryA.Result.ComparePrint(queryB.Result, 10.0)
 			if !c {
 				ok = false
@@ -292,7 +284,7 @@ func (t *Test) GetResults() bool {
 func (t *Test) SaveLatestCSV() {
 	version := t.config.Versions[len(t.config.Versions)-1]
 	for _, orgs := range t.organizations {
-		res := average(t.results[version][orgs])
+		res := test.Average(t.results[version][orgs])
 		if err := res.SaveAllCSV(fmt.Sprintf("plot_%s_", strings.Replace(orgs, ",", "_", -1))); err != nil {
 			panic(err)
 		}
@@ -304,38 +296,10 @@ func (t *Test) StoreLatestToPrometheus(promConfig regression.PromConfig, ciConfi
 	version := t.config.Versions[len(t.config.Versions)-1]
 	cli := prometheus.NewPromClient(Kind, promConfig)
 	for _, orgs := range t.organizations {
-		res := average(t.results[version][orgs])
+		res := test.Average(t.results[version][orgs])
 		if err := cli.Dump(res, version, orgs, ciConfig.Branch, ciConfig.Commit); err != nil {
 			return err
 		}
 	}
 	return nil
 }
-
-func average(pr []*Result) *regression.Result {
-	if len(pr) == 0 {
-		return nil
-	}
-
-	results := make([]*regression.Result, 0, len(pr))
-	for _, r := range pr {
-		results = append(results, r.Result)
-	}
-
-	return regression.Average(results)
-}
-
-func loadOrganizationsYaml(file string) ([]string, error) {
-	text, err := ioutil.ReadFile(file)
-	if err != nil {
-		return nil, err
-	}
-
-	var res []string
-	err = yaml.Unmarshal(text, &res)
-	if err != nil {
-		return nil, err
-	}
-
-	return res, nil
-}

test/metadata-retrieval/command.go

@@ -0,0 +1,51 @@
+package metadataretrieval
+
+import (
+	"os"
+	"os/exec"
+	"syscall"
+)
+
+// Command wraps a metadata-retrieval server instance.
+type Command struct {
+	cmd    *exec.Cmd
+	binary string
+	orgs   string
+	dir    string
+}
+
+// NewCommand creates a new metadata-retrieval command struct.
+func NewCommand(binary, orgs string) *Command {
+	return &Command{
+		cmd:    new(exec.Cmd),
+		binary: binary,
+		orgs:   orgs,
+	}
+}
+
+// Run runs metadata-retrieval util to discover and download organizations
+func (c *Command) Run(envs map[string]string) error {
+	c.cmd = exec.Command(
+		c.binary,
+		"ghsync",
+		"--version", "0",
+		"--orgs", c.orgs,
+		"--no-forks",
+	)
+	c.cmd.Stdout = os.Stdout
+	c.cmd.Stderr = os.Stderr
+	c.cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	for k, v := range envs {
+		c.cmd.Env = append(c.cmd.Env, k+"="+v)
+	}
+
+	return c.cmd.Run()
+}
+
+// Rusage returns usage counters
+func (c *Command) Rusage() *syscall.Rusage {
+	rusage, _ := c.cmd.ProcessState.SysUsage().(*syscall.Rusage)
+	return rusage
+}