Merge pull request vitessio#5983 from planetscale/morgo-update-makefile

Misc cleanups: Remove deadcode in tests, makefile

.github/workflows/sonar_analysis.yml

@@ -2,7 +2,7 @@ name: sonar_analysis
 on:
   push:
     branches:
-      - 'master'
+      - 'sonartest'
 jobs:
 
   build:

Makefile

@@ -74,14 +74,6 @@ install: build
 	# binaries
 	mkdir -p "$${PREFIX}/bin"
 	cp "$${VTROOT}/bin/"{mysqlctld,vtctld,vtctlclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/"
-	# config files
-	mkdir -p "$${PREFIX}/src/vitess.io/vitess"
-	cp -R config "$${PREFIX}/src/vitess.io/vitess/"
-	# also symlink config files in the old location
-	ln -sf src/vitess.io/vitess/config "$${PREFIX}/config"
-	# vtctld web UI files
-	mkdir -p "$${PREFIX}/src/vitess.io/vitess/web/vtctld2"
-	cp -R web/vtctld2/app "$${PREFIX}/src/vitess.io/vitess/web/vtctld2/"
 
 parser:
 	make -C go/vt/sqlparser
@@ -289,12 +281,6 @@ docker_test:
 docker_unit_test:
 	go run test.go -flavor $(flavor) unit
 
-# This can be used to rebalance the total average runtime of each group of
-# tests in Travis. The results are saved in test/config.json, which you can
-# then commit and push.
-rebalance_tests:
-	go run test.go -rebalance 5
-
 # Release a version.
 # This will generate a tar.gz file into the releases folder with the current source
 release: docker_base

build.env

@@ -17,7 +17,7 @@
 source ./tools/shell_functions.inc
 
 go version >/dev/null 2>&1 || fail "Go is not installed or is not in \$PATH. See https://vitess.io/contributing/build-from-source for install instructions."
-goversion_min 1.13 || fail "Go version reported: ${go version}. Version 1.13+ required. See https://vitess.io/contributing/build-from-source for install instructions."
+goversion_min 1.13 || fail "Go version reported: `go version`. Version 1.13+ required. See https://vitess.io/contributing/build-from-source for install instructions."
 
 mkdir -p dist
 mkdir -p bin

test.go

@@ -39,7 +39,6 @@ package main
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -86,7 +85,6 @@ var (
 	shard          = flag.Int("shard", -1, "if N>=0, run the tests whose Shard field matches N")
 	tag            = flag.String("tag", "", "if provided, only run tests with the given tag. Can't be combined with -shard or explicit test list")
 	exclude        = flag.String("exclude", "", "if provided, exclude tests containing any of the given tags (comma delimited)")
-	reshard        = flag.Int("rebalance", 0, "if N>0, check the stats and group tests into N similarly-sized bins by average run time")
 	keepData       = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders")
 	printLog       = flag.Bool("print-log", false, "print the log of each failed test (or all tests if -log-pass) to the console")
 	follow         = flag.Bool("follow", false, "print test output as it runs, instead of waiting to see if it passes or fails")
@@ -299,22 +297,6 @@ func main() {
 		log.Fatalf("Can't parse config file: %v", err)
 	}
 
-	// Resharding.
-	if *reshard > 0 {
-		if err := reshardTests(&config, *reshard); err != nil {
-			log.Fatalf("resharding error: %v", err)
-		}
-		log.Printf("Saving updated config...")
-		data, err := json.MarshalIndent(config, "", "\t")
-		if err != nil {
-			log.Fatalf("can't save new config: %v", err)
-		}
-		if err := ioutil.WriteFile(configFileName, data, 0644); err != nil {
-			log.Fatalf("can't write new config: %v", err)
-		}
-		return
-	}
-
 	flavors := []string{"local"}
 
 	if *docker {
@@ -697,105 +679,6 @@ func updateTestStats(name string, update func(*TestStats)) {
 	}
 }
 
-func reshardTests(config *Config, numShards int) error {
-	var stats Stats
-
-	var data []byte
-	if *remoteStats != "" {
-		log.Printf("Using remote stats for resharding: %v", *remoteStats)
-		resp, err := http.Get(*remoteStats)
-		if err != nil {
-			return err
-		}
-		defer resp.Body.Close()
-		if data, err = ioutil.ReadAll(resp.Body); err != nil {
-			return err
-		}
-	} else {
-		var err error
-		data, err = ioutil.ReadFile(statsFileName)
-		if err != nil {
-			return errors.New("can't read stats file")
-		}
-	}
-
-	if err := json.Unmarshal(data, &stats); err != nil {
-		return fmt.Errorf("can't parse stats file: %v", err)
-	}
-
-	// Sort tests by PassTime.
-	var tests []TestStats
-	var totalTime int64
-	for name, test := range stats.TestStats {
-		test.name = name
-		tests = append(tests, test)
-		totalTime += int64(test.PassTime)
-	}
-	sort.Sort(ByPassTime(tests))
-
-	// Group into shards.
-	max := totalTime / int64(numShards)
-	shards := make([][]TestStats, numShards)
-	sums := make([]int64, numShards)
-	// First pass: greedy approximation.
-firstPass:
-	for len(tests) > 0 {
-		v := int64(tests[0].PassTime)
-
-		for n := range shards {
-			if sums[n]+v < max {
-				shards[n] = append(shards[n], tests[0])
-				sums[n] += v
-				tests = tests[1:]
-				continue firstPass
-			}
-		}
-		// None of the bins has room. Go to second pass.
-		break
-	}
-	// Second pass: distribute the remainder.
-	for len(tests) > 0 {
-		nmin := 0
-		min := sums[0]
-
-		for n := range sums {
-			if sums[n] < min {
-				nmin = n
-				min = sums[n]
-			}
-		}
-
-		shards[nmin] = append(shards[nmin], tests[0])
-		sums[nmin] += int64(tests[0].PassTime)
-		tests = tests[1:]
-	}
-
-	// Update config and print results.
-	for i, tests := range shards {
-		for _, t := range tests {
-			ct, ok := config.Tests[t.name]
-			if !ok {
-				log.Printf("WARNING: skipping unknown test: %v", t.name)
-				continue
-			}
-			ct.Shard = i
-			if ct.Args == nil {
-				ct.Args = []string{}
-			}
-			if ct.Command == nil {
-				ct.Command = []string{}
-			}
-			if ct.Tags == nil {
-				ct.Tags = []string{}
-			}
-			log.Printf("%v:\t%v\n", t.name, t.PassTime)
-		}
-		log.Printf("Shard %v total: %v\n", i, time.Duration(sums[i]))
-	}
-
-	return nil
-}
-
 type ByPassTime []TestStats
 
 func (a ByPassTime) Len() int           { return len(a) }

tools/dependency_check.sh

@@ -24,6 +24,6 @@ function fail() {
 # These binaries are required to 'make test'
 # mysqld might be in /usr/sbin which will not be in the default PATH
 PATH="/usr/sbin:$PATH"
-for binary in mysqld consul etcd etcdctl zksrv.sh javadoc mvn ant curl wget zip unzip; do
+for binary in k3s mysqld consul etcd etcdctl zksrv.sh javadoc mvn ant curl wget zip unzip; do
   command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. See https://vitess.io/contributing/build-from-source for install instructions."
 done;