Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Misc cleanups: Remove deadcode in tests, makefile #5983

Merged
merged 5 commits into from
Apr 1, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Remove deadcode in tests
Signed-off-by: Morgan Tocker <tocker@gmail.com>
  • Loading branch information
morgo committed Mar 27, 2020
commit 03c084689322892a92bb1226437014a93f6db236
14 changes: 0 additions & 14 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,6 @@ install: build
# binaries
mkdir -p "$${PREFIX}/bin"
cp "$${VTROOT}/bin/"{mysqlctld,vtctld,vtctlclient,vtgate,vttablet,vtworker,vtbackup} "$${PREFIX}/bin/"
# config files
mkdir -p "$${PREFIX}/src/vitess.io/vitess"
cp -R config "$${PREFIX}/src/vitess.io/vitess/"
# also symlink config files in the old location
ln -sf src/vitess.io/vitess/config "$${PREFIX}/config"
# vtctld web UI files
mkdir -p "$${PREFIX}/src/vitess.io/vitess/web/vtctld2"
cp -R web/vtctld2/app "$${PREFIX}/src/vitess.io/vitess/web/vtctld2/"

parser:
make -C go/vt/sqlparser
Expand Down Expand Up @@ -289,12 +281,6 @@ docker_test:
docker_unit_test:
go run test.go -flavor $(flavor) unit

# This can be used to rebalance the total average runtime of each group of
# tests in Travis. The results are saved in test/config.json, which you can
# then commit and push.
rebalance_tests:
go run test.go -rebalance 5

# Release a version.
# This will generate a tar.gz file into the releases folder with the current source
release: docker_base
Expand Down
117 changes: 0 additions & 117 deletions test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ package main
import (
"bytes"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
Expand Down Expand Up @@ -86,7 +85,6 @@ var (
shard = flag.Int("shard", -1, "if N>=0, run the tests whose Shard field matches N")
tag = flag.String("tag", "", "if provided, only run tests with the given tag. Can't be combined with -shard or explicit test list")
exclude = flag.String("exclude", "", "if provided, exclude tests containing any of the given tags (comma delimited)")
reshard = flag.Int("rebalance", 0, "if N>0, check the stats and group tests into N similarly-sized bins by average run time")
keepData = flag.Bool("keep-data", false, "don't delete the per-test VTDATAROOT subfolders")
printLog = flag.Bool("print-log", false, "print the log of each failed test (or all tests if -log-pass) to the console")
follow = flag.Bool("follow", false, "print test output as it runs, instead of waiting to see if it passes or fails")
Expand Down Expand Up @@ -299,22 +297,6 @@ func main() {
log.Fatalf("Can't parse config file: %v", err)
}

// Resharding.
if *reshard > 0 {
if err := reshardTests(&config, *reshard); err != nil {
log.Fatalf("resharding error: %v", err)
}
log.Printf("Saving updated config...")
data, err := json.MarshalIndent(config, "", "\t")
if err != nil {
log.Fatalf("can't save new config: %v", err)
}
if err := ioutil.WriteFile(configFileName, data, 0644); err != nil {
log.Fatalf("can't write new config: %v", err)
}
return
}

flavors := []string{"local"}

if *docker {
Expand Down Expand Up @@ -697,105 +679,6 @@ func updateTestStats(name string, update func(*TestStats)) {
}
}

func reshardTests(config *Config, numShards int) error {
var stats Stats

var data []byte
if *remoteStats != "" {
log.Printf("Using remote stats for resharding: %v", *remoteStats)
resp, err := http.Get(*remoteStats)
if err != nil {
return err
}
defer resp.Body.Close()
if data, err = ioutil.ReadAll(resp.Body); err != nil {
return err
}
} else {
var err error
data, err = ioutil.ReadFile(statsFileName)
if err != nil {
return errors.New("can't read stats file")
}
}

if err := json.Unmarshal(data, &stats); err != nil {
return fmt.Errorf("can't parse stats file: %v", err)
}

// Sort tests by PassTime.
var tests []TestStats
var totalTime int64
for name, test := range stats.TestStats {
test.name = name
tests = append(tests, test)
totalTime += int64(test.PassTime)
}
sort.Sort(ByPassTime(tests))

// Group into shards.
max := totalTime / int64(numShards)
shards := make([][]TestStats, numShards)
sums := make([]int64, numShards)
// First pass: greedy approximation.
firstPass:
for len(tests) > 0 {
v := int64(tests[0].PassTime)

for n := range shards {
if sums[n]+v < max {
shards[n] = append(shards[n], tests[0])
sums[n] += v
tests = tests[1:]
continue firstPass
}
}
// None of the bins has room. Go to second pass.
break
}
// Second pass: distribute the remainder.
for len(tests) > 0 {
nmin := 0
min := sums[0]

for n := range sums {
if sums[n] < min {
nmin = n
min = sums[n]
}
}

shards[nmin] = append(shards[nmin], tests[0])
sums[nmin] += int64(tests[0].PassTime)
tests = tests[1:]
}

// Update config and print results.
for i, tests := range shards {
for _, t := range tests {
ct, ok := config.Tests[t.name]
if !ok {
log.Printf("WARNING: skipping unknown test: %v", t.name)
continue
}
ct.Shard = i
if ct.Args == nil {
ct.Args = []string{}
}
if ct.Command == nil {
ct.Command = []string{}
}
if ct.Tags == nil {
ct.Tags = []string{}
}
log.Printf("%v:\t%v\n", t.name, t.PassTime)
}
log.Printf("Shard %v total: %v\n", i, time.Duration(sums[i]))
}

return nil
}

type ByPassTime []TestStats

func (a ByPassTime) Len() int { return len(a) }
Expand Down