Skip to content

Commit

Permalink
Improve testing
Browse files Browse the repository at this point in the history 
Init Start stop a test cluster
  • Loading branch information
svaroqui committed Feb 15, 2017
1 parent 0951851 commit 3006684
Show file tree
Hide file tree
Showing 5 changed files with 304 additions and 109 deletions.
68 changes: 53 additions & 15 deletions cluster/provisioning.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package cluster

import (
"bytes"
"errors"
"os"
"os/exec"
"os/user"
"strconv"
"time"

"github.com/tanji/replication-manager/misc"
)

func (cluster *Cluster) RejoinMysqldump(source *ServerMonitor, dest *ServerMonitor) error {
Expand All @@ -32,6 +35,7 @@ func (cluster *Cluster) RejoinMysqldump(source *ServerMonitor, dest *ServerMonit

func (cluster *Cluster) InitClusterSemiSync() error {
for k, server := range cluster.servers {
cluster.LogPrintf("INFO : Starting Server %s", cluster.cfgGroup+strconv.Itoa(k))
cluster.initMariaDB(server, cluster.cfgGroup+strconv.Itoa(k), "semisync.cnf")
}

Expand All @@ -49,37 +53,71 @@ func (cluster *Cluster) ShutdownClusterSemiSync() error {
func (cluster *Cluster) initMariaDB(server *ServerMonitor, name string, conf string) error {
path := cluster.conf.HttpRoot + "/tests/" + name
os.RemoveAll(path)
if _, err := os.Stat(path); os.IsNotExist(err) {
os.MkdirAll(path, 0711)
usr, err := user.Current()
if err != nil {
cluster.LogPrintf("ERRROR : %s", err)
return err
}
usr, err := user.Current()

err = misc.CopyDir(cluster.conf.HttpRoot+"/tests/data", path)
if err != nil {
cluster.LogPrintf("ERRROR : %s", err)
return err
}
installDB := exec.Command(cluster.conf.MariaDBBinaryPath+"/scripts/mysql_install_db", "--datadir="+path, "--user="+usr.Username)
/*
if _, err := os.Stat(path); os.IsNotExist(err) {
os.MkdirAll(path, 0711)
var outrun bytes.Buffer
installDB.Stdout = &outrun
} else {
cluster.LogPrintf("ERRROR : %s", err)
return err
}
installDB := exec.Command(cluster.conf.MariaDBBinaryPath+"/scripts/mysql_install_db", "--datadir="+path, "--user="+usr.Username)
cluster.LogPrintf("INFO : %s", installDB.Path)
var outrun bytes.Buffer
installDB.Stdout = &outrun
cmdrunErr := installDB.Run()
if cmdrunErr != nil {
cluster.LogPrintf("ERRROR : %s", cmdrunErr)
return cmdrunErr
}
cluster.LogPrintf("PROVISIONING : %s", outrun.String())
cmdrunErr := installDB.Run()
if cmdrunErr != nil {
cluster.LogPrintf("ERRROR : %s", cmdrunErr)
return cmdrunErr
}
cluster.LogPrintf("PROVISIONING : %s", outrun.String())
*/

mariadbdCmd := exec.Command(cluster.conf.MariaDBBinaryPath+"/mysqld", "--defaults-file="+path+"/../etc/"+conf, "--port="+server.Port, "--server-id="+server.Port, "--datadir="+path, "--port="+server.Port, "--user="+usr.Username, "--pid="+path+"/"+name+".pid")

mariadbdCmd := exec.Command(cluster.conf.MariaDBBinaryPath+"/mysqld", "--defaults-file="+path+"../"+conf, "--port="+server.Port, "--server-id="+server.Port, "--datadir="+path, "--port="+server.Port, "--user="+cluster.dbUser, "--user="+usr.Username)
mariadbdCmd.Process.Kill()
cluster.LogPrintf("%s %s", mariadbdCmd.Path, mariadbdCmd.Args)
go mariadbdCmd.Run()
server.Process = mariadbdCmd.Process

var err2 error
exitloop := 0
for exitloop < 30 {
time.Sleep(time.Millisecond * 2000)
cluster.LogPrint("Waiting startup ..")
_, err2 = os.Stat(path + "/" + name + ".pid")
if err2 == nil {
exitloop = 30
}
exitloop++

}
if exitloop < 30 {
cluster.LogPrintf("MariaDB started.", err)
} else {
cluster.LogPrintf("MariaDB start timeout.", err)
return errors.New("Failed to start")
}

return nil
}

func (cluster *Cluster) killMariaDB(server *ServerMonitor) error {
server.Process.Kill()
return nil
}

func (cluster *Cluster) StartAllNodes() error {
return nil
}
225 changes: 135 additions & 90 deletions cluster/regressiontest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,11 @@ func (cluster *Cluster) testFailoverReplAllDelayInteractive() bool {
}

func (cluster *Cluster) testFailoverReplAllDelayAuto() bool {
cluster.LogPrintf("TESTING : InitClusterSemiSync")
cluster.InitClusterSemiSync()
cluster.Bootstrap()
cluster.wait_failover_end()
cluster.ShutdownClusterSemiSync()
return false
}

Expand Down Expand Up @@ -861,115 +866,142 @@ func (cluster *Cluster) getTestResultLabel(res bool) string {
}
}

func (cluster *Cluster) RunAllTests() bool {

func (cluster *Cluster) RunAllTests(test string) bool {
var allTests = map[string]string{}
cluster.cleanall = true
cluster.Bootstrap()
cluster.wait_failover_end()

ret := true
var res bool

res = cluster.testSwitchOverLongTransactionNoRplCheckNoSemiSync()
allTests["1 Switchover Concurrent Long Transaction <cluster.conf.ReadOnly=false> <cluster.conf.RplChecks=true>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
cluster.LogPrintf("TESTING : %s", test)
if test == "testFailoverReplAllDelayAuto" || test == "ALL" {
res = cluster.testFailoverReplAllDelayAuto()
allTests["1 Failover all slaves delay <cluster.conf.RplChecks=false> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOverLongQueryNoRplCheckNoSemiSync()
allTests["1 Switchover Concurrent Long Query <cluster.conf.ReadOnly=false> <cluster.conf.RplChecks=true>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
cluster.cleanall = true
cluster.Bootstrap()
cluster.wait_failover_end()
if test == "testSwitchOverLongTransactionNoRplCheckNoSemiSync" || test == "ALL" {
res = cluster.testSwitchOverLongTransactionNoRplCheckNoSemiSync()
allTests["1 Switchover Concurrent Long Transaction <cluster.conf.ReadOnly=false> <cluster.conf.RplChecks=true>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOverNoReadOnlyNoRplCheck()
allTests["1 Switchover <cluster.conf.ReadOnly=false> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOverLongQueryNoRplCheckNoSemiSync" || test == "ALL" {
res = cluster.testSwitchOverLongQueryNoRplCheckNoSemiSync()
allTests["1 Switchover Concurrent Long Query <cluster.conf.ReadOnly=false> <cluster.conf.RplChecks=true>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOverReadOnlyNoRplCheck()
allTests["1 Switchover <cluster.conf.ReadOnly=true> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOverNoReadOnlyNoRplCheck" || test == "ALL" {
res = cluster.testSwitchOverNoReadOnlyNoRplCheck()
allTests["1 Switchover <cluster.conf.ReadOnly=false> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOver2TimesReplicationOkNoSemiSyncNoRplCheck()
allTests["2 Switchover Replication Ok <2 threads benchmark> <semisync=false> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOverReadOnlyNoRplCheck" || test == "ALL" {
res = cluster.testSwitchOverReadOnlyNoRplCheck()
allTests["1 Switchover <cluster.conf.ReadOnly=true> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOver2TimesReplicationOkSemiSyncNoRplCheck()
allTests["2 Switchover Replication Ok <2 threads benchmark> <semisync=true> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOver2TimesReplicationOkNoSemiSyncNoRplCheck" || test == "ALL" {
res = cluster.testSwitchOver2TimesReplicationOkNoSemiSyncNoRplCheck()
allTests["2 Switchover Replication Ok <2 threads benchmark> <semisync=false> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOverBackPreferedMasterNoRplCheckSemiSync()
allTests["2 Switchover Back Prefered Master <semisync=true> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOver2TimesReplicationOkSemiSyncNoRplCheck" || test == "ALL" {
res = cluster.testSwitchOver2TimesReplicationOkSemiSyncNoRplCheck()
allTests["2 Switchover Replication Ok <2 threads benchmark> <semisync=true> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOverAllSlavesStopRplCheckNoSemiSync()
allTests["Can't Switchover All Slaves Stop <semisync=false> <cluster.conf.RplChecks=true>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOverBackPreferedMasterNoRplCheckSemiSync" || test == "ALL" {
res = cluster.testSwitchOverBackPreferedMasterNoRplCheckSemiSync()
allTests["2 Switchover Back Prefered Master <semisync=true> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOverAllSlavesStopNoSemiSyncNoRplCheck()
allTests["Can Switchover All Slaves Stop <semisync=false> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOverAllSlavesStopRplCheckNoSemiSync" || test == "ALL" {
res = cluster.testSwitchOverAllSlavesStopRplCheckNoSemiSync()
allTests["Can't Switchover All Slaves Stop <semisync=false> <cluster.conf.RplChecks=true>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOverAllSlavesDelayRplCheckNoSemiSync()
allTests["Can't Switchover All Slaves Delay <semisync=false> <cluster.conf.RplChecks=true>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOverAllSlavesStopNoSemiSyncNoRplCheck" || test == "ALL" {
res = cluster.testSwitchOverAllSlavesStopNoSemiSyncNoRplCheck()
allTests["Can Switchover All Slaves Stop <semisync=false> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSwitchOverAllSlavesDelayNoRplChecksNoSemiSync()
allTests["Can Switchover All Slaves Delay <semisync=false> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOverAllSlavesDelayRplCheckNoSemiSync" || test == "ALL" {
res = cluster.testSwitchOverAllSlavesDelayRplCheckNoSemiSync()
allTests["Can't Switchover All Slaves Delay <semisync=false> <cluster.conf.RplChecks=true>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testSlaReplAllSlavesStopNoSemiSync()
allTests["SLA Decrease Can't Switchover All Slaves Stop <Semisync=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSwitchOverAllSlavesDelayNoRplChecksNoSemiSync" || test == "ALL" {
res = cluster.testSwitchOverAllSlavesDelayNoRplChecksNoSemiSync()
allTests["Can Switchover All Slaves Delay <semisync=false> <cluster.conf.RplChecks=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testFailOverNoRplChecksNoSemiSync()
allTests["1 Failover <cluster.conf.RplChecks=false> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testSlaReplAllSlavesStopNoSemiSync" || test == "ALL" {
res = cluster.testSlaReplAllSlavesStopNoSemiSync()
allTests["SLA Decrease Can't Switchover All Slaves Stop <Semisync=false>"] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testFailOverAllSlavesDelayNoRplChecksNoSemiSync()
allTests["1 Failover All Slave Delay <cluster.conf.RplChecks=false> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testFailOverNoRplChecksNoSemiSync" || test == "ALL" {
res = cluster.testFailOverNoRplChecksNoSemiSync()
allTests["1 Failover <cluster.conf.RplChecks=false> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testFailOverAllSlavesDelayRplChecksNoSemiSync()
allTests["1 Failover All Slave Delay <cluster.conf.RplChecks=true> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testFailOverAllSlavesDelayNoRplChecksNoSemiSync" || test == "ALL" {
res = cluster.testFailOverAllSlavesDelayNoRplChecksNoSemiSync()
allTests["1 Failover All Slave Delay <cluster.conf.RplChecks=false> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testNumberFailOverLimitReach()
allTests["1 Failover Number of Failover Reach <cluster.conf.RplChecks=false> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testFailOverAllSlavesDelayRplChecksNoSemiSync" || test == "ALL" {
res = cluster.testFailOverAllSlavesDelayRplChecksNoSemiSync()
allTests["1 Failover All Slave Delay <cluster.conf.RplChecks=true> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

res = cluster.testFailOverTimeNotReach()
allTests["1 Failover Before Time Limit <cluster.conf.RplChecks=false> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
if test == "testNumberFailOverLimitReach" || test == "ALL" {
res = cluster.testNumberFailOverLimitReach()
allTests["1 Failover Number of Failover Reach <cluster.conf.RplChecks=false> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}
if test == "testFailOverTimeNotReach" || test == "ALL" {
res = cluster.testFailOverTimeNotReach()
allTests["1 Failover Before Time Limit <cluster.conf.RplChecks=false> <Semisync=false> "] = cluster.getTestResultLabel(res)
if res == false {
ret = res
}
}

keys := make([]string, 0, len(allTests))
Expand Down Expand Up @@ -1054,7 +1086,20 @@ func (cluster *Cluster) RunSysbench() error {
return nil
}

func (cluster *Cluster) StartAllNodes() error {

func (cluster *Cluster) DelayAllSlaves() error {
for _, s := range cluster.slaves {
dbhelper.StopSlave(s.Conn)
}
result, err := dbhelper.WriteConcurrent2(cluster.master.DSN, 10)
if err != nil {
cluster.LogPrintf("BENCH : %s %s", err.Error(), result)
}
dbhelper.InjectLongTrx(cluster.master.Conn, 10)
time.Sleep(10 * time.Second)
for _, s := range cluster.slaves {
dbhelper.StartSlave(s.Conn)
}
return nil
}

func
Loading

0 comments on commit 3006684

Please sign in to comment.