Merge pull request vitessio#2072 from alainjobart/rebuild

Removing dangerous repair code.

go/cmd/topo2topo/topo2topo.go

@@ -24,8 +24,6 @@ var doShards = flag.Bool("do-shards", false, "copies the shard information")
 var doShardReplications = flag.Bool("do-shard-replications", false, "copies the shard replication information")
 var doTablets = flag.Bool("do-tablets", false, "copies the tablet information")
 
-var deleteKeyspaceShards = flag.Bool("delete-keyspace-shards", false, "when copying shards, first removes the destination shards (will nuke the replication graph)")
-
 func main() {
 	defer exit.RecoverAll()
 	defer logutil.Flush()
@@ -50,7 +48,7 @@ func main() {
 		helpers.CopyKeyspaces(ctx, fromTS.Impl, toTS.Impl)
 	}
 	if *doShards {
-		helpers.CopyShards(ctx, fromTS.Impl, toTS.Impl, *deleteKeyspaceShards)
+		helpers.CopyShards(ctx, fromTS.Impl, toTS.Impl)
 	}
 	if *doShardReplications {
 		helpers.CopyShardReplications(ctx, fromTS.Impl, toTS.Impl)

go/vt/etcdtopo/keyspace.go

@@ -7,10 +7,8 @@ package etcdtopo
 import (
 	"encoding/json"
 	"fmt"
-	"sync"
 
 	"github.com/coreos/go-etcd/etcd"
-	"github.com/youtube/vitess/go/vt/concurrency"
 	"github.com/youtube/vitess/go/vt/topo"
 	"golang.org/x/net/context"
 
@@ -86,32 +84,6 @@ func (s *Server) GetKeyspaces(ctx context.Context) ([]string, error) {
 	return getNodeNames(resp)
 }
 
-// DeleteKeyspaceShards implements topo.Server.
-func (s *Server) DeleteKeyspaceShards(ctx context.Context, keyspace string) error {
-	shards, err := s.GetShardNames(ctx, keyspace)
-	if err != nil {
-		return err
-	}
-
-	wg := sync.WaitGroup{}
-	rec := concurrency.AllErrorRecorder{}
-	global := s.getGlobal()
-	for _, shard := range shards {
-		wg.Add(1)
-		go func(shard string) {
-			defer wg.Done()
-			_, err := global.Delete(shardDirPath(keyspace, shard), true /* recursive */)
-			rec.RecordError(convertError(err))
-		}(shard)
-	}
-	wg.Wait()
-
-	if err = rec.Error(); err != nil {
-		return err
-	}
-	return nil
-}
-
 // DeleteKeyspace implements topo.Server.
 func (s *Server) DeleteKeyspace(ctx context.Context, keyspace string) error {
 	_, err := s.getGlobal().Delete(keyspaceDirPath(keyspace), true /* recursive */)

go/vt/topo/helpers/copy.go

@@ -54,7 +54,7 @@ func CopyKeyspaces(ctx context.Context, fromTS, toTS topo.Impl) {
 }
 
 // CopyShards will create the shards in the destination topo
-func CopyShards(ctx context.Context, fromTS, toTS topo.Impl, deleteKeyspaceShards bool) {
+func CopyShards(ctx context.Context, fromTS, toTS topo.Impl) {
 	keyspaces, err := fromTS.GetKeyspaces(ctx)
 	if err != nil {
 		log.Fatalf("fromTS.GetKeyspaces: %v", err)
@@ -72,13 +72,6 @@ func CopyShards(ctx context.Context, fromTS, toTS topo.Impl, deleteKeyspaceShard
 				return
 			}
 
-			if deleteKeyspaceShards {
-				if err := toTS.DeleteKeyspaceShards(ctx, keyspace); err != nil {
-					rec.RecordError(fmt.Errorf("DeleteKeyspaceShards(%v): %v", keyspace, err))
-					return
-				}
-			}
-
 			for _, shard := range shards {
 				wg.Add(1)
 				go func(keyspace, shard string) {

go/vt/topo/helpers/copy_test.go

@@ -110,15 +110,15 @@ func TestBasic(t *testing.T) {
 	CopyKeyspaces(ctx, fromTS, toTS)
 
 	// check shard copy
-	CopyShards(ctx, fromTS, toTS, true)
+	CopyShards(ctx, fromTS, toTS)
 	shards, err := toTS.GetShardNames(ctx, "test_keyspace")
 	if err != nil {
 		t.Fatalf("toTS.GetShardNames failed: %v", err)
 	}
 	if len(shards) != 1 || shards[0] != "0" {
 		t.Fatalf("unexpected shards: %v", shards)
 	}
-	CopyShards(ctx, fromTS, toTS, false)
+	CopyShards(ctx, fromTS, toTS)
 	s, _, err := toTS.GetShard(ctx, "test_keyspace", "0")
 	if err != nil {
 		t.Fatalf("cannot read shard: %v", err)

go/vt/topo/helpers/tee.go

@@ -279,19 +279,6 @@ func (tee *Tee) GetKeyspaces(ctx context.Context) ([]string, error) {
 	return tee.readFrom.GetKeyspaces(ctx)
 }
 
-// DeleteKeyspaceShards is part of the topo.Server interface
-func (tee *Tee) DeleteKeyspaceShards(ctx context.Context, keyspace string) error {
-	if err := tee.primary.DeleteKeyspaceShards(ctx, keyspace); err != nil {
-		return err
-	}
-
-	if err := tee.secondary.DeleteKeyspaceShards(ctx, keyspace); err != nil {
-		// not critical enough to fail
-		log.Warningf("secondary.DeleteKeyspaceShards(%v) failed: %v", keyspace, err)
-	}
-	return nil
-}
-
 //
 // Shard management, global.
 //

go/vt/topo/helpers/tee_test.go

@@ -20,7 +20,7 @@ func TestTee(t *testing.T) {
 	// create the setup, copy the data
 	fromTS, toTS := createSetup(ctx, t)
 	CopyKeyspaces(ctx, fromTS, toTS)
-	CopyShards(ctx, fromTS, toTS, true)
+	CopyShards(ctx, fromTS, toTS)
 	CopyTablets(ctx, fromTS, toTS)
 
 	// create a tee and check it implements the interface

go/vt/topo/keyspace.go

@@ -220,20 +220,6 @@ func (ts Server) FindAllShardsInKeyspace(ctx context.Context, keyspace string) (
 	return result, nil
 }
 
-// DeleteKeyspaceShards wraps the underlying Impl.DeleteKeyspaceShards
-// and dispatches the event.
-func (ts Server) DeleteKeyspaceShards(ctx context.Context, keyspace string) error {
-	if err := ts.Impl.DeleteKeyspaceShards(ctx, keyspace); err != nil {
-		return err
-	}
-	event.Dispatch(&events.KeyspaceChange{
-		KeyspaceName: keyspace,
-		Keyspace:     nil,
-		Status:       "deleted all shards",
-	})
-	return nil
-}
-
 // DeleteKeyspace wraps the underlying Impl.DeleteKeyspace
 // and dispatches the event.
 func (ts Server) DeleteKeyspace(ctx context.Context, keyspace string) error {

go/vt/topo/server.go

@@ -106,10 +106,6 @@ type Impl interface {
 	// GetKeyspaces returns the known keyspace names. They shall be sorted.
 	GetKeyspaces(ctx context.Context) ([]string, error)
 
-	// DeleteKeyspaceShards deletes all the shards in a keyspace.
-	// Use with caution.
-	DeleteKeyspaceShards(ctx context.Context, keyspace string) error
-
 	//
 	// Shard management, global.
 	//
@@ -136,8 +132,8 @@ type Impl interface {
 	GetShard(ctx context.Context, keyspace, shard string) (*topodatapb.Shard, int64, error)
 
 	// GetShardNames returns the known shards in a keyspace.
-	// Can return ErrNoNode if the keyspace wasn't created,
-	// or if DeleteKeyspaceShards was called. They shall be sorted.
+	// Can return ErrNoNode if the keyspace wasn't created.
+	// They shall be sorted.
 	GetShardNames(ctx context.Context, keyspace string) ([]string, error)
 
 	// DeleteShard deletes the provided shard.

go/vt/topo/test/faketopo/faketopo.go

@@ -82,11 +82,6 @@ func (ft FakeTopo) GetKeyspaces(ctx context.Context) ([]string, error) {
 	return nil, errNotImplemented
 }
 
-// DeleteKeyspaceShards is part of the topo.Server interface.
-func (ft FakeTopo) DeleteKeyspaceShards(ctx context.Context, keyspace string) error {
-	return errNotImplemented
-}
-
 // CreateShard is part of the topo.Server interface.
 func (ft FakeTopo) CreateShard(ctx context.Context, keyspace, shard string, value *topodatapb.Shard) error {
 	return errNotImplemented

go/vt/topo/test/keyspace.go

@@ -6,7 +6,6 @@
 package test
 
 import (
-	"reflect"
 	"testing"
 
 	"github.com/youtube/vitess/go/vt/topo"
@@ -78,18 +77,11 @@ func checkKeyspace(t *testing.T, ts topo.Impl) {
 		t.Errorf("GetKeyspaces: want %v, got %v", []string{"test_keyspace", "test_keyspace2"}, keyspaces)
 	}
 
-	// Call delete shards and make sure the keyspace still exists.
-	if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace2"); err != nil {
-		t.Errorf("DeleteKeyspaceShards: %v", err)
-	}
+	// re-read and update.
 	storedK, storedVersion, err := ts.GetKeyspace(ctx, "test_keyspace2")
 	if err != nil {
 		t.Fatalf("GetKeyspace: %v", err)
 	}
-	if !reflect.DeepEqual(storedK, k) {
-		t.Fatalf("returned keyspace doesn't match: got %v expected %v", storedK, k)
-	}
-
 	storedK.ShardingColumnName = "other_id"
 	var newServedFroms []*topodatapb.Keyspace_ServedFrom
 	for _, ksf := range storedK.ServedFroms {

go/vt/topo/test/shard.go

@@ -51,14 +51,7 @@ func checkShard(t *testing.T, ts topo.Impl) {
 		t.Fatalf("CreateShard: %v", err)
 	}
 
-	// Delete ALL shards.
-	if err := ts.DeleteKeyspaceShards(ctx, "test_keyspace"); err != nil {
-		t.Fatalf("DeleteKeyspaceShards: %v", err)
-	}
-	if err := ts.CreateShard(ctx, "test_keyspace", "b0-c0", shard); err != nil {
-		t.Fatalf("CreateShard: %v", err)
-	}
-
+	// Test getting an invalid shard returns ErrNoNode.
 	if _, _, err := ts.GetShard(ctx, "test_keyspace", "666"); err != topo.ErrNoNode {
 		t.Errorf("GetShard(666): %v", err)
 	}

go/vt/vtctl/vtctl.go

@@ -295,9 +295,6 @@ var commands = []commandGroup{
 	},
 	{
 		"Generic", []command{
-			{"RebuildReplicationGraph", commandRebuildReplicationGraph,
-				"<cell1>,<cell2>... <keyspace1>,<keyspace2>,...",
-				"HIDDEN This takes the Thor's hammer approach of recovery and should only be used in emergencies.  cell1,cell2,... are the canonical source of data for the system. This function uses that canonical data to recover the replication graph, at which point further auditing with Validate can reveal any remaining issues."},
 			{"Validate", commandValidate,
 				"[-ping-tablets]",
 				"Validates that all nodes reachable from the global replication graph and that all tablets in all discoverable cells are consistent."},
@@ -1739,24 +1736,6 @@ func commandValidate(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.
 	return wr.Validate(ctx, *pingTablets)
 }
 
-func commandRebuildReplicationGraph(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
-	// This is sort of a nuclear option.
-	if err := subFlags.Parse(args); err != nil {
-		return err
-	}
-	if subFlags.NArg() < 2 {
-		return fmt.Errorf("The <cell> and <keyspace> arguments are both required for the RebuildReplicationGraph command. To specify multiple cells, separate the cell names with commas. Similarly, to specify multiple keyspaces, separate the keyspace names with commas.")
-	}
-
-	cells := strings.Split(subFlags.Arg(0), ",")
-	keyspaceParams := strings.Split(subFlags.Arg(1), ",")
-	keyspaces, err := keyspaceParamsToKeyspaces(ctx, wr, keyspaceParams)
-	if err != nil {
-		return err
-	}
-	return wr.RebuildReplicationGraph(ctx, cells, keyspaces)
-}
-
 func commandListAllTablets(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag.FlagSet, args []string) error {
 	if err := subFlags.Parse(args); err != nil {
 		return err

go/vt/wrangler/rebuild.go

@@ -5,10 +5,6 @@
 package wrangler
 
 import (
-	"fmt"
-	"sync"
-
-	"github.com/youtube/vitess/go/vt/topo"
 	"github.com/youtube/vitess/go/vt/topotools"
 	"golang.org/x/net/context"
 )
@@ -17,97 +13,3 @@ import (
 func (wr *Wrangler) RebuildKeyspaceGraph(ctx context.Context, keyspace string, cells []string) error {
 	return topotools.RebuildKeyspace(ctx, wr.logger, wr.ts, keyspace, cells)
 }
-
-func strInList(sl []string, s string) bool {
-	for _, x := range sl {
-		if x == s {
-			return true
-		}
-	}
-	return false
-}
-
-// RebuildReplicationGraph is a quick and dirty tool to resurrect the TopologyServer data from the
-// canonical data stored in the tablet nodes.
-//
-// cells: local vt cells to scan for all tablets
-// keyspaces: list of keyspaces to rebuild
-func (wr *Wrangler) RebuildReplicationGraph(ctx context.Context, cells []string, keyspaces []string) error {
-	if cells == nil || len(cells) == 0 {
-		return fmt.Errorf("must specify cells to rebuild replication graph")
-	}
-	if keyspaces == nil || len(keyspaces) == 0 {
-		return fmt.Errorf("must specify keyspaces to rebuild replication graph")
-	}
-
-	allTablets := make([]*topo.TabletInfo, 0, 1024)
-	for _, cell := range cells {
-		tablets, err := topotools.GetAllTablets(ctx, wr.ts, cell)
-		if err != nil {
-			return err
-		}
-		allTablets = append(allTablets, tablets...)
-	}
-
-	for _, keyspace := range keyspaces {
-		wr.logger.Infof("delete keyspace shards: %v", keyspace)
-		if err := wr.ts.DeleteKeyspaceShards(ctx, keyspace); err != nil {
-			return err
-		}
-	}
-
-	keyspacesToRebuild := make(map[string]bool)
-	shardsCreated := make(map[string]bool)
-	hasErr := false
-	mu := sync.Mutex{}
-	wg := sync.WaitGroup{}
-	for _, ti := range allTablets {
-		wg.Add(1)
-		go func(ti *topo.TabletInfo) {
-			defer wg.Done()
-			if !strInList(keyspaces, ti.Keyspace) {
-				return
-			}
-			mu.Lock()
-			keyspacesToRebuild[ti.Keyspace] = true
-			shardPath := ti.Keyspace + "/" + ti.Shard
-			if !shardsCreated[shardPath] {
-				if err := wr.ts.CreateShard(ctx, ti.Keyspace, ti.Shard); err != nil && err != topo.ErrNodeExists {
-					wr.logger.Warningf("failed re-creating shard %v: %v", shardPath, err)
-					hasErr = true
-				} else {
-					shardsCreated[shardPath] = true
-				}
-			}
-			mu.Unlock()
-			err := topo.UpdateTabletReplicationData(ctx, wr.ts, ti.Tablet)
-			if err != nil {
-				mu.Lock()
-				hasErr = true
-				mu.Unlock()
-				wr.logger.Warningf("failed updating replication data: %v", err)
-			}
-		}(ti)
-	}
-	wg.Wait()
-
-	for keyspace := range keyspacesToRebuild {
-		wg.Add(1)
-		go func(keyspace string) {
-			defer wg.Done()
-			if err := wr.RebuildKeyspaceGraph(ctx, keyspace, nil); err != nil {
-				mu.Lock()
-				hasErr = true
-				mu.Unlock()
-				wr.logger.Warningf("RebuildKeyspaceGraph(%v) failed: %v", keyspace, err)
-				return
-			}
-		}(keyspace)
-	}
-	wg.Wait()
-
-	if hasErr {
-		return fmt.Errorf("some errors occurred rebuilding replication graph, consult log")
-	}
-	return nil
-}

go/vt/zktopo/keyspace.go

@@ -120,12 +120,3 @@ func (zkts *Server) GetKeyspaces(ctx context.Context) ([]string, error) {
 		return nil, convertError(err)
 	}
 }
-
-// DeleteKeyspaceShards is part of the topo.Server interface
-func (zkts *Server) DeleteKeyspaceShards(ctx context.Context, keyspace string) error {
-	shardsPath := path.Join(GlobalKeyspacesPath, keyspace, "shards")
-	if err := zk.DeleteRecursive(zkts.zconn, shardsPath, -1); err != nil && err != zookeeper.ErrNoNode {
-		return convertError(err)
-	}
-	return nil
-}