From 8a0b6b104246f8429ac0488949888fe9df8f09a4 Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Tue, 24 Mar 2020 17:27:28 -0400 Subject: [PATCH 01/13] Extract struct parsing logic from main Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose.go | 88 ++++++++++++++------ examples/compose/vtcompose/vtcompose_test.go | 29 +++++++ 2 files changed, 93 insertions(+), 24 deletions(-) create mode 100644 examples/compose/vtcompose/vtcompose_test.go diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index 912081b66e6..ab086dad2e6 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -34,21 +34,30 @@ import ( "vitess.io/vitess/go/vt/log" ) +const ( + // TODO: Make these ints insteads of strings. + DEFAULT_WEB_PORT = "8080" + DEFAULT_GRPC_PORT = "15999" + DEFAULT_MYSQL_PORT = "15306" + DEFAULT_KEYSPACE_DATA = "test_keyspace:2:1:create_messages.sql,create_tokens.sql;unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql" + DEFAULT_CELL = "test" + DEFAULT_EXTERNAL_DB_DATA = "" + DEFAULT_TOPOLOGY_FLAGS = "-topo_implementation consul -topo_global_server_address consul1:8500 -topo_global_root vitess/global" +) + var ( tabletsUsed = 0 tablesPath = "tables/" baseDockerComposeFile = flag.String("base_yaml", "vtcompose/docker-compose.base.yml", "Starting docker-compose yaml") baseVschemaFile = flag.String("base_vschema", "vtcompose/base_vschema.json", "Starting vschema json") - topologyFlags = flag.String("topologyFlags", - "-topo_implementation consul -topo_global_server_address consul1:8500 -topo_global_root vitess/global", - "Vitess Topology Flags config") - webPort = flag.String("webPort", "8080", "Web port to be used") - gRpcPort = flag.String("gRpcPort", "15999", "gRPC port to be used") - mySqlPort = flag.String("mySqlPort", "15306", "mySql port to be used") - cell = flag.String("cell", "test", "Vitess Cell name") - keyspaceData = flag.String("keyspaceData", "test_keyspace:2:1:create_messages.sql,create_tokens.sql;unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql", "List of keyspace_name/external_db_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name separated by ';'") - externalDbData = flag.String("externalDbData", "", "List of Data corresponding to external DBs. List of ,,,,, separated by ';'") + topologyFlags = flag.String("topologyFlags", DEFAULT_TOPOLOGY_FLAGS, "Vitess Topology Flags config") + webPort = flag.String("webPort", DEFAULT_WEB_PORT, "Web port to be used") + gRpcPort = flag.String("gRpcPort", DEFAULT_GRPC_PORT, "gRPC port to be used") + mySqlPort = flag.String("mySqlPort", DEFAULT_MYSQL_PORT, "mySql port to be used") + cell = flag.String("cell", DEFAULT_CELL, "Vitess Cell name") + keyspaceData = flag.String("keyspaceData", DEFAULT_KEYSPACE_DATA, "List of keyspace_name/external_db_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name separated by ';'") + externalDbData = flag.String("externalDbData", DEFAULT_EXTERNAL_DB_DATA, "List of Data corresponding to external DBs. List of ,,,,, separated by ';'") ) type keyspaceInfo struct { @@ -82,17 +91,10 @@ func newKeyspaceInfo(keyspace string, shards int, replicaTablets int, schemaFile return k } -func newExternalDbInfo(dbName, dbHost, dbPort, dbUser, dbPass, dbCharset string) externalDbInfo { - d := externalDbInfo{dbName: dbName, dbHost: dbHost, dbPort: dbPort, dbUser: dbUser, dbPass: dbPass, dbCharset: dbCharset} - return d -} - -func main() { - flag.Parse() +func parseKeyspaceInfo(keyspaceData string) map[string]keyspaceInfo { keyspaceInfoMap := make(map[string]keyspaceInfo) - externalDbInfoMap := make(map[string]externalDbInfo) - for _, v := range strings.Split(*keyspaceData, ";") { + for _, v := range strings.Split(keyspaceData, ";") { tokens := strings.Split(v, ":") shards, _ := strconv.Atoi(tokens[1]) replicaTablets, _ := strconv.Atoi(tokens[2]) @@ -106,13 +108,31 @@ func main() { } } - for _, v := range strings.Split(*externalDbData, ";") { + return keyspaceInfoMap +} + +func newExternalDbInfo(dbName, dbHost, dbPort, dbUser, dbPass, dbCharset string) externalDbInfo { + return externalDbInfo{dbName: dbName, dbHost: dbHost, dbPort: dbPort, dbUser: dbUser, dbPass: dbPass, dbCharset: dbCharset} +} + +func parseExternalDbData(externalDbData string) map[string]externalDbInfo { + externalDbInfoMap := make(map[string]externalDbInfo) + for _, v := range strings.Split(externalDbData, ";") { tokens := strings.Split(v, ":") if len(tokens) > 1 { externalDbInfoMap[tokens[0]] = newExternalDbInfo(tokens[0], tokens[1], tokens[2], tokens[3], tokens[4], tokens[5]) } } + return externalDbInfoMap +} + +func main() { + flag.Parse() + keyspaceInfoMap := parseKeyspaceInfo(*keyspaceData) + externalDbInfoMap := parseExternalDbData(*externalDbData) + + // Write schemaFile. for k, v := range keyspaceInfoMap { if _, ok := externalDbInfoMap[k]; !ok { v.schemaFile = createFile(fmt.Sprintf("%s%s_schema_file.sql", tablesPath, v.keyspace)) @@ -453,9 +473,18 @@ func generateDefaultTablet(tabAlias, shard, role, keyspace string, dbInfo extern return data } -// Generate Vtctld -func generateVtctld() string { - data := fmt.Sprintf(` +type vtctldOptions struct { + webPort int + gRpcPort int + topologyFlags string + cell string +} + +func generateVtctld2(opts vtctldOptions) string { + webPort := strconv.Itoa(opts.webPort) + gRpcPort := strconv.Itoa(opts.gRpcPort) + + return fmt.Sprintf(` - op: add path: /services/vtctld value: @@ -482,9 +511,20 @@ func generateVtctld() string { - consul1 - consul2 - consul3 -`, *webPort, *gRpcPort, *topologyFlags, *cell) +`, webPort, gRpcPort, topologyFlags, cell) +} - return data +// Generate Vtctld +// Deprecated: Replaced by generateVtctld2. +func generateVtctld() string { + webPort2, _ := strconv.Atoi(*webPort) + gRpcPort2, _ := strconv.Atoi(*gRpcPort) + return generateVtctld2(vtctldOptions{ + webPort: webPort2, + gRpcPort: gRpcPort2, + topologyFlags: *topologyFlags, + cell: *cell, + }) } // Generate Vtgate diff --git a/examples/compose/vtcompose/vtcompose_test.go b/examples/compose/vtcompose/vtcompose_test.go new file mode 100644 index 00000000000..74574fdb282 --- /dev/null +++ b/examples/compose/vtcompose/vtcompose_test.go @@ -0,0 +1,29 @@ +/* + * Copyright 2019 The Vitess Authors. + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + + * http://www.apache.org/licenses/LICENSE-2.0 + + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "testing" +) + +func TestPatchedDockerComposeFileIsValid(t *testing.T) { + keyspaceInfoMap := make(map[string]keyspaceInfo) + externalDbInfoMap := make(map[string]externalDbInfo) + + baseFile := readFile(*baseDockerComposeFile) + patchedFile := applyDockerComposePatches(baseFile, keyspaceInfoMap, externalDbInfoMap) +} From 5dacb1331711db1f632dbdebcd370d80cee95dc6 Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 11:13:20 -0400 Subject: [PATCH 02/13] Invert dependencies for generateVt* funcs Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose.go | 35 ++++++++++++++++++------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index ab086dad2e6..06afee37ee9 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -473,14 +473,15 @@ func generateDefaultTablet(tabAlias, shard, role, keyspace string, dbInfo extern return data } -type vtctldOptions struct { +type vtOptions struct { webPort int gRpcPort int + mySqlPort int topologyFlags string cell string } -func generateVtctld2(opts vtctldOptions) string { +func generateVtctld2(opts vtOptions) string { webPort := strconv.Itoa(opts.webPort) gRpcPort := strconv.Itoa(opts.gRpcPort) @@ -511,7 +512,7 @@ func generateVtctld2(opts vtctldOptions) string { - consul1 - consul2 - consul3 -`, webPort, gRpcPort, topologyFlags, cell) +`, webPort, gRpcPort, opts.topologyFlags, opts.cell) } // Generate Vtctld @@ -519,7 +520,7 @@ func generateVtctld2(opts vtctldOptions) string { func generateVtctld() string { webPort2, _ := strconv.Atoi(*webPort) gRpcPort2, _ := strconv.Atoi(*gRpcPort) - return generateVtctld2(vtctldOptions{ + return generateVtctld2(vtOptions{ webPort: webPort2, gRpcPort: gRpcPort2, topologyFlags: *topologyFlags, @@ -527,9 +528,12 @@ func generateVtctld() string { }) } -// Generate Vtgate -func generateVtgate() string { - data := fmt.Sprintf(` +func generateVtgate2(opts vtOptions) string { + webPort := strconv.Itoa(opts.webPort) + gRpcPort := strconv.Itoa(opts.gRpcPort) + mySqlPort := strconv.Itoa(opts.mySqlPort) + + return fmt.Sprintf(` - op: add path: /services/vtgate value: @@ -557,9 +561,22 @@ func generateVtgate() string { - .:/script depends_on: - vtctld -`, *webPort, *gRpcPort, *mySqlPort, *topologyFlags, *cell) +`, webPort, gRpcPort, mySqlPort, opts.topologyFlags, opts.cell) +} - return data +// Generate Vtgate +// Deprecated: Replaced by generateVtgate2. +func generateVtgate() string { + webPort2, _ := strconv.Atoi(*webPort) + gRpcPort2, _ := strconv.Atoi(*gRpcPort) + mySqlPort2, _ := strconv.Atoi(*mySqlPort) + return generateVtgate2(vtOptions{ + webPort: webPort2, + gRpcPort: gRpcPort2, + mySqlPort: mySqlPort2, + topologyFlags: *topologyFlags, + cell: *cell, + }) } func generateVtwork() string { From 1271c9f651b9f06014ac7d5f8e8096ec9bb5030d Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 12:16:42 -0400 Subject: [PATCH 03/13] Convert stringly typed vars to strongly typed vars This will enforce validity of the generated file. Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose.go | 149 +++++++++++++----------- 1 file changed, 84 insertions(+), 65 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index 06afee37ee9..69bfa9076a7 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -35,14 +35,13 @@ import ( ) const ( - // TODO: Make these ints insteads of strings. - DEFAULT_WEB_PORT = "8080" - DEFAULT_GRPC_PORT = "15999" - DEFAULT_MYSQL_PORT = "15306" - DEFAULT_KEYSPACE_DATA = "test_keyspace:2:1:create_messages.sql,create_tokens.sql;unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql" - DEFAULT_CELL = "test" - DEFAULT_EXTERNAL_DB_DATA = "" - DEFAULT_TOPOLOGY_FLAGS = "-topo_implementation consul -topo_global_server_address consul1:8500 -topo_global_root vitess/global" + DefaultWebPort = 8080 + DefaultGrpcPort = 15999 + DefaultMysqlPort = 15306 + DefaultKeyspaceData = "test_keyspace:2:1:create_messages.sql,create_tokens.sql;unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql" + DefaultCell = "test" + DefaultExternalDbData = "" + DefaultTopologyFlags = "-topo_implementation consul -topo_global_server_address consul1:8500 -topo_global_root vitess/global" ) var ( @@ -51,13 +50,13 @@ var ( baseDockerComposeFile = flag.String("base_yaml", "vtcompose/docker-compose.base.yml", "Starting docker-compose yaml") baseVschemaFile = flag.String("base_vschema", "vtcompose/base_vschema.json", "Starting vschema json") - topologyFlags = flag.String("topologyFlags", DEFAULT_TOPOLOGY_FLAGS, "Vitess Topology Flags config") - webPort = flag.String("webPort", DEFAULT_WEB_PORT, "Web port to be used") - gRpcPort = flag.String("gRpcPort", DEFAULT_GRPC_PORT, "gRPC port to be used") - mySqlPort = flag.String("mySqlPort", DEFAULT_MYSQL_PORT, "mySql port to be used") - cell = flag.String("cell", DEFAULT_CELL, "Vitess Cell name") - keyspaceData = flag.String("keyspaceData", DEFAULT_KEYSPACE_DATA, "List of keyspace_name/external_db_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name separated by ';'") - externalDbData = flag.String("externalDbData", DEFAULT_EXTERNAL_DB_DATA, "List of Data corresponding to external DBs. List of ,,,,, separated by ';'") + topologyFlags = flag.String("topologyFlags", DefaultTopologyFlags, "Vitess Topology Flags config") + webPort = flag.Int("webPort", DefaultWebPort, "Web port to be used") + gRpcPort = flag.Int("gRpcPort", DefaultGrpcPort, "gRPC port to be used") + mySqlPort = flag.Int("mySqlPort", DefaultMysqlPort, "mySql port to be used") + cell = flag.String("cell", DefaultCell, "Vitess Cell name") + keyspaceData = flag.String("keyspaceData", DefaultKeyspaceData, "List of keyspace_name/external_db_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name separated by ';'") + externalDbData = flag.String("externalDbData", DefaultExternalDbData, "List of Data corresponding to external DBs. List of ,,,,, separated by ';'") ) type keyspaceInfo struct { @@ -407,7 +406,7 @@ func applyDefaultDockerPatches(dockerComposeFile []byte) []byte { } func applyDockerComposePatches(dockerComposeFile []byte, keyspaceInfoMap map[string]keyspaceInfo, externalDbInfoMap map[string]externalDbInfo) []byte { - //Vtctld, vtgate, vtwork patches + // Vtctld, vtgate, vtwork patches. dockerComposeFile = applyDefaultDockerPatches(dockerComposeFile) for _, keyspaceData := range keyspaceInfoMap { dockerComposeFile = applyKeyspaceDependentPatches(dockerComposeFile, keyspaceData, externalDbInfoMap) @@ -428,49 +427,61 @@ func applyTabletPatches(dockerComposeFile []byte, tabAlias int, shard string, ke return dockerComposeFile } -// Default Tablet -func generateDefaultTablet(tabAlias, shard, role, keyspace string, dbInfo externalDbInfo) string { +func generateDefaultTablet2(tabAlias int, shard, role, keyspace string, dbInfo externalDbInfo, opts vtOptions) string { externalDb := "0" if dbInfo.dbName != "" { externalDb = "1" } - data := fmt.Sprintf(` + + return fmt.Sprintf(` - op: add - path: /services/vttablet%[1]s + path: /services/vttablet%[1]d value: image: vitess/base ports: - - "15%[1]s:%[4]s" - - "%[5]s" + - "15%[1]d:%[4]d" + - "%[5]d" - "3306" volumes: - ".:/script" environment: - TOPOLOGY_FLAGS=%[7]s - - WEB_PORT=%[4]s - - GRPC_PORT=%[5]s + - WEB_PORT=%[4]d + - GRPC_PORT=%[5]d - CELL=%[8]s - KEYSPACE=%[6]s - SHARD=%[2]s - ROLE=%[3]s - - VTHOST=vttablet%[1]s + - VTHOST=vttablet%[1]d - EXTERNAL_DB=%[9]s - DB_PORT=%[10]s - DB_HOST=%[11]s - DB_USER=%[12]s - DB_PASS=%[13]s - DB_CHARSET=%[14]s - command: ["sh", "-c", "/script/vttablet-up.sh %[1]s"] + command: ["sh", "-c", "/script/vttablet-up.sh %[1]d"] depends_on: - vtctld healthcheck: - test: ["CMD-SHELL","curl localhost:%[4]s/debug/health"] + test: ["CMD-SHELL","curl localhost:%[4]d/debug/health"] interval: 30s timeout: 10s retries: 15 -`, tabAlias, shard, role, *webPort, *gRpcPort, keyspace, *topologyFlags, *cell, externalDb, dbInfo.dbPort, dbInfo.dbHost, dbInfo.dbUser, dbInfo.dbPass, dbInfo.dbCharset) +`, + tabAlias, shard, role, opts.webPort, opts.gRpcPort, keyspace, opts.topologyFlags, opts.cell, externalDb, + dbInfo.dbPort, dbInfo.dbHost, dbInfo.dbUser, dbInfo.dbPass, dbInfo.dbCharset) +} - return data +// Default Tablet +// Deprecated: Replaced by generateDefaultTablet2. +func generateDefaultTablet(tabAlias, shard, role, keyspace string, dbInfo externalDbInfo) string { + tabAlias2, _ := strconv.Atoi(tabAlias) + return generateDefaultTablet2(tabAlias2, shard, role, keyspace, dbInfo, vtOptions{ + webPort: *webPort, + gRpcPort: *gRpcPort, + topologyFlags: *topologyFlags, + cell: *cell, + }) } type vtOptions struct { @@ -518,36 +529,30 @@ func generateVtctld2(opts vtOptions) string { // Generate Vtctld // Deprecated: Replaced by generateVtctld2. func generateVtctld() string { - webPort2, _ := strconv.Atoi(*webPort) - gRpcPort2, _ := strconv.Atoi(*gRpcPort) return generateVtctld2(vtOptions{ - webPort: webPort2, - gRpcPort: gRpcPort2, + webPort: *webPort, + gRpcPort: *gRpcPort, topologyFlags: *topologyFlags, cell: *cell, }) } func generateVtgate2(opts vtOptions) string { - webPort := strconv.Itoa(opts.webPort) - gRpcPort := strconv.Itoa(opts.gRpcPort) - mySqlPort := strconv.Itoa(opts.mySqlPort) - return fmt.Sprintf(` - op: add path: /services/vtgate value: image: vitess/base ports: - - "15099:%[1]s" - - "%[2]s" - - "15306:%[3]s" + - "15099:%[1]d" + - "%[2]d" + - "15306:%[3]d" command: ["sh", "-c", "/script/run-forever.sh $$VTROOT/bin/vtgate \ %[4]s \ -logtostderr=true \ - -port %[1]s \ - -grpc_port %[2]s \ - -mysql_server_port %[3]s \ + -port %[1]d \ + -grpc_port %[2]d \ + -mysql_server_port %[3]d \ -mysql_auth_server_impl none \ -cell %[5]s \ -cells_to_watch %[5]s \ @@ -561,51 +566,57 @@ func generateVtgate2(opts vtOptions) string { - .:/script depends_on: - vtctld -`, webPort, gRpcPort, mySqlPort, opts.topologyFlags, opts.cell) +`, opts.webPort, opts.gRpcPort, opts.mySqlPort, opts.topologyFlags, opts.cell) } // Generate Vtgate // Deprecated: Replaced by generateVtgate2. func generateVtgate() string { - webPort2, _ := strconv.Atoi(*webPort) - gRpcPort2, _ := strconv.Atoi(*gRpcPort) - mySqlPort2, _ := strconv.Atoi(*mySqlPort) return generateVtgate2(vtOptions{ - webPort: webPort2, - gRpcPort: gRpcPort2, - mySqlPort: mySqlPort2, + webPort: *webPort, + gRpcPort: *gRpcPort, + mySqlPort: *mySqlPort, topologyFlags: *topologyFlags, cell: *cell, }) } -func generateVtwork() string { - data := fmt.Sprintf(` +func generateVtwork2(opts vtOptions) string { + return fmt.Sprintf(` - op: add path: /services/vtwork value: image: vitess/base ports: - - "15100:%[1]s" - - "%[2]s" + - "15100:%[1]d" + - "%[2]d" command: ["sh", "-c", "$$VTROOT/bin/vtworker \ %[3]s \ -cell %[4]s \ -logtostderr=true \ -service_map 'grpc-vtworker' \ - -port %[1]s \ - -grpc_port %[2]s \ + -port %[1]d \ + -grpc_port %[2]d \ -use_v3_resharding_mode=true \ -pid_file $$VTDATAROOT/tmp/vtwork.pid \ "] depends_on: - vtctld -`, *webPort, *gRpcPort, *topologyFlags, *cell) +`, + opts.webPort, opts.gRpcPort, opts.topologyFlags, opts.cell) +} - return data +// Deprecated: Replaced by generateVtwork2. +func generateVtwork() string { + return generateVtwork2(vtOptions{ + webPort: *webPort, + gRpcPort: *gRpcPort, + topologyFlags: *topologyFlags, + cell: *cell, + }) } -func generateSchemaload(tabletAliases []string, postLoadFile string, keyspace string, dbInfo externalDbInfo) string { +func generateSchemaload2(tabletAliases []string, postLoadFile string, keyspace string, dbInfo externalDbInfo, opts vtOptions) string { targetTab := tabletAliases[0] schemaFileName := fmt.Sprintf("%s_schema_file.sql", keyspace) externalDb := "0" @@ -617,10 +628,8 @@ func generateSchemaload(tabletAliases []string, postLoadFile string, keyspace st // Formatting for list in yaml for i, tabletId := range tabletAliases { - //tabletAliases[i] = "\"vttablet" + tabletId + "\"" tabletAliases[i] = "vttablet" + tabletId + ": " + "{condition : service_healthy}" } - //dependsOn := "[" + strings.Join(tabletAliases, ", ") + "]" dependsOn := "depends_on: {" + strings.Join(tabletAliases, ", ") + "}" data := fmt.Sprintf(` @@ -632,8 +641,8 @@ func generateSchemaload(tabletAliases []string, postLoadFile string, keyspace st - ".:/script" environment: - TOPOLOGY_FLAGS=%[3]s - - WEB_PORT=%[4]s - - GRPC_PORT=%[5]s + - WEB_PORT=%[4]d + - GRPC_PORT=%[5]d - CELL=%[6]s - KEYSPACE=%[7]s - TARGETTAB=%[6]s-0000000%[2]s @@ -644,12 +653,22 @@ func generateSchemaload(tabletAliases []string, postLoadFile string, keyspace st - EXTERNAL_DB=%[10]s command: ["sh", "-c", "/script/schemaload.sh"] %[1]s -`, dependsOn, targetTab, *topologyFlags, *webPort, *gRpcPort, *cell, keyspace, postLoadFile, schemaFileName, externalDb) +`, dependsOn, targetTab, opts.topologyFlags, opts.webPort, opts.gRpcPort, opts.cell, keyspace, postLoadFile, schemaFileName, externalDb) return data } -func generatePrimaryVIndex(tableName, column string, name string) string { +// Deprecated: Replaced by generateSchemaload2. +func generateSchemaload(tabletAliases []string, postLoadFile string, keyspace string, dbInfo externalDbInfo) string { + return generateSchemaload2(tabletAliases, postLoadFile, keyspace, dbInfo, vtOptions{ + webPort: *webPort, + gRpcPort: *gRpcPort, + topologyFlags: *topologyFlags, + cell: *cell, + }) +} + +func generatePrimaryVIndex(tableName, column, name string) string { data := fmt.Sprintf(` [{"op": "add", "path": "/tables/%[1]s", From 4a6e8eb2c2ed1df95a0f322f761030e428fdc942 Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 13:29:50 -0400 Subject: [PATCH 04/13] Drill down options to templating funcs Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose.go | 69 ++++++++++++++++++++----- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index 69bfa9076a7..c6d14b588da 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -78,6 +78,14 @@ type externalDbInfo struct { dbCharset string } +type vtOptions struct { + webPort int + gRpcPort int + mySqlPort int + topologyFlags string + cell string +} + func newKeyspaceInfo(keyspace string, shards int, replicaTablets int, schemaFiles []string, lookupKeyspace string) keyspaceInfo { k := keyspaceInfo{keyspace: keyspace, shards: shards, replicaTablets: replicaTablets, schemaFileNames: schemaFiles, lookupKeyspace: lookupKeyspace} if len(strings.TrimSpace(lookupKeyspace)) == 0 { @@ -130,6 +138,13 @@ func main() { flag.Parse() keyspaceInfoMap := parseKeyspaceInfo(*keyspaceData) externalDbInfoMap := parseExternalDbData(*externalDbData) + vtOpts := vtOptions{ + webPort: *webPort, + gRpcPort: *gRpcPort, + mySqlPort: *mySqlPort, + topologyFlags: *topologyFlags, + cell: *cell, + } // Write schemaFile. for k, v := range keyspaceInfoMap { @@ -167,7 +182,7 @@ func main() { // Docker Compose File Patches dockerComposeFile := readFile(*baseDockerComposeFile) - dockerComposeFile = applyDockerComposePatches(dockerComposeFile, keyspaceInfoMap, externalDbInfoMap) + dockerComposeFile = applyDockerComposePatches2(dockerComposeFile, keyspaceInfoMap, externalDbInfoMap, vtOpts) writeFile(dockerComposeFile, "docker-compose.yml") } @@ -354,7 +369,7 @@ func writeFile(file []byte, fileName string) { } } -func applyKeyspaceDependentPatches(dockerComposeFile []byte, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo) []byte { +func applyKeyspaceDependentPatches2(dockerComposeFile []byte, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, opts vtOptions) []byte { var externalDbInfo externalDbInfo if val, ok := externalDbInfoMap[keyspaceData.keyspace]; ok { externalDbInfo = val @@ -373,7 +388,8 @@ func applyKeyspaceDependentPatches(dockerComposeFile []byte, keyspaceData keyspa masterTablets = append(masterTablets, strconv.Itoa((i+1)*100+1)) } - dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateSchemaload(masterTablets, "", keyspaceData.keyspace, externalDbInfo)) + schemaLoad := generateSchemaload2(masterTablets, "", keyspaceData.keyspace, externalDbInfo, opts) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, schemaLoad) // Append Master and Replica Tablets if keyspaceData.shards < 2 { @@ -398,13 +414,46 @@ func applyKeyspaceDependentPatches(dockerComposeFile []byte, keyspaceData keyspa return dockerComposeFile } +// Deprecated: Replaced by applyKeyspaceDependentPatches2. +func applyKeyspaceDependentPatches(dockerComposeFile []byte, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo) []byte { + return applyKeyspaceDependentPatches2(dockerComposeFile, keyspaceData, externalDbInfoMap, vtOptions{ + webPort: *webPort, + gRpcPort: *gRpcPort, + mySqlPort: *mySqlPort, + topologyFlags: *topologyFlags, + cell: *cell, + }) +} + +// Deprecated: Replaced by applyDefaultDockerPatches2. func applyDefaultDockerPatches(dockerComposeFile []byte) []byte { - dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtctld()) - dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtgate()) - dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtwork()) + return applyDefaultDockerPatches2(dockerComposeFile, vtOptions{ + webPort: *webPort, + gRpcPort: *gRpcPort, + mySqlPort: *mySqlPort, + topologyFlags: *topologyFlags, + cell: *cell, + }) +} + +func applyDefaultDockerPatches2(dockerComposeFile []byte, opts vtOptions) []byte { + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtctld2(opts)) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtgate2(opts)) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtwork2(opts)) return dockerComposeFile } +func applyDockerComposePatches2(dockerComposeFile []byte, keyspaceInfoMap map[string]keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, vtOpts vtOptions) []byte { + // Vtctld, vtgate, vtwork patches. + dockerComposeFile = applyDefaultDockerPatches2(dockerComposeFile, vtOpts) + for _, keyspaceData := range keyspaceInfoMap { + dockerComposeFile = applyKeyspaceDependentPatches2(dockerComposeFile, keyspaceData, externalDbInfoMap, vtOpts) + } + + return dockerComposeFile +} + +// Deprecated: Replaced by applyDockerComposePatches2. func applyDockerComposePatches(dockerComposeFile []byte, keyspaceInfoMap map[string]keyspaceInfo, externalDbInfoMap map[string]externalDbInfo) []byte { // Vtctld, vtgate, vtwork patches. dockerComposeFile = applyDefaultDockerPatches(dockerComposeFile) @@ -484,14 +533,6 @@ func generateDefaultTablet(tabAlias, shard, role, keyspace string, dbInfo extern }) } -type vtOptions struct { - webPort int - gRpcPort int - mySqlPort int - topologyFlags string - cell string -} - func generateVtctld2(opts vtOptions) string { webPort := strconv.Itoa(opts.webPort) gRpcPort := strconv.Itoa(opts.gRpcPort) From b2bdc8a9a47972670db1532a79be759999530961 Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 13:38:30 -0400 Subject: [PATCH 05/13] Replace old func implementations that used flag values directly Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose.go | 128 ++++-------------------- 1 file changed, 20 insertions(+), 108 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index c6d14b588da..b5736b88810 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -182,7 +182,7 @@ func main() { // Docker Compose File Patches dockerComposeFile := readFile(*baseDockerComposeFile) - dockerComposeFile = applyDockerComposePatches2(dockerComposeFile, keyspaceInfoMap, externalDbInfoMap, vtOpts) + dockerComposeFile = applyDockerComposePatches(dockerComposeFile, keyspaceInfoMap, externalDbInfoMap, vtOpts) writeFile(dockerComposeFile, "docker-compose.yml") } @@ -369,7 +369,7 @@ func writeFile(file []byte, fileName string) { } } -func applyKeyspaceDependentPatches2(dockerComposeFile []byte, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, opts vtOptions) []byte { +func applyKeyspaceDependentPatches(dockerComposeFile []byte, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, opts vtOptions) []byte { var externalDbInfo externalDbInfo if val, ok := externalDbInfoMap[keyspaceData.keyspace]; ok { externalDbInfo = val @@ -388,13 +388,13 @@ func applyKeyspaceDependentPatches2(dockerComposeFile []byte, keyspaceData keysp masterTablets = append(masterTablets, strconv.Itoa((i+1)*100+1)) } - schemaLoad := generateSchemaload2(masterTablets, "", keyspaceData.keyspace, externalDbInfo, opts) + schemaLoad := generateSchemaload(masterTablets, "", keyspaceData.keyspace, externalDbInfo, opts) dockerComposeFile = applyInMemoryPatch(dockerComposeFile, schemaLoad) // Append Master and Replica Tablets if keyspaceData.shards < 2 { tabAlias = tabAlias + 100 - dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap) + dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts) } else { // Determine shard range for i := 0; i < keyspaceData.shards; i++ { @@ -406,7 +406,7 @@ func applyKeyspaceDependentPatches2(dockerComposeFile []byte, keyspaceData keysp shard = fmt.Sprintf("%x-%x", interval*(i), interval*(i+1)) } tabAlias = tabAlias + 100 - dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap) + dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts) } } @@ -414,69 +414,36 @@ func applyKeyspaceDependentPatches2(dockerComposeFile []byte, keyspaceData keysp return dockerComposeFile } -// Deprecated: Replaced by applyKeyspaceDependentPatches2. -func applyKeyspaceDependentPatches(dockerComposeFile []byte, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo) []byte { - return applyKeyspaceDependentPatches2(dockerComposeFile, keyspaceData, externalDbInfoMap, vtOptions{ - webPort: *webPort, - gRpcPort: *gRpcPort, - mySqlPort: *mySqlPort, - topologyFlags: *topologyFlags, - cell: *cell, - }) -} - -// Deprecated: Replaced by applyDefaultDockerPatches2. -func applyDefaultDockerPatches(dockerComposeFile []byte) []byte { - return applyDefaultDockerPatches2(dockerComposeFile, vtOptions{ - webPort: *webPort, - gRpcPort: *gRpcPort, - mySqlPort: *mySqlPort, - topologyFlags: *topologyFlags, - cell: *cell, - }) -} - -func applyDefaultDockerPatches2(dockerComposeFile []byte, opts vtOptions) []byte { - dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtctld2(opts)) - dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtgate2(opts)) - dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtwork2(opts)) +func applyDefaultDockerPatches(dockerComposeFile []byte, opts vtOptions) []byte { + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtctld(opts)) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtgate(opts)) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateVtwork(opts)) return dockerComposeFile } -func applyDockerComposePatches2(dockerComposeFile []byte, keyspaceInfoMap map[string]keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, vtOpts vtOptions) []byte { +func applyDockerComposePatches(dockerComposeFile []byte, keyspaceInfoMap map[string]keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, vtOpts vtOptions) []byte { // Vtctld, vtgate, vtwork patches. - dockerComposeFile = applyDefaultDockerPatches2(dockerComposeFile, vtOpts) + dockerComposeFile = applyDefaultDockerPatches(dockerComposeFile, vtOpts) for _, keyspaceData := range keyspaceInfoMap { - dockerComposeFile = applyKeyspaceDependentPatches2(dockerComposeFile, keyspaceData, externalDbInfoMap, vtOpts) + dockerComposeFile = applyKeyspaceDependentPatches(dockerComposeFile, keyspaceData, externalDbInfoMap, vtOpts) } return dockerComposeFile } -// Deprecated: Replaced by applyDockerComposePatches2. -func applyDockerComposePatches(dockerComposeFile []byte, keyspaceInfoMap map[string]keyspaceInfo, externalDbInfoMap map[string]externalDbInfo) []byte { - // Vtctld, vtgate, vtwork patches. - dockerComposeFile = applyDefaultDockerPatches(dockerComposeFile) - for _, keyspaceData := range keyspaceInfoMap { - dockerComposeFile = applyKeyspaceDependentPatches(dockerComposeFile, keyspaceData, externalDbInfoMap) - } - - return dockerComposeFile -} - -func applyTabletPatches(dockerComposeFile []byte, tabAlias int, shard string, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo) []byte { +func applyTabletPatches(dockerComposeFile []byte, tabAlias int, shard string, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, opts vtOptions) []byte { var dbInfo externalDbInfo if val, ok := externalDbInfoMap[keyspaceData.keyspace]; ok { dbInfo = val } - dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultTablet(strconv.Itoa(tabAlias+1), shard, "master", keyspaceData.keyspace, dbInfo)) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultTablet(tabAlias+1, shard, "master", keyspaceData.keyspace, dbInfo, opts)) for i := 0; i < keyspaceData.replicaTablets; i++ { - dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultTablet(strconv.Itoa(tabAlias+2+i), shard, "replica", keyspaceData.keyspace, dbInfo)) + dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultTablet(tabAlias+2+i, shard, "replica", keyspaceData.keyspace, dbInfo, opts)) } return dockerComposeFile } -func generateDefaultTablet2(tabAlias int, shard, role, keyspace string, dbInfo externalDbInfo, opts vtOptions) string { +func generateDefaultTablet(tabAlias int, shard, role, keyspace string, dbInfo externalDbInfo, opts vtOptions) string { externalDb := "0" if dbInfo.dbName != "" { externalDb = "1" @@ -521,19 +488,7 @@ func generateDefaultTablet2(tabAlias int, shard, role, keyspace string, dbInfo e dbInfo.dbPort, dbInfo.dbHost, dbInfo.dbUser, dbInfo.dbPass, dbInfo.dbCharset) } -// Default Tablet -// Deprecated: Replaced by generateDefaultTablet2. -func generateDefaultTablet(tabAlias, shard, role, keyspace string, dbInfo externalDbInfo) string { - tabAlias2, _ := strconv.Atoi(tabAlias) - return generateDefaultTablet2(tabAlias2, shard, role, keyspace, dbInfo, vtOptions{ - webPort: *webPort, - gRpcPort: *gRpcPort, - topologyFlags: *topologyFlags, - cell: *cell, - }) -} - -func generateVtctld2(opts vtOptions) string { +func generateVtctld(opts vtOptions) string { webPort := strconv.Itoa(opts.webPort) gRpcPort := strconv.Itoa(opts.gRpcPort) @@ -567,18 +522,7 @@ func generateVtctld2(opts vtOptions) string { `, webPort, gRpcPort, opts.topologyFlags, opts.cell) } -// Generate Vtctld -// Deprecated: Replaced by generateVtctld2. -func generateVtctld() string { - return generateVtctld2(vtOptions{ - webPort: *webPort, - gRpcPort: *gRpcPort, - topologyFlags: *topologyFlags, - cell: *cell, - }) -} - -func generateVtgate2(opts vtOptions) string { +func generateVtgate(opts vtOptions) string { return fmt.Sprintf(` - op: add path: /services/vtgate @@ -610,19 +554,7 @@ func generateVtgate2(opts vtOptions) string { `, opts.webPort, opts.gRpcPort, opts.mySqlPort, opts.topologyFlags, opts.cell) } -// Generate Vtgate -// Deprecated: Replaced by generateVtgate2. -func generateVtgate() string { - return generateVtgate2(vtOptions{ - webPort: *webPort, - gRpcPort: *gRpcPort, - mySqlPort: *mySqlPort, - topologyFlags: *topologyFlags, - cell: *cell, - }) -} - -func generateVtwork2(opts vtOptions) string { +func generateVtwork(opts vtOptions) string { return fmt.Sprintf(` - op: add path: /services/vtwork @@ -647,17 +579,7 @@ func generateVtwork2(opts vtOptions) string { opts.webPort, opts.gRpcPort, opts.topologyFlags, opts.cell) } -// Deprecated: Replaced by generateVtwork2. -func generateVtwork() string { - return generateVtwork2(vtOptions{ - webPort: *webPort, - gRpcPort: *gRpcPort, - topologyFlags: *topologyFlags, - cell: *cell, - }) -} - -func generateSchemaload2(tabletAliases []string, postLoadFile string, keyspace string, dbInfo externalDbInfo, opts vtOptions) string { +func generateSchemaload(tabletAliases []string, postLoadFile string, keyspace string, dbInfo externalDbInfo, opts vtOptions) string { targetTab := tabletAliases[0] schemaFileName := fmt.Sprintf("%s_schema_file.sql", keyspace) externalDb := "0" @@ -699,16 +621,6 @@ func generateSchemaload2(tabletAliases []string, postLoadFile string, keyspace s return data } -// Deprecated: Replaced by generateSchemaload2. -func generateSchemaload(tabletAliases []string, postLoadFile string, keyspace string, dbInfo externalDbInfo) string { - return generateSchemaload2(tabletAliases, postLoadFile, keyspace, dbInfo, vtOptions{ - webPort: *webPort, - gRpcPort: *gRpcPort, - topologyFlags: *topologyFlags, - cell: *cell, - }) -} - func generatePrimaryVIndex(tableName, column, name string) string { data := fmt.Sprintf(` [{"op": "add", From 80b34f1d136ec8404d6edc7ce14f6ce8bc95beab Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 13:56:02 -0400 Subject: [PATCH 06/13] Break up long lines to improve readability Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose.go | 79 ++++++++++++++++++++----- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index b5736b88810..394f3ba4c61 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -42,6 +42,14 @@ const ( DefaultCell = "test" DefaultExternalDbData = "" DefaultTopologyFlags = "-topo_implementation consul -topo_global_server_address consul1:8500 -topo_global_root vitess/global" + + topologyFlagsUsage = "Vitess Topology Flags config" + webPortUsage = "Web port to be used" + gRpcPortUsage = "gRPC port to be used" + mySqlPortUsage = "mySql port to be used" + cellUsage = "Vitess Cell name" + keyspaceDataUsage = "List of keyspace_name/external_db_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name separated by ';'" + externalDbDataUsage = "List of Data corresponding to external DBs. List of ,,,,, separated by ';'" ) var ( @@ -50,13 +58,13 @@ var ( baseDockerComposeFile = flag.String("base_yaml", "vtcompose/docker-compose.base.yml", "Starting docker-compose yaml") baseVschemaFile = flag.String("base_vschema", "vtcompose/base_vschema.json", "Starting vschema json") - topologyFlags = flag.String("topologyFlags", DefaultTopologyFlags, "Vitess Topology Flags config") - webPort = flag.Int("webPort", DefaultWebPort, "Web port to be used") - gRpcPort = flag.Int("gRpcPort", DefaultGrpcPort, "gRPC port to be used") - mySqlPort = flag.Int("mySqlPort", DefaultMysqlPort, "mySql port to be used") - cell = flag.String("cell", DefaultCell, "Vitess Cell name") - keyspaceData = flag.String("keyspaceData", DefaultKeyspaceData, "List of keyspace_name/external_db_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name separated by ';'") - externalDbData = flag.String("externalDbData", DefaultExternalDbData, "List of Data corresponding to external DBs. List of ,,,,, separated by ';'") + topologyFlags = flag.String("topologyFlags", DefaultTopologyFlags, topologyFlagsUsage) + webPort = flag.Int("webPort", DefaultWebPort, webPortUsage) + gRpcPort = flag.Int("gRpcPort", DefaultGrpcPort, gRpcPortUsage) + mySqlPort = flag.Int("mySqlPort", DefaultMysqlPort, mySqlPortUsage) + cell = flag.String("cell", DefaultCell, cellUsage) + keyspaceData = flag.String("keyspaceData", DefaultKeyspaceData, keyspaceDataUsage) + externalDbData = flag.String("externalDbData", DefaultExternalDbData, externalDbDataUsage) ) type keyspaceInfo struct { @@ -86,8 +94,20 @@ type vtOptions struct { cell string } -func newKeyspaceInfo(keyspace string, shards int, replicaTablets int, schemaFiles []string, lookupKeyspace string) keyspaceInfo { - k := keyspaceInfo{keyspace: keyspace, shards: shards, replicaTablets: replicaTablets, schemaFileNames: schemaFiles, lookupKeyspace: lookupKeyspace} +func newKeyspaceInfo( + keyspace string, + shards int, + replicaTablets int, + schemaFiles []string, + lookupKeyspace string, +) keyspaceInfo { + k := keyspaceInfo{ + keyspace: keyspace, + shards: shards, + replicaTablets: replicaTablets, + schemaFileNames: schemaFiles, + lookupKeyspace: lookupKeyspace, + } if len(strings.TrimSpace(lookupKeyspace)) == 0 { k.useLookups = false } else { @@ -326,7 +346,12 @@ func addTablesVschemaPatch(vSchemaFile []byte, schemaFileNames []string) ([]byte return vSchemaFile, primaryTableColumns } -func addLookupDataToVschema(vSchemaFile []byte, schemaFileNames []string, primaryTableColumns map[string]string, keyspace string) []byte { +func addLookupDataToVschema( + vSchemaFile []byte, + schemaFileNames []string, + primaryTableColumns map[string]string, + keyspace string, +) []byte { for _, fileName := range schemaFileNames { tableName := fileName[7 : len(fileName)-4] lookupTableOwner := "" @@ -369,7 +394,12 @@ func writeFile(file []byte, fileName string) { } } -func applyKeyspaceDependentPatches(dockerComposeFile []byte, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, opts vtOptions) []byte { +func applyKeyspaceDependentPatches( + dockerComposeFile []byte, + keyspaceData keyspaceInfo, + externalDbInfoMap map[string]externalDbInfo, + opts vtOptions, +) []byte { var externalDbInfo externalDbInfo if val, ok := externalDbInfoMap[keyspaceData.keyspace]; ok { externalDbInfo = val @@ -421,7 +451,12 @@ func applyDefaultDockerPatches(dockerComposeFile []byte, opts vtOptions) []byte return dockerComposeFile } -func applyDockerComposePatches(dockerComposeFile []byte, keyspaceInfoMap map[string]keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, vtOpts vtOptions) []byte { +func applyDockerComposePatches( + dockerComposeFile []byte, + keyspaceInfoMap map[string]keyspaceInfo, + externalDbInfoMap map[string]externalDbInfo, + vtOpts vtOptions, +) []byte { // Vtctld, vtgate, vtwork patches. dockerComposeFile = applyDefaultDockerPatches(dockerComposeFile, vtOpts) for _, keyspaceData := range keyspaceInfoMap { @@ -431,7 +466,14 @@ func applyDockerComposePatches(dockerComposeFile []byte, keyspaceInfoMap map[str return dockerComposeFile } -func applyTabletPatches(dockerComposeFile []byte, tabAlias int, shard string, keyspaceData keyspaceInfo, externalDbInfoMap map[string]externalDbInfo, opts vtOptions) []byte { +func applyTabletPatches( + dockerComposeFile []byte, + tabAlias int, + shard string, + keyspaceData keyspaceInfo, + externalDbInfoMap map[string]externalDbInfo, + opts vtOptions, +) []byte { var dbInfo externalDbInfo if val, ok := externalDbInfoMap[keyspaceData.keyspace]; ok { dbInfo = val @@ -575,11 +617,16 @@ func generateVtwork(opts vtOptions) string { "] depends_on: - vtctld -`, - opts.webPort, opts.gRpcPort, opts.topologyFlags, opts.cell) +`, opts.webPort, opts.gRpcPort, opts.topologyFlags, opts.cell) } -func generateSchemaload(tabletAliases []string, postLoadFile string, keyspace string, dbInfo externalDbInfo, opts vtOptions) string { +func generateSchemaload( + tabletAliases []string, + postLoadFile string, + keyspace string, + dbInfo externalDbInfo, + opts vtOptions, +) string { targetTab := tabletAliases[0] schemaFileName := fmt.Sprintf("%s_schema_file.sql", keyspace) externalDb := "0" From 165d270cbc9ad07f994bc6f893153118ceb8efdd Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 13:59:39 -0400 Subject: [PATCH 07/13] Remove some leftover string conversions and inline some vars Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose.go | 29 ++++++++----------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index 394f3ba4c61..0a6cd1e073d 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -531,17 +531,14 @@ func generateDefaultTablet(tabAlias int, shard, role, keyspace string, dbInfo ex } func generateVtctld(opts vtOptions) string { - webPort := strconv.Itoa(opts.webPort) - gRpcPort := strconv.Itoa(opts.gRpcPort) - return fmt.Sprintf(` - op: add path: /services/vtctld value: image: vitess/base ports: - - "15000:%[1]s" - - "%[2]s" + - "15000:%[1]d" + - "%[2]d" command: ["sh", "-c", " $$VTROOT/bin/vtctld \ %[3]s \ -cell %[4]s \ @@ -551,8 +548,8 @@ func generateVtctld(opts vtOptions) string { -backup_storage_implementation file \ -file_backup_storage_root $$VTDATAROOT/backups \ -logtostderr=true \ - -port %[1]s \ - -grpc_port %[2]s \ + -port %[1]d \ + -grpc_port %[2]d \ -pid_file $$VTDATAROOT/tmp/vtctld.pid "] volumes: @@ -561,7 +558,7 @@ func generateVtctld(opts vtOptions) string { - consul1 - consul2 - consul3 -`, webPort, gRpcPort, opts.topologyFlags, opts.cell) +`, opts.webPort, opts.gRpcPort, opts.topologyFlags, opts.cell) } func generateVtgate(opts vtOptions) string { @@ -642,7 +639,7 @@ func generateSchemaload( } dependsOn := "depends_on: {" + strings.Join(tabletAliases, ", ") + "}" - data := fmt.Sprintf(` + return fmt.Sprintf(` - op: add path: /services/schemaload_%[7]s value: @@ -664,12 +661,10 @@ func generateSchemaload( command: ["sh", "-c", "/script/schemaload.sh"] %[1]s `, dependsOn, targetTab, opts.topologyFlags, opts.webPort, opts.gRpcPort, opts.cell, keyspace, postLoadFile, schemaFileName, externalDb) - - return data } func generatePrimaryVIndex(tableName, column, name string) string { - data := fmt.Sprintf(` + return fmt.Sprintf(` [{"op": "add", "path": "/tables/%[1]s", "value": @@ -681,12 +676,10 @@ func generatePrimaryVIndex(tableName, column, name string) string { ]} }] `, tableName, column, name) - - return data } func generateVschemaLookupHash(tableName, tableKeyspace, from, to, owner string) string { - data := fmt.Sprintf(` + return fmt.Sprintf(` [{"op": "add", "path": "/vindexes/%[1]s", "value": @@ -701,12 +694,10 @@ func generateVschemaLookupHash(tableName, tableKeyspace, from, to, owner string) } }] `, tableName, tableKeyspace, from, to, owner) - - return data } func addToColumnVIndexes(tableName, column, referenceName string) string { - data := fmt.Sprintf(` + return fmt.Sprintf(` [{"op": "add", "path": "/tables/%[1]s/column_vindexes/-", "value": @@ -716,6 +707,4 @@ func addToColumnVIndexes(tableName, column, referenceName string) string { } }] `, tableName, column, referenceName) - - return data } From 00cd02d24637af6deea227207861394ecfeb57c7 Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 14:04:42 -0400 Subject: [PATCH 08/13] Move some declarations - Moved vtOptions struct closer to flags declaration - Grouped struct parsing functions Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index 0a6cd1e073d..7167736dbd1 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -67,6 +67,14 @@ var ( externalDbData = flag.String("externalDbData", DefaultExternalDbData, externalDbDataUsage) ) +type vtOptions struct { + webPort int + gRpcPort int + mySqlPort int + topologyFlags string + cell string +} + type keyspaceInfo struct { keyspace string shards int @@ -86,14 +94,6 @@ type externalDbInfo struct { dbCharset string } -type vtOptions struct { - webPort int - gRpcPort int - mySqlPort int - topologyFlags string - cell string -} - func newKeyspaceInfo( keyspace string, shards int, @@ -118,6 +118,10 @@ func newKeyspaceInfo( return k } +func newExternalDbInfo(dbName, dbHost, dbPort, dbUser, dbPass, dbCharset string) externalDbInfo { + return externalDbInfo{dbName: dbName, dbHost: dbHost, dbPort: dbPort, dbUser: dbUser, dbPass: dbPass, dbCharset: dbCharset} +} + func parseKeyspaceInfo(keyspaceData string) map[string]keyspaceInfo { keyspaceInfoMap := make(map[string]keyspaceInfo) @@ -138,10 +142,6 @@ func parseKeyspaceInfo(keyspaceData string) map[string]keyspaceInfo { return keyspaceInfoMap } -func newExternalDbInfo(dbName, dbHost, dbPort, dbUser, dbPass, dbCharset string) externalDbInfo { - return externalDbInfo{dbName: dbName, dbHost: dbHost, dbPort: dbPort, dbUser: dbUser, dbPass: dbPass, dbCharset: dbCharset} -} - func parseExternalDbData(externalDbData string) map[string]externalDbInfo { externalDbInfoMap := make(map[string]externalDbInfo) for _, v := range strings.Split(externalDbData, ";") { From 9c29135c911e5579f5ab4b8c0eaf1a4d968e66ad Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 14:39:57 -0400 Subject: [PATCH 09/13] Clean up and remove random prints Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index 7167736dbd1..cf0cfd9e2cd 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -130,7 +130,6 @@ func parseKeyspaceInfo(keyspaceData string) map[string]keyspaceInfo { shards, _ := strconv.Atoi(tokens[1]) replicaTablets, _ := strconv.Atoi(tokens[2]) schemaFileNames := strings.Split(tokens[3], ",") - print(shards) if len(tokens) > 4 { keyspaceInfoMap[tokens[0]] = newKeyspaceInfo(tokens[0], shards, replicaTablets, schemaFileNames, tokens[4]) @@ -327,7 +326,6 @@ func getKeyColumns(sqlFile string) string { r, _ := regexp.Compile("[^PRIMARY] (KEY|UNIQUE KEY) .*\\((.*)\\).*") rs := r.FindStringSubmatch(string(sqlFileData)) - print(rs[2]) // replace all ` from column names if exists return strings.ReplaceAll(rs[2], "`", "") } @@ -370,8 +368,8 @@ func addLookupDataToVschema( vSchemaFile = applyJsonInMemoryPatch(vSchemaFile, addToColumnVIndexes(lookupTableOwner, firstColumnName, tableName)) // Generate Vschema lookup hash types - vSchemaFile = applyJsonInMemoryPatch(vSchemaFile, - generateVschemaLookupHash(tableName, keyspace, firstColumnName, primaryTableColumns[lookupTableOwner], lookupTableOwner)) + lookupHash := generateVschemaLookupHash(tableName, keyspace, firstColumnName, primaryTableColumns[lookupTableOwner], lookupTableOwner) + vSchemaFile = applyJsonInMemoryPatch(vSchemaFile, lookupHash) } return vSchemaFile @@ -525,9 +523,7 @@ func generateDefaultTablet(tabAlias int, shard, role, keyspace string, dbInfo ex interval: 30s timeout: 10s retries: 15 -`, - tabAlias, shard, role, opts.webPort, opts.gRpcPort, keyspace, opts.topologyFlags, opts.cell, externalDb, - dbInfo.dbPort, dbInfo.dbHost, dbInfo.dbUser, dbInfo.dbPass, dbInfo.dbCharset) +`, tabAlias, shard, role, opts.webPort, opts.gRpcPort, keyspace, opts.topologyFlags, opts.cell, externalDb, dbInfo.dbPort, dbInfo.dbHost, dbInfo.dbUser, dbInfo.dbPass, dbInfo.dbCharset) } func generateVtctld(opts vtOptions) string { From 76446029f240ef42dd8d4a1fca3234178cfd814a Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 14:43:10 -0400 Subject: [PATCH 10/13] Add a smoke test for docker compose file generation Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose_test.go | 345 ++++++++++++++++++- 1 file changed, 340 insertions(+), 5 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose_test.go b/examples/compose/vtcompose/vtcompose_test.go index 74574fdb282..8cd01610064 100644 --- a/examples/compose/vtcompose/vtcompose_test.go +++ b/examples/compose/vtcompose/vtcompose_test.go @@ -17,13 +17,348 @@ package main import ( + "github.com/stretchr/testify/assert" "testing" ) -func TestPatchedDockerComposeFileIsValid(t *testing.T) { - keyspaceInfoMap := make(map[string]keyspaceInfo) - externalDbInfoMap := make(map[string]externalDbInfo) +var ( + testVtOpts = vtOptions{ + webPort: DefaultWebPort, + gRpcPort: DefaultGrpcPort, + mySqlPort: DefaultMysqlPort, + topologyFlags: DefaultTopologyFlags, + cell: DefaultCell, + } + testComposeFile = readFile("./docker-compose.base.yml") + testKeyspaceInfoMap = parseKeyspaceInfo(DefaultKeyspaceData) + testExternalDbInfoMap = parseExternalDbData(DefaultExternalDbData) + referenceFile string +) + +func TestGeneratesCorrectFileWithDefaultOpts(t *testing.T) { + baseFile := testComposeFile + finalFile := applyDockerComposePatches(baseFile, testKeyspaceInfoMap, testExternalDbInfoMap, testVtOpts) + + assert.YAMLEq(t, referenceFile, string(finalFile)) +} - baseFile := readFile(*baseDockerComposeFile) - patchedFile := applyDockerComposePatches(baseFile, keyspaceInfoMap, externalDbInfoMap) +func init() { + referenceFile = ` +services: + consul1: + command: agent -server -bootstrap-expect 3 -ui -disable-host-node-id -client 0.0.0.0 + hostname: consul1 + image: consul:latest + ports: + - 8400:8400 + - 8500:8500 + - 8600:8600 + consul2: + command: agent -server -retry-join consul1 -disable-host-node-id + depends_on: + - consul1 + expose: + - "8400" + - "8500" + - "8600" + hostname: consul2 + image: consul:latest + consul3: + command: agent -server -retry-join consul1 -disable-host-node-id + depends_on: + - consul1 + expose: + - "8400" + - "8500" + - "8600" + hostname: consul3 + image: consul:latest + schemaload_test_keyspace: + command: + - sh + - -c + - /script/schemaload.sh + depends_on: + vttablet101: + condition: service_healthy + vttablet201: + condition: service_healthy + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - TARGETTAB=test-0000000101 + - SLEEPTIME=15 + - VSCHEMA_FILE=test_keyspace_vschema.json + - SCHEMA_FILES=test_keyspace_schema_file.sql + - POST_LOAD_FILE= + - EXTERNAL_DB=0 + image: vitess/base + volumes: + - .:/script + schemaload_unsharded_keyspace: + command: + - sh + - -c + - /script/schemaload.sh + depends_on: + vttablet301: + condition: service_healthy + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=unsharded_keyspace + - TARGETTAB=test-0000000301 + - SLEEPTIME=15 + - VSCHEMA_FILE=unsharded_keyspace_vschema.json + - SCHEMA_FILES=unsharded_keyspace_schema_file.sql + - POST_LOAD_FILE= + - EXTERNAL_DB=0 + image: vitess/base + volumes: + - .:/script + vtctld: + command: + - sh + - -c + - ' $$VTROOT/bin/vtctld -topo_implementation consul -topo_global_server_address + consul1:8500 -topo_global_root vitess/global -cell test -workflow_manager_init + -workflow_manager_use_election -service_map ''grpc-vtctl'' -backup_storage_implementation + file -file_backup_storage_root $$VTDATAROOT/backups -logtostderr=true -port + 8080 -grpc_port 15999 -pid_file $$VTDATAROOT/tmp/vtctld.pid ' + depends_on: + - consul1 + - consul2 + - consul3 + image: vitess/base + ports: + - 15000:8080 + - "15999" + volumes: + - .:/script + vtgate: + command: + - sh + - -c + - '/script/run-forever.sh $$VTROOT/bin/vtgate -topo_implementation consul -topo_global_server_address + consul1:8500 -topo_global_root vitess/global -logtostderr=true -port 8080 -grpc_port + 15999 -mysql_server_port 15306 -mysql_auth_server_impl none -cell test -cells_to_watch + test -tablet_types_to_wait MASTER,REPLICA,RDONLY -gateway_implementation discoverygateway + -service_map ''grpc-vtgateservice'' -pid_file $$VTDATAROOT/tmp/vtgate.pid -normalize_queries=true ' + depends_on: + - vtctld + image: vitess/base + ports: + - 15099:8080 + - "15999" + - 15306:15306 + volumes: + - .:/script + vttablet101: + command: + - sh + - -c + - /script/vttablet-up.sh 101 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - SHARD=-80 + - ROLE=master + - VTHOST=vttablet101 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15101:8080 + - "15999" + - "3306" + volumes: + - .:/script + vttablet102: + command: + - sh + - -c + - /script/vttablet-up.sh 102 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - SHARD=-80 + - ROLE=replica + - VTHOST=vttablet102 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15102:8080 + - "15999" + - "3306" + volumes: + - .:/script + vttablet201: + command: + - sh + - -c + - /script/vttablet-up.sh 201 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - SHARD=80- + - ROLE=master + - VTHOST=vttablet201 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15201:8080 + - "15999" + - "3306" + volumes: + - .:/script + vttablet202: + command: + - sh + - -c + - /script/vttablet-up.sh 202 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - SHARD=80- + - ROLE=replica + - VTHOST=vttablet202 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15202:8080 + - "15999" + - "3306" + volumes: + - .:/script + vttablet301: + command: + - sh + - -c + - /script/vttablet-up.sh 301 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=unsharded_keyspace + - SHARD=- + - ROLE=master + - VTHOST=vttablet301 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15301:8080 + - "15999" + - "3306" + volumes: + - .:/script + vtwork: + command: + - sh + - -c + - '$$VTROOT/bin/vtworker -topo_implementation consul -topo_global_server_address + consul1:8500 -topo_global_root vitess/global -cell test -logtostderr=true -service_map + ''grpc-vtworker'' -port 8080 -grpc_port 15999 -use_v3_resharding_mode=true -pid_file + $$VTDATAROOT/tmp/vtwork.pid ' + depends_on: + - vtctld + image: vitess/base + ports: + - 15100:8080 + - "15999" +version: "2.1" +` } From b3a6383d1276be5b2e7825b6d1e39e0676b83fc8 Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Wed, 25 Mar 2020 15:40:41 -0400 Subject: [PATCH 11/13] Add another test Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose_test.go | 36 ++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/examples/compose/vtcompose/vtcompose_test.go b/examples/compose/vtcompose/vtcompose_test.go index 8cd01610064..9a5de6f2ecf 100644 --- a/examples/compose/vtcompose/vtcompose_test.go +++ b/examples/compose/vtcompose/vtcompose_test.go @@ -17,7 +17,9 @@ package main import ( + "fmt" "github.com/stretchr/testify/assert" + "strconv" "testing" ) @@ -35,11 +37,41 @@ var ( referenceFile string ) -func TestGeneratesCorrectFileWithDefaultOpts(t *testing.T) { +func TestGenerateCorrectFileWithDefaultOpts(t *testing.T) { baseFile := testComposeFile finalFile := applyDockerComposePatches(baseFile, testKeyspaceInfoMap, testExternalDbInfoMap, testVtOpts) - assert.YAMLEq(t, referenceFile, string(finalFile)) + yamlString := string(finalFile) + assert.YAMLEq(t, referenceFile, yamlString) +} + +func TestOptsAppliedThroughoutGeneratedFile(t *testing.T) { + baseFile := testComposeFile + options := vtOptions{ + webPort: 55_555, + gRpcPort: 66_666, + mySqlPort: 77_777, + topologyFlags: "-custom -flags", + cell: "custom cell", + } + finalFile := applyDockerComposePatches(baseFile, testKeyspaceInfoMap, testExternalDbInfoMap, options) + yamlString := string(finalFile) + + // These asserts are not exhaustive, but should cover most cases. + assert.NotContains(t, yamlString, strconv.Itoa(DefaultWebPort)) + assert.Contains(t, yamlString, strconv.Itoa(options.webPort)) + + assert.NotContains(t, yamlString, strconv.Itoa(DefaultGrpcPort)) + assert.Contains(t, yamlString, strconv.Itoa(options.gRpcPort)) + + assert.NotContains(t, yamlString, ":"+strconv.Itoa(DefaultMysqlPort)) + assert.Contains(t, yamlString, ":"+strconv.Itoa(options.webPort)) + + assert.NotContains(t, yamlString, fmt.Sprintf("-cell %s", DefaultCell)) + assert.Contains(t, yamlString, fmt.Sprintf("-cell %s", options.cell)) + + assert.Contains(t, yamlString, fmt.Sprintf("- TOPOLOGY_FLAGS=%s", options.topologyFlags)) + assert.NotContains(t, yamlString, DefaultTopologyFlags) } func init() { From 17e65f512769cf55351244c86b9b9444b9a5245b Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Tue, 31 Mar 2020 09:36:15 -0400 Subject: [PATCH 12/13] Clean up Signed-off-by: Keefer Rourke --- .../compose/vtcompose/docker-compose.test.yml | 318 +++++++++++++++++ examples/compose/vtcompose/vtcompose.go | 35 +- examples/compose/vtcompose/vtcompose_test.go | 334 +----------------- 3 files changed, 346 insertions(+), 341 deletions(-) create mode 100644 examples/compose/vtcompose/docker-compose.test.yml diff --git a/examples/compose/vtcompose/docker-compose.test.yml b/examples/compose/vtcompose/docker-compose.test.yml new file mode 100644 index 00000000000..1e55db8e9fd --- /dev/null +++ b/examples/compose/vtcompose/docker-compose.test.yml @@ -0,0 +1,318 @@ +# This file was generated by running `go run vtcompose/vtcompose.go` with no arguments. + +services: + consul1: + command: agent -server -bootstrap-expect 3 -ui -disable-host-node-id -client 0.0.0.0 + hostname: consul1 + image: consul:latest + ports: + - 8400:8400 + - 8500:8500 + - 8600:8600 + consul2: + command: agent -server -retry-join consul1 -disable-host-node-id + depends_on: + - consul1 + expose: + - "8400" + - "8500" + - "8600" + hostname: consul2 + image: consul:latest + consul3: + command: agent -server -retry-join consul1 -disable-host-node-id + depends_on: + - consul1 + expose: + - "8400" + - "8500" + - "8600" + hostname: consul3 + image: consul:latest + schemaload_test_keyspace: + command: + - sh + - -c + - /script/schemaload.sh + depends_on: + vttablet101: + condition: service_healthy + vttablet201: + condition: service_healthy + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - TARGETTAB=test-0000000101 + - SLEEPTIME=15 + - VSCHEMA_FILE=test_keyspace_vschema.json + - SCHEMA_FILES=test_keyspace_schema_file.sql + - POST_LOAD_FILE= + - EXTERNAL_DB=0 + image: vitess/base + volumes: + - .:/script + schemaload_unsharded_keyspace: + command: + - sh + - -c + - /script/schemaload.sh + depends_on: + vttablet301: + condition: service_healthy + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=unsharded_keyspace + - TARGETTAB=test-0000000301 + - SLEEPTIME=15 + - VSCHEMA_FILE=unsharded_keyspace_vschema.json + - SCHEMA_FILES=unsharded_keyspace_schema_file.sql + - POST_LOAD_FILE= + - EXTERNAL_DB=0 + image: vitess/base + volumes: + - .:/script + vtctld: + command: + - sh + - -c + - ' $$VTROOT/bin/vtctld -topo_implementation consul -topo_global_server_address + consul1:8500 -topo_global_root vitess/global -cell test -workflow_manager_init + -workflow_manager_use_election -service_map ''grpc-vtctl'' -backup_storage_implementation + file -file_backup_storage_root $$VTDATAROOT/backups -logtostderr=true -port + 8080 -grpc_port 15999 -pid_file $$VTDATAROOT/tmp/vtctld.pid ' + depends_on: + - consul1 + - consul2 + - consul3 + image: vitess/base + ports: + - 15000:8080 + - "15999" + volumes: + - .:/script + vtgate: + command: + - sh + - -c + - '/script/run-forever.sh $$VTROOT/bin/vtgate -topo_implementation consul -topo_global_server_address + consul1:8500 -topo_global_root vitess/global -logtostderr=true -port 8080 -grpc_port + 15999 -mysql_server_port 15306 -mysql_auth_server_impl none -cell test -cells_to_watch + test -tablet_types_to_wait MASTER,REPLICA,RDONLY -gateway_implementation discoverygateway + -service_map ''grpc-vtgateservice'' -pid_file $$VTDATAROOT/tmp/vtgate.pid -normalize_queries=true ' + depends_on: + - vtctld + image: vitess/base + ports: + - 15099:8080 + - "15999" + - 15306:15306 + volumes: + - .:/script + vttablet101: + command: + - sh + - -c + - /script/vttablet-up.sh 101 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - SHARD=-80 + - ROLE=master + - VTHOST=vttablet101 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15101:8080 + - "15999" + - "3306" + volumes: + - .:/script + vttablet102: + command: + - sh + - -c + - /script/vttablet-up.sh 102 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - SHARD=-80 + - ROLE=replica + - VTHOST=vttablet102 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15102:8080 + - "15999" + - "3306" + volumes: + - .:/script + vttablet201: + command: + - sh + - -c + - /script/vttablet-up.sh 201 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - SHARD=80- + - ROLE=master + - VTHOST=vttablet201 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15201:8080 + - "15999" + - "3306" + volumes: + - .:/script + vttablet202: + command: + - sh + - -c + - /script/vttablet-up.sh 202 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=test_keyspace + - SHARD=80- + - ROLE=replica + - VTHOST=vttablet202 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15202:8080 + - "15999" + - "3306" + volumes: + - .:/script + vttablet301: + command: + - sh + - -c + - /script/vttablet-up.sh 301 + depends_on: + - vtctld + environment: + - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 + -topo_global_root vitess/global + - WEB_PORT=8080 + - GRPC_PORT=15999 + - CELL=test + - KEYSPACE=unsharded_keyspace + - SHARD=- + - ROLE=master + - VTHOST=vttablet301 + - EXTERNAL_DB=0 + - DB_PORT= + - DB_HOST= + - DB_USER= + - DB_PASS= + - DB_CHARSET= + healthcheck: + interval: 30s + retries: 15 + test: + - CMD-SHELL + - curl localhost:8080/debug/health + timeout: 10s + image: vitess/base + ports: + - 15301:8080 + - "15999" + - "3306" + volumes: + - .:/script + vtwork: + command: + - sh + - -c + - '$$VTROOT/bin/vtworker -topo_implementation consul -topo_global_server_address + consul1:8500 -topo_global_root vitess/global -cell test -logtostderr=true -service_map + ''grpc-vtworker'' -port 8080 -grpc_port 15999 -use_v3_resharding_mode=true -pid_file + $$VTDATAROOT/tmp/vtwork.pid ' + depends_on: + - vtctld + image: vitess/base + ports: + - 15100:8080 + - "15999" +version: "2.1" \ No newline at end of file diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index cf0cfd9e2cd..eb9cd7f091e 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -36,20 +36,19 @@ import ( const ( DefaultWebPort = 8080 + webPortUsage = "Web port to be used" DefaultGrpcPort = 15999 + gRpcPortUsage = "gRPC port to be used" DefaultMysqlPort = 15306 + mySqlPortUsage = "mySql port to be used" DefaultKeyspaceData = "test_keyspace:2:1:create_messages.sql,create_tokens.sql;unsharded_keyspace:0:0:create_dinosaurs.sql,create_eggs.sql" + keyspaceDataUsage = "List of keyspace_name/external_db_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name separated by ';'" DefaultCell = "test" + cellUsage = "Vitess Cell name" DefaultExternalDbData = "" + externalDbDataUsage = "List of Data corresponding to external DBs. List of ,,,,, separated by ';'" DefaultTopologyFlags = "-topo_implementation consul -topo_global_server_address consul1:8500 -topo_global_root vitess/global" - - topologyFlagsUsage = "Vitess Topology Flags config" - webPortUsage = "Web port to be used" - gRpcPortUsage = "gRPC port to be used" - mySqlPortUsage = "mySql port to be used" - cellUsage = "Vitess Cell name" - keyspaceDataUsage = "List of keyspace_name/external_db_name:num_of_shards:num_of_replica_tablets:schema_files:lookup_keyspace_name separated by ';'" - externalDbDataUsage = "List of Data corresponding to external DBs. List of ,,,,, separated by ';'" + topologyFlagsUsage = "Vitess Topology Flags config" ) var ( @@ -119,7 +118,14 @@ func newKeyspaceInfo( } func newExternalDbInfo(dbName, dbHost, dbPort, dbUser, dbPass, dbCharset string) externalDbInfo { - return externalDbInfo{dbName: dbName, dbHost: dbHost, dbPort: dbPort, dbUser: dbUser, dbPass: dbPass, dbCharset: dbCharset} + return externalDbInfo{ + dbName: dbName, + dbHost: dbHost, + dbPort: dbPort, + dbUser: dbUser, + dbPass: dbPass, + dbCharset: dbCharset, + } } func parseKeyspaceInfo(keyspaceData string) map[string]keyspaceInfo { @@ -146,7 +152,8 @@ func parseExternalDbData(externalDbData string) map[string]externalDbInfo { for _, v := range strings.Split(externalDbData, ";") { tokens := strings.Split(v, ":") if len(tokens) > 1 { - externalDbInfoMap[tokens[0]] = newExternalDbInfo(tokens[0], tokens[1], tokens[2], tokens[3], tokens[4], tokens[5]) + externalDbInfoMap[tokens[0]] = + newExternalDbInfo(tokens[0], tokens[1], tokens[2], tokens[3], tokens[4], tokens[5]) } } @@ -169,7 +176,7 @@ func main() { for k, v := range keyspaceInfoMap { if _, ok := externalDbInfoMap[k]; !ok { v.schemaFile = createFile(fmt.Sprintf("%s%s_schema_file.sql", tablesPath, v.keyspace)) - appendtoSqlFile(v.schemaFileNames, v.schemaFile) + appendToSqlFile(v.schemaFileNames, v.schemaFile) closeFile(v.schemaFile) } } @@ -191,8 +198,8 @@ func main() { vSchemaFile, primaryTableColumns = addTablesVschemaPatch(vSchemaFile, keyspaceData.schemaFileNames) if keyspaceData.useLookups { - lookupKeyspace := keyspaceInfoMap[keyspaceData.lookupKeyspace] - vSchemaFile = addLookupDataToVschema(vSchemaFile, lookupKeyspace.schemaFileNames, primaryTableColumns, lookupKeyspace.keyspace) + lookup := keyspaceInfoMap[keyspaceData.lookupKeyspace] + vSchemaFile = addLookupDataToVschema(vSchemaFile, lookup.schemaFileNames, primaryTableColumns, lookup.keyspace) } } @@ -280,7 +287,7 @@ func handleError(err error) { } } -func appendtoSqlFile(schemaFileNames []string, f *os.File) { +func appendToSqlFile(schemaFileNames []string, f *os.File) { for _, file := range schemaFileNames { data, err := ioutil.ReadFile(tablesPath + file) _, err = f.Write(data) diff --git a/examples/compose/vtcompose/vtcompose_test.go b/examples/compose/vtcompose/vtcompose_test.go index 9a5de6f2ecf..4b3691c8e5c 100644 --- a/examples/compose/vtcompose/vtcompose_test.go +++ b/examples/compose/vtcompose/vtcompose_test.go @@ -24,6 +24,11 @@ import ( ) var ( + referenceYaml = string(readFile("./docker-compose.test.yml")) + testComposeFile = readFile("./docker-compose.base.yml") + testKeyspaceInfoMap = parseKeyspaceInfo(DefaultKeyspaceData) + testExternalDbInfoMap = parseExternalDbData(DefaultExternalDbData) + testVtOpts = vtOptions{ webPort: DefaultWebPort, gRpcPort: DefaultGrpcPort, @@ -31,10 +36,6 @@ var ( topologyFlags: DefaultTopologyFlags, cell: DefaultCell, } - testComposeFile = readFile("./docker-compose.base.yml") - testKeyspaceInfoMap = parseKeyspaceInfo(DefaultKeyspaceData) - testExternalDbInfoMap = parseExternalDbData(DefaultExternalDbData) - referenceFile string ) func TestGenerateCorrectFileWithDefaultOpts(t *testing.T) { @@ -42,7 +43,7 @@ func TestGenerateCorrectFileWithDefaultOpts(t *testing.T) { finalFile := applyDockerComposePatches(baseFile, testKeyspaceInfoMap, testExternalDbInfoMap, testVtOpts) yamlString := string(finalFile) - assert.YAMLEq(t, referenceFile, yamlString) + assert.YAMLEq(t, referenceYaml, yamlString) } func TestOptsAppliedThroughoutGeneratedFile(t *testing.T) { @@ -72,325 +73,4 @@ func TestOptsAppliedThroughoutGeneratedFile(t *testing.T) { assert.Contains(t, yamlString, fmt.Sprintf("- TOPOLOGY_FLAGS=%s", options.topologyFlags)) assert.NotContains(t, yamlString, DefaultTopologyFlags) -} - -func init() { - referenceFile = ` -services: - consul1: - command: agent -server -bootstrap-expect 3 -ui -disable-host-node-id -client 0.0.0.0 - hostname: consul1 - image: consul:latest - ports: - - 8400:8400 - - 8500:8500 - - 8600:8600 - consul2: - command: agent -server -retry-join consul1 -disable-host-node-id - depends_on: - - consul1 - expose: - - "8400" - - "8500" - - "8600" - hostname: consul2 - image: consul:latest - consul3: - command: agent -server -retry-join consul1 -disable-host-node-id - depends_on: - - consul1 - expose: - - "8400" - - "8500" - - "8600" - hostname: consul3 - image: consul:latest - schemaload_test_keyspace: - command: - - sh - - -c - - /script/schemaload.sh - depends_on: - vttablet101: - condition: service_healthy - vttablet201: - condition: service_healthy - environment: - - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 - -topo_global_root vitess/global - - WEB_PORT=8080 - - GRPC_PORT=15999 - - CELL=test - - KEYSPACE=test_keyspace - - TARGETTAB=test-0000000101 - - SLEEPTIME=15 - - VSCHEMA_FILE=test_keyspace_vschema.json - - SCHEMA_FILES=test_keyspace_schema_file.sql - - POST_LOAD_FILE= - - EXTERNAL_DB=0 - image: vitess/base - volumes: - - .:/script - schemaload_unsharded_keyspace: - command: - - sh - - -c - - /script/schemaload.sh - depends_on: - vttablet301: - condition: service_healthy - environment: - - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 - -topo_global_root vitess/global - - WEB_PORT=8080 - - GRPC_PORT=15999 - - CELL=test - - KEYSPACE=unsharded_keyspace - - TARGETTAB=test-0000000301 - - SLEEPTIME=15 - - VSCHEMA_FILE=unsharded_keyspace_vschema.json - - SCHEMA_FILES=unsharded_keyspace_schema_file.sql - - POST_LOAD_FILE= - - EXTERNAL_DB=0 - image: vitess/base - volumes: - - .:/script - vtctld: - command: - - sh - - -c - - ' $$VTROOT/bin/vtctld -topo_implementation consul -topo_global_server_address - consul1:8500 -topo_global_root vitess/global -cell test -workflow_manager_init - -workflow_manager_use_election -service_map ''grpc-vtctl'' -backup_storage_implementation - file -file_backup_storage_root $$VTDATAROOT/backups -logtostderr=true -port - 8080 -grpc_port 15999 -pid_file $$VTDATAROOT/tmp/vtctld.pid ' - depends_on: - - consul1 - - consul2 - - consul3 - image: vitess/base - ports: - - 15000:8080 - - "15999" - volumes: - - .:/script - vtgate: - command: - - sh - - -c - - '/script/run-forever.sh $$VTROOT/bin/vtgate -topo_implementation consul -topo_global_server_address - consul1:8500 -topo_global_root vitess/global -logtostderr=true -port 8080 -grpc_port - 15999 -mysql_server_port 15306 -mysql_auth_server_impl none -cell test -cells_to_watch - test -tablet_types_to_wait MASTER,REPLICA,RDONLY -gateway_implementation discoverygateway - -service_map ''grpc-vtgateservice'' -pid_file $$VTDATAROOT/tmp/vtgate.pid -normalize_queries=true ' - depends_on: - - vtctld - image: vitess/base - ports: - - 15099:8080 - - "15999" - - 15306:15306 - volumes: - - .:/script - vttablet101: - command: - - sh - - -c - - /script/vttablet-up.sh 101 - depends_on: - - vtctld - environment: - - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 - -topo_global_root vitess/global - - WEB_PORT=8080 - - GRPC_PORT=15999 - - CELL=test - - KEYSPACE=test_keyspace - - SHARD=-80 - - ROLE=master - - VTHOST=vttablet101 - - EXTERNAL_DB=0 - - DB_PORT= - - DB_HOST= - - DB_USER= - - DB_PASS= - - DB_CHARSET= - healthcheck: - interval: 30s - retries: 15 - test: - - CMD-SHELL - - curl localhost:8080/debug/health - timeout: 10s - image: vitess/base - ports: - - 15101:8080 - - "15999" - - "3306" - volumes: - - .:/script - vttablet102: - command: - - sh - - -c - - /script/vttablet-up.sh 102 - depends_on: - - vtctld - environment: - - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 - -topo_global_root vitess/global - - WEB_PORT=8080 - - GRPC_PORT=15999 - - CELL=test - - KEYSPACE=test_keyspace - - SHARD=-80 - - ROLE=replica - - VTHOST=vttablet102 - - EXTERNAL_DB=0 - - DB_PORT= - - DB_HOST= - - DB_USER= - - DB_PASS= - - DB_CHARSET= - healthcheck: - interval: 30s - retries: 15 - test: - - CMD-SHELL - - curl localhost:8080/debug/health - timeout: 10s - image: vitess/base - ports: - - 15102:8080 - - "15999" - - "3306" - volumes: - - .:/script - vttablet201: - command: - - sh - - -c - - /script/vttablet-up.sh 201 - depends_on: - - vtctld - environment: - - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 - -topo_global_root vitess/global - - WEB_PORT=8080 - - GRPC_PORT=15999 - - CELL=test - - KEYSPACE=test_keyspace - - SHARD=80- - - ROLE=master - - VTHOST=vttablet201 - - EXTERNAL_DB=0 - - DB_PORT= - - DB_HOST= - - DB_USER= - - DB_PASS= - - DB_CHARSET= - healthcheck: - interval: 30s - retries: 15 - test: - - CMD-SHELL - - curl localhost:8080/debug/health - timeout: 10s - image: vitess/base - ports: - - 15201:8080 - - "15999" - - "3306" - volumes: - - .:/script - vttablet202: - command: - - sh - - -c - - /script/vttablet-up.sh 202 - depends_on: - - vtctld - environment: - - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 - -topo_global_root vitess/global - - WEB_PORT=8080 - - GRPC_PORT=15999 - - CELL=test - - KEYSPACE=test_keyspace - - SHARD=80- - - ROLE=replica - - VTHOST=vttablet202 - - EXTERNAL_DB=0 - - DB_PORT= - - DB_HOST= - - DB_USER= - - DB_PASS= - - DB_CHARSET= - healthcheck: - interval: 30s - retries: 15 - test: - - CMD-SHELL - - curl localhost:8080/debug/health - timeout: 10s - image: vitess/base - ports: - - 15202:8080 - - "15999" - - "3306" - volumes: - - .:/script - vttablet301: - command: - - sh - - -c - - /script/vttablet-up.sh 301 - depends_on: - - vtctld - environment: - - TOPOLOGY_FLAGS=-topo_implementation consul -topo_global_server_address consul1:8500 - -topo_global_root vitess/global - - WEB_PORT=8080 - - GRPC_PORT=15999 - - CELL=test - - KEYSPACE=unsharded_keyspace - - SHARD=- - - ROLE=master - - VTHOST=vttablet301 - - EXTERNAL_DB=0 - - DB_PORT= - - DB_HOST= - - DB_USER= - - DB_PASS= - - DB_CHARSET= - healthcheck: - interval: 30s - retries: 15 - test: - - CMD-SHELL - - curl localhost:8080/debug/health - timeout: 10s - image: vitess/base - ports: - - 15301:8080 - - "15999" - - "3306" - volumes: - - .:/script - vtwork: - command: - - sh - - -c - - '$$VTROOT/bin/vtworker -topo_implementation consul -topo_global_server_address - consul1:8500 -topo_global_root vitess/global -cell test -logtostderr=true -service_map - ''grpc-vtworker'' -port 8080 -grpc_port 15999 -use_v3_resharding_mode=true -pid_file - $$VTDATAROOT/tmp/vtwork.pid ' - depends_on: - - vtctld - image: vitess/base - ports: - - 15100:8080 - - "15999" -version: "2.1" -` -} +} \ No newline at end of file From f413c6134f8f5c9631d40f249de4435e6e7ab88e Mon Sep 17 00:00:00 2001 From: Keefer Rourke Date: Tue, 31 Mar 2020 11:52:27 -0400 Subject: [PATCH 13/13] Format vtcompose Signed-off-by: Keefer Rourke --- examples/compose/vtcompose/vtcompose_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/compose/vtcompose/vtcompose_test.go b/examples/compose/vtcompose/vtcompose_test.go index 4b3691c8e5c..fefd595bcc5 100644 --- a/examples/compose/vtcompose/vtcompose_test.go +++ b/examples/compose/vtcompose/vtcompose_test.go @@ -73,4 +73,4 @@ func TestOptsAppliedThroughoutGeneratedFile(t *testing.T) { assert.Contains(t, yamlString, fmt.Sprintf("- TOPOLOGY_FLAGS=%s", options.topologyFlags)) assert.NotContains(t, yamlString, DefaultTopologyFlags) -} \ No newline at end of file +}