Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Materialize workflow support for reference tables #16787

Merged
merged 14 commits into from
Sep 24, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Move cli command to only use --reference-tables
Signed-off-by: Rohit Nayak <rohit@planetscale.com>
  • Loading branch information
rohit-nayak-ps committed Sep 24, 2024
commit 4b5d07ce23237ada36fa6e4b2103c29d64740af6
7 changes: 2 additions & 5 deletions go/cmd/vtctldclient/command/vreplication/common/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,8 @@ var (
MySQLServerVersion string
TruncateUILen int
TruncateErrLen int

// IsReference and Tables are used while materializing reference tables.
IsReference bool
Tables []string
ConfigOverrides []string
ReferenceTables []string
ConfigOverrides []string
}{}
)

Expand Down
11 changes: 3 additions & 8 deletions go/cmd/vtctldclient/command/vreplication/materialize/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata"
)

const missingCreateParams = "either --table-settings (for a regular Materialize workflow) or (--reference and --tables, if materializing reference tables) must be specified"
const missingCreateParams = "either --table-settings or --reference-tables must be specified"

var (
createOptions = struct {
Expand Down Expand Up @@ -87,18 +87,14 @@ should be copied as-is from the source keyspace. Here's an example value for tab
if createOptions.TableSettings.val != nil {
hasTableSettings = true
}
if common.CreateOptions.IsReference && len(common.CreateOptions.Tables) > 0 {
if len(common.CreateOptions.ReferenceTables) > 0 {
isReference = true
}
switch {
case !hasTableSettings && !isReference:
return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, missingCreateParams)
case hasTableSettings && isReference:
return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cannot specify both --table-settings and --reference/--tables")
case common.CreateOptions.IsReference && len(common.CreateOptions.Tables) == 0:
return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cannot specify --reference without --tables")
case !common.CreateOptions.IsReference && len(common.CreateOptions.Tables) > 0:
return vterrors.Errorf(vtrpcpb.Code_FAILED_PRECONDITION, "cannot specify --tables without --reference")
}
return nil
},
Expand Down Expand Up @@ -132,8 +128,7 @@ func commandCreate(cmd *cobra.Command, args []string) error {
Cell: strings.Join(common.CreateOptions.Cells, ","),
TabletTypes: topoproto.MakeStringTypeCSV(common.CreateOptions.TabletTypes),
TabletSelectionPreference: tsp,
IsReference: common.CreateOptions.IsReference,
Tables: common.CreateOptions.Tables,
ReferenceTables: common.CreateOptions.ReferenceTables,
WorkflowOptions: workflowOptions,
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@ func registerCommands(root *cobra.Command) {
create.Flags().StringVar(&common.CreateOptions.MySQLServerVersion, "mysql_server_version", fmt.Sprintf("%s-Vitess", config.DefaultMySQLVersion), "Configure the MySQL version to use for example for the parser.")
create.Flags().IntVar(&common.CreateOptions.TruncateUILen, "sql-max-length-ui", 512, "truncate queries in debug UIs to the given length (default 512)")
create.Flags().IntVar(&common.CreateOptions.TruncateErrLen, "sql-max-length-errors", 0, "truncate queries in error logs to the given length (default unlimited)")
create.Flags().BoolVar(&common.CreateOptions.IsReference, "reference", false, "Specify this when materializing reference tables: tables that exist in full on every target shard.")
create.Flags().StringSliceVarP(&common.CreateOptions.Tables, "tables", "t", nil, "Used with --reference to specify the reference tables to materialize on every target shard.")
create.Flags().StringSliceVarP(&common.CreateOptions.ReferenceTables, "reference-tables", "r", nil, "Used to specify the reference tables to materialize on every target shard.")
base.AddCommand(create)

// Generic workflow commands.
Expand Down
29 changes: 10 additions & 19 deletions go/vt/proto/vtctldata/vtctldata.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

60 changes: 12 additions & 48 deletions go/vt/proto/vtctldata/vtctldata_vtproto.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 2 additions & 3 deletions proto/vtctldata.proto
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,8 @@ message MaterializeSettings {
bool atomic_copy = 16;
WorkflowOptions workflow_options = 17;

// IsReference and Tables are set if the materialization is for reference tables.
bool is_reference = 18;
repeated string tables = 19;
// ReferenceTables is set to a csv list of tables, if the materialization is for reference tables.
repeated string reference_tables = 18;
}

/* Data types for VtctldServer */
Expand Down