Skip to content

Commit

Permalink
sql: add cadence-sql-tool and refactor cadence-cassandra-tool (cadenc…
Browse files Browse the repository at this point in the history
  • Loading branch information
venkat1109 authored Apr 29, 2019
1 parent f750712 commit 41e7412
Show file tree
Hide file tree
Showing 40 changed files with 2,848 additions and 1,301 deletions.
38 changes: 25 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,16 @@ copyright: cmd/tools/copyright/licensegen.go
cadence-cassandra-tool: dep-ensured $(TOOLS_SRC)
go build -i -o cadence-cassandra-tool cmd/tools/cassandra/main.go

cadence-sql-tool: dep-ensured $(TOOLS_SRC)
go build -i -o cadence-sql-tool cmd/tools/sql/main.go

cadence: dep-ensured $(TOOLS_SRC)
go build -i -o cadence cmd/tools/cli/main.go

cadence-server: dep-ensured $(ALL_SRC)
go build -i -o cadence-server cmd/server/cadence.go cmd/server/server.go

bins_nothrift: lint copyright cadence-cassandra-tool cadence cadence-server
bins_nothrift: lint copyright cadence-cassandra-tool cadence-sql-tool cadence cadence-server

bins: thriftc bins_nothrift

Expand Down Expand Up @@ -198,37 +201,46 @@ fmt:

clean:
rm -f cadence
rm -f cadence-sql-tool
rm -f cadence-cassandra-tool
rm -f cadence-server
rm -Rf $(BUILD)

install-schema: bins
./cadence-cassandra-tool --ep 127.0.0.1 create -k cadence --rf 1
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence setup-schema -v 0.0
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence update-schema -d ./schema/cassandra/cadence/versioned
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence setup-schema -v 0.0
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence update-schema -d ./schema/cassandra/cadence/versioned
./cadence-cassandra-tool --ep 127.0.0.1 create -k cadence_visibility --rf 1
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_visibility setup-schema -v 0.0
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_visibility update-schema -d ./schema/cassandra/visibility/versioned
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_visibility setup-schema -v 0.0
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_visibility update-schema -d ./schema/cassandra/visibility/versioned

install-schema-mysql: bins
./cadence-sql-tool --ep 127.0.0.1 create --db cadence
./cadence-sql-tool --ep 127.0.0.1 --db cadence setup-schema -v 0.0
./cadence-sql-tool --ep 127.0.0.1 --db cadence update-schema -d ./schema/mysql/v57/cadence/versioned
./cadence-sql-tool --ep 127.0.0.1 create --db cadence_visibility
./cadence-sql-tool --ep 127.0.0.1 --db cadence_visibility setup-schema -v 0.0
./cadence-sql-tool --ep 127.0.0.1 --db cadence_visibility update-schema -d ./schema/mysql/v57/visibility/versioned

start: bins
./cadence-server start

install-schema-cdc: bins
@echo Setting up cadence_active key space
./cadence-cassandra-tool --ep 127.0.0.1 create -k cadence_active --rf 1
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_active setup-schema -v 0.0
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_active update-schema -d ./schema/cassandra/cadence/versioned
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_active setup-schema -v 0.0
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_active update-schema -d ./schema/cassandra/cadence/versioned
./cadence-cassandra-tool --ep 127.0.0.1 create -k cadence_visibility_active --rf 1
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_visibility_active setup-schema -v 0.0
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_visibility_active update-schema -d ./schema/cassandra/visibility/versioned
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_visibility_active setup-schema -v 0.0
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_visibility_active update-schema -d ./schema/cassandra/visibility/versioned

@echo Setting up cadence_standby key space
./cadence-cassandra-tool --ep 127.0.0.1 create -k cadence_standby --rf 1
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_standby setup-schema -v 0.0
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_standby update-schema -d ./schema/cassandra/cadence/versioned
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_standby setup-schema -v 0.0
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_standby update-schema -d ./schema/cassandra/cadence/versioned
./cadence-cassandra-tool --ep 127.0.0.1 create -k cadence_visibility_standby --rf 1
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_visibility_standby setup-schema -v 0.0
./cadence-cassandra-tool -ep 127.0.0.1 -k cadence_visibility_standby update-schema -d ./schema/cassandra/visibility/versioned
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_visibility_standby setup-schema -v 0.0
./cadence-cassandra-tool --ep 127.0.0.1 -k cadence_visibility_standby update-schema -d ./schema/cassandra/visibility/versioned

start-cdc-active: bins
./cadence-server --zone active start
Expand Down
30 changes: 30 additions & 0 deletions cmd/tools/sql/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
"github.com/uber/cadence/tools/sql"
"os"
)

func main() {
sql.RunTool(os.Args)
}
17 changes: 10 additions & 7 deletions common/persistence/cassandra/cassandraHelpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/gocql/gocql"
log "github.com/sirupsen/logrus"
"github.com/uber/cadence/tools/cassandra"
"github.com/uber/cadence/tools/common/schema"
)

const cassandraPersistenceName = "cassandra"
Expand Down Expand Up @@ -111,14 +112,16 @@ func LoadCassandraSchema(
tmpFile.Close()

config := &cassandra.SetupSchemaConfig{
BaseConfig: cassandra.BaseConfig{
CassHosts: strings.Join(hosts, ","),
CassPort: port,
CassKeyspace: keyspace,
CQLClientConfig: cassandra.CQLClientConfig{
Hosts: strings.Join(hosts, ","),
Port: port,
Keyspace: keyspace,
},
SetupConfig: schema.SetupConfig{
SchemaFilePath: tmpFile.Name(),
Overwrite: override,
DisableVersioning: true,
},
SchemaFilePath: tmpFile.Name(),
Overwrite: override,
DisableVersioning: true,
}

err = cassandra.SetupSchema(config)
Expand Down
2 changes: 1 addition & 1 deletion schema/mysql/v57/cadence/database.sql
Original file line number Diff line number Diff line change
@@ -1 +1 @@
CREATE DATABASE cadence;
CREATE DATABASE cadence character set utf8;
248 changes: 248 additions & 0 deletions schema/mysql/v57/cadence/versioned/v0.1/base.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,248 @@
CREATE TABLE domains(
shard_id INT NOT NULL DEFAULT 54321,
id BINARY(16) NOT NULL,
name VARCHAR(255) UNIQUE NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
is_global TINYINT(1) NOT NULL,
PRIMARY KEY(shard_id, id)
);

CREATE TABLE domain_metadata (
notification_version BIGINT NOT NULL
);

INSERT INTO domain_metadata (notification_version) VALUES (1);

CREATE TABLE shards (
shard_id INT NOT NULL,
--
range_id BIGINT NOT NULL,
data BLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (shard_id)
);

CREATE TABLE transfer_tasks(
shard_id INT NOT NULL,
task_id BIGINT NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (shard_id, task_id)
);

CREATE TABLE executions(
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
--
next_event_id BIGINT NOT NULL,
last_write_version BIGINT NOT NULL,
data BLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (shard_id, domain_id, workflow_id, run_id)
);

CREATE TABLE current_executions(
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
--
run_id BINARY(16) NOT NULL,
create_request_id VARCHAR(64) NOT NULL,
state INT NOT NULL,
close_status INT NOT NULL,
start_version BIGINT NOT NULL,
last_write_version BIGINT NOT NULL,
PRIMARY KEY (shard_id, domain_id, workflow_id)
);

CREATE TABLE buffered_events (
id BIGINT AUTO_INCREMENT NOT NULL,
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
--
data MEDIUMBLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (id)
);

CREATE INDEX buffered_events_by_events_ids ON buffered_events(shard_id, domain_id, workflow_id, run_id);

CREATE TABLE tasks (
domain_id BINARY(16) NOT NULL,
task_list_name VARCHAR(255) NOT NULL,
task_type TINYINT NOT NULL, -- {Activity, Decision}
task_id BIGINT NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (domain_id, task_list_name, task_type, task_id)
);

CREATE TABLE task_lists (
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
name VARCHAR(255) NOT NULL,
task_type TINYINT NOT NULL, -- {Activity, Decision}
--
range_id BIGINT NOT NULL,
data BLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (shard_id, domain_id, name, task_type)
);

CREATE TABLE replication_tasks (
shard_id INT NOT NULL,
task_id BIGINT NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (shard_id, task_id)
);

CREATE TABLE timer_tasks (
shard_id INT NOT NULL,
visibility_timestamp DATETIME(6) NOT NULL,
task_id BIGINT NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (shard_id, visibility_timestamp, task_id)
);

-- Deprecated in favor of history eventsV2
CREATE TABLE events (
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
first_event_id BIGINT NOT NULL,
--
batch_version BIGINT,
range_id BIGINT NOT NULL,
tx_id BIGINT NOT NULL,
data MEDIUMBLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (domain_id, workflow_id, run_id, first_event_id)
);

CREATE TABLE activity_info_maps (
-- each row corresponds to one key of one map<string, ActivityInfo>
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
schedule_id BIGINT NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16),
last_heartbeat_details BLOB,
last_heartbeat_updated_time DATETIME(6) NOT NULL,
PRIMARY KEY (shard_id, domain_id, workflow_id, run_id, schedule_id)
);

CREATE TABLE timer_info_maps (
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
timer_id VARCHAR(255) NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16),
PRIMARY KEY (shard_id, domain_id, workflow_id, run_id, timer_id)
);

CREATE TABLE child_execution_info_maps (
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
initiated_id BIGINT NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16),
PRIMARY KEY (shard_id, domain_id, workflow_id, run_id, initiated_id)
);

CREATE TABLE request_cancel_info_maps (
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
initiated_id BIGINT NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16),
PRIMARY KEY (shard_id, domain_id, workflow_id, run_id, initiated_id)
);

CREATE TABLE signal_info_maps (
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
initiated_id BIGINT NOT NULL,
--
data BLOB NOT NULL,
data_encoding VARCHAR(16),
PRIMARY KEY (shard_id, domain_id, workflow_id, run_id, initiated_id)
);

CREATE TABLE buffered_replication_task_maps (
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
first_event_id BIGINT NOT NULL,
--
version BIGINT NOT NULL,
next_event_id BIGINT NOT NULL,
history MEDIUMBLOB,
history_encoding VARCHAR(16) NOT NULL,
new_run_history BLOB,
new_run_history_encoding VARCHAR(16) NOT NULL DEFAULT 'json',
event_store_version INT NOT NULL, -- indiciates which version of event store to query
new_run_event_store_version INT NOT NULL, -- indiciates which version of event store to query for new run(continueAsNew)
PRIMARY KEY (shard_id, domain_id, workflow_id, run_id, first_event_id)
);

CREATE TABLE signals_requested_sets (
shard_id INT NOT NULL,
domain_id BINARY(16) NOT NULL,
workflow_id VARCHAR(255) NOT NULL,
run_id BINARY(16) NOT NULL,
signal_id VARCHAR(64) NOT NULL,
--
PRIMARY KEY (shard_id, domain_id, workflow_id, run_id, signal_id)
);

-- history eventsV2: history_node stores history event data
CREATE TABLE history_node (
shard_id INT NOT NULL,
tree_id BINARY(16) NOT NULL,
branch_id BINARY(16) NOT NULL,
node_id BIGINT NOT NULL,
txn_id BIGINT NOT NULL,
--
data MEDIUMBLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (shard_id, tree_id, branch_id, node_id, txn_id)
);

-- history eventsV2: history_tree stores branch metadata
CREATE TABLE history_tree (
shard_id INT NOT NULL,
tree_id BINARY(16) NOT NULL,
branch_id BINARY(16) NOT NULL,
--
in_progress BOOLEAN NOT NULL,
data BLOB NOT NULL,
data_encoding VARCHAR(16) NOT NULL,
PRIMARY KEY (shard_id, tree_id, branch_id)
);
Loading

0 comments on commit 41e7412

Please sign in to comment.