Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sink: support maxwell data format #869

Merged
merged 23 commits into from
Sep 3, 2020
Merged
Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
70be8dd
add maxwell protocl
shldreams Aug 17, 2020
db0b6f6
add maxwell protocol
shldreams Aug 21, 2020
ba3b752
fix
shldreams Aug 21, 2020
088a481
Merge branch 'master' into feture_maxwell
shldreams Aug 21, 2020
6210a23
fix dml insert type
shldreams Aug 21, 2020
ce43308
Merge branch 'feture_maxwell' of https://github.com/shldreams/ticdc i…
shldreams Aug 21, 2020
e2b867f
Merge branch 'master' into feture_maxwell
shldreams Aug 26, 2020
4145c3f
Merge branch 'feture_maxwell' of https://github.com/shldreams/ticdc i…
shldreams Aug 26, 2020
8663682
fix column type
shldreams Aug 26, 2020
d503a8e
Merge branch 'master' into feture_maxwell
shldreams Aug 26, 2020
9ca33d1
add ddl type to maxwell type
shldreams Aug 26, 2020
efadd63
fix ci
Aug 27, 2020
c79dad6
Merge branch 'master' into feture_maxwell
Aug 27, 2020
96349fe
fix test case
shldreams Aug 28, 2020
a11a7ef
Merge branch 'master' into feture_maxwell
shldreams Aug 28, 2020
c1cb4cb
Merge branch 'feture_maxwell' of https://github.com/shldreams/ticdc i…
shldreams Aug 28, 2020
9d8a4f6
Merge branch 'feture_maxwell' of https://github.com/shldreams/ticdc i…
shldreams Aug 28, 2020
d74c415
fix test case
shldreams Aug 31, 2020
b79b50f
Merge branch 'master' into feture_maxwell
Sep 2, 2020
124d3cf
fix ddl typecode,column typecode
shldreams Sep 2, 2020
2441915
Merge branch 'feture_maxwell' of https://github.com/shldreams/ticdc i…
shldreams Sep 2, 2020
7c1043c
fix
shldreams Sep 2, 2020
b900bc9
Merge branch 'master' into feture_maxwell
ti-srebot Sep 3, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix ddl typecode,column typecode
  • Loading branch information
shldreams committed Sep 2, 2020
commit 124d3cf072d8b52be230d7bc186ad448a23896f6
46 changes: 20 additions & 26 deletions cdc/sink/codec/maxwell.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (

"github.com/pingcap/errors"
model2 "github.com/pingcap/parser/model"
"github.com/pingcap/parser/mysql"
"github.com/pingcap/ticdc/cdc/model"
)

Expand Down Expand Up @@ -84,7 +85,6 @@ func rowEventToMaxwellMessage(e *model.RowChangedEvent) (*mqMessageKey, *maxwell
Ts: e.CommitTs,
Database: e.Table.Schema,
Table: e.Table.Table,
Type: "update",
Data: make(map[string]interface{}),
Old: make(map[string]interface{}),
}
Expand Down Expand Up @@ -408,21 +408,21 @@ func NewMaxwellEventBatchDecoder(key []byte, value []byte) (EventBatchDecoder, e

//ddl typecode from parser/model/ddl.go
func ddlToMaxwellType(ddlType model2.ActionType) string {
if ddlType >= 5 && ddlType <= 20 {
if ddlType >= model2.ActionAddColumn && ddlType <= model2.ActionDropTablePartition {
return "table-alter"
}
switch ddlType {
case 3:
case model2.ActionCreateTable:
return "table-create"
case 4:
case model2.ActionDropTable:
return "table-drop"
case 22, 23, 27, 28, 29, 33, 37, 38, 41, 42:
return "table-alter"
case 1:
case model2.ActionCreateSchema:
return "database-create"
case 2:
case model2.ActionDropSchema:
return "database-drop"
case 26:
case model2.ActionModifySchemaCharsetAndCollate:
return "database-alter"
default:
return ddlType.String()
Expand All @@ -432,37 +432,31 @@ func ddlToMaxwellType(ddlType model2.ActionType) string {
//Convert column type code to maxwell column type
func columnToMaxwellType(columnType byte) (string, error) {
switch columnType {
// tinyint,smallint,mediumint,int
case 1, 2, 3, 9:
case mysql.TypeTiny, mysql.TypeShort, mysql.TypeLong, mysql.TypeInt24:
return "int", nil
// bigint
case 8:
case mysql.TypeLonglong:
return "bigint", nil
// tinytext,text,mediumtext,longtext,varchar,char
case 249, 252, 250, 251, 254, 15:
case mysql.TypeTinyBlob, mysql.TypeBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeString, mysql.TypeVarchar:
return "string", nil
// date
case 10:
case mysql.TypeDate:
return "date", nil
// datetime,timestamp
case 7, 12:
case mysql.TypeTimestamp, mysql.TypeDatetime:
return "datetime", nil
case 11:
case mysql.TypeDuration:
return "time", nil
case 13:
case mysql.TypeYear:
return "year", nil
case 247:
case mysql.TypeEnum:
return "enum", nil
case 248:
case mysql.TypeSet:
return "set", nil
case 16:
case mysql.TypeBit:
return "bit", nil
case 245:
case mysql.TypeJSON:
return "json", nil
// float,double
case 4, 5:
case mysql.TypeFloat, mysql.TypeDouble:
return "float", nil
case 246:
case mysql.TypeNewDecimal:
return "decimal", nil
default:
return "", errors.Errorf("unsupported column type - %v", columnType)
Expand Down