From 93eedddb64669e218f31cff48059066b03fabf83 Mon Sep 17 00:00:00 2001 From: Martin Sucha <2007393+martin-sucha@users.noreply.github.com> Date: Fri, 4 Dec 2020 15:29:55 +0100 Subject: [PATCH 01/17] Add Go version to issue template (#1510) Some issues might be reproducible only using specific Go version. If someone is not using latest Go version, we might want to ask them to upgrade and try to reproduce. This change is inspired by the following comment: https://github.com/gocql/gocql/issues/1481#issuecomment-728758618 --- .github/issue_template.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/issue_template.md b/.github/issue_template.md index 523427ace..7391cd8b7 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -6,6 +6,9 @@ Please answer these questions before submitting your issue. Thanks! ### What version of Gocql are you using? +### What version of Go are you using? + + ### What did you do? From 3402507adfc48391a2eeb2dd02b1c77f3592928e Mon Sep 17 00:00:00 2001 From: Alexey Romanovsky Date: Tue, 8 Dec 2020 15:36:18 +0500 Subject: [PATCH 02/17] Add port into error message --- control.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control.go b/control.go index aa5cf3570..48ab457d2 100644 --- a/control.go +++ b/control.go @@ -177,7 +177,7 @@ func (c *controlConn) shuffleDial(endpoints []*HostInfo) (*Conn, error) { return conn, nil } - Logger.Printf("gocql: unable to dial control conn %v: %v\n", host.ConnectAddress(), err) + Logger.Printf("gocql: unable to dial control conn %v:%v: %v\n", host.ConnectAddress(), host.Port(), err) } return nil, err From d4d1da99cbed935a2a2d7bbd951aa7e2af0ed5cf Mon Sep 17 00:00:00 2001 From: Alexey Romanovsky Date: Tue, 8 Dec 2020 23:06:52 +0500 Subject: [PATCH 03/17] Update AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index c8d69f766..5cf500661 100644 --- a/AUTHORS +++ b/AUTHORS @@ -116,3 +116,4 @@ Rintaro Okamura Yura Sokolov ; Jorge Bay Dmitriy Kozlov +Alexey Romanovsky From 179bae5f16901945ab66d7c2d9088dc4282281c4 Mon Sep 17 00:00:00 2001 From: Dima Date: Tue, 15 Dec 2020 17:39:51 +0300 Subject: [PATCH 04/17] Do not allocate extra bytes in unmarshalUUID (#1505) If unmarshalUUID called with value of type gocql.UUID, we already have allocated bytes for UUID, so do not allocate extra UUID, but just copy data to value. Also add ability to unmarshal directly to array of 16 bytes. --- marshal.go | 20 +++++++++++++++----- marshal_test.go | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/marshal.go b/marshal.go index e95c1c8f9..9aa8274e4 100644 --- a/marshal.go +++ b/marshal.go @@ -1711,7 +1711,7 @@ func marshalUUID(info TypeInfo, value interface{}) ([]byte, error) { } func unmarshalUUID(info TypeInfo, data []byte, value interface{}) error { - if data == nil || len(data) == 0 { + if len(data) == 0 { switch v := value.(type) { case *string: *v = "" @@ -1726,9 +1726,22 @@ func unmarshalUUID(info TypeInfo, data []byte, value interface{}) error { return nil } + if len(data) != 16 { + return unmarshalErrorf("unable to parse UUID: UUIDs must be exactly 16 bytes long") + } + + switch v := value.(type) { + case *[16]byte: + copy((*v)[:], data) + return nil + case *UUID: + copy((*v)[:], data) + return nil + } + u, err := UUIDFromBytes(data) if err != nil { - return unmarshalErrorf("Unable to parse UUID: %s", err) + return unmarshalErrorf("unable to parse UUID: %s", err) } switch v := value.(type) { @@ -1738,9 +1751,6 @@ func unmarshalUUID(info TypeInfo, data []byte, value interface{}) error { case *[]byte: *v = u[:] return nil - case *UUID: - *v = u - return nil } return unmarshalErrorf("can not unmarshal X %s into %T", info, value) } diff --git a/marshal_test.go b/marshal_test.go index 84100185f..2bb47df7b 100644 --- a/marshal_test.go +++ b/marshal_test.go @@ -2172,3 +2172,17 @@ func TestReadCollectionSize(t *testing.T) { }) } } + +func BenchmarkUnmarshalUUID(b *testing.B) { + b.ReportAllocs() + src := make([]byte, 16) + dst := UUID{} + var ti TypeInfo = NativeType{} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := unmarshalUUID(ti, src, &dst); err != nil { + b.Fatal(err) + } + } +} From e49edf966d9036342234c0f42c9643fbffdefaf5 Mon Sep 17 00:00:00 2001 From: Martin Sucha Date: Tue, 15 Dec 2020 17:53:27 +0100 Subject: [PATCH 05/17] Fix TestMarshal_Decode In 179bae5f16901945ab66d7c2d9088dc4282281c4 we changed the error message from upper case to lower case, but the test was not updated. I don't know whether I missed failing CI build when merging or whether CI did not run at all. Anyway, this commit fixes the test. --- marshal_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marshal_test.go b/marshal_test.go index 2bb47df7b..0fb3f3635 100644 --- a/marshal_test.go +++ b/marshal_test.go @@ -93,7 +93,7 @@ var marshalTests = []struct { []byte{0xb8, 0xe8, 0x56, 0x2c, 0xc, 0xd0}, []byte{0xb8, 0xe8, 0x56, 0x2c, 0xc, 0xd0}, MarshalError("can not marshal []byte 6 bytes long into timeuuid, must be exactly 16 bytes long"), - UnmarshalError("Unable to parse UUID: UUIDs must be exactly 16 bytes long"), + UnmarshalError("unable to parse UUID: UUIDs must be exactly 16 bytes long"), }, { NativeType{proto: 2, typ: TypeInt}, From 43a8f9bf6605c25fcaa921fb7c9f8b3d6024f05d Mon Sep 17 00:00:00 2001 From: Martin Sucha Date: Sun, 24 Jan 2021 22:11:04 +0100 Subject: [PATCH 06/17] Fix TLS host verification when ServerName is not provided in TLS config Enabling host verification in SslOptions fails if ServerName is not set in the configured *tls.Config with: gocql: unable to create session: control: unable to connect to initial hosts: tls: either ServerName or InsecureSkipVerify must be specified in the tls.Config Before https://github.com/gocql/gocql/pull/1368 we used tls.DialWithDialer that sets the ServerName in the tls.Config to the host part of the connect address if ServerName was not explicitly provided. After that MR, we wrap the connection returned by the dialer ourselves and the ServerName is not set. This commit restores the original behavior to set the ServerName to host part of connect address if it is not provided explicitly in the TLS config. Thanks Suhail Patel for reporting the issue: https://github.com/gocql/gocql/pull/1493 --- conn.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/conn.go b/conn.go index de9e8231b..037e6f928 100644 --- a/conn.go +++ b/conn.go @@ -213,14 +213,26 @@ func (s *Session) dialWithoutObserver(ctx context.Context, host *HostInfo, cfg * dialer = d } - conn, err := dialer.DialContext(ctx, "tcp", host.HostnameAndPort()) + addr := host.HostnameAndPort() + conn, err := dialer.DialContext(ctx, "tcp", addr) if err != nil { return nil, err } if cfg.tlsConfig != nil { // the TLS config is safe to be reused by connections but it must not // be modified after being used. - tconn := tls.Client(conn, cfg.tlsConfig) + tlsConfig := cfg.tlsConfig + if !tlsConfig.InsecureSkipVerify && tlsConfig.ServerName == "" { + colonPos := strings.LastIndex(addr, ":") + if colonPos == -1 { + colonPos = len(addr) + } + hostname := addr[:colonPos] + // clone config to avoid modifying the shared one. + tlsConfig = tlsConfig.Clone() + tlsConfig.ServerName = hostname + } + tconn := tls.Client(conn, tlsConfig) if err := tconn.Handshake(); err != nil { conn.Close() return nil, err From a6c12035b8c1d556413f93f1db5a6c61a91cdd2f Mon Sep 17 00:00:00 2001 From: Jaume Marhuenda Date: Tue, 26 Jan 2021 03:09:02 -0500 Subject: [PATCH 07/17] fix compileMetadata bug for materialized views (#1525) Fixes #1524 for materialised views --- cassandra_test.go | 46 +++++++++++++++++++++++++++++++++++++--------- common_test.go | 14 ++++++++++++++ metadata.go | 6 +++--- 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/cassandra_test.go b/cassandra_test.go index d06274bcd..ba2a88081 100644 --- a/cassandra_test.go +++ b/cassandra_test.go @@ -2248,10 +2248,10 @@ func TestMaterializedViewMetadata(t *testing.T) { if materializedViews == nil { t.Fatal("failed to query view metadata, nil returned") } - if len(materializedViews) != 1 { - t.Fatal("expected one view") + if len(materializedViews) != 2 { + t.Fatal("expected two views") } - expectedView := MaterializedViewMetadata{ + expectedView1 := MaterializedViewMetadata{ Keyspace: "gocql_test", Name: "view_view", baseTableName: "view_table", @@ -2268,12 +2268,33 @@ func TestMaterializedViewMetadata(t *testing.T) { IncludeAllColumns: false, MaxIndexInterval: 2048, MemtableFlushPeriodInMs: 0, MinIndexInterval: 128, ReadRepairChance: 0, SpeculativeRetry: "99PERCENTILE", } + expectedView2 := MaterializedViewMetadata{ + Keyspace: "gocql_test", + Name: "view_view2", + baseTableName: "view_table2", + BloomFilterFpChance: 0.01, + Caching: map[string]string{"keys": "ALL", "rows_per_partition": "NONE"}, + Comment: "", + Compaction: map[string]string{"class": "org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy", "max_threshold": "32", "min_threshold": "4"}, + Compression: map[string]string{"chunk_length_in_kb": "64", "class": "org.apache.cassandra.io.compress.LZ4Compressor"}, + CrcCheckChance: 1, + DcLocalReadRepairChance: 0.1, + DefaultTimeToLive: 0, + Extensions: map[string]string{}, + GcGraceSeconds: 864000, + IncludeAllColumns: false, MaxIndexInterval: 2048, MemtableFlushPeriodInMs: 0, MinIndexInterval: 128, ReadRepairChance: 0, + SpeculativeRetry: "99PERCENTILE", + } - expectedView.BaseTableId = materializedViews[0].BaseTableId - expectedView.Id = materializedViews[0].Id - - if !reflect.DeepEqual(materializedViews[0], expectedView) { - t.Fatalf("materialized view is %+v, but expected %+v", materializedViews[0], expectedView) + expectedView1.BaseTableId = materializedViews[0].BaseTableId + expectedView1.Id = materializedViews[0].Id + if !reflect.DeepEqual(materializedViews[0], expectedView1) { + t.Fatalf("materialized view is %+v, but expected %+v", materializedViews[0], expectedView1) + } + expectedView2.BaseTableId = materializedViews[1].BaseTableId + expectedView2.Id = materializedViews[1].Id + if !reflect.DeepEqual(materializedViews[1], expectedView2) { + t.Fatalf("materialized view is %+v, but expected %+v", materializedViews[1], expectedView2) } } @@ -2503,11 +2524,18 @@ func TestKeyspaceMetadata(t *testing.T) { if flagCassVersion.Major >= 3 { materializedView, found := keyspaceMetadata.MaterializedViews["view_view"] if !found { - t.Fatal("failed to find the materialized view in metadata") + t.Fatal("failed to find materialized view view_view in metadata") } if materializedView.BaseTable.Name != "view_table" { t.Fatalf("expected name: %s, materialized view base table name: %s", "view_table", materializedView.BaseTable.Name) } + materializedView, found = keyspaceMetadata.MaterializedViews["view_view2"] + if !found { + t.Fatal("failed to find materialized view view_view2 in metadata") + } + if materializedView.BaseTable.Name != "view_table2" { + t.Fatalf("expected name: %s, materialized view base table name: %s", "view_table2", materializedView.BaseTable.Name) + } } } diff --git a/common_test.go b/common_test.go index c7b912fe0..e751566ba 100644 --- a/common_test.go +++ b/common_test.go @@ -195,6 +195,13 @@ func createMaterializedViews(t *testing.T, session *Session) { PRIMARY KEY (userid));`).Exec(); err != nil { t.Fatalf("failed to create materialized view with err: %v", err) } + if err := session.Query(`CREATE TABLE IF NOT EXISTS gocql_test.view_table2 ( + userid text, + year int, + month int, + PRIMARY KEY (userid));`).Exec(); err != nil { + t.Fatalf("failed to create materialized view with err: %v", err) + } if err := session.Query(`CREATE MATERIALIZED VIEW IF NOT EXISTS gocql_test.view_view AS SELECT year, month, userid FROM gocql_test.view_table @@ -202,6 +209,13 @@ func createMaterializedViews(t *testing.T, session *Session) { PRIMARY KEY (userid, year);`).Exec(); err != nil { t.Fatalf("failed to create materialized view with err: %v", err) } + if err := session.Query(`CREATE MATERIALIZED VIEW IF NOT EXISTS gocql_test.view_view2 AS + SELECT year, month, userid + FROM gocql_test.view_table2 + WHERE year IS NOT NULL AND month IS NOT NULL AND userid IS NOT NULL + PRIMARY KEY (userid, year);`).Exec(); err != nil { + t.Fatalf("failed to create materialized view with err: %v", err) + } } func createFunctions(t *testing.T, session *Session) { diff --git a/metadata.go b/metadata.go index e586dd48f..226ce4726 100644 --- a/metadata.go +++ b/metadata.go @@ -347,9 +347,9 @@ func compileMetadata( keyspace.UserTypes[types[i].Name] = &types[i] } keyspace.MaterializedViews = make(map[string]*MaterializedViewMetadata, len(materializedViews)) - for _, materializedView := range materializedViews { - materializedView.BaseTable = keyspace.Tables[materializedView.baseTableName] - keyspace.MaterializedViews[materializedView.Name] = &materializedView + for i, _ := range materializedViews { + materializedViews[i].BaseTable = keyspace.Tables[materializedViews[i].baseTableName] + keyspace.MaterializedViews[materializedViews[i].Name] = &materializedViews[i] } // add columns from the schema data From 994808f8e61e812510a6432bb11880416c7310a5 Mon Sep 17 00:00:00 2001 From: Jaume Marhuenda Date: Tue, 26 Jan 2021 03:10:24 -0500 Subject: [PATCH 08/17] Add myself to AUTHORS (#1527) --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 5cf500661..009dc759d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -117,3 +117,4 @@ Yura Sokolov ; Jorge Bay Dmitriy Kozlov Alexey Romanovsky +Jaume Marhuenda Beltran From f242734b1558a112c9268e4f84d4d292ac87e2fa Mon Sep 17 00:00:00 2001 From: Jaume Marhuenda Date: Tue, 26 Jan 2021 13:56:05 -0500 Subject: [PATCH 09/17] fix compileMetadata bug for aggregate views (#1528) Fixes #1526 --- cassandra_test.go | 25 +++++++++++++++++++------ common_test.go | 9 +++++++++ metadata.go | 8 ++++---- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/cassandra_test.go b/cassandra_test.go index ba2a88081..1d645266f 100644 --- a/cassandra_test.go +++ b/cassandra_test.go @@ -2310,10 +2310,9 @@ func TestAggregateMetadata(t *testing.T) { if aggregates == nil { t.Fatal("failed to query aggregate metadata, nil returned") } - if len(aggregates) != 1 { - t.Fatal("expected only a single aggregate") + if len(aggregates) != 2 { + t.Fatal("expected two aggregates") } - aggregate := aggregates[0] expectedAggregrate := AggregateMetadata{ Keyspace: "gocql_test", @@ -2338,8 +2337,12 @@ func TestAggregateMetadata(t *testing.T) { expectedAggregrate.InitCond = string([]byte{0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0}) } - if !reflect.DeepEqual(aggregate, expectedAggregrate) { - t.Fatalf("aggregate is %+v, but expected %+v", aggregate, expectedAggregrate) + if !reflect.DeepEqual(aggregates[0], expectedAggregrate) { + t.Fatalf("aggregate 'average' is %+v, but expected %+v", aggregates[0], expectedAggregrate) + } + expectedAggregrate.Name = "average2" + if !reflect.DeepEqual(aggregates[1], expectedAggregrate) { + t.Fatalf("aggregate 'average2' is %+v, but expected %+v", aggregates[1], expectedAggregrate) } } @@ -2486,7 +2489,17 @@ func TestKeyspaceMetadata(t *testing.T) { aggregate, found := keyspaceMetadata.Aggregates["average"] if !found { - t.Fatal("failed to find the aggreate in metadata") + t.Fatal("failed to find the aggregate 'average' in metadata") + } + if aggregate.FinalFunc.Name != "avgfinal" { + t.Fatalf("expected final function %s, but got %s", "avgFinal", aggregate.FinalFunc.Name) + } + if aggregate.StateFunc.Name != "avgstate" { + t.Fatalf("expected state function %s, but got %s", "avgstate", aggregate.StateFunc.Name) + } + aggregate, found = keyspaceMetadata.Aggregates["average2"] + if !found { + t.Fatal("failed to find the aggregate 'average2' in metadata") } if aggregate.FinalFunc.Name != "avgfinal" { t.Fatalf("expected final function %s, but got %s", "avgFinal", aggregate.FinalFunc.Name) diff --git a/common_test.go b/common_test.go index e751566ba..28bab4cf1 100644 --- a/common_test.go +++ b/common_test.go @@ -249,6 +249,15 @@ func createAggregate(t *testing.T, session *Session) { `).Exec(); err != nil { t.Fatalf("failed to create aggregate with err: %v", err) } + if err := session.Query(` + CREATE OR REPLACE AGGREGATE gocql_test.average2(int) + SFUNC avgState + STYPE tuple + FINALFUNC avgFinal + INITCOND (0,0); + `).Exec(); err != nil { + t.Fatalf("failed to create aggregate with err: %v", err) + } } func staticAddressTranslator(newAddr net.IP, newPort int) AddressTranslator { diff --git a/metadata.go b/metadata.go index 226ce4726..44595ffee 100644 --- a/metadata.go +++ b/metadata.go @@ -324,10 +324,10 @@ func compileMetadata( keyspace.Functions[functions[i].Name] = &functions[i] } keyspace.Aggregates = make(map[string]*AggregateMetadata, len(aggregates)) - for _, aggregate := range aggregates { - aggregate.FinalFunc = *keyspace.Functions[aggregate.finalFunc] - aggregate.StateFunc = *keyspace.Functions[aggregate.stateFunc] - keyspace.Aggregates[aggregate.Name] = &aggregate + for i, _ := range aggregates { + aggregates[i].FinalFunc = *keyspace.Functions[aggregates[i].finalFunc] + aggregates[i].StateFunc = *keyspace.Functions[aggregates[i].stateFunc] + keyspace.Aggregates[aggregates[i].Name] = &aggregates[i] } keyspace.Views = make(map[string]*ViewMetadata, len(views)) for i := range views { From b234f8761be4acb56a86126fa43f8ee818d9156d Mon Sep 17 00:00:00 2001 From: Martin Sucha Date: Sat, 24 Oct 2020 19:46:06 +0200 Subject: [PATCH 10/17] Document type conversions for Marshal and Unmarshal It was not documented anywhere how cql data types map to Go data types. I haven't included data for exact cases when type aliases are supported. --- marshal.go | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/marshal.go b/marshal.go index 9aa8274e4..59453dc74 100644 --- a/marshal.go +++ b/marshal.go @@ -44,6 +44,51 @@ type Unmarshaler interface { // Marshal returns the CQL encoding of the value for the Cassandra // internal type described by the info parameter. +// +// nil is serialized as CQL null. +// If value implements Marshaler, its MarshalCQL method is called to marshal the data. +// If value is a pointer, the pointed-to value is marshaled. +// +// Supported conversions are as follows, other type combinations may be added in the future: +// +// CQL type | Go type (value) | Note +// varchar, ascii, blob, text | string, []byte | +// boolean | bool | +// tinyint, smallint, int | integer types | +// tinyint, smallint, int | string | formatted as base 10 number +// bigint, counter | integer types | +// bigint, counter | big.Int | +// bigint, counter | string | formatted as base 10 number +// float | float32 | +// double | float64 | +// decimal | inf.Dec | +// time | int64 | nanoseconds since start of day +// time | time.Duration | duration since start of day +// timestamp | int64 | milliseconds since Unix epoch +// timestamp | time.Time | +// list, set | slice, array | +// list, set | map[X]struct{} | +// map | map[X]Y | +// uuid, timeuuid | gocql.UUID | +// uuid, timeuuid | []byte | raw UUID bytes, length must be 16 bytes +// uuid, timeuuid | string | hex representation, see ParseUUID +// varint | integer types | +// varint | big.Int | +// varint | string | value of number in decimal notation +// inet | net.IP | +// inet | string | IPv4 or IPv6 address string +// tuple | slice, array | +// tuple | struct | fields are marshaled in order of declaration +// user-defined type | gocql.UDTMarshaler | MarshalUDT is called +// user-defined type | map[string]interface{} | +// user-defined type | struct | struct fields' cql tags are used for column names +// date | int64 | milliseconds since Unix epoch to start of day (in UTC) +// date | time.Time | start of day (in UTC) +// date | string | parsed using "2006-01-02" format +// duration | int64 | duration in nanoseconds +// duration | time.Duration | +// duration | gocql.Duration | +// duration | string | parsed with time.ParseDuration func Marshal(info TypeInfo, value interface{}) ([]byte, error) { if info.Version() < protoVersion1 { panic("protocol version not set") @@ -118,6 +163,44 @@ func Marshal(info TypeInfo, value interface{}) ([]byte, error) { // Unmarshal parses the CQL encoded data based on the info parameter that // describes the Cassandra internal data type and stores the result in the // value pointed by value. +// +// If value implements Unmarshaler, it's UnmarshalCQL method is called to +// unmarshal the data. +// If value is a pointer to pointer, it is set to nil if the CQL value is +// null. Otherwise, nulls are unmarshalled as zero value. +// +// Supported conversions are as follows, other type combinations may be added in the future: +// +// CQL type | Go type (value) | Note +// varchar, ascii, blob, text | *string | +// varchar, ascii, blob, text | *[]byte | non-nil buffer is reused +// bool | *bool | +// tinyint, smallint, int, bigint, counter | *integer types | +// tinyint, smallint, int, bigint, counter | *big.Int | +// tinyint, smallint, int, bigint, counter | *string | formatted as base 10 number +// float | *float32 | +// double | *float64 | +// decimal | *inf.Dec | +// time | *int64 | nanoseconds since start of day +// time | *time.Duration | +// timestamp | *int64 | milliseconds since Unix epoch +// timestamp | *time.Time | +// list, set | *slice, *array | +// map | *map[X]Y | +// uuid, timeuuid | *string | see UUID.String +// uuid, timeuuid | *[]byte | raw UUID bytes +// uuid, timeuuid | *gocql.UUID | +// timeuuid | *time.Time | timestamp of the UUID +// inet | *net.IP | +// inet | *string | IPv4 or IPv6 address string +// tuple | *slice, *array | +// tuple | *struct | struct fields are set in order of declaration +// user-defined types | gocql.UDTUnmarshaler | UnmarshalUDT is called +// user-defined types | *map[string]interface{} | +// user-defined types | *struct | cql tag is used to determine field name +// date | *time.Time | time of beginning of the day (in UTC) +// date | *string | formatted with 2006-01-02 format +// duration | *gocql.Duration | func Unmarshal(info TypeInfo, data []byte, value interface{}) error { if v, ok := value.(Unmarshaler); ok { return v.UnmarshalCQL(info, data) From 9e3bb6cbd8817fc0752ee6f4461a412f79496344 Mon Sep 17 00:00:00 2001 From: Martin Sucha Date: Sun, 25 Oct 2020 11:37:28 +0100 Subject: [PATCH 11/17] Start errors with lowercase letter (#1506) Error messages shouldn't be capitalized. https://github.com/golang/go/wiki/CodeReviewComments#error-strings --- control.go | 2 +- frame.go | 4 ++-- marshal.go | 2 +- metadata.go | 18 +++++++++--------- token.go | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/control.go b/control.go index 48ab457d2..de9f4145c 100644 --- a/control.go +++ b/control.go @@ -125,7 +125,7 @@ func hostInfo(addr string, defaultPort int) ([]*HostInfo, error) { if err != nil { return nil, err } else if len(ips) == 0 { - return nil, fmt.Errorf("No IP's returned from DNS lookup for %q", addr) + return nil, fmt.Errorf("no IP's returned from DNS lookup for %q", addr) } // Filter to v4 addresses if any present diff --git a/frame.go b/frame.go index 5fc948895..24fa49463 100644 --- a/frame.go +++ b/frame.go @@ -854,7 +854,7 @@ func (w *writePrepareFrame) writeFrame(f *framer, streamID int) error { if f.proto > protoVersion4 { flags |= flagWithPreparedKeyspace } else { - panic(fmt.Errorf("The keyspace can only be set with protocol 5 or higher")) + panic(fmt.Errorf("the keyspace can only be set with protocol 5 or higher")) } } if f.proto > protoVersion4 { @@ -1502,7 +1502,7 @@ func (f *framer) writeQueryParams(opts *queryParams) { if f.proto > protoVersion4 { flags |= flagWithKeyspace } else { - panic(fmt.Errorf("The keyspace can only be set with protocol 5 or higher")) + panic(fmt.Errorf("the keyspace can only be set with protocol 5 or higher")) } } diff --git a/marshal.go b/marshal.go index 59453dc74..f3b2ac7df 100644 --- a/marshal.go +++ b/marshal.go @@ -2143,7 +2143,7 @@ func marshalUDT(info TypeInfo, value interface{}) ([]byte, error) { case Marshaler: return v.MarshalCQL(info) case unsetColumn: - return nil, unmarshalErrorf("Invalid request: UnsetValue is unsupported for user defined types") + return nil, unmarshalErrorf("invalid request: UnsetValue is unsupported for user defined types") case UDTMarshaler: var buf []byte for _, e := range udt.Elements { diff --git a/metadata.go b/metadata.go index 44595ffee..6cd2b4bc0 100644 --- a/metadata.go +++ b/metadata.go @@ -559,7 +559,7 @@ func getKeyspaceMetadata(session *Session, keyspaceName string) (*KeyspaceMetada iter.Scan(&keyspace.DurableWrites, &replication) err := iter.Close() if err != nil { - return nil, fmt.Errorf("Error querying keyspace schema: %v", err) + return nil, fmt.Errorf("error querying keyspace schema: %v", err) } keyspace.StrategyClass = replication["class"] @@ -585,13 +585,13 @@ func getKeyspaceMetadata(session *Session, keyspaceName string) (*KeyspaceMetada iter.Scan(&keyspace.DurableWrites, &keyspace.StrategyClass, &strategyOptionsJSON) err := iter.Close() if err != nil { - return nil, fmt.Errorf("Error querying keyspace schema: %v", err) + return nil, fmt.Errorf("error querying keyspace schema: %v", err) } err = json.Unmarshal(strategyOptionsJSON, &keyspace.StrategyOptions) if err != nil { return nil, fmt.Errorf( - "Invalid JSON value '%s' as strategy_options for in keyspace '%s': %v", + "invalid JSON value '%s' as strategy_options for in keyspace '%s': %v", strategyOptionsJSON, keyspace.Name, err, ) } @@ -703,7 +703,7 @@ func getTableMetadata(session *Session, keyspaceName string) ([]TableMetadata, e if err != nil { iter.Close() return nil, fmt.Errorf( - "Invalid JSON value '%s' as key_aliases for in table '%s': %v", + "invalid JSON value '%s' as key_aliases for in table '%s': %v", keyAliasesJSON, table.Name, err, ) } @@ -716,7 +716,7 @@ func getTableMetadata(session *Session, keyspaceName string) ([]TableMetadata, e if err != nil { iter.Close() return nil, fmt.Errorf( - "Invalid JSON value '%s' as column_aliases for in table '%s': %v", + "invalid JSON value '%s' as column_aliases for in table '%s': %v", columnAliasesJSON, table.Name, err, ) } @@ -728,7 +728,7 @@ func getTableMetadata(session *Session, keyspaceName string) ([]TableMetadata, e err := iter.Close() if err != nil && err != ErrNotFound { - return nil, fmt.Errorf("Error querying table schema: %v", err) + return nil, fmt.Errorf("error querying table schema: %v", err) } return tables, nil @@ -777,7 +777,7 @@ func (s *Session) scanColumnMetadataV1(keyspace string) ([]ColumnMetadata, error err := json.Unmarshal(indexOptionsJSON, &column.Index.Options) if err != nil { return nil, fmt.Errorf( - "Invalid JSON value '%s' as index_options for column '%s' in table '%s': %v", + "invalid JSON value '%s' as index_options for column '%s' in table '%s': %v", indexOptionsJSON, column.Name, column.Table, @@ -837,7 +837,7 @@ func (s *Session) scanColumnMetadataV2(keyspace string) ([]ColumnMetadata, error err := json.Unmarshal(indexOptionsJSON, &column.Index.Options) if err != nil { return nil, fmt.Errorf( - "Invalid JSON value '%s' as index_options for column '%s' in table '%s': %v", + "invalid JSON value '%s' as index_options for column '%s' in table '%s': %v", indexOptionsJSON, column.Name, column.Table, @@ -915,7 +915,7 @@ func getColumnMetadata(session *Session, keyspaceName string) ([]ColumnMetadata, } if err != nil && err != ErrNotFound { - return nil, fmt.Errorf("Error querying column schema: %v", err) + return nil, fmt.Errorf("error querying column schema: %v", err) } return columns, nil diff --git a/token.go b/token.go index 9ae69b67a..7471299a9 100644 --- a/token.go +++ b/token.go @@ -153,7 +153,7 @@ func newTokenRing(partitioner string, hosts []*HostInfo) (*tokenRing, error) { } else if strings.HasSuffix(partitioner, "RandomPartitioner") { tokenRing.partitioner = randomPartitioner{} } else { - return nil, fmt.Errorf("Unsupported partitioner '%s'", partitioner) + return nil, fmt.Errorf("unsupported partitioner '%s'", partitioner) } for _, host := range hosts { From e4931c4dd1990cd654f6a09c032d5672c53d5d14 Mon Sep 17 00:00:00 2001 From: Piotr Dulikowski Date: Wed, 27 Jan 2021 10:45:44 +0100 Subject: [PATCH 12/17] tuple_test: check unmarshaled results in nested collection test The TestTuple_NestedCollection is supposed to test marshaling and unmarshaling of tuples nested in collections. It first writes a row and then reads it, but it does not check that the unmarshaled value is the same as the one used for writing. This commit adds this check to this test. --- tuple_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tuple_test.go b/tuple_test.go index a2ff31437..5e7a4edfa 100644 --- a/tuple_test.go +++ b/tuple_test.go @@ -280,6 +280,11 @@ func TestTuple_NestedCollection(t *testing.T) { if err != nil { t.Fatal(err) } + + resVal := reflect.ValueOf(res).Elem().Interface() + if !reflect.DeepEqual(test.val, resVal) { + t.Fatalf("unmarshaled value not equal to the original value: expected %#v, got %#v", test.val, resVal) + } }) } } From 745b2e381711f94e69cb1fea2758ebcf49d104f5 Mon Sep 17 00:00:00 2001 From: Piotr Dulikowski Date: Wed, 27 Jan 2021 11:35:32 +0100 Subject: [PATCH 13/17] fix deserialization of nested tuples with nullable elements Gocql supports deserialization of collections with tuples that hold their elements as pointers - for example, list> can be deserialized into [][2]*string. Currently, the pointers in the tuple are always set to a non-nil value, even if the tuple element was NULL. This is inconsistent with non-nested column, for which gocql allows to check if the column was NULL. This patch fixes that, and now pointers in nested tuples are set to nil if the corresponding column was NULL. This patch also updates the TestMarshalTuple test, which expected the old behavior. --- marshal.go | 12 ++++++++-- marshal_test.go | 12 +++++----- tuple_test.go | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 7 deletions(-) diff --git a/marshal.go b/marshal.go index f3b2ac7df..1f3c4aa16 100644 --- a/marshal.go +++ b/marshal.go @@ -2075,7 +2075,11 @@ func unmarshalTuple(info TypeInfo, data []byte, value interface{}) error { switch rv.Field(i).Kind() { case reflect.Ptr: - rv.Field(i).Set(reflect.ValueOf(v)) + if p != nil { + rv.Field(i).Set(reflect.ValueOf(v)) + } else { + rv.Field(i).Set(reflect.Zero(reflect.TypeOf(v))) + } default: rv.Field(i).Set(reflect.ValueOf(v).Elem()) } @@ -2105,7 +2109,11 @@ func unmarshalTuple(info TypeInfo, data []byte, value interface{}) error { switch rv.Index(i).Kind() { case reflect.Ptr: - rv.Index(i).Set(reflect.ValueOf(v)) + if p != nil { + rv.Index(i).Set(reflect.ValueOf(v)) + } else { + rv.Index(i).Set(reflect.Zero(reflect.TypeOf(v))) + } default: rv.Index(i).Set(reflect.ValueOf(v).Elem()) } diff --git a/marshal_test.go b/marshal_test.go index 0fb3f3635..d13f0b144 100644 --- a/marshal_test.go +++ b/marshal_test.go @@ -1784,8 +1784,8 @@ func TestMarshalTuple(t *testing.T) { if got.A != "foo" { t.Errorf("expected A string to be %v, got %v", "foo", got.A) } - if *got.B != "" { - t.Errorf("expected B string to be empty, got %v", *got.B) + if got.B != nil { + t.Errorf("expected B string to be nil, got %v", *got.B) } }, }, @@ -1814,7 +1814,9 @@ func TestMarshalTuple(t *testing.T) { check: func(t *testing.T, v interface{}) { got := v.(*[2]*string) checkString(t, "foo", *(got[0])) - checkString(t, "", *(got[1])) + if got[1] != nil { + t.Errorf("expected string to be nil, got %v", *got[1]) + } }, }, } @@ -1870,7 +1872,7 @@ func TestUnmarshalTuple(t *testing.T) { return } - if *tmp.A != "" || *tmp.B != "foo" { + if tmp.A != nil || *tmp.B != "foo" { t.Errorf("unmarshalTest: expected [nil, foo], got [%v, %v]", *tmp.A, *tmp.B) } }) @@ -1900,7 +1902,7 @@ func TestUnmarshalTuple(t *testing.T) { return } - if *tmp[0] != "" || *tmp[1] != "foo" { + if tmp[0] != nil || *tmp[1] != "foo" { t.Errorf("unmarshalTest: expected [nil, foo], got [%v, %v]", *tmp[0], *tmp[1]) } }) diff --git a/tuple_test.go b/tuple_test.go index 5e7a4edfa..50960081f 100644 --- a/tuple_test.go +++ b/tuple_test.go @@ -288,3 +288,61 @@ func TestTuple_NestedCollection(t *testing.T) { }) } } + +func TestTuple_NullableNestedCollection(t *testing.T) { + session := createSession(t) + defer session.Close() + if session.cfg.ProtoVersion < protoVersion3 { + t.Skip("tuple types are only available of proto>=3") + } + + err := createTable(session, `CREATE TABLE gocql_test.nested_tuples_with_nulls( + id int, + val list>>, + + primary key(id))`) + if err != nil { + t.Fatal(err) + } + + type typ struct { + A *string + B *string + } + + ptrStr := func(s string) *string { + ret := new(string) + *ret = s + return ret + } + + tests := []struct { + name string + val interface{} + }{ + {name: "slice", val: [][]*string{{ptrStr("1"), nil}, {nil, ptrStr("2")}}}, + {name: "array", val: [][2]*string{{ptrStr("1"), nil}, {nil, ptrStr("2")}}}, + {name: "struct", val: []typ{{ptrStr("1"), nil}, {nil, ptrStr("2")}}}, + } + + for i, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := session.Query(`INSERT INTO nested_tuples_with_nulls (id, val) VALUES (?, ?);`, i, test.val).Exec(); err != nil { + t.Fatal(err) + } + + rv := reflect.ValueOf(test.val) + res := reflect.New(rv.Type()).Interface() + + err = session.Query(`SELECT val FROM nested_tuples_with_nulls WHERE id=?`, i).Scan(res) + if err != nil { + t.Fatal(err) + } + + resVal := reflect.ValueOf(res).Elem().Interface() + if !reflect.DeepEqual(test.val, resVal) { + t.Fatalf("unmarshaled value not equal to the original value: expected %#v, got %#v", test.val, resVal) + } + }) + } +} From b9495d6179c88ccdbaac2b00ad0b17aabf23910e Mon Sep 17 00:00:00 2001 From: Piotr Dulikowski Date: Wed, 27 Jan 2021 15:14:58 +0100 Subject: [PATCH 14/17] fix unmarshaling of last empty/nil field in tuple Tuples are serialized as a sequence of pairs: length (32bit signed number) + contents. If a tuple element is NULL, its length is set to -1 and it has empty contents. However, if a tuple itself is NULL, the whole tuple is represented by empty bytes. This special case is handled in the following way: when trying to deserialize a tuple element and the length of remaining, unparsed bytes not greater than 4, the tuple element is treated as if it were NULL. However, this check is incorrect - if the last element of the tuple is represented as length = 0 and empty contents, then the check will cause the driver to interpret it as NULL. For example, for type tuple, values (1, "") and (1, NULL) will both be deserialized into (1, NULL). This patch fixes this problem by adjusting the check - a tuple element is considered NULL if there are _less_ than 4 bytes remaining. --- marshal.go | 6 +++--- tuple_test.go | 45 ++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/marshal.go b/marshal.go index 1f3c4aa16..e38633969 100644 --- a/marshal.go +++ b/marshal.go @@ -2035,7 +2035,7 @@ func unmarshalTuple(info TypeInfo, data []byte, value interface{}) error { for i, elem := range tuple.Elems { // each element inside data is a [bytes] var p []byte - if len(data) > 4 { + if len(data) >= 4 { p, data = readBytes(data) } err := Unmarshal(elem, p, v[i]) @@ -2064,7 +2064,7 @@ func unmarshalTuple(info TypeInfo, data []byte, value interface{}) error { for i, elem := range tuple.Elems { var p []byte - if len(data) > 4 { + if len(data) >= 4 { p, data = readBytes(data) } @@ -2098,7 +2098,7 @@ func unmarshalTuple(info TypeInfo, data []byte, value interface{}) error { for i, elem := range tuple.Elems { var p []byte - if len(data) > 4 { + if len(data) >= 4 { p, data = readBytes(data) } diff --git a/tuple_test.go b/tuple_test.go index 50960081f..0f3219bcd 100644 --- a/tuple_test.go +++ b/tuple_test.go @@ -237,6 +237,45 @@ func TestTupleMapScanNotSet(t *testing.T) { } } +func TestTupleLastFieldEmpty(t *testing.T) { + // Regression test - empty value used to be treated as NULL value in the last tuple field + session := createSession(t) + defer session.Close() + if session.cfg.ProtoVersion < protoVersion3 { + t.Skip("tuple types are only available of proto>=3") + } + err := createTable(session, `CREATE TABLE gocql_test.tuple_last_field_empty( + id int, + val frozen>, + + primary key(id))`) + if err != nil { + t.Fatal(err) + } + + if err := session.Query(`INSERT INTO tuple_last_field_empty (id, val) VALUES (?,(?,?));`, 1, "abc", "").Exec(); err != nil { + t.Fatal(err) + } + + var e1, e2 *string + if err := session.Query("SELECT val FROM tuple_last_field_empty WHERE id = ?", 1).Scan(&e1, &e2); err != nil { + t.Fatal(err) + } + + if e1 == nil { + t.Fatal("expected e1 not to be nil") + } + if *e1 != "abc" { + t.Fatalf("expected e1 to be equal to \"abc\", but is %v", *e2) + } + if e2 == nil { + t.Fatal("expected e2 not to be nil") + } + if *e2 != "" { + t.Fatalf("expected e2 to be an empty string, but is %v", *e2) + } +} + func TestTuple_NestedCollection(t *testing.T) { session := createSession(t) defer session.Close() @@ -320,9 +359,9 @@ func TestTuple_NullableNestedCollection(t *testing.T) { name string val interface{} }{ - {name: "slice", val: [][]*string{{ptrStr("1"), nil}, {nil, ptrStr("2")}}}, - {name: "array", val: [][2]*string{{ptrStr("1"), nil}, {nil, ptrStr("2")}}}, - {name: "struct", val: []typ{{ptrStr("1"), nil}, {nil, ptrStr("2")}}}, + {name: "slice", val: [][]*string{{ptrStr("1"), nil}, {nil, ptrStr("2")}, {ptrStr("3"), ptrStr("")}}}, + {name: "array", val: [][2]*string{{ptrStr("1"), nil}, {nil, ptrStr("2")}, {ptrStr("3"), ptrStr("")}}}, + {name: "struct", val: []typ{{ptrStr("1"), nil}, {nil, ptrStr("2")}, {ptrStr("3"), ptrStr("")}}}, } for i, test := range tests { From bcdfcf3b1d3a5be784a542be4c037ddff58be481 Mon Sep 17 00:00:00 2001 From: Piotr Dulikowski Date: Thu, 28 Jan 2021 10:26:29 +0100 Subject: [PATCH 15/17] add myself to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 009dc759d..bfa72dd86 100644 --- a/AUTHORS +++ b/AUTHORS @@ -118,3 +118,4 @@ Jorge Bay Dmitriy Kozlov Alexey Romanovsky Jaume Marhuenda Beltran +Piotr Dulikowski From df312a6b3d64ea0a1d6871e3eace6a7ff30f5ed6 Mon Sep 17 00:00:00 2001 From: Martin Sucha Date: Fri, 29 Jan 2021 09:47:47 +0100 Subject: [PATCH 16/17] Use the same host iterator in speculative execution Before this commit, regular query execution path and speculative execution path did not use the same host iterator. This means that speculative execution could retry on the same host as the original attempt. This could lead to overloading that slow host even more instead of trying on some other host if the policy consistently selects that host (for example based on token). We fix this by sharing the host iterator returned by the host selection policy. The returned iterator could now accessed from multiple goroutines, so we need to synchronize access to it. I chose this instead of implementing synchronization in each policy because I can't fix any user-provided policies. Synchronizing in the driver avoids introducing new data race for user-provided policies as those likely won't be updated. Fixes https://github.com/gocql/gocql/issues/1530 --- policies.go | 5 ++++- query_executor.go | 31 ++++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/policies.go b/policies.go index 62d809c86..485dcef5a 100644 --- a/policies.go +++ b/policies.go @@ -304,7 +304,10 @@ type HostSelectionPolicy interface { KeyspaceChanged(KeyspaceUpdateEvent) Init(*Session) IsLocal(host *HostInfo) bool - //Pick returns an iteration function over selected hosts + // Pick returns an iteration function over selected hosts. + // Multiple attempts of a single query execution won't call the returned NextHost function concurrently, + // so it's safe to have internal state without additional synchronization as long as every call to Pick returns + // a different instance of NextHost. Pick(ExecutableQuery) NextHost } diff --git a/query_executor.go b/query_executor.go index 6dd912db7..755deae5c 100644 --- a/query_executor.go +++ b/query_executor.go @@ -2,6 +2,7 @@ package gocql import ( "context" + "sync" "time" ) @@ -34,14 +35,15 @@ func (q *queryExecutor) attemptQuery(ctx context.Context, qry ExecutableQuery, c return iter } -func (q *queryExecutor) speculate(ctx context.Context, qry ExecutableQuery, sp SpeculativeExecutionPolicy, results chan *Iter) *Iter { +func (q *queryExecutor) speculate(ctx context.Context, qry ExecutableQuery, sp SpeculativeExecutionPolicy, + hostIter NextHost, results chan *Iter) *Iter { ticker := time.NewTicker(sp.Delay()) defer ticker.Stop() for i := 0; i < sp.Attempts(); i++ { select { case <-ticker.C: - go q.run(ctx, qry, results) + go q.run(ctx, qry, hostIter, results) case <-ctx.Done(): return &Iter{err: ctx.Err()} case iter := <-results: @@ -53,11 +55,23 @@ func (q *queryExecutor) speculate(ctx context.Context, qry ExecutableQuery, sp S } func (q *queryExecutor) executeQuery(qry ExecutableQuery) (*Iter, error) { + hostIter := q.policy.Pick(qry) + // check if the query is not marked as idempotent, if // it is, we force the policy to NonSpeculative sp := qry.speculativeExecutionPolicy() if !qry.IsIdempotent() || sp.Attempts() == 0 { - return q.do(qry.Context(), qry), nil + return q.do(qry.Context(), qry, hostIter), nil + } + + // When speculative execution is enabled, we could be accessing the host iterator from multiple goroutines below. + // To ensure we don't call it concurrently, we wrap the returned NextHost function here to synchronize access to it. + var mu sync.Mutex + origHostIter := hostIter + hostIter = func() SelectedHost { + mu.Lock() + defer mu.Unlock() + return origHostIter() } ctx, cancel := context.WithCancel(qry.Context()) @@ -66,12 +80,12 @@ func (q *queryExecutor) executeQuery(qry ExecutableQuery) (*Iter, error) { results := make(chan *Iter, 1) // Launch the main execution - go q.run(ctx, qry, results) + go q.run(ctx, qry, hostIter, results) // The speculative executions are launched _in addition_ to the main // execution, on a timer. So Speculation{2} would make 3 executions running // in total. - if iter := q.speculate(ctx, qry, sp, results); iter != nil { + if iter := q.speculate(ctx, qry, sp, hostIter, results); iter != nil { return iter, nil } @@ -83,8 +97,7 @@ func (q *queryExecutor) executeQuery(qry ExecutableQuery) (*Iter, error) { } } -func (q *queryExecutor) do(ctx context.Context, qry ExecutableQuery) *Iter { - hostIter := q.policy.Pick(qry) +func (q *queryExecutor) do(ctx context.Context, qry ExecutableQuery, hostIter NextHost) *Iter { selectedHost := hostIter() rt := qry.retryPolicy() @@ -153,9 +166,9 @@ func (q *queryExecutor) do(ctx context.Context, qry ExecutableQuery) *Iter { return &Iter{err: ErrNoConnections} } -func (q *queryExecutor) run(ctx context.Context, qry ExecutableQuery, results chan<- *Iter) { +func (q *queryExecutor) run(ctx context.Context, qry ExecutableQuery, hostIter NextHost, results chan<- *Iter) { select { - case results <- q.do(ctx, qry): + case results <- q.do(ctx, qry, hostIter): case <-ctx.Done(): } } From 4364a4b9cfdd5b806725cc7d2e9abf3bdf461166 Mon Sep 17 00:00:00 2001 From: Martin Sucha Date: Tue, 1 Dec 2020 23:15:39 +0100 Subject: [PATCH 17/17] Add package-level documentation (#1513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The examples are regular godoc/go test examples. Because go examples don't support marking the example as skipped, the // Output line before output is intentionally omitted. This way, the examples are compiled by go test, but not executed (as that would fail because we need to access the database). When someone wants to execute the example, adding the Output line will make it executable. Co-authored-by: Matouš Dzivjak --- README.md | 68 +------ doc.go | 307 +++++++++++++++++++++++++++++++- example_batch_test.go | 55 ++++++ example_dynamic_columns_test.go | 101 +++++++++++ example_lwt_batch_test.go | 95 ++++++++++ example_lwt_test.go | 67 +++++++ example_marshaler_test.go | 93 ++++++++++ example_nulls_test.go | 52 ++++++ example_paging_test.go | 70 ++++++++ example_test.go | 65 +++++++ example_udt_map_test.go | 49 +++++ example_udt_marshaler_test.go | 57 ++++++ example_udt_struct_test.go | 54 ++++++ example_udt_unmarshaler_test.go | 59 ++++++ policies.go | 5 +- session.go | 9 + uuid.go | 3 +- 17 files changed, 1138 insertions(+), 71 deletions(-) create mode 100644 example_batch_test.go create mode 100644 example_dynamic_columns_test.go create mode 100644 example_lwt_batch_test.go create mode 100644 example_lwt_test.go create mode 100644 example_marshaler_test.go create mode 100644 example_nulls_test.go create mode 100644 example_paging_test.go create mode 100644 example_test.go create mode 100644 example_udt_map_test.go create mode 100644 example_udt_marshaler_test.go create mode 100644 example_udt_struct_test.go create mode 100644 example_udt_unmarshaler_test.go diff --git a/README.md b/README.md index 980ca8edb..ac37a5616 100644 --- a/README.md +++ b/README.md @@ -114,73 +114,7 @@ statement. Example ------- -```go -/* Before you execute the program, Launch `cqlsh` and execute: -create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; -create table example.tweet(timeline text, id UUID, text text, PRIMARY KEY(id)); -create index on example.tweet(timeline); -*/ -package main - -import ( - "fmt" - "log" - - "github.com/gocql/gocql" -) - -func main() { - // connect to the cluster - cluster := gocql.NewCluster("192.168.1.1", "192.168.1.2", "192.168.1.3") - cluster.Keyspace = "example" - cluster.Consistency = gocql.Quorum - session, _ := cluster.CreateSession() - defer session.Close() - - // insert a tweet - if err := session.Query(`INSERT INTO tweet (timeline, id, text) VALUES (?, ?, ?)`, - "me", gocql.TimeUUID(), "hello world").Exec(); err != nil { - log.Fatal(err) - } - - var id gocql.UUID - var text string - - /* Search for a specific set of records whose 'timeline' column matches - * the value 'me'. The secondary index that we created earlier will be - * used for optimizing the search */ - if err := session.Query(`SELECT id, text FROM tweet WHERE timeline = ? LIMIT 1`, - "me").Consistency(gocql.One).Scan(&id, &text); err != nil { - log.Fatal(err) - } - fmt.Println("Tweet:", id, text) - - // list all tweets - iter := session.Query(`SELECT id, text FROM tweet WHERE timeline = ?`, "me").Iter() - for iter.Scan(&id, &text) { - fmt.Println("Tweet:", id, text) - } - if err := iter.Close(); err != nil { - log.Fatal(err) - } -} -``` - - -Authentication -------- - -```go -cluster := gocql.NewCluster("192.168.1.1", "192.168.1.2", "192.168.1.3") -cluster.Authenticator = gocql.PasswordAuthenticator{ - Username: "user", - Password: "password" -} -cluster.Keyspace = "example" -cluster.Consistency = gocql.Quorum -session, _ := cluster.CreateSession() -defer session.Close() -``` +See [package documentation](https://pkg.go.dev/github.com/gocql/gocql#pkg-examples). Data Binding ------------ diff --git a/doc.go b/doc.go index 5c4b041a1..68faa7f8c 100644 --- a/doc.go +++ b/doc.go @@ -4,6 +4,309 @@ // Package gocql implements a fast and robust Cassandra driver for the // Go programming language. +// +// Connecting to the cluster +// +// Pass a list of initial node IP addresses to NewCluster to create a new cluster configuration: +// +// cluster := gocql.NewCluster("192.168.1.1", "192.168.1.2", "192.168.1.3") +// +// Port can be specified as part of the address, the above is equivalent to: +// +// cluster := gocql.NewCluster("192.168.1.1:9042", "192.168.1.2:9042", "192.168.1.3:9042") +// +// It is recommended to use the value set in the Cassandra config for broadcast_address or listen_address, +// an IP address not a domain name. This is because events from Cassandra will use the configured IP +// address, which is used to index connected hosts. If the domain name specified resolves to more than 1 IP address +// then the driver may connect multiple times to the same host, and will not mark the node being down or up from events. +// +// Then you can customize more options (see ClusterConfig): +// +// cluster.Keyspace = "example" +// cluster.Consistency = gocql.Quorum +// cluster.ProtocolVersion = 4 +// +// The driver tries to automatically detect the protocol version to use if not set, but you might want to set the +// protocol version explicitly, as it's not defined which version will be used in certain situations (for example +// during upgrade of the cluster when some of the nodes support different set of protocol versions than other nodes). +// +// When ready, create a session from the configuration. Don't forget to Close the session once you are done with it: +// +// session, err := cluster.CreateSession() +// if err != nil { +// return err +// } +// defer session.Close() +// +// Authentication +// +// CQL protocol uses a SASL-based authentication mechanism and so consists of an exchange of server challenges and +// client response pairs. The details of the exchanged messages depend on the authenticator used. +// +// To use authentication, set ClusterConfig.Authenticator or ClusterConfig.AuthProvider. +// +// PasswordAuthenticator is provided to use for username/password authentication: +// +// cluster := gocql.NewCluster("192.168.1.1", "192.168.1.2", "192.168.1.3") +// cluster.Authenticator = gocql.PasswordAuthenticator{ +// Username: "user", +// Password: "password" +// } +// session, err := cluster.CreateSession() +// if err != nil { +// return err +// } +// defer session.Close() +// +// Transport layer security +// +// It is possible to secure traffic between the client and server with TLS. +// +// To use TLS, set the ClusterConfig.SslOpts field. SslOptions embeds *tls.Config so you can set that directly. +// There are also helpers to load keys/certificates from files. +// +// Warning: Due to backward-compatibility reasons, the tls.Config's InsecureSkipVerify is set to +// !SslOptions.EnableHostVerification, so by default host verification is disabled. Most users using TLS should set +// SslOptions.EnableHostVerification to true. +// +// cluster := gocql.NewCluster("192.168.1.1", "192.168.1.2", "192.168.1.3") +// cluster.SslOpts = &gocql.SslOptions{ +// EnableHostVerification: true, +// } +// session, err := cluster.CreateSession() +// if err != nil { +// return err +// } +// defer session.Close() +// +// Executing queries +// +// Create queries with Session.Query. Query values must not be reused between different executions and must not be +// modified after starting execution of the query. +// +// To execute a query without reading results, use Query.Exec: +// +// err := session.Query(`INSERT INTO tweet (timeline, id, text) VALUES (?, ?, ?)`, +// "me", gocql.TimeUUID(), "hello world").WithContext(ctx).Exec() +// +// Single row can be read by calling Query.Scan: +// +// err := session.Query(`SELECT id, text FROM tweet WHERE timeline = ? LIMIT 1`, +// "me").WithContext(ctx).Consistency(gocql.One).Scan(&id, &text) +// +// Multiple rows can be read using Iter.Scanner: +// +// scanner := session.Query(`SELECT id, text FROM tweet WHERE timeline = ?`, +// "me").WithContext(ctx).Iter().Scanner() +// for scanner.Next() { +// var ( +// id gocql.UUID +// text string +// ) +// err = scanner.Scan(&id, &text) +// if err != nil { +// log.Fatal(err) +// } +// fmt.Println("Tweet:", id, text) +// } +// // scanner.Err() closes the iterator, so scanner nor iter should be used afterwards. +// if err := scanner.Err(); err != nil { +// log.Fatal(err) +// } +// +// See Example for complete example. +// +// Prepared statements +// +// The driver automatically prepares DML queries (SELECT/INSERT/UPDATE/DELETE/BATCH statements) and maintains a cache +// of prepared statements. +// CQL protocol does not support preparing other query types. +// +// When using CQL protocol >= 4, it is possible to use gocql.UnsetValue as the bound value of a column. +// This will cause the database to ignore writing the column. +// The main advantage is the ability to keep the same prepared statement even when you don't +// want to update some fields, where before you needed to make another prepared statement. +// +// Executing multiple queries concurrently +// +// Session is safe to use from multiple goroutines, so to execute multiple concurrent queries, just execute them +// from several worker goroutines. Gocql provides synchronously-looking API (as recommended for Go APIs) and the queries +// are executed asynchronously at the protocol level. +// +// results := make(chan error, 2) +// go func() { +// results <- session.Query(`INSERT INTO tweet (timeline, id, text) VALUES (?, ?, ?)`, +// "me", gocql.TimeUUID(), "hello world 1").Exec() +// }() +// go func() { +// results <- session.Query(`INSERT INTO tweet (timeline, id, text) VALUES (?, ?, ?)`, +// "me", gocql.TimeUUID(), "hello world 2").Exec() +// }() +// +// Nulls +// +// Null values are are unmarshalled as zero value of the type. If you need to distinguish for example between text +// column being null and empty string, you can unmarshal into *string variable instead of string. +// +// var text *string +// err := scanner.Scan(&text) +// if err != nil { +// // handle error +// } +// if text != nil { +// // not null +// } +// else { +// // null +// } +// +// See Example_nulls for full example. +// +// Reusing slices +// +// The driver reuses backing memory of slices when unmarshalling. This is an optimization so that a buffer does not +// need to be allocated for every processed row. However, you need to be careful when storing the slices to other +// memory structures. +// +// scanner := session.Query(`SELECT myints FROM table WHERE pk = ?`, "key").WithContext(ctx).Iter().Scanner() +// var myInts []int +// for scanner.Next() { +// // This scan reuses backing store of myInts for each row. +// err = scanner.Scan(&myInts) +// if err != nil { +// log.Fatal(err) +// } +// } +// +// When you want to save the data for later use, pass a new slice every time. A common pattern is to declare the +// slice variable within the scanner loop: +// +// scanner := session.Query(`SELECT myints FROM table WHERE pk = ?`, "key").WithContext(ctx).Iter().Scanner() +// for scanner.Next() { +// var myInts []int +// // This scan always gets pointer to fresh myInts slice, so does not reuse memory. +// err = scanner.Scan(&myInts) +// if err != nil { +// log.Fatal(err) +// } +// } +// +// Paging +// +// The driver supports paging of results with automatic prefetch, see ClusterConfig.PageSize, Session.SetPrefetch, +// Query.PageSize, and Query.Prefetch. +// +// It is also possible to control the paging manually with Query.PageState (this disables automatic prefetch). +// Manual paging is useful if you want to store the page state externally, for example in a URL to allow users +// browse pages in a result. You might want to sign/encrypt the paging state when exposing it externally since +// it contains data from primary keys. +// +// Paging state is specific to the CQL protocol version and the exact query used. It is meant as opaque state that +// should not be modified. If you send paging state from different query or protocol version, then the behaviour +// is not defined (you might get unexpected results or an error from the server). For example, do not send paging state +// returned by node using protocol version 3 to a node using protocol version 4. Also, when using protocol version 4, +// paging state between Cassandra 2.2 and 3.0 is incompatible (https://issues.apache.org/jira/browse/CASSANDRA-10880). +// +// The driver does not check whether the paging state is from the same protocol version/statement. +// You might want to validate yourself as this could be a problem if you store paging state externally. +// For example, if you store paging state in a URL, the URLs might become broken when you upgrade your cluster. +// +// Call Query.PageState(nil) to fetch just the first page of the query results. Pass the page state returned by +// Iter.PageState to Query.PageState of a subsequent query to get the next page. If the length of slice returned +// by Iter.PageState is zero, there are no more pages available (or an error occurred). +// +// Using too low values of PageSize will negatively affect performance, a value below 100 is probably too low. +// While Cassandra returns exactly PageSize items (except for last page) in a page currently, the protocol authors +// explicitly reserved the right to return smaller or larger amount of items in a page for performance reasons, so don't +// rely on the page having the exact count of items. +// +// See Example_paging for an example of manual paging. +// +// Dynamic list of columns +// +// There are certain situations when you don't know the list of columns in advance, mainly when the query is supplied +// by the user. Iter.Columns, Iter.RowData, Iter.MapScan and Iter.SliceMap can be used to handle this case. +// +// See Example_dynamicColumns. +// +// Batches +// +// The CQL protocol supports sending batches of DML statements (INSERT/UPDATE/DELETE) and so does gocql. +// Use Session.NewBatch to create a new batch and then fill-in details of individual queries. +// Then execute the batch with Session.ExecuteBatch. +// +// Logged batches ensure atomicity, either all or none of the operations in the batch will succeed, but they have +// overhead to ensure this property. +// Unlogged batches don't have the overhead of logged batches, but don't guarantee atomicity. +// Updates of counters are handled specially by Cassandra so batches of counter updates have to use CounterBatch type. +// A counter batch can only contain statements to update counters. +// +// For unlogged batches it is recommended to send only single-partition batches (i.e. all statements in the batch should +// involve only a single partition). +// Multi-partition batch needs to be split by the coordinator node and re-sent to +// correct nodes. +// With single-partition batches you can send the batch directly to the node for the partition without incurring the +// additional network hop. +// +// It is also possible to pass entire BEGIN BATCH .. APPLY BATCH statement to Query.Exec. +// There are differences how those are executed. +// BEGIN BATCH statement passed to Query.Exec is prepared as a whole in a single statement. +// Session.ExecuteBatch prepares individual statements in the batch. +// If you have variable-length batches using the same statement, using Session.ExecuteBatch is more efficient. +// +// See Example_batch for an example. +// +// Lightweight transactions +// +// Query.ScanCAS or Query.MapScanCAS can be used to execute a single-statement lightweight transaction (an +// INSERT/UPDATE .. IF statement) and reading its result. See example for Query.MapScanCAS. +// +// Multiple-statement lightweight transactions can be executed as a logged batch that contains at least one conditional +// statement. All the conditions must return true for the batch to be applied. You can use Session.ExecuteBatchCAS and +// Session.MapExecuteBatchCAS when executing the batch to learn about the result of the LWT. See example for +// Session.MapExecuteBatchCAS. +// +// Retries and speculative execution +// +// Queries can be marked as idempotent. Marking the query as idempotent tells the driver that the query can be executed +// multiple times without affecting its result. Non-idempotent queries are not eligible for retrying nor speculative +// execution. +// +// Idempotent queries are retried in case of errors based on the configured RetryPolicy. +// +// Queries can be retried even before they fail by setting a SpeculativeExecutionPolicy. The policy can +// cause the driver to retry on a different node if the query is taking longer than a specified delay even before the +// driver receives an error or timeout from the server. When a query is speculatively executed, the original execution +// is still executing. The two parallel executions of the query race to return a result, the first received result will +// be returned. +// +// User-defined types +// +// UDTs can be mapped (un)marshaled from/to map[string]interface{} a Go struct (or a type implementing +// UDTUnmarshaler, UDTMarshaler, Unmarshaler or Marshaler interfaces). +// +// For structs, cql tag can be used to specify the CQL field name to be mapped to a struct field: +// +// type MyUDT struct { +// FieldA int32 `cql:"a"` +// FieldB string `cql:"b"` +// } +// +// See Example_userDefinedTypesMap, Example_userDefinedTypesStruct, ExampleUDTMarshaler, ExampleUDTUnmarshaler. +// +// Metrics and tracing +// +// It is possible to provide observer implementations that could be used to gather metrics: +// +// - QueryObserver for monitoring individual queries. +// - BatchObserver for monitoring batch queries. +// - ConnectObserver for monitoring new connections from the driver to the database. +// - FrameHeaderObserver for monitoring individual protocol frames. +// +// CQL protocol also supports tracing of queries. When enabled, the database will write information about +// internal events that happened during execution of the query. You can use Query.Trace to request tracing and receive +// the session ID that the database used to store the trace information in system_traces.sessions and +// system_traces.events tables. NewTraceWriter returns an implementation of Tracer that writes the events to a writer. +// Gathering trace information might be essential for debugging and optimizing queries, but writing traces has overhead, +// so this feature should not be used on production systems with very high load unless you know what you are doing. package gocql // import "github.com/gocql/gocql" - -// TODO(tux21b): write more docs. diff --git a/example_batch_test.go b/example_batch_test.go new file mode 100644 index 000000000..b3e767ac4 --- /dev/null +++ b/example_batch_test.go @@ -0,0 +1,55 @@ +package gocql_test + +import ( + "context" + "fmt" + "github.com/gocql/gocql" + "log" +) + +// Example_batch demonstrates how to execute a batch of statements. +func Example_batch() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create table example.batches(pk int, ck int, description text, PRIMARY KEY(pk, ck)); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + ctx := context.Background() + + b := session.NewBatch(gocql.UnloggedBatch).WithContext(ctx) + b.Entries = append(b.Entries, gocql.BatchEntry{ + Stmt: "INSERT INTO example.batches (pk, ck, description) VALUES (?, ?, ?)", + Args: []interface{}{1, 2, "1.2"}, + Idempotent: true, + }) + b.Entries = append(b.Entries, gocql.BatchEntry{ + Stmt: "INSERT INTO example.batches (pk, ck, description) VALUES (?, ?, ?)", + Args: []interface{}{1, 3, "1.3"}, + Idempotent: true, + }) + err = session.ExecuteBatch(b) + if err != nil { + log.Fatal(err) + } + + scanner := session.Query("SELECT pk, ck, description FROM example.batches").Iter().Scanner() + for scanner.Next() { + var pk, ck int32 + var description string + err = scanner.Scan(&pk, &ck, &description) + if err != nil { + log.Fatal(err) + } + fmt.Println(pk, ck, description) + } + // 1 2 1.2 + // 1 3 1.3 +} diff --git a/example_dynamic_columns_test.go b/example_dynamic_columns_test.go new file mode 100644 index 000000000..6b4d3dae6 --- /dev/null +++ b/example_dynamic_columns_test.go @@ -0,0 +1,101 @@ +package gocql_test + +import ( + "context" + "fmt" + "github.com/gocql/gocql" + "log" + "os" + "reflect" + "text/tabwriter" +) + +// Example_dynamicColumns demonstrates how to handle dynamic column list. +func Example_dynamicColumns() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create table example.table1(pk text, ck int, value1 text, value2 int, PRIMARY KEY(pk, ck)); + insert into example.table1 (pk, ck, value1, value2) values ('a', 1, 'b', 2); + insert into example.table1 (pk, ck, value1, value2) values ('c', 3, 'd', 4); + insert into example.table1 (pk, ck, value1, value2) values ('c', 5, null, null); + create table example.table2(pk int, value1 timestamp, PRIMARY KEY(pk)); + insert into example.table2 (pk, value1) values (1, '2020-01-02 03:04:05'); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + printQuery := func(ctx context.Context, session *gocql.Session, stmt string, values ...interface{}) error { + iter := session.Query(stmt, values...).WithContext(ctx).Iter() + fmt.Println(stmt) + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', + 0) + for i, columnInfo := range iter.Columns() { + if i > 0 { + fmt.Fprint(w, "\t| ") + } + fmt.Fprintf(w, "%s (%s)", columnInfo.Name, columnInfo.TypeInfo) + } + + for { + rd, err := iter.RowData() + if err != nil { + return err + } + if !iter.Scan(rd.Values...) { + break + } + fmt.Fprint(w, "\n") + for i, val := range rd.Values { + if i > 0 { + fmt.Fprint(w, "\t| ") + } + + fmt.Fprint(w, reflect.Indirect(reflect.ValueOf(val)).Interface()) + } + } + + fmt.Fprint(w, "\n") + w.Flush() + fmt.Println() + + return iter.Close() + } + + ctx := context.Background() + + err = printQuery(ctx, session, "SELECT * FROM table1") + if err != nil { + log.Fatal(err) + } + + err = printQuery(ctx, session, "SELECT value2, pk, ck FROM table1") + if err != nil { + log.Fatal(err) + } + + err = printQuery(ctx, session, "SELECT * FROM table2") + if err != nil { + log.Fatal(err) + } + // SELECT * FROM table1 + // pk (varchar) | ck (int) | value1 (varchar) | value2 (int) + // a | 1 | b | 2 + // c | 3 | d | 4 + // c | 5 | | 0 + // + // SELECT value2, pk, ck FROM table1 + // value2 (int) | pk (varchar) | ck (int) + // 2 | a | 1 + // 4 | c | 3 + // 0 | c | 5 + // + // SELECT * FROM table2 + // pk (int) | value1 (timestamp) + // 1 | 2020-01-02 03:04:05 +0000 UTC +} diff --git a/example_lwt_batch_test.go b/example_lwt_batch_test.go new file mode 100644 index 000000000..2ed6f0fd5 --- /dev/null +++ b/example_lwt_batch_test.go @@ -0,0 +1,95 @@ +package gocql_test + +import ( + "context" + "fmt" + "github.com/gocql/gocql" + "log" +) + +// ExampleSession_MapExecuteBatchCAS demonstrates how to execute a batch lightweight transaction. +func ExampleSession_MapExecuteBatchCAS() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create table example.my_lwt_batch_table(pk text, ck text, version int, value text, PRIMARY KEY(pk, ck)); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + ctx := context.Background() + + err = session.Query("INSERT INTO example.my_lwt_batch_table (pk, ck, version, value) VALUES (?, ?, ?, ?)", + "pk1", "ck1", 1, "a").WithContext(ctx).Exec() + if err != nil { + log.Fatal(err) + } + + err = session.Query("INSERT INTO example.my_lwt_batch_table (pk, ck, version, value) VALUES (?, ?, ?, ?)", + "pk1", "ck2", 1, "A").WithContext(ctx).Exec() + if err != nil { + log.Fatal(err) + } + + executeBatch := func(ck2Version int) { + b := session.NewBatch(gocql.LoggedBatch) + b.Entries = append(b.Entries, gocql.BatchEntry{ + Stmt: "UPDATE my_lwt_batch_table SET value=? WHERE pk=? AND ck=? IF version=?", + Args: []interface{}{"b", "pk1", "ck1", 1}, + }) + b.Entries = append(b.Entries, gocql.BatchEntry{ + Stmt: "UPDATE my_lwt_batch_table SET value=? WHERE pk=? AND ck=? IF version=?", + Args: []interface{}{"B", "pk1", "ck2", ck2Version}, + }) + m := make(map[string]interface{}) + applied, iter, err := session.MapExecuteBatchCAS(b.WithContext(ctx), m) + if err != nil { + log.Fatal(err) + } + fmt.Println(applied, m) + + m = make(map[string]interface{}) + for iter.MapScan(m) { + fmt.Println(m) + m = make(map[string]interface{}) + } + + if err := iter.Close(); err != nil { + log.Fatal(err) + } + } + + printState := func() { + scanner := session.Query("SELECT ck, value FROM example.my_lwt_batch_table WHERE pk = ?", "pk1"). + WithContext(ctx).Iter().Scanner() + for scanner.Next() { + var ck, value string + err = scanner.Scan(&ck, &value) + if err != nil { + log.Fatal(err) + } + fmt.Println(ck, value) + } + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + } + + executeBatch(0) + printState() + executeBatch(1) + printState() + + // false map[ck:ck1 pk:pk1 version:1] + // map[[applied]:false ck:ck2 pk:pk1 version:1] + // ck1 a + // ck2 A + // true map[] + // ck1 b + // ck2 B +} diff --git a/example_lwt_test.go b/example_lwt_test.go new file mode 100644 index 000000000..5ffe8346f --- /dev/null +++ b/example_lwt_test.go @@ -0,0 +1,67 @@ +package gocql_test + +import ( + "context" + "fmt" + "github.com/gocql/gocql" + "log" +) + +// ExampleQuery_MapScanCAS demonstrates how to execute a single-statement lightweight transaction. +func ExampleQuery_MapScanCAS() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create table example.my_lwt_table(pk int, version int, value text, PRIMARY KEY(pk)); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + ctx := context.Background() + + err = session.Query("INSERT INTO example.my_lwt_table (pk, version, value) VALUES (?, ?, ?)", + 1, 1, "a").WithContext(ctx).Exec() + if err != nil { + log.Fatal(err) + } + m := make(map[string]interface{}) + applied, err := session.Query("UPDATE example.my_lwt_table SET value = ? WHERE pk = ? IF version = ?", + "b", 1, 0).WithContext(ctx).MapScanCAS(m) + if err != nil { + log.Fatal(err) + } + fmt.Println(applied, m) + + var value string + err = session.Query("SELECT value FROM example.my_lwt_table WHERE pk = ?", 1).WithContext(ctx). + Scan(&value) + if err != nil { + log.Fatal(err) + } + fmt.Println(value) + + m = make(map[string]interface{}) + applied, err = session.Query("UPDATE example.my_lwt_table SET value = ? WHERE pk = ? IF version = ?", + "b", 1, 1).WithContext(ctx).MapScanCAS(m) + if err != nil { + log.Fatal(err) + } + fmt.Println(applied, m) + + var value2 string + err = session.Query("SELECT value FROM example.my_lwt_table WHERE pk = ?", 1).WithContext(ctx). + Scan(&value2) + if err != nil { + log.Fatal(err) + } + fmt.Println(value2) + // false map[version:1] + // a + // true map[] + // b +} diff --git a/example_marshaler_test.go b/example_marshaler_test.go new file mode 100644 index 000000000..1d88d92ca --- /dev/null +++ b/example_marshaler_test.go @@ -0,0 +1,93 @@ +package gocql_test + +import ( + "context" + "fmt" + "github.com/gocql/gocql" + "log" + "strconv" + "strings" +) + +// MyMarshaler implements Marshaler and Unmarshaler. +// It represents a version number stored as string. +type MyMarshaler struct { + major, minor, patch int +} + +func (m MyMarshaler) MarshalCQL(info gocql.TypeInfo) ([]byte, error) { + return gocql.Marshal(info, fmt.Sprintf("%d.%d.%d", m.major, m.minor, m.patch)) +} + +func (m *MyMarshaler) UnmarshalCQL(info gocql.TypeInfo, data []byte) error { + var s string + err := gocql.Unmarshal(info, data, &s) + if err != nil { + return err + } + parts := strings.SplitN(s, ".", 3) + if len(parts) != 3 { + return fmt.Errorf("parse version %q: %d parts instead of 3", s, len(parts)) + } + major, err := strconv.Atoi(parts[0]) + if err != nil { + return fmt.Errorf("parse version %q major number: %v", s, err) + } + minor, err := strconv.Atoi(parts[1]) + if err != nil { + return fmt.Errorf("parse version %q minor number: %v", s, err) + } + patch, err := strconv.Atoi(parts[2]) + if err != nil { + return fmt.Errorf("parse version %q patch number: %v", s, err) + } + m.major = major + m.minor = minor + m.patch = patch + return nil +} + +// Example_marshalerUnmarshaler demonstrates how to implement a Marshaler and Unmarshaler. +func Example_marshalerUnmarshaler() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create table example.my_marshaler_table(pk int, value text, PRIMARY KEY(pk)); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + ctx := context.Background() + + value := MyMarshaler{ + major: 1, + minor: 2, + patch: 3, + } + err = session.Query("INSERT INTO example.my_marshaler_table (pk, value) VALUES (?, ?)", + 1, value).WithContext(ctx).Exec() + if err != nil { + log.Fatal(err) + } + var stringValue string + err = session.Query("SELECT value FROM example.my_marshaler_table WHERE pk = 1").WithContext(ctx). + Scan(&stringValue) + if err != nil { + log.Fatal(err) + } + fmt.Println(stringValue) + var unmarshaledValue MyMarshaler + err = session.Query("SELECT value FROM example.my_marshaler_table WHERE pk = 1").WithContext(ctx). + Scan(&unmarshaledValue) + if err != nil { + log.Fatal(err) + } + fmt.Println(unmarshaledValue) + // 1.2.3 + // {1 2 3} +} diff --git a/example_nulls_test.go b/example_nulls_test.go new file mode 100644 index 000000000..3352f45e4 --- /dev/null +++ b/example_nulls_test.go @@ -0,0 +1,52 @@ +package gocql_test + +import ( + "fmt" + "github.com/gocql/gocql" + "log" +) + +// Example_nulls demonstrates how to distinguish between null and zero value when needed. +// +// Null values are unmarshalled as zero value of the type. If you need to distinguish for example between text +// column being null and empty string, you can unmarshal into *string field. +func Example_nulls() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create table example.stringvals(id int, value text, PRIMARY KEY(id)); + insert into example.stringvals (id, value) values (1, null); + insert into example.stringvals (id, value) values (2, ''); + insert into example.stringvals (id, value) values (3, 'hello'); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + scanner := session.Query(`SELECT id, value FROM stringvals`).Iter().Scanner() + for scanner.Next() { + var ( + id int32 + val *string + ) + err := scanner.Scan(&id, &val) + if err != nil { + log.Fatal(err) + } + if val != nil { + fmt.Printf("Row %d is %q\n", id, *val) + } else { + fmt.Printf("Row %d is null\n", id) + } + + } + err = scanner.Err() + if err != nil { + log.Fatal(err) + } + // Row 1 is null + // Row 2 is "" + // Row 3 is "hello" +} diff --git a/example_paging_test.go b/example_paging_test.go new file mode 100644 index 000000000..0a9f6770e --- /dev/null +++ b/example_paging_test.go @@ -0,0 +1,70 @@ +package gocql_test + +import ( + "fmt" + "github.com/gocql/gocql" + "log" +) + +// Example_paging demonstrates how to manually fetch pages and use page state. +// +// See also package documentation about paging. +func Example_paging() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create table example.itoa(id int, description text, PRIMARY KEY(id)); + insert into example.itoa (id, description) values (1, 'one'); + insert into example.itoa (id, description) values (2, 'two'); + insert into example.itoa (id, description) values (3, 'three'); + insert into example.itoa (id, description) values (4, 'four'); + insert into example.itoa (id, description) values (5, 'five'); + insert into example.itoa (id, description) values (6, 'six'); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + var pageState []byte + for { + // We use PageSize(2) for the sake of example, use larger values in production (default is 5000) for performance + // reasons. + iter := session.Query(`SELECT id, description FROM itoa`).PageSize(2).PageState(pageState).Iter() + nextPageState := iter.PageState() + scanner := iter.Scanner() + for scanner.Next() { + var ( + id int + description string + ) + err = scanner.Scan(&id, &description) + if err != nil { + log.Fatal(err) + } + fmt.Println(id, description) + } + err = scanner.Err() + if err != nil { + log.Fatal(err) + } + fmt.Printf("next page state: %+v\n", nextPageState) + if len(nextPageState) == 0 { + break + } + pageState = nextPageState + } + // 5 five + // 1 one + // next page state: [4 0 0 0 1 0 240 127 255 255 253 0] + // 2 two + // 4 four + // next page state: [4 0 0 0 4 0 240 127 255 255 251 0] + // 6 six + // 3 three + // next page state: [4 0 0 0 3 0 240 127 255 255 249 0] + // next page state: [] +} diff --git a/example_test.go b/example_test.go new file mode 100644 index 000000000..589ec57a7 --- /dev/null +++ b/example_test.go @@ -0,0 +1,65 @@ +package gocql_test + +import ( + "context" + "fmt" + "github.com/gocql/gocql" + "log" +) + +func Example() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create table example.tweet(timeline text, id UUID, text text, PRIMARY KEY(id)); + create index on example.tweet(timeline); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.Consistency = gocql.Quorum + // connect to the cluster + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + ctx := context.Background() + + // insert a tweet + if err := session.Query(`INSERT INTO tweet (timeline, id, text) VALUES (?, ?, ?)`, + "me", gocql.TimeUUID(), "hello world").WithContext(ctx).Exec(); err != nil { + log.Fatal(err) + } + + var id gocql.UUID + var text string + + /* Search for a specific set of records whose 'timeline' column matches + * the value 'me'. The secondary index that we created earlier will be + * used for optimizing the search */ + if err := session.Query(`SELECT id, text FROM tweet WHERE timeline = ? LIMIT 1`, + "me").WithContext(ctx).Consistency(gocql.One).Scan(&id, &text); err != nil { + log.Fatal(err) + } + fmt.Println("Tweet:", id, text) + fmt.Println() + + // list all tweets + scanner := session.Query(`SELECT id, text FROM tweet WHERE timeline = ?`, + "me").WithContext(ctx).Iter().Scanner() + for scanner.Next() { + err = scanner.Scan(&id, &text) + if err != nil { + log.Fatal(err) + } + fmt.Println("Tweet:", id, text) + } + // scanner.Err() closes the iterator, so scanner nor iter should be used afterwards. + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + // Tweet: cad53821-3731-11eb-971c-708bcdaada84 hello world + // + // Tweet: cad53821-3731-11eb-971c-708bcdaada84 hello world + // Tweet: d577ab85-3731-11eb-81eb-708bcdaada84 hello world +} diff --git a/example_udt_map_test.go b/example_udt_map_test.go new file mode 100644 index 000000000..10184568e --- /dev/null +++ b/example_udt_map_test.go @@ -0,0 +1,49 @@ +package gocql_test + +import ( + "context" + "fmt" + "github.com/gocql/gocql" + "log" +) + +// Example_userDefinedTypesMap demonstrates how to work with user-defined types as maps. +// See also Example_userDefinedTypesStruct and examples for UDTMarshaler and UDTUnmarshaler if you want to map to structs. +func Example_userDefinedTypesMap() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create type example.my_udt (field_a text, field_b int); + create table example.my_udt_table(pk int, value frozen, PRIMARY KEY(pk)); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + ctx := context.Background() + + value := map[string]interface{}{ + "field_a": "a value", + "field_b": 42, + } + err = session.Query("INSERT INTO example.my_udt_table (pk, value) VALUES (?, ?)", + 1, value).WithContext(ctx).Exec() + if err != nil { + log.Fatal(err) + } + + var readValue map[string]interface{} + + err = session.Query("SELECT value FROM example.my_udt_table WHERE pk = 1").WithContext(ctx).Scan(&readValue) + if err != nil { + log.Fatal(err) + } + fmt.Println(readValue["field_a"]) + fmt.Println(readValue["field_b"]) + // a value + // 42 +} diff --git a/example_udt_marshaler_test.go b/example_udt_marshaler_test.go new file mode 100644 index 000000000..ea9d71ccf --- /dev/null +++ b/example_udt_marshaler_test.go @@ -0,0 +1,57 @@ +package gocql_test + +import ( + "context" + "github.com/gocql/gocql" + "log" +) + +// MyUDTMarshaler implements UDTMarshaler. +type MyUDTMarshaler struct { + fieldA string + fieldB int32 +} + +// MarshalUDT marshals the selected field to bytes. +func (m MyUDTMarshaler) MarshalUDT(name string, info gocql.TypeInfo) ([]byte, error) { + switch name { + case "field_a": + return gocql.Marshal(info, m.fieldA) + case "field_b": + return gocql.Marshal(info, m.fieldB) + default: + // If you want to be strict and return error un unknown field, you can do so here instead. + // Returning nil, nil will set the value of unknown fields to null, which might be handy if you want + // to be forward-compatible when a new field is added to the UDT. + return nil, nil + } +} + +// ExampleUDTMarshaler demonstrates how to implement a UDTMarshaler. +func ExampleUDTMarshaler() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create type example.my_udt (field_a text, field_b int); + create table example.my_udt_table(pk int, value frozen, PRIMARY KEY(pk)); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + ctx := context.Background() + + value := MyUDTMarshaler{ + fieldA: "a value", + fieldB: 42, + } + err = session.Query("INSERT INTO example.my_udt_table (pk, value) VALUES (?, ?)", + 1, value).WithContext(ctx).Exec() + if err != nil { + log.Fatal(err) + } +} diff --git a/example_udt_struct_test.go b/example_udt_struct_test.go new file mode 100644 index 000000000..1f298bd2d --- /dev/null +++ b/example_udt_struct_test.go @@ -0,0 +1,54 @@ +package gocql_test + +import ( + "context" + "fmt" + "github.com/gocql/gocql" + "log" +) + +type MyUDT struct { + FieldA string `cql:"field_a"` + FieldB int32 `cql:"field_b"` +} + +// Example_userDefinedTypesStruct demonstrates how to work with user-defined types as structs. +// See also examples for UDTMarshaler and UDTUnmarshaler if you need more control/better performance. +func Example_userDefinedTypesStruct() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create type example.my_udt (field_a text, field_b int); + create table example.my_udt_table(pk int, value frozen, PRIMARY KEY(pk)); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + ctx := context.Background() + + value := MyUDT{ + FieldA: "a value", + FieldB: 42, + } + err = session.Query("INSERT INTO example.my_udt_table (pk, value) VALUES (?, ?)", + 1, value).WithContext(ctx).Exec() + if err != nil { + log.Fatal(err) + } + + var readValue MyUDT + + err = session.Query("SELECT value FROM example.my_udt_table WHERE pk = 1").WithContext(ctx).Scan(&readValue) + if err != nil { + log.Fatal(err) + } + fmt.Println(readValue.FieldA) + fmt.Println(readValue.FieldB) + // a value + // 42 +} diff --git a/example_udt_unmarshaler_test.go b/example_udt_unmarshaler_test.go new file mode 100644 index 000000000..a5debcb68 --- /dev/null +++ b/example_udt_unmarshaler_test.go @@ -0,0 +1,59 @@ +package gocql_test + +import ( + "context" + "fmt" + "github.com/gocql/gocql" + "log" +) + +// MyUDTUnmarshaler implements UDTUnmarshaler. +type MyUDTUnmarshaler struct { + fieldA string + fieldB int32 +} + +// UnmarshalUDT unmarshals the field identified by name into MyUDTUnmarshaler. +func (m *MyUDTUnmarshaler) UnmarshalUDT(name string, info gocql.TypeInfo, data []byte) error { + switch name { + case "field_a": + return gocql.Unmarshal(info, data, &m.fieldA) + case "field_b": + return gocql.Unmarshal(info, data, &m.fieldB) + default: + // If you want to be strict and return error un unknown field, you can do so here instead. + // Returning nil will ignore unknown fields, which might be handy if you want + // to be forward-compatible when a new field is added to the UDT. + return nil + } +} + +// ExampleUDTUnmarshaler demonstrates how to implement a UDTUnmarshaler. +func ExampleUDTUnmarshaler() { + /* The example assumes the following CQL was used to setup the keyspace: + create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 }; + create type example.my_udt (field_a text, field_b int); + create table example.my_udt_table(pk int, value frozen, PRIMARY KEY(pk)); + insert into example.my_udt_table (pk, value) values (1, {field_a: 'a value', field_b: 42}); + */ + cluster := gocql.NewCluster("localhost:9042") + cluster.Keyspace = "example" + cluster.ProtoVersion = 4 + session, err := cluster.CreateSession() + if err != nil { + log.Fatal(err) + } + defer session.Close() + + ctx := context.Background() + + var value MyUDTUnmarshaler + err = session.Query("SELECT value FROM example.my_udt_table WHERE pk = 1").WithContext(ctx).Scan(&value) + if err != nil { + log.Fatal(err) + } + fmt.Println(value.fieldA) + fmt.Println(value.fieldB) + // a value + // 42 +} diff --git a/policies.go b/policies.go index 485dcef5a..cfcb3de6d 100644 --- a/policies.go +++ b/policies.go @@ -1,9 +1,12 @@ // Copyright (c) 2012 The gocql Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//This file will be the future home for more policies + package gocql +//This file will be the future home for more policies + + import ( "context" "errors" diff --git a/session.go b/session.go index 901ee7f63..efd32afa3 100644 --- a/session.go +++ b/session.go @@ -1102,12 +1102,17 @@ func (q *Query) speculativeExecutionPolicy() SpeculativeExecutionPolicy { return q.spec } +// IsIdempotent returns whether the query is marked as idempotent. +// Non-idempotent query won't be retried. +// See "Retries and speculative execution" in package docs for more details. func (q *Query) IsIdempotent() bool { return q.idempotent } // Idempotent marks the query as being idempotent or not depending on // the value. +// Non-idempotent query won't be retried. +// See "Retries and speculative execution" in package docs for more details. func (q *Query) Idempotent(value bool) *Query { q.idempotent = value return q @@ -1203,6 +1208,10 @@ func (q *Query) Scan(dest ...interface{}) error { // statement containing an IF clause). If the transaction fails because // the existing values did not match, the previous values will be stored // in dest. +// +// As for INSERT .. IF NOT EXISTS, previous values will be returned as if +// SELECT * FROM. So using ScanCAS with INSERT is inherently prone to +// column mismatching. Use MapScanCAS to capture them safely. func (q *Query) ScanCAS(dest ...interface{}) (applied bool, err error) { q.disableSkipMetadata = true iter := q.Iter() diff --git a/uuid.go b/uuid.go index 13ad38379..09727a017 100644 --- a/uuid.go +++ b/uuid.go @@ -2,11 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +package gocql + // The uuid package can be used to generate and parse universally unique // identifiers, a standardized format in the form of a 128 bit number. // // http://tools.ietf.org/html/rfc4122 -package gocql import ( "crypto/rand"