Skip to content

Commit

Permalink
Add support for varints.
Browse files Browse the repository at this point in the history 
Similar to other integer types except you must demarshal varints
greater than 8 bytes using a *big.Int. It is possible to detect
unset varint fields when fetching using a **big.Int double pointer.
Maybe all types should support this as a way to detect unset cells as
opposed to cells with a value of zero?

Varints are signed integers encoded using two's complement. When marshaling,
gocql trims the encoded bytes down to the minimum required to encode the
integer value (e.g. so if you encode a varint value that can fit in one byte,
gocql will actually only send 1 byte).
  • Loading branch information
Muir Manders committed Aug 7, 2014
1 parent 5cb2a37 commit 7eda240
Show file tree
Hide file tree
Showing 4 changed files with 340 additions and 13 deletions.
115 changes: 109 additions & 6 deletions cassandra_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,17 @@ package gocql
import (
"bytes"
"flag"
"math"
"math/big"
"reflect"
"speter.net/go/exp/math/dec/inf"
"strconv"
"strings"
"sync"
"testing"
"time"
"unicode"

"speter.net/go/exp/math/dec/inf"
)

var (
Expand Down Expand Up @@ -351,16 +354,23 @@ func TestSliceMap(t *testing.T) {
testbigint bigint,
testblob blob,
testbool boolean,
testfloat float,
testdouble double,
testfloat float,
testdouble double,
testint int,
testdecimal decimal,
testset set<int>,
testmap map<varchar, varchar>
testmap map<varchar, varchar>,
testvarint varint
)`).Exec(); err != nil {
t.Fatal("create table:", err)
}
m := make(map[string]interface{})

bigInt := new(big.Int)
if _, ok := bigInt.SetString("830169365738487321165427203929228", 10); !ok {
t.Fatal("Failed setting bigint by string")
}

m["testuuid"] = TimeUUID()
m["testvarchar"] = "Test VarChar"
m["testbigint"] = time.Now().Unix()
Expand All @@ -373,9 +383,10 @@ func TestSliceMap(t *testing.T) {
m["testdecimal"] = inf.NewDec(100, 0)
m["testset"] = []int{1, 2, 3, 4, 5, 6, 7, 8, 9}
m["testmap"] = map[string]string{"field1": "val1", "field2": "val2", "field3": "val3"}
m["testvarint"] = bigInt
sliceMap := []map[string]interface{}{m}
if err := session.Query(`INSERT INTO slice_map_table (testuuid, testtimestamp, testvarchar, testbigint, testblob, testbool, testfloat, testdouble, testint, testdecimal, testset, testmap) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
m["testuuid"], m["testtimestamp"], m["testvarchar"], m["testbigint"], m["testblob"], m["testbool"], m["testfloat"], m["testdouble"], m["testint"], m["testdecimal"], m["testset"], m["testmap"]).Exec(); err != nil {
if err := session.Query(`INSERT INTO slice_map_table (testuuid, testtimestamp, testvarchar, testbigint, testblob, testbool, testfloat, testdouble, testint, testdecimal, testset, testmap, testvarint) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
m["testuuid"], m["testtimestamp"], m["testvarchar"], m["testbigint"], m["testblob"], m["testbool"], m["testfloat"], m["testdouble"], m["testint"], m["testdecimal"], m["testset"], m["testmap"], m["testvarint"]).Exec(); err != nil {
t.Fatal("insert:", err)
}
if returned, retErr := session.Query(`SELECT * FROM slice_map_table`).Iter().SliceMap(); retErr != nil {
Expand Down Expand Up @@ -421,6 +432,12 @@ func TestSliceMap(t *testing.T) {
if !reflect.DeepEqual(sliceMap[0]["testmap"], returned[0]["testmap"]) {
t.Fatal("returned testmap did not match")
}

expectedBigInt := sliceMap[0]["testvarint"].(*big.Int)
returnedBigInt := returned[0]["testvarint"].(*big.Int)
if expectedBigInt.Cmp(returnedBigInt) != 0 {
t.Fatal("returned testvarint did not match")
}
}

// Test for MapScan()
Expand Down Expand Up @@ -925,3 +942,89 @@ func TestMarshalFloat64Ptr(t *testing.T) {
t.Fatal("insert float64:", err)
}
}

func TestVarint(t *testing.T) {
session := createSession(t)
defer session.Close()

if err := session.Query("CREATE TABLE varint_test (id varchar, test varint, test2 varint, primary key (id))").Exec(); err != nil {
t.Fatal("create table:", err)
}

if err := session.Query(`INSERT INTO varint_test (id, test) VALUES (?, ?)`, "id", 0).Exec(); err != nil {
t.Fatalf("insert varint: %v", err)
}

var result int
if err := session.Query("SELECT test FROM varint_test").Scan(&result); err != nil {
t.Fatalf("select from varint_test failed: %v", err)
}

if result != 0 {
t.Errorf("Expected 0, was %d", result)
}

if err := session.Query(`INSERT INTO varint_test (id, test) VALUES (?, ?)`, "id", -1).Exec(); err != nil {
t.Fatalf("insert varint: %v", err)
}

if err := session.Query("SELECT test FROM varint_test").Scan(&result); err != nil {
t.Fatalf("select from varint_test failed: %v", err)
}

if result != -1 {
t.Errorf("Expected -1, was %d", result)
}

if err := session.Query(`INSERT INTO varint_test (id, test) VALUES (?, ?)`, "id", int64(math.MaxInt32)+1).Exec(); err != nil {
t.Fatalf("insert varint: %v", err)
}

var result64 int64
if err := session.Query("SELECT test FROM varint_test").Scan(&result64); err != nil {
t.Fatalf("select from varint_test failed: %v", err)
}

if result64 != int64(math.MaxInt32)+1 {
t.Errorf("Expected %d, was %d", int64(math.MaxInt32)+1, result64)
}

biggie := new(big.Int)
biggie.SetString("36893488147419103232", 10) // > 2**64
if err := session.Query(`INSERT INTO varint_test (id, test) VALUES (?, ?)`, "id", biggie).Exec(); err != nil {
t.Fatalf("insert varint: %v", err)
}

resultBig := new(big.Int)
if err := session.Query("SELECT test FROM varint_test").Scan(resultBig); err != nil {
t.Fatalf("select from varint_test failed: %v", err)
}

if resultBig.String() != biggie.String() {
t.Errorf("Expected %s, was %s", biggie.String(), resultBig.String())
}

err := session.Query("SELECT test FROM varint_test").Scan(&result64)
if err == nil || strings.Index(err.Error(), "out of range") == -1 {
t.Errorf("expected our of range error since value is too big for int64")
}

// value not set in cassandra, leave bind variable empty
resultBig = new(big.Int)
if err := session.Query("SELECT test2 FROM varint_test").Scan(resultBig); err != nil {
t.Fatalf("select from varint_test failed: %v", err)
}

if resultBig.Int64() != 0 {
t.Errorf("Expected %s, was %s", biggie.String(), resultBig.String())
}

// can use double pointer to explicitly detect value is not set in cassandra
if err := session.Query("SELECT test2 FROM varint_test").Scan(&resultBig); err != nil {
t.Fatalf("select from varint_test failed: %v", err)
}

if resultBig != nil {
t.Errorf("Expected %v, was %v", nil, *resultBig)
}
}
6 changes: 5 additions & 1 deletion helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@
package gocql

import (
"math/big"
"reflect"
"speter.net/go/exp/math/dec/inf"
"strings"
"time"

"speter.net/go/exp/math/dec/inf"
)

type RowData struct {
Expand Down Expand Up @@ -48,6 +50,8 @@ func goType(t *TypeInfo) reflect.Type {
return reflect.SliceOf(goType(t.Elem))
case TypeMap:
return reflect.MapOf(goType(t.Key), goType(t.Elem))
case TypeVarint:
return reflect.TypeOf(*new(*big.Int))
default:
return nil
}
Expand Down
106 changes: 102 additions & 4 deletions marshal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@ package gocql

import (
"bytes"
"encoding/binary"
"fmt"
"math"
"math/big"
"reflect"
"speter.net/go/exp/math/dec/inf"
"time"

"speter.net/go/exp/math/dec/inf"
)

var (
Expand Down Expand Up @@ -59,6 +61,8 @@ func Marshal(info *TypeInfo, value interface{}) ([]byte, error) {
return marshalMap(info, value)
case TypeUUID, TypeTimeUUID:
return marshalUUID(info, value)
case TypeVarint:
return marshalVarint(info, value)
}
// TODO(tux21b): add the remaining types
return nil, fmt.Errorf("can not marshal %T into %s", value, info)
Expand All @@ -80,6 +84,8 @@ func Unmarshal(info *TypeInfo, data []byte, value interface{}) error {
return unmarshalInt(info, data, value)
case TypeBigInt, TypeCounter:
return unmarshalBigInt(info, data, value)
case TypeVarint:
return unmarshalVarint(info, data, value)
case TypeFloat:
return unmarshalFloat(info, data, value)
case TypeDouble:
Expand Down Expand Up @@ -266,6 +272,8 @@ func marshalBigInt(info *TypeInfo, value interface{}) ([]byte, error) {
return encBigInt(int64(v)), nil
case uint8:
return encBigInt(int64(v)), nil
case *big.Int:
return encBigInt2C(v), nil
}
rv := reflect.ValueOf(value)
switch rv.Type().Kind() {
Expand All @@ -289,6 +297,13 @@ func encBigInt(x int64) []byte {
byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x)}
}

func bytesToInt64(data []byte) (ret int64) {
for i := range data {
ret |= int64(data[i]) << (8 * uint(len(data)-i-1))
}
return ret
}

func unmarshalBigInt(info *TypeInfo, data []byte, value interface{}) error {
return unmarshalIntlike(info, decBigInt(data), data, value)
}
Expand All @@ -297,6 +312,72 @@ func unmarshalInt(info *TypeInfo, data []byte, value interface{}) error {
return unmarshalIntlike(info, int64(decInt(data)), data, value)
}

func unmarshalVarint(info *TypeInfo, data []byte, value interface{}) error {
switch value.(type) {
case *big.Int, **big.Int:
return unmarshalIntlike(info, 0, data, value)
}

if len(data) > 8 {
return unmarshalErrorf("unmarshal int: varint value %v out of range for %T (use big.Int)", data, value)
}

int64Val := bytesToInt64(data)
if len(data) < 8 && data[0]&0x80 > 0 {
int64Val -= (1 << uint(len(data)*8))
}
return unmarshalIntlike(info, int64Val, data, value)
}

func marshalVarint(info *TypeInfo, value interface{}) ([]byte, error) {
var (
retBytes []byte
err error
)

switch v := value.(type) {
case uint64:
if v > uint64(math.MaxInt64) {
retBytes = make([]byte, 9)
binary.BigEndian.PutUint64(retBytes[1:], v)
} else {
retBytes = make([]byte, 8)
binary.BigEndian.PutUint64(retBytes, v)
}
default:
retBytes, err = marshalBigInt(info, value)
}

if err == nil {
// trim down to most significant byte
i := 0
for ; i < len(retBytes)-1; i++ {
b0 := retBytes[i]
if b0 != 0 && b0 != 0xFF {
break
}

b1 := retBytes[i+1]
if b0 == 0 && b1 != 0 {
if b1&0x80 == 0 {
i++
}
break
}

if b0 == 0xFF && b1 != 0xFF {
if b1&0x80 > 0 {
i++
}
break
}
}
retBytes = retBytes[i:]
}

return retBytes, err
}

func unmarshalIntlike(info *TypeInfo, int64Val int64, data []byte, value interface{}) error {
switch v := value.(type) {
case *int:
Expand Down Expand Up @@ -356,12 +437,24 @@ func unmarshalIntlike(info *TypeInfo, int64Val int64, data []byte, value interfa
}
*v = uint8(int64Val)
return nil
case *big.Int:
decBigInt2C(data, v)
return nil
case **big.Int:
if len(data) == 0 {
*v = nil
} else {
*v = decBigInt2C(data, nil)
}
return nil
}

rv := reflect.ValueOf(value)
if rv.Kind() != reflect.Ptr {
return unmarshalErrorf("can not unmarshal into non-pointer %T", value)
}
rv = rv.Elem()

switch rv.Type().Kind() {
case reflect.Int:
if ^uint(0) == math.MaxUint32 && (int64Val < math.MinInt32 || int64Val > math.MaxInt32) {
Expand Down Expand Up @@ -592,7 +685,7 @@ func unmarshalDecimal(info *TypeInfo, data []byte, value interface{}) error {
case **inf.Dec:
if len(data) > 4 {
scale := decInt(data[0:4])
unscaled := decBigInt2C(data[4:])
unscaled := decBigInt2C(data[4:], nil)
*v = inf.NewDecBig(unscaled, inf.Scale(scale))
return nil
} else if len(data) == 0 {
Expand All @@ -608,8 +701,11 @@ func unmarshalDecimal(info *TypeInfo, data []byte, value interface{}) error {
// decBigInt2C sets the value of n to the big-endian two's complement
// value stored in the given data. If data[0]&80 != 0, the number
// is negative. If data is empty, the result will be 0.
func decBigInt2C(data []byte) *big.Int {
n := new(big.Int).SetBytes(data)
func decBigInt2C(data []byte, n *big.Int) *big.Int {
if n == nil {
n = new(big.Int)
}
n.SetBytes(data)
if len(data) > 0 && data[0]&0x80 > 0 {
n.Sub(n, new(big.Int).Lsh(bigOne, uint(len(data))*8))
}
Expand Down Expand Up @@ -1044,6 +1140,8 @@ func (t Type) String() string {
return "map"
case TypeSet:
return "set"
case TypeVarint:
return "varint"
default:
return "unknown"
}
Expand Down
Loading

0 comments on commit 7eda240

Please sign in to comment.