Skip to content

Commit

Permalink
Add support by excluding specific fields from indexes by passing
Browse files Browse the repository at this point in the history 
"excludeFromIndex" []string parameter to the
ProtoMessageToDatastoreEntity function.
  • Loading branch information
@Kami
Kami committed Aug 14, 2019
1 parent d137525 commit ff392f0
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 19 deletions.
39 changes: 33 additions & 6 deletions datastore-translator/translator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@ import (
)

// ProtoMessageToDatastoreEntity will generate an Entity Protobuf that datastore understands
func ProtoMessageToDatastoreEntity(src proto.Message, snakeCase bool) (entity datastore.Entity, err error) {
func ProtoMessageToDatastoreEntity(src proto.Message, snakeCase bool, excludeFromIndex []string) (entity datastore.Entity, err error) {
srcValues := reflect.ValueOf(src).Elem()
properties := make(map[string]*datastore.Value)

for i := 0; i < srcValues.NumField(); i++ {
fName := srcValues.Type().Field(i).Name
if !strings.ContainsAny(fName, "XXX_") {
var value *datastore.Value
if value, err = toDatastoreValue(srcValues.Field(i), snakeCase); err != nil {
if value, err = toDatastoreValue(fName, srcValues.Field(i), snakeCase, excludeFromIndex); err != nil {
return
} else {
if value != nil {
Expand Down Expand Up @@ -117,7 +117,18 @@ func DatastoreEntityToProtoMessage(src *datastore.Entity, dst proto.Message, sna
return err
}

func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, error) {
func toDatastoreValue(fName string, fValue reflect.Value, snakeCase bool, excludeFromIndex []string) (*datastore.Value, error) {
// NOTE: excludeFieldFromIndex needs to contain original Protobuf model field names which can also include underscores
var origFName string

if snakeCase {
origFName = toSnakeCase(fName)
} else {
origFName = fName
}

excludeFieldFromIndex := contains(excludeFromIndex, origFName)

value := &datastore.Value{}
var err error
switch fValue.Kind() {
Expand Down Expand Up @@ -148,7 +159,7 @@ func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, e
size := fValue.Len()
values := make([]*datastore.Value, 0)
for i := 0; i < size; i++ {
val, err := toDatastoreValue(fValue.Index(i), snakeCase)
val, err := toDatastoreValue(fName, fValue.Index(i), snakeCase, nil)
if err != nil {
return nil, err
}
Expand All @@ -167,7 +178,7 @@ func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, e
for _, key := range mapValues.MapKeys() {
k := fmt.Sprint(key)
//TODO what if there is an error?
v, _ := toDatastoreValue(mapValues.MapIndex(key), snakeCase)
v, _ := toDatastoreValue(fName, mapValues.MapIndex(key), snakeCase, nil)
//fmt.Printf("key; %v, value: %v\n",k,v)
properties[k] = v
}
Expand Down Expand Up @@ -203,7 +214,7 @@ func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, e
// translate any imported protobuf's that we defined ourself
if !fValue.IsNil() && fValue.IsValid() {
if importedProto, ok := reflect.ValueOf(fValue.Interface()).Interface().(proto.Message); ok {
entityOfImportedProto, err := ProtoMessageToDatastoreEntity(importedProto, snakeCase)
entityOfImportedProto, err := ProtoMessageToDatastoreEntity(importedProto, snakeCase, excludeFromIndex)
if err != nil {
return nil, err
}
Expand All @@ -220,6 +231,13 @@ func toDatastoreValue(fValue reflect.Value, snakeCase bool) (*datastore.Value, e
log.Println(errString)
err = errors.New(errString)
}

if excludeFieldFromIndex == true {
value.ExcludeFromIndexes = true
} else {
value.ExcludeFromIndexes = false
}

return value, err
}

Expand Down Expand Up @@ -311,3 +329,12 @@ func toSnakeCase(name string) string {
snake = matchAllCap.ReplaceAllString(snake, "${1}_${2}")
return strings.ToLower(snake)
}

func contains(a []string, x string) bool {
for _, n := range a {
if x == n {
return true
}
}
return false
}
6 changes: 3 additions & 3 deletions datastore-translator/translator_integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func TestIntegration(t *testing.T) {

log.Printf("original proto: %v", srcProto)
// 4. translate the source protobuf to datastore.Entity
translatedSrcProto, err := ProtoMessageToDatastoreEntity(srcProto, true)
translatedSrcProto, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)
assert.NilError(t, err)

// 5. save the translated protobuf to datastore
Expand Down Expand Up @@ -124,7 +124,7 @@ func TestEmptyProtoMessage(t *testing.T) {
key := datastore.NameKey("Example_DB_Model", "complex_proto_empty", nil)

srcProto := &example.ExampleDBModel{}
translatedProto, err := ProtoMessageToDatastoreEntity(srcProto, false)
translatedProto, err := ProtoMessageToDatastoreEntity(srcProto, false, nil)
assert.NilError(t, err)

_, err = client.PutEntity(ctx, key, &translatedProto)
Expand Down Expand Up @@ -155,7 +155,7 @@ func TestProtoWithNilPointer(t *testing.T) {
srcProto := &example.ExampleDBModel{
TimestampKey: ptypes.TimestampNow(),
}
translatedProto, err := ProtoMessageToDatastoreEntity(srcProto, false)
translatedProto, err := ProtoMessageToDatastoreEntity(srcProto, false, nil)
assert.NilError(t, err)

_, err = client.PutEntity(ctx, key, &translatedProto)
Expand Down
48 changes: 42 additions & 6 deletions datastore-translator/translator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func TestNestedModel(t *testing.T) {
StringKey: "some random string",
Int32Key: 22,
}
entity, err := ProtoMessageToDatastoreEntity(srcProto, true)
entity, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)
// make sure there is no error
assert.NilError(t, err)
dstProto := &example.ExampleNestedModel{}
Expand All @@ -27,6 +27,42 @@ func TestNestedModel(t *testing.T) {
assert.DeepEqual(t, srcProto.GetStringKey(), dstProto.GetStringKey())
}

func TestProtoMessageToDatastoreEntityWithExcludeFieldsFromIndex(t *testing.T) {
srcProto := &example.ExampleDBModel{
StringKey: "some random string key for testing",
BoolKey: true,
Int32Key: int32(32),
Int64Key: 64,
FloatKey: float32(10.14),
DoubleKey: float64(10.2121),
BytesKey: []byte("this is a byte array"),
}

// No exclude from index fields specified
entity, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)
assert.NilError(t, err)
assert.Equal(t, entity.Properties["string_key"].ExcludeFromIndexes, false)
assert.Equal(t, entity.Properties["bytes_key"].ExcludeFromIndexes, false)
assert.Equal(t, entity.Properties["float_key"].ExcludeFromIndexes, false)

assert.Equal(t, entity.Properties["bool_key"].ExcludeFromIndexes, false)
assert.Equal(t, entity.Properties["int32_key"].ExcludeFromIndexes, false)
assert.Equal(t, entity.Properties["double_key"].ExcludeFromIndexes, false)

// Some exclude from index fields specified
excludeFromIndex := [...]string{"string_key", "bytes_key", "float_key"}

entity, err = ProtoMessageToDatastoreEntity(srcProto, true, excludeFromIndex[:])
assert.NilError(t, err)
assert.Equal(t, entity.Properties["string_key"].ExcludeFromIndexes, true)
assert.Equal(t, entity.Properties["bytes_key"].ExcludeFromIndexes, true)
assert.Equal(t, entity.Properties["float_key"].ExcludeFromIndexes, true)

assert.Equal(t, entity.Properties["bool_key"].ExcludeFromIndexes, false)
assert.Equal(t, entity.Properties["int32_key"].ExcludeFromIndexes, false)
assert.Equal(t, entity.Properties["double_key"].ExcludeFromIndexes, false)
}

func TestFullyPopulatedModel(t *testing.T) {
srcProto := &example.ExampleDBModel{
StringKey: "some random string key for testing",
Expand Down Expand Up @@ -73,7 +109,7 @@ func TestFullyPopulatedModel(t *testing.T) {
},
TimestampKey: ptypes.TimestampNow(),
}
entity, err := ProtoMessageToDatastoreEntity(srcProto, true)
entity, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)

// make sure there is no error
assert.NilError(t, err)
Expand Down Expand Up @@ -125,7 +161,7 @@ func TestPartialModel(t *testing.T) {
"struct-key-null": {Kind: &structpb.Value_NullValue{}},
},
}
entity, err := ProtoMessageToDatastoreEntity(partialProto, true)
entity, err := ProtoMessageToDatastoreEntity(partialProto, true, nil)
assert.NilError(t, err, err)
log.Println(entity)
dstProto := &structpb.Struct{}
Expand All @@ -140,7 +176,7 @@ func TestUnSupportedTypes(t *testing.T) {
srcProto := &unsupported.Model{
Uint32Key: uint32(10),
}
_, err := ProtoMessageToDatastoreEntity(srcProto, false)
_, err := ProtoMessageToDatastoreEntity(srcProto, false, nil)
assert.Error(t, err, "[toDatastoreValue]: datatype[uint32] not supported")
}

Expand All @@ -149,7 +185,7 @@ func TestPMtoDE(t *testing.T) {
StringKey: "some random string",
Int32Key: 22,
}
entity, err := ProtoMessageToDatastoreEntity(srcProto, true)
entity, err := ProtoMessageToDatastoreEntity(srcProto, true, nil)
assert.NilError(t, err)
log.Println(entity)
}
Expand Down Expand Up @@ -317,7 +353,7 @@ func TestProtoWithCustomImport(t *testing.T) {
},
}

dstEntity, err := ProtoMessageToDatastoreEntity(srcProto, false)
dstEntity, err := ProtoMessageToDatastoreEntity(srcProto, false, nil)
assert.NilError(t, err)
// our interest here only to compare the ComplexArrayKey
assert.DeepEqual(t, srcEntity.GetProperties()["ComplexArrayKey"], dstEntity.GetProperties()["ComplexArrayKey"])
Expand Down
8 changes: 4 additions & 4 deletions datastore-translator/ts_struct_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func TestAddTSSupport(t *testing.T) {
StartedOn: ptypes.TimestampNow(),
}
log.Println("Source: ", src)
srcEntity, err := ProtoMessageToDatastoreEntity(src, false)
srcEntity, err := ProtoMessageToDatastoreEntity(src, false, nil)

assert.NilError(t, err)
log.Println("Source Datastore Entity: ", srcEntity)
Expand Down Expand Up @@ -50,7 +50,7 @@ func TestAddStructSupport(t *testing.T) {
},
}
log.Println("Source: ", src)
srcEntity, err := ProtoMessageToDatastoreEntity(src, false)
srcEntity, err := ProtoMessageToDatastoreEntity(src, false, nil)

assert.NilError(t, err)
log.Println("Source Datastore Entity: ", srcEntity)
Expand All @@ -75,7 +75,7 @@ func TestSliceofNestedMessages(t *testing.T) {
},
}
log.Println("Source: ", src)
srcEntity, err := ProtoMessageToDatastoreEntity(src, false)
srcEntity, err := ProtoMessageToDatastoreEntity(src, false, nil)
assert.NilError(t, err)

log.Println("Source Datastore Entity: ", srcEntity)
Expand All @@ -97,7 +97,7 @@ func TestNestedMessages(t *testing.T) {
},
}
log.Println("Source: ", src)
srcEntity, err := ProtoMessageToDatastoreEntity(src, false)
srcEntity, err := ProtoMessageToDatastoreEntity(src, false, nil)
assert.NilError(t, err)

log.Println("Source Datastore Entity: ", srcEntity)
Expand Down

0 comments on commit ff392f0

Please sign in to comment.